From 434e53cba2b14c1a4f04a0b3fb557507d0d9a77f Mon Sep 17 00:00:00 2001 From: Apollon Oikonomopoulos Date: Sat, 12 Aug 2017 12:12:15 -0400 Subject: New upstream version 3.2.16 --- SConstruct | 74 +- buildscripts/buildlogger.py | 12 + buildscripts/clang_format.py | 101 +- buildscripts/generate_compile_expansions.py | 4 +- buildscripts/hang_analyzer.py | 73 +- buildscripts/package_test/.kitchen.yml | 81 ++ buildscripts/package_test/metadata.rb | 2 + .../package_test/recipes/install_mongodb.rb | 120 ++ .../test/recipes/service/install_mongodb_spec.rb | 180 +++ buildscripts/packager-enterprise.py | 16 +- buildscripts/packager.py | 105 +- buildscripts/resmoke.py | 56 +- .../resmokeconfig/suites/multiversion_auth.yml | 37 + .../resmokeconfig/suites/no_passthrough.yml | 3 + .../resmokeconfig/suites/replica_sets_legacy.yml | 8 + .../sharded_collections_jscore_passthrough.yml | 6 +- .../resmokeconfig/suites/sharding_auth.yml | 1 + .../resmokeconfig/suites/sharding_auth_audit.yml | 1 + .../suites/sharding_continuous_config_stepdown.yml | 8 + .../suites/sharding_jscore_passthrough.yml | 5 +- buildscripts/resmokelib/__init__.py | 2 + buildscripts/resmokelib/logging/buildlogger.py | 6 +- buildscripts/resmokelib/logging/handlers.py | 63 +- buildscripts/resmokelib/reportfile.py | 29 + buildscripts/resmokelib/sighandler.py | 110 ++ buildscripts/resmokelib/testing/executor.py | 5 +- .../resmokelib/testing/fixtures/replicaset.py | 8 +- .../resmokelib/testing/fixtures/shardedcluster.py | 5 +- .../resmokelib/testing/fixtures/standalone.py | 5 +- buildscripts/resmokelib/testing/job.py | 16 +- buildscripts/resmokelib/testing/report.py | 205 ++-- buildscripts/resmokelib/testing/testgroup.py | 62 +- buildscripts/resmokelib/utils/timer.py | 66 +- buildscripts/smoke.py | 124 ++- etc/evergreen.yml | 234 +++- etc/longevity.yml | 132 ++- etc/perf.yml | 266 +---- etc/scons/gold_incremental_link.vars | 1 + etc/scons/mongodbtoolchain_clang.vars | 22 + etc/scons/mongodbtoolchain_gcc.vars | 22 + etc/system_perf.yml | 362 ++++-- jstests/auth/commands_builtin_roles.js | 10 +- jstests/auth/commands_user_defined_roles.js | 93 +- jstests/auth/server-4892.js | 6 +- jstests/concurrency/fsm_all.js | 7 +- jstests/concurrency/fsm_all_composed.js | 3 + jstests/concurrency/fsm_all_replication.js | 3 + jstests/concurrency/fsm_all_sharded_replication.js | 1 + ...ll_sharded_replication_legacy_config_servers.js | 1 + ...lication_legacy_config_servers_with_balancer.js | 6 + .../fsm_all_sharded_replication_with_balancer.js | 6 + .../concurrency/fsm_workloads/create_database.js | 150 +++ .../sharded_moveChunk_drop_shard_key_index.js | 84 ++ jstests/core/currentop.js | 114 +- jstests/core/dbadmin.js | 8 +- jstests/core/diagdata.js | 33 + jstests/core/drop2.js | 57 - jstests/core/evalh.js | 18 + jstests/core/evalj.js | 13 + jstests/core/hostinfo.js | 10 + jstests/core/index_partial_write_ops.js | 18 +- jstests/core/killop_drop_collection.js | 54 + jstests/core/list_collections1.js | 3 +- jstests/core/list_indexes.js | 3 +- jstests/core/list_indexes_invalidation.js | 37 + jstests/core/list_namespaces_invalidation.js | 73 ++ jstests/core/max_doc_size.js | 6 +- jstests/core/ref.js | 5 + jstests/core/server22053.js | 19 + jstests/core/server25192.js | 12 + jstests/core/tailable_skip_limit.js | 7 + jstests/libs/check_log.js | 84 ++ jstests/libs/chunk_manipulation_util.js | 6 +- jstests/libs/election_timing_test.js | 227 ---- jstests/libs/parallelTester.js | 6 +- jstests/libs/test_background_ops.js | 4 + jstests/libs/write_concern_util.js | 73 ++ jstests/multiVersion/libs/multi_rs.js | 3 +- .../multiVersion/minor_version_tags_new_old_new.js | 274 +---- .../multiVersion/minor_version_tags_old_new_old.js | 274 +---- jstests/noPassthrough/command_line_parsing.js | 4 +- jstests/noPassthrough/indexbg1.js | 18 +- jstests/noPassthrough/indexbg2.js | 20 +- jstests/noPassthrough/read_majority_reads.js | 270 +++++ jstests/noPassthrough/shell_history.js | 126 +++ jstests/noPassthrough/wt_index_option_defaults.js | 6 +- .../noPassthroughWithMongod/connections_opened.js | 6 +- jstests/noPassthroughWithMongod/ftdc_params.js | 4 +- jstests/noPassthroughWithMongod/indexbg_drop.js | 2 +- jstests/noPassthroughWithMongod/ttl_sharded.js | 30 +- jstests/replsets/chaining_removal.js | 42 +- .../replsets/disallow_adding_initialized_node1.js | 15 +- .../replsets/disallow_adding_initialized_node2.js | 20 +- .../replsets/do_not_sync_from_stale_sync_source.js | 44 + jstests/replsets/double_rollback.js | 32 +- jstests/replsets/double_rollback_early.js | 139 +++ jstests/replsets/drain.js | 19 +- jstests/replsets/election_timing.js | 143 --- jstests/replsets/last_vote.js | 229 ++++ jstests/replsets/libs/tags.js | 323 ++++++ jstests/replsets/maintenance.js | 4 + jstests/replsets/maxSyncSourceLagSecs.js | 41 +- jstests/replsets/no_chaining.js | 10 +- .../no_flapping_during_network_partition.js | 60 + .../oplog_replay_on_startup_update_and_delete.js | 48 + .../priority_takeover_cascading_priorities.js | 20 +- .../priority_takeover_two_nodes_equal_priority.js | 62 +- .../replsets/protocol_version_upgrade_downgrade.js | 9 +- jstests/replsets/read_after_optime.js | 29 +- jstests/replsets/read_committed.js | 165 ++- jstests/replsets/read_committed_after_rollback.js | 6 +- jstests/replsets/read_committed_stale_history.js | 149 +++ .../read_committed_with_catalog_changes.js | 343 ++++++ .../replsets/reconfig_without_increased_queues.js | 10 + jstests/replsets/replsetfreeze.js | 4 + jstests/replsets/request_primary_stepdown.js | 46 + jstests/replsets/resync_with_write_load.js | 2 +- .../rollback_after_sync_source_selection.js | 113 ++ jstests/replsets/rollback_empty_ns.js | 93 -- jstests/replsets/rollback_empty_o.js | 93 -- jstests/replsets/rollback_empty_o2.js | 94 -- jstests/replsets/rollback_too_new.js | 21 +- ...rollback_with_socket_error_then_steady_state.js | 119 ++ jstests/replsets/rslib.js | 176 +++ jstests/replsets/server8070.js | 13 +- jstests/replsets/shutdown_primary.js | 68 ++ jstests/replsets/slavedelay3.js | 7 +- jstests/replsets/stepdown.js | 88 +- .../replsets/stepdown_needs_electable_secondary.js | 143 +++ jstests/replsets/stepdown_needs_majority.js | 110 ++ jstests/replsets/stepdown_wrt_electable.js | 40 - jstests/replsets/sync2.js | 109 +- jstests/replsets/tags.js | 258 +---- .../temp_namespace_restart_as_standalone.js | 87 ++ jstests/replsets/write_concern_after_stepdown.js | 91 ++ .../write_concern_after_stepdown_and_stepup.js | 116 ++ jstests/sharding/authCommands.js | 7 +- jstests/sharding/cleanup_orphaned_basic.js | 4 +- jstests/sharding/conn_pool_stats.js | 43 +- jstests/sharding/cursor1.js | 5 - .../migration_critical_section_concurrency.js | 63 ++ jstests/sharding/move_stale_mongos.js | 14 +- jstests/sharding/shard6.js | 34 +- jstests/sharding/shard_kill_and_pooling.js | 4 +- jstests/sharding/sharding_rs2.js | 7 +- jstests/sharding/split_stale_mongos.js | 8 +- .../sharding/sync_cluster_config/sccc_read_pref.js | 18 + jstests/slow1/replsets_priority1.js | 2 +- jstests/ssl/ssl_fips.js | 5 +- rpm/mongodb-enterprise-unstable.spec | 6 +- rpm/mongodb-enterprise.spec | 6 +- rpm/mongodb-org-unstable.spec | 6 +- rpm/mongodb-org.spec | 6 +- site_scons/site_tools/compilation_db.py | 34 +- site_scons/site_tools/incremental_link.py | 45 + site_scons/site_tools/split_dwarf.py | 65 ++ src/mongo/SConscript | 1 + src/mongo/base/error_codes.err | 5 + src/mongo/base/string_data.h | 24 +- src/mongo/bson/timestamp.cpp | 8 + src/mongo/bson/timestamp.h | 3 + src/mongo/client/SConscript | 1 + src/mongo/client/authenticate.cpp | 6 +- src/mongo/client/authenticate.h | 2 +- src/mongo/client/authenticate_test.cpp | 8 +- src/mongo/client/connpool.cpp | 66 +- src/mongo/client/connpool.h | 28 +- src/mongo/client/dbclient.cpp | 16 +- src/mongo/client/dbclientcursor.cpp | 3 - src/mongo/client/dbclientcursor.h | 7 - src/mongo/client/mongo_uri.cpp | 19 +- src/mongo/client/mongo_uri_connect.cpp | 6 +- src/mongo/client/mongo_uri_test.cpp | 13 + src/mongo/client/native_sasl_client_session.cpp | 6 +- src/mongo/client/sasl_client_authenticate.cpp | 2 +- src/mongo/client/sasl_client_authenticate.h | 2 +- src/mongo/client/sasl_client_authenticate_impl.cpp | 7 +- src/mongo/client/sasl_client_session.h | 1 + .../client/sasl_scramsha1_client_conversation.cpp | 46 +- .../client/sasl_scramsha1_client_conversation.h | 12 +- src/mongo/client/scram_sha1_client_cache.cpp | 70 ++ src/mongo/client/scram_sha1_client_cache.h | 88 ++ src/mongo/client/syncclusterconnection.cpp | 19 +- src/mongo/client/syncclusterconnection.h | 2 +- src/mongo/crypto/SConscript | 41 +- src/mongo/crypto/crypto.h | 51 - src/mongo/crypto/crypto_openssl.cpp | 76 -- src/mongo/crypto/crypto_test.cpp | 521 --------- src/mongo/crypto/crypto_tom.cpp | 88 -- src/mongo/crypto/mechanism_scram.cpp | 292 +++-- src/mongo/crypto/mechanism_scram.h | 104 +- src/mongo/crypto/sha1_block.cpp | 88 ++ src/mongo/crypto/sha1_block.h | 113 ++ src/mongo/crypto/sha1_block_openssl.cpp | 96 ++ src/mongo/crypto/sha1_block_test.cpp | 565 ++++++++++ src/mongo/crypto/sha1_block_tom.cpp | 80 ++ src/mongo/db/SConscript | 2 +- src/mongo/db/auth/SConscript | 17 + src/mongo/db/auth/action_set.cpp | 6 + src/mongo/db/auth/action_set.h | 1 + src/mongo/db/auth/authorization_session.cpp | 7 +- src/mongo/db/auth/authorization_session.h | 26 +- .../db/auth/authorization_session_for_test.cpp | 83 ++ src/mongo/db/auth/authorization_session_for_test.h | 85 ++ src/mongo/db/auth/privilege_parser.cpp | 9 +- src/mongo/db/auth/role_graph_builtin_roles.cpp | 44 +- .../db/auth/sasl_plain_server_conversation.cpp | 18 +- .../db/auth/sasl_scramsha1_server_conversation.cpp | 55 +- src/mongo/db/auth/sasl_scramsha1_test.cpp | 501 +++++++++ src/mongo/db/catalog/apply_ops.cpp | 2 +- src/mongo/db/catalog/coll_mod.cpp | 243 ++-- src/mongo/db/catalog/coll_mod.h | 9 + src/mongo/db/catalog/collection.cpp | 16 +- src/mongo/db/catalog/collection.h | 28 +- src/mongo/db/catalog/database.cpp | 74 +- src/mongo/db/catalog/database.h | 14 +- src/mongo/db/catalog/database_holder.cpp | 98 +- src/mongo/db/catalog/database_holder.h | 9 +- src/mongo/db/catalog/index_catalog.cpp | 27 +- src/mongo/db/catalog/index_create.cpp | 38 +- src/mongo/db/catalog/rename_collection.cpp | 5 +- src/mongo/db/commands/apply_ops.cpp | 41 +- src/mongo/db/commands/find_cmd.cpp | 35 +- src/mongo/db/commands/generic.cpp | 56 + src/mongo/db/commands/getmore_cmd.cpp | 2 +- src/mongo/db/commands/list_databases.cpp | 14 +- src/mongo/db/concurrency/lock_manager.cpp | 13 +- src/mongo/db/concurrency/lock_manager_test.cpp | 61 + src/mongo/db/concurrency/lock_state_test.cpp | 82 ++ src/mongo/db/db.cpp | 60 +- src/mongo/db/db_raii.cpp | 6 +- src/mongo/db/dbcommands.cpp | 19 +- src/mongo/db/dbhelpers.cpp | 6 + src/mongo/db/exec/geo_near.cpp | 90 +- src/mongo/db/ftdc/SConscript | 23 +- src/mongo/db/ftdc/config.h | 2 +- src/mongo/db/ftdc/controller.cpp | 13 + src/mongo/db/ftdc/controller.h | 18 +- src/mongo/db/ftdc/file_writer.cpp | 18 +- src/mongo/db/ftdc/file_writer_test.cpp | 42 +- src/mongo/db/ftdc/ftdc_commands.cpp | 113 ++ src/mongo/db/ftdc/ftdc_mongod.cpp | 14 +- src/mongo/db/ftdc/ftdc_system_stats.cpp | 61 + src/mongo/db/ftdc/ftdc_system_stats.h | 64 ++ src/mongo/db/ftdc/ftdc_system_stats_freebsd.cpp | 37 + src/mongo/db/ftdc/ftdc_system_stats_linux.cpp | 127 +++ src/mongo/db/ftdc/ftdc_system_stats_openbsd.cpp | 37 + src/mongo/db/ftdc/ftdc_system_stats_osx.cpp | 37 + src/mongo/db/ftdc/ftdc_system_stats_solaris.cpp | 37 + src/mongo/db/ftdc/ftdc_system_stats_windows.cpp | 154 +++ src/mongo/db/ftdc/ftdc_test.cpp | 2 + src/mongo/db/index/index_access_method.cpp | 10 +- src/mongo/db/index/index_access_method.h | 9 +- src/mongo/db/instance.cpp | 4 +- src/mongo/db/mongod_options.cpp | 10 +- src/mongo/db/pipeline/SConscript | 1 + src/mongo/db/pipeline/document_source.h | 6 +- src/mongo/db/pipeline/pipeline.cpp | 84 +- src/mongo/db/pipeline/pipeline_test.cpp | 221 +++- src/mongo/db/query/find.cpp | 38 +- src/mongo/db/query/lite_parsed_query.cpp | 15 +- src/mongo/db/query/lite_parsed_query.h | 5 - src/mongo/db/repl/SConscript | 1 + src/mongo/db/repl/applier.h | 2 +- src/mongo/db/repl/bgsync.cpp | 334 ++++-- src/mongo/db/repl/bgsync.h | 14 +- src/mongo/db/repl/data_replicator.cpp | 6 +- src/mongo/db/repl/freshness_checker.cpp | 8 +- src/mongo/db/repl/freshness_checker_test.cpp | 6 +- src/mongo/db/repl/last_vote.cpp | 20 +- src/mongo/db/repl/last_vote.h | 11 +- src/mongo/db/repl/master_slave.cpp | 13 +- src/mongo/db/repl/oplog.cpp | 23 +- src/mongo/db/repl/oplog.h | 8 +- src/mongo/db/repl/oplogreader.cpp | 15 +- src/mongo/db/repl/oplogreader.h | 7 +- src/mongo/db/repl/repl_set_request_votes_args.cpp | 14 +- src/mongo/db/repl/repl_set_request_votes_args.h | 8 +- src/mongo/db/repl/replication_coordinator.h | 27 +- .../repl/replication_coordinator_external_state.h | 5 + ...replication_coordinator_external_state_impl.cpp | 34 +- .../replication_coordinator_external_state_impl.h | 1 + ...replication_coordinator_external_state_mock.cpp | 3 + .../replication_coordinator_external_state_mock.h | 1 + src/mongo/db/repl/replication_coordinator_impl.cpp | 233 ++-- src/mongo/db/repl/replication_coordinator_impl.h | 54 +- .../db/repl/replication_coordinator_impl_elect.cpp | 6 +- .../replication_coordinator_impl_elect_test.cpp | 1 + .../repl/replication_coordinator_impl_elect_v1.cpp | 4 +- .../replication_coordinator_impl_heartbeat.cpp | 77 +- ...lication_coordinator_impl_heartbeat_v1_test.cpp | 2 +- .../db/repl/replication_coordinator_impl_test.cpp | 722 ++++++++---- src/mongo/db/repl/replication_coordinator_mock.cpp | 14 +- src/mongo/db/repl/replication_coordinator_mock.h | 14 +- src/mongo/db/repl/replication_executor.cpp | 13 + src/mongo/db/repl/replication_executor.h | 1 + src/mongo/db/repl/replication_executor_test.cpp | 160 +++ src/mongo/db/repl/replset_commands.cpp | 21 +- src/mongo/db/repl/rs_rollback.cpp | 436 +++----- src/mongo/db/repl/rs_rollback.h | 85 +- src/mongo/db/repl/rs_rollback_test.cpp | 243 ++-- src/mongo/db/repl/sync_source_feedback.cpp | 19 +- src/mongo/db/repl/sync_tail.cpp | 73 +- src/mongo/db/repl/sync_tail.h | 17 +- src/mongo/db/repl/sync_tail_test.cpp | 284 ++++- src/mongo/db/repl/topology_coordinator.h | 26 +- src/mongo/db/repl/topology_coordinator_impl.cpp | 118 +- src/mongo/db/repl/topology_coordinator_impl.h | 17 +- .../db/repl/topology_coordinator_impl_test.cpp | 37 + src/mongo/db/repl/vote_requester.cpp | 13 +- src/mongo/db/repl/vote_requester.h | 6 +- src/mongo/db/s/migration_source_manager.cpp | 100 +- src/mongo/db/stats/SConscript | 21 + src/mongo/db/stats/fill_locker_info_test.cpp | 136 +++ src/mongo/db/storage/mmap_v1/btree/btree_logic.cpp | 4 - .../storage/mmap_v1/catalog/namespace_details.cpp | 1 - .../catalog/namespace_details_collection_entry.cpp | 24 +- .../mmap_v1/mmap_v1_database_catalog_entry.cpp | 27 +- .../mmap_v1/mmap_v1_database_catalog_entry.h | 3 + .../db/storage/mmap_v1/record_store_v1_simple.cpp | 3 +- src/mongo/db/storage/snapshot.h | 4 + .../db/storage/wiredtiger/wiredtiger_index.cpp | 72 +- src/mongo/db/storage/wiredtiger/wiredtiger_index.h | 3 + .../db/storage/wiredtiger/wiredtiger_kv_engine.cpp | 46 +- .../db/storage/wiredtiger/wiredtiger_kv_engine.h | 6 +- .../storage/wiredtiger/wiredtiger_record_store.cpp | 40 +- .../storage/wiredtiger/wiredtiger_record_store.h | 12 +- .../wiredtiger/wiredtiger_recovery_unit.cpp | 6 +- .../wiredtiger/wiredtiger_session_cache.cpp | 43 +- .../storage/wiredtiger/wiredtiger_session_cache.h | 11 +- .../db/storage/wiredtiger/wiredtiger_util.cpp | 4 +- src/mongo/db/write_concern.cpp | 5 + src/mongo/dbtests/SConscript | 1 + src/mongo/dbtests/replica_set_tests.cpp | 127 +++ src/mongo/executor/SConscript | 1 + src/mongo/executor/async_mock_stream_factory.cpp | 4 +- src/mongo/executor/async_mock_stream_factory.h | 59 + src/mongo/executor/async_stream_test.cpp | 6 +- src/mongo/executor/connection_pool.cpp | 118 +- src/mongo/executor/connection_pool.h | 14 +- src/mongo/executor/connection_pool_stats.cpp | 61 +- src/mongo/executor/connection_pool_stats.h | 19 +- src/mongo/executor/connection_pool_test.cpp | 327 +++++- .../executor/connection_pool_test_fixture.cpp | 34 +- src/mongo/executor/connection_pool_test_fixture.h | 2 + src/mongo/executor/network_interface.h | 5 + src/mongo/executor/network_interface_asio.cpp | 5 + src/mongo/executor/network_interface_asio.h | 2 + src/mongo/executor/network_interface_asio_auth.cpp | 10 +- .../executor/network_interface_asio_command.cpp | 6 +- .../network_interface_asio_integration_test.cpp | 10 +- .../executor/network_interface_asio_operation.cpp | 3 + src/mongo/executor/network_interface_asio_test.cpp | 40 + src/mongo/executor/network_interface_impl.h | 1 + src/mongo/executor/network_interface_mock.h | 2 + src/mongo/executor/thread_pool_task_executor.cpp | 23 +- src/mongo/executor/thread_pool_task_executor.h | 5 + src/mongo/installer/msi/SConscript | 2 +- src/mongo/logger/logstream_builder.cpp | 16 +- src/mongo/logger/logstream_builder.h | 9 +- src/mongo/logger/message_event_utf8_encoder.cpp | 34 +- src/mongo/logger/ramlog.cpp | 13 +- src/mongo/logger/ramlog.h | 5 + src/mongo/logger/rotatable_file_writer.cpp | 31 +- src/mongo/s/commands/SConscript | 1 + src/mongo/s/commands/cluster_ftdc_commands.cpp | 94 ++ .../s/commands/cluster_list_databases_cmd.cpp | 13 +- src/mongo/s/commands/cluster_pipeline_cmd.cpp | 4 +- src/mongo/s/query/SConscript | 1 + src/mongo/s/query/async_results_merger.cpp | 11 +- src/mongo/s/query/async_results_merger.h | 9 +- src/mongo/s/query/async_results_merger_test.cpp | 44 +- src/mongo/s/query/cluster_query_knobs.cpp | 39 + src/mongo/s/query/cluster_query_knobs.h | 40 + src/mongo/s/query/router_stage_merge.cpp | 4 + src/mongo/s/sharding_initialization.cpp | 4 + src/mongo/scripting/SConscript | 2 + src/mongo/scripting/deadline_monitor.cpp | 43 + src/mongo/scripting/deadline_monitor.h | 36 +- src/mongo/scripting/deadline_monitor_test.cpp | 1 + src/mongo/scripting/engine.cpp | 20 +- src/mongo/scripting/engine.h | 6 +- src/mongo/scripting/mozjs/implscope.cpp | 72 +- src/mongo/scripting/mozjs/implscope.h | 9 +- src/mongo/scripting/mozjs/mongo.cpp | 22 +- src/mongo/scripting/mozjs/mongo.h | 3 +- src/mongo/scripting/mozjs/objectwrapper.cpp | 117 +- src/mongo/scripting/mozjs/objectwrapper.h | 2 +- src/mongo/scripting/mozjs/proxyscope.cpp | 9 +- src/mongo/scripting/mozjs/proxyscope.h | 5 +- src/mongo/scripting/mozjs/valuewriter.cpp | 4 + src/mongo/scripting/utils.cpp | 4 + src/mongo/shell/assert.js | 29 +- src/mongo/shell/db.js | 2 +- src/mongo/shell/linenoise.cpp | 14 +- src/mongo/shell/replsettest.js | 182 ++- src/mongo/shell/shell_utils_launcher.cpp | 7 +- src/mongo/shell/types.js | 2 +- src/mongo/shell/utils.js | 10 +- src/mongo/stdx/thread.h | 35 +- src/mongo/stdx/type_traits.h | 56 + src/mongo/util/SConscript | 41 + src/mongo/util/exit_code.h | 1 + src/mongo/util/log.h | 16 + src/mongo/util/net/SConscript | 1 + src/mongo/util/net/message_port.cpp | 15 +- src/mongo/util/net/sock.cpp | 189 ++-- src/mongo/util/net/sock.h | 2 +- src/mongo/util/net/ssl_manager.cpp | 38 +- src/mongo/util/net/ssl_options.cpp | 7 + src/mongo/util/net/ssl_options.h | 2 + src/mongo/util/perfctr_collect.cpp | 505 +++++++++ src/mongo/util/perfctr_collect.h | 296 +++++ src/mongo/util/perfctr_collect_test.cpp | 416 +++++++ src/mongo/util/processinfo_linux.cpp | 2 +- src/mongo/util/procparser.cpp | 612 +++++++++++ src/mongo/util/procparser.h | 120 ++ src/mongo/util/procparser_test.cpp | 526 +++++++++ .../util/signal_handlers_synchronous_test.cpp | 2 +- .../boost-1.56.0/libs/thread/src/win32/thread.cpp | 5 +- src/third_party/mozjs-38/extract.sh | 1 + .../mozjs-38/extract/js/public/HashTable.h | 25 +- .../mozjs-38/extract/js/src/jit/CodeGenerator.cpp | 6 +- .../mozjs-38/extract/js/src/jit/ValueNumbering.cpp | 1 + .../extract/js/src/jit/none/MacroAssembler-none.h | 6 - src/third_party/mozjs-38/extract/js/src/jsapi.h | 8 - src/third_party/mozjs-38/extract/js/src/jscntxt.h | 4 +- src/third_party/mozjs-38/extract/js/src/jsobj.cpp | 5 +- .../mozjs-38/extract/js/src/jsscript.cpp | 5 +- src/third_party/mozjs-38/extract/js/src/jsscript.h | 2 +- .../mozjs-38/extract/js/src/jswatchpoint.cpp | 4 +- .../mozjs-38/extract/js/src/selfhosted.out.h | 6 +- .../mozjs-38/extract/js/src/vm/Shape.cpp | 4 +- .../mozjs-38/extract/js/src/vm/StructuredClone.cpp | 15 +- .../mozjs-38/extract/js/src/vm/TypeInference.cpp | 6 + .../mozjs-38/extract/js/src/vm/TypeInference.h | 3 + src/third_party/mozjs-38/extract/mfbt/Opaque.h | 44 + src/third_party/mozjs-38/extract/mfbt/moz.build | 1 + src/third_party/mozjs-38/get_sources.sh | 4 +- src/third_party/mozjs-38/include/js/HashTable.h | 25 +- src/third_party/mozjs-38/include/jsapi.h | 8 - src/third_party/mozjs-38/include/mozilla/Opaque.h | 44 + src/third_party/mozjs-38/mongodb.patch | 24 + .../platform/arm64/linux/build/js-confdefs.h | 6 +- .../platform/arm64/linux/include/js-config.h | 2 +- .../platform/i386/linux/build/js-confdefs.h | 6 +- .../platform/i386/linux/include/js-config.h | 2 +- .../platform/i386/openbsd/build/js-confdefs.h | 6 +- .../platform/i386/openbsd/include/js-config.h | 2 +- .../platform/i386/windows/build/js-confdefs.h | 6 +- .../platform/i386/windows/include/js-config.h | 2 +- .../platform/x86_64/freebsd/build/js-confdefs.h | 6 +- .../platform/x86_64/freebsd/include/js-config.h | 2 +- .../platform/x86_64/linux/build/js-confdefs.h | 6 +- .../platform/x86_64/linux/include/js-config.h | 2 +- .../platform/x86_64/openbsd/build/js-confdefs.h | 6 +- .../platform/x86_64/openbsd/include/js-config.h | 2 +- .../platform/x86_64/osx/build/js-confdefs.h | 6 +- .../platform/x86_64/osx/include/js-config.h | 2 +- .../platform/x86_64/solaris/build/js-confdefs.h | 6 +- .../platform/x86_64/solaris/include/js-config.h | 2 +- .../platform/x86_64/windows/build/js-confdefs.h | 6 +- .../platform/x86_64/windows/include/js-config.h | 2 +- src/third_party/wiredtiger/.gitignore | 37 +- src/third_party/wiredtiger/NEWS | 415 +++++++ src/third_party/wiredtiger/README | 6 +- src/third_party/wiredtiger/RELEASE_INFO | 4 +- src/third_party/wiredtiger/SConstruct | 3 +- .../wiredtiger/api/leveldb/basho/perf_count.cc | 9 +- src/third_party/wiredtiger/api/leveldb/hyper_wt.cc | 18 +- .../wiredtiger/api/leveldb/leveldb_wt.h | 1 + src/third_party/wiredtiger/bench/wtperf/config.c | 140 ++- .../wiredtiger/bench/wtperf/idle_table_cycle.c | 6 +- src/third_party/wiredtiger/bench/wtperf/misc.c | 4 +- .../bench/wtperf/runners/500m-btree-50r50u.wtperf | 2 +- .../bench/wtperf/runners/500m-btree-80r20u.wtperf | 2 +- .../wtperf/runners/500m-btree-populate.wtperf | 2 +- .../bench/wtperf/runners/500m-btree-rdonly.wtperf | 2 +- .../bench/wtperf/runners/checkpoint-stress.wtperf | 2 +- .../bench/wtperf/runners/evict-btree-1.wtperf | 2 +- .../wtperf/runners/evict-btree-readonly.wtperf | 2 +- .../wtperf/runners/evict-btree-stress-multi.wtperf | 2 +- .../bench/wtperf/runners/evict-btree-stress.wtperf | 2 +- .../bench/wtperf/runners/evict-btree.wtperf | 2 +- .../bench/wtperf/runners/evict-lsm-1.wtperf | 2 +- .../bench/wtperf/runners/evict-lsm-readonly.wtperf | 2 +- .../bench/wtperf/runners/evict-lsm.wtperf | 2 +- .../wiredtiger/bench/wtperf/runners/log.wtperf | 2 +- .../bench/wtperf/runners/many-table-stress.wtperf | 19 + .../wtperf/runners/mongodb-secondary-apply.wtperf | 2 +- .../runners/multi-btree-read-heavy-stress.wtperf | 2 +- .../bench/wtperf/runners/multi-btree-stress.wtperf | 2 +- .../runners/multi-btree-zipfian-populate.wtperf | 2 +- .../runners/multi-btree-zipfian-workload.wtperf | 2 +- .../wiredtiger/bench/wtperf/runners/wtperf_run.sh | 5 + .../bench/wtperf/stress/btree-split-stress.wtperf | 5 +- src/third_party/wiredtiger/bench/wtperf/track.c | 4 +- src/third_party/wiredtiger/bench/wtperf/wtperf.c | 346 ++++-- src/third_party/wiredtiger/bench/wtperf/wtperf.h | 4 + .../wiredtiger/bench/wtperf/wtperf_opt.i | 12 +- src/third_party/wiredtiger/build_posix/Make.base | 2 + .../wiredtiger/build_posix/Make.subdirs | 2 + .../build_posix/aclocal/ax_check_compile_flag.m4 | 74 ++ .../wiredtiger/build_posix/aclocal/options.m4 | 2 +- .../wiredtiger/build_posix/aclocal/strict.m4 | 5 + .../wiredtiger/build_posix/aclocal/version-set.m4 | 8 +- .../wiredtiger/build_posix/aclocal/version.m4 | 2 +- .../wiredtiger/build_posix/configure.ac.in | 65 +- src/third_party/wiredtiger/dist/api_config.py | 3 +- src/third_party/wiredtiger/dist/api_data.py | 62 +- src/third_party/wiredtiger/dist/api_err.py | 2 +- src/third_party/wiredtiger/dist/filelist | 4 +- src/third_party/wiredtiger/dist/flags.py | 13 +- .../wiredtiger/dist/package/wiredtiger.spec | 2 +- src/third_party/wiredtiger/dist/s_all | 28 +- src/third_party/wiredtiger/dist/s_define.list | 12 +- src/third_party/wiredtiger/dist/s_docs | 3 +- src/third_party/wiredtiger/dist/s_funcs.list | 1 - src/third_party/wiredtiger/dist/s_prototypes | 2 +- src/third_party/wiredtiger/dist/s_python | 4 +- src/third_party/wiredtiger/dist/s_stat | 3 - src/third_party/wiredtiger/dist/s_string.ok | 44 +- src/third_party/wiredtiger/dist/s_style | 19 +- src/third_party/wiredtiger/dist/s_void | 60 +- src/third_party/wiredtiger/dist/stat_data.py | 15 +- src/third_party/wiredtiger/examples/c/Makefile.am | 1 - src/third_party/wiredtiger/examples/c/ex_all.c | 10 +- src/third_party/wiredtiger/examples/c/ex_async.c | 6 +- src/third_party/wiredtiger/examples/c/ex_backup.c | 28 +- .../wiredtiger/examples/c/ex_data_source.c | 16 + src/third_party/wiredtiger/examples/c/ex_encrypt.c | 4 +- .../wiredtiger/examples/c/ex_file_system.c | 159 +-- src/third_party/wiredtiger/examples/c/ex_log.c | 12 +- src/third_party/wiredtiger/examples/c/ex_scope.c | 217 ---- src/third_party/wiredtiger/examples/c/ex_sync.c | 20 +- src/third_party/wiredtiger/examples/c/ex_thread.c | 7 +- .../java/com/wiredtiger/examples/ex_log.java | 75 +- .../ext/compressors/zlib/zlib_compress.c | 57 +- .../ext/compressors/zstd/zstd_compress.c | 37 +- .../wiredtiger/ext/datasources/helium/helium.c | 11 +- .../wiredtiger/ext/encryptors/rotn/rotn_encrypt.c | 42 +- .../wiredtiger/ext/test/fail_fs/Makefile.am | 9 + .../wiredtiger/ext/test/fail_fs/fail_fs.c | 847 ++++++++++++++ .../wiredtiger/ext/test/kvs_bdb/kvs_bdb.c | 3 +- src/third_party/wiredtiger/import.data | 6 + src/third_party/wiredtiger/lang/java/Makefile.am | 1 + src/third_party/wiredtiger/lang/java/java_doc.i | 1 + .../src/com/wiredtiger/db/PackOutputStream.java | 23 +- src/third_party/wiredtiger/lang/java/wiredtiger.i | 78 +- src/third_party/wiredtiger/lang/python/Makefile.am | 3 +- src/third_party/wiredtiger/lang/python/setup.py | 5 +- .../wiredtiger/lang/python/setup_pip.py | 408 +++++++ .../wiredtiger/lang/python/wiredtiger/pip_init.py | 48 + src/third_party/wiredtiger/src/async/async_api.c | 17 +- src/third_party/wiredtiger/src/async/async_op.c | 3 +- .../wiredtiger/src/async/async_worker.c | 2 +- src/third_party/wiredtiger/src/block/block_addr.c | 3 +- src/third_party/wiredtiger/src/block/block_ckpt.c | 1 - .../wiredtiger/src/block/block_compact.c | 22 +- src/third_party/wiredtiger/src/block/block_ext.c | 15 +- src/third_party/wiredtiger/src/block/block_open.c | 6 + src/third_party/wiredtiger/src/block/block_read.c | 4 +- src/third_party/wiredtiger/src/block/block_slvg.c | 1 - src/third_party/wiredtiger/src/block/block_vrfy.c | 2 +- src/third_party/wiredtiger/src/block/block_write.c | 4 +- src/third_party/wiredtiger/src/bloom/bloom.c | 11 +- src/third_party/wiredtiger/src/btree/bt_compact.c | 17 +- src/third_party/wiredtiger/src/btree/bt_curnext.c | 15 +- src/third_party/wiredtiger/src/btree/bt_curprev.c | 19 +- src/third_party/wiredtiger/src/btree/bt_cursor.c | 707 +++++++----- src/third_party/wiredtiger/src/btree/bt_debug.c | 63 +- src/third_party/wiredtiger/src/btree/bt_delete.c | 7 +- src/third_party/wiredtiger/src/btree/bt_discard.c | 94 +- src/third_party/wiredtiger/src/btree/bt_handle.c | 181 ++- src/third_party/wiredtiger/src/btree/bt_io.c | 9 +- src/third_party/wiredtiger/src/btree/bt_misc.c | 1 + src/third_party/wiredtiger/src/btree/bt_ovfl.c | 8 +- src/third_party/wiredtiger/src/btree/bt_page.c | 28 +- src/third_party/wiredtiger/src/btree/bt_random.c | 432 ++++++++ src/third_party/wiredtiger/src/btree/bt_read.c | 17 +- .../wiredtiger/src/btree/bt_rebalance.c | 17 +- src/third_party/wiredtiger/src/btree/bt_ret.c | 173 +-- src/third_party/wiredtiger/src/btree/bt_slvg.c | 47 +- src/third_party/wiredtiger/src/btree/bt_split.c | 336 +++--- src/third_party/wiredtiger/src/btree/bt_stat.c | 5 +- src/third_party/wiredtiger/src/btree/bt_sync.c | 20 +- src/third_party/wiredtiger/src/btree/bt_vrfy.c | 46 +- src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c | 8 +- src/third_party/wiredtiger/src/btree/bt_walk.c | 113 +- src/third_party/wiredtiger/src/btree/col_modify.c | 14 +- src/third_party/wiredtiger/src/btree/col_srch.c | 7 +- src/third_party/wiredtiger/src/btree/row_key.c | 12 +- src/third_party/wiredtiger/src/btree/row_modify.c | 31 +- src/third_party/wiredtiger/src/btree/row_srch.c | 230 +--- src/third_party/wiredtiger/src/cache/cache_las.c | 5 +- .../wiredtiger/src/checksum/power8/crc32_wrapper.c | 4 +- .../wiredtiger/src/checksum/software/checksum.c | 1 + .../wiredtiger/src/checksum/zseries/crc32-s390x.c | 26 +- src/third_party/wiredtiger/src/config/config.c | 2 +- src/third_party/wiredtiger/src/config/config_api.c | 41 +- .../wiredtiger/src/config/config_collapse.c | 1 + src/third_party/wiredtiger/src/config/config_def.c | 323 +++--- src/third_party/wiredtiger/src/config/config_ext.c | 77 +- src/third_party/wiredtiger/src/conn/conn_api.c | 130 ++- src/third_party/wiredtiger/src/conn/conn_cache.c | 9 +- .../wiredtiger/src/conn/conn_cache_pool.c | 68 +- src/third_party/wiredtiger/src/conn/conn_ckpt.c | 28 +- src/third_party/wiredtiger/src/conn/conn_dhandle.c | 182 +-- src/third_party/wiredtiger/src/conn/conn_handle.c | 34 +- src/third_party/wiredtiger/src/conn/conn_log.c | 88 +- src/third_party/wiredtiger/src/conn/conn_open.c | 61 +- src/third_party/wiredtiger/src/conn/conn_stat.c | 35 +- src/third_party/wiredtiger/src/conn/conn_sweep.c | 44 +- src/third_party/wiredtiger/src/cursor/cur_backup.c | 28 +- src/third_party/wiredtiger/src/cursor/cur_config.c | 2 - src/third_party/wiredtiger/src/cursor/cur_ds.c | 20 +- src/third_party/wiredtiger/src/cursor/cur_file.c | 169 +-- src/third_party/wiredtiger/src/cursor/cur_index.c | 55 +- src/third_party/wiredtiger/src/cursor/cur_join.c | 39 +- src/third_party/wiredtiger/src/cursor/cur_json.c | 186 ++-- src/third_party/wiredtiger/src/cursor/cur_log.c | 66 +- .../wiredtiger/src/cursor/cur_metadata.c | 4 +- src/third_party/wiredtiger/src/cursor/cur_stat.c | 5 +- src/third_party/wiredtiger/src/cursor/cur_std.c | 8 +- src/third_party/wiredtiger/src/cursor/cur_table.c | 71 +- src/third_party/wiredtiger/src/docs/Doxyfile | 8 + .../wiredtiger/src/docs/command-line.dox | 46 + src/third_party/wiredtiger/src/docs/cursor-ops.dox | 4 - .../wiredtiger/src/docs/cursor-random.dox | 5 - .../wiredtiger/src/docs/file-formats.dox | 6 +- .../wiredtiger/src/docs/programming.dox | 3 +- src/third_party/wiredtiger/src/docs/spell.ok | 2 + src/third_party/wiredtiger/src/docs/top/main.dox | 12 +- .../wiredtiger/src/docs/transactions.dox | 2 +- .../wiredtiger/src/docs/tune-compression.dox | 62 -- .../src/docs/tune-page-size-and-comp.dox | 426 +++++++ .../wiredtiger/src/docs/tune-page-sizes.dox | 142 --- src/third_party/wiredtiger/src/docs/upgrading.dox | 93 +- src/third_party/wiredtiger/src/docs/wtperf.dox | 8 +- src/third_party/wiredtiger/src/evict/evict_file.c | 23 +- src/third_party/wiredtiger/src/evict/evict_lru.c | 910 ++++++++++----- src/third_party/wiredtiger/src/evict/evict_page.c | 76 +- src/third_party/wiredtiger/src/evict/evict_stat.c | 2 +- src/third_party/wiredtiger/src/include/api.h | 48 +- src/third_party/wiredtiger/src/include/bitstring.i | 2 +- src/third_party/wiredtiger/src/include/btmem.h | 162 ++- src/third_party/wiredtiger/src/include/btree.h | 53 +- src/third_party/wiredtiger/src/include/btree.i | 155 +-- src/third_party/wiredtiger/src/include/buf.i | 37 +- src/third_party/wiredtiger/src/include/cache.h | 17 +- src/third_party/wiredtiger/src/include/cache.i | 17 +- src/third_party/wiredtiger/src/include/cell.i | 16 +- src/third_party/wiredtiger/src/include/column.i | 27 +- src/third_party/wiredtiger/src/include/compact.h | 4 +- src/third_party/wiredtiger/src/include/config.h | 67 +- .../wiredtiger/src/include/connection.h | 50 +- src/third_party/wiredtiger/src/include/cursor.h | 12 +- src/third_party/wiredtiger/src/include/cursor.i | 62 +- src/third_party/wiredtiger/src/include/dhandle.h | 22 +- src/third_party/wiredtiger/src/include/error.h | 6 +- src/third_party/wiredtiger/src/include/extern.h | 153 +-- .../wiredtiger/src/include/extern_posix.h | 21 +- .../wiredtiger/src/include/extern_win.h | 9 +- src/third_party/wiredtiger/src/include/flags.h | 93 +- src/third_party/wiredtiger/src/include/gcc.h | 2 - src/third_party/wiredtiger/src/include/hardware.h | 20 +- src/third_party/wiredtiger/src/include/intpack.i | 30 +- src/third_party/wiredtiger/src/include/lint.h | 16 +- src/third_party/wiredtiger/src/include/log.h | 26 +- src/third_party/wiredtiger/src/include/lsm.h | 36 +- src/third_party/wiredtiger/src/include/meta.h | 7 +- src/third_party/wiredtiger/src/include/misc.h | 22 +- src/third_party/wiredtiger/src/include/misc.i | 106 +- src/third_party/wiredtiger/src/include/msvc.h | 2 - src/third_party/wiredtiger/src/include/mutex.h | 73 +- src/third_party/wiredtiger/src/include/mutex.i | 42 +- src/third_party/wiredtiger/src/include/os.h | 10 +- .../wiredtiger/src/include/os_windows.h | 22 - src/third_party/wiredtiger/src/include/packing.i | 57 +- src/third_party/wiredtiger/src/include/schema.h | 211 +++- src/third_party/wiredtiger/src/include/serial.i | 30 +- src/third_party/wiredtiger/src/include/session.h | 49 +- src/third_party/wiredtiger/src/include/stat.h | 17 +- .../wiredtiger/src/include/thread_group.h | 2 +- src/third_party/wiredtiger/src/include/txn.h | 25 +- src/third_party/wiredtiger/src/include/txn.i | 23 +- .../wiredtiger/src/include/verify_build.h | 22 +- .../wiredtiger/src/include/wiredtiger.in | 691 +++++++----- .../wiredtiger/src/include/wiredtiger_ext.h | 36 +- .../wiredtiger/src/include/wt_internal.h | 6 +- src/third_party/wiredtiger/src/log/log.c | 132 ++- src/third_party/wiredtiger/src/log/log_slot.c | 394 ++++--- src/third_party/wiredtiger/src/lsm/lsm_cursor.c | 147 ++- .../wiredtiger/src/lsm/lsm_cursor_bulk.c | 4 +- src/third_party/wiredtiger/src/lsm/lsm_manager.c | 68 +- src/third_party/wiredtiger/src/lsm/lsm_merge.c | 20 +- src/third_party/wiredtiger/src/lsm/lsm_meta.c | 13 +- src/third_party/wiredtiger/src/lsm/lsm_stat.c | 14 +- src/third_party/wiredtiger/src/lsm/lsm_tree.c | 136 ++- src/third_party/wiredtiger/src/lsm/lsm_work_unit.c | 79 +- src/third_party/wiredtiger/src/lsm/lsm_worker.c | 25 +- src/third_party/wiredtiger/src/meta/meta_apply.c | 6 +- src/third_party/wiredtiger/src/meta/meta_ckpt.c | 10 +- src/third_party/wiredtiger/src/meta/meta_ext.c | 4 +- src/third_party/wiredtiger/src/meta/meta_table.c | 18 +- src/third_party/wiredtiger/src/meta/meta_track.c | 2 +- src/third_party/wiredtiger/src/meta/meta_turtle.c | 41 +- .../wiredtiger/src/os_common/filename.c | 19 +- .../wiredtiger/src/os_common/os_alloc.c | 2 + .../wiredtiger/src/os_common/os_errno.c | 2 +- .../wiredtiger/src/os_common/os_fs_inmemory.c | 2 - .../wiredtiger/src/os_common/os_fstream.c | 2 +- .../wiredtiger/src/os_common/os_getopt.c | 1 + .../wiredtiger/src/os_common/os_strtouq.c | 1 + src/third_party/wiredtiger/src/os_posix/os_dir.c | 2 - src/third_party/wiredtiger/src/os_posix/os_fs.c | 19 +- .../wiredtiger/src/os_posix/os_getenv.c | 1 + .../wiredtiger/src/os_posix/os_mtx_cond.c | 53 +- .../wiredtiger/src/os_posix/os_setvbuf.c | 2 + src/third_party/wiredtiger/src/os_posix/os_sleep.c | 1 + .../wiredtiger/src/os_posix/os_snprintf.c | 27 + .../wiredtiger/src/os_posix/os_thread.c | 25 +- src/third_party/wiredtiger/src/os_posix/os_time.c | 1 + src/third_party/wiredtiger/src/os_posix/os_yield.c | 9 + src/third_party/wiredtiger/src/os_win/os_dir.c | 2 - src/third_party/wiredtiger/src/os_win/os_fs.c | 36 +- .../wiredtiger/src/os_win/os_mtx_cond.c | 49 +- .../wiredtiger/src/os_win/os_snprintf.c | 50 +- src/third_party/wiredtiger/src/os_win/os_thread.c | 20 +- src/third_party/wiredtiger/src/os_win/os_utf8.c | 2 - .../wiredtiger/src/os_win/os_vsnprintf.c | 41 - src/third_party/wiredtiger/src/os_win/os_yield.c | 8 + .../wiredtiger/src/reconcile/rec_track.c | 20 +- .../wiredtiger/src/reconcile/rec_write.c | 1161 +++++++++++--------- .../wiredtiger/src/schema/schema_alter.c | 180 +++ .../wiredtiger/src/schema/schema_create.c | 5 +- .../wiredtiger/src/schema/schema_drop.c | 8 +- .../wiredtiger/src/schema/schema_list.c | 2 +- .../wiredtiger/src/schema/schema_rename.c | 2 +- .../wiredtiger/src/schema/schema_util.c | 6 +- .../wiredtiger/src/schema/schema_worker.c | 20 +- .../wiredtiger/src/session/session_api.c | 250 +++-- .../wiredtiger/src/session/session_compact.c | 246 +++-- .../wiredtiger/src/session/session_dhandle.c | 115 +- .../wiredtiger/src/session/session_salvage.c | 2 +- src/third_party/wiredtiger/src/support/cond_auto.c | 80 +- src/third_party/wiredtiger/src/support/crypto.c | 1 - src/third_party/wiredtiger/src/support/err.c | 123 ++- src/third_party/wiredtiger/src/support/hazard.c | 314 ++++-- src/third_party/wiredtiger/src/support/hex.c | 10 +- src/third_party/wiredtiger/src/support/mtx_rw.c | 499 +++++---- src/third_party/wiredtiger/src/support/rand.c | 15 + src/third_party/wiredtiger/src/support/scratch.c | 9 +- src/third_party/wiredtiger/src/support/stat.c | 60 +- .../wiredtiger/src/support/thread_group.c | 89 +- src/third_party/wiredtiger/src/txn/txn.c | 188 +++- src/third_party/wiredtiger/src/txn/txn_ckpt.c | 547 +++++---- src/third_party/wiredtiger/src/txn/txn_log.c | 21 +- src/third_party/wiredtiger/src/txn/txn_nsnap.c | 49 +- src/third_party/wiredtiger/src/txn/txn_recover.c | 6 +- src/third_party/wiredtiger/src/utilities/util.h | 4 +- .../wiredtiger/src/utilities/util_alter.c | 53 + .../wiredtiger/src/utilities/util_backup.c | 15 +- .../wiredtiger/src/utilities/util_compact.c | 14 +- .../wiredtiger/src/utilities/util_create.c | 12 +- .../wiredtiger/src/utilities/util_drop.c | 10 +- .../wiredtiger/src/utilities/util_dump.c | 242 +++- .../wiredtiger/src/utilities/util_list.c | 21 +- .../wiredtiger/src/utilities/util_load.c | 17 +- .../wiredtiger/src/utilities/util_load_json.c | 27 +- .../wiredtiger/src/utilities/util_loadtext.c | 13 +- .../wiredtiger/src/utilities/util_main.c | 142 ++- .../wiredtiger/src/utilities/util_misc.c | 5 +- .../wiredtiger/src/utilities/util_printlog.c | 14 +- .../wiredtiger/src/utilities/util_read.c | 19 +- .../wiredtiger/src/utilities/util_rebalance.c | 30 +- .../wiredtiger/src/utilities/util_rename.c | 15 +- .../wiredtiger/src/utilities/util_salvage.c | 30 +- .../wiredtiger/src/utilities/util_stat.c | 11 +- .../wiredtiger/src/utilities/util_truncate.c | 52 + .../wiredtiger/src/utilities/util_upgrade.c | 30 +- .../wiredtiger/src/utilities/util_verify.c | 41 +- .../wiredtiger/src/utilities/util_write.c | 28 +- src/third_party/wiredtiger/test/bloom/test_bloom.c | 15 +- .../wiredtiger/test/checkpoint/checkpointer.c | 26 +- .../wiredtiger/test/checkpoint/smoke.sh | 4 +- .../wiredtiger/test/checkpoint/test_checkpoint.c | 21 +- .../wiredtiger/test/checkpoint/test_checkpoint.h | 1 - .../wiredtiger/test/checkpoint/workers.c | 22 +- src/third_party/wiredtiger/test/csuite/Makefile.am | 22 +- .../wiredtiger/test/csuite/rwlock/main.c | 184 ++++ .../wiredtiger/test/csuite/scope/main.c | 288 +++++ .../test/csuite/wt1965_col_efficiency/main.c | 3 +- .../test/csuite/wt2246_col_append/main.c | 17 +- .../test/csuite/wt2323_join_visibility/main.c | 39 +- .../test/csuite/wt2447_join_main_table/main.c | 12 +- .../test/csuite/wt2535_insert_race/main.c | 5 +- .../test/csuite/wt2592_join_schema/main.c | 9 +- .../test/csuite/wt2834_join_bloom_fix/main.c | 26 +- .../wiredtiger/test/csuite/wt2853_perf/main.c | 21 +- .../test/csuite/wt2909_checkpoint_integrity/main.c | 669 +++++++++++ .../wiredtiger/test/csuite/wt3120_filesys/main.c | 99 ++ .../test/csuite/wt3135_search_near_collator/main.c | 360 ++++++ .../test/csuite/wt3184_dup_index_collator/main.c | 168 +++ .../wiredtiger/test/cursor_order/cursor_order.c | 14 +- .../test/cursor_order/cursor_order_file.c | 30 +- .../test/cursor_order/cursor_order_ops.c | 20 +- src/third_party/wiredtiger/test/fops/file.c | 33 +- src/third_party/wiredtiger/test/fops/t.c | 21 +- src/third_party/wiredtiger/test/format/backup.c | 4 +- src/third_party/wiredtiger/test/format/config.c | 190 +++- src/third_party/wiredtiger/test/format/config.h | 24 +- src/third_party/wiredtiger/test/format/format.h | 7 +- src/third_party/wiredtiger/test/format/ops.c | 445 +++++--- src/third_party/wiredtiger/test/format/rebalance.c | 16 +- src/third_party/wiredtiger/test/format/salvage.c | 25 +- src/third_party/wiredtiger/test/format/t.c | 15 +- src/third_party/wiredtiger/test/format/util.c | 91 +- src/third_party/wiredtiger/test/format/wts.c | 145 ++- src/third_party/wiredtiger/test/huge/huge.c | 7 +- .../java/com/wiredtiger/test/CursorTest03.java | 175 +++ .../java/com/wiredtiger/test/WiredTigerSuite.java | 1 + src/third_party/wiredtiger/test/manydbs/manydbs.c | 10 +- src/third_party/wiredtiger/test/mciproject.yml | 2 +- .../wiredtiger/test/readonly/readonly.c | 52 +- .../wiredtiger/test/recovery/random-abort.c | 54 +- .../wiredtiger/test/recovery/truncated-log.c | 24 +- src/third_party/wiredtiger/test/salvage/salvage.c | 33 +- src/third_party/wiredtiger/test/suite/run.py | 3 +- .../wiredtiger/test/suite/test_alter01.py | 169 +++ .../wiredtiger/test/suite/test_async01.py | 2 +- .../wiredtiger/test/suite/test_async02.py | 2 +- .../wiredtiger/test/suite/test_backup03.py | 2 +- .../wiredtiger/test/suite/test_backup04.py | 2 +- .../wiredtiger/test/suite/test_bug011.py | 17 +- .../wiredtiger/test/suite/test_collator.py | 32 +- .../wiredtiger/test/suite/test_compact02.py | 24 +- .../wiredtiger/test/suite/test_compress01.py | 20 +- .../wiredtiger/test/suite/test_config03.py | 12 +- .../wiredtiger/test/suite/test_config04.py | 4 - .../wiredtiger/test/suite/test_cursor01.py | 1 + .../wiredtiger/test/suite/test_cursor07.py | 2 +- .../wiredtiger/test/suite/test_cursor08.py | 20 +- .../wiredtiger/test/suite/test_cursor10.py | 127 +++ .../wiredtiger/test/suite/test_cursor11.py | 159 +++ .../wiredtiger/test/suite/test_cursor_random.py | 49 + src/third_party/wiredtiger/test/suite/test_dump.py | 54 +- .../wiredtiger/test/suite/test_encrypt01.py | 39 +- .../wiredtiger/test/suite/test_encrypt02.py | 48 +- .../wiredtiger/test/suite/test_encrypt03.py | 37 +- .../wiredtiger/test/suite/test_encrypt04.py | 45 +- .../wiredtiger/test/suite/test_encrypt05.py | 39 +- .../wiredtiger/test/suite/test_encrypt06.py | 39 +- .../wiredtiger/test/suite/test_encrypt07.py | 35 +- .../wiredtiger/test/suite/test_hazard.py | 58 + .../wiredtiger/test/suite/test_index02.py | 71 +- .../wiredtiger/test/suite/test_inmem01.py | 51 +- .../wiredtiger/test/suite/test_intpack.py | 4 +- .../wiredtiger/test/suite/test_join01.py | 11 +- .../wiredtiger/test/suite/test_join03.py | 30 +- .../wiredtiger/test/suite/test_join04.py | 30 +- .../wiredtiger/test/suite/test_join07.py | 30 +- .../wiredtiger/test/suite/test_jsondump02.py | 18 + .../wiredtiger/test/suite/test_lsm01.py | 2 +- .../wiredtiger/test/suite/test_nsnap04.py | 37 +- .../wiredtiger/test/suite/test_overwrite.py | 67 +- .../wiredtiger/test/suite/test_perf001.py | 3 +- .../wiredtiger/test/suite/test_readonly01.py | 3 +- .../wiredtiger/test/suite/test_reconfig01.py | 12 + .../wiredtiger/test/suite/test_reconfig02.py | 3 +- .../wiredtiger/test/suite/test_reconfig04.py | 45 + .../wiredtiger/test/suite/test_schema05.py | 30 +- .../wiredtiger/test/suite/test_schema07.py | 3 +- .../wiredtiger/test/suite/test_shared_cache01.py | 2 +- .../wiredtiger/test/suite/test_shared_cache02.py | 2 +- .../wiredtiger/test/suite/test_stat02.py | 2 +- .../wiredtiger/test/suite/test_sweep01.py | 7 +- .../wiredtiger/test/suite/test_truncate01.py | 10 +- .../wiredtiger/test/suite/test_truncate02.py | 3 +- .../wiredtiger/test/suite/test_txn02.py | 26 +- .../wiredtiger/test/suite/test_txn04.py | 22 +- .../wiredtiger/test/suite/test_txn05.py | 28 +- .../wiredtiger/test/suite/test_txn06.py | 4 +- .../wiredtiger/test/suite/test_txn07.py | 56 +- .../wiredtiger/test/suite/test_txn08.py | 2 +- .../wiredtiger/test/suite/test_txn09.py | 17 +- .../wiredtiger/test/suite/test_txn11.py | 2 +- .../wiredtiger/test/suite/test_txn13.py | 2 +- .../wiredtiger/test/suite/test_txn15.py | 2 +- .../wiredtiger/test/suite/test_util14.py | 92 ++ .../wiredtiger/test/suite/test_util15.py | 71 ++ .../wiredtiger/test/suite/test_util16.py | 71 ++ .../wiredtiger/test/suite/test_util17.py | 57 + src/third_party/wiredtiger/test/suite/wtdataset.py | 96 +- src/third_party/wiredtiger/test/suite/wttest.py | 87 +- .../wiredtiger/test/syscall/Makefile.am | 7 + src/third_party/wiredtiger/test/syscall/syscall.py | 857 +++++++++++++++ .../wiredtiger/test/syscall/wt2336_base/base.run | 180 +++ .../wiredtiger/test/syscall/wt2336_base/main.c | 69 ++ src/third_party/wiredtiger/test/thread/file.c | 35 +- src/third_party/wiredtiger/test/thread/rw.c | 41 +- src/third_party/wiredtiger/test/thread/stats.c | 3 +- src/third_party/wiredtiger/test/thread/t.c | 14 +- src/third_party/wiredtiger/test/utility/misc.c | 42 +- .../wiredtiger/test/utility/parse_opts.c | 11 +- .../wiredtiger/test/utility/test_util.h | 9 +- src/third_party/wiredtiger/test/utility/thread.c | 7 +- .../wiredtiger/test/windows/windows_shim.h | 11 +- .../wiredtiger/test/wtperf/test_conf_dump.py | 296 +++++ .../wiredtiger/tools/wtstats/stat_data.py | 10 + .../wiredtiger/tools/wtstats/wtstats.html.template | 27 +- version.json | 4 +- 912 files changed, 36192 insertions(+), 13814 deletions(-) create mode 100644 buildscripts/package_test/.kitchen.yml create mode 100644 buildscripts/package_test/metadata.rb create mode 100644 buildscripts/package_test/recipes/install_mongodb.rb create mode 100644 buildscripts/package_test/test/recipes/service/install_mongodb_spec.rb create mode 100644 buildscripts/resmokeconfig/suites/multiversion_auth.yml create mode 100644 buildscripts/resmokelib/reportfile.py create mode 100644 buildscripts/resmokelib/sighandler.py create mode 100644 etc/scons/gold_incremental_link.vars create mode 100644 etc/scons/mongodbtoolchain_clang.vars create mode 100644 etc/scons/mongodbtoolchain_gcc.vars create mode 100644 jstests/concurrency/fsm_workloads/create_database.js create mode 100644 jstests/concurrency/fsm_workloads/sharded_moveChunk_drop_shard_key_index.js create mode 100644 jstests/core/diagdata.js delete mode 100644 jstests/core/drop2.js create mode 100644 jstests/core/evalh.js create mode 100644 jstests/core/evalj.js create mode 100644 jstests/core/killop_drop_collection.js create mode 100644 jstests/core/list_indexes_invalidation.js create mode 100644 jstests/core/list_namespaces_invalidation.js create mode 100644 jstests/core/server22053.js create mode 100644 jstests/core/server25192.js create mode 100644 jstests/libs/check_log.js delete mode 100644 jstests/libs/election_timing_test.js create mode 100644 jstests/libs/write_concern_util.js create mode 100644 jstests/noPassthrough/read_majority_reads.js create mode 100644 jstests/noPassthrough/shell_history.js create mode 100644 jstests/replsets/do_not_sync_from_stale_sync_source.js create mode 100644 jstests/replsets/double_rollback_early.js delete mode 100644 jstests/replsets/election_timing.js create mode 100644 jstests/replsets/last_vote.js create mode 100644 jstests/replsets/libs/tags.js create mode 100644 jstests/replsets/no_flapping_during_network_partition.js create mode 100644 jstests/replsets/oplog_replay_on_startup_update_and_delete.js create mode 100644 jstests/replsets/read_committed_stale_history.js create mode 100644 jstests/replsets/read_committed_with_catalog_changes.js create mode 100644 jstests/replsets/request_primary_stepdown.js create mode 100644 jstests/replsets/rollback_after_sync_source_selection.js delete mode 100644 jstests/replsets/rollback_empty_ns.js delete mode 100644 jstests/replsets/rollback_empty_o.js delete mode 100644 jstests/replsets/rollback_empty_o2.js create mode 100644 jstests/replsets/rollback_with_socket_error_then_steady_state.js create mode 100644 jstests/replsets/shutdown_primary.js create mode 100644 jstests/replsets/stepdown_needs_electable_secondary.js create mode 100644 jstests/replsets/stepdown_needs_majority.js delete mode 100644 jstests/replsets/stepdown_wrt_electable.js create mode 100644 jstests/replsets/temp_namespace_restart_as_standalone.js create mode 100644 jstests/replsets/write_concern_after_stepdown.js create mode 100644 jstests/replsets/write_concern_after_stepdown_and_stepup.js create mode 100644 jstests/sharding/migration_critical_section_concurrency.js create mode 100644 jstests/sharding/sync_cluster_config/sccc_read_pref.js create mode 100644 site_scons/site_tools/incremental_link.py create mode 100644 site_scons/site_tools/split_dwarf.py create mode 100644 src/mongo/client/scram_sha1_client_cache.cpp create mode 100644 src/mongo/client/scram_sha1_client_cache.h delete mode 100644 src/mongo/crypto/crypto.h delete mode 100644 src/mongo/crypto/crypto_openssl.cpp delete mode 100644 src/mongo/crypto/crypto_test.cpp delete mode 100644 src/mongo/crypto/crypto_tom.cpp create mode 100644 src/mongo/crypto/sha1_block.cpp create mode 100644 src/mongo/crypto/sha1_block.h create mode 100644 src/mongo/crypto/sha1_block_openssl.cpp create mode 100644 src/mongo/crypto/sha1_block_test.cpp create mode 100644 src/mongo/crypto/sha1_block_tom.cpp create mode 100644 src/mongo/db/auth/authorization_session_for_test.cpp create mode 100644 src/mongo/db/auth/authorization_session_for_test.h create mode 100644 src/mongo/db/auth/sasl_scramsha1_test.cpp create mode 100644 src/mongo/db/ftdc/ftdc_commands.cpp create mode 100644 src/mongo/db/ftdc/ftdc_system_stats.cpp create mode 100644 src/mongo/db/ftdc/ftdc_system_stats.h create mode 100644 src/mongo/db/ftdc/ftdc_system_stats_freebsd.cpp create mode 100644 src/mongo/db/ftdc/ftdc_system_stats_linux.cpp create mode 100644 src/mongo/db/ftdc/ftdc_system_stats_openbsd.cpp create mode 100644 src/mongo/db/ftdc/ftdc_system_stats_osx.cpp create mode 100644 src/mongo/db/ftdc/ftdc_system_stats_solaris.cpp create mode 100644 src/mongo/db/ftdc/ftdc_system_stats_windows.cpp create mode 100644 src/mongo/db/stats/fill_locker_info_test.cpp create mode 100644 src/mongo/dbtests/replica_set_tests.cpp create mode 100644 src/mongo/s/commands/cluster_ftdc_commands.cpp create mode 100644 src/mongo/s/query/cluster_query_knobs.cpp create mode 100644 src/mongo/s/query/cluster_query_knobs.h create mode 100644 src/mongo/scripting/deadline_monitor.cpp create mode 100644 src/mongo/stdx/type_traits.h create mode 100644 src/mongo/util/perfctr_collect.cpp create mode 100644 src/mongo/util/perfctr_collect.h create mode 100644 src/mongo/util/perfctr_collect_test.cpp create mode 100644 src/mongo/util/procparser.cpp create mode 100644 src/mongo/util/procparser.h create mode 100644 src/mongo/util/procparser_test.cpp create mode 100644 src/third_party/mozjs-38/extract/mfbt/Opaque.h create mode 100644 src/third_party/mozjs-38/include/mozilla/Opaque.h create mode 100644 src/third_party/wiredtiger/bench/wtperf/runners/many-table-stress.wtperf create mode 100644 src/third_party/wiredtiger/build_posix/aclocal/ax_check_compile_flag.m4 mode change 100644 => 100755 src/third_party/wiredtiger/dist/s_void delete mode 100644 src/third_party/wiredtiger/examples/c/ex_scope.c create mode 100644 src/third_party/wiredtiger/ext/test/fail_fs/Makefile.am create mode 100644 src/third_party/wiredtiger/ext/test/fail_fs/fail_fs.c create mode 100644 src/third_party/wiredtiger/import.data create mode 100644 src/third_party/wiredtiger/lang/python/setup_pip.py create mode 100644 src/third_party/wiredtiger/lang/python/wiredtiger/pip_init.py create mode 100644 src/third_party/wiredtiger/src/btree/bt_random.c delete mode 100644 src/third_party/wiredtiger/src/docs/tune-compression.dox create mode 100644 src/third_party/wiredtiger/src/docs/tune-page-size-and-comp.dox delete mode 100644 src/third_party/wiredtiger/src/docs/tune-page-sizes.dox create mode 100644 src/third_party/wiredtiger/src/os_posix/os_snprintf.c delete mode 100644 src/third_party/wiredtiger/src/os_win/os_vsnprintf.c create mode 100644 src/third_party/wiredtiger/src/schema/schema_alter.c create mode 100644 src/third_party/wiredtiger/src/utilities/util_alter.c create mode 100644 src/third_party/wiredtiger/src/utilities/util_truncate.c create mode 100644 src/third_party/wiredtiger/test/csuite/rwlock/main.c create mode 100644 src/third_party/wiredtiger/test/csuite/scope/main.c create mode 100644 src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c create mode 100644 src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c create mode 100644 src/third_party/wiredtiger/test/csuite/wt3135_search_near_collator/main.c create mode 100644 src/third_party/wiredtiger/test/csuite/wt3184_dup_index_collator/main.c create mode 100644 src/third_party/wiredtiger/test/java/com/wiredtiger/test/CursorTest03.java create mode 100644 src/third_party/wiredtiger/test/suite/test_alter01.py create mode 100644 src/third_party/wiredtiger/test/suite/test_cursor10.py create mode 100644 src/third_party/wiredtiger/test/suite/test_cursor11.py create mode 100644 src/third_party/wiredtiger/test/suite/test_hazard.py create mode 100644 src/third_party/wiredtiger/test/suite/test_reconfig04.py create mode 100644 src/third_party/wiredtiger/test/suite/test_util14.py create mode 100644 src/third_party/wiredtiger/test/suite/test_util15.py create mode 100644 src/third_party/wiredtiger/test/suite/test_util16.py create mode 100644 src/third_party/wiredtiger/test/suite/test_util17.py create mode 100644 src/third_party/wiredtiger/test/syscall/Makefile.am create mode 100644 src/third_party/wiredtiger/test/syscall/syscall.py create mode 100644 src/third_party/wiredtiger/test/syscall/wt2336_base/base.run create mode 100644 src/third_party/wiredtiger/test/syscall/wt2336_base/main.c create mode 100644 src/third_party/wiredtiger/test/wtperf/test_conf_dump.py diff --git a/SConstruct b/SConstruct index 7eb1536dc6e..3f487e34caf 100644 --- a/SConstruct +++ b/SConstruct @@ -716,6 +716,10 @@ env_vars.Add('VERBOSE', default='auto', ) +env_vars.Add('WINDOWS_OPENSSL_BIN', + help='Sets the path to the openssl binaries for packaging', + default='c:/openssl/bin') + # don't run configure if user calls --help if GetOption('help'): Return() @@ -830,8 +834,8 @@ envDict = dict(BUILD_ROOT=buildDir, UNITTEST_LIST='$BUILD_ROOT/unittests.txt', INTEGRATION_TEST_ALIAS='integration_tests', INTEGRATION_TEST_LIST='$BUILD_ROOT/integration_tests.txt', - CONFIGUREDIR=sconsDataDir.Dir('sconf_temp'), - CONFIGURELOG=sconsDataDir.File('config.log'), + CONFIGUREDIR='$BUILD_ROOT/scons/$VARIANT_DIR/sconf_temp', + CONFIGURELOG='$BUILD_ROOT/scons/config.log', INSTALL_DIR=installDir, CONFIG_HEADER_DEFINES={}, LIBDEPS_TAG_EXPANSIONS=[], @@ -1429,6 +1433,7 @@ elif env.TargetOSIs('windows'): 'crypt32.lib', 'kernel32.lib', 'shell32.lib', + 'pdh.lib', 'version.lib', 'winmm.lib', 'ws2_32.lib', @@ -2265,6 +2270,41 @@ def doConfigure(myenv): myenv = conf.Finish() + def CheckCXX14EnableIfT(context): + test_body = """ + #include + #include + + template + struct scons { + bool hasSupport() { return false; } + }; + + template <> + struct scons> { + bool hasSupport() { return true; } + }; + + int main(int argc, char **argv) { + scons<> SCons; + return SCons.hasSupport() ? EXIT_SUCCESS : EXIT_FAILURE; + } + """ + context.Message('Checking for C++14 std::enable_if_t support...') + ret = context.TryCompile(textwrap.dedent(test_body), '.cpp') + context.Result(ret) + return ret + + # Check for std::enable_if_t support without using the __cplusplus macro + conf = Configure(myenv, help=False, custom_tests = { + 'CheckCXX14EnableIfT' : CheckCXX14EnableIfT, + }) + + if conf.CheckCXX14EnableIfT(): + conf.env.SetConfigHeaderDefine('MONGO_CONFIG_HAVE_STD_ENABLE_IF_T') + + myenv = conf.Finish() + def CheckCXX14MakeUnique(context): test_body = """ #include @@ -2333,6 +2373,18 @@ def doConfigure(myenv): }) libdeps.setup_conftests(conf) + def addOpenSslLibraryToDistArchive(file_name): + openssl_bin_path = os.path.normpath(env['WINDOWS_OPENSSL_BIN'].lower()) + full_file_name = os.path.join(openssl_bin_path, file_name) + if os.path.exists(full_file_name): + env.Append(ARCHIVE_ADDITIONS=[full_file_name]) + env.Append(ARCHIVE_ADDITION_DIR_MAP={ + openssl_bin_path: "bin" + }) + return True + else: + return False + if has_option( "ssl" ): sslLibName = "ssl" cryptoLibName = "crypto" @@ -2340,6 +2392,12 @@ def doConfigure(myenv): sslLibName = "ssleay32" cryptoLibName = "libeay32" + # Add the SSL binaries to the zip file distribution + files = ['ssleay32.dll', 'libeay32.dll'] + for extra_file in files: + if not addOpenSslLibraryToDistArchive(extra_file): + print("WARNING: Cannot find SSL library '%s'" % extra_file) + # Used to import system certificate keychains if conf.env.TargetOSIs('osx'): conf.env.AppendUnique(FRAMEWORKS=[ @@ -2565,10 +2623,20 @@ def doConfigure(myenv): env = doConfigure( env ) +# If the flags in the environment are configured for -gsplit-dwarf, +# inject the necessary emitter. +split_dwarf = Tool('split_dwarf') +if split_dwarf.exists(env): + split_dwarf(env) + # Load the compilation_db tool. We want to do this after configure so we don't end up with # compilation database entries for the configure tests, which is weird. env.Tool("compilation_db") +incremental_link = Tool('incremental_link') +if incremental_link.exists(env): + incremental_link(env) + def checkErrorCodes(): import buildscripts.errorcodes as x if x.checkErrorCodes() == False: @@ -2584,7 +2652,7 @@ def doLint( env , target , source ): raise Exception("ESLint errors") import buildscripts.clang_format - if not buildscripts.clang_format.lint(None, []): + if not buildscripts.clang_format.lint_all(None): raise Exception("clang-format lint errors") import buildscripts.lint diff --git a/buildscripts/buildlogger.py b/buildscripts/buildlogger.py index 48eb468a6e2..531c9cc9e9a 100644 --- a/buildscripts/buildlogger.py +++ b/buildscripts/buildlogger.py @@ -238,6 +238,12 @@ def run_and_echo(command): """ proc = subprocess.Popen(command) + # We write the pid of the spawned process as the first line of buildlogger.py's stdout because + # smoke.py expects to use it to terminate processes individually if already running inside a job + # object. + sys.stdout.write("[buildlogger.py] pid: %d\n" % (proc.pid)) + sys.stdout.flush() + def handle_sigterm(signum, frame): try: proc.send_signal(signum) @@ -415,6 +421,12 @@ def loop_and_callback(command, callback): stderr=subprocess.STDOUT, ) + # We write the pid of the spawned process as the first line of buildlogger.py's stdout because + # smoke.py expects to use it to terminate processes individually if already running inside a job + # object. + sys.stdout.write("[buildlogger.py] pid: %d\n" % (proc.pid)) + sys.stdout.flush() + def handle_sigterm(signum, frame): try: proc.send_signal(signum) diff --git a/buildscripts/clang_format.py b/buildscripts/clang_format.py index a756ff58ae5..99c3e9cf849 100755 --- a/buildscripts/clang_format.py +++ b/buildscripts/clang_format.py @@ -31,7 +31,6 @@ from multiprocessing import cpu_count if __name__ == "__main__" and __package__ is None: sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__))))) -from buildscripts.resmokelib.utils import globstar from buildscripts import moduleconfig @@ -183,7 +182,6 @@ def get_clang_format_from_linux_cache(dest_file): # Destination Path shutil.move("llvm/Release/bin/clang-format", dest_file) - class ClangFormat(object): """Class encapsulates finding a suitable copy of clang-format, and linting/formating an individual file @@ -318,8 +316,6 @@ def parallel_process(items, func): except NotImplementedError: cpus = 1 - # print("Running across %d cpus" % (cpus)) - task_queue = Queue.Queue() # Use a list so that worker function will capture this variable @@ -412,18 +408,16 @@ class Repo(object): def __init__(self, path): self.path = path - # Get candidate files - self.candidate_files = self._get_candidate_files() - self.root = self._get_root() def _callgito(self, args): """Call git for this repository """ # These two flags are the equivalent of -C in newer versions of Git - # but we use these to support versions back to ~1.8 + # but we use these to support versions pre 1.8.5 but it depends on the command + # and what the current directory is return callo(['git', '--git-dir', os.path.join(self.path, ".git"), - '--work-tree', self.path] + args) + '--work-tree', self.path] + args) def _get_local_dir(self, path): """Get a directory path relative to the git root directory @@ -433,13 +427,10 @@ class Repo(object): return path def get_candidates(self, candidates): - """Get the set of candidate files to check by doing an intersection - between the input list, and the list of candidates in the repository + """Get the set of candidate files to check by querying the repository Returns the full path to the file for clang-format to consume. """ - # NOTE: Files may have an absolute root (i.e. leading /) - if candidates is not None and len(candidates) > 0: candidates = [self._get_local_dir(f) for f in candidates] valid_files = list(set(candidates).intersection(self.get_candidate_files())) @@ -448,6 +439,7 @@ class Repo(object): # Get the full file name here valid_files = [os.path.normpath(os.path.join(self.root, f)) for f in valid_files] + return valid_files def get_root(self): @@ -462,15 +454,10 @@ class Repo(object): return gito.rstrip() - def get_candidate_files(self): - """Get a list of candidate files - """ - return self._get_candidate_files() - - def _get_candidate_files(self): - """Query git to get a list of all files in the repo to consider for analysis + def _git_ls_files(self, cmd): + """Run git-ls-files and filter the list of files to a valid candidate list """ - gito = self._callgito(["ls-files"]) + gito = self._callgito(cmd) # This allows us to pick all the interesting files # in the mongo and mongo-enterprise repos @@ -485,24 +472,48 @@ class Repo(object): return file_list + def get_candidate_files(self): + """Query git to get a list of all files in the repo to consider for analysis + """ + return self._git_ls_files(["ls-files", "--cached"]) + + def get_working_tree_candidate_files(self): + """Query git to get a list of all files in the working tree to consider for analysis + """ + return self._git_ls_files(["ls-files", "--cached", "--others"]) + + def get_working_tree_candidates(self): + """Get the set of candidate files to check by querying the repository + + Returns the full path to the file for clang-format to consume. + """ + valid_files = list(self.get_working_tree_candidate_files()) + + # Get the full file name here + valid_files = [os.path.normpath(os.path.join(self.root, f)) for f in valid_files] -def expand_file_string(glob_pattern): - """Expand a string that represents a set of files + # Filter out files that git thinks exist but were removed. + valid_files = [f for f in valid_files if os.path.exists(f)] + + return valid_files + +def get_files_to_check_working_tree(): + """Get a list of files to check form the working tree. + This will pick up files not managed by git. """ - return [os.path.abspath(f) for f in globstar.iglob(glob_pattern)] + repos = get_repos() -def get_files_to_check(files): - """Filter the specified list of files to check down to the actual - list of files that need to be checked.""" - candidates = [] + valid_files = list(itertools.chain.from_iterable([r.get_working_tree_candidates() for r in repos])) - # Get a list of candidate_files - candidates = [expand_file_string(f) for f in files] - candidates = list(itertools.chain.from_iterable(candidates)) + return valid_files +def get_files_to_check(): + """Get a list of files that need to be checked + based on which files are managed by git. + """ repos = get_repos() - valid_files = list(itertools.chain.from_iterable([r.get_candidates(candidates) for r in repos])) + valid_files = list(itertools.chain.from_iterable([r.get_candidates(None) for r in repos])) return valid_files @@ -552,10 +563,19 @@ def lint_patch(clang_format, infile): if files: _lint_files(clang_format, files) -def lint(clang_format, glob): +def lint(clang_format): """Lint files command entry point """ - files = get_files_to_check(glob) + files = get_files_to_check() + + _lint_files(clang_format, files) + + return True + +def lint_all(clang_format): + """Lint files command entry point based on working tree + """ + files = get_files_to_check_working_tree() _lint_files(clang_format, files) @@ -572,18 +592,17 @@ def _format_files(clang_format, files): print("ERROR: failed to format files") sys.exit(1) -def format_func(clang_format, glob): +def format_func(clang_format): """Format files command entry point """ - files = get_files_to_check(glob) + files = get_files_to_check() _format_files(clang_format, files) def usage(): """Print usage """ - print("clang-format.py supports 3 commands [ lint, lint-patch, format ]. Run " - " -? for more information") + print("clang-format.py supports 4 commands [ lint, lint-all, lint-patch, format ].") def main(): """Main entry point @@ -597,11 +616,13 @@ def main(): command = args[1] if command == "lint": - lint(options.clang_format, args[2:]) + lint(options.clang_format) + elif command == "lint-all": + lint_all(options.clang_format) elif command == "lint-patch": lint_patch(options.clang_format, args[2:]) elif command == "format": - format_func(options.clang_format, args[2:]) + format_func(options.clang_format) else: usage() else: diff --git a/buildscripts/generate_compile_expansions.py b/buildscripts/generate_compile_expansions.py index ab1661bcfea..907a8c27c94 100755 --- a/buildscripts/generate_compile_expansions.py +++ b/buildscripts/generate_compile_expansions.py @@ -52,8 +52,8 @@ if not version_parts: exit(1) if version_parts[0]: - print "suffix: latest" - print "src_suffix: latest" + print "suffix: v3.2-latest" + print "src_suffix: v3.2-latest" else: print "suffix: {0}".format(version_line) print "src_suffix: r{0}".format(version_line) diff --git a/buildscripts/hang_analyzer.py b/buildscripts/hang_analyzer.py index ea708cc0ecd..ca1baa00f39 100755 --- a/buildscripts/hang_analyzer.py +++ b/buildscripts/hang_analyzer.py @@ -24,6 +24,12 @@ import threading import time from distutils import spawn from optparse import OptionParser +_is_windows = (sys.platform == "win32") + +if _is_windows: + import win32event + import win32api + if sys.platform == "win32": import win32process @@ -365,7 +371,7 @@ def get_hang_analyzers(): elif sys.platform.startswith("sunos"): dbg = GDBDumper() ps = SolarisProcessList() - elif os.name == 'nt' or (os.name == "posix" and sys.platform == "cygwin"): + elif _is_windows or sys.platform == "cygwin": dbg = WindowsDumper() ps = WindowsProcessList() elif sys.platform == "darwin": @@ -383,10 +389,37 @@ def is_interesting_process(p): return False -def signal_process(pid): - """Signal python process with SIGUSR1, N/A on Windows""" +def signal_event_object(pid): + """Signal the Windows event object""" + + # Use unique event_name created. + event_name = "Global\\Mongo_Python_" + str(pid) + + try: + desired_access = win32event.EVENT_MODIFY_STATE + inherit_handle = False + task_timeout_handle = win32event.OpenEvent(desired_access, + inherit_handle, + event_name) + except win32event.error as err: + print "Exception from win32event.OpenEvent with error: " + str(err) + return + + try: + win32event.SetEvent(task_timeout_handle) + except win32event.error as err: + print "Exception from win32event.SetEvent with error: " + str(err) + finally: + win32api.CloseHandle(task_timeout_handle) + + print "Waiting for process to report" + time.sleep(5) + + +def signal_process(pid, signalnum): + """Signal process with signal, N/A on Windows""" try: - os.kill(pid, signal.SIGUSR1) + os.kill(pid, signalnum) print "Waiting for python process to report" time.sleep(5) @@ -417,8 +450,13 @@ def main(): print "OS: " + platform.platform() try: - distro = platform.linux_distribution() - print "Linux Distribution: " + str(distro) + if _is_windows or sys.platform == "cygwin": + distro = platform.win32_ver() + print "Windows Distribution: " + str(distro) + else: + distro = platform.linux_distribution() + print "Linux Distribution: " + str(distro) + except AttributeError: print "Cannot determine Linux distro since Python is too old" @@ -457,20 +495,29 @@ def main(): for process in processes_orig: sys.stdout.write("Ignoring process %d of %s\n" % (process[0], process[1])) else: - # Dump all other processes first since signaling the python script interrupts it + # Dump python processes by signalling them. The resmoke.py process will generate + # the report.json, when signalled, so we do this before attaching to other processes. + for process in [a for a in processes if a[1].startswith("python")]: + # On Windows, we set up an event object to wait on a signal. For Cygwin, we register + # a signal handler to wait for the signal since it supports POSIX signals. + if _is_windows: + sys.stdout.write("Calling SetEvent to signal python process %s with PID %d\n" % + (process[1], process[0])) + signal_event_object(process[0]) + else: + sys.stdout.write("Sending signal SIGUSR1 to python process %s with PID %d\n" % + (process[1], process[0])) + signal_process(process[0], signal.SIGUSR1) + + # Dump all other processes for process in [a for a in processes if not a[1].startswith("python")]: sys.stdout.write("Dumping process %d of %s\n" % (process[0], process[1])) dbg.dump_info(process[0], process[1], sys.stdout) - for process in [a for a in processes if a[1].startswith("python")]: - signal_process(process[0]) - - dbg.dump_info(process[0], process[1], sys.stdout) - # Suspend the timer so we can exit cleanly timer.cancel() - sys.stdout.write("Done analyzing processes for hangs\n") + sys.stdout.write("Done analyzing all processes for hangs\n") if __name__ == "__main__": main() diff --git a/buildscripts/package_test/.kitchen.yml b/buildscripts/package_test/.kitchen.yml new file mode 100644 index 00000000000..e9b6989d221 --- /dev/null +++ b/buildscripts/package_test/.kitchen.yml @@ -0,0 +1,81 @@ +--- +driver: + name: ec2 + region: us-east-1 + subnet_id: <%= ENV['KITCHEN_SUBNET'] %> + security_group_ids: + - <%= ENV['KITCHEN_SECURITY_GROUP'] %> + aws_ssh_key_id: <%= ENV['KITCHEN_SSH_KEY_ID'] %> + interface: dns + associate_public_ip: true + +verifier: + name: inspec + sudo: true + +provisioner: + name: chef_solo + require_chef_omnibus: 12.6.0 + +platforms: + - name: amazon + driver: + image_id: ami-c481fad3 + transport: + username: ec2-user + - name: debian71 + driver: + image_id: ami-4b124a22 + transport: + username: admin + - name: debian81 + driver: + image_id: ami-896d85e2 + transport: + username: admin + - name: rhel62 + driver: + # Use rhel 6.3 because chef-solo fails on rhel 6.2 + image_id: ami-a35a33ca + transport: + username: root + - name: rhel70 + driver: + image_id: ami-60a1e808 + transport: + username: root + - name: suse11 + driver: + image_id: ami-7f2e6015 + transport: + username: ec2-user + - name: suse12 + driver: + image_id: ami-aeb532c6 + transport: + username: ec2-user + - name: ubuntu1204 + driver: + image_id: ami-3fec7956 + transport: + username: ubuntu + - name: ubuntu1404 + driver: + image_id: ami-1d8c9574 + transport: + username: ubuntu + - name: ubuntu1604 + driver: + image_id: ami-64140d0e + transport: + username: ubuntu + +transport: + ssh_key: ~/.ssh/kitchen.pem + +suites: + - name: service + run_list: + - recipe[package_test::install_mongodb] + attributes: + artifacts_url: <%= ENV['KITCHEN_ARTIFACTS_URL'] %> diff --git a/buildscripts/package_test/metadata.rb b/buildscripts/package_test/metadata.rb new file mode 100644 index 00000000000..3bbfb147acc --- /dev/null +++ b/buildscripts/package_test/metadata.rb @@ -0,0 +1,2 @@ +name 'package_test' +version '0.1.0' diff --git a/buildscripts/package_test/recipes/install_mongodb.rb b/buildscripts/package_test/recipes/install_mongodb.rb new file mode 100644 index 00000000000..627aa09b8af --- /dev/null +++ b/buildscripts/package_test/recipes/install_mongodb.rb @@ -0,0 +1,120 @@ +artifacts_tarball = 'artifacts.tgz' +user = node['current_user'] +homedir = node['etc']['passwd'][user]['dir'] + +ruby_block 'allow sudo over tty' do + block do + file = Chef::Util::FileEdit.new('/etc/sudoers') + file.search_file_replace_line(/Defaults\s+requiretty/, '#Defaults requiretty') + file.search_file_replace_line(/Defaults\s+requiretty/, '#Defaults !visiblepw') + file.write_file + end +end + +# These files limits processes. They therefore interferere with `ulimit -u` when present. +if platform_family? 'rhel' + file '/etc/security/limits.d/90-nproc.conf' do + action :delete + end + file '/etc/security/limits.d/20-nproc.conf' do + action :delete + end +end + +remote_file "#{homedir}/#{artifacts_tarball}" do + source node['artifacts_url'] +end + +execute 'extract artifacts' do + command "tar xzvf #{artifacts_tarball}" + cwd homedir +end + +if platform_family? 'debian' + execute 'apt-get update' do + command 'apt-get update' + end + + package 'openssl' + + # dpkg returns 1 if dependencies are not satisfied, which they will not be + # for enterprise builds. We install dependencies in the next block. + execute 'install mongod' do + command 'dpkg -i `find . -name "*server*.deb"`' + cwd homedir + returns [0, 1] + end + + # yum and zypper fetch dependencies automatically, but dpkg does not. + # Installing the dependencies explicitly is fragile, so we reply on apt-get + # to install dependencies after the fact. + execute 'install dependencies' do + command 'apt-get update && apt-get -y -f install' + end + + execute 'install mongo shell' do + command 'dpkg -i `find . -name "*shell*.deb"`' + cwd homedir + end +end + +if platform_family? 'rhel' + execute 'install mongod' do + command 'yum install -y `find . -name "*server*.rpm"`' + cwd homedir + end + + execute 'install mongo shell' do + command 'yum install -y `find . -name "*shell*.rpm"`' + cwd homedir + end +end + +if platform_family? 'suse' + bash 'wait for zypper lock to be released' do + code <<-EOD + retry_counter=0 + # We also need to make sure another instance of zypper isn't running while + # we do our install, so just run zypper refresh until it doesn't fail. + # Waiting for 2 minutes is copied from an internal project where we do this. + until [ "$retry_counter" -ge "12" ]; do + zypper refresh && exit 0 + retry_counter=$(($retry_counter + 1)) + [ "$retry_counter" = "12" ] && break + sleep 10 + done + exit 1 + EOD + end + + execute 'install mongod' do + command 'zypper -n install `find . -name "*server*.rpm"`' + cwd homedir + end + + execute 'install mongo' do + command 'zypper -n install `find . -name "*shell*.rpm"`' + cwd homedir + end +end + +inspec_wait = < 'unlimited', + 'Max cpu time' => 'unlimited', + 'Max address space' => 'unlimited', + 'Max open files' => '64000', + 'Max resident set' => 'unlimited', + 'Max processes' => '64000' +} +ulimits_cmd = 'cat /proc/$(pgrep mongod)/limits' + +ulimits.each do |limit, value| + describe command("#{ulimits_cmd} | grep \"#{limit}\"") do + its('stdout') { should match(/#{limit}\s+#{value}/) } + end +end + +############################################################ +# This section verifies reads and writes. +# - insert a document into the database +# - verify that findOne() returns a matching document +############################################################ + +describe command('mongo --eval "db.smoke.insert({answer: 42})"') do + its('exit_status') { should eq 0 } + its('stdout') { should match(/.+WriteResult\({ "nInserted" : 1 }\).+/m) } +end + +# read a document from the db +describe command('mongo --eval "db.smoke.findOne()"') do + its('exit_status') { should eq 0 } + its('stdout') { should match(/.+"answer" : 42.+/m) } +end + +############################################################ +# This section verifies uninstall. +############################################################ + +if rpm + describe command('rpm -e $(rpm -qa | grep "mongodb.*server" | awk \'{print $1}\')') do + its('exit_status') { should eq 0 } + end +elsif deb + describe command('dpkg -r $(dpkg -l | grep "mongodb.*server" | awk \'{print $2}\')') do + its('exit_status') { should eq 0 } + end +end + +# make sure we cleaned up +%w( + /lib/systemd/system/mongod.service + /usr/bin/mongod +).each do |filename| + describe file(filename) do + it { should_not exist } + end +end diff --git a/buildscripts/packager-enterprise.py b/buildscripts/packager-enterprise.py index f9437fd935f..e7ae856a9ad 100755 --- a/buildscripts/packager-enterprise.py +++ b/buildscripts/packager-enterprise.py @@ -26,20 +26,15 @@ # apt-get install dpkg-dev rpm debhelper fakeroot ia32-libs createrepo git-core libsnmp15 # echo "Now put the dist gnupg signing keys in ~root/.gnupg" -import argparse import errno -import getopt from glob import glob import packager import os import re import shutil -import stat -import subprocess import sys import tempfile import time -import urlparse # The MongoDB names for the architectures we support. ARCH_CHOICES=["x86_64", "ppc64le"] @@ -143,8 +138,6 @@ def main(argv): os.chdir(prefix) try: - # Download the binaries. - urlfmt="http://downloads.mongodb.com/linux/mongodb-linux-%s-enterprise-%s-%s.tgz" made_pkg = False # Build a package for each distro/spec/arch tuple, and @@ -154,12 +147,9 @@ def main(argv): for build_os in distro.build_os(arch): if build_os in args.distros or not args.distros: - if args.tarball: - filename = tarfile(build_os, arch, spec) - packager.ensure_dir(filename) - shutil.copyfile(args.tarball,filename) - else: - packager.httpget(urlfmt % (arch, build_os, spec.version()), packager.ensure_dir(tarfile(build_os, arch, spec))) + filename = tarfile(build_os, arch, spec) + packager.ensure_dir(filename) + shutil.copyfile(args.tarball,filename) repo = make_package(distro, build_os, arch, spec, srcdir) make_repo(repo, distro, build_os, spec) diff --git a/buildscripts/packager.py b/buildscripts/packager.py index 29fb4fdfa43..adae9fe5f6d 100755 --- a/buildscripts/packager.py +++ b/buildscripts/packager.py @@ -28,18 +28,14 @@ import argparse import errno -import getopt -import httplib2 from glob import glob import os import re import shutil -import stat import subprocess import sys import tempfile import time -import urlparse # The MongoDB names for the architectures we support. ARCH_CHOICES=["x86_64"] @@ -54,11 +50,15 @@ class Spec(object): self.gitspec = gitspec self.rel = rel - # Nightly version numbers can be in the form: 3.0.7-pre-, or 3.0.7-5-g3b67ac + # Commit-triggerd version numbers can be in the form: 3.0.7-pre-, or 3.0.7-5-g3b67ac + # Patch builds version numbers are in the form: 3.5.5-64-g03945fa-patch-58debcdb3ff1223c9d00005b # def is_nightly(self): return bool(re.search("-$", self.version())) or bool(re.search("\d-\d+-g[0-9a-f]+$", self.version())) + def is_patch(self): + return bool(re.search("\d-\d+-g[0-9a-f]+-patch-[0-9a-f]+$", self.version())) + def is_rc(self): return bool(re.search("-rc\d+$", self.version())) @@ -68,6 +68,12 @@ class Spec(object): def version(self): return self.ver + def patch_id(self): + if self.is_patch(): + return re.sub(r'.*-([0-9a-f]+$)', r'\1', self.version()) + else: + return "none" + def metadata_gitspec(self): """Git revision to use for spec+control+init+manpage files. The default is the release tag for the version being packaged.""" @@ -85,35 +91,57 @@ class Spec(object): return "-org" if int(self.ver.split(".")[1])%2==0 else "-org-unstable" def prelease(self): - # "N" is either passed in on the command line, or "1" + # NOTE: This is only called for RPM packages, and only after + # pversion() below has been called. If you want to change this format + # and want DEB packages to match, make sure to update pversion() + # below # - # 1) Standard release - "N" - # 2) Nightly (snapshot) - "0.N.YYYYMMDDlatest" - # 3) RC's - "0.N.rcX" + # "N" is either passed in on the command line, or "1" if self.rel: - corenum = self.rel + corenum = self.rel else: - corenum = 1 - # RC's + corenum = 1 + + # Version suffix for RPM packages: + # 1) RC's - "0.N.rcX" + # 2) Nightly (snapshot) - "0.N.latest" + # 3) Patch builds - "0.N.patch." + # 4) Standard release - "N" if self.is_rc(): - return "0.%s.%s" % (corenum, re.sub('.*-','',self.version())) - # Nightlies + return "0.%s.%s" % (corenum, re.sub('.*-','',self.version())) elif self.is_nightly(): - return "0.%s.%s" % (corenum, time.strftime("%Y%m%d")) + return "0.%s.latest" % (corenum) + elif self.is_patch(): + return "0.%s.patch.%s" % (corenum, self.patch_id()) else: - return str(corenum) + return str(corenum) def pversion(self, distro): # Note: Debian packages have funny rules about dashes in # version numbers, and RPM simply forbids dashes. pversion # will be the package's version number (but we need to know # our upstream version too). + + # For RPM packages this just returns X.Y.X because of the + # aforementioned rules, and prelease (above) adds a suffix later, + # so detect this case early + if re.search("(suse|redhat|fedora|centos|amazon)", distro.name()): + return re.sub("-.*", "", self.version()) + + # For DEB packages, this code sets the full version. If you change + # this format and want RPM packages to match make sure you change + # prelease above as well if re.search("^(debian|ubuntu)", distro.name()): - return re.sub("-", "~", self.ver) - elif re.search("(suse|redhat|fedora|centos|amazon)", distro.name()): - return re.sub("-.*", "", self.ver) - else: - raise Exception("BUG: unsupported platform?") + if self.is_nightly(): + ver = re.sub("-.*", "-latest", self.ver) + elif self.is_patch(): + ver = re.sub("-.*", "", self.ver) + "-patch-" + self.patch_id() + else: + ver = self.ver + + return re.sub("-", "~", ver) + + raise Exception("BUG: unsupported platform?") def branch(self): """Return the major and minor portions of the specified version. @@ -275,7 +303,7 @@ def get_args(distros, arch_choices): parser.add_argument("-d", "--distros", help="Distros to build for", choices=distro_choices, required=False, default=[], action='append') parser.add_argument("-p", "--prefix", help="Directory to build into", required=False) parser.add_argument("-a", "--arches", help="Architecture to build", choices=arch_choices, default=[], required=False, action='append') - parser.add_argument("-t", "--tarball", help="Local tarball to package instead of downloading (only valid with one distro/arch combination)", required=False, type=lambda x: is_valid_file(parser, x)) + parser.add_argument("-t", "--tarball", help="Local tarball to package", required=True, type=lambda x: is_valid_file(parser, x)) args = parser.parse_args() @@ -304,9 +332,6 @@ def main(argv): os.chdir(prefix) try: - # Download the binaries. - urlfmt="http://downloads.mongodb.org/linux/mongodb-linux-%s-%s-%s.tgz" - # Build a package for each distro/spec/arch tuple, and # accumulate the repository-layout directories. for (distro, arch) in crossproduct(distros, args.arches): @@ -314,12 +339,9 @@ def main(argv): for build_os in distro.build_os(arch): if build_os in args.distros or not args.distros: - if args.tarball: - filename = tarfile(build_os, arch, spec) - ensure_dir(filename) - shutil.copyfile(args.tarball,filename) - else: - httpget(urlfmt % (arch, build_os, spec.version()), ensure_dir(tarfile(build_os, arch, spec))) + filename = tarfile(build_os, arch, spec) + ensure_dir(filename) + shutil.copyfile(args.tarball,filename) repo = make_package(distro, build_os, arch, spec, srcdir) make_repo(repo, distro, build_os, spec) @@ -371,27 +393,6 @@ def setupdir(distro, build_os, arch, spec): # or dst/x86_64/redhat/rhel55/mongodb-org-unstable/ return "dst/%s/%s/%s/%s%s-%s/" % (arch, distro.name(), build_os, distro.pkgbase(), spec.suffix(), spec.pversion(distro)) -def httpget(url, filename): - """Download the contents of url to filename, return filename.""" - print "Fetching %s to %s." % (url, filename) - conn = None - u=urlparse.urlparse(url) - assert(u.scheme=='http') - try: - h = httplib2.Http(cache = os.environ["HOME"] + "/.cache") - resp, content = h.request(url, "GET") - t=filename+'.TMP' - if resp.status==200: - with open(t, 'w') as f: - f.write(content) - else: - raise Exception("HTTP error %d" % resp.status) - os.rename(t, filename) - finally: - if conn: - conn.close() - return filename - def unpack_binaries_into(build_os, arch, spec, where): """Unpack the tarfile for (build_os, arch, spec) into directory where.""" rootdir=os.getcwd() diff --git a/buildscripts/resmoke.py b/buildscripts/resmoke.py index a6cb03cb620..837b0ec3b26 100755 --- a/buildscripts/resmoke.py +++ b/buildscripts/resmoke.py @@ -6,13 +6,10 @@ Command line utility for executing MongoDB tests of all kinds. from __future__ import absolute_import -import json import os.path import random -import signal import sys import time -import traceback # Get relative imports to work when the package is not installed on the PYTHONPATH. if __name__ == "__main__" and __package__ is None: @@ -115,21 +112,6 @@ def _dump_suite_config(suite, logging_config): return "\n".join(sb) -def _write_report_file(suites, pathname): - """ - Writes the report.json file if requested. - """ - - reports = [] - for suite in suites: - for group in suite.test_groups: - reports.extend(group.get_reports()) - - combined_report_dict = resmokelib.testing.report.TestReport.combine(*reports).as_dict() - with open(pathname, "w") as fp: - json.dump(combined_report_dict, fp) - - def main(): start_time = time.time() @@ -151,6 +133,11 @@ def main(): interrupted = False suites = resmokelib.parser.get_suites(values, args) + + # Register a signal handler or Windows event object so we can write the report file if the task + # times out. + resmokelib.sighandler.register(resmoke_logger, suites) + try: for suite in suites: resmoke_logger.info(_dump_suite_config(suite, logging_config)) @@ -178,39 +165,8 @@ def main(): if not interrupted: resmokelib.logging.flush.stop_thread() - if resmokelib.config.REPORT_FILE is not None: - _write_report_file(suites, resmokelib.config.REPORT_FILE) + resmokelib.reportfile.write(suites) if __name__ == "__main__": - - def _dump_stacks(signum, frame): - """ - Signal handler that will dump the stacks of all threads. - """ - - header_msg = "Dumping stacks due to SIGUSR1 signal" - - sb = [] - sb.append("=" * len(header_msg)) - sb.append(header_msg) - sb.append("=" * len(header_msg)) - - frames = sys._current_frames() - sb.append("Total threads: %d" % (len(frames))) - sb.append("") - - for thread_id in frames: - stack = frames[thread_id] - sb.append("Thread %d:" % (thread_id)) - sb.append("".join(traceback.format_stack(stack))) - - sb.append("=" * len(header_msg)) - print "\n".join(sb) - - try: - signal.signal(signal.SIGUSR1, _dump_stacks) - except AttributeError: - print "Cannot catch signals on Windows" - main() diff --git a/buildscripts/resmokeconfig/suites/multiversion_auth.yml b/buildscripts/resmokeconfig/suites/multiversion_auth.yml new file mode 100644 index 00000000000..4db3f87f7af --- /dev/null +++ b/buildscripts/resmokeconfig/suites/multiversion_auth.yml @@ -0,0 +1,37 @@ +# Section that is ignored by resmoke.py. +config_variables: +- &keyFile jstests/libs/authTestsKey +- &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly + +selector: + js_test: + roots: + - jstests/multiVersion/*.js + exclude_files: + # Needs extra work to support MONGODB-CR + - jstests/multiVersion/upgrade_cluster_v5_to_v6.js + # Uses threads which don't propagate jsTest() information + - jstests/multiVersion/mixed_storage_version_replication.js + # Uses ToolTest, which doesn't start servers with keyFile + - jstests/multiVersion/transitioning_to_and_from_WT.js + # TODO: SERVER-21578 + - jstests/multiVersion/balancer_multiVersion_detect.js + +# Multiversion tests start their own mongod's. +executor: + js_test: + config: + shell_options: + global_vars: + TestData: + auth: true + authMechanism: SCRAM-SHA-1 + keyFile: *keyFile + keyFileData: *keyFileData + authenticationDatabase: local + authenticationMechanism: SCRAM-SHA-1 + password: *keyFileData + username: __system + nodb: '' + readMode: legacy + writeMode: legacy diff --git a/buildscripts/resmokeconfig/suites/no_passthrough.yml b/buildscripts/resmokeconfig/suites/no_passthrough.yml index b4406cea871..2afbe6ae044 100644 --- a/buildscripts/resmokeconfig/suites/no_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/no_passthrough.yml @@ -2,6 +2,9 @@ selector: js_test: roots: - jstests/noPassthrough/*.js + exclude_files: + # Disable inmem_full as per SERVER-27014 + - jstests/noPassthrough/inmem_full.js # noPassthrough tests start their own mongod's. executor: diff --git a/buildscripts/resmokeconfig/suites/replica_sets_legacy.yml b/buildscripts/resmokeconfig/suites/replica_sets_legacy.yml index 44b1b7a572d..11b160e4925 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_legacy.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_legacy.yml @@ -15,6 +15,14 @@ selector: - jstests/replsets/read_majority_two_arbs.js # The combination of new bridges and PV0 can lead to an improper spanning tree in sync2.js. - jstests/replsets/sync2.js + # PV0 does not persist the last vote + - jstests/replsets/last_vote.js + # PV0's w:majority guarantees aren't strong enough for this test to pass. + - jstests/replsets/write_concern_after_stepdown_and_stepup.js + # This test expects the server to log a PV1-only vote-not-granted reason + - jstests/replsets/no_flapping_during_network_partition.js + # This test requires terms. + - jstests/replsets/rollback_too_new.js executor: js_test: diff --git a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml index 6b51a5f17f5..8096b406f4d 100644 --- a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml @@ -12,15 +12,16 @@ selector: - jstests/core/check_shard_index.js # checkShardingIndex. - jstests/core/collection_truncate.js # emptycapped. - jstests/core/compact_keeps_indexes.js # compact. - - jstests/core/capped_update.js # uses godinsert and can't run under replication. + - jstests/core/currentop.js # uses fsync. - jstests/core/auth_copydb.js # copyDatabase. - jstests/core/copydb.js # copyDatabase. - jstests/core/dbadmin.js # "local" database. - jstests/core/dbhash.js # dbhash. - jstests/core/dbhash2.js # dbhash. + - jstests/core/diagdata.js # Command not supported in mongos - jstests/core/dropdb_race.js # syncdelay. - jstests/core/evalb.js # profiling. - - jstests/core/fsync.js # fsync. + - jstests/core/fsync.js # uses fsync. - jstests/core/geo_haystack*.js # geoSearch. - jstests/core/geo_s2cursorlimitskip.js # profiling. - jstests/core/geo_update_btree2.js # notablescan. @@ -68,6 +69,7 @@ selector: - jstests/core/eval0.js - jstests/core/eval7.js - jstests/core/eval9.js + - jstests/core/killop_drop_collection.js # Uses fsyncLock. executor: js_test: diff --git a/buildscripts/resmokeconfig/suites/sharding_auth.yml b/buildscripts/resmokeconfig/suites/sharding_auth.yml index d99760820e7..1a988d5ab3b 100644 --- a/buildscripts/resmokeconfig/suites/sharding_auth.yml +++ b/buildscripts/resmokeconfig/suites/sharding_auth.yml @@ -24,6 +24,7 @@ selector: - jstests/sharding/migration_with_source_deletes.js # SERVER-21713 - jstests/sharding/migration_sets_fromMigrate_flag.js # SERVER-21713 - jstests/sharding/donor_shard_abort_and_start_new_migration.js # SERVER-21713 + - jstests/sharding/migration_critical_section_concurrency.js # SERVER-21713 executor: js_test: diff --git a/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml b/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml index 61e50340b93..daa287b3f64 100644 --- a/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml +++ b/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml @@ -24,6 +24,7 @@ selector: - jstests/sharding/migration_with_source_deletes.js # SERVER-21713 - jstests/sharding/migration_sets_fromMigrate_flag.js # SERVER-21713 - jstests/sharding/donor_shard_abort_and_start_new_migration.js # SERVER-21713 + - jstests/sharding/migration_critical_section_concurrency.js # SERVER-21713 executor: js_test: diff --git a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml index 43fbcaa94f6..0295f0e4f9d 100644 --- a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml +++ b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml @@ -88,6 +88,14 @@ selector: # for this command - jstests/sharding/major_version_check.js - jstests/sharding/replset_config/ssv_config_check.js + # Runs replSetGetStatus -- via awaitLastOpCommitted -- directly against the config server: + # retries aren't supported. + - jstests/sharding/coll_epoch_test1.js + - jstests/sharding/move_stale_mongos.js + - jstests/sharding/shard4.js + - jstests/sharding/shard5.js + - jstests/sharding/split_stale_mongos.js + - jstests/sharding/zero_shard_version.js # Already stop or blackholes the primary of the CSRS config shard - jstests/sharding/all_config_hosts_down.js - jstests/sharding/all_config_servers_blackholed_from_mongos.js diff --git a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml index 56a5b581ce8..98553af4366 100644 --- a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml @@ -14,12 +14,14 @@ selector: - jstests/core/check_shard_index.js # checkShardingIndex. - jstests/core/collection_truncate.js # emptycapped. - jstests/core/compact_keeps_indexes.js # compact. + - jstests/core/currentop.js # uses fsync. - jstests/core/dbadmin.js # "local" database. - jstests/core/dbhash.js # dbhash. - jstests/core/dbhash2.js # dbhash. + - jstests/core/diagdata.js # Command not supported in mongos - jstests/core/dropdb_race.js # syncdelay. - jstests/core/evalb.js # profiling. - - jstests/core/fsync.js # fsync. + - jstests/core/fsync.js # uses fsync. - jstests/core/geo_haystack*.js # geoSearch. - jstests/core/geo_s2cursorlimitskip.js # profiling. - jstests/core/geo_update_btree2.js # notablescan. @@ -41,6 +43,7 @@ selector: # TODO: SERVER-18292 remove once OP_COMMAND is implemented in mongos. - jstests/core/invalid_db_name.js - jstests/core/validate_cmd_ns.js + - jstests/core/killop_drop_collection.js # Uses fsyncLock. executor: js_test: diff --git a/buildscripts/resmokelib/__init__.py b/buildscripts/resmokelib/__init__.py index 06b0539e25b..d8019dcf404 100644 --- a/buildscripts/resmokelib/__init__.py +++ b/buildscripts/resmokelib/__init__.py @@ -3,5 +3,7 @@ from __future__ import absolute_import from . import errors from . import logging from . import parser +from . import reportfile +from . import sighandler from . import testing from . import utils diff --git a/buildscripts/resmokelib/logging/buildlogger.py b/buildscripts/resmokelib/logging/buildlogger.py index c5f5d40401b..f058372df8c 100644 --- a/buildscripts/resmokelib/logging/buildlogger.py +++ b/buildscripts/resmokelib/logging/buildlogger.py @@ -184,7 +184,7 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler): raise NotImplementedError("_append_logs must be implemented by _BaseBuildloggerHandler" " subclasses") - def flush_with_lock(self, close_called): + def _flush_buffer_with_lock(self, buf, close_called): """ Ensures all logging output has been flushed to the buildlogger server. @@ -194,7 +194,7 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler): called. """ - self.retry_buffer.extend(self.buffer) + self.retry_buffer.extend(buf) if self._append_logs(self.retry_buffer): self.retry_buffer = [] @@ -209,8 +209,6 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler): loggers._BUILDLOGGER_FALLBACK.info(message) self.retry_buffer = [] - self.buffer = [] - class BuildloggerTestHandler(_BaseBuildloggerHandler): """ diff --git a/buildscripts/resmokelib/logging/handlers.py b/buildscripts/resmokelib/logging/handlers.py index 6ede7d38b3c..3d71399bfa5 100644 --- a/buildscripts/resmokelib/logging/handlers.py +++ b/buildscripts/resmokelib/logging/handlers.py @@ -43,10 +43,12 @@ class BufferedHandler(logging.Handler): self.capacity = capacity self.interval_secs = interval_secs - self.buffer = [] - self._lock = threading.Lock() - self._timer = None # Defer creation until actually begin to log messages. + self.__emit_lock = threading.Lock() # Prohibits concurrent access to 'self.__emit_buffer'. + self.__emit_buffer = [] + + self.__flush_lock = threading.Lock() # Serializes callers of self.flush(). + self.__timer = None # Defer creation until we actually begin to log messages. def _new_timer(self): """ @@ -54,7 +56,7 @@ class BufferedHandler(logging.Handler): flush() method after 'interval_secs' seconds. """ - return timer.AlarmClock(self.interval_secs, self.flush, args=[self]) + return timer.AlarmClock(self.interval_secs, self.flush) def process_record(self, record): """ @@ -79,37 +81,45 @@ class BufferedHandler(logging.Handler): will expire after another 'interval_secs' seconds. """ - with self._lock: - self.buffer.append(self.process_record(record)) - if len(self.buffer) >= self.capacity: - if self._timer is not None: - self._timer.snooze() - self.flush_with_lock(False) - if self._timer is not None: - self._timer.reset() + if self.__timer is None: + self.__timer = self._new_timer() + self.__timer.start() - if self._timer is None: - self._timer = self._new_timer() - self._timer.start() + with self.__emit_lock: + self.__emit_buffer.append(self.process_record(record)) + if len(self.__emit_buffer) >= self.capacity: + # Trigger the timer thread to cause it to flush the buffer early. + self.__timer.trigger() - def flush(self, close_called=False): + def flush(self): """ Ensures all logging output has been flushed. """ - with self._lock: - if self.buffer: - self.flush_with_lock(close_called) + self.__flush(close_called=False) - def flush_with_lock(self, close_called): + def __flush(self, close_called): """ Ensures all logging output has been flushed. + """ + + with self.__emit_lock: + buf = self.__emit_buffer + self.__emit_buffer = [] + + # The buffer 'buf' is flushed without holding 'self.__emit_lock' to avoid causing callers of + # self.emit() to block behind the completion of a potentially long-running flush operation. + if buf: + with self.__flush_lock: + self._flush_buffer_with_lock(buf, close_called) - This version resets the buffers back to an empty list and is - intended to be overridden by subclasses. + def _flush_buffer_with_lock(self, buf, close_called): + """ + Ensures all logging output has been flushed. """ - self.buffer = [] + raise NotImplementedError("_flush_buffer_with_lock must be implemented by BufferedHandler" + " subclasses") def close(self): """ @@ -118,9 +128,10 @@ class BufferedHandler(logging.Handler): Stops the timer and flushes the buffer. """ - if self._timer is not None: - self._timer.dismiss() - self.flush(close_called=True) + if self.__timer is not None: + self.__timer.dismiss() + + self.__flush(close_called=True) logging.Handler.close(self) diff --git a/buildscripts/resmokelib/reportfile.py b/buildscripts/resmokelib/reportfile.py new file mode 100644 index 00000000000..7bf288069e2 --- /dev/null +++ b/buildscripts/resmokelib/reportfile.py @@ -0,0 +1,29 @@ +""" +Manages interactions with the report.json file. +""" + +from __future__ import absolute_import + +import json + +from . import config +from .testing import report as _report + + +def write(suites): + """ + Writes the combined report of all executions if --reportFile was + specified on the command line. + """ + + if config.REPORT_FILE is None: + return + + reports = [] + for suite in suites: + for group in suite.test_groups: + reports.extend(group.get_reports()) + + combined_report_dict = _report.TestReport.combine(*reports).as_dict() + with open(config.REPORT_FILE, "w") as fp: + json.dump(combined_report_dict, fp) diff --git a/buildscripts/resmokelib/sighandler.py b/buildscripts/resmokelib/sighandler.py new file mode 100644 index 00000000000..cbce75ca0f7 --- /dev/null +++ b/buildscripts/resmokelib/sighandler.py @@ -0,0 +1,110 @@ +""" +Utility to support asynchronously signaling the current process. +""" + +from __future__ import absolute_import + +import atexit +import os +import signal +import sys +import threading +import traceback + +_is_windows = (sys.platform == "win32") +if _is_windows: + import win32api + import win32event + +from . import reportfile + + +def register(logger, suites): + """ + On Windows, set up an event object to wait for signal, otherwise, register a signal handler + for the SIGUSR1 signal. + """ + + def _handle_sigusr1(signum, frame): + """ + Signal handler that will dump the stacks of all threads and + then write out the report file. + """ + + header_msg = "Dumping stacks due to SIGUSR1 signal" + + _dump_stacks(logger, header_msg) + reportfile.write(suites) + + def _handle_set_event(event_handle): + """ + Windows event object handler that will dump the stacks of all threads and then write out + the report file. + """ + + while True: + try: + # Wait for task time out to dump stacks. + ret = win32event.WaitForSingleObject(event_handle, win32event.INFINITE) + if ret != win32event.WAIT_OBJECT_0: + logger.error("_handle_set_event WaitForSingleObject failed: %d" % ret) + return + except win32event.error as err: + logger.error("Exception from win32event.WaitForSingleObject with error: %s" % err) + else: + header_msg = "Dumping stacks due to signal from win32event.SetEvent" + + _dump_stacks(logger, header_msg) + reportfile.write(suites) + + + # On Windows spawn a thread to wait on an event object for signal to dump stacks. For Cygwin + # platforms, we use a signal handler since it supports POSIX signals. + if _is_windows: + # Create unique event_name. + event_name = "Global\\Mongo_Python_" + str(os.getpid()) + + try: + security_attributes = None + manual_reset = False + initial_state = False + task_timeout_handle = win32event.CreateEvent(security_attributes, + manual_reset, + initial_state, + event_name) + except win32event.error as err: + logger.error("Exception from win32event.CreateEvent with error: %s" % err) + return + + # Register to close event object handle on exit. + atexit.register(win32api.CloseHandle, task_timeout_handle) + + # Create thread. + event_handler_thread = threading.Thread(target=_handle_set_event, + kwargs={"event_handle": task_timeout_handle}, + name="windows_event_handler_thread") + event_handler_thread.daemon = True + event_handler_thread.start() + else: + # Otherwise register a signal handler + signal.signal(signal.SIGUSR1, _handle_sigusr1) + + +def _dump_stacks(logger, header_msg): + """ + Signal handler that will dump the stacks of all threads. + """ + + sb = [] + sb.append(header_msg) + + frames = sys._current_frames() + sb.append("Total threads: %d" % (len(frames))) + sb.append("") + + for thread_id in frames: + stack = frames[thread_id] + sb.append("Thread %d:" % (thread_id)) + sb.append("".join(traceback.format_stack(stack))) + + logger.info("\n".join(sb)) diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py index 5d79abd6ac6..40759539305 100644 --- a/buildscripts/resmokelib/testing/executor.py +++ b/buildscripts/resmokelib/testing/executor.py @@ -78,7 +78,10 @@ class TestGroupExecutor(object): num_repeats = _config.REPEAT while num_repeats > 0: test_queue = self._make_test_queue() - self._test_group.record_start() + + partial_reports = [job.report for job in self._jobs] + self._test_group.record_start(partial_reports) + (report, interrupted) = self._run_tests(test_queue) self._test_group.record_end(report) diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py index c1d6521d5e9..3ac4b870678 100644 --- a/buildscripts/resmokelib/testing/fixtures/replicaset.py +++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py @@ -104,23 +104,25 @@ class ReplicaSetFixture(interface.ReplFixture): # Wait for the primary to be elected. client = utils.new_mongo_client(port=self.port) while True: + self.logger.info("Waiting for primary on port %d to be elected.", self.port) is_master = client.admin.command("isMaster")["ismaster"] if is_master: break - self.logger.info("Waiting for primary on port %d to be elected.", self.port) time.sleep(0.1) # Wait a little bit before trying again. + self.logger.info("Primary on port %d successfully elected.", self.port) # Wait for the secondaries to become available. for secondary in self.get_secondaries(): client = utils.new_mongo_client(port=secondary.port, read_preference=pymongo.ReadPreference.SECONDARY) while True: + self.logger.info("Waiting for secondary on port %d to become available.", + secondary.port) is_secondary = client.admin.command("isMaster")["secondary"] if is_secondary: break - self.logger.info("Waiting for secondary on port %d to become available.", - secondary.port) time.sleep(0.1) # Wait a little bit before trying again. + self.logger.info("Secondary on port %d is now available.", secondary.port) def teardown(self): running_at_start = self.is_running() diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py index e6523449636..282319a235c 100644 --- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py +++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py @@ -301,9 +301,10 @@ class _MongoSFixture(interface.Fixture): # be established. while True: # Check whether the mongos exited for some reason. - if self.mongos.poll() is not None: + exit_code = self.mongos.poll() + if exit_code is not None: raise errors.ServerFailure("Could not connect to mongos on port %d, process ended" - " unexpectedly." % (self.port)) + " unexpectedly with code %d." % (self.port, exit_code)) try: # Use a shorter connection timeout to more closely satisfy the requested deadline. diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index 87e661e2fab..edb38177fa2 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -127,9 +127,10 @@ class MongoDFixture(interface.Fixture): # be established. while True: # Check whether the mongod exited for some reason. - if self.mongod.poll() is not None: + exit_code = self.mongod.poll() + if exit_code is not None: raise errors.ServerFailure("Could not connect to mongod on port %d, process ended" - " unexpectedly." % (self.port)) + " unexpectedly with code %d." % (self.port, exit_code)) try: # Use a shorter connection timeout to more closely satisfy the requested deadline. diff --git a/buildscripts/resmokelib/testing/job.py b/buildscripts/resmokelib/testing/job.py index 4ad35542982..d1856551fd1 100644 --- a/buildscripts/resmokelib/testing/job.py +++ b/buildscripts/resmokelib/testing/job.py @@ -113,14 +113,14 @@ class Job(object): raise except errors.ServerFailure: - self.logger.error("%s marked as a failure by a hook's before_test.", - test.shortDescription()) + self.logger.exception("%s marked as a failure by a hook's before_test.", + test.shortDescription()) self._fail_test(test, sys.exc_info(), return_code=2) raise errors.StopExecution("A hook's before_test failed") except errors.TestFailure: - self.logger.error("%s marked as a failure by a hook's before_test.", - test.shortDescription()) + self.logger.exception("%s marked as a failure by a hook's before_test.", + test.shortDescription()) self._fail_test(test, sys.exc_info(), return_code=1) if config.FAIL_FAST: raise errors.StopExecution("A hook's before_test failed") @@ -147,14 +147,14 @@ class Job(object): raise except errors.ServerFailure: - self.logger.error("%s marked as a failure by a hook's after_test.", - test.shortDescription()) + self.logger.exception("%s marked as a failure by a hook's after_test.", + test.shortDescription()) self.report.setFailure(test, return_code=2) raise errors.StopExecution("A hook's after_test failed") except errors.TestFailure: - self.logger.error("%s marked as a failure by a hook's after_test.", - test.shortDescription()) + self.logger.exception("%s marked as a failure by a hook's after_test.", + test.shortDescription()) self.report.setFailure(test, return_code=1) if config.FAIL_FAST: raise errors.StopExecution("A hook's after_test failed") diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py index a5dc8744ddb..703076bff56 100644 --- a/buildscripts/resmokelib/testing/report.py +++ b/buildscripts/resmokelib/testing/report.py @@ -6,6 +6,7 @@ and timing information for the report.json file. from __future__ import absolute_import import copy +import threading import time import unittest @@ -30,6 +31,8 @@ class TestReport(unittest.TestResult): self.build_id = build_id self.build_config = build_config + self._lock = threading.Lock() + self.reset() @classmethod @@ -50,33 +53,35 @@ class TestReport(unittest.TestResult): if not isinstance(report, TestReport): raise TypeError("reports must be a list of TestReport instances") - for test_info in report.test_infos: - # If the user triggers a KeyboardInterrupt exception while a test is running, then - # it is possible for 'test_info' to be modified by a job thread later on. We make a - # shallow copy in order to ensure 'num_failed' is consistent with the actual number - # of tests that have status equal to "failed". - test_info = copy.copy(test_info) + with report._lock: + for test_info in report.test_infos: + # If the user triggers a KeyboardInterrupt exception while a test is running, + # then it is possible for 'test_info' to be modified by a job thread later on. + # We make a shallow copy in order to ensure 'num_interrupted' is consistent with + # the actual number of tests that have status equal to "timeout". + test_info = copy.copy(test_info) - # TestReport.addXX() may not have been called. - if test_info.status is None or test_info.return_code is None: - # Mark the test as having failed if it was interrupted. It might have passed if - # the suite ran to completion, but we wouldn't know for sure. - test_info.status = "fail" - test_info.return_code = -2 + # TestReport.addXX() may not have been called. + if test_info.status is None or test_info.return_code is None: + # Mark the test as having timed out if it was interrupted. It might have + # passed if the suite ran to completion, but we wouldn't know for sure. + test_info.status = "timeout" + test_info.return_code = -2 - # TestReport.stopTest() may not have been called. - if test_info.end_time is None: - # Use the current time as the time that the test finished running. - test_info.end_time = combining_time + # TestReport.stopTest() may not have been called. + if test_info.end_time is None: + # Use the current time as the time that the test finished running. + test_info.end_time = combining_time - combined_report.test_infos.append(test_info) + combined_report.test_infos.append(test_info) - combined_report.num_dynamic += report.num_dynamic + combined_report.num_dynamic += report.num_dynamic # Recompute number of success, failures, and errors. combined_report.num_succeeded = len(combined_report.get_successful()) combined_report.num_failed = len(combined_report.get_failed()) combined_report.num_errored = len(combined_report.get_errored()) + combined_report.num_interrupted = len(combined_report.get_interrupted()) return combined_report @@ -89,12 +94,10 @@ class TestReport(unittest.TestResult): test_info = _TestInfo(test.id(), dynamic) test_info.start_time = time.time() - self.test_infos.append(test_info) basename = test.basename() if dynamic: command = "(dynamic test case)" - self.num_dynamic += 1 else: command = test.as_command() self.logger.info("Running %s...\n%s", basename, command) @@ -117,6 +120,11 @@ class TestReport(unittest.TestResult): test.shortDescription(), test_info.url_endpoint) + with self._lock: + self.test_infos.append(test_info) + if dynamic: + self.num_dynamic += 1 + # Set up the test-specific logger. logger_name = "%s:%s" % (test.logger.name, test.short_name()) logger = logging.loggers.new_logger(logger_name, parent=test.logger) @@ -126,6 +134,8 @@ class TestReport(unittest.TestResult): build_config=self.build_config, test_id=test_id) + # TestReport.combine() doesn't access the '__original_loggers' attribute, so we don't bother + # protecting it with the lock. self.__original_loggers[test_info.test_id] = test.logger test.logger = logger @@ -136,8 +146,9 @@ class TestReport(unittest.TestResult): unittest.TestResult.stopTest(self, test) - test_info = self._find_test_info(test) - test_info.end_time = time.time() + with self._lock: + test_info = self._find_test_info(test) + test_info.end_time = time.time() time_taken = test_info.end_time - test_info.start_time self.logger.info("%s ran in %0.2f seconds.", test.basename(), time_taken) @@ -147,6 +158,9 @@ class TestReport(unittest.TestResult): logging.flush.close_later(test.logger) # Restore the original logger for the test. + # + # TestReport.combine() doesn't access the '__original_loggers' attribute, so we don't bother + # protecting it with the lock. test.logger = self.__original_loggers.pop(test.id()) def addError(self, test, err): @@ -156,28 +170,32 @@ class TestReport(unittest.TestResult): """ unittest.TestResult.addError(self, test, err) - self.num_errored += 1 - test_info = self._find_test_info(test) - test_info.status = "error" - test_info.return_code = test.return_code + with self._lock: + self.num_errored += 1 + + test_info = self._find_test_info(test) + test_info.status = "error" + test_info.return_code = test.return_code def setError(self, test): """ Used to change the outcome of an existing test to an error. """ - test_info = self._find_test_info(test) - if test_info.end_time is None: - raise ValueError("stopTest was not called on %s" % (test.basename())) + with self._lock: + test_info = self._find_test_info(test) + if test_info.end_time is None: + raise ValueError("stopTest was not called on %s" % (test.basename())) - test_info.status = "error" - test_info.return_code = 2 + test_info.status = "error" + test_info.return_code = 2 # Recompute number of success, failures, and errors. self.num_succeeded = len(self.get_successful()) self.num_failed = len(self.get_failed()) self.num_errored = len(self.get_errored()) + self.num_interrupted = len(self.get_interrupted()) def addFailure(self, test, err): """ @@ -186,28 +204,32 @@ class TestReport(unittest.TestResult): """ unittest.TestResult.addFailure(self, test, err) - self.num_failed += 1 - test_info = self._find_test_info(test) - test_info.status = "fail" - test_info.return_code = test.return_code + with self._lock: + self.num_failed += 1 + + test_info = self._find_test_info(test) + test_info.status = "fail" + test_info.return_code = test.return_code def setFailure(self, test, return_code=1): """ Used to change the outcome of an existing test to a failure. """ - test_info = self._find_test_info(test) - if test_info.end_time is None: - raise ValueError("stopTest was not called on %s" % (test.basename())) + with self._lock: + test_info = self._find_test_info(test) + if test_info.end_time is None: + raise ValueError("stopTest was not called on %s" % (test.basename())) - test_info.status = "error" - test_info.return_code = return_code + test_info.status = "fail" + test_info.return_code = return_code # Recompute number of success, failures, and errors. self.num_succeeded = len(self.get_successful()) self.num_failed = len(self.get_failed()) self.num_errored = len(self.get_errored()) + self.num_interrupted = len(self.get_interrupted()) def addSuccess(self, test): """ @@ -215,38 +237,57 @@ class TestReport(unittest.TestResult): """ unittest.TestResult.addSuccess(self, test) - self.num_succeeded += 1 - test_info = self._find_test_info(test) - test_info.status = "pass" - test_info.return_code = test.return_code + with self._lock: + self.num_succeeded += 1 + + test_info = self._find_test_info(test) + test_info.status = "pass" + test_info.return_code = test.return_code def wasSuccessful(self): """ Returns true if all tests executed successfully. """ - return self.num_failed == self.num_errored == 0 + + with self._lock: + return self.num_failed == self.num_errored == self.num_interrupted == 0 def get_successful(self): """ Returns the status and timing information of the tests that executed successfully. """ - return [test_info for test_info in self.test_infos if test_info.status == "pass"] + + with self._lock: + return [test_info for test_info in self.test_infos if test_info.status == "pass"] def get_failed(self): """ Returns the status and timing information of the tests that raised a failureException during their execution. """ - return [test_info for test_info in self.test_infos if test_info.status == "fail"] + + with self._lock: + return [test_info for test_info in self.test_infos if test_info.status == "fail"] def get_errored(self): """ Returns the status and timing information of the tests that raised a non-failureException during their execution. """ - return [test_info for test_info in self.test_infos if test_info.status == "error"] + + with self._lock: + return [test_info for test_info in self.test_infos if test_info.status == "error"] + + def get_interrupted(self): + """ + Returns the status and timing information of the tests that had + their execution interrupted. + """ + + with self._lock: + return [test_info for test_info in self.test_infos if test_info.status == "timeout"] def as_dict(self): """ @@ -256,42 +297,56 @@ class TestReport(unittest.TestResult): """ results = [] - for test_info in self.test_infos: - # Don't distinguish between failures and errors. - status = "pass" if test_info.status == "pass" else "fail" - - result = { - "test_file": test_info.test_id, - "status": status, - "exit_code": test_info.return_code, - "start": test_info.start_time, - "end": test_info.end_time, - "elapsed": test_info.end_time - test_info.start_time, + with self._lock: + for test_info in self.test_infos: + status = test_info.status + if status == "error": + # Don't distinguish between failures and errors. + status = "fail" + elif status == "timeout": + # Until EVG-1536 is completed, we shouldn't distinguish between failures and + # interrupted tests in the report.json file. In Evergreen, the behavior to sort + # tests with the "timeout" test status after tests with the "pass" test status + # effectively hides interrupted tests from the test results sidebar unless + # sorting by the time taken. + status = "fail" + + result = { + "test_file": test_info.test_id, + "status": status, + "exit_code": test_info.return_code, + "start": test_info.start_time, + "end": test_info.end_time, + "elapsed": test_info.end_time - test_info.start_time, + } + + if test_info.url_endpoint is not None: + result["url"] = test_info.url_endpoint + result["url_raw"] = test_info.url_endpoint + "?raw=1" + + results.append(result) + + return { + "results": results, + "failures": self.num_failed + self.num_errored + self.num_interrupted, } - if test_info.url_endpoint is not None: - result["url"] = test_info.url_endpoint - result["url_raw"] = test_info.url_endpoint + "?raw=1" - - results.append(result) - - return { - "results": results, - "failures": self.num_failed + self.num_errored, - } - def reset(self): """ Resets the test report back to its initial state. """ - self.test_infos = [] + with self._lock: + self.test_infos = [] - self.num_dynamic = 0 - self.num_succeeded = 0 - self.num_failed = 0 - self.num_errored = 0 + self.num_dynamic = 0 + self.num_succeeded = 0 + self.num_failed = 0 + self.num_errored = 0 + self.num_interrupted = 0 + # TestReport.combine() doesn't access the '__original_loggers' attribute, so we don't bother + # protecting it with the lock. self.__original_loggers = {} def _find_test_info(self, test): diff --git a/buildscripts/resmokelib/testing/testgroup.py b/buildscripts/resmokelib/testing/testgroup.py index 688d56c296d..866cc85a4c0 100644 --- a/buildscripts/resmokelib/testing/testgroup.py +++ b/buildscripts/resmokelib/testing/testgroup.py @@ -5,8 +5,10 @@ about when and how they execute. from __future__ import absolute_import +import itertools import time +from . import report as _report from . import summary as _summary @@ -30,17 +32,30 @@ class TestGroup(object): self._end_times = [] self._reports = [] + # We keep a reference to the TestReports from the currently running jobs so that we can + # report intermediate results. + self._partial_reports = None + def get_reports(self): """ - Returns the list of reports. + Returns the list of reports. If there's an execution currently + in progress, then a report for the partial results is included + in the returned list. """ + + if self._partial_reports is not None: + active_report = _report.TestReport.combine(*self._partial_reports) + return self._reports + [active_report] + return self._reports - def record_start(self): + def record_start(self, partial_reports): """ - Records the start time of an execution. + Records the start time of an execution and stores the + TestReports for currently running jobs. """ self._start_times.append(time.time()) + self._partial_reports = partial_reports def record_end(self, report): """ @@ -48,13 +63,24 @@ class TestGroup(object): """ self._end_times.append(time.time()) self._reports.append(report) + self._partial_reports = None def summarize_latest(self, sb): """ Returns a summary of the latest execution of the group and appends a summary of that execution onto the string builder 'sb'. + + If there's an execution currently in progress, then the partial + summary of that execution is appended to 'sb'. """ - return self._summarize_execution(-1, sb) + + if self._partial_reports is None: + return self._summarize_execution(-1, sb) + + active_report = _report.TestReport.combine(*self._partial_reports) + # Use the current time as the time that the test group finished running. + end_time = time.time() + return self._summarize_report(active_report, self._start_times[-1], end_time, sb) def summarize(self, sb): """ @@ -103,10 +129,26 @@ class TestGroup(object): string builder 'sb'. """ - report = self._reports[iteration] - time_taken = self._end_times[iteration] - self._start_times[iteration] + return self._summarize_report(self._reports[iteration], + self._start_times[iteration], + self._end_times[iteration], + sb) + + def _summarize_report(self, report, start_time, end_time, sb): + """ + Returns the summary information of the execution given by + 'report' that started at 'start_time' and finished at + 'end_time', and appends a summary of that execution onto the + string builder 'sb'. + """ + + time_taken = end_time - start_time - num_run = report.num_succeeded + report.num_errored + report.num_failed + # Tests that were interrupted are treated as failures because (1) the test has already been + # started and therefore isn't skipped and (2) the test has yet to finish and therefore + # cannot be said to have succeeded. + num_failed = report.num_failed + report.num_interrupted + num_run = report.num_succeeded + report.num_errored + num_failed num_skipped = len(self.tests) + report.num_dynamic - num_run if report.num_succeeded == num_run and num_skipped == 0: @@ -114,14 +156,14 @@ class TestGroup(object): return _summary.Summary(num_run, time_taken, num_run, 0, 0, 0) summary = _summary.Summary(num_run, time_taken, report.num_succeeded, num_skipped, - report.num_failed, report.num_errored) + num_failed, report.num_errored) sb.append("%d test(s) ran in %0.2f seconds" " (%d succeeded, %d were skipped, %d failed, %d errored)" % summary) - if report.num_failed > 0: + if num_failed > 0: sb.append("The following tests failed (with exit code):") - for test_info in report.get_failed(): + for test_info in itertools.chain(report.get_failed(), report.get_interrupted()): sb.append(" %s (%d)" % (test_info.test_id, test_info.return_code)) if report.num_errored > 0: diff --git a/buildscripts/resmokelib/utils/timer.py b/buildscripts/resmokelib/utils/timer.py index 80531d5db5c..a32ed99b670 100644 --- a/buildscripts/resmokelib/utils/timer.py +++ b/buildscripts/resmokelib/utils/timer.py @@ -1,10 +1,10 @@ """ Alternative to the threading.Timer class. -Enables a timer to be restarted without needing to construct a new thread -each time. This is necessary to execute periodic actions, e.g. flushing -log messages to buildlogger, while avoiding errors related to "can't start -new thread" that would otherwise occur on Windows. +Enables a timer to be restarted without needing to construct a new +thread each time. This is necessary to execute periodic actions, e.g. +flushing log messages to buildlogger, while avoiding errors related to +"can't start new thread" that would otherwise occur on Windows. """ from __future__ import absolute_import @@ -32,8 +32,8 @@ class AlarmClock(threading.Thread): self.args = args if args is not None else [] self.kwargs = kwargs if kwargs is not None else {} - self.lock = threading.Lock() - self.cond = threading.Condition(self.lock) + self.__lock = threading.Lock() + self.__cond = threading.Condition(self.__lock) self.snoozed = False # canceled for one execution self.dismissed = False # canceled for all time @@ -44,9 +44,9 @@ class AlarmClock(threading.Thread): Disables the timer. """ - with self.lock: + with self.__lock: self.dismissed = True - self.cond.notify_all() + self.__cond.notify_all() self.join() # Tidy up the started thread. @@ -54,24 +54,25 @@ class AlarmClock(threading.Thread): def snooze(self): """ - Skips the next execution of 'func' if it has not already started. + Skips the next execution of 'func' if it has not already + started. """ - with self.lock: + with self.__lock: if self.dismissed: raise ValueError("Timer cannot be snoozed if it has been dismissed") self.snoozed = True self.restarted = False - self.cond.notify_all() + self.__cond.notify_all() def reset(self): """ - Restarts the timer, causing it to wait 'interval' seconds before calling - 'func' again. + Restarts the timer, causing it to wait 'interval' seconds before + calling 'func' again. """ - with self.lock: + with self.__lock: if self.dismissed: raise ValueError("Timer cannot be reset if it has been dismissed") @@ -79,26 +80,43 @@ class AlarmClock(threading.Thread): raise ValueError("Timer cannot be reset if it has not been snoozed") self.restarted = True - self.cond.notify_all() + self.__cond.notify_all() + + def trigger(self): + """ + Signals the timer, causing 'func' to execute sooner than waiting + for 'interval' seconds to pass. + """ + + with self.__lock: + if self.dismissed: + raise ValueError("Timer cannot be triggered if it has been dismissed") + + if self.snoozed: + raise ValueError("Timer cannot be triggered if it has been snoozed") + + self.__cond.notify_all() def run(self): """ - Repeatedly calls 'func' with a delay of 'interval' seconds between executions. + Repeatedly calls 'func' with a delay of 'interval' seconds + between executions. - If the timer is snoozed before 'func' is called, then it waits to be reset. - After it has been reset, the timer will again wait 'interval' seconds and - then try to call 'func'. + If the timer is snoozed before 'func' is called, then it waits + to be reset. After it has been reset, the timer will again wait + 'interval' seconds and then try to call 'func'. - If the timer is dismissed, then no subsequent executions of 'func' are made. + If the timer is dismissed, then no subsequent executions of + 'func' are made. """ while True: - with self.lock: + with self.__lock: if self.dismissed: return # Wait for the specified amount of time. - self.cond.wait(self.interval) + self.__cond.wait(self.interval) if self.dismissed: return @@ -106,7 +124,7 @@ class AlarmClock(threading.Thread): # If the timer was snoozed, then it should wait to be reset. if self.snoozed: while not self.restarted: - self.cond.wait() + self.__cond.wait() if self.dismissed: return @@ -120,6 +138,6 @@ class AlarmClock(threading.Thread): self.func(*self.args, **self.kwargs) # Reacquire the lock. - with self.lock: + with self.__lock: # Ignore snoozes that took place while the function was being executed. self.snoozed = False diff --git a/buildscripts/smoke.py b/buildscripts/smoke.py index 93fa2b38ad3..38675139ab9 100755 --- a/buildscripts/smoke.py +++ b/buildscripts/smoke.py @@ -36,6 +36,7 @@ from datetime import datetime from itertools import izip import glob +import logging from optparse import OptionParser import os import pprint @@ -75,6 +76,12 @@ except: except: json = None +# Get relative imports to work when the package is not installed on the PYTHONPATH. +if __name__ == "__main__" and __package__ is None: + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(os.path.realpath(__file__))))) + +from buildscripts.resmokelib.core import pipe + # TODO clean this up so we don't need globals... mongo_repo = os.getcwd() #'./' @@ -167,6 +174,10 @@ class mongod(NullMongod): self.proc = None self.auth = False + self.job_object = None + self._inner_proc_pid = None + self._stdout_pipe = None + def ensure_test_dirs(self): utils.ensureDir(smoke_db_prefix + "/tmp/unittest/") utils.ensureDir(smoke_db_prefix + "/data/") @@ -262,6 +273,28 @@ class mongod(NullMongod): print "running " + " ".join(argv) self.proc = self._start(buildlogger(argv, is_global=True)) + # If the mongod process is spawned under buildlogger.py, then the first line of output + # should include the pid of the underlying mongod process. If smoke.py didn't create its own + # job object because it is already inside one, then the pid is used to attempt to terminate + # the underlying mongod process. + first_line = self.proc.stdout.readline() + match = re.search("^\[buildlogger.py\] pid: (?P[0-9]+)$", first_line.rstrip()) + if match is not None: + self._inner_proc_pid = int(match.group("pid")) + else: + # The first line of output didn't include the pid of the underlying mongod process. We + # write the first line of output to smoke.py's stdout to ensure the message doesn't get + # lost since it's possible that buildlogger.py isn't being used. + sys.stdout.write(first_line) + + logger = logging.Logger("", level=logging.DEBUG) + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(logging.Formatter(fmt="%(message)s")) + logger.addHandler(handler) + + self._stdout_pipe = pipe.LoggerPipe(logger, logging.INFO, self.proc.stdout) + self._stdout_pipe.wait_until_started() + if not self.did_mongod_start(self.port): raise Exception("Failed to start mongod") @@ -275,12 +308,15 @@ class mongod(NullMongod): synced = synced and "syncedTo" in source and source["syncedTo"] def _start(self, argv): - """In most cases, just call subprocess.Popen(). On windows, - add the started process to a new Job Object, so that any - child processes of this process can be killed with a single - call to TerminateJobObject (see self.stop()). + """In most cases, just call subprocess.Popen(). On Windows, this + method also assigns the started process to a job object if a new + one was created. This ensures that any child processes of this + process can be killed with a single call to TerminateJobObject + (see self.stop()). """ + creation_flags = 0 + if os.sys.platform == "win32": # Create a job object with the "kill on job close" # flag; this is inherited by child processes (ie @@ -288,28 +324,30 @@ class mongod(NullMongod): # and lets us terminate the whole tree of processes # rather than orphaning the mongod. import win32job + import win32process - # Magic number needed to allow job reassignment in Windows 7 - # see: MSDN - Process Creation Flags - ms684863 - CREATE_BREAKAWAY_FROM_JOB = 0x01000000 + # Don't create a job object if the current process is already inside one. + if not win32job.IsProcessInJob(win32process.GetCurrentProcess(), None): + self.job_object = win32job.CreateJobObject(None, '') - proc = Popen(argv, creationflags=CREATE_BREAKAWAY_FROM_JOB) + job_info = win32job.QueryInformationJobObject( + self.job_object, win32job.JobObjectExtendedLimitInformation) + job_info['BasicLimitInformation']['LimitFlags'] |= \ + win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + win32job.SetInformationJobObject( + self.job_object, + win32job.JobObjectExtendedLimitInformation, + job_info) - self.job_object = win32job.CreateJobObject(None, '') + # Magic number needed to allow job reassignment in Windows 7 + # see: MSDN - Process Creation Flags - ms684863 + creation_flags |= win32process.CREATE_BREAKAWAY_FROM_JOB - job_info = win32job.QueryInformationJobObject( - self.job_object, win32job.JobObjectExtendedLimitInformation) - job_info['BasicLimitInformation']['LimitFlags'] |= win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE - win32job.SetInformationJobObject( - self.job_object, - win32job.JobObjectExtendedLimitInformation, - job_info) + proc = Popen(argv, creationflags=creation_flags, stdout=PIPE, stderr=None, bufsize=0) + if self.job_object is not None: win32job.AssignProcessToJobObject(self.job_object, proc._handle) - else: - proc = Popen(argv) - return proc def stop(self): @@ -317,11 +355,53 @@ class mongod(NullMongod): print >> sys.stderr, "probable bug: self.proc unset in stop()" return try: - if os.sys.platform == "win32": + if os.sys.platform == "win32" and self.job_object is not None: + # If smoke.py created its own job object, then we clean up the spawned processes by + # terminating it. import win32job win32job.TerminateJobObject(self.job_object, -1) # Windows doesn't seem to kill the process immediately, so give it some time to die time.sleep(5) + elif os.sys.platform == "win32": + # If smoke.py didn't create its own job object, then we attempt to clean up the + # spawned processes by terminating them individually. + import win32api + import win32con + import win32event + import win32process + import winerror + + def win32_terminate(handle): + # Adapted from implementation of Popen.terminate() in subprocess.py of Python + # 2.7 because earlier versions do not catch exceptions. + try: + win32process.TerminateProcess(handle, -1) + except win32process.error as err: + # ERROR_ACCESS_DENIED (winerror=5) is received when the process has + # already died. + if err.winerror != winerror.ERROR_ACCESS_DENIED: + raise + return_code = win32process.GetExitCodeProcess(handle) + if return_code == win32con.STILL_ACTIVE: + raise + + # Terminate the mongod process underlying buildlogger.py if one exists. + if self._inner_proc_pid is not None: + # The PROCESS_TERMINATE privilege is necessary to call TerminateProcess() and + # the SYNCHRONIZE privilege is necessary to call WaitForSingleObject(). See + # https://msdn.microsoft.com/en-us/library/windows/desktop/ms684880(v=vs.85).aspx + # for more details. + required_access = win32con.PROCESS_TERMINATE | win32con.SYNCHRONIZE + inner_proc_handle = win32api.OpenProcess(required_access, + False, + self._inner_proc_pid) + try: + win32_terminate(inner_proc_handle) + win32event.WaitForSingleObject(inner_proc_handle, win32event.INFINITE) + finally: + win32api.CloseHandle(inner_proc_handle) + + win32_terminate(self.proc._handle) elif hasattr(self.proc, "terminate"): # This method added in Python 2.6 self.proc.terminate() @@ -331,6 +411,10 @@ class mongod(NullMongod): print >> sys.stderr, "error shutting down mongod" print >> sys.stderr, e self.proc.wait() + + if self._stdout_pipe is not None: + self._stdout_pipe.wait_until_finished() + sys.stderr.flush() sys.stdout.flush() diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 0dc36dd54cf..9bef88f9a0e 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -385,6 +385,57 @@ functions: rm -rf src /data/db/* mongo-coredumps.tgz ~/.aws + "set up kitchen credentials": + command: shell.exec + params: + silent: true + script: | + set -o errexit + + mkdir -p ~/.ssh ~/.aws + echo -n "${kitchen_private_key}" > ~/.ssh/kitchen.pem + chmod 0600 ~/.ssh/kitchen.pem + + cat < ~/.aws/config + [default] + region = us-east-1 + EOF + + cat < ~/.aws/credentials + [default] + aws_access_key_id = ${kitchen_aws_key} + aws_secret_access_key = ${kitchen_aws_secret} + EOF + + "run kitchen": + command: shell.exec + type: test + params: + shell: bash + working_dir: src/buildscripts/package_test + script: | + set -o errexit + + export KITCHEN_ARTIFACTS_URL="https://s3.amazonaws.com/mciuploads/${project}/${build_variant}/${revision}/artifacts/${build_id}.tgz" + export KITCHEN_SECURITY_GROUP="${kitchen_security_group}" + export KITCHEN_SSH_KEY_ID="${kitchen_ssh_key_id}" + export KITCHEN_SUBNET="${kitchen_subnet}" + + for i in {1..3} + do + if ! kitchen verify "${packager_distro}"; then + verified="false" + kitchen destroy "${packager_distro}" || true + sleep 30 + else + verified="true" + break + fi + done + + kitchen destroy "${packager_distro}" || true + test "$verified" = "true" + pre: - command: shell.track - command: shell.exec @@ -594,6 +645,15 @@ tasks: ${python|python} ${packager_script} --prefix `pwd`/.. --distros ${packager_distro} --tarball `pwd`/../mongodb-binaries.tgz -s ${version} -m HEAD -a ${packager_arch} cd .. fi + + # Create separate shell archive + mkdir -p shell-archive/build + cd shell-archive + ${platform_decompress|tar xzvf} ../mongodb-binaries.tgz + find . -mindepth 3 ! -name "mongo${exe}" -type f -exec rm {} \; # delete bin/* except bin/mongo + ${python|python} ../buildscripts/make_archive.py -o mongodb-shell.${ext|tgz} $(find mongodb-* -type f) + cd .. + - command: archive.targz_pack params: target: "artifacts.tgz" @@ -632,6 +692,16 @@ tasks: permissions: public-read content_type: ${content_type|application/x-gzip} display_name: Binaries + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/shell-archive/mongodb-shell.${ext|tgz} + remote_file: mongodb-mongo-v3.2/${build_variant}/${revision}/binaries/mongo-shell-${build_id}.${ext|tgz} + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/x-gzip} + display_name: Shell - command: s3.put params: aws_key: ${aws_key} @@ -1292,6 +1362,17 @@ tasks: resmoke_args: --suites=multiversion run_multiple_jobs: true +- <<: *task_template + name: multiversion_auth + commands: + - func: "do setup" + - func: "do multiversion setup" + - func: "run tests" + vars: + path_prefix: PATH=$PATH:/data/multiversion + resmoke_args: --suites=multiversion_auth + run_multiple_jobs: true + - <<: *task_template name: noPassthrough commands: @@ -1656,6 +1737,7 @@ tasks: - <<: *task_template name: sharding + exec_timeout_secs: 28800 # 8 hour timeout for the task overall commands: - func: "do setup" - func: "run tests" @@ -1665,6 +1747,7 @@ tasks: - <<: *task_template name: sharding_WT + exec_timeout_secs: 28800 # 8 hour timeout for the task overall commands: - func: "do setup" - func: "run tests" @@ -1699,6 +1782,7 @@ tasks: - <<: *task_template name: sharding_WT_ese + exec_timeout_secs: 28800 # 8 hour timeout for the task overall commands: - func: "do setup" - func: "run tests" @@ -1717,6 +1801,7 @@ tasks: - <<: *task_template name: sharding_auth + exec_timeout_secs: 28800 # 8 hour timeout for the task overall commands: - func: "do setup" - func: "run tests" @@ -1728,6 +1813,7 @@ tasks: name: sharding_auth_audit_WT depends_on: - name: sharding_auth + exec_timeout_secs: 28800 # 8 hour timeout for the task overall commands: - func: "do setup" - func: "run tests" @@ -1842,6 +1928,14 @@ tasks: resmoke_args: --suites=tool --storageEngine=wiredTiger run_multiple_jobs: true +- name: package + depends_on: + - name: compile + commands: + - func: "fetch artifacts" + - func: "set up kitchen credentials" + - func: "run kitchen" + - name: push patchable: false depends_on: @@ -1850,6 +1944,13 @@ tasks: commands: - func: "fetch artifacts" - func: "fetch binaries" + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + remote_file: mongodb-mongo-v3.2/${build_variant}/${revision}/binaries/mongo-shell-${build_id}.${ext|tgz} + bucket: mciuploads + local_file: src/mongo-shell.tgz - command: s3.get params: aws_key: ${aws_key} @@ -1896,23 +1997,16 @@ tasks: set -o errexit set -o verbose - mv mongo-binaries.tgz mongodb-binaries.${ext|tgz} - mv mongo-debugsymbols.tgz debugsymbols-*.${ext|tgz} || true - cp mongodb-*.${ext|tgz} mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz} - cp debugsymbols-*.${ext|tgz} mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz} || true + mv mongo-binaries.tgz mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz} + mv mongo-shell.tgz mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz} + mv mongo-debugsymbols.tgz mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz} || true mv distsrc.${ext|tgz} mongodb-src-${src_suffix}.${ext|tar.gz} || true /usr/bin/find build/ -type f | grep msi$ | xargs -I original_filename cp original_filename mongodb-win32-${push_arch}-${suffix}.msi || true - # Extract shell and banner files out of scons-made archive for separate shell archive - ${platform_decompress|tar --wildcards -zxvf} mongodb-binaries.${ext|tgz} mongodb-*/bin/mongo${exe} mongodb-*/\[A-Z\]* - - # Create new shell archive (make_archive.py uses filename to determine what format archive to create) - ${python|python} buildscripts/make_archive.py -o mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz} $(ls -d mongodb-*/bin/mongo${exe} mongodb-*/[A-Z]*) - notary-client.py --key-name "server-3.2" --auth-token-file ${workdir}/src/signing_auth_token --comment "Evergreen Automatic Signing ${revision} - ${build_variant} - ${branch_name}" --notary-url http://notary-service.build.10gen.cc:5000 --skip-missing mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz} mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz} mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz} mongodb-win32-${push_arch}-${suffix}.msi mongodb-src-${src_suffix}.${ext|tar.gz} if [ "${has_packages|}" = "true" ]; then - CURATOR_RELEASE="fab90fac3cb45c0ef411e647b3d621cbe053fe19" + CURATOR_RELEASE="88f34a9f1c79db7ea9597b6e85eb5995d03714ba" curl -L -O http://boxes.10gen.com/build/curator/curator-dist-rhel70-$CURATOR_RELEASE.tar.gz tar -zxvf curator-dist-rhel70-$CURATOR_RELEASE.tar.gz ./curator repo --config ./etc/repo_config.yaml --distro ${packager_distro} --edition ${repo_edition} --version ${version} --arch ${packager_arch} --packages repo @@ -2822,6 +2916,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: ubuntu1404 @@ -2930,6 +3027,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: ubuntu1604 @@ -3038,15 +3138,19 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: enterprise-linux-64-amazon-ami - display_name: Enterprise Amazon Linux + display_name: "Enterprise Amazon Linux" modules: - enterprise - mongo-tools run_on: - linux-64-amzn-test + batchtime: 1440 # 1 day expansions: tooltags: "-tags 'ssl sasl'" push_path: linux @@ -3114,6 +3218,9 @@ buildvariants: - name: ssl - name: sslSpecial - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: amazon @@ -3225,6 +3332,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push distros: - rhel70 @@ -3424,6 +3534,7 @@ buildvariants: - name: mmap - name: mongosTest - name: multiversion + - name: multiversion_auth - name: noPassthrough - name: noPassthrough_WT - name: noPassthroughWithMongod @@ -3468,7 +3579,7 @@ buildvariants: - rhel70 - name: windows-64-2k8-debug - display_name: Windows 2008R2 DEBUG + display_name: "! Windows 2008R2 DEBUG" modules: - mongo-tools run_on: @@ -3564,7 +3675,7 @@ buildvariants: - name: unittests - name: enterprise-windows-64 - display_name: Enterprise Windows + display_name: "* Enterprise Windows" modules: - enterprise - mongo-tools @@ -3811,13 +3922,14 @@ buildvariants: modules: - mongo-tools run_on: - - osx-1010 + - macos-1012 batchtime: 1440 # 1 day expansions: push_path: osx push_bucket: downloads.mongodb.org push_name: osx push_arch: x86_64 + gorootvars: CGO_CFLAGS=-mmacosx-version-min=10.7 CGO_LDFLAGS=-mmacosx-version-min=10.7 compile_flags: --allocator=system -j$(sysctl -n hw.logicalcpu) --release --osx-version-min=10.7 --libc++ num_jobs_available: 1 tasks: @@ -3899,7 +4011,7 @@ buildvariants: modules: - mongo-tools run_on: - - osx-1010 + - macos-1012 batchtime: 1440 # 1 day expansions: push_path: osx @@ -3907,7 +4019,8 @@ buildvariants: push_name: osx-ssl push_arch: x86_64 tooltags: "-tags ssl" - compile_flags: --ssl --allocator=system -j$(sysctl -n hw.logicalcpu) --release --osx-version-min=10.7 --libc++ + gorootvars: CGO_CPPFLAGS=-I/opt/mongodbtoolchain/v2/include CGO_CFLAGS=-mmacosx-version-min=10.7 CGO_LDFLAGS=-mmacosx-version-min=10.7 + compile_flags: --ssl --allocator=system -j$(sysctl -n hw.logicalcpu) --release --osx-version-min=10.7 --libc++ CPPPATH=/opt/mongodbtoolchain/v2/include num_jobs_available: 1 tasks: - name: compile @@ -3984,17 +4097,18 @@ buildvariants: - rhel70 - name: osx-107-debug - display_name: OS X 10.7 DEBUG + display_name: "* OS X 10.7 DEBUG" modules: - mongo-tools run_on: - - osx-1010 + - macos-1012 expansions: push_path: osx push_bucket: downloads.mongodb.org push_name: osx-debug push_arch: x86_64 num_jobs_available: 1 + gorootvars: CGO_CFLAGS=-mmacosx-version-min=10.7 CGO_LDFLAGS=-mmacosx-version-min=10.7 compile_flags: --dbg=on --opt=on --allocator=system -j$(sysctl -n hw.logicalcpu) --osx-version-min=10.7 --libc++ tasks: - name: compile @@ -4029,7 +4143,7 @@ buildvariants: - enterprise - mongo-tools run_on: - - osx-1010 + - macos-1012 batchtime: 1440 # 1 day expansions: push_path: osx @@ -4037,7 +4151,8 @@ buildvariants: push_name: osx push_arch: x86_64-enterprise tooltags: "-tags 'ssl sasl'" - compile_flags: --ssl --allocator=system -j$(sysctl -n hw.logicalcpu) --release --osx-version-min=10.7 --libc++ + gorootvars: CGO_CPPFLAGS=-I/opt/mongodbtoolchain/v2/include CGO_CFLAGS=-mmacosx-version-min=10.7 CGO_LDFLAGS=-mmacosx-version-min=10.7 + compile_flags: --ssl --allocator=system -j$(sysctl -n hw.logicalcpu) --release --osx-version-min=10.7 --libc++ CPPPATH=/opt/mongodbtoolchain/v2/include num_jobs_available: 1 tasks: - name: compile @@ -4054,8 +4169,6 @@ buildvariants: - name: replicasets_auth - name: sasl - name: sharding_csrs_upgrade_auth_WT - - name: snmp - - name: snmp_WT - name: ssl - name: sslSpecial - name: unittests @@ -4117,7 +4230,7 @@ buildvariants: - name: push - name: enterprise-rhel-62-64-bit - display_name: Enterprise RHEL 6.2 + display_name: "! Enterprise RHEL 6.2" modules: - enterprise - mongo-tools @@ -4205,6 +4318,7 @@ buildvariants: - name: mmap - name: mongosTest - name: multiversion + - name: multiversion_auth - name: noPassthrough - name: noPassthroughWithMongod - name: noPassthroughWithMongod_WT @@ -4258,6 +4372,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push distros: - rhel70 @@ -4308,6 +4425,9 @@ buildvariants: - name: ssl - name: sslSpecial - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: enterprise-rhel-71-ppc64le @@ -4662,6 +4782,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push distros: - rhel70 @@ -4771,6 +4894,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push ########################################### @@ -4824,6 +4950,9 @@ buildvariants: - name: ssl - name: sslSpecial - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: enterprise-ubuntu1404-64 @@ -4873,6 +5002,9 @@ buildvariants: - name: ssl - name: sslSpecial - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: enterprise-ubuntu1604-64 @@ -4922,6 +5054,9 @@ buildvariants: - name: ssl - name: sslSpecial - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push @@ -4977,6 +5112,9 @@ buildvariants: - name: ssl - name: sslSpecial - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: suse12 @@ -5086,6 +5224,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push @@ -5136,6 +5277,9 @@ buildvariants: - name: ssl - name: sslSpecial - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: suse11 @@ -5245,6 +5389,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push ########################################### @@ -5252,7 +5399,7 @@ buildvariants: ########################################### - name: solaris-64-bit - display_name: Solaris + display_name: "* Solaris" modules: - mongo-tools run_on: @@ -5381,6 +5528,9 @@ buildvariants: - name: ssl - name: sslSpecial - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: enterprise-debian81-64 @@ -5430,6 +5580,9 @@ buildvariants: - name: ssl - name: sslSpecial - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push @@ -5543,6 +5696,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push - name: debian81 @@ -5655,6 +5811,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: package + distros: + - ubuntu1604-packer - name: push @@ -5890,20 +6049,21 @@ buildvariants: # Experimental buildvariants # ########################################### -- name: ubuntu1410-debug-asan - display_name: ~ ASAN Enterprise SSL Ubuntu 14.10 DEBUG +- name: ubuntu1604-debug-asan + display_name: ~ ASAN Enterprise SSL Ubuntu 16.04 DEBUG modules: - enterprise - mongo-tools run_on: - - ubuntu1410-build + - ubuntu1604-build stepback: false batchtime: 1440 # 1 day expansions: tooltags: "-tags 'ssl'" - asan_symbolizer: ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer-3.5 + # We need llvm-symbolizer in the PATH for ASAN for clang-3.7 or later. + path_prefix: PATH=$PATH:/usr/lib/llvm-3.8/bin enable_lsan: LSAN_OPTIONS="suppressions=etc/lsan.suppressions" ASAN_OPTIONS=detect_leaks=1 - compile_flags: CC=/usr/bin/clang CXX=/usr/bin/clang++ --dbg=on --opt=on --allocator=system --sanitize=address --ssl -j$(grep -c ^processor /proc/cpuinfo) --nostrip VARIANT_DIR=build + compile_flags: CC=/usr/bin/clang-3.8 CXX=/usr/bin/clang++-3.8 CPPDEFINES="_GLIBCXX_USE_CXX11_ABI=0" --dbg=on --opt=on --allocator=system --sanitize=address --ssl -j$(grep -c ^processor /proc/cpuinfo) --nostrip VARIANT_DIR=build num_jobs_available: $(grep -c ^processor /proc/cpuinfo) test_flags: -j1 # Avoid starting too many mongod's under ASAN build. tasks: @@ -5967,6 +6127,7 @@ buildvariants: - name: mmap - name: mongosTest - name: multiversion + - name: multiversion_auth - name: noPassthrough - name: noPassthroughWithMongod - name: noPassthroughWithMongod_WT @@ -6018,19 +6179,20 @@ buildvariants: - name: tool_WT - name: unittests -- name: ubuntu1410-asan - display_name: ~ ASAN SSL Ubuntu 14.10 +- name: ubuntu1604-asan + display_name: ~ ASAN SSL Ubuntu 16.04 modules: - mongo-tools run_on: - - ubuntu1410-build + - ubuntu1604-build stepback: false batchtime: 1440 # 1 day expansions: tooltags: "-tags 'ssl'" - asan_symbolizer: ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer-3.5 + # We need llvm-symbolizer in the PATH for ASAN for clang-3.7 or later. + path_prefix: PATH=$PATH:/usr/lib/llvm-3.8/bin enable_lsan: LSAN_OPTIONS="suppressions=etc/lsan.suppressions" ASAN_OPTIONS=detect_leaks=1 - compile_flags: CC=/usr/bin/clang CXX=/usr/bin/clang++ --opt=on --allocator=system --sanitize=address --ssl -j$(grep -c ^processor /proc/cpuinfo) --nostrip VARIANT_DIR=build + compile_flags: CC=/usr/bin/clang-3.8 CXX=/usr/bin/clang++-3.8 CPPDEFINES="_GLIBCXX_USE_CXX11_ABI=0" --opt=on --allocator=system --sanitize=address --ssl -j$(grep -c ^processor /proc/cpuinfo) --nostrip VARIANT_DIR=build num_jobs_available: $(grep -c ^processor /proc/cpuinfo) test_flags: -j1 # Avoid starting too many mongod's under ASAN build. tasks: @@ -6039,4 +6201,4 @@ buildvariants: - name: jstestfuzz_WT - name: jstestfuzz_replication_WT - name: jstestfuzz_sharded_WT - - name: unittests \ No newline at end of file + - name: unittests diff --git a/etc/longevity.yml b/etc/longevity.yml index 0b820841616..b5cb674a354 100644 --- a/etc/longevity.yml +++ b/etc/longevity.yml @@ -43,8 +43,8 @@ post: set -e set -o verbose # call terraform destroy twice to avoid AWS timeout - yes yes | ./terraform destroy - yes yes | ./terraform destroy + yes yes | ./terraform destroy --var-file=cluster.json + yes yes | ./terraform destroy --var-file=cluster.json # clean all file to be safe rm -rf * echo "Cluster DESTROYED." @@ -57,66 +57,99 @@ functions: rm -rf ./* mkdir src mkdir work - mkdir bin - mkdir keys - pwd - ls - command: manifest.load - command: git.get_project params: directory: src revisions: # for each module include revision as : ${_rev} dsi: ${dsi_rev} - YCSB: ${YCSB_rev} - - command: git.apply_patch + - command: shell.exec params: - directory: src + working_dir: work + script: | + cat > bootstrap.yml < runtime.yml < secret - chmod 400 secret - echo "${ec2_pem}" > keys/aws.pem - chmod 400 keys/aws.pem + # AWS ssh secret key + echo "${ec2_pem}" > aws_ssh_key.pem + chmod 400 aws_ssh_key.pem + + cat > runtime_secret.yml < : ${_rev} dsi: ${dsi_rev} workloads: ${workloads_rev} - YCSB: ${YCSB_rev} - - command: git.apply_patch + - command: shell.exec params: - directory: src + working_dir: work + script: | + cat > bootstrap.yml < runtime.yml < secret - chmod 400 secret - echo "${ec2_pem}" > keys/aws.pem - chmod 400 keys/aws.pem + # AWS ssh secret key + echo "${ec2_pem}" > aws_ssh_key.pem + chmod 400 aws_ssh_key.pem + + cat > runtime_secret.yml < 1) || privileges.length > 1) { - for (var j = 0; j < privileges.length; j++) { - var p = privileges[j]; + if ((testcase.privileges.length == 1 && testcase.privileges[0].actions.length > 1) || + testcase.privileges.length > 1) { + for (var j = 0; j < testcase.privileges.length; j++) { + var p = testcase.privileges[j]; var resource = p.resource; var actions = p.actions; + // A particular privilege can explicitly specify that it should not be removed when + // testing for authorization failure. This accommodates special-case behavior for + // views in conjunction with the create and collMod commands. + if (p.removeWhenTestingAuthzFailure === false) { + continue; + } + for (var k = 0; k < actions.length; k++) { var privDoc = { resource: resource, @@ -114,27 +120,58 @@ function runOneTest(conn, t) { } } - msg = testProperAuthorization(conn, t, testcase); + // Test for proper authorization with the privileges specified in the test case. + msg = testProperAuthorization(conn, t, testcase, testcase.privileges); if (msg) { failures.push(t.testname + ": " + msg); } - // test resource pattern where collection is "" - privileges.forEach(function(j) { - if (j.resource.collection && !j.resource.collection.startsWith('system.')) { - j.resource.collection = ""; - } - }); - msg = testProperAuthorization(conn, t, testcase); + + var specialResource = function(resource) { + if (!resource) + return true; + + // Tests which use {db: "local", collection: "oplog.rs"} will not work with + // {db: "", collection: "oplog.rs"}. oplog.rs is special, and does not match with + // forDatabaseName or anyNormalResource ResourcePatterns. The same is true of + // oplog.$main, but oplog.$main is also an illegal collection name on any database + // other than local. The other collections checked for here in the local database have + // the same property as oplog.rs. + return !resource.db || !resource.collection || + resource.collection.startsWith("system.") || resource.db == "local"; + }; + + // Test for proper authorization with the test case's privileges where non-system + // collections are modified to be the empty string. + msg = testProperAuthorization( + conn, + t, + testcase, + testcase.privileges.map(function(priv) { + // Make a copy of the privilege so as not to modify the original array. + var modifiedPrivilege = Object.extend({}, priv, true); + if (modifiedPrivilege.resource.collection && !specialResource(priv.resource)) { + modifiedPrivilege.resource.collection = ""; + } + return modifiedPrivilege; + })); if (msg) { failures.push(t.testname + ": " + msg); } - // test resource pattern where database is "" - privileges.forEach(function(j) { - if (j.resource.db) { - j.resource.db = ""; - } - }); - msg = testProperAuthorization(conn, t, testcase); + + // Test for proper authorization with the test case's privileges where the database is the + // empty string. + msg = testProperAuthorization(conn, + t, + testcase, + testcase.privileges.map(function(priv) { + // Make a copy of the privilege so as not to modify the + // original array. + var modifiedPrivilege = Object.extend({}, priv, true); + if (!specialResource(priv.resource)) { + modifiedPrivilege.resource.db = ""; + } + return modifiedPrivilege; + })); if (msg) { failures.push(t.testname + ": " + msg); } diff --git a/jstests/auth/server-4892.js b/jstests/auth/server-4892.js index ef0c95c868e..4ca6318230f 100644 --- a/jstests/auth/server-4892.js +++ b/jstests/auth/server-4892.js @@ -74,11 +74,7 @@ withMongod({auth: ""}, cursor.next(); expectNumLiveCursors(mongod, 1); - cursor = null; - // NOTE(schwerin): We assume that after setting cursor = null, there are no remaining - // references - // to the cursor, and that gc() will deterministically garbage collect it. - gc(); + cursor.close(); // NOTE(schwerin): dbKillCursors gets piggybacked on subsequent messages on the // connection, so we diff --git a/jstests/concurrency/fsm_all.js b/jstests/concurrency/fsm_all.js index 35031becb89..4c45ac7b99d 100644 --- a/jstests/concurrency/fsm_all.js +++ b/jstests/concurrency/fsm_all.js @@ -4,7 +4,12 @@ load('jstests/concurrency/fsm_libs/runner.js'); var dir = 'jstests/concurrency/fsm_workloads'; -var blacklist = [].map(function(file) { +var blacklist = [ + // Disabled due to MongoDB restrictions and/or workload restrictions + + // This workload assumes it is running against a sharded cluster. + 'sharded_moveChunk_drop_shard_key_index.js', +].map(function(file) { return dir + '/' + file; }); diff --git a/jstests/concurrency/fsm_all_composed.js b/jstests/concurrency/fsm_all_composed.js index 159ff0919c9..c32b039dd32 100644 --- a/jstests/concurrency/fsm_all_composed.js +++ b/jstests/concurrency/fsm_all_composed.js @@ -16,6 +16,9 @@ var blacklist = [ // is slow and the composer doesn't honor iteration counts: 'remove_single_document_eval_nolock.js', 'update_simple_eval_nolock.js', + + // This workload assumes it is running against a sharded cluster. + 'sharded_moveChunk_drop_shard_key_index.js', ].map(function(file) { return dir + '/' + file; }); diff --git a/jstests/concurrency/fsm_all_replication.js b/jstests/concurrency/fsm_all_replication.js index 31770af9f80..93be7b7dfe2 100644 --- a/jstests/concurrency/fsm_all_replication.js +++ b/jstests/concurrency/fsm_all_replication.js @@ -9,6 +9,9 @@ var blacklist = [ 'agg_group_external.js', // uses >100MB of data, which can overwhelm test hosts 'agg_sort_external.js', // uses >100MB of data, which can overwhelm test hosts 'findAndModify_update_grow.js', // can cause OOM kills on test hosts + + // This workload assumes it is running against a sharded cluster. + 'sharded_moveChunk_drop_shard_key_index.js', ].map(function(file) { return dir + '/' + file; }); diff --git a/jstests/concurrency/fsm_all_sharded_replication.js b/jstests/concurrency/fsm_all_sharded_replication.js index d4068148d47..435277a2a47 100644 --- a/jstests/concurrency/fsm_all_sharded_replication.js +++ b/jstests/concurrency/fsm_all_sharded_replication.js @@ -9,6 +9,7 @@ var blacklist = [ 'distinct.js', // SERVER-13116 distinct isn't sharding aware 'distinct_noindex.js', // SERVER-13116 distinct isn't sharding aware 'distinct_projection.js', // SERVER-13116 distinct isn't sharding aware + 'create_database.js', // SERVER-17397 Drops of sharded namespaces may not fully succeed 'drop_database.js', // SERVER-17397 Drops of sharded namespaces may not fully succeed // Disabled due to SERVER-3645, '.count() can be wrong on sharded collections'. diff --git a/jstests/concurrency/fsm_all_sharded_replication_legacy_config_servers.js b/jstests/concurrency/fsm_all_sharded_replication_legacy_config_servers.js index 28151a05643..e4355a62789 100644 --- a/jstests/concurrency/fsm_all_sharded_replication_legacy_config_servers.js +++ b/jstests/concurrency/fsm_all_sharded_replication_legacy_config_servers.js @@ -9,6 +9,7 @@ var blacklist = [ 'distinct.js', // SERVER-13116 distinct isn't sharding aware 'distinct_noindex.js', // SERVER-13116 distinct isn't sharding aware 'distinct_projection.js', // SERVER-13116 distinct isn't sharding aware + 'create_database.js', // SERVER-17397 Drops of sharded namespaces may not fully succeed 'drop_database.js', // SERVER-17397 Drops of sharded namespaces may not fully succeed // Disabled due to SERVER-3645, '.count() can be wrong on sharded collections'. diff --git a/jstests/concurrency/fsm_all_sharded_replication_legacy_config_servers_with_balancer.js b/jstests/concurrency/fsm_all_sharded_replication_legacy_config_servers_with_balancer.js index c903331e971..167efc452ed 100644 --- a/jstests/concurrency/fsm_all_sharded_replication_legacy_config_servers_with_balancer.js +++ b/jstests/concurrency/fsm_all_sharded_replication_legacy_config_servers_with_balancer.js @@ -9,6 +9,7 @@ var blacklist = [ 'distinct.js', // SERVER-13116 distinct isn't sharding aware 'distinct_noindex.js', // SERVER-13116 distinct isn't sharding aware 'distinct_projection.js', // SERVER-13116 distinct isn't sharding aware + 'create_database.js', // SERVER-17397 Drops of sharded namespaces may not fully succeed 'drop_database.js', // SERVER-17397 Drops of sharded namespaces may not fully succeed 'remove_where.js', // SERVER-14669 Multi-removes that use $where miscount removed documents @@ -73,6 +74,11 @@ var blacklist = [ 'rename_collection_dbname_droptarget.js', 'rename_collection_droptarget.js', + // This workload assumes that the distributed lock can always be acquired when running the split + // command in its setup() function; however, a LockBusy error may be returned if the balancer is + // running. + 'sharded_moveChunk_drop_shard_key_index.js', + 'update_simple_eval.js', // eval doesn't work with sharded collections 'update_simple_eval_nolock.js', // eval doesn't work with sharded collections 'update_upsert_multi.js', // our update queries lack shard keys diff --git a/jstests/concurrency/fsm_all_sharded_replication_with_balancer.js b/jstests/concurrency/fsm_all_sharded_replication_with_balancer.js index 444f7eab3cb..9e63ec701ee 100644 --- a/jstests/concurrency/fsm_all_sharded_replication_with_balancer.js +++ b/jstests/concurrency/fsm_all_sharded_replication_with_balancer.js @@ -9,6 +9,7 @@ var blacklist = [ 'distinct.js', // SERVER-13116 distinct isn't sharding aware 'distinct_noindex.js', // SERVER-13116 distinct isn't sharding aware 'distinct_projection.js', // SERVER-13116 distinct isn't sharding aware + 'create_database.js', // SERVER-17397 Drops of sharded namespaces may not fully succeed 'drop_database.js', // SERVER-17397 Drops of sharded namespaces may not fully succeed 'remove_where.js', // SERVER-14669 Multi-removes that use $where miscount removed documents @@ -74,6 +75,11 @@ var blacklist = [ 'rename_collection_dbname_droptarget.js', 'rename_collection_droptarget.js', + // This workload assumes that the distributed lock can always be acquired when running the split + // command in its setup() function; however, a LockBusy error may be returned if the balancer is + // running. + 'sharded_moveChunk_drop_shard_key_index.js', + 'update_simple_eval.js', // eval doesn't work with sharded collections 'update_simple_eval_nolock.js', // eval doesn't work with sharded collections 'update_upsert_multi.js', // our update queries lack shard keys diff --git a/jstests/concurrency/fsm_workloads/create_database.js b/jstests/concurrency/fsm_workloads/create_database.js new file mode 100644 index 00000000000..6b996064d4d --- /dev/null +++ b/jstests/concurrency/fsm_workloads/create_database.js @@ -0,0 +1,150 @@ +'use strict'; + +/** + * create_database.js + * + * Repeatedly creates and drops a database, with the focus on creation using different name casing. + * Create using all different methods, implicitly by inserting, creating views/indexes etc. + * + * Each thread uses its own database, though sometimes threads may try to create databases with + * names that only differ in case, expecting the appriopriate error code. + */ +var $config = (function() { + + var data = { + checkCommandResult: function checkCommandResult(mayFailWithDatabaseDifferCase, res) { + if (mayFailWithDatabaseDifferCase && !res.ok) + assertAlways.commandFailedWithCode(res, ErrorCodes.DatabaseDifferCase); + else + assertAlways.commandWorked(res); + return res; + }, + + checkWriteResult: function checkWriteResult(mayFailWithDatabaseDifferCase, res) { + if (mayFailWithDatabaseDifferCase && res.hasWriteError()) { + assertAlways.eq(res.getWriteError().code, ErrorCodes.DatabaseDifferCase); + } else + assertAlways.writeOK(res); + return res; + } + }; + + var states = (function() { + function init(db, collName) { + var uniqueNr = this.tid; + var semiUniqueNr = Math.floor(uniqueNr / 2); + + // The semiUniqueDBName may clash and result in a DatabaseDifferCas error on + // creation, + // while the uniqueDBName does not clash. The unique and created variables track + // this. + this.semiUniqueDBName = + (this.tid % 2 ? 'create_database' : 'CREATE_DATABASE') + semiUniqueNr; + this.uniqueDBName = 'CreateDatabase' + uniqueNr; + this.myDB = db.getSiblingDB(this.uniqueDBName); + this.created = false; + this.unique = true; + } + + function useSemiUniqueDBName(db, collName) { + this.myDB = db.getSiblingDB(this.semiUniqueDBName); + this.unique = false; + } + + function createCollection(db, collName) { + this.created = + this.checkCommandResult(!this.unique, this.myDB.createCollection(collName)).ok; + } + + function createIndex(db, collName) { + var background = Math.random > 0.5; + var res = this.myDB.getCollection(collName).createIndex({x: 1}, {background}); + this.created |= + this.checkCommandResult(!this.unique, res).createdCollectionAutomatically; + } + + function insert(db, collName) { + this.created |= this.checkWriteResult(!this.created && !this.unique, + this.myDB.getCollection(collName).insert({x: 1})) + .nInserted == 1; + } + + function upsert(db, collName) { + this.created |= this.checkWriteResult(!this.created && !this.unique, + this.myDB.getCollection(collName).update( + {x: 1}, {x: 2}, {upsert: 1})).nUpserted == 1; + } + + function drop(db, collName) { + if (this.created) + assertAlways(this.myDB.getCollection(collName).drop()); + } + + function dropDatabase(db, collName) { + if (this.created) + assertAlways.commandWorked(this.myDB.dropDatabase()); + } + + function listDatabases(db, collName) { + for (var database of db.adminCommand({listDatabases: 1}).databases) { + var res = db.getSiblingDB(database.name).runCommand({listCollections: 1}); + assertAlways.commandWorked(res); + assertAlways.neq(database.name, this.myDB.toString(), "this DB shouldn't exist"); + } + } + + function listDatabasesNameOnly(db, collName) { + for (var database of db.adminCommand({listDatabases: 1, nameOnly: 1}).databases) { + var res = db.getSiblingDB(database.name).runCommand({listCollections: 1}); + assertAlways.commandWorked(res); + assertAlways.neq(database.name, this.myDB.toString(), "this DB shouldn't exist"); + } + } + + return { + init: init, + useSemiUniqueDBName: useSemiUniqueDBName, + createCollection: createCollection, + createIndex: createIndex, + insert: insert, + upsert: upsert, + drop: drop, + dropDatabase: dropDatabase, + listDatabases: listDatabases, + listDatabasesNameOnly: listDatabasesNameOnly, + }; + })(); + + var transitions = { + init: { + useSemiUniqueDBName: 0.25, + createCollection: 0.375, + createIndex: 0.125, + insert: 0.125, + upsert: 0.125 + }, + useSemiUniqueDBName: {createCollection: 1.00}, + createCollection: {dropDatabase: 0.25, createIndex: 0.25, insert: 0.25, upsert: 0.25}, + createIndex: {insert: 0.25, upsert: 0.25, dropDatabase: 0.5}, + insert: {dropDatabase: 0.2, drop: 0.05, insert: 0.5, upsert: 0.25}, + upsert: {dropDatabase: 0.2, drop: 0.05, insert: 0.25, upsert: 0.5}, + drop: {dropDatabase: 0.75, init: 0.25}, // OK to leave the empty database behind sometimes + dropDatabase: {init: 0.75, listDatabases: 0.15, listDatabasesNameOnly: 0.10}, + listDatabases: {init: 0.75, listDatabases: 0.15, listDatabasesNameOnly: 0.10}, + listDatabasesNameOnly: {init: 0.75, listDatabases: 0.10, listDatabasesNameOnly: 0.15}, + }; + + return { + data: data, + // We only run a few iterations to reduce the amount of data cumulatively + // written to disk by mmapv1. For example, setting 10 threads and 180 + // iterations (with an expected 6 transitions per create/drop roundtrip) + // causes this workload to write at least 32MB (.ns and .0 files) * 10 threads + // * 30 iterations worth of data to disk, or about 10GB, which can be slow on + // test hosts. + threadCount: 10, + iterations: 180, + states: states, + transitions: transitions, + }; +})(); diff --git a/jstests/concurrency/fsm_workloads/sharded_moveChunk_drop_shard_key_index.js b/jstests/concurrency/fsm_workloads/sharded_moveChunk_drop_shard_key_index.js new file mode 100644 index 00000000000..c476b7a0dab --- /dev/null +++ b/jstests/concurrency/fsm_workloads/sharded_moveChunk_drop_shard_key_index.js @@ -0,0 +1,84 @@ +'use strict'; + +/** + * sharded_moveChunk_drop_shard_key_index.js + * + * Tests that dropping the shard key index while migrating a chunk doesn't cause the shard to abort. + * + * This workload was designed to reproduce SERVER-24994. + */ + +var $config = (function() { + + var data = { + numSplitPoints: 100, + shardKey: {key: 1} + }; + + var states = { + + init: function init(db, collName) { + // No-op + }, + + moveChunk: function moveChunk(db, collName) { + var configDB = db.getSiblingDB('config'); + var shards = configDB.shards.aggregate([{$sample: {size: 1}}]).toArray(); + assertAlways.eq(1, shards.length, tojson(shards)); + + var shardName = shards[0]._id; + var chunkBoundary = Random.randInt(this.numSplitPoints); + + // We don't assert that the command succeeded when migrating a chunk because it's + // possible another thread has already started migrating a chunk. + db.adminCommand({ + moveChunk: db[collName].getFullName(), + find: {key: chunkBoundary}, + to: shardName, + _waitForDelete: true, + }); + }, + + dropIndex: function dropIndex(db, collName) { + // We don't assert that the command succeeded when dropping an index because it's + // possible another thread has already dropped this index. + db[collName].dropIndex(this.shardKey); + + // Re-create the index that was dropped. + assertAlways.commandWorked(db[collName].createIndex(this.shardKey)); + } + + }; + + var transitions = { + init: {moveChunk: 0.5, dropIndex: 0.5}, + moveChunk: {moveChunk: 0.5, dropIndex: 0.5}, + dropIndex: {moveChunk: 0.5, dropIndex: 0.5} + }; + + function setup(db, collName, cluster) { + var bulk = db[collName].initializeUnorderedBulkOp(); + for (var i = 0; i < this.numSplitPoints; ++i) { + bulk.insert({key: i}); + } + + var res = bulk.execute(); + assertAlways.writeOK(res); + assertAlways.eq(this.numSplitPoints, res.nInserted, tojson(res)); + + for (i = 0; i < this.numSplitPoints; ++i) { + assertWhenOwnColl.commandWorked( + db.adminCommand({split: db[collName].getFullName(), middle: {key: i}})); + } + } + + return { + threadCount: 10, + iterations: 100, + data: data, + states: states, + transitions: transitions, + setup: setup + }; + +})(); diff --git a/jstests/core/currentop.js b/jstests/core/currentop.js index ef948d415be..296ad69355c 100644 --- a/jstests/core/currentop.js +++ b/jstests/core/currentop.js @@ -1,82 +1,32 @@ -print("BEGIN currentop.js"); - -// test basic currentop functionality + querying of nested documents -t = db.jstests_currentop; -t.drop(); - -for (i = 0; i < 100; i++) { - t.save({"num": i}); -} - -print("count:" + t.count()); - -function ops(q) { - printjson(db.currentOp().inprog); - return db.currentOp(q).inprog; -} - -print("start shell"); - -// sleep for a second for each (of 100) documents; can be killed in between documents & test should -// complete before 100 seconds -s1 = startParallelShell("db.jstests_currentop.count( { '$where': function() { sleep(1000); } } )"); - -print("sleep"); -sleep(1000); - -print("inprog:"); -printjson(db.currentOp().inprog); -print(); -sleep(1); -print("inprog:"); -printjson(db.currentOp().inprog); -print(); - -// need to wait for read to start -print("wait have some ops"); -assert.soon(function() { - return ops({"locks.Collection": "r", "ns": "test.jstests_currentop"}).length + - ops({"locks.Collection": "R", "ns": "test.jstests_currentop"}).length >= - 1; -}, "have_some_ops"); -print("ok"); - -s2 = startParallelShell("db.jstests_currentop.update({ '$where': function() { sleep(150); } }," + - " { '$inc': {num: 1} }, false, true );"); - -o = []; - -function f() { - o = ops({"ns": "test.jstests_currentop"}); - - printjson(o); - - var writes = ops({"locks.Collection": "w", "ns": "test.jstests_currentop"}).length; - - var readops = ops({"locks.Collection": "r", "ns": "test.jstests_currentop"}); - print("readops:"); - printjson(readops); - var reads = readops.length; - - print("total: " + o.length + " w: " + writes + " r:" + reads); - - return o.length > writes && o.length > reads; -} - -print("go"); - -assert.soon(f, "f"); - -// avoid waiting for the operations to complete (if soon succeeded) -for (var i in o) { - db.killOp(o[i].opid); -} - -start = new Date(); - -// The operations running in the parallel shells may or may not have been killed. -s1({checkExitSuccess: false}); -s2({checkExitSuccess: false}); - -// don't want to pass if timeout killed the js function -assert((new Date()) - start < 30000); +/** + * Tests that long-running operations show up in currentOp and report the locks they are holding. + */ +(function() { + "use strict"; + const coll = db.jstests_currentop; + coll.drop(); + + // We fsync+lock the server to cause all subsequent write operations to block. + assert.commandWorked(db.fsyncLock()); + + const awaitInsertShell = startParallelShell(function() { + assert.writeOK(db.jstests_currentop.insert({})); + }); + + // Wait until the write appears in the currentOp output reporting that it is waiting for a lock. + assert.soon( + function() { + return db.currentOp({ + ns: coll.getFullName(), + "locks.Global": "w", + "waitingForLock": true, + }).inprog.length === 1; + }, + function() { + return "Failed to find blocked insert in currentOp() output: " + tojson(db.currentOp()); + }); + + // Unlock the server and make sure the write finishes. + assert.commandWorked(db.fsyncUnlock()); + awaitInsertShell(); +}()); diff --git a/jstests/core/dbadmin.js b/jstests/core/dbadmin.js index 061b9ff979e..2afc958c526 100644 --- a/jstests/core/dbadmin.js +++ b/jstests/core/dbadmin.js @@ -7,8 +7,12 @@ load('jstests/aggregation/extras/utils.js'); t.save({x: 1}); t.save({x: 1}); - var res = db._adminCommand("listDatabases"); - assert(res.databases && res.databases.length > 0, "listDatabases 1 " + tojson(res)); + var res = db.adminCommand("listDatabases"); + assert(res.databases && res.databases.length > 0, "listDatabases: " + tojson(res)); + + var res = db.adminCommand({listDatabases: 1, nameOnly: true}); + assert(res.databases && res.databases.length > 0 && res.totalSize === undefined, + "listDatabases nameOnly: " + tojson(res)); var now = new Date(); var x = db._adminCommand("ismaster"); diff --git a/jstests/core/diagdata.js b/jstests/core/diagdata.js new file mode 100644 index 00000000000..490e4a3eb2b --- /dev/null +++ b/jstests/core/diagdata.js @@ -0,0 +1,33 @@ +// Test that verifies getDiagnosticData returns FTDC data + +(function() { + "use strict"; + + // Verify we require admin database + assert.commandFailed(db.diagdata.runCommand("getDiagnosticData")); + + // We need to retry a few times if run this test immediately after mongod is started as FTDC may + // not have run yet. + var foundGoodDocument = false; + + for (var i = 0; i < 60; ++i) { + var result = db.adminCommand("getDiagnosticData"); + assert.commandWorked(result); + + var data = result.data; + + if (!data.hasOwnProperty("start")) { + // Wait a little longer for FTDC to start + sleep(500); + } else { + // Check for a few common properties to ensure we got data + assert(data.hasOwnProperty("serverStatus"), + "does not have 'serverStatus' in '" + tojson(data) + "'"); + assert(data.hasOwnProperty("end"), "does not have 'end' in '" + tojson(data) + "'"); + foundGoodDocument = true; + } + } + assert(foundGoodDocument, + "getDiagnosticData failed to return a non-empty command, is FTDC running?"); + +})(); diff --git a/jstests/core/drop2.js b/jstests/core/drop2.js deleted file mode 100644 index 54f6b8b78ab..00000000000 --- a/jstests/core/drop2.js +++ /dev/null @@ -1,57 +0,0 @@ -var coll = db.jstests_drop2; -coll.drop(); - -function debug(x) { - printjson(x); -} - -coll.save({}); - -function getOpId(drop) { - var inProg = db.currentOp().inprog; - debug(inProg); - for (var id in inProg) { - var op = inProg[id]; - if (drop) { - if (op.query && op.query.drop && op.query.drop == coll.getName()) { - return op.opid; - } - } else { - if (op.query && op.query.query && op.query.query.$where && op.ns == (coll + "")) { - return op.opid; - } - } - } - return null; -} - -var awaitCount = startParallelShell( - "print(\"Count thread started\");" + "db.getMongo().getCollection(\"" + (coll + "") + "\")" + - ".count( { $where: function() {" + "while( 1 ) { sleep( 1 ); } } } );" + - "print(\"Count thread terminating\");"); -countOpId = null; -assert.soon(function() { - countOpId = getOpId(false); - return countOpId; -}); - -var awaitDrop = - startParallelShell("print(\"Drop thread started\");" + "print(\"drop result: \" + " + - "db.getMongo().getCollection(\"" + (coll + "") + "\")" + ".drop() );" + - "print(\"Drop thread terminating\")"); -dropOpId = null; -assert.soon(function() { - dropOpId = getOpId(true); - return dropOpId; -}); - -db.killOp(dropOpId); -db.killOp(countOpId); - -var exitCode = awaitCount({checkExitSuccess: false}); -assert.neq(0, exitCode, "expected shell to exit abnormally due to JS execution being terminated"); - -// The drop operation may or may not have been killed. -awaitDrop({checkExitSuccess: false}); - -coll.drop(); // in SERVER-1818, this fails diff --git a/jstests/core/evalh.js b/jstests/core/evalh.js new file mode 100644 index 00000000000..e1058fbdce4 --- /dev/null +++ b/jstests/core/evalh.js @@ -0,0 +1,18 @@ +/** + * Test that db.eval does not support auth. + */ +(function() { + 'use strict'; + + assert.writeOK(db.evalprep.insert({}), "db must exist for eval to succeed"); + assert(db.evalprep.drop()); + + // The db.auth method call getMongo().auth but catches the exception. + assert.eq(0, db.eval('db.auth("reader", "reader")')); + + // Call the native implementation auth function and verify it does not exist under the db.eval + // javascript context. + assert.throws(function() { + db.eval('db.getMongo().auth("reader", "reader")'); + }); +})(); diff --git a/jstests/core/evalj.js b/jstests/core/evalj.js new file mode 100644 index 00000000000..f2326fff365 --- /dev/null +++ b/jstests/core/evalj.js @@ -0,0 +1,13 @@ +(function() { + "use strict"; + + db.col.insert({data: 5}); + + db.eval("print(5)"); + + db.system.js.insert({_id: "foo", value: Code("db.col.drop()")}); + + db.eval("print(5)"); + + assert.eq(5, db.col.findOne()["data"]); +})(); diff --git a/jstests/core/hostinfo.js b/jstests/core/hostinfo.js index 6d27b195f39..110fa8f9d09 100644 --- a/jstests/core/hostinfo.js +++ b/jstests/core/hostinfo.js @@ -31,3 +31,13 @@ if (hostinfo.os.type != "") { assert.neq(hostinfo.system.cpuArch, "" || null, "Missing CPU Architecture"); assert.neq(hostinfo.system.numaEnabled, "" || null, "Missing NUMA flag"); } + +var buildInfo = assert.commandWorked(db.runCommand({buildInfo: 1})); +if (buildInfo.buildEnvironment && buildInfo.buildEnvironment.target_arch) { + var targetArch = buildInfo.buildEnvironment.target_arch; + if (targetArch == "i386") + assert.eq(hostinfo.system.cpuAddrSize, 32); + else + assert.eq(hostinfo.system.cpuAddrSize, 64); + assert.eq(hostinfo.system.cpuAddrSize, buildInfo.bits); +} diff --git a/jstests/core/index_partial_write_ops.js b/jstests/core/index_partial_write_ops.js index b962347a26d..22c4d0dd519 100644 --- a/jstests/core/index_partial_write_ops.js +++ b/jstests/core/index_partial_write_ops.js @@ -19,7 +19,7 @@ coll.drop(); // Create partial index. - assert.commandWorked(coll.ensureIndex({x: 1}, {partialFilterExpression: {a: 1}})); + assert.commandWorked(coll.ensureIndex({x: 1}, {unique: true, partialFilterExpression: {a: 1}})); assert.writeOK(coll.insert({_id: 1, x: 5, a: 2, b: 1})); // Not in index. assert.writeOK(coll.insert({_id: 2, x: 6, a: 1, b: 1})); // In index. @@ -59,4 +59,20 @@ // Delete that does affect partial index. assert.writeOK(coll.remove({x: 6})); assert.eq(0, getNumKeys("x_1")); + + // Documents with duplicate keys that straddle the index. + assert.writeOK(coll.insert({_id: 3, x: 1, a: 1})); // In index. + assert.writeOK(coll.insert({_id: 4, x: 1, a: 0})); // Not in index. + assert.writeErrorWithCode( + coll.insert({_id: 5, x: 1, a: 1}), + ErrorCodes.DuplicateKey); // Duplicate key constraint prevents insertion. + + // Only _id 3 is in the index. + assert.eq(1, getNumKeys("x_1")); + + // Remove _id 4, _id 3 should remain in index. + assert.writeOK(coll.remove({_id: 4})); + + // _id 3 is still in the index. + assert.eq(1, getNumKeys("x_1")); })(); diff --git a/jstests/core/killop_drop_collection.js b/jstests/core/killop_drop_collection.js new file mode 100644 index 00000000000..7138ee8eda6 --- /dev/null +++ b/jstests/core/killop_drop_collection.js @@ -0,0 +1,54 @@ +/** + * A killOp command issued against a collection drop should not interfere with the drop and allow it + * to complete. Interrupting a collection drop could leave the database in an inconsistent state. + * This test confirms that killOp won't interrupt a collection drop, and that the drop occurs + * successfully. + */ +(function() { + "use strict"; + + var collectionName = "killop_drop"; + var collection = db.getCollection(collectionName); + collection.drop(); + assert.writeOK(collection.insert({x: 1})); + + // Attempt to fsyncLock the database, aborting early if the storage engine doesn't support it. + var storageEngine = jsTest.options().storageEngine; + var fsyncRes = db.fsyncLock(); + if (!fsyncRes.ok) { + assert.commandFailedWithCode(fsyncRes, ErrorCodes.CommandNotSupported); + jsTest.log("Skipping test on storage engine " + storageEngine + + ", which does not support fsyncLock."); + return; + } + + // Kick off a drop on the collection. + var useDefaultPort = null; + var noConnect = false; + var awaitDropCommand = startParallelShell(function() { + assert.commandWorked(db.getSiblingDB("test").runCommand({drop: "killop_drop"})); + }, useDefaultPort, noConnect); + + // Wait for the drop operation to appear in the db.currentOp() output. + var dropCommandOpId = null; + assert.soon(function() { + var dropOpsInProgress = db.currentOp().inprog.filter(function(op) { + return op.query && op.query.drop === collection.getName(); + }); + if (dropOpsInProgress.length > 0) { + dropCommandOpId = dropOpsInProgress[0].opid; + } + return dropCommandOpId; + }); + + // Issue a killOp for the drop command, then unlock the server. We expect that the drop + // operation was *not* killed, and that the collection was dropped successfully. + assert.commandWorked(db.killOp(dropCommandOpId)); + assert.commandWorked(db.fsyncUnlock()); + awaitDropCommand(); + + // Ensure that the collection has been dropped. + assert.eq(-1, + db.getCollectionNames().indexOf(collectionName), + "Expected collection to not appear in listCollections output after being dropped"); +}()); diff --git a/jstests/core/list_collections1.js b/jstests/core/list_collections1.js index 04acb82290b..897ebfa1ca9 100644 --- a/jstests/core/list_collections1.js +++ b/jstests/core/list_collections1.js @@ -277,8 +277,7 @@ res = mydb.runCommand("listCollections", {cursor: {batchSize: 0}}); cursor = new DBCommandCursor(mydb.getMongo(), res, 2); - cursor = null; - gc(); // Shell will send a killCursors message when cleaning up underlying cursor. + cursor.close(); cursor = new DBCommandCursor(mydb.getMongo(), res, 2); assert.throws(function() { cursor.hasNext(); diff --git a/jstests/core/list_indexes.js b/jstests/core/list_indexes.js index 520406be59f..de0f4473980 100644 --- a/jstests/core/list_indexes.js +++ b/jstests/core/list_indexes.js @@ -164,8 +164,7 @@ res = coll.runCommand("listIndexes", {cursor: {batchSize: 0}}); cursor = new DBCommandCursor(coll.getDB().getMongo(), res, 2); - cursor = null; - gc(); // Shell will send a killCursors message when cleaning up underlying cursor. + cursor.close(); cursor = new DBCommandCursor(coll.getDB().getMongo(), res, 2); assert.throws(function() { cursor.hasNext(); diff --git a/jstests/core/list_indexes_invalidation.js b/jstests/core/list_indexes_invalidation.js new file mode 100644 index 00000000000..1a7770b75fb --- /dev/null +++ b/jstests/core/list_indexes_invalidation.js @@ -0,0 +1,37 @@ +// SERVER-24963/SERVER-27930 Missing invalidation for system.indexes writes +(function() { + 'use strict'; + var collName = 'system_indexes_invalidations'; + var collNameRenamed = 'renamed_collection'; + var coll = db[collName]; + var collRenamed = db[collNameRenamed]; + + function testIndexInvalidation(isRename) { + coll.drop(); + collRenamed.drop(); + assert.commandWorked(coll.createIndexes([{a: 1}, {b: 1}, {c: 1}])); + + // Get the first two indexes. Use find on 'system.indexes' on MMAPv1, listIndexes otherwise. + var cmd = db.system.indexes.count() ? {find: 'system.indexes'} : {listIndexes: collName}; + Object.extend(cmd, {batchSize: 2}); + var res = db.runCommand(cmd); + assert.commandWorked(res, 'could not run ' + tojson(cmd)); + printjson(res); + + // Ensure the cursor has data, rename or drop the collection, and exhaust the cursor. + var cursor = new DBCommandCursor(db.getMongo(), res); + var errMsg = + 'expected more data from command ' + tojson(cmd) + ', with result ' + tojson(res); + assert(cursor.hasNext(), errMsg); + if (isRename) { + assert.commandWorked(coll.renameCollection(collNameRenamed)); + } else { + assert(coll.drop()); + } + assert.gt(cursor.itcount(), 0, errMsg); + } + + // Test that we invalidate indexes for both collection drops and renames. + testIndexInvalidation(false); + testIndexInvalidation(true); +}()); diff --git a/jstests/core/list_namespaces_invalidation.js b/jstests/core/list_namespaces_invalidation.js new file mode 100644 index 00000000000..c257421b376 --- /dev/null +++ b/jstests/core/list_namespaces_invalidation.js @@ -0,0 +1,73 @@ +// SERVER-27996/SERVER-28022 Missing invalidation for system.namespaces writes +(function() { + 'use strict'; + var dbInvalidName = 'system_namespaces_invalidations'; + var dbInvalid = db.getSiblingDB(dbInvalidName); + var num_collections = 3; + var DROP = 1; + var RENAME = 2; + var MOVE = 3; + function testNamespaceInvalidation(namespaceAction, batchSize) { + dbInvalid.dropDatabase(); + + // Create enough collections to necessitate multiple cursor batches. + for (var i = 0; i < num_collections; i++) { + assert.commandWorked(dbInvalid.createCollection('coll' + i.toString())); + } + + // Get the first two namespaces. Use find on 'system.namespaces' on MMAPv1, listCollections + // otherwise. + var cmd = dbInvalid.system.indexes.count() ? {find: 'system.namespaces'} + : {listCollections: dbInvalidName}; + Object.extend(cmd, {batchSize: batchSize}); + var res = dbInvalid.runCommand(cmd); + assert.commandWorked(res, 'could not run ' + tojson(cmd)); + printjson(res); + + // Ensure the cursor has data, invalidate the namespace, and exhaust the cursor. + var cursor = new DBCommandCursor(dbInvalid.getMongo(), res); + var errMsg = + 'expected more data from command ' + tojson(cmd) + ', with result ' + tojson(res); + assert(cursor.hasNext(), errMsg); + if (namespaceAction == RENAME) { + // Rename the collection to something that does not fit in the previously allocated + // memory for the record. + assert.commandWorked( + dbInvalid['coll1'].renameCollection('coll1' + + 'lkdsahflaksjdhfsdkljhfskladhfkahfsakfla' + + 'skfjhaslfaslfkhasklfjhsakljhdsjksahkldjslh')); + } else if (namespaceAction == DROP) { + assert(dbInvalid['coll1'].drop()); + } else if (namespaceAction == MOVE) { + var modCmd = { + collMod: 'coll1', + validator: { + $or: [ + {phone: {$type: "string"}}, + {email: {$regex: /@mongodb\.com$/}}, + {status: {$in: ["Unknown", "Incomplete"]}}, + {address: {$type: "string"}}, + {ssn: {$type: "string"}}, + {favoriteBook: {$type: "string"}}, + {favoriteColor: {$type: "string"}}, + {favoriteBeverage: {$type: "string"}}, + {favoriteDay: {$type: "string"}}, + {favoriteFood: {$type: "string"}}, + {favoriteSport: {$type: "string"}}, + {favoriteMovie: {$type: "string"}}, + {favoriteShow: {$type: "string"}} + ] + } + }; + assert.commandWorked(dbInvalid.runCommand(modCmd)); + } + assert.gt(cursor.itcount(), 0, errMsg); + } + // Test that we invalidate the old namespace record ID when we remove, rename, or move a + // namespace record. + for (var j = 2; j < 7; j++) { + testNamespaceInvalidation(DROP, j); + testNamespaceInvalidation(RENAME, j); + testNamespaceInvalidation(MOVE, j); + } +}()); diff --git a/jstests/core/max_doc_size.js b/jstests/core/max_doc_size.js index 03deeafb307..11bdc66d398 100644 --- a/jstests/core/max_doc_size.js +++ b/jstests/core/max_doc_size.js @@ -1,11 +1,7 @@ var maxBsonObjectSize = db.isMaster().maxBsonObjectSize; var docOverhead = Object.bsonsize({_id: new ObjectId(), x: ''}); var maxStrSize = maxBsonObjectSize - docOverhead; - -var maxStr = 'a'; -while (maxStr.length < maxStrSize) - maxStr += 'a'; - +var maxStr = 'a'.repeat(maxStrSize); var coll = db.max_doc_size; coll.drop(); diff --git a/jstests/core/ref.js b/jstests/core/ref.js index 02c4cb92a07..63c6bf0b472 100644 --- a/jstests/core/ref.js +++ b/jstests/core/ref.js @@ -10,6 +10,11 @@ var other = { }; db.otherthings.save(other); +// Verify that the DBPointer prototype is not serializable +assert.throws(function() { + db.things.save({a: DBPointer.prototype}); +}); + db.things.save({name: "abc"}); x = db.things.findOne(); x.o = new DBPointer("otherthings", other._id); diff --git a/jstests/core/server22053.js b/jstests/core/server22053.js new file mode 100644 index 00000000000..5549dbc95ee --- /dev/null +++ b/jstests/core/server22053.js @@ -0,0 +1,19 @@ +(function() { + + "use strict"; + var t = db.jstests_server22053; + + /* eslint-disable no-sparse-arrays */ + var s0 = [, , 3, , , 6]; + t.coll.insert({mys: s0}); + + var cur = t.coll.find(); + var doc = cur.next(); + assert.eq(6, doc['mys'].length); + assert.eq(undefined, doc['mys'][0]); + assert.eq(undefined, doc['mys'][1]); + assert.eq(3, doc['mys'][2]); + assert.eq(undefined, doc['mys'][3]); + assert.eq(undefined, doc['mys'][4]); + assert.eq(6, doc['mys'][5]); +}()); diff --git a/jstests/core/server25192.js b/jstests/core/server25192.js new file mode 100644 index 00000000000..e07cfdcf50c --- /dev/null +++ b/jstests/core/server25192.js @@ -0,0 +1,12 @@ +(function() { + + "use strict"; + + var x = {}; + + assert.doesNotThrow(function() { + Object.extend(x, {a: null}, true); + }, [], "Extending an object with a null field does not throw"); + + assert.eq(x.a, null); +}()); diff --git a/jstests/core/tailable_skip_limit.js b/jstests/core/tailable_skip_limit.js index 2f3dbd19aea..7ed0b8ae7c2 100644 --- a/jstests/core/tailable_skip_limit.js +++ b/jstests/core/tailable_skip_limit.js @@ -79,4 +79,11 @@ assert.eq(t.find().addOption(2).itcount(), 0); assert.writeOK(t.insert({a: 1})); assert.eq(t.find().addOption(2).itcount(), 1); + + // Tests that find with 'awaitData' option but without 'tailable' option results in an + // assertion. + assert.throws(function() { + var cursor = t.find().addOption(DBQuery.Option.awaitData); + cursor.next(); + }); })(); diff --git a/jstests/libs/check_log.js b/jstests/libs/check_log.js new file mode 100644 index 00000000000..edfd8b1d5cf --- /dev/null +++ b/jstests/libs/check_log.js @@ -0,0 +1,84 @@ +/* + * Helper functions which connect to a server, and check its logs for particular strings. + */ +var checkLog; + +(function() { + "use strict"; + + if (checkLog) { + return; // Protect against this file being double-loaded. + } + + checkLog = (function() { + var getGlobalLog = function(conn) { + var cmdRes; + try { + cmdRes = conn.adminCommand({getLog: 'global'}); + } catch (e) { + // Retry with network errors. + print("checkLog ignoring failure: " + e); + return null; + } + + return assert.commandWorked(cmdRes).log; + }; + + /* + * Calls the 'getLog' function at regular intervals on the provided connection 'conn' until + * the provided 'msg' is found in the logs, or 5 minutes have elapsed. Throws an exception + * on timeout. + */ + var contains = function(conn, msg) { + assert.soon( + function() { + var logMessages = getGlobalLog(conn); + if (logMessages === null) { + return false; + } + for (var i = 0; i < logMessages.length; i++) { + if (logMessages[i].indexOf(msg) != -1) { + return true; + } + } + return false; + }, + 'Could not find log entries containing the following message: ' + msg, + 5 * 60 * 1000, + 300); + }; + + /* + * Calls the 'getLog' function at regular intervals on the provided connection 'conn' until + * the provided 'msg' is found in the logs exactly 'expectedCount' times, or 5 minutes have + * elapsed. + * Throws an exception on timeout. + */ + var containsWithCount = function(conn, msg, expectedCount) { + var count = 0; + assert.soon( + function() { + var logMessages = getGlobalLog(conn); + if (logMessages === null) { + return false; + } + for (var i = 0; i < logMessages.length; i++) { + if (logMessages[i].indexOf(msg) != -1) { + count++; + } + } + + return expectedCount === count; + }, + 'Expected ' + expectedCount + ', but instead saw ' + count + + ' log entries containing the following message: ' + msg, + 5 * 60 * 1000, + 300); + }; + + return { + contains: contains, + containsWithCount: containsWithCount + }; + })(); +})(); diff --git a/jstests/libs/chunk_manipulation_util.js b/jstests/libs/chunk_manipulation_util.js index ab56b061c9a..79359eb7e10 100644 --- a/jstests/libs/chunk_manipulation_util.js +++ b/jstests/libs/chunk_manipulation_util.js @@ -101,7 +101,8 @@ function configureMoveChunkFailPoint(shardConnection, stepNumber, mode) { assert(stepNumber >= 1); assert(stepNumber <= 6); var admin = shardConnection.getDB('admin'); - admin.runCommand({configureFailPoint: 'moveChunkHangAtStep' + stepNumber, mode: mode}); + assert.commandWorked( + admin.runCommand({configureFailPoint: 'moveChunkHangAtStep' + stepNumber, mode: mode})); } // @@ -166,7 +167,8 @@ function configureMigrateFailPoint(shardConnection, stepNumber, mode) { assert(stepNumber >= 1); assert(stepNumber <= 5); var admin = shardConnection.getDB('admin'); - admin.runCommand({configureFailPoint: 'migrateThreadHangAtStep' + stepNumber, mode: mode}); + assert.commandWorked(admin.runCommand( + {configureFailPoint: 'migrateThreadHangAtStep' + stepNumber, mode: mode})); } // diff --git a/jstests/libs/election_timing_test.js b/jstests/libs/election_timing_test.js deleted file mode 100644 index d9be493c381..00000000000 --- a/jstests/libs/election_timing_test.js +++ /dev/null @@ -1,227 +0,0 @@ -/** - * ElectionTimingTest - set up a ReplSetTest and use default or provided functions to - * trigger an election. The time it takes to discover a new primary is recorded. - */ -var ElectionTimingTest = function(opts) { - // How many times do we start a new ReplSetTest. - this.testRuns = opts.testRuns || 1; - - // How many times do we step down during a ReplSetTest"s lifetime. - this.testCycles = opts.testCycles || 1; - - // The config is set to two electable nodes since we use waitForMemberState - // to wait for the electable secondary to become primary. - this.nodes = opts.nodes || [{}, {}, {rsConfig: {arbiterOnly: true}}]; - - // The name of the replica set and of the collection. - this.name = opts.name || "election_timing"; - - // Pass additional replicaSet config options. - this.settings = opts.settings || {}; - - // pv1 is the default in master and here. - this.protocolVersion = opts.hasOwnProperty("protocolVersion") ? opts.protocolVersion : 1; - - // A function that runs after the ReplSetTest is initialized. - this.testSetup = opts.testSetup || Function.prototype; - - // A function that triggers election, default is to kill the mongod process. - this.electionTrigger = opts.electionTrigger || this.stopPrimary; - - // A function that waits for new primary to be elected. - this.waitForNewPrimary = opts.waitForNewPrimary || this.waitForNewPrimary; - - // A function that cleans up after the election trigger. - this.testReset = opts.testReset || this.stopPrimaryReset; - - // The interval passed to stepdown that primaries may not seek re-election. - // We also have to wait out this interval before allowing another stepdown. - this.stepDownGuardTime = opts.stepDownGuardTime || 60; - - // Test results will be stored in these arrays. - this.testResults = []; - this.testErrors = []; - - this._runTimingTest(); -}; - -ElectionTimingTest.prototype._runTimingTest = function() { - for (var run = 0; run < this.testRuns; run++) { - var collectionName = "test." + this.name; - var cycleData = { - testRun: run, - results: [] - }; - - jsTestLog("Starting ReplSetTest for test " + this.name + " run: " + run); - this.rst = - new ReplSetTest({name: this.name, nodes: this.nodes, nodeOptions: {verbose: ""}}); - this.rst.startSet(); - - // Get the replset config and apply the settings object. - var conf = this.rst.getReplSetConfig(); - conf.settings = conf.settings || {}; - conf.settings = Object.merge(conf.settings, this.settings); - - // Explicitly setting protocolVersion. - conf.protocolVersion = this.protocolVersion; - this.rst.initiate(conf); - - // Run the user supplied testSetup() method. Typical uses would be to set up - // bridging, or wait for a particular state after initiate(). - try { - this.testSetup(); - } catch (e) { - // If testSetup() fails, we are in an unknown state, log and return. - this.testErrors.push({testRun: run, status: "testSetup() failed", error: e}); - this.rst.stopSet(); - return; - } - - // Create and populate a collection. - var primary = this.rst.getPrimary(); - - this.electionTimeoutLimitMillis = - ElectionTimingTest.calculateElectionTimeoutLimitMillis(primary); - jsTestLog('Election timeout limit: ' + this.electionTimeoutLimitMillis + ' ms'); - - var coll = primary.getCollection(collectionName); - for (var i = 0; i < 100; i++) { - assert.writeOK(coll.insert({_id: i, x: i * 3, arbitraryStr: "this is a string"})); - } - - // Run the election tests on this ReplSetTest instance. - var secondary; - for (var cycle = 0; cycle < this.testCycles; cycle++) { - // Wait for replication. - this.rst.awaitSecondaryNodes(); - this.rst.awaitReplication(); - primary = this.rst.getPrimary(); - secondary = this.rst.getSecondary(); - - jsTestLog("Starting test: " + this.name + " run: " + run + " cycle: " + cycle); - var isMasterResult = primary.getDB("admin").isMaster(); - assert.commandWorked(isMasterResult, "isMaster() failed"); - var oldElectionId = isMasterResult.electionId; - assert.neq(undefined, oldElectionId, "isMaster() failed to return a valid electionId"); - - // Time the new election. - var stepDownTime = Date.now(); - - // Run the specified election trigger method. Default is to sigstop the primary. - try { - this.electionTrigger(); - } catch (e) { - // Left empty on purpose. - } - - // Wait for the electable secondary to become primary. - try { - this.waitForNewPrimary(this.rst, secondary); - } catch (e) { - // If we didn"t find a primary, save the error, break so this - // ReplSetTest is stopped. We can"t continue from a flaky state. - this.testErrors.push( - {testRun: run, cycle: cycle, status: "new primary not elected", error: e}); - break; - } - - var electionCompleteTime = Date.now(); - - // Verify we had an election and we have a new primary. - var newPrimary = this.rst.getPrimary(); - isMasterResult = newPrimary.getDB("admin").isMaster(); - assert.commandWorked(isMasterResult, "isMaster() failed"); - var newElectionId = isMasterResult.electionId; - assert.neq(undefined, newElectionId, "isMaster() failed to return a valid electionId"); - - if (bsonWoCompare(oldElectionId, newElectionId) !== 0) { - this.testErrors.push({ - testRun: run, - cycle: cycle, - status: "electionId not changed, no election was triggered" - }); - break; - } - - if (primary.host === newPrimary.host) { - this.testErrors.push( - {testRun: run, cycle: cycle, status: "Previous primary was re-elected"}); - break; - } - - cycleData.results.push((electionCompleteTime - stepDownTime) / 1000); - - // If we are running another test on this ReplSetTest, call the reset function. - if (cycle + 1 < this.testCycles) { - try { - this.testReset(); - } catch (e) { - this.testErrors.push( - {testRun: run, cycle: cycle, status: "testReset() failed", error: e}); - break; - } - } - } - this.testResults.push(cycleData); - this.rst.stopSet(); - } -}; - -ElectionTimingTest.prototype.stopPrimary = function() { - this.originalPrimary = this.rst.getNodeId(this.rst.getPrimary()); - this.rst.stop(this.originalPrimary); -}; - -ElectionTimingTest.prototype.stopPrimaryReset = function() { - this.rst.restart(this.originalPrimary); -}; - -ElectionTimingTest.prototype.stepDownPrimary = function() { - var adminDB = this.rst.getPrimary().getDB("admin"); - adminDB.runCommand({replSetStepDown: this.stepDownGuardTime, force: true}); -}; - -ElectionTimingTest.prototype.stepDownPrimaryReset = function() { - sleep(this.stepDownGuardTime * 1000); -}; - -ElectionTimingTest.prototype.waitForNewPrimary = function(rst, secondary) { - assert.commandWorked(secondary.adminCommand({ - replSetTest: 1, - waitForMemberState: ReplSetTest.State.PRIMARY, - timeoutMillis: 60 * 1000 - }), - "node " + secondary.host + " failed to become primary"); -}; - -/** - * Calculates upper limit for actual failover time in milliseconds. - */ -ElectionTimingTest.calculateElectionTimeoutLimitMillis = function(primary) { - var configResult = assert.commandWorked(primary.adminCommand({replSetGetConfig: 1})); - var config = configResult.config; - // Protocol version is 0 if missing from config. - var protocolVersion = config.hasOwnProperty("protocolVersion") ? config.protocolVersion : 0; - var electionTimeoutMillis = 0; - var electionTimeoutOffsetLimitFraction = 0; - if (protocolVersion === 0) { - electionTimeoutMillis = 30000; // from TopologyCoordinatorImpl::VoteLease::leaseTime - electionTimeoutOffsetLimitFraction = 0; - } else { - electionTimeoutMillis = config.settings.electionTimeoutMillis; - var getParameterResult = assert.commandWorked(primary.adminCommand({ - getParameter: 1, - replElectionTimeoutOffsetLimitFraction: 1, - })); - electionTimeoutOffsetLimitFraction = - getParameterResult.replElectionTimeoutOffsetLimitFraction; - } - var assertSoonIntervalMillis = 200; // from assert.js - var applierDrainWaitMillis = 1000; // from SyncTail::tryPopAndWaitForMore() - var padding = 5000; // extra padding for slow hosts - var electionTimeoutLimitMillis = - (1 + electionTimeoutOffsetLimitFraction) * electionTimeoutMillis + applierDrainWaitMillis + - assertSoonIntervalMillis + padding; - return electionTimeoutLimitMillis; -}; diff --git a/jstests/libs/parallelTester.js b/jstests/libs/parallelTester.js index b043e3de78d..ed55078934b 100644 --- a/jstests/libs/parallelTester.js +++ b/jstests/libs/parallelTester.js @@ -139,7 +139,6 @@ if (typeof _threadInject != "undefined") { "killop.js", "run_program1.js", "notablescan.js", - "drop2.js", "dropdb_race.js", "fsync2.js", // May be placed in serialTestsArr once SERVER-4243 is fixed. "bench_test1.js", @@ -149,7 +148,6 @@ if (typeof _threadInject != "undefined") { // this has a chance to see the message "connections_opened.js", // counts connections, globally "opcounters_write_cmd.js", - "currentop.js", // SERVER-8673, plus rwlock yielding issues "set_param1.js", // changes global state "geo_update_btree2.js", // SERVER-11132 test disables table scans "update_setOnInsert.js", // SERVER-9982 @@ -168,8 +166,10 @@ if (typeof _threadInject != "undefined") { // some tests can't be run in parallel with each other var serialTestsArr = [ + // These tests use fsyncLock. parallelFilesDir + "/fsync.js", - parallelFilesDir + "/auth1.js", + parallelFilesDir + "/currentop.js", + parallelFilesDir + "/killop_drop_collection.js", // These tests expect the profiler to be on or off at specific points // during the test run. diff --git a/jstests/libs/test_background_ops.js b/jstests/libs/test_background_ops.js index ce21a636ff6..f30314234da 100644 --- a/jstests/libs/test_background_ops.js +++ b/jstests/libs/test_background_ops.js @@ -84,6 +84,10 @@ var getResult = function(mongo, name) { * Overrides the parallel shell code in mongo */ function startParallelShell(jsCode, port) { + if (TestData) { + jsCode = "TestData = " + tojson(TestData) + ";" + jsCode; + } + var x; if (port) { x = startMongoProgramNoConnect("mongo", "--port", port, "--eval", jsCode); diff --git a/jstests/libs/write_concern_util.js b/jstests/libs/write_concern_util.js new file mode 100644 index 00000000000..526acb9b6b5 --- /dev/null +++ b/jstests/libs/write_concern_util.js @@ -0,0 +1,73 @@ +/** + * Utilities for testing writeConcern. + */ + +load("jstests/libs/check_log.js"); + +// Stops replication on the given server(s). +function stopServerReplication(conn) { + if (conn.length) { + conn.forEach(function(n) { + stopServerReplication(n); + }); + return; + } + + // Clear ramlog so checkLog can't find log messages from previous times this fail point was + // enabled. + assert.commandWorked(conn.adminCommand({clearLog: 'global'})); + var errMsg = 'Failed to enable stopReplProducer failpoint.'; + assert.commandWorked( + conn.adminCommand({configureFailPoint: 'stopReplProducer', mode: 'alwaysOn'}), errMsg); + + // Wait until the fail point is actually hit. + checkLog.contains(conn, 'bgsync - stopReplProducer fail point enabled'); +} + +// Stops replication at all replicaset secondaries. +function stopReplicationOnSecondaries(rs) { + stopServerReplication(rs.getSecondaries()); +} + +// Stops replication at all shard secondaries. +function stopReplicationOnSecondariesOfAllShards(st) { + st._rsObjects.forEach(stopReplicationOnSecondaries); +} + +// Restarts replication on the given server(s). +function restartServerReplication(conn) { + if (conn.length) { + conn.forEach(function(n) { + restartServerReplication(n); + }); + return; + } + + var errMsg = 'Failed to disable stopReplProducer failpoint.'; + assert.commandWorked( + conn.getDB('admin').runCommand({configureFailPoint: 'stopReplProducer', mode: 'off'}), + errMsg); +} + +// Restarts replication at all nodes in a replicaset. +function restartReplSetReplication(rs) { + restartServerReplication(rs.nodes); +} + +// Restarts replication at all replicaset secondaries. +function restartReplicationOnSecondaries(rs) { + restartServerReplication(rs.getSecondaries()); +} + +// Restarts replication at all nodes in a sharded cluster. +function restartReplicationOnAllShards(st) { + st._rsObjects.forEach(restartReplSetReplication); + restartReplSetReplication(st.configRS); +} + +// Asserts that a writeConcernError was received. +function assertWriteConcernError(res) { + assert(res.writeConcernError, "No writeConcernError received, got: " + tojson(res)); + assert(res.writeConcernError.code); + assert(res.writeConcernError.errmsg); +} diff --git a/jstests/multiVersion/libs/multi_rs.js b/jstests/multiVersion/libs/multi_rs.js index 109db580453..673be42d3df 100644 --- a/jstests/multiVersion/libs/multi_rs.js +++ b/jstests/multiVersion/libs/multi_rs.js @@ -45,6 +45,7 @@ ReplSetTest.prototype.upgradeNode = function(node, opts, user, pwd) { if (user != undefined) { assert.eq(1, node.getDB("admin").auth(user, pwd)); } + jsTest.authenticate(node); var isMaster = node.getDB('admin').runCommand({isMaster: 1}); @@ -71,7 +72,7 @@ ReplSetTest.prototype.stepdown = function(nodeId) { var node = this.nodes[nodeId]; try { - node.getDB("admin").runCommand({replSetStepDown: 50, force: true}); + node.getDB("admin").runCommand({replSetStepDown: 300, secondaryCatchUpPeriodSecs: 60}); assert(false); } catch (ex) { print('Caught exception after stepDown cmd: ' + tojson(ex)); diff --git a/jstests/multiVersion/minor_version_tags_new_old_new.js b/jstests/multiVersion/minor_version_tags_new_old_new.js index f39b3da4c68..7adb4d868a9 100644 --- a/jstests/multiVersion/minor_version_tags_new_old_new.js +++ b/jstests/multiVersion/minor_version_tags_new_old_new.js @@ -1,271 +1,17 @@ - (function() { 'use strict'; + load("jstests/replsets/libs/tags.js"); + // 3.2.1 is the final version to use the old style replSetUpdatePosition command. var oldVersion = "3.2.1"; var newVersion = "latest"; - var nodes = { - n1: {binVersion: newVersion}, - n2: {binVersion: oldVersion}, - n3: {binVersion: newVersion}, - n4: {binVersion: oldVersion}, - n5: {binVersion: newVersion} - }; - var host = getHostName(); - var name = 'tags'; - - var replTest = new ReplSetTest({name: name, nodes: nodes, useBridge: true}); - var nodes = replTest.nodeList(); - var conns = replTest.startSet(); - var port = replTest.ports; - replTest.initiate({ - _id: name, - members: [ - { - _id: 0, - host: nodes[0], - tags: { - server: '0', - dc: 'ny', - ny: '1', - rack: 'ny.rk1', - }, - }, - { - _id: 1, - host: nodes[1], - priority: 2, - tags: { - server: '1', - dc: 'ny', - ny: '2', - rack: 'ny.rk1', - }, - }, - { - _id: 2, - host: nodes[2], - priority: 3, - tags: { - server: '2', - dc: 'ny', - ny: '3', - rack: 'ny.rk2', 2: 'this', - }, - }, - { - _id: 3, - host: nodes[3], - tags: { - server: '3', - dc: 'sf', - sf: '1', - rack: 'sf.rk1', - }, - }, - { - _id: 4, - host: nodes[4], - tags: { - server: '4', - dc: 'sf', - sf: '2', - rack: 'sf.rk2', - }, - }, - ], - settings: { - getLastErrorModes: { - '2 dc and 3 server': { - dc: 2, - server: 3, - }, - '1 and 2': { - 2: 1, - server: 1, - }, - '2': { - 2: 1, - }, - '3 and 4': { - sf: 2, - }, - '3 or 4': { - sf: 1, - }, - }, - }, - }); - - replTest.awaitReplication(); - - // Create collection to guard against timeouts due to file allocation. - assert.commandWorked(replTest.getPrimary().getDB('foo').createCollection('bar')); - replTest.awaitReplication(); - - var ensurePrimary = function(nodeId, expectedWritableNodes) { - jsTestLog('Node ' + nodeId + ' (' + replTest.nodes[nodeId].host + ') should be primary.'); - replTest.waitForState(replTest.nodes[nodeId], ReplSetTest.State.PRIMARY, 60 * 1000); - primary = replTest.getPrimary(); - primary.forceWriteMode('commands'); - var writeConcern = { - writeConcern: {w: expectedWritableNodes, wtimeout: 30 * 1000} - }; - assert.writeOK(primary.getDB('foo').bar.insert({x: 100}, writeConcern)); - return primary; - }; - - // 2 should eventually stage a priority takeover from the primary. - var primary = ensurePrimary(2, 3); - - jsTestLog('primary is now 2'); - var config = assert.commandWorked(primary.adminCommand({replSetGetConfig: 1})).config; - jsTestLog('test configuration = ' + tojson(config)); - - jsTestLog('Setting up partitions: [0-1-2] [3] [4]'); - conns[0].disconnect(conns[3]); - conns[0].disconnect(conns[4]); - conns[1].disconnect(conns[3]); - conns[1].disconnect(conns[4]); - conns[2].disconnect(conns[3]); - conns[2].disconnect(conns[4]); - conns[3].disconnect(conns[4]); - jsTestLog('Done setting up partitions'); - - jsTestLog('partitions: nodes with each set of brackets [N1, N2, N3] form a complete network.'); - jsTestLog('partitions: [0-1-2] [3] [4] (only nodes 0 and 1 can replicate from primary node 2'); - - var doc = { - x: 1 - }; - - // This timeout should be shorter in duration than the server parameter maxSyncSourceLagSecs. - // Some writes are expected to block for this 'timeout' duration before failing. - // Depending on the order of heartbeats (containing last committed op time) received - // by a node, it might hang up on its sync source. This may cause some of the write concern - // tests to fail. - var timeout = 20 * 1000; - - jsTestLog('test1'); - primary = ensurePrimary(2, 3); - - jsTestLog('Non-existent write concern should be rejected.'); - options = { - writeConcern: {w: 'blahblah', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - var result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); - assert.neq(null, result.getWriteConcernError()); - assert.eq(ErrorCodes.UnknownReplWriteConcern, - result.getWriteConcernError().code, - tojson(result.getWriteConcernError())); - - jsTestLog('Write concern "3 or 4" should fail - 3 and 4 are not connected to the primary.'); - var options = { - writeConcern: {w: '3 or 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - result = primary.getDB('foo').bar.insert(doc, options); - assert.neq(null, result.getWriteConcernError()); - assert(result.getWriteConcernError().errInfo.wtimeout); - - conns[1].reconnect(conns[4]); - jsTestLog('partitions: [0-1-2] [1-4] [3] ' + - '(all nodes besides node 3 can replicate from primary node 2)'); - primary = ensurePrimary(2, 4); - - jsTestLog('Write concern "3 or 4" should work - 4 is now connected to the primary ' + - primary.host + ' via node 1 ' + replTest.nodes[1].host); - options = { - writeConcern: {w: '3 or 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "3 and 4" should fail - 3 is not connected to the primary.'); - options = { - writeConcern: {w: '3 and 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); - assert.neq(null, result.getWriteConcernError()); - assert(result.getWriteConcernError().errInfo.wtimeout, tojson(result.getWriteConcernError())); - - conns[3].reconnect(conns[4]); - jsTestLog('partitions: [0-1-2] [1-4] [3-4] ' + - '(all secondaries can replicate from primary node 2)'); - primary = ensurePrimary(2, 5); - - jsTestLog('31004 should sync from 31001 (31026)'); - jsTestLog('31003 should sync from 31004 (31024)'); - jsTestLog('Write concern "3 and 4" should work - ' + - 'nodes 3 and 4 are connected to primary via node 1.'); - options = { - writeConcern: {w: '3 and 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "2" - writes to primary only.'); - options = { - writeConcern: {w: '2', wtimeout: 0} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "1 and 2"'); - options = { - writeConcern: {w: '1 and 2', wtimeout: 0} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "2 dc and 3 server"'); - primary = ensurePrimary(2, 5); - options = { - writeConcern: {w: '2 dc and 3 server', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Bringing down current primary node 2 ' + primary.host + - ' to allow next higher priority node 1 ' + replTest.nodes[1].host + - ' to become primary.'); - - // Is this necessary since 3 will be connected to the new primary via node 4? - conns[1].reconnect(conns[3]); - - conns[2].disconnect(conns[0]); - conns[2].disconnect(conns[1]); - - // Is this necessary when we partition node 2 off from the rest of the nodes? - replTest.stop(2); - jsTestLog('partitions: [0-1] [2] [1-3-4] ' + - '(all secondaries except down node 2 can replicate from new primary node 1)'); - - // Node 1 with slightly higher priority will take over. - jsTestLog('1 must become primary here because otherwise the other members will take too ' + - 'long timing out their old sync threads'); - primary = ensurePrimary(1, 4); - - jsTestLog('Write concern "3 and 4" should still work with new primary node 1 ' + primary.host); - options = { - writeConcern: {w: '3 and 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "2" should fail because node 2 ' + replTest.nodes[2].host + - ' is down.'); - options = { - writeConcern: {w: '2', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); - assert.neq(null, result.getWriteConcernError()); - assert(result.getWriteConcernError().errInfo.wtimeout); - - replTest.stopSet(); - jsTestLog('tags.js SUCCESS'); + var nodes = [ + {binVersion: newVersion}, + {binVersion: oldVersion}, + {binVersion: newVersion}, + {binVersion: oldVersion}, + {binVersion: newVersion} + ]; + new TagsTest({nodes: nodes, forceWriteMode: 'commands'}).run(); }()); diff --git a/jstests/multiVersion/minor_version_tags_old_new_old.js b/jstests/multiVersion/minor_version_tags_old_new_old.js index 056aab2972a..2c4eb59c245 100644 --- a/jstests/multiVersion/minor_version_tags_old_new_old.js +++ b/jstests/multiVersion/minor_version_tags_old_new_old.js @@ -1,271 +1,17 @@ - (function() { 'use strict'; + load("jstests/replsets/libs/tags.js"); + // 3.2.1 is the final version to use the old style replSetUpdatePosition command. var oldVersion = "3.2.1"; var newVersion = "latest"; - var nodes = { - n1: {binVersion: oldVersion}, - n2: {binVersion: newVersion}, - n3: {binVersion: oldVersion}, - n4: {binVersion: newVersion}, - n5: {binVersion: oldVersion} - }; - var host = getHostName(); - var name = 'tags'; - - var replTest = new ReplSetTest({name: name, nodes: nodes, useBridge: true}); - var nodes = replTest.nodeList(); - var conns = replTest.startSet(); - var port = replTest.ports; - replTest.initiate({ - _id: name, - members: [ - { - _id: 0, - host: nodes[0], - tags: { - server: '0', - dc: 'ny', - ny: '1', - rack: 'ny.rk1', - }, - }, - { - _id: 1, - host: nodes[1], - priority: 2, - tags: { - server: '1', - dc: 'ny', - ny: '2', - rack: 'ny.rk1', - }, - }, - { - _id: 2, - host: nodes[2], - priority: 3, - tags: { - server: '2', - dc: 'ny', - ny: '3', - rack: 'ny.rk2', 2: 'this', - }, - }, - { - _id: 3, - host: nodes[3], - tags: { - server: '3', - dc: 'sf', - sf: '1', - rack: 'sf.rk1', - }, - }, - { - _id: 4, - host: nodes[4], - tags: { - server: '4', - dc: 'sf', - sf: '2', - rack: 'sf.rk2', - }, - }, - ], - settings: { - getLastErrorModes: { - '2 dc and 3 server': { - dc: 2, - server: 3, - }, - '1 and 2': { - 2: 1, - server: 1, - }, - '2': { - 2: 1, - }, - '3 and 4': { - sf: 2, - }, - '3 or 4': { - sf: 1, - }, - }, - }, - }); - - replTest.awaitReplication(); - - // Create collection to guard against timeouts due to file allocation. - assert.commandWorked(replTest.getPrimary().getDB('foo').createCollection('bar')); - replTest.awaitReplication(); - - var ensurePrimary = function(nodeId, expectedWritableNodes) { - jsTestLog('Node ' + nodeId + ' (' + replTest.nodes[nodeId].host + ') should be primary.'); - replTest.waitForState(replTest.nodes[nodeId], ReplSetTest.State.PRIMARY, 60 * 1000); - primary = replTest.getPrimary(); - primary.forceWriteMode('commands'); - var writeConcern = { - writeConcern: {w: expectedWritableNodes, wtimeout: 30 * 1000} - }; - assert.writeOK(primary.getDB('foo').bar.insert({x: 100}, writeConcern)); - return primary; - }; - - // 2 should eventually stage a priority takeover from the primary. - var primary = ensurePrimary(2, 3); - - jsTestLog('primary is now 2'); - var config = assert.commandWorked(primary.adminCommand({replSetGetConfig: 1})).config; - jsTestLog('test configuration = ' + tojson(config)); - - jsTestLog('Setting up partitions: [0-1-2] [3] [4]'); - conns[0].disconnect(conns[3]); - conns[0].disconnect(conns[4]); - conns[1].disconnect(conns[3]); - conns[1].disconnect(conns[4]); - conns[2].disconnect(conns[3]); - conns[2].disconnect(conns[4]); - conns[3].disconnect(conns[4]); - jsTestLog('Done setting up partitions'); - - jsTestLog('partitions: nodes with each set of brackets [N1, N2, N3] form a complete network.'); - jsTestLog('partitions: [0-1-2] [3] [4] (only nodes 0 and 1 can replicate from primary node 2'); - - var doc = { - x: 1 - }; - - // This timeout should be shorter in duration than the server parameter maxSyncSourceLagSecs. - // Some writes are expected to block for this 'timeout' duration before failing. - // Depending on the order of heartbeats (containing last committed op time) received - // by a node, it might hang up on its sync source. This may cause some of the write concern - // tests to fail. - var timeout = 20 * 1000; - - jsTestLog('test1'); - primary = ensurePrimary(2, 3); - - jsTestLog('Non-existent write concern should be rejected.'); - options = { - writeConcern: {w: 'blahblah', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - var result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); - assert.neq(null, result.getWriteConcernError()); - assert.eq(ErrorCodes.UnknownReplWriteConcern, - result.getWriteConcernError().code, - tojson(result.getWriteConcernError())); - - jsTestLog('Write concern "3 or 4" should fail - 3 and 4 are not connected to the primary.'); - var options = { - writeConcern: {w: '3 or 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - result = primary.getDB('foo').bar.insert(doc, options); - assert.neq(null, result.getWriteConcernError()); - assert(result.getWriteConcernError().errInfo.wtimeout); - - conns[1].reconnect(conns[4]); - jsTestLog('partitions: [0-1-2] [1-4] [3] ' + - '(all nodes besides node 3 can replicate from primary node 2)'); - primary = ensurePrimary(2, 4); - - jsTestLog('Write concern "3 or 4" should work - 4 is now connected to the primary ' + - primary.host + ' via node 1 ' + replTest.nodes[1].host); - options = { - writeConcern: {w: '3 or 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "3 and 4" should fail - 3 is not connected to the primary.'); - options = { - writeConcern: {w: '3 and 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); - assert.neq(null, result.getWriteConcernError()); - assert(result.getWriteConcernError().errInfo.wtimeout, tojson(result.getWriteConcernError())); - - conns[3].reconnect(conns[4]); - jsTestLog('partitions: [0-1-2] [1-4] [3-4] ' + - '(all secondaries can replicate from primary node 2)'); - primary = ensurePrimary(2, 5); - - jsTestLog('31004 should sync from 31001 (31026)'); - jsTestLog('31003 should sync from 31004 (31024)'); - jsTestLog('Write concern "3 and 4" should work - ' + - 'nodes 3 and 4 are connected to primary via node 1.'); - options = { - writeConcern: {w: '3 and 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "2" - writes to primary only.'); - options = { - writeConcern: {w: '2', wtimeout: 0} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "1 and 2"'); - options = { - writeConcern: {w: '1 and 2', wtimeout: 0} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "2 dc and 3 server"'); - primary = ensurePrimary(2, 5); - options = { - writeConcern: {w: '2 dc and 3 server', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Bringing down current primary node 2 ' + primary.host + - ' to allow next higher priority node 1 ' + replTest.nodes[1].host + - ' to become primary.'); - - // Is this necessary since 3 will be connected to the new primary via node 4? - conns[1].reconnect(conns[3]); - - conns[2].disconnect(conns[0]); - conns[2].disconnect(conns[1]); - - // Is this necessary when we partition node 2 off from the rest of the nodes? - replTest.stop(2); - jsTestLog('partitions: [0-1] [2] [1-3-4] ' + - '(all secondaries except down node 2 can replicate from new primary node 1)'); - - // Node 1 with slightly higher priority will take over. - jsTestLog('1 must become primary here because otherwise the other members will take too ' + - 'long timing out their old sync threads'); - primary = ensurePrimary(1, 4); - - jsTestLog('Write concern "3 and 4" should still work with new primary node 1 ' + primary.host); - options = { - writeConcern: {w: '3 and 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "2" should fail because node 2 ' + replTest.nodes[2].host + - ' is down.'); - options = { - writeConcern: {w: '2', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); - assert.neq(null, result.getWriteConcernError()); - assert(result.getWriteConcernError().errInfo.wtimeout); - - replTest.stopSet(); - jsTestLog('tags.js SUCCESS'); + var nodes = [ + {binVersion: oldVersion}, + {binVersion: newVersion}, + {binVersion: oldVersion}, + {binVersion: newVersion}, + {binVersion: oldVersion} + ]; + new TagsTest({nodes: nodes, forceWriteMode: 'commands'}).run(); }()); diff --git a/jstests/noPassthrough/command_line_parsing.js b/jstests/noPassthrough/command_line_parsing.js index d2e02bb9865..06246e72084 100644 --- a/jstests/noPassthrough/command_line_parsing.js +++ b/jstests/noPassthrough/command_line_parsing.js @@ -25,7 +25,7 @@ var m2expected = { var m2result = m2.getDB("admin").runCommand("getCmdLineOpts"); // remove variables that depend on the way the test is started. -delete m2result.parsed.nopreallocj; +delete m2result.parsed.storage.mmapv1; delete m2result.parsed.setParameter; delete m2result.parsed.storage.engine; delete m2result.parsed.storage.journal; @@ -49,7 +49,7 @@ var m3expected = { var m3result = m3.getDB("admin").runCommand("getCmdLineOpts"); // remove variables that depend on the way the test is started. -delete m3result.parsed.nopreallocj; +delete m3result.parsed.storage.mmapv1; delete m3result.parsed.setParameter; delete m3result.parsed.storage.engine; delete m3result.parsed.storage.journal; diff --git a/jstests/noPassthrough/indexbg1.js b/jstests/noPassthrough/indexbg1.js index 62e3617fe9f..00670f3f2db 100644 --- a/jstests/noPassthrough/indexbg1.js +++ b/jstests/noPassthrough/indexbg1.js @@ -14,13 +14,15 @@ var resetParallel = function() { parallel().drop(); }; +// Return the PID to call `waitpid` on for clean shutdown. var doParallel = function(work) { resetParallel(); print("doParallel: " + work); - startMongoProgramNoConnect("mongo", - "--eval", - work + "; db." + baseName + "_parallelStatus.save( {done:1} );", - db.getMongo().host); + return startMongoProgramNoConnect( + "mongo", + "--eval", + work + "; db." + baseName + "_parallelStatus.save( {done:1} );", + db.getMongo().host); }; var doneParallel = function() { @@ -34,6 +36,7 @@ var waitParallel = function() { }; var size = 400 * 1000; +var bgIndexBuildPid; while (1) { // if indexing finishes before we can run checks, try indexing w/ more data print("size: " + size); @@ -48,7 +51,7 @@ while (1) { // if indexing finishes before we can run checks, try indexing w/ m assert.writeOK(bulk.execute()); assert.eq(size, t.count()); - doParallel(fullName + ".ensureIndex( {i:1}, {background:true} )"); + bgIndexBuildPid = doParallel(fullName + ".ensureIndex( {i:1}, {background:true} )"); try { // wait for indexing to start print("wait for indexing to start"); @@ -98,12 +101,17 @@ while (1) { // if indexing finishes before we can run checks, try indexing w/ m break; } print("indexing finished too soon, retrying..."); + // Although the index build finished, ensure the shell has exited. + waitProgram(bgIndexBuildPid); size *= 2; assert(size < 200000000, "unable to run checks in parallel with index creation"); } print("our tests done, waiting for parallel to finish"); waitParallel(); +// Ensure the shell has exited cleanly. Otherwise the test harness may send a SIGTERM which can lead +// to a false test failure. +waitProgram(bgIndexBuildPid); print("finished"); assert.eq(1, t.count({i: -10})); diff --git a/jstests/noPassthrough/indexbg2.js b/jstests/noPassthrough/indexbg2.js index 5fcd975a98f..e9ac45c8b78 100644 --- a/jstests/noPassthrough/indexbg2.js +++ b/jstests/noPassthrough/indexbg2.js @@ -14,12 +14,14 @@ var resetParallel = function() { parallel().drop(); }; +// Return the PID to call `waitpid` on for clean shutdown. var doParallel = function(work) { resetParallel(); - startMongoProgramNoConnect("mongo", - "--eval", - work + "; db." + baseName + "_parallelStatus.save( {done:1} );", - db.getMongo().host); + return startMongoProgramNoConnect( + "mongo", + "--eval", + work + "; db." + baseName + "_parallelStatus.save( {done:1} );", + db.getMongo().host); }; var doneParallel = function() { @@ -35,6 +37,7 @@ var waitParallel = function() { var doTest = function() { "use strict"; var size = 10000; + var bgIndexBuildPid; while (1) { // if indexing finishes before we can run checks, try indexing w/ more data print("size: " + size); var fullName = "db." + baseName; @@ -46,7 +49,8 @@ var doTest = function() { } assert.eq(size, t.count()); - doParallel(fullName + ".ensureIndex( {i:1}, {background:true, unique:true} )"); + bgIndexBuildPid = + doParallel(fullName + ".ensureIndex( {i:1}, {background:true, unique:true} )"); try { // wait for indexing to start assert.soon(function() { @@ -59,13 +63,19 @@ var doTest = function() { // wait for parallel status to update to reflect indexing status sleep(1000); if (!doneParallel()) { + waitProgram(bgIndexBuildPid); throw e; } } if (!doneParallel()) { + // Ensure the shell has exited cleanly. Otherwise the test harness may send a SIGTERM + // which can lead to a false test failure. + waitProgram(bgIndexBuildPid); break; } print("indexing finished too soon, retrying..."); + // Although the index build finished, ensure the shell has exited. + waitProgram(bgIndexBuildPid); size *= 2; assert(size < 5000000, "unable to run checks in parallel with index creation"); } diff --git a/jstests/noPassthrough/read_majority_reads.js b/jstests/noPassthrough/read_majority_reads.js new file mode 100644 index 00000000000..5c169e3bed7 --- /dev/null +++ b/jstests/noPassthrough/read_majority_reads.js @@ -0,0 +1,270 @@ +/** + * Tests that read operations with readConcern majority only see committed data. + * + * The following read operations are tested: + * - find command + * - aggregation + * - distinct + * - group + * - count + * - parallelCollectionScan + * - geoNear + * - geoSearch + * + * Each operation is tested on a single node, and (if supported) through mongos on both sharded and + * unsharded collections. Mongos doesn't directly handle readConcern majority, but these tests + * should ensure that it correctly propagates the setting to the shards when running commands. + */ + +(function() { + 'use strict'; + + function makeCursor(mongo, result) { + return new DBCommandCursor(mongo, result); + } + + // These test cases are functions that return a cursor of the documents in collections without + // fetching them yet. + var cursorTestCases = { + find: function(coll) { + return makeCursor(coll.getMongo(), + assert.commandWorked(coll.runCommand( + 'find', {readConcern: {level: 'majority'}, batchSize: 0}))); + }, + aggregate: function(coll) { + return makeCursor( + coll.getMongo(), + assert.commandWorked(coll.runCommand( + 'aggregate', + {readConcern: {level: 'majority'}, cursor: {batchSize: 0}, pipeline: []}))); + }, + parallelCollectionScan: function(coll) { + var res = coll.runCommand('parallelCollectionScan', + {readConcern: {level: 'majority'}, numCursors: 1}); + assert.commandWorked(res); + assert.eq(res.cursors.length, 1, tojson(res)); + return makeCursor(coll.getMongo(), res.cursors[0]); + }, + }; + + // These test cases have a run method that will be passed a collection with a single object with + // _id: 1 and a state field that equals either "before" or "after". The collection will also + // contain both a 2dsphere and a geoHaystack index to enable testing commands that depend on + // them. The return value from the run method is expected to be the value of expectedBefore or + // expectedAfter depending on the state of the state field. + var nonCursorTestCases = { + count_before: { + run: function(coll) { + var res = coll.runCommand( + 'count', {readConcern: {level: 'majority'}, query: {state: 'before'}}); + assert.commandWorked(res); + return res.n; + }, + expectedBefore: 1, + expectedAfter: 0, + }, + count_after: { + run: function(coll) { + var res = coll.runCommand( + 'count', {readConcern: {level: 'majority'}, query: {state: 'after'}}); + assert.commandWorked(res); + return res.n; + }, + expectedBefore: 0, + expectedAfter: 1, + }, + distinct: { + run: function(coll) { + var res = + coll.runCommand('distinct', {readConcern: {level: 'majority'}, key: 'state'}); + assert.commandWorked(res); + assert.eq(res.values.length, 1, tojson(res)); + return res.values[0]; + }, + expectedBefore: 'before', + expectedAfter: 'after', + }, + geoNear: { + run: function(coll) { + var res = coll.runCommand('geoNear', + { + readConcern: {level: 'majority'}, + near: [0, 0], + spherical: true, + }); + assert.commandWorked(res); + assert.eq(res.results.length, 1, tojson(res)); + return res.results[0].obj.state; + }, + expectedBefore: 'before', + expectedAfter: 'after', + }, + geoSearch: { + run: function(coll) { + var res = coll.runCommand('geoSearch', + { + readConcern: {level: 'majority'}, + near: [0, 0], + search: {_id: 1}, // Needed due to SERVER-23158. + maxDistance: 1, + }); + assert.commandWorked(res); + assert.eq(res.results.length, 1, tojson(res)); + return res.results[0].state; + }, + expectedBefore: 'before', + expectedAfter: 'after', + }, + group: { + run: function(coll) { + var res = coll.runCommand({ + 'group': { + ns: coll.getName(), + key: {_id: 1}, + initial: {}, + $reduce: function(curr, result) { + result.state = curr.state; + }, + }, + readConcern: {level: 'majority'}, + }); + assert.commandWorked(res); + assert.eq(res.retval.length, 1, tojson(res)); + return res.retval[0].state; + }, + expectedBefore: 'before', + expectedAfter: 'after', + }, + }; + + function runTests(coll, mongodConnection) { + function makeSnapshot() { + return assert.commandWorked(mongodConnection.adminCommand("makeSnapshot")).name; + } + function setCommittedSnapshot(snapshot) { + assert.commandWorked(mongodConnection.adminCommand({"setCommittedSnapshot": snapshot})); + } + + for (var testName in cursorTestCases) { + jsTestLog('Running ' + testName + ' against ' + coll.toString()); + var getCursor = cursorTestCases[testName]; + + // Setup initial state. + assert.writeOK(coll.remove({})); + assert.writeOK(coll.save({_id: 1, state: 'before'})); + setCommittedSnapshot(makeSnapshot()); + + // Check initial conditions. + assert.eq(getCursor(coll).next().state, 'before'); + + // Change state without making it committed. + assert.writeOK(coll.save({_id: 1, state: 'after'})); + + // Cursor still sees old state. + assert.eq(getCursor(coll).next().state, 'before'); + + // Create a cursor before the update is visible. + var oldCursor = getCursor(coll); + + // Making a snapshot doesn't make the update visible yet. + var snapshot = makeSnapshot(); + assert.eq(getCursor(coll).next().state, 'before'); + + // Setting it as committed does for both new and old cursors. + setCommittedSnapshot(snapshot); + assert.eq(getCursor(coll).next().state, 'after'); + assert.eq(oldCursor.next().state, 'after'); + } + + assert.commandWorked(coll.ensureIndex({point: '2dsphere'})); + assert.commandWorked(coll.ensureIndex({point: 'geoHaystack', _id: 1}, {bucketSize: 1})); + for (var testName in nonCursorTestCases) { + jsTestLog('Running ' + testName + ' against ' + coll.toString()); + var getResult = nonCursorTestCases[testName].run; + var expectedBefore = nonCursorTestCases[testName].expectedBefore; + var expectedAfter = nonCursorTestCases[testName].expectedAfter; + + // Setup initial state. + assert.writeOK(coll.remove({})); + assert.writeOK(coll.save({_id: 1, state: 'before', point: [0, 0]})); + setCommittedSnapshot(makeSnapshot()); + + // Check initial conditions. + assert.eq(getResult(coll), expectedBefore); + + // Change state without making it committed. + assert.writeOK(coll.save({_id: 1, state: 'after', point: [0, 0]})); + + // Cursor still sees old state. + assert.eq(getResult(coll), expectedBefore); + + // Making a snapshot doesn't make the update visible yet. + var snapshot = makeSnapshot(); + assert.eq(getResult(coll), expectedBefore); + + // Setting it as committed does. + setCommittedSnapshot(snapshot); + assert.eq(getResult(coll), expectedAfter); + } + } + + var mongod = MongoRunner.runMongod({setParameter: 'testingSnapshotBehaviorInIsolation=true'}); + assert.neq( + null, + mongod, + 'mongod was unable to start with the testingSnapshotBehaviorInIsolation parameter enabled'); + + if (!mongod.adminCommand('serverStatus').storageEngine.supportsCommittedReads) { + print("Skipping read_majority_reads.js since storageEngine doesn't support it."); + return; + } + + (function testSingleNode() { + var db = mongod.getDB("singleNode"); + runTests(db.collection, mongod); + })(); + + var shardingTest = new ShardingTest({ + shards: 0, // We use the existing mongod. + mongos: 1, + }); + shardingTest.adminCommand({addShard: mongod.host}); + + // Remove tests of commands that aren't supported at all through mongos, even on unsharded + // collections. + ['parallelCollectionScan', 'geoSearch'].forEach(function(cmd) { + // Make sure it really isn't supported. + assert.eq(shardingTest.getDB('test').coll.runCommand(cmd).code, ErrorCodes.CommandNotFound); + delete cursorTestCases[cmd]; + delete nonCursorTestCases[cmd]; + }); + + (function testUnshardedDBThroughMongos() { + var db = shardingTest.getDB("throughMongos"); + runTests(db.unshardedDB, mongod); + })(); + + shardingTest.adminCommand({enableSharding: 'throughMongos'}); + + (function testUnshardedCollectionThroughMongos() { + var db = shardingTest.getDB("throughMongos"); + runTests(db.unshardedCollection, mongod); + })(); + + (function testShardedCollectionThroughMongos() { + var db = shardingTest.getDB("throughMongos"); + var collection = db.shardedCollection; + shardingTest.adminCommand({shardCollection: collection.getFullName(), key: {_id: 1}}); + + // The group command isn't supported on sharded collections. It also uses a weird syntax so + // code to check that it isn't supported can't be reused with other commands. + assert.eq(collection.runCommand({'group': {ns: collection.getName()}}).code, + ErrorCodes.IllegalOperation); + delete nonCursorTestCases.group; + + runTests(collection, mongod); + })(); + + shardingTest.stop(); + MongoRunner.stopMongod(mongod); +})(); diff --git a/jstests/noPassthrough/shell_history.js b/jstests/noPassthrough/shell_history.js new file mode 100644 index 00000000000..bbc8f9538af --- /dev/null +++ b/jstests/noPassthrough/shell_history.js @@ -0,0 +1,126 @@ +// Test that when running the shell for the first time creates the ~/.dbshell file, and it has +// appropriate permissions (where relevant). + +(function() { + "use strict"; + + // Use dataPath because it includes the trailing "/" or "\". + var tmpHome = MongoRunner.dataPath; + // Ensure it exists and is a dir (eg. if running without resmoke.py and /data/db doesn't exist). + mkdir(tmpHome); + removeFile(tmpHome + ".dbshell"); + + var args = []; + var cmdline = "mongo --nodb"; + var redirection = ""; + var env = {}; + if (_isWindows()) { + args.push("cmd.exe"); + args.push("/c"); + + // Input is set to NUL. The output must also be redirected to NUL, otherwise running the + // jstest manually has strange terminal IO behaviour. + redirection = "< NUL > NUL"; + + // USERPROFILE set to the tmp homedir. + // Since NUL is a character device, isatty() will return true, which means that .mongorc.js + // will be created in the HOMEDRIVE + HOMEPATH location, so we must set them also. + if (tmpHome.match("^[a-zA-Z]:")) { + var tmpHomeDrive = tmpHome.substr(0, 2); + var tmpHomePath = tmpHome.substr(2); + } else { + var _pwd = pwd(); + assert(_pwd.match("^[a-zA-Z]:"), "pwd must include drive"); + var tmpHomeDrive = _pwd.substr(0, 2); + var tmpHomePath = tmpHome; + } + env = { + USERPROFILE: tmpHome, + HOMEDRIVE: tmpHomeDrive, + HOMEPATH: tmpHomePath + }; + + } else { + args.push("sh"); + args.push("-c"); + + // Use the mongo shell from the current dir, same as resmoke.py does. + // Doesn't handle resmoke's --mongo= option. + cmdline = "./" + cmdline; + + // Set umask to 0 prior to running the shell. + cmdline = "umask 0 ; " + cmdline; + + // stdin is /dev/null. + redirection = "< /dev/null"; + + // HOME set to the tmp homedir. + if (!tmpHome.startsWith("/")) { + tmpHome = pwd() + "/" + tmpHome; + } + env = { + HOME: tmpHome + }; + } + + // Workaround for SERVER-18877 not being fixed in earlier versions. + if (_isWindows()) { + for (var envvar in env) { + cmdline = "set " + envvar + "=" + env[envvar] + "&&" + cmdline; + } + } else { + for (var envvar in env) { + cmdline = envvar + "='" + env[envvar] + "' ; " + cmdline; + } + } + + // Add redirection to cmdline, and add cmdline to args. + cmdline += " " + redirection; + args.push(cmdline); + jsTestLog("Running args:\n " + tojson(args) + "\nwith env:\n " + tojson(env)); + var rc = runProgram.apply(this, args); + + assert.eq(rc, 0); + + var files = listFiles(tmpHome); + jsTestLog(tojson(files)); + + var findFile = function(baseName) { + for (var i = 0; i < files.length; i++) { + if (files[i].baseName === baseName) { + return files[i]; + } + } + return undefined; + }; + + var targetFile = ".dbshell"; + var file = findFile(targetFile); + + assert.neq(typeof(file), "undefined", targetFile + " should exist, but it doesn't"); + assert.eq(file.isDirectory, false, targetFile + " should not be a directory, but it is"); + assert.eq(file.size, 0, targetFile + " should be empty, but it isn't"); + + if (!_isWindows()) { + // On Unix, check that the file has the correct mode (permissions). + // The shell has no way to stat a file. + // There is no stat utility in POSIX. + // `ls -l` is POSIX, so this is the best that we have. + // Check for exactly "-rw-------". + clearRawMongoProgramOutput(); + var rc = runProgram("ls", "-l", file.name); + assert.eq(rc, 0); + // Before SERVER-22992 is fixed: + var output = null; + assert.soon(function() { + output = rawMongoProgramOutput(); + return output != ""; + }); + // After SERVER-22992 is fixed: + // var output = rawMongoProgramOutput(); + var fields = output.split(" "); + // First field is the prefix, second field is the `ls -l` permissions. + assert.eq(fields[1].substr(0, 10), "-rw-------", targetFile + " has bad permissions"); + } + +})(); diff --git a/jstests/noPassthrough/wt_index_option_defaults.js b/jstests/noPassthrough/wt_index_option_defaults.js index 2516cc28d24..b7574d3206a 100644 --- a/jstests/noPassthrough/wt_index_option_defaults.js +++ b/jstests/noPassthrough/wt_index_option_defaults.js @@ -31,9 +31,9 @@ // Use different values for the same configuration string key to test that index-specific // options override collection-wide options, and that collection-wide options override // system-wide options. - var systemWideConfigString = 'split_pct=25,'; - var collectionWideConfigString = 'split_pct=30,'; - var indexSpecificConfigString = 'split_pct=35,'; + var systemWideConfigString = 'split_pct=70,'; + var collectionWideConfigString = 'split_pct=75,'; + var indexSpecificConfigString = 'split_pct=80,'; // Start up a mongod with system-wide defaults for index options and create a collection without // any additional options. Tests than an index without any additional options should take on the diff --git a/jstests/noPassthroughWithMongod/connections_opened.js b/jstests/noPassthroughWithMongod/connections_opened.js index 2ec192ed1e2..d3d141ab5a1 100644 --- a/jstests/noPassthroughWithMongod/connections_opened.js +++ b/jstests/noPassthroughWithMongod/connections_opened.js @@ -20,7 +20,8 @@ var signalCollection = 'keepRunning'; function createPersistentConnection() { assert.soon(function() { try { - return new Mongo(db.getMongo().host); + permConns.push(new Mongo(db.getMongo().host)); + return true; } catch (x) { return false; } @@ -95,7 +96,4 @@ jsTestLog("Testing that current connections counter went down after temporary co waitForConnections(originalConnInfo.current + numPerTypeToCreate, originalConnInfo.totalCreated + numPerTypeToCreate * 2); -persistent = null; -gc(); - MongoRunner.stopMongod(mongo); diff --git a/jstests/noPassthroughWithMongod/ftdc_params.js b/jstests/noPassthroughWithMongod/ftdc_params.js index 5fae9e77c49..fd029a264c2 100644 --- a/jstests/noPassthroughWithMongod/ftdc_params.js +++ b/jstests/noPassthroughWithMongod/ftdc_params.js @@ -19,7 +19,7 @@ // Verify the defaults are as we documented them assert.eq(getparam("diagnosticDataCollectionEnabled"), true); assert.eq(getparam("diagnosticDataCollectionPeriodMillis"), 1000); - assert.eq(getparam("diagnosticDataCollectionDirectorySizeMB"), 100); + assert.eq(getparam("diagnosticDataCollectionDirectorySizeMB"), 200); assert.eq(getparam("diagnosticDataCollectionFileSizeMB"), 10); assert.eq(getparam("diagnosticDataCollectionSamplesPerChunk"), 300); assert.eq(getparam("diagnosticDataCollectionSamplesPerInterimUpdate"), 10); @@ -53,7 +53,7 @@ // Reset assert.commandWorked(setparam({"diagnosticDataCollectionFileSizeMB": 10})); - assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 100})); + assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 200})); assert.commandWorked(setparam({"diagnosticDataCollectionPeriodMillis": 1000})); assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerChunk": 300})); assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerInterimUpdate": 10})); diff --git a/jstests/noPassthroughWithMongod/indexbg_drop.js b/jstests/noPassthroughWithMongod/indexbg_drop.js index ac5783525a6..a81f2f97e61 100644 --- a/jstests/noPassthroughWithMongod/indexbg_drop.js +++ b/jstests/noPassthroughWithMongod/indexbg_drop.js @@ -51,7 +51,7 @@ var bulk = masterDB.getCollection(collection).initializeUnorderedBulkOp(); for (i = 0; i < size; ++i) { bulk.insert({i: Random.rand()}); } -assert.writeOK(bulk.execute()); +assert.writeOK(bulk.execute({w: 2, wtimeout: replTest.kDefaultTimeoutMS})); jsTest.log("Starting background indexing for test of: " + tojson(dc)); // Add another index to be sure the drop command works. diff --git a/jstests/noPassthroughWithMongod/ttl_sharded.js b/jstests/noPassthroughWithMongod/ttl_sharded.js index 577a968eed9..8f37c616f6f 100644 --- a/jstests/noPassthroughWithMongod/ttl_sharded.js +++ b/jstests/noPassthroughWithMongod/ttl_sharded.js @@ -36,15 +36,14 @@ t.ensureIndex({x: 1}, {expireAfterSeconds: 20000}); s.adminCommand({split: ns, middle: {_id: 12}}); s.adminCommand({moveChunk: ns, find: {_id: 0}, to: s.getOther(s.getServer(dbname)).name}); -// one shard will lose 12/12 docs, the other 6/12, so count will go -// from 24 -> 18 or 12 -> 6 -assert.soon(function() { - return t.count() < 7; -}, "TTL index on x didn't delete enough", 70 * 1000); - -// ensure that count ultimately ends up at 6 -assert.eq(0, t.find({x: {$lt: new Date(now - 20000000)}}).count()); -assert.eq(6, t.count()); +// Check that all expired documents are deleted. +assert.soon( + function() { + return t.count() === 6 && t.find({x: {$lt: new Date(now - 20000000)}}).count() === 0; + }, + "TTL index did not successfully delete expired documents, all documents: " + + tojson(t.find().toArray()), + 70 * 1000); // now lets check things explicily on each shard var shard0 = s._connections[0].getDB(dbname); @@ -73,10 +72,13 @@ s.getDB(dbname).runCommand({collMod: coll, index: {keyPattern: {x: 1}, expireAft assert.eq(10000, getTTLTime(shard0.getCollection(coll), {x: 1})); assert.eq(10000, getTTLTime(shard1.getCollection(coll), {x: 1})); -assert.soon(function() { - return t.count() < 6; -}, "new expireAfterSeconds value not taking effect", 70 * 1000); -assert.eq(0, t.find({x: {$lt: new Date(now - 10000000)}}).count()); -assert.eq(3, t.count()); +// Check that all expired documents are deleted. +assert.soon( + function() { + return t.count() === 3 && t.find({x: {$lt: new Date(now - 10000000)}}).count() === 0; + }, + "new expireAfterSeconds did not successfully delete expired documents, all documents: " + + tojson(t.find().toArray()), + 70 * 1000); s.stop(); diff --git a/jstests/replsets/chaining_removal.js b/jstests/replsets/chaining_removal.js index dbc80148745..2c42b3f83e4 100644 --- a/jstests/replsets/chaining_removal.js +++ b/jstests/replsets/chaining_removal.js @@ -2,6 +2,8 @@ (function() { "use strict"; + load("jstests/replsets/rslib.js"); + var numNodes = 5; var host = getHostName(); var name = "chaining_removal"; @@ -19,33 +21,25 @@ {_id: 4, host: nodes[4].host, priority: 0}, ], }); - replTest.waitForState(nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); - replTest.awaitNodesAgreeOnPrimary(); + replTest.awaitNodesAgreeOnPrimary(replTest.kDefaultTimeoutMS, nodes, 0); var primary = replTest.getPrimary(); replTest.awaitReplication(); + // When setting up chaining on slow machines, we do not want slow writes or delayed heartbeats + // to cause our nodes to invalidate the sync source provided in the 'replSetSyncFrom' command. + // To achieve this, we disable the server parameter 'maxSyncSourceLagSecs' (see + // repl_settings_init.cpp and TopologyCoordinatorImpl::Options) in + // TopologyCoordinatorImpl::shouldChangeSyncSource(). + assert.commandWorked(nodes[1].getDB('admin').runCommand( + {configureFailPoint: 'disableMaxSyncSourceLagSecs', mode: 'alwaysOn'})); + assert.commandWorked(nodes[4].getDB('admin').runCommand( + {configureFailPoint: 'disableMaxSyncSourceLagSecs', mode: 'alwaysOn'})); + // Force node 1 to sync directly from node 0. - assert.commandWorked(nodes[1].getDB("admin").runCommand({"replSetSyncFrom": nodes[0].host})); - var res; - assert.soon( - function() { - res = nodes[1].getDB("admin").runCommand({"replSetGetStatus": 1}); - return res.syncingTo === nodes[0].host; - }, - function() { - return "node 1 failed to start syncing from node 0: " + tojson(res); - }); + syncFrom(nodes[1], nodes[0], replTest); // Force node 4 to sync through node 1. - assert.commandWorked(nodes[4].getDB("admin").runCommand({"replSetSyncFrom": nodes[1].host})); - assert.soon( - function() { - res = nodes[4].getDB("admin").runCommand({"replSetGetStatus": 1}); - return res.syncingTo === nodes[1].host; - }, - function() { - return "node 4 failed to start chaining through node 1: " + tojson(res); - }); + syncFrom(nodes[4], nodes[1], replTest); // write that should reach all nodes var timeout = 60 * 1000; @@ -54,6 +48,12 @@ }; assert.writeOK(primary.getDB(name).foo.insert({x: 1}, options)); + // Re-enable 'maxSyncSourceLagSecs' checking on sync source. + assert.commandWorked(nodes[1].getDB('admin').runCommand( + {configureFailPoint: 'disableMaxSyncSourceLagSecs', mode: 'off'})); + assert.commandWorked(nodes[4].getDB('admin').runCommand( + {configureFailPoint: 'disableMaxSyncSourceLagSecs', mode: 'off'})); + var config = primary.getDB("local").system.replset.findOne(); config.members.pop(); config.version++; diff --git a/jstests/replsets/disallow_adding_initialized_node1.js b/jstests/replsets/disallow_adding_initialized_node1.js index 8d4491975b6..ba39e6010ad 100644 --- a/jstests/replsets/disallow_adding_initialized_node1.js +++ b/jstests/replsets/disallow_adding_initialized_node1.js @@ -3,8 +3,10 @@ // Initialize two replica sets A and B with the same name: A_0; B_0 // Add B_0 to the replica set A. This operation should fail on replica set A should fail on // detecting an inconsistent replica set ID in the heartbeat response metadata from B_0. + (function() { 'use strict'; + load("jstests/libs/check_log.js"); var name = 'disallow_adding_initialized_node1'; var replSetA = new ReplSetTest({name: name, nodes: [{rsConfig: {_id: 10}}, ]}); @@ -45,20 +47,9 @@ assert.eq(primaryB, newPrimaryB); // Mismatch replica set IDs in heartbeat responses should be logged. - var checkLog = function(node, msg) { - assert.soon(function() { - var logMessages = assert.commandWorked(node.adminCommand({getLog: 'global'})).log; - for (var i = 0; i < logMessages.length; i++) { - if (logMessages[i].indexOf(msg) != -1) { - return true; - } - } - return false; - }, 'Did not see a log entry containing the following message: ' + msg, 10000, 1000); - }; var msgB = "replica set IDs do not match, ours: " + configB.settings.replicaSetId + "; remote node's: " + configA.settings.replicaSetId; - checkLog(primaryB, msgB); + checkLog.contains(primaryB, msgB); var statusA = assert.commandWorked(primaryA.adminCommand({replSetGetStatus: 1})); var statusB = assert.commandWorked(primaryB.adminCommand({replSetGetStatus: 1})); diff --git a/jstests/replsets/disallow_adding_initialized_node2.js b/jstests/replsets/disallow_adding_initialized_node2.js index c4125f7c069..f54ef947e56 100644 --- a/jstests/replsets/disallow_adding_initialized_node2.js +++ b/jstests/replsets/disallow_adding_initialized_node2.js @@ -8,8 +8,10 @@ // This test requires users to persist across a restart. // @tags: [requires_persistence] + (function() { 'use strict'; + load("jstests/libs/check_log.js"); var name = 'disallow_adding_initialized_node2'; var replSetA = new ReplSetTest( @@ -52,23 +54,12 @@ assert.eq(primaryB, newPrimaryB); // Mismatch replica set IDs in heartbeat responses should be logged. - var checkLog = function(node, msg) { - assert.soon(function() { - var logMessages = assert.commandWorked(node.adminCommand({getLog: 'global'})).log; - for (var i = 0; i < logMessages.length; i++) { - if (logMessages[i].indexOf(msg) != -1) { - return true; - } - } - return false; - }, 'Did not see a log entry containing the following message: ' + msg, 10000, 1000); - }; var msgA = "replica set IDs do not match, ours: " + configA.settings.replicaSetId + "; remote node's: " + configB.settings.replicaSetId; var msgB = "replica set IDs do not match, ours: " + configB.settings.replicaSetId + "; remote node's: " + configA.settings.replicaSetId; - checkLog(primaryA, msgA); - checkLog(primaryB, msgB); + checkLog.contains(primaryA, msgA); + checkLog.contains(primaryB, msgB); var statusA = assert.commandWorked(primaryA.adminCommand({replSetGetStatus: 1})); var statusB = assert.commandWorked(primaryB.adminCommand({replSetGetStatus: 1})); @@ -83,9 +74,6 @@ assert.eq(12, statusA.members[2]._id); assert.eq(primaryB.host, statusA.members[2].name); assert.eq(ReplSetTest.State.DOWN, statusA.members[2].state); - assert(statusA.members[2].lastHeartbeatMessage.indexOf(msgA) != -1 || - statusA.members[2].lastHeartbeatMessage.indexOf( - 'no response within election timeout period') != -1); // Replica set B's config should remain unchanged. assert.eq(1, statusB.members.length); diff --git a/jstests/replsets/do_not_sync_from_stale_sync_source.js b/jstests/replsets/do_not_sync_from_stale_sync_source.js new file mode 100644 index 00000000000..bf6957a0e06 --- /dev/null +++ b/jstests/replsets/do_not_sync_from_stale_sync_source.js @@ -0,0 +1,44 @@ +/* + * This tests that nodes do not sync from other nodes that are behind them. The test sets up a + * 3-node replica set and then stops replication at one node so it starts to lag. The test then + * uses 'replSetSyncFrom' to force the up to date node to sync from the lagging node. After it + * receives its first batch, it errors saying that it cannot sync from a node behind it. + */ + +(function() { + 'use strict'; + + load("jstests/libs/check_log.js"); + load("jstests/replsets/rslib.js"); + + var name = "do_not_sync_from_stale_sync_source"; + var collName = "test.coll"; + + var rst = new ReplSetTest({ + name: name, + nodes: [{}, {rsConfig: {priority: 0}}, {rsConfig: {priority: 0}}], + useBridge: true, + settings: {chainingAllowed: false} + }); + var nodes = rst.startSet(); + rst.initiate(); + + jsTestLog("Make sure node 0 is primary."); + assert.eq(nodes[0], rst.getPrimary()); + // Wait for all data bearing nodes to get up to date. + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: 0}, {writeConcern: {w: 3, wtimeout: rst.kDefaultTimeoutMS}})); + + jsTestLog("Stop node 2 from syncing so it starts lagging."); + assert.commandWorked(nodes[2].getDB('admin').runCommand( + {configureFailPoint: 'stopReplProducer', mode: 'alwaysOn'})); + checkLog.contains(nodes[2], 'stopReplProducer fail point enabled'); + + jsTestLog("Do a write that replicates to [0,1]."); + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: 1}, {writeConcern: {w: 2, wtimeout: rst.kDefaultTimeoutMS}})); + + jsTestLog("Tell node 1 to sync from node 0 which is now behind."); + assert.commandWorked(nodes[1].adminCommand({"replSetSyncFrom": nodes[0].host})); + checkLog.contains(nodes[1], "is not greater than our last fetched OpTime"); +}()); diff --git a/jstests/replsets/double_rollback.js b/jstests/replsets/double_rollback.js index 2286b80e315..1f37bd93349 100644 --- a/jstests/replsets/double_rollback.js +++ b/jstests/replsets/double_rollback.js @@ -9,8 +9,13 @@ * it rolled back, which could then lead to a double-rollback when node 2 was reconnected * to node 1 and tried to apply its oplog despite not being in a consistent state. */ + (function() { 'use strict'; + load("jstests/libs/check_log.js"); + + load("jstests/libs/check_log.js"); + load("jstests/replsets/rslib.js"); var name = "double_rollback"; var dbName = "test"; @@ -40,16 +45,8 @@ }); } - function stepUp(rst, node) { - var primary = rst.getPrimary(); - if (primary != node) { - assert.commandWorked(primary.adminCommand({replSetStepDown: 1})); - } - waitForState(node, ReplSetTest.State.PRIMARY); - } - jsTestLog("Make sure node 0 is primary."); - stepUp(rst, nodes[0]); + rst.stepUp(nodes[0]); assert.eq(nodes[0], rst.getPrimary()); // Wait for all data bearing nodes to get up to date. assert.writeOK(nodes[0].getDB(dbName).getCollection(collName).insert( @@ -100,18 +97,7 @@ jsTestLog("Wait for failpoint on node 2 to pause rollback before it finishes"); // Wait for fail point message to be logged. - var checkLog = function(node, msg) { - assert.soon(function() { - var logMessages = assert.commandWorked(node.adminCommand({getLog: 'global'})).log; - for (var i = 0; i < logMessages.length; i++) { - if (logMessages[i].indexOf(msg) != -1) { - return true; - } - } - return false; - }, 'Did not see a log entry containing the following message: ' + msg, timeout); - }; - checkLog(nodes[2], 'rollback - rollbackHangBeforeFinish fail point enabled'); + checkLog.contains(nodes[2], 'rollback - rollbackHangBeforeFinish fail point enabled'); jsTestLog("Repartition to: [1,3,4] and [0,2]."); nodes[2].disconnect(nodes[1]); @@ -128,8 +114,8 @@ // for a sync source it can use to reach minvalid and get back into SECONDARY state. Node 0 // is the only node it can reach, but since node 0 doesn't contain node 2's minvalid oplog entry // node 2 will refuse to use it as a sync source. - checkLog(nodes[2], - "remote oplog does not contain entry with optime matching our required optime"); + checkLog.contains( + nodes[2], "remote oplog does not contain entry with optime matching our required optime"); var node0RBID = nodes[0].adminCommand('replSetGetRBID').rbid; var node1RBID = nodes[1].adminCommand('replSetGetRBID').rbid; diff --git a/jstests/replsets/double_rollback_early.js b/jstests/replsets/double_rollback_early.js new file mode 100644 index 00000000000..4af501701c6 --- /dev/null +++ b/jstests/replsets/double_rollback_early.js @@ -0,0 +1,139 @@ +/* SERVER-27050 This test causes node 2 to enter rollback, then fail after setting minValid, but + * before truncating the oplog. It will then choose the same sync source (1) and retry the rollback. + * The upstream node itself rolls back at this point. Node 2 should detect this case and fail the + * rollback and refuse to choose node 1 as its sync source because it doesn't have the minValid. + */ + +(function() { + 'use strict'; + + load("jstests/libs/check_log.js"); + load("jstests/replsets/rslib.js"); + + var collName = "test.coll"; + var counter = 0; + + var rst = new ReplSetTest({ + nodes: [ + {}, + {}, + {rsConfig: {priority: 0}}, + {rsConfig: {arbiterOnly: true}}, + {rsConfig: {arbiterOnly: true}} + ], + useBridge: true + }); + var nodes = rst.startSet(); + rst.initiate(); + + jsTestLog("Make sure node 0 is primary."); + rst.stepUp(nodes[0]); + assert.eq(nodes[0], rst.getPrimary()); + // Wait for all data bearing nodes to get up to date. + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: counter++}, {writeConcern: {w: 3, wtimeout: rst.kDefaultTimeoutMs}})); + + jsTestLog("Create two partitions: [1] and [0,2,3,4]."); + nodes[1].disconnect(nodes[0]); + nodes[1].disconnect(nodes[2]); + nodes[1].disconnect(nodes[3]); + nodes[1].disconnect(nodes[4]); + + jsTestLog("Do a write that is replicated to [0,2,3,4]."); + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: counter++}, {writeConcern: {w: 2, wtimeout: rst.kDefaultTimeoutMs}})); + + jsTestLog("Repartition to: [0,2] and [1,3,4]."); + nodes[1].reconnect(nodes[3]); + nodes[1].reconnect(nodes[4]); + nodes[3].disconnect(nodes[0]); + nodes[3].disconnect(nodes[2]); + nodes[4].disconnect(nodes[0]); + nodes[4].disconnect(nodes[2]); + + jsTestLog("Ensure that 0 steps down and that 1 becomes primary."); + waitForState(nodes[0], ReplSetTest.State.SECONDARY); + waitForState(nodes[1], ReplSetTest.State.PRIMARY); + assert.eq(nodes[1], rst.getPrimary()); + + jsTestLog("Do a write to node 1 on the [1,3,4] side of the partition."); + // We track this object to ensure it gets rolled back on node 1 later. + assert.writeOK(nodes[1].getCollection(collName).insert({a: counter++, tracked: true})); + + // Turn on failpoint on node 2 to pause rollback after oplog is truncated and minValid is set. + assert.commandWorked(nodes[2].adminCommand( + {configureFailPoint: 'rollbackHangThenFailAfterWritingMinValid', mode: 'alwaysOn'})); + + jsTestLog("Repartition to: [0] and [1,2,3,4]."); + nodes[2].disconnect(nodes[0]); + nodes[2].reconnect(nodes[1]); + nodes[2].reconnect(nodes[3]); + nodes[2].reconnect(nodes[4]); + + jsTestLog("Wait for node 2 to go into ROLLBACK and start syncing from node 1."); + // Since nodes 1 and 2 have now diverged, node 2 should go into rollback. + waitForState(nodes[2], ReplSetTest.State.ROLLBACK); + rst.awaitSyncSource(nodes[2], nodes[1]); + + jsTestLog("Wait for failpoint on node 2 to pause rollback after it writes minValid"); + // Wait for fail point message to be logged. + checkLog.contains(nodes[2], + 'rollback - rollbackHangThenFailAfterWritingMinValid fail point enabled'); + + // Switch failpoints, causing rollback to fail then pause when it retries. It is important to + // enable the new one before disabling the current one. + assert.commandWorked( + nodes[2].adminCommand({configureFailPoint: 'rollbackHangBeforeStart', mode: 'alwaysOn'})); + assert.commandWorked(nodes[2].adminCommand( + {configureFailPoint: 'rollbackHangThenFailAfterWritingMinValid', mode: 'off'})); + jsTestLog("Wait for failpoint on node 2 to pause rollback after it restarts"); + // Wait for fail point message to be logged. + checkLog.contains(nodes[2], 'rollback - rollbackHangBeforeStart fail point enabled'); + + jsTestLog("Repartition to: [0,3,4] and [1,2]."); + nodes[3].disconnect(nodes[1]); + nodes[3].reconnect(nodes[0]); + nodes[4].disconnect(nodes[1]); + nodes[4].reconnect(nodes[0]); + + jsTestLog("Ensure that 0 becomes primary."); + waitForState(nodes[0], ReplSetTest.State.PRIMARY); + waitForState(nodes[1], ReplSetTest.State.SECONDARY); + assert.eq(nodes[0], rst.getPrimary()); + // Do a write so that node 0 is definitely ahead of node 1. + assert.writeOK(nodes[0].getCollection(collName).insert({a: counter++})); + + jsTestLog("Repartition to: [0,1,3,4] and [2] so 1 rolls back and replicates from 0."); + assert.eq(nodes[1].getCollection(collName).count({tracked: true}), 1); + nodes[1].reconnect(nodes[0]); + waitForState(nodes[1], ReplSetTest.State.SECONDARY); + jsTestLog("w:2 write to node 0"); + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: counter++}, {writeConcern: {w: 2, wtimeout: rst.kDefaultTimeoutMs}})); + reconnect(nodes[1]); // rollback drops connections. + assert.eq(nodes[1].getCollection(collName).count({tracked: true}), 0); + + // Turn off failpoint on node 2 to allow rollback to finish its attempt at rollback from node 1. + // It should fail with a rbid error and get stuck. + jsTestLog("Repartition to: [0,3,4] and [1,2]."); + nodes[1].reconnect(nodes[2]); + assert.commandWorked( + nodes[2].adminCommand({configureFailPoint: 'rollbackHangBeforeStart', mode: 'off'})); + + jsTestLog("Wait for node 2 exit ROLLBACK state and go into RECOVERING"); + waitForState(nodes[2], ReplSetTest.State.RECOVERING); + + // At this point node 2 has truncated its oplog back to the common point and is looking + // for a sync source it can use to reach minvalid and get back into SECONDARY state. Node 1 + // is the only node it can reach, but since node 1 doesn't contain node 2's minvalid oplog entry + // node 2 will refuse to use it as a sync source. + checkLog.contains(nodes[2], "Upstream node rolled back. Need to retry our rollback."); + waitForState(nodes[2], ReplSetTest.State.RECOVERING); + + // This log message means that it will not be willing to use node 1 as the sync source when it + // retries. + checkLog.contains( + nodes[2], "remote oplog does not contain entry with optime matching our required optime"); + + rst.stopSet(); +}()); diff --git a/jstests/replsets/drain.js b/jstests/replsets/drain.js index 7f170c485d9..59082255c1d 100644 --- a/jstests/replsets/drain.js +++ b/jstests/replsets/drain.js @@ -25,7 +25,6 @@ var primary = replSet.getPrimary(); var secondary = replSet.getSecondary(); - var isPV0 = replSet.getReplSetConfigFromNode().protocolVersion != 1; // Do an initial insert to prevent the secondary from going into recovery var numDocuments = 20; @@ -43,7 +42,7 @@ } assert.writeOK(bulk.execute()); jsTestLog('Number of documents inserted into collection on primary: ' + numDocuments); - assert.eq(numDocuments, primary.getDB("foo").foo.find().itcount()); + assert.eq(numDocuments, primary.getDB("foo").foo.count()); assert.soon(function() { var serverStatus = secondary.getDB('foo').serverStatus(); @@ -52,13 +51,12 @@ jsTestLog('Number of operations buffered on secondary since stopping applier: ' + bufferCountChange); return bufferCountChange >= numDocuments - 1; - }, 'secondary did not buffer operations for new inserts on primary', 30000, 1000); + }, 'secondary did not buffer operations for new inserts on primary', 300000, 1000); // Kill primary; secondary will enter drain mode to catch up primary.getDB("admin").shutdownServer({force: true}); - var electionTimeout = (isPV0 ? 60 : 20) * 1000; // Timeout in milliseconds - replSet.waitForState(secondary, ReplSetTest.State.PRIMARY, electionTimeout); + replSet.waitForState(secondary, ReplSetTest.State.PRIMARY); // Ensure new primary is not yet writable jsTestLog('New primary should not be writable yet'); @@ -75,12 +73,7 @@ "find failed with unexpected error code: " + tojson(res)); // Nor should it be readable with the slaveOk bit. secondary.slaveOk = true; - res = secondary.getDB("foo").runCommand({find: "foo"}); - assert.commandFailed(res); - assert.eq(ErrorCodes.NotMasterOrSecondary, - res.code, - "find failed with unexpected error code: " + tojson(res)); - secondary.slaveOk = false; + assert.commandWorked(secondary.getDB("foo").runCommand({find: "foo"})); assert.commandFailedWithCode( secondary.adminCommand({ @@ -100,7 +93,7 @@ assert.commandWorked( secondary.adminCommand({ replSetTest: 1, - waitForDrainFinish: 5000, + waitForDrainFinish: 30000, }), 'replSetTest waitForDrainFinish should work when draining is allowed to complete'); @@ -110,5 +103,5 @@ // Check for at least two entries. There was one prior to freezing op application on the // secondary and we cannot guarantee all writes reached the secondary's op queue prior to // shutting down the original primary. - assert.gte(primary.getDB("foo").foo.find().itcount(), 2); + assert.gte(primary.getDB("foo").foo.count(), 2); })(); diff --git a/jstests/replsets/election_timing.js b/jstests/replsets/election_timing.js deleted file mode 100644 index 48139e6bac0..00000000000 --- a/jstests/replsets/election_timing.js +++ /dev/null @@ -1,143 +0,0 @@ -// Run a small set of tests using the ElectionTimingTest framework. While this -// reports the timing of the election, we are using it to check if any errors happen -// during different election cycles. -(function() { - "use strict"; - load("jstests/libs/election_timing_test.js"); - var testStart = Date.now(); - - var testCases = [ - { - name: "testV1Stop", - description: "protocolVersion 1, primary is stopped", - protocolVersion: 1, - // testRuns is the number of times a new ReplSetTest will be used. - testRuns: 1, - // testCycles is the number of election cycles that will be run per ReplSetTest lifespan. - testCycles: 5, - // testSetup is run after the replSet is initiated. - // Function.prototype is the default. - testSetup: Function.prototype, - // Trigger an election by stepping down, stopping, or partitioning the primary. - // stopPrimary is the default. - electionTrigger: ElectionTimingTest.prototype.stopPrimary, - // After the election has completed, make the old primary available again. - // stopPrimaryReset is the default. - testReset: ElectionTimingTest.prototype.stopPrimaryReset - }, - - /* - This test case is flakey since waiting for the old primary to shutdown can take longer than - the - allowed timeout, even if a new primary was elected during the shutdown time. - - { - name: "testV1StopTimeout1500", - description: "protocolVersion 1, primary is stopped, electionTimeoutMillis set to 1500", - protocolVersion: 1, - testRuns: 1, - testCycles: 5, - // The settings object is merged into the replset config settings object. - settings: {electionTimeoutMillis: 1500} - }, - */ - - { - name: "testV1StepDown", - description: "protocolVersion 1, primary is stepped down", - protocolVersion: 1, - testRuns: 1, - testCycles: 5, - electionTrigger: ElectionTimingTest.prototype.stepDownPrimary, - testReset: ElectionTimingTest.prototype.stepDownPrimaryReset, - }, - - { - name: "testV1StepDown1500", - description: "protocolVersion 1, primary is stepped down", - protocolVersion: 1, - testRuns: 1, - testCycles: 5, - electionTrigger: ElectionTimingTest.prototype.stepDownPrimary, - testReset: ElectionTimingTest.prototype.stepDownPrimaryReset, - // The settings object is merged into the replset config settings object. - settings: {electionTimeoutMillis: 1500} - }, - - { - name: "testV1StepDownLargeCluster", - description: "protocolVersion 1, primary is stepped down, 7 electable nodes", - protocolVersion: 1, - nodes: 7, - testRuns: 1, - testCycles: 5, - electionTrigger: ElectionTimingTest.prototype.stepDownPrimary, - testReset: function() {}, - waitForNewPrimary: function(rst, secondary) { - rst.getPrimary(); - } - }, - - { - name: "testV0Stop", - description: "protocolVersion 0, primary is stopped", - protocolVersion: 0, - testRuns: 1, - testCycles: 1 - }, - - { - name: "testV0StepDown", - description: "protocolVersion 0, primary is stepped down", - protocolVersion: 0, - testRuns: 1, - testCycles: 2, - stepDownGuardTime: 30, - // There is a guard time in pv0 that prevents an election right - // after initiating. - testSetup: function() { - sleep(30 * 1000); - }, - electionTrigger: ElectionTimingTest.prototype.stepDownPrimary, - testReset: ElectionTimingTest.prototype.stepDownPrimaryReset - }, - - ]; - - testCases.forEach(function(tc) { - var testRun = new ElectionTimingTest(tc); - tc.testResults = testRun.testResults; - tc.electionTimeoutLimitMillis = testRun.electionTimeoutLimitMillis; - - if (testRun.testErrors.length) { - // Stop tests if we encounter an error. - // Dump available information for debugging. - jsTestLog("Errors from: " + tc.name); - printjson(tc); - printjson(testRun.testErrors); - throw new Error(testRun.testErrors[0].status); - } - // Print results of current test in case - // we need to analyze a failed test later. - jsTestLog("Raw Results: " + tc.name); - printjson(tc.testResults); - }); - - testCases.forEach(function(tc) { - var allResults = []; - tc.testResults.forEach(function(tr) { - allResults = allResults.concat(tr.results); - }); - - var resAvg = Array.avg(allResults); - var resMin = Math.min(... allResults); - var resMax = Math.max(... allResults); - var resStdDev = Array.stdDev(allResults); - - jsTestLog("Results: " + tc.name + " Average over " + allResults.length + " runs: " + - resAvg + " Min: " + resMin + " Max: " + resMax + " StdDev: " + - resStdDev.toFixed(4)); - }); - - jsTestLog("Tests completed in: " + (Date.now() - testStart) / 1000 + " seconds"); -}()); diff --git a/jstests/replsets/last_vote.js b/jstests/replsets/last_vote.js new file mode 100644 index 00000000000..9b05766cae6 --- /dev/null +++ b/jstests/replsets/last_vote.js @@ -0,0 +1,229 @@ +// Tests that the last vote document is stored during elections and that it is loaded and used on +// startup. +// +// The test first runs a few elections and checks that the lastVote document is set correctly +// after each one. +// +// The test then restarts one node as a standalone, changes its last vote doc, and stops the +// other node. It then restarts the first node as a replicaset and manually runs +// replSetRequestVotes commands against it and checks that its response is correct. +// +// @tags: [requires_persistence] + +(function() { + "use strict"; + load("jstests/replsets/rslib.js"); // For getLatestOp() + + var name = "last_vote"; + var rst = new ReplSetTest({ + name: name, + nodes: 2, + }); + rst.startSet(); + + // Lower the election timeout to make the test run faster since it waits for multiple elections. + var conf = rst.getReplSetConfig(); + conf.settings = { + electionTimeoutMillis: 6000, + }; + rst.initiate(conf); + + const lastVoteNS = 'local.replset.election'; + + function getLastVoteDoc(conn) { + assert.eq( + conn.getCollection(lastVoteNS).find().itcount(), 1, 'last vote should be singleton'); + return conn.getCollection(lastVoteNS).findOne(); + } + + function setLastVoteDoc(conn, term, candidate) { + var newLastVote = { + term: term, + candidateIndex: rst.getNodeId(candidate) + }; + return assert.writeOK(conn.getCollection(lastVoteNS).update({}, newLastVote)); + } + + function assertNodeHasLastVote(node, term, candidate) { + var lastVoteDoc = getLastVoteDoc(node); + assert.eq(lastVoteDoc.term, term, node.host + " had wrong last vote term."); + assert.eq(lastVoteDoc.candidateIndex, + rst.getNodeId(candidate), + node.host + " had wrong last vote candidate."); + } + + function assertCurrentTerm(node, term) { + var stat = assert.commandWorked(node.adminCommand({replSetGetStatus: 1})); + assert.eq(stat.term, term, "Term changed when it should not have"); + } + + jsTestLog("Test that last vote is set on successive elections"); + + // Run a few successive elections, alternating who becomes primary. + var numElections = 3; + for (var i = 0; i < numElections; i++) { + var primary = rst.getPrimary(); + var secondary = rst.getSecondary(); + var term = getLatestOp(primary).t; + + jsTestLog("Last vote should have term: " + term + " and candidate: " + primary.host + + ", index: " + rst.getNodeId(primary)); + rst.nodes.forEach(function(node) { + assertNodeHasLastVote(node, term, primary); + }); + + assert.throws(function() { + primary.adminCommand({replSetStepDown: 60 * 10, force: true}); + }); + + // Make sure a new primary has been established. + rst.waitForState(primary, ReplSetTest.State.SECONDARY); + rst.waitForState(secondary, ReplSetTest.State.PRIMARY); + + // Reset election timeout for the old primary. + assert.commandWorked(primary.adminCommand({replSetFreeze: 0})); + } + + var term = getLatestOp(rst.getPrimary()).t + 100; + + jsTestLog("Test that last vote is loaded on startup"); + + // Ensure that all ops are replicated before stepping up node 1. + rst.awaitReplication(); + + // We cannot reconfig node 0 to have priority 0 if it is currently the primary, + // so we make sure node 1 is primary. + jsTestLog("Stepping up node 1"); + rst.stepUp(rst.nodes[1]); + + jsTestLog("Reconfiguring cluster to make node 0 unelectable so it stays SECONDARY on restart"); + conf = rst.getReplSetConfigFromNode(); + conf.version++; + conf.members[0].priority = 0; + reconfig(rst, conf); + rst.awaitNodesAgreeOnConfigVersion(); + + jsTestLog("Restarting node 0 as a standalone"); + var node0 = rst.restart(0, {noReplSet: true}); // Restart as a standalone node. + jsTestLog("Stopping node 1"); + rst.stop(1); // Stop node 1 so that node 0 controls the term by itself. + jsTestLog("Setting the lastVote on node 0 to term: " + term + " candidate: " + + rst.nodes[0].host + ", index: 0"); + setLastVoteDoc(node0, term, rst.nodes[0]); + + jsTestLog("Restarting node 0 in replica set mode"); + node0 = rst.restart(0); // Restart in replSet mode again. + assertCurrentTerm(node0, term); + + jsTestLog("Manually sending node 0 a dryRun replSetRequestVotes command, " + + "expecting failure in old term"); + var response = assert.commandWorked(node0.adminCommand({ + replSetRequestVotes: 1, + setName: name, + dryRun: true, + term: term - 1, + candidateIndex: 1, + configVersion: 2, + lastCommittedOp: getLatestOp(node0) + })); + assert.eq(response.term, + term, + "replSetRequestVotes response had the wrong term: " + tojson(response)); + assert(!response.voteGranted, + "node granted vote in term before last vote doc: " + tojson(response)); + assert.eq(response.reason, + "candidate's term is lower than mine", + "replSetRequestVotes response had the wrong reason: " + tojson(response)); + assertNodeHasLastVote(node0, term, rst.nodes[0]); + assertCurrentTerm(node0, term); + + jsTestLog("Manually sending node 0 a dryRun replSetRequestVotes command in same term, " + + "expecting success but no recording of lastVote"); + response = assert.commandWorked(node0.adminCommand({ + replSetRequestVotes: 1, + setName: name, + dryRun: true, + term: term, + candidateIndex: 1, + configVersion: 2, + lastCommittedOp: getLatestOp(node0) + })); + assert.eq(response.term, + term, + "replSetRequestVotes response had the wrong term: " + tojson(response)); + assert(response.voteGranted, + "node failed to grant dryRun vote in term equal to last vote doc: " + tojson(response)); + assert.eq(response.reason, + "", + "replSetRequestVotes response had the wrong reason: " + tojson(response)); + assertNodeHasLastVote(node0, term, rst.nodes[0]); + assertCurrentTerm(node0, term); + + jsTestLog( + "Manually sending node 0 a replSetRequestVotes command, expecting failure in same term"); + response = assert.commandWorked(node0.adminCommand({ + replSetRequestVotes: 1, + setName: name, + dryRun: false, + term: term, + candidateIndex: 1, + configVersion: 2, + lastCommittedOp: getLatestOp(node0) + })); + assert.eq(response.term, + term, + "replSetRequestVotes response had the wrong term: " + tojson(response)); + assert(!response.voteGranted, + "node granted vote in term of last vote doc: " + tojson(response)); + assert.eq(response.reason, + "already voted for another candidate this term", + "replSetRequestVotes response had the wrong reason: " + tojson(response)); + assertNodeHasLastVote(node0, term, rst.nodes[0]); + assertCurrentTerm(node0, term); + + jsTestLog("Manually sending node 0 a replSetRequestVotes command, " + + "expecting success with a recording of the new lastVote"); + response = assert.commandWorked(node0.adminCommand({ + replSetRequestVotes: 1, + setName: name, + dryRun: false, + term: term + 1, + candidateIndex: 1, + configVersion: 2, + lastCommittedOp: getLatestOp(node0) + })); + assert.eq(response.term, + term + 1, + "replSetRequestVotes response had the wrong term: " + tojson(response)); + assert(response.voteGranted, + "node failed to grant vote in term greater than last vote doc: " + tojson(response)); + assert.eq(response.reason, + "", + "replSetRequestVotes response had the wrong reason: " + tojson(response)); + assertNodeHasLastVote(node0, term + 1, rst.nodes[1]); + assertCurrentTerm(node0, term + 1); + + jsTestLog("Manually sending node 0 a dryRun replSetRequestVotes command in future term, " + + "expecting success but no recording of lastVote"); + response = assert.commandWorked(node0.adminCommand({ + replSetRequestVotes: 1, + setName: name, + dryRun: true, + term: term + 2, + candidateIndex: 1, + configVersion: 2, + lastCommittedOp: getLatestOp(node0) + })); + assert.eq(response.term, + term + 2, + "replSetRequestVotes response had the wrong term: " + tojson(response)); + assert(response.voteGranted, + "node failed to grant vote in term greater than last vote doc: " + tojson(response)); + assert.eq(response.reason, + "", + "replSetRequestVotes response had the wrong reason: " + tojson(response)); + assertNodeHasLastVote(node0, term + 1, rst.nodes[1]); + assertCurrentTerm(node0, term + 2); + + rst.stopSet(); +})(); diff --git a/jstests/replsets/libs/tags.js b/jstests/replsets/libs/tags.js new file mode 100644 index 00000000000..80ea4264fc3 --- /dev/null +++ b/jstests/replsets/libs/tags.js @@ -0,0 +1,323 @@ +/** + * Sets up a test for replica set tags sets. + * + * https://docs.mongodb.com/v3.0/tutorial/configure-replica-set-tag-sets/ + */ +var TagsTest = function(options) { + 'use strict'; + + if (!(this instanceof TagsTest)) { + return new TagsTest(options); + } + + // Capture the 'this' reference + var self = this; + + self.options = options; + + /** + * Runs the test. + */ + this.run = function() { + var options = this.options; + + load('jstests/replsets/rslib.js'); + + var nodes = options.nodes; + var host = getHostName(); + var name = 'tags'; + var forceWriteMode = options.forceWriteMode; + + var replTest = new ReplSetTest({name: name, nodes: {n0: nodes[0]}, useBridge: true}); + replTest.startSet(); + replTest.initiate(); + + // If provided in 'options', we set the featureCompatibilityVersion. We do this prior to + // adding any other members to the replica set. This effectively allows us to emulate + // upgrading some of our nodes to the latest version while performing write operations under + // different network partition scenarios. + if (options.setFeatureCompatibilityVersion) { + assert.commandWorked(replTest.getPrimary().adminCommand( + {setFeatureCompatibilityVersion: options.setFeatureCompatibilityVersion})); + } + + for (var i = 1; i < nodes.length; ++i) { + replTest.add(nodes[i]); + } + + const conns = replTest.nodes; + nodes = replTest.nodeList(); + var port = replTest.ports; + var nextVersion = replTest.getReplSetConfigFromNode().version + 1; + const replSetConfig = { + _id: name, + members: [ + { + _id: 0, + host: nodes[0], + tags: { + server: '0', + dc: 'ny', + ny: '1', + rack: 'ny.rk1', + }, + }, + { + _id: 1, + host: nodes[1], + priority: 2, + tags: { + server: '1', + dc: 'ny', + ny: '2', + rack: 'ny.rk1', + }, + }, + { + _id: 2, + host: nodes[2], + priority: 3, + tags: { + server: '2', + dc: 'ny', + ny: '3', + rack: 'ny.rk2', 2: 'this', + }, + }, + { + _id: 3, + host: nodes[3], + tags: { + server: '3', + dc: 'sf', + sf: '1', + rack: 'sf.rk1', + }, + }, + { + _id: 4, + host: nodes[4], + tags: { + server: '4', + dc: 'sf', + sf: '2', + rack: 'sf.rk2', + }, + }, + ], + settings: { + getLastErrorModes: { + '2 dc and 3 server': { + dc: 2, + server: 3, + }, + '1 and 2': { + 2: 1, + server: 1, + }, + '2': { + 2: 1, + }, + '3 and 4': { + sf: 2, + }, + '3 or 4': { + sf: 1, + }, + }, + }, + version: nextVersion, + }; + + reconfig(replTest, replSetConfig); + + replTest.waitForState(replTest.nodes[2], ReplSetTest.State.PRIMARY); + replTest.awaitReplication(); + + // Create collection to guard against timeouts due to file allocation. + assert.commandWorked(replTest.getPrimary().getDB('foo').createCollection('bar')); + replTest.awaitReplication(); + + // nodeId is the index of the node that we expect to see as primary. + // expectedNodesAgreeOnPrimary is a set of nodes that should agree that 'nodeId' is the + // primary. + // expectedWritableNodesCount is the number of nodes we can expect to write to. Defaults to + // expectedNodesAgreeOnPrimary.length. + var ensurePrimary = function( + nodeId, expectedNodesAgreeOnPrimary, expectedWritableNodesCount) { + expectedWritableNodesCount = + expectedWritableNodesCount || expectedNodesAgreeOnPrimary.length; + jsTestLog('ensurePrimary - Node ' + nodeId + ' (' + replTest.nodes[nodeId].host + + ') should be primary.'); + replTest.awaitNodesAgreeOnPrimary( + replTest.kDefaultTimeoutMS, expectedNodesAgreeOnPrimary, nodeId); + jsTestLog('ensurePrimary - Nodes ' + tojson(expectedNodesAgreeOnPrimary) + + ' agree that ' + nodeId + ' (' + replTest.nodes[nodeId].host + + ') should be primary.'); + primary = replTest.getPrimary(); + if (forceWriteMode) { + primary.forceWriteMode(forceWriteMode); + } + var writeConcern = { + writeConcern: + {w: expectedWritableNodesCount, wtimeout: replTest.kDefaultTimeoutMS} + }; + assert.writeOK(primary.getDB('foo').bar.insert({x: 100}, writeConcern)); + jsTestLog('ensurePrimary - Successfully written a document to primary node (' + + replTest.nodes[nodeId].host + ') using a write concern of w:' + + expectedWritableNodesCount); + return primary; + }; + + // 2 should eventually stage a priority takeover from the primary. + var primary = ensurePrimary(2, replTest.nodes); + + jsTestLog('primary is now 2'); + var config = assert.commandWorked(primary.adminCommand({replSetGetConfig: 1})).config; + jsTestLog('test configuration = ' + tojson(config)); + + jsTestLog('Setting up partitions: [0-1-2] [3] [4]'); + conns[0].disconnect(conns[3]); + conns[0].disconnect(conns[4]); + conns[1].disconnect(conns[3]); + conns[1].disconnect(conns[4]); + conns[2].disconnect(conns[3]); + conns[2].disconnect(conns[4]); + conns[3].disconnect(conns[4]); + jsTestLog('Done setting up partitions'); + + jsTestLog( + 'partitions: nodes with each set of brackets [N1, N2, N3] form a complete network.'); + jsTestLog( + 'partitions: [0-1-2] [3] [4] (only nodes 0 and 1 can replicate from primary node 2'); + + var doc = { + x: 1 + }; + + // This timeout should be shorter in duration than the server parameter + // maxSyncSourceLagSecs. + // Some writes are expected to block for this 'timeout' duration before failing. + // Depending on the order of heartbeats (containing last committed op time) received + // by a node, it might hang up on its sync source. This may cause some of the write concern + // tests to fail. + var timeout = 20 * 1000; + + jsTestLog('test1'); + primary = ensurePrimary(2, replTest.nodes.slice(0, 3)); + + jsTestLog('Non-existent write concern should be rejected.'); + options = { + writeConcern: {w: 'blahblah', wtimeout: timeout} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + var result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); + assert.neq(null, result.getWriteConcernError()); + assert.eq(ErrorCodes.UnknownReplWriteConcern, + result.getWriteConcernError().code, + tojson(result.getWriteConcernError())); + + jsTestLog('Write concern "3 or 4" should fail - 3 and 4 are not connected to the primary.'); + var options = { + writeConcern: {w: '3 or 4', wtimeout: timeout} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + result = primary.getDB('foo').bar.insert(doc, options); + assert.neq(null, result.getWriteConcernError()); + assert(result.getWriteConcernError().errInfo.wtimeout); + + conns[1].reconnect(conns[4]); + jsTestLog('partitions: [0-1-2] [1-4] [3] ' + + '(all nodes besides node 3 can replicate from primary node 2)'); + primary = ensurePrimary(2, replTest.nodes.slice(0, 3), 4); + + jsTestLog('Write concern "3 or 4" should work - 4 is now connected to the primary ' + + primary.host + ' via node 1 ' + replTest.nodes[1].host); + options = { + writeConcern: {w: '3 or 4', wtimeout: timeout} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); + + jsTestLog('Write concern "3 and 4" should fail - 3 is not connected to the primary.'); + options = { + writeConcern: {w: '3 and 4', wtimeout: timeout} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); + assert.neq(null, result.getWriteConcernError()); + assert(result.getWriteConcernError().errInfo.wtimeout, + tojson(result.getWriteConcernError())); + + conns[3].reconnect(conns[4]); + jsTestLog('partitions: [0-1-2] [1-4] [3-4] ' + + '(all secondaries can replicate from primary node 2)'); + primary = ensurePrimary(2, replTest.nodes.slice(0, 3), replTest.nodes.length); + + jsTestLog('Write concern "3 and 4" should work - ' + + 'nodes 3 and 4 are connected to primary via node 1.'); + options = { + writeConcern: {w: '3 and 4', wtimeout: timeout} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); + + jsTestLog('Write concern "2" - writes to primary only.'); + options = { + writeConcern: {w: '2', wtimeout: 0} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); + + jsTestLog('Write concern "1 and 2"'); + options = { + writeConcern: {w: '1 and 2', wtimeout: 0} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); + + jsTestLog('Write concern "2 dc and 3 server"'); + primary = ensurePrimary(2, replTest.nodes.slice(0, 3), replTest.nodes.length); + options = { + writeConcern: {w: '2 dc and 3 server', wtimeout: timeout} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); + + jsTestLog('Bringing down current primary node 2 ' + primary.host + + ' to allow next higher priority node 1 ' + replTest.nodes[1].host + + ' to become primary.'); + + conns[1].reconnect(conns[3]); + conns[2].disconnect(conns[0]); + conns[2].disconnect(conns[1]); + jsTestLog('partitions: [0-1] [2] [1-3-4] ' + + '(all secondaries except down node 2 can replicate from new primary node 1)'); + + // Node 1 with slightly higher priority will take over. + jsTestLog('1 must become primary here because otherwise the other members will take too ' + + 'long timing out their old sync threads'); + primary = ensurePrimary(1, replTest.nodes.slice(0, 2), 4); + + jsTestLog('Write concern "3 and 4" should still work with new primary node 1 ' + + primary.host); + options = { + writeConcern: {w: '3 and 4', wtimeout: timeout} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); + + jsTestLog('Write concern "2" should fail because node 2 ' + replTest.nodes[2].host + + ' is down.'); + options = { + writeConcern: {w: '2', wtimeout: timeout} + }; + assert.writeOK(primary.getDB('foo').bar.insert(doc)); + result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); + assert.neq(null, result.getWriteConcernError()); + assert(result.getWriteConcernError().errInfo.wtimeout); + + replTest.stopSet(); + }; + +}; diff --git a/jstests/replsets/maintenance.js b/jstests/replsets/maintenance.js index 7e49e07e396..b1fe94efc0e 100644 --- a/jstests/replsets/maintenance.js +++ b/jstests/replsets/maintenance.js @@ -75,6 +75,10 @@ assert.soon(function() { return !im.secondary && !im.ismaster; }); +var recv = conns[1].getDB("admin").runCommand({find: "foo"}); +assert.commandFailed(recv); +assert.eq(recv.errmsg, "node is recovering"); + print("now getmore shouldn't work"); var ex = assert.throws(function() { lastDoc = null; diff --git a/jstests/replsets/maxSyncSourceLagSecs.js b/jstests/replsets/maxSyncSourceLagSecs.js index 6bcec4a3f71..a00ad8a7bef 100644 --- a/jstests/replsets/maxSyncSourceLagSecs.js +++ b/jstests/replsets/maxSyncSourceLagSecs.js @@ -4,6 +4,8 @@ // @tags: [requires_fsync] (function() { "use strict"; + load("jstests/replsets/rslib.js"); + var name = "maxSyncSourceLagSecs"; var replTest = new ReplSetTest({ name: name, @@ -23,36 +25,33 @@ }); replTest.awaitNodesAgreeOnPrimary(); + // Disable maxSyncSourceLagSecs behavior until we've established the spanning tree we want. + replTest.nodes.forEach(function(node) { + assert.commandWorked(node.getDB('admin').runCommand( + {configureFailPoint: 'disableMaxSyncSourceLagSecs', mode: 'alwaysOn'})); + }); + var master = replTest.getPrimary(); var slaves = replTest.liveNodes.slaves; - assert.commandWorked(slaves[0].getDB("admin").runCommand({replSetSyncFrom: master.name})); - assert.commandWorked(slaves[1].getDB("admin").runCommand({replSetSyncFrom: master.name})); + syncFrom(slaves[0], master, replTest); + syncFrom(slaves[1], master, replTest); master.getDB("foo").bar.save({a: 1}); replTest.awaitReplication(); + jsTestLog("Setting sync target of slave 2 to slave 1"); + syncFrom(slaves[1], slaves[0], replTest); + printjson(replTest.status()); + // need to put at least maxSyncSourceLagSecs b/w first op and subsequent ops // so that the shouldChangeSyncSource logic goes into effect sleep(4000); - jsTestLog("Setting sync target of slave 2 to slave 1"); - assert.soon(function() { - // We do a write each time and have this in a try...catch block due to the fallout of - // SERVER-24114. If that timeout occurs, then we search for another sync source, however we - // will not find one unless more writes have come in. Additionally, it is possible that - // slaves[1] will switch to sync from slaves[0] after slaves[1] replicates a write from - // the primary but before slaves[0] replicates it. slaves[1] will then have to roll back - // which would cause a network error. - try { - slaves[1].getDB("admin").runCommand({replSetSyncFrom: slaves[0].name}); - var res = slaves[1].getDB("admin").runCommand({"replSetGetStatus": 1}); - master.getDB("foo").bar.insert({a: 1}); - return res.syncingTo === slaves[0].name; - } catch (e) { - print("Exception in assert.soon, retrying: " + e); - return false; - } - }, "sync target not changed to other slave", 100 * 1000, 2 * 1000); - printjson(replTest.status()); + // Re-enable maxSyncSourceLagSecs behavior now that we have the spanning tree we want and are + // ready to test that behavior. + replTest.nodes.forEach(function(node) { + assert.commandWorked(node.getDB('admin').runCommand( + {configureFailPoint: 'disableMaxSyncSourceLagSecs', mode: 'off'})); + }); jsTestLog("Lock slave 1 and add some docs. Force sync target for slave 2 to change to primary"); assert.commandWorked(slaves[0].getDB("admin").runCommand({fsync: 1, lock: 1})); diff --git a/jstests/replsets/no_chaining.js b/jstests/replsets/no_chaining.js index ad086c72f9a..07a67c3ea2b 100644 --- a/jstests/replsets/no_chaining.js +++ b/jstests/replsets/no_chaining.js @@ -1,3 +1,4 @@ +load("jstests/replsets/rslib.js"); function myprint(x) { print("chaining output: " + x); @@ -38,14 +39,7 @@ var checkNoChaining = function() { }; var forceSync = function() { - var config; - try { - config = nodes[2].getDB("local").system.replset.findOne(); - } catch (e) { - config = nodes[2].getDB("local").system.replset.findOne(); - } - var targetHost = config.members[1].host; - printjson(nodes[2].getDB("admin").runCommand({replSetSyncFrom: targetHost})); + syncFrom(nodes[2], nodes[1], replTest); assert.soon(function() { return nodes[2].getDB("test").foo.findOne() != null; }, 'Check for data after force sync'); diff --git a/jstests/replsets/no_flapping_during_network_partition.js b/jstests/replsets/no_flapping_during_network_partition.js new file mode 100644 index 00000000000..1fb987aef99 --- /dev/null +++ b/jstests/replsets/no_flapping_during_network_partition.js @@ -0,0 +1,60 @@ +/* + * Test that arbiters vote no in elections if they can see a healthy primary of equal or greater + * priority to the candidate, preventing flapping during certain kinds of network partitions. + * + * 1. Initiate a 3-node replica set with one arbiter (PSA) and a higher priority primary. + * 2. Create a network partition between the primary and secondary. + * 3. Wait long enough for the secondary to call for an election. + * 4. Verify the primary and secondary did not change. + * 5. Heal the partition. + * 6. Verify the primary and secondary did not change and are in the initial term. + */ + +(function() { + "use strict"; + + load("jstests/libs/check_log.js"); + + var name = "no_flapping_during_network_partition"; + + var replTest = new ReplSetTest({name: name, nodes: 3, useBridge: true}); + var nodes = replTest.startSet(); + var config = replTest.getReplSetConfig(); + config.members[0].priority = 5; + config.members[2].arbiterOnly = true; + config.settings = { + electionTimeoutMillis: 2000 + }; + replTest.initiate(config); + + function getTerm(node) { + return node.adminCommand({replSetGetStatus: 1}).term; + } + + replTest.waitForState(nodes[0], ReplSetTest.State.PRIMARY); + + var primary = replTest.getPrimary(); + var secondary = replTest.getSecondary(); + var initialTerm = getTerm(primary); + + jsTestLog("Create a network partition between the primary and secondary."); + primary.disconnect(secondary); + + jsTestLog("Wait long enough for the secondary to call for an election."); + checkLog.contains(secondary, "can see a healthy primary of equal or greater priority"); + + jsTestLog("Verify the primary and secondary do not change during the partition."); + assert.eq(primary, replTest.getPrimary()); + assert.eq(secondary, replTest.getSecondary()); + + jsTestLog("Heal the partition."); + primary.reconnect(secondary); + + jsTestLog("Verify the primary and secondary did not change and are in the initial term."); + assert.eq(primary, replTest.getPrimary()); + assert.eq(secondary, replTest.getSecondary()); + assert.eq(initialTerm, getTerm(primary)); + assert.eq(initialTerm, getTerm(secondary)); + + replTest.stopSet(); +})(); diff --git a/jstests/replsets/oplog_replay_on_startup_update_and_delete.js b/jstests/replsets/oplog_replay_on_startup_update_and_delete.js new file mode 100644 index 00000000000..14d99b330a4 --- /dev/null +++ b/jstests/replsets/oplog_replay_on_startup_update_and_delete.js @@ -0,0 +1,48 @@ +// SERVER-7200 On startup, replica set nodes delete oplog state past the oplog delete point and +// apply any remaining unapplied ops before coming up as a secondary. This test specifically tests +// having an update and a delete of the same document in the same batch. This is a bit of an edge +// case because if the delete has been applied already, the update won't find any documents. +// +// @tags: [requires_persistence] +(function() { + "use strict"; + + var ns = "test.coll"; + var id = ObjectId(); + + var rst = new ReplSetTest({ + nodes: 1, + }); + + rst.startSet(); + rst.initiate(); + + var conn = rst.getPrimary(); // Waits for PRIMARY state. + + // Do the insert update and delete operations. + var coll = conn.getCollection(ns); + assert.writeOK(coll.insert({_id: id})); + assert.writeOK(coll.update({_id: id}, {$inc: {a: 1}})); + assert.writeOK(coll.remove({_id: id})); + assert.eq(coll.findOne({_id: id}), null); + + // Set the appliedThrough point back to the insert so the update and delete are replayed. + conn = rst.restart(0, {noReplSet: true}); // Restart as a standalone node. + assert.neq(null, conn, "failed to restart"); + var oplog = conn.getCollection('local.oplog.rs'); + oplog.find().forEach(printjsononeline); + assert.eq(oplog.count({ns: ns, op: 'i'}), 1); + var insertOp = oplog.findOne({ns: ns, op: 'i'}); + var term = 't' in insertOp ? insertOp.t : -1; + var minValidColl = conn.getCollection('local.replset.minvalid'); + assert.writeOK(minValidColl.update({}, {$set: {begin: {ts: insertOp.ts, t: term}}})); + printjson({minValidDoc: minValidColl.findOne()}); + + // Make sure it starts up fine again and doesn't have the document. + conn = rst.restart(0); // Restart in replSet mode again. + conn = rst.getPrimary(); // Waits for PRIMARY state. + coll = conn.getCollection(ns); + assert.eq(coll.findOne({_id: id}), null); + + rst.stopSet(); +})(); diff --git a/jstests/replsets/priority_takeover_cascading_priorities.js b/jstests/replsets/priority_takeover_cascading_priorities.js index df119abb669..e210d250f10 100644 --- a/jstests/replsets/priority_takeover_cascading_priorities.js +++ b/jstests/replsets/priority_takeover_cascading_priorities.js @@ -21,25 +21,15 @@ replSet.startSet(); replSet.initiate(); - var waitForPrimary = function(i) { - // Refresh connections to nodes. - replSet.status(); - assert.commandWorked( - replSet.nodes[i].adminCommand({ - replSetTest: 1, - waitForMemberState: ReplSetTest.State.PRIMARY, - timeoutMillis: 60 * 1000, - }), - 'node ' + i + ' ' + replSet.nodes[i].host + ' failed to become primary'); - }; - - waitForPrimary(0); + replSet.waitForState(0, ReplSetTest.State.PRIMARY); + // Wait until all nodes get the "no-op" of "new primary" after initial sync. + waitUntilAllNodesCaughtUp(replSet.nodes); replSet.stop(0); - waitForPrimary(1); + replSet.waitForState(1, ReplSetTest.State.PRIMARY); replSet.stop(1); - waitForPrimary(2); + replSet.waitForState(2, ReplSetTest.State.PRIMARY); // Cannot stop any more nodes because we will not have a majority. })(); diff --git a/jstests/replsets/priority_takeover_two_nodes_equal_priority.js b/jstests/replsets/priority_takeover_two_nodes_equal_priority.js index 42134f9430e..10f31796b2a 100644 --- a/jstests/replsets/priority_takeover_two_nodes_equal_priority.js +++ b/jstests/replsets/priority_takeover_two_nodes_equal_priority.js @@ -1,25 +1,29 @@ -// 2 nodes with non-default priority. -// 3-node replica set with priorities 3, 3 and 1 (default) -// Start replica set. Ensure that highest priority node becomes primary eventually. -// Step down the primary and confirm that the next highest priority node becomes primary. +/** + * Test to ensure that nodes with the highest priorities eventually become PRIMARY. + * + * 1. Initiate a 3 node replica set with node priorities of 3, 3 and 1 (default) + * 2. Make sure that one of the highest priority nodes becomes PRIMARY. + * 3. Step down the PRIMARY and confirm that the other high priority node becomes PRIMARY. + */ load('jstests/replsets/rslib.js'); (function() { 'use strict'; var name = 'priority_takeover_two_nodes_equal_priority'; - var replSet = new ReplSetTest( - {name: name, nodes: [{rsConfig: {priority: 3}}, {rsConfig: {priority: 3}}, {}, ]}); - replSet.startSet(); - replSet.initiate(); + var replTest = new ReplSetTest( + {name: name, nodes: [{rsConfig: {priority: 3}}, {rsConfig: {priority: 3}}, {}]}); + replTest.startSet(); + replTest.initiate(); + jsTestLog("Waiting for one of the high priority nodes to become PRIMARY."); var primary; var primaryIndex = -1; var defaultPriorityNodeIndex = 2; assert.soon( function() { - primary = replSet.getPrimary(); - replSet.nodes.find(function(node, index, array) { + primary = replTest.getPrimary(); + replTest.nodes.find(function(node, index, array) { if (primary.host == node.host) { primaryIndex = index; return true; @@ -28,27 +32,25 @@ load('jstests/replsets/rslib.js'); }); return primaryIndex !== defaultPriorityNodeIndex; }, - 'neither of the priority 3 nodes was elected primary', - 60000, // timeout - 1000 // interval + 'Neither of the high priority nodes was elected primary.', + replTest.kDefaultTimeoutMS, // timeout + 1000 // interval ); - try { - assert.commandWorked(primary.getDB('admin').runCommand({replSetStepDown: 90})); - } catch (x) { - // expected - } - var newPrimaryIndex = primaryIndex === 0 ? 1 : 0; - - // Refresh connections to nodes. - replSet.status(); - - assert.commandWorked(replSet.nodes[newPrimaryIndex].adminCommand({ - replSetTest: 1, - waitForMemberState: ReplSetTest.State.PRIMARY, - timeoutMillis: 60 * 1000, - }), - 'node ' + newPrimaryIndex + ' ' + replSet.nodes[newPrimaryIndex].host + - ' failed to become primary'); + jsTestLog("Stepping down the current primary."); + assert.throws(function() { + assert.commandWorked( + primary.adminCommand({replSetStepDown: 10 * 60, secondaryCatchUpPeriodSecs: 10 * 60})); + }); + + // Make sure the primary has stepped down. + assert.neq(primary, replTest.getPrimary()); + + // We expect the other high priority node to eventually become primary. + var expectedNewPrimaryIndex = (primaryIndex === 0) ? 1 : 0; + + jsTestLog("Waiting for the other high priority node to become PRIMARY."); + var expectedNewPrimary = replTest.nodes[expectedNewPrimaryIndex]; + replTest.waitForState(expectedNewPrimary, ReplSetTest.State.PRIMARY); })(); diff --git a/jstests/replsets/protocol_version_upgrade_downgrade.js b/jstests/replsets/protocol_version_upgrade_downgrade.js index 58210853c91..06bd26f2d94 100644 --- a/jstests/replsets/protocol_version_upgrade_downgrade.js +++ b/jstests/replsets/protocol_version_upgrade_downgrade.js @@ -35,7 +35,8 @@ load("jstests/replsets/rslib.js"); // Do a write, this will set up sync sources on secondaries. print("do a write"); - assert.writeOK(primaryColl.bar.insert({x: 1}, {writeConcern: {w: 3}})); + assert.writeOK( + primaryColl.bar.insert({x: 1}, {writeConcern: {w: 3, wtimeout: rst.kDefaultTimeoutMS}})); // Check optime format in protocol version 0, which is a Timestamp. var res = primary.adminCommand({replSetGetStatus: 1}); assert.commandWorked(res); @@ -56,7 +57,8 @@ load("jstests/replsets/rslib.js"); // This write may block until all nodes finish upgrade, because replSetUpdatePosition may be // rejected by the primary for mismatched config version before secondaries get reconfig. // This will make secondaries wait for 0.5 seconds and retry. - assert.writeOK(primaryColl.bar.insert({x: 2}, {writeConcern: {w: 3}})); + assert.writeOK( + primaryColl.bar.insert({x: 2}, {writeConcern: {w: 3, wtimeout: rst.kDefaultTimeoutMS}})); // Check optime format in protocol version 1, which is an object including the term. res = primary.adminCommand({replSetGetStatus: 1}); @@ -79,7 +81,8 @@ load("jstests/replsets/rslib.js"); conf.protocolVersion = 0; conf.version++; reconfig(rst, conf); - assert.writeOK(primaryColl.bar.insert({x: 3}, {writeConcern: {w: 3}})); + assert.writeOK( + primaryColl.bar.insert({x: 3}, {writeConcern: {w: 3, wtimeout: rst.kDefaultTimeoutMS}})); // Check optime format in protocol version 0, which is a Timestamp. res = primary.adminCommand({replSetGetStatus: 1}); diff --git a/jstests/replsets/read_after_optime.js b/jstests/replsets/read_after_optime.js index 30cf7782679..7c5217660f6 100644 --- a/jstests/replsets/read_after_optime.js +++ b/jstests/replsets/read_after_optime.js @@ -2,6 +2,7 @@ (function() { "use strict"; + load("jstests/libs/check_log.js"); var replTest = new ReplSetTest({nodes: 2}); replTest.startSet(); @@ -33,30 +34,6 @@ assert.gt(timeoutResult.waitedMS, 500); }; - var countLogMessages = function(msg) { - var total = 0; - var logMessages = assert.commandWorked(testDB.adminCommand({getLog: 'global'})).log; - for (var i = 0; i < logMessages.length; i++) { - if (logMessages[i].indexOf(msg) != -1) { - total++; - } - } - return total; - }; - - var checkLog = function(msg, expectedCount) { - var count; - assert.soon( - function() { - count = countLogMessages(msg); - return expectedCount == count; - }, - 'Expected ' + expectedCount + ', but instead saw ' + count + - ' log entries containing the following message: ' + msg, - 60000, - 300); - }; - // Run the time out test 3 times with replication debug log level increased to 2 // for first and last run. The time out message should be logged twice. testDB.setLogLevel(2, 'command'); @@ -65,7 +42,7 @@ var msg = 'Command on database ' + testDB.getName() + ' timed out waiting for read concern to be satisfied. Command:'; - checkLog(msg, 1); + checkLog.containsWithCount(testDB.getMongo(), msg, 1); // Read concern timed out message should not be logged. runTimeoutTest(); @@ -74,7 +51,7 @@ runTimeoutTest(); testDB.setLogLevel(0, 'command'); - checkLog(msg, 2); + checkLog.containsWithCount(testDB.getMongo(), msg, 2); // Test read on future afterOpTime that will eventually occur. var insertFunc = startParallelShell( diff --git a/jstests/replsets/read_committed.js b/jstests/replsets/read_committed.js index 26611a04f0a..5d1d906c823 100644 --- a/jstests/replsets/read_committed.js +++ b/jstests/replsets/read_committed.js @@ -1,6 +1,4 @@ /** - * @tags: [requires_journaling] - * * Test basic read committed functionality, including: * - Writes with writeConcern 'majority' should be visible once the write completes. * - With the only data-bearing secondary down, committed reads should not include newly inserted @@ -11,6 +9,46 @@ (function() { "use strict"; + const majorityWriteConcern = { + writeConcern: {w: "majority", wtimeout: 60 * 1000} + }; + + // Each test case includes a 'prepareCollection' method that sets up the initial state starting + // with an empty collection, a 'write' method that does some write, and two arrays, + // 'expectedBefore' and 'expectedAfter' that describe the expected contents of the collection + // before and after the write. The 'prepareCollection' and 'write' methods should leave the + // collection either empty or with a single document with _id: 1. + const testCases = { + insert: { + prepareCollection: function(coll) {}, // No-op + write: function(coll, writeConcern) { + assert.writeOK(coll.insert({_id: 1}, writeConcern)); + }, + expectedBefore: [], + expectedAfter: [{_id: 1}], + }, + update: { + prepareCollection: function(coll) { + assert.writeOK(coll.insert({_id: 1, state: 'before'}, majorityWriteConcern)); + }, + write: function(coll, writeConcern) { + assert.writeOK(coll.update({_id: 1}, {$set: {state: 'after'}}, writeConcern)); + }, + expectedBefore: [{_id: 1, state: 'before'}], + expectedAfter: [{_id: 1, state: 'after'}], + }, + remove: { + prepareCollection: function(coll) { + assert.writeOK(coll.insert({_id: 1}, majorityWriteConcern)); + }, + write: function(coll, writeConcern) { + assert.writeOK(coll.remove({_id: 1}, writeConcern)); + }, + expectedBefore: [{_id: 1}], + expectedAfter: [], + }, + }; + // Set up a set and grab things for later. var name = "read_committed"; var replTest = @@ -29,56 +67,117 @@ throw e; } - replTest.initiate({ + var nodes = replTest.nodeList(); + var config = { "_id": name, "members": [ {"_id": 0, "host": nodes[0]}, {"_id": 1, "host": nodes[1], priority: 0}, {"_id": 2, "host": nodes[2], arbiterOnly: true} ] - }); + }; + replTest.initiate(config); // Get connections and collection. var primary = replTest.getPrimary(); var secondary = replTest.liveNodes.slaves[0]; - var secondaryId = replTest.getNodeId(secondary); - var db = primary.getDB(name); - var t = db[name]; + var coll = primary.getDB(name)[name]; + var secondaryColl = secondary.getDB(name)[name]; - function doDirtyRead() { - var res = t.runCommand('find', {"readConcern": {"level": "local"}}); - assert.commandWorked(res); - return new DBCommandCursor(db.getMongo(), res).toArray()[0].state; + function log(arg) { + jsTest.log(tojson(arg)); + } + + function doRead(coll, readConcern) { + readConcern.maxTimeMS = 3000; + var res = assert.commandWorked(coll.runCommand('find', readConcern)); + return new DBCommandCursor(coll.getMongo(), res).toArray(); + } + + function doDirtyRead(coll) { + log("doing dirty read"); + var ret = doRead(coll, {"readConcern": {"level": "local"}}); + log("done doing dirty read."); + return ret; + } + + function doCommittedRead(coll) { + log("doing committed read"); + var ret = doRead(coll, {"readConcern": {"level": "majority"}}); + log("done doing committed read."); + return ret; } - function doCommittedRead() { - var res = t.runCommand('find', {"readConcern": {"level": "majority"}}); + function readLatestOplogEntry(readConcernLevel) { + var oplog = primary.getDB('local').oplog.rs; + var res = oplog.runCommand('find', + { + "readConcern": {"level": readConcernLevel}, + "maxTimeMS": 3000, + sort: {$natural: -1}, + limit: 1, + }); assert.commandWorked(res); - return new DBCommandCursor(db.getMongo(), res).toArray()[0].state; + return new DBCommandCursor(coll.getMongo(), res).toArray()[0]; } - // Do a write, wait for it to replicate, and ensure it is visible. - assert.writeOK( - t.save({_id: 1, state: 0}, {writeConcern: {w: "majority", wtimeout: 60 * 1000}})); - assert.eq(doDirtyRead(), 0); - assert.eq(doCommittedRead(), 0); + for (var testName in testCases) { + jsTestLog('Running test ' + testName); + var test = testCases[testName]; + + const setUpInitialState = function setUpInitialState() { + assert.writeOK(coll.remove({}, majorityWriteConcern)); + test.prepareCollection(coll); + // Do some sanity checks. + assert.eq(doDirtyRead(coll), test.expectedBefore); + assert.eq(doCommittedRead(coll), test.expectedBefore); + }; - replTest.stop(secondaryId); + // Writes done with majority write concern must be immediately visible to both dirty and + // committed reads. + setUpInitialState(); + test.write(coll, majorityWriteConcern); + assert.eq(doDirtyRead(coll), test.expectedAfter); + assert.eq(doCommittedRead(coll), test.expectedAfter); - // Do a write and ensure it is only visible to dirty reads - assert.writeOK(t.save({_id: 1, state: 1})); - assert.eq(doDirtyRead(), 1); - assert.eq(doCommittedRead(), 0); + // Return to the initial state, then stop the secondary from applying new writes to prevent + // them from becoming committed. + setUpInitialState(); + assert.commandWorked( + secondary.adminCommand({configureFailPoint: "rsSyncApplyStop", mode: "alwaysOn"})); + const initialOplogTs = readLatestOplogEntry('local').ts; - // Try the committed read again after sleeping to ensure it doesn't only work for queries - // immediately after the write. - sleep(1000); - assert.eq(doCommittedRead(), 0); + // Writes done without majority write concern must be immediately visible to dirty read + // and hidden from committed reads until they have been replicated. The rules for seeing + // an oplog entry for a write are the same as for the write itself. + test.write(coll, {}); + assert.eq(doDirtyRead(coll), test.expectedAfter); + assert.neq(readLatestOplogEntry('local').ts, initialOplogTs); + assert.eq(doCommittedRead(coll), test.expectedBefore); + assert.eq(readLatestOplogEntry('majority').ts, initialOplogTs); - // Restart the node and ensure the committed view is updated. - replTest.restart(secondaryId); - db.getLastError("majority", 60 * 1000); - assert.eq(doDirtyRead(), 1); - assert.eq(doCommittedRead(), 1); + // Try the committed read again after sleeping to ensure it doesn't only work for + // queries immediately after the write. + sleep(1000); + assert.eq(doCommittedRead(coll), test.expectedBefore); + assert.eq(readLatestOplogEntry('majority').ts, initialOplogTs); + // Restart oplog application on the secondary and ensure the committed view is updated. + assert.commandWorked( + secondary.adminCommand({configureFailPoint: "rsSyncApplyStop", mode: "off"})); + coll.getDB().getLastError("majority", 60 * 1000); + assert.eq(doCommittedRead(coll), test.expectedAfter); + assert.neq(readLatestOplogEntry('majority').ts, initialOplogTs); + + // The secondary will be able to make the write committed soon after the primary, but there + // is no way to block until it does. + try { + assert.soon(function() { + return friendlyEqual(doCommittedRead(secondaryColl), test.expectedAfter); + }); + } catch (e) { + // generate useful error messages on failures. + assert.eq(doCommittedRead(secondaryColl), test.expectedAfter); + } + } }()); diff --git a/jstests/replsets/read_committed_after_rollback.js b/jstests/replsets/read_committed_after_rollback.js index 68ba334ac9c..4101d9a242c 100644 --- a/jstests/replsets/read_committed_after_rollback.js +++ b/jstests/replsets/read_committed_after_rollback.js @@ -149,8 +149,10 @@ load("jstests/replsets/rslib.js"); // For startSetIfSupportsReadMajority. // now be visible as a committed read to both oldPrimary and newPrimary. assert.commandWorked( pureSecondary.adminCommand({configureFailPoint: "rsSyncApplyStop", mode: "off"})); - assert.commandWorked( - newPrimaryColl.runCommand({getLastError: 1, w: 'majority', wtimeout: 30000})); + // Do a write to the new primary so that the old primary can establish a sync source to learn + // about the new commit. + assert.writeOK(newPrimary.getDB(name).unrelatedCollection.insert( + {a: 1}, {writeConcern: {w: 'majority', wtimeout: replTest.kDefaultTimeoutMS}})); assert.eq(doCommittedRead(newPrimaryColl), 'new'); assert.eq(doCommittedRead(oldPrimaryColl), 'new'); }()); diff --git a/jstests/replsets/read_committed_stale_history.js b/jstests/replsets/read_committed_stale_history.js new file mode 100644 index 00000000000..a92595bf439 --- /dev/null +++ b/jstests/replsets/read_committed_stale_history.js @@ -0,0 +1,149 @@ +/* + * Tests that a node on a stale branch of history won't incorrectly mark its ops as committed even + * when hearing about a commit point with a higher optime from a new primary. + */ +(function() { + 'use strict'; + + load("jstests/libs/check_log.js"); + load("jstests/libs/write_concern_util.js"); + load("jstests/replsets/rslib.js"); + + var name = "readCommittedStaleHistory"; + var dbName = "wMajorityCheck"; + var collName = "stepdown"; + + var rst = new ReplSetTest({ + name: name, + nodes: [{}, {}, {rsConfig: {priority: 0}}, ], + nodeOptions: {enableMajorityReadConcern: ""}, + useBridge: true + }); + + if (!startSetIfSupportsReadMajority(rst)) { + jsTest.log("skipping test since storage engine doesn't support committed reads"); + return; + } + + var nodes = rst.nodes; + rst.initiate(); + + /** + * Waits for the given node to be in state primary *and* have finished drain mode and thus + * be available for writes. + */ + function waitForPrimary(node) { + assert.soon(function() { + return node.adminCommand('ismaster').ismaster; + }); + } + + // Asserts that the given document is not visible in the committed snapshot on the given node. + function checkDocNotCommitted(node, doc) { + var docs = + node.getDB(dbName).getCollection(collName).find(doc).readConcern('majority').toArray(); + assert.eq(0, docs.length, tojson(docs)); + } + + jsTestLog("Make sure node 0 is primary."); + rst.stepUp(nodes[0]); + var primary = rst.getPrimary(); + var secondaries = rst.getSecondaries(); + assert.eq(nodes[0], primary); + // Wait for all data bearing nodes to get up to date. + assert.writeOK(nodes[0].getDB(dbName).getCollection(collName).insert( + {a: 1}, {writeConcern: {w: 3, wtimeout: rst.kDefaultTimeoutMS}})); + + // Stop the secondaries from replicating. + stopServerReplication(secondaries); + // Stop the primary from being able to complete stepping down. + assert.commandWorked( + nodes[0].adminCommand({configureFailPoint: 'blockHeartbeatStepdown', mode: 'alwaysOn'})); + + jsTestLog("Do a write that won't ever reach a majority of nodes"); + assert.writeOK(nodes[0].getDB(dbName).getCollection(collName).insert({a: 2})); + + // Ensure that the write that was just done is not visible in the committed snapshot. + checkDocNotCommitted(nodes[0], {a: 2}); + + // Prevent the primary from rolling back later on. + assert.commandWorked( + nodes[0].adminCommand({configureFailPoint: 'rollbackHangBeforeStart', mode: 'alwaysOn'})); + + jsTest.log("Disconnect primary from all secondaries"); + nodes[0].disconnect(nodes[1]); + nodes[0].disconnect(nodes[2]); + + // Ensure the soon-to-be primary cannot see the write from the old primary. + assert.eq(null, nodes[1].getDB(dbName).getCollection(collName).findOne({a: 2})); + + jsTest.log("Wait for a new primary to be elected"); + // Allow the secondaries to replicate again. + restartServerReplication(secondaries); + + waitForPrimary(nodes[1]); + + jsTest.log("Do a write to the new primary"); + assert.writeOK(nodes[1].getDB(dbName).getCollection(collName).insert( + {a: 3}, {writeConcern: {w: 2, wtimeout: rst.kDefaultTimeoutMS}})); + + // Ensure the new primary still cannot see the write from the old primary. + assert.eq(null, nodes[1].getDB(dbName).getCollection(collName).findOne({a: 2})); + + // Ensure the stale primary still hasn't committed the write it did that never reached + // the other nodes. + checkDocNotCommitted(nodes[0], {a: 2}); + + jsTest.log("Reconnect the old primary to the rest of the nodes"); + nodes[1].reconnect(nodes[0]); + nodes[2].reconnect(nodes[0]); + + // Sleep 10 seconds to allow some heartbeats to be processed, so we can verify that the + // heartbeats don't cause the stale primary to incorrectly advance the commit point. + sleep(10000); + + // Ensure the new primary still cannot see the write from the old primary. + assert.eq(null, nodes[1].getDB(dbName).getCollection(collName).findOne({a: 2})); + + // Ensure the stale primary still hasn't committed the write it did that never reached + // the other nodes. + checkDocNotCommitted(nodes[0], {a: 2}); + + jsTest.log("Allow the old primary to finish stepping down and become secondary"); + var res = null; + try { + res = nodes[0].adminCommand({configureFailPoint: 'blockHeartbeatStepdown', mode: 'off'}); + } catch (e) { + // Expected - once we disable the fail point the stepdown will proceed and it's racy whether + // the stepdown closes all connections before or after the configureFailPoint command + // returns + } + if (res) { + assert.commandWorked(res); + } + rst.waitForState(nodes[0], ReplSetTest.State.SECONDARY); + reconnect(nodes[0]); + + // At this point the former primary will attempt to go into rollback, but the + // 'rollbackHangBeforeStart' will prevent it from doing so. + checkDocNotCommitted(nodes[0], {a: 2}); + checkLog.contains(nodes[0], 'rollback - rollbackHangBeforeStart fail point enabled'); + checkDocNotCommitted(nodes[0], {a: 2}); + + jsTest.log("Allow the original primary to roll back its write and catch up to the new primary"); + assert.commandWorked( + nodes[0].adminCommand({configureFailPoint: 'rollbackHangBeforeStart', mode: 'off'})); + + assert.soonNoExcept(function() { + return null == nodes[0].getDB(dbName).getCollection(collName).findOne({a: 2}); + }, "Original primary never rolled back its write"); + + rst.awaitReplication(); + + // Ensure that the old primary got the write that the new primary did and sees it as committed. + assert.neq( + null, + nodes[0].getDB(dbName).getCollection(collName).find({a: 3}).readConcern('majority').next()); + + rst.stopSet(); +}()); diff --git a/jstests/replsets/read_committed_with_catalog_changes.js b/jstests/replsets/read_committed_with_catalog_changes.js new file mode 100644 index 00000000000..dae14da31de --- /dev/null +++ b/jstests/replsets/read_committed_with_catalog_changes.js @@ -0,0 +1,343 @@ +/** + * Test read committed functionality when mixed with catalog changes. Since we don't support + * multiple versions of the catalog, operations that modify the catalog may need to lock out + * committed readers until the modification is in the committed snapshot. + * + * The following replicated operations are tested here: + * - creating a collection in an existing db + * - creating a collection in a new db + * - dropping a collection + * - dropping a db + * - dropping a collection and creating one with the same name + * - dropping a db and creating one with the same name + * - renaming a collection to a new, unused name + * - renaming a collection on top of an existing collection + * - creating a foreground index + * - creating a background index + * - dropping an index + * + * The following non-replicated operations are tested here: + * - repair database + * - reindex collection + * - compact collection + */ + +load("jstests/libs/parallelTester.js"); // For ScopedThread. +load("jstests/replsets/rslib.js"); // For startSetIfSupportsReadMajority. + +(function() { + "use strict"; + + // Each test case includes a 'prepare' method that sets up the initial state starting with a + // database that has been dropped, a 'performOp' method that does some operation, and two + // arrays, 'blockedCollections' and 'unblockedCollections', that list the collections that + // should be blocked or unblocked between the time the operation is performed until it is + // committed. If the operation is local only and isn't replicated, the test case should include + // a 'localOnly' field set to true. Test cases are not allowed to touch any databases other than + // the one passed in. + const testCases = { + createCollectionInExistingDB: { + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + }, + performOp: function(db) { + assert.writeOK(db.coll.insert({_id: 1})); + }, + blockedCollections: ['coll'], + unblockedCollections: ['other'], + }, + createCollectionInNewDB: { + prepare: function(db) {}, + performOp: function(db) { + assert.writeOK(db.coll.insert({_id: 1})); + }, + blockedCollections: ['coll'], + unblockedCollections: ['otherDoesNotExist'], // Only existent collections are blocked. + }, + dropCollection: { + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + assert.writeOK(db.coll.insert({_id: 1})); + }, + performOp: function(db) { + assert(db.coll.drop()); + }, + blockedCollections: [], + unblockedCollections: ['coll', 'other'], + }, + dropDB: { + prepare: function(db) { + assert.writeOK(db.coll.insert({_id: 1})); + }, + performOp: function(db) { + assert.commandWorked(db.dropDatabase()); + }, + blockedCollections: [], + unblockedCollections: ['coll'], + }, + dropAndRecreateCollection: { + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + assert.writeOK(db.coll.insert({_id: 1})); + }, + performOp: function(db) { + assert(db.coll.drop()); + assert.writeOK(db.coll.insert({_id: 1})); + }, + blockedCollections: ['coll'], + unblockedCollections: ['other'], + }, + dropAndRecreateDB: { + prepare: function(db) { + assert.writeOK(db.coll.insert({_id: 1})); + }, + performOp: function(db) { + assert.commandWorked(db.dropDatabase()); + assert.writeOK(db.coll.insert({_id: 1})); + }, + blockedCollections: ['coll'], + unblockedCollections: ['otherDoesNotExist'], + }, + renameCollectionToNewName: { + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + assert.writeOK(db.from.insert({_id: 1})); + }, + performOp: function(db) { + assert.commandWorked(db.from.renameCollection('coll')); + }, + blockedCollections: ['coll'], + unblockedCollections: ['other', 'from' /*doesNotExist*/], + }, + renameCollectionToExistingName: { + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + assert.writeOK(db.from.insert({_id: 'from'})); + assert.writeOK(db.coll.insert({_id: 'coll'})); + }, + performOp: function(db) { + assert.commandWorked(db.from.renameCollection('coll', true)); + }, + blockedCollections: ['coll'], + unblockedCollections: ['other', 'from' /*doesNotExist*/], + }, + createIndexForeground: { + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + assert.writeOK(db.coll.insert({_id: 1})); + }, + performOp: function(db) { + assert.commandWorked(db.coll.ensureIndex({x: 1}, {background: false})); + }, + blockedCollections: ['coll'], + unblockedCollections: ['other'], + }, + createIndexBackground: { + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + assert.writeOK(db.coll.insert({_id: 1})); + }, + performOp: function(db) { + assert.commandWorked(db.coll.ensureIndex({x: 1}, {background: true})); + }, + blockedCollections: ['coll'], + unblockedCollections: ['other'], + }, + dropIndex: { + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + assert.writeOK(db.coll.insert({_id: 1})); + assert.commandWorked(db.coll.ensureIndex({x: 1})); + }, + performOp: function(db) { + assert.commandWorked(db.coll.dropIndex({x: 1})); + }, + blockedCollections: ['coll'], + unblockedCollections: ['other'], + }, + + // Remaining cases are local-only operations. + repairDatabase: { + prepare: function(db) { + assert.writeOK(db.coll.insert({_id: 1})); + }, + performOp: function(db) { + assert.commandWorked(db.repairDatabase()); + }, + blockedCollections: ['coll'], + unblockedCollections: ['otherDoesNotExist'], + localOnly: true, + }, + reIndex: { + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + assert.writeOK(db.coll.insert({_id: 1})); + assert.commandWorked(db.coll.ensureIndex({x: 1})); + }, + performOp: function(db) { + assert.commandWorked(db.coll.reIndex()); + }, + blockedCollections: ['coll'], + unblockedCollections: ['other'], + localOnly: true, + }, + compact: { + // At least on WiredTiger, compact is fully inplace so it doesn't need to block readers. + prepare: function(db) { + assert.writeOK(db.other.insert({_id: 1})); + assert.writeOK(db.coll.insert({_id: 1})); + assert.commandWorked(db.coll.ensureIndex({x: 1})); + }, + performOp: function(db) { + var res = db.coll.runCommand('compact', {force: true}); + if (res.code != ErrorCodes.CommandNotSupported) { + // It is fine for a storage engine to support snapshots but not compact. Since + // compact doesn't block any collections we are fine with doing a no-op here. + // Other errors should fail the test. + assert.commandWorked(res); + } + + }, + blockedCollections: [], + unblockedCollections: ['coll', 'other'], + localOnly: true, + }, + }; + + // Assertion helpers. These must get all state as arguments rather than through closure since + // they may be passed in to a ScopedThread. + function assertReadsBlock(coll) { + var res = + coll.runCommand('find', {"readConcern": {"level": "majority"}, "maxTimeMS": 1000}); + assert.commandFailedWithCode(res, + ErrorCodes.ExceededTimeLimit, + "Expected read of " + coll.getFullName() + " to block"); + } + + function assertReadsSucceed(coll, timeoutMs = 1000) { + var res = coll.runCommand('find', + {"readConcern": {"level": "majority"}, "maxTimeMS": timeoutMs}); + assert.commandWorked(res, 'reading from ' + coll.getFullName()); + // Exhaust the cursor to avoid leaking cursors on the server. + new DBCommandCursor(coll.getMongo(), res).itcount(); + } + + // Set up a set and grab things for later. + var name = "read_committed_with_catalog_changes"; + var replTest = + new ReplSetTest({name: name, nodes: 3, nodeOptions: {enableMajorityReadConcern: ''}}); + + if (!startSetIfSupportsReadMajority(replTest)) { + jsTest.log("skipping test since storage engine doesn't support committed reads"); + return; + } + + var nodes = replTest.nodeList(); + var config = { + "_id": name, + "members": [ + {"_id": 0, "host": nodes[0]}, + {"_id": 1, "host": nodes[1], priority: 0}, + {"_id": 2, "host": nodes[2], arbiterOnly: true} + ] + }; + replTest.initiate(config); + + // Get connections. + var primary = replTest.getPrimary(); + var secondary = replTest.liveNodes.slaves[0]; + + // This is the DB that all of the tests will use. + var mainDB = primary.getDB('mainDB'); + + // This DB won't be used by any tests so it should always be unblocked. + var otherDB = primary.getDB('otherDB'); + var otherDBCollection = otherDB.collection; + assert.writeOK( + otherDBCollection.insert({}, {writeConcern: {w: "majority", wtimeout: 60 * 1000}})); + assertReadsSucceed(otherDBCollection); + + for (var testName in testCases) { + jsTestLog('Running test ' + testName); + var test = testCases[testName]; + + const setUpInitialState = function setUpInitialState() { + assert.commandWorked(mainDB.dropDatabase()); + test.prepare(mainDB); + mainDB.getLastError('majority', 60 * 1000); + // Do some sanity checks. + assertReadsSucceed(otherDBCollection); + test.blockedCollections.forEach((name) => assertReadsSucceed(mainDB[name])); + test.unblockedCollections.forEach((name) => assertReadsSucceed(mainDB[name])); + }; + + // All operations, whether replicated or not, must become visible automatically as long as + // the secondary is keeping up. + setUpInitialState(); + test.performOp(mainDB); + assertReadsSucceed(otherDBCollection); + test.blockedCollections.forEach((name) => assertReadsSucceed(mainDB[name])); + test.unblockedCollections.forEach((name) => assertReadsSucceed(mainDB[name])); + + // Return to the initial state, then stop the secondary from applying new writes to prevent + // them from becoming committed. + setUpInitialState(); + assert.commandWorked( + secondary.adminCommand({configureFailPoint: "rsSyncApplyStop", mode: "alwaysOn"})); + + // If the tested operation isn't replicated, do a write to the side collection before + // performing the operation. This will ensure that the operation happens after an + // uncommitted write which prevents it from immediately being marked as committed. + if (test.localOnly) { + assert.writeOK(otherDBCollection.insert({})); + } + + // Perform the op and ensure that blocked collections block and unblocked ones don't. + test.performOp(mainDB); + assertReadsSucceed(otherDBCollection); + test.blockedCollections.forEach((name) => assertReadsBlock(mainDB[name])); + test.unblockedCollections.forEach((name) => assertReadsSucceed(mainDB[name])); + + // Use background threads to test that reads that start blocked can complete if the + // operation they are waiting on becomes committed while the read is still blocked. + // We don't do this when testing auth because ScopedThread's don't propagate auth + // credentials. + var threads = jsTest.options().auth ? [] : test.blockedCollections.map((name) => { + // This function must get all inputs as arguments and can't use closure because it + // is used in a ScopedThread. + function bgThread(host, collection, assertReadsSucceed) { + // Use a longer timeout since we expect to block for a little while (at least 2 + // seconds). + assertReadsSucceed(new Mongo(host).getCollection(collection), 30 * 1000); + } + var thread = new ScopedThread( + bgThread, primary.host, mainDB[name].getFullName(), assertReadsSucceed); + thread.start(); + return thread; + }); + sleep(1000); // Give the reads a chance to block. + + try { + // Try the committed read again after sleeping to ensure that it still blocks even if it + // isn't immediately after the operation. + test.blockedCollections.forEach((name) => assertReadsBlock(mainDB[name])); + + // Restart oplog application on the secondary and ensure the blocked collections become + // unblocked. + assert.commandWorked( + secondary.adminCommand({configureFailPoint: "rsSyncApplyStop", mode: "off"})); + mainDB.getLastError("majority", 60 * 1000); + test.blockedCollections.forEach((name) => assertReadsSucceed(mainDB[name])); + + // Wait for the threads to complete and report any errors encountered from running them. + threads.forEach((thread) => { + thread.join(); + thread.join = () => {}; // Make join a no-op for the finally below. + assert(!thread.hasFailed(), "One of the threads failed. See above for details."); + }); + } finally { + // Make sure we wait for all threads to finish. + threads.forEach(thread => thread.join()); + } + } +}()); diff --git a/jstests/replsets/reconfig_without_increased_queues.js b/jstests/replsets/reconfig_without_increased_queues.js index 8b71817166d..39ef7a7e60c 100644 --- a/jstests/replsets/reconfig_without_increased_queues.js +++ b/jstests/replsets/reconfig_without_increased_queues.js @@ -69,6 +69,16 @@ replTest.awaitSecondaryNodes(); + // We cannot reconfig nodes[2] to have priority 0 if it is currently the primary. After the + // first reconfig, it will be unelectable so this only needs to be done once. + if (replTest.getPrimary() === replTest.nodes[2]) { + jsTestLog("Stepping down node 2 before reconfig"); + assert.throws(function() { + replTest.nodes[2].adminCommand({replSetStepDown: 5, force: true}); + }); + replTest.waitForState(replTest.nodes[2], ReplSetTest.State.SECONDARY); + } + // ** Setup different priorities var c = replTest.getReplSetConfigFromNode(); c.members[0].priority = 99; diff --git a/jstests/replsets/replsetfreeze.js b/jstests/replsets/replsetfreeze.js index aff26f4e20a..dcdc738f43c 100644 --- a/jstests/replsets/replsetfreeze.js +++ b/jstests/replsets/replsetfreeze.js @@ -53,6 +53,10 @@ var r = replTest.initiate(config); var master = replTest.getPrimary(); var secondary = replTest.getSecondary(); +assert.commandFailedWithCode(master.getDB("admin").runCommand({replSetFreeze: 30}), + ErrorCodes.NotSecondary, + 'replSetFreeze should return error when run on primary'); + replTest.awaitSecondaryNodes(); print("2: step down m1"); diff --git a/jstests/replsets/request_primary_stepdown.js b/jstests/replsets/request_primary_stepdown.js new file mode 100644 index 00000000000..02050bd55f4 --- /dev/null +++ b/jstests/replsets/request_primary_stepdown.js @@ -0,0 +1,46 @@ +// 3-node replica set - one arbiter and two electable nodes with different priorities. +// Wait for replica set to stabilize with the higher priority node as primary. +// Step down the high priority node. Wait for the lower priority electable node to become primary. +// Eventually the high priority node will run a priority takeover election to become primary. During +// this election that node should make sure that it does not error in _requestRemotePrimaryStepDown. +(function() { + 'use strict'; + load('jstests/replsets/rslib.js'); + + var name = 'request_primary_stepdown'; + var replSet = new ReplSetTest( + {name: name, nodes: [{rsConfig: {priority: 3}}, {}, {rsConfig: {arbiterOnly: true}}]}); + replSet.startSet(); + var conf = replSet.getReplSetConfig(); + conf.protocolVersion = 0; + replSet.initiate(conf); + + replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replSet.awaitSecondaryNodes(); + replSet.awaitReplication(); + var primary = replSet.getPrimary(); + + assert.commandWorked( + replSet.nodes[0].adminCommand({setParameter: 1, logComponentVerbosity: {executor: 4}})); + clearRawMongoProgramOutput(); + + // Primary should step down long enough for election to occur on secondary. + var stepDownException = assert.throws(function() { + var result = primary.adminCommand({replSetStepDown: 70, secondaryCatchUpPeriodSecs: 60}); + print('replSetStepDown did not throw exception but returned: ' + tojson(result)); + }); + assert.neq(-1, + tojson(stepDownException).indexOf('error doing query'), + 'replSetStepDown did not disconnect client'); + + // Wait for node 1 to be promoted to primary after node 0 stepped down. + replSet.waitForState(replSet.nodes[1], ReplSetTest.State.PRIMARY, 60 * 1000); + + // Eventually node 0 will stand for election again because it has a higher priorty. + replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY, 100 * 1000); + var logContents = rawMongoProgramOutput(); + assert.eq(logContents.indexOf("stepdown period must be longer than secondaryCatchUpPeriodSecs"), + -1, + "_requestRemotePrimaryStepDown sent an invalid replSetStepDown command"); + +})(); diff --git a/jstests/replsets/resync_with_write_load.js b/jstests/replsets/resync_with_write_load.js index 9d8031a1007..1a782ffacbe 100644 --- a/jstests/replsets/resync_with_write_load.js +++ b/jstests/replsets/resync_with_write_load.js @@ -90,7 +90,7 @@ print("waiting for load generation to finish"); loadGen(); // load must stop before we await replication. -replTest.awaitReplication(60 * 1000); +replTest.awaitReplication(240 * 1000); // Make sure oplogs match try { diff --git a/jstests/replsets/rollback_after_sync_source_selection.js b/jstests/replsets/rollback_after_sync_source_selection.js new file mode 100644 index 00000000000..817da77164e --- /dev/null +++ b/jstests/replsets/rollback_after_sync_source_selection.js @@ -0,0 +1,113 @@ +/* + * This tests that nodes get a new sync source if their sync source rolls back between when it is + * chosen as a sync source and when it is first used. The test sets up a five node replicaset + * and creates a simple rollback scenario. Before node 0 goes into rollback, however, we pause + * the oplog fetcher on node 2, which is syncing from node 0. After the rollback occurs we let + * the oplog fetcher continue and after it gets back its first batch from the sync source it + * realizes that its sync source has rolled back and it errors before getting a new sync source. + */ + +(function() { + 'use strict'; + + load("jstests/libs/check_log.js"); + load("jstests/replsets/rslib.js"); + + var name = "rollback_after_sync_source_selection"; + var collName = "test.coll"; + + var rst = new ReplSetTest({ + name: name, + nodes: [ + {}, + {}, + {rsConfig: {priority: 0}}, + {rsConfig: {arbiterOnly: true}}, + {rsConfig: {arbiterOnly: true}} + ], + useBridge: true + }); + var nodes = rst.startSet(); + rst.initiate(); + + jsTestLog("Make sure node 0 is primary."); + rst.stepUp(nodes[0]); + assert.eq(nodes[0], rst.getPrimary()); + // Wait for all data bearing nodes to get up to date. + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: 0}, {writeConcern: {w: 3, wtimeout: rst.kDefaultTimeoutMS}})); + + jsTestLog("Create two partitions: [1] and [0,2,3,4]."); + nodes[1].disconnect(nodes[0]); + nodes[1].disconnect(nodes[2]); + nodes[1].disconnect(nodes[3]); + nodes[1].disconnect(nodes[4]); + + jsTestLog("Do a write that replicates to [0,2,3,4]."); + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: 1}, {writeConcern: {w: 2, wtimeout: rst.kDefaultTimeoutMS}})); + + jsTestLog("Pausing node 2's oplog fetcher before first fetch."); + assert.commandWorked(nodes[2].getDB('admin').runCommand( + {configureFailPoint: 'fetcherHangBeforeStart', mode: 'alwaysOn'})); + syncFrom(nodes[2], nodes[0], rst); + checkLog.contains(nodes[2], 'fetcherHangBeforeStart fail point enabled'); + + jsTestLog("Do a write on partition [0,2,3,4]; it won't replicate due to the failpoint."); + assert.writeOK(nodes[0].getCollection(collName).insert({a: 2})); + + jsTestLog("Repartition to: [0,2] and [1,3,4]."); + nodes[1].reconnect(nodes[3]); + nodes[1].reconnect(nodes[4]); + nodes[3].disconnect(nodes[0]); + nodes[3].disconnect(nodes[2]); + nodes[4].disconnect(nodes[0]); + nodes[4].disconnect(nodes[2]); + + jsTestLog("Ensure that 0 steps down and that 1 becomes primary."); + waitForState(nodes[0], ReplSetTest.State.SECONDARY); + waitForState(nodes[1], ReplSetTest.State.PRIMARY); + assert.eq(nodes[1], rst.getPrimary()); + + jsTestLog("Do a write to node 1 on the [1,3,4] side of the partition."); + assert.writeOK(nodes[1].getCollection(collName).insert({a: 3})); + + jsTestLog("Remove the partition but maintain that node 2 only talks to node 0."); + var node0RBID = nodes[0].adminCommand('replSetGetRBID').rbid; + nodes[0].reconnect(nodes[1]); + nodes[0].reconnect(nodes[3]); + nodes[0].reconnect(nodes[4]); + + jsTestLog("Wait for node 0 to go into ROLLBACK"); + // Wait for a rollback to happen. + assert.soonNoExcept(function() { + var node0RBIDNew = nodes[0].adminCommand('replSetGetRBID').rbid; + return node0RBIDNew !== node0RBID; + }); + waitForState(nodes[0], ReplSetTest.State.SECONDARY); + + // At this point nodes 0 and 1 should have the same data. + assert.neq(null, + nodes[0].getCollection(collName).findOne({a: 0}), + "Node " + nodes[0].host + + " did not contain initial op that should be present on all nodes"); + assert.eq(null, + nodes[0].getCollection(collName).findOne({a: 1}), + "Node " + nodes[0].host + " contained op that should have been rolled back"); + assert.eq(null, + nodes[0].getCollection(collName).findOne({a: 2}), + "Node " + nodes[0].host + " contained op that should have been rolled back"); + assert.neq(null, + nodes[0].getCollection(collName).findOne({a: 3}), + "Node " + nodes[0].host + " did not contain op from after rollback"); + + jsTestLog("Let oplog fetcher continue and error that the sync source rolled back."); + // Turn off failpoint on node 2 to allow it to continue fetching. + assert.commandWorked(nodes[2].getDB('admin').runCommand( + {configureFailPoint: 'fetcherHangBeforeStart', mode: 'off'})); + checkLog.contains(nodes[2], + "Upstream node rolled back after verifying that it had our MinValid point."); + + rst.awaitSecondaryNodes(); + rst.stopSet(); +}()); diff --git a/jstests/replsets/rollback_empty_ns.js b/jstests/replsets/rollback_empty_ns.js deleted file mode 100644 index f6a07319eb4..00000000000 --- a/jstests/replsets/rollback_empty_ns.js +++ /dev/null @@ -1,93 +0,0 @@ -// test that a rollback of an op with empty ns causes a message to be logged -// -// If all data-bearing nodes in a replica set are using an ephemeral storage engine, the set will -// not be able to survive a scenario where all data-bearing nodes are down simultaneously. In such a -// scenario, none of the members will have any data, and upon restart will each look for a member to -// inital sync from, so no primary will be elected. This test induces such a scenario, so cannot be -// run on ephemeral storage engines. -// @tags: [requires_persistence] - -// function to check the logs for an entry -doesEntryMatch = function(array, regex) { - var found = false; - for (i = 0; i < array.length; i++) { - if (regex.test(array[i])) { - found = true; - } - } - return found; -}; - -// set up a set and grab things for later -var name = "rollback_empty_ns"; -var replTest = new ReplSetTest({name: name, nodes: 3}); -var nodes = replTest.nodeList(); -var conns = replTest.startSet(); -replTest.initiate({ - "_id": name, - "members": [ - {"_id": 0, "host": nodes[0], priority: 3}, - {"_id": 1, "host": nodes[1]}, - {"_id": 2, "host": nodes[2], arbiterOnly: true} - ] -}); -var a_conn = conns[0]; -var b_conn = conns[1]; -var AID = replTest.getNodeId(a_conn); -var BID = replTest.getNodeId(b_conn); - -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); - -// get master and do an initial write -var master = replTest.getPrimary(); -assert(master === conns[0], "conns[0] assumed to be master"); -assert(a_conn.host === master.host, "a_conn assumed to be master"); -var options = { - writeConcern: {w: 2, wtimeout: 60000}, - upsert: true -}; -assert.writeOK(a_conn.getDB(name).foo.insert({x: 1}, options)); - -// shut down master -replTest.stop(AID); - -// insert a fake oplog entry with an empty ns -master = replTest.getPrimary(); -assert(b_conn.host === master.host, "b_conn assumed to be master"); -options = { - writeConcern: {w: 1, wtimeout: 60000}, - upsert: true -}; -// another insert to set minvalid ahead -assert.writeOK(b_conn.getDB(name).foo.insert({x: 123})); -var oplog_entry = b_conn.getDB("local").oplog.rs.find().sort({$natural: -1})[0]; -oplog_entry["ts"] = Timestamp(oplog_entry["ts"].t, oplog_entry["ts"].i + 1); -oplog_entry["ns"] = ""; -assert.writeOK(b_conn.getDB("local").oplog.rs.insert(oplog_entry)); - -// shut down B and bring back the original master -replTest.stop(BID); -replTest.restart(AID); -master = replTest.getPrimary(); -assert(a_conn.host === master.host, "a_conn assumed to be master"); - -// do a write so that B will have to roll back -options = { - writeConcern: {w: 1, wtimeout: 60000}, - upsert: true -}; -assert.writeOK(a_conn.getDB(name).foo.insert({x: 2}, options)); - -// restart B, which should rollback and log a message about not rolling back empty ns'd oplog entry -replTest.restart(BID); -var msg = RegExp("ignoring op on rollback no ns TODO : "); -assert.soon(function() { - try { - var log = b_conn.getDB("admin").adminCommand({getLog: "global"}).log; - return doesEntryMatch(log, msg); - } catch (e) { - return false; - } -}, "Did not see a log entry about skipping the empty ns'd oplog entry during rollback"); - -replTest.stopSet(); diff --git a/jstests/replsets/rollback_empty_o.js b/jstests/replsets/rollback_empty_o.js deleted file mode 100644 index f3468fcde5e..00000000000 --- a/jstests/replsets/rollback_empty_o.js +++ /dev/null @@ -1,93 +0,0 @@ -// test that a rollback of an op with empty o causes a message to be logged -// -// If all data-bearing nodes in a replica set are using an ephemeral storage engine, the set will -// not be able to survive a scenario where all data-bearing nodes are down simultaneously. In such a -// scenario, none of the members will have any data, and upon restart will each look for a member to -// inital sync from, so no primary will be elected. This test induces such a scenario, so cannot be -// run on ephemeral storage engines. -// @tags: [requires_persistence] - -// function to check the logs for an entry -doesEntryMatch = function(array, regex) { - var found = false; - for (i = 0; i < array.length; i++) { - if (regex.test(array[i])) { - found = true; - } - } - return found; -}; - -// set up a set and grab things for later -var name = "rollback_empty_o"; -var replTest = new ReplSetTest({name: name, nodes: 3}); -var nodes = replTest.nodeList(); -var conns = replTest.startSet(); -replTest.initiate({ - "_id": name, - "members": [ - {"_id": 0, "host": nodes[0], priority: 3}, - {"_id": 1, "host": nodes[1]}, - {"_id": 2, "host": nodes[2], arbiterOnly: true} - ] -}); -var a_conn = conns[0]; -var b_conn = conns[1]; -var AID = replTest.getNodeId(a_conn); -var BID = replTest.getNodeId(b_conn); - -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); - -// get master and do an initial write -var master = replTest.getPrimary(); -assert(master === conns[0], "conns[0] assumed to be master"); -assert(a_conn.host === master.host, "a_conn assumed to be master"); -var options = { - writeConcern: {w: 2, wtimeout: 60000}, - upsert: true -}; -assert.writeOK(a_conn.getDB(name).foo.insert({x: 1}, options)); - -// shut down master -replTest.stop(AID); - -// insert a fake oplog entry with an empty o -master = replTest.getPrimary(); -assert(b_conn.host === master.host, "b_conn assumed to be master"); -options = { - writeConcern: {w: 1, wtimeout: 60000}, - upsert: true -}; -// another insert to set minvalid ahead -assert.writeOK(b_conn.getDB(name).foo.insert({x: 123})); -var oplog_entry = b_conn.getDB("local").oplog.rs.find().sort({$natural: -1})[0]; -oplog_entry["ts"] = Timestamp(oplog_entry["ts"].t, oplog_entry["ts"].i + 1); -oplog_entry["o"] = {}; -assert.writeOK(b_conn.getDB("local").oplog.rs.insert(oplog_entry)); - -// shut down B and bring back the original master -replTest.stop(BID); -replTest.restart(AID); -master = replTest.getPrimary(); -assert(a_conn.host === master.host, "a_conn assumed to be master"); - -// do a write so that B will have to roll back -options = { - writeConcern: {w: 1, wtimeout: 60000}, - upsert: true -}; -assert.writeOK(a_conn.getDB(name).foo.insert({x: 2}, options)); - -// restart B, which should rollback and log a message about not rolling back empty o'd oplog entry -replTest.restart(BID); -var msg = RegExp("ignoring op on rollback : "); -assert.soon(function() { - try { - var log = b_conn.getDB("admin").adminCommand({getLog: "global"}).log; - return doesEntryMatch(log, msg); - } catch (e) { - return false; - } -}, "Did not see a log entry about skipping the empty o'd oplog entry during rollback"); - -replTest.stopSet(); diff --git a/jstests/replsets/rollback_empty_o2.js b/jstests/replsets/rollback_empty_o2.js deleted file mode 100644 index 56eb8512575..00000000000 --- a/jstests/replsets/rollback_empty_o2.js +++ /dev/null @@ -1,94 +0,0 @@ -// test that a rollback of an update with empty o2 causes a message to be logged -// -// If all data-bearing nodes in a replica set are using an ephemeral storage engine, the set will -// not be able to survive a scenario where all data-bearing nodes are down simultaneously. In such a -// scenario, none of the members will have any data, and upon restart will each look for a member to -// inital sync from, so no primary will be elected. This test induces such a scenario, so cannot be -// run on ephemeral storage engines. -// @tags: [requires_persistence] - -// function to check the logs for an entry -doesEntryMatch = function(array, regex) { - var found = false; - for (i = 0; i < array.length; i++) { - if (regex.test(array[i])) { - found = true; - } - } - return found; -}; - -// set up a set and grab things for later -var name = "rollback_empty_o2"; -var replTest = new ReplSetTest({name: name, nodes: 3}); -var nodes = replTest.nodeList(); -var conns = replTest.startSet(); -replTest.initiate({ - "_id": name, - "members": [ - {"_id": 0, "host": nodes[0], priority: 3}, - {"_id": 1, "host": nodes[1]}, - {"_id": 2, "host": nodes[2], arbiterOnly: true} - ] -}); -var a_conn = conns[0]; -var b_conn = conns[1]; -var AID = replTest.getNodeId(a_conn); -var BID = replTest.getNodeId(b_conn); - -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); - -// get master and do an initial write -var master = replTest.getPrimary(); -assert(master === conns[0], "conns[0] assumed to be master"); -assert(a_conn.host === master.host, "a_conn assumed to be master"); -var options = { - writeConcern: {w: 2, wtimeout: 60000}, - upsert: true -}; -assert.writeOK(a_conn.getDB(name).foo.insert({x: 1}, options)); - -// shut down master -replTest.stop(AID); - -// insert a fake oplog entry with an empty o2 -master = replTest.getPrimary(); -assert(b_conn.host === master.host, "b_conn assumed to be master"); -options = { - writeConcern: {w: 1, wtimeout: 60000}, - upsert: true -}; -// another insert to set minvalid ahead -assert.writeOK(b_conn.getDB(name).foo.insert({x: 123})); -var oplog_entry = b_conn.getDB("local").oplog.rs.find().sort({$natural: -1})[0]; -oplog_entry["ts"] = Timestamp(oplog_entry["ts"].t, oplog_entry["ts"].i + 1); -oplog_entry["op"] = "u"; -oplog_entry["o2"] = {}; -assert.writeOK(b_conn.getDB("local").oplog.rs.insert(oplog_entry)); - -// shut down B and bring back the original master -replTest.stop(BID); -replTest.restart(AID); -master = replTest.getPrimary(); -assert(a_conn.host === master.host, "a_conn assumed to be master"); - -// do a write so that B will have to roll back -options = { - writeConcern: {w: 1, wtimeout: 60000}, - upsert: true -}; -assert.writeOK(a_conn.getDB(name).foo.insert({x: 2}, options)); - -// restart B, which should rollback and log a message about not rolling back empty o2'd oplog entry -replTest.restart(BID); -var msg = RegExp("ignoring op on rollback : "); -assert.soon(function() { - try { - var log = b_conn.getDB("admin").adminCommand({getLog: "global"}).log; - return doesEntryMatch(log, msg); - } catch (e) { - return false; - } -}, "Did not see a log entry about skipping the empty o2'd oplog entry during rollback"); - -replTest.stopSet(); diff --git a/jstests/replsets/rollback_too_new.js b/jstests/replsets/rollback_too_new.js index 5619df42e18..c7fe4fa2320 100644 --- a/jstests/replsets/rollback_too_new.js +++ b/jstests/replsets/rollback_too_new.js @@ -9,6 +9,8 @@ (function() { "use strict"; + load("jstests/replsets/rslib.js"); + // set up a set and grab things for later var name = "rollback_too_new"; var replTest = new ReplSetTest({name: name, nodes: 3}); @@ -43,9 +45,22 @@ replTest.stop(CID); - // do one write to master - // in order to trigger a rollback on C - assert.writeOK(master.getDB(name).foo.insert({x: 2}, options)); + // We bump the term to make sure node 0's oplog is ahead of node 2's. + var term = getLatestOp(conns[0]).t; + try { + assert.commandWorked(conns[0].adminCommand({replSetStepDown: 1, force: true})); + } catch (e) { + if (e.message.indexOf("error doing query: failed") < 0) { + throw e; + } + } + + // After stepping down due to the higher term, it will eventually get reelected. + replTest.waitForState(conns[0], ReplSetTest.State.PRIMARY); + // Wait for the node to increase its term. + assert.soon(function() { + return getLatestOp(conns[0]).t > term; + }); // Node C should connect to new master as a sync source because chaining is disallowed. // C is ahead of master but it will still connect to it. diff --git a/jstests/replsets/rollback_with_socket_error_then_steady_state.js b/jstests/replsets/rollback_with_socket_error_then_steady_state.js new file mode 100644 index 00000000000..25a5c863970 --- /dev/null +++ b/jstests/replsets/rollback_with_socket_error_then_steady_state.js @@ -0,0 +1,119 @@ +// This test causes node 2 to enter rollback and then fail with a SocketException before updating +// MinValid or altering durable state in any way. It will then choose a sync source from which it +// is able to stitch the oplog and therefore doesn't need to roll back. Prior to SERVER-27282, the +// node would be "stuck" with state=ROLLBACK while it was doing steady-state replication, with no +// way to reach SECONDARY without restarting the process. +(function() { + 'use strict'; + + load("jstests/libs/check_log.js"); + load("jstests/replsets/rslib.js"); + + var collName = "test.coll"; + var counter = 0; + + var rst = new ReplSetTest({ + name: 'rollback_with_socket_error_then_steady_state', + nodes: [ + // Primary flops between nodes 0 and 1. + {}, + {}, + // Node 2 is the node under test. + {rsConfig: {priority: 0}}, + // Arbiters to sway elections. + {rsConfig: {arbiterOnly: true}}, + {rsConfig: {arbiterOnly: true}} + ], + useBridge: true + }); + var nodes = rst.startSet(); + rst.initiate(); + + jsTestLog("Make sure node 0 is primary."); + rst.stepUp(nodes[0]); + assert.eq(nodes[0], rst.getPrimary()); + // Wait for all data bearing nodes to get up to date. + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: counter++}, {writeConcern: {w: 3, wtimeout: 5 * 60 * 1000}})); + + jsTestLog("Create two partitions: [1] and [0,2,3,4]."); + nodes[1].disconnect(nodes[0]); + nodes[1].disconnect(nodes[2]); + nodes[1].disconnect(nodes[3]); + nodes[1].disconnect(nodes[4]); + + jsTestLog("Do a write that is replicated to [0,2,3,4]."); + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: counter++}, {writeConcern: {w: 2, wtimeout: 5 * 60 * 1000}})); + + jsTestLog("Repartition to: [0,2] and [1,3,4]."); + nodes[1].reconnect(nodes[3]); + nodes[1].reconnect(nodes[4]); + nodes[3].disconnect(nodes[0]); + nodes[3].disconnect(nodes[2]); + nodes[4].disconnect(nodes[0]); + nodes[4].disconnect(nodes[2]); + + jsTestLog("Ensure that 0 steps down and that 1 becomes primary."); + waitForState(nodes[0], ReplSetTest.State.SECONDARY); + waitForState(nodes[1], ReplSetTest.State.PRIMARY); + assert.eq(nodes[1], rst.getPrimary()); + + jsTestLog("Do a write to node 1 on the [1,3,4] side of the partition."); + assert.writeOK(nodes[1].getCollection(collName).insert({a: counter++})); + + // Turn on failpoint on node 2 to pause rollback before doing anything. + assert.commandWorked( + nodes[2].adminCommand({configureFailPoint: 'rollbackHangBeforeStart', mode: 'alwaysOn'})); + + jsTestLog("Repartition to: [0] and [1,2,3,4]."); + nodes[2].disconnect(nodes[0]); + nodes[2].reconnect(nodes[1]); + nodes[2].reconnect(nodes[3]); + nodes[2].reconnect(nodes[4]); + + jsTestLog("Wait for node 2 to decide to go into ROLLBACK and start syncing from node 1."); + // Since nodes 1 and 2 have now diverged, node 2 should go into rollback. The failpoint will + // stop it from actually transitioning to rollback, so the checkLog bellow will ensure that we + // have decided to rollback, but haven't actually started yet. + rst.awaitSyncSource(nodes[2], nodes[1]); + + jsTestLog("Wait for failpoint on node 2 to pause rollback before it starts"); + // Wait for fail point message to be logged. + checkLog.contains(nodes[2], 'rollback - rollbackHangBeforeStart fail point enabled'); + + jsTestLog("Repartition to: [1] and [0,2,3,4]."); + nodes[1].disconnect(nodes[3]); + nodes[1].disconnect(nodes[4]); + nodes[2].disconnect(nodes[1]); + nodes[2].reconnect(nodes[0]); + nodes[3].reconnect(nodes[0]); + nodes[3].reconnect(nodes[2]); + nodes[4].reconnect(nodes[0]); + nodes[4].reconnect(nodes[2]); + + // Turn off failpoint on node 2 to allow rollback against node 1 to fail with a network error. + assert.commandWorked( + nodes[2].adminCommand({configureFailPoint: 'rollbackHangBeforeStart', mode: 'off'})); + + // Make node 0 ahead of node 2 again so node 2 will pick it as a sync source. + + jsTestLog("waiting for node 0 to be primary"); + waitForState(nodes[1], ReplSetTest.State.SECONDARY); + waitForState(nodes[0], ReplSetTest.State.PRIMARY); + assert.eq(nodes[0], rst.getPrimary()); + + jsTestLog("w:2 write to node 0 (replicated to node 2)"); + assert.writeOK(nodes[0].getCollection(collName).insert( + {a: counter++}, {writeConcern: {w: 2, wtimeout: 5 * 60 * 1000}})); + + // At this point node 2 has failed rollback before making any durable changes, including writing + // to minValid. That means that it is free to pick any sync source and will pick node 0 where it + // can pick up where it left off without rolling back. Ensure that it is able to reach SECONDARY + // and doesn't do steady-state replication in ROLLBACK state. + jsTestLog("Wait for node 2 to go into SECONDARY"); + assert.neq(nodes[2].adminCommand('replSetGetStatus').myState, + ReplSetTest.State.ROLLBACK, + "node 2 is doing steady-state replication with state=ROLLBACK!"); + waitForState(nodes[2], ReplSetTest.State.SECONDARY); +}()); diff --git a/jstests/replsets/rslib.js b/jstests/replsets/rslib.js index 48e934a88d8..b97f79c00d1 100644 --- a/jstests/replsets/rslib.js +++ b/jstests/replsets/rslib.js @@ -1,11 +1,50 @@ +var syncFrom; var wait, occasionally, reconnect, getLatestOp, waitForAllMembers, reconfig, awaitOpTime; var waitUntilAllNodesCaughtUp; +var waitForState; +var awaitRSClientHosts; +var getLastOpTime; +var startSetIfSupportsReadMajority; (function() { "use strict"; + load("jstests/libs/write_concern_util.js"); + var count = 0; var w = 0; + /** + * A wrapper around `replSetSyncFrom` to ensure that the desired sync source is ahead of the + * syncing node so that the syncing node can choose to sync from the desired sync source. + * It first stops replication on the syncing node so that it can do a write on the desired + * sync source and make sure it's ahead. When replication is restarted, the desired sync + * source will be a valid sync source for the syncing node. + */ + syncFrom = function(syncingNode, desiredSyncSource, rst) { + jsTestLog("Forcing " + syncingNode.name + " to sync from " + desiredSyncSource.name); + + // Ensure that 'desiredSyncSource' doesn't already have the dummy write sitting around from + // a previous syncFrom attempt. + var dummyName = "dummyForSyncFrom"; + rst.getPrimary().getDB(dummyName).getCollection(dummyName).drop(); + assert.soonNoExcept(function() { + return desiredSyncSource.getDB(dummyName).getCollection(dummyName).findOne() == null; + }); + + stopServerReplication(syncingNode); + + assert.writeOK(rst.getPrimary().getDB(dummyName).getCollection(dummyName).insert({a: 1})); + // Wait for 'desiredSyncSource' to get the dummy write we just did so we know it's + // definitely ahead of 'syncingNode' before we call replSetSyncFrom. + assert.soonNoExcept(function() { + return desiredSyncSource.getDB(dummyName).getCollection(dummyName).findOne({a: 1}); + }); + + assert.commandWorked(syncingNode.adminCommand({replSetSyncFrom: desiredSyncSource.name})); + restartServerReplication(syncingNode); + rst.awaitSyncSource(syncingNode, desiredSyncSource); + }; + wait = function(f, msg) { w++; var n = 0; @@ -192,4 +231,141 @@ var waitUntilAllNodesCaughtUp; timeout); }; + /** + * Waits for the given node to reach the given state, ignoring network errors. + */ + waitForState = function(node, state) { + assert.soonNoExcept(function() { + assert.commandWorked(node.adminCommand( + {replSetTest: 1, waitForMemberState: state, timeoutMillis: 60 * 1000 * 5})); + return true; + }); + }; + + /** + * Starts each node in the given replica set if the storage engine supports readConcern + *'majority'. + * Returns true if the replica set was started successfully and false otherwise. + * + * @param replSetTest - The instance of {@link ReplSetTest} to start + * @param options - The options passed to {@link ReplSetTest.startSet} + */ + startSetIfSupportsReadMajority = function(replSetTest, options) { + try { + replSetTest.startSet(options); + } catch (e) { + var conn = MongoRunner.runMongod(); + if (!conn.getDB("admin").serverStatus().storageEngine.supportsCommittedReads) { + MongoRunner.stopMongod(conn); + return false; + } + throw e; + } + return true; + }; + + /** + * Waits for the specified hosts to enter a certain state. + */ + awaitRSClientHosts = function(conn, host, hostOk, rs, timeout) { + var hostCount = host.length; + if (hostCount) { + for (var i = 0; i < hostCount; i++) { + awaitRSClientHosts(conn, host[i], hostOk, rs); + } + + return; + } + + timeout = timeout || 5 * 60 * 1000; + + if (hostOk == undefined) + hostOk = { + ok: true + }; + if (host.host) + host = host.host; + if (rs) + rs = rs.name; + + print("Awaiting " + host + " to be " + tojson(hostOk) + " for " + conn + " (rs: " + rs + + ")"); + + var tests = 0; + + assert.soon(function() { + var rsClientHosts = conn.adminCommand('connPoolStats').replicaSets; + if (tests++ % 10 == 0) { + printjson(rsClientHosts); + } + + for (var rsName in rsClientHosts) { + if (rs && rs != rsName) + continue; + + for (var i = 0; i < rsClientHosts[rsName].hosts.length; i++) { + var clientHost = rsClientHosts[rsName].hosts[i]; + if (clientHost.addr != host) + continue; + + // Check that *all* host properties are set correctly + var propOk = true; + for (var prop in hostOk) { + // Use special comparator for tags because isMaster can return the fields in + // different order. The fields of the tags should be treated like a set of + // strings and 2 tags should be considered the same if the set is equal. + if (prop == 'tags') { + if (!clientHost.tags) { + propOk = false; + break; + } + + for (var hostTag in hostOk.tags) { + if (clientHost.tags[hostTag] != hostOk.tags[hostTag]) { + propOk = false; + break; + } + } + + for (var clientTag in clientHost.tags) { + if (clientHost.tags[clientTag] != hostOk.tags[clientTag]) { + propOk = false; + break; + } + } + + continue; + } + + if (isObject(hostOk[prop])) { + if (!friendlyEqual(hostOk[prop], clientHost[prop])) { + propOk = false; + break; + } + } else if (clientHost[prop] != hostOk[prop]) { + propOk = false; + break; + } + } + + if (propOk) { + return true; + } + } + } + + return false; + }, 'timed out waiting for replica set client to recognize hosts', timeout); + }; + + /** + * Returns the last opTime of the connection based from replSetGetStatus. Can only + * be used on replica set nodes. + */ + getLastOpTime = function(conn) { + var replSetStatus = + assert.commandWorked(conn.getDB("admin").runCommand({replSetGetStatus: 1})); + var connStatus = replSetStatus.members.filter(m => m.self)[0]; + return connStatus.optime; + }; }()); diff --git a/jstests/replsets/server8070.js b/jstests/replsets/server8070.js index e91e95e99a4..5d8435a656e 100644 --- a/jstests/replsets/server8070.js +++ b/jstests/replsets/server8070.js @@ -1,3 +1,5 @@ +load("jstests/replsets/rslib.js"); + // Test for SERVER-8070: Flush buffer before changing sync targets to prevent unnecessary rollbacks // This test writes 50 ops to one secondary's data (member2) and 25 ops to the other secondary's // data (member3), then puts 50 more ops in member3's buffer and makes sure that member3 doesn't try @@ -40,8 +42,9 @@ master.getDB("foo").bar.insert({x: 1}); replSet.awaitReplication(); jsTest.log("Make sure 2 & 3 are syncing from the primary"); -member2.adminCommand({replSetSyncFrom: getHostName() + ":" + replSet.ports[0]}); -member3.adminCommand({replSetSyncFrom: getHostName() + ":" + replSet.ports[0]}); +assert.eq(master, replSet.nodes[0]); +syncFrom(replSet.nodes[1], master, replSet); +syncFrom(replSet.nodes[2], master, replSet); jsTest.log("Stop 2's replication"); member2.runCommand({configureFailPoint: 'rsSyncApplyStop', mode: 'alwaysOn'}); @@ -100,8 +103,8 @@ assert(syncingTo !== getHostName() + ":" + replSet.ports[1], "node 3 is syncing jsTest.log("Pause 3's bgsync thread"); var rsBgSyncProduceResult3 = - member3.runCommand({configureFailPoint: 'rsBgSyncProduce', mode: 'alwaysOn'}); -assert.eq(1, rsBgSyncProduceResult3.ok, "member 3 rsBgSyncProduce admin command failed"); + member3.runCommand({configureFailPoint: 'stopReplProducer', mode: 'alwaysOn'}); +assert.eq(1, rsBgSyncProduceResult3.ok, "member 3 stopReplProducer admin command failed"); // count documents in member 3 assert.eq(26, @@ -123,7 +126,7 @@ assert.soon(function() { }, "Replication member 3 did not apply ops 25-75"); jsTest.log("Start 3's bgsync thread"); -member3.runCommand({configureFailPoint: 'rsBgSyncProduce', mode: 'off'}); +member3.runCommand({configureFailPoint: 'stopReplProducer', mode: 'off'}); jsTest.log("Node 3 shouldn't hit rollback"); var end = (new Date()).getTime() + 10000; diff --git a/jstests/replsets/shutdown_primary.js b/jstests/replsets/shutdown_primary.js new file mode 100644 index 00000000000..66d429dec89 --- /dev/null +++ b/jstests/replsets/shutdown_primary.js @@ -0,0 +1,68 @@ +/** + * Test that the shutdown command called on a primary node waits for a majority of secondaries to + * catch up before taking effect, and will fail otherwise. + * + * 1. Initiate a 3-node replica set + * 2. Block replication to secondaries + * 3. Write to primary + * 4. Try to shut down primary and expect failure + * 5. Try to shut down primary in a parallel shell and expect success + * 6. Resume replication on secondaries + * 7. Try to create a new connection to the shut down primary and expect an error + * + */ +(function() { + load("jstests/libs/write_concern_util.js"); // for stopReplicationOnSecondaries, + // restartReplicationOnSecondaries + + "use strict"; + + var name = "shutdown_primary"; + + var replTest = new ReplSetTest({name: name, nodes: 3}); + replTest.startSet(); + replTest.initiate(); + + var primary = replTest.getPrimary(); + var testDB = primary.getDB(name); + var timeout = 5 * 60 * 1000; + assert.writeOK(testDB.foo.insert({x: 1}, {writeConcern: {w: 3, wtimeout: timeout}})); + + jsTestLog("Blocking replication to secondaries."); + stopReplicationOnSecondaries(replTest); + + jsTestLog("Executing write to primary."); + assert.writeOK(testDB.foo.insert({x: 2})); + + jsTestLog("Attempting to shut down primary."); + assert.commandFailedWithCode(primary.adminCommand({shutdown: 1}), + ErrorCodes.ExceededTimeLimit, + "shut down did not fail with 'ExceededTimeLimit'"); + + jsTestLog("Verifying primary did not shut down."); + assert.writeOK(testDB.foo.insert({x: 3})); + + jsTestLog("Shutting down primary in a parallel shell"); + var awaitShell = startParallelShell(function() { + db.adminCommand({shutdown: 1, timeoutSecs: 60}); + }, primary.port); + + jsTestLog("Resuming replication."); + restartReplicationOnSecondaries(replTest); + + jsTestLog("Verifying primary shut down and cannot be connected to."); + // Successfully starting shutdown throws a network error. + var exitCode = awaitShell({checkExitSuccess: false}); + assert.neq(0, exitCode, "expected shutdown to close the shell's connection"); + assert.soonNoExcept(function() { + // The parallel shell exits while shutdown is in progress, and if this happens early enough, + // the primary can still accept connections despite successfully starting to shutdown. + // So, retry connecting until connections cannot be established and an error is thrown. + assert.throws(function() { + new Mongo(primary.host); + }); + return true; + }, "expected primary node to shut down and not be connectable"); + + replTest.stopSet(); +})(); diff --git a/jstests/replsets/slavedelay3.js b/jstests/replsets/slavedelay3.js index 2ce6e9b2a80..dc3bb0786d9 100644 --- a/jstests/replsets/slavedelay3.js +++ b/jstests/replsets/slavedelay3.js @@ -29,12 +29,7 @@ nodes[0].disconnect(nodes[2]); master.foo.insert({x: 1}); -assert.commandWorked(nodes[1].getDB("admin").runCommand({"replSetSyncFrom": nodes[0].host})); -var res; -assert.soon(function() { - res = nodes[1].getDB("admin").runCommand({"replSetGetStatus": 1}); - return res.syncingTo === nodes[0].host; -}, "node 4 failed to start chaining: " + tojson(res)); +syncFrom(nodes[1], nodes[0], replTest); // make sure the record still appears in the remote slave assert.soon(function() { diff --git a/jstests/replsets/stepdown.js b/jstests/replsets/stepdown.js index 4b736b3b93a..5a8388da4d4 100644 --- a/jstests/replsets/stepdown.js +++ b/jstests/replsets/stepdown.js @@ -28,40 +28,66 @@ print("\ndo a write"); assert.writeOK(master.getDB("foo").bar.insert({x: 1})); replTest.awaitReplication(); -// lock secondaries -print("\nlock secondaries"); -replTest.liveNodes.slaves.forEach(function(slave) { - printjson(assert.commandWorked(slave.getDB("admin").runCommand({fsync: 1, lock: 1}))); -}); - -print("\nwaiting several seconds before stepdown"); - -sleep(2000); - -for (var i = 0; i < 11; i++) { - // do another write - assert.writeOK(master.getDB("foo").bar.insert({x: i})); - sleep(1000); +// In the event of any error, we have to unlock any nodes that we have fsyncLocked. +function unlockNodes(nodes) { + jsTestLog('Unlocking nodes: ' + tojson(nodes)); + nodes.forEach(function(node) { + try { + jsTestLog('Unlocking node: ' + node); + assert.commandWorked(node.getDB("admin").fsyncUnlock()); + } catch (e) { + jsTestLog('Failed to unlock node: ' + node + ': ' + tojson(e) + + '. Ignoring unlock error and moving on to next node.'); + } + }); } -print("\n do stepdown that should not work"); - -// this should fail, so we don't need to try/catch -printjson(assert.commandFailed(master.getDB("admin").runCommand({replSetStepDown: 10}))); - -print("\n do stepdown that should work"); -assert.throws(function() { - assert.commandFailed(master.getDB("admin").runCommand({replSetStepDown: 50, force: true})); -}); - -var r2 = assert.commandWorked(master.getDB("admin").runCommand({ismaster: 1})); -assert.eq(r2.ismaster, false); -assert.eq(r2.secondary, true); +var lockedNodes = []; +try { + // lock secondaries + jsTestLog('Locking nodes: ' + tojson(replTest.liveNodes.slaves)); + replTest.liveNodes.slaves.forEach(function(node) { + jsTestLog('Locking node: ' + node); + jsTestLog( + 'fsync lock ' + node + ' result: ' + + tojson(assert.commandWorked(node.getDB("admin").runCommand({fsync: 1, lock: 1})))); + lockedNodes.push(node); + }); + + jsTestLog('Stepping down primary: ' + master); + + for (var i = 0; i < 11; i++) { + // do another write + assert.writeOK(master.getDB("foo").bar.insert({x: i})); + } -print("\nunlock"); -replTest.liveNodes.slaves.forEach(function(slave) { - printjson(assert.commandWorked(slave.getDB("admin").fsyncUnlock())); -}); + jsTestLog('Do stepdown of primary ' + master + ' that should not work'); + + // this should fail, so we don't need to try/catch + jsTestLog( + 'Step down ' + master + ' expected error: ' + + tojson(assert.commandFailed(master.getDB("admin").runCommand({replSetStepDown: 10})))); + + // The server will disconnect the client on a successful forced stepdown so we use the + // presence of an exception to confirm the forced stepdown result. + jsTestLog('Do stepdown of primary ' + master + ' that should work'); + var exceptionFromForcedStepDown = assert.throws(function() { + master.getDB("admin") + .runCommand({replSetStepDown: ReplSetTest.kDefaultTimeoutMS, force: true}); + }); + jsTestLog('Forced stepdown ' + master + ' expected failure: ' + + tojson(exceptionFromForcedStepDown)); + + jsTestLog('Checking isMaster on ' + master); + var r2 = assert.commandWorked(master.getDB("admin").runCommand({ismaster: 1})); + jsTestLog('Result from running isMaster on ' + master + ': ' + tojson(r2)); + assert.eq(r2.ismaster, false); + assert.eq(r2.secondary, true); +} catch (e) { + throw e; +} finally { + unlockNodes(lockedNodes); +} print("\nreset stepped down time"); assert.commandWorked(master.getDB("admin").runCommand({replSetFreeze: 0})); diff --git a/jstests/replsets/stepdown_needs_electable_secondary.js b/jstests/replsets/stepdown_needs_electable_secondary.js new file mode 100644 index 00000000000..9a59ff0fd58 --- /dev/null +++ b/jstests/replsets/stepdown_needs_electable_secondary.js @@ -0,0 +1,143 @@ +/** + * Test to ensure that replSetStepDown called on a primary will only succeed if a majority of nodes + * are caught up to it and that at least one node in this majority is electable. Tests this with a + * 5 node replica set. + * + * 1. Initiate a 5-node replica set + * 2. Disable replication to all secondaries + * 3. Execute some writes on primary + * 4. Try to step down primary and expect to fail + * 5. Enable replication to one unelectable secondary, secondary B + * 6. Await replication to secondary B by executing primary write with writeConcern:2 + * 7. Try to step down primary and expect failure + * 8. Enable replication to a different unelectable secondary, secondary C + * 9. Await replication to secondary C by executing primary write with writeConcern:3 + * 10. Try to step down primary and expect failure + * 11. Enable replication to an electable secondary, secondary A + * 12. Await replication to secondary A by executing primary write with writeConcern:4 + * 13. Try to step down primary and expect success + * 14. Assert that original primary is now a secondary + * + */ +(function() { + load("jstests/libs/write_concern_util.js"); // for stopReplicationOnSecondaries, + // restartServerReplication, + // restartReplSetReplication + + 'use strict'; + + var name = 'stepdown_needs_electable_secondary'; + + var replTest = new ReplSetTest({name: name, nodes: 5}); + var nodes = replTest.nodeList(); + + replTest.startSet(); + replTest.initiate({ + "_id": name, + "members": [ + {"_id": 0, "host": nodes[0]}, + {"_id": 1, "host": nodes[1]}, + {"_id": 2, "host": nodes[2]}, + {"_id": 3, "host": nodes[3], "priority": 0}, // unelectable + {"_id": 4, "host": nodes[4], "priority": 0} // unelectable + ], + "settings": {"chainingAllowed": false} + }); + + function assertStepDownFailsWithExceededTimeLimit(node) { + assert.commandFailedWithCode( + node.getDB("admin").runCommand({replSetStepDown: 5, secondaryCatchUpPeriodSecs: 5}), + ErrorCodes.ExceededTimeLimit, + "step down did not fail with 'ExceededTimeLimit'"); + } + + function assertStepDownSucceeds(node) { + assert.throws(function() { + node.adminCommand({replSetStepDown: 60, secondaryCatchUpPeriodSecs: 60}); + }); + } + + var primary = replTest.getPrimary(); + + jsTestLog("Blocking writes to all secondaries."); + stopReplicationOnSecondaries(replTest); + + jsTestLog("Doing a write to primary."); + var testDB = replTest.getPrimary().getDB('testdb'); + var coll = testDB.stepdown_needs_electable_secondary; + var timeout = 5 * 60 * 1000; + assert.writeOK( + coll.insert({"dummy_key": "dummy_val"}, {writeConcern: {w: 1, wtimeout: timeout}})); + + // Try to step down with only the primary caught up (1 node out of 5). + // stepDown should fail. + jsTestLog("Trying to step down primary with only 1 node out of 5 caught up."); + assertStepDownFailsWithExceededTimeLimit(primary); + + // Get the two unelectable secondaries + var secondaryB_unelectable = replTest.nodes[3]; + var secondaryC_unelectable = replTest.nodes[4]; + + // Get an electable secondary + var secondaryA_electable = replTest.getSecondaries().find(function(s) { + var nodeId = replTest.getNodeId(s); + return (nodeId !== 3 && nodeId !== 4); // nodes 3 and 4 are set to be unelectable + }); + + // Enable writes to Secondary B (unelectable). Await replication. + // (2 out of 5 nodes caught up, 0 electable) + // stepDown should fail due to no caught up majority. + jsTestLog("Re-enabling writes to unelectable secondary: node #" + + replTest.getNodeId(secondaryB_unelectable) + ", " + secondaryB_unelectable); + restartServerReplication(secondaryB_unelectable); + + // Wait for this secondary to catch up by issuing a write that must be replicated to 2 nodes + assert.writeOK( + coll.insert({"dummy_key": "dummy_val"}, {writeConcern: {w: 2, wtimeout: timeout}})); + + // Try to step down and fail + jsTestLog("Trying to step down primary with only 2 nodes out of 5 caught up."); + assertStepDownFailsWithExceededTimeLimit(primary); + + // Enable writes to Secondary C (unelectable). Await replication. + // (3 out of 5 nodes caught up, 0 electable) + // stepDown should fail due to caught up majority without electable node. + jsTestLog("Re-enabling writes to unelectable secondary: node #" + + replTest.getNodeId(secondaryC_unelectable) + ", " + secondaryC_unelectable); + restartServerReplication(secondaryC_unelectable); + + // Wait for this secondary to catch up by issuing a write that must be replicated to 3 nodes + assert.writeOK( + coll.insert({"dummy_key": "dummy_val"}, {writeConcern: {w: 3, wtimeout: timeout}})); + + // Try to step down and fail + jsTestLog("Trying to step down primary with a caught up majority that " + + "doesn't contain an electable node."); + assertStepDownFailsWithExceededTimeLimit(primary); + + // Enable writes to Secondary A (electable). Await replication. + // (4 out of 5 nodes caught up, 1 electable) + // stepDown should succeed due to caught up majority containing an electable node. + jsTestLog("Re-enabling writes to electable secondary: node #" + + replTest.getNodeId(secondaryA_electable) + ", " + secondaryA_electable); + restartServerReplication(secondaryA_electable); + + // Wait for this secondary to catch up by issuing a write that must be replicated to 4 nodes + assert.writeOK( + coll.insert({"dummy_key": "dummy_val"}, {writeConcern: {w: 4, wtimeout: timeout}})); + + // Try to step down. We expect success, so catch the exception thrown by 'replSetStepDown'. + jsTestLog("Trying to step down primary with a caught up majority that " + + "does contain an electable node."); + + assertStepDownSucceeds(primary); + + // Make sure that original primary has transitioned to SECONDARY state + jsTestLog("Wait for PRIMARY " + primary.host + " to completely step down."); + replTest.waitForState(primary, ReplSetTest.State.SECONDARY); + + // Disable all fail points for clean shutdown + restartReplSetReplication(replTest); + replTest.stopSet(); + +}()); diff --git a/jstests/replsets/stepdown_needs_majority.js b/jstests/replsets/stepdown_needs_majority.js new file mode 100644 index 00000000000..51c6dac4def --- /dev/null +++ b/jstests/replsets/stepdown_needs_majority.js @@ -0,0 +1,110 @@ +/** + * Test to ensure that 'replSetStepDown' called on a primary will fail if an electable node is + * caught up to it but less than a majority of nodes are caught up to it. Additionally tests that + * step down will then succeed once a majority has caught up. Tests this with a 5 node replica set. + * + * 1. Initiate a 5-node replica set + * 2. Disable replication to all secondaries + * 3. Execute a write on primary with writeConcern:1 + * 4. Try to step down primary and expect to fail + * 5. Enable replication to one secondary (Secondary A) + * 6. Await replication to Secondary A by executing primary write with writeConcern:2 + * 7. Try to step down primary and expect failure + * 8. Enable replication to a different secondary (Secondary B) + * 9. Await replication to Secondary B by executing primary write with writeConcern:3 + * 10. Try to step down primary and expect success + * + */ +(function() { + load("jstests/libs/write_concern_util.js"); // for stopReplicationOnSecondaries, // + // restartServerReplication, + // restartReplSetReplication + + 'use strict'; + + function assertStepDownFailsWithExceededTimeLimit(node) { + assert.commandFailedWithCode( + node.getDB("admin").runCommand({replSetStepDown: 5, secondaryCatchUpPeriodSecs: 5}), + ErrorCodes.ExceededTimeLimit, + "step down did not fail with 'ExceededTimeLimit'"); + } + + function assertStepDownSucceeds(node) { + assert.throws(function() { + node.adminCommand({replSetStepDown: 60, secondaryCatchUpPeriodSecs: 60}); + }); + } + + function nodeIdStr(repltest, node) { + return "node #" + repltest.getNodeId(node) + ", " + node.host; + } + + // + // Test setup + // + var name = 'stepdown_needs_majority'; + var replTest = new ReplSetTest({name: name, nodes: 5, settings: {chainingAllowed: false}}); + + replTest.startSet(); + replTest.initiate(); + + var primary = replTest.getPrimary(); + var testDB = primary.getDB('testdb'); + var coll = testDB[name]; + var dummy_doc = { + "dummy_key": "dummy_val" + }; + var timeout = 5 * 60 * 1000; + + // + // Block writes to all secondaries + // + jsTestLog("Blocking writes to all secondaries."); + stopReplicationOnSecondaries(replTest); + + // + // Write to the primary and attempt stepdown + // + jsTestLog("Issuing a write to the primary(" + primary.host + ") with write_concern:1"); + assert.writeOK(coll.insert(dummy_doc, {writeConcern: {w: 1, wtimeout: timeout}})); + + jsTestLog("Trying to step down primary with only 1 node out of 5 caught up."); + assertStepDownFailsWithExceededTimeLimit(primary); + + // + // Re-enable writes to Secondary A and attempt stepdown + // + var secondaryA = replTest.getSecondaries()[0]; + jsTestLog("Reenabling writes to one secondary (" + nodeIdStr(replTest, secondaryA) + ")"); + restartServerReplication(secondaryA); + + jsTestLog("Issuing a write to the primary with write_concern:2"); + assert.writeOK(coll.insert(dummy_doc, {writeConcern: {w: 2, wtimeout: timeout}})); + + jsTestLog("Trying to step down primary with only 2 nodes out of 5 caught up."); + assertStepDownFailsWithExceededTimeLimit(primary); + + // + // Re-enable writes to Secondary B and attempt stepdown + // + var secondaryB = replTest.getSecondaries()[1]; + jsTestLog("Reenabling writes to another secondary (" + nodeIdStr(replTest, secondaryB) + ")"); + restartServerReplication(secondaryB); + + jsTestLog("Issuing a write to the primary with write_concern:3"); + assert.writeOK(coll.insert(dummy_doc, {writeConcern: {w: 3, wtimeout: timeout}})); + + jsTestLog("Trying to step down primary with 3 nodes out of 5 caught up."); + assertStepDownSucceeds(primary); + + jsTestLog("Waiting for PRIMARY(" + primary.host + ") to step down & become SECONDARY."); + replTest.waitForState(primary, ReplSetTest.State.SECONDARY); + + // + // Disable failpoints and stop replica set + // + jsTestLog("Disabling all fail points to allow for clean shutdown"); + restartReplSetReplication(replTest); + replTest.stopSet(); + +}()); diff --git a/jstests/replsets/stepdown_wrt_electable.js b/jstests/replsets/stepdown_wrt_electable.js deleted file mode 100644 index c929f2a2c56..00000000000 --- a/jstests/replsets/stepdown_wrt_electable.js +++ /dev/null @@ -1,40 +0,0 @@ -// Test that replSetStepDown filters out non-electable nodes -var replTest = new ReplSetTest({name: 'testSet', nodes: 2}); -var nodes = replTest.startSet(); - -// setup config -var c = replTest.getReplSetConfig(); -c.members[1].priority = 0; // not electable -replTest.initiate(c); - -var master = replTest.getPrimary(); -var testDB = master.getDB('test'); -var firstPrimary = testDB.isMaster().primary; - -// do a write to allow stepping down of the primary; -// otherwise, the primary will refuse to step down -testDB.foo.insert({x: 1}); -replTest.awaitReplication(); - -// stepdown should fail since there is no-one to elect within 10 secs -testDB.adminCommand({replSetStepDown: 5}); -assert(master.getDB("a").isMaster().ismaster, "not master"); - -// step down the primary asyncronously so it doesn't kill this test -var wait = startParallelShell("db.adminCommand({replSetStepDown:1000, force:true})", master.port); -var exitCode = wait({checkExitSuccess: false}); -assert.neq(0, exitCode, "expected replSetStepDown to close the shell's connection"); - -// check that the old primary is no longer master -assert.soon(function() { - try { - var isMaster = master.getDB("a").isMaster(); - printjson(isMaster); - return !(isMaster.ismaster); - } catch (e) { - return false; - } -}, "they shouldn't be master, but are"); - -// stop -replTest.stopSet(); diff --git a/jstests/replsets/sync2.js b/jstests/replsets/sync2.js index 481f59a13d9..0948ed70c4d 100644 --- a/jstests/replsets/sync2.js +++ b/jstests/replsets/sync2.js @@ -1,56 +1,53 @@ -var replTest = new ReplSetTest({name: 'sync2', nodes: 5, useBridge: true}); -var nodes = replTest.nodeList(); -var conns = replTest.startSet({oplogSize: "2"}); -replTest.initiate({ - "_id": "sync2", - "members": [ - {"_id": 0, host: nodes[0], priority: 2}, - {"_id": 1, host: nodes[1]}, - {"_id": 2, host: nodes[2]}, - {"_id": 3, host: nodes[3]}, - {"_id": 4, host: nodes[4]} - ] -}); - -var master = replTest.getPrimary(); -jsTestLog("Replica set test initialized"); - -// initial sync -master.getDB("foo").bar.insert({x: 1}); -replTest.awaitReplication(); - -conns[0].disconnect(conns[4]); -conns[1].disconnect(conns[2]); -conns[2].disconnect(conns[3]); -conns[3].disconnect(conns[1]); - -// 4 is connected to 2 -conns[4].disconnect(conns[1]); -conns[4].disconnect(conns[3]); - -assert.soon(function() { - master = replTest.getPrimary(); - return master === conns[0]; -}, 60 * 1000, "node 0 did not become primary quickly enough"); - -replTest.awaitReplication(); -jsTestLog("Checking that ops still replicate correctly"); -var option = { - writeConcern: {w: 5, wtimeout: 30000} -}; -// In PV0, this write can fail as a result of a bad spanning tree. If 2 was syncing from 4 prior to -// bridging, it will not change sync sources and receive the write in time. This was not a problem -// in 3.0 because the old version of mongobridge caused all the nodes to restart during -// partitioning, forcing the set to rebuild the spanning tree. -assert.writeOK(master.getDB("foo").bar.insert({x: 1}, option)); - -// 4 is connected to 3 -conns[4].disconnect(conns[2]); -conns[4].reconnect(conns[3]); - -option = { - writeConcern: {w: 5, wtimeout: 30000} -}; -assert.writeOK(master.getDB("foo").bar.insert({x: 1}, option)); - -replTest.stopSet(); +// Tests that nodes sync from each other properly and that nodes find new sync sources when they +// are disconnected from their current sync source. + +(function() { + 'use strict'; + + var replTest = new ReplSetTest({ + name: 'sync2', + nodes: [{rsConfig: {priority: 5}}, {arbiter: true}, {}, {}, {}], + useBridge: true + }); + var conns = replTest.startSet(); + replTest.initiate(); + + var master = replTest.getPrimary(); + jsTestLog("Replica set test initialized"); + + master.getDB("foo").bar.insert({x: 1}); + replTest.awaitReplication(); + + conns[0].disconnect(conns[4]); + conns[1].disconnect(conns[2]); + conns[2].disconnect(conns[3]); + conns[3].disconnect(conns[1]); + + // 4 is connected to 2 + conns[4].disconnect(conns[1]); + conns[4].disconnect(conns[3]); + + assert.soon(function() { + master = replTest.getPrimary(); + return master === conns[0]; + }, replTest.kDefaultTimeoutMS, "node 0 did not become primary quickly enough"); + + replTest.awaitReplication(); + jsTestLog("Checking that ops still replicate correctly"); + var option = { + writeConcern: {w: conns.length - 1, wtimeout: replTest.kDefaultTimeoutMS} + }; + // In PV0, this write can fail as a result of a bad spanning tree. If 2 was syncing from 4 prior + // to bridging, it will not change sync sources and receive the write in time. This was not a + // problem in 3.0 because the old version of mongobridge caused all the nodes to restart during + // partitioning, forcing the set to rebuild the spanning tree. + assert.writeOK(master.getDB("foo").bar.insert({x: 1}, option)); + + // 4 is connected to 3 + conns[4].disconnect(conns[2]); + conns[4].reconnect(conns[3]); + + assert.writeOK(master.getDB("foo").bar.insert({x: 1}, option)); + + replTest.stopSet(); +}()); diff --git a/jstests/replsets/tags.js b/jstests/replsets/tags.js index 55a0c4e2927..2de03f22d4f 100644 --- a/jstests/replsets/tags.js +++ b/jstests/replsets/tags.js @@ -1,260 +1,8 @@ (function() { 'use strict'; - var num = 5; - var host = getHostName(); - var name = 'tags'; + load("jstests/replsets/libs/tags.js"); - var replTest = new ReplSetTest({name: name, nodes: num, useBridge: true}); - var nodes = replTest.nodeList(); - var conns = replTest.startSet(); - var port = replTest.ports; - replTest.initiate({ - _id: name, - members: [ - { - _id: 0, - host: nodes[0], - tags: { - server: '0', - dc: 'ny', - ny: '1', - rack: 'ny.rk1', - }, - }, - { - _id: 1, - host: nodes[1], - priority: 2, - tags: { - server: '1', - dc: 'ny', - ny: '2', - rack: 'ny.rk1', - }, - }, - { - _id: 2, - host: nodes[2], - priority: 3, - tags: { - server: '2', - dc: 'ny', - ny: '3', - rack: 'ny.rk2', 2: 'this', - }, - }, - { - _id: 3, - host: nodes[3], - tags: { - server: '3', - dc: 'sf', - sf: '1', - rack: 'sf.rk1', - }, - }, - { - _id: 4, - host: nodes[4], - tags: { - server: '4', - dc: 'sf', - sf: '2', - rack: 'sf.rk2', - }, - }, - ], - settings: { - getLastErrorModes: { - '2 dc and 3 server': { - dc: 2, - server: 3, - }, - '1 and 2': { - 2: 1, - server: 1, - }, - '2': { - 2: 1, - }, - '3 and 4': { - sf: 2, - }, - '3 or 4': { - sf: 1, - }, - }, - }, - }); - - replTest.awaitReplication(); - - // Create collection to guard against timeouts due to file allocation. - assert.commandWorked(replTest.getPrimary().getDB('foo').createCollection('bar')); - replTest.awaitReplication(); - - var ensurePrimary = function(nodeId, expectedWritableNodes) { - jsTestLog('Node ' + nodeId + ' (' + replTest.nodes[nodeId].host + ') should be primary.'); - replTest.waitForState(replTest.nodes[nodeId], ReplSetTest.State.PRIMARY, 60 * 1000); - primary = replTest.getPrimary(); - var writeConcern = { - writeConcern: {w: expectedWritableNodes, wtimeout: 30 * 1000} - }; - assert.writeOK(primary.getDB('foo').bar.insert({x: 100}, writeConcern)); - return primary; - }; - - // 2 should eventually stage a priority takeover from the primary. - var primary = ensurePrimary(2, 3); - - jsTestLog('primary is now 2'); - var config = assert.commandWorked(primary.adminCommand({replSetGetConfig: 1})).config; - jsTestLog('test configuration = ' + tojson(config)); - - jsTestLog('Setting up partitions: [0-1-2] [3] [4]'); - conns[0].disconnect(conns[3]); - conns[0].disconnect(conns[4]); - conns[1].disconnect(conns[3]); - conns[1].disconnect(conns[4]); - conns[2].disconnect(conns[3]); - conns[2].disconnect(conns[4]); - conns[3].disconnect(conns[4]); - jsTestLog('Done setting up partitions'); - - jsTestLog('partitions: nodes with each set of brackets [N1, N2, N3] form a complete network.'); - jsTestLog('partitions: [0-1-2] [3] [4] (only nodes 0 and 1 can replicate from primary node 2'); - - var doc = { - x: 1 - }; - - // This timeout should be shorter in duration than the server parameter maxSyncSourceLagSecs. - // Some writes are expected to block for this 'timeout' duration before failing. - // Depending on the order of heartbeats (containing last committed op time) received - // by a node, it might hang up on its sync source. This may cause some of the write concern - // tests to fail. - var timeout = 20 * 1000; - - jsTestLog('test1'); - primary = ensurePrimary(2, 3); - - jsTestLog('Non-existent write concern should be rejected.'); - options = { - writeConcern: {w: 'blahblah', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - var result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); - assert.neq(null, result.getWriteConcernError()); - assert.eq(ErrorCodes.UnknownReplWriteConcern, - result.getWriteConcernError().code, - tojson(result.getWriteConcernError())); - - jsTestLog('Write concern "3 or 4" should fail - 3 and 4 are not connected to the primary.'); - var options = { - writeConcern: {w: '3 or 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - result = primary.getDB('foo').bar.insert(doc, options); - assert.neq(null, result.getWriteConcernError()); - assert(result.getWriteConcernError().errInfo.wtimeout); - - conns[1].reconnect(conns[4]); - jsTestLog('partitions: [0-1-2] [1-4] [3] ' + - '(all nodes besides node 3 can replicate from primary node 2)'); - primary = ensurePrimary(2, 4); - - jsTestLog('Write concern "3 or 4" should work - 4 is now connected to the primary ' + - primary.host + ' via node 1 ' + replTest.nodes[1].host); - options = { - writeConcern: {w: '3 or 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "3 and 4" should fail - 3 is not connected to the primary.'); - options = { - writeConcern: {w: '3 and 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); - assert.neq(null, result.getWriteConcernError()); - assert(result.getWriteConcernError().errInfo.wtimeout, tojson(result.getWriteConcernError())); - - conns[3].reconnect(conns[4]); - jsTestLog('partitions: [0-1-2] [1-4] [3-4] ' + - '(all secondaries can replicate from primary node 2)'); - primary = ensurePrimary(2, 5); - - jsTestLog('31004 should sync from 31001 (31026)'); - jsTestLog('31003 should sync from 31004 (31024)'); - jsTestLog('Write concern "3 and 4" should work - ' + - 'nodes 3 and 4 are connected to primary via node 1.'); - options = { - writeConcern: {w: '3 and 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "2" - writes to primary only.'); - options = { - writeConcern: {w: '2', wtimeout: 0} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "1 and 2"'); - options = { - writeConcern: {w: '1 and 2', wtimeout: 0} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "2 dc and 3 server"'); - primary = ensurePrimary(2, 5); - options = { - writeConcern: {w: '2 dc and 3 server', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Bringing down current primary node 2 ' + primary.host + - ' to allow next higher priority node 1 ' + replTest.nodes[1].host + - ' to become primary.'); - - // Is this necessary since 3 will be connected to the new primary via node 4? - conns[1].reconnect(conns[3]); - - conns[2].disconnect(conns[0]); - conns[2].disconnect(conns[1]); - - // Is this necessary when we partition node 2 off from the rest of the nodes? - replTest.stop(2); - jsTestLog('partitions: [0-1] [2] [1-3-4] ' + - '(all secondaries except down node 2 can replicate from new primary node 1)'); - - // Node 1 with slightly higher priority will take over. - jsTestLog('1 must become primary here because otherwise the other members will take too ' + - 'long timing out their old sync threads'); - primary = ensurePrimary(1, 4); - - jsTestLog('Write concern "3 and 4" should still work with new primary node 1 ' + primary.host); - options = { - writeConcern: {w: '3 and 4', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - assert.writeOK(primary.getDB('foo').bar.insert(doc, options)); - - jsTestLog('Write concern "2" should fail because node 2 ' + replTest.nodes[2].host + - ' is down.'); - options = { - writeConcern: {w: '2', wtimeout: timeout} - }; - assert.writeOK(primary.getDB('foo').bar.insert(doc)); - result = assert.writeError(primary.getDB('foo').bar.insert(doc, options)); - assert.neq(null, result.getWriteConcernError()); - assert(result.getWriteConcernError().errInfo.wtimeout); - - replTest.stopSet(); - jsTestLog('tags.js SUCCESS'); + var nodes = [{}, {}, {}, {}, {}]; + new TagsTest({nodes: nodes}).run(); }()); diff --git a/jstests/replsets/temp_namespace_restart_as_standalone.js b/jstests/replsets/temp_namespace_restart_as_standalone.js new file mode 100644 index 00000000000..063b1f3bbf4 --- /dev/null +++ b/jstests/replsets/temp_namespace_restart_as_standalone.js @@ -0,0 +1,87 @@ +/** + * Tests that temporary collections are not dropped when a member of a replica set is started up as + * a stand-alone mongod, i.e. without the --replSet parameter. + * + * @tags: [requires_persistence] + */ +(function() { + var rst = new ReplSetTest({nodes: 2}); + rst.startSet(); + + // Rig the election so that the first node becomes the primary and remains primary despite the + // secondary being terminated during this test. + var replSetConfig = rst.getReplSetConfig(); + replSetConfig.members[1].priority = 0; + replSetConfig.members[1].votes = 0; + rst.initiate(replSetConfig); + + var primaryConn = rst.getPrimary(); + var secondaryConn = rst.getSecondary(); + + var primaryDB = primaryConn.getDB("test"); + var secondaryDB = secondaryConn.getDB("test"); + + // Create a temporary collection and wait until the operation has replicated to the secondary. + assert.commandWorked(primaryDB.runCommand({ + create: "temp_collection", + temp: true, + })); + + // Verify that the temporary collection exists on the primary and has temp=true. + var primaryCollectionInfos = primaryDB.getCollectionInfos({name: "temp_collection"}); + assert.eq(1, primaryCollectionInfos.length, "'temp_collection' wasn't created on the primary"); + assert.eq("temp_collection", + primaryCollectionInfos[0].name, + "'temp_collection' wasn't created on the primary"); + assert.eq(true, + primaryCollectionInfos[0].options.temp, + "'temp_collection' wasn't created as temporary on the primary: " + + tojson(primaryCollectionInfos[0].options)); + + rst.awaitReplication(); + + // Verify that the temporary collection exists on the secondary and has temp=true. + var secondaryCollectionInfos = secondaryDB.getCollectionInfos({name: "temp_collection"}); + assert.eq( + 1, secondaryCollectionInfos.length, "'temp_collection' wasn't created on the secondary"); + assert.eq("temp_collection", + secondaryCollectionInfos[0].name, + "'temp_collection' wasn't created on the secondary"); + assert.eq(true, + secondaryCollectionInfos[0].options.temp, + "'temp_collection' wasn't created as temporary on the secondary: " + + tojson(secondaryCollectionInfos[0].options)); + + // Shut down the secondary and restart it as a stand-alone mongod. + var secondaryNodeId = rst.getNodeId(secondaryDB.getMongo()); + rst.stop(secondaryNodeId); + + secondaryConn = MongoRunner.runMongod({dbpath: secondaryConn.dbpath, noCleanData: true}); + assert.neq(null, secondaryConn, "secondary failed to start up as a stand-alone mongod"); + secondaryDB = secondaryConn.getDB("test"); + + // Verify that the temporary collection still exists on the secondary and has temp=true. + secondaryCollectionInfos = secondaryDB.getCollectionInfos({name: "temp_collection"}); + assert.eq(1, + secondaryCollectionInfos.length, + "'temp_collection' was dropped after restarting the secondary as a stand-alone"); + assert.eq("temp_collection", + secondaryCollectionInfos[0].name, + "'temp_collection' was dropped after restarting the secondary as a stand-alone"); + assert.eq(true, + secondaryCollectionInfos[0].options.temp, + "'temp_collection' is no longer temporary after restarting the secondary as a" + + " stand-alone: " + tojson(secondaryCollectionInfos[0].options)); + + // Shut down the secondary and restart it as a member of the replica set. + MongoRunner.stopMongod(secondaryConn); + + var restart = true; + rst.start(secondaryNodeId, {}, restart); + + // Verify that writes are replicated to the temporary collection and can successfully be applied + // by the secondary after having restarted it. + assert.writeOK(primaryDB.temp_collection.insert({}, {writeConcern: {w: 2, wtimeout: 60000}})); + + rst.stopSet(); +})(); diff --git a/jstests/replsets/write_concern_after_stepdown.js b/jstests/replsets/write_concern_after_stepdown.js new file mode 100644 index 00000000000..57cc44fda79 --- /dev/null +++ b/jstests/replsets/write_concern_after_stepdown.js @@ -0,0 +1,91 @@ +/* + * Tests that heartbeats containing writes from a different branch of history can't cause a stale + * primary to incorrectly acknowledge a w:majority write that's about to be rolled back. + */ +(function() { + 'use strict'; + + load("jstests/replsets/rslib.js"); + load("jstests/libs/write_concern_util.js"); + + var name = "writeConcernStepDownAndBackUp"; + var dbName = "wMajorityCheck"; + var collName = "stepdownAndBackUp"; + + var rst = new ReplSetTest( + {name: name, nodes: [{}, {}, {rsConfig: {priority: 0}}, ], useBridge: true}); + var nodes = rst.startSet(); + rst.initiate(); + + function waitForPrimary(node) { + assert.soon(function() { + return node.adminCommand('ismaster').ismaster; + }); + } + + jsTestLog("Make sure node 0 is primary."); + rst.stepUp(nodes[0]); + var primary = rst.getPrimary(); + var secondaries = rst.getSecondaries(); + assert.eq(nodes[0], primary); + // Wait for all data bearing nodes to get up to date. + assert.writeOK(nodes[0].getDB(dbName).getCollection(collName).insert( + {a: 1}, {writeConcern: {w: 3, wtimeout: rst.kDefaultTimeoutMS}})); + + // Stop the secondaries from replicating. + stopServerReplication(secondaries); + // Stop the primary from being able to complete stepping down. + assert.commandWorked( + nodes[0].adminCommand({configureFailPoint: 'blockHeartbeatStepdown', mode: 'alwaysOn'})); + + jsTestLog("Do w:majority write that will block waiting for replication."); + var doMajorityWrite = function() { + var res = db.getSiblingDB('wMajorityCheck') + .stepdownAndBackUp.insert({a: 2}, {writeConcern: {w: 'majority'}}); + assert.writeErrorWithCode(res, ErrorCodes.PrimarySteppedDown); + }; + + var joinMajorityWriter = startParallelShell(doMajorityWrite, nodes[0].port); + + jsTest.log("Disconnect primary from all secondaries"); + nodes[0].disconnect(nodes[1]); + nodes[0].disconnect(nodes[2]); + + jsTest.log("Wait for a new primary to be elected"); + // Allow the secondaries to replicate again. + restartServerReplication(secondaries); + + waitForPrimary(nodes[1]); + + jsTest.log("Do a write to the new primary"); + assert.writeOK(nodes[1].getDB(dbName).getCollection(collName).insert( + {a: 3}, {writeConcern: {w: 2, wtimeout: rst.kDefaultTimeoutMS}})); + + jsTest.log("Reconnect the old primary to the rest of the nodes"); + // Only allow the old primary to connect to the other nodes, not the other way around. + // This is so that the old priamry will detect that it needs to step down and step itself down, + // rather than one of the other nodes detecting this and sending it a replSetStepDown command, + // which would cause the old primary to kill all operations and close all connections, making + // the way that the insert in the parallel shell fails be nondeterministic. Rather than + // handling all possible failure modes in the parallel shell, allowing heartbeat connectivity in + // only one direction makes it easier for the test to fail deterministically. + nodes[1].acceptConnectionsFrom(nodes[0]); + nodes[2].acceptConnectionsFrom(nodes[0]); + + joinMajorityWriter(); + + // Allow the old primary to finish stepping down so that shutdown can finish. + var res = null; + try { + res = nodes[0].adminCommand({configureFailPoint: 'blockHeartbeatStepdown', mode: 'off'}); + } catch (e) { + // Expected - once we disable the fail point the stepdown will proceed and it's racy whether + // the stepdown closes all connections before or after the configureFailPoint command + // returns + } + if (res) { + assert.commandWorked(res); + } + + rst.stopSet(); +}()); diff --git a/jstests/replsets/write_concern_after_stepdown_and_stepup.js b/jstests/replsets/write_concern_after_stepdown_and_stepup.js new file mode 100644 index 00000000000..22167ee356c --- /dev/null +++ b/jstests/replsets/write_concern_after_stepdown_and_stepup.js @@ -0,0 +1,116 @@ +/* + * Tests that heartbeats containing writes from a different branch of history can't cause a stale + * primary to incorrectly acknowledge a w:majority write that's about to be rolled back, even if the + * stale primary is re-elected primary before waiting for the write concern acknowledgement. + */ +(function() { + 'use strict'; + + var name = "writeConcernStepDownAndBackUp"; + var dbName = "wMajorityCheck"; + var collName = "stepdownAndBackUp"; + + var rst = new ReplSetTest( + {name: name, nodes: [{}, {}, {rsConfig: {priority: 0}}, ], useBridge: true}); + var nodes = rst.startSet(); + rst.initiate(); + + var timeout = 5 * 60 * 1000; + + function waitForState(node, state) { + assert.soonNoExcept(function() { + assert.commandWorked(node.adminCommand( + {replSetTest: 1, waitForMemberState: state, timeoutMillis: timeout})); + return true; + }); + } + + function waitForPrimary(node) { + assert.soon(function() { + return node.adminCommand('ismaster').ismaster; + }); + } + + jsTestLog("Make sure node 0 is primary."); + rst.stepUp(nodes[0]); + var primary = rst.getPrimary(); + var secondaries = rst.getSecondaries(); + assert.eq(nodes[0], primary); + // Wait for all data bearing nodes to get up to date. + assert.writeOK(nodes[0].getDB(dbName).getCollection(collName).insert( + {a: 1}, {writeConcern: {w: 3, wtimeout: timeout}})); + + // Stop the secondaries from replicating. + secondaries.forEach(function(node) { + assert.commandWorked( + node.adminCommand({configureFailPoint: 'rsSyncApplyStop', mode: 'alwaysOn'})); + }); + // Stop the primary from calling into awaitReplication() + assert.commandWorked(nodes[0].adminCommand( + {configureFailPoint: 'hangBeforeWaitingForWriteConcern', mode: 'alwaysOn'})); + + jsTestLog("Do w:majority write that won't enter awaitReplication() until after the primary " + + "has stepped down and back up"); + var doMajorityWrite = function() { + assert.commandWorked(db.adminCommand({ismaster: 1})); + + assert.throws(function() { + db.getSiblingDB('wMajorityCheck') + .stepdownAndBackUp.insert({a: 2}, {writeConcern: {w: 'majority'}}); + }); + }; + + var joinMajorityWriter = startParallelShell(doMajorityWrite, nodes[0].port); + + jsTest.log("Disconnect primary from all secondaries"); + nodes[0].disconnect(nodes[1]); + nodes[0].disconnect(nodes[2]); + + jsTest.log("Wait for a new primary to be elected"); + // Allow the secondaries to replicate again. + secondaries.forEach(function(node) { + assert.commandWorked( + node.adminCommand({configureFailPoint: 'rsSyncApplyStop', mode: 'off'})); + }); + + waitForPrimary(nodes[1]); + + jsTest.log("Do a write to the new primary"); + assert.writeOK(nodes[1].getDB(dbName).getCollection(collName).insert( + {a: 3}, {writeConcern: {w: 2, wtimeout: timeout}})); + + jsTest.log("Reconnect the old primary to the rest of the nodes"); + nodes[0].reconnect(nodes[1]); + nodes[0].reconnect(nodes[2]); + + jsTest.log("Wait for the old primary to step down, roll back its write, and apply the " + + "new writes from the new primary"); + waitForState(nodes[0], ReplSetTest.State.SECONDARY); + rst.awaitReplication(); + + // At this point all 3 nodes should have the same data + assert.soonNoExcept(function() { + nodes.forEach(function(node) { + assert.eq(null, + node.getDB(dbName).getCollection(collName).findOne({a: 2}), + "Node " + node.host + " contained op that should have been rolled back"); + assert.neq(null, + node.getDB(dbName).getCollection(collName).findOne({a: 3}), + "Node " + node.host + + " was missing op from branch of history that should have persisted"); + }); + return true; + }); + + jsTest.log("Make the original primary become primary once again"); + rst.stepUp(nodes[0]); + + jsTest.log("Unblock the thread waiting for replication of the now rolled-back write, ensure " + + "that the write concern failed"); + assert.commandWorked(nodes[0].adminCommand( + {configureFailPoint: 'hangBeforeWaitingForWriteConcern', mode: 'off'})); + + joinMajorityWriter(); + + rst.stopSet(); +}()); diff --git a/jstests/sharding/authCommands.js b/jstests/sharding/authCommands.js index 2fb42801a40..c22e7aee6a8 100644 --- a/jstests/sharding/authCommands.js +++ b/jstests/sharding/authCommands.js @@ -71,6 +71,8 @@ var doTest = function() { assert.eq(expectedDocs, testDB.foo.count()); // Wait for the balancer to start back up + assert.writeOK( + configDB.settings.update({_id: 'balancer'}, {$set: {_waitForDelete: true}}, true)); st.startBalancer(); // Make sure we've done at least some splitting, so the balancer will work @@ -83,11 +85,6 @@ var doTest = function() { return x < 2 && configDB.locks.findOne({_id: 'test.foo'}).state == 0; }, "no balance happened", 5 * 60 * 1000); - assert.soon(function() { - print("Waiting for migration cleanup to occur..."); - return testDB.foo.find().itcount() == testDB.foo.count(); - }); - var map = function() { emit(this.i, this.j); }; diff --git a/jstests/sharding/cleanup_orphaned_basic.js b/jstests/sharding/cleanup_orphaned_basic.js index 3ed9015941b..5a119ec3fff 100644 --- a/jstests/sharding/cleanup_orphaned_basic.js +++ b/jstests/sharding/cleanup_orphaned_basic.js @@ -63,10 +63,10 @@ // Ping shard[1] so it will be aware that it is sharded. Otherwise cleanupOrphaned // may fail. assert.commandWorked(mongosAdmin.runCommand( - {moveChunk: coll.getFullName(), find: {_id: 1}, to: shards[1]._id})); + {moveChunk: coll.getFullName(), find: {_id: 1}, to: shards[1]._id, _waitForDelete: true})); assert.commandWorked(mongosAdmin.runCommand( - {moveChunk: coll.getFullName(), find: {_id: 1}, to: shards[0]._id})); + {moveChunk: coll.getFullName(), find: {_id: 1}, to: shards[0]._id, _waitForDelete: true})); // Collection's home is shard0, there are no chunks assigned to shard1. st.shard1.getCollection(ns).insert({}); diff --git a/jstests/sharding/conn_pool_stats.js b/jstests/sharding/conn_pool_stats.js index 872c20602a5..04952a273b6 100644 --- a/jstests/sharding/conn_pool_stats.js +++ b/jstests/sharding/conn_pool_stats.js @@ -1,19 +1,32 @@ // Tests for the connPoolStats command. +(function() { + "use strict"; + // Create a cluster with 2 shards. + var cluster = new ShardingTest({shards: 2}); -// Create a cluster with 2 shards. -var cluster = new ShardingTest({shards: 2}); + // Needed because the command was expanded post 3.2 + var version = cluster.s.getDB("admin").runCommand({buildinfo: 1}).versionArray; + var post32 = (version[0] > 4) || ((version[0] == 3) && (version[1] > 2)); -// Run the connPoolStats command -stats = cluster.s.getDB("admin").runCommand({connPoolStats: 1}); + // Run the connPoolStats command + var stats = cluster.s.getDB("admin").runCommand({connPoolStats: 1}); -// Validate output -printjson(stats); -assert.commandWorked(stats); -assert("replicaSets" in stats); -assert("hosts" in stats); -assert("numClientConnections" in stats); -assert("numAScopedConnections" in stats); -assert("totalInUse" in stats); -assert("totalAvailable" in stats); -assert("totalCreated" in stats); -assert.lte(stats["totalInUse"] + stats["totalAvailable"], stats["totalCreated"], tojson(stats)); + // Validate output + printjson(stats); + assert.commandWorked(stats); + assert("replicaSets" in stats); + assert("hosts" in stats); + assert("numClientConnections" in stats); + assert("numAScopedConnections" in stats); + assert("totalInUse" in stats); + assert("totalAvailable" in stats); + assert("totalCreated" in stats); + assert.lte(stats["totalInUse"] + stats["totalAvailable"], stats["totalCreated"], tojson(stats)); + if (post32) { + assert("pools" in stats); + assert("totalRefreshing" in stats); + assert.lte(stats["totalInUse"] + stats["totalAvailable"] + stats["totalRefreshing"], + stats["totalCreated"], + tojson(stats)); + } +})(); diff --git a/jstests/sharding/cursor1.js b/jstests/sharding/cursor1.js index e7d54ac85e4..eccc9a70c31 100644 --- a/jstests/sharding/cursor1.js +++ b/jstests/sharding/cursor1.js @@ -44,8 +44,6 @@ assert.eq(numObjs, cursor3.itcount(), "c3"); // Test that a cursor with a 1 second timeout eventually times out. - gc(); - gc(); var cur = db.foo.find().batchSize(2); assert(cur.next(), "T1"); assert(cur.next(), "T2"); @@ -65,9 +63,6 @@ } }, "cursor failed to time out", /*timeout*/ 30000, /*interval*/ 5000); - gc(); - gc(); - s.stop(); })(); diff --git a/jstests/sharding/migration_critical_section_concurrency.js b/jstests/sharding/migration_critical_section_concurrency.js new file mode 100644 index 00000000000..059d6a0e32e --- /dev/null +++ b/jstests/sharding/migration_critical_section_concurrency.js @@ -0,0 +1,63 @@ +// This test ensures that if one collection is its migration critical section, this won't stall +// operations for other sharded or unsharded collections + +load('./jstests/libs/chunk_manipulation_util.js'); + +(function() { + 'use strict'; + + var staticMongod = MongoRunner.runMongod({}); // For startParallelOps. + + var st = new ShardingTest({mongos: 1, shards: 2}); + assert.commandWorked(st.s0.adminCommand({enableSharding: 'TestDB'})); + st.ensurePrimaryShard('TestDB', st.shard0.shardName); + + var testDB = st.s0.getDB('TestDB'); + + assert.commandWorked(st.s0.adminCommand({shardCollection: 'TestDB.Coll0', key: {Key: 1}})); + assert.commandWorked(st.s0.adminCommand({split: 'TestDB.Coll0', middle: {Key: 0}})); + + var coll0 = testDB.Coll0; + assert.writeOK(coll0.insert({Key: -1, Value: '-1'})); + assert.writeOK(coll0.insert({Key: 1, Value: '1'})); + + assert.commandWorked(st.s0.adminCommand({shardCollection: 'TestDB.Coll1', key: {Key: 1}})); + assert.commandWorked(st.s0.adminCommand({split: 'TestDB.Coll1', middle: {Key: 0}})); + + var coll1 = testDB.Coll1; + assert.writeOK(coll1.insert({Key: -1, Value: '-1'})); + assert.writeOK(coll1.insert({Key: 1, Value: '1'})); + + // Ensure that coll0 has chunks on both shards so we can test queries against both donor and + // recipient for Coll1's migration below + assert.commandWorked( + st.s0.adminCommand({moveChunk: 'TestDB.Coll0', find: {Key: 1}, to: st.shard1.shardName})); + + // Pause the move chunk operation in the critical section + pauseMigrateAtStep(st.shard1, migrateStepNames.done); + var joinMoveChunk = moveChunkParallel( + staticMongod, st.s0.host, {Key: 1}, null, 'TestDB.Coll1', st.shard1.shardName); + + // Wait till the donor reaches the critical section + waitForMoveChunkStep(st.shard0, moveChunkStepNames.reachedSteadyState); + + // Ensure that operations for 'Coll0' are not stalled + assert.eq(1, coll0.find({Key: {$lte: -1}}).maxTimeMS(5000).itcount()); + assert.eq(1, coll0.find({Key: {$gte: 1}}).maxTimeMS(5000).itcount()); + assert.writeOK(coll0.insert({Key: -2, Value: '-2'}, {writeConcern: {wtimeout: 5000}})); + assert.writeOK(coll0.insert({Key: 2, Value: '2'}, {writeConcern: {wtimeout: 5000}})); + assert.eq(2, coll0.find({Key: {$lte: -1}}).maxTimeMS(5000).itcount()); + assert.eq(2, coll0.find({Key: {$gte: 1}}).maxTimeMS(5000).itcount()); + + // Ensure that operations for non-sharded collections are not stalled + var collUnsharded = testDB.CollUnsharded; + assert.eq(0, collUnsharded.find({}).maxTimeMS(5000).itcount()); + assert.writeOK( + collUnsharded.insert({TestKey: 0, Value: 'Zero'}, {writeConcern: {wtimeout: 5000}})); + assert.eq(1, collUnsharded.find({}).maxTimeMS(5000).itcount()); + + unpauseMigrateAtStep(st.shard1, migrateStepNames.done); + joinMoveChunk(); + + st.stop(); +})(); diff --git a/jstests/sharding/move_stale_mongos.js b/jstests/sharding/move_stale_mongos.js index 44dd7a7649f..9df8c80af03 100644 --- a/jstests/sharding/move_stale_mongos.js +++ b/jstests/sharding/move_stale_mongos.js @@ -15,10 +15,20 @@ var curShardIndex = 0; for (var i = 0; i < 100; i += 10) { assert.commandWorked(st.s0.getDB('admin').runCommand({split: testNs, middle: {_id: i}})); + if (st.configRS) { + st.configRS.awaitLastOpCommitted(); // Ensure that other mongos sees the split + } var nextShardIndex = (curShardIndex + 1) % shards.length; - assert.commandWorked(st.s1.getDB('admin').runCommand( - {moveChunk: testNs, find: {_id: i + 5}, to: shards[nextShardIndex]})); + assert.commandWorked(st.s1.getDB('admin').runCommand({ + moveChunk: testNs, + find: {_id: i + 5}, + to: shards[nextShardIndex], + _waitForDelete: true + })); curShardIndex = nextShardIndex; + if (st.configRS) { + st.configRS.awaitLastOpCommitted(); // Ensure that other mongos sees the move + } } st.stop(); diff --git a/jstests/sharding/shard6.js b/jstests/sharding/shard6.js index 3b4f3e416eb..7d3736c5c21 100644 --- a/jstests/sharding/shard6.js +++ b/jstests/sharding/shard6.js @@ -1,8 +1,8 @@ // shard6.js -summary = ""; +var summary = ""; -s = new ShardingTest({name: "shard6", shards: 2}); +var s = new ShardingTest({name: "shard6", shards: 2}); s.config.settings.update({_id: "balancer"}, {$set: {stopped: true}}, true); @@ -10,29 +10,33 @@ s.adminCommand({enablesharding: "test"}); s.ensurePrimaryShard('test', 'shard0001'); s.adminCommand({shardcollection: "test.data", key: {num: 1}}); -db = s.getDB("test"); +var db = s.getDB("test"); function poolStats(where) { var total = 0; var msg = "poolStats " + where + " "; - var x = db.runCommand("connPoolStats").hosts; - for (var h in x) { - var z = x[h]; - msg += z.created + " "; - total += z.created; + var stats = db.runCommand("connPoolStats"); + for (var h in stats.hosts) { + if (!stats.hosts.hasOwnProperty(h)) { + continue; + } + var host = stats.hosts[h]; + msg += host.created + " "; + total += host.created; } - printjson(x); + printjson(stats.hosts); print("****\n" + msg + "\n*****"); summary += msg + "\n"; + + assert.eq(total, stats.totalCreated, "mismatched number of total connections created"); + return total; } poolStats("at start"); // we want a lot of data, so lets make a 50k string to cheat :) -bigString = ""; -while (bigString.length < 50000) - bigString += "this is a big string. "; +var bigString = "this is a big string. ".repeat(50000); // ok, now lets insert a some data var num = 0; @@ -42,7 +46,7 @@ for (; num < 100; num++) { assert.eq(100, db.data.find().toArray().length, "basic find after setup"); -connBefore = poolStats("setup done"); +poolStats("setup done"); // limit @@ -54,7 +58,7 @@ for (var i = 1; i < 10; i++) { poolStats("after loop : " + i); } -assert.eq(connBefore, poolStats("limit test done"), "limit test conns"); +poolStats("limit test done"); function assertOrder(start, num) { var a = db.data.find().skip(start).limit(num).sort({num: 1}).map(function(z) { @@ -99,7 +103,7 @@ poolStats("after checking itcount"); // --- test save support --- -o = db.data.findOne(); +var o = db.data.findOne(); o.x = 16; db.data.save(o); o = db.data.findOne({_id: o._id}); diff --git a/jstests/sharding/shard_kill_and_pooling.js b/jstests/sharding/shard_kill_and_pooling.js index 6b8397f9e37..369ebe6c3ee 100644 --- a/jstests/sharding/shard_kill_and_pooling.js +++ b/jstests/sharding/shard_kill_and_pooling.js @@ -44,10 +44,8 @@ for (var test = 0; test < 2; test++) { jsTest.log("Returning the connections back to the pool."); for (var i = 0; i < conns.length; i++) { - conns[i] = null; + conns[i].close(); } - // Make sure we return connections back to the pool - gc(); // Don't make test fragile by linking to format of shardConnPoolStats, but this is useful if // something goes wrong. diff --git a/jstests/sharding/sharding_rs2.js b/jstests/sharding/sharding_rs2.js index ac186e6478b..c8663ee8dc8 100644 --- a/jstests/sharding/sharding_rs2.js +++ b/jstests/sharding/sharding_rs2.js @@ -136,10 +136,9 @@ assert.eq(100, ts.find().itcount(), "B5"); assert.eq(100, ts.find().batchSize(5).itcount(), "B6"); - t.find().batchSize(3).next(); - gc(); - gc(); - gc(); + var cursor = t.find().batchSize(3); + cursor.next(); + cursor.close(); // --- sharded ---- diff --git a/jstests/sharding/split_stale_mongos.js b/jstests/sharding/split_stale_mongos.js index e40112f0cd0..2a2a9b50cc8 100644 --- a/jstests/sharding/split_stale_mongos.js +++ b/jstests/sharding/split_stale_mongos.js @@ -12,7 +12,13 @@ assert.commandWorked(admin.runCommand({shardCollection: testNs, key: {_id: 1}})) for (var i = 0; i < 100; i += 10) { assert.commandWorked(st.s0.getDB('admin').runCommand({split: testNs, middle: {_id: i}})); + if (st.configRS) { + st.configRS.awaitLastOpCommitted(); // Ensure that other mongos sees the previous split + } assert.commandWorked(st.s1.getDB('admin').runCommand({split: testNs, middle: {_id: i + 5}})); + if (st.configRS) { + st.configRS.awaitLastOpCommitted(); // Ensure that other mongos sees the previous split + } } -st.stop(); \ No newline at end of file +st.stop(); diff --git a/jstests/sharding/sync_cluster_config/sccc_read_pref.js b/jstests/sharding/sync_cluster_config/sccc_read_pref.js new file mode 100644 index 00000000000..24e606677c0 --- /dev/null +++ b/jstests/sharding/sync_cluster_config/sccc_read_pref.js @@ -0,0 +1,18 @@ +/** + * Simple test that makes sure that running a command with read preference settings on a SCCC setup + * works. + */ +(function() { + 'use strict'; + + // Start a sharding cluster with a single shard, which has one node + var st = new ShardingTest({shards: 1, config: 3, other: {sync: true}}); + st.stopBalancer(); + + st.s.setReadPref('secondaryPreferred'); + var count = st.s.getDB('config').mongos.count(); + + assert.eq(1, count); + + st.stop(); +})(); diff --git a/jstests/slow1/replsets_priority1.js b/jstests/slow1/replsets_priority1.js index 7e5d562214b..4bbb856a4b6 100644 --- a/jstests/slow1/replsets_priority1.js +++ b/jstests/slow1/replsets_priority1.js @@ -73,7 +73,7 @@ var checkPrimaryIs = function(node) { }, 15); return ok; - }, node.host + '==1', 60000, 1000); + }, node.host + '==1', 240000, 1000); everyoneOkSoon(); }; diff --git a/jstests/ssl/ssl_fips.js b/jstests/ssl/ssl_fips.js index 47f148340d9..9b3e4b94c96 100644 --- a/jstests/ssl/ssl_fips.js +++ b/jstests/ssl/ssl_fips.js @@ -24,10 +24,7 @@ if (mongo != 0) { print("mongod failed to start, checking for FIPS support"); mongoOutput = rawMongoProgramOutput(); assert(mongoOutput.match(/this version of mongodb was not compiled with FIPS support/) || - mongoOutput.match(/FIPS_mode_set:fips mode not supported/) || - // Ubuntu 16.04's OpenSSL produces an unexpected error message, remove this check when - // SERVER-24350 is resolved - mongoOutput.match(/error:00000000:lib\(0\):func\(0\):reason\(0\)/)); + mongoOutput.match(/FIPS_mode_set:fips mode not supported/)); } else { // verify that auth works, SERVER-18051 md.getDB("admin").createUser({user: "root", pwd: "root", roles: ["root"]}); diff --git a/rpm/mongodb-enterprise-unstable.spec b/rpm/mongodb-enterprise-unstable.spec index cd4c66607b8..e5556b9a1cd 100644 --- a/rpm/mongodb-enterprise-unstable.spec +++ b/rpm/mongodb-enterprise-unstable.spec @@ -34,7 +34,7 @@ This metapackage will install the mongo shell, import/export tools, other client %package server Summary: MongoDB database server (enterprise) Group: Applications/Databases -Requires: openssl, net-snmp, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi +Requires: openssl %{?el6:>= 1.0.1}, net-snmp, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi Conflicts: mongo-10gen, mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools %description server @@ -58,7 +58,7 @@ This package contains the MongoDB server software, default configuration files, %package shell Summary: MongoDB shell client (enterprise) Group: Applications/Databases -Requires: openssl, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi +Requires: openssl %{?el6:>= 1.0.1}, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi Conflicts: mongo-10gen, mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools %description shell @@ -105,7 +105,7 @@ This package contains mongos, the MongoDB sharded cluster query router. %package tools Summary: MongoDB tools (enterprise) Group: Applications/Databases -Requires: openssl, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi +Requires: openssl %{?el6:>= 1.0.1}, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi Conflicts: mongo-10gen, mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools %description tools diff --git a/rpm/mongodb-enterprise.spec b/rpm/mongodb-enterprise.spec index 95fd1410306..3373b1bb19d 100644 --- a/rpm/mongodb-enterprise.spec +++ b/rpm/mongodb-enterprise.spec @@ -35,7 +35,7 @@ This metapackage will install the mongo shell, import/export tools, other client %package server Summary: MongoDB database server (enterprise) Group: Applications/Databases -Requires: openssl, net-snmp, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi +Requires: openssl %{?el6:>= 1.0.1}, net-snmp, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi Conflicts: mongo-10gen, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools Obsoletes: mongo-10gen-enterprise-server Provides: mongo-10gen-enterprise-server @@ -61,7 +61,7 @@ This package contains the MongoDB server software, default configuration files, %package shell Summary: MongoDB shell client (enterprise) Group: Applications/Databases -Requires: openssl, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi +Requires: openssl %{?el6:>= 1.0.1}, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi Conflicts: mongo-10gen, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools Obsoletes: mongo-10gen-enterprise-shell Provides: mongo-10gen-enterprise-shell @@ -112,7 +112,7 @@ This package contains mongos, the MongoDB sharded cluster query router. %package tools Summary: MongoDB tools (enterprise) Group: Applications/Databases -Requires: openssl, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi +Requires: openssl %{?el6:>= 1.0.1}, cyrus-sasl, cyrus-sasl-plain, cyrus-sasl-gssapi Conflicts: mongo-10gen, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools Obsoletes: mongo-10gen-enterprise-tools Provides: mongo-10gen-enterprise-tools diff --git a/rpm/mongodb-org-unstable.spec b/rpm/mongodb-org-unstable.spec index 27b2266bd3e..39e305c3afc 100644 --- a/rpm/mongodb-org-unstable.spec +++ b/rpm/mongodb-org-unstable.spec @@ -33,7 +33,7 @@ This metapackage will install the mongo shell, import/export tools, other client %package server Summary: MongoDB database server Group: Applications/Databases -Requires: openssl +Requires: openssl %{?el6:>= 1.0.1} Conflicts: mongo-10gen, mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools %description server @@ -57,7 +57,7 @@ This package contains the MongoDB server software, default configuration files, %package shell Summary: MongoDB shell client Group: Applications/Databases -Requires: openssl +Requires: openssl %{?el6:>= 1.0.1} Conflicts: mongo-10gen, mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools %description shell @@ -104,7 +104,7 @@ This package contains mongos, the MongoDB sharded cluster query router. %package tools Summary: MongoDB tools Group: Applications/Databases -Requires: openssl +Requires: openssl %{?el6:>= 1.0.1} Conflicts: mongo-10gen, mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools %description tools diff --git a/rpm/mongodb-org.spec b/rpm/mongodb-org.spec index 00e30576b5c..40d2808060b 100644 --- a/rpm/mongodb-org.spec +++ b/rpm/mongodb-org.spec @@ -35,7 +35,7 @@ This metapackage will install the mongo shell, import/export tools, other client %package server Summary: MongoDB database server Group: Applications/Databases -Requires: openssl +Requires: openssl %{?el6:>= 1.0.1} Conflicts: mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools Obsoletes: mongo-10gen-server Provides: mongo-10gen-server @@ -61,7 +61,7 @@ This package contains the MongoDB server software, default configuration files, %package shell Summary: MongoDB shell client Group: Applications/Databases -Requires: openssl +Requires: openssl %{?el6:>= 1.0.1} Conflicts: mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools Obsoletes: mongo-10gen-shell Provides: mongo-10gen-shell @@ -112,7 +112,7 @@ This package contains mongos, the MongoDB sharded cluster query router. %package tools Summary: MongoDB tools Group: Applications/Databases -Requires: openssl +Requires: openssl %{?el6:>= 1.0.1} Conflicts: mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools Obsoletes: mongo-10gen-tools Provides: mongo-10gen-tools diff --git a/site_scons/site_tools/compilation_db.py b/site_scons/site_tools/compilation_db.py index a4c82eebd53..5c0d90e5f41 100644 --- a/site_scons/site_tools/compilation_db.py +++ b/site_scons/site_tools/compilation_db.py @@ -30,6 +30,16 @@ import itertools # communicate more gracefully? __COMPILATION_DB_ENTRIES=[] +# Cribbed from Tool/cc.py and Tool/c++.py. It would be better if +# we could obtain this from SCons. +_CSuffixes = ['.c'] +if not SCons.Util.case_sensitive_suffixes('.c', '.C'): + _CSuffixes.append('.C') + +_CXXSuffixes = ['.cpp', '.cc', '.cxx', '.c++', '.C++'] +if SCons.Util.case_sensitive_suffixes('.c', '.C'): + _CXXSuffixes.append('.C') + # We make no effort to avoid rebuilding the entries. Someday, perhaps we could and even # integrate with the cache, but there doesn't seem to be much call for it. class __CompilationDbNode(SCons.Node.Node): @@ -99,20 +109,15 @@ def generate(env, **kwargs): static_obj, shared_obj = SCons.Tool.createObjBuilders(env) - # TODO: Is there a way to obtain the configured suffixes for C and C++ - # from the existing obj builders? Seems unfortunate to re-iterate them. - CSuffixes = ['.c'] - CXXSuffixes = ['.cc', '.cxx', '.cpp'] - env['COMPILATIONDB_COMSTR'] = kwargs.get( 'COMPILATIONDB_COMSTR', 'Building compilation database $TARGET') components_by_suffix = itertools.chain( - itertools.product(CSuffixes, [ + itertools.product(_CSuffixes, [ (static_obj, SCons.Defaults.StaticObjectEmitter, '$CCCOM'), (shared_obj, SCons.Defaults.SharedObjectEmitter, '$SHCCCOM'), ]), - itertools.product(CXXSuffixes, [ + itertools.product(_CXXSuffixes, [ (static_obj, SCons.Defaults.StaticObjectEmitter, '$CXXCOM'), (shared_obj, SCons.Defaults.SharedObjectEmitter, '$SHCXXCOM'), ]), @@ -122,13 +127,14 @@ def generate(env, **kwargs): suffix = entry[0] builder, base_emitter, command = entry[1] - builder.add_emitter( - suffix, SCons.Builder.ListEmitter( - [ - makeEmitCompilationDbEntry(command), - base_emitter, - ] - )) + # Assumes a dictionary emitter + emitter = builder.emitter[suffix] + builder.emitter[suffix] = SCons.Builder.ListEmitter( + [ + emitter, + makeEmitCompilationDbEntry(command), + ] + ) env['BUILDERS']['__COMPILATIONDB_Entry'] = SCons.Builder.Builder( action=SCons.Action.Action(CompilationDbEntryAction, None), diff --git a/site_scons/site_tools/incremental_link.py b/site_scons/site_tools/incremental_link.py new file mode 100644 index 00000000000..cf74ef9674c --- /dev/null +++ b/site_scons/site_tools/incremental_link.py @@ -0,0 +1,45 @@ +# Copyright 2017 MongoDB Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import SCons + +def _tag_as_precious(target, source, env): + env.Precious(target) + return target, source + +def generate(env): + builders = env['BUILDERS'] + for builder in ('Program', 'SharedLibrary', 'LoadableModule'): + emitter = builders[builder].emitter + builders[builder].emitter = SCons.Builder.ListEmitter([ + emitter, + _tag_as_precious, + ]) + +def exists(env): + # By default, the windows linker is incremental, so unless + # overridden in the environment with /INCREMENTAL:NO, the tool is + # in play. + if env.TargetOSIs('windows') and not "/INCREMENTAL:NO" in env['LINKFLAGS']: + return True + + # On posix platforms, excluding darwin, we may have enabled + # incremental linking. Check for the relevant flags. + if env.TargetOSIs('posix') and \ + not env.TargetOSIs('darwin') and \ + "-fuse-ld=gold" in env['LINKFLAGS'] and \ + "-Wl,--incremental" in env['LINKFLAGS']: + return True + + return False diff --git a/site_scons/site_tools/split_dwarf.py b/site_scons/site_tools/split_dwarf.py new file mode 100644 index 00000000000..95130c9e9a3 --- /dev/null +++ b/site_scons/site_tools/split_dwarf.py @@ -0,0 +1,65 @@ +# Copyright 2017 MongoDB Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import SCons + +_splitDwarfFlag = '-gsplit-dwarf' + +# Cribbed from Tool/cc.py and Tool/c++.py. It would be better if +# we could obtain this from SCons. +_CSuffixes = ['.c'] +if not SCons.Util.case_sensitive_suffixes('.c', '.C'): + _CSuffixes.append('.C') + +_CXXSuffixes = ['.cpp', '.cc', '.cxx', '.c++', '.C++'] +if SCons.Util.case_sensitive_suffixes('.c', '.C'): + _CXXSuffixes.append('.C') + +def _dwo_emitter(target, source, env): + new_targets = [] + for t in target: + base, ext = SCons.Util.splitext(str(t)) + if not any(ext == env[osuffix] for osuffix in ['OBJSUFFIX', 'SHOBJSUFFIX']): + continue + # TODO: Move 'dwo' into DWOSUFFIX so it can be customized? For + # now, GCC doesn't let you control the output filename, so it + # doesn't matter. + dwotarget = (t.builder.target_factory or env.File)(base + ".dwo") + new_targets.append(dwotarget) + targets = target + new_targets + return (targets, source) + +def generate(env): + suffixes = [] + if _splitDwarfFlag in env['CCFLAGS']: + suffixes = _CSuffixes + _CXXSuffixes + else: + if _splitDwarfFlag in env['CFLAGS']: + suffixes.extend(_CSuffixes) + if _splitDwarfFlag in env['CXXFLAGS']: + suffixes.extend(_CXXSuffixes) + + for object_builder in SCons.Tool.createObjBuilders(env): + emitterdict = object_builder.builder.emitter + for suffix in emitterdict.iterkeys(): + if not suffix in suffixes: + continue + base = emitterdict[suffix] + emitterdict[suffix] = SCons.Builder.ListEmitter([ + base, + _dwo_emitter, + ]) + +def exists(env): + return any(_splitDwarfFlag in env[f] for f in ['CCFLAGS', 'CFLAGS', 'CXXFLAGS']) diff --git a/src/mongo/SConscript b/src/mongo/SConscript index e4b6c1dc770..18f75acdaeb 100644 --- a/src/mongo/SConscript +++ b/src/mongo/SConscript @@ -218,6 +218,7 @@ config_header_substs = ( ('@mongo_config_have_memset_s@', 'MONGO_CONFIG_HAVE_MEMSET_S'), ('@mongo_config_have_posix_monotonic_clock@', 'MONGO_CONFIG_HAVE_POSIX_MONOTONIC_CLOCK'), ('@mongo_config_have_std_is_trivially_copyable@', 'MONGO_CONFIG_HAVE_STD_IS_TRIVIALLY_COPYABLE'), + ('@mongo_config_have_std_enable_if_t@', 'MONGO_CONFIG_HAVE_STD_ENABLE_IF_T'), ('@mongo_config_have_std_make_unique@', 'MONGO_CONFIG_HAVE_STD_MAKE_UNIQUE'), ('@mongo_config_have_std_align@', 'MONGO_CONFIG_HAVE_STD_ALIGN'), ('@mongo_config_have_strnlen@', 'MONGO_CONFIG_HAVE_STRNLEN'), diff --git a/src/mongo/base/error_codes.err b/src/mongo/base/error_codes.err index 7657112cac6..692fae39172 100644 --- a/src/mongo/base/error_codes.err +++ b/src/mongo/base/error_codes.err @@ -155,6 +155,11 @@ error_code("OplogOutOfOrder", 152) error_code("CanRepairToDowngrade", 157) error_code("MustUpgrade", 158) error_code("QueryPlanKilled", 173) +error_code("FileOpenFailed", 174) +error_code("ZoneNotFound", 175) +error_code("RangeOverlapConflict", 176) +error_code("WindowsPdhError", 177) +error_code("BadPerfCounterPath", 178) error_code("ReceivedOpReplyMessage", 198); # Non-sequential error codes (for compatibility only) diff --git a/src/mongo/base/string_data.h b/src/mongo/base/string_data.h index e7a3d677594..ef09cbe827f 100644 --- a/src/mongo/base/string_data.h +++ b/src/mongo/base/string_data.h @@ -35,6 +35,7 @@ #include #include +#include "mongo/stdx/type_traits.h" #define MONGO_INCLUDE_INVARIANT_H_WHITELISTED #include "mongo/util/invariant.h" #undef MONGO_INCLUDE_INVARIANT_H_WHITELISTED @@ -59,6 +60,9 @@ class StringData { StringData(const char* c, size_t len, TrustedInitTag) : _data(c), _size(len) {} public: + // Iterator type + using const_iterator = const char*; + /** Constructs an empty StringData. */ StringData() = default; @@ -98,6 +102,23 @@ public: invariant(_data || (_size == 0)); } + /** + * Constructs a StringData with begin and end iterators. begin points to the beginning of the + * string. end points to the position past the end of the string. In a null-terminated string, + * end points to the null-terminator. + * + * We template the second parameter to ensure if StringData is called with 0 in the second + * parameter, the (ptr,len) constructor is chosen instead. + */ + template < + typename InputIt, + typename = stdx::enable_if_t::value>> + StringData(InputIt begin, InputIt end) { + invariant(begin && end); + _data = begin; + _size = std::distance(begin, end); + } + /** * Returns -1, 0, or 1 if 'this' is less, equal, or greater than 'other' in * lexicographical order. @@ -171,9 +192,6 @@ public: // // iterators // - - typedef const char* const_iterator; - const_iterator begin() const { return rawData(); } diff --git a/src/mongo/bson/timestamp.cpp b/src/mongo/bson/timestamp.cpp index 2abd3a8990a..232aff04128 100644 --- a/src/mongo/bson/timestamp.cpp +++ b/src/mongo/bson/timestamp.cpp @@ -34,6 +34,7 @@ #include #include +#include "mongo/bson/bsonobjbuilder.h" #include "mongo/util/time_support.h" namespace mongo { @@ -70,4 +71,11 @@ std::string Timestamp::toString() const { ss << std::hex << secs << ':' << i; return ss.str(); } + + +BSONObj Timestamp::toBSON() const { + BSONObjBuilder bldr; + bldr.append("", *this); + return bldr.obj(); +} } diff --git a/src/mongo/bson/timestamp.h b/src/mongo/bson/timestamp.h index 1dc04973efb..39c0eb572a8 100644 --- a/src/mongo/bson/timestamp.h +++ b/src/mongo/bson/timestamp.h @@ -34,6 +34,8 @@ namespace mongo { +class BSONObj; + /** * Timestamp: A combination of a count of seconds since the POSIX epoch plus an ordinal value. */ @@ -112,6 +114,7 @@ public: // Append the BSON representation of this Timestamp to the given BufBuilder with the given // name. This lives here because Timestamp manages its own serialization format. void append(BufBuilder& builder, const StringData& fieldName) const; + BSONObj toBSON() const; private: std::tuple tie() const { diff --git a/src/mongo/client/SConscript b/src/mongo/client/SConscript index d112f34fe14..a8a5367e2fe 100644 --- a/src/mongo/client/SConscript +++ b/src/mongo/client/SConscript @@ -60,6 +60,7 @@ saslClientSource = [ 'sasl_client_session.cpp', 'sasl_plain_client_conversation.cpp', 'sasl_scramsha1_client_conversation.cpp', + 'scram_sha1_client_cache.cpp', ] # Add in actual sasl dependencies if sasl is enabled, otherwise diff --git a/src/mongo/client/authenticate.cpp b/src/mongo/client/authenticate.cpp index 58002af5607..6db235f552b 100644 --- a/src/mongo/client/authenticate.cpp +++ b/src/mongo/client/authenticate.cpp @@ -237,7 +237,7 @@ void authX509(RunCommandHook runCommand, void auth(RunCommandHook runCommand, const BSONObj& params, - StringData hostname, + const HostAndPort& hostname, StringData clientName, AuthCompletionHandler handler) { std::string mechanism; @@ -273,7 +273,7 @@ bool needsFallback(const AuthResponse& response) { void asyncAuth(RunCommandHook runCommand, const BSONObj& params, - StringData hostname, + const HostAndPort& hostname, StringData clientName, AuthCompletionHandler handler) { auth(runCommand, @@ -297,7 +297,7 @@ void asyncAuth(RunCommandHook runCommand, } // namespace void authenticateClient(const BSONObj& params, - StringData hostname, + const HostAndPort& hostname, StringData clientName, RunCommandHook runCommand, AuthCompletionHandler handler) { diff --git a/src/mongo/client/authenticate.h b/src/mongo/client/authenticate.h index 65c649c49c2..f8d7434f30a 100644 --- a/src/mongo/client/authenticate.h +++ b/src/mongo/client/authenticate.h @@ -92,7 +92,7 @@ extern const char* const kMechanismScramSha1; * tantamount to authentication failure, but may also indicate more serious problems. */ void authenticateClient(const BSONObj& params, - StringData hostname, + const HostAndPort& hostname, StringData clientSubjectName, RunCommandHook runCommand, AuthCompletionHandler handler = AuthCompletionHandler()); diff --git a/src/mongo/client/authenticate_test.cpp b/src/mongo/client/authenticate_test.cpp index 0de2a1e9670..6a6596d52d8 100644 --- a/src/mongo/client/authenticate_test.cpp +++ b/src/mongo/client/authenticate_test.cpp @@ -173,13 +173,13 @@ public: TEST_F(AuthClientTest, MongoCR) { auto params = loadMongoCRConversation(); - auth::authenticateClient(std::move(params), "", "", _runCommandCallback); + auth::authenticateClient(std::move(params), HostAndPort(), "", _runCommandCallback); } TEST_F(AuthClientTest, asyncMongoCR) { auto params = loadMongoCRConversation(); auth::authenticateClient(std::move(params), - "", + HostAndPort(), "", _runCommandCallback, [this](auth::AuthResponse response) { ASSERT(response.isOK()); }); @@ -188,13 +188,13 @@ TEST_F(AuthClientTest, asyncMongoCR) { #ifdef MONGO_CONFIG_SSL TEST_F(AuthClientTest, X509) { auto params = loadX509Conversation(); - auth::authenticateClient(std::move(params), "", _username, _runCommandCallback); + auth::authenticateClient(std::move(params), HostAndPort(), _username, _runCommandCallback); } TEST_F(AuthClientTest, asyncX509) { auto params = loadX509Conversation(); auth::authenticateClient(std::move(params), - "", + HostAndPort(), _username, _runCommandCallback, [this](auth::AuthResponse response) { ASSERT(response.isOK()); }); diff --git a/src/mongo/client/connpool.cpp b/src/mongo/client/connpool.cpp index 2a019db9787..598d58afcd1 100644 --- a/src/mongo/client/connpool.cpp +++ b/src/mongo/client/connpool.cpp @@ -62,6 +62,11 @@ PoolForHost::~PoolForHost() { } void PoolForHost::clear() { + if (!_parentDestroyed) { + logNoCache() << "Dropping all pooled connections to " << _hostName << "(with timeout of " + << _socketTimeout << " seconds)"; + } + while (!_pool.empty()) { StoredConnection sc = _pool.top(); delete sc.conn; @@ -80,9 +85,19 @@ void PoolForHost::done(DBConnectionPool* pool, DBClientBase* c) { if (isFailed || // Another (later) connection was reported as broken to this host - (c->getSockCreationMicroSec() < _minValidCreationTimeMicroSec) || + (c->getSockCreationMicroSec() < _minValidCreationTimeMicroSec)) { + logNoCache() << "Ending connection to host " << _hostName << "(with timeout of " + << _socketTimeout << " seconds)" + << " due to bad connection status; " << openConnections() + << " connections to that host remain open"; + pool->onDestroy(c); + delete c; + } else if (_maxPoolSize >= 0 && static_cast(_pool.size()) >= _maxPoolSize) { // We have a pool size that we need to enforce - (_maxPoolSize >= 0 && static_cast(_pool.size()) >= _maxPoolSize)) { + logNoCache() << "Ending idle connection to host " << _hostName << "(with timeout of " + << _socketTimeout << " seconds)" + << " because the pool meets constraints; " << openConnections() + << " connections to that host remain open"; pool->onDestroy(c); delete c; } else { @@ -95,9 +110,11 @@ void PoolForHost::reportBadConnectionAt(uint64_t microSec) { if (microSec != DBClientBase::INVALID_SOCK_CREATION_TIME && microSec > _minValidCreationTimeMicroSec) { _minValidCreationTimeMicroSec = microSec; - log() << "Detected bad connection created at " << _minValidCreationTimeMicroSec - << " microSec, clearing pool for " << _hostName << " of " << _pool.size() - << " connections" << endl; + + logNoCache() << "Detected bad connection created at " << _minValidCreationTimeMicroSec + << " microSec, clearing pool for " << _hostName << " of " << openConnections() + << " connections" << endl; + clear(); } } @@ -130,6 +147,9 @@ DBClientBase* PoolForHost::get(DBConnectionPool* pool, double socketTimeout) { } void PoolForHost::flush() { + log() << "Dropping all pooled connections to " << _hostName << "(with timeout of " + << _socketTimeout << " seconds)"; + while (!_pool.empty()) { StoredConnection c = _pool.top(); _pool.pop(); @@ -196,10 +216,17 @@ DBClientBase* DBConnectionPool::_get(const string& ident, double socketTimeout) stdx::lock_guard L(_mutex); PoolForHost& p = _pools[PoolKey(ident, socketTimeout)]; p.setMaxPoolSize(_maxPoolSize); + p.setSocketTimeout(socketTimeout); p.initializeHostName(ident); return p.get(this, socketTimeout); } +int DBConnectionPool::openConnections(const string& ident, double socketTimeout) { + stdx::lock_guard L(_mutex); + PoolForHost& p = _pools[PoolKey(ident, socketTimeout)]; + return p.openConnections(); +} + DBClientBase* DBConnectionPool::_finishCreate(const string& host, double socketTimeout, DBClientBase* conn) { @@ -219,11 +246,17 @@ DBClientBase* DBConnectionPool::_finishCreate(const string& host, throw; } + log() << "Successfully connected to " << host << " (" << openConnections(host, socketTimeout) + << " connections now open to " << host << " with a " << socketTimeout + << " second timeout)"; + return conn; } DBClientBase* DBConnectionPool::get(const ConnectionString& url, double socketTimeout) { - DBClientBase* c = _get(url.toString(), socketTimeout); + auto host = url.toString(); + + DBClientBase* c = _get(host, socketTimeout); if (c) { try { onHandedOut(c); @@ -236,7 +269,7 @@ DBClientBase* DBConnectionPool::get(const ConnectionString& url, double socketTi string errmsg; c = url.connect(errmsg, socketTimeout); - uassert(13328, _name + ": connect failed " + url.toString() + " : " + errmsg, c); + uassert(13328, _name + ": connect failed " + host + " : " + errmsg, c); return _finishCreate(url.toString(), socketTimeout, c); } @@ -262,6 +295,7 @@ DBClientBase* DBConnectionPool::get(const string& host, double socketTimeout) { host, 11002, str::stream() << _name << " error: " << errmsg); + return _finishCreate(host, socketTimeout, c); } @@ -284,7 +318,13 @@ void DBConnectionPool::release(const string& host, DBClientBase* c) { DBConnectionPool::~DBConnectionPool() { - // connection closing is handled by ~PoolForHost + // Do not log in destruction, because global connection pools get + // destroyed after the logging framework. + stdx::lock_guard L(_mutex); + for (PoolMap::iterator i = _pools.begin(); i != _pools.end(); i++) { + PoolForHost& p = i->second; + p._parentDestroyed = true; + } } void DBConnectionPool::flush() { @@ -361,11 +401,11 @@ void DBConnectionPool::appendConnectionStats(executor::ConnectionPoolStats* stat invariant(uri.isOK()); HostAndPort host = uri.getValue().getServers().front(); - executor::ConnectionStatsPerHost hostStats{ - static_cast(i->second.numInUse()), - static_cast(i->second.numAvailable()), - static_cast(i->second.numCreated())}; - stats->updateStatsForHost(host, hostStats); + executor::ConnectionStatsPer hostStats{static_cast(i->second.numInUse()), + static_cast(i->second.numAvailable()), + static_cast(i->second.numCreated()), + 0}; + stats->updateStatsForHost("global", host, hostStats); } } } diff --git a/src/mongo/client/connpool.h b/src/mongo/client/connpool.h index 509965971ff..969a905df5f 100644 --- a/src/mongo/client/connpool.h +++ b/src/mongo/client/connpool.h @@ -57,12 +57,15 @@ public: // Sentinel value indicating pool has no cleanup limit static const int kPoolSizeUnlimited; + friend class DBConnectionPool; + PoolForHost() : _created(0), _minValidCreationTimeMicroSec(0), _type(ConnectionString::INVALID), _maxPoolSize(kPoolSizeUnlimited), - _checkedOut(0) {} + _checkedOut(0), + _parentDestroyed(false) {} ~PoolForHost(); @@ -80,6 +83,13 @@ public: _maxPoolSize = maxPoolSize; } + /** + * Sets the socket timeout on this host, for reporting purposes only. + */ + void setSocketTimeout(double socketTimeout) { + _socketTimeout = socketTimeout; + } + int numAvailable() const { return (int)_pool.size(); } @@ -88,6 +98,13 @@ public: return _checkedOut; } + /** + * Returns the number of open connections in this pool. + */ + int openConnections() const { + return _checkedOut + (int)_pool.size(); + } + void createdOne(DBClientBase* base); long long numCreated() const { return _created; @@ -140,6 +157,7 @@ private: }; std::string _hostName; + double _socketTimeout; std::stack _pool; int64_t _created; @@ -151,6 +169,9 @@ private: // The number of currently active connections from this pool int _checkedOut; + + // Whether our parent DBConnectionPool object is in destruction + bool _parentDestroyed; }; class DBConnectionHook { @@ -197,6 +218,11 @@ public: return _maxPoolSize; } + /** + * Returns the number of connections to the given host pool. + */ + int openConnections(const std::string& ident, double socketTimeout); + /** * Sets the maximum number of connections pooled per-host. * diff --git a/src/mongo/client/dbclient.cpp b/src/mongo/client/dbclient.cpp index 706c5ed8838..6fd1c849c8d 100644 --- a/src/mongo/client/dbclient.cpp +++ b/src/mongo/client/dbclient.cpp @@ -536,7 +536,7 @@ void DBClientWithCommands::_auth(const BSONObj& params) { auth::authenticateClient( params, - HostAndPort(getServerAddress()).host(), + HostAndPort(getServerAddress()), clientName, [this](RemoteCommandRequest request, auth::AuthCompletionHandler handler) { BSONObj info; @@ -678,9 +678,11 @@ bool DBClientWithCommands::eval(const string& dbname, const string& jscode) { list DBClientWithCommands::getDatabaseNames() { BSONObj info; - uassert(10005, - "listdatabases failed", - runCommand("admin", BSON("listDatabases" << 1), info, QueryOption_SlaveOk)); + uassert( + 10005, + "listdatabases failed", + runCommand( + "admin", BSON("listDatabases" << 1 << "nameOnly" << true), info, QueryOption_SlaveOk)); uassert(10006, "listDatabases.databases not array", info["databases"].type() == Array); list names; @@ -922,6 +924,12 @@ Status DBClientConnection::connect(const HostAndPort& serverAddress) { return swIsMasterReply.getStatus(); } + // Ensure that the isMaster response is "ok:1". + auto isMasterStatus = getStatusFromCommandResult(swIsMasterReply.getValue().data); + if (!isMasterStatus.isOK()) { + return isMasterStatus; + } + auto swProtocolSet = rpc::parseProtocolSetFromIsMasterReply(swIsMasterReply.getValue().data); if (!swProtocolSet.isOK()) { return swProtocolSet.getStatus(); diff --git a/src/mongo/client/dbclientcursor.cpp b/src/mongo/client/dbclientcursor.cpp index 2e0a5bdade3..d76277d49ca 100644 --- a/src/mongo/client/dbclientcursor.cpp +++ b/src/mongo/client/dbclientcursor.cpp @@ -324,8 +324,6 @@ void DBClientCursor::dataReceived(bool& retry, string& host) { /** If true, safe to call next(). Requests more from server if necessary. */ bool DBClientCursor::more() { - _assertIfNull(); - if (!_putBack.empty()) return true; @@ -343,7 +341,6 @@ bool DBClientCursor::more() { } BSONObj DBClientCursor::next() { - DEV _assertIfNull(); if (!_putBack.empty()) { BSONObj ret = _putBack.top(); _putBack.pop(); diff --git a/src/mongo/client/dbclientcursor.h b/src/mongo/client/dbclientcursor.h index a085d90dbd9..a0d9d5e00b5 100644 --- a/src/mongo/client/dbclientcursor.h +++ b/src/mongo/client/dbclientcursor.h @@ -70,7 +70,6 @@ public: then perhaps stop. */ int objsLeftInBatch() const { - _assertIfNull(); return _putBack.size() + batch.nReturned - batch.pos; } bool moreInCurrentBatch() { @@ -143,7 +142,6 @@ public: ResultFlag_ErrSet is the possible exception to that */ bool hasResultFlag(int flag) { - _assertIfNull(); return (resultFlags & flag) != 0; } @@ -290,11 +288,6 @@ private: void requestMore(); - // Don't call from a virtual function - void _assertIfNull() const { - uassert(13348, "connection died", this); - } - // init pieces void _assembleInit(Message& toSend); }; diff --git a/src/mongo/client/mongo_uri.cpp b/src/mongo/client/mongo_uri.cpp index 443a35f9858..cbf438441f0 100644 --- a/src/mongo/client/mongo_uri.cpp +++ b/src/mongo/client/mongo_uri.cpp @@ -56,14 +56,21 @@ const char kMongoDBURL[] = "(?:([^:]+)(?::([^@]+))?@)?" // servers: grabs all host:port or UNIX socket names - "((?:(?:[^\\/]+|/.+.sock?),?)+)" + "((?:[^\\/]+|/.+\\.sock)(?:(?:[^\\/]+|/.+\\.sock),)*)" - // database: matches anything but the chars that cannot - // be part of a MongoDB database name. - "(?:/([^/\\.\\ \"*<>:\\|\\?]*))?" + // database and options are grouped together + "(?:/" + + // database: matches anything but the chars that cannot be part of a MongoDB database name which + // are (in order) - forward slash, back slash, dot, space, double-quote, dollar sign, asterisk, + // less than, greater than, colon, pipe, question mark. + "([^/\\\\\\.\\ \"\\$*<>:\\|\\?]*)?" // options - "(?:\\?(?:(.+=.+)&?)+)*"; + "(?:\\?([^&=?]+=[^&=?]+(?:&[^&=?]+=[^&=?]+)*))?" + + // close db/options group + ")?"; } // namespace @@ -86,7 +93,7 @@ StatusWith MongoURI::parse(const std::string& url) { str::stream() << "Failed to parse mongodb:// URL: " << url); } - // We have 5 top level captures, plus the whole input. + // We have the whole input plus 5 top level captures (user, password, host, db, options). invariant(matches.size() == 6); if (!matches[3].matched) { diff --git a/src/mongo/client/mongo_uri_connect.cpp b/src/mongo/client/mongo_uri_connect.cpp index ff9c048b610..cdb4766fb69 100644 --- a/src/mongo/client/mongo_uri_connect.cpp +++ b/src/mongo/client/mongo_uri_connect.cpp @@ -162,15 +162,15 @@ BSONObj MongoURI::_makeAuthObjFromOptions(int maxWireVersion) const { } DBClientBase* MongoURI::connect(std::string& errmsg, double socketTimeout) const { - auto ret = _connectString.connect(errmsg, socketTimeout); + auto ret = std::unique_ptr(_connectString.connect(errmsg, socketTimeout)); if (!ret) { - return ret; + return nullptr; } if (!_user.empty()) { ret->auth(_makeAuthObjFromOptions(ret->getMaxWireVersion())); } - return ret; + return ret.release(); } } // namespace mongo diff --git a/src/mongo/client/mongo_uri_test.cpp b/src/mongo/client/mongo_uri_test.cpp index 89cf855ffed..7a3d5f9c812 100644 --- a/src/mongo/client/mongo_uri_test.cpp +++ b/src/mongo/client/mongo_uri_test.cpp @@ -61,6 +61,8 @@ const URITestCase validCases[] = { {"mongodb://127.0.0.1/dbName?foo=a&c=b", "", "", kMaster, "", 1, 2, "dbName"}, + {"mongodb://localhost/?foo=bar", "", "", kMaster, "", 1, 1, ""}, + {"mongodb://user:pwd@127.0.0.1:1234", "user", "pwd", kMaster, "", 1, 0, ""}, {"mongodb://user@127.0.0.1:1234", "user", "", kMaster, "", 1, 0, ""}, @@ -269,9 +271,20 @@ const URITestCase validCases[] = { const InvalidURITestCase invalidCases[] = { + // No host. {"mongodb://"}, + // Needs a "/" after the hosts and before the options. {"mongodb://localhost:27017,localhost:27018?replicaSet=missingSlash"}, + + // Host list must actually be comma separated. + {"mongodb://localhost:27017localhost:27018"}, + + // Domain sockets have to end in ".sock". + {"mongodb:///notareal/domainsock"}, + + // Options can't have multiple question marks. Only one. + {"mongodb://localhost:27017/?foo=a?c=b&d=e?asdf=foo"}, }; TEST(MongoURI, GoodTrickyURIs) { diff --git a/src/mongo/client/native_sasl_client_session.cpp b/src/mongo/client/native_sasl_client_session.cpp index 7976c09e413..8f8d4fe91af 100644 --- a/src/mongo/client/native_sasl_client_session.cpp +++ b/src/mongo/client/native_sasl_client_session.cpp @@ -33,6 +33,7 @@ #include "mongo/client/sasl_client_conversation.h" #include "mongo/client/sasl_plain_client_conversation.h" #include "mongo/client/sasl_scramsha1_client_conversation.h" +#include "mongo/client/scram_sha1_client_cache.h" #include "mongo/util/mongoutils/str.h" namespace mongo { @@ -47,6 +48,9 @@ MONGO_INITIALIZER(NativeSaslClientContext)(InitializerContext* context) { return Status::OK(); } +// Global cache for SCRAM-SHA-1 credentials +SCRAMSHA1ClientCache* scramsha1ClientCache = new SCRAMSHA1ClientCache; + } // namespace NativeSaslClientSession::NativeSaslClientSession() @@ -63,7 +67,7 @@ Status NativeSaslClientSession::initialize() { if (mechanism == "PLAIN") { _saslConversation.reset(new SaslPLAINClientConversation(this)); } else if (mechanism == "SCRAM-SHA-1") { - _saslConversation.reset(new SaslSCRAMSHA1ClientConversation(this)); + _saslConversation.reset(new SaslSCRAMSHA1ClientConversation(this, scramsha1ClientCache)); } else { return Status(ErrorCodes::BadValue, mongoutils::str::stream() << "SASL mechanism " << mechanism diff --git a/src/mongo/client/sasl_client_authenticate.cpp b/src/mongo/client/sasl_client_authenticate.cpp index ecb7dfd7d1e..1d305d44d05 100644 --- a/src/mongo/client/sasl_client_authenticate.cpp +++ b/src/mongo/client/sasl_client_authenticate.cpp @@ -39,7 +39,7 @@ namespace mongo { using namespace mongoutils; void (*saslClientAuthenticate)(auth::RunCommandHook runCommand, - StringData hostname, + const HostAndPort& hostname, const BSONObj& saslParameters, auth::AuthCompletionHandler handler) = nullptr; diff --git a/src/mongo/client/sasl_client_authenticate.h b/src/mongo/client/sasl_client_authenticate.h index 8e52bee407d..cde16cd3afe 100644 --- a/src/mongo/client/sasl_client_authenticate.h +++ b/src/mongo/client/sasl_client_authenticate.h @@ -68,7 +68,7 @@ class BSONObj; * returned. */ extern void (*saslClientAuthenticate)(auth::RunCommandHook runCommand, - StringData hostname, + const HostAndPort& hostname, const BSONObj& saslParameters, auth::AuthCompletionHandler handler); diff --git a/src/mongo/client/sasl_client_authenticate_impl.cpp b/src/mongo/client/sasl_client_authenticate_impl.cpp index aa4d7e64838..35b575e545a 100644 --- a/src/mongo/client/sasl_client_authenticate_impl.cpp +++ b/src/mongo/client/sasl_client_authenticate_impl.cpp @@ -116,7 +116,7 @@ Status extractPassword(const BSONObj& saslParameters, * Returns Status::OK() on success. */ Status configureSession(SaslClientSession* session, - StringData hostname, + const HostAndPort& hostname, StringData targetDatabase, const BSONObj& saslParameters) { std::string mechanism; @@ -134,10 +134,11 @@ Status configureSession(SaslClientSession* session, session->setParameter(SaslClientSession::parameterServiceName, value); status = bsonExtractStringFieldWithDefault( - saslParameters, saslCommandServiceHostnameFieldName, hostname, &value); + saslParameters, saslCommandServiceHostnameFieldName, hostname.host(), &value); if (!status.isOK()) return status; session->setParameter(SaslClientSession::parameterServiceHostname, value); + session->setParameter(SaslClientSession::parameterServiceHostAndPort, hostname.toString()); status = bsonExtractStringField(saslParameters, saslCommandUserFieldName, &value); if (!status.isOK()) @@ -247,7 +248,7 @@ void asyncSaslConversation(auth::RunCommandHook runCommand, * "client". */ void saslClientAuthenticateImpl(auth::RunCommandHook runCommand, - StringData hostname, + const HostAndPort& hostname, const BSONObj& saslParameters, auth::AuthCompletionHandler handler) { int saslLogLevel = getSaslClientLogLevel(saslParameters); diff --git a/src/mongo/client/sasl_client_session.h b/src/mongo/client/sasl_client_session.h index c93a843c9d7..3cda8fba89c 100644 --- a/src/mongo/client/sasl_client_session.h +++ b/src/mongo/client/sasl_client_session.h @@ -63,6 +63,7 @@ public: enum Parameter { parameterServiceName = 0, parameterServiceHostname, + parameterServiceHostAndPort, parameterMechanism, parameterUser, parameterPassword, diff --git a/src/mongo/client/sasl_scramsha1_client_conversation.cpp b/src/mongo/client/sasl_scramsha1_client_conversation.cpp index e7d8643668a..fb6fb1702ea 100644 --- a/src/mongo/client/sasl_scramsha1_client_conversation.cpp +++ b/src/mongo/client/sasl_scramsha1_client_conversation.cpp @@ -34,6 +34,7 @@ #include "mongo/base/parse_number.h" #include "mongo/client/sasl_client_session.h" +#include "mongo/client/scram_sha1_client_cache.h" #include "mongo/platform/random.h" #include "mongo/util/base64.h" #include "mongo/util/mongoutils/str.h" @@ -46,13 +47,12 @@ using std::unique_ptr; using std::string; SaslSCRAMSHA1ClientConversation::SaslSCRAMSHA1ClientConversation( - SaslClientSession* saslClientSession) - : SaslClientConversation(saslClientSession), _step(0), _authMessage(""), _clientNonce("") {} - -SaslSCRAMSHA1ClientConversation::~SaslSCRAMSHA1ClientConversation() { - // clear the _saltedPassword memory - memset(_saltedPassword, 0, scram::hashSize); -} + SaslClientSession* saslClientSession, SCRAMSHA1ClientCache* clientCache) + : SaslClientConversation(saslClientSession), + _step(0), + _authMessage(), + _clientCache(clientCache), + _clientNonce() {} StatusWith SaslSCRAMSHA1ClientConversation::step(StringData inputData, std::string* outputData) { @@ -159,7 +159,7 @@ StatusWith SaslSCRAMSHA1ClientConversation::_secondStep(const std::vector< } std::string salt = input[1].substr(2); - int iterationCount; + size_t iterationCount; Status status = parseNumberFromStringWithBase(input[2].substr(2), 10, &iterationCount); if (status != Status::OK()) { @@ -178,14 +178,28 @@ StatusWith SaslSCRAMSHA1ClientConversation::_secondStep(const std::vector< return StatusWith(ex.toStatus()); } - scram::generateSaltedPassword( - _saslClientSession->getParameter(SaslClientSession::parameterPassword), - reinterpret_cast(decodedSalt.c_str()), - decodedSalt.size(), - iterationCount, - _saltedPassword); + scram::SCRAMPresecrets presecrets( + _saslClientSession->getParameter(SaslClientSession::parameterPassword).toString(), + std::vector(decodedSalt.begin(), decodedSalt.end()), + iterationCount); + + StatusWith targetHost = HostAndPort::parse( + _saslClientSession->getParameter(SaslClientSession::parameterServiceHostAndPort)); + + if (targetHost.isOK()) { + _credentials = _clientCache->getCachedSecrets(targetHost.getValue(), presecrets); + + if (!_credentials) { + _credentials = scram::generateSecrets(presecrets); + + _clientCache->setCachedSecrets( + std::move(targetHost.getValue()), std::move(presecrets), _credentials); + } + } else { + _credentials = scram::generateSecrets(presecrets); + } - std::string clientProof = scram::generateClientProof(_saltedPassword, _authMessage); + std::string clientProof = scram::generateClientProof(_credentials, _authMessage); StringBuilder sb; sb << "c=biws,r=" << nonce << ",p=" << clientProof; @@ -224,7 +238,7 @@ StatusWith SaslSCRAMSHA1ClientConversation::_thirdStep(const std::vector. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/client/scram_sha1_client_cache.h" + +namespace mongo { + +scram::SCRAMSecrets SCRAMSHA1ClientCache::getCachedSecrets( + const HostAndPort& target, const scram::SCRAMPresecrets& presecrets) const { + const stdx::lock_guard lock(_hostToSecretsMutex); + + // Search the cache for a record associated with the host we're trying to connect to. + auto foundSecret = _hostToSecrets.find(target); + if (foundSecret != _hostToSecrets.end()) { + // Presecrets contain parameters provided by the server, which may change. If the + // cached presecrets don't match the presecrets we have on hand, we must not return the + // stale cached secrets. We'll need to rerun the SCRAM computation. + const scram::SCRAMPresecrets& foundPresecrets = foundSecret->second.first; + if (foundPresecrets == presecrets) { + return foundSecret->second.second; + } + } + return {}; +} + +void SCRAMSHA1ClientCache::setCachedSecrets(HostAndPort target, + scram::SCRAMPresecrets presecrets, + scram::SCRAMSecrets secrets) { + const stdx::lock_guard lock(_hostToSecretsMutex); + + decltype(_hostToSecrets)::iterator it; + bool insertionSuccessful; + auto cacheRecord = std::make_pair(std::move(presecrets), std::move(secrets)); + // Insert the presecrets, and the secrets we computed for them into the cache + std::tie(it, insertionSuccessful) = _hostToSecrets.emplace(std::move(target), cacheRecord); + // If there was already a cache entry for the target HostAndPort, we should overwrite it. + // We have fresher presecrets and secrets. + if (!insertionSuccessful) { + it->second = std::move(cacheRecord); + } +} + +} // namespace mongo diff --git a/src/mongo/client/scram_sha1_client_cache.h b/src/mongo/client/scram_sha1_client_cache.h new file mode 100644 index 00000000000..ee595094ec5 --- /dev/null +++ b/src/mongo/client/scram_sha1_client_cache.h @@ -0,0 +1,88 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ +#pragma once + +#include +#include + +#include "mongo/crypto/mechanism_scram.h" +#include "mongo/platform/unordered_map.h" +#include "mongo/stdx/mutex.h" +#include "mongo/util/net/hostandport.h" + +namespace mongo { + +/** + * A cache for the intermediate steps of the SCRAM-SHA-1 computation. + * + * Clients wishing to authenticate to a server using SCRAM-SHA-1 + * must produce a set of credential objects from their password, + * a salt, and an iteration count. The computation to generate these + * is very expensive, proportional to the iteration count. The high + * cost of this computation prevents brute force attacks on + * intercepted SCRAM authentication data, or a stolen password + * database. The inputs to the function are unlikely to frequently + * change. Caching the relationship between the inputs and the + * resulting output should make repeated authentication attempts + * to a single server much faster. + * + * This is explicitly permitted by RFC5802, section 5.1: + * + * "Note that a client implementation MAY cache + * ClientKey&ServerKey (or just SaltedPassword) for later + * reauthentication to the same service, as it is likely that the + * server is going to advertise the same salt value upon + * reauthentication. This might be useful for mobile clients where + * CPU usage is a concern." + */ +class SCRAMSHA1ClientCache { +public: + /** + * Returns precomputed SCRAMSecrets, if one has already been + * stored for the specified hostname and the provided presecrets + * match those recorded for the hostname. Otherwise, no secrets + * are returned. + */ + scram::SCRAMSecrets getCachedSecrets(const HostAndPort& target, + const scram::SCRAMPresecrets& presecrets) const; + + /** + * Records a set of precomputed SCRAMSecrets for the specified + * host, along with the presecrets used to generate them. + */ + void setCachedSecrets(HostAndPort target, + scram::SCRAMPresecrets presecrets, + scram::SCRAMSecrets secrets); + +private: + mutable stdx::mutex _hostToSecretsMutex; + std::unordered_map> + _hostToSecrets; +}; + +} // namespace mongo diff --git a/src/mongo/client/syncclusterconnection.cpp b/src/mongo/client/syncclusterconnection.cpp index c15cf186267..a6b1662d4f0 100644 --- a/src/mongo/client/syncclusterconnection.cpp +++ b/src/mongo/client/syncclusterconnection.cpp @@ -276,9 +276,7 @@ BSONObj SyncClusterConnection::findOne(const string& ns, const BSONObj* fieldsToReturn, int queryOptions) { if (ns.find(".$cmd") != string::npos) { - string cmdName = query.obj.firstElementFieldName(); - - int lockType = _lockType(cmdName); + int lockType = _lockType(query); if (lockType > 0) { // write $cmd string errmsg; @@ -381,10 +379,10 @@ unique_ptr SyncClusterConnection::query(const string& ns, int batchSize) { _lastErrors.clear(); if (ns.find(".$cmd") != string::npos) { - string cmdName = query.obj.firstElementFieldName(); - int lockType = _lockType(cmdName); + int lockType = _lockType(query); uassert(13054, - (string) "write $cmd not supported in SyncClusterConnection::query for:" + cmdName, + str::stream() << "write $cmd not supported in SyncClusterConnection::query for: " + << query.toString(), lockType <= 0); } @@ -622,7 +620,14 @@ void SyncClusterConnection::say(Message& toSend, bool isRetry, string* actualSer _checkLast(); } -int SyncClusterConnection::_lockType(const string& name) { +int SyncClusterConnection::_lockType(const Query& query) { + string name = query.obj.firstElementFieldName(); + + if (name == "query") { + // actual command is embedded in the query object + name = query.obj["query"].Obj().firstElementFieldName(); + } + { stdx::lock_guard lk(_mutex); map::iterator i = _lockTypes.find(name); diff --git a/src/mongo/client/syncclusterconnection.h b/src/mongo/client/syncclusterconnection.h index 40373d4ae19..06fba3d0be2 100644 --- a/src/mongo/client/syncclusterconnection.h +++ b/src/mongo/client/syncclusterconnection.h @@ -198,7 +198,7 @@ private: const BSONObj* fieldsToReturn, int queryOptions, int batchSize); - int _lockType(const std::string& name); + int _lockType(const Query& query); void _checkLast(); void _connect(const std::string& host); diff --git a/src/mongo/crypto/SConscript b/src/mongo/crypto/SConscript index f66e88be2b3..5b8ed85a684 100644 --- a/src/mongo/crypto/SConscript +++ b/src/mongo/crypto/SConscript @@ -8,18 +8,41 @@ env.SConscript( ], ) -env.Library('crypto_tom', - ['crypto_tom.cpp'], - LIBDEPS=['tom/tomcrypt']) +env.Library('sha1_block', + source=[ + 'sha1_block.cpp' + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + ]) + +env.Library( + target='sha1_block_tom', + source=[ + 'sha1_block_tom.cpp' + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + 'sha1_block', + 'tom/tomcrypt' + ]) -env.Library('crypto_openssl', - ['crypto_openssl.cpp']) +env.Library('sha1_block_openssl', + source=[ + 'sha1_block_openssl.cpp' + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + 'sha1_block', + ]) env.Library('scramauth', ['mechanism_scram.cpp'], LIBDEPS=['$BUILD_DIR/mongo/base', - 'crypto_${MONGO_CRYPTO}']) + '$BUILD_DIR/mongo/base/secure_allocator', + '$BUILD_DIR/mongo/util/secure_zero_memory', + 'sha1_block_${MONGO_CRYPTO}']) -env.CppUnitTest('crypto_test', - ['crypto_test.cpp'], - LIBDEPS=['crypto_${MONGO_CRYPTO}']) +env.CppUnitTest('sha1_block_test', + ['sha1_block_test.cpp'], + LIBDEPS=['sha1_block_${MONGO_CRYPTO}']) diff --git a/src/mongo/crypto/crypto.h b/src/mongo/crypto/crypto.h deleted file mode 100644 index 7b08e371a84..00000000000 --- a/src/mongo/crypto/crypto.h +++ /dev/null @@ -1,51 +0,0 @@ -/** -* Copyright (C) 2014 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -* -* As a special exception, the copyright holders give permission to link the -* code of portions of this program with the OpenSSL library under certain -* conditions as described in each individual source file and distribute -* linked combinations including the program with the OpenSSL library. You -* must comply with the GNU Affero General Public License in all respects for -* all of the code used other than as permitted herein. If you modify file(s) -* with this exception, you may extend this exception to your version of the -* file(s), but you are not obligated to do so. If you do not wish to do so, -* delete this exception statement from your version. If you delete this -* exception statement from all source files in the program, then also delete -* it in the license file. -*/ - -#pragma once - -#include - -namespace mongo { -namespace crypto { -/* - * Computes a SHA-1 hash of 'input'. - */ -bool sha1(const unsigned char* input, const size_t inputLen, unsigned char* output); - -/* - * Computes a HMAC SHA-1 keyed hash of 'input' using the key 'key' - */ -bool hmacSha1(const unsigned char* key, - const size_t keyLen, - const unsigned char* input, - const size_t inputLen, - unsigned char* output, - unsigned int* outputLen); - -} // namespace crypto -} // namespace mongo diff --git a/src/mongo/crypto/crypto_openssl.cpp b/src/mongo/crypto/crypto_openssl.cpp deleted file mode 100644 index 4dc1e5d02c4..00000000000 --- a/src/mongo/crypto/crypto_openssl.cpp +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (C) 2014 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/platform/basic.h" - -#include "mongo/config.h" -#include "mongo/util/scopeguard.h" - -#ifndef MONGO_CONFIG_SSL -#error This file should only be included in SSL-enabled builds -#endif - -#include -#include -#include - -namespace mongo { -namespace crypto { -/* - * Computes a SHA-1 hash of 'input'. - */ -bool sha1(const unsigned char* input, const size_t inputLen, unsigned char* output) { - EVP_MD_CTX digestCtx; - EVP_MD_CTX_init(&digestCtx); - ON_BLOCK_EXIT(EVP_MD_CTX_cleanup, &digestCtx); - - if (1 != EVP_DigestInit_ex(&digestCtx, EVP_sha1(), NULL)) { - return false; - } - - if (1 != EVP_DigestUpdate(&digestCtx, input, inputLen)) { - return false; - } - - return (1 == EVP_DigestFinal_ex(&digestCtx, output, NULL)); -} - -/* - * Computes a HMAC SHA-1 keyed hash of 'input' using the key 'key' - */ -bool hmacSha1(const unsigned char* key, - const size_t keyLen, - const unsigned char* input, - const size_t inputLen, - unsigned char* output, - unsigned int* outputLen) { - return HMAC(EVP_sha1(), key, keyLen, input, inputLen, output, outputLen); -} - -} // namespace crypto -} // namespace mongo diff --git a/src/mongo/crypto/crypto_test.cpp b/src/mongo/crypto/crypto_test.cpp deleted file mode 100644 index 52d488b92cf..00000000000 --- a/src/mongo/crypto/crypto_test.cpp +++ /dev/null @@ -1,521 +0,0 @@ -/** - * Copyright (C) 2014 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/crypto/crypto.h" -#include "mongo/unittest/unittest.h" - -namespace mongo { -namespace { -const int digestLen = 20; - -// SHA-1 test vectors from http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA_All.pdf -const struct { - const char* msg; - unsigned char hash[digestLen]; -} sha1Tests[] = {{"abc", - {0xa9, - 0x99, - 0x3e, - 0x36, - 0x47, - 0x06, - 0x81, - 0x6a, - 0xba, - 0x3e, - 0x25, - 0x71, - 0x78, - 0x50, - 0xc2, - 0x6c, - 0x9c, - 0xd0, - 0xd8, - 0x9d}}, - - {"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", - {0x84, - 0x98, - 0x3E, - 0x44, - 0x1C, - 0x3B, - 0xD2, - 0x6E, - 0xBA, - 0xAE, - 0x4A, - 0xA1, - 0xF9, - 0x51, - 0x29, - 0xE5, - 0xE5, - 0x46, - 0x70, - 0xF1}}}; - -TEST(CryptoVectors, SHA1) { - unsigned char sha1Result[digestLen]; - size_t numTests = sizeof(sha1Tests) / sizeof(sha1Tests[0]); - for (size_t i = 0; i < numTests; i++) { - ASSERT(crypto::sha1(reinterpret_cast(sha1Tests[i].msg), - strlen(sha1Tests[i].msg), - sha1Result)) - << "Failed SHA1 iteration " << i; - ASSERT(0 == memcmp(sha1Tests[i].hash, sha1Result, digestLen)) << "Failed SHA1 iteration " - << i; - } -} - -const int maxKeySize = 80; -const int maxDataSize = 54; -// HMAC-SHA-1 test vectors from http://tools.ietf.org/html/rfc2202.html -const struct { - unsigned char key[maxKeySize]; - int keyLen; - unsigned char data[maxDataSize]; - int dataLen; - unsigned char hash[digestLen]; -} hmacSha1Tests[] = { - // RFC test case 1 - {{0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b, - 0x0b}, - 20, - {0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, - 8, - {0xb6, - 0x17, - 0x31, - 0x86, - 0x55, - 0x05, - 0x72, - 0x64, - 0xe2, - 0x8b, - 0xc0, - 0xb6, - 0xfb, - 0x37, - 0x8c, - 0x8e, - 0xf1, - 0x46, - 0xbe, - 0x00}}, - - // RFC test case 3 - {{0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa}, - 20, - {0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd, - 0xdd}, - 50, - {0x12, - 0x5d, - 0x73, - 0x42, - 0xb9, - 0xac, - 0x11, - 0xcd, - 0x91, - 0xa3, - 0x9a, - 0xf4, - 0x8a, - 0xa1, - 0x7b, - 0x4f, - 0x63, - 0xf1, - 0x75, - 0xd3}}, - - // RFC test case 4 - {{0x01, - 0x02, - 0x03, - 0x04, - 0x05, - 0x06, - 0x07, - 0x08, - 0x09, - 0x0a, - 0x0b, - 0x0c, - 0x0d, - 0x0e, - 0x0f, - 0x10, - 0x11, - 0x12, - 0x13, - 0x14, - 0x15, - 0x16, - 0x17, - 0x18, - 0x19}, - 25, - {0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd, - 0xcd}, - 50, - {0x4c, - 0x90, - 0x07, - 0xf4, - 0x02, - 0x62, - 0x50, - 0xc6, - 0xbc, - 0x84, - 0x14, - 0xf9, - 0xbf, - 0x50, - 0xc8, - 0x6c, - 0x2d, - 0x72, - 0x35, - 0xda}}, - - // RFC test case 6 - {{0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa, - 0xaa}, - 80, - {0x54, - 0x65, - 0x73, - 0x74, - 0x20, - 0x55, - 0x73, - 0x69, - 0x6e, - 0x67, - 0x20, - 0x4c, - 0x61, - 0x72, - 0x67, - 0x65, - 0x72, - 0x20, - 0x54, - 0x68, - 0x61, - 0x6e, - 0x20, - 0x42, - 0x6c, - 0x6f, - 0x63, - 0x6b, - 0x2d, - 0x53, - 0x69, - 0x7a, - 0x65, - 0x20, - 0x4b, - 0x65, - 0x79, - 0x20, - 0x2d, - 0x20, - 0x48, - 0x61, - 0x73, - 0x68, - 0x20, - 0x4b, - 0x65, - 0x79, - 0x20, - 0x46, - 0x69, - 0x72, - 0x73, - 0x74}, - 54, - {0xaa, - 0x4a, - 0xe5, - 0xe1, - 0x52, - 0x72, - 0xd0, - 0x0e, - 0x95, - 0x70, - 0x56, - 0x37, - 0xce, - 0x8a, - 0x3b, - 0x55, - 0xed, - 0x40, - 0x21, - 0x12}}}; - -TEST(CryptoVectors, HMACSHA1) { - unsigned char hmacSha1Result[digestLen]; - unsigned int hashLen = digestLen; - - size_t numTests = sizeof(hmacSha1Tests) / sizeof(hmacSha1Tests[0]); - for (size_t i = 0; i < numTests; i++) { - ASSERT(crypto::hmacSha1(hmacSha1Tests[i].key, - hmacSha1Tests[i].keyLen, - hmacSha1Tests[i].data, - hmacSha1Tests[i].dataLen, - hmacSha1Result, - &hashLen)) - << "Failed HMAC-SHA1 iteration " << i; - ASSERT(0 == memcmp(hmacSha1Tests[i].hash, hmacSha1Result, digestLen)) - << "Failed HMAC-SHA1 iteration " << i; - } -} -} // namespace -} // namespace mongo diff --git a/src/mongo/crypto/crypto_tom.cpp b/src/mongo/crypto/crypto_tom.cpp deleted file mode 100644 index a629dff945d..00000000000 --- a/src/mongo/crypto/crypto_tom.cpp +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (C) 2014 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/platform/basic.h" - -#include "mongo/config.h" - -#ifdef MONGO_CONFIG_SSL -#error This file should not be included if compiling with SSL support -#endif - -#include "mongo/crypto/tom/tomcrypt.h" - -namespace mongo { -namespace crypto { -/* - * Computes a SHA-1 hash of 'input'. - */ -bool sha1(const unsigned char* input, const size_t inputLen, unsigned char* output) { - hash_state hashState; - if (sha1_init(&hashState) != CRYPT_OK) { - return false; - } - if (sha1_process(&hashState, input, inputLen) != CRYPT_OK) { - return false; - } - if (sha1_done(&hashState, output) != CRYPT_OK) { - return false; - } - - return true; -} - -/* - * Computes a HMAC SHA-1 keyed hash of 'input' using the key 'key' - */ -bool hmacSha1(const unsigned char* key, - const size_t keyLen, - const unsigned char* input, - const size_t inputLen, - unsigned char* output, - unsigned int* outputLen) { - if (!key || !input || !output) { - return false; - } - - static int hashId = -1; - if (hashId == -1) { - register_hash(&sha1_desc); - hashId = find_hash("sha1"); - } - - unsigned long sha1HashLen = 20; - if (hmac_memory(hashId, key, keyLen, input, inputLen, output, &sha1HashLen) != CRYPT_OK) { - return false; - } - - *outputLen = sha1HashLen; - return true; -} - -} // namespace crypto -} // namespace mongo diff --git a/src/mongo/crypto/mechanism_scram.cpp b/src/mongo/crypto/mechanism_scram.cpp index 086e9943c4c..c18bc39cc51 100644 --- a/src/mongo/crypto/mechanism_scram.cpp +++ b/src/mongo/crypto/mechanism_scram.cpp @@ -32,29 +32,54 @@ #include -#include "mongo/crypto/crypto.h" #include "mongo/platform/random.h" #include "mongo/util/base64.h" +#include "mongo/util/secure_zero_memory.h" namespace mongo { namespace scram { using std::unique_ptr; +namespace { +/** + * Compare two arrays of bytes for equality in constant time. + * + * This means that the function runs for the same amount of time even if they differ. Unlike memcmp, + * this function does not exit on the first difference. + * + * Returns true if the two arrays are equal. + * + * TODO: evaluate if LTO inlines or changes the code flow of this function. + */ +NOINLINE_DECL +bool consttimeMemEqual(volatile const unsigned char* s1, // NOLINT - using volatile to + volatile const unsigned char* s2, // NOLINT - disable compiler optimizations + size_t length) { + unsigned int ret = 0; + + for (size_t i = 0; i < length; ++i) { + ret |= s1[i] ^ s2[i]; + } + + return (1 & ((ret - 1) >> 8)); +} +} // namespace + // Compute the SCRAM step Hi() as defined in RFC5802 -static void HMACIteration(const unsigned char input[], - size_t inputLen, - const unsigned char salt[], - size_t saltLen, - unsigned int iterationCount, - unsigned char output[]) { - unsigned char intermediateDigest[hashSize]; - unsigned char startKey[hashSize]; - // Placeholder for HMAC return size, will always be scram::hashSize for HMAC SHA-1 - unsigned int hashLen = 0; - - uassert(17450, "invalid salt length provided", saltLen + 4 == hashSize); - memcpy(startKey, salt, saltLen); +static SHA1Block HMACIteration(const unsigned char input[], + size_t inputLen, + const unsigned char salt[], + size_t saltLen, + unsigned int iterationCount) { + SHA1Block output; + SHA1Block intermediateDigest; + // Reserve a 20 byte block for the initial key. We use 16 byte salts, and must reserve an extra + // 4 bytes for a suffix mandated by RFC5802. + std::array startKey; + + uassert(17450, "invalid salt length provided", saltLen + 4 == startKey.size()); + std::copy(salt, salt + saltLen, startKey.begin()); startKey[saltLen] = 0; startKey[saltLen + 1] = 0; @@ -62,72 +87,62 @@ static void HMACIteration(const unsigned char input[], startKey[saltLen + 3] = 1; // U1 = HMAC(input, salt + 0001) - fassert(17494, crypto::hmacSha1(input, inputLen, startKey, saltLen + 4, output, &hashLen)); - - memcpy(intermediateDigest, output, hashSize); + output = SHA1Block::computeHmac(input, inputLen, startKey.data(), startKey.size()); + intermediateDigest = output; // intermediateDigest contains Ui and output contains the accumulated XOR:ed result for (size_t i = 2; i <= iterationCount; i++) { - unsigned char intermediateOutput[hashSize]; - fassert(17495, - crypto::hmacSha1( - input, inputLen, intermediateDigest, hashSize, intermediateOutput, &hashLen)); - memcpy(intermediateDigest, intermediateOutput, hashSize); - for (size_t k = 0; k < hashSize; k++) { - output[k] ^= intermediateDigest[k]; - } + intermediateDigest = SHA1Block::computeHmac( + input, inputLen, intermediateDigest.data(), intermediateDigest.size()); + output.xorInline(intermediateDigest); } + + return output; } // Iterate the hash function to generate SaltedPassword -void generateSaltedPassword(StringData hashedPassword, - const unsigned char* salt, - const int saltLen, - const int iterationCount, - unsigned char saltedPassword[hashSize]) { +SHA1Block generateSaltedPassword(const SCRAMPresecrets& presecrets) { // saltedPassword = Hi(hashedPassword, salt) - HMACIteration(reinterpret_cast(hashedPassword.rawData()), - hashedPassword.size(), - salt, - saltLen, - iterationCount, - saltedPassword); + SHA1Block saltedPassword = + HMACIteration(reinterpret_cast(presecrets.hashedPassword.c_str()), + presecrets.hashedPassword.size(), + presecrets.salt.data(), + presecrets.salt.size(), + presecrets.iterationCount); + + return saltedPassword; +} + +SCRAMSecrets generateSecrets(const SCRAMPresecrets& presecrets) { + SHA1Block saltedPassword = generateSaltedPassword(presecrets); + return generateSecrets(saltedPassword); } -void generateSecrets(const std::string& hashedPassword, - const unsigned char salt[], - size_t saltLen, - size_t iterationCount, - unsigned char storedKey[hashSize], - unsigned char serverKey[hashSize]) { - unsigned char saltedPassword[hashSize]; - unsigned char clientKey[hashSize]; - unsigned int hashLen = 0; - - generateSaltedPassword(hashedPassword, salt, saltLen, iterationCount, saltedPassword); - - // clientKey = HMAC(saltedPassword, "Client Key") - fassert(17498, - crypto::hmacSha1(saltedPassword, - hashSize, - reinterpret_cast(clientKeyConst.data()), - clientKeyConst.size(), - clientKey, - &hashLen)); - - // storedKey = H(clientKey) - fassert(17499, crypto::sha1(clientKey, hashSize, storedKey)); - - // serverKey = HMAC(saltedPassword, "Server Key") - fassert(17500, - crypto::hmacSha1(saltedPassword, - hashSize, - reinterpret_cast(serverKeyConst.data()), - serverKeyConst.size(), - serverKey, - &hashLen)); +SCRAMSecrets generateSecrets(const SHA1Block& saltedPassword) { + auto generateAndStoreSecrets = [&saltedPassword]( + SHA1Block& clientKey, SHA1Block& storedKey, SHA1Block& serverKey) { + + // ClientKey := HMAC(saltedPassword, "Client Key") + clientKey = + SHA1Block::computeHmac(saltedPassword.data(), + saltedPassword.size(), + reinterpret_cast(clientKeyConst.data()), + clientKeyConst.size()); + + // StoredKey := H(clientKey) + storedKey = SHA1Block::computeHash(clientKey.data(), clientKey.size()); + + // ServerKey := HMAC(SaltedPassword, "Server Key") + serverKey = + SHA1Block::computeHmac(saltedPassword.data(), + saltedPassword.size(), + reinterpret_cast(serverKeyConst.data()), + serverKeyConst.size()); + }; + return SCRAMSecrets(std::move(generateAndStoreSecrets)); } + BSONObj generateCredentials(const std::string& hashedPassword, int iterationCount) { const int saltLenQWords = 2; @@ -142,115 +157,76 @@ BSONObj generateCredentials(const std::string& hashedPassword, int iterationCoun base64::encode(reinterpret_cast(userSalt), sizeof(userSalt)); // Compute SCRAM secrets serverKey and storedKey - unsigned char storedKey[hashSize]; - unsigned char serverKey[hashSize]; - - generateSecrets(hashedPassword, - reinterpret_cast(userSalt), - saltLenQWords * sizeof(uint64_t), - iterationCount, - storedKey, - serverKey); + auto secrets = generateSecrets( + SCRAMPresecrets(hashedPassword, + std::vector(reinterpret_cast(userSalt), + reinterpret_cast(userSalt) + + saltLenQWords * sizeof(uint64_t)), + iterationCount)); - std::string encodedStoredKey = base64::encode(reinterpret_cast(storedKey), hashSize); - std::string encodedServerKey = base64::encode(reinterpret_cast(serverKey), hashSize); + std::string encodedStoredKey = secrets->storedKey.toString(); + std::string encodedServerKey = secrets->serverKey.toString(); return BSON(iterationCountFieldName << iterationCount << saltFieldName << encodedUserSalt << storedKeyFieldName << encodedStoredKey << serverKeyFieldName << encodedServerKey); } -std::string generateClientProof(const unsigned char saltedPassword[hashSize], +std::string generateClientProof(const SCRAMSecrets& clientCredentials, const std::string& authMessage) { - // ClientKey := HMAC(saltedPassword, "Client Key") - unsigned char clientKey[hashSize]; - unsigned int hashLen = 0; - fassert(18689, - crypto::hmacSha1(saltedPassword, - hashSize, - reinterpret_cast(clientKeyConst.data()), - clientKeyConst.size(), - clientKey, - &hashLen)); - - // StoredKey := H(clientKey) - unsigned char storedKey[hashSize]; - fassert(18701, crypto::sha1(clientKey, hashSize, storedKey)); - // ClientSignature := HMAC(StoredKey, AuthMessage) - unsigned char clientSignature[hashSize]; - fassert(18702, - crypto::hmacSha1(storedKey, - hashSize, - reinterpret_cast(authMessage.c_str()), - authMessage.size(), - clientSignature, - &hashLen)); - - // ClientProof := ClientKey XOR ClientSignature - unsigned char clientProof[hashSize]; - for (size_t i = 0; i < hashSize; i++) { - clientProof[i] = clientKey[i] ^ clientSignature[i]; - } - - return base64::encode(reinterpret_cast(clientProof), hashSize); + SHA1Block clientSignature = + SHA1Block::computeHmac(clientCredentials->storedKey.data(), + clientCredentials->storedKey.size(), + reinterpret_cast(authMessage.c_str()), + authMessage.size()); + + clientSignature.xorInline(clientCredentials->clientKey); + return clientSignature.toString(); } -/** - * Compare two arrays of bytes for equality in constant time. - * - * This means that the function runs for the same amount of time even if they differ. Unlike memcmp, - * this function does not exit on the first difference. - * - * Returns true if the two arrays are equal. - * - * TODO: evaluate if LTO inlines or changes the code flow of this function. - */ -NOINLINE_DECL -bool memequal(volatile const unsigned char* s1, volatile const unsigned char* s2, size_t length) { - unsigned char ret = 0; - - for (size_t i = 0; i < length; ++i) { - ret |= s1[i] ^ s2[i]; - } - - return ret == 0; -} - -bool verifyServerSignature(const unsigned char saltedPassword[hashSize], +bool verifyServerSignature(const SCRAMSecrets& clientCredentials, const std::string& authMessage, const std::string& receivedServerSignature) { - // ServerKey := HMAC(SaltedPassword, "Server Key") - unsigned int hashLen; - unsigned char serverKey[hashSize]; - fassert(18703, - crypto::hmacSha1(saltedPassword, - hashSize, - reinterpret_cast(serverKeyConst.data()), - serverKeyConst.size(), - serverKey, - &hashLen)); - // ServerSignature := HMAC(ServerKey, AuthMessage) - unsigned char serverSignature[hashSize]; - fassert(18704, - crypto::hmacSha1(serverKey, - hashSize, - reinterpret_cast(authMessage.c_str()), - authMessage.size(), - serverSignature, - &hashLen)); - - std::string encodedServerSignature = - base64::encode(reinterpret_cast(serverSignature), sizeof(serverSignature)); + SHA1Block serverSignature = + SHA1Block::computeHmac(clientCredentials->serverKey.data(), + clientCredentials->serverKey.size(), + reinterpret_cast(authMessage.c_str()), + authMessage.size()); + + std::string encodedServerSignature = serverSignature.toString(); if (encodedServerSignature.size() != receivedServerSignature.size()) { return false; } - return memequal(reinterpret_cast(encodedServerSignature.c_str()), - reinterpret_cast(receivedServerSignature.c_str()), - encodedServerSignature.size()); + return consttimeMemEqual( + reinterpret_cast(encodedServerSignature.c_str()), + reinterpret_cast(receivedServerSignature.c_str()), + encodedServerSignature.size()); +} + +bool verifyClientProof(StringData clientProof, StringData storedKey, StringData authMessage) { + // ClientSignature := HMAC(StoredKey, AuthMessage) + SHA1Block clientSignature = + SHA1Block::computeHmac(reinterpret_cast(storedKey.rawData()), + storedKey.size(), + reinterpret_cast(authMessage.rawData()), + authMessage.size()); + + auto clientProofSHA1Status = SHA1Block::fromBuffer( + reinterpret_cast(clientProof.rawData()), clientProof.size()); + uassertStatusOK(clientProofSHA1Status); + clientSignature.xorInline(clientProofSHA1Status.getValue()); + + // StoredKey := H(clientSignature) + SHA1Block computedStoredKey = + SHA1Block::computeHash(clientSignature.data(), clientSignature.size()); + + return consttimeMemEqual(reinterpret_cast(storedKey.rawData()), + computedStoredKey.data(), + computedStoredKey.size()); } } // namespace scram diff --git a/src/mongo/crypto/mechanism_scram.h b/src/mongo/crypto/mechanism_scram.h index 65e77b38f74..92b79efffb9 100644 --- a/src/mongo/crypto/mechanism_scram.h +++ b/src/mongo/crypto/mechanism_scram.h @@ -30,12 +30,13 @@ #include +#include "mongo/base/secure_allocator.h" #include "mongo/base/status.h" +#include "mongo/crypto/sha1_block.h" #include "mongo/db/jsobj.h" namespace mongo { namespace scram { -const unsigned int hashSize = 20; const std::string serverKeyConst = "Server Key"; const std::string clientKeyConst = "Client Key"; @@ -45,25 +46,90 @@ const std::string saltFieldName = "salt"; const std::string storedKeyFieldName = "storedKey"; const std::string serverKeyFieldName = "serverKey"; +/* + * The precursors necessary to perform the computation which produces SCRAMSecrets. + * These are the original password, its salt, and the number of times it must be + * hashed to produce the SaltedPassword used to generate the rest of the SCRAMSecrets. + */ +struct SCRAMPresecrets { + SCRAMPresecrets(std::string hashedPassword, + std::vector salt, + size_t iterationCount) + : hashedPassword(std::move(hashedPassword)), + salt(std::move(salt)), + iterationCount(iterationCount) {} + + std::string hashedPassword; + std::vector salt; + size_t iterationCount; +}; + +inline bool operator==(const SCRAMPresecrets& lhs, const SCRAMPresecrets& rhs) { + return lhs.hashedPassword == rhs.hashedPassword && lhs.salt == rhs.salt && + lhs.iterationCount == rhs.iterationCount; +} + /* * Computes the SaltedPassword from password, salt and iterationCount. */ -void generateSaltedPassword(StringData hashedPassword, - const unsigned char* salt, - const int saltLen, - const int iterationCount, - unsigned char saltedPassword[hashSize]); +SHA1Block generateSaltedPassword(const SCRAMPresecrets& presecrets); /* - * Computes the SCRAM secrets storedKey and serverKey using the salt 'salt' + * Stores all of the keys, generated from a password, needed for a client or server to perform a + * SCRAM handshake. + * These keys are reference counted, and allocated using the SecureAllocator. + * May be unpopulated. SCRAMSecrets created via the default constructor are unpopulated. + * The behavior is undefined if the accessors are called when unpopulated. + */ +class SCRAMSecrets { +private: + struct SCRAMSecretsHolder { + SHA1Block clientKey; + SHA1Block storedKey; + SHA1Block serverKey; + }; + +public: + // Creates an unpopulated SCRAMSecrets object. + SCRAMSecrets() = default; + + // Creates a populated SCRAMSecrets object. First, allocates secure storage, then provides it + // to a callback, which fills the memory. + template + explicit SCRAMSecrets(T initializationFun) + : _ptr(std::make_shared>()) { + initializationFun((*this)->clientKey, (*this)->storedKey, (*this)->serverKey); + } + + // Returns true if the underlying shared_pointer is populated. + explicit operator bool() const { + return static_cast(_ptr); + } + + const SecureHandle& operator*() const { + invariant(_ptr); + return *_ptr; + } + + const SecureHandle& operator->() const { + invariant(_ptr); + return *_ptr; + } + +private: + std::shared_ptr> _ptr; +}; + +/* + * Computes the SCRAM secrets clientKey, storedKey, and serverKey using the salt 'salt' * and iteration count 'iterationCount' as defined in RFC5802 (server side). */ -void generateSecrets(const std::string& hashedPassword, - const unsigned char salt[], - size_t saltLen, - size_t iterationCount, - unsigned char storedKey[hashSize], - unsigned char serverKey[hashSize]); +SCRAMSecrets generateSecrets(const SCRAMPresecrets& presecrets); + +/* + * Computes the ClientKey and StoredKey from SaltedPassword (client side). + */ +SCRAMSecrets generateSecrets(const SHA1Block& saltedPassword); /* * Generates the user salt and the SCRAM secrets storedKey and serverKey as @@ -72,9 +138,9 @@ void generateSecrets(const std::string& hashedPassword, BSONObj generateCredentials(const std::string& hashedPassword, int iterationCount); /* - * Computes the ClientProof from SaltedPassword and authMessage (client side). + * Computes the ClientProof from ClientKey, StoredKey, and authMessage (client side). */ -std::string generateClientProof(const unsigned char saltedPassword[hashSize], +std::string generateClientProof(const SCRAMSecrets& clientCredentials, const std::string& authMessage); /* @@ -89,8 +155,14 @@ bool validatePassword(const std::string& hashedPassword, /* * Verifies ServerSignature (client side). */ -bool verifyServerSignature(const unsigned char saltedPassword[hashSize], +bool verifyServerSignature(const SCRAMSecrets& clientCredentials, const std::string& authMessage, const std::string& serverSignature); + +/* + * Verifies ClientProof (server side). + */ +bool verifyClientProof(StringData clientProof, StringData storedKey, StringData authMessage); + } // namespace scram } // namespace mongo diff --git a/src/mongo/crypto/sha1_block.cpp b/src/mongo/crypto/sha1_block.cpp new file mode 100644 index 00000000000..43565419a7d --- /dev/null +++ b/src/mongo/crypto/sha1_block.cpp @@ -0,0 +1,88 @@ +/** + * Copyright (C) 2017 MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/crypto/sha1_block.h" + +#include "mongo/bson/bsonmisc.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/util/base64.h" +#include "mongo/util/mongoutils/str.h" + +namespace mongo { + +const size_t SHA1Block::kHashLength = 20; + +SHA1Block::SHA1Block(HashType hash) : _hash(std::move(hash)) {} + +StatusWith SHA1Block::fromBuffer(const uint8_t* input, size_t inputLen) { + if (inputLen != kHashLength) { + return {ErrorCodes::InvalidLength, + str::stream() << "Unsupported SHA1Hash hash length: " << inputLen}; + } + + HashType newHash; + memcpy(newHash.data(), input, inputLen); + return SHA1Block(newHash); +} + +StatusWith SHA1Block::fromBinData(const BSONBinData& binData) { + if (binData.type != BinDataGeneral) { + return {ErrorCodes::UnsupportedFormat, "SHA1Block only accepts BinDataGeneral type"}; + } + + if (binData.length != kHashLength) { + return {ErrorCodes::UnsupportedFormat, + str::stream() << "Unsupported SHA1Block hash length: " << binData.length}; + } + + HashType newHash; + memcpy(newHash.data(), binData.data, binData.length); + return SHA1Block(newHash); +} + +std::string SHA1Block::toString() const { + return base64::encode(reinterpret_cast(_hash.data()), _hash.size()); +} + +void SHA1Block::appendAsBinData(BSONObjBuilder& builder, StringData fieldName) { + builder.appendBinData(fieldName, _hash.size(), BinDataGeneral, _hash.data()); +} + +void SHA1Block::xorInline(const SHA1Block& other) { + for (size_t x = 0; x < _hash.size(); x++) { + _hash[x] ^= other._hash[x]; + } +} + +bool SHA1Block::operator==(const SHA1Block& rhs) const { + return rhs._hash == this->_hash; +} + +} // namespace mongo diff --git a/src/mongo/crypto/sha1_block.h b/src/mongo/crypto/sha1_block.h new file mode 100644 index 00000000000..266f6602e6c --- /dev/null +++ b/src/mongo/crypto/sha1_block.h @@ -0,0 +1,113 @@ +/** +* Copyright (C) 2017 MongoDB Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include +#include +#include + +#include "mongo/base/status_with.h" + +namespace mongo { + +struct BSONBinData; +class BSONObjBuilder; + +/** + * Data structure with fixed sized byte array that can be used as HMAC key or result of a SHA1 + * computation. + */ +class SHA1Block { +public: + static const size_t kHashLength; // 20 + using HashType = std::array; + + SHA1Block() = default; + SHA1Block(HashType rawHash); + static StatusWith fromBuffer(const uint8_t* input, size_t inputLen); + + /** + * Computes a SHA-1 hash of 'input'. + */ + static SHA1Block computeHash(const uint8_t* input, size_t inputLen); + + /** + * Computes a HMAC SHA-1 keyed hash of 'input' using the key 'key' + */ + static SHA1Block computeHmac(const uint8_t* key, + size_t keyLen, + const uint8_t* input, + size_t inputLen) { + SHA1Block output; + SHA1Block::computeHmac(key, keyLen, input, inputLen, &output); + return output; + } + + /** + * Computes a HMAC SHA-1 keyed hash of 'input' using the key 'key'. Writes the results into + * a pre-allocated SHA1Block. This lets us allocate SHA1Blocks with the SecureAllocator. + */ + static void computeHmac(const uint8_t* key, + size_t keyLen, + const uint8_t* input, + size_t inputLen, + SHA1Block* const output); + + const uint8_t* data() const { + return _hash.data(); + } + + size_t size() const { + return _hash.size(); + } + + /** + * Make a new SHA1Block from a BSON BinData value. + */ + static StatusWith fromBinData(const BSONBinData& binData); + + /** + * Append this to a builder using the given name as a BSON BinData type value. + */ + void appendAsBinData(BSONObjBuilder& builder, StringData fieldName); + + /** + * Do a bitwise xor against another SHA1Block and replace the current contents of this block + * with the result. + */ + void xorInline(const SHA1Block& other); + + std::string toString() const; + bool operator==(const SHA1Block& rhs) const; + +private: + HashType _hash; +}; + +} // namespace mongo diff --git a/src/mongo/crypto/sha1_block_openssl.cpp b/src/mongo/crypto/sha1_block_openssl.cpp new file mode 100644 index 00000000000..ce0cab8e9e6 --- /dev/null +++ b/src/mongo/crypto/sha1_block_openssl.cpp @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2014 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/crypto/sha1_block.h" + +#include "mongo/config.h" +#include "mongo/stdx/memory.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/scopeguard.h" + +#ifndef MONGO_CONFIG_SSL +#error This file should only be included in SSL-enabled builds +#endif + +#include +#include +#include +#include +#if OPENSSL_VERSION_NUMBER < 0x10100000L +namespace { +// Copies of OpenSSL after 1.1.0 define new EVP digest routines. We must +// polyfill used definitions to interact with older OpenSSL versions. +EVP_MD_CTX* EVP_MD_CTX_new() { + void* ret = OPENSSL_malloc(sizeof(EVP_MD_CTX)); + + if (ret != NULL) { + memset(ret, 0, sizeof(EVP_MD_CTX)); + } + return static_cast(ret); +} + +void EVP_MD_CTX_free(EVP_MD_CTX* ctx) { + EVP_MD_CTX_cleanup(ctx); + OPENSSL_free(ctx); +} +} // namespace +#endif + +namespace mongo { +/* + * Computes a SHA-1 hash of 'input'. + */ +SHA1Block SHA1Block::computeHash(const uint8_t* input, size_t inputLen) { + HashType output; + + EVP_MD_CTX digestCtx; + EVP_MD_CTX_init(&digestCtx); + ON_BLOCK_EXIT(EVP_MD_CTX_cleanup, &digestCtx); + + fassert(40379, + EVP_DigestInit_ex(&digestCtx, EVP_sha1(), NULL) == 1 && + EVP_DigestUpdate(&digestCtx, input, inputLen) == 1 && + EVP_DigestFinal_ex(&digestCtx, output.data(), NULL) == 1); + return SHA1Block(output); +} + +/* + * Computes a HMAC SHA-1 keyed hash of 'input' using the key 'key', writes output into 'output'. + */ +void SHA1Block::computeHmac(const uint8_t* key, + size_t keyLen, + const uint8_t* input, + size_t inputLen, + SHA1Block* const output) { + fassert(40380, + HMAC(EVP_sha1(), key, keyLen, input, inputLen, output->_hash.data(), NULL) != NULL); +} + +} // namespace mongo diff --git a/src/mongo/crypto/sha1_block_test.cpp b/src/mongo/crypto/sha1_block_test.cpp new file mode 100644 index 00000000000..651b2e52bfe --- /dev/null +++ b/src/mongo/crypto/sha1_block_test.cpp @@ -0,0 +1,565 @@ +/** + * Copyright (C) 2014 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/bson/bsonmisc.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/crypto/sha1_block.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace { + +// SHA-1 test vectors from http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA_All.pdf +const struct { + const char* msg; + SHA1Block hash; +} sha1Tests[] = {{"abc", + SHA1Block::HashType{0xa9, + 0x99, + 0x3e, + 0x36, + 0x47, + 0x06, + 0x81, + 0x6a, + 0xba, + 0x3e, + 0x25, + 0x71, + 0x78, + 0x50, + 0xc2, + 0x6c, + 0x9c, + 0xd0, + 0xd8, + 0x9d}}, + + {"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + SHA1Block::HashType{0x84, + 0x98, + 0x3E, + 0x44, + 0x1C, + 0x3B, + 0xD2, + 0x6E, + 0xBA, + 0xAE, + 0x4A, + 0xA1, + 0xF9, + 0x51, + 0x29, + 0xE5, + 0xE5, + 0x46, + 0x70, + 0xF1}}}; + +TEST(CryptoVectors, SHA1) { + size_t numTests = sizeof(sha1Tests) / sizeof(sha1Tests[0]); + for (size_t i = 0; i < numTests; i++) { + SHA1Block result = SHA1Block::computeHash( + reinterpret_cast(sha1Tests[i].msg), strlen(sha1Tests[i].msg)); + ASSERT(sha1Tests[i].hash == result) << "Failed SHA1 iteration " << i; + } +} + +const int maxKeySize = 80; +const int maxDataSize = 54; +// HMAC-SHA-1 test vectors from http://tools.ietf.org/html/rfc2202.html +const struct { + unsigned char key[maxKeySize]; + int keyLen; + unsigned char data[maxDataSize]; + int dataLen; + SHA1Block hash; +} hmacSha1Tests[] = { + // RFC test case 1 + {{0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b, + 0x0b}, + 20, + {0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65}, + 8, + SHA1Block::HashType{0xb6, + 0x17, + 0x31, + 0x86, + 0x55, + 0x05, + 0x72, + 0x64, + 0xe2, + 0x8b, + 0xc0, + 0xb6, + 0xfb, + 0x37, + 0x8c, + 0x8e, + 0xf1, + 0x46, + 0xbe, + 0x00}}, + + // RFC test case 3 + {{0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa}, + 20, + {0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd, + 0xdd}, + 50, + SHA1Block::HashType{0x12, + 0x5d, + 0x73, + 0x42, + 0xb9, + 0xac, + 0x11, + 0xcd, + 0x91, + 0xa3, + 0x9a, + 0xf4, + 0x8a, + 0xa1, + 0x7b, + 0x4f, + 0x63, + 0xf1, + 0x75, + 0xd3}}, + + // RFC test case 4 + {{0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0a, + 0x0b, + 0x0c, + 0x0d, + 0x0e, + 0x0f, + 0x10, + 0x11, + 0x12, + 0x13, + 0x14, + 0x15, + 0x16, + 0x17, + 0x18, + 0x19}, + 25, + {0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd, + 0xcd}, + 50, + SHA1Block::HashType{0x4c, + 0x90, + 0x07, + 0xf4, + 0x02, + 0x62, + 0x50, + 0xc6, + 0xbc, + 0x84, + 0x14, + 0xf9, + 0xbf, + 0x50, + 0xc8, + 0x6c, + 0x2d, + 0x72, + 0x35, + 0xda}}, + + // RFC test case 6 + {{0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa, + 0xaa}, + 80, + {0x54, + 0x65, + 0x73, + 0x74, + 0x20, + 0x55, + 0x73, + 0x69, + 0x6e, + 0x67, + 0x20, + 0x4c, + 0x61, + 0x72, + 0x67, + 0x65, + 0x72, + 0x20, + 0x54, + 0x68, + 0x61, + 0x6e, + 0x20, + 0x42, + 0x6c, + 0x6f, + 0x63, + 0x6b, + 0x2d, + 0x53, + 0x69, + 0x7a, + 0x65, + 0x20, + 0x4b, + 0x65, + 0x79, + 0x20, + 0x2d, + 0x20, + 0x48, + 0x61, + 0x73, + 0x68, + 0x20, + 0x4b, + 0x65, + 0x79, + 0x20, + 0x46, + 0x69, + 0x72, + 0x73, + 0x74}, + 54, + SHA1Block::HashType{0xaa, + 0x4a, + 0xe5, + 0xe1, + 0x52, + 0x72, + 0xd0, + 0x0e, + 0x95, + 0x70, + 0x56, + 0x37, + 0xce, + 0x8a, + 0x3b, + 0x55, + 0xed, + 0x40, + 0x21, + 0x12}}}; + +TEST(CryptoVectors, HMACSHA1) { + size_t numTests = sizeof(hmacSha1Tests) / sizeof(hmacSha1Tests[0]); + for (size_t i = 0; i < numTests; i++) { + SHA1Block result = SHA1Block::computeHmac(hmacSha1Tests[i].key, + hmacSha1Tests[i].keyLen, + hmacSha1Tests[i].data, + hmacSha1Tests[i].dataLen); + ASSERT(hmacSha1Tests[i].hash == result) << "Failed HMAC-SHA1 iteration " << i; + } +} + +TEST(SHA1Block, BinDataRoundTrip) { + SHA1Block::HashType rawHash; + rawHash.fill(0); + for (size_t i = 0; i < rawHash.size(); i++) { + rawHash[i] = i; + } + + SHA1Block testHash(rawHash); + + BSONObjBuilder builder; + testHash.appendAsBinData(builder, "hash"); + auto newObj = builder.done(); + + auto hashElem = newObj["hash"]; + ASSERT_EQ(BinData, hashElem.type()); + ASSERT_EQ(BinDataGeneral, hashElem.binDataType()); + + int binLen = 0; + auto rawBinData = hashElem.binData(binLen); + ASSERT_EQ(SHA1Block::kHashLength, static_cast(binLen)); + + auto newHashStatus = + SHA1Block::fromBinData(BSONBinData(rawBinData, binLen, hashElem.binDataType())); + ASSERT_OK(newHashStatus.getStatus()); + ASSERT_TRUE(testHash == newHashStatus.getValue()); +} + +TEST(SHA1Block, CanOnlyConstructFromBinGeneral) { + std::string dummy(SHA1Block::kHashLength, 'x'); + + auto newHashStatus = SHA1Block::fromBinData(BSONBinData(dummy.c_str(), dummy.size(), newUUID)); + ASSERT_EQ(ErrorCodes::UnsupportedFormat, newHashStatus.getStatus()); +} + +TEST(SHA1Block, FromBinDataShouldRegectWrongSize) { + std::string dummy(SHA1Block::kHashLength - 1, 'x'); + + auto newHashStatus = + SHA1Block::fromBinData(BSONBinData(dummy.c_str(), dummy.size(), BinDataGeneral)); + ASSERT_EQ(ErrorCodes::UnsupportedFormat, newHashStatus.getStatus()); +} + +TEST(SHA1Block, FromBufferShouldRejectWrongLength) { + std::string dummy(SHA1Block::kHashLength - 1, 'x'); + + auto newHashStatus = + SHA1Block::fromBuffer(reinterpret_cast(dummy.c_str()), dummy.size()); + ASSERT_EQ(ErrorCodes::InvalidLength, newHashStatus.getStatus()); +} + + +} // namespace +} // namespace mongo diff --git a/src/mongo/crypto/sha1_block_tom.cpp b/src/mongo/crypto/sha1_block_tom.cpp new file mode 100644 index 00000000000..7d32dbc2f53 --- /dev/null +++ b/src/mongo/crypto/sha1_block_tom.cpp @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2014 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/crypto/sha1_block.h" + +#include "mongo/config.h" +#include "mongo/util/assert_util.h" + +#ifdef MONGO_CONFIG_SSL +#error This file should not be included if compiling with SSL support +#endif + +#include "mongo/crypto/tom/tomcrypt.h" + +namespace mongo { + +/* + * Computes a SHA-1 hash of 'input'. + */ +SHA1Block SHA1Block::computeHash(const uint8_t* input, size_t inputLen) { + HashType output; + + hash_state hashState; + fassert(40381, + sha1_init(&hashState) == CRYPT_OK && + sha1_process(&hashState, input, inputLen) == CRYPT_OK && + sha1_done(&hashState, output.data()) == CRYPT_OK); + return SHA1Block(output); +} + +/* + * Computes a HMAC SHA-1 keyed hash of 'input' using the key 'key' + */ +void SHA1Block::computeHmac(const uint8_t* key, + size_t keyLen, + const uint8_t* input, + size_t inputLen, + SHA1Block* const output) { + invariant(key && input); + + static int hashId = -1; + if (hashId == -1) { + register_hash(&sha1_desc); + hashId = find_hash("sha1"); + } + + unsigned long sha1HashLen = 20; + fassert(40382, + hmac_memory(hashId, key, keyLen, input, inputLen, output->_hash.data(), &sha1HashLen) == + CRYPT_OK); +} + +} // namespace mongo diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index 628bb239813..a8ce1b54416 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -618,7 +618,6 @@ serverOnlyFiles = [ "repl/storage_interface_impl.cpp", "repl/sync_source_feedback.cpp", "service_context_d.cpp", - "stats/fill_locker_info.cpp", "stats/lock_server_status_section.cpp", "stats/range_deleter_server_status.cpp", "stats/snapshots.cpp", @@ -689,6 +688,7 @@ serveronlyLibdeps = [ "s/sharding", "startup_warnings_mongod", "stats/counters", + "stats/fill_locker_info", "stats/top", "storage/devnull/storage_devnull", "storage/ephemeral_for_test/storage_ephemeral_for_test", diff --git a/src/mongo/db/auth/SConscript b/src/mongo/db/auth/SConscript index 90c7a01e72c..1201d5e8d0e 100644 --- a/src/mongo/db/auth/SConscript +++ b/src/mongo/db/auth/SConscript @@ -178,3 +178,20 @@ env.CppUnitTest('authorization_manager_test', 'authorization_manager_test.cpp', LIBDEPS=['authcore', 'authmocks', 'saslauth']) env.CppUnitTest('authorization_session_test', 'authorization_session_test.cpp', LIBDEPS=['authcore', 'authmocks', 'saslauth']) + +env.Library( + target='authorization_session_for_test', + source=[ + 'authorization_session_for_test.cpp', + ], + LIBDEPS=[ + 'authcore', + ] +) + +env.CppUnitTest('sasl_scramsha1_test', + 'sasl_scramsha1_test.cpp', + LIBDEPS=[ + 'saslauth', + '$BUILD_DIR/mongo/client/sasl_client', + ]) diff --git a/src/mongo/db/auth/action_set.cpp b/src/mongo/db/auth/action_set.cpp index 6f89222679b..b9b11981aad 100644 --- a/src/mongo/db/auth/action_set.cpp +++ b/src/mongo/db/auth/action_set.cpp @@ -41,6 +41,12 @@ namespace mongo { +ActionSet::ActionSet(std::initializer_list actions) { + for (auto& action : actions) { + addAction(action); + } +} + void ActionSet::addAction(const ActionType& action) { if (action == ActionType::anyAction) { addAllActions(); diff --git a/src/mongo/db/auth/action_set.h b/src/mongo/db/auth/action_set.h index 339a68f019c..3bea5d9fdc1 100644 --- a/src/mongo/db/auth/action_set.h +++ b/src/mongo/db/auth/action_set.h @@ -44,6 +44,7 @@ namespace mongo { class ActionSet { public: ActionSet() : _actions(0) {} + ActionSet(std::initializer_list actions); void addAction(const ActionType& action); void addAllActionsFromSet(const ActionSet& actionSet); diff --git a/src/mongo/db/auth/authorization_session.cpp b/src/mongo/db/auth/authorization_session.cpp index 4f80f19439e..676fccd070d 100644 --- a/src/mongo/db/auth/authorization_session.cpp +++ b/src/mongo/db/auth/authorization_session.cpp @@ -467,7 +467,12 @@ static int buildResourceSearchList(const ResourcePattern& target, resourceSearchList[size++] = ResourcePattern::forAnyResource(); if (target.isExactNamespacePattern()) { if (!target.ns().isSystem()) { - resourceSearchList[size++] = ResourcePattern::forAnyNormalResource(); + // Some databases should not be matchable with ResourcePattern::forAnyNormalResource. + // 'local' and 'config' are used to store special system collections, which user level + // administrators should not be able to manipulate. + if (target.ns().db() != "local" && target.ns().db() != "config") { + resourceSearchList[size++] = ResourcePattern::forAnyNormalResource(); + } resourceSearchList[size++] = ResourcePattern::forDatabaseName(target.ns().db()); } resourceSearchList[size++] = ResourcePattern::forCollectionName(target.ns().coll()); diff --git a/src/mongo/db/auth/authorization_session.h b/src/mongo/db/auth/authorization_session.h index 7b5ce4799fa..61c6093bcf7 100644 --- a/src/mongo/db/auth/authorization_session.h +++ b/src/mongo/db/auth/authorization_session.h @@ -93,7 +93,7 @@ public: // Takes ownership of the externalState. explicit AuthorizationSession(std::unique_ptr externalState); - ~AuthorizationSession(); + virtual ~AuthorizationSession(); AuthorizationManager& getAuthorizationManager(); @@ -256,16 +256,24 @@ public: // called. bool isImpersonating() const; -private: - // If any users authenticated on this session are marked as invalid this updates them with - // up-to-date information. May require a read lock on the "admin" db to read the user data. - void _refreshUserInfoAsNeeded(OperationContext* txn); - +protected: // Builds a vector of all roles held by users who are authenticated on this connection. The // vector is stored in _authenticatedRoleNames. This function is called when users are // logged in or logged out, as well as when the user cache is determined to be out of date. void _buildAuthenticatedRolesVector(); + // All Users who have been authenticated on this connection. + UserSet _authenticatedUsers; + + // The roles of the authenticated users. This vector is generated when the authenticated + // users set is changed. + std::vector _authenticatedRoleNames; + +private: + // If any users authenticated on this session are marked as invalid this updates them with + // up-to-date information. May require a read lock on the "admin" db to read the user data. + void _refreshUserInfoAsNeeded(OperationContext* txn); + // Checks if this connection is authorized for the given Privilege, ignoring whether or not // we should even be doing authorization checks in general. Note: this may acquire a read // lock on the admin database (to update out-of-date user privilege information). @@ -273,12 +281,6 @@ private: std::unique_ptr _externalState; - // All Users who have been authenticated on this connection. - UserSet _authenticatedUsers; - // The roles of the authenticated users. This vector is generated when the authenticated - // users set is changed. - std::vector _authenticatedRoleNames; - // A vector of impersonated UserNames and a vector of those users' RoleNames. // These are used in the auditing system. They are not used for authz checks. std::vector _impersonatedUserNames; diff --git a/src/mongo/db/auth/authorization_session_for_test.cpp b/src/mongo/db/auth/authorization_session_for_test.cpp new file mode 100644 index 00000000000..a38891a3f23 --- /dev/null +++ b/src/mongo/db/auth/authorization_session_for_test.cpp @@ -0,0 +1,83 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/auth/authorization_session_for_test.h" + +#include +#include + +#include "mongo/db/auth/privilege.h" +#include "mongo/db/auth/user.h" +#include "mongo/db/auth/user_name.h" +#include "mongo/db/auth/user_set.h" +#include "mongo/stdx/memory.h" + +namespace mongo { +const StringData AuthorizationSessionForTest::kTestDBName = + StringData("authorizationSessionForTestDB"); + +AuthorizationSessionForTest::~AuthorizationSessionForTest() { + revokeAllPrivileges(); +} + +void AuthorizationSessionForTest::assumePrivilegesForDB(Privilege privilege, StringData dbName) { + assumePrivilegesForDB(std::vector{privilege}, dbName); +} + +void AuthorizationSessionForTest::assumePrivilegesForDB(PrivilegeVector privileges, + StringData dbName) { + auto user = stdx::make_unique(UserName("authorizationSessionForTestUser", dbName)); + user->addPrivileges(privileges); + + _authenticatedUsers.add(user.get()); + _testUsers.emplace_back(std::move(user)); + _buildAuthenticatedRolesVector(); +} + +void AuthorizationSessionForTest::revokePrivilegesForDB(StringData dbName) { + _authenticatedUsers.removeByDBName(dbName); + _testUsers.erase(std::remove_if(_testUsers.begin(), + _testUsers.end(), + [&](const std::unique_ptr& user) { + return dbName == user->getName().getDB(); + }), + _testUsers.end()); +} + +void AuthorizationSessionForTest::revokeAllPrivileges() { + _testUsers.erase(std::remove_if(_testUsers.begin(), + _testUsers.end(), + [&](const std::unique_ptr& user) { + _authenticatedUsers.removeByDBName(user->getName().getDB()); + return true; + }), + _testUsers.end()); +} +} // namespace mongo diff --git a/src/mongo/db/auth/authorization_session_for_test.h b/src/mongo/db/auth/authorization_session_for_test.h new file mode 100644 index 00000000000..84ec29df5ea --- /dev/null +++ b/src/mongo/db/auth/authorization_session_for_test.h @@ -0,0 +1,85 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/auth/authz_session_external_state.h" +#include "mongo/db/auth/user.h" + +namespace mongo { + +class AuthorizationSessionForTest : public AuthorizationSession { + MONGO_DISALLOW_COPYING(AuthorizationSessionForTest); + +public: + // A database name used for testing purposes, deliberately named to minimize collisions with + // other test users. + static const StringData kTestDBName; + + /** + * Construct a new AuthorizationSessionForTest, taking ownership of 'externalState'. + */ + explicit AuthorizationSessionForTest(std::unique_ptr externalState) + : AuthorizationSession(std::move(externalState)) {} + + /** + * Cleans up any privileges granted via assumePrivilegesForDB(). + */ + ~AuthorizationSessionForTest(); + + /** + * Grants this session all privileges in 'privileges' for the database named 'dbName'. Any prior + * privileges granted on 'dbName' via a call to this method are erased. + * + * Do not use this method if also adding users via addAndAuthorizeUser() in the same database. + */ + void assumePrivilegesForDB(PrivilegeVector privilege, StringData dbName = kTestDBName); + void assumePrivilegesForDB(Privilege privilege, StringData dbName = kTestDBName); + + /** + * Revoke all privileges granted via assumePrivilegesForDB() on the database named 'dbName'. + * + * Do not use this method if also adding users via addAndAuthorizeUser() in the same database. + */ + void revokePrivilegesForDB(StringData dbName); + + /** + * Revokes all privileges granted via assumePrivilegesForDB() on every database. + * + * Do not use this method if also adding users via addAndAuthorizeUser() in the same database. + */ + void revokeAllPrivileges(); + +private: + std::vector> _testUsers; +}; +} // namespace mongo diff --git a/src/mongo/db/auth/privilege_parser.cpp b/src/mongo/db/auth/privilege_parser.cpp index 140fbde2d7f..d1131c8881f 100644 --- a/src/mongo/db/auth/privilege_parser.cpp +++ b/src/mongo/db/auth/privilege_parser.cpp @@ -93,8 +93,13 @@ bool ParsedResource::isValid(std::string* errMsg) const { } if (isCollectionSet() && (!NamespaceString::validCollectionName(getCollection()) && !getCollection().empty())) { - *errMsg = stream() << getCollection() << " is not a valid collection name"; - return false; + // local.oplog.$main is a real collection that the server will create. But, collection + // names with a '$' character are illegal. We must make an exception for this collection + // here so we can grant users access to it. + if (!(getDb() == "local" && getCollection() == "oplog.$main")) { + *errMsg = stream() << getCollection() << " is not a valid collection name"; + return false; + } } return true; } diff --git a/src/mongo/db/auth/role_graph_builtin_roles.cpp b/src/mongo/db/auth/role_graph_builtin_roles.cpp index 8a7cf4f3b8a..ba0785aef09 100644 --- a/src/mongo/db/auth/role_graph_builtin_roles.cpp +++ b/src/mongo/db/auth/role_graph_builtin_roles.cpp @@ -361,14 +361,17 @@ void addClusterMonitorPrivileges(PrivilegeVector* privileges) { Privilege::addPrivilegeToPrivilegeVector( privileges, Privilege(ResourcePattern::forAnyNormalResource(), clusterMonitorRoleDatabaseActions)); - addReadOnlyDbPrivileges(privileges, "config"); Privilege::addPrivilegeToPrivilegeVector( privileges, - Privilege(ResourcePattern::forExactNamespace(NamespaceString("local.system.replset")), - ActionType::find)); + Privilege(ResourcePattern::forDatabaseName("config"), clusterMonitorRoleDatabaseActions)); Privilege::addPrivilegeToPrivilegeVector( privileges, - Privilege(ResourcePattern::forExactNamespace(NamespaceString("local.sources")), + Privilege(ResourcePattern::forDatabaseName("local"), clusterMonitorRoleDatabaseActions)); + addReadOnlyDbPrivileges(privileges, "local"); + addReadOnlyDbPrivileges(privileges, "config"); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forExactNamespace(NamespaceString("local", "system.replset")), ActionType::find)); Privilege::addPrivilegeToPrivilegeVector( privileges, @@ -391,22 +394,26 @@ void addClusterManagerPrivileges(PrivilegeVector* privileges) { Privilege::addPrivilegeToPrivilegeVector( privileges, Privilege(ResourcePattern::forAnyNormalResource(), clusterManagerRoleDatabaseActions)); - addReadOnlyDbPrivileges(privileges, "config"); - ActionSet configSettingsActions; - configSettingsActions << ActionType::insert << ActionType::update << ActionType::remove; Privilege::addPrivilegeToPrivilegeVector( privileges, - Privilege(ResourcePattern::forExactNamespace(NamespaceString("config", "settings")), - configSettingsActions)); + Privilege(ResourcePattern::forDatabaseName("config"), clusterManagerRoleDatabaseActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, + Privilege(ResourcePattern::forDatabaseName("local"), clusterManagerRoleDatabaseActions)); + Privilege::addPrivilegeToPrivilegeVector( privileges, Privilege(ResourcePattern::forExactNamespace(NamespaceString("local", "system.replset")), readRoleActions)); + addReadOnlyDbPrivileges(privileges, "config"); + + ActionSet writeActions; + writeActions << ActionType::insert << ActionType::update << ActionType::remove; Privilege::addPrivilegeToPrivilegeVector( - privileges, - Privilege(ResourcePattern::forExactNamespace(NamespaceString("config", "tags")), - configSettingsActions)); + privileges, Privilege(ResourcePattern::forDatabaseName("config"), writeActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, Privilege(ResourcePattern::forDatabaseName("local"), writeActions)); } void addClusterAdminPrivileges(PrivilegeVector* privileges) { @@ -433,6 +440,12 @@ void addBackupPrivileges(PrivilegeVector* privileges) { Privilege::addPrivilegeToPrivilegeVector( privileges, Privilege(ResourcePattern::forClusterResource(), clusterActions)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, Privilege(ResourcePattern::forDatabaseName("config"), ActionType::find)); + + Privilege::addPrivilegeToPrivilegeVector( + privileges, Privilege(ResourcePattern::forDatabaseName("local"), ActionType::find)); + Privilege::addPrivilegeToPrivilegeVector( privileges, Privilege(ResourcePattern::forCollectionName("system.indexes"), ActionType::find)); @@ -503,6 +516,12 @@ void addRestorePrivileges(PrivilegeVector* privileges) { Privilege::addPrivilegeToPrivilegeVector( privileges, Privilege(ResourcePattern::forAnyResource(), ActionType::listCollections)); + Privilege::addPrivilegeToPrivilegeVector( + privileges, Privilege(ResourcePattern::forDatabaseName("config"), actions)); + + Privilege::addPrivilegeToPrivilegeVector( + privileges, Privilege(ResourcePattern::forDatabaseName("local"), actions)); + // Privileges for user/role management Privilege::addPrivilegeToPrivilegeVector( privileges, Privilege(ResourcePattern::forAnyNormalResource(), userAdminRoleActions)); @@ -560,6 +579,7 @@ void addRootRolePrivileges(PrivilegeVector* privileges) { addUserAdminAnyDbPrivileges(privileges); addDbAdminAnyDbPrivileges(privileges); addReadWriteAnyDbPrivileges(privileges); + addBackupPrivileges(privileges); addRestorePrivileges(privileges); Privilege::addPrivilegeToPrivilegeVector( privileges, Privilege(ResourcePattern::forAnyResource(), ActionType::validate)); diff --git a/src/mongo/db/auth/sasl_plain_server_conversation.cpp b/src/mongo/db/auth/sasl_plain_server_conversation.cpp index b5f0b9e3c8f..d1e993db4a1 100644 --- a/src/mongo/db/auth/sasl_plain_server_conversation.cpp +++ b/src/mongo/db/auth/sasl_plain_server_conversation.cpp @@ -80,18 +80,16 @@ StatusWith SaslPLAINServerConversation::step(StringData inputData, std::st } } else { // Handle schemaVersion28SCRAM (SCRAM only mode) - unsigned char storedKey[scram::hashSize]; - unsigned char serverKey[scram::hashSize]; - - scram::generateSecrets( + std::string decodedSalt = base64::decode(creds.scram.salt); + scram::SCRAMSecrets secrets = scram::generateSecrets(scram::SCRAMPresecrets( authDigest, - reinterpret_cast(base64::decode(creds.scram.salt).c_str()), - 16, - creds.scram.iterationCount, - storedKey, - serverKey); + std::vector(reinterpret_cast(decodedSalt.c_str()), + reinterpret_cast(decodedSalt.c_str()) + + 16), + creds.scram.iterationCount)); if (creds.scram.storedKey != - base64::encode(reinterpret_cast(storedKey), scram::hashSize)) { + base64::encode(reinterpret_cast(secrets->storedKey.data()), + secrets->storedKey.size())) { return StatusWith(ErrorCodes::AuthenticationFailed, mongoutils::str::stream() << "Incorrect user name or password"); } diff --git a/src/mongo/db/auth/sasl_scramsha1_server_conversation.cpp b/src/mongo/db/auth/sasl_scramsha1_server_conversation.cpp index 9fd8496b7bc..5809306456d 100644 --- a/src/mongo/db/auth/sasl_scramsha1_server_conversation.cpp +++ b/src/mongo/db/auth/sasl_scramsha1_server_conversation.cpp @@ -35,8 +35,8 @@ #include #include -#include "mongo/crypto/crypto.h" #include "mongo/crypto/mechanism_scram.h" +#include "mongo/crypto/sha1_block.h" #include "mongo/db/auth/sasl_options.h" #include "mongo/platform/random.h" #include "mongo/util/base64.h" @@ -275,60 +275,23 @@ StatusWith SaslSCRAMSHA1ServerConversation::_secondStep(const std::vector< // ClientKey := ClientSignature XOR ClientProof // ServerSignature := HMAC(ServerKey, AuthMessage) - unsigned int hashLen = 0; - unsigned char clientSignature[scram::hashSize]; - - std::string decodedStoredKey = base64::decode(_creds.scram.storedKey); - // ClientSignature := HMAC(StoredKey, AuthMessage) - fassert(18662, - crypto::hmacSha1(reinterpret_cast(decodedStoredKey.c_str()), - scram::hashSize, - reinterpret_cast(_authMessage.c_str()), - _authMessage.size(), - clientSignature, - &hashLen)); - - fassert(18658, hashLen == scram::hashSize); - - try { - clientProof = base64::decode(clientProof); - } catch (const DBException& ex) { - return StatusWith(ex.toStatus()); - } - const unsigned char* decodedClientProof = - reinterpret_cast(clientProof.c_str()); - - // ClientKey := ClientSignature XOR ClientProof - unsigned char clientKey[scram::hashSize]; - for (size_t i = 0; i < scram::hashSize; i++) { - clientKey[i] = clientSignature[i] ^ decodedClientProof[i]; - } - - // StoredKey := H(ClientKey) - unsigned char computedStoredKey[scram::hashSize]; - fassert(18659, crypto::sha1(clientKey, scram::hashSize, computedStoredKey)); - - if (memcmp(decodedStoredKey.c_str(), computedStoredKey, scram::hashSize) != 0) { + if (!scram::verifyClientProof( + base64::decode(clientProof), base64::decode(_creds.scram.storedKey), _authMessage)) { return StatusWith(ErrorCodes::AuthenticationFailed, mongoutils::str::stream() << "SCRAM-SHA-1 authentication failed, storedKey mismatch"); } // ServerSignature := HMAC(ServerKey, AuthMessage) - unsigned char serverSignature[scram::hashSize]; std::string decodedServerKey = base64::decode(_creds.scram.serverKey); - fassert(18660, - crypto::hmacSha1(reinterpret_cast(decodedServerKey.c_str()), - scram::hashSize, - reinterpret_cast(_authMessage.c_str()), - _authMessage.size(), - serverSignature, - &hashLen)); - - fassert(18661, hashLen == scram::hashSize); + SHA1Block serverSignature = + SHA1Block::computeHmac(reinterpret_cast(decodedServerKey.c_str()), + decodedServerKey.size(), + reinterpret_cast(_authMessage.c_str()), + _authMessage.size()); StringBuilder sb; - sb << "v=" << base64::encode(reinterpret_cast(serverSignature), scram::hashSize); + sb << "v=" << serverSignature.toString(); *outputData = sb.str(); return StatusWith(false); diff --git a/src/mongo/db/auth/sasl_scramsha1_test.cpp b/src/mongo/db/auth/sasl_scramsha1_test.cpp new file mode 100644 index 00000000000..66cddabe02e --- /dev/null +++ b/src/mongo/db/auth/sasl_scramsha1_test.cpp @@ -0,0 +1,501 @@ +/* + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/client/native_sasl_client_session.h" +#include "mongo/client/scram_sha1_client_cache.h" +#include "mongo/crypto/mechanism_scram.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authz_manager_external_state_mock.h" +#include "mongo/db/auth/authz_session_external_state_mock.h" +#include "mongo/db/auth/native_sasl_authentication_session.h" +#include "mongo/db/auth/sasl_scramsha1_server_conversation.h" +#include "mongo/db/service_context_noop.h" +#include "mongo/stdx/memory.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/base64.h" +#include "mongo/util/password_digest.h" + +namespace mongo { + +BSONObj generateSCRAMUserDocument(StringData username, StringData password) { + const size_t scramIterationCount = 10000; + std::string database = "test"; + + std::string digested = createPasswordDigest(username, password); + BSONObj scramCred = scram::generateCredentials(digested, scramIterationCount); + return BSON("_id" << (str::stream() << database << "." << username).operator std::string() + << AuthorizationManager::USER_NAME_FIELD_NAME << username + << AuthorizationManager::USER_DB_FIELD_NAME << database << "credentials" + << BSON("SCRAM-SHA-1" << scramCred) << "roles" << BSONArray() << "privileges" + << BSONArray()); +} + +BSONObj generateMONGODBCRUserDocument(StringData username, StringData password) { + std::string database = "test"; + + std::string digested = createPasswordDigest(username, password); + return BSON("_id" << (str::stream() << database << "." << username).operator std::string() + << AuthorizationManager::USER_NAME_FIELD_NAME << username + << AuthorizationManager::USER_DB_FIELD_NAME << database << "credentials" + << BSON("MONGODB-CR" << digested) << "roles" << BSONArray() << "privileges" + << BSONArray()); +} + +std::string corruptEncodedPayload(const std::string& message, + std::string::const_iterator begin, + std::string::const_iterator end) { + std::string raw = base64::decode( + message.substr(std::distance(message.begin(), begin), std::distance(begin, end))); + if (raw[0] == std::numeric_limits::max()) { + raw[0] -= 1; + } else { + raw[0] += 1; + } + return base64::encode(raw); +} + +class SaslTestState { +public: + enum Participant { kClient, kServer }; + SaslTestState() : SaslTestState(kClient, 0) {} + SaslTestState(Participant participant, size_t stage) : participant(participant), stage(stage) {} + +private: + // Define members here, so that they can be used in declaration of lens(). In C++14, lens() + // can be declared with a return of decltype(auto), without a trailing return type, and these + // members can go at the end of the class. + Participant participant; + size_t stage; + +public: + std::tuple lens() const { + return std::tie(stage, participant); + } + + friend bool operator==(const SaslTestState& lhs, const SaslTestState& rhs) { + return lhs.lens() == rhs.lens(); + } + + friend bool operator<(const SaslTestState& lhs, const SaslTestState& rhs) { + return lhs.lens() < rhs.lens(); + } + + void next() { + if (participant == kClient) { + participant = kServer; + } else { + participant = kClient; + stage++; + } + } + + std::string toString() const { + std::stringstream ss; + if (participant == kClient) { + ss << "Client"; + } else { + ss << "Server"; + } + ss << "Step" << stage; + + return ss.str(); + } +}; + +class SCRAMMutators { +public: + SCRAMMutators() {} + + void setMutator(SaslTestState state, stdx::function fun) { + mutators.insert(std::make_pair(state, fun)); + } + + void execute(SaslTestState state, std::string& str) { + auto it = mutators.find(state); + if (it != mutators.end()) { + it->second(str); + } + } + +private: + std::map> mutators; +}; + +struct SCRAMStepsResult { + SCRAMStepsResult() : outcome(SaslTestState::kClient, 1), status(Status::OK()) {} + SCRAMStepsResult(SaslTestState outcome, Status status) : outcome(outcome), status(status) {} + bool operator==(const SCRAMStepsResult& other) const { + return outcome == other.outcome && status.code() == other.status.code() && + status.reason() == other.status.reason(); + } + SaslTestState outcome; + Status status; + + friend std::ostream& operator<<(std::ostream& os, const SCRAMStepsResult& result) { + return os << "{outcome: " << result.outcome.toString() << ", status: " << result.status + << "}"; + } +}; + +SCRAMStepsResult runSteps(NativeSaslAuthenticationSession* saslServerSession, + NativeSaslClientSession* saslClientSession, + SCRAMMutators interposers = SCRAMMutators{}) { + SCRAMStepsResult result{}; + std::string clientOutput = ""; + std::string serverOutput = ""; + + for (size_t step = 1; step <= 3; step++) { + ASSERT_FALSE(saslClientSession->isDone()); + ASSERT_FALSE(saslServerSession->isDone()); + + // Client step + result.status = saslClientSession->step(serverOutput, &clientOutput); + if (result.status != Status::OK()) { + return result; + } + std::cout << result.outcome.toString() << ": " << clientOutput << std::endl; + interposers.execute(result.outcome, clientOutput); + result.outcome.next(); + + // Server step + result.status = saslServerSession->step(clientOutput, &serverOutput); + if (result.status != Status::OK()) { + return result; + } + interposers.execute(result.outcome, serverOutput); + std::cout << result.outcome.toString() << ": " << serverOutput << std::endl; + result.outcome.next(); + } + ASSERT_TRUE(saslClientSession->isDone()); + ASSERT_TRUE(saslServerSession->isDone()); + + return result; +} + +class SCRAMSHA1Fixture : public mongo::unittest::Test { +protected: + const SCRAMStepsResult goalState = + SCRAMStepsResult(SaslTestState(SaslTestState::kClient, 4), Status::OK()); + + ServiceContextNoop serviceContext; + ServiceContextNoop::UniqueClient client; + ServiceContextNoop::UniqueOperationContext txn; + + AuthzManagerExternalStateMock* authzManagerExternalState; + std::unique_ptr authzManager; + std::unique_ptr authzSession; + + std::unique_ptr saslServerSession; + std::unique_ptr saslClientSession; + + void setUp() { + client = serviceContext.makeClient("test"); + txn = serviceContext.makeOperationContext(client.get()); + + auto uniqueAuthzManagerExternalStateMock = + stdx::make_unique(); + authzManagerExternalState = uniqueAuthzManagerExternalStateMock.get(); + authzManager = + stdx::make_unique(std::move(uniqueAuthzManagerExternalStateMock)); + authzSession = stdx::make_unique( + stdx::make_unique(authzManager.get())); + + saslServerSession = stdx::make_unique(authzSession.get()); + saslServerSession->setOpCtxt(txn.get()); + saslServerSession->start("test", "SCRAM-SHA-1", "mongodb", "MockServer.test", 1, false); + saslClientSession = stdx::make_unique(); + saslClientSession->setParameter(NativeSaslClientSession::parameterMechanism, "SCRAM-SHA-1"); + saslClientSession->setParameter(NativeSaslClientSession::parameterServiceName, "mongodb"); + saslClientSession->setParameter(NativeSaslClientSession::parameterServiceHostname, + "MockServer.test"); + saslClientSession->setParameter(NativeSaslClientSession::parameterServiceHostAndPort, + "MockServer.test:27017"); + } +}; + +/*TEST_F(SCRAMSHA1Fixture, testServerStep1DoesNotIncludeNonceFromClientStep1) { + authzManagerExternalState->insertPrivilegeDocument( + txn.get(), generateSCRAMUserDocument("sajack", "sajack"), BSONObj()); + + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "sajack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("sajack", "sajack")); + + ASSERT_OK(saslClientSession->initialize()); + + SCRAMMutators mutator; + mutator.setMutator(SaslTestState(SaslTestState::kServer, 1), [](std::string& serverMessage) { + std::string::iterator nonceBegin = serverMessage.begin() + serverMessage.find("r="); + std::string::iterator nonceEnd = std::find(nonceBegin, serverMessage.end(), ','); + serverMessage = serverMessage.replace(nonceBegin, nonceEnd, "r="); + + }); + ASSERT_EQ(SCRAMStepsResult(SaslTestState(SaslTestState::kClient, 2), + Status(ErrorCodes::BadValue, + "Server SCRAM-SHA-1 nonce does not match client nonce: r=")), + runSteps(saslServerSession.get(), saslClientSession.get(), mutator)); +}*/ + +TEST_F(SCRAMSHA1Fixture, testClientStep2DoesNotIncludeNonceFromServerStep1) { + authzManagerExternalState->insertPrivilegeDocument( + txn.get(), generateSCRAMUserDocument("sajack", "sajack"), BSONObj()); + + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "sajack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("sajack", "sajack")); + + ASSERT_OK(saslClientSession->initialize()); + + SCRAMMutators mutator; + mutator.setMutator(SaslTestState(SaslTestState::kClient, 2), + [](std::string& clientMessage) { + std::string::iterator nonceBegin = + clientMessage.begin() + clientMessage.find("r="); + std::string::iterator nonceEnd = + std::find(nonceBegin, clientMessage.end(), ','); + clientMessage = clientMessage.replace(nonceBegin, nonceEnd, "r="); + }); + ASSERT_EQ(SCRAMStepsResult( + SaslTestState(SaslTestState::kServer, 2), + Status(ErrorCodes::BadValue, "Incorrect SCRAM-SHA-1 client|server nonce: r=")), + runSteps(saslServerSession.get(), saslClientSession.get(), mutator)); +} + +TEST_F(SCRAMSHA1Fixture, testClientStep2GivesBadProof) { + authzManagerExternalState->insertPrivilegeDocument( + txn.get(), generateSCRAMUserDocument("sajack", "sajack"), BSONObj()); + + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "sajack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("sajack", "sajack")); + + ASSERT_OK(saslClientSession->initialize()); + + SCRAMMutators mutator; + mutator.setMutator( + SaslTestState(SaslTestState::kClient, 2), + [](std::string& clientMessage) { + std::string::iterator proofBegin = clientMessage.begin() + clientMessage.find("p=") + 2; + std::string::iterator proofEnd = std::find(proofBegin, clientMessage.end(), ','); + clientMessage = clientMessage.replace( + proofBegin, proofEnd, corruptEncodedPayload(clientMessage, proofBegin, proofEnd)); + + }); + + ASSERT_EQ(SCRAMStepsResult(SaslTestState(SaslTestState::kServer, 2), + Status(ErrorCodes::AuthenticationFailed, + "SCRAM-SHA-1 authentication failed, storedKey mismatch")), + runSteps(saslServerSession.get(), saslClientSession.get(), mutator)); +} + +TEST_F(SCRAMSHA1Fixture, testServerStep2GivesBadVerifier) { + authzManagerExternalState->insertPrivilegeDocument( + txn.get(), generateSCRAMUserDocument("sajack", "sajack"), BSONObj()); + + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "sajack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("sajack", "sajack")); + + ASSERT_OK(saslClientSession->initialize()); + + std::string encodedVerifier; + SCRAMMutators mutator; + mutator.setMutator( + SaslTestState(SaslTestState::kServer, 2), + [&encodedVerifier](std::string& serverMessage) { + std::string::iterator verifierBegin = + serverMessage.begin() + serverMessage.find("v=") + 2; + std::string::iterator verifierEnd = std::find(verifierBegin, serverMessage.end(), ','); + encodedVerifier = corruptEncodedPayload(serverMessage, verifierBegin, verifierEnd); + + serverMessage = serverMessage.replace(verifierBegin, verifierEnd, encodedVerifier); + + }); + + auto result = runSteps(saslServerSession.get(), saslClientSession.get(), mutator); + + ASSERT_EQ( + SCRAMStepsResult( + SaslTestState(SaslTestState::kClient, 3), + Status(ErrorCodes::BadValue, + str::stream() << "Client failed to verify SCRAM-SHA-1 ServerSignature, received " + << encodedVerifier)), + result); +} + + +TEST_F(SCRAMSHA1Fixture, testSCRAM) { + authzManagerExternalState->insertPrivilegeDocument( + txn.get(), generateSCRAMUserDocument("sajack", "sajack"), BSONObj()); + + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "sajack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("sajack", "sajack")); + + ASSERT_OK(saslClientSession->initialize()); + + ASSERT_EQ(goalState, runSteps(saslServerSession.get(), saslClientSession.get())); +} + +TEST_F(SCRAMSHA1Fixture, testNULLInPassword) { + authzManagerExternalState->insertPrivilegeDocument( + txn.get(), generateSCRAMUserDocument("sajack", "saj\0ack"), BSONObj()); + + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "sajack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("sajack", "saj\0ack")); + + ASSERT_OK(saslClientSession->initialize()); + + ASSERT_EQ(goalState, runSteps(saslServerSession.get(), saslClientSession.get())); +} + + +TEST_F(SCRAMSHA1Fixture, testCommasInUsernameAndPassword) { + authzManagerExternalState->insertPrivilegeDocument( + txn.get(), generateSCRAMUserDocument("s,a,jack", "s,a,jack"), BSONObj()); + + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "s,a,jack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("s,a,jack", "s,a,jack")); + + ASSERT_OK(saslClientSession->initialize()); + + ASSERT_EQ(goalState, runSteps(saslServerSession.get(), saslClientSession.get())); +} + +TEST_F(SCRAMSHA1Fixture, testIncorrectUser) { + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "sajack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("sajack", "sajack")); + + ASSERT_OK(saslClientSession->initialize()); + + ASSERT_EQ(SCRAMStepsResult(SaslTestState(SaslTestState::kServer, 1), + Status(ErrorCodes::UserNotFound, "Could not find user sajack@test")), + runSteps(saslServerSession.get(), saslClientSession.get())); +} + +TEST_F(SCRAMSHA1Fixture, testIncorrectPassword) { + authzManagerExternalState->insertPrivilegeDocument( + txn.get(), generateSCRAMUserDocument("sajack", "sajack"), BSONObj()); + + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "sajack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("sajack", "invalidPassword")); + + ASSERT_OK(saslClientSession->initialize()); + + ASSERT_EQ(SCRAMStepsResult(SaslTestState(SaslTestState::kServer, 2), + Status(ErrorCodes::AuthenticationFailed, + "SCRAM-SHA-1 authentication failed, storedKey mismatch")), + runSteps(saslServerSession.get(), saslClientSession.get())); +} + +TEST_F(SCRAMSHA1Fixture, testMONGODBCR) { + authzManagerExternalState->insertPrivilegeDocument( + txn.get(), generateMONGODBCRUserDocument("sajack", "sajack"), BSONObj()); + + saslClientSession->setParameter(NativeSaslClientSession::parameterUser, "sajack"); + saslClientSession->setParameter(NativeSaslClientSession::parameterPassword, + createPasswordDigest("sajack", "sajack")); + + ASSERT_OK(saslClientSession->initialize()); + + ASSERT_EQ(goalState, runSteps(saslServerSession.get(), saslClientSession.get())); +} + +TEST(SCRAMSHA1Cache, testGetFromEmptyCache) { + SCRAMSHA1ClientCache cache; + std::string saltStr("saltsaltsaltsalt"); + std::vector salt(saltStr.begin(), saltStr.end()); + HostAndPort host("localhost:27017"); + + ASSERT_FALSE(cache.getCachedSecrets(host, scram::SCRAMPresecrets("aaa", salt, 10000))); +} + + +TEST(SCRAMSHA1Cache, testSetAndGet) { + SCRAMSHA1ClientCache cache; + std::string saltStr("saltsaltsaltsalt"); + std::string badSaltStr("s@lts@lts@lts@lt"); + std::vector salt(saltStr.begin(), saltStr.end()); + std::vector badSalt(badSaltStr.begin(), badSaltStr.end()); + HostAndPort host("localhost:27017"); + + auto secret = scram::generateSecrets(scram::SCRAMPresecrets("aaa", salt, 10000)); + cache.setCachedSecrets(host, scram::SCRAMPresecrets("aaa", salt, 10000), secret); + auto cachedSecret = cache.getCachedSecrets(host, scram::SCRAMPresecrets("aaa", salt, 10000)); + ASSERT_TRUE(cachedSecret); + ASSERT_TRUE(secret->clientKey == cachedSecret->clientKey); + ASSERT_TRUE(secret->serverKey == cachedSecret->serverKey); + ASSERT_TRUE(secret->storedKey == cachedSecret->storedKey); +} + + +TEST(SCRAMSHA1Cache, testSetAndGetWithDifferentParameters) { + SCRAMSHA1ClientCache cache; + std::string saltStr("saltsaltsaltsalt"); + std::string badSaltStr("s@lts@lts@lts@lt"); + std::vector salt(saltStr.begin(), saltStr.end()); + std::vector badSalt(badSaltStr.begin(), badSaltStr.end()); + HostAndPort host("localhost:27017"); + + auto secret = scram::generateSecrets(scram::SCRAMPresecrets("aaa", salt, 10000)); + cache.setCachedSecrets(host, scram::SCRAMPresecrets("aaa", salt, 10000), secret); + + ASSERT_FALSE(cache.getCachedSecrets(HostAndPort("localhost:27018"), + scram::SCRAMPresecrets("aaa", salt, 10000))); + ASSERT_FALSE(cache.getCachedSecrets(host, scram::SCRAMPresecrets("aab", salt, 10000))); + ASSERT_FALSE(cache.getCachedSecrets(host, scram::SCRAMPresecrets("aaa", badSalt, 10000))); + ASSERT_FALSE(cache.getCachedSecrets(host, scram::SCRAMPresecrets("aaa", salt, 10001))); +} + + +TEST(SCRAMSHA1Cache, testSetAndReset) { + SCRAMSHA1ClientCache cache; + StringData saltStr("saltsaltsaltsalt"); + std::vector salt(saltStr.begin(), saltStr.end()); + HostAndPort host("localhost:27017"); + + auto secret = scram::generateSecrets(scram::SCRAMPresecrets("aaa", salt, 10000)); + cache.setCachedSecrets(host, scram::SCRAMPresecrets("aaa", salt, 10000), secret); + auto newSecret = scram::generateSecrets(scram::SCRAMPresecrets("aab", salt, 10000)); + cache.setCachedSecrets(host, scram::SCRAMPresecrets("aab", salt, 10000), newSecret); + + ASSERT_FALSE(cache.getCachedSecrets(host, scram::SCRAMPresecrets("aaa", salt, 10000))); + auto cachedSecret = cache.getCachedSecrets(host, scram::SCRAMPresecrets("aab", salt, 10000)); + ASSERT_TRUE(cachedSecret); + ASSERT_TRUE(newSecret->clientKey == cachedSecret->clientKey); + ASSERT_TRUE(newSecret->serverKey == cachedSecret->serverKey); + ASSERT_TRUE(newSecret->storedKey == cachedSecret->storedKey); +} + +} // namespace mongo diff --git a/src/mongo/db/catalog/apply_ops.cpp b/src/mongo/db/catalog/apply_ops.cpp index 6d00127b9af..0afea74e776 100644 --- a/src/mongo/db/catalog/apply_ops.cpp +++ b/src/mongo/db/catalog/apply_ops.cpp @@ -132,7 +132,7 @@ Status applyOps(OperationContext* txn, try { MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { if (*opType == 'c') { - status = repl::applyCommand_inlock(txn, temp); + status = repl::applyCommand_inlock(txn, temp, true); break; } else { OldClientContext ctx(txn, ns); diff --git a/src/mongo/db/catalog/coll_mod.cpp b/src/mongo/db/catalog/coll_mod.cpp index 0f2304a6f49..700d50beb45 100644 --- a/src/mongo/db/catalog/coll_mod.cpp +++ b/src/mongo/db/catalog/coll_mod.cpp @@ -41,6 +41,99 @@ #include "mongo/db/service_context.h" namespace mongo { + +struct CollModRequest { + const IndexDescriptor* idx = nullptr; + BSONElement indexExpireAfterSeconds = {}; + BSONElement collValidator = {}; + std::string collValidationAction = {}; + std::string collValidationLevel = {}; + BSONElement usePowerOf2Sizes = {}; + BSONElement noPadding = {}; +}; + +StatusWith parseCollModRequest(OperationContext* txn, + const NamespaceString& nss, + Collection* coll, + const BSONObj& cmdObj) { + CollModRequest cmr; + + BSONForEach(e, cmdObj) { + if (str::equals("collMod", e.fieldName())) { + // no-op + } else if (str::startsWith(e.fieldName(), "$")) { + // no-op ignore top-level fields prefixed with $. They are for the command processor + } else if (LiteParsedQuery::cmdOptionMaxTimeMS == e.fieldNameStringData()) { + // no-op + } else if (str::equals("index", e.fieldName())) { + BSONObj indexObj = e.Obj(); + BSONObj keyPattern = indexObj.getObjectField("keyPattern"); + + if (keyPattern.isEmpty()) { + return Status(ErrorCodes::InvalidOptions, "no keyPattern specified"); + } + cmr.indexExpireAfterSeconds = indexObj["expireAfterSeconds"]; + if (cmr.indexExpireAfterSeconds.eoo()) { + return Status(ErrorCodes::InvalidOptions, "no expireAfterSeconds field"); + } + if (!cmr.indexExpireAfterSeconds.isNumber()) { + return Status(ErrorCodes::InvalidOptions, + "expireAfterSeconds field must be a number"); + } + + const IndexDescriptor* idx = + coll->getIndexCatalog()->findIndexByKeyPattern(txn, keyPattern); + if (idx == NULL) { + return Status(ErrorCodes::IndexNotFound, + str::stream() << "cannot find index " << keyPattern << " for ns " + << nss.ns()); + } + cmr.idx = idx; + + BSONElement oldExpireSecs = cmr.idx->infoObj().getField("expireAfterSeconds"); + + if (oldExpireSecs.eoo()) { + return Status(ErrorCodes::InvalidOptions, "no expireAfterSeconds field to update"); + } + if (!oldExpireSecs.isNumber()) { + return Status(ErrorCodes::InvalidOptions, + "existing expireAfterSeconds field is not a number"); + } + } else if (str::equals("validator", e.fieldName())) { + auto statusW = coll->parseValidator(e.Obj()); + if (!statusW.isOK()) + return statusW.getStatus(); + + cmr.collValidator = e; + } else if (str::equals("validationLevel", e.fieldName())) { + auto statusW = coll->parseValidationLevel(e.String()); + if (!statusW.isOK()) + return statusW.getStatus(); + + cmr.collValidationLevel = e.String(); + } else if (str::equals("validationAction", e.fieldName())) { + auto statusW = coll->parseValidationAction(e.String()); + if (!statusW.isOK()) + return statusW.getStatus(); + + cmr.collValidationAction = e.String(); + } else { + // As of SERVER-17312 we only support these two options. When SERVER-17320 is + // resolved this will need to be enhanced to handle other options. + const StringData name = e.fieldNameStringData(); + if (name == "usePowerOf2Sizes") + cmr.usePowerOf2Sizes = e; + else if (name == "noPadding") + cmr.noPadding = e; + else + return Status(ErrorCodes::InvalidOptions, + str::stream() << "unknown option to collMod: " << name); + } + } + + return {std::move(cmr)}; +} + Status collMod(OperationContext* txn, const NamespaceString& nss, const BSONObj& cmdObj, @@ -49,7 +142,7 @@ Status collMod(OperationContext* txn, ScopedTransaction transaction(txn, MODE_IX); AutoGetDb autoDb(txn, dbName, MODE_X); Database* const db = autoDb.getDb(); - Collection* coll = db ? db->getCollection(nss) : NULL; + Collection* coll = db ? db->getCollection(nss) : nullptr; // This can kill all cursors so don't allow running it while a background operation is in // progress. @@ -71,122 +164,80 @@ Status collMod(OperationContext* txn, << nss.ns()); } + auto statusW = parseCollModRequest(txn, nss, coll, cmdObj); + if (!statusW.isOK()) { + return statusW.getStatus(); + } + + CollModRequest cmr = statusW.getValue(); + WriteUnitOfWork wunit(txn); - Status errorStatus = Status::OK(); + if (!cmr.indexExpireAfterSeconds.eoo()) { + BSONElement& newExpireSecs = cmr.indexExpireAfterSeconds; + BSONElement oldExpireSecs = cmr.idx->infoObj().getField("expireAfterSeconds"); + + if (oldExpireSecs != newExpireSecs) { + result->appendAs(oldExpireSecs, "expireAfterSeconds_old"); + // Change the value of "expireAfterSeconds" on disk. + coll->getCatalogEntry()->updateTTLSetting( + txn, cmr.idx->indexName(), newExpireSecs.safeNumberLong()); + // Notify the index catalog that the definition of this index changed. + cmr.idx = coll->getIndexCatalog()->refreshEntry(txn, cmr.idx); + result->appendAs(newExpireSecs, "expireAfterSeconds_new"); + } + } - BSONForEach(e, cmdObj) { - if (str::equals("collMod", e.fieldName())) { - // no-op - } else if (str::startsWith(e.fieldName(), "$")) { - // no-op ignore top-level fields prefixed with $. They are for the command processor - } else if (LiteParsedQuery::cmdOptionMaxTimeMS == e.fieldNameStringData()) { - // no-op - } else if (str::equals("index", e.fieldName())) { - BSONObj indexObj = e.Obj(); - BSONObj keyPattern = indexObj.getObjectField("keyPattern"); + if (!cmr.collValidator.eoo()) + coll->setValidator(txn, cmr.collValidator.Obj()); - if (keyPattern.isEmpty()) { - errorStatus = Status(ErrorCodes::InvalidOptions, "no keyPattern specified"); - continue; - } + if (!cmr.collValidationAction.empty()) + coll->setValidationAction(txn, cmr.collValidationAction); - BSONElement newExpireSecs = indexObj["expireAfterSeconds"]; - if (newExpireSecs.eoo()) { - errorStatus = Status(ErrorCodes::InvalidOptions, "no expireAfterSeconds field"); - continue; - } - if (!newExpireSecs.isNumber()) { - errorStatus = - Status(ErrorCodes::InvalidOptions, "expireAfterSeconds field must be a number"); - continue; - } + if (!cmr.collValidationLevel.empty()) + coll->setValidationLevel(txn, cmr.collValidationLevel); - const IndexDescriptor* idx = - coll->getIndexCatalog()->findIndexByKeyPattern(txn, keyPattern); - if (idx == NULL) { - errorStatus = Status(ErrorCodes::InvalidOptions, - str::stream() << "cannot find index " << keyPattern - << " for ns " << nss.ns()); - continue; - } - BSONElement oldExpireSecs = idx->infoObj().getField("expireAfterSeconds"); - if (oldExpireSecs.eoo()) { - errorStatus = - Status(ErrorCodes::InvalidOptions, "no expireAfterSeconds field to update"); - continue; - } - if (!oldExpireSecs.isNumber()) { - errorStatus = Status(ErrorCodes::InvalidOptions, - "existing expireAfterSeconds field is not a number"); - continue; - } + auto setCollectionOption = [&](BSONElement& COElement) { + typedef CollectionOptions CO; + const StringData name = COElement.fieldNameStringData(); - if (oldExpireSecs != newExpireSecs) { - result->appendAs(oldExpireSecs, "expireAfterSeconds_old"); - // Change the value of "expireAfterSeconds" on disk. - coll->getCatalogEntry()->updateTTLSetting( - txn, idx->indexName(), newExpireSecs.numberLong()); - // Notify the index catalog that the definition of this index changed. - idx = coll->getIndexCatalog()->refreshEntry(txn, idx); - result->appendAs(newExpireSecs, "expireAfterSeconds_new"); - } - } else if (str::equals("validator", e.fieldName())) { - auto status = coll->setValidator(txn, e.Obj()); - if (!status.isOK()) - errorStatus = std::move(status); - } else if (str::equals("validationLevel", e.fieldName())) { - auto status = coll->setValidationLevel(txn, e.String()); - if (!status.isOK()) - errorStatus = std::move(status); - } else if (str::equals("validationAction", e.fieldName())) { - auto status = coll->setValidationAction(txn, e.String()); - if (!status.isOK()) - errorStatus = std::move(status); - } else { - // As of SERVER-17312 we only support these two options. When SERVER-17320 is - // resolved this will need to be enhanced to handle other options. - typedef CollectionOptions CO; - const StringData name = e.fieldNameStringData(); - const int flag = (name == "usePowerOf2Sizes") - ? CO::Flag_UsePowerOf2Sizes - : (name == "noPadding") ? CO::Flag_NoPadding : 0; - if (!flag) { - errorStatus = Status(ErrorCodes::InvalidOptions, - str::stream() << "unknown option to collMod: " << name); - continue; - } + int flag = (name == "usePowerOf2Sizes") ? CO::Flag_UsePowerOf2Sizes + : (name == "noPadding") ? CO::Flag_NoPadding : 0; - CollectionCatalogEntry* cce = coll->getCatalogEntry(); + CollectionCatalogEntry* cce = coll->getCatalogEntry(); - const int oldFlags = cce->getCollectionOptions(txn).flags; - const bool oldSetting = oldFlags & flag; - const bool newSetting = e.trueValue(); + const int oldFlags = cce->getCollectionOptions(txn).flags; + const bool oldSetting = oldFlags & flag; + const bool newSetting = COElement.trueValue(); - result->appendBool(name.toString() + "_old", oldSetting); - result->appendBool(name.toString() + "_new", newSetting); + result->appendBool(name.toString() + "_old", oldSetting); + result->appendBool(name.toString() + "_new", newSetting); - const int newFlags = newSetting ? (oldFlags | flag) // set flag - : (oldFlags & ~flag); // clear flag + const int newFlags = newSetting ? (oldFlags | flag) // set flag + : (oldFlags & ~flag); // clear flag - // NOTE we do this unconditionally to ensure that we note that the user has - // explicitly set flags, even if they are just setting the default. - cce->updateFlags(txn, newFlags); + // NOTE we do this unconditionally to ensure that we note that the user has + // explicitly set flags, even if they are just setting the default. + cce->updateFlags(txn, newFlags); - const CollectionOptions newOptions = cce->getCollectionOptions(txn); - invariant(newOptions.flags == newFlags); - invariant(newOptions.flagsSet); - } + const CollectionOptions newOptions = cce->getCollectionOptions(txn); + invariant(newOptions.flags == newFlags); + invariant(newOptions.flagsSet); + }; + + if (!cmr.usePowerOf2Sizes.eoo()) { + setCollectionOption(cmr.usePowerOf2Sizes); } - if (!errorStatus.isOK()) { - return errorStatus; + if (!cmr.noPadding.eoo()) { + setCollectionOption(cmr.noPadding); } getGlobalServiceContext()->getOpObserver()->onCollMod( txn, (dbName.toString() + ".$cmd").c_str(), cmdObj); wunit.commit(); + return Status::OK(); } } // namespace mongo diff --git a/src/mongo/db/catalog/coll_mod.h b/src/mongo/db/catalog/coll_mod.h index eb8644b74d1..bb86bbc50d1 100644 --- a/src/mongo/db/catalog/coll_mod.h +++ b/src/mongo/db/catalog/coll_mod.h @@ -27,13 +27,22 @@ */ #include "mongo/base/status.h" +#include "mongo/base/status_with.h" namespace mongo { class BSONObj; class BSONObjBuilder; +class Collection; class NamespaceString; class OperationContext; +struct CollModRequest; + +StatusWith parseCollModRequest(OperationContext* txn, + const NamespaceString& nss, + Collection* coll, + const BSONObj& cmdObj); + /** * Performs the collection modification described in "cmdObj" on the collection "ns". */ diff --git a/src/mongo/db/catalog/collection.cpp b/src/mongo/db/catalog/collection.cpp index 0043b20a4c5..04e94eab47d 100644 --- a/src/mongo/db/catalog/collection.cpp +++ b/src/mongo/db/catalog/collection.cpp @@ -180,9 +180,9 @@ Collection::Collection(OperationContext* txn, _validatorDoc(_details->getCollectionOptions(txn).validator.getOwned()), _validator(uassertStatusOK(parseValidator(_validatorDoc))), _validationAction(uassertStatusOK( - _parseValidationAction(_details->getCollectionOptions(txn).validationAction))), + parseValidationAction(_details->getCollectionOptions(txn).validationAction))), _validationLevel(uassertStatusOK( - _parseValidationLevel(_details->getCollectionOptions(txn).validationLevel))), + parseValidationLevel(_details->getCollectionOptions(txn).validationLevel))), _cursorManager(fullNS), _cappedNotifier(_recordStore->isCapped() ? new CappedInsertNotifier() : nullptr), _mustTakeCappedLockOnInsert(isCapped() && !_ns.isSystemDotProfile() && !_ns.isOplog()) { @@ -823,7 +823,7 @@ Status Collection::setValidator(OperationContext* txn, BSONObj validatorDoc) { return Status::OK(); } -StatusWith Collection::_parseValidationLevel(StringData newLevel) { +StatusWith Collection::parseValidationLevel(StringData newLevel) { if (newLevel == "") { // default return STRICT_V; @@ -839,7 +839,7 @@ StatusWith Collection::_parseValidationLevel(String } } -StatusWith Collection::_parseValidationAction(StringData newAction) { +StatusWith Collection::parseValidationAction(StringData newAction) { if (newAction == "") { // default return ERROR_V; @@ -878,7 +878,7 @@ StringData Collection::getValidationAction() const { Status Collection::setValidationLevel(OperationContext* txn, StringData newLevel) { invariant(txn->lockState()->isCollectionLockedForMode(ns().toString(), MODE_X)); - StatusWith status = _parseValidationLevel(newLevel); + StatusWith status = parseValidationLevel(newLevel); if (!status.isOK()) { return status.getStatus(); } @@ -893,7 +893,7 @@ Status Collection::setValidationLevel(OperationContext* txn, StringData newLevel Status Collection::setValidationAction(OperationContext* txn, StringData newAction) { invariant(txn->lockState()->isCollectionLockedForMode(ns().toString(), MODE_X)); - StatusWith status = _parseValidationAction(newAction); + StatusWith status = parseValidationAction(newAction); if (!status.isOK()) { return status.getStatus(); } @@ -1047,4 +1047,8 @@ Status Collection::touch(OperationContext* txn, return Status::OK(); } + +UpdateNotifier* Collection::getUpdateNotifier() { + return this; +} } diff --git a/src/mongo/db/catalog/collection.h b/src/mongo/db/catalog/collection.h index 6823045b952..8f516f72bbd 100644 --- a/src/mongo/db/catalog/collection.h +++ b/src/mongo/db/catalog/collection.h @@ -38,6 +38,7 @@ #include "mongo/base/status_with.h" #include "mongo/base/string_data.h" #include "mongo/bson/mutable/damage_vector.h" +#include "mongo/db/catalog/coll_mod.h" #include "mongo/db/catalog/collection_info_cache.h" #include "mongo/db/catalog/cursor_manager.h" #include "mongo/db/catalog/index_catalog.h" @@ -362,6 +363,16 @@ public: */ void temp_cappedTruncateAfter(OperationContext* txn, RecordId end, bool inclusive); + enum ValidationAction { WARN, ERROR_V }; + enum ValidationLevel { OFF, MODERATE, STRICT_V }; + + /** + * Returns a non-ok Status if validator is not legal for this collection. + */ + StatusWithMatchExpression parseValidator(const BSONObj& validator) const; + + static StatusWith parseValidationLevel(StringData); + static StatusWith parseValidationAction(StringData); /** * Sets the validator for this collection. * @@ -422,16 +433,17 @@ public: */ void notifyCappedWaitersIfNeeded(); -private: /** - * Returns a non-ok Status if document does not pass this collection's validator. + * This function is necessary for a 3.2 backport. We have a better fix for the + * underlying issue in later versions. */ - Status checkValidation(OperationContext* txn, const BSONObj& document) const; + UpdateNotifier* getUpdateNotifier(); +private: /** - * Returns a non-ok Status if validator is not legal for this collection. + * Returns a non-ok Status if document does not pass this collection's validator. */ - StatusWithMatchExpression parseValidator(const BSONObj& validator) const; + Status checkValidation(OperationContext* txn, const BSONObj& document) const; Status recordStoreGoingToMove(OperationContext* txn, const RecordId& oldLocation, @@ -470,11 +482,9 @@ private: BSONObj _validatorDoc; // Points into _validatorDoc. Null means no filter. std::unique_ptr _validator; - enum ValidationAction { WARN, ERROR_V } _validationAction; - enum ValidationLevel { OFF, MODERATE, STRICT_V } _validationLevel; - static StatusWith _parseValidationLevel(StringData); - static StatusWith _parseValidationAction(StringData); + ValidationAction _validationAction; + ValidationLevel _validationLevel; // this is mutable because read only users of the Collection class // use it keep state. This seems valid as const correctness of Collection diff --git a/src/mongo/db/catalog/database.cpp b/src/mongo/db/catalog/database.cpp index a54a5e4798c..13b4b7faa01 100644 --- a/src/mongo/db/catalog/database.cpp +++ b/src/mongo/db/catalog/database.cpp @@ -40,16 +40,14 @@ #include "mongo/db/audit.h" #include "mongo/db/auth/auth_index_d.h" #include "mongo/db/background.h" -#include "mongo/db/clientcursor.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/collection_catalog_entry.h" #include "mongo/db/catalog/collection_options.h" #include "mongo/db/catalog/database_catalog_entry.h" #include "mongo/db/catalog/database_holder.h" +#include "mongo/db/clientcursor.h" #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/dbhelpers.h" -#include "mongo/db/service_context.h" -#include "mongo/db/service_context_d.h" #include "mongo/db/index/index_access_method.h" #include "mongo/db/instance.h" #include "mongo/db/introspect.h" @@ -57,10 +55,12 @@ #include "mongo/db/repl/oplog.h" #include "mongo/db/repl/replication_coordinator_global.h" #include "mongo/db/server_parameters.h" +#include "mongo/db/service_context.h" +#include "mongo/db/service_context_d.h" #include "mongo/db/stats/top.h" -#include "mongo/db/storage/storage_options.h" -#include "mongo/db/storage/storage_engine.h" #include "mongo/db/storage/recovery_unit.h" +#include "mongo/db/storage/storage_engine.h" +#include "mongo/db/storage/storage_options.h" #include "mongo/util/log.h" namespace mongo { @@ -215,35 +215,6 @@ Database::Database(OperationContext* txn, StringData name, DatabaseCatalogEntry* } } - -/*static*/ -string Database::duplicateUncasedName(const string& name, set* duplicates) { - if (duplicates) { - duplicates->clear(); - } - - set allShortNames; - dbHolder().getAllShortNames(allShortNames); - - for (const auto& dbname : allShortNames) { - if (strcasecmp(dbname.c_str(), name.c_str())) - continue; - - if (strcmp(dbname.c_str(), name.c_str()) == 0) - continue; - - if (duplicates) { - duplicates->insert(dbname); - } else { - return dbname; - } - } - if (duplicates) { - return duplicates->empty() ? "" : *duplicates->begin(); - } - return ""; -} - void Database::clearTmpCollections(OperationContext* txn) { invariant(txn->lockState()->isDbLockedForMode(name(), MODE_X)); @@ -344,14 +315,8 @@ void Database::getStats(OperationContext* opCtx, BSONObjBuilder* output, double } Status Database::dropCollection(OperationContext* txn, StringData fullns) { - invariant(txn->lockState()->isDbLockedForMode(name(), MODE_X)); - - LOG(1) << "dropCollection: " << fullns << endl; - massertNamespaceNotIndex(fullns, "dropCollection"); - - Collection* collection = getCollection(fullns); - if (!collection) { - // collection doesn't exist + if (!getCollection(fullns)) { + // Collection doesn't exist so don't bother validating if it can be dropped. return Status::OK(); } @@ -370,9 +335,24 @@ Status Database::dropCollection(OperationContext* txn, StringData fullns) { } } + return dropCollectionEvenIfSystem(txn, nss); +} + +Status Database::dropCollectionEvenIfSystem(OperationContext* txn, const NamespaceString& fullns) { + invariant(txn->lockState()->isDbLockedForMode(name(), MODE_X)); + + LOG(1) << "dropCollection: " << fullns; + + Collection* collection = getCollection(fullns); + if (!collection) { + return Status::OK(); // Post condition already met. + } + + massertNamespaceNotIndex(fullns.toString(), "dropCollection"); + BackgroundOperation::assertNoBgOpInProgForNs(fullns); - audit::logDropCollection(&cc(), fullns); + audit::logDropCollection(&cc(), fullns.toString()); Status s = collection->getIndexCatalog()->dropAllIndexes(txn, true); if (!s.isOK()) { @@ -384,13 +364,13 @@ Status Database::dropCollection(OperationContext* txn, StringData fullns) { verify(collection->_details->getTotalIndexCount(txn) == 0); LOG(1) << "\t dropIndexes done" << endl; - Top::get(txn->getClient()->getServiceContext()).collectionDropped(fullns); + Top::get(txn->getClient()->getServiceContext()).collectionDropped(fullns.toString()); // We want to destroy the Collection object before telling the StorageEngine to destroy the // RecordStore. - _clearCollectionCache(txn, fullns, "collection dropped"); + _clearCollectionCache(txn, fullns.toString(), "collection dropped"); - s = _dbEntry->dropCollection(txn, fullns); + s = _dbEntry->dropCollection(txn, fullns.toString()); if (!s.isOK()) return s; @@ -407,7 +387,7 @@ Status Database::dropCollection(OperationContext* txn, StringData fullns) { } } - getGlobalServiceContext()->getOpObserver()->onDropCollection(txn, nss); + getGlobalServiceContext()->getOpObserver()->onDropCollection(txn, fullns); return Status::OK(); } diff --git a/src/mongo/db/catalog/database.h b/src/mongo/db/catalog/database.h index 53c3cb5712d..77b6ccc865b 100644 --- a/src/mongo/db/catalog/database.h +++ b/src/mongo/db/catalog/database.h @@ -144,7 +144,12 @@ public: const DatabaseCatalogEntry* getDatabaseCatalogEntry() const; + /** + * dropCollection() will refuse to drop system collections. Use dropCollectionEvenIfSystem() if + * that is required. + */ Status dropCollection(OperationContext* txn, StringData fullns); + Status dropCollectionEvenIfSystem(OperationContext* txn, const NamespaceString& fullns); Collection* createCollection(OperationContext* txn, StringData ns, @@ -167,15 +172,6 @@ public: StringData toNS, bool stayTemp); - /** - * @return name of an existing database with same text name but different - * casing, if one exists. Otherwise the empty std::string is returned. If - * 'duplicates' is specified, it is filled with all duplicate names. - // TODO move??? - */ - static std::string duplicateUncasedName(const std::string& name, - std::set* duplicates = 0); - static Status validateDBName(StringData dbname); const std::string& getSystemIndexesName() const { diff --git a/src/mongo/db/catalog/database_holder.cpp b/src/mongo/db/catalog/database_holder.cpp index f4a2cf62970..b6633f5c328 100644 --- a/src/mongo/db/catalog/database_holder.cpp +++ b/src/mongo/db/catalog/database_holder.cpp @@ -35,14 +35,15 @@ #include "mongo/db/audit.h" #include "mongo/db/auth/auth_index_d.h" #include "mongo/db/background.h" -#include "mongo/db/client.h" -#include "mongo/db/clientcursor.h" #include "mongo/db/catalog/database.h" #include "mongo/db/catalog/database_catalog_entry.h" -#include "mongo/db/service_context.h" +#include "mongo/db/client.h" +#include "mongo/db/clientcursor.h" #include "mongo/db/operation_context.h" +#include "mongo/db/service_context.h" #include "mongo/db/storage/storage_engine.h" #include "mongo/util/log.h" +#include "mongo/util/scopeguard.h" namespace mongo { @@ -91,56 +92,81 @@ Database* DatabaseHolder::get(OperationContext* txn, StringData ns) const { return NULL; } +std::set DatabaseHolder::_getNamesWithConflictingCasing_inlock(StringData name) { + std::set duplicates; + + for (const auto& nameAndPointer : _dbs) { + // A name that's equal with case-insensitive match must be identical, or it's a duplicate. + if (name.equalCaseInsensitive(nameAndPointer.first) && name != nameAndPointer.first) + duplicates.insert(nameAndPointer.first); + } + return duplicates; +} + +std::set DatabaseHolder::getNamesWithConflictingCasing(StringData name) { + stdx::lock_guard lk(_m); + return _getNamesWithConflictingCasing_inlock(name); +} + Database* DatabaseHolder::openDb(OperationContext* txn, StringData ns, bool* justCreated) { const StringData dbname = _todb(ns); invariant(txn->lockState()->isDbLockedForMode(dbname, MODE_X)); - Database* db = get(txn, ns); - if (db) { - if (justCreated) { - *justCreated = false; - } + if (justCreated) + *justCreated = false; // Until proven otherwise. - return db; - } + stdx::unique_lock lk(_m); - // Check casing - const string duplicate = Database::duplicateUncasedName(dbname.toString()); - if (!duplicate.empty()) { - stringstream ss; - ss << "db already exists with different case already have: [" << duplicate - << "] trying to create [" << dbname.toString() << "]"; - uasserted(ErrorCodes::DatabaseDifferCase, ss.str()); - } + // The following will insert a nullptr for dbname, which will treated the same as a non- + // existant database by the get method, yet still counts in getNamesWithConflictingCasing. + if (auto db = _dbs[dbname]) + return db; + // We've inserted a nullptr entry for dbname: make sure to remove it on unsuccessful exit. + auto removeDbGuard = MakeGuard([this, &lk, dbname] { + if (!lk.owns_lock()) + lk.lock(); + _dbs.erase(dbname); + }); + + // Check casing in lock to avoid transient duplicates. + auto duplicates = _getNamesWithConflictingCasing_inlock(dbname); + uassert(ErrorCodes::DatabaseDifferCase, + str::stream() << "db already exists with different case already have: [" + << *duplicates.cbegin() << "] trying to create [" << dbname.toString() + << "]", + duplicates.empty()); + + + // Do the catalog lookup and database creation outside of the scoped lock, because these may + // block. Only one thread can be inside this method for the same DB name, because of the + // requirement for X-lock on the database when we enter. So there is no way we can insert two + // different databases for the same name. + lk.unlock(); StorageEngine* storageEngine = getGlobalServiceContext()->getGlobalStorageEngine(); - invariant(storageEngine); - DatabaseCatalogEntry* entry = storageEngine->getDatabaseCatalogEntry(txn, dbname); - invariant(entry); - const bool exists = entry->exists(); - if (!exists) { - audit::logCreateDatabase(&cc(), dbname); - } - if (justCreated) { - *justCreated = !exists; + if (!entry->exists()) { + audit::logCreateDatabase(&cc(), dbname); + if (justCreated) + *justCreated = true; } - // Do this outside of the scoped lock, because database creation does transactional - // operations which may block. Only one thread can be inside this method for the same DB - // name, because of the requirement for X-lock on the database when we enter. So there is - // no way we can insert two different databases for the same name. - db = new Database(txn, dbname, entry); + auto newDb = stdx::make_unique(txn, dbname, entry); - stdx::lock_guard lk(_m); - _dbs[dbname] = db; + // Finally replace our nullptr entry with the new Database pointer. + removeDbGuard.Dismiss(); + lk.lock(); + auto it = _dbs.find(dbname); + invariant(it != _dbs.end() && it->second == nullptr); + Database* newDbPointer = newDb.release(); + _dbs[dbname] = newDbPointer; + invariant(_getNamesWithConflictingCasing_inlock(dbname.toString()).empty()); - return db; + return newDbPointer; } void DatabaseHolder::close(OperationContext* txn, StringData ns) { - // TODO: This should be fine if only a DB X-lock invariant(txn->lockState()->isW()); const StringData dbName = _todb(ns); diff --git a/src/mongo/db/catalog/database_holder.h b/src/mongo/db/catalog/database_holder.h index ce99747937b..91850197771 100644 --- a/src/mongo/db/catalog/database_holder.h +++ b/src/mongo/db/catalog/database_holder.h @@ -29,6 +29,7 @@ #pragma once #include +#include #include "mongo/base/string_data.h" #include "mongo/db/namespace_string.h" @@ -87,9 +88,15 @@ public: } } + /** + * Returns the set of existing database names that differ only in casing. + */ + std::set getNamesWithConflictingCasing(StringData name); + private: - typedef StringMap DBs; + std::set _getNamesWithConflictingCasing_inlock(StringData name); + typedef StringMap DBs; mutable SimpleMutex _m; DBs _dbs; }; diff --git a/src/mongo/db/catalog/index_catalog.cpp b/src/mongo/db/catalog/index_catalog.cpp index b572a66426e..be51e69ef83 100644 --- a/src/mongo/db/catalog/index_catalog.cpp +++ b/src/mongo/db/catalog/index_catalog.cpp @@ -133,6 +133,17 @@ IndexCatalogEntry* IndexCatalog::_setupInMemoryStructures(OperationContext* txn, Status status = _isSpecOk(descriptor->infoObj()); if (!status.isOK() && status != ErrorCodes::IndexAlreadyExists) { + if (_collection->ns().ns() == "admin.system.version") { + auto indexNameElem = descriptor->infoObj()["name"]; + + if (indexNameElem.type() == BSONType::String && + indexNameElem.valueStringData() == StringData("incompatible_with_version_32")) { + severe() + << "Cannot start mongod when the featureCompatibilityVersion is higher than " + "3.2. See http://dochub.mongodb.org/core/3.4-feature-compatibility."; + fassertFailedNoTrace(40352); + } + } severe() << "Found an invalid index " << descriptor->infoObj() << " on the " << _collection->ns().ns() << " collection: " << status.reason(); fassertFailedNoTrace(28782); @@ -393,6 +404,8 @@ void IndexCatalog::IndexBuildBlock::fail() { void IndexCatalog::IndexBuildBlock::success() { Collection* collection = _catalog->_collection; fassert(17207, collection->ok()); + NamespaceString ns(_indexNamespace); + invariant(_txn->lockState()->isDbLockedForMode(ns.db(), MODE_X)); collection->getCatalogEntry()->indexBuildSuccess(_txn, _indexName); @@ -402,6 +415,8 @@ void IndexCatalog::IndexBuildBlock::success() { fassert(17331, entry && entry == _entry); OperationContext* txn = _txn; + LOG(2) << "marking index " << _indexName << " as ready in snapshot id " + << txn->recoveryUnit()->getSnapshotId(); _txn->recoveryUnit()->onCommit([txn, entry, collection] { // Note: this runs after the WUOW commits but before we release our X lock on the // collection. This means that any snapshot created after this must include the full index, @@ -842,13 +857,12 @@ Status IndexCatalog::_dropIndex(OperationContext* txn, IndexCatalogEntry* entry) invariant(_entries.release(entry->descriptor()) == entry); txn->recoveryUnit()->registerChange(new IndexRemoveChange(txn, _collection, &_entries, entry)); + _collection->infoCache()->droppedIndex(txn, indexName); entry = NULL; _deleteIndexFromDisk(txn, indexName, indexNamespace); _checkMagic(); - _collection->infoCache()->droppedIndex(txn, indexName); - return Status::OK(); } @@ -1154,9 +1168,12 @@ Status IndexCatalog::_unindexRecord(OperationContext* txn, options.logIfError = logIfError; options.dupsAllowed = isDupsAllowed(index->descriptor()); - // For unindex operations, dupsAllowed=false really means that it is safe to delete anything - // that matches the key, without checking the RecordID, since dups are impossible. We need - // to disable this behavior for in-progress indexes. See SERVER-17487 for more details. + // On WiredTiger, we do blind unindexing of records for efficiency. However, when duplicates + // are allowed in unique indexes, WiredTiger does not do blind unindexing, and instead confirms + // that the recordid matches the element we are removing. + // We need to disable blind-deletes for in-progress indexes, in order to force recordid-matching + // for unindex operations, since initial sync can build an index over a collection with + // duplicates. See SERVER-17487 for more details. options.dupsAllowed = options.dupsAllowed || !index->isReady(txn); int64_t removed; diff --git a/src/mongo/db/catalog/index_create.cpp b/src/mongo/db/catalog/index_create.cpp index b18e90ac880..a5cbd7a920b 100644 --- a/src/mongo/db/catalog/index_create.cpp +++ b/src/mongo/db/catalog/index_create.cpp @@ -47,6 +47,7 @@ #include "mongo/db/query/internal_plans.h" #include "mongo/db/repl/replication_coordinator_global.h" #include "mongo/db/operation_context.h" +#include "mongo/db/server_parameters.h" #include "mongo/stdx/mutex.h" #include "mongo/util/fail_point.h" #include "mongo/util/fail_point_service.h" @@ -63,6 +64,30 @@ using std::endl; MONGO_FP_DECLARE(crashAfterStartingIndexBuild); +std::atomic maxIndexBuildMemoryUsageMegabytes(500); // NOLINT + +class ExportedMaxIndexBuildMemoryUsageParameter + : public ExportedServerParameter { +public: + ExportedMaxIndexBuildMemoryUsageParameter() + : ExportedServerParameter( + ServerParameterSet::getGlobal(), + "maxIndexBuildMemoryUsageMegabytes", + &maxIndexBuildMemoryUsageMegabytes) {} + + virtual Status validate(const std::int32_t& potentialNewValue) { + if (potentialNewValue < 100) { + return Status( + ErrorCodes::BadValue, + "maxIndexBuildMemoryUsageMegabytes must be greater than or equal to 100 MB"); + } + + return Status::OK(); + } + +} exportedMaxIndexBuildMemoryUsageParameter; + + /** * On rollback sets MultiIndexBlock::_needToCleanup to true. */ @@ -169,6 +194,14 @@ Status MultiIndexBlock::init(const std::vector& indexSpecs) { _buildInBackground = (_buildInBackground && info["background"].trueValue()); } + std::vector indexInfoObjs; + indexInfoObjs.reserve(indexSpecs.size()); + std::size_t eachIndexBuildMaxMemoryUsageBytes = 0; + if (!indexSpecs.empty()) { + eachIndexBuildMaxMemoryUsageBytes = + std::size_t(maxIndexBuildMemoryUsageMegabytes) * 1024 * 1024 / indexSpecs.size(); + } + for (size_t i = 0; i < indexSpecs.size(); i++) { BSONObj info = indexSpecs[i]; StatusWith statusWithInfo = @@ -192,7 +225,7 @@ Status MultiIndexBlock::init(const std::vector& indexSpecs) { if (!_buildInBackground) { // Bulk build process requires foreground building as it assumes nothing is changing // under it. - index.bulk = index.real->initiateBulk(); + index.bulk = index.real->initiateBulk(eachIndexBuildMaxMemoryUsageBytes); } const IndexDescriptor* descriptor = index.block->getEntry()->descriptor(); @@ -203,7 +236,8 @@ Status MultiIndexBlock::init(const std::vector& indexSpecs) { log() << "build index on: " << ns << " properties: " << descriptor->toString(); if (index.bulk) - log() << "\t building index using bulk method"; + log() << "\t building index using bulk method; build may temporarily use up to " + << eachIndexBuildMaxMemoryUsageBytes / 1024 / 1024 << " megabytes of RAM"; index.filterExpression = index.block->getEntry()->getFilterExpression(); diff --git a/src/mongo/db/catalog/rename_collection.cpp b/src/mongo/db/catalog/rename_collection.cpp index 6607f49e98f..d8e8cfe0c85 100644 --- a/src/mongo/db/catalog/rename_collection.cpp +++ b/src/mongo/db/catalog/rename_collection.cpp @@ -41,6 +41,8 @@ #include "mongo/db/catalog/index_catalog.h" #include "mongo/db/catalog/index_create.h" #include "mongo/db/client.h" +#include "mongo/db/concurrency/write_conflict_exception.h" +#include "mongo/db/curop.h" #include "mongo/db/db_raii.h" #include "mongo/db/index/index_descriptor.h" #include "mongo/db/index_builder.h" @@ -115,7 +117,7 @@ Status renameCollection(OperationContext* txn, Database* const targetDB = dbHolder().openDb(txn, target.db()); - { + MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { WriteUnitOfWork wunit(txn); // Check if the target namespace exists and if dropTarget is true. @@ -149,6 +151,7 @@ Status renameCollection(OperationContext* txn, wunit.commit(); } + MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "renameCollection", target.ns()); // If we get here, we are renaming across databases, so we must copy all the data and // indexes, then remove the source collection. diff --git a/src/mongo/db/commands/apply_ops.cpp b/src/mongo/db/commands/apply_ops.cpp index f4f0b930a8c..77e89db3aca 100644 --- a/src/mongo/db/commands/apply_ops.cpp +++ b/src/mongo/db/commands/apply_ops.cpp @@ -121,22 +121,31 @@ public: txn->setWriteConcern(wcResult.getValue()); setupSynchronousCommit(txn); + auto applyOpsRes = [&]() { + try { + // Note: this scope guard must go out of scope before the waitForWriteConcern below! + auto client = txn->getClient(); + auto lastOpAtOperationStart = repl::ReplClientInfo::forClient(client).getLastOp(); + ScopeGuard lastOpSetterGuard = + MakeObjGuard(repl::ReplClientInfo::forClient(client), + &repl::ReplClientInfo::setLastOpToSystemLastOpTime, + txn); + + auto res = appendCommandStatus(result, applyOps(txn, dbname, cmdObj, &result)); + + if (repl::ReplClientInfo::forClient(client).getLastOp() != lastOpAtOperationStart) { + // If this operation has already generated a new lastOp, don't bother setting it + // here. No-op applyOps will not generate a new lastOp, so we still need the + // guard to fire in that case. + lastOpSetterGuard.Dismiss(); + } - auto client = txn->getClient(); - auto lastOpAtOperationStart = repl::ReplClientInfo::forClient(client).getLastOp(); - ScopeGuard lastOpSetterGuard = - MakeObjGuard(repl::ReplClientInfo::forClient(client), - &repl::ReplClientInfo::setLastOpToSystemLastOpTime, - txn); - - auto applyOpsStatus = appendCommandStatus(result, applyOps(txn, dbname, cmdObj, &result)); - - if (repl::ReplClientInfo::forClient(client).getLastOp() != lastOpAtOperationStart) { - // If this operation has already generated a new lastOp, don't bother setting it - // here. No-op applyOps will not generate a new lastOp, so we still need the guard to - // fire in that case. - lastOpSetterGuard.Dismiss(); - } + return res; + } catch (const DBException& e) { + appendCommandStatus(result, e.toStatus()); + return false; + } + }(); WriteConcernResult res; auto waitForWCStatus = @@ -146,7 +155,7 @@ public: &res); appendCommandWCStatus(result, waitForWCStatus); - return applyOpsStatus; + return applyOpsRes; } private: diff --git a/src/mongo/db/commands/find_cmd.cpp b/src/mongo/db/commands/find_cmd.cpp index ddd40f0ed09..4e33d11ef2f 100644 --- a/src/mongo/db/commands/find_cmd.cpp +++ b/src/mongo/db/commands/find_cmd.cpp @@ -41,17 +41,17 @@ #include "mongo/db/commands.h" #include "mongo/db/db_raii.h" #include "mongo/db/exec/working_set_common.h" -#include "mongo/db/service_context.h" #include "mongo/db/matcher/extensions_callback_real.h" -#include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/query/cursor_response.h" #include "mongo/db/query/explain.h" #include "mongo/db/query/find.h" #include "mongo/db/query/find_common.h" #include "mongo/db/query/get_executor.h" +#include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/s/operation_shard_version.h" #include "mongo/db/s/sharding_state.h" #include "mongo/db/server_parameters.h" +#include "mongo/db/service_context.h" #include "mongo/db/stats/counters.h" #include "mongo/s/stale_exception.h" #include "mongo/util/log.h" @@ -243,37 +243,34 @@ public: ShardingState* const shardingState = ShardingState::get(txn); - if (OperationShardVersion::get(txn).hasShardVersion() && shardingState->enabled()) { - ChunkVersion receivedVersion = OperationShardVersion::get(txn).getShardVersion(nss); - ChunkVersion latestVersion; - // Wait for migration completion to get the correct chunk version. + // Parse, canonicalize, plan, transcribe, and get a plan executor. + boost::optional optionalCtx; + try { + optionalCtx.emplace(txn, nss); + } catch (const StaleConfigException& sce) { + // Wait for migration completion to get the correct chunk version const int maxTimeoutSec = 30; int timeoutSec = cq->getParsed().getMaxTimeMS() / 1000; if (!timeoutSec || timeoutSec > maxTimeoutSec) { timeoutSec = maxTimeoutSec; } - if (!shardingState->waitTillNotInCriticalSection(timeoutSec)) { - uasserted(ErrorCodes::LockTimeout, "Timeout while waiting for migration commit"); + if (shardingState->waitTillNotInCriticalSection(maxTimeoutSec)) { + ChunkVersion unused; + shardingState->refreshMetadataIfNeeded( + txn, nss.ns(), sce.getVersionReceived(), &unused); } - - // If the received version is newer than the version cached in 'shardingState', then we - // have to refresh 'shardingState' from the config servers. We do this before acquiring - // locks so that we don't hold locks while waiting on the network. - uassertStatusOK(shardingState->refreshMetadataIfNeeded( - txn, nss.ns(), receivedVersion, &latestVersion)); + throw; } - // Acquire locks. - AutoGetCollectionForRead ctx(txn, nss); + const auto& ctx = *optionalCtx; Collection* collection = ctx.getCollection(); const int dbProfilingLevel = ctx.getDb() ? ctx.getDb()->getProfilingLevel() : serverGlobalParams.defaultProfile; - // It is possible that the sharding version will change during yield while we are - // retrieving a plan executor. If this happens we will throw an error and mongos will - // retry. + // It is possible that the sharding version will change during yield while we are retrieving + // a plan executor. If this happens we will throw an error and mongos will retry. const ChunkVersion shardingVersionAtStart = shardingState->getVersion(nss.ns()); // Get the execution plan for the query. diff --git a/src/mongo/db/commands/generic.cpp b/src/mongo/db/commands/generic.cpp index 47f47442452..bc8963273da 100644 --- a/src/mongo/db/commands/generic.cpp +++ b/src/mongo/db/commands/generic.cpp @@ -32,6 +32,7 @@ #include +#include "mongo/bson/util/bson_extract.h" #include "mongo/bson/util/builder.h" #include "mongo/client/dbclient_rs.h" #include "mongo/db/auth/action_set.h" @@ -448,6 +449,61 @@ public: } getLogCmd; +class ClearLogCmd : public Command { +public: + ClearLogCmd() : Command("clearLog") {} + + virtual bool slaveOk() const { + return true; + } + virtual bool adminOnly() const { + return true; + } + virtual bool isWriteCommandForConfigServer() const { + return false; + } + Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) override { + // No access control needed since this command is a testing-only command that must be + // enabled at the command line. + return Status::OK(); + } + virtual void help(stringstream& help) const { + help << "{ clearLog : 'global' }"; + } + + virtual bool run(OperationContext* opCtx, + const string& dbname, + BSONObj& cmdObj, + int, + string& errmsg, + BSONObjBuilder& result) { + std::string logName; + Status status = bsonExtractStringField(cmdObj, "clearLog", &logName); + if (!status.isOK()) { + return appendCommandStatus(result, status); + } + + if (logName != "global") { + return appendCommandStatus( + result, Status(ErrorCodes::InvalidOptions, "Only the 'global' log can be cleared")); + } + RamLog* ramlog = RamLog::getIfExists(logName); + invariant(ramlog); + ramlog->clear(); + return true; + } +}; + +MONGO_INITIALIZER(RegisterClearLogCmd)(InitializerContext* context) { + if (Command::testCommandsEnabled) { + // Leaked intentionally: a Command registers itself when constructed. + new ClearLogCmd(); + } + return Status::OK(); +} + class CmdGetCmdLineOpts : Command { public: CmdGetCmdLineOpts() : Command("getCmdLineOpts") {} diff --git a/src/mongo/db/commands/getmore_cmd.cpp b/src/mongo/db/commands/getmore_cmd.cpp index bb545b2ebff..5aca8c8ad1b 100644 --- a/src/mongo/db/commands/getmore_cmd.cpp +++ b/src/mongo/db/commands/getmore_cmd.cpp @@ -301,7 +301,7 @@ public: auto planSummary = Explain::getPlanSummary(exec); { - stdx::lock_guard(*txn->getClient()); + stdx::lock_guard lk(*txn->getClient()); CurOp::get(txn)->setPlanSummary_inlock(planSummary); } diff --git a/src/mongo/db/commands/list_databases.cpp b/src/mongo/db/commands/list_databases.cpp index 93d07479b8f..ee049091efc 100644 --- a/src/mongo/db/commands/list_databases.cpp +++ b/src/mongo/db/commands/list_databases.cpp @@ -37,6 +37,9 @@ #include "mongo/db/storage/storage_engine.h" namespace mongo { +namespace { +static const StringData kNameOnlyField{"nameOnly"}; +} // namespace using std::set; using std::string; @@ -61,7 +64,8 @@ public: return false; } virtual void help(stringstream& help) const { - help << "list databases on this server"; + help << "{ listDatabases:1, [filter: ] [, nameOnly: true ] }\n" + "list databases on this server"; } virtual void addRequiredPrivileges(const std::string& dbname, const BSONObj& cmdObj, @@ -79,6 +83,8 @@ public: int, string& errmsg, BSONObjBuilder& result) { + bool nameOnly = jsobj[kNameOnlyField].trueValue(); + vector dbNames; StorageEngine* storageEngine = getGlobalServiceContext()->getGlobalStorageEngine(); storageEngine->listDatabases(&dbNames); @@ -93,7 +99,7 @@ public: BSONObjBuilder b; b.append("name", dbname); - { + if (!nameOnly) { ScopedTransaction transaction(txn, MODE_IS); Lock::DBLock dbLock(txn->lockState(), dbname, MODE_IS); @@ -117,7 +123,9 @@ public: } result.append("databases", dbInfos); - result.append("totalSize", double(totalSize)); + if (!nameOnly) { + result.append("totalSize", double(totalSize)); + } return true; } } cmdListDatabases; diff --git a/src/mongo/db/concurrency/lock_manager.cpp b/src/mongo/db/concurrency/lock_manager.cpp index 6d2450086fe..b8ab480e6a2 100644 --- a/src/mongo/db/concurrency/lock_manager.cpp +++ b/src/mongo/db/concurrency/lock_manager.cpp @@ -754,6 +754,7 @@ void LockManager::_onLockModeChanged(LockHead* lock, bool checkConflictQueue) { LockRequest* iterNext = NULL; + bool newlyCompatibleFirst = false; // Set on enabling compatibleFirst mode. for (LockRequest* iter = lock->conflictList._front; (iter != NULL) && checkConflictQueue; iter = iterNext) { invariant(iter->status == LockRequest::STATUS_WAITING); @@ -765,11 +766,11 @@ void LockManager::_onLockModeChanged(LockHead* lock, bool checkConflictQueue) { if (conflicts(iter->mode, lock->grantedModes)) { // If iter doesn't have a previous pointer, this means that it is at the front of the // queue. If we continue scanning the queue beyond this point, we will starve it by - // granting more and more requests. - if (!iter->prev) { + // granting more and more requests. However, if we newly transition to compatibleFirst + // mode, grant any waiting compatible requests. + if (!iter->prev && !newlyCompatibleFirst) { break; } - continue; } @@ -782,13 +783,13 @@ void LockManager::_onLockModeChanged(LockHead* lock, bool checkConflictQueue) { lock->decConflictModeCount(iter->mode); if (iter->compatibleFirst) { - lock->compatibleFirstCount++; + newlyCompatibleFirst |= (lock->compatibleFirstCount++ == 0); } iter->notify->notify(lock->resourceId, LOCK_OK); - // Small optimization - nothing is compatible with MODE_X, so no point in looking - // further in the conflict queue. + // Small optimization - nothing is compatible with a newly granted MODE_X, so no point in + // looking further in the conflict queue. Conflicting MODE_X requests are skipped above. if (iter->mode == MODE_X) { break; } diff --git a/src/mongo/db/concurrency/lock_manager_test.cpp b/src/mongo/db/concurrency/lock_manager_test.cpp index 218dd6227ad..4fcda52ae89 100644 --- a/src/mongo/db/concurrency/lock_manager_test.cpp +++ b/src/mongo/db/concurrency/lock_manager_test.cpp @@ -26,6 +26,7 @@ * it in the license file. */ +#include "mongo/db/concurrency/lock_manager_defs.h" #include "mongo/db/concurrency/lock_manager_test_help.h" #include "mongo/unittest/unittest.h" @@ -743,6 +744,66 @@ TEST(LockManager, CompatibleFirstImmediateGrant) { ASSERT(lockMgr.unlock(&requestX)); } +TEST(LockManager, CompatibleFirstGrantAlreadyQueued) { + LockManager lockMgr; + const ResourceId resId(RESOURCE_GLOBAL, 0); + + // This tests the following behaviors (alternatives indicated with '|'): + // Lock held in X, queue: S X|IX IS, where S is compatibleFirst. + // Once X unlocks|downgrades both the S and IS requests should proceed. + + + enum UnblockMethod { kDowngrading, kUnlocking }; + LockMode conflictingModes[2] = {MODE_IX, MODE_X}; + UnblockMethod unblockMethods[2] = {kDowngrading, kUnlocking}; + + for (LockMode writerMode : conflictingModes) { + for (UnblockMethod unblockMethod : unblockMethods) { + MMAPV1LockerImpl locker1; + LockRequestCombo request1(&locker1); + + MMAPV1LockerImpl locker2; + LockRequestCombo request2(&locker2); + request2.compatibleFirst = true; + + MMAPV1LockerImpl locker3; + LockRequestCombo request3(&locker3); + + MMAPV1LockerImpl locker4; + LockRequestCombo request4(&locker4); + + // Hold the lock in X and establish the S IX|X IS queue. + ASSERT(LOCK_OK == lockMgr.lock(resId, &request1, MODE_X)); + ASSERT(LOCK_WAITING == lockMgr.lock(resId, &request2, MODE_S)); + ASSERT(LOCK_WAITING == lockMgr.lock(resId, &request3, writerMode)); + ASSERT(LOCK_WAITING == lockMgr.lock(resId, &request4, MODE_IS)); + + // Now unlock the initial X, so all readers should be able to proceed, while the writer + // remains queued. + if (unblockMethod == kUnlocking) { + ASSERT(lockMgr.unlock(&request1)); + } else { + invariant(unblockMethod == kDowngrading); + lockMgr.downgrade(&request1, MODE_S); + } + ASSERT(request2.lastResult == LOCK_OK); + ASSERT(request3.lastResult == LOCK_INVALID); + ASSERT(request4.lastResult == LOCK_OK); + + // Now unlock the readers, and the writer succeeds as well. + ASSERT(lockMgr.unlock(&request2)); + ASSERT(lockMgr.unlock(&request4)); + if (unblockMethod == kDowngrading) { + ASSERT(lockMgr.unlock(&request1)); + } + ASSERT(request3.lastResult == LOCK_OK); + + // Unlock the writer + ASSERT(lockMgr.unlock(&request3)); + } + } +} + TEST(LockManager, CompatibleFirstDelayedGrant) { LockManager lockMgr; const ResourceId resId(RESOURCE_GLOBAL, 0); diff --git a/src/mongo/db/concurrency/lock_state_test.cpp b/src/mongo/db/concurrency/lock_state_test.cpp index ec44c9dc096..c3a29275416 100644 --- a/src/mongo/db/concurrency/lock_state_test.cpp +++ b/src/mongo/db/concurrency/lock_state_test.cpp @@ -30,11 +30,13 @@ #include "mongo/platform/basic.h" +#include #include #include #include "mongo/config.h" #include "mongo/db/concurrency/lock_manager_test_help.h" +#include "mongo/db/concurrency/locker.h" #include "mongo/unittest/unittest.h" #include "mongo/util/log.h" #include "mongo/util/timer.h" @@ -361,5 +363,85 @@ TEST(Locker, PerformanceLocker) { } #endif // MONGO_CONFIG_DEBUG_BUILD +namespace { +/** + * Helper function to determine if 'lockerInfo' contains a lock with ResourceId 'resourceId' and + * lock mode 'mode' within 'lockerInfo.locks'. + */ +bool lockerInfoContainsLock(const Locker::LockerInfo& lockerInfo, + const ResourceId& resourceId, + const LockMode& mode) { + return (1U == std::count_if(lockerInfo.locks.begin(), + lockerInfo.locks.end(), + [&resourceId, &mode](const Locker::OneLock& lock) { + return lock.resourceId == resourceId && lock.mode == mode; + })); +} +} // namespace + +TEST(LockerImpl, GetLockerInfoShouldReportHeldLocks) { + const ResourceId globalId(RESOURCE_GLOBAL, ResourceId::SINGLETON_GLOBAL); + const ResourceId dbId(RESOURCE_DATABASE, std::string("TestDB")); + const ResourceId collectionId(RESOURCE_COLLECTION, std::string("TestDB.collection")); + + // Take an exclusive lock on the collection. + DefaultLockerImpl locker; + ASSERT_EQ(LOCK_OK, locker.lockGlobal(MODE_IX)); + ASSERT_EQ(LOCK_OK, locker.lock(dbId, MODE_IX)); + ASSERT_EQ(LOCK_OK, locker.lock(collectionId, MODE_X)); + + // Assert it shows up in the output of getLockerInfo(). + Locker::LockerInfo lockerInfo; + locker.getLockerInfo(&lockerInfo); + + ASSERT(lockerInfoContainsLock(lockerInfo, globalId, MODE_IX)); + ASSERT(lockerInfoContainsLock(lockerInfo, dbId, MODE_IX)); + ASSERT(lockerInfoContainsLock(lockerInfo, collectionId, MODE_X)); + ASSERT_EQ(3U, lockerInfo.locks.size()); + + ASSERT(locker.unlockAll()); +} + +TEST(LockerImpl, GetLockerInfoShouldReportPendingLocks) { + const ResourceId globalId(RESOURCE_GLOBAL, ResourceId::SINGLETON_GLOBAL); + const ResourceId dbId(RESOURCE_DATABASE, std::string("TestDB")); + const ResourceId collectionId(RESOURCE_COLLECTION, std::string("TestDB.collection")); + + // Take an exclusive lock on the collection. + DefaultLockerImpl successfulLocker; + ASSERT_EQ(LOCK_OK, successfulLocker.lockGlobal(MODE_IX)); + ASSERT_EQ(LOCK_OK, successfulLocker.lock(dbId, MODE_IX)); + ASSERT_EQ(LOCK_OK, successfulLocker.lock(collectionId, MODE_X)); + + // Now attempt to get conflicting locks. + DefaultLockerImpl conflictingLocker; + ASSERT_EQ(LOCK_OK, conflictingLocker.lockGlobal(MODE_IS)); + ASSERT_EQ(LOCK_OK, conflictingLocker.lock(dbId, MODE_IS)); + ASSERT_EQ(LOCK_WAITING, conflictingLocker.lockBegin(collectionId, MODE_IS)); + + // Assert the held locks show up in the output of getLockerInfo(). + Locker::LockerInfo lockerInfo; + conflictingLocker.getLockerInfo(&lockerInfo); + ASSERT(lockerInfoContainsLock(lockerInfo, globalId, MODE_IS)); + ASSERT(lockerInfoContainsLock(lockerInfo, dbId, MODE_IS)); + ASSERT(lockerInfoContainsLock(lockerInfo, collectionId, MODE_IS)); + ASSERT_EQ(3U, lockerInfo.locks.size()); + + // Assert it reports that it is waiting for the collection lock. + ASSERT_EQ(collectionId, lockerInfo.waitingResource); + + // Make sure it no longer reports waiting once unlocked. + ASSERT(successfulLocker.unlockAll()); + + const unsigned timeoutMs = 0; + const bool checkDeadlock = false; + ASSERT_EQ(LOCK_OK, + conflictingLocker.lockComplete(collectionId, MODE_IS, timeoutMs, checkDeadlock)); + + conflictingLocker.getLockerInfo(&lockerInfo); + ASSERT_FALSE(lockerInfo.waitingResource.isValid()); + + ASSERT(conflictingLocker.unlockAll()); +} } // namespace mongo diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index e18b60efb17..6b0dda65150 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -149,7 +149,10 @@ void (*snmpInit)() = NULL; extern int diagLogging; -static const NamespaceString startupLogCollectionName("local.startup_log"); +namespace { + +const NamespaceString startupLogCollectionName("local.startup_log"); +const NamespaceString kSystemReplSetCollection("local.system.replset"); #ifdef _WIN32 ntservice::NtServiceDefaultStrings defaultServiceStrings = { @@ -215,7 +218,7 @@ public: } }; -static void logStartup(OperationContext* txn) { +void logStartup(OperationContext* txn) { BSONObjBuilder toLog; stringstream id; id << getHostNameCached() << "-" << jsTime().asInt64(); @@ -255,7 +258,7 @@ static void logStartup(OperationContext* txn) { wunit.commit(); } -static void checkForIdIndexes(OperationContext* txn, Database* db) { +void checkForIdIndexes(OperationContext* txn, Database* db) { if (db->name() == "local") { // we do not need an _id index on anything in the local database return; @@ -293,13 +296,10 @@ static void checkForIdIndexes(OperationContext* txn, Database* db) { * @returns the number of documents in local.system.replset or 0 if this was started with * --replset. */ -static unsigned long long checkIfReplMissingFromCommandLine(OperationContext* txn) { - // This is helpful for the query below to work as you can't open files when readlocked - ScopedTransaction transaction(txn, MODE_X); - Lock::GlobalWrite lk(txn->lockState()); +unsigned long long checkIfReplMissingFromCommandLine(OperationContext* txn) { if (!repl::getGlobalReplicationCoordinator()->getSettings().usingReplSets()) { DBDirectClient c(txn); - return c.count("local.system.replset"); + return c.count(kSystemReplSetCollection.ns()); } return 0; } @@ -311,7 +311,7 @@ static unsigned long long checkIfReplMissingFromCommandLine(OperationContext* tx * start up or get promoted to be replica set primaries, newer nodes clear the temp flags left by * these versions. */ -static bool isSubjectToSERVER23299(OperationContext* txn) { +bool isSubjectToSERVER23299(OperationContext* txn) { dbHolder().openDb(txn, startupLogCollectionName.db()); AutoGetCollectionForRead autoColl(txn, startupLogCollectionName); // No startup log or an empty one means either that the user was not running an affected @@ -353,7 +353,7 @@ static bool isSubjectToSERVER23299(OperationContext* txn) { return true; } -static void handleSERVER23299ForDb(OperationContext* txn, Database* db) { +void handleSERVER23299ForDb(OperationContext* txn, Database* db) { log() << "Scanning " << db->name() << " db for SERVER-23299 eligibility"; const auto dbEntry = db->getDatabaseCatalogEntry(); list collNames; @@ -374,7 +374,22 @@ static void handleSERVER23299ForDb(OperationContext* txn, Database* db) { log() << "Done scanning " << db->name() << " for SERVER-23299 eligibility"; } -static void repairDatabasesAndCheckVersion(OperationContext* txn) { +/** + * Check that the oplog is capped, and abort the process if it is not. + * Caller must lock DB before calling this function. + */ +void checkForCappedOplog(OperationContext* txn, Database* db) { + const NamespaceString oplogNss(repl::rsOplogName); + invariant(txn->lockState()->isDbLockedForMode(oplogNss.db(), MODE_IS)); + Collection* oplogCollection = db->getCollection(oplogNss); + if (oplogCollection && !oplogCollection->isCapped()) { + severe() << "The oplog collection " << oplogNss + << " is not capped; a capped oplog is a requirement for replication to function."; + fassertFailedNoTrace(40116); + } +} + +void repairDatabasesAndCheckVersion(OperationContext* txn) { LOG(1) << "enter repairDatabases (to check pdfile version #)" << endl; ScopedTransaction transaction(txn, MODE_X); @@ -397,6 +412,13 @@ static void repairDatabasesAndCheckVersion(OperationContext* txn) { const repl::ReplSettings& replSettings = repl::getGlobalReplicationCoordinator()->getSettings(); + // We open the "local" database before calling checkIfReplMissingFromCommandLine() to ensure the + // in-memory catalog entries for the 'kSystemReplSetCollection' collection have been populated + // if the collection exists. If the "local" database didn't exist at this point yet, then it + // will be created. + Lock::DBLock dbLock(txn->lockState(), kSystemReplSetCollection.db(), MODE_X); + dbHolder().openDb(txn, kSystemReplSetCollection.db()); + // On replica set members we only clear temp collections on DBs other than "local" during // promotion to primary. On pure slaves, they are only cleared when the oplog tells them // to. The local DB is special because it is not replicated. See SERVER-10927 for more @@ -480,6 +502,11 @@ static void repairDatabasesAndCheckVersion(OperationContext* txn) { if (replSettings.usingReplSets()) { // We only care about the _id index if we are in a replset checkForIdIndexes(txn, db); + // Ensure oplog is capped (mmap does not guarantee order of inserts on noncapped + // collections) + if (db->name() == "local") { + checkForCappedOplog(txn, db); + } } if (shouldDoCleanupForSERVER23299) { @@ -503,7 +530,7 @@ static void repairDatabasesAndCheckVersion(OperationContext* txn) { LOG(1) << "done repairDatabases" << endl; } -static void _initWireSpec() { +void _initWireSpec() { WireSpec& spec = WireSpec::instance(); // accept from any version spec.minWireVersionIncoming = RELEASE_2_4_AND_BEFORE; @@ -513,8 +540,7 @@ static void _initWireSpec() { spec.maxWireVersionOutgoing = FIND_COMMAND; } - -static void _initAndListen(int listenPort) { +void _initAndListen(int listenPort) { Client::initThread("initandlisten"); _initWireSpec(); @@ -721,6 +747,8 @@ static void _initAndListen(int listenPort) { "data." << startupWarningsLog; } + logStartup(startupOpCtx.get()); + getDeleter()->startWorkers(); restartInProgressIndexesFromLastShutdown(startupOpCtx.get()); @@ -757,8 +785,6 @@ static void _initAndListen(int listenPort) { uassertStatusOK(ShardingStateRecovery::recover(startupOpCtx.get())); } - logStartup(startupOpCtx.get()); - // MessageServer::run will return when exit code closes its socket and we don't need the // operation context anymore startupOpCtx.reset(); @@ -785,6 +811,8 @@ ExitCode initAndListen(int listenPort) { } } +} // namespace + #if defined(_WIN32) ExitCode initService() { ntservice::reportStatus(SERVICE_RUNNING); diff --git a/src/mongo/db/db_raii.cpp b/src/mongo/db/db_raii.cpp index b0da72c5bb8..4e7045061c1 100644 --- a/src/mongo/db/db_raii.cpp +++ b/src/mongo/db/db_raii.cpp @@ -134,8 +134,10 @@ void AutoGetCollectionForRead::_ensureMajorityCommittedSnapshotIsValid(const Nam uassertStatusOK(_txn->recoveryUnit()->setReadFromMajorityCommittedSnapshot()); - stdx::lock_guard lk(*_txn->getClient()); - CurOp::get(_txn)->yielded(); + { + stdx::lock_guard lk(*_txn->getClient()); + CurOp::get(_txn)->yielded(); + } // Relock. _autoColl.emplace(_txn, nss, MODE_IS); diff --git a/src/mongo/db/dbcommands.cpp b/src/mongo/db/dbcommands.cpp index e65da17dd07..53b92f56bd3 100644 --- a/src/mongo/db/dbcommands.cpp +++ b/src/mongo/db/dbcommands.cpp @@ -92,6 +92,7 @@ #include "mongo/rpc/reply_builder_interface.h" #include "mongo/rpc/metadata.h" #include "mongo/rpc/metadata/config_server_metadata.h" +#include "mongo/rpc/metadata/repl_set_metadata.h" #include "mongo/rpc/metadata/server_selection_metadata.h" #include "mongo/rpc/metadata/sharding_metadata.h" #include "mongo/rpc/protocol.h" @@ -141,7 +142,7 @@ public: BSONObjBuilder& result) { bool force = cmdObj.hasField("force") && cmdObj["force"].trueValue(); - long long timeoutSecs = 0; + long long timeoutSecs = 10; if (cmdObj.hasField("timeoutSecs")) { timeoutSecs = cmdObj["timeoutSecs"].numberLong(); } @@ -1257,7 +1258,16 @@ void Command::execCommand(OperationContext* txn, replCoord->getReplicationMode() == repl::ReplicationCoordinator::modeReplSet && !replCoord->canAcceptWritesForDatabase(dbname) && !replCoord->getMemberState().secondary()) { - uasserted(ErrorCodes::NotMasterOrSecondary, "node is recovering"); + uassert(ErrorCodes::NotMasterOrSecondary, + "node is recovering", + !replCoord->getMemberState().recovering()); + uassert(ErrorCodes::NotMasterOrSecondary, + "node is not in primary or recovering state", + replCoord->getMemberState().primary()); + // Check ticket SERVER-21432, slaveOk commands are allowed in drain mode + uassert(ErrorCodes::NotMasterOrSecondary, + "node is in drain mode", + commandIsOverriddenToRunOnSecondary || commandCanRunOnSecondary); } } @@ -1466,8 +1476,9 @@ bool Command::run(OperationContext* txn, if (isReplSet) { repl::OpTime lastOpTimeFromClient = repl::ReplClientInfo::forClient(txn->getClient()).getLastOp(); - replCoord->prepareReplResponseMetadata(request, lastOpTimeFromClient, &metadataBob); - + if (request.getMetadata().hasField(rpc::kReplSetMetadataFieldName)) { + replCoord->prepareReplMetadata(lastOpTimeFromClient, &metadataBob); + } // For commands from mongos, append some info to help getLastError(w) work. // TODO: refactor out of here as part of SERVER-18326 if (isShardingAware || serverGlobalParams.configsvr) { diff --git a/src/mongo/db/dbhelpers.cpp b/src/mongo/db/dbhelpers.cpp index e0d6da23040..d3c62ea0ef8 100644 --- a/src/mongo/db/dbhelpers.cpp +++ b/src/mongo/db/dbhelpers.cpp @@ -351,6 +351,12 @@ long long Helpers::removeRange(OperationContext* txn, IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByKeyPattern(txn, indexKeyPattern.toBSON()); + if (!desc) { + warning(LogComponent::kSharding) << "shard key index " << indexKeyPattern.toBSON() + << " on '" << ns << "' was dropped"; + return -1; + } + unique_ptr exec( InternalPlanner::indexScan(txn, collection, diff --git a/src/mongo/db/exec/geo_near.cpp b/src/mongo/db/exec/geo_near.cpp index c4ae337ab16..71a9b50e749 100644 --- a/src/mongo/db/exec/geo_near.cpp +++ b/src/mongo/db/exec/geo_near.cpp @@ -264,8 +264,13 @@ class GeoNear2DStage::DensityEstimator { public: DensityEstimator(PlanStage::Children* children, const IndexDescriptor* twoDindex, - const GeoNearParams* nearParams) - : _children(children), _twoDIndex(twoDindex), _nearParams(nearParams), _currentLevel(0) { + const GeoNearParams* nearParams, + const R2Annulus& fullBounds) + : _children(children), + _twoDIndex(twoDindex), + _nearParams(nearParams), + _fullBounds(fullBounds), + _currentLevel(0) { GeoHashConverter::Parameters hashParams; Status status = GeoHashConverter::parseParameters(_twoDIndex->infoObj(), &hashParams); // The index status should always be valid. @@ -292,7 +297,8 @@ private: PlanStage::Children* _children; // Points to PlanStage::_children in the NearStage. const IndexDescriptor* _twoDIndex; // Not owned here. const GeoNearParams* _nearParams; // Not owned here. - IndexScan* _indexScan = nullptr; // Owned in PlanStage::_children. + const R2Annulus& _fullBounds; + IndexScan* _indexScan = nullptr; // Owned in PlanStage::_children. unique_ptr _converter; GeoHash _centroidCell; unsigned _currentLevel; @@ -359,6 +365,39 @@ PlanStage::StageState GeoNear2DStage::DensityEstimator::work(OperationContext* t if (state == PlanStage::IS_EOF) { // We ran through the neighbors but found nothing. + // + // Before going to the next-coarsest level, check whether our search area contains the + // entire search annulus, since we don't want to spend time doing density estimation over + // areas that are much larger than the requested $maxDistance. + // + // The search area consists of four cells with side length S. Within its cell, the closest + // vertex to the search point must be the vertex shared with the other three cells. If the + // search point lies in the upper left cell, this means that it must lie in the lower right + // quadrant of that cell. Furthermore, this lower-right quadrant has a side-length of S/2. + // + // +-----------+-----------+ + // | | | + // | S/2 | | + // + +-----+ | + // | | o | | + // | | | | + // +-----+-----+-----------+ + // | | | + // | | | + // | | | + // | | | + // | | | + // +-----------+-----------+ + // S + // + // As long as the outer radius of the search annulus is less than S/2, it must be entirely + // contained within these four cells. + if (_fullBounds.getOuter() < (0.5 * _converter->sizeEdge(_currentLevel))) { + // We're covering the entire search annulus. Return EOF to indicate we're done. + *estimatedDistance = 0.5 * _converter->sizeEdge(_currentLevel); + return PlanStage::IS_EOF; + } + if (_currentLevel > 0u) { // Advance to the next level and search again. _currentLevel--; @@ -391,7 +430,8 @@ PlanStage::StageState GeoNear2DStage::initialize(OperationContext* txn, Collection* collection, WorkingSetID* out) { if (!_densityEstimator) { - _densityEstimator.reset(new DensityEstimator(&_children, _twoDIndex, &_nearParams)); + _densityEstimator.reset( + new DensityEstimator(&_children, _twoDIndex, &_nearParams, _fullBounds)); } double estimatedDistance; @@ -793,11 +833,13 @@ public: DensityEstimator(PlanStage::Children* children, const IndexDescriptor* s2Index, const GeoNearParams* nearParams, - const S2IndexingParams& indexParams) + const S2IndexingParams& indexParams, + const R2Annulus& fullBounds) : _children(children), _s2Index(s2Index), _nearParams(nearParams), _indexParams(indexParams), + _fullBounds(fullBounds), _currentLevel(0) { // cellId.AppendVertexNeighbors(level, output) requires level < finest, // so we use the minimum of max_level - 1 and the user specified finest @@ -820,6 +862,7 @@ private: const IndexDescriptor* _s2Index; // Not owned here. const GeoNearParams* _nearParams; // Not owned here. const S2IndexingParams _indexParams; + const R2Annulus& _fullBounds; int _currentLevel; IndexScan* _indexScan = nullptr; // Owned in PlanStage::_children. }; @@ -873,6 +916,41 @@ PlanStage::StageState GeoNear2DSphereStage::DensityEstimator::work(OperationCont if (state == PlanStage::IS_EOF) { // We ran through the neighbors but found nothing. + // + // Before going to the next-coarsest level, check whether our search area contains the + // entire search annulus, since we don't want to spend time doing density estimation over + // areas that are much larger than the requested $maxDistance. + // + // The search area consists of four cells at level L. Within its cell, the closest vertex to + // the search point must be the vertex shared with the other three cells. If the search + // point lies in the upper left cell, this means that it must lie in the lower right + // sub-cell at level L+1. + // + // +-----------+-----------+ + // | | | + // | S | | + // + +-----+ | + // | | o | | + // | | | | + // +-----+-----+-----------+ + // | | | + // | | | + // | | | + // | | | + // | | | + // +-----------+-----------+ + // + // In the diagram above, S is the width of the cell at level L+1. We can determine a lower + // bound for the width any cell at this level, i.e. S > minWidth(L+1). As long as the outer + // radius of the search annulus is less than minWidth(L+1), it must be entirely contained + // within these four level L cells. + if (_fullBounds.getOuter() < + (S2::kMinWidth.GetValue(_currentLevel + 1) * kRadiusOfEarthInMeters)) { + // We're covering the entire search annulus. Return EOF to indicate we're done. + *estimatedDistance = S2::kMinWidth.GetValue(_currentLevel + 1) * kRadiusOfEarthInMeters; + return PlanStage::IS_EOF; + } + if (_currentLevel > 0) { // Advance to the next level and search again. _currentLevel--; @@ -907,7 +985,7 @@ PlanStage::StageState GeoNear2DSphereStage::initialize(OperationContext* txn, WorkingSetID* out) { if (!_densityEstimator) { _densityEstimator.reset( - new DensityEstimator(&_children, _s2Index, &_nearParams, _indexParams)); + new DensityEstimator(&_children, _s2Index, &_nearParams, _indexParams, _fullBounds)); } double estimatedDistance; diff --git a/src/mongo/db/ftdc/SConscript b/src/mongo/db/ftdc/SConscript index 67601b2d3ab..5b8c2d58455 100644 --- a/src/mongo/db/ftdc/SConscript +++ b/src/mongo/db/ftdc/SConscript @@ -32,21 +32,32 @@ ftdcEnv.Library( ], ) +platform_libs = [] + +if env.TargetOSIs('linux'): + platform_libs = [ + '$BUILD_DIR/mongo/util/procparser' + ] +elif env.TargetOSIs('windows'): + platform_libs = [ + '$BUILD_DIR/mongo/util/perfctr_collect' + ] + env.Library( target='ftdc_mongod', source=[ + 'ftdc_commands.cpp', 'ftdc_mongod.cpp', + 'ftdc_system_stats.cpp', + 'ftdc_system_stats_${TARGET_OS}.cpp', ], LIBDEPS=[ '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/server_parameters', + '$BUILD_DIR/mongo/db/storage/storage_options', + '$BUILD_DIR/mongo/util/processinfo', 'ftdc' - ], - LIBDEPS_TAGS=[ - # Needs 'mongo::storageGlobalParams' in storage_options.cpp which is in serveronly - # And this library is already part of serveronly - 'incomplete', - ], + ] + platform_libs, ) env.CppUnitTest( diff --git a/src/mongo/db/ftdc/config.h b/src/mongo/db/ftdc/config.h index ef7ee8b9143..feb57aaa1e6 100644 --- a/src/mongo/db/ftdc/config.h +++ b/src/mongo/db/ftdc/config.h @@ -86,7 +86,7 @@ struct FTDCConfig { static const bool kEnabledDefault = true; static const std::uint64_t kPeriodMillisDefault; - static const std::uint64_t kMaxDirectorySizeBytesDefault = 100 * 1024 * 1024; + static const std::uint64_t kMaxDirectorySizeBytesDefault = 200 * 1024 * 1024; static const std::uint64_t kMaxFileSizeBytesDefault = 10 * 1024 * 1024; static const std::uint64_t kMaxFileUniqifier = 65000; diff --git a/src/mongo/db/ftdc/controller.cpp b/src/mongo/db/ftdc/controller.cpp index 65819c7e380..c13f6fc7fcf 100644 --- a/src/mongo/db/ftdc/controller.cpp +++ b/src/mongo/db/ftdc/controller.cpp @@ -99,6 +99,13 @@ void FTDCController::addOnRotateCollector(std::unique_ptr lock(_mutex); + return _mostRecentPeriodicDocument.getOwned(); + } +} + void FTDCController::start() { log() << "Initializing full-time diagnostic data capture with directory '" << _path.generic_string() << "'"; @@ -211,6 +218,12 @@ void FTDCController::doLoop() { client, std::get<0>(collectSample), std::get<1>(collectSample)); uassertStatusOK(s); + + // Store a reference to the most recent document from the periodic collectors + { + stdx::lock_guard lock(_mutex); + _mostRecentPeriodicDocument = std::get<0>(collectSample); + } } } } catch (...) { diff --git a/src/mongo/db/ftdc/controller.h b/src/mongo/db/ftdc/controller.h index 3db82b6b171..44b5e838506 100644 --- a/src/mongo/db/ftdc/controller.h +++ b/src/mongo/db/ftdc/controller.h @@ -43,6 +43,8 @@ namespace mongo { +class ServiceContext; + /** * Responsible for periodic collection of samples, writing them to disk, * and rotation. @@ -118,6 +120,16 @@ public: */ void stop(); + /** + * Get the FTDCController from ServiceContext. + */ + static FTDCController* get(ServiceContext* serviceContext); + + /** + * Get a reference to most recent document from the periodic collectors. + */ + BSONObj getMostRecentPeriodicDocument(); + private: /** * Do periodic statistics collection, and all other work on the background thread. @@ -162,7 +174,7 @@ private: // Directory to store files const boost::filesystem::path _path; - // Mutex to protect the condvar, and configuration changes. + // Mutex to protect the condvar, configuration changes, and most recent periodic document. stdx::mutex _mutex; stdx::condition_variable _condvar; @@ -176,6 +188,10 @@ private: // Set of periodic collectors FTDCCollectorCollection _periodicCollectors; + // Last seen sample document from periodic collectors + // Owned + BSONObj _mostRecentPeriodicDocument; + // Set of file rotation collectors FTDCCollectorCollection _rotateCollectors; diff --git a/src/mongo/db/ftdc/file_writer.cpp b/src/mongo/db/ftdc/file_writer.cpp index 576f946700a..1f290249c67 100644 --- a/src/mongo/db/ftdc/file_writer.cpp +++ b/src/mongo/db/ftdc/file_writer.cpp @@ -57,9 +57,6 @@ Status FTDCFileWriter::open(const boost::filesystem::path& file) { _archiveFile = file; - // Disable file buffering - _archiveStream.rdbuf()->pubsetbuf(0, 0); - // Ideally, we create a file from scratch via O_CREAT but there is not portable way via C++ // iostreams to do this. _archiveStream.open(_archiveFile.c_str(), @@ -87,9 +84,6 @@ Status FTDCFileWriter::writeInterimFileBuffer(ConstDataRange buf) { // Fixed size interim stream std::ofstream interimStream; - // Disable file buffering - interimStream.rdbuf()->pubsetbuf(0, 0); - // Open up a temporary interim file interimStream.open(_interimTempFile.c_str(), std::ios_base::out | std::ios_base::binary | std::ios_base::trunc); @@ -134,6 +128,18 @@ Status FTDCFileWriter::writeArchiveFileBuffer(ConstDataRange buf) { << _archiveFile.generic_string()}; } + // Flush the stream explictly, this is preferred over "pubsetbuf(0,0)" which has implementation + // defined behavior of "before any I/O has occurred". + _archiveStream.flush(); + + if (_archiveStream.fail()) { + return { + ErrorCodes::FileStreamFailed, + str::stream() + << "Failed to flush to archive file buffer for full-time diagnostic data capture: " + << _archiveFile.generic_string()}; + } + _size += buf.length(); return Status::OK(); diff --git a/src/mongo/db/ftdc/file_writer_test.cpp b/src/mongo/db/ftdc/file_writer_test.cpp index 6555a13c115..62443894254 100644 --- a/src/mongo/db/ftdc/file_writer_test.cpp +++ b/src/mongo/db/ftdc/file_writer_test.cpp @@ -45,6 +45,7 @@ namespace mongo { const char* kTestFile = "metrics.test"; +const char* kTestFileCopy = "metrics.test.copy"; // File Sanity check TEST(FTDCFileTest, TestFileBasicMetadata) { @@ -160,7 +161,27 @@ public: } private: - void validate(bool forceCompress = true) { + void validate() { + // Verify we are flushing writes correctly by copying the file, and then reading it. + auto tempfile(boost::filesystem::path(_tempdir.path()) / kTestFileCopy); + boost::filesystem::copy_file(_path, tempfile); + + // Read the file to make sure it is correct. + // We do not verify contents because the compressor may not have flushed the final records + // which is expected. + { + FTDCFileReader reader; + + ASSERT_OK(reader.open(tempfile)); + + auto sw = reader.hasNext(); + while (sw.isOK() && sw.getValue()) { + sw = reader.hasNext(); + } + + ASSERT_OK(sw); + } + _writer.close(); ValidateDocumentList(_path, _docs); @@ -244,6 +265,25 @@ TEST(FTDCFileTest, TestFull) { } } +// Test a large documents so that we cause multiple 4kb buffers to flush on Windows. +TEST(FTDCFileTest, TestLargeDocuments) { + FileTestTie c; + + for (int j = 0; j < 5; j++) { + for (size_t i = 0; i <= FTDCConfig::kMaxSamplesPerArchiveMetricChunkDefault; i++) { + BSONObjBuilder b; + b.append("name", "joe"); + for (size_t k = 0; k < 200; k++) { + b.appendNumber( + "num", + static_cast(i * j + 5000 - (sin(static_cast(k)) * 100))); + } + + c.addSample(b.obj()); + } + } +} + // Test a bad file TEST(FTDCFileTest, TestBadFile) { unittest::TempDir tempdir("metrics_testpath"); diff --git a/src/mongo/db/ftdc/ftdc_commands.cpp b/src/mongo/db/ftdc/ftdc_commands.cpp new file mode 100644 index 00000000000..13487171aa6 --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_commands.cpp @@ -0,0 +1,113 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/base/init.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/client.h" +#include "mongo/db/commands.h" +#include "mongo/db/ftdc/controller.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/operation_context.h" + +namespace mongo { +namespace { + +/** + * Get the most recent document FTDC collected from its periodic collectors. + * + * Document will be empty if FTDC has never run. + */ +class GetDiagnosticDataCommand final : public Command { +public: + GetDiagnosticDataCommand() : Command("getDiagnosticData") {} + + bool adminOnly() const override { + return true; + } + + void help(std::stringstream& help) const override { + help << "get latest diagnostic data collection snapshot"; + } + + bool slaveOk() const override { + return true; + } + + bool isWriteCommandForConfigServer() const override { + return false; + } + + Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) override { + if (!AuthorizationSession::get(client)->isAuthorizedForActionsOnResource( + ResourcePattern::forClusterResource(), ActionType::serverStatus)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + + if (!AuthorizationSession::get(client)->isAuthorizedForActionsOnResource( + ResourcePattern::forClusterResource(), ActionType::replSetGetStatus)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + + if (!AuthorizationSession::get(client)->isAuthorizedForActionsOnResource( + ResourcePattern::forExactNamespace(NamespaceString("local", "oplog.rs")), + ActionType::collStats)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + + return Status::OK(); + } + + bool run(OperationContext* txn, + const std::string& db, + BSONObj& cmdObj, + int options, + std::string& errmsg, + BSONObjBuilder& result) override { + result.append( + "data", FTDCController::get(txn->getServiceContext())->getMostRecentPeriodicDocument()); + + return true; + } +}; + +Command* ftdcCommand; + +MONGO_INITIALIZER(CreateDiagnosticDataCommand)(InitializerContext* context) { + ftdcCommand = new GetDiagnosticDataCommand(); + + return Status::OK(); +} + +} // namespace + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_mongod.cpp b/src/mongo/db/ftdc/ftdc_mongod.cpp index e328dbd3dff..f9861186af0 100644 --- a/src/mongo/db/ftdc/ftdc_mongod.cpp +++ b/src/mongo/db/ftdc/ftdc_mongod.cpp @@ -28,6 +28,8 @@ #include "mongo/platform/basic.h" +#include "mongo/db/ftdc/ftdc_mongod.h" + #include #include #include @@ -35,11 +37,11 @@ #include "mongo/base/init.h" #include "mongo/base/status.h" #include "mongo/bson/bsonobjbuilder.h" - #include "mongo/db/commands.h" #include "mongo/db/ftdc/collector.h" #include "mongo/db/ftdc/config.h" #include "mongo/db/ftdc/controller.h" +#include "mongo/db/ftdc/ftdc_system_stats.h" #include "mongo/db/jsobj.h" #include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/repl/replication_coordinator_global.h" @@ -288,7 +290,6 @@ private: } // namespace - // Register the FTDC system // Note: This must be run before the server parameters are parsed during startup // so that the FTDCController is initialized. @@ -310,6 +311,8 @@ void startFTDC() { // Install periodic collectors // These are collected on the period interval in FTDCConfig. + // NOTE: For each command here, there must be an equivalent privilege check in + // GetDiagnosticDataCommand // CmdServerStatus controller->addPeriodicCollector(stdx::make_unique( @@ -327,6 +330,9 @@ void startFTDC() { "collStats", "local.oplog.rs.stats", "local.oplog.rs", BSONObj())); } + // Install System Metric Collector as a periodic collector + installSystemMetricsCollector(controller.get()); + // Install file rotation collectors // These are collected on each file rotation. @@ -358,4 +364,8 @@ void stopFTDC() { } } +FTDCController* FTDCController::get(ServiceContext* serviceContext) { + return getFTDCController(serviceContext).get(); +} + } // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_system_stats.cpp b/src/mongo/db/ftdc/ftdc_system_stats.cpp new file mode 100644 index 00000000000..42a2892d964 --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_system_stats.cpp @@ -0,0 +1,61 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/ftdc/ftdc_system_stats.h" + +#include + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/ftdc/collector.h" + +namespace mongo { + +namespace { + +/** + * Name of FTDC collector to create. + */ +auto kSystemMetricsCollector = "systemMetrics"; + +} // namespace + +std::string SystemMetricsCollector::name() const { + return kSystemMetricsCollector; +} + +void SystemMetricsCollector::processStatusErrors(Status s, BSONObjBuilder* builder) { + if (!s.isOK()) { + builder->append("error", s.toString()); + } +} + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_system_stats.h b/src/mongo/db/ftdc/ftdc_system_stats.h new file mode 100644 index 00000000000..20d21ef4f39 --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_system_stats.h @@ -0,0 +1,64 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include + +#include "mongo/base/status.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/ftdc/controller.h" +#include "mongo/db/ftdc/controller.h" + +namespace mongo { + +class FTDCController; + +/** + * Base class for system metrics collectors. Sets collector name to a common name all system metrics + * collectors to use. + */ +class SystemMetricsCollector : public FTDCCollectorInterface { +public: + std::string name() const final; + +protected: + /** + * Convert any errors we see into BSON for the user to see in the final FTDC document. It is + * acceptable for the collector to fail, but we do not want to shutdown the FTDC loop because + * of it. We assume that the BSONBuilder is not corrupt on non-OK Status but nothing else with + * regards to the final document output. + */ + static void processStatusErrors(Status s, BSONObjBuilder* builder); +}; + + +/** + * Install a system metrics collector if it exists as a periodic collector. + */ +void installSystemMetricsCollector(FTDCController* controller); + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_system_stats_freebsd.cpp b/src/mongo/db/ftdc/ftdc_system_stats_freebsd.cpp new file mode 100644 index 00000000000..d06092f9ba9 --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_system_stats_freebsd.cpp @@ -0,0 +1,37 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/ftdc/ftdc_system_stats.h" + +namespace mongo { + +void installSystemMetricsCollector(FTDCController* controller) {} + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_system_stats_linux.cpp b/src/mongo/db/ftdc/ftdc_system_stats_linux.cpp new file mode 100644 index 00000000000..5ed3c6e89ba --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_system_stats_linux.cpp @@ -0,0 +1,127 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/ftdc/ftdc_system_stats.h" + +#include +#include + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/ftdc/collector.h" +#include "mongo/db/ftdc/controller.h" +#include "mongo/stdx/memory.h" +#include "mongo/util/processinfo.h" +#include "mongo/util/procparser.h" + +namespace mongo { + +namespace { + +static const std::vector kCpuKeys{ + "btime", "cpu", "ctxt", "processes", "procs_blocked", "procs_running"}; + +static const std::vector kMemKeys{ + "MemTotal", + "MemFree", + "Cached", + "Dirty", + "Buffers", + "SwapTotal", + "SwapCached", + "SwapFree", + "Active", + "Inactive", + "Active(anon)", + "Inactive(anon)", + "Active(file)", + "Inactive(file)", +}; + +/** + * Collect metrics from the Linux /proc file system. + */ +class LinuxSystemMetricsCollector final : public SystemMetricsCollector { +public: + LinuxSystemMetricsCollector() : _disks(procparser::findPhysicalDisks("/sys/block")) { + for (const auto& disk : _disks) { + _disksStringData.emplace_back(disk); + } + } + + void collect(OperationContext* txn, BSONObjBuilder& builder) override { + { + BSONObjBuilder subObjBuilder(builder.subobjStart("cpu")); + + // Include the number of cpus to simplify client calculations + ProcessInfo p; + subObjBuilder.append("num_cpus", p.getNumCores()); + + processStatusErrors( + procparser::parseProcStatFile("/proc/stat", kCpuKeys, &subObjBuilder), + &subObjBuilder); + subObjBuilder.doneFast(); + } + + { + BSONObjBuilder subObjBuilder(builder.subobjStart("memory")); + processStatusErrors( + procparser::parseProcMemInfoFile("/proc/meminfo", kMemKeys, &subObjBuilder), + &subObjBuilder); + subObjBuilder.doneFast(); + } + + // Skip the disks section if we could not find any disks. + // This can happen when we do not have permission to /sys/block for instance. + if (!_disksStringData.empty()) { + BSONObjBuilder subObjBuilder(builder.subobjStart("disks")); + processStatusErrors(procparser::parseProcDiskStatsFile( + "/proc/diskstats", _disksStringData, &subObjBuilder), + &subObjBuilder); + subObjBuilder.doneFast(); + } + } + +private: + // List of physical disks to collect stats from as string from findPhysicalDisks. + std::vector _disks; + + // List of physical disks to collect stats from as StringData to pass to parseProcDiskStatsFile. + std::vector _disksStringData; +}; + +} // namespace + +void installSystemMetricsCollector(FTDCController* controller) { + controller->addPeriodicCollector(stdx::make_unique()); +} + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_system_stats_openbsd.cpp b/src/mongo/db/ftdc/ftdc_system_stats_openbsd.cpp new file mode 100644 index 00000000000..d06092f9ba9 --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_system_stats_openbsd.cpp @@ -0,0 +1,37 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/ftdc/ftdc_system_stats.h" + +namespace mongo { + +void installSystemMetricsCollector(FTDCController* controller) {} + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_system_stats_osx.cpp b/src/mongo/db/ftdc/ftdc_system_stats_osx.cpp new file mode 100644 index 00000000000..d06092f9ba9 --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_system_stats_osx.cpp @@ -0,0 +1,37 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/ftdc/ftdc_system_stats.h" + +namespace mongo { + +void installSystemMetricsCollector(FTDCController* controller) {} + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_system_stats_solaris.cpp b/src/mongo/db/ftdc/ftdc_system_stats_solaris.cpp new file mode 100644 index 00000000000..d06092f9ba9 --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_system_stats_solaris.cpp @@ -0,0 +1,37 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/ftdc/ftdc_system_stats.h" + +namespace mongo { + +void installSystemMetricsCollector(FTDCController* controller) {} + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_system_stats_windows.cpp b/src/mongo/db/ftdc/ftdc_system_stats_windows.cpp new file mode 100644 index 00000000000..e684ebcf326 --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_system_stats_windows.cpp @@ -0,0 +1,154 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kFTDC + +#include "mongo/platform/basic.h" + +#include "mongo/db/ftdc/ftdc_system_stats.h" + +#include +#include + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/ftdc/collector.h" +#include "mongo/db/ftdc/controller.h" +#include "mongo/stdx/memory.h" +#include "mongo/util/log.h" +#include "mongo/util/perfctr_collect.h" + +namespace mongo { + +namespace { + +const std::vector kCpuCounters = { + "\\Processor(_Total)\\% Idle Time", + "\\Processor(_Total)\\% Interrupt Time", + "\\Processor(_Total)\\% Privileged Time", + "\\Processor(_Total)\\% Processor Time", + "\\Processor(_Total)\\% User Time", + "\\Processor(_Total)\\Interrupts/sec", + "\\System\\Context Switches/sec", + "\\System\\Processes", + "\\System\\Processor Queue Length", + "\\System\\System Up Time", + "\\System\\Threads", +}; + +const std::vector kMemoryCounters = { + "\\Memory\\Available Bytes", + "\\Memory\\Cache Bytes", + "\\Memory\\Cache Faults/sec", + "\\Memory\\Committed Bytes", + "\\Memory\\Commit Limit", + "\\Memory\\Page Reads/sec", + "\\Memory\\Page Writes/sec", + "\\Memory\\Pages Input/sec", + "\\Memory\\Pages Output/sec", + "\\Memory\\Pool Nonpaged Bytes", + "\\Memory\\Pool Paged Bytes", + "\\Memory\\Pool Paged Resident Bytes", + "\\Memory\\System Cache Resident Bytes", + "\\Memory\\System Code Total Bytes", + +}; + +const std::vector kDiskCounters = { + "\\PhysicalDisk(*)\\% Disk Read Time", + "\\PhysicalDisk(*)\\% Disk Write Time", + "\\PhysicalDisk(*)\\Avg. Disk Read Queue Length", + "\\PhysicalDisk(*)\\Avg. Disk Write Queue Length", + "\\PhysicalDisk(*)\\Disk Read Bytes/sec", + "\\PhysicalDisk(*)\\Disk Write Bytes/sec", + "\\PhysicalDisk(*)\\Disk Reads/sec", + "\\PhysicalDisk(*)\\Disk Writes/sec", + "\\PhysicalDisk(*)\\Current Disk Queue Length", +}; + + +/** + * Collect metrics from Windows Performance Counters. + */ +class WindowsSystemMetricsCollector final : public SystemMetricsCollector { +public: + WindowsSystemMetricsCollector(std::unique_ptr collector) + : _collector(std::move(collector)) {} + + void collect(OperationContext* txn, BSONObjBuilder& builder) override { + processStatusErrors(_collector->collect(&builder), &builder); + } + +private: + std::unique_ptr _collector; +}; + + +StatusWith> createCollector() { + PerfCounterCollection collection; + + Status s = collection.addCountersGroup("cpu", kCpuCounters); + if (!s.isOK()) { + return s; + } + + // TODO: Should we capture the Heap Counters for the current process? + s = collection.addCountersGroup("memory", kMemoryCounters); + if (!s.isOK()) { + return s; + } + + s = collection.addCountersGroupedByInstanceName("disks", kDiskCounters); + if (!s.isOK()) { + return s; + } + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + if (!swCollector.getStatus().isOK()) { + return swCollector.getStatus(); + } + + return {std::move(swCollector.getValue())}; +} + +} // namespace + +void installSystemMetricsCollector(FTDCController* controller) { + auto swCollector = createCollector(); + if (!swCollector.getStatus().isOK()) { + warning() << "Failed to initialize Performance Counters for FTDC: " + << swCollector.getStatus(); + return; + } + + controller->addPeriodicCollector( + stdx::make_unique(std::move(swCollector.getValue()))); +} + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_test.cpp b/src/mongo/db/ftdc/ftdc_test.cpp index 33fcce62eff..28910def1a1 100644 --- a/src/mongo/db/ftdc/ftdc_test.cpp +++ b/src/mongo/db/ftdc/ftdc_test.cpp @@ -58,6 +58,8 @@ void ValidateDocumentList(const boost::filesystem::path& p, const std::vector IndexAccessMethod::initiateBulk() { - return std::unique_ptr(new BulkBuilder(this, _descriptor)); +std::unique_ptr IndexAccessMethod::initiateBulk( + size_t maxMemoryUsageBytes) { + return std::unique_ptr(new BulkBuilder(this, _descriptor, maxMemoryUsageBytes)); } IndexAccessMethod::BulkBuilder::BulkBuilder(const IndexAccessMethod* index, - const IndexDescriptor* descriptor) + const IndexDescriptor* descriptor, + size_t maxMemoryUsageBytes) : _sorter(Sorter::make( SortOptions() .TempDir(storageGlobalParams.dbpath + "/_tmp") .ExtSortAllowed() - .MaxMemoryUsageBytes(100 * 1024 * 1024), + .MaxMemoryUsageBytes(maxMemoryUsageBytes), BtreeExternalSortComparison(descriptor->keyPattern(), descriptor->version()))), _real(index) {} diff --git a/src/mongo/db/index/index_access_method.h b/src/mongo/db/index/index_access_method.h index 401f0dbe0e4..8ea1d58fde8 100644 --- a/src/mongo/db/index/index_access_method.h +++ b/src/mongo/db/index/index_access_method.h @@ -202,7 +202,9 @@ public: using Sorter = mongo::Sorter; - BulkBuilder(const IndexAccessMethod* index, const IndexDescriptor* descriptor); + BulkBuilder(const IndexAccessMethod* index, + const IndexDescriptor* descriptor, + size_t maxMemoryUsageBytes); std::unique_ptr _sorter; const IndexAccessMethod* _real; @@ -216,8 +218,11 @@ public: * This can return NULL, meaning bulk mode is not available. * * It is only legal to initiate bulk when the index is new and empty. + * + * maxMemoryUsageBytes: amount of memory consumed before the external sorter starts spilling to + * disk */ - std::unique_ptr initiateBulk(); + std::unique_ptr initiateBulk(size_t maxMemoryUsageBytes); /** * Call this when you are ready to finish your bulk work. diff --git a/src/mongo/db/instance.cpp b/src/mongo/db/instance.cpp index 2bd2d003b49..ef292b96f35 100644 --- a/src/mongo/db/instance.cpp +++ b/src/mongo/db/instance.cpp @@ -886,7 +886,7 @@ bool receivedGetMore(OperationContext* txn, DbResponse& dbresponse, Message& m, curop.debug().cursorid = cursorid; { - stdx::lock_guard(*txn->getClient()); + stdx::lock_guard lk(*txn->getClient()); CurOp::get(txn)->setNS_inlock(ns); } @@ -1158,7 +1158,7 @@ void receivedInsert(OperationContext* txn, const NamespaceString& nsString, Mess DbMessage d(m); const char* ns = d.getns(); { - stdx::lock_guard(*txn->getClient()); + stdx::lock_guard lk(*txn->getClient()); CurOp::get(txn)->setNS_inlock(nsString.ns()); } diff --git a/src/mongo/db/mongod_options.cpp b/src/mongo/db/mongod_options.cpp index e8facbaa850..610adbdbc3d 100644 --- a/src/mongo/db/mongod_options.cpp +++ b/src/mongo/db/mongod_options.cpp @@ -44,9 +44,9 @@ #include "mongo/db/server_options.h" #include "mongo/db/server_options_helpers.h" #include "mongo/db/storage/mmap_v1/mmap_v1_options.h" -#include "mongo/s/catalog/catalog_manager.h" #include "mongo/logger/console_appender.h" #include "mongo/logger/message_event_utf8_encoder.h" +#include "mongo/s/catalog/catalog_manager.h" #include "mongo/util/log.h" #include "mongo/util/mongoutils/str.h" #include "mongo/util/net/ssl_options.h" @@ -332,12 +332,12 @@ Status addMongodOptions(moe::OptionSection* options) { "storage.mmapv1.journal.commitIntervalMs"); // Deprecated option that we don't want people to use for performance reasons - storage_options.addOptionChaining("nopreallocj", + storage_options.addOptionChaining("storage.mmapv1.journal.nopreallocj", "nopreallocj", moe::Switch, "don't preallocate journal files") .hidden() - .setSources(moe::SourceAllLegacy); + .setSources(moe::SourceAll); #if defined(__linux__) general_options.addOptionChaining( @@ -1032,8 +1032,8 @@ Status storeMongodOptions(const moe::Environment& params, const std::vector(); } - if (params.count("nopreallocj")) { - mmapv1GlobalOptions.preallocj = !params["nopreallocj"].as(); + if (params.count("storage.mmapv1.journal.nopreallocj")) { + mmapv1GlobalOptions.preallocj = !params["storage.mmapv1.journal.nopreallocj"].as(); } if (params.count("net.http.RESTInterfaceEnabled")) { diff --git a/src/mongo/db/pipeline/SConscript b/src/mongo/db/pipeline/SConscript index 1e70c1fa376..715cd170c48 100644 --- a/src/mongo/db/pipeline/SConscript +++ b/src/mongo/db/pipeline/SConscript @@ -174,5 +174,6 @@ env.CppUnitTest( 'pipeline', '$BUILD_DIR/mongo/db/auth/authorization_manager_mock_init', '$BUILD_DIR/mongo/db/service_context', + '$BUILD_DIR/mongo/db/auth/authorization_session_for_test' ], ) diff --git a/src/mongo/db/pipeline/document_source.h b/src/mongo/db/pipeline/document_source.h index 6a2845ecddc..a6b81feb420 100644 --- a/src/mongo/db/pipeline/document_source.h +++ b/src/mongo/db/pipeline/document_source.h @@ -30,12 +30,12 @@ #include "mongo/platform/basic.h" -#include #include +#include +#include #include #include #include -#include #include #include @@ -502,7 +502,7 @@ private: typedef std::vector> Accumulators; - typedef std::unordered_map GroupsMap; + typedef boost::unordered_map GroupsMap; GroupsMap groups; /* diff --git a/src/mongo/db/pipeline/pipeline.cpp b/src/mongo/db/pipeline/pipeline.cpp index 04ed07ae797..eba7877ed5a 100644 --- a/src/mongo/db/pipeline/pipeline.cpp +++ b/src/mongo/db/pipeline/pipeline.cpp @@ -331,45 +331,67 @@ Status Pipeline::checkAuthForCommand(ClientBasic* client, std::vector privileges; - if (cmdObj.getFieldDotted("pipeline.0.$indexStats")) { - Privilege::addPrivilegeToPrivilegeVector( - &privileges, - Privilege(ResourcePattern::forAnyNormalResource(), ActionType::indexStats)); - } else { - // If no source requiring an alternative permission scheme is specified then default to - // requiring find() privileges on the given namespace. + BSONElement pipelineElem = cmdObj["pipeline"]; + if (pipelineElem.type() != BSONType::Array) { + return Status(ErrorCodes::TypeMismatch, "'pipeline' must be specified as an array"); + } + + BSONObj pipeline = pipelineElem.embeddedObject(); + if (pipeline.isEmpty()) { + // The pipeline is empty, so we require only the find action. Privilege::addPrivilegeToPrivilegeVector(&privileges, Privilege(inputResource, ActionType::find)); - } + } else { + if (pipeline.firstElementType() != BSONType::Object) { + // The pipeline contains something that's not an object. + return Status(ErrorCodes::TypeMismatch, + "'pipeline' cannot contain non-object elements"); + } - BSONObj pipeline = cmdObj.getObjectField("pipeline"); - BSONForEach(stageElem, pipeline) { - BSONObj stage = stageElem.embeddedObjectUserCheck(); - StringData stageName = stage.firstElementFieldName(); - if (stageName == "$out" && stage.firstElementType() == String) { - NamespaceString outputNs(db, stage.firstElement().str()); - uassert(17139, - mongoutils::str::stream() << "Invalid $out target namespace, " << outputNs.ns(), - outputNs.isValid()); - - ActionSet actions; - actions.addAction(ActionType::remove); - actions.addAction(ActionType::insert); - if (shouldBypassDocumentValidationForCommand(cmdObj)) { - actions.addAction(ActionType::bypassDocumentValidation); - } + // We treat the first stage in the pipeline specially, as $indexStats changes the authz + // requirements when appearing in this position. + BSONObj firstPipelineStage = pipeline.firstElement().embeddedObject(); + if (str::equals("$indexStats", firstPipelineStage.firstElementFieldName())) { Privilege::addPrivilegeToPrivilegeVector( - &privileges, Privilege(ResourcePattern::forExactNamespace(outputNs), actions)); - } else if (stageName == "$lookup" && stage.firstElementType() == Object) { - NamespaceString fromNs(db, stage.firstElement()["from"].str()); - Privilege::addPrivilegeToPrivilegeVector( - &privileges, - Privilege(ResourcePattern::forExactNamespace(fromNs), ActionType::find)); + &privileges, Privilege(inputResource, ActionType::indexStats)); + } else { + // If no source requiring an alternative permission scheme is specified then default to + // requiring find() privileges on the given namespace. + Privilege::addPrivilegeToPrivilegeVector(&privileges, + Privilege(inputResource, ActionType::find)); + } + + BSONObj pipelineStages = cmdObj.getObjectField("pipeline"); + BSONForEach(stageElem, pipelineStages) { + BSONObj stage = stageElem.embeddedObjectUserCheck(); + StringData stageName = stage.firstElementFieldName(); + if (stageName == "$out" && stage.firstElementType() == String) { + NamespaceString outputNs(db, stage.firstElement().str()); + uassert(17139, + mongoutils::str::stream() << "Invalid $out target namespace, " + << outputNs.ns(), + outputNs.isValid()); + + ActionSet actions; + actions.addAction(ActionType::remove); + actions.addAction(ActionType::insert); + if (shouldBypassDocumentValidationForCommand(cmdObj)) { + actions.addAction(ActionType::bypassDocumentValidation); + } + Privilege::addPrivilegeToPrivilegeVector( + &privileges, Privilege(ResourcePattern::forExactNamespace(outputNs), actions)); + } else if (stageName == "$lookup" && stage.firstElementType() == Object) { + NamespaceString fromNs(db, stage.firstElement()["from"].str()); + Privilege::addPrivilegeToPrivilegeVector( + &privileges, + Privilege(ResourcePattern::forExactNamespace(fromNs), ActionType::find)); + } } } - if (AuthorizationSession::get(client)->isAuthorizedForPrivileges(privileges)) + if (AuthorizationSession::get(client)->isAuthorizedForPrivileges(privileges)) { return Status::OK(); + } return Status(ErrorCodes::Unauthorized, "unauthorized"); } diff --git a/src/mongo/db/pipeline/pipeline_test.cpp b/src/mongo/db/pipeline/pipeline_test.cpp index 333fc5963de..edf9b30ca87 100644 --- a/src/mongo/db/pipeline/pipeline_test.cpp +++ b/src/mongo/db/pipeline/pipeline_test.cpp @@ -28,12 +28,20 @@ #include "mongo/platform/basic.h" +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_session_for_test.h" +#include "mongo/db/auth/authz_manager_external_state_mock.h" +#include "mongo/db/auth/authz_session_external_state_mock.h" +#include "mongo/db/operation_context_noop.h" #include "mongo/db/pipeline/document.h" #include "mongo/db/pipeline/expression_context.h" #include "mongo/db/pipeline/field_path.h" #include "mongo/db/pipeline/pipeline.h" -#include "mongo/db/operation_context_noop.h" +#include "mongo/db/service_context.h" +#include "mongo/db/service_context_noop.h" #include "mongo/dbtests/dbtests.h" +#include "mongo/stdx/memory.h" namespace mongo { bool isMongos() { @@ -638,4 +646,215 @@ public: SuiteInstance myall; +namespace AuthzTests { +const NamespaceString testFooNss("test.foo"); +const NamespaceString testBarNss("test.bar"); +const std::string testFooDb = testFooNss.db().toString(); +const std::string testBarDb = testBarNss.db().toString(); +const ResourcePattern testFooCollResource(ResourcePattern::forExactNamespace(testFooNss)); +const ResourcePattern testBarCollResource(ResourcePattern::forExactNamespace(testBarNss)); + +class PipelineAuthzTest : public ::mongo::unittest::Test { +public: + AuthzManagerExternalStateMock* managerState; + OperationContextNoop txn; + AuthzSessionExternalStateMock* sessionState; + std::unique_ptr authzManager; + std::unique_ptr service; + ServiceContext::UniqueClient client; + AuthorizationSessionForTest* authzSession; + + void setUp() override { + auto localManagerState = stdx::make_unique(); + managerState = localManagerState.get(); + managerState->setAuthzVersion(AuthorizationManager::schemaVersion26Final); + authzManager = stdx::make_unique(std::move(localManagerState)); + authzManager->setAuthEnabled(true); + auto localSessionState = + stdx::make_unique(authzManager.get()); + sessionState = localSessionState.get(); + service = stdx::make_unique(); + client = service->makeClient("PipelineAuthzTest"); + auto authzSessionForTest = + stdx::make_unique(std::move(localSessionState)); + authzSession = authzSessionForTest.get(); + AuthorizationSession::set(client.get(), std::move(authzSessionForTest)); + } +}; + +TEST_F(PipelineAuthzTest, CheckAuthForCommandFailsIfPipelineIsNotAnArray) { + BSONObj cmdObjIntPipeline = BSON("aggregate" << testFooNss.coll() << "pipeline" << 7); + ASSERT_EQ(ErrorCodes::TypeMismatch, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObjIntPipeline)); + + BSONObj cmdObjObjPipeline = BSON("aggregate" << testFooNss.coll() << "pipeline" << BSONObj()); + ASSERT_EQ(ErrorCodes::TypeMismatch, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObjObjPipeline)); + + BSONObj cmdObjNoPipeline = BSON("aggregate" << testFooNss.coll()); + ASSERT_EQ(ErrorCodes::TypeMismatch, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObjNoPipeline)); +} + +TEST_F(PipelineAuthzTest, CheckAuthForCommandFailsIfPipelineFirstStageIsNotAnObject) { + BSONObj cmdObjFirstStageInt = + BSON("aggregate" << testFooNss.coll() << "pipeline" << BSON_ARRAY(7)); + ASSERT_EQ(ErrorCodes::TypeMismatch, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObjFirstStageInt)); + + BSONObj cmdObjFirstStageArray = + BSON("aggregate" << testFooNss.coll() << "pipeline" << BSON_ARRAY(BSONArray())); + ASSERT_EQ(ErrorCodes::TypeMismatch, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObjFirstStageArray)); +} + +TEST_F(PipelineAuthzTest, CannotAggregateEmptyPipelineWithoutFindAction) { + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << BSONArray()); + ASSERT_EQ(ErrorCodes::Unauthorized, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, CanAggregateEmptyPipelineWithFindAction) { + authzSession->assumePrivilegesForDB( + Privilege(testFooCollResource, ActionSet{ActionType::find})); + + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << BSONArray()); + ASSERT_OK(Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, CannotAggregateWithoutFindActionIfFirstStageNotIndexStats) { + authzSession->assumePrivilegesForDB( + Privilege(testFooCollResource, ActionSet{ActionType::indexStats})); + + BSONArray pipeline = BSON_ARRAY(BSON("$limit" << 1) << BSON("$indexStats" << BSONObj())); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline); + ASSERT_EQ(ErrorCodes::Unauthorized, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, CanAggregateWithFindActionIfFirstStageNotIndexStats) { + authzSession->assumePrivilegesForDB( + Privilege(testFooCollResource, ActionSet{ActionType::find})); + + BSONArray pipeline = BSON_ARRAY(BSON("$limit" << 1) << BSON("$indexStats" << BSONObj())); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline); + ASSERT_OK(Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, CannotAggregateIndexStatsWithoutIndexStatsAction) { + authzSession->assumePrivilegesForDB( + Privilege(testFooCollResource, ActionSet{ActionType::find})); + + BSONArray pipeline = BSON_ARRAY(BSON("$indexStats" << BSONObj())); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline); + ASSERT_EQ(ErrorCodes::Unauthorized, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, CanAggregateIndexStatsWithIndexStatsAction) { + authzSession->assumePrivilegesForDB( + Privilege(testFooCollResource, ActionSet{ActionType::indexStats})); + + BSONArray pipeline = BSON_ARRAY(BSON("$indexStats" << BSONObj())); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline); + ASSERT_OK(Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, CheckAuthForCommandFailsIfOutNamespaceIsNotValid) { + BSONArray pipeline = BSON_ARRAY(BSON("$out" + << "")); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline); + ASSERT_THROWS_CODE( + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj), UserException, 17139); +} + +TEST_F(PipelineAuthzTest, CannotAggregateOutWithoutInsertAndRemoveOnTargetNamespace) { + // We only have find on the aggregation namespace. + authzSession->assumePrivilegesForDB( + Privilege(testFooCollResource, ActionSet{ActionType::find})); + + BSONArray pipeline = BSON_ARRAY(BSON("$out" << testBarNss.coll())); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline); + ASSERT_EQ(ErrorCodes::Unauthorized, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); + + // We have insert but not remove on the $out namespace. + authzSession->assumePrivilegesForDB( + {Privilege(testFooCollResource, ActionSet{ActionType::find}), + Privilege(testBarCollResource, ActionSet{ActionType::insert})}); + ASSERT_EQ(ErrorCodes::Unauthorized, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); + + // We have remove but not insert on the $out namespace. + authzSession->assumePrivilegesForDB( + {Privilege(testFooCollResource, ActionSet{ActionType::find}), + Privilege(testBarCollResource, ActionSet{ActionType::remove})}); + ASSERT_EQ(ErrorCodes::Unauthorized, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, CanAggregateOutWithInsertAndRemoveOnTargetNamespace) { + authzSession->assumePrivilegesForDB( + {Privilege(testFooCollResource, ActionSet{ActionType::find}), + Privilege(testBarCollResource, ActionSet{ActionType::insert, ActionType::remove})}); + + BSONArray pipeline = BSON_ARRAY(BSON("$out" << testBarNss.coll())); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline); + ASSERT_OK(Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); + + BSONObj cmdObjNoBypassDocumentValidation = + BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline + << "bypassDocumentValidation" << false); + ASSERT_OK( + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObjNoBypassDocumentValidation)); +} + +TEST_F(PipelineAuthzTest, + CannotAggregateOutBypassingValidationWithoutBypassDocumentValidationOnTargetNamespace) { + authzSession->assumePrivilegesForDB( + {Privilege(testFooCollResource, ActionSet{ActionType::find}), + Privilege(testBarCollResource, ActionSet{ActionType::insert, ActionType::remove})}); + + BSONArray pipeline = BSON_ARRAY(BSON("$out" << testBarNss.coll())); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline + << "bypassDocumentValidation" << true); + ASSERT_EQ(ErrorCodes::Unauthorized, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, + CanAggregateOutBypassingValidationWithBypassDocumentValidationOnTargetNamespace) { + authzSession->assumePrivilegesForDB( + {Privilege(testFooCollResource, ActionSet{ActionType::find}), + Privilege(testBarCollResource, + ActionSet{ActionType::insert, + ActionType::remove, + ActionType::bypassDocumentValidation})}); + + BSONArray pipeline = BSON_ARRAY(BSON("$out" << testBarNss.coll())); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline + << "bypassDocumentValidation" << true); + ASSERT_OK(Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, CannotAggregateLookupWithoutFindOnJoinedNamespace) { + authzSession->assumePrivilegesForDB( + Privilege(testFooCollResource, ActionSet{ActionType::find})); + + BSONArray pipeline = BSON_ARRAY(BSON("$lookup" << BSON("from" << testBarNss.coll()))); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline); + ASSERT_EQ(ErrorCodes::Unauthorized, + Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} + +TEST_F(PipelineAuthzTest, CanAggregateLookupWithFindOnJoinedNamespace) { + authzSession->assumePrivilegesForDB( + {Privilege(testFooCollResource, ActionSet{ActionType::find}), + Privilege(testBarCollResource, ActionSet{ActionType::find})}); + + BSONArray pipeline = BSON_ARRAY(BSON("$lookup" << BSON("from" << testBarNss.coll()))); + BSONObj cmdObj = BSON("aggregate" << testFooNss.coll() << "pipeline" << pipeline); + ASSERT_OK(Pipeline::checkAuthForCommand(client.get(), testFooDb, cmdObj)); +} +} // namespace AuthzTests } // namespace PipelineTests diff --git a/src/mongo/db/query/find.cpp b/src/mongo/db/query/find.cpp index 813b019aba7..dc2784b2dd8 100644 --- a/src/mongo/db/query/find.cpp +++ b/src/mongo/db/query/find.cpp @@ -42,7 +42,6 @@ #include "mongo/db/db_raii.h" #include "mongo/db/exec/filter.h" #include "mongo/db/exec/working_set_common.h" -#include "mongo/db/service_context.h" #include "mongo/db/keypattern.h" #include "mongo/db/matcher/extensions_callback_real.h" #include "mongo/db/query/explain.h" @@ -54,6 +53,7 @@ #include "mongo/db/s/sharding_state.h" #include "mongo/db/server_options.h" #include "mongo/db/server_parameters.h" +#include "mongo/db/service_context.h" #include "mongo/db/storage/storage_options.h" #include "mongo/s/chunk_version.h" #include "mongo/s/stale_exception.h" @@ -171,7 +171,7 @@ void endQueryOp(OperationContext* txn, if (dbProfilingLevel > 0 || curop->elapsedMillis() > serverGlobalParams.slowMS || logger::globalLogDomain()->shouldLog(commandLogComponent, logLevelOne)) { // Generate plan summary string. - stdx::lock_guard(*txn->getClient()); + stdx::lock_guard lk(*txn->getClient()); curop->setPlanSummary_inlock(Explain::getPlanSummary(&exec)); } @@ -389,7 +389,7 @@ QueryResult::View getMore(OperationContext* txn, auto planSummary = Explain::getPlanSummary(exec); { - stdx::lock_guard(*txn->getClient()); + stdx::lock_guard lk(*txn->getClient()); curop.setPlanSummary_inlock(planSummary); } @@ -525,13 +525,38 @@ std::string runQuery(OperationContext* txn, LOG(5) << "Running query:\n" << cq->toString(); LOG(2) << "Running query: " << cq->toStringShort(); + ShardingState* const shardingState = ShardingState::get(txn); + // Parse, canonicalize, plan, transcribe, and get a plan executor. - AutoGetCollectionForRead ctx(txn, nss); + boost::optional optionalCtx; + try { + optionalCtx.emplace(txn, nss); + } catch (const StaleConfigException& sce) { + // Wait for migration completion to get the correct chunk version + const int maxTimeoutSec = 30; + int timeoutSec = cq->getParsed().getMaxTimeMS() / 1000; + if (!timeoutSec || timeoutSec > maxTimeoutSec) { + timeoutSec = maxTimeoutSec; + } + + if (shardingState->waitTillNotInCriticalSection(maxTimeoutSec)) { + ChunkVersion unused; + shardingState->refreshMetadataIfNeeded( + txn, nss.ns(), sce.getVersionReceived(), &unused); + } + throw; + } + + const auto& ctx = *optionalCtx; Collection* collection = ctx.getCollection(); const int dbProfilingLevel = ctx.getDb() ? ctx.getDb()->getProfilingLevel() : serverGlobalParams.defaultProfile; + // It is possible that the sharding version will change during yield while we are retrieving a + // plan executor. If this happens we will throw an error and mongos will retry. + const ChunkVersion shardingVersionAtStart = shardingState->getVersion(nss.ns()); + // We have a parsed query. Time to get the execution plan for it. std::unique_ptr exec = uassertStatusOK( getExecutorFind(txn, collection, nss, std::move(cq), PlanExecutor::YIELD_AUTO)); @@ -568,11 +593,6 @@ std::string runQuery(OperationContext* txn, return ""; } - ShardingState* const shardingState = ShardingState::get(txn); - - // We freak out later if this changes before we're done with the query. - const ChunkVersion shardingVersionAtStart = shardingState->getVersion(nss.ns()); - // Handle query option $maxTimeMS (not used with commands). curop.setMaxTimeMicros(static_cast(pq.getMaxTimeMS()) * 1000); txn->checkForInterrupt(); // May trigger maxTimeAlwaysTimeOut fail point. diff --git a/src/mongo/db/query/lite_parsed_query.cpp b/src/mongo/db/query/lite_parsed_query.cpp index a4f9696b7a0..7dfdd093f12 100644 --- a/src/mongo/db/query/lite_parsed_query.cpp +++ b/src/mongo/db/query/lite_parsed_query.cpp @@ -375,7 +375,7 @@ StatusWith> LiteParsedQuery::makeFromFindCommand(Nam pq->addMetaProjection(); - Status validateStatus = pq->validateFindCmd(); + Status validateStatus = pq->validate(); if (!validateStatus.isOK()) { return validateStatus; } @@ -662,6 +662,11 @@ Status LiteParsedQuery::validate() const { } } + // AwaitData is only valid with a tailable cursor. + if (_awaitData && !_tailable) { + return Status(ErrorCodes::BadValue, "Cannot set awaitData without tailable"); + } + return Status::OK(); } @@ -975,14 +980,6 @@ void LiteParsedQuery::addMetaProjection() { } } -Status LiteParsedQuery::validateFindCmd() { - if (isAwaitData() && !isTailable()) { - return Status(ErrorCodes::BadValue, "Cannot set awaitData without tailable"); - } - - return validate(); -} - boost::optional LiteParsedQuery::getEffectiveBatchSize() const { return _batchSize ? _batchSize : _ntoreturn; } diff --git a/src/mongo/db/query/lite_parsed_query.h b/src/mongo/db/query/lite_parsed_query.h index 0287ad159e6..57c1ccd0a04 100644 --- a/src/mongo/db/query/lite_parsed_query.h +++ b/src/mongo/db/query/lite_parsed_query.h @@ -331,11 +331,6 @@ private: */ void addMetaProjection(); - /** - * Returns OK if this is valid in the find command context. - */ - Status validateFindCmd(); - const NamespaceString _nss; BSONObj _filter; diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript index 007a37c5c25..af7515855f9 100644 --- a/src/mongo/db/repl/SConscript +++ b/src/mongo/db/repl/SConscript @@ -250,6 +250,7 @@ env.Library('topology_coordinator_impl', 'repl_settings', 'rslog', 'topology_coordinator', + '$BUILD_DIR/mongo/util/fail_point', ]) env.CppUnitTest('repl_set_heartbeat_response_test', diff --git a/src/mongo/db/repl/applier.h b/src/mongo/db/repl/applier.h index a8e48c0edb9..d035e0d80ef 100644 --- a/src/mongo/db/repl/applier.h +++ b/src/mongo/db/repl/applier.h @@ -64,7 +64,7 @@ public: /** * Type of function to to apply a single operation. In production, this function - * would have the same outcome as calling SyncTail::syncApply() ('convertUpdatesToUpserts' + * would have the same outcome as calling SyncTail::syncApply() ('inSteadyStateReplication' * value will be embedded in the function implementation). */ using ApplyOperationFn = stdx::function; diff --git a/src/mongo/db/repl/bgsync.cpp b/src/mongo/db/repl/bgsync.cpp index a0312fa30f2..98d86f0c7c0 100644 --- a/src/mongo/db/repl/bgsync.cpp +++ b/src/mongo/db/repl/bgsync.cpp @@ -52,9 +52,12 @@ #include "mongo/db/repl/rollback_source_impl.h" #include "mongo/db/repl/rs_rollback.h" #include "mongo/db/repl/rs_sync.h" +#include "mongo/db/repl/storage_interface.h" #include "mongo/db/stats/timer_stats.h" #include "mongo/executor/network_interface_factory.h" +#include "mongo/rpc/get_status_from_command_result.h" #include "mongo/rpc/metadata/repl_set_metadata.h" +#include "mongo/rpc/metadata/server_selection_metadata.h" #include "mongo/stdx/memory.h" #include "mongo/util/concurrency/thread_pool.h" #include "mongo/util/exit.h" @@ -115,7 +118,7 @@ Status checkRemoteOplogStart(stdx::function()> getNextOperat } // namespace -MONGO_FP_DECLARE(rsBgSyncProduce); +MONGO_FP_DECLARE(stopReplProducer); BackgroundSync* BackgroundSync::s_instance = 0; stdx::mutex BackgroundSync::s_mutex; @@ -131,6 +134,12 @@ static ServerStatusMetricField displayOpsRead("repl.network.ops", &op static Counter64 networkByteStats; static ServerStatusMetricField displayBytesRead("repl.network.bytes", &networkByteStats); +// Failpoint which causes rollback to hang before starting. +MONGO_FP_DECLARE(rollbackHangBeforeStart); + +// Failpoint which causes the oplog fetcher to hang before the first fetch. +MONGO_FP_DECLARE(fetcherHangBeforeStart); + // The count of items in the buffer static Counter64 bufferCountGauge; static ServerStatusMetricField displayBufferCount("repl.buffer.count", @@ -154,6 +163,8 @@ size_t getSize(const BSONObj& o) { } } // namespace +const NamespaceString BackgroundSync::kLocalOplogNss("local.oplog.rs"); + BackgroundSync::BackgroundSync() : _buffer(bufferMaxSizeGauge, &getSize), _threadPoolTaskExecutor(makeThreadPool(), @@ -242,6 +253,8 @@ void BackgroundSync::_producerThread() { return; } + invariant(!state.rollback()); + // We need to wait until initial sync has started. if (_replCoord->getMyLastAppliedOpTime().isNull()) { sleepsecs(1); @@ -258,6 +271,21 @@ void BackgroundSync::_producerThread() { } void BackgroundSync::_produce(OperationContext* txn) { + if (MONGO_FAIL_POINT(stopReplProducer)) { + // This log output is used in js tests so please leave it. + log() << "bgsync - stopReplProducer fail point " + "enabled. Blocking until fail point is disabled."; + + // TODO(SERVER-27120): Remove the return statement and uncomment the while loop. + // Currently we cannot block here or we prevent primaries from being fully elected since + // we'll never call _signalNoNewDataForApplier. + // while (MONGO_FAIL_POINT(stopReplProducer) && !inShutdown()) { + // mongo::sleepsecs(1); + // } + mongo::sleepsecs(1); + return; + } + // this oplog reader does not do a handshake because we don't want the server it's syncing // from to track how far it has synced { @@ -276,16 +304,13 @@ void BackgroundSync::_produce(OperationContext* txn) { } } - while (MONGO_FAIL_POINT(rsBgSyncProduce)) { - sleepmillis(0); - } - - + HostAndPort oldSyncSource; // find a target to sync from the last optime fetched OpTime lastOpTimeFetched; { stdx::unique_lock lock(_mutex); lastOpTimeFetched = _lastOpTimeFetched; + oldSyncSource = _syncSourceHost; _syncSourceHost = HostAndPort(); } OplogReader syncSourceReader; @@ -296,7 +321,9 @@ void BackgroundSync::_produce(OperationContext* txn) { minValid = minValidSaved; } } - syncSourceReader.connectToSyncSource(txn, lastOpTimeFetched, minValid, _replCoord); + + int rbid; + syncSourceReader.connectToSyncSource(txn, lastOpTimeFetched, minValid, _replCoord, &rbid); // no server found if (syncSourceReader.getHost().empty()) { @@ -305,6 +332,17 @@ void BackgroundSync::_produce(OperationContext* txn) { return; } + // If our sync source has not changed, it is likely caused by our heartbeat data map being + // out of date. In that case we sleep for 1 second to reduce the amount we spin waiting + // for our map to update. + if (syncSourceReader.getHost() == oldSyncSource) { + log() << "Chose same sync source candidate as last time, " << oldSyncSource + << ". Sleeping for 1 second to avoid immediately choosing a new sync source for " + "the same reason as last time."; + + sleepsecs(1); + } + long long lastHashFetched; { stdx::lock_guard lock(_mutex); @@ -317,7 +355,7 @@ void BackgroundSync::_produce(OperationContext* txn) { _replCoord->signalUpstreamUpdater(); } - const Milliseconds oplogSocketTimeout(OplogReader::kSocketTimeout); + const Milliseconds kRollbackOplogSocketTimeout(10 * 60 * 1000); const auto isV1ElectionProtocol = _replCoord->isV1ElectionProtocol(); // Under protocol version 1, make the awaitData timeout (maxTimeMS) dependent on the election @@ -348,7 +386,8 @@ void BackgroundSync::_produce(OperationContext* txn) { lastOpTimeFetched, lastHashFetched, fetcherMaxTimeMS, - &fetcherReturnStatus); + &fetcherReturnStatus, + rbid); BSONObjBuilder cmdBob; @@ -365,16 +404,32 @@ void BackgroundSync::_produce(OperationContext* txn) { metadataBob.append(rpc::kReplSetMetadataFieldName, 1); } + if (MONGO_FAIL_POINT(fetcherHangBeforeStart)) { + // This log output is used in js tests so please leave it. + log() << "BackgroundSync - fetcherHangBeforeStart fail point " + "enabled. Blocking until fail point is disabled."; + while (MONGO_FAIL_POINT(fetcherHangBeforeStart) && !inShutdown()) { + mongo::sleepsecs(1); + } + + // If the sync source candidate rolls back while in this fail point, it will close all + // connections and the next request will fail. + // We manually drop all connections here so that the following Fetcher request succeeds. + _threadPoolTaskExecutor.dropConnections(source); + } + auto dbName = nsToDatabase(rsOplogName); auto cmdObj = cmdBob.obj(); auto metadataObj = metadataBob.obj(); + // 5 seconds more than the find command's 1 minute maxTimeMs + const Milliseconds oplogQueryNetworkTimeout = duration_cast(Seconds(65)); Fetcher fetcher(&_threadPoolTaskExecutor, source, dbName, cmdObj, fetcherCallback, metadataObj, - _replCoord->getConfig().getElectionTimeoutPeriod()); + oplogQueryNetworkTimeout); LOG(1) << "scheduling fetcher to read remote oplog on " << source << " starting at " << cmdObj["filter"]; @@ -409,10 +464,10 @@ void BackgroundSync::_produce(OperationContext* txn) { ConnectionPool connectionPool(messagingPortTags); std::unique_ptr connection; auto getConnection = - [&connection, &connectionPool, oplogSocketTimeout, source]() -> DBClientBase* { + [&connection, &connectionPool, kRollbackOplogSocketTimeout, source]() -> DBClientBase* { if (!connection.get()) { connection.reset(new ConnectionPool::ConnectionPtr( - &connectionPool, source, Date_t::now(), oplogSocketTimeout)); + &connectionPool, source, Date_t::now(), kRollbackOplogSocketTimeout)); }; return connection->get(); }; @@ -436,32 +491,55 @@ void BackgroundSync::_produce(OperationContext* txn) { } } } - // check that we are at minvalid, otherwise we cannot roll back as we may be in an - // inconsistent state - const auto minValid = getMinValid(txn); - if (lastApplied < minValid) { - fassertNoTrace(18750, - Status(ErrorCodes::UnrecoverableRollbackError, - str::stream() - << "need to rollback, but in inconsistent state. " - << "minvalid: " << minValid.toString() - << " > our last optime: " << lastApplied.toString())); - } - _rollback(txn, source, getConnection); + _rollback(txn, source, rbid, getConnection); stop(); } else if (!fetcherReturnStatus.isOK()) { warning() << "Fetcher error querying oplog: " << fetcherReturnStatus.toString(); } } +void BackgroundSync::_lastAppliedFetcherCallback(const StatusWith& result, + OpTime lastOpTimeFetched, + Status* returnStatus) { + if (!result.isOK()) { + *returnStatus = result.getStatus(); + return; + } + + const auto& queryResponse = result.getValue(); + if (queryResponse.documents.empty()) { + *returnStatus = Status(ErrorCodes::InvalidSyncSource, "Upstream node had an empty oplog."); + return; + } + + const auto& remoteLastAppliedDocument = queryResponse.documents.front(); + const auto remoteLastAppliedOpTime = OpTime::parseFromOplogEntry(remoteLastAppliedDocument); + if (!remoteLastAppliedOpTime.isOK()) { + *returnStatus = Status(ErrorCodes::InvalidBSON, + str::stream() << "Received invalid oplog entry from upstream node: " + << remoteLastAppliedDocument.toString() << ". Error: " + << remoteLastAppliedOpTime.getStatus().toString()); + return; + } + if (remoteLastAppliedOpTime.getValue() <= lastOpTimeFetched) { + *returnStatus = Status(ErrorCodes::InvalidSyncSource, + str::stream() << "Upstream node's last applied OpTime " + << remoteLastAppliedOpTime.getValue().toString() + << " is not greater than our last fetched OpTime " + << lastOpTimeFetched.toString()); + return; + } +} + void BackgroundSync::_fetcherCallback(const StatusWith& result, BSONObjBuilder* bob, const HostAndPort& source, OpTime lastOpTimeFetched, long long lastFetchedHash, Milliseconds fetcherMaxTimeMS, - Status* returnStatus) { + Status* returnStatus, + int rbid) { // if target cut connections between connecting and querying (for // example, because it stepped down) we might not have a cursor if (!result.isOK()) { @@ -481,26 +559,6 @@ void BackgroundSync::_fetcherCallback(const StatusWith& bool syncSourceHasSyncSource = false; OpTime sourcesLastOp; - // Forward metadata (containing liveness information) to replication coordinator. - bool receivedMetadata = - queryResponse.otherFields.metadata.hasElement(rpc::kReplSetMetadataFieldName); - if (receivedMetadata) { - auto metadataResult = - rpc::ReplSetMetadata::readFromMetadata(queryResponse.otherFields.metadata); - if (!metadataResult.isOK()) { - error() << "invalid replication metadata from sync source " << source << ": " - << metadataResult.getStatus() << ": " << queryResponse.otherFields.metadata; - return; - } - const auto& metadata = metadataResult.getValue(); - _replCoord->processReplSetMetadata(metadata); - if (metadata.getPrimaryIndex() != rpc::ReplSetMetadata::kNoPrimary) { - _replCoord->cancelAndRescheduleElectionTimeout(); - } - syncSourceHasSyncSource = metadata.getSyncSourceIndex() != -1; - sourcesLastOp = metadata.getLastOpVisible(); - } - const auto& documents = queryResponse.documents; auto firstDocToApply = documents.cbegin(); auto lastDocToApply = documents.cend(); @@ -515,6 +573,64 @@ void BackgroundSync::_fetcherCallback(const StatusWith& // Check start of remote oplog and, if necessary, stop fetcher to execute rollback. if (queryResponse.first) { + // Once we establish our cursor, we need to ensure that our upstream node hasn't rolled back + // since that could cause it to not have our required minValid point. The cursor will be + // killed if the upstream node rolls back so we don't need to keep checking. This must be + // blocking since the Fetcher doesn't give us a way to defer sending the getmores after we + // return. + auto handle = _threadPoolTaskExecutor.scheduleRemoteCommand( + {source, "admin", BSON("replSetGetRBID" << 1)}, + [&](const executor::TaskExecutor::RemoteCommandCallbackArgs& rbidReply) { + *returnStatus = rbidReply.response.getStatus(); + if (!returnStatus->isOK()) + return; + + const auto& rbidReplyObj = rbidReply.response.getValue().data; + *returnStatus = getStatusFromCommandResult(rbidReplyObj); + if (!returnStatus->isOK()) + return; + + const auto rbidElem = rbidReplyObj["rbid"]; + if (rbidElem.type() != NumberInt) { + *returnStatus = + Status(ErrorCodes::InvalidSyncSource, + str::stream() << "Upstream node returned an " + << "rbid with invalid type " << rbidElem.type()); + return; + } + if (rbidElem.Int() != rbid) { + *returnStatus = Status(ErrorCodes::InvalidSyncSource, + "Upstream node rolled back after verifying " + "that it had our MinValid point. Retrying."); + } + }); + if (!handle.isOK()) { + *returnStatus = handle.getStatus(); + return; + } + + _threadPoolTaskExecutor.wait(handle.getValue()); + if (!returnStatus->isOK()) + return; + + // Check that the upstream last applied OpTime is newer than our last fetched OpTime. + Fetcher lastAppliedFetcher(&_threadPoolTaskExecutor, + source, + kLocalOplogNss.db().toString(), + BSON("find" << kLocalOplogNss.coll() << "limit" << 1 << "sort" + << BSON("$natural" << -1)), + stdx::bind(&BackgroundSync::_lastAppliedFetcherCallback, + this, + stdx::placeholders::_1, + lastOpTimeFetched, + returnStatus), + rpc::ServerSelectionMetadata(true, boost::none).toBSON(), + Seconds(30)); + lastAppliedFetcher.schedule(); + lastAppliedFetcher.wait(); + if (!returnStatus->isOK()) + return; + auto getNextOperation = [&firstDocToApply, lastDocToApply]() -> StatusWith { if (firstDocToApply == lastDocToApply) { return Status(ErrorCodes::OplogStartMissing, "remote oplog start missing"); @@ -539,6 +655,32 @@ void BackgroundSync::_fetcherCallback(const StatusWith& return; } + if (MONGO_FAIL_POINT(stopReplProducer)) { + return; + } + + // Process replset metadata. It is important that this happen after we've validated the + // first batch, so we don't progress our knowledge of the commit point from a + // response that triggers a rollback. + bool receivedMetadata = + queryResponse.otherFields.metadata.hasElement(rpc::kReplSetMetadataFieldName); + if (receivedMetadata) { + auto metadataResult = + rpc::ReplSetMetadata::readFromMetadata(queryResponse.otherFields.metadata); + if (!metadataResult.isOK()) { + error() << "invalid replication metadata from sync source " << source << ": " + << metadataResult.getStatus() << ": " << queryResponse.otherFields.metadata; + return; + } + const auto& metadata = metadataResult.getValue(); + _replCoord->processReplSetMetadata(metadata, true /*advance commit point*/); + if (metadata.getPrimaryIndex() != rpc::ReplSetMetadata::kNoPrimary) { + _replCoord->cancelAndRescheduleElectionTimeout(); + } + syncSourceHasSyncSource = metadata.getSyncSourceIndex() != -1; + sourcesLastOp = metadata.getLastOpVisible(); + } + // The count of the bytes of the documents read off the network. int networkDocumentBytes = 0; Timestamp lastTS; @@ -702,40 +844,86 @@ void BackgroundSync::consume() { void BackgroundSync::_rollback(OperationContext* txn, const HostAndPort& source, + boost::optional requiredRBID, stdx::function getConnection) { - // Abort only when syncRollback detects we are in a unrecoverable state. - // In other cases, we log the message contained in the error status and retry later. - auto status = syncRollback(txn, - OplogInterfaceLocal(txn, rsOplogName), - RollbackSourceImpl(getConnection, source, rsOplogName), - _replCoord); - if (status.isOK()) { - // When the syncTail thread sees there is no new data by adding something to the buffer. - _signalNoNewDataForApplier(); - // Wait until the buffer is empty. - // This is an indication that syncTail has removed the sentinal marker from the buffer - // and reset its local lastAppliedOpTime via the replCoord. - while (!_buffer.empty()) { - sleepmillis(10); - if (inShutdown()) { - return; - } + if (MONGO_FAIL_POINT(rollbackHangBeforeStart)) { + // This log output is used in js tests so please leave it. + log() << "rollback - rollbackHangBeforeStart fail point " + "enabled. Blocking until fail point is disabled."; + while (MONGO_FAIL_POINT(rollbackHangBeforeStart) && !inShutdown()) { + mongo::sleepsecs(1); } + } - // It is now safe to clear the ROLLBACK state, which may result in the applier thread - // transitioning to SECONDARY. This is safe because the applier thread has now reloaded - // the new rollback minValid from the database. - if (!_replCoord->setFollowerMode(MemberState::RS_RECOVERING)) { - warning() << "Failed to transition into " << MemberState(MemberState::RS_RECOVERING) - << "; expected to be in state " << MemberState(MemberState::RS_ROLLBACK) - << " but found self in " << _replCoord->getMemberState(); + // Set state to ROLLBACK while we are in this function. This prevents serving reads, even from + // the oplog. This can fail if we are elected PRIMARY, in which case we better not do any + // rolling back. If we successfully enter ROLLBACK we will only exit this function fatally or + // after transitioning to RECOVERING. We always transition to RECOVERING regardless of success + // or (recoverable) failure since we may be in an inconsistent state. If rollback failed before + // writing anything, SyncTail will quickly take us to SECONDARY since are are still at our + // original MinValid, which is fine because we may choose a sync source that doesn't require + // rollback. If it failed after we wrote to MinValid, then we will pick a sync source that will + // cause us to roll back to the same common point, which is fine. If we succeeded, we will be + // consistent as soon as we apply up to/through MinValid and SyncTail will make us SECONDARY + // then. + { + log() << "rollback 0"; + Lock::GlobalWrite globalWrite(txn->lockState()); + if (!_replCoord->setFollowerMode(MemberState::RS_ROLLBACK)) { + log() << "Cannot transition from " << _replCoord->getMemberState().toString() << " to " + << MemberState(MemberState::RS_ROLLBACK).toString(); + return; } - return; } - if (ErrorCodes::UnrecoverableRollbackError == status.code()) { - fassertNoTrace(28723, status); + + try { + auto status = syncRollback(txn, + OplogInterfaceLocal(txn, rsOplogName), + RollbackSourceImpl(getConnection, source, rsOplogName), + requiredRBID, + _replCoord); + + // Abort only when syncRollback detects we are in a unrecoverable state. + // WARNING: these statuses sometimes have location codes which are lost with uassertStatusOK + // so we need to check here first. + if (ErrorCodes::UnrecoverableRollbackError == status.code()) { + severe() << "Unable to complete rollback. A full resync may be needed: " << status; + fassertFailedNoTrace(28723); + } + + // In other cases, we log the message contained in the error status and retry later. + uassertStatusOK(status); + } catch (const DBException& ex) { + // UnrecoverableRollbackError should only come from a returned status which is handled + // above. + invariant(ex.getCode() != ErrorCodes::UnrecoverableRollbackError); + + warning() << "rollback cannot complete at this time (retrying later): " << ex + << " appliedThrough=" << _replCoord->getMyLastAppliedOpTime() + << " minvalid=" << getMinValid(txn); + + // Sleep a bit to allow upstream node to coalesce, if that was the cause of the failure. If + // we failed in a way that will keep failing, but wasn't flagged as a fatal failure, this + // will also prevent us from hot-looping and putting too much load on upstream nodes. + sleepsecs(5); // 5 seconds was chosen as a completely arbitrary amount of time. + } catch (...) { + std::terminate(); + } + + // At this point we are about to leave rollback. Before we do, wait for any writes done + // as part of rollback to be durable, and then do any necessary checks that we didn't + // wind up rolling back something illegal. We must wait for the rollback to be durable + // so that if we wind up shutting down uncleanly in response to something we rolled back + // we know that we won't wind up right back in the same situation when we start back up + // because the rollback wasn't durable. + txn->recoveryUnit()->waitUntilDurable(); + + if (!_replCoord->setFollowerMode(MemberState::RS_RECOVERING)) { + severe() << "Failed to transition into " << MemberState(MemberState::RS_RECOVERING) + << "; expected to be in state " << MemberState(MemberState::RS_ROLLBACK) + << " but found self in " << _replCoord->getMemberState(); + fassertFailedNoTrace(40364); } - warning() << "rollback cannot proceed at this time (retrying later): " << status; } HostAndPort BackgroundSync::getSyncTarget() { diff --git a/src/mongo/db/repl/bgsync.h b/src/mongo/db/repl/bgsync.h index 489790f988d..20ea6629ba0 100644 --- a/src/mongo/db/repl/bgsync.h +++ b/src/mongo/db/repl/bgsync.h @@ -77,6 +77,8 @@ public: */ class BackgroundSync : public BackgroundSyncInterface { public: + static const NamespaceString kLocalOplogNss; + // Allow index prefetching to be turned on/off enum IndexPrefetchConfig { UNINITIALIZED = 0, @@ -187,7 +189,16 @@ private: OpTime lastOpTimeFetched, long long lastFetchedHash, Milliseconds fetcherMaxTimeMS, - Status* returnStatus); + Status* returnStatus, + int rbid); + + /** + * A callback to a Fetcher that checks that the remote last applied OpTime is newer than the + * local last fetched OpTime. + */ + void _lastAppliedFetcherCallback(const StatusWith& result, + OpTime lastOpTimeFetched, + Status* returnStatus); /** * Executes a rollback. @@ -195,6 +206,7 @@ private: */ void _rollback(OperationContext* txn, const HostAndPort& source, + boost::optional requiredRBID, stdx::function getConnection); /** diff --git a/src/mongo/db/repl/data_replicator.cpp b/src/mongo/db/repl/data_replicator.cpp index aed481df509..145ffc2e92c 100644 --- a/src/mongo/db/repl/data_replicator.cpp +++ b/src/mongo/db/repl/data_replicator.cpp @@ -386,10 +386,12 @@ Status DatabasesCloner::start() { _status = Status::OK(); log() << "starting cloning of all databases"; - // Schedule listDatabase command which will kick off the database cloner per result db. + // Schedule listDatabase command which will kick off the database cloner per result db. We only + // retrieve database names since computing & fetching all database stats can be costly on the + // remote node when there are a large number of collections. Request listDBsReq(_source, "admin", - BSON("listDatabases" << true), + BSON("listDatabases" << true << "nameOnly" << true), rpc::ServerSelectionMetadata(true, boost::none).toBSON()); CBHStatus s = _exec->scheduleRemoteCommand( listDBsReq, diff --git a/src/mongo/db/repl/freshness_checker.cpp b/src/mongo/db/repl/freshness_checker.cpp index 71b7b2e419b..ef1afe02f92 100644 --- a/src/mongo/db/repl/freshness_checker.cpp +++ b/src/mongo/db/repl/freshness_checker.cpp @@ -149,7 +149,8 @@ void FreshnessChecker::Algorithm::processResponse(const RemoteCommandRequest& re LOG(2) << "FreshnessChecker: Got response from " << request.target << " of " << res; if (res["fresher"].trueValue()) { - log() << "not electing self, we are not freshest"; + log() << "not electing self, " << request.target.toString() + << " knows a node is fresher than us"; _abortReason = FresherNodeFound; return; } @@ -162,10 +163,15 @@ void FreshnessChecker::Algorithm::processResponse(const RemoteCommandRequest& re } Timestamp remoteTime(res["opTime"].date()); if (remoteTime == _lastOpTimeApplied) { + log() << "not electing self, " << request.target.toString() + << " has same OpTime as us: " << remoteTime.toBSON(); _abortReason = FreshnessTie; } if (remoteTime > _lastOpTimeApplied) { // something really wrong (rogue command?) + log() << "not electing self, " << request.target.toString() + << " has newer OpTime than us. Our OpTime: " << _lastOpTimeApplied.toBSON() + << ", their OpTime: " << remoteTime.toBSON(); _abortReason = FresherNodeFound; return; } diff --git a/src/mongo/db/repl/freshness_checker_test.cpp b/src/mongo/db/repl/freshness_checker_test.cpp index a8a269b627b..606be981611 100644 --- a/src/mongo/db/repl/freshness_checker_test.cpp +++ b/src/mongo/db/repl/freshness_checker_test.cpp @@ -262,7 +262,8 @@ TEST_F(FreshnessCheckerTest, ElectNotElectingSelfWeAreNotFreshest) { stopCapturingLogMessages(); ASSERT_EQUALS(shouldAbortElection(), FreshnessChecker::FresherNodeFound); - ASSERT_EQUALS(1, countLogLinesContaining("not electing self, we are not freshest")); + ASSERT_EQUALS( + 1, countLogLinesContaining("not electing self, h1:27017 knows a node is fresher than us")); } TEST_F(FreshnessCheckerTest, ElectNotElectingSelfWeAreNotFreshestOpTime) { @@ -471,7 +472,8 @@ TEST_F(FreshnessCheckerTest, ElectNotElectingSelfWeAreNotFreshestManyNodes) { waitOnChecker(); stopCapturingLogMessages(); ASSERT_EQUALS(shouldAbortElection(), FreshnessChecker::FresherNodeFound); - ASSERT_EQUALS(1, countLogLinesContaining("not electing self, we are not freshest")); + ASSERT_EQUALS( + 1, countLogLinesContaining("not electing self, h1:27017 knows a node is fresher than us")); } TEST_F(FreshnessCheckerTest, ElectNotElectingSelfWeAreNotFreshestOpTimeManyNodes) { diff --git a/src/mongo/db/repl/last_vote.cpp b/src/mongo/db/repl/last_vote.cpp index 0b125624019..e7cc78ace9f 100644 --- a/src/mongo/db/repl/last_vote.cpp +++ b/src/mongo/db/repl/last_vote.cpp @@ -38,27 +38,31 @@ namespace { const std::string kCandidateIndexFieldName = "candidateIndex"; const std::string kTermFieldName = "term"; +const std::string kIdFieldName = "_id"; -const std::string kLegalFieldNames[] = { - kCandidateIndexFieldName, kTermFieldName, -}; +const std::string kLegalFieldNames[] = {kCandidateIndexFieldName, kTermFieldName, kIdFieldName}; } // namespace -Status LastVote::initialize(const BSONObj& argsObj) { - Status status = bsonCheckOnlyHasFields("VotedFar", argsObj, kLegalFieldNames); +LastVote::LastVote(long long term, long long candidateIndex) + : _candidateIndex(candidateIndex), _term(term) {} + +StatusWith LastVote::readFromLastVote(const BSONObj& doc) { + Status status = bsonCheckOnlyHasFields("LastVote", doc, kLegalFieldNames); if (!status.isOK()) return status; - status = bsonExtractIntegerField(argsObj, kTermFieldName, &_term); + long long term; + status = bsonExtractIntegerField(doc, kTermFieldName, &term); if (!status.isOK()) return status; - status = bsonExtractIntegerField(argsObj, kCandidateIndexFieldName, &_candidateIndex); + long long candidateIndex; + status = bsonExtractIntegerField(doc, kCandidateIndexFieldName, &candidateIndex); if (!status.isOK()) return status; - return Status::OK(); + return LastVote{term, candidateIndex}; } void LastVote::setTerm(long long term) { diff --git a/src/mongo/db/repl/last_vote.h b/src/mongo/db/repl/last_vote.h index 0466ffccbec..db1a9ab75e0 100644 --- a/src/mongo/db/repl/last_vote.h +++ b/src/mongo/db/repl/last_vote.h @@ -28,17 +28,20 @@ #pragma once +#include "mongo/base/status_with.h" + namespace mongo { class BSONObj; class BSONObjBuilder; -class Status; namespace repl { class LastVote { public: - Status initialize(const BSONObj& argsObj); + LastVote(long long term, long long candidateIndex); + + static StatusWith readFromLastVote(const BSONObj& doc); long long getTerm() const; long long getCandidateIndex() const; @@ -48,8 +51,8 @@ public: BSONObj toBSON() const; private: - long long _candidateIndex = -1; - long long _term = -1; + long long _candidateIndex; + long long _term; }; } // namespace repl diff --git a/src/mongo/db/repl/master_slave.cpp b/src/mongo/db/repl/master_slave.cpp index 1cb12ba561e..dcd5d77f395 100644 --- a/src/mongo/db/repl/master_slave.cpp +++ b/src/mongo/db/repl/master_slave.cpp @@ -537,7 +537,7 @@ bool ReplSource::handleDuplicateDbName(OperationContext* txn, // missing from master after optime "ts". return false; } - if (Database::duplicateUncasedName(db).empty()) { + if (dbHolder().getNamesWithConflictingCasing(db).empty()) { // No duplicate database names are present. return true; } @@ -547,7 +547,7 @@ bool ReplSource::handleDuplicateDbName(OperationContext* txn, { // This is always a GlobalWrite lock (so no ns/db used from the context) invariant(txn->lockState()->isW()); - Lock::TempRelease(txn->lockState()); + Lock::TempRelease tempRelease(txn->lockState()); // We always log an operation after executing it (never before), so // a database list will always be valid as of an oplog entry generated @@ -594,8 +594,7 @@ bool ReplSource::handleDuplicateDbName(OperationContext* txn, } // Check for duplicates again, since we released the lock above. - set duplicates; - Database::duplicateUncasedName(db, &duplicates); + auto duplicates = dbHolder().getNamesWithConflictingCasing(db); // The database is present on the master and no conflicting databases // are present on the master. Drop any local conflicts. @@ -610,20 +609,20 @@ bool ReplSource::handleDuplicateDbName(OperationContext* txn, massert(14034, "Duplicate database names present after attempting to delete duplicates", - Database::duplicateUncasedName(db).empty()); + dbHolder().getNamesWithConflictingCasing(db).empty()); return true; } void ReplSource::applyCommand(OperationContext* txn, const BSONObj& op) { try { - Status status = applyCommand_inlock(txn, op); + Status status = applyCommand_inlock(txn, op, true); if (!status.isOK()) { SyncTail sync(nullptr, SyncTail::MultiSyncApplyFunc()); sync.setHostname(hostName); if (sync.shouldRetry(txn, op)) { uassert(28639, "Failure retrying initial sync update", - applyCommand_inlock(txn, op).isOK()); + applyCommand_inlock(txn, op, true).isOK()); } } } catch (UserException& e) { diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp index d98db32fd43..63858050014 100644 --- a/src/mongo/db/repl/oplog.cpp +++ b/src/mongo/db/repl/oplog.cpp @@ -644,7 +644,8 @@ std::map opsMap = { {[](OperationContext* txn, const char* ns, BSONObj& cmd) -> Status { BSONObjBuilder resultWeDontCareAbout; return collMod(txn, parseNs(ns, cmd), cmd, &resultWeDontCareAbout); - }}}, + }, + {ErrorCodes::IndexNotFound, ErrorCodes::NamespaceNotFound}}}, {"dropDatabase", {[](OperationContext* txn, const char* ns, BSONObj& cmd) -> Status { return dropDatabase(txn, NamespaceString(ns).db().toString()); }, @@ -712,7 +713,7 @@ std::map opsMap = { Status applyOperation_inlock(OperationContext* txn, Database* db, const BSONObj& op, - bool convertUpdateToUpsert) { + bool inSteadyStateReplication) { LOG(3) << "applying op: " << op << endl; OpCounters* opCounters = txn->writesAreReplicated() ? &globalOpCounters : &replOpCounters; @@ -794,6 +795,7 @@ Status applyOperation_inlock(OperationContext* txn, // Wait for thread to start and register itself IndexBuilder::waitForBgIndexStarting(); } + txn->recoveryUnit()->abandonSnapshot(); } else { IndexBuilder builder(o); Status status = builder.buildInForeground(txn, db); @@ -866,7 +868,7 @@ Status applyOperation_inlock(OperationContext* txn, OpDebug debug; BSONObj updateCriteria = o2; - const bool upsert = valueB || convertUpdateToUpsert; + const bool upsert = valueB || inSteadyStateReplication; uassert(ErrorCodes::NoSuchKey, str::stream() << "Failed to apply update due to missing _id: " << op.toString(), @@ -949,7 +951,9 @@ Status applyOperation_inlock(OperationContext* txn, return Status::OK(); } -Status applyCommand_inlock(OperationContext* txn, const BSONObj& op) { +Status applyCommand_inlock(OperationContext* txn, + const BSONObj& op, + bool inSteadyStateReplication) { const char* names[] = {"o", "ns", "op"}; BSONElement fields[3]; op.getFields(3, names, fields); @@ -972,6 +976,14 @@ Status applyCommand_inlock(OperationContext* txn, const BSONObj& op) { const char* ns = fieldNs.valuestrsafe(); + // Applying renameCollection during initial sync might lead to data corruption, so we restart + // the initial sync. + if (!inSteadyStateReplication && o.firstElementFieldName() == std::string("renameCollection")) { + return Status(ErrorCodes::OplogOperationUnsupported, + str::stream() + << "Applying renameCollection not supported in initial sync: " << op); + } + // Applying commands in repl is done under Global W-lock, so it is safe to not // perform the current DB checks after reacquiring the lock. invariant(txn->lockState()->isW()); @@ -1190,8 +1202,7 @@ void SnapshotThread::run() { invariant(!opTimeOfSnapshot.isNull()); } - _manager->createSnapshot(txn.get(), name); - replCoord->onSnapshotCreate(opTimeOfSnapshot, name); + replCoord->createSnapshot(txn.get(), opTimeOfSnapshot, name); } catch (const WriteConflictException& wce) { log() << "skipping storage snapshot pass due to write conflict"; continue; diff --git a/src/mongo/db/repl/oplog.h b/src/mongo/db/repl/oplog.h index b022aa0e545..e3251102fef 100644 --- a/src/mongo/db/repl/oplog.h +++ b/src/mongo/db/repl/oplog.h @@ -128,20 +128,22 @@ void oplogCheckCloseDatabase(OperationContext* txn, Database* db); /** * Take a non-command op and apply it locally * Used for applying from an oplog - * @param convertUpdateToUpsert convert some updates to upserts for idempotency reasons + * @param inSteadyStateReplication convert some updates to upserts for idempotency reasons * Returns failure status if the op was an update that could not be applied. */ Status applyOperation_inlock(OperationContext* txn, Database* db, const BSONObj& op, - bool convertUpdateToUpsert = false); + bool inSteadyStateReplication = false); /** * Take a command op and apply it locally * Used for applying from an oplog + * inSteadyStateReplication indicates whether we are in steady state replication, rather than + * initial sync. * Returns failure status if the op that could not be applied. */ -Status applyCommand_inlock(OperationContext* txn, const BSONObj& op); +Status applyCommand_inlock(OperationContext* txn, const BSONObj& op, bool inSteadyStateReplication); /** * Initializes the global Timestamp with the value from the timestamp of the last oplog entry. diff --git a/src/mongo/db/repl/oplogreader.cpp b/src/mongo/db/repl/oplogreader.cpp index 63832b201af..f00b95e147a 100644 --- a/src/mongo/db/repl/oplogreader.cpp +++ b/src/mongo/db/repl/oplogreader.cpp @@ -165,7 +165,8 @@ Status OplogReader::_compareRequiredOpTimeWithQueryResponse(const OpTime& requir void OplogReader::connectToSyncSource(OperationContext* txn, const OpTime& lastOpTimeFetched, const OpTime& requiredOpTime, - ReplicationCoordinator* replCoord) { + ReplicationCoordinator* replCoord, + int* rbidOut) { const Timestamp sentinelTimestamp(duration_cast(Milliseconds(curTimeMillis64())), 0); const OpTime sentinel(sentinelTimestamp, std::numeric_limits::max()); OpTime oldestOpTimeSeen = sentinel; @@ -209,6 +210,18 @@ void OplogReader::connectToSyncSource(OperationContext* txn, replCoord->blacklistSyncSource(candidate, Date_t::now() + Seconds(10)); continue; } + + if (rbidOut) { + BSONObj reply; + if (!conn()->runCommand("admin", BSON("replSetGetRBID" << 1), reply)) { + log() << "Error getting rbid from " << candidate << ": " << reply; + resetConnection(); + replCoord->blacklistSyncSource(candidate, Date_t::now() + Seconds(10)); + continue; + } + *rbidOut = reply["rbid"].Int(); + } + // Read the first (oldest) op and confirm that it's not newer than our last // fetched op. Otherwise, we have fallen off the back of that source's oplog. BSONObj remoteOldestOp(findOne(rsOplogName.c_str(), Query())); diff --git a/src/mongo/db/repl/oplogreader.h b/src/mongo/db/repl/oplogreader.h index 1434125697a..3c983b007fc 100644 --- a/src/mongo/db/repl/oplogreader.h +++ b/src/mongo/db/repl/oplogreader.h @@ -153,12 +153,17 @@ public: * is left unconnected, where this->conn() equals NULL. * In the process of connecting, this function may add items to the repl coordinator's * sync source blacklist. + * + * If the rbidOut param is non-null it will be set to the rbid of the server before any data is + * fetched from it. + * * This function may throw DB exceptions. */ void connectToSyncSource(OperationContext* txn, const OpTime& lastOpTimeFetched, const OpTime& requiredOpTime, - ReplicationCoordinator* replCoord); + ReplicationCoordinator* replCoord, + int* rbidOut = nullptr); private: /** diff --git a/src/mongo/db/repl/repl_set_request_votes_args.cpp b/src/mongo/db/repl/repl_set_request_votes_args.cpp index 0041e78c4b2..3d55af3f93c 100644 --- a/src/mongo/db/repl/repl_set_request_votes_args.cpp +++ b/src/mongo/db/repl/repl_set_request_votes_args.cpp @@ -40,7 +40,9 @@ const std::string kCandidateIndexFieldName = "candidateIndex"; const std::string kCommandName = "replSetRequestVotes"; const std::string kConfigVersionFieldName = "configVersion"; const std::string kDryRunFieldName = "dryRun"; -const std::string kLastCommittedOpFieldName = "lastCommittedOp"; +// The underlying field name is inaccurate, but changing it requires a fair amount of cross +// compatibility work for no real benefit. +const std::string kLastDurableOpTimeFieldName = "lastCommittedOp"; const std::string kOkFieldName = "ok"; const std::string kReasonFieldName = "reason"; const std::string kSetNameFieldName = "setName"; @@ -52,7 +54,7 @@ const std::string kLegalArgsFieldNames[] = { kCommandName, kConfigVersionFieldName, kDryRunFieldName, - kLastCommittedOpFieldName, + kLastDurableOpTimeFieldName, kSetNameFieldName, kTermFieldName, }; @@ -89,7 +91,7 @@ Status ReplSetRequestVotesArgs::initialize(const BSONObj& argsObj) { if (!status.isOK()) return status; - status = bsonExtractOpTimeField(argsObj, kLastCommittedOpFieldName, &_lastCommittedOp); + status = bsonExtractOpTimeField(argsObj, kLastDurableOpTimeFieldName, &_lastDurableOpTime); if (!status.isOK()) return status; @@ -112,8 +114,8 @@ long long ReplSetRequestVotesArgs::getConfigVersion() const { return _cfgver; } -OpTime ReplSetRequestVotesArgs::getLastCommittedOp() const { - return _lastCommittedOp; +OpTime ReplSetRequestVotesArgs::getLastDurableOpTime() const { + return _lastDurableOpTime; } bool ReplSetRequestVotesArgs::isADryRun() const { @@ -127,7 +129,7 @@ void ReplSetRequestVotesArgs::addToBSON(BSONObjBuilder* builder) const { builder->append(kTermFieldName, _term); builder->appendIntOrLL(kCandidateIndexFieldName, _candidateIndex); builder->appendIntOrLL(kConfigVersionFieldName, _cfgver); - _lastCommittedOp.append(builder, kLastCommittedOpFieldName); + _lastDurableOpTime.append(builder, kLastDurableOpTimeFieldName); } Status ReplSetRequestVotesResponse::initialize(const BSONObj& argsObj) { diff --git a/src/mongo/db/repl/repl_set_request_votes_args.h b/src/mongo/db/repl/repl_set_request_votes_args.h index 12541650798..7f1cf3bc72e 100644 --- a/src/mongo/db/repl/repl_set_request_votes_args.h +++ b/src/mongo/db/repl/repl_set_request_votes_args.h @@ -46,7 +46,7 @@ public: long long getTerm() const; long long getCandidateIndex() const; long long getConfigVersion() const; - OpTime getLastCommittedOp() const; + OpTime getLastDurableOpTime() const; bool isADryRun() const; void addToBSON(BSONObjBuilder* builder) const; @@ -56,9 +56,9 @@ private: long long _term = -1; // Current known term of the command issuer. // replSet config index of the member who sent the replSetRequestVotesCmd. long long _candidateIndex = -1; - long long _cfgver = -1; // replSet config version known to the command issuer. - OpTime _lastCommittedOp; // The last known committed op of the command issuer. - bool _dryRun = false; // Indicates this is a pre-election check when true. + long long _cfgver = -1; // replSet config version known to the command issuer. + OpTime _lastDurableOpTime; // The last known durable op of the command issuer. + bool _dryRun = false; // Indicates this is a pre-election check when true. }; class ReplSetRequestVotesResponse { diff --git a/src/mongo/db/repl/replication_coordinator.h b/src/mongo/db/repl/replication_coordinator.h index dd55fcca799..d7eba5a493c 100644 --- a/src/mongo/db/repl/replication_coordinator.h +++ b/src/mongo/db/repl/replication_coordinator.h @@ -466,14 +466,13 @@ public: virtual void processReplSetGetConfig(BSONObjBuilder* result) = 0; /** - * Processes the ReplSetMetadata returned from a command run against another replica set - * member and updates protocol version 1 information (most recent optime that is committed, - * member id of the current PRIMARY, the current config version and the current term). - * - * TODO(dannenberg): Move this method to be testing only if it does not end up being used - * to process the find and getmore metadata responses from the DataReplicator. + * Processes the ReplSetMetadata returned from a command run against another + * replica set member and so long as the config version in the metadata matches the replica set + * config version this node currently has, updates the current term and optionally updates + * this node's notion of the commit point. */ - virtual void processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata) = 0; + virtual void processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata, + bool advanceCommitPoint) = 0; /** * Elections under protocol version 1 are triggered by a timer. @@ -685,9 +684,8 @@ public: /** * Prepares a metadata object describing the current term, primary, and lastOp information. */ - virtual void prepareReplResponseMetadata(const rpc::RequestInterface& request, - const OpTime& lastOpTimeFromClient, - BSONObjBuilder* builder) = 0; + virtual void prepareReplMetadata(const OpTime& lastOpTimeFromClient, + BSONObjBuilder* builder) const = 0; /** * Returns true if the V1 election protocol is being used and false otherwise. @@ -745,9 +743,12 @@ public: virtual void forceSnapshotCreation() = 0; /** - * Called when a new snapshot is created. + * Creates a new snapshot in the storage engine and registers it for use in the replication + * coordinator. */ - virtual void onSnapshotCreate(OpTime timeOfSnapshot, SnapshotName name) = 0; + virtual void createSnapshot(OperationContext* txn, + OpTime timeOfSnapshot, + SnapshotName name) = 0; /** * Blocks until either the current committed snapshot is at least as high as 'untilSnapshot', @@ -765,7 +766,7 @@ public: /** * Gets the latest OpTime of the currentCommittedSnapshot. */ - virtual OpTime getCurrentCommittedSnapshotOpTime() = 0; + virtual OpTime getCurrentCommittedSnapshotOpTime() const = 0; /** * Appends connection information to the provided BSONObjBuilder. diff --git a/src/mongo/db/repl/replication_coordinator_external_state.h b/src/mongo/db/repl/replication_coordinator_external_state.h index b595f606a61..de8ee0011e1 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state.h +++ b/src/mongo/db/repl/replication_coordinator_external_state.h @@ -233,6 +233,11 @@ public: */ virtual void updateCommittedSnapshot(SnapshotName newCommitPoint) = 0; + /** + * Creates a new snapshot. + */ + virtual void createSnapshot(OperationContext* txn, SnapshotName name) = 0; + /** * Signals the SnapshotThread, if running, to take a forced snapshot even if the global * timestamp hasn't changed. diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp index 128e18b9f53..f0e4cb4e0ee 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp @@ -297,9 +297,7 @@ StatusWith ReplicationCoordinatorExternalStateImpl::loadLocalLastVoteD << "Did not find replica set lastVote document in " << lastVoteCollectionName); } - LastVote lastVote; - lastVote.initialize(lastVoteObj); - return StatusWith(lastVote); + return LastVote::readFromLastVote(lastVoteObj); } MONGO_WRITE_CONFLICT_RETRY_LOOP_END( txn, "load replica set lastVote", lastVoteCollectionName); @@ -315,12 +313,29 @@ Status ReplicationCoordinatorExternalStateImpl::storeLocalLastVoteDocument( MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { ScopedTransaction transaction(txn, MODE_IX); Lock::DBLock dbWriteLock(txn->lockState(), lastVoteDatabaseName, MODE_X); - Helpers::putSingleton(txn, lastVoteCollectionName, lastVoteObj); - return Status::OK(); + + // If there is no last vote document, we want to store one. Otherwise, we only want to + // replace it if the new last vote document would have a higher term. We both check + // the term of the current last vote document and insert the new document under the + // DBLock to synchronize the two operations. + BSONObj result; + bool exists = Helpers::getSingleton(txn, lastVoteCollectionName, result); + if (!exists) { + Helpers::putSingleton(txn, lastVoteCollectionName, lastVoteObj); + } else { + StatusWith oldLastVoteDoc = LastVote::readFromLastVote(result); + if (!oldLastVoteDoc.isOK()) { + return oldLastVoteDoc.getStatus(); + } + if (lastVote.getTerm() > oldLastVoteDoc.getValue().getTerm()) { + Helpers::putSingleton(txn, lastVoteCollectionName, lastVoteObj); + } + } } MONGO_WRITE_CONFLICT_RETRY_LOOP_END( txn, "save replica set lastVote", lastVoteCollectionName); - MONGO_UNREACHABLE; + txn->recoveryUnit()->waitUntilDurable(); + return Status::OK(); } catch (const DBException& ex) { return ex.toStatus(); } @@ -533,6 +548,13 @@ void ReplicationCoordinatorExternalStateImpl::updateCommittedSnapshot(SnapshotNa manager->setCommittedSnapshot(newCommitPoint); } +void ReplicationCoordinatorExternalStateImpl::createSnapshot(OperationContext* txn, + SnapshotName name) { + auto manager = getGlobalServiceContext()->getGlobalStorageEngine()->getSnapshotManager(); + invariant(manager); // This should never be called if there is no SnapshotManager. + manager->createSnapshot(txn, name); +} + void ReplicationCoordinatorExternalStateImpl::forceSnapshotCreation() { if (_snapshotThread) _snapshotThread->forceSnapshot(); diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.h b/src/mongo/db/repl/replication_coordinator_external_state_impl.h index 26667fbba96..fb3135b8592 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_impl.h +++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.h @@ -79,6 +79,7 @@ public: virtual void dropAllTempCollections(OperationContext* txn); void dropAllSnapshots() final; void updateCommittedSnapshot(SnapshotName newCommitPoint) final; + void createSnapshot(OperationContext* txn, SnapshotName name) final; void forceSnapshotCreation() final; virtual bool snapshotsEnabled() const; virtual void notifyOplogMetadataWaiters(); diff --git a/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp b/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp index 2918d8cdcad..a05a7e7ae06 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp +++ b/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp @@ -216,6 +216,9 @@ void ReplicationCoordinatorExternalStateMock::dropAllSnapshots() {} void ReplicationCoordinatorExternalStateMock::updateCommittedSnapshot(SnapshotName newCommitPoint) { } +void ReplicationCoordinatorExternalStateMock::createSnapshot(OperationContext* txn, + SnapshotName name) {} + void ReplicationCoordinatorExternalStateMock::forceSnapshotCreation() {} bool ReplicationCoordinatorExternalStateMock::snapshotsEnabled() const { diff --git a/src/mongo/db/repl/replication_coordinator_external_state_mock.h b/src/mongo/db/repl/replication_coordinator_external_state_mock.h index 929772f5820..1434fc34bab 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_mock.h +++ b/src/mongo/db/repl/replication_coordinator_external_state_mock.h @@ -80,6 +80,7 @@ public: virtual void dropAllTempCollections(OperationContext* txn); virtual void dropAllSnapshots(); virtual void updateCommittedSnapshot(SnapshotName newCommitPoint); + virtual void createSnapshot(OperationContext* txn, SnapshotName name); virtual void forceSnapshotCreation(); virtual bool snapshotsEnabled() const; virtual void notifyOplogMetadataWaiters(); diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp index 5dcf7b9b1c3..88e9310d141 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl.cpp @@ -329,7 +329,7 @@ Date_t ReplicationCoordinatorImpl::getPriorityTakeover_forTest() const { return _priorityTakeoverWhen; } -OpTime ReplicationCoordinatorImpl::getCurrentCommittedSnapshotOpTime() { +OpTime ReplicationCoordinatorImpl::getCurrentCommittedSnapshotOpTime() const { stdx::lock_guard lk(_mutex); if (_currentCommittedSnapshot) { return _currentCommittedSnapshot->opTime; @@ -348,7 +348,13 @@ void ReplicationCoordinatorImpl::_updateLastVote(const LastVote& lastVote) { bool ReplicationCoordinatorImpl::_startLoadLocalConfig(OperationContext* txn) { StatusWith lastVote = _externalState->loadLocalLastVoteDocument(txn); if (!lastVote.isOK()) { - log() << "Did not find local voted for document at startup; " << lastVote.getStatus(); + if (lastVote.getStatus() == ErrorCodes::NoMatchingDocument) { + log() << "Did not find local voted for document at startup."; + } else { + severe() << "Error loading local voted for document at startup; " + << lastVote.getStatus(); + fassertFailedNoTrace(40367); + } } else { LastVote vote = lastVote.getValue(); _replExecutor.scheduleWork( @@ -1439,10 +1445,38 @@ ReplicationCoordinator::StatusAndDuration ReplicationCoordinatorImpl::_awaitRepl return StatusAndDuration(Status::OK(), Milliseconds(timer->millis())); } - if (replMode == modeReplSet && !_memberState.primary()) { - return StatusAndDuration( - Status(ErrorCodes::NotMaster, "Not master while waiting for replication"), - Milliseconds(timer->millis())); + auto checkForStepDown = [&]() -> Status { + if (replMode == modeReplSet && !_memberState.primary()) { + return {ErrorCodes::NotMaster, "Primary stepped down while waiting for replication"}; + } + + // Relax term checking under 3.2 because some commands (eg. createIndexes) might not return + // a term in the response metadata to mongos which may pass the no-term OpTime back to + // mongod eventually. + if (opTime.getTerm() != OpTime::kUninitializedTerm && + _cachedTerm != OpTime::kUninitializedTerm && opTime.getTerm() != _cachedTerm) { + return { + ErrorCodes::NotMaster, + str::stream() << "Term changed from " << opTime.getTerm() << " to " << _cachedTerm + << " while waiting for replication, indicating that this node must " + "have stepped down."}; + } + + if (_stepDownPending) { + return {ErrorCodes::NotMaster, + "Received stepdown request while waiting for replication"}; + } + return Status::OK(); + }; + + Status stepdownStatus = checkForStepDown(); + if (!stepdownStatus.isOK()) { + return StatusAndDuration(stepdownStatus, Milliseconds(timer->millis())); + } + + auto interruptStatus = txn->checkForInterruptNoAssert(); + if (!interruptStatus.isOK()) { + return StatusAndDuration(interruptStatus, Milliseconds(timer->millis())); } if (writeConcern.wMode.empty()) { @@ -1464,6 +1498,7 @@ ReplicationCoordinator::StatusAndDuration ReplicationCoordinatorImpl::_awaitRepl return StatusAndDuration(interruptedStatus, elapsed); } + if (!waitInfo.master) { return StatusAndDuration(Status(ErrorCodes::NotMaster, "Not master anymore while waiting for replication" @@ -1500,6 +1535,11 @@ ReplicationCoordinator::StatusAndDuration ReplicationCoordinatorImpl::_awaitRepl } else { condVar.wait_for(*lock, waitTime); } + + stepdownStatus = checkForStepDown(); + if (!stepdownStatus.isOK()) { + return StatusAndDuration(stepdownStatus, elapsed); + } } Status status = _checkIfWriteConcernCanBeSatisfied_inlock(writeConcern); @@ -1638,7 +1678,8 @@ void ReplicationCoordinatorImpl::_stepDownContinue( return; } bool forceNow = now >= waitUntil ? force : false; - if (_topCoord->stepDown(stepDownUntil, forceNow, getMyLastAppliedOpTime())) { + if (_topCoord->stepDown( + stepDownUntil, forceNow, getMyLastAppliedOpTime(), getLastCommittedOpTime())) { // Schedule work to (potentially) step back up once the stepdown period has ended. _replExecutor.scheduleWorkAt(stepDownUntil, stdx::bind(&ReplicationCoordinatorImpl::_handleTimePassing, @@ -1845,32 +1886,35 @@ int ReplicationCoordinatorImpl::_getMyId_inlock() const { } bool ReplicationCoordinatorImpl::prepareReplSetUpdatePositionCommand(BSONObjBuilder* cmdBuilder) { - stdx::lock_guard lock(_mutex); - invariant(_rsConfig.isInitialized()); - // Do not send updates if we have been removed from the config. - if (_selfIndex == -1) { - return false; - } - cmdBuilder->append("replSetUpdatePosition", 1); - // Create an array containing objects each live member connected to us and for ourself. - BSONArrayBuilder arrayBuilder(cmdBuilder->subarrayStart("optimes")); - for (SlaveInfoVector::iterator itr = _slaveInfo.begin(); itr != _slaveInfo.end(); ++itr) { - if (itr->lastAppliedOpTime.isNull()) { - // Don't include info on members we haven't heard from yet. - continue; - } - // Don't include members we think are down. - if (!itr->self && itr->down) { - continue; + { + stdx::lock_guard lock(_mutex); + invariant(_rsConfig.isInitialized()); + // Do not send updates if we have been removed from the config. + if (_selfIndex == -1) { + return false; } + cmdBuilder->append("replSetUpdatePosition", 1); + // Create an array containing objects each live member connected to us and for ourself. + BSONArrayBuilder arrayBuilder(cmdBuilder->subarrayStart("optimes")); + for (SlaveInfoVector::iterator itr = _slaveInfo.begin(); itr != _slaveInfo.end(); ++itr) { + if (itr->lastAppliedOpTime.isNull()) { + // Don't include info on members we haven't heard from yet. + continue; + } + // Don't include members we think are down. + if (!itr->self && itr->down) { + continue; + } - BSONObjBuilder entry(arrayBuilder.subobjStart()); - itr->lastDurableOpTime.append(&entry, "durableOpTime"); - itr->lastAppliedOpTime.append(&entry, "appliedOpTime"); - entry.append("memberId", itr->memberId); - entry.append("cfgver", _rsConfig.getConfigVersion()); + BSONObjBuilder entry(arrayBuilder.subobjStart()); + itr->lastDurableOpTime.append(&entry, "durableOpTime"); + itr->lastAppliedOpTime.append(&entry, "appliedOpTime"); + entry.append("memberId", itr->memberId); + entry.append("cfgver", _rsConfig.getConfigVersion()); + } } - + // Add metadata to command. Old style parsing logic will reject the metadata. + prepareReplMetadata(OpTime(), cmdBuilder); return true; } @@ -1991,10 +2035,11 @@ void ReplicationCoordinatorImpl::processReplSetGetConfig(BSONObjBuilder* result) result->append("config", _rsConfig.toBSON()); } -void ReplicationCoordinatorImpl::processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata) { +void ReplicationCoordinatorImpl::processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata, + bool advanceCommitPoint) { EventHandle evh; - _scheduleWorkAndWaitForCompletion([this, &evh, &replMetadata](const CallbackArgs& args) { - evh = _processReplSetMetadata_incallback(replMetadata); + _scheduleWorkAndWaitForCompletion([&](const CallbackArgs& args) { + evh = _processReplSetMetadata_incallback(replMetadata, advanceCommitPoint); }); if (evh.isValid()) { _replExecutor.waitForEvent(evh); @@ -2007,11 +2052,13 @@ void ReplicationCoordinatorImpl::cancelAndRescheduleElectionTimeout() { } EventHandle ReplicationCoordinatorImpl::_processReplSetMetadata_incallback( - const rpc::ReplSetMetadata& replMetadata) { + const rpc::ReplSetMetadata& replMetadata, bool advanceCommitPoint) { if (replMetadata.getConfigVersion() != _rsConfig.getConfigVersion()) { return EventHandle(); } - _setLastCommittedOpTime(replMetadata.getLastOpCommitted()); + if (advanceCommitPoint) { + _setLastCommittedOpTime(replMetadata.getLastOpCommitted()); + } return _updateTerm_incallback(replMetadata.getTerm()); } @@ -2147,6 +2194,16 @@ void ReplicationCoordinatorImpl::_processReplSetFreeze_finish( return; } + if (_topCoord->getRole() != TopologyCoordinator::Role::follower) { + *result = Status(ErrorCodes::NotSecondary, + str::stream() + << "cannot freeze node when primary or running for election. state: " + << (_topCoord->getRole() == TopologyCoordinator::Role::leader + ? "Primary" + : "Running-Election")); + return; + } + _topCoord->prepareFreezeResponse(_replExecutor.now(), secs, response); if (_topCoord->getRole() == TopologyCoordinator::Role::candidate) { @@ -2321,12 +2378,14 @@ Status ReplicationCoordinatorImpl::processReplSetReconfig(OperationContext* txn, return status; } + auto finishEvh = uassertStatusOK(_replExecutor.makeEvent()); const stdx::function reconfigFinishFn( stdx::bind(&ReplicationCoordinatorImpl::_finishReplSetReconfig, this, stdx::placeholders::_1, newConfig, - myIndex.getValue())); + myIndex.getValue(), + finishEvh)); // If it's a force reconfig, the primary node may not be electable after the configuration // change. In case we are that primary node, finish the reconfig under the global lock, @@ -2338,14 +2397,15 @@ Status ReplicationCoordinatorImpl::processReplSetReconfig(OperationContext* txn, } fassert(18824, cbh.getStatus()); configStateGuard.Dismiss(); - _replExecutor.wait(cbh.getValue()); + _replExecutor.waitForEvent(finishEvh); return Status::OK(); } void ReplicationCoordinatorImpl::_finishReplSetReconfig( const ReplicationExecutor::CallbackArgs& cbData, const ReplicaSetConfig& newConfig, - int myIndex) { + int myIndex, + ReplicationExecutor::EventHandle finishedEvent) { stdx::unique_lock lk(_mutex); invariant(_rsConfigState == kConfigReconfiguring); invariant(_rsConfig.isInitialized()); @@ -2353,8 +2413,18 @@ void ReplicationCoordinatorImpl::_finishReplSetReconfig( const PostMemberStateUpdateAction action = _setCurrentRSConfig_inlock(cbData, newConfig, myIndex); lk.unlock(); - _resetElectionInfoOnProtocolVersionUpgrade(oldConfig, newConfig); - _performPostMemberStateUpdateAction(action); + auto evh = _resetElectionInfoOnProtocolVersionUpgrade(oldConfig, newConfig); + if (evh) { + _replExecutor.onEvent( + evh, + [this, action, finishedEvent](const ReplicationExecutor::CallbackArgs& cbArgs) { + _performPostMemberStateUpdateAction(action); + _replExecutor.signalEvent(finishedEvent); + }); + } else { + _performPostMemberStateUpdateAction(action); + _replExecutor.signalEvent(finishedEvent); + } } Status ReplicationCoordinatorImpl::processReplSetInitiate(OperationContext* txn, @@ -2475,11 +2545,8 @@ void ReplicationCoordinatorImpl::_finishReplSetInitiate( stdx::unique_lock lk(_mutex); invariant(_rsConfigState == kConfigInitiating); invariant(!_rsConfig.isInitialized()); - const ReplicaSetConfig oldConfig = _rsConfig; - const PostMemberStateUpdateAction action = - _setCurrentRSConfig_inlock(cbData, newConfig, myIndex); + auto action = _setCurrentRSConfig_inlock(cbData, newConfig, myIndex); lk.unlock(); - _resetElectionInfoOnProtocolVersionUpgrade(oldConfig, newConfig); _performPostMemberStateUpdateAction(action); } @@ -2517,6 +2584,7 @@ ReplicationCoordinatorImpl::_updateMemberStateFromTopologyCoordinator_inlock() { info->condVar->notify_all(); } _canAcceptNonLocalWrites = false; + _stepDownPending = false; result = kActionCloseAllConnections; } else { result = kActionFollowerModeStateChange; @@ -3093,7 +3161,7 @@ bool ReplicationCoordinatorImpl::shouldChangeSyncSource(const HostAndPort& curre } void ReplicationCoordinatorImpl::_updateLastCommittedOpTime_inlock() { - if (!_getMemberState_inlock().primary()) { + if (!_getMemberState_inlock().primary() || _stepDownPending) { return; } @@ -3200,10 +3268,8 @@ Status ReplicationCoordinatorImpl::processReplSetRequestVotes( return cbh.getStatus(); } _replExecutor.wait(cbh.getValue()); - if (response->getVoteGranted()) { - LastVote lastVote; - lastVote.setTerm(args.getTerm()); - lastVote.setCandidateIndex(args.getCandidateIndex()); + if (!args.isADryRun() && response->getVoteGranted()) { + LastVote lastVote{args.getTerm(), args.getCandidateIndex()}; Status status = _externalState->storeLocalLastVoteDocument(txn, lastVote); if (!status.isOK()) { @@ -3266,37 +3332,35 @@ void ReplicationCoordinatorImpl::_processReplSetDeclareElectionWinner_finish( *result = _topCoord->processReplSetDeclareElectionWinner(args, responseTerm); } -void ReplicationCoordinatorImpl::prepareReplResponseMetadata(const rpc::RequestInterface& request, - const OpTime& lastOpTimeFromClient, - BSONObjBuilder* builder) { - if (request.getMetadata().hasField(rpc::kReplSetMetadataFieldName)) { - rpc::ReplSetMetadata metadata; - CBHStatus cbh = _replExecutor.scheduleWork( - stdx::bind(&ReplicationCoordinatorImpl::_prepareReplResponseMetadata_finish, - this, - stdx::placeholders::_1, - lastOpTimeFromClient, - &metadata)); +void ReplicationCoordinatorImpl::prepareReplMetadata(const OpTime& lastOpTimeFromClient, + BSONObjBuilder* builder) const { + rpc::ReplSetMetadata metadata; - if (cbh.getStatus() == ErrorCodes::ShutdownInProgress) { - return; - } - - fassert(28709, cbh.getStatus()); - _replExecutor.wait(cbh.getValue()); + CBHStatus cbh = _replExecutor.scheduleWork( + stdx::bind(&ReplicationCoordinatorImpl::_prepareReplResponseMetadata_finish, + this, + stdx::placeholders::_1, + lastOpTimeFromClient, + &metadata)); - metadata.writeToMetadata(builder); + if (cbh.getStatus() == ErrorCodes::ShutdownInProgress) { + return; } + + fassert(28709, cbh.getStatus()); + _replExecutor.wait(cbh.getValue()); + + metadata.writeToMetadata(builder); } void ReplicationCoordinatorImpl::_prepareReplResponseMetadata_finish( const ReplicationExecutor::CallbackArgs& cbData, const OpTime& lastOpTimeFromClient, - rpc::ReplSetMetadata* metadata) { + rpc::ReplSetMetadata* metadata) const { OpTime lastReadableOpTime = getCurrentCommittedSnapshotOpTime(); OpTime lastVisibleOpTime = std::max(lastOpTimeFromClient, lastReadableOpTime); - _topCoord->prepareReplResponseMetadata(metadata, lastVisibleOpTime, _lastCommittedOpTime); + _topCoord->prepareReplMetadata(metadata, lastVisibleOpTime, _lastCommittedOpTime); } bool ReplicationCoordinatorImpl::isV1ElectionProtocol() { @@ -3503,7 +3567,7 @@ EventHandle ReplicationCoordinatorImpl::_updateTerm_incallback( if (localUpdateTermResult == TopologyCoordinator::UpdateTermResult::kTriggerStepDown) { log() << "stepping down from primary, because a new term has begun: " << term; _topCoord->prepareForStepDown(); - return _stepDownStart(); + return _stepDownStart(false); } return EventHandle(); } @@ -3541,9 +3605,11 @@ size_t ReplicationCoordinatorImpl::getNumUncommittedSnapshots() { return _uncommittedSnapshotsSize.load(); } -void ReplicationCoordinatorImpl::onSnapshotCreate(OpTime timeOfSnapshot, SnapshotName name) { +void ReplicationCoordinatorImpl::createSnapshot(OperationContext* txn, + OpTime timeOfSnapshot, + SnapshotName name) { stdx::lock_guard lock(_mutex); - + _externalState->createSnapshot(txn, name); auto snapshotInfo = SnapshotInfo{timeOfSnapshot, name}; if (timeOfSnapshot <= _lastCommittedOpTime) { @@ -3609,35 +3675,40 @@ void ReplicationCoordinatorImpl::waitForElectionDryRunFinish_forTest() { } } -void ReplicationCoordinatorImpl::_resetElectionInfoOnProtocolVersionUpgrade( +EventHandle ReplicationCoordinatorImpl::_resetElectionInfoOnProtocolVersionUpgrade( const ReplicaSetConfig& oldConfig, const ReplicaSetConfig& newConfig) { // On protocol version upgrade, reset last vote as if I just learned the term 0 from other // nodes. if (!oldConfig.isInitialized() || oldConfig.getProtocolVersion() >= newConfig.getProtocolVersion()) { - return; + return {}; } invariant(newConfig.getProtocolVersion() == 1); // Write last vote - auto cbStatus = _replExecutor.scheduleDBWork([this](const CallbackArgs& cbData) { + auto evhStatus = _replExecutor.makeEvent(); + if (evhStatus.getStatus() == ErrorCodes::ShutdownInProgress) { + return {}; + } + invariant(evhStatus.isOK()); + auto evh = evhStatus.getValue(); + + auto cbStatus = _replExecutor.scheduleDBWork([this, evh](const CallbackArgs& cbData) { if (cbData.status == ErrorCodes::CallbackCanceled) { return; } invariant(cbData.txn); - LastVote lastVote; - lastVote.setTerm(OpTime::kInitialTerm); - lastVote.setCandidateIndex(-1); + LastVote lastVote{OpTime::kInitialTerm, -1}; auto status = _externalState->storeLocalLastVoteDocument(cbData.txn, lastVote); invariant(status.isOK()); + _replExecutor.signalEvent(evh); }); if (cbStatus.getStatus() == ErrorCodes::ShutdownInProgress) { - return; + return {}; } - invariant(cbStatus.isOK()); - _replExecutor.wait(cbStatus.getValue()); + return evh; } CallbackHandle ReplicationCoordinatorImpl::_scheduleWork(const CallbackFn& work) { diff --git a/src/mongo/db/repl/replication_coordinator_impl.h b/src/mongo/db/repl/replication_coordinator_impl.h index 33b93a70027..5f22864311a 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.h +++ b/src/mongo/db/repl/replication_coordinator_impl.h @@ -217,7 +217,8 @@ public: virtual void processReplSetGetConfig(BSONObjBuilder* result) override; - virtual void processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata) override; + virtual void processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata, + bool advanceCommitPoint) override; virtual void cancelAndRescheduleElectionTimeout() override; @@ -290,9 +291,8 @@ public: virtual Status processReplSetDeclareElectionWinner(const ReplSetDeclareElectionWinnerArgs& args, long long* responseTerm) override; - void prepareReplResponseMetadata(const rpc::RequestInterface&, - const OpTime& lastOpTimeFromClient, - BSONObjBuilder* builder) override; + virtual void prepareReplMetadata(const OpTime& lastOpTimeFromClient, + BSONObjBuilder* builder) const override; virtual Status processHeartbeatV1(const ReplSetHeartbeatArgsV1& args, ReplSetHeartbeatResponse* response) override; @@ -315,9 +315,11 @@ public: virtual void forceSnapshotCreation() override; - virtual void onSnapshotCreate(OpTime timeOfSnapshot, SnapshotName name) override; + virtual void createSnapshot(OperationContext* txn, + OpTime timeOfSnapshot, + SnapshotName name) override; - virtual OpTime getCurrentCommittedSnapshotOpTime() override; + virtual OpTime getCurrentCommittedSnapshotOpTime() const override; virtual void waitUntilSnapshotCommitted(OperationContext* txn, const SnapshotName& untilSnapshot) override; @@ -663,11 +665,11 @@ private: Status* result); /** - * Bottom half of prepareReplResponseMetadata. + * Bottom half of prepareReplMetadata. */ void _prepareReplResponseMetadata_finish(const ReplicationExecutor::CallbackArgs& cbData, const OpTime& lastOpTimeFromClient, - rpc::ReplSetMetadata* metadata); + rpc::ReplSetMetadata* metadata) const; /** * Scheduled to cause the ReplicationCoordinator to reconsider any state that might * need to change as a result of time passing - for instance becoming PRIMARY when a single @@ -891,7 +893,8 @@ private: */ void _finishReplSetReconfig(const ReplicationExecutor::CallbackArgs& cbData, const ReplicaSetConfig& newConfig, - int myIndex); + int myIndex, + ReplicationExecutor::EventHandle finishedEvent); /** * Changes _rsConfigState to newState, and notify any waiters. @@ -1025,7 +1028,10 @@ private: */ void _requestRemotePrimaryStepdown(const HostAndPort& target); - ReplicationExecutor::EventHandle _stepDownStart(); + /** + * Schedules stepdown to run with the global exclusive lock. + */ + ReplicationExecutor::EventHandle _stepDownStart(bool hasMutex); /** * Completes a step-down of the current node. Must be run with a global @@ -1064,9 +1070,11 @@ private: * Utility method that schedules or performs actions specified by a HeartbeatResponseAction * returned by a TopologyCoordinator::processHeartbeatResponse(V1) call with the given * value of "responseStatus". + * 'hasMutex' is true if the caller is holding _mutex. TODO(SERVER-27083): Remove this. */ void _handleHeartbeatResponseAction(const HeartbeatResponseAction& action, - const StatusWith& responseStatus); + const StatusWith& responseStatus, + bool hasMutex); /** * Bottom half of processHeartbeat(), which runs in the replication executor. @@ -1116,11 +1124,13 @@ private: /** * Callback that processes the ReplSetMetadata returned from a command run against another - * replica set member and updates protocol version 1 information (most recent optime that is - * committed, member id of the current PRIMARY, the current config version and the current term) + * replica set member and so long as the config version in the metadata matches the replica set + * config version this node currently has, updates the current term and optionally updates + * this node's notion of the commit point. * Returns the finish event which is invalid if the process has already finished. */ - EventHandle _processReplSetMetadata_incallback(const rpc::ReplSetMetadata& replMetadata); + EventHandle _processReplSetMetadata_incallback(const rpc::ReplSetMetadata& replMetadata, + bool advanceCommitPoint); /** * Blesses a snapshot to be used for new committed reads. @@ -1176,11 +1186,11 @@ private: void _startElectSelfIfEligibleV1(bool isPriorityTakeover); /** - * Reset the term of last vote to 0 to prevent any node from voting for term 0. - * Blocking until last vote write finishes. Must be called without holding _mutex. + * Resets the term of last vote to 0 to prevent any node from voting for term 0. + * Returns the event handle that indicates when last vote write finishes. */ - void _resetElectionInfoOnProtocolVersionUpgrade(const ReplicaSetConfig& oldConfig, - const ReplicaSetConfig& newConfig); + EventHandle _resetElectionInfoOnProtocolVersionUpgrade(const ReplicaSetConfig& oldConfig, + const ReplicaSetConfig& newConfig); /** * Schedules work and returns handle to callback. @@ -1282,6 +1292,14 @@ private: // TODO: ideally this should only change on rollbacks NOT on mongod restarts also. int _rbid; // (M) + // Indicates that we've received a request to stepdown from PRIMARY (likely via a heartbeat) + // TODO(SERVER-27083): This bool is redundant of the same-named bool in TopologyCoordinatorImpl, + // but due to mutex ordering between _mutex and _topoMutex we can't inspect the + // TopologyCoordinator field in awaitReplication() where this bool is used. Once we get rid + // of topoMutex and start guarding access to the TopologyCoordinator via _mutex we should + // consolidate the two bools. + bool _stepDownPending = false; // (M) + // list of information about clients waiting on replication. Does *not* own the WaiterInfos. std::vector _replicationWaiterList; // (M) diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect.cpp index a2cd0eb386e..b8b413c7d79 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_elect.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_elect.cpp @@ -202,7 +202,11 @@ void ReplicationCoordinatorImpl::_onFreshnessCheckComplete() { return; } - log() << "running for election"; + log() << "running for election" + << (abortReason == FreshnessChecker::FreshnessTie + ? "; slept last election, so running regardless of possible tie" + : ""); + // Secure our vote for ourself first if (!_topCoord->voteForMyself(now)) { return; diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect_test.cpp index 6dac4852d0f..1a3838b7c11 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_elect_test.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_elect_test.cpp @@ -515,6 +515,7 @@ TEST_F(ReplCoordElectTest, StepsDownRemoteIfNodeHasHigherPriorityThanCurrentPrim auto&& request = noi->getRequest(); log() << request.target << " processing " << request.cmdObj; ASSERT_EQUALS("replSetStepDown", request.cmdObj.firstElement().fieldNameStringData()); + ASSERT_EQUALS(1LL, request.cmdObj["secondaryCatchUpPeriodSecs"].safeNumberLong()); auto target = request.target; ASSERT_EQUALS(HostAndPort("node2", 12345), target); auto response = makeResponseStatus(BSON("ok" << 1)); diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp index 97b545a1292..b52c5a124e9 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp @@ -188,9 +188,7 @@ void ReplicationCoordinatorImpl::_onDryRunComplete(long long originalTerm) { _topCoord->voteForMyselfV1(); // Store the vote in persistent storage. - LastVote lastVote; - lastVote.setTerm(originalTerm + 1); - lastVote.setCandidateIndex(_selfIndex); + LastVote lastVote{originalTerm + 1, _selfIndex}; auto cbStatus = _replExecutor.scheduleDBWork( [this, lastVote](const ReplicationExecutor::CallbackArgs& cbData) { diff --git a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp index 4c23bab4f5b..aa3548335c1 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp @@ -64,6 +64,8 @@ typedef ReplicationExecutor::CallbackHandle CBHandle; using executor::RemoteCommandRequest; +MONGO_FP_DECLARE(blockHeartbeatStepdown); + void ReplicationCoordinatorImpl::_doMemberHeartbeat(ReplicationExecutor::CallbackArgs cbData, const HostAndPort& target, int targetIndex) { @@ -152,7 +154,9 @@ void ReplicationCoordinatorImpl::_handleHeartbeatResponse( if (replMetadata.isOK()) { // Asynchronous stepdown could happen, but it will be queued in executor after // this function, so we cannot and don't need to wait for it to finish. - _processReplSetMetadata_incallback(replMetadata.getValue()); + // Arbiters are the only nodes allowed to advance their commit point via heartbeats. + bool advanceCommitPoint = getMemberState().arbiter(); + _processReplSetMetadata_incallback(replMetadata.getValue(), advanceCommitPoint); } } const Date_t now = _replExecutor.now(); @@ -164,10 +168,11 @@ void ReplicationCoordinatorImpl::_handleHeartbeatResponse( networkTime = cbData.response.getValue().elapsedMillis; // TODO(sz) Because the term is duplicated in ReplSetMetaData, we can get rid of this // and update tests. - _updateTerm_incallback(hbStatusResponse.getValue().getTerm()); - // Postpone election timeout if we have a successful heartbeat response from the primary. const auto& hbResponse = hbStatusResponse.getValue(); - if (hbResponse.hasState() && hbResponse.getState().primary()) { + _updateTerm_incallback(hbResponse.getTerm()); + // Postpone election timeout if we have a successful heartbeat response from the primary. + if (hbResponse.hasState() && hbResponse.getState().primary() && + hbResponse.getTerm() == _topCoord->getTerm()) { cancelAndRescheduleElectionTimeout(); } } else { @@ -206,7 +211,7 @@ void ReplicationCoordinatorImpl::_handleHeartbeatResponse( _scheduleHeartbeatToTarget( target, targetIndex, std::max(now, action.getNextHeartbeatStartDate())); - _handleHeartbeatResponseAction(action, hbStatusResponse); + _handleHeartbeatResponseAction(action, hbStatusResponse, false /*we're not holding _mutex*/); } void ReplicationCoordinatorImpl::_updateOpTimesFromHeartbeat_inlock(int targetIndex, @@ -226,11 +231,13 @@ void ReplicationCoordinatorImpl::_updateOpTimesFromHeartbeat_inlock(int targetIn void ReplicationCoordinatorImpl::_handleHeartbeatResponseAction( const HeartbeatResponseAction& action, - const StatusWith& responseStatus) { + const StatusWith& responseStatus, + bool hasMutex) { switch (action.getAction()) { case HeartbeatResponseAction::NoAction: // Update the cached member state if different than the current topology member state if (_memberState != _topCoord->getMemberState()) { + invariant(!hasMutex); stdx::unique_lock lk(_mutex); const PostMemberStateUpdateAction postUpdateAction = _updateMemberStateFromTopologyCoordinator_inlock(); @@ -250,7 +257,7 @@ void ReplicationCoordinatorImpl::_handleHeartbeatResponseAction( log() << "Stepping down from primary in response to heartbeat"; _topCoord->prepareForStepDown(); // Don't need to wait for stepdown to finish. - _stepDownStart(); + _stepDownStart(hasMutex); break; case HeartbeatResponseAction::StepDownRemotePrimary: { invariant(action.getPrimaryConfigIndex() != _selfIndex); @@ -295,7 +302,13 @@ void remoteStepdownCallback(const ReplicationExecutor::RemoteCommandCallbackArgs } // namespace void ReplicationCoordinatorImpl::_requestRemotePrimaryStepdown(const HostAndPort& target) { - RemoteCommandRequest request(target, "admin", BSON("replSetStepDown" << 1)); + auto secondaryCatchUpPeriod(duration_cast(_rsConfig.getHeartbeatInterval() / 2)); + RemoteCommandRequest request( + target, + "admin", + BSON("replSetStepDown" << 20 << "secondaryCatchUpPeriodSecs" + << std::min(static_cast(secondaryCatchUpPeriod.count()), + 20LL))); log() << "Requesting " << target << " step down from primary"; CBHStatus cbh = _replExecutor.scheduleRemoteCommand(request, remoteStepdownCallback); @@ -304,11 +317,19 @@ void ReplicationCoordinatorImpl::_requestRemotePrimaryStepdown(const HostAndPort } } -ReplicationExecutor::EventHandle ReplicationCoordinatorImpl::_stepDownStart() { +ReplicationExecutor::EventHandle ReplicationCoordinatorImpl::_stepDownStart(bool hasMutex) { + { + boost::optional> lk; + if (!hasMutex) { + lk.emplace(_mutex); + } + _stepDownPending = true; + } auto finishEvent = _makeEvent(); if (!finishEvent) { return finishEvent; } + _replExecutor.scheduleWorkWithGlobalExclusiveLock(stdx::bind( &ReplicationCoordinatorImpl::_stepDownFinish, this, stdx::placeholders::_1, finishEvent)); return finishEvent; @@ -321,13 +342,28 @@ void ReplicationCoordinatorImpl::_stepDownFinish( return; } invariant(cbData.txn); + + if (MONGO_FAIL_POINT(blockHeartbeatStepdown)) { + // Must reschedule rather than block so we don't take up threads in the replication + // executor. + sleepmillis(10); + _replExecutor.scheduleWorkWithGlobalExclusiveLock( + stdx::bind(&ReplicationCoordinatorImpl::_stepDownFinish, + this, + stdx::placeholders::_1, + finishedEvent)); + + return; + } + // TODO Add invariant that we've got global shared or global exclusive lock, when supported // by lock manager. stdx::unique_lock lk(_mutex); - _topCoord->stepDownIfPending(); - const PostMemberStateUpdateAction action = _updateMemberStateFromTopologyCoordinator_inlock(); - lk.unlock(); - _performPostMemberStateUpdateAction(action); + if (_topCoord->stepDownIfPending()) { + const auto action = _updateMemberStateFromTopologyCoordinator_inlock(); + lk.unlock(); + _performPostMemberStateUpdateAction(action); + } _replExecutor.signalEvent(finishedEvent); } @@ -526,8 +562,15 @@ void ReplicationCoordinatorImpl::_heartbeatReconfigFinish( const PostMemberStateUpdateAction action = _setCurrentRSConfig_inlock(cbData, newConfig, myIndexValue); lk.unlock(); - _resetElectionInfoOnProtocolVersionUpgrade(oldConfig, newConfig); - _performPostMemberStateUpdateAction(action); + auto evh = _resetElectionInfoOnProtocolVersionUpgrade(oldConfig, newConfig); + if (evh) { + _replExecutor.onEvent(evh, + [this, action](const ReplicationExecutor::CallbackArgs& cbArgs) { + _performPostMemberStateUpdateAction(action); + }); + } else { + _performPostMemberStateUpdateAction(action); + } } void ReplicationCoordinatorImpl::_trackHeartbeatHandle(const StatusWith& handle) { @@ -623,7 +666,9 @@ void ReplicationCoordinatorImpl::_handleLivenessTimeout( _topCoord->setMemberAsDown(now, memberIndex, _getMyLastDurableOpTime_inlock()); // Don't mind potential asynchronous stepdown as this is the last step of // liveness check. - _handleHeartbeatResponseAction(action, makeStatusWith()); + _handleHeartbeatResponseAction(action, + makeStatusWith(), + true /*we're holding _mutex*/); } } } diff --git a/src/mongo/db/repl/replication_coordinator_impl_heartbeat_v1_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_heartbeat_v1_test.cpp index 2233e21cc21..40a30c48e5a 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_heartbeat_v1_test.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_heartbeat_v1_test.cpp @@ -351,7 +351,7 @@ TEST_F(ReplCoordHBV1Test, ArbiterRecordsCommittedOpTimeFromHeartbeatMetadata) { << 1 << "primaryIndex" << 1 << "term" << committedOpTime.getTerm() << "syncSourceIndex" << 1))); ASSERT_OK(metadata.getStatus()); - getReplCoord()->processReplSetMetadata(metadata.getValue()); + getReplCoord()->processReplSetMetadata(metadata.getValue(), true); ASSERT_EQ(getReplCoord()->getMyLastAppliedOpTime().getTimestamp(), expected.getTimestamp()); }; diff --git a/src/mongo/db/repl/replication_coordinator_impl_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_test.cpp index f730e4deccd..a9b1d297be0 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_test.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_test.cpp @@ -44,8 +44,10 @@ #include "mongo/db/repl/optime.h" #include "mongo/db/repl/read_concern_args.h" #include "mongo/db/repl/read_concern_response.h" +#include "mongo/db/repl/repl_client_info.h" #include "mongo/db/repl/repl_set_heartbeat_args.h" #include "mongo/db/repl/repl_set_heartbeat_args_v1.h" +#include "mongo/db/repl/repl_set_request_votes_args.h" #include "mongo/db/repl/repl_settings.h" #include "mongo/db/repl/replica_set_config.h" #include "mongo/db/repl/replication_coordinator.h" // ReplSetReconfigArgs @@ -80,15 +82,15 @@ using executor::RemoteCommandResponse; typedef ReplicationCoordinator::ReplSetReconfigArgs ReplSetReconfigArgs; Status kInterruptedStatus(ErrorCodes::Interrupted, "operation was interrupted"); -// Helper class to wrap Timestamp as an OpTime with term 0. -struct OpTimeWithTermZero { - OpTimeWithTermZero(unsigned int sec, unsigned int i) : timestamp(sec, i) {} +// Helper class to wrap Timestamp as an OpTime with term 1. +struct OpTimeWithTermOne { + OpTimeWithTermOne(unsigned int sec, unsigned int i) : timestamp(sec, i) {} operator OpTime() const { - return OpTime(timestamp, 0); + return OpTime(timestamp, 1); } operator boost::optional() const { - return OpTime(timestamp, 0); + return OpTime(timestamp, 1); } OpTime asOpTime() const { @@ -673,7 +675,7 @@ TEST_F(ReplCoordTest, RollBackIDShouldIncreaseByOneWhenIncrementRollbackIDIsCall TEST_F(ReplCoordTest, NodeReturnsImmediatelyWhenAwaitReplicationIsRanAgainstAStandaloneNode) { init(""); OperationContextNoop txn; - OpTimeWithTermZero time(100, 1); + OpTimeWithTermOne time(100, 1); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoWaiting; @@ -691,7 +693,7 @@ TEST_F(ReplCoordTest, NodeReturnsImmediatelyWhenAwaitReplicationIsRanAgainstAMas settings.setMaster(true); init(settings); OperationContextNoop txn; - OpTimeWithTermZero time(100, 1); + OpTimeWithTermOne time(100, 1); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoWaiting; @@ -718,7 +720,7 @@ TEST_F(ReplCoordTest, NodeReturnsNotMasterWhenRunningAwaitReplicationAgainstASec HostAndPort("node1", 12345)); OperationContextNoop txn; - OpTimeWithTermZero time(100, 1); + OpTimeWithTermOne time(100, 1); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoWaiting; @@ -731,7 +733,7 @@ TEST_F(ReplCoordTest, NodeReturnsNotMasterWhenRunningAwaitReplicationAgainstASec ASSERT_EQUALS(ErrorCodes::NotMaster, statusAndDur.status); } -TEST_F(ReplCoordTest, NodeReturnsOkWhenRunningAwaitReplicationAgainstPrimaryWithWZero) { +TEST_F(ReplCoordTest, NodeReturnsOkWhenRunningAwaitReplicationAgainstPrimaryWithWTermOne) { assertStartSuccess(BSON("_id" << "mySet" << "version" << 2 << "members" @@ -746,7 +748,7 @@ TEST_F(ReplCoordTest, NodeReturnsOkWhenRunningAwaitReplicationAgainstPrimaryWith HostAndPort("node1", 12345)); OperationContextNoop txn; - OpTimeWithTermZero time(100, 1); + OpTimeWithTermOne time(100, 1); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoWaiting; @@ -755,8 +757,8 @@ TEST_F(ReplCoordTest, NodeReturnsOkWhenRunningAwaitReplicationAgainstPrimaryWith // Become primary. ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); ASSERT(getReplCoord()->getMemberState().primary()); @@ -786,12 +788,12 @@ TEST_F(ReplCoordTest, << "_id" << 3))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoWaiting; @@ -859,12 +861,12 @@ TEST_F(ReplCoordTest, NodeReturnsWriteConcernFailedUntilASufficientNumberOfNodes << "_id" << 3))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoWaiting; @@ -1028,7 +1030,7 @@ TEST_F( getReplCoord()->setLastDurableOptime_forTest(2, 1, time1); getReplCoord()->setLastAppliedOptime_forTest(2, 2, time1); getReplCoord()->setLastDurableOptime_forTest(2, 2, time1); - getReplCoord()->onSnapshotCreate(time1, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time1, SnapshotName(1)); statusAndDur = getReplCoord()->awaitReplication(&txn, time1, majorityWriteConcern); ASSERT_OK(statusAndDur.status); @@ -1055,6 +1057,8 @@ TEST_F( // another name if we didn't get a high enough one. } + auto zeroOpTimeInCurrentTerm = OpTime(Timestamp(0, 0), 1); + ReplClientInfo::forClient(txn.getClient()).setLastOp(zeroOpTimeInCurrentTerm); statusAndDur = getReplCoord()->awaitReplicationOfLastOpForClient(&txn, majorityWriteConcern); ASSERT_EQUALS(ErrorCodes::WriteConcernFailed, statusAndDur.status); statusAndDur = getReplCoord()->awaitReplicationOfLastOpForClient(&txn, multiDCWriteConcern); @@ -1063,7 +1067,7 @@ TEST_F( ASSERT_OK(statusAndDur.status); // All modes satisfied - getReplCoord()->onSnapshotCreate(time1, getReplCoord()->reserveSnapshotName(nullptr)); + getReplCoord()->createSnapshot(&txn, time1, getReplCoord()->reserveSnapshotName(nullptr)); statusAndDur = getReplCoord()->awaitReplicationOfLastOpForClient(&txn, majorityWriteConcern); ASSERT_OK(statusAndDur.status); @@ -1157,14 +1161,14 @@ TEST_F(ReplCoordTest, NodeReturnsOkWhenAWriteConcernWithNoTimeoutHasBeenSatisfie << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); ReplicationAwaiter awaiter(getReplCoord(), &txn); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoTimeout; @@ -1216,14 +1220,14 @@ TEST_F(ReplCoordTest, NodeReturnsWriteConcernFailedWhenAWriteConcernTimesOutBefo << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); ReplicationAwaiter awaiter(getReplCoord(), &txn); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); WriteConcernOptions writeConcern; writeConcern.wTimeout = 50; @@ -1257,14 +1261,14 @@ TEST_F(ReplCoordTest, << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); ReplicationAwaiter awaiter(getReplCoord(), &txn); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoTimeout; @@ -1299,14 +1303,14 @@ TEST_F(ReplCoordTest, NodeReturnsNotMasterWhenSteppingDownBeforeSatisfyingAWrite << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); ReplicationAwaiter awaiter(getReplCoord(), &txn); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoTimeout; @@ -1339,14 +1343,14 @@ TEST_F(ReplCoordTest, << "node3"))), HostAndPort("node1")); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); ReplicationAwaiter awaiter(getReplCoord(), &txn); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); WriteConcernOptions writeConcern; writeConcern.wTimeout = WriteConcernOptions::kNoTimeout; @@ -1377,7 +1381,6 @@ private: virtual void setUp() { ReplCoordTest::setUp(); init("mySet/test1:1234,test2:1234,test3:1234"); - assertStartSuccess(BSON("_id" << "mySet" << "version" << 1 << "members" @@ -1393,6 +1396,7 @@ private: } }; + TEST_F(ReplCoordTest, NodeReturnsBadValueWhenUpdateTermIsRunAgainstANonReplNode) { init(ReplSettings()); ASSERT_TRUE(ReplicationCoordinator::modeNone == getReplCoord()->getReplicationMode()); @@ -1522,7 +1526,7 @@ TEST_F(ReplCoordTest, ConcurrentStepDownShouldNotSignalTheSameFinishEventMoreTha TEST_F(StepDownTest, NodeReturnsNotMasterWhenAskedToStepDownAsANonPrimaryNode) { OperationContextReplMock txn; - OpTimeWithTermZero optime1(100, 1); + OpTimeWithTermOne optime1(100, 1); // All nodes are caught up getReplCoord()->setMyLastAppliedOpTime(optime1); getReplCoord()->setMyLastDurableOpTime(optime1); @@ -1537,7 +1541,7 @@ TEST_F(StepDownTest, NodeReturnsNotMasterWhenAskedToStepDownAsANonPrimaryNode) { TEST_F(StepDownTest, NodeReturnsExceededTimeLimitWhenStepDownFailsToObtainTheGlobalLockWithinTheAllottedTime) { OperationContextReplMock txn; - OpTimeWithTermZero optime1(100, 1); + OpTimeWithTermOne optime1(100, 1); // All nodes are caught up getReplCoord()->setMyLastAppliedOpTime(optime1); getReplCoord()->setMyLastDurableOpTime(optime1); @@ -1554,43 +1558,170 @@ TEST_F(StepDownTest, ASSERT_TRUE(getReplCoord()->getMemberState().primary()); } -TEST_F(StepDownTest, - NodeTransitionsToSecondaryImmediatelyWhenStepDownIsRunAndAnUpToDateElectableNodeExists) { +/* Step Down Test for a 5-node replica set */ +class StepDownTestFiveNode : public StepDownTest { +protected: + /* + * Simulate a round of heartbeat requests from the primary by manually setting + * the heartbeat response messages from each node. 'numNodesCaughtUp' will + * determine how many nodes return an optime that is up to date with the + * primary's optime. Sets electability of all caught up nodes to 'caughtUpAreElectable' + */ + void simulateHeartbeatResponses(OpTime optimePrimary, + OpTime optimeLagged, + int numNodesCaughtUp, + bool caughtUpAreElectable) { + int hbNum = 1; + while (getNet()->hasReadyRequests()) { + NetworkInterfaceMock::NetworkOperationIterator noi = getNet()->getNextReadyRequest(); + RemoteCommandRequest request = noi->getRequest(); + + // Only process heartbeat requests. + ASSERT_EQ(request.cmdObj.firstElement().fieldNameStringData().toString(), + "replSetHeartbeat"); + + ReplSetHeartbeatArgsV1 hbArgs; + ASSERT_OK(hbArgs.initialize(request.cmdObj)); + + log() << request.target.toString() << " processing " << request.cmdObj; + + // Catch up 'numNodesCaughtUp' nodes out of 5. + OpTime optimeResponse = (hbNum <= numNodesCaughtUp) ? optimePrimary : optimeLagged; + bool isElectable = (hbNum <= numNodesCaughtUp) ? caughtUpAreElectable : true; + + ReplSetHeartbeatResponse hbResp; + hbResp.setSetName(hbArgs.getSetName()); + hbResp.setState(MemberState::RS_SECONDARY); + hbResp.setConfigVersion(hbArgs.getConfigVersion()); + hbResp.setDurableOpTime(optimeResponse); + hbResp.setAppliedOpTime(optimeResponse); + hbResp.setElectable(isElectable); + BSONObjBuilder respObj; + respObj << "ok" << 1; + hbResp.addToBSON(&respObj, false); + getNet()->scheduleResponse(noi, getNet()->now(), makeResponseStatus(respObj.obj())); + hbNum += 1; + } + } + +private: + virtual void setUp() { + ReplCoordTest::setUp(); + init("mySet/test1:1234,test2:1234,test3:1234,test4:1234,test5:1234"); + + assertStartSuccess(BSON("_id" + << "mySet" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 0 << "host" + << "test1:1234") + << BSON("_id" << 1 << "host" + << "test2:1234") + << BSON("_id" << 2 << "host" + << "test3:1234") + << BSON("_id" << 3 << "host" + << "test4:1234") + << BSON("_id" << 4 << "host" + << "test5:1234"))), + HostAndPort("test1", 1234)); + ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); + myRid = getReplCoord()->getMyRID(); + } +}; + +TEST_F( + StepDownTestFiveNode, + NodeReturnsExceededTimeLimitWhenStepDownIsRunAndCaughtUpMajorityExistsButWithoutElectableNode) { OperationContextReplMock txn; - OpTimeWithTermZero optime1(100, 1); + OpTime optimeLagged(Timestamp(100, 1), 1); + OpTime optimePrimary(Timestamp(100, 2), 1); + // All nodes are caught up - getReplCoord()->setMyLastAppliedOpTime(optime1); - getReplCoord()->setMyLastDurableOpTime(optime1); - ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 1, optime1)); - ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 2, optime1)); + getReplCoord()->setMyLastAppliedOpTime(optimePrimary); + getReplCoord()->setMyLastDurableOpTime(optimePrimary); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 1, optimeLagged)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 2, optimeLagged)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 3, optimeLagged)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 4, optimeLagged)); simulateSuccessfulV1Election(); enterNetwork(); getNet()->runUntil(getNet()->now() + Seconds(2)); ASSERT(getNet()->hasReadyRequests()); - NetworkInterfaceMock::NetworkOperationIterator noi = getNet()->getNextReadyRequest(); - RemoteCommandRequest request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; - ReplSetHeartbeatArgsV1 hbArgs; - if (hbArgs.initialize(request.cmdObj).isOK()) { - ReplSetHeartbeatResponse hbResp; - hbResp.setSetName(hbArgs.getSetName()); - hbResp.setState(MemberState::RS_SECONDARY); - hbResp.setConfigVersion(hbArgs.getConfigVersion()); - hbResp.setDurableOpTime(optime1); - hbResp.setAppliedOpTime(optime1); - BSONObjBuilder respObj; - respObj << "ok" << 1; - hbResp.addToBSON(&respObj, false); - getNet()->scheduleResponse(noi, getNet()->now(), makeResponseStatus(respObj.obj())); - } - while (getNet()->hasReadyRequests()) { - getNet()->blackHole(getNet()->getNextReadyRequest()); - } + + // Make sure a majority are caught up (i.e. 3 out of 5). We catch up two secondaries since + // the primary counts as one towards majority + int numNodesCaughtUp = 2; + simulateHeartbeatResponses(optimePrimary, optimeLagged, numNodesCaughtUp, false); + getNet()->runReadyNetworkOperations(); + exitNetwork(); + + ASSERT_TRUE(getReplCoord()->getMemberState().primary()); + auto status = getReplCoord()->stepDown(&txn, false, Milliseconds(0), Milliseconds(1000)); + ASSERT_EQUALS(ErrorCodes::ExceededTimeLimit, status); + ASSERT_TRUE(getReplCoord()->getMemberState().primary()); +} + +TEST_F(StepDownTestFiveNode, + NodeReturnsExceededTimeLimitWhenStepDownIsRunAndNoCaughtUpMajorityExists) { + OperationContextReplMock txn; + OpTime optimeLagged(Timestamp(100, 1), 1); + OpTime optimePrimary(Timestamp(100, 2), 1); + + // All nodes are caught up + getReplCoord()->setMyLastAppliedOpTime(optimePrimary); + getReplCoord()->setMyLastDurableOpTime(optimePrimary); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 1, optimeLagged)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 2, optimeLagged)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 3, optimeLagged)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 4, optimeLagged)); + + simulateSuccessfulV1Election(); + + enterNetwork(); + getNet()->runUntil(getNet()->now() + Seconds(2)); + ASSERT(getNet()->hasReadyRequests()); + + // Make sure less than a majority are caught up (i.e. 2 out of 5) We catch up one secondary + // since the primary counts as one towards majority + int numNodesCaughtUp = 1; + simulateHeartbeatResponses(optimePrimary, optimeLagged, numNodesCaughtUp, true); getNet()->runReadyNetworkOperations(); exitNetwork(); + ASSERT_TRUE(getReplCoord()->getMemberState().primary()); + auto status = getReplCoord()->stepDown(&txn, false, Milliseconds(0), Milliseconds(1000)); + ASSERT_EQUALS(ErrorCodes::ExceededTimeLimit, status); + ASSERT_TRUE(getReplCoord()->getMemberState().primary()); +} + +TEST_F( + StepDownTestFiveNode, + NodeTransitionsToSecondaryImmediatelyWhenStepDownIsRunAndAnUpToDateMajorityWithElectableNodeExists) { + OperationContextReplMock txn; + OpTime optimeLagged(Timestamp(100, 1), 1); + OpTime optimePrimary(Timestamp(100, 2), 1); + + // All nodes are caught up + getReplCoord()->setMyLastAppliedOpTime(optimePrimary); + getReplCoord()->setMyLastDurableOpTime(optimePrimary); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 1, optimeLagged)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 2, optimeLagged)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 3, optimeLagged)); + ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 4, optimeLagged)); + + simulateSuccessfulV1Election(); + + enterNetwork(); + getNet()->runUntil(getNet()->now() + Seconds(2)); + ASSERT(getNet()->hasReadyRequests()); + + // Make sure a majority are caught up (i.e. 3 out of 5). We catch up two secondaries since + // the primary counts as one towards majority + int numNodesCaughtUp = 2; + simulateHeartbeatResponses(optimePrimary, optimeLagged, numNodesCaughtUp, true); + getNet()->runReadyNetworkOperations(); + exitNetwork(); ASSERT_TRUE(getReplCoord()->getMemberState().primary()); ASSERT_OK(getReplCoord()->stepDown(&txn, false, Milliseconds(0), Milliseconds(1000))); @@ -1631,8 +1762,8 @@ TEST_F(ReplCoordTest, NodeBecomesPrimaryAgainWhenStepDownTimeoutExpiresInASingle TEST_F(StepDownTest, NodeReturnsExceededTimeLimitWhenNoSecondaryIsCaughtUpWithinStepDownsSecondaryCatchUpPeriod) { OperationContextReplMock txn; - OpTimeWithTermZero optime1(100, 1); - OpTimeWithTermZero optime2(100, 2); + OpTimeWithTermOne optime1(100, 1); + OpTimeWithTermOne optime2(100, 2); // No secondary is caught up auto repl = getReplCoord(); repl->setMyLastAppliedOpTime(optime2); @@ -1666,8 +1797,9 @@ TEST_F(StepDownTest, TEST_F(StepDownTest, NodeTransitionsToSecondaryWhenASecondaryCatchesUpAfterTheFirstRoundOfHeartbeats) { OperationContextReplMock txn; - OpTimeWithTermZero optime1(100, 1); - OpTimeWithTermZero optime2(100, 2); + OpTime optime1(Timestamp(100, 1), 1); + OpTime optime2(Timestamp(100, 2), 1); + // No secondary is caught up auto repl = getReplCoord(); repl->setMyLastAppliedOpTime(optime2); @@ -1677,6 +1809,8 @@ TEST_F(StepDownTest, simulateSuccessfulV1Election(); + ASSERT_TRUE(getReplCoord()->getMemberState().primary()); + // Step down where the secondary actually has to catch up before the stepDown can succeed. // On entering the network, _stepDownContinue should cancel the heartbeats scheduled for // T + 2 seconds and send out a new round of heartbeats immediately. @@ -1713,6 +1847,7 @@ TEST_F(StepDownTest, log() << "Blackholing network request " << noi->getRequest().cmdObj; getNet()->blackHole(noi); } + getNet()->runReadyNetworkOperations(); exitNetwork(); @@ -1724,8 +1859,9 @@ TEST_F(StepDownTest, TEST_F(StepDownTest, NodeTransitionsToSecondaryWhenASecondaryCatchesUpDuringStepDownsSecondaryCatchupPeriod) { OperationContextReplMock txn; - OpTimeWithTermZero optime1(100, 1); - OpTimeWithTermZero optime2(100, 2); + OpTime optime1(Timestamp(100, 1), 1); + OpTime optime2(Timestamp(100, 2), 1); + // No secondary is caught up auto repl = getReplCoord(); repl->setMyLastAppliedOpTime(optime2); @@ -1808,8 +1944,8 @@ TEST_F(StepDownTest, TEST_F(StepDownTest, NodeReturnsInterruptedWhenInterruptedDuringStepDown) { const unsigned int opID = 100; OperationContextReplMock txn{opID}; - OpTimeWithTermZero optime1(100, 1); - OpTimeWithTermZero optime2(100, 2); + OpTimeWithTermOne optime1(100, 1); + OpTimeWithTermOne optime2(100, 2); // No secondary is caught up auto repl = getReplCoord(); repl->setMyLastAppliedOpTime(optime2); @@ -1906,7 +2042,7 @@ TEST_F(ReplCoordTest, NodeIncludesOtherMembersProgressInUpdatePositionCommand) { getReplCoord()->prepareReplSetUpdatePositionCommand(&cmdBuilder); BSONObj cmd = cmdBuilder.done(); - ASSERT_EQUALS(2, cmd.nFields()); + ASSERT_EQUALS(3, cmd.nFields()); ASSERT_EQUALS("replSetUpdatePosition", cmd.firstElement().fieldNameStringData()); std::set memberIds; @@ -1959,9 +2095,9 @@ TEST_F(ReplCoordTest, NodeIncludesOtherMembersProgressInOldUpdatePositionCommand << "test2:1234") << BSON("_id" << 2 << "host" << "test3:1234"))), HostAndPort("test1", 1234)); - OpTimeWithTermZero optime1(100, 1); - OpTimeWithTermZero optime2(100, 2); - OpTimeWithTermZero optime3(2, 1); + OpTimeWithTermOne optime1(100, 1); + OpTimeWithTermOne optime2(100, 2); + OpTimeWithTermOne optime3(2, 1); getReplCoord()->setMyLastAppliedOpTime(optime1); getReplCoord()->setMyLastDurableOpTime(optime1); ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(1, 1, optime2)); @@ -1991,7 +2127,7 @@ TEST_F(ReplCoordTest, NodeIncludesOtherMembersProgressInOldUpdatePositionCommand ASSERT_EQUALS(2, memberId); ASSERT_EQUALS(optime3.timestamp, entry["optime"]["ts"].timestamp()); } - ASSERT_EQUALS(0, entry["optime"]["t"].Number()); + ASSERT_EQUALS(1, entry["optime"]["t"].Number()); } ASSERT_EQUALS(3U, memberIds.size()); // Make sure we saw all 3 nodes } @@ -2011,8 +2147,8 @@ TEST_F(ReplCoordTest, HostAndPort("test2", 1234)); OperationContextNoop txn; getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); // Can't unset maintenance mode if it was never set to begin with. Status status = getReplCoord()->setMaintenanceMode(false); @@ -2035,8 +2171,8 @@ TEST_F(ReplCoordTest, HostAndPort("test2", 1234)); OperationContextNoop txn; getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); // valid set ASSERT_OK(getReplCoord()->setMaintenanceMode(true)); ASSERT_TRUE(getReplCoord()->getMemberState().recovering()); @@ -2064,8 +2200,8 @@ TEST_F(ReplCoordTest, AllowAsManyUnsetMaintenanceModesAsThereHaveBeenSetMaintena HostAndPort("test2", 1234)); OperationContextNoop txn; getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); // Can set multiple times ASSERT_OK(getReplCoord()->setMaintenanceMode(true)); ASSERT_OK(getReplCoord()->setMaintenanceMode(true)); @@ -2095,8 +2231,8 @@ TEST_F(ReplCoordTest, SettingAndUnsettingMaintenanceModeShouldNotAffectRollbackS HostAndPort("test2", 1234)); OperationContextNoop txn; getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); // From rollback, entering and exiting maintenance mode doesn't change perceived // state. @@ -2134,8 +2270,8 @@ TEST_F(ReplCoordTest, DoNotAllowMaintenanceModeWhilePrimary) { HostAndPort("test2", 1234)); OperationContextNoop txn; getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); // Can't modify maintenance mode when PRIMARY simulateSuccessfulV1Election(); @@ -2167,8 +2303,8 @@ TEST_F(ReplCoordTest, DoNotAllowSettingMaintenanceModeWhileConductingAnElection) HostAndPort("test2", 1234)); OperationContextNoop txn; getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); // TODO this election shouldn't have to happen. simulateSuccessfulV1Election(); @@ -2227,8 +2363,8 @@ TEST_F(ReplCoordTest, HostAndPort("node1", 12345)); OperationContextNoop txn; - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); getReplCoord()->setMyLastAppliedOpTime(time2); getReplCoord()->setMyLastDurableOpTime(time2); @@ -2271,8 +2407,8 @@ TEST_F(ReplCoordTest, HostAndPort("node1", 12345)); OperationContextNoop txn; - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); getReplCoord()->setMyLastAppliedOpTime(time2); getReplCoord()->setMyLastDurableOpTime(time2); @@ -2301,8 +2437,8 @@ TEST_F(ReplCoordTest, NodeDoesNotIncludeItselfWhenRunningGetHostsWrittenToInMast OperationContextNoop txn; OID client = OID::gen(); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); getExternalState()->setClientHostAndPort(clientHost); HandshakeArgs handshake; @@ -2451,12 +2587,12 @@ TEST_F(ReplCoordTest, DoNotProcessSelfWhenUpdatePositionContainsInfoAboutSelf) { << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); - OpTime time1({100, 1}, 2); - OpTime time2({100, 2}, 2); + OpTime time1({100, 1}, 1); + OpTime time2({100, 2}, 1); getReplCoord()->setMyLastAppliedOpTime(time1); getReplCoord()->setMyLastDurableOpTime(time1); @@ -2497,13 +2633,13 @@ TEST_F(ReplCoordTest, DoNotProcessSelfWhenOldUpdatePositionContainsInfoAboutSelf << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); - OpTimeWithTermZero staleTime(10, 0); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); + OpTimeWithTermOne staleTime(10, 0); getReplCoord()->setMyLastAppliedOpTime(time1); getReplCoord()->setMyLastDurableOpTime(time1); @@ -2541,12 +2677,12 @@ TEST_F(ReplCoordTest, DoNotProcessUpdatePositionWhenItsConfigVersionIsIncorrect) << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); - OpTime time1({100, 1}, 3); - OpTime time2({100, 2}, 3); + OpTime time1({100, 1}, 1); + OpTime time2({100, 2}, 1); getReplCoord()->setMyLastAppliedOpTime(time1); getReplCoord()->setMyLastDurableOpTime(time1); @@ -2586,13 +2722,13 @@ TEST_F(ReplCoordTest, DoNotProcessOldUpdatePositionWhenItsConfigVersionIsIncorre << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); - OpTimeWithTermZero staleTime(10, 0); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); + OpTimeWithTermOne staleTime(10, 0); getReplCoord()->setMyLastAppliedOpTime(time1); getReplCoord()->setMyLastDurableOpTime(time1); @@ -2629,12 +2765,12 @@ TEST_F(ReplCoordTest, DoNotProcessUpdatePositionOfMembersWhoseIdsAreNotInTheConf << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); - OpTime time1({100, 1}, 2); - OpTime time2({100, 2}, 2); + OpTime time1({100, 1}, 1); + OpTime time2({100, 2}, 1); getReplCoord()->setMyLastAppliedOpTime(time1); getReplCoord()->setMyLastDurableOpTime(time1); @@ -2672,13 +2808,13 @@ TEST_F(ReplCoordTest, DoNotProcessOldUpdatePositionOfMembersWhoseIdsAreNotInTheC << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); - OpTimeWithTermZero staleTime(10, 0); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); + OpTimeWithTermOne staleTime(10, 0); getReplCoord()->setMyLastAppliedOpTime(time1); getReplCoord()->setMyLastDurableOpTime(time1); @@ -2714,13 +2850,13 @@ TEST_F(ReplCoordTest, << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); simulateSuccessfulV1Election(); - OpTimeWithTermZero time1(100, 1); - OpTimeWithTermZero time2(100, 2); - OpTimeWithTermZero staleTime(10, 0); + OpTimeWithTermOne time1(100, 1); + OpTimeWithTermOne time2(100, 2); + OpTimeWithTermOne staleTime(10, 0); getReplCoord()->setMyLastAppliedOpTime(time1); getReplCoord()->setMyLastDurableOpTime(time1); @@ -2779,11 +2915,11 @@ TEST_F(ReplCoordTest, AwaitReplicationShouldResolveAsNormalDuringAReconfig) { << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 2)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 2)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 2)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 2)); simulateSuccessfulV1Election(); - OpTimeWithTermZero time(100, 2); + OpTimeWithTermOne time(100, 2); // 3 nodes waiting for time WriteConcernOptions writeConcern; @@ -2861,11 +2997,11 @@ TEST_F( << "_id" << 2))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 2)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 2)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 2)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 2)); simulateSuccessfulV1Election(); - OpTimeWithTermZero time(100, 2); + OpTimeWithTermOne time(100, 2); // 3 nodes waiting for time WriteConcernOptions writeConcern; @@ -2922,15 +3058,15 @@ TEST_F(ReplCoordTest, << "_id" << 4))), HostAndPort("node1", 12345)); ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 1)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 1)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 1)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 1)); simulateSuccessfulV1Election(); OpTime time(Timestamp(100, 2), 1); getReplCoord()->setMyLastAppliedOpTime(time); getReplCoord()->setMyLastDurableOpTime(time); - getReplCoord()->onSnapshotCreate(time, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time, SnapshotName(1)); ASSERT_OK(getReplCoord()->setLastAppliedOptime_forTest(2, 1, time)); @@ -3021,7 +3157,7 @@ TEST_F(ReplCoordTest, ASSERT_EQUALS(ErrorCodes::WriteConcernFailed, getReplCoord()->awaitReplication(&txn, time, majorityWriteConcern).status); - getReplCoord()->onSnapshotCreate(time, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time, SnapshotName(1)); ASSERT_OK(getReplCoord()->awaitReplication(&txn, time, majorityWriteConcern).status); } @@ -3094,13 +3230,13 @@ TEST_F(ReplCoordTest, NodeReturnsShutdownInProgressWhenWaitingUntilAnOpTimeDurin << "_id" << 0))), HostAndPort("node1", 12345)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(10, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(10, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(10, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(10, 0)); shutdown(); auto result = getReplCoord()->waitUntilOpTime( - &txn, ReadConcernArgs(OpTimeWithTermZero(50, 0), ReadConcernLevel::kLocalReadConcern)); + &txn, ReadConcernArgs(OpTimeWithTermOne(50, 0), ReadConcernLevel::kLocalReadConcern)); ASSERT_TRUE(result.didWait()); ASSERT_EQUALS(ErrorCodes::ShutdownInProgress, result.getStatus()); @@ -3115,13 +3251,13 @@ TEST_F(ReplCoordTest, NodeReturnsInterruptedWhenWaitingUntilAnOpTimeIsInterrupte << "_id" << 0))), HostAndPort("node1", 12345)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(10, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(10, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(10, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(10, 0)); txn.setCheckForInterruptStatus(Status(ErrorCodes::Interrupted, "test")); auto result = getReplCoord()->waitUntilOpTime( - &txn, ReadConcernArgs(OpTimeWithTermZero(50, 0), ReadConcernLevel::kLocalReadConcern)); + &txn, ReadConcernArgs(OpTimeWithTermOne(50, 0), ReadConcernLevel::kLocalReadConcern)); ASSERT_TRUE(result.didWait()); ASSERT_EQUALS(ErrorCodes::Interrupted, result.getStatus()); @@ -3151,10 +3287,10 @@ TEST_F(ReplCoordTest, NodeReturnsOkImmediatelyWhenWaitingUntilOpTimePassesAnOpTi << "_id" << 0))), HostAndPort("node1", 12345)); - getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermZero(100, 0)); - getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermZero(100, 0)); + getReplCoord()->setMyLastAppliedOpTime(OpTimeWithTermOne(100, 0)); + getReplCoord()->setMyLastDurableOpTime(OpTimeWithTermOne(100, 0)); auto result = getReplCoord()->waitUntilOpTime( - &txn, ReadConcernArgs(OpTimeWithTermZero(50, 0), ReadConcernLevel::kLocalReadConcern)); + &txn, ReadConcernArgs(OpTimeWithTermOne(50, 0), ReadConcernLevel::kLocalReadConcern)); ASSERT_TRUE(result.didWait()); ASSERT_OK(result.getStatus()); @@ -3170,7 +3306,7 @@ TEST_F(ReplCoordTest, NodeReturnsOkImmediatelyWhenWaitingUntilOpTimePassesAnOpTi HostAndPort("node1", 12345)); - OpTimeWithTermZero time(100, 0); + OpTimeWithTermOne time(100, 0); getReplCoord()->setMyLastAppliedOpTime(time); getReplCoord()->setMyLastDurableOpTime(time); auto result = getReplCoord()->waitUntilOpTime( @@ -3185,7 +3321,7 @@ TEST_F(ReplCoordTest, init(ReplSettings()); OperationContextNoop txn; auto result = getReplCoord()->waitUntilOpTime( - &txn, ReadConcernArgs(OpTimeWithTermZero(50, 0), ReadConcernLevel::kLocalReadConcern)); + &txn, ReadConcernArgs(OpTimeWithTermOne(50, 0), ReadConcernLevel::kLocalReadConcern)); ASSERT_FALSE(result.didWait()); ASSERT_EQUALS(ErrorCodes::NotAReplicaSet, result.getStatus()); @@ -3261,7 +3397,7 @@ TEST_F(ReplCoordTest, ReadAfterCommittedGreaterOpTime) { getReplCoord()->setMyLastAppliedOpTime(OpTime(Timestamp(100, 0), 1)); getReplCoord()->setMyLastDurableOpTime(OpTime(Timestamp(100, 0), 1)); - getReplCoord()->onSnapshotCreate(OpTime(Timestamp(100, 0), 1), SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, OpTime(Timestamp(100, 0), 1), SnapshotName(1)); auto result = getReplCoord()->waitUntilOpTime( &txn, ReadConcernArgs(OpTime(Timestamp(50, 0), 1), ReadConcernLevel::kMajorityReadConcern)); @@ -3281,7 +3417,7 @@ TEST_F(ReplCoordTest, ReadAfterCommittedEqualOpTime) { OpTime time(Timestamp(100, 0), 1); getReplCoord()->setMyLastAppliedOpTime(time); getReplCoord()->setMyLastDurableOpTime(time); - getReplCoord()->onSnapshotCreate(time, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time, SnapshotName(1)); auto result = getReplCoord()->waitUntilOpTime( &txn, ReadConcernArgs(time, ReadConcernLevel::kMajorityReadConcern)); @@ -3307,7 +3443,7 @@ TEST_F(ReplCoordTest, ReadAfterCommittedDeferredGreaterOpTime) { // Not guaranteed to be scheduled after waitUntil blocks... getReplCoord()->setMyLastAppliedOpTime(committedOpTime); getReplCoord()->setMyLastDurableOpTime(committedOpTime); - getReplCoord()->onSnapshotCreate(committedOpTime, SnapshotName(1)); + getReplCoord()->createSnapshot(nullptr, committedOpTime, SnapshotName(1)); }); auto result = getReplCoord()->waitUntilOpTime( @@ -3339,7 +3475,7 @@ TEST_F(ReplCoordTest, ReadAfterCommittedDeferredEqualOpTime) { // Not guaranteed to be scheduled after waitUntil blocks... getReplCoord()->setMyLastAppliedOpTime(opTimeToWait); getReplCoord()->setMyLastDurableOpTime(opTimeToWait); - getReplCoord()->onSnapshotCreate(opTimeToWait, SnapshotName(1)); + getReplCoord()->createSnapshot(nullptr, opTimeToWait, SnapshotName(1)); }); auto result = getReplCoord()->waitUntilOpTime( @@ -3372,7 +3508,7 @@ TEST_F(ReplCoordTest, IgnoreTheContentsOfMetadataWhenItsConfigVersionDoesNotMatc "lastOpCommitted" << BSON("ts" << Timestamp(10, 0) << "t" << 2) << "lastOpVisible" << BSON("ts" << Timestamp(10, 0) << "t" << 2) << "configVersion" << 1 << "primaryIndex" << 2 << "term" << 2 << "syncSourceIndex" << 1))); - getReplCoord()->processReplSetMetadata(metadata.getValue()); + getReplCoord()->processReplSetMetadata(metadata.getValue(), true); ASSERT_EQUALS(OpTime(Timestamp(0, 0), 0), getReplCoord()->getLastCommittedOpTime()); // higher configVersion @@ -3382,7 +3518,7 @@ TEST_F(ReplCoordTest, IgnoreTheContentsOfMetadataWhenItsConfigVersionDoesNotMatc << BSON("ts" << Timestamp(10, 0) << "t" << 2) << "lastOpVisible" << BSON("ts" << Timestamp(10, 0) << "t" << 2) << "configVersion" << 100 << "primaryIndex" << 2 << "term" << 2 << "syncSourceIndex" << 1))); - getReplCoord()->processReplSetMetadata(metadata2.getValue()); + getReplCoord()->processReplSetMetadata(metadata2.getValue(), true); ASSERT_EQUALS(OpTime(Timestamp(0, 0), 0), getReplCoord()->getLastCommittedOpTime()); } @@ -3409,7 +3545,7 @@ TEST_F(ReplCoordTest, UpdateLastCommittedOpTimeWhenTheLastCommittedOpTimeFromMet ASSERT_EQUALS(1, getReplCoord()->getTerm()); OpTime time(Timestamp(10, 0), 1); - getReplCoord()->onSnapshotCreate(time, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time, SnapshotName(1)); // higher OpTime, should change StatusWith metadata = rpc::ReplSetMetadata::readFromMetadata(BSON( @@ -3417,7 +3553,7 @@ TEST_F(ReplCoordTest, UpdateLastCommittedOpTimeWhenTheLastCommittedOpTimeFromMet "lastOpCommitted" << BSON("ts" << Timestamp(10, 0) << "t" << 1) << "lastOpVisible" << BSON("ts" << Timestamp(10, 0) << "t" << 1) << "configVersion" << 2 << "primaryIndex" << 2 << "term" << 1 << "syncSourceIndex" << 1))); - getReplCoord()->processReplSetMetadata(metadata.getValue()); + getReplCoord()->processReplSetMetadata(metadata.getValue(), true); ASSERT_EQUALS(OpTime(Timestamp(10, 0), 1), getReplCoord()->getLastCommittedOpTime()); ASSERT_EQUALS(OpTime(Timestamp(10, 0), 1), getReplCoord()->getCurrentCommittedSnapshotOpTime()); @@ -3427,7 +3563,7 @@ TEST_F(ReplCoordTest, UpdateLastCommittedOpTimeWhenTheLastCommittedOpTimeFromMet "lastOpCommitted" << BSON("ts" << Timestamp(9, 0) << "t" << 1) << "lastOpVisible" << BSON("ts" << Timestamp(9, 0) << "t" << 1) << "configVersion" << 2 << "primaryIndex" << 2 << "term" << 1 << "syncSourceIndex" << 1))); - getReplCoord()->processReplSetMetadata(metadata2.getValue()); + getReplCoord()->processReplSetMetadata(metadata2.getValue(), true); ASSERT_EQUALS(OpTime(Timestamp(10, 0), 1), getReplCoord()->getLastCommittedOpTime()); } @@ -3458,7 +3594,7 @@ TEST_F(ReplCoordTest, UpdateTermWhenTheTermFromMetadataIsNewerButNeverUpdateCurr "lastOpCommitted" << BSON("ts" << Timestamp(10, 0) << "t" << 3) << "lastOpVisible" << BSON("ts" << Timestamp(10, 0) << "t" << 3) << "configVersion" << 2 << "primaryIndex" << 2 << "term" << 3 << "syncSourceIndex" << 1))); - getReplCoord()->processReplSetMetadata(metadata.getValue()); + getReplCoord()->processReplSetMetadata(metadata.getValue(), true); ASSERT_EQUALS(OpTime(Timestamp(10, 0), 3), getReplCoord()->getLastCommittedOpTime()); ASSERT_EQUALS(3, getReplCoord()->getTerm()); ASSERT_EQUALS(-1, getTopoCoord().getCurrentPrimaryIndex()); @@ -3469,7 +3605,7 @@ TEST_F(ReplCoordTest, UpdateTermWhenTheTermFromMetadataIsNewerButNeverUpdateCurr "lastOpCommitted" << BSON("ts" << Timestamp(11, 0) << "t" << 3) << "lastOpVisible" << BSON("ts" << Timestamp(11, 0) << "t" << 3) << "configVersion" << 2 << "primaryIndex" << 1 << "term" << 2 << "syncSourceIndex" << 1))); - getReplCoord()->processReplSetMetadata(metadata2.getValue()); + getReplCoord()->processReplSetMetadata(metadata2.getValue(), true); ASSERT_EQUALS(OpTime(Timestamp(11, 0), 3), getReplCoord()->getLastCommittedOpTime()); ASSERT_EQUALS(3, getReplCoord()->getTerm()); ASSERT_EQUALS(-1, getTopoCoord().getCurrentPrimaryIndex()); @@ -3480,14 +3616,14 @@ TEST_F(ReplCoordTest, UpdateTermWhenTheTermFromMetadataIsNewerButNeverUpdateCurr "lastOpCommitted" << BSON("ts" << Timestamp(11, 0) << "t" << 3) << "lastOpVisible" << BSON("ts" << Timestamp(11, 0) << "t" << 3) << "configVersion" << 2 << "primaryIndex" << 1 << "term" << 3 << "syncSourceIndex" << 1))); - getReplCoord()->processReplSetMetadata(metadata3.getValue()); + getReplCoord()->processReplSetMetadata(metadata3.getValue(), true); ASSERT_EQUALS(OpTime(Timestamp(11, 0), 3), getReplCoord()->getLastCommittedOpTime()); ASSERT_EQUALS(3, getReplCoord()->getTerm()); ASSERT_EQUALS(-1, getTopoCoord().getCurrentPrimaryIndex()); } TEST_F(ReplCoordTest, - TermAndLastCommittedOpTimeUpdateWhenHeartbeatResponseWithMetadataHasFresherValues) { + LastCommittedOpTimeNotUpdatedEvenWhenHeartbeatResponseWithMetadataHasFresherValues) { // Ensure that the metadata is processed if it is contained in a heartbeat response. assertStartSuccess(BSON("_id" << "mySet" @@ -3507,7 +3643,61 @@ TEST_F(ReplCoordTest, auto replCoord = getReplCoord(); auto config = replCoord->getConfig(); - // Higher term - should update term and lastCommittedOpTime. + // Higher term - should update term but not last committed optime. + StatusWith metadata = rpc::ReplSetMetadata::readFromMetadata(BSON( + rpc::kReplSetMetadataFieldName + << BSON("lastOpCommitted" << BSON("ts" << Timestamp(10, 0) << "t" << 3) << "lastOpVisible" + << BSON("ts" << Timestamp(10, 0) << "t" << 3) << "configVersion" + << config.getConfigVersion() << "primaryIndex" << 1 << "term" << 3 + << "syncSourceIndex" << 1))); + BSONObjBuilder metadataBuilder; + ASSERT_OK(metadata.getValue().writeToMetadata(&metadataBuilder)); + auto metadataObj = metadataBuilder.obj(); + + auto net = getNet(); + net->enterNetwork(); + + ASSERT_TRUE(net->hasReadyRequests()); + auto noi = net->getNextReadyRequest(); + const auto& request = noi->getRequest(); + ASSERT_EQUALS(HostAndPort("node2", 12345), request.target); + ASSERT_EQUALS("replSetHeartbeat", request.cmdObj.firstElement().fieldNameStringData()); + + ReplSetHeartbeatResponse hbResp; + hbResp.setConfigVersion(config.getConfigVersion()); + hbResp.setSetName(config.getReplSetName()); + hbResp.setState(MemberState::RS_SECONDARY); + net->scheduleResponse(noi, net->now(), makeResponseStatus(hbResp.toBSON(true), metadataObj)); + net->runReadyNetworkOperations(); + net->exitNetwork(); + + ASSERT_EQUALS(OpTime(Timestamp(0, 0), 0), getReplCoord()->getLastCommittedOpTime()); + ASSERT_EQUALS(3, getReplCoord()->getTerm()); + ASSERT_EQUALS(-1, getTopoCoord().getCurrentPrimaryIndex()); +} + +TEST_F(ReplCoordTest, TermAndLastCommittedOpTimeUpdatedFromHeartbeatWhenArbiter) { + // Ensure that the metadata is processed if it is contained in a heartbeat response. + assertStartSuccess(BSON("_id" + << "mySet" + << "version" << 2 << "members" + << BSON_ARRAY(BSON("host" + << "node1:12345" + << "_id" << 0 << "arbiterOnly" << true) + << BSON("host" + << "node2:12345" + << "_id" << 1)) << "protocolVersion" << 1), + HostAndPort("node1", 12345)); + ASSERT_EQUALS(OpTime(Timestamp(0, 0), 0), getReplCoord()->getLastCommittedOpTime()); + OperationContextNoop txn; + getReplCoord()->updateTerm(&txn, 1); + ASSERT_EQUALS(1, getReplCoord()->getTerm()); + + auto replCoord = getReplCoord(); + auto config = replCoord->getConfig(); + + // Higher term - should update term and lastCommittedOpTime since arbiters learn of the + // commit point via heartbeats. StatusWith metadata = rpc::ReplSetMetadata::readFromMetadata(BSON( rpc::kReplSetMetadataFieldName << BSON("lastOpCommitted" << BSON("ts" << Timestamp(10, 0) << "t" << 3) << "lastOpVisible" @@ -3703,7 +3893,7 @@ TEST_F(ReplCoordTest, } TEST_F(ReplCoordTest, - CancelAndRescheduleElectionTimeoutWhenProcessingHeartbeatResponseFromPrimary) { + RescheduleElectionTimeoutWhenProcessingHeartbeatResponseFromPrimaryInSameTerm) { assertStartSuccess(BSON("_id" << "mySet" << "protocolVersion" << 1 << "version" << 2 << "members" @@ -3735,6 +3925,8 @@ TEST_F(ReplCoordTest, ReplSetHeartbeatResponse hbResp; hbResp.setSetName("mySet"); hbResp.setState(MemberState::RS_PRIMARY); + hbResp.setTerm(replCoord->getTerm()); + // Heartbeat response is scheduled with a delay so that we can be sure that // the election was rescheduled due to the heartbeat response. auto heartbeatWhen = net->now() + Seconds(1); @@ -3748,6 +3940,54 @@ TEST_F(ReplCoordTest, replCoord->getElectionTimeout_forTest()); } +TEST_F(ReplCoordTest, + DontRescheduleElectionTimeoutWhenProcessingHeartbeatResponseFromPrimaryInDiffertTerm) { + assertStartSuccess(BSON("_id" + << "mySet" + << "protocolVersion" << 1 << "version" << 2 << "members" + << BSON_ARRAY(BSON("host" + << "node1:12345" + << "_id" << 0) + << BSON("host" + << "node2:12345" + << "_id" << 1))), + HostAndPort("node1", 12345)); + + ReplicationCoordinatorImpl* replCoord = getReplCoord(); + ASSERT_TRUE(replCoord->setFollowerMode(MemberState::RS_SECONDARY)); + + auto electionTimeoutWhen = replCoord->getElectionTimeout_forTest(); + ASSERT_NOT_EQUALS(Date_t(), electionTimeoutWhen); + + auto net = getNet(); + net->enterNetwork(); + ASSERT_TRUE(net->hasReadyRequests()); + auto noi = net->getNextReadyRequest(); + auto&& request = noi->getRequest(); + log() << "processing " << request.cmdObj; + ASSERT_EQUALS(HostAndPort("node2", 12345), request.target); + + ASSERT_EQUALS("replSetHeartbeat", request.cmdObj.firstElement().fieldNameStringData()); + + // Respond to node1's heartbeat command to indicate that node2 is PRIMARY. + ReplSetHeartbeatResponse hbResp; + hbResp.setSetName("mySet"); + hbResp.setState(MemberState::RS_PRIMARY); + hbResp.setTerm(replCoord->getTerm() - 1); + + // Heartbeat response is scheduled with a delay so that we can be sure that + // the election was rescheduled due to the heartbeat response. + auto heartbeatWhen = net->now() + Seconds(1); + net->scheduleResponse(noi, heartbeatWhen, makeResponseStatus(hbResp.toBSON(true))); + net->runUntil(heartbeatWhen); + ASSERT_EQUALS(heartbeatWhen, net->now()); + net->runReadyNetworkOperations(); + net->exitNetwork(); + + ASSERT_GREATER_THAN(heartbeatWhen + replCoord->getConfig().getElectionTimeoutPeriod(), + replCoord->getElectionTimeout_forTest()); +} + TEST_F(ReplCoordTest, CancelAndRescheduleElectionTimeoutWhenProcessingHeartbeatResponseWithoutState) { assertStartSuccess(BSON("_id" @@ -3812,9 +4052,9 @@ TEST_F(ReplCoordTest, AdvanceCommittedSnapshotToMostRecentSnapshotPriorToOpTimeW OpTime time5(Timestamp(100, 5), 1); OpTime time6(Timestamp(100, 6), 1); - getReplCoord()->onSnapshotCreate(time1, SnapshotName(1)); - getReplCoord()->onSnapshotCreate(time2, SnapshotName(2)); - getReplCoord()->onSnapshotCreate(time5, SnapshotName(3)); + getReplCoord()->createSnapshot(&txn, time1, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time2, SnapshotName(2)); + getReplCoord()->createSnapshot(&txn, time5, SnapshotName(3)); // ensure current snapshot follows price is right rules (closest but not greater than) getReplCoord()->setMyLastAppliedOpTime(time3); @@ -3844,9 +4084,9 @@ TEST_F(ReplCoordTest, DoNotAdvanceCommittedSnapshotWhenAnOpTimeIsNewerThanOurLat OpTime time5(Timestamp(100, 5), 1); OpTime time6(Timestamp(100, 6), 1); - getReplCoord()->onSnapshotCreate(time1, SnapshotName(1)); - getReplCoord()->onSnapshotCreate(time2, SnapshotName(2)); - getReplCoord()->onSnapshotCreate(time5, SnapshotName(3)); + getReplCoord()->createSnapshot(&txn, time1, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time2, SnapshotName(2)); + getReplCoord()->createSnapshot(&txn, time5, SnapshotName(3)); // ensure current snapshot will not advance beyond existing snapshots getReplCoord()->setMyLastAppliedOpTime(time6); @@ -3874,16 +4114,16 @@ TEST_F(ReplCoordTest, OpTime time5(Timestamp(100, 5), 1); OpTime time6(Timestamp(100, 6), 1); - getReplCoord()->onSnapshotCreate(time1, SnapshotName(1)); - getReplCoord()->onSnapshotCreate(time2, SnapshotName(2)); - getReplCoord()->onSnapshotCreate(time5, SnapshotName(3)); + getReplCoord()->createSnapshot(&txn, time1, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time2, SnapshotName(2)); + getReplCoord()->createSnapshot(&txn, time5, SnapshotName(3)); getReplCoord()->setMyLastAppliedOpTime(time6); getReplCoord()->setMyLastDurableOpTime(time6); ASSERT_EQUALS(time5, getReplCoord()->getCurrentCommittedSnapshotOpTime()); // ensure current snapshot updates on new snapshot if we are that far - getReplCoord()->onSnapshotCreate(time6, SnapshotName(4)); + getReplCoord()->createSnapshot(&txn, time6, SnapshotName(4)); ASSERT_EQUALS(time6, getReplCoord()->getCurrentCommittedSnapshotOpTime()); } @@ -3906,9 +4146,9 @@ TEST_F(ReplCoordTest, ZeroCommittedSnapshotWhenAllSnapshotsAreDropped) { OpTime time5(Timestamp(100, 5), 1); OpTime time6(Timestamp(100, 6), 1); - getReplCoord()->onSnapshotCreate(time1, SnapshotName(1)); - getReplCoord()->onSnapshotCreate(time2, SnapshotName(2)); - getReplCoord()->onSnapshotCreate(time5, SnapshotName(3)); + getReplCoord()->createSnapshot(&txn, time1, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time2, SnapshotName(2)); + getReplCoord()->createSnapshot(&txn, time5, SnapshotName(3)); // ensure dropping all snapshots should reset the current committed snapshot getReplCoord()->dropAllSnapshots(); @@ -3930,7 +4170,7 @@ TEST_F(ReplCoordTest, DoNotAdvanceCommittedSnapshotWhenAppliedOpTimeChanges) { OpTime time1(Timestamp(100, 1), 1); OpTime time2(Timestamp(100, 2), 1); - getReplCoord()->onSnapshotCreate(time1, SnapshotName(1)); + getReplCoord()->createSnapshot(&txn, time1, SnapshotName(1)); getReplCoord()->setMyLastAppliedOpTime(time1); ASSERT_EQUALS(OpTime(), getReplCoord()->getCurrentCommittedSnapshotOpTime()); @@ -4089,6 +4329,37 @@ TEST_F(ReplCoordTest, OnlyForwardSyncProgressForOtherNodesWhenTheNodesAreBelieve ASSERT_EQUALS(1U, memberIds4.size()); } +TEST_F(ReplCoordTest, NewStyleUpdatePositionCmdHasMetadata) { + assertStartSuccess( + BSON("_id" + << "mySet" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 0 << "host" + << "test1:1234") + << BSON("_id" << 1 << "host" + << "test2:1234") << BSON("_id" << 2 << "host" + << "test3:1234")) + << "protocolVersion" << 1 << "settings" + << BSON("electionTimeoutMillis" << 2000 << "heartbeatIntervalMillis" << 40000)), + HostAndPort("test1", 1234)); + OpTime optime(Timestamp(100, 2), 0); + getReplCoord()->setMyLastAppliedOpTime(optime); + getReplCoord()->setMyLastDurableOpTime(optime); + + // Set last committed optime via metadata. + OperationContextReplMock txn; + rpc::ReplSetMetadata syncSourceMetadata(optime.getTerm(), optime, optime, 1, OID(), -1, 1); + getReplCoord()->processReplSetMetadata(syncSourceMetadata, true); + getReplCoord()->createSnapshot(&txn, optime, SnapshotName(1)); + + BSONObjBuilder cmdBuilder; + ASSERT_TRUE(getReplCoord()->prepareReplSetUpdatePositionCommand(&cmdBuilder)); + BSONObj cmd = cmdBuilder.obj(); + auto metadata = unittest::assertGet(rpc::ReplSetMetadata::readFromMetadata(cmd)); + ASSERT_EQUALS(metadata.getTerm(), getReplCoord()->getTerm()); + ASSERT_EQUALS(metadata.getLastOpVisible(), optime); +} + TEST_F(ReplCoordTest, StepDownWhenHandleLivenessTimeoutMarksAMajorityOfVotingNodesDown) { assertStartSuccess( BSON("_id" @@ -4417,6 +4688,89 @@ TEST_F(ReplCoordTest, PopulateUnsetWriteConcernOptionsSyncModeReturnsInputIfWMod getReplCoord()->populateUnsetWriteConcernOptionsSyncMode(wc).syncMode); } +TEST_F(ReplCoordTest, NodeStoresElectionVotes) { + assertStartSuccess(BSON("_id" + << "mySet" + << "version" << 2 << "members" + << BSON_ARRAY(BSON("host" + << "node1:12345" + << "_id" << 0) + << BSON("host" + << "node2:12345" + << "_id" << 1) << BSON("host" + << "node3:12345" + << "_id" << 2))), + HostAndPort("node1", 12345)); + auto time = OpTime(Timestamp(100, 0), 1); + ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); + getReplCoord()->setMyLastAppliedOpTime(time); + getReplCoord()->setMyLastDurableOpTime(time); + simulateSuccessfulV1Election(); + + OperationContextReplMock txn; + + ReplSetRequestVotesArgs args; + ASSERT_OK(args.initialize(BSON( + "replSetRequestVotes" << 1 << "setName" + << "mySet" + << "term" << 7LL << "candidateIndex" << 2LL << "configVersion" << 2LL + << "dryRun" << false << "lastCommittedOp" << time.toBSON()))); + ReplSetRequestVotesResponse response; + + ASSERT_OK(getReplCoord()->processReplSetRequestVotes(&txn, args, &response)); + ASSERT_EQUALS("", response.getReason()); + ASSERT_TRUE(response.getVoteGranted()); + + auto lastVote = getExternalState()->loadLocalLastVoteDocument(&txn); + ASSERT_OK(lastVote.getStatus()); + + // This is not a dry-run election so the last vote should include the new term and candidate. + ASSERT_EQUALS(lastVote.getValue().getTerm(), 7); + ASSERT_EQUALS(lastVote.getValue().getCandidateIndex(), 2); +} + +TEST_F(ReplCoordTest, NodeDoesNotStoreDryRunVotes) { + assertStartSuccess(BSON("_id" + << "mySet" + << "version" << 2 << "members" + << BSON_ARRAY(BSON("host" + << "node1:12345" + << "_id" << 0) + << BSON("host" + << "node2:12345" + << "_id" << 1) << BSON("host" + << "node3:12345" + << "_id" << 2))), + HostAndPort("node1", 12345)); + auto time = OpTime(Timestamp(100, 0), 1); + ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); + getReplCoord()->setMyLastAppliedOpTime(time); + getReplCoord()->setMyLastDurableOpTime(time); + simulateSuccessfulV1Election(); + + OperationContextReplMock txn; + + ReplSetRequestVotesArgs args; + ASSERT_OK(args.initialize(BSON( + "replSetRequestVotes" << 1 << "setName" + << "mySet" + << "term" << 7LL << "candidateIndex" << 2LL << "configVersion" << 2LL + << "dryRun" << true << "lastCommittedOp" << time.toBSON()))); + ReplSetRequestVotesResponse response; + + ASSERT_OK(getReplCoord()->processReplSetRequestVotes(&txn, args, &response)); + ASSERT_EQUALS("", response.getReason()); + ASSERT_TRUE(response.getVoteGranted()); + + auto lastVote = getExternalState()->loadLocalLastVoteDocument(&txn); + ASSERT_OK(lastVote.getStatus()); + + // This is a dry-run election so the last vote should not be updated with the new term and + // candidate. + ASSERT_EQUALS(lastVote.getValue().getTerm(), 1); + ASSERT_EQUALS(lastVote.getValue().getCandidateIndex(), 0); +} + // TODO(schwerin): Unit test election id updating } // namespace } // namespace repl diff --git a/src/mongo/db/repl/replication_coordinator_mock.cpp b/src/mongo/db/repl/replication_coordinator_mock.cpp index 7d3f987878f..184cf75eda1 100644 --- a/src/mongo/db/repl/replication_coordinator_mock.cpp +++ b/src/mongo/db/repl/replication_coordinator_mock.cpp @@ -236,7 +236,8 @@ void ReplicationCoordinatorMock::processReplSetGetConfig(BSONObjBuilder* result) // TODO } -void ReplicationCoordinatorMock::processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata) {} +void ReplicationCoordinatorMock::processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata, + bool advanceCommitPoint) {} void ReplicationCoordinatorMock::cancelAndRescheduleElectionTimeout() {} @@ -381,9 +382,8 @@ Status ReplicationCoordinatorMock::processReplSetDeclareElectionWinner( return Status::OK(); } -void ReplicationCoordinatorMock::prepareReplResponseMetadata(const rpc::RequestInterface& request, - const OpTime& lastOpTimeFromClient, - BSONObjBuilder* builder) {} +void ReplicationCoordinatorMock::prepareReplMetadata(const OpTime& lastOpTimeFromClient, + BSONObjBuilder* builder) const {} Status ReplicationCoordinatorMock::processHeartbeatV1(const ReplSetHeartbeatArgsV1& args, ReplSetHeartbeatResponse* response) { @@ -414,11 +414,13 @@ SnapshotName ReplicationCoordinatorMock::reserveSnapshotName(OperationContext* t void ReplicationCoordinatorMock::forceSnapshotCreation() {} -void ReplicationCoordinatorMock::onSnapshotCreate(OpTime timeOfSnapshot, SnapshotName name) {} +void ReplicationCoordinatorMock::createSnapshot(OperationContext* txn, + OpTime timeOfSnapshot, + SnapshotName name){}; void ReplicationCoordinatorMock::dropAllSnapshots() {} -OpTime ReplicationCoordinatorMock::getCurrentCommittedSnapshotOpTime() { +OpTime ReplicationCoordinatorMock::getCurrentCommittedSnapshotOpTime() const { return OpTime(); } diff --git a/src/mongo/db/repl/replication_coordinator_mock.h b/src/mongo/db/repl/replication_coordinator_mock.h index 15315cd5a8f..b5500265045 100644 --- a/src/mongo/db/repl/replication_coordinator_mock.h +++ b/src/mongo/db/repl/replication_coordinator_mock.h @@ -150,7 +150,8 @@ public: virtual void processReplSetGetConfig(BSONObjBuilder* result); - virtual void processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata); + void processReplSetMetadata(const rpc::ReplSetMetadata& replMetadata, + bool advanceCommitPoint) override; virtual void cancelAndRescheduleElectionTimeout() override; @@ -217,9 +218,8 @@ public: virtual Status processReplSetDeclareElectionWinner(const ReplSetDeclareElectionWinnerArgs& args, long long* responseTerm); - void prepareReplResponseMetadata(const rpc::RequestInterface& request, - const OpTime& lastOpTimeFromClient, - BSONObjBuilder* builder) override; + void prepareReplMetadata(const OpTime& lastOpTimeFromClient, + BSONObjBuilder* builder) const override; virtual Status processHeartbeatV1(const ReplSetHeartbeatArgsV1& args, ReplSetHeartbeatResponse* response); @@ -238,11 +238,13 @@ public: virtual void forceSnapshotCreation() override; - virtual void onSnapshotCreate(OpTime timeOfSnapshot, SnapshotName name); + virtual void createSnapshot(OperationContext* txn, + OpTime timeOfSnapshot, + SnapshotName name) override; virtual void dropAllSnapshots() override; - virtual OpTime getCurrentCommittedSnapshotOpTime() override; + virtual OpTime getCurrentCommittedSnapshotOpTime() const override; virtual void waitUntilSnapshotCommitted(OperationContext* txn, const SnapshotName& untilSnapshot) override; diff --git a/src/mongo/db/repl/replication_executor.cpp b/src/mongo/db/repl/replication_executor.cpp index 4081e3fb876..2b1c7c6124f 100644 --- a/src/mongo/db/repl/replication_executor.cpp +++ b/src/mongo/db/repl/replication_executor.cpp @@ -274,6 +274,7 @@ StatusWith ReplicationExecutor::onEvent( queue = &event->_waiters; } else { queue = &_readyQueue; + _networkInterface->signalWorkAvailable(); } return enqueueWork_inlock(queue, work); } @@ -382,6 +383,7 @@ StatusWith ReplicationExecutor::scheduleWor ++insertBefore; _sleepersQueue.splice(insertBefore, temp, temp.begin()); ++_counterScheduledWorkAts; + _networkInterface->signalWorkAvailable(); return cbHandle; } @@ -426,7 +428,9 @@ void ReplicationExecutor::_doOperation(OperationContext* txn, return; Callback* callback = _getCallbackFromHandle(cbHandle); const WorkQueue::iterator iter = callback->_iter; + callback->_isRemoved = true; iter->callback = CallbackHandle(); + iter->isNetworkOperation = false; _freeQueue.splice(_freeQueue.begin(), *workQueue, iter); lk.unlock(); { @@ -493,7 +497,10 @@ ReplicationExecutor::getWork() { } const WorkItem work = *_readyQueue.begin(); const CallbackHandle cbHandle = work.callback; + Callback* callback = _getCallbackFromHandle(cbHandle); + callback->_isRemoved = true; _readyQueue.begin()->callback = CallbackHandle(); + _readyQueue.begin()->isNetworkOperation = false; _freeQueue.splice(_freeQueue.begin(), _readyQueue, _readyQueue.begin()); return std::make_pair(work, cbHandle); } @@ -590,6 +597,7 @@ ReplicationExecutor::Callback::Callback(ReplicationExecutor* executor, _callbackFn(callbackFn), _isCanceled(false), _isSleeper(false), + _isRemoved(false), _iter(iter), _finishedEvent(finishedEvent) {} @@ -602,6 +610,11 @@ bool ReplicationExecutor::Callback::isCanceled() const { void ReplicationExecutor::Callback::cancel() { stdx::unique_lock lk(_executor->_mutex); + // If this element has already been removed from the queues, + // the cancel is too late and has no effect. + if (_isRemoved) + return; + _isCanceled = true; if (_isSleeper) { diff --git a/src/mongo/db/repl/replication_executor.h b/src/mongo/db/repl/replication_executor.h index c46ef4a659a..9402a66f364 100644 --- a/src/mongo/db/repl/replication_executor.h +++ b/src/mongo/db/repl/replication_executor.h @@ -363,6 +363,7 @@ private: CallbackFn _callbackFn; bool _isCanceled; bool _isSleeper; + bool _isRemoved; WorkQueue::iterator _iter; EventHandle _finishedEvent; }; diff --git a/src/mongo/db/repl/replication_executor_test.cpp b/src/mongo/db/repl/replication_executor_test.cpp index 998a80a549e..60259be0b52 100644 --- a/src/mongo/db/repl/replication_executor_test.cpp +++ b/src/mongo/db/repl/replication_executor_test.cpp @@ -52,6 +52,8 @@ namespace repl { namespace { using executor::NetworkInterfaceMock; +using executor::RemoteCommandResponse; +using unittest::assertGet; const int64_t prngSeed = 1; @@ -171,6 +173,164 @@ TEST_F(ReplicationExecutorTest, CancelBeforeRunningFutureWork) { ASSERT_EQUALS(1, executor.getDiagnosticBSON().getFieldDotted("queues.ready").Int()); } +// Equivalent to EventChainAndWaitingTest::onGo +TEST_F(ReplicationExecutorTest, ScheduleCallbackOnFutureEvent) { + launchExecutorThread(); + getNet()->exitNetwork(); + + ReplicationExecutor& executor = getReplExecutor(); + // We signal this "ping" event and the executor will signal "pong" event in return. + auto ping = assertGet(executor.makeEvent()); + auto pong = assertGet(executor.makeEvent()); + auto fn = [&executor, pong](const ReplicationExecutor::CallbackArgs& cbData) { + ASSERT_OK(cbData.status); + executor.signalEvent(pong); + }; + + // Wait for a future event. + executor.onEvent(ping, fn); + ASSERT_EQUALS(0, executor.getDiagnosticBSON().getFieldDotted("queues.ready").Int()); + executor.signalEvent(ping); + executor.waitForEvent(pong); +} + +// Equivalent to EventChainAndWaitingTest::onGoAfterTriggered +TEST_F(ReplicationExecutorTest, ScheduleCallbackOnSignaledEvent) { + launchExecutorThread(); + getNet()->exitNetwork(); + + ReplicationExecutor& executor = getReplExecutor(); + // We signal this "ping" event and the executor will signal "pong" event in return. + auto ping = assertGet(executor.makeEvent()); + auto pong = assertGet(executor.makeEvent()); + auto fn = [&executor, pong](const ReplicationExecutor::CallbackArgs& cbData) { + ASSERT_OK(cbData.status); + executor.signalEvent(pong); + }; + + // Wait for a signaled event. + executor.signalEvent(ping); + executor.onEvent(ping, fn); + executor.waitForEvent(pong); +} + +TEST_F(ReplicationExecutorTest, ScheduleCallbackAtNow) { + launchExecutorThread(); + getNet()->exitNetwork(); + + ReplicationExecutor& executor = getReplExecutor(); + auto finishEvent = assertGet(executor.makeEvent()); + auto fn = [&executor, finishEvent](const ReplicationExecutor::CallbackArgs& cbData) { + ASSERT_OK(cbData.status); + executor.signalEvent(finishEvent); + }; + + auto cb = executor.scheduleWorkAt(getNet()->now(), fn); + executor.waitForEvent(finishEvent); +} + +TEST_F(ReplicationExecutorTest, ScheduleCallbackAtAFutureTime) { + launchExecutorThread(); + getNet()->exitNetwork(); + + ReplicationExecutor& executor = getReplExecutor(); + auto finishEvent = assertGet(executor.makeEvent()); + auto fn = [&executor, finishEvent](const ReplicationExecutor::CallbackArgs& cbData) { + ASSERT_OK(cbData.status); + executor.signalEvent(finishEvent); + }; + + auto now = getNet()->now(); + now += Milliseconds(1000); + auto cb = executor.scheduleWorkAt(now, fn); + + getNet()->enterNetwork(); + getNet()->runUntil(now); + getNet()->exitNetwork(); + + executor.waitForEvent(finishEvent); +} + + +TEST_F(ReplicationExecutorTest, TestForCancelRace) { + launchExecutorThread(); + getNet()->exitNetwork(); + + unittest::Barrier enterCallback(2U), runCallback(2U); + + ReplicationExecutor& executor = getReplExecutor(); + bool firstEventDone = false; + bool firstEventCanceled = false; + auto fn = [&executor, &enterCallback, &runCallback, &firstEventDone, &firstEventCanceled]( + const executor::TaskExecutor::RemoteCommandCallbackArgs& cbData) { + // This barrier lets the test code wait until we're in the callback. + enterCallback.countDownAndWait(); + // This barrier lets the test code keep us in the callback until it has run the cancel. + runCallback.countDownAndWait(); + firstEventCanceled = !cbData.response.getStatus().isOK(); + firstEventDone = true; + }; + + // First, schedule a network event to run. + const executor::RemoteCommandRequest request( + HostAndPort("test1", 1234), "mydb", BSON("nothing" << 0)); + auto firstCallback = assertGet(executor.scheduleRemoteCommand(request, fn)); + + // Now let the request happen. + // We need to run the network on another thread, because the test + // fixture will hang waiting for the callbacks to complete. + auto timeThread = stdx::thread([this] { + getNet()->enterNetwork(); + ASSERT(getNet()->hasReadyRequests()); + auto noi = getNet()->getNextReadyRequest(); + getNet()->scheduleResponse(noi, getNet()->now(), RemoteCommandResponse()); + getNet()->runReadyNetworkOperations(); + getNet()->exitNetwork(); + }); + + // Wait until we're in the callback. + enterCallback.countDownAndWait(); + + // Schedule a different network event to run. + bool secondEventDone = false; + bool secondEventCanceled = false; + auto fn2 = [&executor, &secondEventDone, &secondEventCanceled]( + const executor::TaskExecutor::RemoteCommandCallbackArgs& cbData) { + secondEventCanceled = !cbData.response.getStatus().isOK(); + secondEventDone = true; + }; + auto secondCallback = assertGet(executor.scheduleRemoteCommand(request, fn2)); + ASSERT_FALSE(firstEventDone); // The first event should be stuck at runCallback barrier. + // Cancel the first callback. This cancel should have no effect as the callback has + // already been started. + executor.cancel(firstCallback); + + // Let the first callback continue to completion. + runCallback.countDownAndWait(); + + // Now the time thread can continue. + timeThread.join(); + + // The first event should be done, the second event should be pending. + ASSERT(firstEventDone); + ASSERT_FALSE(secondEventDone); + + // Run the network thread, which should run the second request. + { + getNet()->enterNetwork(); + // The second request should be ready. + ASSERT(getNet()->hasReadyRequests()) << "Second request is not ready (cancelled?)"; + auto noi = getNet()->getNextReadyRequest(); + getNet()->scheduleResponse(noi, getNet()->now(), RemoteCommandResponse()); + getNet()->runReadyNetworkOperations(); + getNet()->exitNetwork(); + } + + // The second callback should have run without being canceled. + ASSERT_TRUE(secondEventDone); + ASSERT_FALSE(secondEventCanceled); +} + } // namespace } // namespace repl } // namespace mongo diff --git a/src/mongo/db/repl/replset_commands.cpp b/src/mongo/db/repl/replset_commands.cpp index ee0aa50a6f0..0c202e906a1 100644 --- a/src/mongo/db/repl/replset_commands.cpp +++ b/src/mongo/db/repl/replset_commands.cpp @@ -58,6 +58,7 @@ #include "mongo/db/service_context.h" #include "mongo/db/storage/storage_engine.h" #include "mongo/executor/network_interface.h" +#include "mongo/rpc/metadata/repl_set_metadata.h" #include "mongo/util/fail_point_service.h" #include "mongo/util/log.h" #include "mongo/util/scopeguard.h" @@ -664,7 +665,9 @@ public: int, string& errmsg, BSONObjBuilder& result) { - Status status = getGlobalReplicationCoordinator()->checkReplEnabledForCommand(&result); + auto replCoord = repl::ReplicationCoordinator::get(txn->getClient()->getServiceContext()); + + Status status = replCoord->checkReplEnabledForCommand(&result); if (!status.isOK()) return appendCommandStatus(result, status); @@ -673,6 +676,14 @@ public: if (cmdObj.hasField("handshake")) return true; + auto metadataResult = rpc::ReplSetMetadata::readFromMetadata(cmdObj); + if (metadataResult.isOK()) { + // New style update position command has metadata, which may inform the + // upstream of a higher term. + auto metadata = metadataResult.getValue(); + replCoord->processReplSetMetadata(metadata, false /*don't advance the commit point*/); + } + // In the case of an update from a member with an invalid replica set config, // we return our current config version. long long configVersion = -1; @@ -681,9 +692,8 @@ public: status = args.initialize(cmdObj); if (status.isOK()) { - // v3.2.2+ style replSetUpdatePosition command. - status = getGlobalReplicationCoordinator()->processReplSetUpdatePosition( - args, &configVersion); + // v3.2.4+ style replSetUpdatePosition command. + status = replCoord->processReplSetUpdatePosition(args, &configVersion); if (status == ErrorCodes::InvalidReplicaSetConfig) { result.append("configVersion", configVersion); @@ -696,8 +706,7 @@ public: if (!status.isOK()) return appendCommandStatus(result, status); - status = getGlobalReplicationCoordinator()->processReplSetUpdatePosition( - oldArgs, &configVersion); + status = replCoord->processReplSetUpdatePosition(oldArgs, &configVersion); if (status == ErrorCodes::InvalidReplicaSetConfig) { result.append("configVersion", configVersion); diff --git a/src/mongo/db/repl/rs_rollback.cpp b/src/mongo/db/repl/rs_rollback.cpp index c05c5a7d69c..f860b2e005f 100644 --- a/src/mongo/db/repl/rs_rollback.cpp +++ b/src/mongo/db/repl/rs_rollback.cpp @@ -62,8 +62,10 @@ #include "mongo/db/repl/roll_back_local_operations.h" #include "mongo/db/repl/rollback_source.h" #include "mongo/db/repl/rslog.h" +#include "mongo/util/exit.h" #include "mongo/util/fail_point_service.h" #include "mongo/util/log.h" +#include "mongo/util/scopeguard.h" /* Scenarios * @@ -119,59 +121,50 @@ namespace repl { // Failpoint which causes rollback to hang before finishing. MONGO_FP_DECLARE(rollbackHangBeforeFinish); +MONGO_FP_DECLARE(rollbackHangThenFailAfterWritingMinValid); -namespace { - -class RSFatalException : public std::exception { -public: - RSFatalException(std::string m = "replica set fatal exception") : msg(m) {} - virtual ~RSFatalException() throw(){}; - virtual const char* what() const throw() { - return msg.c_str(); - } - -private: - std::string msg; -}; - -struct DocID { - // ns and _id both point into ownedObj's buffer - BSONObj ownedObj; - const char* ns; - BSONElement _id; - bool operator<(const DocID& other) const { - int comp = strcmp(ns, other.ns); - if (comp < 0) - return true; - if (comp > 0) - return false; - return _id < other._id; - } -}; - -struct FixUpInfo { - // note this is a set -- if there are many $inc's on a single document we need to rollback, - // we only need to refetch it once. - set toRefetch; +using namespace rollback_internal; - // collections to drop - set toDrop; +bool DocID::operator<(const DocID& other) const { + int comp = strcmp(ns, other.ns); + if (comp < 0) + return true; + if (comp > 0) + return false; - // Indexes to drop. - // Key is collection namespace. Value is name of index to drop. - multimap indexesToDrop; + return _id < other._id; +} - set collectionsToResyncData; - set collectionsToResyncMetadata; +bool DocID::operator==(const DocID& other) const { + // Since this is only used for tests, going with the simple impl that reuses operator< which is + // used in the real code. + return !(*this < other || other < *this); +} - OpTime commonPoint; - RecordId commonPointOurDiskloc; +void FixUpInfo::removeAllDocsToRefetchFor(const std::string& collection) { + docsToRefetch.erase(docsToRefetch.lower_bound(DocID::minFor(collection.c_str())), + docsToRefetch.upper_bound(DocID::maxFor(collection.c_str()))); +} - int rbid; // remote server's current rollback sequence # -}; +void FixUpInfo::removeRedundantOperations() { + // These loops and their bodies can be done in any order. The final result of the FixUpInfo + // members will be the same either way. + for (const auto& collection : collectionsToDrop) { + removeAllDocsToRefetchFor(collection); + indexesToDrop.erase(collection); + collectionsToResyncMetadata.erase(collection); + } + for (const auto& collection : collectionsToResyncData) { + removeAllDocsToRefetchFor(collection); + indexesToDrop.erase(collection); + collectionsToResyncMetadata.erase(collection); + collectionsToDrop.erase(collection); + } +} -Status refetch(FixUpInfo& fixUpInfo, const BSONObj& ourObj) { +Status rollback_internal::updateFixUpInfoFromLocalOplogEntry(FixUpInfo& fixUpInfo, + const BSONObj& ourObj) { const char* op = ourObj.getStringField("op"); if (*op == 'n') return Status::OK(); @@ -183,14 +176,14 @@ Status refetch(FixUpInfo& fixUpInfo, const BSONObj& ourObj) { doc.ownedObj = ourObj.getOwned(); doc.ns = doc.ownedObj.getStringField("ns"); if (*doc.ns == '\0') { - warning() << "ignoring op on rollback no ns TODO : " << doc.ownedObj.toString(); - return Status::OK(); + throw RSFatalException(str::stream() + << "local op on rollback has no ns: " << doc.ownedObj.toString()); } BSONObj obj = doc.ownedObj.getObjectField(*op == 'u' ? "o2" : "o"); if (obj.isEmpty()) { - warning() << "ignoring op on rollback : " << doc.ownedObj.toString(); - return Status::OK(); + throw RSFatalException(str::stream() + << "local op on rollback has no object field: " << (doc.ownedObj)); } if (*op == 'c') { @@ -208,7 +201,7 @@ Status refetch(FixUpInfo& fixUpInfo, const BSONObj& ourObj) { // Create collection operation // { ts: ..., h: ..., op: "c", ns: "foo.$cmd", o: { create: "abc", ... } } string ns = nss.db().toString() + '.' + obj["create"].String(); // -> foo.abc - fixUpInfo.toDrop.insert(ns); + fixUpInfo.collectionsToDrop.insert(ns); return Status::OK(); } else if (cmdname == "drop") { string ns = nss.db().toString() + '.' + first.valuestr(); @@ -270,7 +263,7 @@ Status refetch(FixUpInfo& fixUpInfo, const BSONObj& ourObj) { severe() << message; return Status(ErrorCodes::UnrecoverableRollbackError, message); } - auto subStatus = refetch(fixUpInfo, subopElement.Obj()); + auto subStatus = updateFixUpInfoFromLocalOplogEntry(fixUpInfo, subopElement.Obj()); if (!subStatus.isOK()) { return subStatus; } @@ -328,13 +321,57 @@ Status refetch(FixUpInfo& fixUpInfo, const BSONObj& ourObj) { throw RSFatalException(); } - fixUpInfo.toRefetch.insert(doc); + fixUpInfo.docsToRefetch.insert(doc); return Status::OK(); } +namespace { + +/** + * This must be called before making any changes to our local data and after fetching any + * information from the upstream node. If any information is fetched from the upstream node after we + * have written locally, the function must be called again. + */ +void checkRbidAndUpdateMinValid(OperationContext* txn, + const int rbid, + const RollbackSource& rollbackSource) { + // It is important that the steps are performed in order to avoid racing with upstream rollbacks + // + // 1) Get the last doc in their oplog. + // 2) Get their RBID and fail if it has changed. + // 3) Set our minValid to the previously fetched OpTime of the top of their oplog. + + const auto newMinValidDoc = rollbackSource.getLastOperation(); + if (newMinValidDoc.isEmpty()) { + uasserted(40361, "rollback error newest oplog entry on source is missing or empty"); + } + if (rbid != rollbackSource.getRollbackId()) { + // Our source rolled back itself so the data we received isn't necessarily consistent. + uasserted(40365, "rollback rbid on source changed during rollback, canceling this attempt"); + } + + // we have items we are writing that aren't from a point-in-time. thus best not to come + // online until we get to that point in freshness. + OpTime minValid = fassertStatusOK(28774, OpTime::parseFromOplogEntry(newMinValidDoc)); + log() << "Setting minvalid to " << minValid; + setAppliedThrough(txn, {}); // Use top of oplog. + setMinValid(txn, minValid); + + if (MONGO_FAIL_POINT(rollbackHangThenFailAfterWritingMinValid)) { + // This log output is used in js tests so please leave it. + log() << "rollback - rollbackHangThenFailAfterWritingMinValid fail point " + "enabled. Blocking until fail point is disabled."; + while (MONGO_FAIL_POINT(rollbackHangThenFailAfterWritingMinValid)) { + invariant(!inShutdown()); // It is an error to shutdown while enabled. + mongo::sleepsecs(1); + } + uasserted(40378, + "failing rollback due to rollbackHangThenFailAfterWritingMinValid fail point"); + } +} void syncFixUp(OperationContext* txn, - FixUpInfo& fixUpInfo, + const FixUpInfo& fixUpInfo, const RollbackSource& rollbackSource, ReplicationCoordinator* replCoord) { // fetch all first so we needn't handle interruption in a fancy way @@ -344,73 +381,45 @@ void syncFixUp(OperationContext* txn, // namespace -> doc id -> doc map> goodVersions; - BSONObj newMinValid; - // fetch all the goodVersions of each document from current primary - DocID doc; unsigned long long numFetched = 0; - try { - for (set::iterator it = fixUpInfo.toRefetch.begin(); it != fixUpInfo.toRefetch.end(); - it++) { - doc = *it; - - verify(!doc._id.eoo()); + for (auto&& doc : fixUpInfo.docsToRefetch) { + invariant(!doc._id.eoo()); // This is checked when we insert to the set. - { - // TODO : slow. lots of round trips. - numFetched++; - BSONObj good = rollbackSource.findOne(NamespaceString(doc.ns), doc._id.wrap()); - totalSize += good.objsize(); - uassert(13410, "replSet too much data to roll back", totalSize < 300 * 1024 * 1024); - - // note good might be eoo, indicating we should delete it - goodVersions[doc.ns][doc] = good; + try { + // TODO : slow. lots of round trips. + numFetched++; + BSONObj good = rollbackSource.findOne(NamespaceString(doc.ns), doc._id.wrap()); + totalSize += good.objsize(); + if (totalSize >= 300 * 1024 * 1024) { + throw RSFatalException("replSet too much data to roll back"); } + + // Note good might be empty, indicating we should delete it. + goodVersions[doc.ns][doc] = good; + } catch (const DBException& ex) { + log() << "rollback couldn't re-get from ns: " << doc.ns << " _id: " << (doc._id) << ' ' + << numFetched << '/' << fixUpInfo.docsToRefetch.size() << ": " << (ex); + throw; } - newMinValid = rollbackSource.getLastOperation(); - if (newMinValid.isEmpty()) { - error() << "rollback error newMinValid empty?"; - return; - } - } catch (const DBException& e) { - LOG(1) << "rollback re-get objects: " << e.toString(); - error() << "rollback couldn't re-get ns:" << doc.ns << " _id:" << doc._id << ' ' - << numFetched << '/' << fixUpInfo.toRefetch.size(); - throw e; } log() << "rollback 3.5"; - if (fixUpInfo.rbid != rollbackSource.getRollbackId()) { - // Our source rolled back itself so the data we received isn't necessarily consistent. - warning() << "rollback rbid on source changed during rollback, " - << "cancelling this attempt"; - return; - } + checkRbidAndUpdateMinValid(txn, fixUpInfo.rbid, rollbackSource); // update them log() << "rollback 4 n:" << goodVersions.size(); - bool warn = false; - invariant(!fixUpInfo.commonPointOurDiskloc.isNull()); - // we have items we are writing that aren't from a point-in-time. thus best not to come - // online until we get to that point in freshness. - // TODO this is still wrong because we don't record that we are in rollback, and we can't really - // recover. - OpTime minValid = fassertStatusOK(28774, OpTime::parseFromOplogEntry(newMinValid)); - log() << "minvalid=" << minValid; - setAppliedThrough(txn, {}); // Use top of oplog. - setMinValid(txn, minValid); - // any full collection resyncs required? if (!fixUpInfo.collectionsToResyncData.empty() || !fixUpInfo.collectionsToResyncMetadata.empty()) { for (const string& ns : fixUpInfo.collectionsToResyncData) { log() << "rollback 4.1.1 coll resync " << ns; - fixUpInfo.indexesToDrop.erase(ns); - fixUpInfo.collectionsToResyncMetadata.erase(ns); + invariant(!fixUpInfo.indexesToDrop.count(ns)); + invariant(!fixUpInfo.collectionsToResyncMetadata.count(ns)); const NamespaceString nss(ns); @@ -421,7 +430,7 @@ void syncFixUp(OperationContext* txn, Database* db = dbHolder().openDb(txn, nss.db().toString()); invariant(db); WriteUnitOfWork wunit(txn); - db->dropCollection(txn, ns); + fassertStatusOK(40359, db->dropCollectionEvenIfSystem(txn, nss)); wunit.commit(); } @@ -443,9 +452,11 @@ void syncFixUp(OperationContext* txn, auto infoResult = rollbackSource.getCollectionInfo(nss); if (!infoResult.isOK()) { - // Collection dropped by "them" so we should drop it too. - log() << ns << " not found on remote host, dropping"; - fixUpInfo.toDrop.insert(ns); + // Collection dropped by "them" so we can't correctly change it here. If we get to + // the roll-forward phase, we will drop it then. If the drop is rolled-back upstream + // and we restart, we will be expected to still have the collection. + log() << ns << " not found on remote host, so not rolling back collmod operation." + " We will drop the collection soon."; continue; } @@ -496,56 +507,28 @@ void syncFixUp(OperationContext* txn, // we did more reading from primary, so check it again for a rollback (which would mess // us up), and make minValid newer. log() << "rollback 4.2"; - - string err; - try { - newMinValid = rollbackSource.getLastOperation(); - if (newMinValid.isEmpty()) { - err = "can't get minvalid from sync source"; - } else { - OpTime minValid = fassertStatusOK(28775, OpTime::parseFromOplogEntry(newMinValid)); - log() << "minvalid=" << minValid; - setMinValid(txn, minValid); - setAppliedThrough(txn, fixUpInfo.commonPoint); - } - } catch (const DBException& e) { - err = "can't get/set minvalid: "; - err += e.what(); - } - if (fixUpInfo.rbid != rollbackSource.getRollbackId()) { - // our source rolled back itself. so the data we received isn't necessarily - // consistent. however, we've now done writes. thus we have a problem. - err += "rbid at primary changed during resync/rollback"; - } - if (!err.empty()) { - severe() << "rolling back : " << err << ". A full resync will be necessary."; - // TODO: reset minvalid so that we are permanently in fatal state - // TODO: don't be fatal, but rather, get all the data first. - throw RSFatalException(); - } - log() << "rollback 4.3"; + checkRbidAndUpdateMinValid(txn, fixUpInfo.rbid, rollbackSource); } log() << "rollback 4.6"; - // drop collections to drop before doing individual fixups - that might make things faster - // below actually if there were subsequent inserts to rollback - for (set::iterator it = fixUpInfo.toDrop.begin(); it != fixUpInfo.toDrop.end(); it++) { + // drop collections to drop before doing individual fixups + for (set::iterator it = fixUpInfo.collectionsToDrop.begin(); + it != fixUpInfo.collectionsToDrop.end(); + it++) { log() << "rollback drop: " << *it; - fixUpInfo.indexesToDrop.erase(*it); + invariant(!fixUpInfo.indexesToDrop.count(*it)); ScopedTransaction transaction(txn, MODE_IX); const NamespaceString nss(*it); Lock::DBLock dbLock(txn->lockState(), nss.db(), MODE_X); Database* db = dbHolder().get(txn, nsToDatabaseSubstring(*it)); if (db) { - WriteUnitOfWork wunit(txn); - Helpers::RemoveSaver removeSaver("rollback", "", *it); // perform a collection scan and write all documents in the collection to disk std::unique_ptr exec(InternalPlanner::collectionScan( - txn, *it, db->getCollection(*it), PlanExecutor::YIELD_MANUAL)); + txn, *it, db->getCollection(*it), PlanExecutor::YIELD_AUTO)); BSONObj curObj; PlanExecutor::ExecState execState; while (PlanExecutor::ADVANCED == (execState = exec->getNext(&curObj, NULL))) { @@ -570,7 +553,8 @@ void syncFixUp(OperationContext* txn, throw RSFatalException(); } - db->dropCollection(txn, *it); + WriteUnitOfWork wunit(txn); + fassertStatusOK(40360, db->dropCollectionEvenIfSystem(txn, nss)); wunit.commit(); } } @@ -622,9 +606,8 @@ void syncFixUp(OperationContext* txn, // while rolling back createCollection operations. const auto& ns = nsAndGoodVersionsByDocID.first; unique_ptr removeSaver; - if (!fixUpInfo.toDrop.count(ns)) { - removeSaver.reset(new Helpers::RemoveSaver("rollback", "", ns)); - } + invariant(!fixUpInfo.collectionsToDrop.count(ns)); + removeSaver.reset(new Helpers::RemoveSaver("rollback", "", ns)); const auto& goodVersionsByDocID = nsAndGoodVersionsByDocID.second; for (const auto& idAndDoc : goodVersionsByDocID) { @@ -639,10 +622,7 @@ void syncFixUp(OperationContext* txn, BSONObj pattern = doc._id.wrap(); // { _id : ... } try { verify(doc.ns && *doc.ns); - if (fixUpInfo.collectionsToResyncData.count(doc.ns)) { - // We just synced this entire collection. - continue; - } + invariant(!fixUpInfo.collectionsToResyncData.count(doc.ns)); // TODO: Lots of overhead in context. This can be faster. const NamespaceString docNss(doc.ns); @@ -715,8 +695,13 @@ void syncFixUp(OperationContext* txn, } } } catch (const DBException& e) { - error() << "rolling back capped collection rec " << doc.ns << ' ' - << e.toString(); + // Replicated capped collections have many ways to become + // inconsistent. We rely on age-out to make these problems go away + // eventually. + warning() << "ignoring failure to roll back change to capped " + << "collection " << doc.ns << " with _id " + << idAndDoc.first._id.toString( + /*includeFieldName*/ false) << ": " << e; } } else { deleteObjects(txn, @@ -727,25 +712,6 @@ void syncFixUp(OperationContext* txn, true, // justone true); // god } - // did we just empty the collection? if so let's check if it even - // exists on the source. - if (collection->numRecords(txn) == 0) { - try { - NamespaceString nss(doc.ns); - auto infoResult = rollbackSource.getCollectionInfo(nss); - if (!infoResult.isOK()) { - // we should drop - WriteUnitOfWork wunit(txn); - ctx.db()->dropCollection(txn, doc.ns); - wunit.commit(); - } - } catch (const DBException& ex) { - // Failed to run listCollections command on sync source. - // This isn't *that* big a deal, but is bad. - warning() << "rollback error querying for existence of " << doc.ns - << " at the primary, ignoring: " << ex; - } - } } } else { // TODO faster... @@ -766,8 +732,8 @@ void syncFixUp(OperationContext* txn, } } catch (const DBException& e) { log() << "exception in rollback ns:" << doc.ns << ' ' << pattern.toString() << ' ' - << e.toString() << " ndeletes:" << deletes; - warn = true; + << e << " ndeletes:" << deletes; + throw; } } } @@ -795,113 +761,80 @@ void syncFixUp(OperationContext* txn, Status status = getGlobalAuthorizationManager()->initialize(txn); if (!status.isOK()) { - warning() << "Failed to reinitialize auth data after rollback: " << status; - warn = true; + severe() << "Failed to reinitialize auth data after rollback: " << status; + fassertFailedNoTrace(40366); } // Reload the lastAppliedOpTime and lastDurableOpTime value in the replcoord and the // lastAppliedHash value in bgsync to reflect our new last op. replCoord->resetLastOpTimesFromOplog(txn); - - // done - if (warn) - warning() << "issues during syncRollback, see log"; - else - log() << "rollback done"; + log() << "rollback done"; } Status _syncRollback(OperationContext* txn, const OplogInterface& localOplog, const RollbackSource& rollbackSource, - ReplicationCoordinator* replCoord, - const SleepSecondsFn& sleepSecondsFn) { + boost::optional requiredRBID, + ReplicationCoordinator* replCoord) { invariant(!txn->lockState()->isLocked()); - log() << "rollback 0"; - - /** by doing this, we will not service reads (return an error as we aren't in secondary - * state. that perhaps is moot because of the write lock above, but that write lock - * probably gets deferred or removed or yielded later anyway. - * - * also, this is better for status reporting - we know what is happening. - */ - { - Lock::GlobalWrite globalWrite(txn->lockState()); - if (!replCoord->setFollowerMode(MemberState::RS_ROLLBACK)) { - return Status(ErrorCodes::OperationFailed, - str::stream() << "Cannot transition from " - << replCoord->getMemberState().toString() << " to " - << MemberState(MemberState::RS_ROLLBACK).toString()); - } - } - FixUpInfo how; log() << "rollback 1"; how.rbid = rollbackSource.getRollbackId(); - { - log() << "rollback 2 FindCommonPoint"; - try { - auto processOperationForFixUp = - [&how](const BSONObj& operation) { return refetch(how, operation); }; - auto res = syncRollBackLocalOperations( - localOplog, rollbackSource.getOplog(), processOperationForFixUp); - if (!res.isOK()) { - const auto status = res.getStatus(); - switch (status.code()) { - case ErrorCodes::OplogStartMissing: - case ErrorCodes::UnrecoverableRollbackError: - sleepSecondsFn(Seconds(1)); - return status; - default: - throw RSFatalException(status.toString()); - } - } else { - how.commonPoint = res.getValue().first; - how.commonPointOurDiskloc = res.getValue().second; + if (requiredRBID) { + uassert(40362, + "Upstream node rolled back. Need to retry our rollback.", + how.rbid == *requiredRBID); + } + + log() << "rollback 2 FindCommonPoint"; + try { + auto processOperationForFixUp = [&how](const BSONObj& operation) { + return updateFixUpInfoFromLocalOplogEntry(how, operation); + }; + auto res = syncRollBackLocalOperations( + localOplog, rollbackSource.getOplog(), processOperationForFixUp); + if (!res.isOK()) { + const auto status = res.getStatus(); + switch (status.code()) { + case ErrorCodes::OplogStartMissing: + case ErrorCodes::UnrecoverableRollbackError: + return status; + default: + throw RSFatalException(status.toString()); } - } catch (const RSFatalException& e) { - error() << string(e.what()); - return Status(ErrorCodes::UnrecoverableRollbackError, - str::stream() - << "need to rollback, but unable to determine common point between" - " local and remote oplog: " << e.what(), - 18752); - } catch (const DBException& e) { - warning() << "rollback 2 exception " << e.toString() << "; sleeping 1 min"; - - sleepSecondsFn(Seconds(60)); - throw; } + + how.commonPoint = res.getValue().first; + how.commonPointOurDiskloc = res.getValue().second; + how.removeRedundantOperations(); + } catch (const RSFatalException& e) { + return Status(ErrorCodes::UnrecoverableRollbackError, + str::stream() + << "need to rollback, but unable to determine common point between" + " local and remote oplog: " << e.what(), + 18752); } + log() << "rollback common point is " << how.commonPoint; log() << "rollback 3 fixup"; - - replCoord->incrementRollbackID(); try { + ON_BLOCK_EXIT([&] { replCoord->incrementRollbackID(); }); syncFixUp(txn, how, rollbackSource, replCoord); } catch (const RSFatalException& e) { - error() << "exception during rollback: " << e.what(); - return Status(ErrorCodes::UnrecoverableRollbackError, - str::stream() << "exception during rollback: " << e.what(), - 18753); - } catch (...) { - replCoord->incrementRollbackID(); - - throw; + return Status(ErrorCodes::UnrecoverableRollbackError, e.what(), 18753); } - replCoord->incrementRollbackID(); if (MONGO_FAIL_POINT(rollbackHangBeforeFinish)) { // This log output is used in js tests so please leave it. log() << "rollback - rollbackHangBeforeFinish fail point " "enabled. Blocking until fail point is disabled."; while (MONGO_FAIL_POINT(rollbackHangBeforeFinish)) { + invariant(!inShutdown()); // It is an error to shutdown while enabled. mongo::sleepsecs(1); } } - // Success; leave "ROLLBACK" state intact until applier thread has reloaded the new minValid. - // Otherwise, the applier could transition the node to SECONDARY with an out-of-date minValid. return Status::OK(); } @@ -910,8 +843,8 @@ Status _syncRollback(OperationContext* txn, Status syncRollback(OperationContext* txn, const OplogInterface& localOplog, const RollbackSource& rollbackSource, - ReplicationCoordinator* replCoord, - const SleepSecondsFn& sleepSecondsFn) { + boost::optional requiredRBID, + ReplicationCoordinator* replCoord) { invariant(txn); invariant(replCoord); @@ -919,22 +852,11 @@ Status syncRollback(OperationContext* txn, DisableDocumentValidation validationDisabler(txn); txn->setReplicatedWrites(false); - Status status = _syncRollback(txn, localOplog, rollbackSource, replCoord, sleepSecondsFn); + Status status = _syncRollback(txn, localOplog, rollbackSource, requiredRBID, replCoord); log() << "rollback finished" << rsLog; return status; } -Status syncRollback(OperationContext* txn, - const OplogInterface& localOplog, - const RollbackSource& rollbackSource, - ReplicationCoordinator* replCoord) { - return syncRollback(txn, - localOplog, - rollbackSource, - replCoord, - [](Seconds seconds) { sleepsecs(durationCount(seconds)); }); -} - } // namespace repl } // namespace mongo diff --git a/src/mongo/db/repl/rs_rollback.h b/src/mongo/db/repl/rs_rollback.h index c88e9fd27c0..8ee7dd04367 100644 --- a/src/mongo/db/repl/rs_rollback.h +++ b/src/mongo/db/repl/rs_rollback.h @@ -28,12 +28,10 @@ #pragma once -#include "mongo/base/disallow_copying.h" #include "mongo/base/status.h" -#include "mongo/base/status_with.h" #include "mongo/db/jsobj.h" -#include "mongo/stdx/functional.h" -#include "mongo/util/time_support.h" +#include "mongo/db/record_id.h" +#include "mongo/db/repl/optime.h" namespace mongo { @@ -44,7 +42,6 @@ class OperationContext; namespace repl { class OplogInterface; -class OpTime; class ReplicationCoordinator; class RollbackSource; @@ -58,7 +55,7 @@ class RollbackSource; * - undo operations by fetching all documents affected, then replaying * the sync source's oplog until we reach the time in the oplog when we fetched the last * document. - * This function can throw std::exception on failures. + * This function can throw exceptions on failures. * This function runs a command on the sync source to detect if the sync source rolls back * while our rollback is in progress. * @@ -69,21 +66,77 @@ class RollbackSource; * supports fetching documents and copying collections. * @param replCoord Used to track the rollback ID and to change the follower state * - * Failures: Most failures are returned as a status but some failures throw an std::exception. + * If requiredRBID is supplied, we error if the upstream node has a different RBID (ie it rolled + * back) after fetching any information from it. + * + * Failures: If a Status with code UnrecoverableRollbackError is returned, the caller must exit + * fatally. All other errors should be considered recoverable regardless of whether reported as a + * status or exception. */ - -using SleepSecondsFn = stdx::function; - -Status syncRollback(OperationContext* txn, - const OplogInterface& localOplog, - const RollbackSource& rollbackSource, - ReplicationCoordinator* replCoord, - const SleepSecondsFn& sleepSecondsFn); - Status syncRollback(OperationContext* txn, const OplogInterface& localOplog, const RollbackSource& rollbackSource, + boost::optional requiredRBID, ReplicationCoordinator* replCoord); +/** + * This namespace contains internal details of the rollback system. It is only exposed in a header + * for unittesting. Nothing here should be used outside of rs_rollback.cpp or its unittest. + */ +namespace rollback_internal { + +struct DocID { + BSONObj ownedObj; + const char* ns; + BSONElement _id; + bool operator<(const DocID& other) const; + bool operator==(const DocID& other) const; + + static DocID minFor(const char* ns) { + auto obj = BSON("" << MINKEY); + return {obj, ns, obj.firstElement()}; + } + + static DocID maxFor(const char* ns) { + auto obj = BSON("" << MAXKEY); + return {obj, ns, obj.firstElement()}; + } +}; + +struct FixUpInfo { + // note this is a set -- if there are many $inc's on a single document we need to rollback, + // we only need to refetch it once. + std::set docsToRefetch; + + // Key is collection namespace. Value is name of index to drop. + std::multimap indexesToDrop; + + std::set collectionsToDrop; + std::set collectionsToResyncData; + std::set collectionsToResyncMetadata; + + OpTime commonPoint; + RecordId commonPointOurDiskloc; + + int rbid; // remote server's current rollback sequence # + + void removeAllDocsToRefetchFor(const std::string& collection); + void removeRedundantOperations(); +}; + +// Indicates that rollback cannot complete and the server must abort. +class RSFatalException : public std::exception { +public: + RSFatalException(std::string m = "replica set fatal exception") : msg(m) {} + virtual const char* what() const throw() { + return msg.c_str(); + } + +private: + std::string msg; +}; + +Status updateFixUpInfoFromLocalOplogEntry(FixUpInfo& fixUpInfo, const BSONObj& ourObj); +} // namespace rollback_internal } // namespace repl } // namespace mongo diff --git a/src/mongo/db/repl/rs_rollback_test.cpp b/src/mongo/db/repl/rs_rollback_test.cpp index 3f768d79f4b..3a324aff179 100644 --- a/src/mongo/db/repl/rs_rollback_test.cpp +++ b/src/mongo/db/repl/rs_rollback_test.cpp @@ -33,6 +33,7 @@ #include #include +#include "mongo/bson/json.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/database.h" #include "mongo/db/catalog/database_holder.h" @@ -63,6 +64,7 @@ namespace { using namespace mongo; using namespace mongo::repl; +using namespace mongo::repl::rollback_internal; const OplogInterfaceMock::Operations kEmptyMockOperations; @@ -173,8 +175,6 @@ void RSRollbackTest::tearDown() { setGlobalReplicationCoordinator(nullptr); } -void noSleep(Seconds seconds) {} - TEST_F(RSRollbackTest, InconsistentMinValid) { repl::setAppliedThrough(_txn.get(), OpTime(Timestamp(Seconds(0), 0), 0)); repl::setMinValid(_txn.get(), OpTime(Timestamp(Seconds(1), 0), 0)); @@ -182,36 +182,12 @@ TEST_F(RSRollbackTest, InconsistentMinValid) { OplogInterfaceMock(kEmptyMockOperations), RollbackSourceMock(std::unique_ptr( new OplogInterfaceMock(kEmptyMockOperations))), - _coordinator, - noSleep); + {}, + _coordinator); ASSERT_EQUALS(ErrorCodes::UnrecoverableRollbackError, status.code()); ASSERT_EQUALS(18752, status.location()); } -TEST_F(RSRollbackTest, SetFollowerModeFailed) { - class ReplicationCoordinatorSetFollowerModeMock : public ReplicationCoordinatorMock { - public: - ReplicationCoordinatorSetFollowerModeMock() - : ReplicationCoordinatorMock(createReplSettings()) {} - MemberState getMemberState() const override { - return MemberState::RS_DOWN; - } - bool setFollowerMode(const MemberState& newState) override { - return false; - } - }; - _coordinator = new ReplicationCoordinatorSetFollowerModeMock(); - setGlobalReplicationCoordinator(_coordinator); - - ASSERT_EQUALS(ErrorCodes::OperationFailed, - syncRollback(_txn.get(), - OplogInterfaceMock(kEmptyMockOperations), - RollbackSourceMock(std::unique_ptr( - new OplogInterfaceMock(kEmptyMockOperations))), - _coordinator, - noSleep).code()); -} - TEST_F(RSRollbackTest, OplogStartMissing) { OpTime ts(Timestamp(Seconds(1), 0), 0); auto operation = @@ -223,8 +199,8 @@ TEST_F(RSRollbackTest, OplogStartMissing) { RollbackSourceMock(std::unique_ptr(new OplogInterfaceMock({ operation, }))), - _coordinator, - noSleep).code()); + {}, + _coordinator).code()); } TEST_F(RSRollbackTest, NoRemoteOpLog) { @@ -235,8 +211,8 @@ TEST_F(RSRollbackTest, NoRemoteOpLog) { OplogInterfaceMock({operation}), RollbackSourceMock(std::unique_ptr( new OplogInterfaceMock(kEmptyMockOperations))), - _coordinator, - noSleep); + {}, + _coordinator); ASSERT_EQUALS(ErrorCodes::UnrecoverableRollbackError, status.code()); ASSERT_EQUALS(18752, status.location()); } @@ -257,12 +233,36 @@ TEST_F(RSRollbackTest, RemoteGetRollbackIdThrows) { OplogInterfaceMock({operation}), RollbackSourceLocal(std::unique_ptr( new OplogInterfaceMock(kEmptyMockOperations))), - _coordinator, - noSleep), + {}, + _coordinator), UserException, ErrorCodes::UnknownError); } +TEST_F(RSRollbackTest, RemoteGetRollbackIdDiffersFromRequiredRBID) { + OpTime ts(Timestamp(Seconds(1), 0), 0); + auto operation = + std::make_pair(BSON("ts" << ts.getTimestamp() << "h" << ts.getTerm()), RecordId()); + + class RollbackSourceLocal : public RollbackSourceMock { + public: + RollbackSourceLocal(std::unique_ptr oplog) + : RollbackSourceMock(std::move(oplog)) {} + int getRollbackId() const override { + return 2; + } + }; + + ASSERT_THROWS_CODE(syncRollback(_txn.get(), + OplogInterfaceMock({operation}), + RollbackSourceLocal(std::unique_ptr( + new OplogInterfaceMock(kEmptyMockOperations))), + {1}, + _coordinator), + UserException, + ErrorCodes::Error(40362)); +} + TEST_F(RSRollbackTest, BothOplogsAtCommonPoint) { createOplog(_txn.get()); OpTime ts(Timestamp(Seconds(1), 0), 1); @@ -274,8 +274,8 @@ TEST_F(RSRollbackTest, BothOplogsAtCommonPoint) { RollbackSourceMock(std::unique_ptr(new OplogInterfaceMock({ operation, }))), - _coordinator, - noSleep)); + {}, + _coordinator)); } /** @@ -341,8 +341,8 @@ int _testRollbackDelete(OperationContext* txn, ASSERT_OK(syncRollback(txn, OplogInterfaceMock({deleteOperation, commonOperation}), rollbackSource, - coordinator, - noSleep)); + {}, + coordinator)); ASSERT_TRUE(rollbackSource.called); Lock::DBLock dbLock(txn->lockState(), "test", MODE_S); @@ -415,8 +415,8 @@ TEST_F(RSRollbackTest, RollbackInsertDocumentWithNoId) { auto status = syncRollback(_txn.get(), OplogInterfaceMock({insertDocumentOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep); + {}, + _coordinator); stopCapturingLogMessages(); ASSERT_EQUALS(ErrorCodes::UnrecoverableRollbackError, status.code()); ASSERT_EQUALS(18752, status.location()); @@ -474,8 +474,8 @@ TEST_F(RSRollbackTest, RollbackCreateIndexCommand) { _txn.get(), OplogInterfaceMock({insertDocumentOperation, insertDocumentOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep)); + {}, + _coordinator)); stopCapturingLogMessages(); ASSERT_EQUALS(1, countLogLinesContaining("rollback drop index: collection: test.t. index: a_1")); @@ -531,8 +531,8 @@ TEST_F(RSRollbackTest, RollbackCreateIndexCommandIndexNotInCatalog) { ASSERT_OK(syncRollback(_txn.get(), OplogInterfaceMock({insertDocumentOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep)); + {}, + _coordinator)); stopCapturingLogMessages(); ASSERT_EQUALS(1, countLogLinesContaining("rollback drop index: collection: test.t. index: a_1")); @@ -578,8 +578,8 @@ TEST_F(RSRollbackTest, RollbackCreateIndexCommandMissingNamespace) { auto status = syncRollback(_txn.get(), OplogInterfaceMock({insertDocumentOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep); + {}, + _coordinator); stopCapturingLogMessages(); ASSERT_EQUALS(ErrorCodes::UnrecoverableRollbackError, status.code()); ASSERT_EQUALS(18752, status.location()); @@ -622,8 +622,8 @@ TEST_F(RSRollbackTest, RollbackCreateIndexCommandInvalidNamespace) { auto status = syncRollback(_txn.get(), OplogInterfaceMock({insertDocumentOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep); + {}, + _coordinator); stopCapturingLogMessages(); ASSERT_EQUALS(ErrorCodes::UnrecoverableRollbackError, status.code()); ASSERT_EQUALS(18752, status.location()); @@ -665,8 +665,8 @@ TEST_F(RSRollbackTest, RollbackCreateIndexCommandMissingIndexName) { auto status = syncRollback(_txn.get(), OplogInterfaceMock({insertDocumentOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep); + {}, + _coordinator); stopCapturingLogMessages(); ASSERT_EQUALS(ErrorCodes::UnrecoverableRollbackError, status.code()); ASSERT_EQUALS(18752, status.location()); @@ -700,8 +700,8 @@ TEST_F(RSRollbackTest, RollbackUnknownCommand) { RollbackSourceMock(std::unique_ptr(new OplogInterfaceMock({ commonOperation, }))), - _coordinator, - noSleep); + {}, + _coordinator); ASSERT_EQUALS(ErrorCodes::UnrecoverableRollbackError, status.code()); ASSERT_EQUALS(18751, status.location()); } @@ -735,11 +735,51 @@ TEST_F(RSRollbackTest, RollbackDropCollectionCommand) { ASSERT_OK(syncRollback(_txn.get(), OplogInterfaceMock({dropCollectionOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep)); + {}, + _coordinator)); ASSERT_TRUE(rollbackSource.called); } +TEST_F(RSRollbackTest, RollbackDropCollectionCommandFailsIfRBIDChangesWhileSyncingCollection) { + createOplog(_txn.get()); + auto commonOperation = + std::make_pair(BSON("ts" << Timestamp(Seconds(1), 0) << "h" << 1LL), RecordId(1)); + auto dropCollectionOperation = + std::make_pair(BSON("ts" << Timestamp(Seconds(2), 0) << "h" << 1LL << "op" + << "c" + << "ns" + << "test.t" + << "o" << BSON("drop" + << "t")), + RecordId(2)); + class RollbackSourceLocal : public RollbackSourceMock { + public: + RollbackSourceLocal(std::unique_ptr oplog) + : RollbackSourceMock(std::move(oplog)), copyCollectionCalled(false) {} + int getRollbackId() const override { + return copyCollectionCalled ? 1 : 0; + } + void copyCollectionFromRemote(OperationContext* txn, + const NamespaceString& nss) const override { + copyCollectionCalled = true; + } + mutable bool copyCollectionCalled; + }; + RollbackSourceLocal rollbackSource(std::unique_ptr(new OplogInterfaceMock({ + commonOperation, + }))); + + _createCollection(_txn.get(), "test.t", CollectionOptions()); + ASSERT_THROWS_CODE(syncRollback(_txn.get(), + OplogInterfaceMock({dropCollectionOperation, commonOperation}), + rollbackSource, + 0, + _coordinator), + DBException, + 40365); + ASSERT(rollbackSource.copyCollectionCalled); +} + BSONObj makeApplyOpsOplogEntry(Timestamp ts, std::initializer_list ops) { BSONObjBuilder entry; entry << "ts" << ts << "h" << 1LL << "op" @@ -848,8 +888,8 @@ TEST_F(RSRollbackTest, RollbackApplyOpsCommand) { ASSERT_OK(syncRollback(_txn.get(), OplogInterfaceMock({applyOpsOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep)); + {}, + _coordinator)); ASSERT_EQUALS(4U, rollbackSource.searchedIds.size()); ASSERT_EQUALS(1U, rollbackSource.searchedIds.count(1)); ASSERT_EQUALS(1U, rollbackSource.searchedIds.count(2)); @@ -887,8 +927,8 @@ TEST_F(RSRollbackTest, RollbackCreateCollectionCommand) { ASSERT_OK(syncRollback(_txn.get(), OplogInterfaceMock({createCollectionOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep)); + {}, + _coordinator)); { Lock::DBLock dbLock(_txn->lockState(), "test", MODE_S); auto db = dbHolder().get(_txn.get(), "test"); @@ -928,8 +968,8 @@ TEST_F(RSRollbackTest, RollbackCollectionModificationCommand) { ASSERT_OK(syncRollback(_txn.get(), OplogInterfaceMock({collectionModificationOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep)); + {}, + _coordinator)); stopCapturingLogMessages(); ASSERT_TRUE(rollbackSource.called); for (const auto& message : getCapturedLogMessages()) { @@ -967,10 +1007,89 @@ TEST_F(RSRollbackTest, RollbackCollectionModificationCommandInvalidCollectionOpt syncRollback(_txn.get(), OplogInterfaceMock({collectionModificationOperation, commonOperation}), rollbackSource, - _coordinator, - noSleep); + {}, + _coordinator); ASSERT_EQUALS(ErrorCodes::UnrecoverableRollbackError, status.code()); ASSERT_EQUALS(18753, status.location()); } +TEST(RSRollbackTest, LocalEntryWithoutNsIsFatal) { + const auto validOplogEntry = fromjson("{op: 'i', ns: 'test.t', o: {_id:1, a: 1}}"); + FixUpInfo fui; + ASSERT_OK(updateFixUpInfoFromLocalOplogEntry(fui, validOplogEntry)); + ASSERT_THROWS(updateFixUpInfoFromLocalOplogEntry(fui, validOplogEntry.removeField("ns")), + RSFatalException); +} + +TEST(RSRollbackTest, LocalEntryWithoutOIsFatal) { + const auto validOplogEntry = fromjson("{op: 'i', ns: 'test.t', o: {_id:1, a: 1}}"); + FixUpInfo fui; + ASSERT_OK(updateFixUpInfoFromLocalOplogEntry(fui, validOplogEntry)); + ASSERT_THROWS(updateFixUpInfoFromLocalOplogEntry(fui, validOplogEntry.removeField("o")), + RSFatalException); +} + +TEST(RSRollbackTest, LocalEntryWithoutO2IsFatal) { + const auto validOplogEntry = + fromjson("{op: 'u', ns: 'test.t', o2: {_id: 1}, o: {_id:1, a: 1}}"); + FixUpInfo fui; + ASSERT_OK(updateFixUpInfoFromLocalOplogEntry(fui, validOplogEntry)); + ASSERT_THROWS(updateFixUpInfoFromLocalOplogEntry(fui, validOplogEntry.removeField("o2")), + RSFatalException); +} + +// The testcases used here are trying to detect off-by-one errors in +// FixUpInfo::removeAllDocsToRefectchFor. +TEST(FixUpInfoTest, RemoveAllDocsToRefetchForWorks) { + const auto normalHolder = BSON("" << OID::gen()); + const auto normalKey = normalHolder.firstElement(); + + // Can't use ASSERT_EQ with this since it isn't ostream-able. Failures will at least give you + // the size. If that isn't enough, use GDB. + using DocSet = std::set; + + FixUpInfo fui; + fui.docsToRefetch = { + DocID::minFor("a"), + DocID{{}, "a", normalKey}, + DocID::maxFor("a"), + + DocID::minFor("b"), + DocID{{}, "b", normalKey}, + DocID::maxFor("b"), + + DocID::minFor("c"), + DocID{{}, "c", normalKey}, + DocID::maxFor("c"), + }; + + // Remove from the middle. + fui.removeAllDocsToRefetchFor("b"); + ASSERT((fui.docsToRefetch == + DocSet{ + DocID::minFor("a"), + DocID{{}, "a", normalKey}, + DocID::maxFor("a"), + + DocID::minFor("c"), + DocID{{}, "c", normalKey}, + DocID::maxFor("c"), + })) + << "remaining docs: " << fui.docsToRefetch.size(); + + // Remove from the end. + fui.removeAllDocsToRefetchFor("c"); + ASSERT((fui.docsToRefetch == + DocSet{ + DocID::minFor("a"), // This comment helps clang-format. + DocID{{}, "a", normalKey}, + DocID::maxFor("a"), + })) + << "remaining docs: " << fui.docsToRefetch.size(); + + // Everything else. + fui.removeAllDocsToRefetchFor("a"); + ASSERT((fui.docsToRefetch == DocSet{})) << "remaining docs: " << fui.docsToRefetch.size(); +} + } // namespace diff --git a/src/mongo/db/repl/sync_source_feedback.cpp b/src/mongo/db/repl/sync_source_feedback.cpp index 43b84f45e38..beb73bebbdd 100644 --- a/src/mongo/db/repl/sync_source_feedback.cpp +++ b/src/mongo/db/repl/sync_source_feedback.cpp @@ -113,17 +113,14 @@ Status SyncSourceFeedback::updateUpstream(OperationContext* txn, bool oldStyle) return Status::OK(); } BSONObjBuilder cmd; - { - stdx::unique_lock lock(_mtx); - // The command could not be created, likely because this node was removed from the set. - if (!oldStyle) { - if (!replCoord->prepareReplSetUpdatePositionCommand(&cmd)) { - return Status::OK(); - } - } else { - if (!replCoord->prepareOldReplSetUpdatePositionCommand(&cmd)) { - return Status::OK(); - } + // The command could not be created, likely because this node was removed from the set. + if (!oldStyle) { + if (!replCoord->prepareReplSetUpdatePositionCommand(&cmd)) { + return Status::OK(); + } + } else { + if (!replCoord->prepareOldReplSetUpdatePositionCommand(&cmd)) { + return Status::OK(); } } BSONObj res; diff --git a/src/mongo/db/repl/sync_tail.cpp b/src/mongo/db/repl/sync_tail.cpp index a336c8de68d..6c4d88132eb 100644 --- a/src/mongo/db/repl/sync_tail.cpp +++ b/src/mongo/db/repl/sync_tail.cpp @@ -255,7 +255,7 @@ bool SyncTail::peek(BSONObj* op) { // static Status SyncTail::syncApply(OperationContext* txn, const BSONObj& op, - bool convertUpdateToUpsert, + bool inSteadyStateReplication, ApplyOperationInLockFn applyOperationInLock, ApplyCommandInLockFn applyCommandInLock, IncrementOpsAppliedStatsFn incrementOpsAppliedStats) { @@ -291,7 +291,7 @@ Status SyncTail::syncApply(OperationContext* txn, Lock::GlobalWrite globalWriteLock(txn->lockState()); // special case apply for commands to avoid implicit database creation - Status status = applyCommandInLock(txn, op); + Status status = applyCommandInLock(txn, op, inSteadyStateReplication); incrementOpsAppliedStats(); return status; } @@ -304,7 +304,7 @@ Status SyncTail::syncApply(OperationContext* txn, txn->setReplicatedWrites(false); DisableDocumentValidation validationDisabler(txn); - Status status = applyOperationInLock(txn, db, op, convertUpdateToUpsert); + Status status = applyOperationInLock(txn, db, op, inSteadyStateReplication); if (!status.isOK() && status.code() == ErrorCodes::WriteConflict) { throw WriteConflictException(); } @@ -333,20 +333,24 @@ Status SyncTail::syncApply(OperationContext* txn, auto resetLocks = [&](LockMode mode) { collectionLock.reset(); + // Warning: We must reset the pointer to nullptr first, in order to ensure that we + // drop the DB lock before acquiring + // the upgraded one. + dbLock.reset(); dbLock.reset(new Lock::DBLock(txn->lockState(), dbName, mode)); collectionLock.reset(new Lock::CollectionLock(txn->lockState(), ns, mode)); }; resetLocks(MODE_IX); if (!dbHolder().get(txn, dbName)) { - // need to create database, try again + // Need to create database, so reset lock to stronger mode. resetLocks(MODE_X); ctx.reset(new OldClientContext(txn, ns)); } else { ctx.reset(new OldClientContext(txn, ns)); if (!ctx->db()->getCollection(ns)) { - // uh, oh, we need to create collection - // try again + // Need to implicitly create collection. This occurs for 'u' opTypes, + // but not for 'i' nor 'd'. ctx.reset(); resetLocks(MODE_X); ctx.reset(new OldClientContext(txn, ns)); @@ -365,10 +369,12 @@ Status SyncTail::syncApply(OperationContext* txn, return Status(ErrorCodes::BadValue, ss); } -Status SyncTail::syncApply(OperationContext* txn, const BSONObj& op, bool convertUpdateToUpsert) { +Status SyncTail::syncApply(OperationContext* txn, + const BSONObj& op, + bool inSteadyStateReplication) { return syncApply(txn, op, - convertUpdateToUpsert, + inSteadyStateReplication, applyOperation_inlock, applyCommand_inlock, stdx::bind(&Counter64::increment, &opsAppliedStats, 1ULL)); @@ -985,11 +991,12 @@ void multiSyncApply(const std::vector& ops, SyncTail* st) { // allow us to get through the magic barrier txn.lockState()->setIsBatchWriter(true); - bool convertUpdatesToUpserts = true; + // This function is only called in steady state replication. + bool inSteadyStateReplication = true; for (std::vector::const_iterator it = ops.begin(); it != ops.end(); ++it) { try { - const Status s = SyncTail::syncApply(&txn, *it, convertUpdatesToUpserts); + const Status s = SyncTail::syncApply(&txn, *it, inSteadyStateReplication); if (!s.isOK()) { severe() << "Error applying operation (" << it->toString() << "): " << s; fassertFailedNoTrace(16359); @@ -1012,23 +1019,39 @@ void multiInitialSyncApply(const std::vector& ops, SyncTail* st) { initializeWriterThread(); OperationContextImpl txn; - txn.setReplicatedWrites(false); - DisableDocumentValidation validationDisabler(&txn); + Status status = multiInitialSyncApply_noAbort(&txn, ops, st); + fassertNoTrace(15915, status); +} + +Status multiInitialSyncApply_noAbort(OperationContext* txn, + const std::vector& ops, + SyncTail* st) { + txn->setReplicatedWrites(false); + DisableDocumentValidation validationDisabler(txn); // allow us to get through the magic barrier - txn.lockState()->setIsBatchWriter(true); + txn->lockState()->setIsBatchWriter(true); - bool convertUpdatesToUpserts = false; + // This function is only called in initial sync, as its name suggests. + bool inSteadyStateReplication = false; for (std::vector::const_iterator it = ops.begin(); it != ops.end(); ++it) { try { - const Status s = SyncTail::syncApply(&txn, *it, convertUpdatesToUpserts); + const Status s = SyncTail::syncApply(txn, *it, inSteadyStateReplication); if (!s.isOK()) { - if (st->shouldRetry(&txn, *it)) { - const Status s2 = SyncTail::syncApply(&txn, *it, convertUpdatesToUpserts); + // Don't retry on commands. + SyncTail::OplogEntry entry(*it); + if (entry.opType[0] == 'c') { + error() << "Error applying command (" << it->toString() << "): " << s; + return s; + } + + // We might need to fetch the missing docs from the sync source. + if (st->shouldRetry(txn, *it)) { + const Status s2 = SyncTail::syncApply(txn, *it, inSteadyStateReplication); if (!s2.isOK()) { severe() << "Error applying operation (" << it->toString() << "): " << s2; - fassertFailedNoTrace(15915); + return s2; } } @@ -1037,16 +1060,24 @@ void multiInitialSyncApply(const std::vector& ops, SyncTail* st) { // subsequently got deleted and no longer exists on the Sync Target at all } } catch (const DBException& e) { + // SERVER-24927 If we have a NamespaceNotFound exception, then this document will be + // dropped before initial sync ends anyways and we should ignore it. + SyncTail::OplogEntry entry(*it); + if (e.getCode() == ErrorCodes::NamespaceNotFound && + isCrudOpType(entry.opType.rawData())) { + continue; + } + severe() << "writer worker caught exception: " << causedBy(e) << " on: " << it->toString(); if (inShutdown()) { - return; + return Status::OK(); } - - fassertFailedNoTrace(16361); + return e.toStatus(); } } + return Status::OK(); } } // namespace repl diff --git a/src/mongo/db/repl/sync_tail.h b/src/mongo/db/repl/sync_tail.h index 313a16bf48b..7493698b14c 100644 --- a/src/mongo/db/repl/sync_tail.h +++ b/src/mongo/db/repl/sync_tail.h @@ -59,7 +59,7 @@ public: /** * Type of function that takes a non-command op and applies it locally. * Used for applying from an oplog. - * Last boolean argument 'convertUpdateToUpsert' converts some updates to upserts for + * Last boolean argument 'inSteadyStateReplication' converts some updates to upserts for * idempotency reasons. * Returns failure status if the op was an update that could not be applied. */ @@ -71,7 +71,7 @@ public: * Used for applying from an oplog. * Returns failure status if the op that could not be applied. */ - using ApplyCommandInLockFn = stdx::function; + using ApplyCommandInLockFn = stdx::function; /** * Type of function to increment "repl.apply.ops" server status metric. @@ -88,12 +88,12 @@ public: */ static Status syncApply(OperationContext* txn, const BSONObj& o, - bool convertUpdateToUpsert, + bool inSteadyStateReplication, ApplyOperationInLockFn applyOperationInLock, ApplyCommandInLockFn applyCommandInLock, IncrementOpsAppliedStatsFn incrementOpsAppliedStats); - static Status syncApply(OperationContext* txn, const BSONObj& o, bool convertUpdateToUpsert); + static Status syncApply(OperationContext* txn, const BSONObj& o, bool inSteadyStateReplication); void oplogApplication(StorageInterface* storageInterface); bool peek(BSONObj* obj); @@ -191,9 +191,9 @@ public: static int replWriterThreadCount; protected: - // Cap the batches using the limit on journal commits. - // This works out to be 100 MB (64 bit) or 50 MB (32 bit) - static const unsigned int replBatchLimitBytes = dur::UncommittedBytesLimit; + // Cap the batches to 50 MB for 32-bit systems and 100 MB for 64-bit systems. + static const unsigned int replBatchLimitBytes = + (sizeof(void*) == 4) ? 50 * 1024 * 1024 : 100 * 1024 * 1024; static const unsigned int replBatchLimitOperations = 5000; // Apply a batch of operations, using multiple threads. @@ -220,6 +220,9 @@ private: // These free functions are used by the thread pool workers to write ops to the db. void multiSyncApply(const std::vector& ops, SyncTail* st); void multiInitialSyncApply(const std::vector& ops, SyncTail* st); +Status multiInitialSyncApply_noAbort(OperationContext* txn, + const std::vector& ops, + SyncTail* st); } // namespace repl } // namespace mongo diff --git a/src/mongo/db/repl/sync_tail_test.cpp b/src/mongo/db/repl/sync_tail_test.cpp index 29f81337e05..0d0250e4814 100644 --- a/src/mongo/db/repl/sync_tail_test.cpp +++ b/src/mongo/db/repl/sync_tail_test.cpp @@ -30,23 +30,32 @@ #include +#include "mongo/db/catalog/collection_options.h" +#include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/database.h" #include "mongo/db/catalog/database_holder.h" #include "mongo/db/catalog/document_validation.h" +#include "mongo/db/client.h" #include "mongo/db/concurrency/d_concurrency.h" #include "mongo/db/concurrency/write_conflict_exception.h" -#include "mongo/db/client.h" #include "mongo/db/curop.h" +#include "mongo/db/db_raii.h" #include "mongo/db/jsobj.h" +#include "mongo/db/json.h" +#include "mongo/db/query/internal_plans.h" #include "mongo/db/repl/bgsync.h" #include "mongo/db/repl/operation_context_repl_mock.h" +#include "mongo/db/repl/oplog_interface_local.h" #include "mongo/db/repl/replication_coordinator_global.h" #include "mongo/db/repl/replication_coordinator_mock.h" #include "mongo/db/repl/sync_tail.h" #include "mongo/db/service_context.h" #include "mongo/db/storage/storage_options.h" -#include "mongo/unittest/unittest.h" #include "mongo/unittest/temp_dir.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/md5.hpp" +#include "mongo/util/scopeguard.h" + namespace { @@ -88,7 +97,7 @@ void SyncTailTest::setUp() { // go away after the global storage engine is initialized. unittest::TempDir tempDir("sync_tail_test"); mongo::storageGlobalParams.dbpath = tempDir.path(); - mongo::storageGlobalParams.engine = "devnull"; + mongo::storageGlobalParams.engine = "ephemeralForTest"; mongo::storageGlobalParams.engineSetByUser = true; serviceContext->initializeGlobalStorageEngine(); } @@ -101,16 +110,29 @@ void SyncTailTest::setUp() { _txn.reset(new OperationContextReplMock(&cc(), 0)); _opsApplied = 0; _applyOp = - [](OperationContext* txn, Database* db, const BSONObj& op, bool convertUpdateToUpsert) { + [](OperationContext* txn, Database* db, const BSONObj& op, bool inSteadyStateReplication) { return Status::OK(); }; - _applyCmd = [](OperationContext* txn, const BSONObj& op) { return Status::OK(); }; + _applyCmd = [](OperationContext* txn, const BSONObj& op, bool) { return Status::OK(); }; _incOps = [this]() { _opsApplied++; }; } void SyncTailTest::tearDown() { + ON_BLOCK_EXIT([&] { Client::destroy(); }); + + dropAllDatabasesExceptLocal(_txn.get()); { + ScopedTransaction transaction(_txn.get(), MODE_X); Lock::GlobalWrite globalLock(_txn->lockState()); + AutoGetDb autoDBLocal(_txn.get(), "local", MODE_X); + auto localDB = autoDBLocal.getDb(); + if (localDB) { + MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { + // Do not wrap in a WriteUnitOfWork until SERVER-17103 is addressed. + dropDatabase(_txn.get(), localDB); + } + MONGO_WRITE_CONFLICT_RETRY_LOOP_END(_txn.get(), "_dropAllDBs", "local"); + } BSONObjBuilder unused; invariant(mongo::dbHolder().closeAll(_txn.get(), unused, false)); } @@ -118,6 +140,11 @@ void SyncTailTest::tearDown() { setGlobalReplicationCoordinator(nullptr); } +Status failedApplyCommand(OperationContext* txn, const BSONObj& theOperation, bool) { + FAIL("applyCommand unexpectedly invoked."); + return Status::OK(); +} + TEST_F(SyncTailTest, Peek) { BackgroundSyncMock bgsync; SyncTail syncTail(&bgsync, [](const std::vector& ops, SyncTail* st) {}); @@ -159,7 +186,7 @@ TEST_F(SyncTailTest, SyncApplyNoOp) { SyncTail::ApplyOperationInLockFn applyOp = [&](OperationContext* txn, Database* db, const BSONObj& theOperation, - bool convertUpdateToUpsert) { + bool inSteadyStateReplication) { applyOpCalled = true; ASSERT_TRUE(txn); ASSERT_TRUE(txn->lockState()->isDbLockedForMode("test", MODE_X)); @@ -167,17 +194,12 @@ TEST_F(SyncTailTest, SyncApplyNoOp) { ASSERT_TRUE(documentValidationDisabled(txn)); ASSERT_TRUE(db); ASSERT_EQUALS(op, theOperation); - ASSERT_FALSE(convertUpdateToUpsert); + ASSERT_FALSE(inSteadyStateReplication); return Status::OK(); }; - SyncTail::ApplyCommandInLockFn applyCmd = - [&](OperationContext* txn, const BSONObj& theOperation) { - FAIL("applyCommand unexpectedly invoked."); - return Status::OK(); - }; ASSERT_TRUE(_txn->writesAreReplicated()); ASSERT_FALSE(documentValidationDisabled(_txn.get())); - ASSERT_OK(SyncTail::syncApply(_txn.get(), op, false, applyOp, applyCmd, _incOps)); + ASSERT_OK(SyncTail::syncApply(_txn.get(), op, false, applyOp, failedApplyCommand, _incOps)); ASSERT_TRUE(applyOpCalled); ASSERT_EQUALS(1U, _opsApplied); } @@ -191,19 +213,14 @@ TEST_F(SyncTailTest, SyncApplyNoOpApplyOpThrowsException) { SyncTail::ApplyOperationInLockFn applyOp = [&](OperationContext* txn, Database* db, const BSONObj& theOperation, - bool convertUpdateToUpsert) { + bool inSteadyStateReplication) { applyOpCalled++; if (applyOpCalled < 5) { throw WriteConflictException(); } return Status::OK(); }; - SyncTail::ApplyCommandInLockFn applyCmd = - [&](OperationContext* txn, const BSONObj& theOperation) { - FAIL("applyCommand unexpectedly invoked."); - return Status::OK(); - }; - ASSERT_OK(SyncTail::syncApply(_txn.get(), op, false, applyOp, applyCmd, _incOps)); + ASSERT_OK(SyncTail::syncApply(_txn.get(), op, false, applyOp, failedApplyCommand, _incOps)); ASSERT_EQUALS(5, applyOpCalled); ASSERT_EQUALS(1U, _opsApplied); } @@ -217,7 +234,7 @@ void SyncTailTest::_testSyncApplyInsertDocument(LockMode expectedMode) { SyncTail::ApplyOperationInLockFn applyOp = [&](OperationContext* txn, Database* db, const BSONObj& theOperation, - bool convertUpdateToUpsert) { + bool inSteadyStateReplication) { applyOpCalled = true; ASSERT_TRUE(txn); ASSERT_TRUE(txn->lockState()->isDbLockedForMode("test", expectedMode)); @@ -226,17 +243,12 @@ void SyncTailTest::_testSyncApplyInsertDocument(LockMode expectedMode) { ASSERT_TRUE(documentValidationDisabled(txn)); ASSERT_TRUE(db); ASSERT_EQUALS(op, theOperation); - ASSERT_TRUE(convertUpdateToUpsert); + ASSERT_TRUE(inSteadyStateReplication); return Status::OK(); }; - SyncTail::ApplyCommandInLockFn applyCmd = - [&](OperationContext* txn, const BSONObj& theOperation) { - FAIL("applyCommand unexpectedly invoked."); - return Status::OK(); - }; ASSERT_TRUE(_txn->writesAreReplicated()); ASSERT_FALSE(documentValidationDisabled(_txn.get())); - ASSERT_OK(SyncTail::syncApply(_txn.get(), op, true, applyOp, applyCmd, _incOps)); + ASSERT_OK(SyncTail::syncApply(_txn.get(), op, true, applyOp, failedApplyCommand, _incOps)); ASSERT_TRUE(applyOpCalled); ASSERT_EQUALS(1U, _opsApplied); } @@ -259,11 +271,13 @@ TEST_F(SyncTailTest, SyncApplyInsertDocumentCollectionMissing) { TEST_F(SyncTailTest, SyncApplyInsertDocumentCollectionExists) { { Lock::GlobalWrite globalLock(_txn->lockState()); + WriteUnitOfWork wunit(_txn.get()); bool justCreated = false; Database* db = dbHolder().openDb(_txn.get(), "test", &justCreated); ASSERT_TRUE(db); ASSERT_TRUE(justCreated); Collection* collection = db->createCollection(_txn.get(), "test.t"); + wunit.commit(); ASSERT_TRUE(collection); } _testSyncApplyInsertDocument(MODE_IX); @@ -278,7 +292,7 @@ TEST_F(SyncTailTest, SyncApplyIndexBuild) { SyncTail::ApplyOperationInLockFn applyOp = [&](OperationContext* txn, Database* db, const BSONObj& theOperation, - bool convertUpdateToUpsert) { + bool inSteadyStateReplication) { applyOpCalled = true; ASSERT_TRUE(txn); ASSERT_TRUE(txn->lockState()->isDbLockedForMode("test", MODE_X)); @@ -286,17 +300,12 @@ TEST_F(SyncTailTest, SyncApplyIndexBuild) { ASSERT_TRUE(documentValidationDisabled(txn)); ASSERT_TRUE(db); ASSERT_EQUALS(op, theOperation); - ASSERT_FALSE(convertUpdateToUpsert); + ASSERT_FALSE(inSteadyStateReplication); return Status::OK(); }; - SyncTail::ApplyCommandInLockFn applyCmd = - [&](OperationContext* txn, const BSONObj& theOperation) { - FAIL("applyCommand unexpectedly invoked."); - return Status::OK(); - }; ASSERT_TRUE(_txn->writesAreReplicated()); ASSERT_FALSE(documentValidationDisabled(_txn.get())); - ASSERT_OK(SyncTail::syncApply(_txn.get(), op, false, applyOp, applyCmd, _incOps)); + ASSERT_OK(SyncTail::syncApply(_txn.get(), op, false, applyOp, failedApplyCommand, _incOps)); ASSERT_TRUE(applyOpCalled); ASSERT_EQUALS(1U, _opsApplied); } @@ -310,12 +319,12 @@ TEST_F(SyncTailTest, SyncApplyCommand) { SyncTail::ApplyOperationInLockFn applyOp = [&](OperationContext* txn, Database* db, const BSONObj& theOperation, - bool convertUpdateToUpsert) { + bool inSteadyStateReplication) { FAIL("applyOperation unexpectedly invoked."); return Status::OK(); }; SyncTail::ApplyCommandInLockFn applyCmd = - [&](OperationContext* txn, const BSONObj& theOperation) { + [&](OperationContext* txn, const BSONObj& theOperation, bool) { applyCmdCalled = true; ASSERT_TRUE(txn); ASSERT_TRUE(txn->lockState()->isW()); @@ -340,12 +349,12 @@ TEST_F(SyncTailTest, SyncApplyCommandThrowsException) { SyncTail::ApplyOperationInLockFn applyOp = [&](OperationContext* txn, Database* db, const BSONObj& theOperation, - bool convertUpdateToUpsert) { + bool inSteadyStateReplication) { FAIL("applyOperation unexpectedly invoked."); return Status::OK(); }; SyncTail::ApplyCommandInLockFn applyCmd = - [&](OperationContext* txn, const BSONObj& theOperation) { + [&](OperationContext* txn, const BSONObj& theOperation, bool) { applyCmdCalled++; if (applyCmdCalled < 5) { throw WriteConflictException(); @@ -357,4 +366,199 @@ TEST_F(SyncTailTest, SyncApplyCommandThrowsException) { ASSERT_EQUALS(1U, _opsApplied); } +/** + * Creates a command oplog entry with given optime and namespace. + */ +BSONObj makeCommandOplogEntry(OpTime opTime, const NamespaceString& nss, const BSONObj& command) { + BSONObjBuilder bob; + bob.appendElements(opTime.toBSON()); + bob.append("h", 1LL); + bob.append("v", 2); + bob.append("op", "c"); + bob.append("ns", nss.getCommandNS()); + bob.append("o", command); + return bob.obj(); +} + +/** + * Creates a create collection oplog entry with given optime. + */ +BSONObj makeCreateCollectionOplogEntry(OpTime opTime, + const NamespaceString& nss = NamespaceString("test.foo"), + const BSONObj& options = BSONObj()) { + BSONObjBuilder bob; + bob.append("create", nss.coll()); + bob.appendElements(options); + return makeCommandOplogEntry(opTime, nss, bob.obj()); +} + +/** + * Creates an insert oplog entry with given optime and namespace. + */ +BSONObj makeInsertDocumentOplogEntry(OpTime opTime, + const NamespaceString& nss, + const BSONObj& documentToInsert) { + BSONObjBuilder bob; + bob.appendElements(opTime.toBSON()); + bob.append("h", 1LL); + bob.append("op", "i"); + bob.append("ns", nss.ns()); + bob.append("o", documentToInsert); + return bob.obj(); +} + +class IdempotencyTest : public SyncTailTest { +protected: + BSONObj createCollection(); + BSONObj buildIndex(const BSONObj& indexSpec, const BSONObj& options = BSONObj()); + BSONObj dropIndex(const std::string& indexName); + OpTime nextOpTime() { + static long long lastSecond = 1; + return OpTime(Timestamp(Seconds(lastSecond++), 0), 1LL); + } + Status runOp(const BSONObj& entry); + Status runOps(std::initializer_list ops); + // Validate data and indexes. Return the MD5 hash of the documents ordered by _id. + std::string validate(); + + NamespaceString nss{"test.foo"}; + NamespaceString nssIndex{"test.system.indexes"}; +}; + +Status IdempotencyTest::runOp(const BSONObj& op) { + return runOps({op}); +} + +Status IdempotencyTest::runOps(std::initializer_list ops) { + std::vector opsVector(ops); + SyncTail syncTail(nullptr, SyncTail::MultiSyncApplyFunc()); + return multiInitialSyncApply_noAbort(_txn.get(), opsVector, &syncTail); +} + +BSONObj IdempotencyTest::createCollection() { + return makeCreateCollectionOplogEntry(nextOpTime(), nss); +} + +BSONObj IdempotencyTest::buildIndex(const BSONObj& indexSpec, const BSONObj& options) { + BSONObjBuilder bob; + bob.append("v", 1); + bob.append("key", indexSpec); + bob.append("name", std::string(indexSpec.firstElementFieldName()) + "_index"); + bob.append("ns", nss.ns()); + bob.appendElementsUnique(options); + return makeInsertDocumentOplogEntry(nextOpTime(), nssIndex, bob.obj()); +} + +BSONObj IdempotencyTest::dropIndex(const std::string& indexName) { + auto cmd = BSON("deleteIndex" << nss.coll() << "index" << indexName); + return makeCommandOplogEntry(nextOpTime(), nss, cmd); +} + +std::string IdempotencyTest::validate() { + auto collection = AutoGetCollectionForRead(_txn.get(), nss).getCollection(); + if (!collection) { + return "CollectionNotFound"; + } + ValidateResults validateResults; + BSONObjBuilder bob; + + Lock::DBLock lk(_txn->lockState(), nss.db(), MODE_IS); + Lock::CollectionLock lock(_txn->lockState(), nss.ns(), MODE_IS); + ASSERT_OK(collection->validate(_txn.get(), true, true, &validateResults, &bob)); + ASSERT_TRUE(validateResults.valid); + + IndexDescriptor* desc = collection->getIndexCatalog()->findIdIndex(_txn.get()); + ASSERT_TRUE(desc); + auto exec = InternalPlanner::indexScan(_txn.get(), + collection, + desc, + BSONObj(), + BSONObj(), + false, + PlanExecutor::YIELD_MANUAL, + InternalPlanner::FORWARD, + InternalPlanner::IXSCAN_FETCH); + ASSERT(NULL != exec.get()); + md5_state_t st; + md5_init(&st); + + PlanExecutor::ExecState state; + BSONObj c; + while (PlanExecutor::ADVANCED == (state = exec->getNext(&c, NULL))) { + md5_append(&st, (const md5_byte_t*)c.objdata(), c.objsize()); + } + ASSERT_EQUALS(PlanExecutor::IS_EOF, state); + md5digest d; + md5_finish(&st, d); + return digestToString(d); +} + +TEST_F(IdempotencyTest, CollModNamespaceNotFound) { + getGlobalReplicationCoordinator()->setFollowerMode(MemberState::RS_RECOVERING); + + ASSERT_OK(runOp(createCollection())); + ASSERT_OK(runOp(buildIndex(BSON("createdAt" << 1), BSON("expireAfterSeconds" << 3600)))); + + auto indexChange = fromjson("{keyPattern: {createdAt:1}, expireAfterSeconds:4000}}"); + auto collModCmd = BSON("collMod" << nss.coll() << "index" << indexChange); + auto collModOp = makeCommandOplogEntry(nextOpTime(), nss, collModCmd); + auto dropCollOp = makeCommandOplogEntry(nextOpTime(), nss, BSON("drop" << nss.coll())); + + auto ops = {collModOp, dropCollOp}; + + ASSERT_OK(runOps(ops)); + auto hash = validate(); + ASSERT_OK(runOps(ops)); + ASSERT_EQUALS(hash, validate()); +} + +TEST_F(IdempotencyTest, CollModIndexNotFound) { + getGlobalReplicationCoordinator()->setFollowerMode(MemberState::RS_RECOVERING); + + ASSERT_OK(runOp(createCollection())); + ASSERT_OK(runOp(buildIndex(BSON("createdAt" << 1), BSON("expireAfterSeconds" << 3600)))); + + auto indexChange = fromjson("{keyPattern: {createdAt:1}, expireAfterSeconds:4000}}"); + auto collModCmd = BSON("collMod" << nss.coll() << "index" << indexChange); + auto collModOp = makeCommandOplogEntry(nextOpTime(), nss, collModCmd); + auto dropIndexOp = dropIndex("createdAt_index"); + + auto ops = {collModOp, dropIndexOp}; + + ASSERT_OK(runOps(ops)); + auto hash = validate(); + ASSERT_OK(runOps(ops)); + ASSERT_EQUALS(hash, validate()); +} + +TEST_F(IdempotencyTest, ResyncOnRenameCollection) { + ReplicationCoordinator::get(_txn.get())->setFollowerMode(MemberState::RS_RECOVERING); + + auto cmd = BSON("renameCollection" << nss.ns() << "to" + << "test.bar" + << "stayTemp" << false << "dropTarget" << false); + auto op = makeCommandOplogEntry(nextOpTime(), nss, cmd); + ASSERT_EQUALS(runOp(op), ErrorCodes::OplogOperationUnsupported); +} + +TEST_F(IdempotencyTest, MultiInitialSyncApplySkipsDocumentOnNamespaceNotFound) { + BSONObj emptyDoc; + NamespaceString nss("test", "foo"); + NamespaceString badNss("test", "bad"); + auto doc1 = BSON("_id" << 1); + auto doc2 = BSON("_id" << 2); + auto doc3 = BSON("_id" << 3); + auto op0 = makeCreateCollectionOplogEntry(nextOpTime(), nss); + auto op1 = makeInsertDocumentOplogEntry(nextOpTime(), nss, doc1); + auto op2 = makeInsertDocumentOplogEntry(nextOpTime(), badNss, doc2); + auto op3 = makeInsertDocumentOplogEntry(nextOpTime(), nss, doc3); + runOps({op0, op1, op2, op3}); + + OplogInterfaceLocal collectionReader(_txn.get(), nss.ns()); + auto iter = collectionReader.makeIterator(); + ASSERT_EQUALS(doc3, unittest::assertGet(iter->next()).first); + ASSERT_EQUALS(doc1, unittest::assertGet(iter->next()).first); + ASSERT_EQUALS(ErrorCodes::NoSuchKey, iter->next().getStatus()); +} + } // namespace diff --git a/src/mongo/db/repl/topology_coordinator.h b/src/mongo/db/repl/topology_coordinator.h index de01144f796..3a51485052a 100644 --- a/src/mongo/db/repl/topology_coordinator.h +++ b/src/mongo/db/repl/topology_coordinator.h @@ -379,12 +379,24 @@ public: /** * Tries to transition the coordinator from the leader role to the follower role. * - * Fails if "force" is not set and no follower is known to be up. It is illegal - * to call this method if the node is not leader. + * If force==true, step down this node and return true immediately. Else, a step down + * succeeds only if the following conditions are met: * - * Returns whether or not the step down succeeded. + * C1. A majority set of nodes, M, in the replica set have optimes greater than or + * equal to the last applied optime of the primary. + * + * C2. If C1 holds, then there must exist at least one electable secondary node in the + * majority set M. + * + * If C1 and C2 hold, a step down occurs and this method returns true. Else, the step down + * fails and this method returns false. + * + * NOTE: It is illegal to call this method if the node is not a primary. */ - virtual bool stepDown(Date_t until, bool force, const OpTime& lastOpApplied) = 0; + virtual bool stepDown(Date_t until, + bool force, + const OpTime& lastOpApplied, + const OpTime& lastOpCommitted) = 0; /** * Sometimes a request to step down comes in (like via a heartbeat), but we don't have the @@ -410,9 +422,9 @@ public: /** * Prepares a BSONObj describing the current term, primary, and lastOp information. */ - virtual void prepareReplResponseMetadata(rpc::ReplSetMetadata* metadata, - const OpTime& lastVisibleOpTime, - const OpTime& lastCommittedOpTime) const = 0; + virtual void prepareReplMetadata(rpc::ReplSetMetadata* metadata, + const OpTime& lastVisibleOpTime, + const OpTime& lastCommittedOpTime) const = 0; /** * Writes into 'output' all the information needed to generate a summary of the current diff --git a/src/mongo/db/repl/topology_coordinator_impl.cpp b/src/mongo/db/repl/topology_coordinator_impl.cpp index e245ddb3276..53c79c80b03 100644 --- a/src/mongo/db/repl/topology_coordinator_impl.cpp +++ b/src/mongo/db/repl/topology_coordinator_impl.cpp @@ -51,6 +51,7 @@ #include "mongo/db/server_parameters.h" #include "mongo/rpc/metadata/repl_set_metadata.h" #include "mongo/s/catalog/catalog_manager.h" +#include "mongo/util/fail_point_service.h" #include "mongo/util/hex.h" #include "mongo/util/log.h" #include "mongo/util/mongoutils/str.h" @@ -67,6 +68,11 @@ const Seconds TopologyCoordinatorImpl::VoteLease::leaseTime = Seconds(30); // must be before it will call for a priority takeover election. MONGO_EXPORT_STARTUP_SERVER_PARAMETER(priorityTakeoverFreshnessWindowSeconds, int, 2); +// If this fail point is enabled, TopologyCoordinatorImpl::shouldChangeSyncSource() will ignore +// the option TopologyCoordinatorImpl::Options::maxSyncSourceLagSecs. The sync source will not be +// re-evaluated if it lags behind another node by more than 'maxSyncSourceLagSecs' seconds. +MONGO_FP_DECLARE(disableMaxSyncSourceLagSecs); + namespace { template @@ -1361,6 +1367,23 @@ bool TopologyCoordinatorImpl::_aMajoritySeemsToBeUp() const { return vUp * 2 > _rsConfig.getTotalVotingMembers(); } +bool TopologyCoordinatorImpl::_canSeeHealthyPrimaryOfEqualOrGreaterPriority( + const int candidateIndex) const { + const double candidatePriority = _rsConfig.getMemberAt(candidateIndex).getPriority(); + for (auto it = _hbdata.begin(); it != _hbdata.end(); ++it) { + if (!it->up() || it->getState() != MemberState::RS_PRIMARY) { + continue; + } + const int itIndex = indexOfIterator(_hbdata, it); + const double priority = _rsConfig.getMemberAt(itIndex).getPriority(); + if (itIndex != candidateIndex && priority >= candidatePriority) { + return true; + } + } + + return false; +} + bool TopologyCoordinatorImpl::_isOpTimeCloseEnoughToLatestToElect( const OpTime& otherOpTime, const OpTime& ourLastOpApplied) const { const OpTime latestKnownOpTime = _latestKnownOpTime(ourLastOpApplied); @@ -2228,24 +2251,37 @@ void TopologyCoordinatorImpl::processLoseElection() { } } -bool TopologyCoordinatorImpl::stepDown(Date_t until, bool force, const OpTime& lastOpApplied) { - bool canStepDown = force; - for (int i = 0; !canStepDown && i < _rsConfig.getNumMembers(); ++i) { - if (i == _selfIndex) { - continue; - } - UnelectableReasonMask reason = _getUnelectableReason(i, lastOpApplied); - if (!reason && _hbdata[i].getAppliedOpTime() >= lastOpApplied) { - canStepDown = true; - } +bool TopologyCoordinatorImpl::stepDown(Date_t until, + bool force, + const OpTime& lastOpApplied, + const OpTime& lastOpCommitted) { + // force==true overrides all other checks. + if (force) { + _stepDownUntil = until; + _stepDownSelfAndReplaceWith(-1); + return true; } - if (!canStepDown) { + // Ensure a majority of caught up nodes. + if (lastOpCommitted < lastOpApplied) { return false; } - _stepDownUntil = until; - _stepDownSelfAndReplaceWith(-1); - return true; + + // Now make sure we also have at least one caught up node that is also electable. + for (int memberIndex = 0; memberIndex < _rsConfig.getNumMembers(); memberIndex++) { + // ignore your self + if (memberIndex == _selfIndex) { + continue; + } + UnelectableReasonMask reason = _getUnelectableReason(memberIndex, lastOpApplied); + if (!reason && _hbdata.at(memberIndex).getAppliedOpTime() >= lastOpApplied) { + // Found a caught up and electable node, succeed with step down. + _stepDownUntil = until; + _stepDownSelfAndReplaceWith(-1); + return true; + } + } + return false; } void TopologyCoordinatorImpl::setFollowerMode(MemberState::MS newMode) { @@ -2383,32 +2419,40 @@ bool TopologyCoordinatorImpl::shouldChangeSyncSource(const HostAndPort& currentS return true; } - unsigned int currentSecs = currentSourceOpTime.getSecs(); - unsigned int goalSecs = currentSecs + durationCount(_options.maxSyncSourceLagSecs); + if (MONGO_FAIL_POINT(disableMaxSyncSourceLagSecs)) { + log() << "disableMaxSyncSourceLagSecs fail point enabled - not checking the most recent " + "OpTime, " << currentSourceOpTime.toString() << ", of our current sync source, " + << currentSource << ", against the OpTimes of the other nodes in this replica set."; + } else { + unsigned int currentSecs = currentSourceOpTime.getSecs(); + unsigned int goalSecs = currentSecs + durationCount(_options.maxSyncSourceLagSecs); - for (std::vector::const_iterator it = _hbdata.begin(); it != _hbdata.end(); - ++it) { - const int itIndex = indexOfIterator(_hbdata, it); - const MemberConfig& candidateConfig = _rsConfig.getMemberAt(itIndex); - if (it->up() && (candidateConfig.isVoter() || !_selfConfig().isVoter()) && - (candidateConfig.shouldBuildIndexes() || !_selfConfig().shouldBuildIndexes()) && - it->getState().readable() && !_memberIsBlacklisted(candidateConfig, now) && - goalSecs < it->getAppliedOpTime().getSecs()) { - log() << "re-evaluating sync source because our current sync source's most recent " - << "OpTime is " << currentSourceOpTime.toString() << " which is more than " - << _options.maxSyncSourceLagSecs << " behind member " - << candidateConfig.getHostAndPort().toString() << " whose most recent OpTime is " - << it->getAppliedOpTime().toString(); - invariant(itIndex != _selfIndex); - return true; + for (std::vector::const_iterator it = _hbdata.begin(); + it != _hbdata.end(); + ++it) { + const int itIndex = indexOfIterator(_hbdata, it); + const MemberConfig& candidateConfig = _rsConfig.getMemberAt(itIndex); + if (it->up() && (candidateConfig.isVoter() || !_selfConfig().isVoter()) && + (candidateConfig.shouldBuildIndexes() || !_selfConfig().shouldBuildIndexes()) && + it->getState().readable() && !_memberIsBlacklisted(candidateConfig, now) && + goalSecs < it->getAppliedOpTime().getSecs()) { + log() << "re-evaluating sync source because our current sync source's most recent " + << "OpTime is " << currentSourceOpTime.toString() << " which is more than " + << _options.maxSyncSourceLagSecs << " behind member " + << candidateConfig.getHostAndPort().toString() + << " whose most recent OpTime is " << it->getAppliedOpTime().toString(); + invariant(itIndex != _selfIndex); + return true; + } } } + return false; } -void TopologyCoordinatorImpl::prepareReplResponseMetadata(rpc::ReplSetMetadata* metadata, - const OpTime& lastVisibleOpTime, - const OpTime& lastCommittedOpTime) const { +void TopologyCoordinatorImpl::prepareReplMetadata(rpc::ReplSetMetadata* metadata, + const OpTime& lastVisibleOpTime, + const OpTime& lastCommittedOpTime) const { *metadata = rpc::ReplSetMetadata(_term, lastCommittedOpTime, @@ -2442,12 +2486,16 @@ void TopologyCoordinatorImpl::processReplSetRequestVotes(const ReplSetRequestVot } else if (args.getSetName() != _rsConfig.getReplSetName()) { response->setVoteGranted(false); response->setReason("candidate's set name differs from mine"); - } else if (args.getLastCommittedOp() < lastAppliedOpTime) { + } else if (args.getLastDurableOpTime() < lastAppliedOpTime) { response->setVoteGranted(false); response->setReason("candidate's data is staler than mine"); } else if (!args.isADryRun() && _lastVote.getTerm() == args.getTerm()) { response->setVoteGranted(false); response->setReason("already voted for another candidate this term"); + } else if (_selfConfig().isArbiter() && + _canSeeHealthyPrimaryOfEqualOrGreaterPriority(args.getCandidateIndex())) { + response->setVoteGranted(false); + response->setReason("can see a healthy primary of equal or greater priority"); } else { if (!args.isADryRun()) { _lastVote.setTerm(args.getTerm()); diff --git a/src/mongo/db/repl/topology_coordinator_impl.h b/src/mongo/db/repl/topology_coordinator_impl.h index fb4b7786cb3..4361f27f3bc 100644 --- a/src/mongo/db/repl/topology_coordinator_impl.h +++ b/src/mongo/db/repl/topology_coordinator_impl.h @@ -219,12 +219,15 @@ public: virtual void processLoseElection(); virtual Status checkShouldStandForElection(Date_t now, const OpTime& lastOpApplied) const; virtual void setMyHeartbeatMessage(const Date_t now, const std::string& message); - virtual bool stepDown(Date_t until, bool force, const OpTime& lastOpApplied); + virtual bool stepDown(Date_t until, + bool force, + const OpTime& lastOpApplied, + const OpTime& lastOpCommitted); virtual bool stepDownIfPending(); virtual Date_t getStepDownTime() const; - virtual void prepareReplResponseMetadata(rpc::ReplSetMetadata* metadata, - const OpTime& lastVisibleOpTime, - const OpTime& lastCommitttedOpTime) const; + virtual void prepareReplMetadata(rpc::ReplSetMetadata* metadata, + const OpTime& lastVisibleOpTime, + const OpTime& lastCommitttedOpTime) const; Status processReplSetDeclareElectionWinner(const ReplSetDeclareElectionWinnerArgs& args, long long* responseTerm); virtual void processReplSetRequestVotes(const ReplSetRequestVotesArgs& args, @@ -305,6 +308,10 @@ private: // Sees if a majority number of votes are held by members who are currently "up" bool _aMajoritySeemsToBeUp() const; + // Returns true if the node can see a healthy primary of equal or greater priority to the + // candidate. + bool _canSeeHealthyPrimaryOfEqualOrGreaterPriority(const int candidateIndex) const; + // Is otherOpTime close enough (within 10 seconds) to the latest known optime to qualify // for an election bool _isOpTimeCloseEnoughToLatestToElect(const OpTime& otherOpTime, @@ -457,7 +464,7 @@ private: } _voteLease; // V1 last vote info for elections - LastVote _lastVote; + LastVote _lastVote{OpTime::kInitialTerm, -1}; enum class ReadCommittedSupport { kUnknown, diff --git a/src/mongo/db/repl/topology_coordinator_impl_test.cpp b/src/mongo/db/repl/topology_coordinator_impl_test.cpp index 8d3ae2b6516..e35499c7797 100644 --- a/src/mongo/db/repl/topology_coordinator_impl_test.cpp +++ b/src/mongo/db/repl/topology_coordinator_impl_test.cpp @@ -5513,6 +5513,43 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenConfigVersionDoesNotMatch) { ASSERT_FALSE(response.getVoteGranted()); } +TEST_F(TopoCoordTest, ArbiterDoesNotGrantVoteWhenItCanSeeAHealthyPrimaryOfEqualOrGreaterPriority) { + updateConfig(BSON("_id" + << "rs0" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 10 << "host" + << "hself" + << "arbiterOnly" << true) + << BSON("_id" << 20 << "host" + << "h2" + << "priority" << 5) << BSON("_id" << 30 << "host" + << "h3"))), + 0); + heartbeatFromMember(HostAndPort("h2"), + "rs0", + MemberState::RS_PRIMARY, + OpTime(Timestamp(0, 0), 0), + Milliseconds(300)); + heartbeatFromMember(HostAndPort("h3"), + "rs0", + MemberState::RS_SECONDARY, + OpTime(Timestamp(0, 0), 0), + Milliseconds(300)); + + ReplSetRequestVotesArgs args; + args.initialize(BSON("replSetRequestVotes" << 1 << "setName" + << "rs0" + << "term" << 1LL << "candidateIndex" << 2LL + << "configVersion" << 1LL << "lastCommittedOp" + << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); + ReplSetRequestVotesResponse response; + OpTime lastAppliedOpTime; + + getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + ASSERT_EQUALS("can see a healthy primary of equal or greater priority", response.getReason()); + ASSERT_FALSE(response.getVoteGranted()); +} + TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenTermIsStale) { updateConfig(BSON("_id" << "rs0" diff --git a/src/mongo/db/repl/vote_requester.cpp b/src/mongo/db/repl/vote_requester.cpp index 06df2e289c5..eafa68715e2 100644 --- a/src/mongo/db/repl/vote_requester.cpp +++ b/src/mongo/db/repl/vote_requester.cpp @@ -48,12 +48,12 @@ VoteRequester::Algorithm::Algorithm(const ReplicaSetConfig& rsConfig, long long candidateIndex, long long term, bool dryRun, - OpTime lastOplogEntry) + OpTime lastDurableOpTime) : _rsConfig(rsConfig), _candidateIndex(candidateIndex), _term(term), _dryRun(dryRun), - _lastOplogEntry(lastOplogEntry) { + _lastDurableOpTime(lastDurableOpTime) { // populate targets with all voting members that aren't this node long long index = 0; for (auto member = _rsConfig.membersBegin(); member != _rsConfig.membersEnd(); member++) { @@ -75,10 +75,7 @@ std::vector VoteRequester::Algorithm::getRequests() const requestVotesCmdBuilder.append("candidateIndex", _candidateIndex); requestVotesCmdBuilder.append("configVersion", _rsConfig.getConfigVersion()); - BSONObjBuilder lastCommittedOp(requestVotesCmdBuilder.subobjStart("lastCommittedOp")); - lastCommittedOp.append("ts", _lastOplogEntry.getTimestamp()); - lastCommittedOp.append("t", _lastOplogEntry.getTerm()); - lastCommittedOp.done(); + _lastDurableOpTime.append(&requestVotesCmdBuilder, "lastCommittedOp"); const BSONObj requestVotesCmd = requestVotesCmdBuilder.obj(); @@ -150,9 +147,9 @@ StatusWith VoteRequester::start( long long candidateIndex, long long term, bool dryRun, - OpTime lastOplogEntry, + OpTime lastDurableOpTime, const stdx::function& onCompletion) { - _algorithm.reset(new Algorithm(rsConfig, candidateIndex, term, dryRun, lastOplogEntry)); + _algorithm.reset(new Algorithm(rsConfig, candidateIndex, term, dryRun, lastDurableOpTime)); _runner.reset(new ScatterGatherRunner(_algorithm.get())); return _runner->start(executor, onCompletion); } diff --git a/src/mongo/db/repl/vote_requester.h b/src/mongo/db/repl/vote_requester.h index 8ee9b052d4d..433facf345c 100644 --- a/src/mongo/db/repl/vote_requester.h +++ b/src/mongo/db/repl/vote_requester.h @@ -65,7 +65,7 @@ public: long long candidateIndex, long long term, bool dryRun, - OpTime lastOplogEntry); + OpTime lastDurableOpTime); virtual ~Algorithm(); virtual std::vector getRequests() const; virtual void processResponse(const executor::RemoteCommandRequest& request, @@ -89,7 +89,7 @@ public: const long long _candidateIndex; const long long _term; bool _dryRun = false; // this bool indicates this is a mock election when true - const OpTime _lastOplogEntry; + const OpTime _lastDurableOpTime; std::vector _targets; unordered_set _responders; bool _staleTerm = false; @@ -115,7 +115,7 @@ public: long long candidateIndex, long long term, bool dryRun, - OpTime lastOplogEntry, + OpTime lastDurableOpTime, const stdx::function& onCompletion = stdx::function()); /** diff --git a/src/mongo/db/s/migration_source_manager.cpp b/src/mongo/db/s/migration_source_manager.cpp index 96dd8de9b68..d6ea4110beb 100644 --- a/src/mongo/db/s/migration_source_manager.cpp +++ b/src/mongo/db/s/migration_source_manager.cpp @@ -469,6 +469,29 @@ bool MigrationSourceManager::storeCurrentLocs(OperationContext* txn, return true; } +namespace { + +static bool stillSameSession(boost::optional const& memberSessionId, + MigrationSessionId const& argSessionId, + std::string& errmsg) { + if (!memberSessionId) { + errmsg = "not active"; + return false; + } + + // A mongod version < v3.2 will not have sessionId, in which case it is empty and + // ignored. + if (!argSessionId.isEmpty() && !memberSessionId->matches(argSessionId)) { + errmsg = str::stream() << "migration session id changed from " << argSessionId.toString() + << " to " << memberSessionId->toString() + << " while initial clone was active"; + return false; + } + return true; +} + +} // namespace + bool MigrationSourceManager::clone(OperationContext* txn, const MigrationSessionId& sessionId, string& errmsg, @@ -483,16 +506,7 @@ bool MigrationSourceManager::clone(OperationContext* txn, stdx::lock_guard sl(_mutex); - if (!_sessionId) { - errmsg = "not active"; - return false; - } - - // A mongod version < v3.2 will not have sessionId, in which case it is empty and ignored. - if (!sessionId.isEmpty() && !_sessionId->matches(sessionId)) { - errmsg = str::stream() << "requested migration session id " << sessionId.toString() - << " does not match active session id " - << _sessionId->toString(); + if (!stillSameSession(_sessionId, sessionId, errmsg)) { return false; } @@ -507,37 +521,46 @@ bool MigrationSourceManager::clone(OperationContext* txn, static_cast((12 + collection->averageObjectSize(txn)) * cloneLocsRemaining())); } - bool isBufferFilled = false; BSONArrayBuilder clonedDocsArrayBuilder(allocSize); + std::vector cloneLocsTemp; + + // We carve off a limited number of records to look up per externally-visible iteration + // so that, for very big collections, progress meters register activity. + cloneLocsTemp.reserve(1000); + + bool isBufferFilled = false; while (!isBufferFilled) { ScopedTransaction scopedXact(txn, MODE_IS); AutoGetCollection autoColl(txn, _getNS(), MODE_IS); + Collection* collection = autoColl.getCollection(); - stdx::lock_guard sl(_mutex); - - if (!_sessionId) { - errmsg = "not active"; + if (!collection) { + errmsg = str::stream() << "collection " << _getNS().toString() << " does not exist"; return false; } - // A mongod version < v3.2 will not have sessionId, in which case it is empty and ignored. - if (!sessionId.isEmpty() && !_sessionId->matches(sessionId)) { - errmsg = str::stream() << "migration session id changed from " << sessionId.toString() - << " to " << _sessionId->toString() - << " while initial clone was active"; - return false; - } + { + stdx::lock_guard sl(_mutex); - Collection* collection = autoColl.getCollection(); - if (!collection) { - errmsg = str::stream() << "collection " << _nss.toString() << " does not exist"; - return false; - } + if (!stillSameSession(_sessionId, sessionId, errmsg)) { + return false; + } - stdx::lock_guard lk(_cloneLocsMutex); + { + stdx::lock_guard lk(_cloneLocsMutex); + + for (RecordId const& cloneLoc : _cloneLocs) { + cloneLocsTemp.push_back(cloneLoc); + if (cloneLocsTemp.size() == cloneLocsTemp.capacity()) { + break; // enough for now + } + } + } + } - std::set::iterator cloneLocsIter = _cloneLocs.begin(); - for (; cloneLocsIter != _cloneLocs.end(); ++cloneLocsIter) { + // release locks during find ops + auto cloneLocsIter = cloneLocsTemp.begin(); + for (; cloneLocsIter != cloneLocsTemp.end(); ++cloneLocsIter) { if (tracker.intervalHasElapsed()) // should I yield? break; @@ -561,12 +584,23 @@ bool MigrationSourceManager::clone(OperationContext* txn, clonedDocsArrayBuilder.append(doc.value()); } - _cloneLocs.erase(_cloneLocs.begin(), cloneLocsIter); + // reclaim locks and record progress + + stdx::lock_guard sl(_mutex); + + if (!stillSameSession(_sessionId, sessionId, errmsg)) { + return false; + } + + stdx::lock_guard lk(_cloneLocsMutex); - // Note: must be holding _cloneLocsMutex, don't move this inside while condition! + std::for_each(cloneLocsTemp.begin(), + cloneLocsIter, + [&](RecordId const& loc) { _cloneLocs.erase(loc); }); if (_cloneLocs.empty()) { - break; + break; // and return } + cloneLocsTemp.clear(); } result.appendArray("objects", clonedDocsArrayBuilder.arr()); diff --git a/src/mongo/db/stats/SConscript b/src/mongo/db/stats/SConscript index 0dea68e45da..a0ca5534447 100644 --- a/src/mongo/db/stats/SConscript +++ b/src/mongo/db/stats/SConscript @@ -54,3 +54,24 @@ env.Library( '$BUILD_DIR/mongo/util/concurrency/spin_lock', ], ) + +env.Library( + target='fill_locker_info', + source=[ + 'fill_locker_info.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/db/concurrency/lock_manager' + ], +) + +env.CppUnitTest( + target='fill_locker_info_test', + source=[ + 'fill_locker_info_test.cpp', + ], + LIBDEPS=[ + 'fill_locker_info', + ], +) diff --git a/src/mongo/db/stats/fill_locker_info_test.cpp b/src/mongo/db/stats/fill_locker_info_test.cpp new file mode 100644 index 00000000000..40a58879ddf --- /dev/null +++ b/src/mongo/db/stats/fill_locker_info_test.cpp @@ -0,0 +1,136 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/bson/bsontypes.h" +#include "mongo/db/concurrency/lock_manager_defs.h" +#include "mongo/db/stats/fill_locker_info.h" +#include "mongo/unittest/death_test.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace { +using LockerInfo = Locker::LockerInfo; +using OneLock = Locker::OneLock; + +const ResourceId kGlobalId(RESOURCE_GLOBAL, ResourceId::SINGLETON_GLOBAL); + +TEST(FillLockerInfo, DoesReportWaitingForLockIfWaiting) { + LockerInfo info; + info.waitingResource = kGlobalId; + ASSERT_TRUE(info.waitingResource.isValid()); + + BSONObjBuilder infoBuilder; + fillLockerInfo(info, infoBuilder); + const BSONObj infoObj = infoBuilder.done(); + + ASSERT(infoObj["waitingForLock"].type() == BSONType::Bool); + ASSERT_TRUE(infoObj["waitingForLock"].Bool()); +} + +TEST(FillLockerInfo, DoesNotReportWaitingForLockIfNotWaiting) { + LockerInfo info; + info.waitingResource = ResourceId(); // This means it is not waiting for anything. + ASSERT_FALSE(info.waitingResource.isValid()); + + BSONObjBuilder infoBuilder; + fillLockerInfo(info, infoBuilder); + const BSONObj infoObj = infoBuilder.done(); + + ASSERT(infoObj["waitingForLock"].type() == BSONType::Bool); + ASSERT_FALSE(infoObj["waitingForLock"].Bool()); +} + +TEST(FillLockerInfo, DoesReportLockStats) { + LockerInfo info; + SingleThreadedLockStats stats; + stats.recordAcquisition(kGlobalId, MODE_IX); + info.stats = stats; + + BSONObjBuilder infoBuilder; + fillLockerInfo(info, infoBuilder); + const BSONObj infoObj = infoBuilder.done(); + + ASSERT_EQ(infoObj["lockStats"].type(), BSONType::Object); +} + +DEATH_TEST(FillLockerInfo, ShouldFailIfLocksAreNotSortedAppropriately, "Invariant failure") { + LockerInfo info; + // The global lock is supposed to come before the database lock. + info.locks = {OneLock{ResourceId(RESOURCE_DATABASE, std::string("TestDB")), MODE_X}, + OneLock{kGlobalId, MODE_IX}}; + + BSONObjBuilder infoBuilder; + fillLockerInfo(info, infoBuilder); +} + +TEST(FillLockerInfo, DoesReportLocksHeld) { + const ResourceId dbId(RESOURCE_DATABASE, std::string("TestDB")); + LockerInfo info; + info.locks = {OneLock{kGlobalId, MODE_IX}, OneLock{dbId, MODE_IX}}; + + BSONObjBuilder infoBuilder; + fillLockerInfo(info, infoBuilder); + const BSONObj infoObj = infoBuilder.done(); + + ASSERT_EQ(infoObj["locks"].type(), BSONType::Object); + ASSERT_EQ(infoObj["locks"][resourceTypeName(kGlobalId.getType())].type(), BSONType::String); + ASSERT_EQ(infoObj["locks"][resourceTypeName(kGlobalId.getType())].String(), "w"); + ASSERT_EQ(infoObj["locks"][resourceTypeName(dbId.getType())].type(), BSONType::String); + ASSERT_EQ(infoObj["locks"][resourceTypeName(dbId.getType())].String(), "w"); +} + +TEST(FillLockerInfo, ShouldReportMaxTypeHeldForResourceType) { + const ResourceId firstDbId(RESOURCE_DATABASE, std::string("FirstDB")); + const ResourceId secondDbId(RESOURCE_DATABASE, std::string("SecondDB")); + LockerInfo info; + info.locks = { + OneLock{kGlobalId, MODE_IX}, OneLock{firstDbId, MODE_IX}, OneLock{secondDbId, MODE_X}}; + + BSONObjBuilder infoBuilder; + fillLockerInfo(info, infoBuilder); + BSONObj infoObj = infoBuilder.done(); + + ASSERT_EQ(infoObj["locks"].type(), BSONType::Object); + ASSERT_EQ(infoObj["locks"][resourceTypeName(firstDbId.getType())].type(), BSONType::String); + ASSERT_EQ(infoObj["locks"][resourceTypeName(firstDbId.getType())].String(), + "W"); // One is held in IX, one in X, so X should win and be displayed as "W". + + // Ensure it still works if locks are reported in the opposite order. + info.locks = { + OneLock{kGlobalId, MODE_IX}, OneLock{secondDbId, MODE_X}, OneLock{firstDbId, MODE_IX}}; + + ASSERT_EQ(infoObj["locks"].type(), BSONType::Object); + ASSERT_EQ(infoObj["locks"][resourceTypeName(firstDbId.getType())].type(), BSONType::String); + ASSERT_EQ(infoObj["locks"][resourceTypeName(firstDbId.getType())].String(), "W"); +} + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/storage/mmap_v1/btree/btree_logic.cpp b/src/mongo/db/storage/mmap_v1/btree/btree_logic.cpp index 6e2d19d6bce..ffec90efc04 100644 --- a/src/mongo/db/storage/mmap_v1/btree/btree_logic.cpp +++ b/src/mongo/db/storage/mmap_v1/btree/btree_logic.cpp @@ -2310,10 +2310,6 @@ bool BtreeLogic::locate(OperationContext* txn, *bucketLocOut = _locate(txn, getRootLoc(txn), owned, posOut, &found, recordLoc, direction); - if (!found) { - return false; - } - skipUnusedKeys(txn, bucketLocOut, posOut, direction); return found; diff --git a/src/mongo/db/storage/mmap_v1/catalog/namespace_details.cpp b/src/mongo/db/storage/mmap_v1/catalog/namespace_details.cpp index 915e3a7e44d..b8139cf7e89 100644 --- a/src/mongo/db/storage/mmap_v1/catalog/namespace_details.cpp +++ b/src/mongo/db/storage/mmap_v1/catalog/namespace_details.cpp @@ -97,7 +97,6 @@ NamespaceDetails::Extra* NamespaceDetails::allocExtra(OperationContext* txn, Namespace fullns(ns); Namespace extrans(fullns.extraName(i)); // throws UserException if ns name too long - massert(10350, "allocExtra: base ns missing?", this); massert(10351, "allocExtra: extra already exists", ni.details(extrans) == 0); Extra temp; diff --git a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp index 5cfda64a2af..e24b668ac3e 100644 --- a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp +++ b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp @@ -32,6 +32,9 @@ #include "mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/database.h" +#include "mongo/db/catalog/database_holder.h" #include "mongo/db/index/index_descriptor.h" #include "mongo/db/ops/update.h" #include "mongo/db/record_id.h" @@ -251,6 +254,11 @@ Status NamespaceDetailsCollectionCatalogEntry::removeIndex(OperationContext* txn d->idx(getTotalIndexCount(txn)) = IndexDetails(); } + // Someone may be querying the system.indexes namespace directly, so we need to invalidate + // its cursors. + MMAPV1DatabaseCatalogEntry::invalidateSystemCollectionRecord( + txn, NamespaceString(_db->name(), "system.indexes"), infoLocation); + // remove from system.indexes _indexRecordStore->deleteRecord(txn, infoLocation); @@ -360,8 +368,20 @@ void NamespaceDetailsCollectionCatalogEntry::_updateSystemNamespaces(OperationCo RecordData entry = _namespacesRecordStore->dataFor(txn, _namespacesRecordId); const BSONObj newEntry = applyUpdateOperators(entry.releaseToBson(), update); - StatusWith result = _namespacesRecordStore->updateRecord( - txn, _namespacesRecordId, newEntry.objdata(), newEntry.objsize(), false, NULL); + + // Get update notifier + invariant(txn->lockState()->isDbLockedForMode(_db->name(), MODE_X)); + Database* db = dbHolder().get(txn, _db->name()); + Collection* systemCollection = + db->getCollection(NamespaceString(_db->name(), "system.namespaces")); + UpdateNotifier* namespacesNotifier = systemCollection->getUpdateNotifier(); + + StatusWith result = _namespacesRecordStore->updateRecord(txn, + _namespacesRecordId, + newEntry.objdata(), + newEntry.objsize(), + false, + namespacesNotifier); fassert(17486, result.getStatus()); setNamespacesRecordId(txn, result.getValue()); } diff --git a/src/mongo/db/storage/mmap_v1/mmap_v1_database_catalog_entry.cpp b/src/mongo/db/storage/mmap_v1/mmap_v1_database_catalog_entry.cpp index 4b3c592a554..c29d7507c9b 100644 --- a/src/mongo/db/storage/mmap_v1/mmap_v1_database_catalog_entry.cpp +++ b/src/mongo/db/storage/mmap_v1/mmap_v1_database_catalog_entry.cpp @@ -34,6 +34,8 @@ #include +#include "mongo/db/catalog/database.h" +#include "mongo/db/catalog/database_holder.h" #include "mongo/db/catalog/index_catalog_entry.h" #include "mongo/db/index/2d_access_method.h" #include "mongo/db/index/btree_access_method.h" @@ -300,6 +302,9 @@ Status MMAPV1DatabaseCatalogEntry::renameCollection(OperationContext* txn, if (!s.isOK()) return s; } + // Invalidate index record for the old collection. + invalidateSystemCollectionRecord( + txn, NamespaceString(name(), "system.indexes"), record->id); systemIndexRecordStore->deleteRecord(txn, record->id); } @@ -366,6 +371,10 @@ Status MMAPV1DatabaseCatalogEntry::_renameSingleNamespace(OperationContext* txn, RecordId rid = _addNamespaceToNamespaceCollection(txn, toNS, newSpec.isEmpty() ? 0 : &newSpec); + // Invalidate old namespace record + invalidateSystemCollectionRecord( + txn, NamespaceString(name(), "system.namespaces"), oldSpecLocation); + _getNamespaceRecordStore()->deleteRecord(txn, oldSpecLocation); Entry*& entry = _collections[toNS.toString()]; @@ -378,6 +387,17 @@ Status MMAPV1DatabaseCatalogEntry::_renameSingleNamespace(OperationContext* txn, return Status::OK(); } +void MMAPV1DatabaseCatalogEntry::invalidateSystemCollectionRecord( + OperationContext* txn, NamespaceString systemCollectionNamespace, RecordId record) { + // Having to go back up through the DatabaseHolder is a bit of a layering + // violation, but at this point we're not going to add more MMAPv1 specific interfaces. + StringData dbName = systemCollectionNamespace.db(); + invariant(txn->lockState()->isDbLockedForMode(dbName, MODE_X)); + Database* db = dbHolder().get(txn, dbName); + Collection* systemCollection = db->getCollection(systemCollectionNamespace); + systemCollection->getCursorManager()->invalidateDocument(txn, record, INVALIDATION_DELETION); +} + void MMAPV1DatabaseCatalogEntry::appendExtraStats(OperationContext* opCtx, BSONObjBuilder* output, double scale) const { @@ -789,7 +809,12 @@ void MMAPV1DatabaseCatalogEntry::_removeNamespaceFromNamespaceCollection(Operati RecordStoreV1Base* rs = _getNamespaceRecordStore(); invariant(rs); - rs->deleteRecord(txn, entry->second->catalogEntry->getNamespacesRecordId()); + // Invalidate old namespace record + RecordId oldSpecLocation = entry->second->catalogEntry->getNamespacesRecordId(); + invalidateSystemCollectionRecord( + txn, NamespaceString(name(), "system.namespaces"), oldSpecLocation); + + rs->deleteRecord(txn, oldSpecLocation); } CollectionOptions MMAPV1DatabaseCatalogEntry::getCollectionOptions(OperationContext* txn, diff --git a/src/mongo/db/storage/mmap_v1/mmap_v1_database_catalog_entry.h b/src/mongo/db/storage/mmap_v1/mmap_v1_database_catalog_entry.h index e127eece268..bc1229db016 100644 --- a/src/mongo/db/storage/mmap_v1/mmap_v1_database_catalog_entry.h +++ b/src/mongo/db/storage/mmap_v1/mmap_v1_database_catalog_entry.h @@ -127,6 +127,9 @@ public: * exist. */ void createNamespaceForIndex(OperationContext* txn, StringData name); + static void invalidateSystemCollectionRecord(OperationContext* txn, + NamespaceString systemCollectionNamespace, + RecordId record); /** * Ensures data files are compatible, in case we are downgrading from a newer version. Returns diff --git a/src/mongo/db/storage/mmap_v1/record_store_v1_simple.cpp b/src/mongo/db/storage/mmap_v1/record_store_v1_simple.cpp index 6803e5a3bc7..df0e8edd675 100644 --- a/src/mongo/db/storage/mmap_v1/record_store_v1_simple.cpp +++ b/src/mongo/db/storage/mmap_v1/record_store_v1_simple.cpp @@ -452,7 +452,8 @@ Status SimpleRecordStoreV1::compact(OperationContext* txn, _details->setLastExtentSize(txn, 0); // create a new extent so new records go there - increaseStorageSize(txn, _details->lastExtentSize(txn), true); + const bool enforceQuota = false; + increaseStorageSize(txn, _details->lastExtentSize(txn), enforceQuota); wunit.commit(); } diff --git a/src/mongo/db/storage/snapshot.h b/src/mongo/db/storage/snapshot.h index 6ce5b57e51d..b622c14d863 100644 --- a/src/mongo/db/storage/snapshot.h +++ b/src/mongo/db/storage/snapshot.h @@ -57,6 +57,10 @@ public: return _id != other._id; } + std::string toString() const { + return std::to_string(_id); + } + private: uint64_t _id; }; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp index e5c478932ac..27ab575387f 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp @@ -37,22 +37,22 @@ #include #include "mongo/base/checked_cast.h" -#include "mongo/db/json.h" #include "mongo/db/catalog/index_catalog_entry.h" #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/index/index_descriptor.h" +#include "mongo/db/json.h" #include "mongo/db/service_context.h" #include "mongo/db/storage/key_string.h" +#include "mongo/db/storage/storage_options.h" #include "mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h" #include "mongo/db/storage/wiredtiger/wiredtiger_global_options.h" #include "mongo/db/storage/wiredtiger/wiredtiger_record_store.h" #include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h" #include "mongo/db/storage/wiredtiger/wiredtiger_util.h" -#include "mongo/db/storage/storage_options.h" #include "mongo/stdx/memory.h" #include "mongo/util/assert_util.h" -#include "mongo/util/hex.h" #include "mongo/util/fail_point.h" +#include "mongo/util/hex.h" #include "mongo/util/log.h" #include "mongo/util/mongoutils/str.h" @@ -381,7 +381,7 @@ bool WiredTigerIndex::isEmpty(OperationContext* txn) { WT_CURSOR* c = curwrap.get(); if (!c) return true; - int ret = WT_OP_CHECK(c->next(c)); + int ret = WT_READ_CHECK(c->next(c)); if (ret == WT_NOTFOUND) return true; invariantWTOK(ret); @@ -442,7 +442,7 @@ bool WiredTigerIndex::isDup(WT_CURSOR* c, const BSONObj& key, const RecordId& id KeyString data(key, _ordering); WiredTigerItem item(data.getBuffer(), data.getSize()); c->set_key(c, item.Get()); - int ret = WT_OP_CHECK(c->search(c)); + int ret = WT_READ_CHECK(c->search(c)); if (ret == WT_NOTFOUND) { return false; } @@ -490,13 +490,17 @@ protected: // Open cursors can cause bulk open_cursor to fail with EBUSY. // TODO any other cases that could cause EBUSY? WiredTigerSession* outerSession = WiredTigerRecoveryUnit::get(_txn)->getSession(_txn); - outerSession->closeAllCursors(); + outerSession->closeAllCursors(idx->uri()); // Not using cursor cache since we need to set "bulk". WT_CURSOR* cursor; - // We use our own session to ensure we aren't in a transaction. + // Use a different session to ensure we don't hijack an existing transaction. + // Configure the bulk cursor open to fail quickly if it would wait on a checkpoint + // completing - since checkpoints can take a long time, and waiting can result in + // an unexpected pause in building an index. WT_SESSION* session = _session->getSession(); - int err = session->open_cursor(session, idx->uri().c_str(), NULL, "bulk", &cursor); + int err = session->open_cursor( + session, idx->uri().c_str(), NULL, "bulk,checkpoint_wait=false", &cursor); if (!err) return cursor; @@ -805,7 +809,7 @@ protected: void advanceWTCursor() { WT_CURSOR* c = _cursor->get(); - int ret = WT_OP_CHECK(_forward ? c->next(c) : c->prev(c)); + int ret = WT_READ_CHECK(_forward ? c->next(c) : c->prev(c)); if (ret == WT_NOTFOUND) { _cursorAtEof = true; return; @@ -822,7 +826,7 @@ protected: const WiredTigerItem keyItem(query.getBuffer(), query.getSize()); c->set_key(c, keyItem.Get()); - int ret = WT_OP_CHECK(c->search_near(c, &cmp)); + int ret = WT_READ_CHECK(c->search_near(c, &cmp)); if (ret == WT_NOTFOUND) { _cursorAtEof = true; TRACE_CURSOR << "\t not found"; @@ -974,7 +978,7 @@ public: c->set_key(c, keyItem.Get()); // Using search rather than search_near. - int ret = WT_OP_CHECK(c->search(c)); + int ret = WT_READ_CHECK(c->search(c)); if (ret != WT_NOTFOUND) invariantWTOK(ret); _cursorAtEof = ret == WT_NOTFOUND; @@ -989,7 +993,7 @@ public: WiredTigerIndexUnique::WiredTigerIndexUnique(OperationContext* ctx, const std::string& uri, const IndexDescriptor* desc) - : WiredTigerIndex(ctx, uri, desc) {} + : WiredTigerIndex(ctx, uri, desc), _partial(desc->isPartial()) {} std::unique_ptr WiredTigerIndexUnique::newCursor(OperationContext* txn, bool forward) const { @@ -1025,7 +1029,7 @@ Status WiredTigerIndexUnique::_insert(WT_CURSOR* c, // we put them all in the "list" // Note that we can't omit AllZeros when there are multiple ids for a value. When we remove // down to a single value, it will be cleaned up. - ret = WT_OP_CHECK(c->search(c)); + ret = WT_READ_CHECK(c->search(c)); invariantWTOK(ret); WT_ITEM old; @@ -1073,8 +1077,37 @@ void WiredTigerIndexUnique::_unindex(WT_CURSOR* c, WiredTigerItem keyItem(data.getBuffer(), data.getSize()); c->set_key(c, keyItem.Get()); + auto triggerWriteConflictAtPoint = [&keyItem](WT_CURSOR* point) { + // WT_NOTFOUND may occur during a background index build. Insert a dummy value and + // delete it again to trigger a write conflict in case this is being concurrently + // indexed by the background indexer. + point->set_key(point, keyItem.Get()); + point->set_value(point, emptyItem.Get()); + invariantWTOK(WT_OP_CHECK(point->insert(point))); + point->set_key(point, keyItem.Get()); + invariantWTOK(WT_OP_CHECK(point->remove(point))); + }; + if (!dupsAllowed) { - // nice and clear + if (_partial) { + // Check that the record id matches. We may be called to unindex records that are not + // present in the index due to the partial filter expression. + int ret = WT_OP_CHECK(c->search(c)); + if (ret == WT_NOTFOUND) { + triggerWriteConflictAtPoint(c); + return; + } + WT_ITEM value; + invariantWTOK(c->get_value(c, &value)); + BufReader br(value.data, value.size); + fassert(40416, br.remaining()); + if (KeyString::decodeRecordId(&br) != id) { + return; + } + // Ensure there aren't any other values in here. + KeyString::TypeBits::fromBuffer(&br); + fassert(40417, !br.remaining()); + } int ret = WT_OP_CHECK(c->remove(c)); if (ret == WT_NOTFOUND) { return; @@ -1085,16 +1118,9 @@ void WiredTigerIndexUnique::_unindex(WT_CURSOR* c, // dups are allowed, so we have to deal with a vector of RecordIds. - int ret = WT_OP_CHECK(c->search(c)); + int ret = WT_READ_CHECK(c->search(c)); if (ret == WT_NOTFOUND) { - // WT_NOTFOUND is only expected during a background index build. Insert a dummy value and - // delete it again to trigger a write conflict in case this is being concurrently indexed by - // the background indexer. - c->set_key(c, keyItem.Get()); - c->set_value(c, emptyItem.Get()); - invariantWTOK(WT_OP_CHECK(c->insert(c))); - c->set_key(c, keyItem.Get()); - invariantWTOK(WT_OP_CHECK(c->remove(c))); + triggerWriteConflictAtPoint(c); return; } invariantWTOK(ret); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h index 4ea5741e7a5..66284492e69 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h @@ -164,6 +164,9 @@ public: Status _insert(WT_CURSOR* c, const BSONObj& key, const RecordId& id, bool dupsAllowed) override; void _unindex(WT_CURSOR* c, const BSONObj& key, const RecordId& id, bool dupsAllowed) override; + +private: + bool _partial; }; class WiredTigerIndexStandard : public WiredTigerIndex { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index d0d28218770..a98e230c257 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -200,7 +200,7 @@ WiredTigerKVEngine::WiredTigerKVEngine(const std::string& canonicalName, ss << "create,"; ss << "cache_size=" << cacheSizeGB << "G,"; ss << "session_max=20000,"; - ss << "eviction=(threads_max=4),"; + ss << "eviction=(threads_min=4,threads_max=4),"; ss << "config_base=false,"; ss << "statistics=(fast),"; // The setting may have a later setting override it if not using the journal. We make it @@ -345,11 +345,12 @@ int64_t WiredTigerKVEngine::getIdentSize(OperationContext* opCtx, StringData ide Status WiredTigerKVEngine::repairIdent(OperationContext* opCtx, StringData ident) { WiredTigerSession* session = WiredTigerRecoveryUnit::get(opCtx)->getSession(opCtx); - session->closeAllCursors(); + string uri = _uri(ident); + session->closeAllCursors(uri); + _sessionCache->closeAllCursors(uri); if (isEphemeral()) { return Status::OK(); } - string uri = _uri(ident); return _salvageIfNeeded(uri.c_str()); } @@ -530,6 +531,8 @@ Status WiredTigerKVEngine::dropIdent(OperationContext* opCtx, StringData ident) bool WiredTigerKVEngine::_drop(StringData ident) { string uri = _uri(ident); + _sessionCache->closeAllCursors(uri); + WiredTigerSession session(_conn); int ret = session.getSession()->drop( @@ -545,9 +548,9 @@ bool WiredTigerKVEngine::_drop(StringData ident) { // this is expected, queue it up { stdx::lock_guard lk(_identToDropMutex); - _identToDrop.push(uri); + _identToDrop.push_front(uri); } - _sessionCache->closeAllCursors(); + _sessionCache->closeCursorsForQueuedDrops(); return false; } @@ -555,6 +558,28 @@ bool WiredTigerKVEngine::_drop(StringData ident) { return false; } +std::list WiredTigerKVEngine::filterCursorsWithQueuedDrops( + std::list* cache) { + std::list toDrop; + + stdx::lock_guard lk(_identToDropMutex); + if (_identToDrop.empty()) + return toDrop; + + for (auto i = cache->begin(); i != cache->end();) { + if (!i->_cursor || + std::find(_identToDrop.begin(), _identToDrop.end(), std::string(i->_cursor->uri)) == + _identToDrop.end()) { + ++i; + continue; + } + toDrop.push_back(*i); + i = cache->erase(i); + } + + return toDrop; +} + bool WiredTigerKVEngine::haveDropsQueued() const { Date_t now = Date_t::now(); Milliseconds delta = now - _previousCheckedDropsQueued; @@ -565,13 +590,14 @@ bool WiredTigerKVEngine::haveDropsQueued() const { } // We only want to check the queue max once per second or we'll thrash - // This is done in haveDropsQueued, not dropSomeQueuedIdents so we skip the mutex if (delta < Milliseconds(1000)) return false; _previousCheckedDropsQueued = now; - stdx::lock_guard lk(_identToDropMutex); - return !_identToDrop.empty(); + + // Don't wait for the mutex: if we can't get it, report that no drops are queued. + stdx::unique_lock lk(_identToDropMutex, stdx::defer_lock); + return lk.try_lock() && !_identToDrop.empty(); } void WiredTigerKVEngine::dropSomeQueuedIdents() { @@ -597,7 +623,7 @@ void WiredTigerKVEngine::dropSomeQueuedIdents() { if (_identToDrop.empty()) break; uri = _identToDrop.front(); - _identToDrop.pop(); + _identToDrop.pop_front(); } int ret = session.getSession()->drop( session.getSession(), uri.c_str(), "force,checkpoint_wait=false"); @@ -605,7 +631,7 @@ void WiredTigerKVEngine::dropSomeQueuedIdents() { if (ret == EBUSY) { stdx::lock_guard lk(_identToDropMutex); - _identToDrop.push(uri); + _identToDrop.push_back(uri); } else { invariantWTOK(ret); } diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h index 8c5a54f6bb4..0085c5d5cf7 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h @@ -31,7 +31,7 @@ #pragma once -#include +#include #include #include @@ -133,6 +133,8 @@ public: return _conn; } void dropSomeQueuedIdents(); + std::list filterCursorsWithQueuedDrops( + std::list* cache); bool haveDropsQueued() const; void syncSizeInfo(bool sync) const; @@ -177,7 +179,7 @@ private: mutable stdx::mutex _dropAllQueuesMutex; mutable stdx::mutex _identToDropMutex; - std::queue _identToDrop; + std::list _identToDrop; mutable Date_t _previousCheckedDropsQueued; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp index bd6bdceb5e9..3ed0b96f6df 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp @@ -92,6 +92,7 @@ bool shouldUseOplogHack(OperationContext* opCtx, const std::string& uri) { } // namespace MONGO_FP_DECLARE(WTWriteConflictException); +MONGO_FP_DECLARE(WTWriteConflictExceptionForReads); MONGO_FP_DECLARE(WTPausePrimaryOplogDurabilityLoop); const std::string kWiredTigerEngineName = "wiredTiger"; @@ -450,7 +451,7 @@ public: // Nothing after the next line can throw WCEs. // Note that an unpositioned (or eof) WT_CURSOR returns the first/last entry in the // table when you call next/prev. - int advanceRet = WT_OP_CHECK(_forward ? c->next(c) : c->prev(c)); + int advanceRet = WT_READ_CHECK(_forward ? c->next(c) : c->prev(c)); if (advanceRet == WT_NOTFOUND) { _eof = true; return {}; @@ -489,7 +490,7 @@ public: WT_CURSOR* c = _cursor->get(); c->set_key(c, _makeKey(id)); // Nothing after the next line can throw WCEs. - int seekRet = WT_OP_CHECK(c->search(c)); + int seekRet = WT_READ_CHECK(c->search(c)); if (seekRet == WT_NOTFOUND) { _eof = true; return {}; @@ -538,7 +539,7 @@ public: c->set_key(c, _makeKey(_lastReturnedId)); int cmp; - int ret = WT_OP_CHECK(c->search_near(c, &cmp)); + int ret = WT_READ_CHECK(c->search_near(c, &cmp)); if (ret == WT_NOTFOUND) { _eof = true; return !_rs._isCapped; @@ -641,7 +642,7 @@ public: } boost::optional next() final { - int advanceRet = WT_OP_CHECK(_cursor->next(_cursor)); + int advanceRet = WT_READ_CHECK(_cursor->next(_cursor)); if (advanceRet == WT_NOTFOUND) return {}; invariantWTOK(advanceRet); @@ -940,7 +941,7 @@ RecordData WiredTigerRecordStore::dataFor(OperationContext* txn, const RecordId& WT_CURSOR* c = curwrap.get(); invariant(c); c->set_key(c, _makeKey(id)); - int ret = WT_OP_CHECK(c->search(c)); + int ret = WT_READ_CHECK(c->search(c)); massert(28556, "Didn't find RecordId in WiredTigerRecordStore", ret != WT_NOTFOUND); invariantWTOK(ret); return _getData(curwrap); @@ -953,7 +954,7 @@ bool WiredTigerRecordStore::findRecord(OperationContext* txn, WT_CURSOR* c = curwrap.get(); invariant(c); c->set_key(c, _makeKey(id)); - int ret = WT_OP_CHECK(c->search(c)); + int ret = WT_READ_CHECK(c->search(c)); if (ret == WT_NOTFOUND) { return false; } @@ -971,7 +972,7 @@ void WiredTigerRecordStore::deleteRecord(OperationContext* txn, const RecordId& cursor.assertInActiveTxn(); WT_CURSOR* c = cursor.get(); c->set_key(c, _makeKey(id)); - int ret = WT_OP_CHECK(c->search(c)); + int ret = WT_READ_CHECK(c->search(c)); invariantWTOK(ret); WT_ITEM old_value; @@ -1078,7 +1079,7 @@ int64_t WiredTigerRecordStore::cappedDeleteAsNeeded_inlock(OperationContext* txn if (_cappedFirstRecord != RecordId()) { int64_t key = _makeKey(_cappedFirstRecord); truncateEnd->set_key(truncateEnd, key); - ret = WT_OP_CHECK(truncateEnd->search(truncateEnd)); + ret = WT_READ_CHECK(truncateEnd->search(truncateEnd)); if (ret == 0) { positioned = true; savedFirstKey = key; @@ -1087,7 +1088,7 @@ int64_t WiredTigerRecordStore::cappedDeleteAsNeeded_inlock(OperationContext* txn // Advance the cursor truncateEnd until we find a suitable end point for our truncate while ((sizeSaved < sizeOverCap || docsRemoved < docsOverCap) && (docsRemoved < 20000) && - (positioned || (ret = WT_OP_CHECK(truncateEnd->next(truncateEnd))) == 0)) { + (positioned || (ret = WT_READ_CHECK(truncateEnd->next(truncateEnd))) == 0)) { positioned = false; int64_t key; invariantWTOK(truncateEnd->get_key(truncateEnd, &key)); @@ -1122,7 +1123,7 @@ int64_t WiredTigerRecordStore::cappedDeleteAsNeeded_inlock(OperationContext* txn if (docsRemoved > 0) { // if we scanned to the end of the collection or past our insert, go back one if (ret == WT_NOTFOUND || newestIdToDelete >= justInserted) { - ret = WT_OP_CHECK(truncateEnd->prev(truncateEnd)); + ret = WT_READ_CHECK(truncateEnd->prev(truncateEnd)); } invariantWTOK(ret); @@ -1274,7 +1275,7 @@ Status WiredTigerRecordStore::insertRecords(OperationContext* txn, } else if (_isCapped) { stdx::lock_guard lk(_uncommittedRecordIdsMutex); record.id = _nextId(); - _addUncommitedRecordId_inlock(txn, record.id); + _addUncommittedRecordId_inlock(txn, record.id); } else { record.id = _nextId(); } @@ -1324,7 +1325,7 @@ StatusWith WiredTigerRecordStore::insertRecord(OperationContext* txn, } void WiredTigerRecordStore::_dealtWithCappedId(SortedRecordIds::iterator it, bool didCommit) { - invariant(&(*it) != NULL); + invariant(it->isNormal()); stdx::lock_guard lk(_uncommittedRecordIdsMutex); if (didCommit && _isOplog && *it != _oplog_highestSeen) { // Defer removal from _uncommittedRecordIds until it is durable. We don't need to wait for @@ -1379,7 +1380,7 @@ StatusWith WiredTigerRecordStore::updateRecord(OperationContext* txn, WT_CURSOR* c = curwrap.get(); invariant(c); c->set_key(c, _makeKey(id)); - int ret = WT_OP_CHECK(c->search(c)); + int ret = WT_READ_CHECK(c->search(c)); invariantWTOK(ret); WT_ITEM old_value; @@ -1464,7 +1465,7 @@ std::vector> WiredTigerRecordStore::getManyCursors Status WiredTigerRecordStore::truncate(OperationContext* txn) { WiredTigerCursor startWrap(_uri, _tableId, true, txn); WT_CURSOR* start = startWrap.get(); - int ret = WT_OP_CHECK(start->next(start)); + int ret = WT_READ_CHECK(start->next(start)); // Empty collections don't have anything to truncate. if (ret == WT_NOTFOUND) { return Status::OK(); @@ -1621,7 +1622,7 @@ Status WiredTigerRecordStore::oplogDiskLocRegister(OperationContext* txn, const return id.getStatus(); stdx::lock_guard lk(_uncommittedRecordIdsMutex); - _addUncommitedRecordId_inlock(txn, id.getValue()); + _addUncommittedRecordId_inlock(txn, id.getValue()); return Status::OK(); } @@ -1703,11 +1704,10 @@ void WiredTigerRecordStore::waitForAllEarlierOplogWritesToBeVisible(OperationCon } } -void WiredTigerRecordStore::_addUncommitedRecordId_inlock(OperationContext* txn, - const RecordId& id) { - // todo: make this a dassert at some point - // invariant(_uncommittedRecordIds.empty() || _uncommittedRecordIds.back() < id); +void WiredTigerRecordStore::_addUncommittedRecordId_inlock(OperationContext* txn, RecordId id) { + dassert(_uncommittedRecordIds.empty() || _uncommittedRecordIds.back() < id); SortedRecordIds::iterator it = _uncommittedRecordIds.insert(_uncommittedRecordIds.end(), id); + invariant(it->isNormal()); txn->recoveryUnit()->registerChange(new CappedInsertChange(this, it)); _oplog_highestSeen = id; } @@ -1727,7 +1727,7 @@ boost::optional WiredTigerRecordStore::oplogStartHack( int cmp; c->set_key(c, _makeKey(startingPosition)); - int ret = WT_OP_CHECK(c->search_near(c, &cmp)); + int ret = WT_READ_CHECK(c->search_near(c, &cmp)); if (ret == 0 && cmp > 0) ret = c->prev(c); // landed one higher than startingPosition if (ret == WT_NOTFOUND) diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h index 1fab25f8be4..6a439793ee0 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h @@ -47,10 +47,17 @@ /** * Either executes the specified operation and returns it's value or randomly throws a write - * conflict exception if the WTWriteConflictException failpoint is enabled. + * conflict exception if the WTWriteConflictException failpoint is enabled. This is only checked + * on cursor methods that make modifications. */ #define WT_OP_CHECK(x) (((MONGO_FAIL_POINT(WTWriteConflictException))) ? (WT_ROLLBACK) : (x)) +/** + * Identical to WT_OP_CHECK except this is checked on cursor seeks/advancement. + */ +#define WT_READ_CHECK(x) \ + (((MONGO_FAIL_POINT(WTWriteConflictExceptionForReads))) ? (WT_ROLLBACK) : (x)) + namespace mongo { class RecoveryUnit; @@ -264,7 +271,7 @@ private: static RecordId _fromKey(int64_t k); void _dealtWithCappedId(SortedRecordIds::iterator it, bool didCommit); - void _addUncommitedRecordId_inlock(OperationContext* txn, const RecordId& id); + void _addUncommittedRecordId_inlock(OperationContext* txn, RecordId id); RecordId _nextId(); void _setId(RecordId id); @@ -326,6 +333,7 @@ private: // WT failpoint to throw write conflict exceptions randomly MONGO_FP_FORWARD_DECLARE(WTWriteConflictException); +MONGO_FP_FORWARD_DECLARE(WTWriteConflictExceptionForReads); // Prevents oplog writes from being considered durable on the primary. Once activated, new writes // will not be considered durable until deactivated. It is unspecified whether writes that commit diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp index 58fd74c2ee0..6e2d6643d92 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp @@ -206,10 +206,10 @@ void WiredTigerRecoveryUnit::_txnClose(bool commit) { WT_SESSION* s = _session->getSession(); if (commit) { invariantWTOK(s->commit_transaction(s, NULL)); - LOG(3) << "WT commit_transaction"; + LOG(3) << "WT commit_transaction for snapshot id " << _mySnapshotId; } else { invariantWTOK(s->rollback_transaction(s, NULL)); - LOG(3) << "WT rollback_transaction"; + LOG(3) << "WT rollback_transaction for snapshot id " << _mySnapshotId; } _active = false; _mySnapshotId = nextSnapshotId.fetchAndAdd(1); @@ -252,7 +252,7 @@ void WiredTigerRecoveryUnit::_txnOpen(OperationContext* opCtx) { invariantWTOK(s->begin_transaction(s, NULL)); } - LOG(3) << "WT begin_transaction"; + LOG(3) << "WT begin_transaction for snapshot id " << _mySnapshotId; _timer.reset(); _active = true; } diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp index 54fff70be4d..cb1282a6790 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp @@ -106,16 +106,31 @@ void WiredTigerSession::releaseCursor(uint64_t id, WT_CURSOR* cursor) { } } -void WiredTigerSession::closeAllCursors(uint64_t cursorEpoch) { +void WiredTigerSession::closeAllCursors(const std::string& uri) { invariant(_session); - for (CursorCache::iterator i = _cursors.begin(); i != _cursors.end(); ++i) { + + for (auto i = _cursors.begin(); i != _cursors.end();) { + WT_CURSOR* cursor = i->_cursor; + if (cursor && uri == cursor->uri) { + invariantWTOK(cursor->close(cursor)); + i = _cursors.erase(i); + } else + ++i; + } +} + +void WiredTigerSession::closeCursorsForQueuedDrops(uint64_t cursorEpoch, + WiredTigerKVEngine* engine) { + invariant(_session); + + auto toDrop = engine->filterCursorsWithQueuedDrops(&_cursors); + + for (auto i = toDrop.begin(); i != toDrop.end(); i++) { WT_CURSOR* cursor = i->_cursor; if (cursor) { invariantWTOK(cursor->close(cursor)); } } - _cursors.clear(); - _cursorEpoch = cursorEpoch; } namespace { @@ -221,13 +236,20 @@ void WiredTigerSessionCache::waitUntilDurable(bool forceCheckpoint) { _journalListener->onDurable(token); } -void WiredTigerSessionCache::closeAllCursors() { +void WiredTigerSessionCache::closeAllCursors(const std::string& uri) { + stdx::lock_guard lock(_cacheLock); + for (SessionCache::iterator i = _sessions.begin(); i != _sessions.end(); i++) { + (*i)->closeAllCursors(uri); + } +} + +void WiredTigerSessionCache::closeCursorsForQueuedDrops() { // Increment the cursor epoch so that all cursors from this epoch are closed. uint64_t cursorEpoch = _cursorEpoch.addAndFetch(1); stdx::lock_guard lock(_cacheLock); for (SessionCache::iterator i = _sessions.begin(); i != _sessions.end(); i++) { - (*i)->closeAllCursors(cursorEpoch); + (*i)->closeCursorsForQueuedDrops(cursorEpoch, _engine); } } @@ -288,19 +310,22 @@ void WiredTigerSessionCache::releaseSession(WiredTigerSession* session) { return; } - // This checks that we are only caching idle sessions and not something which might hold - // locks or otherwise prevent truncation. { WT_SESSION* ss = session->getSession(); uint64_t range; + // This checks that we are only caching idle sessions and not something which might hold + // locks or otherwise prevent truncation. invariantWTOK(ss->transaction_pinned_range(ss, &range)); invariant(range == 0); + + // Release resources in the session we're about to cache. + invariantWTOK(ss->reset(ss)); } // If the cursor epoch has moved on, close all cursors in the session. uint64_t cursorEpoch = _cursorEpoch.load(); if (session->_getCursorEpoch() != cursorEpoch) - session->closeAllCursors(cursorEpoch); + session->closeCursorsForQueuedDrops(cursorEpoch, _engine); bool returnedToCache = false; uint64_t currentEpoch = _epoch.load(); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h index 4cd482f9772..b0030fe5f20 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h @@ -83,7 +83,9 @@ public: void releaseCursor(uint64_t id, WT_CURSOR* cursor); - void closeAllCursors(uint64_t cursorEpoch = 0); + void closeCursorsForQueuedDrops(uint64_t cursorEpoch, WiredTigerKVEngine* engine); + + void closeAllCursors(const std::string& uri); int cursorsOut() const { return _cursorsOut; @@ -149,11 +151,16 @@ public: */ void closeAll(); + /** + * Closes cached cursors for tables that are queued to be dropped. + */ + void closeCursorsForQueuedDrops(); + /** * Closes all cached cursors and ensures that previously opened cursors will be closed on * release. */ - void closeAllCursors(); + void closeAllCursors(const std::string& uri); /** * Transitions the cache to shutting down mode. Any already released sessions are freed and diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp index b840815e3a5..64eb665f9e2 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp @@ -396,9 +396,9 @@ int WiredTigerUtil::verifyTable(OperationContext* txn, ErrorAccumulator eventHandler(errors); // Try to close as much as possible to avoid EBUSY errors. - WiredTigerRecoveryUnit::get(txn)->getSession(txn)->closeAllCursors(); + WiredTigerRecoveryUnit::get(txn)->getSession(txn)->closeAllCursors(uri); WiredTigerSessionCache* sessionCache = WiredTigerRecoveryUnit::get(txn)->getSessionCache(); - sessionCache->closeAll(); + sessionCache->closeAllCursors(uri); // Open a new session with custom error handlers. WT_CONNECTION* conn = WiredTigerRecoveryUnit::get(txn)->getSessionCache()->conn(); diff --git a/src/mongo/db/write_concern.cpp b/src/mongo/db/write_concern.cpp index 3c9086ca39a..9dc04670dc4 100644 --- a/src/mongo/db/write_concern.cpp +++ b/src/mongo/db/write_concern.cpp @@ -45,6 +45,7 @@ #include "mongo/db/storage/storage_engine.h" #include "mongo/db/write_concern_options.h" #include "mongo/rpc/protocol.h" +#include "mongo/util/fail_point_service.h" #include "mongo/util/log.h" namespace mongo { @@ -72,6 +73,8 @@ namespace { const std::string kLocalDB = "local"; } // namespace +MONGO_FP_DECLARE(hangBeforeWaitingForWriteConcern); + StatusWith extractWriteConcern(OperationContext* txn, const BSONObj& cmdObj, const std::string& dbName) { @@ -234,6 +237,8 @@ Status waitForWriteConcern(OperationContext* txn, // This check does not hold for writes done through dbeval because it runs with a global X lock. dassert(!txn->lockState()->isLocked() || txn->getClient()->isInDirectClient()); + MONGO_FAIL_POINT_PAUSE_WHILE_SET(hangBeforeWaitingForWriteConcern); + // Next handle blocking on disk Timer syncTimer; diff --git a/src/mongo/dbtests/SConscript b/src/mongo/dbtests/SConscript index f56098bfa1f..6f4d062df7c 100644 --- a/src/mongo/dbtests/SConscript +++ b/src/mongo/dbtests/SConscript @@ -109,6 +109,7 @@ dbtest = env.Program( 'query_stage_update.cpp', 'querytests.cpp', 'replica_set_monitor_test.cpp', + 'replica_set_tests.cpp', 'repltests.cpp', 'rollbacktests.cpp', 'socktests.cpp', diff --git a/src/mongo/dbtests/replica_set_tests.cpp b/src/mongo/dbtests/replica_set_tests.cpp new file mode 100644 index 00000000000..0d9a7cc3a3b --- /dev/null +++ b/src/mongo/dbtests/replica_set_tests.cpp @@ -0,0 +1,127 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/client.h" +#include "mongo/db/dbdirectclient.h" +#include "mongo/db/repl/last_vote.h" +#include "mongo/db/repl/replication_coordinator_external_state_impl.h" +#include "mongo/db/service_context.h" +#include "mongo/stdx/memory.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace { + +ServiceContext::UniqueOperationContext makeOpCtx() { + return cc().makeOperationContext(); +} + +class ReplicaSetTest : public mongo::unittest::Test { +protected: + void setUp() { + auto txn = makeOpCtx(); + _replCoordExternalState.reset(new repl::ReplicationCoordinatorExternalStateImpl()); + } + + void tearDown() { + auto txn = makeOpCtx(); + DBDirectClient client(txn.get()); + client.dropCollection("local.replset.election"); + + _replCoordExternalState.reset(); + } + + repl::ReplicationCoordinatorExternalStateImpl* getReplCoordExternalState() { + return _replCoordExternalState.get(); + } + +private: + std::unique_ptr _replCoordExternalState; +}; + +TEST_F(ReplicaSetTest, ReplCoordExternalStateStoresLastVoteWithNewTerm) { + auto txn = makeOpCtx(); + auto replCoordExternalState = getReplCoordExternalState(); + + replCoordExternalState->storeLocalLastVoteDocument(txn.get(), repl::LastVote{2, 1}); + + auto lastVote = replCoordExternalState->loadLocalLastVoteDocument(txn.get()); + ASSERT_OK(lastVote.getStatus()); + ASSERT_EQ(lastVote.getValue().getTerm(), 2); + ASSERT_EQ(lastVote.getValue().getCandidateIndex(), 1); + + replCoordExternalState->storeLocalLastVoteDocument(txn.get(), repl::LastVote{3, 1}); + + lastVote = replCoordExternalState->loadLocalLastVoteDocument(txn.get()); + ASSERT_OK(lastVote.getStatus()); + ASSERT_EQ(lastVote.getValue().getTerm(), 3); + ASSERT_EQ(lastVote.getValue().getCandidateIndex(), 1); +} + +TEST_F(ReplicaSetTest, ReplCoordExternalStateDoesNotStoreLastVoteWithOldTerm) { + auto txn = makeOpCtx(); + auto replCoordExternalState = getReplCoordExternalState(); + + replCoordExternalState->storeLocalLastVoteDocument(txn.get(), repl::LastVote{2, 1}); + + auto lastVote = replCoordExternalState->loadLocalLastVoteDocument(txn.get()); + ASSERT_OK(lastVote.getStatus()); + ASSERT_EQ(lastVote.getValue().getTerm(), 2); + ASSERT_EQ(lastVote.getValue().getCandidateIndex(), 1); + + replCoordExternalState->storeLocalLastVoteDocument(txn.get(), repl::LastVote{1, 1}); + + lastVote = replCoordExternalState->loadLocalLastVoteDocument(txn.get()); + ASSERT_OK(lastVote.getStatus()); + ASSERT_EQ(lastVote.getValue().getTerm(), 2); + ASSERT_EQ(lastVote.getValue().getCandidateIndex(), 1); +} + +TEST_F(ReplicaSetTest, ReplCoordExternalStateDoesNotStoreLastVoteWithEqualTerm) { + auto txn = makeOpCtx(); + auto replCoordExternalState = getReplCoordExternalState(); + + replCoordExternalState->storeLocalLastVoteDocument(txn.get(), repl::LastVote{2, 1}); + + auto lastVote = replCoordExternalState->loadLocalLastVoteDocument(txn.get()); + ASSERT_OK(lastVote.getStatus()); + ASSERT_EQ(lastVote.getValue().getTerm(), 2); + ASSERT_EQ(lastVote.getValue().getCandidateIndex(), 1); + + replCoordExternalState->storeLocalLastVoteDocument(txn.get(), repl::LastVote{2, 2}); + + lastVote = replCoordExternalState->loadLocalLastVoteDocument(txn.get()); + ASSERT_OK(lastVote.getStatus()); + ASSERT_EQ(lastVote.getValue().getTerm(), 2); + ASSERT_EQ(lastVote.getValue().getCandidateIndex(), 1); +} + +} // namespace +} // namespace mongo diff --git a/src/mongo/executor/SConscript b/src/mongo/executor/SConscript index c8ff0e69fd0..ec25fc15952 100644 --- a/src/mongo/executor/SConscript +++ b/src/mongo/executor/SConscript @@ -149,6 +149,7 @@ env.Library( '$BUILD_DIR/mongo/base/system_error', '$BUILD_DIR/mongo/db/auth/authcommon', '$BUILD_DIR/mongo/db/commands/test_commands_enabled', + '$BUILD_DIR/mongo/rpc/command_status', '$BUILD_DIR/mongo/rpc/rpc', '$BUILD_DIR/third_party/shim_asio', 'async_stream', diff --git a/src/mongo/executor/async_mock_stream_factory.cpp b/src/mongo/executor/async_mock_stream_factory.cpp index 2a2d8322b59..9613f19cf91 100644 --- a/src/mongo/executor/async_mock_stream_factory.cpp +++ b/src/mongo/executor/async_mock_stream_factory.cpp @@ -212,7 +212,7 @@ void AsyncMockStreamFactory::MockStream::read(asio::mutable_buffer buf, void AsyncMockStreamFactory::MockStream::pushRead(std::vector toRead) { stdx::unique_lock lk(_mutex); - invariant(_state != kRunning && _state != kCanceled); + invariant(_state != kRunning); _readQueue.emplace(std::move(toRead)); } @@ -222,7 +222,7 @@ void AsyncMockStreamFactory::MockStream::setError(std::error_code ec) { std::vector AsyncMockStreamFactory::MockStream::popWrite() { stdx::unique_lock lk(_mutex); - invariant(_state != kRunning && _state != kCanceled); + invariant(_state != kRunning); auto nextWrite = std::move(_writeQueue.front()); _writeQueue.pop(); return nextWrite; diff --git a/src/mongo/executor/async_mock_stream_factory.h b/src/mongo/executor/async_mock_stream_factory.h index 11c10128065..f7897806281 100644 --- a/src/mongo/executor/async_mock_stream_factory.h +++ b/src/mongo/executor/async_mock_stream_factory.h @@ -50,6 +50,49 @@ namespace executor { class AsyncStreamInterface; +/** + * A factory that produces mock streams to allow for testing of NetworkInterfaceASIO. + * + * The streams produced by this factory simulate a flow of Events (ConnectEvent, + * ReadEvent, WriteEvent). The streams created by this factory will automatically + * pause themselves at each Event, and the caller must unblock them by destroying + * the Event object to continue. + * + * Example use of this factory: + * + * AsyncMockStreamFactory factory(); + * + * // NIA will then call makeStream(...) to create new streams from the + * // factory, or the caller can do this manually. + * + * // Wait for the desired stream to exist + * auto stream = streamFactory.blockUntilStreamExists(host); + * + * // If we do not care to inspect after a certain event, we can skip it: + * ConnectEvent{stream}.skip(); + * + * // To examine the stream at an Event, instantiate the event object. + * // When the Event object goes out of scope the stream will unblock. + * { + * WriteEvent write{stream}; + * + * // Inspect what NIA wrote to this stream: + * auto messageData = stream->popWrite(); + * ... + * } + * + * // The Event object will keep the stream blocked as long as it exists. + * // Use this window to perform operations on the stream or inspect it. + * { + * ReadEvent read{stream}; + * + * // Simulate data sent to this stream over the network + * stream->pushRead( ... ); + * + * // Or, simulate a networking error + * stream->setError( error_code ); + * } + */ class AsyncMockStreamFactory final : public AsyncStreamFactoryInterface { public: AsyncMockStreamFactory() = default; @@ -57,6 +100,22 @@ public: std::unique_ptr makeStream(asio::io_service::strand* strand, const HostAndPort& host) override; + /** + * A mock stream class for testing the egress networking layer. + * + * At the core of this class is an idea of deferring actions and allowing inspection + * of state of the stream before those actions happen. + * + * This class operates on the assumption that two threads are in use: a networking + * thread used by NIA to issue IO calls on the MockStream, and a test thread to + * wait on those calls and react. + * + * When the test thread creates an Event object, the constructor sends it to wait + * on a condition variable. When NIA issues an IO call on the stream, the MockStream + * load the proper handler into a placeholder, and then calls notify() on the + * condition variable. At that point the stream is paused and the test thread + * may operate on it. + */ class MockStream final : public AsyncStreamInterface { public: MockStream(asio::io_service::strand* strand, diff --git a/src/mongo/executor/async_stream_test.cpp b/src/mongo/executor/async_stream_test.cpp index 508e81aa5f5..a34c39f9b1f 100644 --- a/src/mongo/executor/async_stream_test.cpp +++ b/src/mongo/executor/async_stream_test.cpp @@ -134,7 +134,11 @@ TEST(AsyncStreamTest, IsOpen) { server.shutdown(); - ASSERT_FALSE(stream.isOpen()); + // There is nothing we can wait on to determinstically know when + // the socket will transition to closed. Busy wait for that. + while (stream.isOpen()) { + stdx::this_thread::sleep_for(Milliseconds(1)); + } } } // namespace diff --git a/src/mongo/executor/connection_pool.cpp b/src/mongo/executor/connection_pool.cpp index ee243789587..79461a9a5c1 100644 --- a/src/mongo/executor/connection_pool.cpp +++ b/src/mongo/executor/connection_pool.cpp @@ -1,4 +1,5 @@ -/** * Copyright (C) 2015 MongoDB Inc. +/** + * Copyright (C) 2015 MongoDB Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License, version 3, @@ -93,11 +94,23 @@ public: */ size_t availableConnections(const stdx::unique_lock& lk); + /** + * Returns the number of in progress connections in the pool. + */ + size_t refreshingConnections(const stdx::unique_lock& lk); + /** * Returns the total number of connections ever created in this pool. */ size_t createdConnections(const stdx::unique_lock& lk); + /** + * Returns the total number of connections currently open that belong to + * this pool. This is the sum of refreshingConnections, availableConnections, + * and inUseConnections. + */ + size_t openConnections(const stdx::unique_lock& lk); + private: using OwnedConnection = std::unique_ptr; using OwnershipPool = std::unordered_map; @@ -112,7 +125,7 @@ private: void fulfillRequests(stdx::unique_lock& lk); - void spawnConnections(stdx::unique_lock& lk, const HostAndPort& hostAndPort); + void spawnConnections(stdx::unique_lock& lk); void shutdown(); @@ -137,6 +150,7 @@ private: Date_t _requestTimerExpiration; size_t _generation; bool _inFulfillRequests; + bool _inSpawnConnections; size_t _created; @@ -171,12 +185,15 @@ size_t const ConnectionPool::kDefaultMaxConns = std::numeric_limits::max size_t const ConnectionPool::kDefaultMinConns = 1; const Milliseconds ConnectionPool::kDefaultRefreshRequirement = Minutes(1); const Milliseconds ConnectionPool::kDefaultRefreshTimeout = Seconds(20); +size_t const ConnectionPool::kDefaultMaxConnecting = std::numeric_limits::max(); const Status ConnectionPool::kConnectionStateUnknown = Status(ErrorCodes::InternalError, "Connection is in an unknown state"); -ConnectionPool::ConnectionPool(std::unique_ptr impl, Options options) - : _options(std::move(options)), _factory(std::move(impl)) {} +ConnectionPool::ConnectionPool(std::unique_ptr impl, + std::string name, + Options options) + : _name(std::move(name)), _options(std::move(options)), _factory(std::move(impl)) {} ConnectionPool::~ConnectionPool() = default; @@ -221,13 +238,24 @@ void ConnectionPool::appendConnectionStats(ConnectionPoolStats* stats) const { HostAndPort host = kv.first; auto& pool = kv.second; - ConnectionStatsPerHost hostStats{pool->inUseConnections(lk), - pool->availableConnections(lk), - pool->createdConnections(lk)}; - stats->updateStatsForHost(host, hostStats); + ConnectionStatsPer hostStats{pool->inUseConnections(lk), + pool->availableConnections(lk), + pool->createdConnections(lk), + pool->refreshingConnections(lk)}; + stats->updateStatsForHost(_name, host, hostStats); } } +size_t ConnectionPool::getNumConnectionsPerHost(const HostAndPort& hostAndPort) const { + stdx::unique_lock lk(_mutex); + auto iter = _pools.find(hostAndPort); + if (iter != _pools.end()) { + return iter->second->openConnections(lk); + } + + return 0; +} + void ConnectionPool::returnConnection(ConnectionInterface* conn) { stdx::unique_lock lk(_mutex); @@ -244,6 +272,7 @@ ConnectionPool::SpecificPool::SpecificPool(ConnectionPool* parent, const HostAnd _requestTimer(parent->_factory->makeTimer()), _generation(0), _inFulfillRequests(false), + _inSpawnConnections(false), _created(0), _state(State::kRunning) {} @@ -260,26 +289,34 @@ size_t ConnectionPool::SpecificPool::availableConnections( return _readyPool.size(); } +size_t ConnectionPool::SpecificPool::refreshingConnections( + const stdx::unique_lock& lk) { + return _processingPool.size(); +} + size_t ConnectionPool::SpecificPool::createdConnections(const stdx::unique_lock& lk) { return _created; } +size_t ConnectionPool::SpecificPool::openConnections(const stdx::unique_lock& lk) { + return _checkedOutPool.size() + _readyPool.size() + _processingPool.size(); +} + void ConnectionPool::SpecificPool::getConnection(const HostAndPort& hostAndPort, Milliseconds timeout, stdx::unique_lock lk, GetConnectionCallback cb) { - // We need some logic here to handle kNoTimeout, which is defined as -1 Milliseconds. If we just - // added the timeout, we would get a time 1MS in the past, which would immediately timeout - the - // exact opposite of what we want. - auto expiration = (timeout == RemoteCommandRequest::kNoTimeout) - ? RemoteCommandRequest::kNoExpirationDate - : _parent->_factory->now() + timeout; + if (timeout < Milliseconds(0) || timeout > _parent->_options.refreshTimeout) { + timeout = _parent->_options.refreshTimeout; + } + + const auto expiration = _parent->_factory->now() + timeout; _requests.push(make_pair(expiration, std::move(cb))); updateStateInLock(); - spawnConnections(lk, hostAndPort); + spawnConnections(lk); fulfillRequests(lk); } @@ -302,6 +339,8 @@ void ConnectionPool::SpecificPool::returnConnection(ConnectionInterface* connPtr if (!conn->getStatus().isOK()) { // TODO: alert via some callback if the host is bad + log() << "Ending connection to host " << _hostAndPort << " due to bad connection status; " + << openConnections(lk) << " connections to that host remain open"; return; } @@ -312,6 +351,9 @@ void ConnectionPool::SpecificPool::returnConnection(ConnectionInterface* connPtr if (_readyPool.size() + _processingPool.size() + _checkedOutPool.size() >= _parent->_options.minConnections) { // If we already have minConnections, just let the connection lapse + log() << "Ending idle connection to host " << _hostAndPort + << " because the pool meets constraints; " << openConnections(lk) + << " connections to that host remain open"; return; } @@ -328,8 +370,10 @@ void ConnectionPool::SpecificPool::returnConnection(ConnectionInterface* connPtr auto conn = takeFromProcessingPool(connPtr); // If the host and port were dropped, let this lapse - if (conn->getGeneration() != _generation) + if (conn->getGeneration() != _generation) { + spawnConnections(lk); return; + } // If we're in shutdown, we don't need refreshed connections if (_state == State::kInShutdown) @@ -339,6 +383,19 @@ void ConnectionPool::SpecificPool::returnConnection(ConnectionInterface* connPtr // pool if (status.isOK()) { addToReady(lk, std::move(conn)); + spawnConnections(lk); + return; + } + + // If we've exceeded the time limit, start a new connect, rather than + // failing all operations. We do this because the various callers have + // their own time limit which is unrelated to our internal one. + if (status.code() == ErrorCodes::ExceededTimeLimit) { + log() << "Pending connection to host " << _hostAndPort + << " did not complete within the connection timeout," + << " retrying with a new connection;" << openConnections(lk) + << " connections to that host remain open"; + spawnConnections(lk); return; } @@ -402,6 +459,10 @@ void ConnectionPool::SpecificPool::processFailure(const Status& status, // Drop ready connections _readyPool.clear(); + // Log something helpful + log() << "Dropping all pooled connections to " << _hostAndPort + << " due to failed operation on a connection"; + // Migrate processing connections to the dropped pool for (auto&& x : _processingPool) { _droppedProcessingPool[x.first] = std::move(x.second); @@ -456,7 +517,7 @@ void ConnectionPool::SpecificPool::fulfillRequests(stdx::unique_lockgetHostAndPort()); + spawnConnections(lk); } // Drop the bad connection. @@ -486,8 +547,15 @@ void ConnectionPool::SpecificPool::fulfillRequests(stdx::unique_lock& lk, - const HostAndPort& hostAndPort) { +void ConnectionPool::SpecificPool::spawnConnections(stdx::unique_lock& lk) { + // If some other thread (possibly this thread) is spawning connections, + // don't keep padding the callstack. + if (_inSpawnConnections) + return; + + _inSpawnConnections = true; + auto guard = MakeGuard([&] { _inSpawnConnections = false; }); + // We want minConnections <= outstanding requests <= maxConnections auto target = [&] { return std::max( @@ -496,11 +564,12 @@ void ConnectionPool::SpecificPool::spawnConnections(stdx::unique_lock_options.maxConnecting)) { std::unique_ptr handle; try { // make a new connection and put it in processing - handle = _parent->_factory->makeConnection(hostAndPort, _generation); + handle = _parent->_factory->makeConnection(_hostAndPort, _generation); } catch (std::system_error& e) { severe() << "Failed to construct a new connection object: " << e.what(); fassertFailed(40336); @@ -524,8 +593,15 @@ void ConnectionPool::SpecificPool::spawnConnections(stdx::unique_lockgetGeneration() != _generation) { // If the host and port was dropped, let the // connection lapse + spawnConnections(lk); } else if (status.isOK()) { addToReady(lk, std::move(conn)); + spawnConnections(lk); + } else if (status.code() == ErrorCodes::ExceededTimeLimit) { + // If we've exceeded the time limit, restart the connect, rather than + // failing all operations. We do this because the various callers + // have their own time limit which is unrelated to our internal one. + spawnConnections(lk); } else { // If the setup failed, cascade the failure edge processFailure(status, std::move(lk)); diff --git a/src/mongo/executor/connection_pool.h b/src/mongo/executor/connection_pool.h index 83dbe54d240..ad5dec51a04 100644 --- a/src/mongo/executor/connection_pool.h +++ b/src/mongo/executor/connection_pool.h @@ -72,6 +72,7 @@ public: static const size_t kDefaultMinConns; static const Milliseconds kDefaultRefreshRequirement; static const Milliseconds kDefaultRefreshTimeout; + static const size_t kDefaultMaxConnecting; static const Status kConnectionStateUnknown; @@ -91,6 +92,13 @@ public: */ size_t maxConnections = kDefaultMaxConns; + /** + * The maximum number of processing connections for a host. This includes pending + * connections in setup/refresh. It's designed to rate limit connection storms rather than + * steady state processing (as maxConnections does). + */ + size_t maxConnecting = kDefaultMaxConnecting; + /** * Amount of time to wait before timing out a refresh attempt */ @@ -111,6 +119,7 @@ public: }; explicit ConnectionPool(std::unique_ptr impl, + std::string name, Options options = Options{}); ~ConnectionPool(); @@ -121,9 +130,13 @@ public: void appendConnectionStats(ConnectionPoolStats* stats) const; + size_t getNumConnectionsPerHost(const HostAndPort& hostAndPort) const; + private: void returnConnection(ConnectionInterface* connection); + std::string _name; + // Options are set at startup and never changed at run time, so these are // accessed outside the lock const Options _options; @@ -190,7 +203,6 @@ class ConnectionPool::ConnectionInterface : public TimerInterface { public: ConnectionInterface() = default; - virtual ~ConnectionInterface() = default; /** diff --git a/src/mongo/executor/connection_pool_stats.cpp b/src/mongo/executor/connection_pool_stats.cpp index 02ff34da6c1..64cc3bbc6eb 100644 --- a/src/mongo/executor/connection_pool_stats.cpp +++ b/src/mongo/executor/connection_pool_stats.cpp @@ -36,44 +36,73 @@ namespace mongo { namespace executor { -ConnectionStatsPerHost::ConnectionStatsPerHost(size_t nInUse, size_t nAvailable, size_t nCreated) - : inUse(nInUse), available(nAvailable), created(nCreated) {} +ConnectionStatsPer::ConnectionStatsPer(size_t nInUse, + size_t nAvailable, + size_t nCreated, + size_t nRefreshing) + : inUse(nInUse), available(nAvailable), created(nCreated), refreshing(nRefreshing) {} -ConnectionStatsPerHost::ConnectionStatsPerHost() = default; +ConnectionStatsPer::ConnectionStatsPer() = default; -ConnectionStatsPerHost& ConnectionStatsPerHost::operator+=(const ConnectionStatsPerHost& other) { +ConnectionStatsPer& ConnectionStatsPer::operator+=(const ConnectionStatsPer& other) { inUse += other.inUse; available += other.available; created += other.created; + refreshing += other.refreshing; return *this; } -void ConnectionPoolStats::updateStatsForHost(HostAndPort host, ConnectionStatsPerHost newStats) { +void ConnectionPoolStats::updateStatsForHost(std::string pool, + HostAndPort host, + ConnectionStatsPer newStats) { // Update stats for this host. - auto hostStats = mapFindWithDefault(statsByHost, host); - hostStats += newStats; - statsByHost.insert(std::make_pair(host, hostStats)); + statsByPool[pool] += newStats; + statsByHost[host] += newStats; + statsByPoolHost[pool][host] += newStats; // Update total connection stats. totalInUse += newStats.inUse; totalAvailable += newStats.available; totalCreated += newStats.created; + totalRefreshing += newStats.refreshing; } void ConnectionPoolStats::appendToBSON(mongo::BSONObjBuilder& result) { result.appendNumber("totalInUse", totalInUse); result.appendNumber("totalAvailable", totalAvailable); result.appendNumber("totalCreated", totalCreated); + result.appendNumber("totalRefreshing", totalRefreshing); - BSONObjBuilder hostBuilder(result.subobjStart("hosts")); - for (auto&& host : statsByHost) { - BSONObjBuilder hostInfo(hostBuilder.subobjStart(host.first.toString())); - - auto hostStats = host.second; - hostInfo.appendNumber("inUse", hostStats.inUse); - hostInfo.appendNumber("available", hostStats.available); - hostInfo.appendNumber("created", hostStats.created); + { + BSONObjBuilder poolBuilder(result.subobjStart("pools")); + for (auto&& pool : statsByPool) { + BSONObjBuilder poolInfo(poolBuilder.subobjStart(pool.first)); + auto poolStats = pool.second; + poolInfo.appendNumber("poolInUse", poolStats.inUse); + poolInfo.appendNumber("poolAvailable", poolStats.available); + poolInfo.appendNumber("poolCreated", poolStats.created); + poolInfo.appendNumber("poolRefreshing", poolStats.refreshing); + for (auto&& host : statsByPoolHost[pool.first]) { + BSONObjBuilder hostInfo(poolInfo.subobjStart(host.first.toString())); + auto hostStats = host.second; + hostInfo.appendNumber("inUse", hostStats.inUse); + hostInfo.appendNumber("available", hostStats.available); + hostInfo.appendNumber("created", hostStats.created); + hostInfo.appendNumber("refreshing", hostStats.refreshing); + } + } + } + { + BSONObjBuilder hostBuilder(result.subobjStart("hosts")); + for (auto&& host : statsByHost) { + BSONObjBuilder hostInfo(hostBuilder.subobjStart(host.first.toString())); + auto hostStats = host.second; + hostInfo.appendNumber("inUse", hostStats.inUse); + hostInfo.appendNumber("available", hostStats.available); + hostInfo.appendNumber("created", hostStats.created); + hostInfo.appendNumber("refreshing", hostStats.refreshing); + } } } diff --git a/src/mongo/executor/connection_pool_stats.h b/src/mongo/executor/connection_pool_stats.h index 5e9bd0c7633..148dd89facf 100644 --- a/src/mongo/executor/connection_pool_stats.h +++ b/src/mongo/executor/connection_pool_stats.h @@ -36,19 +36,20 @@ namespace mongo { namespace executor { /** - * Holds connection information for a specific remote host. These objects are maintained by + * Holds connection information for a specific pool or remote host. These objects are maintained by * a parent ConnectionPoolStats object and should not need to be created directly. */ -struct ConnectionStatsPerHost { - ConnectionStatsPerHost(size_t nInUse, size_t nAvailable, size_t nCreated); +struct ConnectionStatsPer { + ConnectionStatsPer(size_t nInUse, size_t nAvailable, size_t nCreated, size_t nRefreshing); - ConnectionStatsPerHost(); + ConnectionStatsPer(); - ConnectionStatsPerHost& operator+=(const ConnectionStatsPerHost& other); + ConnectionStatsPer& operator+=(const ConnectionStatsPer& other); size_t inUse = 0u; size_t available = 0u; size_t created = 0u; + size_t refreshing = 0u; }; /** @@ -57,15 +58,19 @@ struct ConnectionStatsPerHost { * Total connection counts will then be updated accordingly. */ struct ConnectionPoolStats { - void updateStatsForHost(HostAndPort host, ConnectionStatsPerHost newStats); + void updateStatsForHost(std::string pool, HostAndPort host, ConnectionStatsPer newStats); void appendToBSON(mongo::BSONObjBuilder& result); size_t totalInUse = 0u; size_t totalAvailable = 0u; size_t totalCreated = 0u; + size_t totalRefreshing = 0u; - std::unordered_map statsByHost; + std::unordered_map statsByPool; + std::unordered_map statsByHost; + std::unordered_map> + statsByPoolHost; }; } // namespace executor diff --git a/src/mongo/executor/connection_pool_test.cpp b/src/mongo/executor/connection_pool_test.cpp index 24949a142c7..8bd79f6f445 100644 --- a/src/mongo/executor/connection_pool_test.cpp +++ b/src/mongo/executor/connection_pool_test.cpp @@ -66,7 +66,7 @@ private: * another. */ TEST_F(ConnectionPoolTest, SameConn) { - ConnectionPool pool(stdx::make_unique()); + ConnectionPool pool(stdx::make_unique(), "test pool"); // Grab and stash an id for the first request size_t conn1Id = 0; @@ -98,7 +98,7 @@ TEST_F(ConnectionPoolTest, SameConn) { * Verify that a failed connection isn't returned to the pool */ TEST_F(ConnectionPoolTest, FailedConnDifferentConn) { - ConnectionPool pool(stdx::make_unique()); + ConnectionPool pool(stdx::make_unique(), "test pool"); // Grab the first connection and indicate that it failed size_t conn1Id = 0; @@ -131,7 +131,7 @@ TEST_F(ConnectionPoolTest, FailedConnDifferentConn) { * connections. */ TEST_F(ConnectionPoolTest, DifferentHostDifferentConn) { - ConnectionPool pool(stdx::make_unique()); + ConnectionPool pool(stdx::make_unique(), "test pool"); // Conn 1 from port 30000 size_t conn1Id = 0; @@ -163,7 +163,7 @@ TEST_F(ConnectionPoolTest, DifferentHostDifferentConn) { * Verify that not returning handle's to the pool spins up new connections. */ TEST_F(ConnectionPoolTest, DifferentConnWithoutReturn) { - ConnectionPool pool(stdx::make_unique()); + ConnectionPool pool(stdx::make_unique(), "test pool"); // Get the first connection, move it out rather than letting it return ConnectionPool::ConnectionHandle conn1; @@ -199,7 +199,7 @@ TEST_F(ConnectionPoolTest, DifferentConnWithoutReturn) { * Note that the lack of pushSetup() calls delays the get. */ TEST_F(ConnectionPoolTest, TimeoutOnSetup) { - ConnectionPool pool(stdx::make_unique()); + ConnectionPool pool(stdx::make_unique(), "test pool"); bool notOk = false; @@ -234,7 +234,7 @@ TEST_F(ConnectionPoolTest, refreshHappens) { ConnectionPool::Options options; options.refreshRequirement = Milliseconds(1000); - ConnectionPool pool(stdx::make_unique(), options); + ConnectionPool pool(stdx::make_unique(), "test pool", options); auto now = Date_t::now(); @@ -270,7 +270,7 @@ TEST_F(ConnectionPoolTest, refreshTimeoutHappens) { ConnectionPool::Options options; options.refreshRequirement = Milliseconds(1000); options.refreshTimeout = Milliseconds(2000); - ConnectionPool pool(stdx::make_unique(), options); + ConnectionPool pool(stdx::make_unique(), "test pool", options); auto now = Date_t::now(); @@ -308,13 +308,14 @@ TEST_F(ConnectionPoolTest, refreshTimeoutHappens) { // see if that pans out. In this case, we'll get a failure on timeout. ConnectionImpl::pushSetup(Status::OK()); pool.get(HostAndPort(), - Milliseconds(10000), + Milliseconds(1000), [&](StatusWith swConn) { ASSERT(!swConn.isOK()); reachedA = true; }); ASSERT(!reachedA); + PoolImpl::setNow(now + Milliseconds(3000)); // Let the refresh timeout PoolImpl::setNow(now + Milliseconds(4000)); @@ -338,7 +339,7 @@ TEST_F(ConnectionPoolTest, refreshTimeoutHappens) { * Verify that requests are served in expiration order, not insertion order */ TEST_F(ConnectionPoolTest, requestsServedByUrgency) { - ConnectionPool pool(stdx::make_unique()); + ConnectionPool pool(stdx::make_unique(), "test pool"); bool reachedA = false; bool reachedB = false; @@ -385,7 +386,7 @@ TEST_F(ConnectionPoolTest, maxPoolRespected) { ConnectionPool::Options options; options.minConnections = 1; options.maxConnections = 2; - ConnectionPool pool(stdx::make_unique(), options); + ConnectionPool pool(stdx::make_unique(), "test pool", options); ConnectionPool::ConnectionHandle conn1; ConnectionPool::ConnectionHandle conn2; @@ -436,6 +437,202 @@ TEST_F(ConnectionPoolTest, maxPoolRespected) { doneWith(conn3); } +/** + * Verify that we respect maxConnecting + */ +TEST_F(ConnectionPoolTest, maxConnectingRespected) { + ConnectionPool::Options options; + options.minConnections = 1; + options.maxConnecting = 2; + ConnectionPool pool(stdx::make_unique(), "test pool", options); + + ConnectionPool::ConnectionHandle conn1; + ConnectionPool::ConnectionHandle conn2; + ConnectionPool::ConnectionHandle conn3; + + // Make 3 requests, each which keep their connection (don't return it to + // the pool) + pool.get(HostAndPort(), + Milliseconds(3000), + [&](StatusWith swConn) { + ASSERT(swConn.isOK()); + + conn3 = std::move(swConn.getValue()); + }); + pool.get(HostAndPort(), + Milliseconds(2000), + [&](StatusWith swConn) { + ASSERT(swConn.isOK()); + + conn2 = std::move(swConn.getValue()); + }); + pool.get(HostAndPort(), + Milliseconds(1000), + [&](StatusWith swConn) { + ASSERT(swConn.isOK()); + + conn1 = std::move(swConn.getValue()); + }); + + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 2u); + ConnectionImpl::pushSetup(Status::OK()); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 2u); + ConnectionImpl::pushSetup(Status::OK()); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 1u); + ConnectionImpl::pushSetup(Status::OK()); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 0u); + + ASSERT(conn1); + ASSERT(conn2); + ASSERT(conn3); + + ASSERT_NE(conn1.get(), conn2.get()); + ASSERT_NE(conn2.get(), conn3.get()); + ASSERT_NE(conn1.get(), conn3.get()); + + doneWith(conn1); + doneWith(conn2); + doneWith(conn3); +} + +/** + * Verify that refresh callbacks block new connections, then trigger new connection spawns after + * they return + */ +TEST_F(ConnectionPoolTest, maxConnectingWithRefresh) { + ConnectionPool::Options options; + options.maxConnecting = 1; + options.refreshRequirement = Milliseconds(1000); + ConnectionPool pool(stdx::make_unique(), "test pool", options); + + auto now = Date_t::now(); + + PoolImpl::setNow(now); + + // Get a connection + ConnectionImpl::pushSetup(Status::OK()); + pool.get(HostAndPort(), + Milliseconds(5000), + [&](StatusWith swConn) { + ASSERT(swConn.isOK()); + doneWith(swConn.getValue()); + }); + + ASSERT_EQ(ConnectionImpl::refreshQueueDepth(), 0u); + + // After 1 second, one refresh has queued + PoolImpl::setNow(now + Milliseconds(1000)); + ASSERT_EQ(ConnectionImpl::refreshQueueDepth(), 1u); + + bool reachedA = false; + + // Try to get another connection + pool.get(HostAndPort(), + Milliseconds(5000), + [&](StatusWith swConn) { + ASSERT(swConn.isOK()); + doneWith(swConn.getValue()); + reachedA = true; + }); + + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 0u); + ASSERT(!reachedA); + ConnectionImpl::pushRefresh(Status::OK()); + ASSERT_EQ(ConnectionImpl::refreshQueueDepth(), 0u); + ASSERT(reachedA); +} + +/** + * Verify that refreshes block new connects, but don't themselves respect maxConnecting + */ +TEST_F(ConnectionPoolTest, maxConnectingWithMultipleRefresh) { + ConnectionPool::Options options; + options.maxConnecting = 2; + options.minConnections = 3; + options.refreshRequirement = Milliseconds(1000); + ConnectionPool pool(stdx::make_unique(), "test pool", options); + + auto now = Date_t::now(); + + PoolImpl::setNow(now); + + // Get us spun up to 3 connections in the pool + pool.get(HostAndPort(), + Milliseconds(5000), + [&](StatusWith swConn) { + ASSERT(swConn.isOK()); + doneWith(swConn.getValue()); + }); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 2u); + ConnectionImpl::pushSetup(Status::OK()); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 2u); + ConnectionImpl::pushSetup(Status::OK()); + ConnectionImpl::pushSetup(Status::OK()); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 0u); + + // Force more than two connections into refresh + PoolImpl::setNow(now + Milliseconds(1500)); + ASSERT_EQ(ConnectionImpl::refreshQueueDepth(), 3u); + + std::array conns; + + // Start 5 new requests + for (size_t i = 0; i < conns.size(); ++i) { + pool.get(HostAndPort(), + Milliseconds(static_cast(1000 + i)), + [&conns, i](StatusWith swConn) { + ASSERT(swConn.isOK()); + conns[i] = std::move(swConn.getValue()); + }); + } + + auto firstNBound = [&](size_t n) { + for (size_t i = 0; i < n; ++i) { + ASSERT(conns[i]); + } + for (size_t i = n; i < conns.size(); ++i) { + ASSERT_FALSE(conns[i]); + } + }; + + // None have started connecting + ASSERT_EQ(ConnectionImpl::refreshQueueDepth(), 3u); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 0u); + firstNBound(0); + + // After one refresh, one refreshed connection gets handed out + ConnectionImpl::pushRefresh(Status::OK()); + ASSERT_EQ(ConnectionImpl::refreshQueueDepth(), 2u); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 0u); + firstNBound(1); + + // After two refresh, one enters the setup queue, one refreshed connection gets handed out + ConnectionImpl::pushRefresh(Status::OK()); + ASSERT_EQ(ConnectionImpl::refreshQueueDepth(), 1u); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 1u); + firstNBound(2); + + // After three refresh, we're done refreshing. Two queued in setup + ConnectionImpl::pushRefresh(Status::OK()); + ASSERT_EQ(ConnectionImpl::refreshQueueDepth(), 0u); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 2u); + firstNBound(3); + + // now pushing setup gets us a new connection + ConnectionImpl::pushSetup(Status::OK()); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 1u); + firstNBound(4); + + // and we're done + ConnectionImpl::pushSetup(Status::OK()); + ASSERT_EQ(ConnectionImpl::setupQueueDepth(), 0u); + firstNBound(5); + + for (auto& conn : conns) { + doneWith(conn); + } +} + /** * Verify that minConnections is respected */ @@ -445,7 +642,7 @@ TEST_F(ConnectionPoolTest, minPoolRespected) { options.maxConnections = 3; options.refreshRequirement = Milliseconds(1000); options.refreshTimeout = Milliseconds(2000); - ConnectionPool pool(stdx::make_unique(), options); + ConnectionPool pool(stdx::make_unique(), "test pool", options); auto now = Date_t::now(); @@ -554,7 +751,7 @@ TEST_F(ConnectionPoolTest, hostTimeoutHappens) { options.refreshRequirement = Milliseconds(5000); options.refreshTimeout = Milliseconds(5000); options.hostTimeout = Milliseconds(1000); - ConnectionPool pool(stdx::make_unique(), options); + ConnectionPool pool(stdx::make_unique(), "test pool", options); auto now = Date_t::now(); @@ -603,7 +800,7 @@ TEST_F(ConnectionPoolTest, hostTimeoutHappensMoreGetsDelay) { options.refreshRequirement = Milliseconds(5000); options.refreshTimeout = Milliseconds(5000); options.hostTimeout = Milliseconds(1000); - ConnectionPool pool(stdx::make_unique(), options); + ConnectionPool pool(stdx::make_unique(), "test pool", options); auto now = Date_t::now(); @@ -665,7 +862,7 @@ TEST_F(ConnectionPoolTest, hostTimeoutHappensCheckoutDelays) { options.refreshRequirement = Milliseconds(5000); options.refreshTimeout = Milliseconds(5000); options.hostTimeout = Milliseconds(1000); - ConnectionPool pool(stdx::make_unique(), options); + ConnectionPool pool(stdx::make_unique(), "test pool", options); auto now = Date_t::now(); @@ -745,7 +942,7 @@ TEST_F(ConnectionPoolTest, dropConnections) { options.maxConnections = 1; options.refreshRequirement = Seconds(1); options.refreshTimeout = Seconds(2); - ConnectionPool pool(stdx::make_unique(), options); + ConnectionPool pool(stdx::make_unique(), "test pool", options); auto now = Date_t::now(); PoolImpl::setNow(now); @@ -830,6 +1027,106 @@ TEST_F(ConnectionPoolTest, dropConnections) { ASSERT(reachedB); } +/** + * Verify that timeouts during setup don't prematurely time out unrelated requests + */ +TEST_F(ConnectionPoolTest, SetupTimeoutsDontTimeoutUnrelatedRequests) { + ConnectionPool::Options options; + + options.maxConnections = 1; + options.refreshTimeout = Seconds(2); + ConnectionPool pool(stdx::make_unique(), "test pool", options); + + auto now = Date_t::now(); + PoolImpl::setNow(now); + + boost::optional> conn1; + pool.get( + HostAndPort(), + Seconds(10), + [&](StatusWith swConn) { conn1 = std::move(swConn); }); + + // initially we haven't called our callback + ASSERT(!conn1); + + PoolImpl::setNow(now + Seconds(1)); + + // Still haven't fired on conn1 + ASSERT(!conn1); + + // Get conn2 (which should have an extra second before the timeout) + boost::optional> conn2; + pool.get( + HostAndPort(), + Seconds(10), + [&](StatusWith swConn) { conn2 = std::move(swConn); }); + + PoolImpl::setNow(now + Seconds(2)); + + ASSERT(conn1); + ASSERT(!conn1->isOK()); + ASSERT(conn1->getStatus().code() == ErrorCodes::ExceededTimeLimit); + + ASSERT(!conn2); +} + +/** + * Verify that timeouts during refresh don't prematurely time out unrelated requests + */ +TEST_F(ConnectionPoolTest, RefreshTimeoutsDontTimeoutRequests) { + ConnectionPool::Options options; + + options.maxConnections = 1; + options.refreshTimeout = Seconds(2); + options.refreshRequirement = Seconds(3); + ConnectionPool pool(stdx::make_unique(), "test pool", options); + + auto now = Date_t::now(); + PoolImpl::setNow(now); + + // Successfully get a new connection + size_t conn1Id = 0; + ConnectionImpl::pushSetup(Status::OK()); + pool.get(HostAndPort(), + Seconds(1), + [&](StatusWith swConn) { + conn1Id = CONN2ID(swConn); + doneWith(swConn.getValue()); + }); + ASSERT(conn1Id); + + // Force it into refresh + PoolImpl::setNow(now + Seconds(3)); + + boost::optional> conn1; + pool.get( + HostAndPort(), + Seconds(10), + [&](StatusWith swConn) { conn1 = std::move(swConn); }); + + // initially we haven't called our callback + ASSERT(!conn1); + + // 1 second later we've triggered a refresh and still haven't called the callback + PoolImpl::setNow(now + Seconds(4)); + ASSERT(!conn1); + + // Get conn2 (which should have an extra second before the timeout) + boost::optional> conn2; + pool.get( + HostAndPort(), + Seconds(10), + [&](StatusWith swConn) { conn2 = std::move(swConn); }); + + PoolImpl::setNow(now + Seconds(5)); + + ASSERT(conn1); + ASSERT(!conn1->isOK()); + ASSERT(conn1->getStatus().code() == ErrorCodes::ExceededTimeLimit); + + ASSERT(!conn2); +} + } // namespace connection_pool_test_details } // namespace executor } // namespace mongo diff --git a/src/mongo/executor/connection_pool_test_fixture.cpp b/src/mongo/executor/connection_pool_test_fixture.cpp index 2d48ad0b5e2..e8aff615448 100644 --- a/src/mongo/executor/connection_pool_test_fixture.cpp +++ b/src/mongo/executor/connection_pool_test_fixture.cpp @@ -116,9 +116,13 @@ void ConnectionImpl::pushSetup(PushSetupCallback status) { _pushSetupQueue.push_back(status); if (_setupQueue.size()) { - _setupQueue.front()->_setupCallback(_setupQueue.front(), _pushSetupQueue.front()()); + auto connPtr = _setupQueue.front(); + auto callback = _pushSetupQueue.front(); _setupQueue.pop_front(); _pushSetupQueue.pop_front(); + + auto cb = connPtr->_setupCallback; + cb(connPtr, callback()); } } @@ -126,13 +130,22 @@ void ConnectionImpl::pushSetup(Status status) { pushSetup([status]() { return status; }); } +size_t ConnectionImpl::setupQueueDepth() { + return _setupQueue.size(); +} + void ConnectionImpl::pushRefresh(PushRefreshCallback status) { _pushRefreshQueue.push_back(status); if (_refreshQueue.size()) { - _refreshQueue.front()->_refreshCallback(_refreshQueue.front(), _pushRefreshQueue.front()()); + auto connPtr = _refreshQueue.front(); + auto callback = _pushRefreshQueue.front(); + _refreshQueue.pop_front(); _pushRefreshQueue.pop_front(); + + auto cb = connPtr->_refreshCallback; + cb(connPtr, callback()); } } @@ -140,6 +153,10 @@ void ConnectionImpl::pushRefresh(Status status) { pushRefresh([status]() { return status; }); } +size_t ConnectionImpl::refreshQueueDepth() { + return _refreshQueue.size(); +} + Date_t ConnectionImpl::getLastUsed() const { return _lastUsed; } @@ -166,9 +183,13 @@ void ConnectionImpl::setup(Milliseconds timeout, SetupCallback cb) { _setupQueue.push_back(this); if (_pushSetupQueue.size()) { - _setupQueue.front()->_setupCallback(_setupQueue.front(), _pushSetupQueue.front()()); + auto connPtr = _setupQueue.front(); + auto callback = _pushSetupQueue.front(); _setupQueue.pop_front(); _pushSetupQueue.pop_front(); + + auto cb = connPtr->_setupCallback; + cb(connPtr, callback()); } } @@ -182,9 +203,14 @@ void ConnectionImpl::refresh(Milliseconds timeout, RefreshCallback cb) { _refreshQueue.push_back(this); if (_pushRefreshQueue.size()) { - _refreshQueue.front()->_refreshCallback(_refreshQueue.front(), _pushRefreshQueue.front()()); + auto connPtr = _refreshQueue.front(); + auto callback = _pushRefreshQueue.front(); + _refreshQueue.pop_front(); _pushRefreshQueue.pop_front(); + + auto cb = connPtr->_refreshCallback; + cb(connPtr, callback()); } } diff --git a/src/mongo/executor/connection_pool_test_fixture.h b/src/mongo/executor/connection_pool_test_fixture.h index 74946ba5cf4..8509e03171c 100644 --- a/src/mongo/executor/connection_pool_test_fixture.h +++ b/src/mongo/executor/connection_pool_test_fixture.h @@ -95,10 +95,12 @@ public: // Push either a callback that returns the status for a setup, or just the Status static void pushSetup(PushSetupCallback status); static void pushSetup(Status status); + static size_t setupQueueDepth(); // Push either a callback that returns the status for a refresh, or just the Status static void pushRefresh(PushRefreshCallback status); static void pushRefresh(Status status); + static size_t refreshQueueDepth(); private: void indicateUsed() override; diff --git a/src/mongo/executor/network_interface.h b/src/mongo/executor/network_interface.h index e35f8ed9c52..3dfae564d91 100644 --- a/src/mongo/executor/network_interface.h +++ b/src/mongo/executor/network_interface.h @@ -151,6 +151,11 @@ public: */ virtual bool onNetworkThread() = 0; + /** + * Drops all connections to the given host in the connection pool. + */ + virtual void dropConnections(const HostAndPort& hostAndPort) = 0; + protected: NetworkInterface(); }; diff --git a/src/mongo/executor/network_interface_asio.cpp b/src/mongo/executor/network_interface_asio.cpp index 3900b92f4e5..60db2b52e95 100644 --- a/src/mongo/executor/network_interface_asio.cpp +++ b/src/mongo/executor/network_interface_asio.cpp @@ -86,6 +86,7 @@ NetworkInterfaceASIO::NetworkInterfaceASIO(Options options) _timerFactory(std::move(_options.timerFactory)), _streamFactory(std::move(_options.streamFactory)), _connectionPool(stdx::make_unique(this), + _options.instanceName, _options.connectionPoolOptions), _isExecutorRunnable(false), _strand(_io_service) {} @@ -392,5 +393,9 @@ bool NetworkInterfaceASIO::onNetworkThread() { [id](const stdx::thread& thread) { return id == thread.get_id(); }); } +void NetworkInterfaceASIO::dropConnections(const HostAndPort& hostAndPort) { + _connectionPool.dropConnections(hostAndPort); +} + } // namespace executor } // namespace mongo diff --git a/src/mongo/executor/network_interface_asio.h b/src/mongo/executor/network_interface_asio.h index b4af49ae602..426515f4ee5 100644 --- a/src/mongo/executor/network_interface_asio.h +++ b/src/mongo/executor/network_interface_asio.h @@ -121,6 +121,8 @@ public: bool inShutdown() const; + void dropConnections(const HostAndPort& hostAndPort) override; + private: using ResponseStatus = TaskExecutor::ResponseStatus; using NetworkInterface::RemoteCommandCompletionFn; diff --git a/src/mongo/executor/network_interface_asio_auth.cpp b/src/mongo/executor/network_interface_asio_auth.cpp index b9a341d13d6..dcf8dc8a5ce 100644 --- a/src/mongo/executor/network_interface_asio_auth.cpp +++ b/src/mongo/executor/network_interface_asio_auth.cpp @@ -40,6 +40,7 @@ #include "mongo/db/commands.h" #include "mongo/db/server_options.h" #include "mongo/rpc/factory.h" +#include "mongo/rpc/get_status_from_command_result.h" #include "mongo/rpc/legacy_request_builder.h" #include "mongo/rpc/reply_interface.h" #include "mongo/stdx/memory.h" @@ -90,6 +91,12 @@ void NetworkInterfaceASIO::_runIsMaster(AsyncOp* op) { auto commandReply = std::move(swCommandReply.getValue()); + // Ensure that the isMaster response is "ok:1". + auto commandStatus = getStatusFromCommandResult(commandReply.data); + if (!commandStatus.isOK()) { + return _completeOperation(op, commandStatus); + } + auto protocolSet = rpc::parseProtocolSetFromIsMasterReply(commandReply.data); if (!protocolSet.isOK()) return _completeOperation(op, protocolSet.getStatus()); @@ -175,8 +182,7 @@ void NetworkInterfaceASIO::_authenticate(AsyncOp* op) { }; auto params = getInternalUserAuthParamsWithFallback(); - auth::authenticateClient( - params, op->request().target.host(), clientName, runCommandHook, authHook); + auth::authenticateClient(params, op->request().target, clientName, runCommandHook, authHook); } } // namespace executor diff --git a/src/mongo/executor/network_interface_asio_command.cpp b/src/mongo/executor/network_interface_asio_command.cpp index 88daf7aafe4..57b77298b42 100644 --- a/src/mongo/executor/network_interface_asio_command.cpp +++ b/src/mongo/executor/network_interface_asio_command.cpp @@ -235,7 +235,11 @@ void NetworkInterfaceASIO::_beginCommunication(AsyncOp* op) { // codepath. if (op->_inSetup) { - log() << "Successfully connected to " << op->request().target.toString(); + auto host = op->request().target; + auto getConnectionDuration = now() - op->start(); + log() << "Successfully connected to " << host << ", took " << getConnectionDuration << " (" + << _connectionPool.getNumConnectionsPerHost(host) << " connections now open to " + << host << ")"; op->_inSetup = false; op->finish(RemoteCommandResponse()); return; diff --git a/src/mongo/executor/network_interface_asio_integration_test.cpp b/src/mongo/executor/network_interface_asio_integration_test.cpp index 8de7c71ebc4..972996304dc 100644 --- a/src/mongo/executor/network_interface_asio_integration_test.cpp +++ b/src/mongo/executor/network_interface_asio_integration_test.cpp @@ -63,7 +63,13 @@ public: void startNet(NetworkInterfaceASIO::Options options = NetworkInterfaceASIO::Options()) { options.streamFactory = stdx::make_unique(); options.timerFactory = stdx::make_unique(); +#ifdef _WIN32 + // Connections won't queue on windows, so attempting to open too many connections + // concurrently will result in refused connections and test failure. + options.connectionPoolOptions.maxConnections = 16u; +#else options.connectionPoolOptions.maxConnections = 256u; +#endif _net = stdx::make_unique(std::move(options)); _net->startup(); } @@ -232,7 +238,7 @@ private: }; TEST_F(NetworkInterfaceASIOIntegrationTest, StressTest) { - const std::size_t numOps = 1000; + const std::size_t numOps = 500; std::vector testResults(numOps, {ErrorCodes::InternalError, "uninitialized"}); ErrorCodes::Error expectedResults[numOps]; CountdownLatch cl(numOps); @@ -249,7 +255,7 @@ TEST_F(NetworkInterfaceASIOIntegrationTest, StressTest) { for (std::size_t i = 0; i < numOps; ++i) { // stagger operations slightly to mitigate connection pool contention - sleepmillis(rng.nextInt32(16)); + sleepmillis(rng.nextInt32(50)); auto r = rng.nextCanonicalDouble(); diff --git a/src/mongo/executor/network_interface_asio_operation.cpp b/src/mongo/executor/network_interface_asio_operation.cpp index 2a75fb2c233..0c3e29d029c 100644 --- a/src/mongo/executor/network_interface_asio_operation.cpp +++ b/src/mongo/executor/network_interface_asio_operation.cpp @@ -204,6 +204,9 @@ NetworkInterfaceASIO::AsyncCommand* NetworkInterfaceASIO::AsyncOp::command() { } void NetworkInterfaceASIO::AsyncOp::finish(const ResponseStatus& status) { + LOG(2) << "Request " << _request.id << " finished with response: " + << (status.getStatus().isOK() ? status.getValue().data.toString() + : status.getStatus().toString()); _onFinish(status); } diff --git a/src/mongo/executor/network_interface_asio_test.cpp b/src/mongo/executor/network_interface_asio_test.cpp index bbe75be5f49..144a0ad91d4 100644 --- a/src/mongo/executor/network_interface_asio_test.cpp +++ b/src/mongo/executor/network_interface_asio_test.cpp @@ -533,6 +533,46 @@ public: } }; +TEST_F(NetworkInterfaceASIOConnectionHookTest, InvalidIsMaster) { + auto validationFailedStatus = + Status(ErrorCodes::InterruptedDueToReplStateChange, "operation was interrupted"); + + start(makeTestHook( + [&](const HostAndPort& remoteHost, const RemoteCommandResponse& isMasterReply) { + return Status(ErrorCodes::UnknownError, "unused"); + }, + [&](const HostAndPort& remoteHost) + -> StatusWith> { return {boost::none}; }, + [&](const HostAndPort& remoteHost, RemoteCommandResponse&& response) { + return Status::OK(); + })); + + auto deferred = startCommand(makeCallbackHandle(), + {testHost, + "blah", + BSON("foo" + << "bar")}); + + auto stream = streamFactory().blockUntilStreamExists(testHost); + + ConnectEvent{stream}.skip(); + + // simulate isMaster reply. + stream->simulateServer(rpc::Protocol::kOpQuery, + [](RemoteCommandRequest request) -> RemoteCommandResponse { + RemoteCommandResponse response; + response.data = BSON("ok" << 0.0 << "errmsg" + << "operation was interrupted" + << "code" << 11602); + return response; + }); + + // we should stop here. + auto& res = deferred.get(); + + ASSERT(res == validationFailedStatus); +} + TEST_F(NetworkInterfaceASIOConnectionHookTest, ValidateHostInvalid) { bool validateCalled = false; bool hostCorrect = false; diff --git a/src/mongo/executor/network_interface_impl.h b/src/mongo/executor/network_interface_impl.h index 5915d2d99d6..76759dd454b 100644 --- a/src/mongo/executor/network_interface_impl.h +++ b/src/mongo/executor/network_interface_impl.h @@ -96,6 +96,7 @@ public: void cancelAllCommands() override {} void setAlarm(Date_t when, const stdx::function& action) override; bool onNetworkThread() override; + void dropConnections(const HostAndPort& hostAndPort) override {} private: /** diff --git a/src/mongo/executor/network_interface_mock.h b/src/mongo/executor/network_interface_mock.h index 439b33e5caf..4096b8f68ac 100644 --- a/src/mongo/executor/network_interface_mock.h +++ b/src/mongo/executor/network_interface_mock.h @@ -104,6 +104,8 @@ public: virtual bool onNetworkThread(); + void dropConnections(const HostAndPort&) override {} + //////////////////////////////////////////////////////////////////////////////// // diff --git a/src/mongo/executor/thread_pool_task_executor.cpp b/src/mongo/executor/thread_pool_task_executor.cpp index 570ac69826e..e33e2bcc487 100644 --- a/src/mongo/executor/thread_pool_task_executor.cpp +++ b/src/mongo/executor/thread_pool_task_executor.cpp @@ -273,6 +273,9 @@ StatusWith ThreadPoolTaskExecutor::scheduleWorkAt( } invariant(now() >= when); stdx::unique_lock lk(_mutex); + if (cbState->canceled.load()) { + return; + } scheduleIntoPool_inlock(&_sleepersQueue, cbState->iter, std::move(lk)); }); @@ -469,22 +472,26 @@ void ThreadPoolTaskExecutor::scheduleIntoPool_inlock(WorkQueue* fromQueue, } void ThreadPoolTaskExecutor::runCallback(std::shared_ptr cbStateArg) { - auto cbStatePtr = cbStateArg.get(); CallbackHandle cbHandle; - setCallbackForHandle(&cbHandle, std::move(cbStateArg)); + setCallbackForHandle(&cbHandle, cbStateArg); CallbackArgs args(this, std::move(cbHandle), - cbStatePtr->canceled.load() + cbStateArg->canceled.load() ? Status({ErrorCodes::CallbackCanceled, "Callback canceled"}) : Status::OK()); - cbStatePtr->callback(std::move(args)); - cbStatePtr->isFinished.store(true); + invariant(!cbStateArg->isFinished.load()); + cbStateArg->callback(std::move(args)); + cbStateArg->isFinished.store(true); stdx::lock_guard lk(_mutex); - _poolInProgressQueue.erase(cbStatePtr->iter); - if (cbStatePtr->finishedCondition) { - cbStatePtr->finishedCondition->notify_all(); + _poolInProgressQueue.erase(cbStateArg->iter); + if (cbStateArg->finishedCondition) { + cbStateArg->finishedCondition->notify_all(); } } +void ThreadPoolTaskExecutor::dropConnections(const HostAndPort& hostAndPort) { + _net->dropConnections(hostAndPort); +} + } // namespace executor } // namespace mongo diff --git a/src/mongo/executor/thread_pool_task_executor.h b/src/mongo/executor/thread_pool_task_executor.h index f4afb7b58c9..ae4ce9bb40e 100644 --- a/src/mongo/executor/thread_pool_task_executor.h +++ b/src/mongo/executor/thread_pool_task_executor.h @@ -88,6 +88,11 @@ public: */ void cancelAllCommands(); + /** + * Drops all connections to the given host on the network interface. + */ + void dropConnections(const HostAndPort& hostAndPort); + private: class CallbackState; class EventState; diff --git a/src/mongo/installer/msi/SConscript b/src/mongo/installer/msi/SConscript index f17d3523464..f8fe85d7bc2 100644 --- a/src/mongo/installer/msi/SConscript +++ b/src/mongo/installer/msi/SConscript @@ -125,7 +125,7 @@ env.Command(objects, ' -dTargetFileName=${SERVER_ARCHIVE}' r' -dSaslSource=c:\sasl\bin' r' -dSnmpSource=c:\snmp\bin' - r' -dSslSource=c:\openssl\bin' + r' -dSslSource=' + env['WINDOWS_OPENSSL_BIN'] + ' -out ' + buildDir + r'\msi\\' ' -arch ' + msi_platform + ' -ext "$WIXUIEXT"' diff --git a/src/mongo/logger/logstream_builder.cpp b/src/mongo/logger/logstream_builder.cpp index 2577704aad0..199991c7dd7 100644 --- a/src/mongo/logger/logstream_builder.cpp +++ b/src/mongo/logger/logstream_builder.cpp @@ -81,12 +81,14 @@ LogstreamBuilder::LogstreamBuilder(MessageLogDomain* domain, LogstreamBuilder::LogstreamBuilder(MessageLogDomain* domain, std::string contextName, LogSeverity severity, - LogComponent component) + LogComponent component, + bool shouldCache) : _domain(domain), _contextName(std::move(contextName)), _severity(std::move(severity)), _component(std::move(component)), - _tee(nullptr) {} + _tee(nullptr), + _shouldCache(shouldCache) {} LogstreamBuilder::LogstreamBuilder(logger::MessageLogDomain* domain, const std::string& contextName, @@ -103,7 +105,8 @@ LogstreamBuilder::LogstreamBuilder(LogstreamBuilder&& other) _baseMessage(std::move(other._baseMessage)), _os(std::move(other._os)), _tee(std::move(other._tee)), - _isTruncatable(other._isTruncatable) {} + _isTruncatable(other._isTruncatable), + _shouldCache(other._shouldCache) {} LogstreamBuilder& LogstreamBuilder::operator=(LogstreamBuilder&& other) { _domain = std::move(other._domain); @@ -114,6 +117,7 @@ LogstreamBuilder& LogstreamBuilder::operator=(LogstreamBuilder&& other) { _os = std::move(other._os); _tee = std::move(other._tee); _isTruncatable = std::move(other._isTruncatable); + _shouldCache = other._shouldCache; return *this; } @@ -134,7 +138,8 @@ LogstreamBuilder::~LogstreamBuilder() { _tee->write(_os->str()); } _os->str(""); - if (isThreadOstreamCacheInitialized && !threadOstreamCache.getMake()->get()) { + if (_shouldCache && isThreadOstreamCacheInitialized && + !threadOstreamCache.getMake()->get()) { *threadOstreamCache.get() = std::move(_os); } } @@ -148,7 +153,8 @@ void LogstreamBuilder::operator<<(Tee* tee) { void LogstreamBuilder::makeStream() { if (!_os) { - if (isThreadOstreamCacheInitialized && threadOstreamCache.getMake()->get()) { + if (_shouldCache && isThreadOstreamCacheInitialized && + threadOstreamCache.getMake()->get()) { _os = std::move(*threadOstreamCache.get()); } else { _os = stdx::make_unique(); diff --git a/src/mongo/logger/logstream_builder.h b/src/mongo/logger/logstream_builder.h index 40f56ccf75f..f829621c8a7 100644 --- a/src/mongo/logger/logstream_builder.h +++ b/src/mongo/logger/logstream_builder.h @@ -72,11 +72,17 @@ public: * "contextName" is a short name of the thread or other context. * "severity" is the logging severity of the message. * "component" is the primary log component of the message. + * + * By default, this class will create one ostream per thread, and it + * will cache that object in a threadlocal and reuse it for subsequent + * logs messages. Set "shouldCache" to false to create a new ostream + * for each instance of this class rather than cacheing. */ LogstreamBuilder(MessageLogDomain* domain, std::string contextName, LogSeverity severity, - LogComponent component); + LogComponent component, + bool shouldCache = true); /** * Deprecated. @@ -236,6 +242,7 @@ private: std::unique_ptr _os; Tee* _tee; bool _isTruncatable = true; + bool _shouldCache; }; diff --git a/src/mongo/logger/message_event_utf8_encoder.cpp b/src/mongo/logger/message_event_utf8_encoder.cpp index 3483a113d1e..2a09e046736 100644 --- a/src/mongo/logger/message_event_utf8_encoder.cpp +++ b/src/mongo/logger/message_event_utf8_encoder.cpp @@ -46,6 +46,14 @@ MessageEventDetailsEncoder::DateFormatter MessageEventDetailsEncoder::getDateFor return _dateFormatter; } +namespace { +#ifdef _WIN32 +const auto kEOL = StringData("\r\n", StringData::LiteralTag()); +#else +const auto kEOL = StringData("\n", StringData::LiteralTag()); +#endif +} // namespace + MessageEventDetailsEncoder::~MessageEventDetailsEncoder() {} std::ostream& MessageEventDetailsEncoder::encode(const MessageEventEphemeral& event, std::ostream& os) { @@ -67,6 +75,26 @@ std::ostream& MessageEventDetailsEncoder::encode(const MessageEventEphemeral& ev } StringData msg = event.getMessage(); + +#ifdef _WIN32 + // We need to translate embedded Unix style line endings into Windows style endings. + std::string tempstr; + size_t embeddedNewLine = msg.find('\n'); + + if (embeddedNewLine != std::string::npos) { + tempstr = msg.toString().replace(embeddedNewLine, 1, "\r\n"); + + embeddedNewLine = tempstr.find('\n', embeddedNewLine + 2); + while (embeddedNewLine != std::string::npos) { + tempstr = tempstr.replace(embeddedNewLine, 1, "\r\n"); + + embeddedNewLine = tempstr.find('\n', embeddedNewLine + 2); + } + + msg = tempstr; + } +#endif + if (event.isTruncatable() && msg.size() > maxLogLine) { os << "warning: log line attempted (" << msg.size() / 1024 << "kB) over max size (" << maxLogLine / 1024 << "kB), printing beginning and end ... "; @@ -76,8 +104,10 @@ std::ostream& MessageEventDetailsEncoder::encode(const MessageEventEphemeral& ev } else { os << msg; } - if (!msg.endsWith(StringData("\n", StringData::LiteralTag()))) - os << '\n'; + + if (!msg.endsWith(kEOL)) + os << kEOL; + return os; } diff --git a/src/mongo/logger/ramlog.cpp b/src/mongo/logger/ramlog.cpp index e9d7685d49f..6ce15dba3a4 100644 --- a/src/mongo/logger/ramlog.cpp +++ b/src/mongo/logger/ramlog.cpp @@ -50,8 +50,7 @@ RM* _named = NULL; } // namespace RamLog::RamLog(const std::string& name) : _name(name), _totalLinesWritten(0), _lastWrite(0) { - h = 0; - n = 0; + clear(); for (int i = 0; i < N; i++) lines[i][C - 1] = 0; } @@ -84,6 +83,16 @@ void RamLog::write(const std::string& str) { h = (h + 1) % N; } +void RamLog::clear() { + stdx::lock_guard lk(_mutex); + _totalLinesWritten = 0; + _lastWrite = 0; + h = 0; + n = 0; + for (int i = 0; i < N; i++) + lines[i][0] = 0; +} + time_t RamLog::LineIterator::lastWrite() { return _ramlog->_lastWrite; } diff --git a/src/mongo/logger/ramlog.h b/src/mongo/logger/ramlog.h index c69fb1e4814..8e087c6d531 100644 --- a/src/mongo/logger/ramlog.h +++ b/src/mongo/logger/ramlog.h @@ -93,6 +93,11 @@ public: */ void write(const std::string& str); + /** + * Empties out the RamLog. + */ + void clear(); + /** * Writes an HTML representation of the log to "s". diff --git a/src/mongo/logger/rotatable_file_writer.cpp b/src/mongo/logger/rotatable_file_writer.cpp index 9b23952ac02..d1c2543f20f 100644 --- a/src/mongo/logger/rotatable_file_writer.cpp +++ b/src/mongo/logger/rotatable_file_writer.cpp @@ -97,8 +97,6 @@ private: virtual std::streamsize xsputn(const char* s, std::streamsize count); virtual int_type overflow(int_type ch = traits_type::eof()); - std::streamsize writeToFile(const char* s, std::streamsize count); - HANDLE _fileHandle; }; @@ -163,7 +161,8 @@ bool Win32FileStreambuf::open(StringData fileName, bool append) { return false; } -std::streamsize Win32FileStreambuf::writeToFile(const char* s, std::streamsize count) { +// Called when strings are written to ostream +std::streamsize Win32FileStreambuf::xsputn(const char* s, std::streamsize count) { DWORD totalBytesWritten = 0; while (count > totalBytesWritten) { @@ -171,32 +170,8 @@ std::streamsize Win32FileStreambuf::writeToFile(const char* s, std::streamsize c if (!WriteFile(_fileHandle, s, count - totalBytesWritten, &bytesWritten, NULL)) { break; } - totalBytesWritten += bytesWritten; - } - - return totalBytesWritten; -} - -// Called when strings are written to ostream -std::streamsize Win32FileStreambuf::xsputn(const char* s, std::streamsize count) { - DWORD totalBytesWritten = 0; - - // Scan for embedded newlines before end - // this should be rare since the newline should only be at the end - const char* startPos = s; - for (int i = 0; i < count; i++) { - if (s[i] == '\n') { - totalBytesWritten += writeToFile(startPos, i - (startPos - s)); - writeToFile("\r\n", 2); - totalBytesWritten += 1; // Caller expected we only wrote 1 char, so tell them so - startPos = &s[i + 1]; - } - } - // Did the string not end on "\n"? Write the remaining, no need for CRLF - // as upper layers are responsible for it - if ((startPos - s) != count) { - totalBytesWritten += writeToFile(startPos, count - (startPos - s)); + totalBytesWritten += bytesWritten; } return totalBytesWritten; diff --git a/src/mongo/s/commands/SConscript b/src/mongo/s/commands/SConscript index 5400114f85d..743918fdabb 100644 --- a/src/mongo/s/commands/SConscript +++ b/src/mongo/s/commands/SConscript @@ -31,6 +31,7 @@ env.Library( 'cluster_find_and_modify_cmd.cpp', 'cluster_flush_router_config_cmd.cpp', 'cluster_fsync_cmd.cpp', + 'cluster_ftdc_commands.cpp', 'cluster_get_last_error_cmd.cpp', 'cluster_get_prev_error_cmd.cpp', 'cluster_get_shard_version_cmd.cpp', diff --git a/src/mongo/s/commands/cluster_ftdc_commands.cpp b/src/mongo/s/commands/cluster_ftdc_commands.cpp new file mode 100644 index 00000000000..29c80482c58 --- /dev/null +++ b/src/mongo/s/commands/cluster_ftdc_commands.cpp @@ -0,0 +1,94 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/base/init.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/client.h" +#include "mongo/db/commands.h" +#include "mongo/db/ftdc/controller.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/operation_context.h" + +namespace mongo { +namespace { + +/** + * getDiagnosticData is a MongoD only command. We implement in MongoS to give users a better error + * message. + */ +class GetDiagnosticDataCommand final : public Command { +public: + GetDiagnosticDataCommand() : Command("getDiagnosticData") {} + + bool adminOnly() const override { + return true; + } + + void help(std::stringstream& help) const override { + help << "get latest diagnostic data collection snapshot"; + } + + bool slaveOk() const override { + return true; + } + + bool isWriteCommandForConfigServer() const override { + return false; + } + + Status checkAuthForCommand(ClientBasic* client, + const std::string& dbname, + const BSONObj& cmdObj) override { + return Status::OK(); + } + + bool run(OperationContext* txn, + const std::string& db, + BSONObj& cmdObj, + int options, + std::string& errmsg, + BSONObjBuilder& result) override { + errmsg = "getDiagnosticData not allowed through mongos"; + + return false; + } +}; + +Command* ftdcCommand; + +MONGO_INITIALIZER(CreateDiagnosticDataCommand)(InitializerContext* context) { + ftdcCommand = new GetDiagnosticDataCommand(); + + return Status::OK(); +} + +} // namespace +} // namespace mongo diff --git a/src/mongo/s/commands/cluster_list_databases_cmd.cpp b/src/mongo/s/commands/cluster_list_databases_cmd.cpp index 23dad76f0a3..b55636965ef 100644 --- a/src/mongo/s/commands/cluster_list_databases_cmd.cpp +++ b/src/mongo/s/commands/cluster_list_databases_cmd.cpp @@ -87,6 +87,8 @@ public: int options, std::string& errmsg, BSONObjBuilder& result) { + const bool nameOnly = cmdObj["nameOnly"].trueValue(); + map sizes; map> dbShardInfo; @@ -152,9 +154,11 @@ public: BSONObjBuilder temp; temp.append("name", name); - temp.appendNumber("sizeOnDisk", size); - temp.appendBool("empty", size == 1); - temp.append("shards", dbShardInfo[name]->obj()); + if (!nameOnly) { + temp.appendNumber("sizeOnDisk", size); + temp.appendBool("empty", size == 1); + temp.append("shards", dbShardInfo[name]->obj()); + } dbListBuilder.append(temp.obj()); } @@ -168,6 +172,9 @@ public: dbListBuilder.done(); + if (nameOnly) + return true; + result.appendNumber("totalSize", totalSize); result.appendNumber("totalSizeMb", totalSize / (1024 * 1024)); diff --git a/src/mongo/s/commands/cluster_pipeline_cmd.cpp b/src/mongo/s/commands/cluster_pipeline_cmd.cpp index 48544a41c3d..16ef0b05a55 100644 --- a/src/mongo/s/commands/cluster_pipeline_cmd.cpp +++ b/src/mongo/s/commands/cluster_pipeline_cmd.cpp @@ -51,6 +51,7 @@ #include "mongo/s/commands/cluster_commands_common.h" #include "mongo/s/config.h" #include "mongo/s/grid.h" +#include "mongo/s/query/cluster_query_knobs.h" #include "mongo/s/query/store_possible_cursor.h" #include "mongo/s/stale_exception.h" #include "mongo/util/log.h" @@ -237,7 +238,8 @@ public: // Run merging command on random shard, unless a stage needs the primary shard. Need to use // ShardConnection so that the merging mongod is sent the config servers on connection init. auto& prng = txn->getClient()->getPrng(); - const auto& mergingShardId = needPrimaryShardMerger + const auto& mergingShardId = + (needPrimaryShardMerger || internalQueryAlwaysMergeOnPrimaryShard) ? conf->getPrimaryId() : shardResults[prng.nextInt32(shardResults.size())].shardTargetId; const auto mergingShard = grid.shardRegistry()->getShard(txn, mergingShardId); diff --git a/src/mongo/s/query/SConscript b/src/mongo/s/query/SConscript index 18022507aa2..fd27f991f01 100644 --- a/src/mongo/s/query/SConscript +++ b/src/mongo/s/query/SConscript @@ -6,6 +6,7 @@ env.Library( target="cluster_query", source=[ "cluster_find.cpp", + "cluster_query_knobs.cpp", ], LIBDEPS=[ '$BUILD_DIR/mongo/db/commands', diff --git a/src/mongo/s/query/async_results_merger.cpp b/src/mongo/s/query/async_results_merger.cpp index 984c2f85184..18f5b3a6c07 100644 --- a/src/mongo/s/query/async_results_merger.cpp +++ b/src/mongo/s/query/async_results_merger.cpp @@ -332,8 +332,9 @@ StatusWith AsyncResultsMerger::nextEvent() for (size_t i = 0; i < _remotes.size(); ++i) { auto& remote = _remotes[i]; - // It is illegal to call this method if there is an error received from any shard. - invariant(remote.status.isOK()); + if (!remote.status.isOK()) { + return remote.status; + } if (!remote.hasNext() && !remote.exhausted() && !remote.cbHandle.isValid()) { // If we already have established a cursor with this remote, and there is no outstanding @@ -474,6 +475,12 @@ void AsyncResultsMerger::handleBatchResponse( return; } else { remote.status = cursorResponseStatus.getStatus(); + if (remote.status == ErrorCodes::CallbackCanceled) { + // This callback should only be canceled as part of the shutdown sequence, so we + // promote a canceled callback error to an error that will make more sense to the + // client. + remote.status = Status(ErrorCodes::ShutdownInProgress, "shutdown in progress"); + } } // Unreachable host errors are swallowed if the 'allowPartialResults' option is set. We diff --git a/src/mongo/s/query/async_results_merger.h b/src/mongo/s/query/async_results_merger.h index 4f1d8e7ca58..eea7c330dc7 100644 --- a/src/mongo/s/query/async_results_merger.h +++ b/src/mongo/s/query/async_results_merger.h @@ -159,11 +159,12 @@ public: * Starts shutting down this ARM. Returns a handle to an event which is signaled when this * cursor is safe to destroy. * - * Returns an invalid handle if the underlying task executor is shutting down. In this case, it - * is legal to destroy the cursor only after the task executor shutdown process is complete. + * Returns an invalid handle if the underlying task executor is shutting down. In this case, + * killing is considered complete and the ARM may be destroyed immediately. * - * An ARM can only be destroyed if either 1) all its results have been exhausted or 2) the kill - * event returned by this method has been signaled. + * When the underlying task executor is *not* shutting down, an ARM can only be destroyed if + * either 1) all its results have been exhausted or 2) the kill event returned by this method + * has been signaled. * * May be called multiple times (idempotent). */ diff --git a/src/mongo/s/query/async_results_merger_test.cpp b/src/mongo/s/query/async_results_merger_test.cpp index 31cf4228bcf..2dd95b9708b 100644 --- a/src/mongo/s/query/async_results_merger_test.cpp +++ b/src/mongo/s/query/async_results_merger_test.cpp @@ -787,7 +787,7 @@ TEST_F(AsyncResultsMergerTest, NextEventAfterTaskExecutorShutdown) { BSONObj findCmd = fromjson("{find: 'testcoll'}"); makeCursorFromFindCmd(findCmd, kTestShardIds); executor->shutdown(); - ASSERT_NOT_OK(arm->nextEvent().getStatus()); + ASSERT_EQ(ErrorCodes::ShutdownInProgress, arm->nextEvent().getStatus()); auto killEvent = arm->kill(); ASSERT_FALSE(killEvent.isValid()); } @@ -1437,6 +1437,48 @@ TEST_F(AsyncResultsMergerTest, GetMoreRequestWithoutAwaitDataCantHaveMaxTime) { executor->waitForEvent(killEvent); } +TEST_F(AsyncResultsMergerTest, ShardCanErrorInBetweenReadyAndNextEvent) { + BSONObj findCmd = fromjson("{find: 'testcoll', tailable: true}"); + makeCursorFromFindCmd(findCmd, {kTestShardIds[0]}); + + ASSERT_FALSE(arm->ready()); + auto readyEvent = unittest::assertGet(arm->nextEvent()); + scheduleErrorResponse({ErrorCodes::BadValue, "bad thing happened"}); + + ASSERT_EQ(ErrorCodes::BadValue, arm->nextEvent().getStatus()); + + // Required to kill the 'arm' on error before destruction. + auto killEvent = arm->kill(); + executor->waitForEvent(killEvent); +} + +TEST_F(AsyncResultsMergerTest, RetryWhenShardHasRetriableErrorInBetweenReadyAndNextEvent) { + BSONObj findCmd = fromjson("{find: 'testcoll'}"); + makeCursorFromFindCmd(findCmd, {kTestShardIds[0]}); + + ASSERT_FALSE(arm->ready()); + + // First attempt returns a retriable error. + auto readyEvent = unittest::assertGet(arm->nextEvent()); + scheduleErrorResponse({ErrorCodes::NotMasterNoSlaveOk, "not master and not slave"}); + + // We expect to be able to retrieve another event, and be waiting on the retry to succeed. + readyEvent = unittest::assertGet(arm->nextEvent()); + std::vector responses; + std::vector batch = {fromjson("{_id: 1}"), fromjson("{_id: 2}")}; + responses.emplace_back(_nss, CursorId(0), batch); + scheduleNetworkResponses(std::move(responses), CursorResponse::ResponseType::InitialResponse); + + executor->waitForEvent(readyEvent); + ASSERT_TRUE(arm->ready()); + ASSERT_EQ(fromjson("{_id: 1}"), *unittest::assertGet(arm->nextReady())); + ASSERT_TRUE(arm->ready()); + ASSERT_EQ(fromjson("{_id: 2}"), *unittest::assertGet(arm->nextReady())); + ASSERT_TRUE(arm->ready()); + ASSERT_TRUE(arm->remotesExhausted()); + ASSERT(!unittest::assertGet(arm->nextReady())); +} + } // namespace } // namespace mongo diff --git a/src/mongo/s/query/cluster_query_knobs.cpp b/src/mongo/s/query/cluster_query_knobs.cpp new file mode 100644 index 00000000000..76e82b4f914 --- /dev/null +++ b/src/mongo/s/query/cluster_query_knobs.cpp @@ -0,0 +1,39 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/s/query/cluster_query_knobs.h" + +#include "mongo/db/server_parameters.h" + +namespace mongo { + +MONGO_EXPORT_SERVER_PARAMETER(internalQueryAlwaysMergeOnPrimaryShard, bool, false); + +} // namespace mongo diff --git a/src/mongo/s/query/cluster_query_knobs.h b/src/mongo/s/query/cluster_query_knobs.h new file mode 100644 index 00000000000..76df18fde4d --- /dev/null +++ b/src/mongo/s/query/cluster_query_knobs.h @@ -0,0 +1,40 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include + +namespace mongo { + +// If set to true on mongos, all aggregations delivered to the mongos which require a merging shard +// will select the primary shard as the merger. False by default, which means that the merging shard +// will be selected randomly amongst the shards participating in the query. +extern std::atomic internalQueryAlwaysMergeOnPrimaryShard; // NOLINT + +} // namespace mongo diff --git a/src/mongo/s/query/router_stage_merge.cpp b/src/mongo/s/query/router_stage_merge.cpp index 527bc0f0063..b4be2ae5b78 100644 --- a/src/mongo/s/query/router_stage_merge.cpp +++ b/src/mongo/s/query/router_stage_merge.cpp @@ -57,6 +57,10 @@ StatusWith> RouterStageMerge::next() { void RouterStageMerge::kill() { auto killEvent = _arm.kill(); + if (!killEvent) { + // Mongos is shutting down. + return; + } _executor->waitForEvent(killEvent); } diff --git a/src/mongo/s/sharding_initialization.cpp b/src/mongo/s/sharding_initialization.cpp index 37257afe244..89348e61407 100644 --- a/src/mongo/s/sharding_initialization.cpp +++ b/src/mongo/s/sharding_initialization.cpp @@ -66,6 +66,7 @@ using executor::ConnectionPool; MONGO_EXPORT_STARTUP_SERVER_PARAMETER(ShardingTaskExecutorPoolHostTimeoutMS, int, -1); MONGO_EXPORT_STARTUP_SERVER_PARAMETER(ShardingTaskExecutorPoolMaxSize, int, -1); +MONGO_EXPORT_STARTUP_SERVER_PARAMETER(ShardingTaskExecutorPoolMaxConnecting, int, -1); MONGO_EXPORT_STARTUP_SERVER_PARAMETER(ShardingTaskExecutorPoolMinSize, int, static_cast(ConnectionPool::kDefaultMinConns)); @@ -181,6 +182,9 @@ Status initializeGlobalShardingState(OperationContext* txn, connPoolOptions.maxConnections = (ShardingTaskExecutorPoolMaxSize != -1) ? ShardingTaskExecutorPoolMaxSize : ConnectionPool::kDefaultMaxConns; + connPoolOptions.maxConnecting = (ShardingTaskExecutorPoolMaxConnecting != -1) + ? ShardingTaskExecutorPoolMaxConnecting + : ConnectionPool::kDefaultMaxConnecting; connPoolOptions.minConnections = ShardingTaskExecutorPoolMinSize; connPoolOptions.refreshRequirement = (ShardingTaskExecutorPoolRefreshRequirementMS != -1) ? Milliseconds(ShardingTaskExecutorPoolRefreshRequirementMS) diff --git a/src/mongo/scripting/SConscript b/src/mongo/scripting/SConscript index 3f1e034fa40..ef524aa852e 100644 --- a/src/mongo/scripting/SConscript +++ b/src/mongo/scripting/SConscript @@ -10,6 +10,7 @@ Import([ env.Library( target='scripting_common', source=[ + 'deadline_monitor.cpp', 'engine.cpp', 'utils.cpp', ], @@ -183,5 +184,6 @@ env.CppUnitTest( 'deadline_monitor_test.cpp', ], LIBDEPS=[ + 'scripting_common', ], ) diff --git a/src/mongo/scripting/deadline_monitor.cpp b/src/mongo/scripting/deadline_monitor.cpp new file mode 100644 index 00000000000..5eb0f52e5de --- /dev/null +++ b/src/mongo/scripting/deadline_monitor.cpp @@ -0,0 +1,43 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/scripting/deadline_monitor.h" + +#include "mongo/db/server_parameters.h" + +namespace mongo { + +MONGO_EXPORT_SERVER_PARAMETER(scriptingEngineInterruptIntervalMS, int, 1000); + +int getScriptingEngineInterruptInterval() { + return scriptingEngineInterruptIntervalMS.load(); +} + +} // namespace mongo diff --git a/src/mongo/scripting/deadline_monitor.h b/src/mongo/scripting/deadline_monitor.h index 89f7e9d1b84..b1c2855dd46 100644 --- a/src/mongo/scripting/deadline_monitor.h +++ b/src/mongo/scripting/deadline_monitor.h @@ -30,6 +30,7 @@ #include #include "mongo/base/disallow_copying.h" +#include "mongo/platform/atomic_word.h" #include "mongo/platform/unordered_map.h" #include "mongo/stdx/condition_variable.h" #include "mongo/stdx/thread.h" @@ -38,6 +39,9 @@ namespace mongo { +// Returns the current interrupt interval from the setParameter value +int getScriptingEngineInterruptInterval(); + /** * DeadlineMonitor * @@ -91,11 +95,18 @@ public: * @param task the task to kill() * @param timeoutMs number of milliseconds before the deadline expires */ - void startDeadline(_Task* const task, uint64_t timeoutMs) { - const auto deadline = Date_t::now() + Milliseconds(timeoutMs); + void startDeadline(_Task* const task, int64_t timeoutMs) { + Date_t deadline; + if (timeoutMs > 0) { + deadline = Date_t::now() + Milliseconds(timeoutMs); + } else { + deadline = Date_t::max(); + } stdx::lock_guard lk(_deadlineMutex); - _tasks[task] = deadline; + if (_tasks.find(task) == _tasks.end()) { + _tasks.emplace(task, deadline); + } if (deadline < _nearestDeadlineWallclock) { _nearestDeadlineWallclock = deadline; @@ -121,14 +132,25 @@ private: */ void deadlineMonitorThread() { stdx::unique_lock lk(_deadlineMutex); + Date_t lastInterruptCycle = Date_t::now(); while (!_inShutdown) { // get the next interval to wait const Date_t now = Date_t::now(); + const auto interruptInterval = Milliseconds{getScriptingEngineInterruptInterval()}; + + if (now - lastInterruptCycle > interruptInterval) { + for (const auto& task : _tasks) { + if (task.second > now) + task.first->interrupt(); + } + lastInterruptCycle = now; + } // wait for a task to be added or a deadline to expire if (_nearestDeadlineWallclock > now) { - if (_nearestDeadlineWallclock == Date_t::max()) { - _newDeadlineAvailable.wait(lk); + if (_nearestDeadlineWallclock == Date_t::max() || + _nearestDeadlineWallclock - now > interruptInterval) { + _newDeadlineAvailable.wait_for(lk, interruptInterval); } else { _newDeadlineAvailable.wait_until(lk, _nearestDeadlineWallclock.toSystemTimePoint()); @@ -138,7 +160,7 @@ private: // set the next interval to wait for deadline completion _nearestDeadlineWallclock = Date_t::max(); - typename TaskDeadlineMap::iterator i = _tasks.begin(); + auto i = _tasks.begin(); while (i != _tasks.end()) { if (i->second < now) { // deadline expired @@ -155,7 +177,7 @@ private: } } - typedef unordered_map<_Task*, Date_t> TaskDeadlineMap; + using TaskDeadlineMap = std::unordered_map<_Task*, Date_t>; TaskDeadlineMap _tasks; // map of running tasks with deadlines stdx::mutex _deadlineMutex; // protects all non-const members, except _monitorThread stdx::condition_variable _newDeadlineAvailable; // Signaled for timeout, start and stop diff --git a/src/mongo/scripting/deadline_monitor_test.cpp b/src/mongo/scripting/deadline_monitor_test.cpp index 98f4008b234..71daefbce8f 100644 --- a/src/mongo/scripting/deadline_monitor_test.cpp +++ b/src/mongo/scripting/deadline_monitor_test.cpp @@ -72,6 +72,7 @@ public: if (_group) _group->noteKill(); } + void interrupt() {} TaskGroup* _group; uint64_t _killed; }; diff --git a/src/mongo/scripting/engine.cpp b/src/mongo/scripting/engine.cpp index 1bee97838f8..7e6628344ab 100644 --- a/src/mongo/scripting/engine.cpp +++ b/src/mongo/scripting/engine.cpp @@ -267,13 +267,11 @@ ScriptingFunction Scope::createFunction(const char* code) { FunctionCacheMap::iterator i = _cachedFunctions.find(code); if (i != _cachedFunctions.end()) return i->second; - // NB: we calculate the function number for v8 so the cache can be utilized to - // lookup the source on an exception, but SpiderMonkey uses the value - // returned by JS_CompileFunction. - ScriptingFunction defaultFunctionNumber = getFunctionCache().size() + 1; - ScriptingFunction actualFunctionNumber = _createFunction(code, defaultFunctionNumber); - _cachedFunctions[code] = actualFunctionNumber; - return actualFunctionNumber; + + // Get a function number, so the cache can be utilized to lookup the source on an exception + ScriptingFunction functionNumber = _createFunction(code); + _cachedFunctions[code] = functionNumber; + return functionNumber; } namespace JSFiles { @@ -507,12 +505,8 @@ public: } protected: - FunctionCacheMap& getFunctionCache() { - return _real->getFunctionCache(); - } - - ScriptingFunction _createFunction(const char* code, ScriptingFunction functionNumber = 0) { - return _real->_createFunction(code, functionNumber); + ScriptingFunction _createFunction(const char* code) { + return _real->_createFunction(code); } private: diff --git a/src/mongo/scripting/engine.h b/src/mongo/scripting/engine.h index f06333e9a45..d31311f0114 100644 --- a/src/mongo/scripting/engine.h +++ b/src/mongo/scripting/engine.h @@ -211,11 +211,7 @@ protected: */ class StoredFuncModLogOpHandler; - virtual FunctionCacheMap& getFunctionCache() { - return _cachedFunctions; - } - virtual ScriptingFunction _createFunction(const char* code, - ScriptingFunction functionNumber = 0) = 0; + virtual ScriptingFunction _createFunction(const char* code) = 0; std::string _localDBName; int64_t _loadedVersion; diff --git a/src/mongo/scripting/mozjs/implscope.cpp b/src/mongo/scripting/mozjs/implscope.cpp index c5bd51ef5cd..3b260c1558c 100644 --- a/src/mongo/scripting/mozjs/implscope.cpp +++ b/src/mongo/scripting/mozjs/implscope.cpp @@ -169,6 +169,10 @@ void MozJSImplScope::kill() { JS_RequestInterruptCallback(_runtime); } +void MozJSImplScope::interrupt() { + JS_RequestInterruptCallback(_runtime); +} + bool MozJSImplScope::isKillPending() const { return _pendingKill.load(); } @@ -483,11 +487,20 @@ BSONObj MozJSImplScope::getObject(const char* field) { void MozJSImplScope::newFunction(StringData raw, JS::MutableHandleValue out) { MozJSEntry entry(this); - std::string code = str::stream() << "____MongoToSM_newFunction_temp = " << raw; + _MozJSCreateFunction(raw, std::move(out)); +} + +void MozJSImplScope::_MozJSCreateFunction(StringData raw, JS::MutableHandleValue fun) { + std::string code = str::stream() + << "(" << parseJSFunctionOrExpression(_context, StringData(raw)) << ")"; JS::CompileOptions co(_context); setCompileOptions(&co); - _checkErrorState(JS::Evaluate(_context, _global, co, code.c_str(), code.length(), out)); + + _checkErrorState(JS::Evaluate(_context, _global, co, code.c_str(), code.length(), fun)); + uassert(10232, + "not a function", + fun.isObject() && JS_ObjectIsFunction(_context, fun.toObjectOrNull())); } BSONObj MozJSImplScope::callThreadArgs(const BSONObj& args) { @@ -528,39 +541,20 @@ bool hasFunctionIdentifier(StringData code) { return code[8] == ' ' || code[8] == '('; } -void MozJSImplScope::_MozJSCreateFunction(const char* raw, - ScriptingFunction functionNumber, - JS::MutableHandleValue fun) { - std::string code = str::stream() << "_funcs" << functionNumber << " = " - << parseJSFunctionOrExpression(_context, StringData(raw)); - - JS::CompileOptions co(_context); - setCompileOptions(&co); - - _checkErrorState(JS::Evaluate(_context, _global, co, code.c_str(), code.length(), fun)); - uassert(10232, - "not a function", - fun.isObject() && JS_ObjectIsFunction(_context, fun.toObjectOrNull())); -} - -ScriptingFunction MozJSImplScope::_createFunction(const char* raw, - ScriptingFunction functionNumber) { +ScriptingFunction MozJSImplScope::_createFunction(const char* raw) { MozJSEntry entry(this); JS::RootedValue fun(_context); - _MozJSCreateFunction(raw, functionNumber, &fun); + _MozJSCreateFunction(raw, &fun); _funcs.emplace_back(_context, fun.get()); - - return functionNumber; + return _funcs.size(); } void MozJSImplScope::setFunction(const char* field, const char* code) { MozJSEntry entry(this); JS::RootedValue fun(_context); - - _MozJSCreateFunction(code, getFunctionCache().size() + 1, &fun); - + _MozJSCreateFunction(code, &fun); ObjectWrapper(_context, _global).setValue(field, fun); } @@ -606,14 +600,16 @@ int MozJSImplScope::invoke(ScriptingFunction func, if (timeoutMs) _engine->getDeadlineMonitor().startDeadline(this, timeoutMs); + else { + _engine->getDeadlineMonitor().startDeadline(this, -1); + } JS::RootedValue out(_context); JS::RootedObject obj(_context, smrecv.toObjectOrNull()); bool success = JS::Call(_context, obj, funcValue, args, &out); - if (timeoutMs) - _engine->getDeadlineMonitor().stopDeadline(this); + _engine->getDeadlineMonitor().stopDeadline(this); _checkErrorState(success); @@ -651,15 +647,17 @@ bool MozJSImplScope::exec(StringData code, if (_checkErrorState(success, reportError, assertOnError)) return false; - if (timeoutMs) + if (timeoutMs) { _engine->getDeadlineMonitor().startDeadline(this, timeoutMs); + } else { + _engine->getDeadlineMonitor().startDeadline(this, -1); + } JS::RootedValue out(_context); success = JS_ExecuteScript(_context, _global, script, &out); - if (timeoutMs) - _engine->getDeadlineMonitor().stopDeadline(this); + _engine->getDeadlineMonitor().stopDeadline(this); if (_checkErrorState(success, reportError, assertOnError)) return false; @@ -706,6 +704,11 @@ void MozJSImplScope::localConnectForDbEval(OperationContext* txn, const char* db // NOTE: order is important here. the following methods must be called after // the above conditional statements. + _connectState = ConnectState::Local; + _localDBName = dbName; + + loadStored(txn); + // install db access functions in the global object installDBAccess(); @@ -713,16 +716,11 @@ void MozJSImplScope::localConnectForDbEval(OperationContext* txn, const char* db _mongoLocalProto.install(_global); execCoreFiles(); - const char* const makeMongo = "_mongo = new Mongo()"; + const char* const makeMongo = "const _mongo = new Mongo()"; exec(makeMongo, "local connect 2", false, true, true, 0); - std::string makeDB = str::stream() << "db = _mongo.getDB(\"" << dbName << "\");"; + std::string makeDB = str::stream() << "const db = _mongo.getDB(\"" << dbName << "\");"; exec(makeDB, "local connect 3", false, true, true, 0); - - _connectState = ConnectState::Local; - _localDBName = dbName; - - loadStored(txn); } void MozJSImplScope::externalSetup() { diff --git a/src/mongo/scripting/mozjs/implscope.h b/src/mongo/scripting/mozjs/implscope.h index 3fac5e5bdef..bfff70a93b0 100644 --- a/src/mongo/scripting/mozjs/implscope.h +++ b/src/mongo/scripting/mozjs/implscope.h @@ -89,6 +89,8 @@ public: void kill(); + void interrupt(); + bool isKillPending() const override; OperationContext* getOpContext() const; @@ -145,8 +147,7 @@ public: void injectNative(const char* field, NativeFunction func, void* data = 0) override; - ScriptingFunction _createFunction(const char* code, - ScriptingFunction functionNumber = 0) override; + ScriptingFunction _createFunction(const char* code) override; void newFunction(StringData code, JS::MutableHandleValue out); @@ -309,9 +310,7 @@ public: } private: - void _MozJSCreateFunction(const char* raw, - ScriptingFunction functionNumber, - JS::MutableHandleValue fun); + void _MozJSCreateFunction(StringData raw, JS::MutableHandleValue fun); /** * This structure exists exclusively to construct the runtime and context diff --git a/src/mongo/scripting/mozjs/mongo.cpp b/src/mongo/scripting/mozjs/mongo.cpp index 31164398961..39dc350d7d5 100644 --- a/src/mongo/scripting/mozjs/mongo.cpp +++ b/src/mongo/scripting/mozjs/mongo.cpp @@ -44,12 +44,14 @@ #include "mongo/scripting/mozjs/valuewriter.h" #include "mongo/scripting/mozjs/wrapconstrainedmethod.h" #include "mongo/stdx/memory.h" +#include "mongo/util/assert_util.h" namespace mongo { namespace mozjs { const JSFunctionSpec MongoBase::methods[] = { - MONGO_ATTACH_JS_CONSTRAINED_METHOD_NO_PROTO(auth, MongoLocalInfo, MongoExternalInfo), + MONGO_ATTACH_JS_CONSTRAINED_METHOD_NO_PROTO(auth, MongoExternalInfo), + MONGO_ATTACH_JS_CONSTRAINED_METHOD_NO_PROTO(close, MongoExternalInfo), MONGO_ATTACH_JS_CONSTRAINED_METHOD_NO_PROTO( copyDatabaseWithSCRAM, MongoLocalInfo, MongoExternalInfo), MONGO_ATTACH_JS_CONSTRAINED_METHOD_NO_PROTO(cursorFromId, MongoLocalInfo, MongoExternalInfo), @@ -89,8 +91,11 @@ const JSFunctionSpec MongoExternalInfo::freeFunctions[4] = { namespace { DBClientBase* getConnection(JS::CallArgs& args) { - return static_cast*>(JS_GetPrivate(args.thisv().toObjectOrNull())) - ->get(); + auto ret = static_cast*>( + JS_GetPrivate(args.thisv().toObjectOrNull()))->get(); + uassert( + ErrorCodes::BadValue, "Trying to get connection for closed Mongo object", ret != nullptr); + return ret; } void setCursor(JS::HandleObject target, @@ -120,6 +125,17 @@ void MongoBase::finalize(JSFreeOp* fop, JSObject* obj) { } } +void MongoBase::Functions::close::call(JSContext* cx, JS::CallArgs args) { + getConnection(args); + + auto thisv = args.thisv().toObjectOrNull(); + auto conn = static_cast*>(JS_GetPrivate(thisv)); + + conn->reset(); + + args.rval().setUndefined(); +} + void MongoBase::Functions::runCommand::call(JSContext* cx, JS::CallArgs args) { if (args.length() != 3) uasserted(ErrorCodes::BadValue, "runCommand needs 3 args"); diff --git a/src/mongo/scripting/mozjs/mongo.h b/src/mongo/scripting/mozjs/mongo.h index c3a82d10998..297b4907c6d 100644 --- a/src/mongo/scripting/mozjs/mongo.h +++ b/src/mongo/scripting/mozjs/mongo.h @@ -46,6 +46,7 @@ struct MongoBase : public BaseInfo { struct Functions { MONGO_DECLARE_JS_FUNCTION(auth); MONGO_DECLARE_JS_FUNCTION(copyDatabaseWithSCRAM); + MONGO_DECLARE_JS_FUNCTION(close); MONGO_DECLARE_JS_FUNCTION(cursorFromId); MONGO_DECLARE_JS_FUNCTION(cursorHandleFromId); MONGO_DECLARE_JS_FUNCTION(find); @@ -63,7 +64,7 @@ struct MongoBase : public BaseInfo { MONGO_DECLARE_JS_FUNCTION(getMaxWireVersion); }; - static const JSFunctionSpec methods[18]; + static const JSFunctionSpec methods[19]; static const char* const className; static const unsigned classFlags = JSCLASS_HAS_PRIVATE; diff --git a/src/mongo/scripting/mozjs/objectwrapper.cpp b/src/mongo/scripting/mozjs/objectwrapper.cpp index 1cecbc169e6..77ea9a1eca8 100644 --- a/src/mongo/scripting/mozjs/objectwrapper.cpp +++ b/src/mongo/scripting/mozjs/objectwrapper.cpp @@ -31,6 +31,7 @@ #include "mongo/scripting/mozjs/objectwrapper.h" #include +#include #include "mongo/base/error_codes.h" #include "mongo/bson/bsonobjbuilder.h" @@ -467,51 +468,60 @@ BSONObj ObjectWrapper::toBSON() { // emplace() inside ValueWriter. The runtime asserts enabled by MozJS's // debug mode will catch runtime errors, but be aware of how difficult this // is to get right and what to look for if one of them bites you. - WriteFieldRecursionFrames frames; - frames.emplace(_context, _object, nullptr, StringData{}); BSONObjBuilder b; - // We special case the _id field in top-level objects and move it to the front. - // This matches other drivers behavior and makes finding the _id field quicker in BSON. - if (hasOwnField(InternedString::_id)) { - _writeField(&b, InternedString::_id, &frames, frames.top().originalBSON); - } + { + // NOTE: Keep the frames in a scope so that it is clear that + // we always destroy them before we destroy 'b'. It is + // important to do so: if 'b' is destroyed before the frames, + // and we don't pop all of the frames (say, due to an + // exeption), then the frame dtors would write to freed + // memory. + WriteFieldRecursionFrames frames; + frames.emplace(_context, _object, nullptr, StringData{}); + + // We special case the _id field in top-level objects and move it to the front. + // This matches other drivers behavior and makes finding the _id field quicker in BSON. + if (hasOwnField(InternedString::_id)) { + _writeField(&b, InternedString::_id, &frames, frames.top().originalBSON); + } - while (frames.size()) { - auto& frame = frames.top(); + while (frames.size()) { + auto& frame = frames.top(); - // If the index is the same as length, we've seen all the keys at this - // level and should go up a level - if (frame.idx == frame.ids.length()) { - frames.pop(); - continue; - } + // If the index is the same as length, we've seen all the keys at this + // level and should go up a level + if (frame.idx == frame.ids.length()) { + frames.pop(); + continue; + } - if (frame.idx == 0 && frame.originalBSON && !frame.altered) { - // If this is our first look at the object and it has an unaltered - // bson behind it, move idx to the end so we'll roll up on the next - // pass through the loop. - frame.subbob_or(&b)->appendElements(*frame.originalBSON); - frame.idx = frame.ids.length(); - continue; - } + if (frame.idx == 0 && frame.originalBSON && !frame.altered) { + // If this is our first look at the object and it has an unaltered + // bson behind it, move idx to the end so we'll roll up on the next + // pass through the loop. + frame.subbob_or(&b)->appendElements(*frame.originalBSON); + frame.idx = frame.ids.length(); + continue; + } - id.set(frame.ids[frame.idx++]); + id.set(frame.ids[frame.idx++]); - if (frames.size() == 1) { - IdWrapper idw(_context, id); + if (frames.size() == 1) { + IdWrapper idw(_context, id); - // TODO: check if it's cheaper to just compare with an interned - // string of "_id" rather than with ascii - if (idw.isString() && idw.equalsAscii("_id")) { - continue; + // TODO: check if it's cheaper to just compare with an interned + // string of "_id" rather than with ascii + if (idw.isString() && idw.equalsAscii("_id")) { + continue; + } } - } - // writeField invokes ValueWriter with the frame stack, which will push - // onto frames for subobjects, which will effectively recurse the loop. - _writeField(frame.subbob_or(&b), JS::HandleId(id), &frames, frame.originalBSON); + // writeField invokes ValueWriter with the frame stack, which will push + // onto frames for subobjects, which will effectively recurse the loop. + _writeField(frame.subbob_or(&b), JS::HandleId(id), &frames, frame.originalBSON); + } } const int sizeWithEOO = b.len() + 1 /*EOO*/ - 4 /*BSONObj::Holder ref count*/; @@ -528,15 +538,42 @@ ObjectWrapper::WriteFieldRecursionFrame::WriteFieldRecursionFrame(JSContext* cx, JSObject* obj, BSONObjBuilder* parent, StringData sd) - : thisv(cx, obj), ids(cx, JS_Enumerate(cx, thisv)) { + : thisv(cx, obj), ids(cx) { + bool isArray = false; if (parent) { - subbob.emplace(JS_IsArrayObject(cx, thisv) ? parent->subarrayStart(sd) - : parent->subobjStart(sd)); + isArray = JS_IsArrayObject(cx, thisv); + + subbob.emplace(isArray ? parent->subarrayStart(sd) : parent->subobjStart(sd)); } - if (!ids) { - throwCurrentJSException( - cx, ErrorCodes::JSInterpreterFailure, "Failure to enumerate object"); + if (isArray) { + uint32_t length; + if (!JS_GetArrayLength(cx, thisv, &length)) { + throwCurrentJSException( + cx, ErrorCodes::JSInterpreterFailure, "Failure to get array length"); + } + + if (!ids.reserve(length)) { + throwCurrentJSException( + cx, ErrorCodes::JSInterpreterFailure, "Failure to reserve array"); + } + + JS::RootedId rid(cx); + for (uint32_t i = 0; i < length; i++) { + rid.set(INT_TO_JSID(i)); + ids.infallibleAppend(rid); + } + } else { + JS::AutoIdArray rids(cx, JS_Enumerate(cx, thisv)); + + if (!ids.reserve(rids.length())) { + throwCurrentJSException( + cx, ErrorCodes::JSInterpreterFailure, "Failure to reserve array"); + } + + for (uint32_t i = 0; i < rids.length(); i++) { + ids.infallibleAppend(rids[i]); + } } if (getScope(cx)->getProto().instanceOf(thisv)) { diff --git a/src/mongo/scripting/mozjs/objectwrapper.h b/src/mongo/scripting/mozjs/objectwrapper.h index 42be21ff36f..983c662efb3 100644 --- a/src/mongo/scripting/mozjs/objectwrapper.h +++ b/src/mongo/scripting/mozjs/objectwrapper.h @@ -198,7 +198,7 @@ private: JS::RootedObject thisv; // ids for the keys of thisv - JS::AutoIdArray ids; + JS::AutoIdVector ids; // Current index of the current key we're working on std::size_t idx = 0; diff --git a/src/mongo/scripting/mozjs/proxyscope.cpp b/src/mongo/scripting/mozjs/proxyscope.cpp index 5e48aabf9cf..bb4cd5c06ff 100644 --- a/src/mongo/scripting/mozjs/proxyscope.cpp +++ b/src/mongo/scripting/mozjs/proxyscope.cpp @@ -229,10 +229,9 @@ void MozJSProxyScope::injectNative(const char* field, NativeFunction func, void* run([&] { _implScope->injectNative(field, func, data); }); } -ScriptingFunction MozJSProxyScope::_createFunction(const char* raw, - ScriptingFunction functionNumber) { +ScriptingFunction MozJSProxyScope::_createFunction(const char* raw) { ScriptingFunction out; - run([&] { out = _implScope->_createFunction(raw, functionNumber); }); + run([&] { out = _implScope->_createFunction(raw); }); return out; } @@ -244,6 +243,10 @@ void MozJSProxyScope::kill() { _implScope->kill(); } +void MozJSProxyScope::interrupt() { + _implScope->interrupt(); +} + /** * Invokes a function on the implementation thread * diff --git a/src/mongo/scripting/mozjs/proxyscope.h b/src/mongo/scripting/mozjs/proxyscope.h index 09a84101dfd..451981330a1 100644 --- a/src/mongo/scripting/mozjs/proxyscope.h +++ b/src/mongo/scripting/mozjs/proxyscope.h @@ -165,8 +165,7 @@ public: void injectNative(const char* field, NativeFunction func, void* data = 0) override; - ScriptingFunction _createFunction(const char* code, - ScriptingFunction functionNumber = 0) override; + ScriptingFunction _createFunction(const char* code) override; OperationContext* getOpContext() const; @@ -175,6 +174,8 @@ public: */ void kill(); + void interrupt(); + private: template void run(Closure&& closure); diff --git a/src/mongo/scripting/mozjs/valuewriter.cpp b/src/mongo/scripting/mozjs/valuewriter.cpp index 0b02a945a41..dc4223050dc 100644 --- a/src/mongo/scripting/mozjs/valuewriter.cpp +++ b/src/mongo/scripting/mozjs/valuewriter.cpp @@ -293,6 +293,10 @@ void ValueWriter::_writeObject(BSONObjBuilder* b, } if (scope->getProto().getJSClass() == jsclass) { + uassert(ErrorCodes::BadValue, + "can't serialize DBPointer prototype", + scope->getProto().getProto() != obj); + JS::RootedValue id(_context); o.getValue("id", &id); diff --git a/src/mongo/scripting/utils.cpp b/src/mongo/scripting/utils.cpp index 10d8fcfe020..a9a9bf0a696 100644 --- a/src/mongo/scripting/utils.cpp +++ b/src/mongo/scripting/utils.cpp @@ -26,8 +26,12 @@ * then also delete it in the license file. */ +#include "mongo/platform/basic.h" + +#include "mongo/bson/json.h" #include "mongo/scripting/engine.h" #include "mongo/util/md5.hpp" +#include "mongo/util/time_support.h" namespace mongo { diff --git a/src/mongo/shell/assert.js b/src/mongo/shell/assert.js index 0d1e225a990..21012a68a35 100644 --- a/src/mongo/shell/assert.js +++ b/src/mongo/shell/assert.js @@ -185,7 +185,7 @@ assert.soon = function(f, msg, timeout /*ms*/, interval) { } var start = new Date(); - timeout = timeout || 30000; + timeout = timeout || 5 * 60 * 1000; interval = interval || 200; var last; while (1) { @@ -452,16 +452,33 @@ assert.writeOK = function(res, msg) { }; assert.writeError = function(res, msg) { + return assert.writeErrorWithCode(res, null, msg); +}; + +assert.writeErrorWithCode = function(res, expectedCode, msg) { var errMsg = null; + var foundCode = null; if (res instanceof WriteResult) { - if (!res.hasWriteError() && !res.hasWriteConcernError()) { + if (res.hasWriteError()) { + foundCode = res.getWriteError().code; + } else if (res.hasWriteConcernError()) { + foundCode = res.getWriteConcernError().code; + } else { errMsg = "no write error: " + tojson(res); } } else if (res instanceof BulkWriteResult) { // Can only happen with bulk inserts - if (!res.hasWriteErrors() && !res.hasWriteConcernError()) { + if (res.hasWriteErrors()) { + if (res.getWriteErrorCount() > 1 && expectedCode != null) { + errMsg = "can't check for specific code when there was more than one write error"; + } else { + foundCode = res.getWriteErrorAt(0).code; + } + } else if (res.hasWriteConcernError()) { + foundCode = res.getWriteConcernError().code; + } else { errMsg = "no write errors: " + tojson(res); } } else if (res instanceof WriteCommandError) { @@ -473,6 +490,12 @@ assert.writeError = function(res, msg) { } } + if (!errMsg && expectedCode) { + if (foundCode != expectedCode) { + errMsg = "found code " + foundCode + " does not match expected code " + expectedCode; + } + } + if (errMsg) { if (msg) errMsg = errMsg + ": " + msg; diff --git a/src/mongo/shell/db.js b/src/mongo/shell/db.js index 9a58b6b114a..54070125f54 100644 --- a/src/mongo/shell/db.js +++ b/src/mongo/shell/db.js @@ -1202,7 +1202,7 @@ var DB; var _defaultWriteConcern = { w: 'majority', - wtimeout: 30 * 1000 + wtimeout: 60 * 1000 }; function getUserObjString(userObj) { diff --git a/src/mongo/shell/linenoise.cpp b/src/mongo/shell/linenoise.cpp index 641a4389e2d..8d32b9294bd 100644 --- a/src/mongo/shell/linenoise.cpp +++ b/src/mongo/shell/linenoise.cpp @@ -2765,7 +2765,17 @@ int linenoiseHistorySetMaxLen(int len) { /* Save the history in the specified file. On success 0 is returned * otherwise -1 is returned. */ int linenoiseHistorySave(const char* filename) { - FILE* fp = fopen(filename, "wt"); + FILE* fp; +#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE || defined(__APPLE__) + int fd = open(filename, O_WRONLY | O_TRUNC | O_CREAT, S_IRUSR | S_IWUSR); + if (fd == -1) { + // report errno somehow? + return -1; + } + fp = fdopen(fd, "wt"); +#else + fp = fopen(filename, "wt"); +#endif // _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE || defined(__APPLE__) if (fp == NULL) { return -1; } @@ -2775,7 +2785,7 @@ int linenoiseHistorySave(const char* filename) { fprintf(fp, "%s\n", history[j]); } } - fclose(fp); + fclose(fp); // Also causes fd to be closed. return 0; } diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js index fb0807909fb..4a64f5e50a0 100644 --- a/src/mongo/shell/replsettest.js +++ b/src/mongo/shell/replsettest.js @@ -89,6 +89,8 @@ var ReplSetTest = function(opts) { var _unbridgedPorts; var _unbridgedNodes; + this.kDefaultTimeoutMS = 5 * 60 * 1000; + // Publicly exposed variables /** @@ -161,7 +163,7 @@ var ReplSetTest = function(opts) { return; } - timeout = timeout || 30000; + timeout = timeout || self.kDefaultTimeoutMS; if (!node.getDB) { node = self.nodes[node]; @@ -179,7 +181,7 @@ var ReplSetTest = function(opts) { var currTime = new Date().getTime(); var status; - assert.soon(function() { + assert.soonNoExcept(function() { try { var conn = _callIsMaster(); if (!conn) { @@ -401,9 +403,9 @@ var ReplSetTest = function(opts) { * Blocks until the secondary nodes have completed recovery and their roles are known. */ this.awaitSecondaryNodes = function(timeout) { - timeout = timeout || 60000; + timeout = timeout || self.kDefaultTimeoutMS; - assert.soon(function() { + assert.soonNoExcept(function() { // Reload who the current slaves are self.getPrimary(timeout); @@ -443,45 +445,60 @@ var ReplSetTest = function(opts) { /** * Blocks until all nodes agree on who the primary is. + * If 'expectedPrimaryNodeId' is provided, ensure that every node is seeing this node as the + * primary. Otherwise, ensure that all the nodes in the set agree with the first node on the + * identity of the primary. */ - this.awaitNodesAgreeOnPrimary = function(timeout) { - timeout = timeout || 60000; + this.awaitNodesAgreeOnPrimary = function(timeout, nodes, expectedPrimaryNodeId) { + timeout = timeout || self.kDefaultTimeoutMS; + nodes = nodes || self.nodes; + expectedPrimaryNodeId = expectedPrimaryNodeId || -1; + if (expectedPrimaryNodeId === -1) { + print("AwaitNodesAgreeOnPrimary: Waiting for nodes to agree on any primary."); + } else { + print("AwaitNodesAgreeOnPrimary: Waiting for nodes to agree on " + + nodes[expectedPrimaryNodeId].name + " as primary."); + } - assert.soon(function() { - try { - var primary = -1; - - for (var i = 0; i < self.nodes.length; i++) { - var replSetGetStatus = - self.nodes[i].getDB("admin").runCommand({replSetGetStatus: 1}); - var nodesPrimary = -1; - for (var j = 0; j < replSetGetStatus.members.length; j++) { - if (replSetGetStatus.members[j].state === ReplSetTest.State.PRIMARY) { - // Node sees two primaries. - if (nodesPrimary !== -1) { - return false; - } - nodesPrimary = j; - } - } - // Node doesn't see a primary. - if (nodesPrimary < 0) { - return false; - } + assert.soonNoExcept(function() { + var primary = expectedPrimaryNodeId; - if (primary < 0) { - // If we haven't seen a primary yet, set it to this. - primary = nodesPrimary; - } else if (primary !== nodesPrimary) { - return false; + for (var i = 0; i < nodes.length; i++) { + var replSetGetStatus = nodes[i].getDB("admin").runCommand({replSetGetStatus: 1}); + var nodesPrimary = -1; + for (var j = 0; j < replSetGetStatus.members.length; j++) { + if (replSetGetStatus.members[j].state === ReplSetTest.State.PRIMARY) { + // Node sees two primaries. + if (nodesPrimary !== -1) { + print("AwaitNodesAgreeOnPrimary: Retrying because " + nodes[i].name + + " thinks both " + nodes[nodesPrimary].name + " and " + + nodes[j].name + " are primary."); + + return false; + } + nodesPrimary = j; } } + // Node doesn't see a primary. + if (nodesPrimary < 0) { + print("AwaitNodesAgreeOnPrimary: Retrying because " + nodes[i].name + + " does not see a primary."); + return false; + } - return true; - } catch (e) { - print("caught exception " + e); - return false; + if (primary < 0) { + // If we haven't seen a primary yet, set it to this. + primary = nodesPrimary; + } else if (primary !== nodesPrimary) { + print("AwaitNodesAgreeOnPrimary: Retrying because " + nodes[i].name + + " thinks the primary is " + nodes[nodesPrimary].name + " instead of " + + nodes[primary].name); + return false; + } } + + print("AwaitNodesAgreeOnPrimary: Nodes agreed on primary " + nodes[primary].name); + return true; }, "Awaiting nodes to agree on primary", timeout); }; @@ -490,10 +507,10 @@ var ReplSetTest = function(opts) { * if primary is available will return a connection to it. Otherwise throws an exception. */ this.getPrimary = function(timeout) { - timeout = timeout || 60000; + timeout = timeout || self.kDefaultTimeoutMS; var primary = null; - assert.soon(function() { + assert.soonNoExcept(function() { primary = _callIsMaster(); return primary; }, "Finding primary", timeout); @@ -503,9 +520,9 @@ var ReplSetTest = function(opts) { this.awaitNoPrimary = function(msg, timeout) { msg = msg || "Timed out waiting for there to be no primary in replset: " + this.name; - timeout = timeout || 30000; + timeout = timeout || self.kDefaultTimeoutMS; - assert.soon(function() { + assert.soonNoExcept(function() { return _callIsMaster() == false; }, msg, timeout); }; @@ -572,7 +589,7 @@ var ReplSetTest = function(opts) { var config = cfg || this.getReplSetConfig(); var cmd = {}; var cmdKey = initCmd || 'replSetInitiate'; - timeout = timeout || 120000; + timeout = timeout || self.kDefaultTimeoutMS; if (jsTestOptions().useLegacyReplicationProtocol && !config.hasOwnProperty("protocolVersion")) { config.protocolVersion = 0; @@ -590,6 +607,49 @@ var ReplSetTest = function(opts) { } }; + /** + * Steps up 'node' as primary. + * Waits for all nodes to reach the same optime before each election. + * Calls awaitReplication() which requires all connections in 'nodes' to be authenticated. + */ + this.stepUp = function(node) { + this.awaitReplication(); + this.awaitNodesAgreeOnPrimary(); + if (this.getPrimary() === node) { + print("Node " + node.host + " is already primary, no need to step it up."); + return; + } + print("Stepping up node " + node.host); + + // Ensure the specified node is primary. + for (var i = 0; i < this.nodes.length; i++) { + var primary = this.getPrimary(); + if (primary === node) { + break; + } + try { + // Make sure the nodes do not step back up for 10 minutes. + assert.commandWorked(primary.adminCommand({replSetStepDown: 10 * 60, force: true})); + } catch (ex) { + print("Caught exception while stepping down node '" + tojson(node.host) + "': " + + tojson(ex)); + } + this.awaitReplication(); + this.awaitNodesAgreeOnPrimary(); + } + + // Reset the rest of the nodes so they can run for election during the test. + for (var i = 0; i < this.nodes.length; i++) { + // Cannot call replSetFreeze on the primary. + if (this.nodes[i] === node) { + continue; + } + assert.commandWorked(this.nodes[i].adminCommand({replSetFreeze: 0})); + } + + assert.eq(this.getPrimary(), node, node.host + " was not primary after stepUp"); + }; + /** * Gets the current replica set config from the specified node index. If no nodeId is specified, * uses the primary node. @@ -625,6 +685,28 @@ var ReplSetTest = function(opts) { } }; + /** + * Blocks until all nodes in the replica set have the same config version as the primary. + **/ + this.awaitNodesAgreeOnConfigVersion = function(timeout) { + timeout = timeout || this.kDefaultTimeoutMS; + + assert.soonNoExcept(function() { + var primaryVersion = self.getPrimary().adminCommand({ismaster: 1}).setVersion; + + for (var i = 0; i < self.nodes.length; i++) { + var version = self.nodes[i].adminCommand({ismaster: 1}).setVersion; + assert.eq(version, + primaryVersion, + "waiting for secondary node " + self.nodes[i].host + + " with config version of " + version + + " to match the version of the primary " + primaryVersion); + } + + return true; + }, "Awaiting nodes to agree on config version", timeout); + }; + /** * Waits for the last oplog entry on the primary to be visible in the committed snapshop view * of the oplog on *all* secondaries. @@ -656,7 +738,7 @@ var ReplSetTest = function(opts) { print("Waiting for op with OpTime " + tojson(opTime) + " to be committed on all secondaries"); - assert.soon(function() { + assert.soonNoExcept(function() { for (var i = 0; i < rst.nodes.length; i++) { var node = rst.nodes[i]; @@ -690,14 +772,14 @@ var ReplSetTest = function(opts) { }; this.awaitReplication = function(timeout) { - timeout = timeout || 30000; + timeout = timeout || self.kDefaultTimeoutMS; var masterLatestOpTime; // Blocking call, which will wait for the last optime written on the master to be available var awaitLastOpTimeWrittenFn = function() { var master = self.getPrimary(); - assert.soon(function() { + assert.soonNoExcept(function() { try { masterLatestOpTime = _getLastOpTime(master); } catch (e) { @@ -706,7 +788,7 @@ var ReplSetTest = function(opts) { } return true; - }, "awaiting oplog query", 30000); + }, "awaiting oplog query", timeout); }; awaitLastOpTimeWrittenFn(); @@ -734,7 +816,7 @@ var ReplSetTest = function(opts) { ", is " + tojson(masterLatestOpTime) + ", last oplog entry is " + tojsononeline(masterOpTime)); - assert.soon(function() { + assert.soonNoExcept(function() { try { print("ReplSetTest awaitReplication: checking secondaries against timestamp " + tojson(masterLatestOpTime)); @@ -752,7 +834,7 @@ var ReplSetTest = function(opts) { ", but expected config version #" + configVersion); if (slaveConfigVersion > configVersion) { - master = this.getPrimary(); + master = self.getPrimary(); configVersion = master.getDB("local")['system.replset'].findOne().version; masterOpTime = _getLastOpTime(master); @@ -1169,7 +1251,7 @@ var ReplSetTest = function(opts) { */ this.waitForMaster = function(timeout) { var master; - assert.soon(function() { + assert.soonNoExcept(function() { return (master = self.getPrimary()); }, "waiting for master", timeout); @@ -1288,7 +1370,7 @@ ReplSetTest.awaitRSClientHosts = function(conn, host, hostOk, rs, timeout) { return; } - timeout = timeout || 60000; + timeout = timeout || 5 * 60 * 1000; if (hostOk == undefined) hostOk = { @@ -1303,7 +1385,7 @@ ReplSetTest.awaitRSClientHosts = function(conn, host, hostOk, rs, timeout) { var tests = 0; - assert.soon(function() { + assert.soonNoExcept(function() { var rsClientHosts = conn.adminCommand('connPoolStats').replicaSets; if (tests++ % 10 == 0) { printjson(rsClientHosts); diff --git a/src/mongo/shell/shell_utils_launcher.cpp b/src/mongo/shell/shell_utils_launcher.cpp index 81793143ec0..d01b2d9ec98 100644 --- a/src/mongo/shell/shell_utils_launcher.cpp +++ b/src/mongo/shell/shell_utils_launcher.cpp @@ -546,7 +546,10 @@ bool wait_for_pid(ProcessId pid, bool block = true, int* exit_code = NULL) { } #else int tmp; - bool ret = (pid.toNative() == waitpid(pid.toNative(), &tmp, (block ? 0 : WNOHANG))); + int ret; + do { + ret = waitpid(pid.toNative(), &tmp, (block ? 0 : WNOHANG)); + } while (ret == -1 && errno == EINTR); if (ret && exit_code) { if (WIFEXITED(tmp)) { *exit_code = WEXITSTATUS(tmp); @@ -556,7 +559,7 @@ bool wait_for_pid(ProcessId pid, bool block = true, int* exit_code = NULL) { MONGO_UNREACHABLE; } } - return ret; + return ret == pid.toNative(); #endif } diff --git a/src/mongo/shell/types.js b/src/mongo/shell/types.js index 066747176bf..622456d0988 100644 --- a/src/mongo/shell/types.js +++ b/src/mongo/shell/types.js @@ -251,7 +251,7 @@ Array.stdDev = function(arr) { Object.extend = function(dst, src, deep) { for (var k in src) { var v = src[k]; - if (deep && typeof(v) == "object") { + if (deep && typeof(v) == "object" && v !== null) { if (v.constructor === ObjectId) { // convert ObjectId properly eval("v = " + tojson(v)); } else if ("floatApprox" in v) { // convert NumberLong properly diff --git a/src/mongo/shell/utils.js b/src/mongo/shell/utils.js index 43925df86a4..2d86a1f6ac3 100644 --- a/src/mongo/shell/utils.js +++ b/src/mongo/shell/utils.js @@ -283,7 +283,15 @@ jsTest.authenticateNodes = function(nodes) { assert.soonNoExcept(function() { for (var i = 0; i < nodes.length; i++) { // Don't try to authenticate to arbiters - res = nodes[i].getDB("admin").runCommand({replSetGetStatus: 1}); + try { + res = nodes[i].getDB("admin").runCommand({replSetGetStatus: 1}); + } catch (e) { + // ReplicaSet tests which don't use auth are allowed to have nodes crash during + // startup. To allow tests which use to behavior to work with auth, + // attempting authentication against a dead node should be non-fatal. + print("Caught exception getting replSetStatus while authenticating: " + e); + continue; + } if (res.myState == 7) { continue; } diff --git a/src/mongo/stdx/thread.h b/src/mongo/stdx/thread.h index 9b1d2f5ae10..822dcaf6b87 100644 --- a/src/mongo/stdx/thread.h +++ b/src/mongo/stdx/thread.h @@ -29,6 +29,8 @@ #pragma once #include +#include +#include #include #include #include @@ -109,7 +111,38 @@ inline void swap(thread& lhs, thread& rhs) BOOST_NOEXCEPT { lhs.swap(rhs); } -namespace this_thread = ::std::this_thread; // NOLINT +namespace this_thread { +using std::this_thread::get_id; // NOLINT +using std::this_thread::yield; // NOLINT + +#ifdef _WIN32 +using std::this_thread::sleep_for; // NOLINT +using std::this_thread::sleep_until; // NOLINT +#else +template +inline void sleep_for(const std::chrono::duration& sleep_duration) { // NOLINT + if (sleep_duration <= sleep_duration.zero()) + return; + + const auto seconds = + std::chrono::duration_cast(sleep_duration); // NOLINT + const auto nanoseconds = + std::chrono::duration_cast(sleep_duration - seconds); // NOLINT + struct timespec sleepVal = {static_cast(seconds.count()), // NOLINT + static_cast(nanoseconds.count())}; + struct timespec remainVal; + while (nanosleep(&sleepVal, &remainVal) == -1 && errno == EINTR) { + sleepVal = remainVal; + } +} + +template +void sleep_until(const std::chrono::time_point& sleep_time) { // NOLINT + const auto now = Clock::now(); + sleep_for(sleep_time - now); +} +#endif +} // namespace this_thread } // namespace stdx } // namespace mongo diff --git a/src/mongo/stdx/type_traits.h b/src/mongo/stdx/type_traits.h new file mode 100644 index 00000000000..9fc95279b98 --- /dev/null +++ b/src/mongo/stdx/type_traits.h @@ -0,0 +1,56 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/config.h" + +#if defined(MONGO_CONFIG_HAVE_STD_ENABLE_IF_T) + +namespace mongo { +namespace stdx { + +using ::std::enable_if_t; + +} // namespace stdx +} // namespace mongo + +#else + +namespace mongo { +namespace stdx { + +template +using enable_if_t = typename std::enable_if::type; + +} // namespace stdx +} // namespace mongo + +#endif diff --git a/src/mongo/util/SConscript b/src/mongo/util/SConscript index 17335efc207..9699c0c5555 100644 --- a/src/mongo/util/SConscript +++ b/src/mongo/util/SConscript @@ -446,3 +446,44 @@ env.CppUnitTest( '$BUILD_DIR/mongo/base', ] ) + +if env.TargetOSIs('linux'): + env.Library( + target='procparser', + source=[ + "procparser.cpp", + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + ], + ) + + env.CppUnitTest( + target='procparser_test', + source=[ + 'procparser_test.cpp', + ], + LIBDEPS=[ + 'procparser', + ]) + +if env.TargetOSIs('windows'): + env.Library( + target='perfctr_collect', + source=[ + "perfctr_collect.cpp", + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/platform/platform', + ], + ) + + env.CppUnitTest( + target='perfctr_collect_test', + source=[ + 'perfctr_collect_test.cpp', + ], + LIBDEPS=[ + 'perfctr_collect', + ]) diff --git a/src/mongo/util/exit_code.h b/src/mongo/util/exit_code.h index 209e27a27a6..004fb579088 100644 --- a/src/mongo/util/exit_code.h +++ b/src/mongo/util/exit_code.h @@ -52,6 +52,7 @@ enum ExitCode : int { EXIT_WINDOWS_SERVICE_STOP = 49, EXIT_POSSIBLE_CORRUPTION = 60, // this means we detected a possible corruption situation, like a buf overflow + EXIT_WATCHDOG = 61, // Internal Storage Node Watchdog has terminated mongod EXIT_UNCAUGHT = 100, // top level exception that wasn't caught EXIT_TEST = 101 }; diff --git a/src/mongo/util/log.h b/src/mongo/util/log.h index b0f53a30b87..3f17151d4f4 100644 --- a/src/mongo/util/log.h +++ b/src/mongo/util/log.h @@ -132,6 +132,22 @@ inline LogstreamBuilder log() { ::MongoLogDefaultComponent_component); } +/** + * Returns a LogstreamBuilder that does not cache its ostream in a threadlocal cache. + * Use this variant when logging from places that may not be able to access threadlocals, + * such as from within other threadlocal-managed objects, or thread_specific_ptr-managed + * objects. + * + * Once SERVER-29377 is completed, this overload can be removed. + */ +inline LogstreamBuilder logNoCache() { + return LogstreamBuilder(logger::globalLogDomain(), + getThreadName(), + logger::LogSeverity::Log(), + ::MongoLogDefaultComponent_component, + false); +} + inline LogstreamBuilder log(logger::LogComponent component) { return LogstreamBuilder( logger::globalLogDomain(), getThreadName(), logger::LogSeverity::Log(), component); diff --git a/src/mongo/util/net/SConscript b/src/mongo/util/net/SConscript index 72d3384fb33..3a72a9b278c 100644 --- a/src/mongo/util/net/SConscript +++ b/src/mongo/util/net/SConscript @@ -38,6 +38,7 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/db/server_options_core', + '$BUILD_DIR/mongo/db/server_parameters', '$BUILD_DIR/mongo/util/background_job', '$BUILD_DIR/mongo/util/concurrency/ticketholder', '$BUILD_DIR/mongo/util/fail_point', diff --git a/src/mongo/util/net/message_port.cpp b/src/mongo/util/net/message_port.cpp index c916c2cfa50..1033ba112dd 100644 --- a/src/mongo/util/net/message_port.cpp +++ b/src/mongo/util/net/message_port.cpp @@ -173,9 +173,22 @@ bool MessagingPort::recv(Message& m) { psock->setHandshakeReceived(); goto again; } + + auto sslMode = sslGlobalParams.sslMode.load(); + uassert(17189, "The server is configured to only allow SSL connections", - sslGlobalParams.sslMode.load() != SSLParams::SSLMode_requireSSL); + sslMode != SSLParams::SSLMode_requireSSL); + + // For users attempting to upgrade their applications from no SSL to SSL, provide + // information about connections that still aren't using SSL (but only once per + // connection) + if (!sslGlobalParams.disableNonSSLConnectionLogging && + (sslMode == SSLParams::SSLMode_preferSSL)) { + LOG(0) << "SSL mode is set to 'preferred' and connection " << connectionId() + << " to " << remote() << " is not using SSL."; + } + #endif // MONGO_CONFIG_SSL } if (static_cast(len) < sizeof(MSGHEADER::Value) || diff --git a/src/mongo/util/net/sock.cpp b/src/mongo/util/net/sock.cpp index 4c34ced371a..4ecde553370 100644 --- a/src/mongo/util/net/sock.cpp +++ b/src/mongo/util/net/sock.cpp @@ -33,7 +33,10 @@ #include "mongo/util/net/sock.h" +#include + #if !defined(_WIN32) +#include #include #include #include @@ -71,6 +74,38 @@ using std::vector; MONGO_FP_DECLARE(throwSockExcep); +namespace { + +// Provides a cross-platform function for setting a file descriptor/socket to non-blocking mode. +bool setBlock(int fd, bool block) { +#ifdef _WIN32 + u_long ioMode = block ? 0 : 1; + return (NO_ERROR == ::ioctlsocket(fd, FIONBIO, &ioMode)); +#else + int flags = fcntl(fd, F_GETFL, fd); + if (block) { + return (-1 != fcntl(fd, F_SETFL, (flags & ~O_NONBLOCK))); + } else { + return (-1 != fcntl(fd, F_SETFL, (flags | O_NONBLOCK))); + } +#endif +} + +void networkWarnWithDescription(const Socket& socket, StringData call, int errorCode = -1) { +#ifdef _WIN32 + if (errorCode == -1) { + errorCode = WSAGetLastError(); + } +#endif + auto ewd = errnoWithDescription(errorCode); + warning() << "Failed to connect to " << socket.remoteAddr().getAddr() << ":" + << socket.remoteAddr().getPort() << ", in(" << call << "), reason: " << ewd; +} + +const double kMaxConnectTimeoutMS = 5000; + +} // namespace + static bool ipv6 = false; void enableIPv6(bool state) { ipv6 = state; @@ -538,53 +573,91 @@ std::string Socket::doSSLHandshake(const char* firstBytes, int len) { } #endif -class ConnectBG : public BackgroundJob { -public: - ConnectBG(int sock, SockAddr remote) : _sock(sock), _remote(remote) {} +bool Socket::connect(SockAddr& remote) { + _remote = remote; - void run() { -#if defined(_WIN32) - if ((_res = _connect()) == SOCKET_ERROR) { - _errnoWithDescription = errnoWithDescription(); + _fd = ::socket(remote.getType(), SOCK_STREAM, 0); + if (_fd == INVALID_SOCKET) { + networkWarnWithDescription(*this, "socket"); + return false; + } + + if (!setBlock(_fd, false)) { + networkWarnWithDescription(*this, "set socket to non-blocking mode"); + return false; + } + + const Milliseconds connectTimeoutMillis(static_cast( + _timeout > 0 ? std::min(kMaxConnectTimeoutMS, (_timeout * 1000)) : kMaxConnectTimeoutMS)); + const Date_t expiration = Date_t::now() + connectTimeoutMillis; + + bool connectSucceeded = ::connect(_fd, _remote.raw(), _remote.addressSize) == 0; + + if (!connectSucceeded) { +#ifdef _WIN32 + if (WSAGetLastError() != WSAEWOULDBLOCK) { + networkWarnWithDescription(*this, "connect"); + return false; } #else - while ((_res = _connect()) == -1) { - const int error = errno; - if (error != EINTR) { - _errnoWithDescription = errnoWithDescription(error); - break; - } + if (errno != EINTR && errno != EINPROGRESS) { + networkWarnWithDescription(*this, "connect"); + return false; } #endif - } - std::string name() const { - return "ConnectBG"; - } - std::string getErrnoWithDescription() const { - return _errnoWithDescription; - } - int inError() const { - return _res; - } + pollfd pfd; + pfd.fd = _fd; + pfd.events = POLLOUT; -private: - int _connect() const { - return ::connect(_sock, _remote.raw(), _remote.addressSize); - } + while (true) { + const auto timeout = std::max(Milliseconds(0), expiration - Date_t::now()); - int _sock; - int _res; - SockAddr _remote; - std::string _errnoWithDescription; -}; + int pollReturn = socketPoll(&pfd, 1, timeout.count()); +#ifdef _WIN32 + if (pollReturn == SOCKET_ERROR) { + networkWarnWithDescription(*this, "poll"); + return false; + } +#else + if (pollReturn == -1) { + if (errno != EINTR) { + networkWarnWithDescription(*this, "poll"); + return false; + } -bool Socket::connect(SockAddr& remote) { - _remote = remote; + // EINTR in poll, try again + continue; + } +#endif + // No activity for the full duration of the timeout. + if (pollReturn == 0) { + warning() << "Failed to connect to " << _remote.getAddr() << ":" + << _remote.getPort() << " after " << connectTimeoutMillis + << " milliseconds, giving up."; + return false; + } - _fd = socket(remote.getType(), SOCK_STREAM, 0); - if (_fd == INVALID_SOCKET) { - LOG(_logLevel) << "ERROR: connect invalid socket " << errnoWithDescription() << endl; + // We had a result, see if there's an error on the socket. + int optVal; + socklen_t optLen = sizeof(optVal); + if (::getsockopt( + _fd, SOL_SOCKET, SO_ERROR, reinterpret_cast(&optVal), &optLen) == -1) { + networkWarnWithDescription(*this, "getsockopt"); + return false; + } + if (optVal != 0) { + networkWarnWithDescription(*this, "checking socket for error after poll", optVal); + return false; + } + + // We had activity and we don't have errors on the socket, we're connected. + break; + } + } + + if (!setBlock(_fd, true)) { + networkWarnWithDescription(*this, "could not set socket to blocking mode"); return false; } @@ -592,25 +665,6 @@ bool Socket::connect(SockAddr& remote) { setTimeout(_timeout); } - static const unsigned int connectTimeoutMillis = 5000; - ConnectBG bg(_fd, remote); - bg.go(); - if (bg.wait(connectTimeoutMillis)) { - if (bg.inError()) { - warning() << "Failed to connect to " << _remote.getAddr() << ":" << _remote.getPort() - << ", reason: " << bg.getErrnoWithDescription() << endl; - close(); - return false; - } - } else { - // time out the connect - close(); - bg.wait(); // so bg stays in scope until bg thread terminates - warning() << "Failed to connect to " << _remote.getAddr() << ":" << _remote.getPort() - << " after " << connectTimeoutMillis << " milliseconds, giving up." << endl; - return false; - } - if (remote.getType() != AF_UNIX) disableNagle(_fd); @@ -637,9 +691,7 @@ int Socket::_send(const char* data, int len, const char* context) { } #endif int ret = ::send(_fd, data, len, portSendFlags); - if (ret < 0) { - handleSendError(ret, context); - } + return ret; } @@ -653,11 +705,15 @@ void Socket::send(const char* data, int len, const char* context) { #else errno = ENETUNREACH; #endif - handleSendError(ret, context); } else { ret = _send(data, len, context); } + if (ret < 0) { + handleSendError(ret, context); + continue; + } + _bytesOut += ret; fassert(16507, ret <= len); @@ -717,15 +773,7 @@ void Socket::send(const vector>& data, const char* context) { } if (ret == -1) { - if (errno != EAGAIN || _timeout == 0) { - LOG(_logLevel) << "Socket " << context << " send() " << errnoWithDescription() - << ' ' << remoteString() << endl; - throw SocketException(SocketException::SEND_ERROR, remoteString()); - } else { - LOG(_logLevel) << "Socket " << context << " send() remote timeout " - << remoteString() << endl; - throw SocketException(SocketException::SEND_TIMEOUT, remoteString()); - } + handleSendError(ret, context); } else { struct iovec*& i = meta.msg_iov; while (ret > 0) { @@ -798,7 +846,7 @@ void Socket::handleSendError(int ret, const char* context) { #endif LOG(_logLevel) << "Socket " << context << " send() timed out " << remoteString() << endl; throw SocketException(SocketException::SEND_TIMEOUT, remoteString()); - } else { + } else if (mongo_errno != EINTR) { LOG(_logLevel) << "Socket " << context << " send() " << errnoWithDescription(mongo_errno) << ' ' << remoteString() << endl; throw SocketException(SocketException::SEND_ERROR, remoteString()); @@ -818,7 +866,6 @@ void Socket::handleRecvError(int ret, int len) { int e = errno; #if defined(EINTR) if (e == EINTR) { - LOG(_logLevel) << "EINTR returned from recv(), retrying"; return; } #endif diff --git a/src/mongo/util/net/sock.h b/src/mongo/util/net/sock.h index 74a36c38c09..399cbc09e9f 100644 --- a/src/mongo/util/net/sock.h +++ b/src/mongo/util/net/sock.h @@ -348,7 +348,7 @@ public: } void handleRecvError(int ret, int len); - MONGO_COMPILER_NORETURN void handleSendError(int ret, const char* context); + void handleSendError(int ret, const char* context); private: void _init(); diff --git a/src/mongo/util/net/ssl_manager.cpp b/src/mongo/util/net/ssl_manager.cpp index 7057bf3c004..9dd51e46fb9 100644 --- a/src/mongo/util/net/ssl_manager.cpp +++ b/src/mongo/util/net/ssl_manager.cpp @@ -43,6 +43,7 @@ #include "mongo/base/init.h" #include "mongo/bson/bsonobjbuilder.h" #include "mongo/config.h" +#include "mongo/db/server_parameters.h" #include "mongo/platform/atomic_word.h" #include "mongo/stdx/memory.h" #include "mongo/util/concurrency/mutex.h" @@ -77,6 +78,41 @@ const SSLParams& getSSLGlobalParams() { return sslGlobalParams; } +/** + * Configurable via --setParameter disableNonSSLConnectionLogging=true. If false (default) + * if the sslMode is set to preferSSL, we will log connections that are not using SSL. + * If true, such log messages will be suppressed. + */ +ExportedServerParameter + disableNonSSLConnectionLoggingParameter(ServerParameterSet::getGlobal(), + "disableNonSSLConnectionLogging", + &sslGlobalParams.disableNonSSLConnectionLogging); + +class OpenSSLCipherConfigParameter + : public ExportedServerParameter { +public: + OpenSSLCipherConfigParameter() + : ExportedServerParameter( + ServerParameterSet::getGlobal(), + "opensslCipherConfig", + &sslGlobalParams.sslCipherConfig) {} + Status validate(const std::string& potentialNewValue) final { + if (!sslGlobalParams.sslCipherConfig.empty()) { + return Status( + ErrorCodes::BadValue, + "opensslCipherConfig setParameter is incompatible with net.ssl.sslCipherConfig"); + } + // Note that there is very little validation that we can do here. + // OpenSSL exposes no API to validate a cipher config string. The only way to figure out + // what a string maps to is to make an SSL_CTX object, set the string on it, then parse the + // resulting STACK_OF object. If provided an invalid entry in the string, it will silently + // ignore it. Because an entry in the string may map to multiple ciphers, or remove ciphers + // from the final set produced by the full string, we can't tell if any entry failed + // to parse. + return Status::OK(); + } +} openSSLCipherConfig; + #ifdef MONGO_CONFIG_SSL // Old copies of OpenSSL will not have constants to disable protocols they don't support. // Define them to values we can OR together safely to generically disable these protocols across @@ -709,7 +745,7 @@ unsigned long long SSLManager::_convertASN1ToMillis(ASN1_TIME* asn1time) { bool SSLManager::_parseAndValidateCertificate(const std::string& keyFile, std::string* subjectName, Date_t* serverCertificateExpirationDate) { - BIO* inBIO = BIO_new(BIO_s_file_internal()); + BIO* inBIO = BIO_new(BIO_s_file()); if (inBIO == NULL) { error() << "failed to allocate BIO object: " << getSSLErrorMessage(ERR_get_error()); return false; diff --git a/src/mongo/util/net/ssl_options.cpp b/src/mongo/util/net/ssl_options.cpp index 2d29e4704f2..a29b2d1805c 100644 --- a/src/mongo/util/net/ssl_options.cpp +++ b/src/mongo/util/net/ssl_options.cpp @@ -257,6 +257,13 @@ Status storeSSLServerOptions(const moe::Environment& params) { } if (params.count("net.ssl.sslCipherConfig")) { + warning() + << "net.ssl.sslCipherConfig is deprecated. It will be removed in a future release."; + if (!sslGlobalParams.sslCipherConfig.empty()) { + return Status(ErrorCodes::BadValue, + "net.ssl.sslCipherConfig is incompatible with the openSSLCipherConfig " + "setParameter"); + } sslGlobalParams.sslCipherConfig = params["net.ssl.sslCipherConfig"].as(); } diff --git a/src/mongo/util/net/ssl_options.h b/src/mongo/util/net/ssl_options.h index e897656ed25..f148c400b9b 100644 --- a/src/mongo/util/net/ssl_options.h +++ b/src/mongo/util/net/ssl_options.h @@ -58,6 +58,8 @@ struct SSLParams { bool sslFIPSMode; // --sslFIPSMode bool sslAllowInvalidCertificates; // --sslAllowInvalidCertificates bool sslAllowInvalidHostnames; // --sslAllowInvalidHostnames + bool disableNonSSLConnectionLogging = + false; // --setParameter disableNonSSLConnectionLogging=true SSLParams() { sslMode.store(SSLMode_disabled); diff --git a/src/mongo/util/perfctr_collect.cpp b/src/mongo/util/perfctr_collect.cpp new file mode 100644 index 00000000000..89ac6694a60 --- /dev/null +++ b/src/mongo/util/perfctr_collect.cpp @@ -0,0 +1,505 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kFTDC + +#include "mongo/platform/basic.h" + +#include "mongo/util/perfctr_collect.h" + +#include "mongo/base/init.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/util/log.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/scopeguard.h" +#include "mongo/util/text.h" + +namespace mongo { + +namespace { + +// Handle to the PDH library so that we can format error messages. +HANDLE hPdhLibrary = nullptr; + +/** + * Load PDH.DLL for good error messages. + */ +MONGO_INITIALIZER(PdhInit)(InitializerContext* context) { + hPdhLibrary = LoadLibraryW(L"pdh.dll"); + if (nullptr == hPdhLibrary) { + DWORD gle = GetLastError(); + return {ErrorCodes::WindowsPdhError, + str::stream() << "LoadLibrary of pdh.dll failed with " + << errnoWithDescription(gle)}; + } + + return Status::OK(); +} + +/** + * Output an error message for ether PDH or the system. + */ +std::string errnoWithPdhDescription(PDH_STATUS status) { + LPWSTR errorText = nullptr; + + if (!FormatMessageW( + FORMAT_MESSAGE_FROM_HMODULE | FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_ALLOCATE_BUFFER, + hPdhLibrary, + status, + 0, + reinterpret_cast( + &errorText), // fudge the type so FormatMessageW uses it as an out parameter. + 0, + nullptr)) { + DWORD gle = GetLastError(); + return str::stream() << "Format message failed with " << gle << " for status " << status; + } + + ScopeGuard errorTextGuard = MakeGuard([errorText]() { LocalFree(errorText); }); + std::string utf8ErrorText = toUtf8String(errorText); + + auto size = utf8ErrorText.find_first_of("\r\n"); + if (size == std::string::npos) { + size = utf8ErrorText.length(); + } + + return utf8ErrorText.substr(0, size); +} + +/** + * Format an error message for a PDH function call failure. + */ +std::string formatFunctionCallError(StringData functionName, PDH_STATUS status) { + return str::stream() << functionName << " failed with '" << errnoWithPdhDescription(status) + << "'"; +} + +/** + * Transform a vector of string data into a vector of strings. + */ +void transformStringDataVector(const std::vector& input, + std::vector* output) { + output->reserve(input.size()); + for (const auto& str : input) { + output->emplace_back(str.toString()); + } +} + +/** + * Check if a counter depends on system ticks per second to compute its value from raw values. This + * is basically any counter that does not use 100NS as a base. FYI, if we query raw count counters, + * we will get the system ticks as a time base. + */ +bool counterHasTickBasedTimeBase(uint32_t type) { + return ((type & PERF_TYPE_COUNTER) == PERF_TYPE_COUNTER) && + ((type & PERF_TIMER_100NS) != PERF_TIMER_100NS); +} + +} // namespace + +StatusWith> PerfCounterCollection::checkCounters( + StringData name, const std::vector& paths) { + if (_counters.find(name.toString()) != _counters.end() || + _nestedCounters.find(name.toString()) != _nestedCounters.end()) { + return Status(ErrorCodes::BadValue, str::stream() << "Duplicate group name for " << name); + } + + std::vector stringPaths; + transformStringDataVector(paths, &stringPaths); + + // While duplicate counter paths are not a problem for PDH, they are a waste of time. + std::sort(stringPaths.begin(), stringPaths.end()); + + if (std::unique(stringPaths.begin(), stringPaths.end()) != stringPaths.end()) { + return Status(ErrorCodes::BadValue, + str::stream() << "Duplicate counters in paths specified"); + } + + return {stringPaths}; +} + +Status PerfCounterCollection::addCountersGroup(StringData name, + const std::vector& paths) { + auto swCounters = checkCounters(name, paths); + if (!swCounters.getStatus().isOK()) { + return swCounters.getStatus(); + } + + _counters.emplace(name.toString(), std::move(swCounters.getValue())); + + return Status::OK(); +} + +Status PerfCounterCollection::addCountersGroupedByInstanceName( + StringData name, const std::vector& paths) { + auto swCounters = checkCounters(name, paths); + if (!swCounters.getStatus().isOK()) { + return swCounters.getStatus(); + } + + _nestedCounters.emplace(name.toString(), std::move(swCounters.getValue())); + + return Status::OK(); +} + +StatusWith> PerfCounterCollector::create( + PerfCounterCollection builder) { + auto pcc = std::unique_ptr(new PerfCounterCollector()); + + Status s = pcc->open(); + if (!s.isOK()) { + return s; + } + + for (const auto& kvp : builder._counters) { + s = pcc->addCountersGroup(kvp.first, kvp.second); + if (!s.isOK()) { + return s; + } + } + + // Sort to enforce predictable output in final document + std::sort(pcc->_counters.begin(), + pcc->_counters.end(), + [](const CounterGroup& a, const CounterGroup& b) { return a.name < b.name; }); + + for (const auto& kvp : builder._nestedCounters) { + s = pcc->addCountersGroupedByInstanceName(kvp.first, kvp.second); + if (!s.isOK()) { + return s; + } + } + + std::sort( + pcc->_nestedCounters.begin(), + pcc->_nestedCounters.end(), + [](const NestedCounterGroup& a, const NestedCounterGroup& b) { return a.name < b.name; }); + + pcc->checkForTicksTimeBase(); + + return {std::move(pcc)}; +} + +PerfCounterCollector::~PerfCounterCollector() { + /*ignore*/ PdhCloseQuery(_query); +} + +Status PerfCounterCollector::open() { + PDH_STATUS status = PdhOpenQueryW(nullptr, NULL, &_query); + if (status != ERROR_SUCCESS) { + return {ErrorCodes::WindowsPdhError, formatFunctionCallError("PdhOpenQueryW", status)}; + } + + return Status::OK(); +} + +StatusWith PerfCounterCollector::addCounter(StringData path) { + PDH_HCOUNTER counter{0}; + + PDH_STATUS status = + PdhAddCounterW(_query, toNativeString(path.toString().c_str()).c_str(), NULL, &counter); + + if (status != ERROR_SUCCESS) { + return {ErrorCodes::WindowsPdhError, formatFunctionCallError("PdhAddCounterW", status)}; + } + + DWORD bufferSize = 0; + + status = PdhGetCounterInfoW(counter, false, &bufferSize, nullptr); + + if (status != PDH_MORE_DATA) { + return {ErrorCodes::WindowsPdhError, formatFunctionCallError("PdhGetCounterInfoW", status)}; + } + + auto buf = stdx::make_unique(bufferSize); + auto counterInfo = reinterpret_cast(buf.get()); + + status = PdhGetCounterInfoW(counter, false, &bufferSize, counterInfo); + + if (status != ERROR_SUCCESS) { + return {ErrorCodes::WindowsPdhError, formatFunctionCallError("PdhGetCounterInfoW", status)}; + } + + // A full qualified path is as such: + // "\\MYMACHINE\\Processor(0)\\% Idle Time" + // MachineName \\ Object Name (Instance Name) \\ CounterName + // Ex: + // MachineName: MYMACHINE + // Object Name: Processor + // InstanceName: 0 + // CounterName: % Idle Time + // We do not want to use Machine Name, but sometimes we want InstanceName + // + std::string firstName = str::stream() << '\\' << toUtf8String(counterInfo->szObjectName) << '\\' + << toUtf8String(counterInfo->szCounterName); + + // Compute a second name + std::string secondName(firstName); + + bool hasSecondValue = false; + + // Only include base for counters that need it + if ((counterInfo->dwType & PERF_COUNTER_PRECISION) == PERF_COUNTER_PRECISION) { + secondName += " Base"; + hasSecondValue = true; + } + + // InstanceName is null for counters without instance names + return {CounterInfo{std::move(firstName), + std::move(secondName), + hasSecondValue, + counterInfo->szInstanceName ? toUtf8String(counterInfo->szInstanceName) + : std::string(), + counterInfo->dwType, + counter}}; +} + +StatusWith> PerfCounterCollector::addCounters( + StringData path) { + std::wstring pathWide = toNativeString(path.toString().c_str()); + DWORD pathListLength = 0; + PDH_STATUS status = PdhExpandCounterPathW(pathWide.c_str(), nullptr, &pathListLength); + + if (status != PDH_MORE_DATA) { + return {ErrorCodes::WindowsPdhError, + str::stream() << formatFunctionCallError("PdhExpandCounterPathW", status) + << " for counter '" << path << "'"}; + } + + auto buf = stdx::make_unique(pathListLength); + + status = PdhExpandCounterPathW(pathWide.c_str(), buf.get(), &pathListLength); + + if (status != ERROR_SUCCESS) { + return {ErrorCodes::WindowsPdhError, + formatFunctionCallError("PdhExpandCounterPathW", status)}; + } + + std::vector counters; + + // Windows' PdhExpandWildCardPathW returns a nullptr terminated array of nullptr separated + // strings. + std::vector counterNames; + + const wchar_t* ptr = buf.get(); + while (ptr && *ptr) { + counterNames.emplace_back(toUtf8String(ptr)); + ptr += wcslen(ptr) + 1; + } + + // Sort to ensure we have a predictable ordering in the final BSON + std::sort(counterNames.begin(), counterNames.end()); + + for (const auto& name : counterNames) { + auto swCounterInfo = addCounter(name); + if (!swCounterInfo.isOK()) { + return swCounterInfo.getStatus(); + } + + counters.emplace_back(std::move(swCounterInfo.getValue())); + } + + return {std::move(counters)}; +} + +Status PerfCounterCollector::addCountersGroup(StringData groupName, + const std::vector& paths) { + CounterGroup group; + group.name = groupName.toString(); + + for (const auto& path : paths) { + auto swCounters = addCounters(path.c_str()); + if (!swCounters.isOK()) { + return swCounters.getStatus(); + } + + auto newCounters = swCounters.getValue(); + + std::copy(newCounters.begin(), newCounters.end(), std::back_inserter(group.counters)); + } + + _counters.emplace_back(group); + + return Status::OK(); +} + +Status PerfCounterCollector::addCountersGroupedByInstanceName( + StringData groupName, const std::vector& paths) { + NestedCounterGroup group; + group.name = groupName.toString(); + + for (const auto& path : paths) { + auto swCounters = addCounters(path.c_str()); + if (!swCounters.isOK()) { + return swCounters.getStatus(); + } + + auto newCounters = swCounters.getValue(); + + for (const auto& counter : newCounters) { + // Verify the counter has an instance name. + if (counter.instanceName.empty()) { + return {ErrorCodes::BadValue, + str::stream() << "Counter '" << counter.firstName + << "' must be an instance specific counter."}; + } + + // Skip counters in the _Total instance category. + if (counter.instanceName == "_Total") { + continue; + } + + group.counters[counter.instanceName].emplace_back(std::move(counter)); + } + } + + _nestedCounters.emplace_back(group); + + return Status::OK(); +} + +Status PerfCounterCollector::collectCounters(const std::vector& counters, + BSONObjBuilder* builder) { + for (const auto& counterInfo : counters) { + DWORD dwType = 0; + + // Elapsed Time is an unusual counter in that being able to control the sample period for + // the counter is uninteresting even though it is computed from two values. Just return the + // computed value instead. + if (counterInfo.type == PERF_ELAPSED_TIME) { + PDH_FMT_COUNTERVALUE counterValue = {0}; + PDH_STATUS status = PdhGetFormattedCounterValue( + counterInfo.handle, PDH_FMT_DOUBLE, &dwType, &counterValue); + if (status != ERROR_SUCCESS) { + return {ErrorCodes::WindowsPdhError, + formatFunctionCallError("PdhGetFormattedCounterValue", status)}; + } + + builder->append(counterInfo.firstName, counterValue.doubleValue); + + } else { + PDH_RAW_COUNTER rawCounter = {0}; + PDH_STATUS status = PdhGetRawCounterValue(counterInfo.handle, &dwType, &rawCounter); + if (status != ERROR_SUCCESS) { + return {ErrorCodes::WindowsPdhError, + formatFunctionCallError("PdhGetRawCounterValue", status)}; + } + + if (counterInfo.hasSecondValue) { + // Precise counters require the second value in the raw counter information + builder->append(counterInfo.firstName, rawCounter.FirstValue); + builder->append(counterInfo.secondName, rawCounter.SecondValue); + } else { + builder->append(counterInfo.firstName, rawCounter.FirstValue); + } + } + } + + return Status::OK(); +} + +void PerfCounterCollector::checkForTicksTimeBase() { + for (const auto& counterGroup : _counters) { + for (const auto& counter : counterGroup.counters) { + if (counterHasTickBasedTimeBase(counter.type)) { + _timeBaseTicksCounter = &counter; + return; + } + } + } + + for (const auto& counterGroup : _nestedCounters) { + for (const auto& instanceNamePair : counterGroup.counters) { + for (const auto& counter : instanceNamePair.second) { + if (counterHasTickBasedTimeBase(counter.type)) { + _timeBaseTicksCounter = &counter; + return; + } + } + } + } +} + +Status PerfCounterCollector::collect(BSONObjBuilder* builder) { + // Ask PDH to collect the counters + PDH_STATUS status = PdhCollectQueryData(_query); + if (status != ERROR_SUCCESS) { + return {ErrorCodes::WindowsPdhError, + formatFunctionCallError("PdhCollectQueryData", status)}; + } + + // Output timebase + // Counters that are based on time either use 100NS or System Ticks Per Second. + // We only need to output system ticks per second once if any counter depends on it. + // This is typically 3320310. + if (_timeBaseTicksCounter) { + int64_t timebase; + + status = PdhGetCounterTimeBase(_timeBaseTicksCounter->handle, &timebase); + if (status != ERROR_SUCCESS) { + return {ErrorCodes::WindowsPdhError, + formatFunctionCallError("PdhGetCounterTimeBase", status)}; + } + + builder->append("timebase", timebase); + } + + // Retrieve all the values that PDH collected for us. + for (const auto& counterGroup : _counters) { + BSONObjBuilder subObjBuilder(builder->subobjStart(counterGroup.name)); + + Status s = collectCounters(counterGroup.counters, &subObjBuilder); + if (!s.isOK()) { + return s; + } + + subObjBuilder.doneFast(); + } + + for (const auto& counterGroup : _nestedCounters) { + BSONObjBuilder subObjBuilder(builder->subobjStart(counterGroup.name)); + + for (const auto& instanceNamePair : counterGroup.counters) { + BSONObjBuilder instSubObjBuilder(builder->subobjStart(instanceNamePair.first)); + + Status s = collectCounters(instanceNamePair.second, &instSubObjBuilder); + if (!s.isOK()) { + return s; + } + + instSubObjBuilder.doneFast(); + } + + subObjBuilder.doneFast(); + } + + return Status::OK(); +} + +} // namespace mongo diff --git a/src/mongo/util/perfctr_collect.h b/src/mongo/util/perfctr_collect.h new file mode 100644 index 00000000000..5cf42409ff5 --- /dev/null +++ b/src/mongo/util/perfctr_collect.h @@ -0,0 +1,296 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/base/status.h" +#include "mongo/base/status_with.h" +#include "mongo/base/string_data.h" +#include "mongo/stdx/memory.h" + +namespace mongo { + +class BSONObjBuilder; + +/** + * PerfCounterCollection contains a set of counters for PerfCounterCollector to collect. This class + * supports adding counters with wildcards. It also optionally supports grouping counters by + * instance name. + */ +class PerfCounterCollection { + MONGO_DISALLOW_COPYING(PerfCounterCollection); + + friend class PerfCounterCollector; + +public: + PerfCounterCollection() = default; + PerfCounterCollection(PerfCounterCollection&& other) + : _counters(std::move(other._counters)), + _nestedCounters(std::move(other._nestedCounters)) {} + PerfCounterCollection& operator=(PerfCounterCollection&& other) { + _counters = std::move(other._counters); + _nestedCounters = std::move(other._nestedCounters); + return *this; + } + + /** + * Add vector of counters grouped under 'name'. + * + * group name - the name of the BSON document to add these counters into. + * paths - a vector of counter paths. These may contain wildcards. + * + * Errors if groupName duplicates an existing group or if paths has duplicate keys. Does not + * validate if the counters exist. + * + * Output document: + * For the following counters in "cpu": + * "\System\Processes" + * "\Processor(_Total)\% Idle Time" + * + * { + * "cpu" : { + * "\System\Processes" : 42, + * "\Processor\% Idle Time" : 12, + * "\Processor\% Idle Time Base" : 53, + * } + * } + */ + Status addCountersGroup(StringData groupName, const std::vector& paths); + + /** + * Add vector of counters grouped under 'name', and grouped by instance name. + * + * group name - the name of the BSON document to add these counters into. + * paths - a vector of counter paths. These may contain wildcards. The '_Total' instance is + * automatically filtered since it can be computed by summing other instances. + * + * Errors if groupName duplicates an existing group or if paths has duplicate keys. Does not + * validate if the counters exist. + * + * Output document: + * For the following counters in "disks": + * "\PhysicalDisk(*)\% Disk Write Time" + * + * { + * "disks" : { + * "0 C:" : { + * "\PhysicalDisk\% Disk Write Time": 42, + * "\PhysicalDisk\% Disk Write Time Base": 32, + * }, + * "1 D:" : { + * "\PhysicalDisk\% Disk Write Time": 43, + * "\PhysicalDisk\% Disk Write Time Base": 37, + * } + * } + * } + */ + Status addCountersGroupedByInstanceName(StringData groupName, + const std::vector& paths); + +private: + /** + * Check for duplicate group and counters. + */ + StatusWith> checkCounters(StringData groupName, + const std::vector& paths); + +private: + // Vector of counters which are not sub-grouped by instance name. + std::unordered_map> _counters; + + // Vector of counters sub grouped by instance name. + std::unordered_map> _nestedCounters; +}; + +/** + * PerfCounterCollector collects a series of counters from a Performance Data Helper (PDH) Query and + * output the raw counter values to BSONObjBuilder. + */ +class PerfCounterCollector { + MONGO_DISALLOW_COPYING(PerfCounterCollector); + +public: + ~PerfCounterCollector(); + PerfCounterCollector(PerfCounterCollector&& other) + : _query(std::move(other._query)), + _counters(std::move(other._counters)), + _nestedCounters(std::move(other._nestedCounters)), + _timeBaseTicksCounter(std::move(other._timeBaseTicksCounter)) {} + + /** + * Create a PerfCounterCollector to collect the performance counters in the specified + * PerfCounterCollection. + */ + static StatusWith> create(PerfCounterCollection builder); + + /** + * Collect the counters from PDH, and output their raw values into builder. The exception is + * elapsed time counters which returns computed values instead of raw values. + * + * For each counters, if the counter is a precision counter (see PERF_COUNTER_PRECISION), the + * second value is output under the name " Base". Also, a single field is output called + * "timebase" if any counter depends on system ticks per second. See counterHasTickBasedTimeBase + * for more details about timebase. + */ + Status collect(BSONObjBuilder* builder); + +private: + /** + * Describes a counter by querying PDH, and contains the necessary information to retrieve a + * counter from PDH. + */ + struct CounterInfo { + /** + * The name of the first value for a counter. This is output as: + * "\\". + */ + std::string firstName; + + /** + * The name of the second value of a counter if the counter is a precision counter. This is + * output as: "\\ Base". + */ + std::string secondName; + + /** + * True if the counter is a precision counter, and its value should be output in the output + * BSON document. + */ + bool hasSecondValue; + + /** + * Instance name of the counter. Empty if the counter has no instance name. + */ + std::string instanceName; + + /** + * Counter Type. See PERF_* constants in winperf.h. + * https://technet.microsoft.com/en-us/library/cc785636(v=ws.10).aspx + */ + uint32_t type; + + /** + * Handle of counter to collect from. + */ + PDH_HCOUNTER handle; + }; + + /** + * A set of counters that are part of "name" in the final bson document. + */ + struct CounterGroup { + /** + * Name of the counter group. + */ + std::string name; + + /** + * Vector of counters in this group. + */ + std::vector counters; + }; + + /** + * A set of counters that are part of "name" and "instanceName" in the final bson document. + */ + struct NestedCounterGroup { + /** + * Name of the counter group. + */ + std::string name; + + /** + * A map of instance name to vector of counters to collect for each instance name. + * Ordered Map to ensure output is well-ordered. + */ + std::map> counters; + }; + +private: + PerfCounterCollector() = default; + + /** + * Open the PDH Query. + */ + Status open(); + + /** + * Add the specified counter group to the PDH Query. + */ + Status addCountersGroup(StringData groupName, const std::vector& paths); + + /** + * Add the specified counter group to the PDH Query grouped by instance name. + */ + Status addCountersGroupedByInstanceName(StringData groupName, + const std::vector& paths); + + /** + * Add a counter to the PDH query and get a description of it. + */ + StatusWith addCounter(StringData path); + + /** + * Add a set of counters to the PDH query, and get descriptions of them. + */ + StatusWith> addCounters(StringData path); + + /** + * Collect a vector of counters and output them to builder. + */ + Status collectCounters(const std::vector& counters, BSONObjBuilder* builder); + + /** + * Check if any of the counters we want depends on system ticks per second as a time base. + */ + void checkForTicksTimeBase(); + +private: + // PDH Query + HQUERY _query{INVALID_HANDLE_VALUE}; + + // Typically: CPU & Memory counters + std::vector _counters; + + // Typically: Disks counters + std::vector _nestedCounters; + + // A counter that uses ticks as a timebase + const CounterInfo* _timeBaseTicksCounter{nullptr}; +}; + +} // namespace mongo diff --git a/src/mongo/util/perfctr_collect_test.cpp b/src/mongo/util/perfctr_collect_test.cpp new file mode 100644 index 00000000000..a5d519f69ad --- /dev/null +++ b/src/mongo/util/perfctr_collect_test.cpp @@ -0,0 +1,416 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kFTDC + +#include "mongo/platform/basic.h" + +#include "mongo/util/perfctr_collect.h" + +#include +#include + +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/log.h" + +namespace mongo { + +namespace { +using StringMap = std::map; + +/** + * Convert BSON document nested up to 3 levels into a map where keys are dot separated paths to + * values. + */ +StringMap toNestedStringMap(BSONObj& obj) { + StringMap map; + + for (const auto& parent : obj) { + if (parent.isABSONObj()) { + std::string parentNamePrefix = std::string(parent.fieldName()) + "."; + + for (const auto& child : parent.Obj()) { + if (child.isABSONObj()) { + std::string childNamePrefix = parentNamePrefix + child.fieldName() + "."; + + for (const auto& grandChild : child.Obj()) { + map[childNamePrefix + grandChild.fieldName()] = grandChild.numberLong(); + } + + } else { + map[parentNamePrefix + child.fieldName()] = child.numberLong(); + } + } + } else { + map[parent.fieldName()] = parent.numberLong(); + } + } + + return map; +} + +#define ASSERT_KEY(_key) ASSERT_TRUE(stringMap.find(_key) != stringMap.end()); +#define ASSERT_NO_KEY(_key) ASSERT_TRUE(stringMap.find(_key) == stringMap.end()); + +#define ASSERT_TIMEBASE ASSERT_KEY("timebase"); +#define ASSERT_NO_TIMEBASE ASSERT_NO_KEY("timebase"); + +#define ASSERT_GROUP_AND_RAW_COUNTER(g, c) \ + ASSERT_KEY(g "." c); \ + ASSERT_NO_KEY(g "." c " Base"); + +#define ASSERT_GROUP_AND_NON_RAW_COUNTER(g, c) \ + ASSERT_KEY(g "." c); \ + ASSERT_KEY(g "." c " Base"); + +#define ASSERT_NESTED_GROUP_AND_RAW_COUNTER(g, p, c) \ + ASSERT_KEY(g "." p "." c); \ + ASSERT_NO_KEY(g "." p "." c " Base"); + +#define ASSERT_NO_NESTED_GROUP_AND_RAW_COUNTER(g, p, c) \ + ASSERT_NO_KEY(g "." p "." c); \ + ASSERT_NO_KEY(g "." p "." c " Base"); + +#define ASSERT_NESTED_GROUP_AND_NON_RAW_COUNTER(g, p, c) \ + ASSERT_KEY(g "." p "." c); \ + ASSERT_KEY(g "." p "." c " Base"); + +#define ASSERT_NO_NESTED_GROUP_AND_NON_RAW_COUNTER(g, p, c) \ + ASSERT_NO_KEY(g "." p "." c); \ + ASSERT_NO_KEY(g "." p "." c " Base"); + +#define COLLECT_COUNTERS_VERBOSE \ + BSONObjBuilder builder; \ + ASSERT_OK(collector->collect(&builder)); \ + auto obj = builder.obj(); \ + log() << "OBJ:" << obj; \ + auto stringMap = toNestedStringMap(obj); \ + for (const auto& kvp : stringMap) { \ + log() << "kvp " << kvp.first << " - " << kvp.second; \ + } + +#define COLLECT_COUNTERS_QUIET \ + BSONObjBuilder builder; \ + ASSERT_OK(collector->collect(&builder)); \ + auto obj = builder.obj(); \ + auto stringMap = toNestedStringMap(obj); + +#define COLLECT_COUNTERS COLLECT_COUNTERS_QUIET + +size_t kDefaultCollectionCount = 2; + +// Simple verification test +TEST(FTDCPerfCollector, TestSingleCounter) { + PerfCounterCollection collection; + // PERF_100NSEC_TIMER + ASSERT_OK(collection.addCountersGroup("cpu", {"\\Processor(0)\\% Idle Time"})); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_OK(swCollector.getStatus()); + auto collector = std::move(swCollector.getValue()); + + for (size_t i = 0; i < kDefaultCollectionCount; i++) { + COLLECT_COUNTERS; + + ASSERT_NO_TIMEBASE; + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\Processor\\% Idle Time"); + } +} + + +// Simple verification test +TEST(FTDCPerfCollector, TestSingleRawCounter) { + PerfCounterCollection collection; + // PERF_COUNTER_RAWCOUNT + ASSERT_OK(collection.addCountersGroup("cpu", {"\\System\\Processes"})); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_OK(swCollector.getStatus()); + auto collector = std::move(swCollector.getValue()); + + for (size_t i = 0; i < kDefaultCollectionCount; i++) { + COLLECT_COUNTERS; + + ASSERT_NO_TIMEBASE; + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\System\\Processes"); + } +} + +// Test negative cases for collection +TEST(FTDCPerfCollector, TestBadCollectionInput) { + PerfCounterCollection collection; + ASSERT_OK(collection.addCountersGroup("cpu", {"\\Processor(0)\\% Idle Time"})); + + // Duplicate group + ASSERT_NOT_OK(collection.addCountersGroup("cpu", {"\\Processor(0)\\% Idle Time"})); + + // Duplicate counter + ASSERT_NOT_OK( + collection.addCountersGroup("cpu2", + { + "\\Processor(0)\\% Idle Time", "\\Processor(0)\\% Idle Time", + })); + + // Duplicate group + ASSERT_NOT_OK( + collection.addCountersGroupedByInstanceName("cpu", {"\\Processor(0)\\% Idle Time"})); + + // Duplicate counter + ASSERT_NOT_OK(collection.addCountersGroupedByInstanceName( + "cpu2", + { + "\\Processor(0)\\% Idle Time", "\\Processor(0)\\% Idle Time", + })); +} + +// Test negative collector input +TEST(FTDCPerfCollector, TestBadCollectorInput) { + // Bad counter name + { + PerfCounterCollection collection; + ASSERT_OK(collection.addCountersGroup("cpu", {"\\Processor(0)\\DOES NOT EXIST"})); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_NOT_OK(swCollector.getStatus()); + } + + // Bad wild card + { + PerfCounterCollection collection; + ASSERT_OK(collection.addCountersGroup("cpu", {"\\Processor(0)\\DOES*"})); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_NOT_OK(swCollector.getStatus()); + } + + // Use addCounterGroup with instance wildcard + { + PerfCounterCollection collection; + ASSERT_OK(collection.addCountersGroup("cpu", {"\\Processor(*)\\\\% Idle Time"})); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_NOT_OK(swCollector.getStatus()); + } + + // Use addCountersGroupedByInstanceName without instance name + { + PerfCounterCollection collection; + ASSERT_OK(collection.addCountersGroupedByInstanceName("cpu", {"\\System\\Processes"})); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_NOT_OK(swCollector.getStatus()); + } +} + +// Test all the different counter types we use in the MongoDB code +TEST(FTDCPerfCollector, TestCounterTypes) { + PerfCounterCollection collection; + ASSERT_OK(collection.addCountersGroup( + "misc", + { + "\\Processor(0)\\% Idle Time", // PERF_100NSEC_TIMER + "\\Processor(0)\\% Processor Time", // PERF_100NSEC_TIMER_INV + "\\System\\Processes", // PERF_COUNTER_RAWCOUNT + "\\System\\System Up Time", // PERF_ELAPSED_TIME + "\\Memory\\Available Bytes", // PERF_COUNTER_LARGE_RAWCOUNT + "\\PhysicalDisk(_Total)\\% Disk Write Time", // PERF_PRECISION_100NS_TIMER + "\\PhysicalDisk(_Total)\\Avg. Disk Bytes/Write", // PERF_AVERAGE_BULK + "\\PhysicalDisk(_Total)\\Avg. Disk Read Queue Length", // PERF_COUNTER_LARGE_QUEUELEN_TYPE + "\\PhysicalDisk(_Total)\\Avg. Disk sec/Write", // PERF_AVERAGE_TIMER + "\\PhysicalDisk(_Total)\\Disk Write Bytes/sec", // PERF_COUNTER_BULK_COUNT + "\\PhysicalDisk(_Total)\\Disk Writes/sec", // PERF_COUNTER_COUNTER + })); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_OK(swCollector.getStatus()); + auto collector = std::move(swCollector.getValue()); + + for (size_t i = 0; i < kDefaultCollectionCount; i++) { + COLLECT_COUNTERS; + + ASSERT_TIMEBASE + ASSERT_GROUP_AND_RAW_COUNTER("misc", "\\Processor\\% Idle Time"); + ASSERT_GROUP_AND_RAW_COUNTER("misc", "\\Processor\\% Processor Time"); + ASSERT_GROUP_AND_RAW_COUNTER("misc", "\\System\\System Up Time"); + ASSERT_GROUP_AND_NON_RAW_COUNTER("misc", "\\PhysicalDisk\\% Disk Write Time"); + ASSERT_GROUP_AND_RAW_COUNTER("misc", "\\PhysicalDisk\\Avg. Disk Bytes/Write"); + ASSERT_GROUP_AND_RAW_COUNTER("misc", "\\PhysicalDisk\\Avg. Disk Read Queue Length"); + ASSERT_GROUP_AND_RAW_COUNTER("misc", "\\PhysicalDisk\\Avg. Disk sec/Write"); + ASSERT_GROUP_AND_RAW_COUNTER("misc", "\\PhysicalDisk\\Disk Write Bytes/sec"); + ASSERT_GROUP_AND_RAW_COUNTER("misc", "\\PhysicalDisk\\Disk Writes/sec"); + + ASSERT_GROUP_AND_RAW_COUNTER("misc", "\\System\\Processes"); + } +} + +// Test multiple counter groups +TEST(FTDCPerfCollector, TestMultipleCounterGroups) { + PerfCounterCollection collection; + ASSERT_OK(collection.addCountersGroup( + "cpu", {"\\Processor(0)\\% Idle Time", "\\Processor(0)\\% Processor Time"})); + ASSERT_OK( + collection.addCountersGroup("sys", {"\\System\\Processes", "\\System\\System Up Time"})); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_OK(swCollector.getStatus()); + auto collector = std::move(swCollector.getValue()); + + for (size_t i = 0; i < kDefaultCollectionCount; i++) { + COLLECT_COUNTERS; + + ASSERT_TIMEBASE + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\Processor\\% Idle Time"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\Processor\\% Processor Time"); + ASSERT_GROUP_AND_RAW_COUNTER("sys", "\\System\\System Up Time"); + + ASSERT_GROUP_AND_RAW_COUNTER("sys", "\\System\\Processes"); + } +} + +// Test multiple nested counter groups +TEST(FTDCPerfCollector, TestMultipleNestedCounterGroups) { + PerfCounterCollection collection; + ASSERT_OK(collection.addCountersGroupedByInstanceName( + "cpu", {"\\Processor(*)\\% Idle Time", "\\Processor(*)\\% Processor Time"})); + ASSERT_OK( + collection.addCountersGroup("sys", {"\\System\\Processes", "\\System\\System Up Time"})); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_OK(swCollector.getStatus()); + auto collector = std::move(swCollector.getValue()); + + for (size_t i = 0; i < kDefaultCollectionCount; i++) { + COLLECT_COUNTERS; + ASSERT_TIMEBASE + + ASSERT_NESTED_GROUP_AND_RAW_COUNTER("cpu", "0", "\\Processor\\% Idle Time"); + ASSERT_NESTED_GROUP_AND_RAW_COUNTER("cpu", "0", "\\Processor\\% Processor Time"); + + ASSERT_NO_NESTED_GROUP_AND_RAW_COUNTER("cpu", "_Total", "\\Processor\\% Idle Time"); + ASSERT_NO_NESTED_GROUP_AND_RAW_COUNTER("cpu", "_Total", "\\Processor\\% Processor Time"); + + ASSERT_GROUP_AND_RAW_COUNTER("sys", "\\System\\System Up Time"); + ASSERT_GROUP_AND_RAW_COUNTER("sys", "\\System\\Processes"); + } +} + +// Test Counters we use in MongoDB +TEST(FTDCPerfCollector, TestLocalCounters) { + PerfCounterCollection collection; + ASSERT_OK(collection.addCountersGroup("cpu", + { + "\\Processor(_Total)\\% Idle Time", + "\\Processor(_Total)\\% Interrupt Time", + "\\Processor(_Total)\\% Privileged Time", + "\\Processor(_Total)\\% Processor Time", + "\\Processor(_Total)\\% User Time", + "\\Processor(_Total)\\Interrupts/sec", + "\\System\\Context Switches/sec", + "\\System\\Processes", + "\\System\\Processor Queue Length", + "\\System\\System Up Time", + "\\System\\Threads", + })); + + // TODO: Should we capture the Heap Counters for the current process? + ASSERT_OK(collection.addCountersGroup("memory", + { + "\\Memory\\Available Bytes", + "\\Memory\\Cache Bytes", + "\\Memory\\Cache Faults/sec", + "\\Memory\\Committed Bytes", + "\\Memory\\Commit Limit", + "\\Memory\\Page Reads/sec", + "\\Memory\\Page Writes/sec", + "\\Memory\\Pages Input/sec", + "\\Memory\\Pages Output/sec", + "\\Memory\\Pool Nonpaged Bytes", + "\\Memory\\Pool Paged Bytes", + "\\Memory\\Pool Paged Resident Bytes", + "\\Memory\\System Cache Resident Bytes", + "\\Memory\\System Code Total Bytes", + })); + + ASSERT_OK(collection.addCountersGroupedByInstanceName( + "disks", + { + "\\PhysicalDisk(*)\\% Disk Read Time", + "\\PhysicalDisk(*)\\% Disk Write Time", + "\\PhysicalDisk(*)\\Avg. Disk Bytes/Read", + "\\PhysicalDisk(*)\\Avg. Disk Bytes/Write", + "\\PhysicalDisk(*)\\Avg. Disk Read Queue Length", + "\\PhysicalDisk(*)\\Avg. Disk Write Queue Length", + "\\PhysicalDisk(*)\\Avg. Disk sec/Read", + "\\PhysicalDisk(*)\\Avg. Disk sec/Write", + "\\PhysicalDisk(*)\\Disk Read Bytes/sec", + "\\PhysicalDisk(*)\\Disk Write Bytes/sec", + "\\PhysicalDisk(*)\\Disk Reads/sec", + "\\PhysicalDisk(*)\\Disk Writes/sec", + "\\PhysicalDisk(*)\\Current Disk Queue Length", + })); + + auto swCollector = PerfCounterCollector::create(std::move(collection)); + ASSERT_OK(swCollector.getStatus()); + auto collector = std::move(swCollector.getValue()); + + for (size_t i = 0; i < kDefaultCollectionCount; i++) { + COLLECT_COUNTERS; + ASSERT_TIMEBASE + + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\Processor\\% Idle Time"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\Processor\\% Interrupt Time"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\Processor\\% Privileged Time"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\Processor\\% Processor Time"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\Processor\\% User Time"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\Processor\\Interrupts/sec"); + + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\System\\Context Switches/sec"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\System\\Processes"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\System\\Processor Queue Length"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\System\\System Up Time"); + ASSERT_GROUP_AND_RAW_COUNTER("cpu", "\\System\\Threads"); + + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Available Bytes"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Cache Bytes"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Cache Faults/sec"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Commit Limit"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Committed Bytes"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Page Reads/sec"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Page Writes/sec"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Pages Input/sec"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Pages Output/sec"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\Pool Paged Resident Bytes"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\System Cache Resident Bytes"); + ASSERT_GROUP_AND_RAW_COUNTER("memory", "\\Memory\\System Code Total Bytes"); + } +} + +} // namespace +} // namespace mongo diff --git a/src/mongo/util/processinfo_linux.cpp b/src/mongo/util/processinfo_linux.cpp index 8addc447ef3..a6a31f0c467 100644 --- a/src/mongo/util/processinfo_linux.cpp +++ b/src/mongo/util/processinfo_linux.cpp @@ -459,7 +459,7 @@ void ProcessInfo::SystemInfo::collectSystemInfo() { osName = distroName; osVersion = distroVersion; memSize = LinuxSysHelper::getSystemMemorySize(); - addrSize = (string(unameData.machine).find("x86_64") != string::npos ? 64 : 32); + addrSize = sizeof(void*) * CHAR_BIT; numCores = cpuCount; pageSize = static_cast(sysconf(_SC_PAGESIZE)); cpuArch = unameData.machine; diff --git a/src/mongo/util/procparser.cpp b/src/mongo/util/procparser.cpp new file mode 100644 index 00000000000..bafc674fc06 --- /dev/null +++ b/src/mongo/util/procparser.cpp @@ -0,0 +1,612 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kFTDC + +#include "mongo/platform/basic.h" + +#include "mongo/util/procparser.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mongo/base/parse_number.h" +#include "mongo/base/status.h" +#include "mongo/base/status_with.h" +#include "mongo/base/string_data.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/util/log.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/scopeguard.h" +#include "mongo/util/text.h" +namespace mongo { + +namespace { + +/** + * Get USER_HZ for the machine. See time(7) for an explanation. + */ +int64_t getTicksPerSecond() { + int64_t ret = sysconf(_SC_CLK_TCK); + return ret; +} + +/** + * Convert USER_HZ to milliseconds. + */ +double convertTicksToMilliSeconds(const int64_t ticks, const int64_t ticksPerSecond) { + return static_cast(ticks) / (static_cast(ticksPerSecond) / 1000.0); +} + +const size_t kFileBufferSize = 16384; +const size_t kFileReadRetryCount = 5; + +constexpr auto kSysBlockDeviceDirectoryName = "device"; + +/** + * Read a file from disk as a string with a null-terminating byte using the POSIX file api. + * + * This function is designed to get all the data it needs from small /proc files in a single read. + * The /proc/stat and /proc/diskstats files can vary in size, but 16kb will cover most cases. + * + * Finally, we return errors instead of throwing to ensure that FTDC can return partial information + * on failure instead of no information. Some container filesystems may overlay /proc so we may not + * be reading directly from the kernel. + */ +StatusWith readFileAsString(StringData filename) { + int fd = open(filename.toString().c_str(), 0); + if (fd == -1) { + int err = errno; + return Status(ErrorCodes::FileOpenFailed, + str::stream() << "Failed to open file " << filename + << " with error: " << errnoWithDescription(err)); + } + auto scopedGuard = MakeGuard([fd] { close(fd); }); + + BufBuilder builder(kFileBufferSize); + std::array buf; + + ssize_t size_read = 0; + + // Read until the end as needed + do { + // Retry if interrupted + size_t retry = 0; + + do { + size_read = read(fd, buf.data(), kFileBufferSize); + + if (size_read == -1) { + int err = errno; + + // Retry if we hit EGAIN or EINTR a few times before giving up + if (retry < kFileReadRetryCount && (err == EAGAIN || err == EINTR)) { + ++retry; + continue; + } + + return Status(ErrorCodes::FileStreamFailed, + str::stream() << "Failed to read file " << filename + << " with error: " << errnoWithDescription(err)); + } + + break; + } while (true); + + if (size_read != 0) { + builder.appendBuf(buf.data(), size_read); + } + } while (size_read != 0); + + // Null terminate the buffer since we are about to convert it to a string + builder.appendChar(0); + + return std::string(builder.buf(), builder.len()); +} + + +const char* const kAdditionCpuFields[] = {"user_ms", + "nice_ms", + "system_ms", + "idle_ms", + "iowait_ms", + "irq_ms", + "softirq_ms", + "steal_ms", + "guest_ms", + "guest_nice_ms"}; +const size_t kAdditionCpuFieldCount = std::extent::value; + +const char* const kDiskFields[] = { + "reads", + "reads_merged", + "read_sectors", + "read_time_ms", + "writes", + "writes_merged", + "write_sectors", + "write_time_ms", + "io_in_progress", + "io_time_ms", + "io_queued_ms", +}; + +const size_t kDiskFieldCount = std::extent::value; + +} // namespace + +namespace procparser { +// Here is an example of the type of string it supports. +// Note: intr output has been trimmed +// +// The cpu field maps up to 10 individual fields depending on the kernel version. For other views, +// this code assumes there is only a single value. +// +// For more information, see: +// Documentation/filesystems/proc.txt in the Linux kernel +// proc(5) man page +// +// > cat /proc/stat +// cpu 41801 9179 32206 831134223 34279 0 947 0 0 0 +// cpu0 2977 450 2475 69253074 1959 0 116 0 0 0 +// cpu1 6213 4261 9400 69177349 845 0 539 0 0 0 +// cpu2 1949 831 3699 69261035 645 0 0 0 0 0 +// cpu3 2222 644 3283 69264801 783 0 0 0 0 0 +// cpu4 16576 607 4757 69232589 8195 0 291 0 0 0 +// cpu5 3742 391 4571 69257332 2322 0 0 0 0 0 +// cpu6 2173 376 743 69284308 400 0 0 0 0 0 +// cpu7 1232 375 704 69285753 218 0 0 0 0 0 +// cpu8 960 127 576 69262851 18107 0 0 0 0 0 +// cpu9 1755 227 744 69283938 362 0 0 0 0 0 +// cpu10 1380 641 678 69285193 219 0 0 0 0 0 +// cpu11 618 244 572 69285995 218 0 0 0 0 0 +// intr 54084718 135 2 .... +// ctxt 190305514 +// btime 1463584038 +// processes 47438 +// procs_running 1 +// procs_blocked 0 +// softirq 102690251 8 26697410 115481 23345078 816026 0 2296 26068778 0 25645174 +// +Status parseProcStat(const std::vector& keys, + StringData data, + int64_t ticksPerSecond, + BSONObjBuilder* builder) { + bool foundKeys = false; + + using string_split_iterator = boost::split_iterator; + + // Split the file by lines. + // token_compress_on means the iterator skips over consecutive '\n'. This should not be a + // problem in normal /proc/stat output. + for (string_split_iterator lineIt = string_split_iterator( + data.begin(), + data.end(), + boost::token_finder([](char c) { return c == '\n'; }, boost::token_compress_on)); + lineIt != string_split_iterator(); + ++lineIt) { + StringData line((*lineIt).begin(), (*lineIt).end()); + + // Split the line by spaces since that is the only delimiter for stat files. + // token_compress_on means the iterator skips over consecutive ' '. This is needed for the + // first line which is "cpu ". + string_split_iterator partIt = string_split_iterator( + line.begin(), + line.end(), + boost::token_finder([](char c) { return c == ' '; }, boost::token_compress_on)); + + // Skip processing this line if we do not have a key. + if (partIt == string_split_iterator()) { + continue; + } + + StringData key((*partIt).begin(), (*partIt).end()); + + ++partIt; + + // Skip processing this line if we only have a key, and no number. + if (partIt == string_split_iterator()) { + continue; + } + + // Check if the key is in the list. /proc/stat will have extra keys, and + // may not have the keys we want. + if (keys.empty() || std::find(keys.begin(), keys.end(), key) != keys.end()) { + foundKeys = true; + + if (key == "cpu") { + // Cpu is 10 fields, we need to chew through all of them. + // Some kernels we support lack the last field or two: guest and/or guest_nice. + for (size_t index = 0; + partIt != string_split_iterator() && index < kAdditionCpuFieldCount; + ++partIt, ++index) { + StringData stringValue((*partIt).begin(), (*partIt).end() - (*partIt).begin()); + + uint64_t value; + + if (!parseNumberFromString(stringValue, &value).isOK()) { + value = 0; + } + + builder->appendNumber(kAdditionCpuFields[index], + convertTicksToMilliSeconds(value, ticksPerSecond)); + } + } else { + StringData stringValue((*partIt).begin(), (*partIt).end() - (*partIt).begin()); + + uint64_t value; + + if (!parseNumberFromString(stringValue, &value).isOK()) { + value = 0; + } + + builder->appendNumber(key, static_cast(value)); + } + } + } + + return foundKeys ? Status::OK() + : Status(ErrorCodes::NoSuchKey, "Failed to find any keys in stat string"); +} + +Status parseProcStatFile(StringData filename, + const std::vector& keys, + BSONObjBuilder* builder) { + auto swString = readFileAsString(filename); + if (!swString.isOK()) { + return swString.getStatus(); + } + + return parseProcStat(keys, swString.getValue(), getTicksPerSecond(), builder); +} + +// Here is an example of the type of string it supports: +// Note: output has been trimmed +// +// For more information, see: +// Documentation/filesystems/proc.txt in the Linux kernel +// proc(5) man page +// +// > cat /proc/meminfo +// MemTotal: 12294392 kB +// MemFree: 3652612 kB +// MemAvailable: 11831704 kB +// Buffers: 568536 kB +// Cached: 6421520 kB +// SwapCached: 0 kB +// HugePages_Total: 0 +// +// Note: HugePages_* do not end in kB, it is not a typo +// +Status parseProcMemInfo(const std::vector& keys, + StringData data, + BSONObjBuilder* builder) { + bool foundKeys = false; + + using string_split_iterator = boost::split_iterator; + + // Split the file by lines. + // token_compress_on means the iterator skips over consecutive '\n'. This should not be a + // problem in normal /proc/memInfo output. + for (string_split_iterator lineIt = string_split_iterator( + data.begin(), + data.end(), + boost::token_finder([](char c) { return c == '\n'; }, boost::token_compress_on)); + lineIt != string_split_iterator(); + ++lineIt) { + StringData line((*lineIt).begin(), (*lineIt).end()); + + // Split the line by spaces and colons since these are the delimiters for meminfo files. + // token_compress_on means the iterator skips over consecutive ' '. This is needed for + // every line. + string_split_iterator partIt = string_split_iterator(line.begin(), + line.end(), + boost::token_finder([](char c) { + return c == ' ' || c == ':'; + }, boost::token_compress_on)); + + // Skip processing this line if we do not have a key. + if (partIt == string_split_iterator()) { + continue; + } + + StringData key((*partIt).begin(), (*partIt).end()); + + ++partIt; + + // Skip processing this line if we only have a key, and no number. + if (partIt == string_split_iterator()) { + continue; + } + + // Check if the key is in the list. /proc/meminfo will have extra keys, and may not have the + // keys we want. + if (keys.empty() || std::find(keys.begin(), keys.end(), key) != keys.end()) { + foundKeys = true; + + StringData stringValue((*partIt).begin(), (*partIt).end()); + + uint64_t value; + + if (!parseNumberFromString(stringValue, &value).isOK()) { + value = 0; + } + + // Check if the line ends in "kB" + ++partIt; + + // If there is one last token, check if it is actually "kB" + if (partIt != string_split_iterator()) { + StringData kb_token((*partIt).begin(), (*partIt).end()); + auto keyWithSuffix = key.toString(); + + if (kb_token == "kB") { + keyWithSuffix.append("_kb"); + } + + builder->appendNumber(keyWithSuffix, static_cast(value)); + } else { + builder->appendNumber(key, static_cast(value)); + } + } + } + + return foundKeys ? Status::OK() + : Status(ErrorCodes::NoSuchKey, "Failed to find any keys in meminfo string"); +} + +Status parseProcMemInfoFile(StringData filename, + const std::vector& keys, + BSONObjBuilder* builder) { + auto swString = readFileAsString(filename); + if (!swString.isOK()) { + return swString.getStatus(); + } + + return parseProcMemInfo(keys, swString.getValue(), builder); +} + +// Here is an example of the type of string it supports: +// +// For more information, see: +// Documentation/iostats.txt in the Linux kernel +// proc(5) man page +// +// > cat /proc/diskstats +// 8 0 sda 120611 33630 6297628 96550 349797 167398 11311562 2453603 0 117514 2554160 +// 8 1 sda1 138 37 8642 315 3 0 18 14 0 292 329 +// 8 2 sda2 120409 33593 6285754 96158 329029 167398 11311544 2450573 0 115611 2550739 +// 8 16 sdb 12707 3876 1525418 57507 997 3561 297576 97976 0 37870 155619 +// 8 17 sdb1 12601 3876 1521090 57424 992 3561 297576 97912 0 37738 155468 +// 11 0 sr0 0 0 0 0 0 0 0 0 0 0 0 +// 253 0 dm-0 154910 0 6279522 177681 506513 0 11311544 5674418 0 117752 5852275 +// 253 1 dm-1 109 0 4584 226 0 0 0 0 0 172 226 +// +Status parseProcDiskStats(const std::vector& disks, + StringData data, + BSONObjBuilder* builder) { + bool foundKeys = false; + std::vector stats; + stats.reserve(kDiskFieldCount); + + using string_split_iterator = boost::split_iterator; + + // Split the file by lines. + // token_compress_on means the iterator skips over consecutive '\n'. This should not be a + // problem in normal /proc/diskstats output. + for (string_split_iterator lineIt = string_split_iterator( + data.begin(), + data.end(), + boost::token_finder([](char c) { return c == '\n'; }, boost::token_compress_on)); + lineIt != string_split_iterator(); + ++lineIt) { + StringData line((*lineIt).begin(), (*lineIt).end()); + + // Skip leading whitespace so that the split_iterator starts on non-whitespace otherwise we + // get an empty first token. Device major numbers (the first number on each line) are right + // aligned to 4 spaces and start from + // single digits. + auto beginNonWhitespace = + std::find_if_not(line.begin(), line.end(), [](char c) { return c == ' '; }); + + // Split the line by spaces since that is the only delimiter for diskstats files. + // token_compress_on means the iterator skips over consecutive ' '. + string_split_iterator partIt = string_split_iterator( + beginNonWhitespace, + line.end(), + boost::token_finder([](char c) { return c == ' '; }, boost::token_compress_on)); + + // Skip processing this line if the line is blank + if (partIt == string_split_iterator()) { + continue; + } + + ++partIt; + + // Skip processing this line if we only have a device major number. + if (partIt == string_split_iterator()) { + continue; + } + + ++partIt; + + // Skip processing this line if we only have a device major minor. + if (partIt == string_split_iterator()) { + continue; + } + + StringData disk((*partIt).begin(), (*partIt).end()); + + // Skip processing this line if we only have a block device name. + if (partIt == string_split_iterator()) { + continue; + } + + ++partIt; + + // Check if the disk is in the list. /proc/diskstats will have extra disks, and may not have + // the disk we want. + if (disks.empty() || std::find(disks.begin(), disks.end(), disk) != disks.end()) { + foundKeys = true; + + stats.clear(); + + // Only generate a disk document if the disk has some activity. For instance, there + // could be a CD-ROM drive that is not used. + bool hasSomeNonZeroStats = false; + + for (size_t index = 0; partIt != string_split_iterator() && index < kDiskFieldCount; + ++partIt, ++index) { + StringData stringValue((*partIt).begin(), (*partIt).end()); + + uint64_t value; + + if (!parseNumberFromString(stringValue, &value).isOK()) { + value = 0; + } + + if (value != 0) { + hasSomeNonZeroStats = true; + } + + stats.push_back(value); + } + + if (hasSomeNonZeroStats) { + // Start a new document with disk as the name. + BSONObjBuilder sub(builder->subobjStart(disk)); + + for (size_t index = 0; index < stats.size() && index < kDiskFieldCount; ++index) { + sub.appendNumber(kDiskFields[index], static_cast(stats[index])); + } + + sub.doneFast(); + } + } + } + + return foundKeys ? Status::OK() + : Status(ErrorCodes::NoSuchKey, "Failed to find any keys in diskstats string"); +} + +Status parseProcDiskStatsFile(StringData filename, + const std::vector& disks, + BSONObjBuilder* builder) { + auto swString = readFileAsString(filename); + if (!swString.isOK()) { + return swString.getStatus(); + } + + return parseProcDiskStats(disks, swString.getValue(), builder); +} + +namespace { + +/** + * Is this a disk that is interesting to us? We only want physical disks, not multiple disk devices, + * LVM2 devices, partitions, or RAM disks. + * + * A physical disk has a symlink to a directory at /sys/block//device. + * + * Note: returns false upon any errors such as access denied. + */ +bool isInterestingDisk(const boost::filesystem::path& path) { + boost::filesystem::path blockDevicePath(path); + blockDevicePath /= kSysBlockDeviceDirectoryName; + + boost::system::error_code ec; + auto statusSysBlock = boost::filesystem::status(blockDevicePath, ec); + if (!boost::filesystem::exists(statusSysBlock)) { + return false; + } + + if (ec) { + warning() << "Error checking directory '" << blockDevicePath.generic_string() + << "': " << ec.message(); + return false; + } + + if (!boost::filesystem::is_directory(statusSysBlock)) { + return false; + } + + return true; +} + +} // namespace + +std::vector findPhysicalDisks(StringData sysBlockPath) { + boost::system::error_code ec; + auto sysBlockPathStr = sysBlockPath.toString(); + + auto statusSysBlock = boost::filesystem::status(sysBlockPathStr, ec); + if (ec) { + warning() << "Error checking directory '" << sysBlockPathStr << "': " << ec.message(); + return {}; + } + + if (!(boost::filesystem::exists(statusSysBlock) && + boost::filesystem::is_directory(statusSysBlock))) { + warning() << "Could not find directory '" << sysBlockPathStr << "': " << ec.message(); + return {}; + } + + std::vector files; + + // Iterate through directories in /sys/block. The directories in this directory can be physical + // block devices (like SSD or HDD) or virtual devices like the LVM2 device mapper or a multiple + // disk device. It does not contain disk partitions. + boost::filesystem::directory_iterator di(sysBlockPathStr, ec); + if (ec) { + warning() << "Error getting directory iterator '" << sysBlockPathStr + << "': " << ec.message(); + return {}; + } + + for (; di != boost::filesystem::directory_iterator(); di++) { + auto path = (*di).path(); + + if (isInterestingDisk(path)) { + files.push_back(path.filename().generic_string()); + } + } + + return files; +} + +} // namespace procparser +} // namespace mongo diff --git a/src/mongo/util/procparser.h b/src/mongo/util/procparser.h new file mode 100644 index 00000000000..2f9e97e343c --- /dev/null +++ b/src/mongo/util/procparser.h @@ -0,0 +1,120 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include +#include +#include + +#include "mongo/base/status.h" +#include "mongo/base/string_data.h" + +namespace mongo { + +class BSONObjBuilder; + +namespace procparser { + +/** + * Reads a string matching /proc/stat format, and writes the values of the specified keys into + * builder. If fields are not in the "data" parameter, they are omitted. Converts fields + * from USER_HZ to milliseconds, and names fields with a "_ms" suffix. If the string is empty, + * corrupt, or missing fields, the builder will simply be missing fields. + * + * keys - sorted vector of field names to include in the output, "cpu" will include the 11 fields + * that make up cpu. If keys is empty, all keys are outputed. + * data - string to parsee + * ticksPerSecond - USER_HZ value + * builder - BSON output + */ +Status parseProcStat(const std::vector& keys, + StringData data, + int64_t ticksPerSecond, + BSONObjBuilder* builder); + +/** +* Read from file, and write the specified list of keys into builder. +* +* See parseProcStat. +* +* Returns Status errors on file reading issues. +*/ +Status parseProcStatFile(StringData filename, + const std::vector& keys, + BSONObjBuilder* builder); + +/** + * Read a string matching /proc/meminfo format, and write the specified list of keys in builder. + * + * keys - list of keys to output in BSON. If keys is empty, all keys are outputed. + * data - string to parsee + * builder - BSON output + */ +Status parseProcMemInfo(const std::vector& keys, + StringData data, + BSONObjBuilder* builder); + +/** + * Read from file, and write the specified list of keys in builder. + */ +Status parseProcMemInfoFile(StringData filename, + const std::vector& keys, + BSONObjBuilder* builder); + +/** + * Read a string matching /proc/diskstats format, and write the specified list of disks in builder. + * + * disks - vector of block devices to include in output. For each disk selected, 11 fields are + * output in a nested document. There is no error if the disk is not found in the data. Also + * a disk is excluded if it has no activity since startup (i.e. an idle CD-ROM drive). If + * disks is empty, all non-zero block devices are outputed (this will include partitions, + * etc). + * data - string to parsee + * builder - BSON output + */ +Status parseProcDiskStats(const std::vector& disks, + StringData data, + BSONObjBuilder* builder); + +/** + * Read from file, and write the specified list of disks in builder. + */ +Status parseProcDiskStatsFile(StringData filename, + const std::vector& disks, + BSONObjBuilder* builder); + +/** + * Get a vector of disks to monitor by enumerating the specified directory. + * + * If the directory does not exist, or otherwise permission is denied, returns an empty vector. + */ +std::vector findPhysicalDisks(StringData directory); + +} // namespace procparser +} // namespace mongo diff --git a/src/mongo/util/procparser_test.cpp b/src/mongo/util/procparser_test.cpp new file mode 100644 index 00000000000..4cbc02df68a --- /dev/null +++ b/src/mongo/util/procparser_test.cpp @@ -0,0 +1,526 @@ +/** + * Copyright (C) 2016 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kFTDC + +#include "mongo/platform/basic.h" + +#include "mongo/util/procparser.h" + +#include +#include + +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/unittest/unittest.h" +#include "mongo/util/log.h" + +namespace mongo { + +namespace { +using StringMap = std::map; + +StringMap toStringMap(BSONObj& obj) { + StringMap map; + + for (const auto& e : obj) { + map[e.fieldName()] = e.numberLong(); + } + + return map; +} + +StringMap toNestedStringMap(BSONObj& obj) { + StringMap map; + + for (const auto& e : obj) { + if (e.isABSONObj()) { + std::string prefix = std::string(e.fieldName()) + "."; + + for (const auto& child : e.Obj()) { + map[prefix + child.fieldName()] = child.numberLong(); + } + } + } + + return map; +} + +#define ASSERT_KEY(_key) ASSERT_TRUE(stringMap.find(_key) != stringMap.end()); +#define ASSERT_NO_KEY(_key) ASSERT_TRUE(stringMap.find(_key) == stringMap.end()); +#define ASSERT_KEY_AND_VALUE(_key, _value) ASSERT_EQUALS(stringMap.at(_key), _value); + +#define ASSERT_PARSE_STAT(_keys, _x) \ + BSONObjBuilder builder; \ + ASSERT_OK(procparser::parseProcStat(_keys, _x, 1000, &builder)); \ + auto obj = builder.obj(); \ + auto stringMap = toStringMap(obj); +#define ASSERT_PARSE_MEMINFO(_keys, _x) \ + BSONObjBuilder builder; \ + ASSERT_OK(procparser::parseProcMemInfo(_keys, _x, &builder)); \ + auto obj = builder.obj(); \ + auto stringMap = toStringMap(obj); +#define ASSERT_PARSE_DISKSTATS(_disks, _x) \ + BSONObjBuilder builder; \ + ASSERT_OK(procparser::parseProcDiskStats(_disks, _x, &builder)); \ + auto obj = builder.obj(); \ + auto stringMap = toNestedStringMap(obj); + +TEST(FTDCProcStat, TestStat) { + std::vector keys{"cpu", "ctxt", "processes"}; + + // Normal case + { + ASSERT_PARSE_STAT( + keys, + "cpu 41801 9179 32206 831134223 34279 0 947 0 0 0\n" + "cpu0 2977 450 2475 69253074 1959 0 116 0 0 0\n" + "cpu1 6213 4261 9400 69177349 845 0 539 0 0 0\n" + "cpu2 1949 831 3699 69261035 645 0 0 0 0 0\n" + "cpu3 2222 644 3283 69264801 783 0 0 0 0 0\n" + "cpu4 16576 607 4757 69232589 8195 0 291 0 0 0\n" + "cpu5 3742 391 4571 69257332 2322 0 0 0 0 0\n" + "cpu6 2173 376 743 69284308 400 0 0 0 0 0\n" + "cpu7 1232 375 704 69285753 218 0 0 0 0 0\n" + "cpu8 960 127 576 69262851 18107 0 0 0 0 0\n" + "cpu9 1755 227 744 69283938 362 0 0 0 0 0\n" + "cpu10 1380 641 678 69285193 219 0 0 0 0 0\n" + "cpu11 618 244 572 69285995 218 0 0 0 0 0\n" + "intr 54084718 135 2 ....\n" + "ctxt 190305514\n" + "btime 1463584038\n" + "processes 47438\n" + "procs_running 1\n" + "procs_blocked 0\n" + "softirq 102690251 8 26697410 115481 23345078 816026 0 2296 26068778 0 25645174\n"); + ASSERT_KEY_AND_VALUE("user_ms", 41801UL); + ASSERT_KEY_AND_VALUE("nice_ms", 9179UL); + ASSERT_KEY_AND_VALUE("system_ms", 32206UL); + ASSERT_KEY_AND_VALUE("idle_ms", 831134223UL); + ASSERT_KEY_AND_VALUE("iowait_ms", 34279UL); + ASSERT_KEY_AND_VALUE("irq_ms", 0UL); + ASSERT_KEY_AND_VALUE("softirq_ms", 947UL); + ASSERT_KEY_AND_VALUE("steal_ms", 0UL); + ASSERT_KEY_AND_VALUE("guest_ms", 0UL); + ASSERT_KEY_AND_VALUE("guest_nice_ms", 0UL); + ASSERT_KEY_AND_VALUE("ctxt", 190305514UL); + ASSERT_KEY_AND_VALUE("processes", 47438UL); + } + + // Missing fields in cpu and others + { + ASSERT_PARSE_STAT(keys, + "cpu 41801 9179 32206\n" + "ctxt 190305514\n"); + ASSERT_KEY_AND_VALUE("user_ms", 41801UL); + ASSERT_KEY_AND_VALUE("nice_ms", 9179UL); + ASSERT_KEY_AND_VALUE("system_ms", 32206UL); + ASSERT_NO_KEY("idle_ms"); + ASSERT_KEY_AND_VALUE("ctxt", 190305514UL); + ASSERT_NO_KEY("processes"); + } + + // Missing fields in cpu and others + { + ASSERT_PARSE_STAT(keys, + "cpu 41801\n" + "ctxt 190305514\n"); + ASSERT_KEY_AND_VALUE("user_ms", 41801UL); + ASSERT_NO_KEY("nice_ms"); + ASSERT_KEY_AND_VALUE("ctxt", 190305514UL); + ASSERT_NO_KEY("processes"); + } + + // Missing fields in cpu + { + ASSERT_PARSE_STAT(keys, + "cpu \n" + "ctxt 190305514\n"); + ASSERT_KEY_AND_VALUE("ctxt", 190305514UL); + ASSERT_NO_KEY("processes"); + } + + // Single string with only cpu and numbers + { + ASSERT_PARSE_STAT(keys, "cpu 41801 9179 32206"); + ASSERT_KEY_AND_VALUE("user_ms", 41801UL); + ASSERT_KEY_AND_VALUE("nice_ms", 9179UL); + ASSERT_KEY_AND_VALUE("system_ms", 32206UL); + ASSERT_NO_KEY("idle_ms"); + } + + // Single string with only cpu + { + BSONObjBuilder builder; + ASSERT_NOT_OK(procparser::parseProcStat(keys, "cpu", 1000, &builder)); + } + + // Single string with only cpu and a number, and empty ctxt + { + ASSERT_PARSE_STAT(keys, + "cpu 123\n" + "ctxt"); + ASSERT_KEY_AND_VALUE("user_ms", 123UL); + } + + // Empty String + { + BSONObjBuilder builder; + ASSERT_NOT_OK(procparser::parseProcStat(keys, "", 1000, &builder)); + } +} + +// Test we can parse the /proc/stat on this machine. Also assert we have the expected fields +// This tests is designed to exercise our parsing code on various Linuxes and fail +// Normally when run in the FTDC loop we return a non-fatal error so we may not notice the failure +// otherwise. +TEST(FTDCProcStat, TestLocalStat) { + std::vector keys{ + "btime", "cpu", "ctxt", "processes", "procs_blocked", "procs_running", + }; + + BSONObjBuilder builder; + + ASSERT_OK(procparser::parseProcStatFile("/proc/stat", keys, &builder)); + + BSONObj obj = builder.obj(); + auto stringMap = toStringMap(obj); + log() << "OBJ:" << obj; + ASSERT_KEY("user_ms"); + ASSERT_KEY("nice_ms"); + ASSERT_KEY("idle_ms"); + ASSERT_KEY("system_ms"); + ASSERT_KEY("iowait_ms"); + ASSERT_KEY("irq_ms"); + ASSERT_KEY("softirq_ms"); + ASSERT_KEY("steal_ms"); + // Needs 2.6.24 - ASSERT_KEY("guest_ms"); + // Needs 2.6.33 - ASSERT_KEY("guest_nice_ms"); + ASSERT_KEY("ctxt"); + ASSERT_KEY("btime"); + ASSERT_KEY("processes"); + ASSERT_KEY("procs_running"); + ASSERT_KEY("procs_blocked"); +} + +TEST(FTDCProcStat, TestLocalNonExistentStat) { + std::vector keys{ + "btime", "cpu", "ctxt", "processes", "procs_blocked", "procs_running", + }; + BSONObjBuilder builder; + + ASSERT_NOT_OK(procparser::parseProcStatFile("/proc/does_not_exist", keys, &builder)); +} + +TEST(FTDCProcMemInfo, TestMemInfo) { + std::vector keys{"Key1", "Key2", "Key3"}; + + // Normal case + { + ASSERT_PARSE_MEMINFO(keys, "Key1: 123 kB\nKey2: 456 kB"); + ASSERT_KEY_AND_VALUE("Key1_kb", 123UL); + ASSERT_KEY_AND_VALUE("Key2_kb", 456UL); + } + + // Space in key name + { + ASSERT_PARSE_MEMINFO(keys, "Key1: 123 kB\nKey 2: 456 kB"); + ASSERT_KEY_AND_VALUE("Key1_kb", 123UL); + ASSERT_NO_KEY("Key2_kb"); + } + + // No newline + { + ASSERT_PARSE_MEMINFO(keys, "Key1: 123 kB Key2: 456 kB"); + ASSERT_KEY_AND_VALUE("Key1_kb", 123UL); + ASSERT_NO_KEY("Key2_kb"); + } + + // Missing colon on first key + { + ASSERT_PARSE_MEMINFO(keys, "Key1 123 kB\nKey2: 456 kB"); + ASSERT_KEY_AND_VALUE("Key1_kb", 123UL); + ASSERT_KEY_AND_VALUE("Key2_kb", 456UL); + } + + // One token missing kB, HugePages is not size in kB + { + ASSERT_PARSE_MEMINFO(keys, "Key1: 123 kB\nKey2: 456\nKey3: 789 kB\nKey4: 789 kB"); + ASSERT_KEY_AND_VALUE("Key1_kb", 123UL); + ASSERT_KEY_AND_VALUE("Key2", 456UL); + ASSERT_KEY_AND_VALUE("Key3_kb", 789UL); + ASSERT_NO_KEY("Key4_kb"); + } + + // Empty string + { + BSONObjBuilder builder; + ASSERT_NOT_OK(procparser::parseProcMemInfo(keys, "", &builder)); + } +} + +// Test we can parse the /proc/meminfo on this machine. Also assert we have the expected fields +// This tests is designed to exercise our parsing code on various Linuxes and fail +// Normally when run in the FTDC loop we return a non-fatal error so we may not notice the failure +// otherwise. +TEST(FTDCProcMemInfo, TestLocalMemInfo) { + std::vector keys{ + "Active", + "Active(anon)", + "Active(file)", + "AnonHugePages", + "AnonPages", + "Bounce", + "Buffers", + "Cached", + "CmaFree", + "CmaTotal", + "CommitLimit", + "Committed_AS", + "Dirty", + "HardwareCorrupted", + "Inactive", + "Inactive(anon)", + "Inactive(file)", + "KernelStack", + "Mapped", + "MemAvailable", + "MemFree", + "MemTotal", + "Mlocked", + "NFS_Unstable", + "PageTables", + "SReclaimable", + "SUnreclaim", + "Shmem", + "Slab", + "SwapCached", + "SwapFree", + "SwapTotal", + "Unevictable", + "VmallocChunk", + "VmallocTotal", + "VmallocUsed", + "Writeback", + "WritebackTmp", + }; + + BSONObjBuilder builder; + + ASSERT_OK(procparser::parseProcMemInfoFile("/proc/meminfo", keys, &builder)); + + BSONObj obj = builder.obj(); + auto stringMap = toStringMap(obj); + log() << "OBJ:" << obj; + ASSERT_KEY("MemTotal_kb"); + ASSERT_KEY("MemFree_kb"); + // Needs in 3.15+ - ASSERT_KEY("MemAvailable_kb"); + ASSERT_KEY("Buffers_kb"); + ASSERT_KEY("Cached_kb"); + ASSERT_KEY("SwapCached_kb"); + ASSERT_KEY("Active_kb"); + ASSERT_KEY("Inactive_kb"); + // Needs 2.6.28+ - ASSERT_KEY("Active(anon)_kb"); + // Needs 2.6.28+ - ASSERT_KEY("Inactive(anon)_kb"); + // Needs 2.6.28+ - ASSERT_KEY("Active(file)_kb"); + // Needs 2.6.28+ - ASSERT_KEY("Inactive(file)_kb"); + // Needs 2.6.28+ - ASSERT_KEY("Unevictable_kb"); + // Needs 2.6.28+ - ASSERT_KEY("Mlocked_kb"); + ASSERT_KEY("SwapTotal_kb"); + ASSERT_KEY("SwapFree_kb"); + ASSERT_KEY("Dirty_kb"); + ASSERT_KEY("Writeback_kb"); + ASSERT_KEY("AnonPages_kb"); + ASSERT_KEY("Mapped_kb"); + // Needs 2.6.32+ - ASSERT_KEY("Shmem_kb"); + ASSERT_KEY("Slab_kb"); + // Needs 2.6.19+ - ASSERT_KEY("SReclaimable_kb"); + // Needs 2.6.19+ - ASSERT_KEY("SUnreclaim_kb"); + // Needs 2.6.32+ - ASSERT_KEY("KernelStack_kb"); + ASSERT_KEY("PageTables_kb"); + ASSERT_KEY("NFS_Unstable_kb"); + ASSERT_KEY("Bounce_kb"); + // Needs 2.6.19+ - ASSERT_KEY("WritebackTmp_kb"); + ASSERT_KEY("CommitLimit_kb"); + ASSERT_KEY("Committed_AS_kb"); + ASSERT_KEY("VmallocTotal_kb"); + ASSERT_KEY("VmallocUsed_kb"); + ASSERT_KEY("VmallocChunk_kb"); + // Needs CONFIG_MEMORY_FAILURE & 2.6.32+ ASSERT_KEY("HardwareCorrupted_kb"); + // Needs CONFIG_TRANSPARENT_HUGEPAGE - ASSERT_KEY("AnonHugePages_kb"); + // Needs CONFIG_CMA & 3.19+ - ASSERT_KEY("CmaTotal_kb"); + // Needs CONFIG_CMA & 3.19+ - ASSERT_KEY("CmaFree_kb"); +} + + +TEST(FTDCProcMemInfo, TestLocalNonExistentMemInfo) { + std::vector keys{}; + BSONObjBuilder builder; + + ASSERT_NOT_OK(procparser::parseProcMemInfoFile("/proc/does_not_exist", keys, &builder)); +} + + +TEST(FTDCProcDiskStats, TestDiskStats) { + std::vector disks{"dm-1", "sda", "sdb"}; + + // Normal case including high device major numbers. + { + ASSERT_PARSE_DISKSTATS( + disks, + " 8 0 sda 120611 33630 6297628 96550 349797 167398 11311562 2453603 0 117514 " + "2554160\n" + " 8 1 sda1 138 37 8642 315 3 0 18 14 0 292 329\n" + " 8 2 sda2 120409 33593 6285754 96158 329029 167398 11311544 2450573 0 115611 " + "2550739\n" + " 8 16 sdb 12707 3876 1525418 57507 997 3561 297576 97976 0 37870 155619\n" + " 8 17 sdb1 12601 3876 1521090 57424 992 3561 297576 97912 0 37738 155468\n" + " 11 0 sr0 0 0 0 0 0 0 0 0 0 0 0\n" + "2253 0 dm-0 154910 0 6279522 177681 506513 0 11311544 5674418 0 117752 5852275\n" + "2253 1 dm-1 109 0 4584 226 0 0 0 0 0 172 226"); + ASSERT_KEY_AND_VALUE("sda.reads", 120611UL); + ASSERT_KEY_AND_VALUE("sda.writes", 349797UL); + ASSERT_KEY_AND_VALUE("sda.io_queued_ms", 2554160UL); + ASSERT_KEY_AND_VALUE("sdb.reads", 12707UL); + ASSERT_KEY_AND_VALUE("sdb.writes", 997UL); + ASSERT_KEY_AND_VALUE("sdb.io_queued_ms", 155619UL); + ASSERT_KEY_AND_VALUE("dm-1.reads", 109UL); + ASSERT_KEY_AND_VALUE("dm-1.writes", 0UL); + ASSERT_KEY_AND_VALUE("dm-1.io_queued_ms", 226UL); + } + + // Exclude a block device without any activity + { + ASSERT_PARSE_DISKSTATS( + disks, + " 8 0 sda 120611 33630 6297628 96550 349797 167398 11311562 2453603 0 117514 " + "2554160\n" + " 8 1 sda1 138 37 8642 315 3 0 18 14 0 292 329\n" + " 8 2 sda2 120409 33593 6285754 96158 329029 167398 11311544 2450573 0 115611 " + "2550739\n" + " 8 16 sdb 0 0 0 0 0 0 0 0 0 0 0\n" + " 8 17 sdb1 12601 3876 1521090 57424 992 3561 297576 97912 0 37738 155468\n" + " 11 0 sr0 0 0 0 0 0 0 0 0 0 0 0\n" + "2253 0 dm-0 154910 0 6279522 177681 506513 0 11311544 5674418 0 117752 5852275\n" + "2253 1 dm-1 109 0 4584 226 0 0 0 0 0 172 226"); + ASSERT_KEY_AND_VALUE("sda.reads", 120611UL); + ASSERT_KEY_AND_VALUE("sda.writes", 349797UL); + ASSERT_KEY_AND_VALUE("sda.io_queued_ms", 2554160UL); + ASSERT_NO_KEY("sdb.reads"); + ASSERT_NO_KEY("sdb.writes"); + ASSERT_NO_KEY("sdb.io_queued_ms"); + ASSERT_KEY_AND_VALUE("dm-1.reads", 109UL); + ASSERT_KEY_AND_VALUE("dm-1.writes", 0UL); + ASSERT_KEY_AND_VALUE("dm-1.io_queued_ms", 226UL); + } + + + // Strings with less numbers + { ASSERT_PARSE_DISKSTATS(disks, "8 0 sda 120611 33630 6297628 96550 349797 "); } + + // Strings with no numbers + { ASSERT_PARSE_DISKSTATS(disks, "8 0 sda"); } + + // Strings that are too short + { + BSONObjBuilder builder; + ASSERT_NOT_OK(procparser::parseProcDiskStats(disks, "8 0", &builder)); + ASSERT_NOT_OK(procparser::parseProcDiskStats(disks, "8", &builder)); + ASSERT_NOT_OK(procparser::parseProcDiskStats(disks, "", &builder)); + } +} + +TEST(FTDCProcDiskStats, TestLocalNonExistentStat) { + std::vector disks{"dm-1", "sda", "sdb"}; + BSONObjBuilder builder; + + ASSERT_NOT_OK(procparser::parseProcDiskStatsFile("/proc/does_not_exist", disks, &builder)); +} + +TEST(FTDCProcDiskStats, TestFindBadPhysicalDiskPaths) { + // Validate nothing goes wrong when we check a non-existent path. + { + auto disks = procparser::findPhysicalDisks("/proc/does_not_exist"); + ASSERT_EQUALS(0UL, disks.size()); + } + + // Validate nothing goes wrong when we check a path we do not have permission. + { + auto disks = procparser::findPhysicalDisks("/sys/kernel/debug"); + ASSERT_EQUALS(0UL, disks.size()); + } +} + +// Test we can parse the /proc/diskstats on this machine. Also assert we have the expected fields +// This tests is designed to exercise our parsing code on various Linuxes and fail +// Normally when run in the FTDC loop we return a non-fatal error so we may not notice the failure +// otherwise. +TEST(FTDCProcDiskStats, TestLocalDiskStats) { + auto disks = procparser::findPhysicalDisks("/sys/block"); + + std::vector disks2; + for (const auto& disk : disks) { + log() << "DISK:" << disk; + disks2.emplace_back(disk); + } + + ASSERT_NOT_EQUALS(0UL, disks.size()); + + BSONObjBuilder builder; + + ASSERT_OK(procparser::parseProcDiskStatsFile("/proc/diskstats", disks2, &builder)); + + BSONObj obj = builder.obj(); + auto stringMap = toNestedStringMap(obj); + log() << "OBJ:" << obj; + + bool foundDisk = false; + + for (const auto& disk : disks) { + std::string prefix(disk); + prefix += "."; + + auto reads = prefix + "reads"; + auto io_queued_ms = prefix + "io_queued_ms"; + + // Make sure that if have the first field, then we have the last field. + if (stringMap.find(reads) != stringMap.end()) { + foundDisk = true; + if (stringMap.find(io_queued_ms) == stringMap.end()) { + FAIL(std::string("Inconsistency for ") + disk); + } + } + } + + if (!foundDisk) { + FAIL("Did not find any interesting disks on this machine."); + } +} + +} // namespace +} // namespace mongo diff --git a/src/mongo/util/signal_handlers_synchronous_test.cpp b/src/mongo/util/signal_handlers_synchronous_test.cpp index f4104bafa16..c3623dde608 100644 --- a/src/mongo/util/signal_handlers_synchronous_test.cpp +++ b/src/mongo/util/signal_handlers_synchronous_test.cpp @@ -59,7 +59,6 @@ IGNORED_SIGNAL(SIGHUP) IGNORED_SIGNAL(SIGPIPE) FATAL_SIGNAL(SIGQUIT) FATAL_SIGNAL(SIGABRT) -FATAL_SIGNAL(SIGFPE) FATAL_SIGNAL(SIGILL) #if not defined(__has_feature) @@ -70,6 +69,7 @@ FATAL_SIGNAL(SIGILL) // These signals trip the leak sanitizer FATAL_SIGNAL(SIGSEGV) FATAL_SIGNAL(SIGBUS) +FATAL_SIGNAL(SIGFPE) #endif DEATH_TEST(FatalTerminateTest, diff --git a/src/third_party/boost-1.56.0/libs/thread/src/win32/thread.cpp b/src/third_party/boost-1.56.0/libs/thread/src/win32/thread.cpp index 54ebbf3e091..eb117d529ee 100644 --- a/src/third_party/boost-1.56.0/libs/thread/src/win32/thread.cpp +++ b/src/third_party/boost-1.56.0/libs/thread/src/win32/thread.cpp @@ -92,7 +92,10 @@ namespace boost } else { - BOOST_VERIFY(false); + // If we attempted to set the data to a null pointer, don't flag an error. + // This happens at shutdown, where boost has a race between thread and + // process exit handlers. + BOOST_VERIFY(!new_data); //boost::throw_exception(thread_resource_error()); } } diff --git a/src/third_party/mozjs-38/extract.sh b/src/third_party/mozjs-38/extract.sh index 101bd56e27c..e955be36387 100755 --- a/src/third_party/mozjs-38/extract.sh +++ b/src/third_party/mozjs-38/extract.sh @@ -182,6 +182,7 @@ cp extract/mfbt/MemoryReporting.h include/mozilla cp extract/mfbt/Move.h include/mozilla cp extract/mfbt/NullPtr.h include/mozilla cp extract/mfbt/NumericLimits.h include/mozilla +cp extract/mfbt/Opaque.h include/mozilla cp extract/mfbt/Pair.h include/mozilla cp extract/mfbt/PodOperations.h include/mozilla cp extract/mfbt/Poison.h include/mozilla diff --git a/src/third_party/mozjs-38/extract/js/public/HashTable.h b/src/third_party/mozjs-38/extract/js/public/HashTable.h index 5fb5012c4a8..36f37d6852a 100644 --- a/src/third_party/mozjs-38/extract/js/public/HashTable.h +++ b/src/third_party/mozjs-38/extract/js/public/HashTable.h @@ -13,6 +13,7 @@ #include "mozilla/Casting.h" #include "mozilla/MemoryReporting.h" #include "mozilla/Move.h" +#include "mozilla/Opaque.h" #include "mozilla/PodOperations.h" #include "mozilla/ReentrancyGuard.h" #include "mozilla/TemplateLib.h" @@ -32,6 +33,8 @@ namespace detail { /*****************************************************************************/ +using Generation = mozilla::Opaque; + // A JS-friendly, STL-like container providing a hash-based map from keys to // values. In particular, HashMap calls constructors and destructors of all // objects added so non-PODs may be used safely. @@ -205,7 +208,9 @@ class HashMap // If |generation()| is the same before and after a HashMap operation, // pointers into the table remain valid. - uint32_t generation() const { return impl.generation(); } + Generation generation() const { + return impl.generation(); + } /************************************************** Shorthand operations */ @@ -436,7 +441,9 @@ class HashSet // If |generation()| is the same before and after a HashSet operation, // pointers into the table remain valid. - uint32_t generation() const { return impl.generation(); } + Generation generation() const { + return impl.generation(); + } /************************************************** Shorthand operations */ @@ -771,7 +778,7 @@ class HashTable : private AllocPolicy Entry* entry_; #ifdef JS_DEBUG const HashTable* table_; - uint32_t generation; + Generation generation; #endif protected: @@ -878,7 +885,7 @@ class HashTable : private AllocPolicy #ifdef JS_DEBUG const HashTable* table_; uint64_t mutationCount; - uint32_t generation; + Generation generation; bool validEntry; #endif @@ -1017,8 +1024,8 @@ class HashTable : private AllocPolicy static const size_t CAP_BITS = 24; public: - Entry* table; // entry storage - uint32_t gen; // entry storage generation number + uint64_t gen; // entry storage generation number + Entry* table; // entry storage uint32_t entryCount; // number of entries in table uint32_t removedCount:CAP_BITS; // removed entry sentinels in table uint32_t hashShift:8; // multiplicative hash shift @@ -1102,8 +1109,8 @@ class HashTable : private AllocPolicy public: explicit HashTable(AllocPolicy ap) : AllocPolicy(ap) - , table(nullptr) , gen(0) + , table(nullptr) , entryCount(0) , removedCount(0) , hashShift(sHashBits) @@ -1529,10 +1536,10 @@ class HashTable : private AllocPolicy return JS_BIT(sHashBits - hashShift); } - uint32_t generation() const + Generation generation() const { MOZ_ASSERT(table); - return gen; + return Generation(gen); } size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const diff --git a/src/third_party/mozjs-38/extract/js/src/jit/CodeGenerator.cpp b/src/third_party/mozjs-38/extract/js/src/jit/CodeGenerator.cpp index 97ac07a65d7..aec5913b352 100644 --- a/src/third_party/mozjs-38/extract/js/src/jit/CodeGenerator.cpp +++ b/src/third_party/mozjs-38/extract/js/src/jit/CodeGenerator.cpp @@ -3464,12 +3464,12 @@ CodeGenerator::visitCallDirectEvalV(LCallDirectEvalV* lir) callVM(DirectEvalValueInfo, lir); } +// Registers safe for use before generatePrologue(). +static const uint32_t EntryTempMask = Registers::TempMask & ~(1 << OsrFrameReg.code()); + void CodeGenerator::generateArgumentsChecks(bool bailout) { - // Registers safe for use before generatePrologue(). - static const uint32_t EntryTempMask = Registers::TempMask & ~(1 << OsrFrameReg.code()); - // This function can be used the normal way to check the argument types, // before entering the function and bailout when arguments don't match. // For debug purpose, this is can also be used to force/check that the diff --git a/src/third_party/mozjs-38/extract/js/src/jit/ValueNumbering.cpp b/src/third_party/mozjs-38/extract/js/src/jit/ValueNumbering.cpp index eb367e132ef..424c2dd8e0a 100644 --- a/src/third_party/mozjs-38/extract/js/src/jit/ValueNumbering.cpp +++ b/src/third_party/mozjs-38/extract/js/src/jit/ValueNumbering.cpp @@ -438,6 +438,7 @@ ValueNumberer::fixupOSROnlyLoop(MBasicBlock* block, MBasicBlock* backedge) graph_.insertBlockBefore(block, fake); fake->setImmediateDominator(fake); fake->addNumDominated(1); + fake->setDomIndex(fake->id()); // Create zero-input phis to use as inputs for any phis in |block|. // Again, this is a little odd, but it's the least-odd thing we can do diff --git a/src/third_party/mozjs-38/extract/js/src/jit/none/MacroAssembler-none.h b/src/third_party/mozjs-38/extract/js/src/jit/none/MacroAssembler-none.h index ef1ec17d766..77ed4382749 100644 --- a/src/third_party/mozjs-38/extract/js/src/jit/none/MacroAssembler-none.h +++ b/src/third_party/mozjs-38/extract/js/src/jit/none/MacroAssembler-none.h @@ -131,12 +131,6 @@ class Assembler : public AssemblerShared static void ToggleCall(CodeLocationLabel, bool) { MOZ_CRASH(); } static uintptr_t GetPointer(uint8_t*) { MOZ_CRASH(); } - - void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end, - const Disassembler::HeapAccess &heapAccess) - { - MOZ_CRASH(); - } }; class Operand diff --git a/src/third_party/mozjs-38/extract/js/src/jsapi.h b/src/third_party/mozjs-38/extract/js/src/jsapi.h index 2b279b52092..0d8cd52b12b 100644 --- a/src/third_party/mozjs-38/extract/js/src/jsapi.h +++ b/src/third_party/mozjs-38/extract/js/src/jsapi.h @@ -275,10 +275,6 @@ class AutoHashMapRooter : protected AutoGCRooter return map.sizeOfIncludingThis(mallocSizeOf); } - uint32_t generation() const { - return map.generation(); - } - /************************************************** Shorthand operations */ bool has(const Lookup& l) const { @@ -390,10 +386,6 @@ class AutoHashSetRooter : protected AutoGCRooter return set.sizeOfIncludingThis(mallocSizeOf); } - uint32_t generation() const { - return set.generation(); - } - /************************************************** Shorthand operations */ bool has(const Lookup& l) const { diff --git a/src/third_party/mozjs-38/extract/js/src/jscntxt.h b/src/third_party/mozjs-38/extract/js/src/jscntxt.h index 7a46270d7c9..f5e38d3b56a 100644 --- a/src/third_party/mozjs-38/extract/js/src/jscntxt.h +++ b/src/third_party/mozjs-38/extract/js/src/jscntxt.h @@ -35,11 +35,11 @@ typedef HashSet ShapeSet; /* Detects cycles when traversing an object graph. */ class AutoCycleDetector { + Generation hashsetGenerationAtInit; JSContext* cx; RootedObject obj; - bool cyclic; - uint32_t hashsetGenerationAtInit; ObjectSet::AddPtr hashsetAddPointer; + bool cyclic; MOZ_DECL_USE_GUARD_OBJECT_NOTIFIER public: diff --git a/src/third_party/mozjs-38/extract/js/src/jsobj.cpp b/src/third_party/mozjs-38/extract/js/src/jsobj.cpp index f086030580d..2c66de6216b 100644 --- a/src/third_party/mozjs-38/extract/js/src/jsobj.cpp +++ b/src/third_party/mozjs-38/extract/js/src/jsobj.cpp @@ -1579,8 +1579,9 @@ CreateThisForFunctionWithGroup(JSContext* cx, HandleObjectGroup group, HandleObj if (!res) return nullptr; - if (newKind != SingletonObject) - newScript->registerNewObject(res); + // Make sure group->newScript is still there. + if (newKind != SingletonObject && group->newScript()) + group->newScript()->registerNewObject(res); return res; } diff --git a/src/third_party/mozjs-38/extract/js/src/jsscript.cpp b/src/third_party/mozjs-38/extract/js/src/jsscript.cpp index 31724b821e4..dfc4079bc1e 100644 --- a/src/third_party/mozjs-38/extract/js/src/jsscript.cpp +++ b/src/third_party/mozjs-38/extract/js/src/jsscript.cpp @@ -116,7 +116,10 @@ Bindings::initWithTemporaryStorage(ExclusiveContext* cx, InternalBindingsHandle // any time, such accesses are mediated by DebugScopeProxy (see // DebugScopeProxy::handleUnaliasedAccess). uint32_t nslots = CallObject::RESERVED_SLOTS; - uint32_t aliasedBodyLevelLexicalBegin = UINT16_MAX; + + // Unless there are aliased body-level lexical bindings at all, set the + // begin index to an impossible slot number. + uint32_t aliasedBodyLevelLexicalBegin = LOCALNO_LIMIT; for (BindingIter bi(self); bi; bi++) { if (bi->aliased()) { // Per ES6, lexical bindings cannot be accessed until diff --git a/src/third_party/mozjs-38/extract/js/src/jsscript.h b/src/third_party/mozjs-38/extract/js/src/jsscript.h index 7a6bff02a68..98676598cd8 100644 --- a/src/third_party/mozjs-38/extract/js/src/jsscript.h +++ b/src/third_party/mozjs-38/extract/js/src/jsscript.h @@ -206,8 +206,8 @@ class Bindings uint16_t numArgs_; uint16_t numBlockScoped_; uint16_t numBodyLevelLexicals_; - uint16_t aliasedBodyLevelLexicalBegin_; uint16_t numUnaliasedBodyLevelLexicals_; + uint32_t aliasedBodyLevelLexicalBegin_; uint32_t numVars_; uint32_t numUnaliasedVars_; diff --git a/src/third_party/mozjs-38/extract/js/src/jswatchpoint.cpp b/src/third_party/mozjs-38/extract/js/src/jswatchpoint.cpp index 4de64410dc9..1861204c8aa 100644 --- a/src/third_party/mozjs-38/extract/js/src/jswatchpoint.cpp +++ b/src/third_party/mozjs-38/extract/js/src/jswatchpoint.cpp @@ -27,15 +27,15 @@ namespace { class AutoEntryHolder { typedef WatchpointMap::Map Map; + Generation gen; Map& map; Map::Ptr p; - uint32_t gen; RootedObject obj; RootedId id; public: AutoEntryHolder(JSContext* cx, Map& map, Map::Ptr p) - : map(map), p(p), gen(map.generation()), obj(cx, p->key().object), id(cx, p->key().id) + : gen(map.generation()), map(map), p(p), obj(cx, p->key().object), id(cx, p->key().id) { MOZ_ASSERT(!p->value().held); p->value().held = true; diff --git a/src/third_party/mozjs-38/extract/js/src/selfhosted.out.h b/src/third_party/mozjs-38/extract/js/src/selfhosted.out.h index 7e7df5d36e0..50d5c0bda65 100644 --- a/src/third_party/mozjs-38/extract/js/src/selfhosted.out.h +++ b/src/third_party/mozjs-38/extract/js/src/selfhosted.out.h @@ -4,16 +4,16 @@ namespace js { namespace selfhosted { - static const unsigned char data[] = { 120, 156, 237, 125, 107, 119, 219, 198, 174, 232, 247, 254, 10, 70, 235, 158, 70, 222, 150, 124, 108, 39, 237, 110, 157, 56, 93, 169, 243, 168, 79, 147, 56, 55, 118, 219, 221, 237, 235, 147, 69, 75, 148, 68, 75, 34, 85, 146, 178, 45, 55, 254, 239, 119, 48, 79, 96, 30, 20, 101, 59, 143, 221, 189, 181, 86, 98, 105, 136, 193, 96, 48, 0, 6, 192, 60, 120, 30, 23, 81, 89, 245, 223, 63, 45, 138, 120, 241, 62, 205, 250, 201, 229, 193, 32, 218, 141, 248, 239, 125, 241, 243, 209, 87, 231, 18, 234, 176, 42, 210, 108, 248, 190, 156, 159, 150, 252, 27, 3, 180, 139, 12, 240, 111, 73, 60, 126, 29, 207, 24, 140, 252, 134, 241, 228, 179, 253, 42, 41, 226, 42, 205, 51, 142, 4, 253, 126, 244, 213, 96, 158, 245, 248, 147, 87, 105, 89, 181, 215, 162, 63, 191, 138, 216, 167, 26, 165, 229, 198, 36, 201, 134, 213, 136, 85, 217, 124, 244, 213, 245, 87, 240, 96, 146, 84, 28, 238, 109, 145, 87, 57, 123, 0, 248, 15, 78, 207, 146, 94, 245, 190, 87, 36, 113, 149, 180, 179, 249, 100, 178, 246, 136, 129, 106, 176, 13, 211, 81, 167, 243, 20, 240, 44, 79, 51, 2, 5, 5, 20, 100, 54, 47, 71, 4, 4, 10, 40, 72, 57, 73, 123, 9, 129, 225, 37, 22, 80, 94, 84, 20, 134, 21, 0, 200, 235, 120, 156, 236, 229, 25, 227, 239, 156, 177, 229, 116, 146, 180, 161, 82, 199, 84, 93, 3, 94, 104, 166, 189, 75, 122, 121, 209, 215, 108, 43, 146, 106, 94, 100, 97, 182, 92, 127, 229, 226, 23, 40, 58, 209, 159, 215, 107, 104, 52, 126, 138, 75, 214, 220, 44, 41, 170, 69, 59, 239, 68, 51, 171, 133, 89, 196, 88, 149, 19, 82, 142, 242, 31, 243, 124, 146, 196, 89, 251, 220, 2, 190, 119, 239, 220, 130, 124, 51, 159, 158, 38, 133, 3, 184, 78, 225, 246, 70, 73, 111, 44, 250, 177, 151, 39, 69, 143, 147, 171, 235, 164, 131, 168, 125, 30, 237, 238, 238, 70, 115, 54, 152, 131, 52, 75, 250, 209, 135, 15, 145, 40, 226, 253, 229, 96, 240, 57, 26, 21, 249, 197, 243, 162, 200, 139, 246, 214, 118, 135, 181, 47, 4, 153, 225, 234, 68, 173, 156, 55, 208, 90, 179, 72, 124, 197, 133, 207, 52, 199, 240, 178, 210, 253, 172, 74, 134, 156, 242, 71, 136, 136, 199, 76, 68, 77, 107, 178, 51, 155, 143, 236, 33, 121, 29, 87, 163, 247, 211, 148, 49, 168, 19, 109, 94, 110, 13, 240, 135, 54, 127, 24, 79, 147, 95, 227, 201, 60, 249, 103, 82, 228, 237, 203, 78, 180, 176, 56, 117, 201, 123, 185, 128, 14, 183, 47, 163, 123, 236, 251, 101, 244, 245, 215, 172, 0, 190, 46, 40, 178, 151, 73, 245, 58, 169, 70, 121, 191, 125, 208, 137, 222, 42, 68, 130, 58, 80, 82, 0, 228, 157, 19, 172, 110, 31, 172, 29, 191, 61, 49, 221, 19, 143, 109, 54, 235, 82, 202, 105, 73, 158, 6, 53, 104, 238, 237, 151, 123, 241, 100, 18, 195, 24, 66, 229, 53, 239, 240, 124, 223, 137, 170, 197, 44, 201, 7, 188, 129, 53, 194, 66, 40, 33, 29, 219, 215, 34, 250, 115, 178, 104, 199, 197, 112, 62, 77, 178, 74, 15, 24, 235, 26, 224, 98, 93, 147, 40, 21, 4, 193, 42, 64, 88, 71, 90, 194, 166, 181, 160, 119, 168, 112, 49, 61, 205, 39, 45, 155, 161, 194, 128, 49, 130, 153, 244, 116, 162, 41, 103, 47, 22, 76, 213, 148, 49, 98, 12, 217, 150, 233, 178, 168, 193, 72, 51, 131, 195, 49, 129, 152, 164, 18, 247, 154, 25, 34, 85, 196, 42, 244, 24, 19, 95, 72, 74, 218, 2, 77, 39, 98, 117, 215, 8, 171, 229, 80, 106, 84, 94, 102, 63, 252, 30, 233, 130, 1, 37, 236, 81, 197, 84, 60, 103, 73, 47, 77, 74, 109, 71, 20, 31, 216, 160, 199, 243, 73, 133, 202, 241, 96, 236, 49, 234, 25, 216, 70, 207, 60, 54, 52, 239, 81, 17, 83, 245, 16, 37, 46, 114, 81, 251, 154, 136, 24, 34, 105, 111, 13, 99, 193, 54, 96, 179, 19, 61, 99, 86, 111, 58, 75, 39, 201, 211, 98, 216, 222, 234, 68, 123, 170, 223, 215, 184, 247, 123, 164, 219, 120, 158, 108, 151, 73, 92, 244, 70, 207, 39, 9, 12, 115, 132, 251, 121, 128, 149, 9, 38, 50, 52, 144, 76, 26, 216, 211, 118, 251, 64, 202, 197, 90, 244, 228, 201, 19, 102, 60, 12, 35, 56, 196, 174, 207, 160, 116, 183, 12, 30, 192, 226, 200, 216, 147, 104, 43, 250, 1, 25, 41, 13, 112, 188, 117, 178, 22, 237, 40, 139, 4, 173, 100, 209, 147, 93, 32, 166, 182, 145, 177, 5, 143, 72, 26, 179, 246, 51, 241, 56, 153, 148, 9, 226, 51, 60, 129, 62, 172, 171, 231, 10, 197, 56, 122, 140, 49, 40, 216, 77, 123, 20, 153, 86, 199, 189, 113, 210, 231, 220, 102, 198, 8, 15, 226, 128, 169, 64, 251, 81, 4, 168, 88, 27, 236, 203, 250, 58, 126, 172, 80, 28, 28, 143, 79, 56, 19, 201, 32, 209, 182, 81, 159, 199, 134, 80, 65, 198, 181, 221, 169, 134, 237, 142, 97, 86, 60, 0, 67, 124, 107, 10, 176, 20, 194, 152, 216, 98, 120, 88, 49, 255, 169, 167, 132, 113, 194, 125, 4, 191, 72, 122, 205, 209, 99, 108, 140, 144, 106, 60, 248, 142, 77, 77, 157, 232, 62, 111, 67, 121, 78, 247, 145, 0, 15, 138, 124, 202, 91, 245, 11, 224, 54, 19, 64, 35, 118, 219, 39, 70, 234, 100, 95, 136, 249, 194, 10, 213, 137, 60, 157, 232, 152, 246, 214, 92, 30, 188, 138, 203, 234, 95, 88, 29, 1, 117, 55, 10, 234, 155, 122, 190, 230, 168, 150, 174, 196, 165, 84, 128, 63, 182, 181, 147, 232, 32, 0, 250, 116, 183, 177, 190, 129, 246, 179, 191, 221, 238, 167, 85, 183, 218, 102, 63, 185, 182, 97, 121, 251, 72, 26, 55, 49, 77, 120, 181, 238, 81, 184, 1, 166, 124, 100, 232, 124, 138, 202, 52, 242, 145, 159, 229, 142, 166, 64, 7, 215, 30, 145, 231, 245, 218, 98, 55, 74, 36, 149, 112, 216, 181, 1, 136, 179, 55, 176, 3, 207, 207, 147, 98, 209, 6, 172, 167, 76, 146, 7, 217, 29, 170, 191, 215, 143, 219, 108, 48, 144, 51, 136, 213, 192, 141, 220, 72, 128, 186, 251, 212, 61, 211, 158, 176, 33, 58, 232, 15, 19, 79, 133, 53, 128, 171, 152, 174, 28, 133, 45, 16, 182, 59, 204, 236, 156, 231, 105, 95, 25, 101, 174, 104, 220, 244, 68, 66, 211, 2, 19, 156, 81, 55, 159, 34, 222, 35, 3, 106, 232, 99, 222, 101, 135, 235, 102, 39, 26, 179, 47, 107, 65, 133, 28, 196, 76, 24, 107, 149, 146, 249, 116, 73, 72, 45, 197, 240, 11, 169, 241, 8, 65, 64, 27, 183, 27, 12, 226, 157, 14, 221, 214, 138, 67, 231, 153, 74, 241, 208, 5, 117, 137, 179, 67, 105, 17, 25, 12, 143, 234, 28, 230, 211, 228, 139, 213, 156, 146, 17, 247, 151, 86, 156, 219, 234, 141, 208, 138, 26, 181, 145, 138, 229, 215, 27, 62, 246, 31, 67, 109, 238, 114, 220, 62, 149, 214, 0, 51, 154, 42, 205, 139, 188, 120, 30, 247, 70, 95, 172, 222, 12, 4, 125, 127, 89, 213, 105, 168, 54, 181, 154, 161, 40, 178, 199, 246, 117, 60, 251, 98, 199, 117, 26, 207, 190, 208, 49, 133, 154, 79, 89, 205, 55, 201, 197, 179, 36, 43, 19, 17, 75, 64, 126, 225, 174, 198, 28, 42, 179, 254, 207, 146, 62, 207, 139, 218, 137, 176, 38, 82, 0, 159, 95, 178, 50, 30, 36, 111, 231, 149, 116, 44, 203, 246, 83, 14, 135, 80, 215, 203, 205, 211, 144, 53, 5, 193, 249, 24, 198, 244, 14, 7, 253, 83, 217, 82, 198, 138, 198, 254, 7, 231, 157, 50, 168, 31, 131, 127, 119, 108, 12, 239, 152, 135, 46, 243, 36, 47, 154, 50, 240, 93, 210, 159, 247, 190, 92, 23, 174, 224, 228, 125, 116, 171, 133, 18, 137, 240, 51, 238, 245, 230, 211, 249, 36, 38, 25, 102, 159, 85, 195, 86, 6, 85, 34, 33, 243, 86, 40, 100, 14, 164, 135, 108, 206, 124, 139, 12, 202, 178, 124, 139, 75, 7, 179, 100, 235, 235, 39, 200, 36, 217, 84, 104, 22, 188, 45, 146, 18, 178, 16, 187, 118, 56, 5, 159, 6, 9, 76, 69, 159, 223, 254, 6, 169, 59, 121, 228, 133, 67, 228, 80, 47, 149, 0, 173, 175, 251, 31, 156, 22, 73, 60, 118, 31, 93, 127, 21, 254, 197, 101, 75, 181, 106, 123, 204, 161, 241, 192, 6, 126, 9, 139, 194, 172, 161, 44, 49, 2, 218, 70, 15, 154, 59, 38, 68, 118, 253, 230, 82, 234, 188, 109, 32, 238, 192, 88, 222, 173, 186, 122, 109, 101, 48, 119, 165, 49, 6, 231, 21, 209, 111, 159, 101, 196, 22, 118, 205, 147, 242, 92, 25, 101, 208, 210, 190, 75, 135, 163, 234, 63, 230, 118, 215, 205, 94, 255, 181, 140, 110, 183, 123, 135, 70, 55, 152, 198, 86, 228, 125, 74, 155, 219, 237, 126, 26, 155, 187, 162, 213, 245, 242, 232, 139, 51, 186, 66, 253, 63, 154, 229, 229, 232, 191, 108, 243, 203, 73, 252, 8, 54, 56, 128, 215, 151, 126, 73, 179, 126, 123, 198, 24, 150, 246, 226, 42, 89, 209, 0, 155, 229, 56, 109, 136, 239, 54, 247, 194, 136, 11, 13, 160, 166, 185, 177, 249, 69, 53, 110, 18, 162, 91, 219, 112, 154, 70, 225, 28, 68, 133, 217, 212, 220, 56, 89, 75, 77, 33, 143, 187, 69, 45, 111, 218, 82, 173, 189, 113, 8, 207, 162, 16, 162, 213, 55, 224, 124, 237, 231, 63, 163, 254, 17, 71, 125, 201, 208, 6, 243, 209, 116, 81, 117, 233, 82, 234, 94, 62, 91, 252, 150, 178, 81, 202, 218, 21, 35, 60, 129, 165, 62, 246, 133, 253, 73, 50, 216, 16, 229, 108, 1, 186, 229, 8, 3, 72, 145, 48, 211, 158, 158, 39, 71, 188, 65, 2, 45, 104, 64, 176, 124, 95, 169, 85, 225, 113, 180, 201, 216, 108, 182, 240, 197, 151, 109, 177, 188, 78, 225, 58, 182, 71, 226, 251, 236, 208, 173, 128, 54, 6, 147, 51, 195, 132, 31, 2, 135, 8, 221, 156, 103, 214, 10, 49, 34, 92, 84, 88, 74, 247, 161, 224, 124, 3, 178, 67, 116, 75, 12, 126, 178, 159, 243, 17, 229, 227, 74, 246, 15, 254, 0, 240, 203, 155, 212, 45, 155, 110, 51, 92, 184, 211, 105, 22, 79, 80, 175, 161, 189, 165, 125, 102, 64, 183, 25, 40, 94, 157, 118, 183, 151, 207, 51, 181, 155, 87, 195, 11, 218, 186, 124, 96, 58, 210, 91, 230, 187, 119, 161, 22, 223, 87, 9, 35, 246, 24, 228, 237, 235, 175, 225, 255, 199, 178, 104, 93, 224, 163, 251, 48, 196, 232, 226, 231, 198, 249, 134, 15, 151, 90, 246, 159, 247, 225, 197, 136, 217, 22, 166, 220, 252, 9, 196, 28, 158, 165, 40, 142, 154, 59, 90, 14, 99, 14, 142, 171, 252, 132, 207, 4, 0, 100, 249, 158, 100, 138, 87, 159, 126, 50, 73, 152, 141, 230, 21, 45, 151, 152, 97, 176, 29, 208, 42, 183, 75, 56, 165, 184, 48, 176, 79, 228, 11, 235, 152, 157, 205, 168, 114, 187, 36, 212, 49, 108, 52, 15, 124, 19, 224, 100, 194, 76, 56, 159, 90, 75, 105, 10, 54, 63, 133, 193, 92, 110, 119, 198, 62, 163, 67, 250, 124, 3, 3, 116, 231, 182, 102, 117, 27, 226, 12, 253, 141, 108, 74, 51, 27, 130, 50, 79, 156, 26, 103, 106, 22, 251, 155, 162, 243, 128, 235, 228, 145, 152, 253, 172, 55, 153, 247, 147, 178, 29, 218, 70, 3, 18, 180, 170, 192, 200, 45, 241, 30, 23, 42, 188, 75, 14, 133, 197, 106, 163, 156, 25, 9, 188, 169, 71, 11, 148, 193, 42, 55, 157, 90, 155, 75, 51, 204, 129, 187, 220, 124, 42, 173, 137, 244, 143, 108, 215, 136, 110, 201, 183, 216, 10, 3, 228, 119, 139, 104, 32, 174, 19, 158, 75, 86, 233, 247, 248, 169, 13, 49, 146, 114, 3, 246, 211, 74, 236, 177, 30, 167, 32, 60, 25, 30, 59, 177, 71, 59, 233, 139, 209, 195, 3, 105, 246, 133, 91, 187, 200, 223, 36, 23, 4, 123, 91, 130, 137, 245, 177, 195, 164, 122, 199, 98, 250, 226, 60, 233, 31, 78, 114, 179, 141, 156, 59, 191, 180, 181, 134, 245, 182, 128, 228, 102, 160, 219, 162, 143, 13, 182, 161, 123, 184, 100, 120, 100, 29, 152, 88, 202, 209, 77, 79, 164, 169, 32, 247, 251, 108, 148, 211, 106, 97, 159, 179, 1, 61, 9, 215, 122, 147, 92, 154, 3, 77, 116, 95, 62, 87, 48, 56, 104, 192, 138, 40, 253, 252, 137, 103, 247, 59, 241, 204, 33, 182, 32, 213, 196, 17, 130, 253, 193, 111, 5, 95, 199, 236, 112, 202, 58, 181, 46, 78, 203, 161, 181, 69, 118, 192, 243, 148, 30, 147, 20, 49, 92, 47, 147, 74, 144, 254, 130, 169, 44, 25, 56, 222, 146, 206, 99, 114, 49, 147, 246, 69, 215, 244, 192, 111, 81, 177, 156, 254, 156, 114, 243, 173, 171, 48, 11, 241, 96, 59, 208, 214, 54, 177, 254, 229, 124, 2, 34, 255, 167, 176, 142, 40, 250, 233, 68, 253, 60, 99, 5, 92, 187, 162, 107, 99, 86, 4, 125, 79, 32, 25, 27, 91, 201, 88, 204, 120, 191, 156, 74, 242, 225, 0, 15, 58, 186, 99, 6, 11, 200, 217, 128, 134, 157, 36, 156, 28, 72, 1, 130, 57, 189, 164, 33, 65, 238, 186, 102, 25, 239, 130, 102, 217, 174, 101, 33, 37, 5, 231, 50, 122, 143, 143, 121, 245, 147, 6, 100, 56, 120, 183, 236, 156, 192, 44, 78, 11, 103, 145, 127, 27, 117, 31, 0, 142, 55, 97, 170, 74, 205, 102, 85, 253, 96, 235, 36, 64, 16, 161, 24, 64, 27, 80, 171, 172, 3, 169, 139, 90, 181, 42, 218, 58, 202, 45, 121, 201, 44, 64, 214, 200, 78, 72, 33, 23, 70, 204, 143, 203, 182, 13, 62, 227, 100, 116, 197, 217, 194, 154, 85, 69, 186, 10, 146, 45, 15, 146, 159, 147, 197, 10, 24, 182, 125, 169, 53, 166, 113, 92, 8, 74, 190, 37, 98, 144, 237, 34, 117, 130, 106, 79, 139, 225, 174, 215, 225, 132, 195, 61, 194, 28, 170, 18, 216, 82, 33, 142, 141, 114, 76, 252, 100, 154, 231, 92, 152, 2, 99, 65, 16, 206, 154, 240, 58, 141, 243, 156, 18, 154, 102, 75, 36, 189, 166, 48, 174, 170, 162, 132, 89, 255, 114, 115, 59, 250, 0, 127, 182, 196, 159, 135, 6, 102, 94, 50, 98, 246, 205, 68, 105, 206, 103, 73, 182, 120, 78, 104, 65, 55, 104, 189, 123, 161, 179, 76, 106, 163, 140, 125, 82, 137, 185, 151, 89, 114, 17, 237, 181, 225, 96, 192, 49, 210, 14, 107, 222, 198, 71, 207, 36, 65, 164, 101, 107, 199, 54, 242, 113, 224, 35, 61, 28, 176, 75, 190, 253, 53, 25, 155, 4, 64, 139, 36, 174, 141, 140, 79, 96, 52, 130, 161, 243, 24, 64, 120, 182, 39, 162, 193, 250, 102, 203, 131, 0, 170, 113, 35, 233, 91, 241, 120, 106, 206, 253, 122, 150, 35, 204, 54, 28, 92, 122, 237, 237, 139, 74, 97, 242, 230, 144, 243, 140, 225, 232, 158, 34, 37, 140, 63, 88, 199, 236, 64, 188, 180, 2, 116, 12, 114, 230, 60, 192, 128, 233, 223, 20, 255, 251, 103, 92, 0, 158, 197, 85, 172, 207, 210, 194, 118, 163, 245, 117, 178, 225, 168, 35, 4, 51, 176, 66, 129, 196, 151, 239, 148, 79, 199, 9, 246, 240, 184, 16, 132, 125, 117, 93, 199, 141, 241, 106, 197, 144, 59, 192, 59, 183, 222, 207, 69, 50, 201, 156, 84, 146, 77, 190, 241, 0, 140, 49, 247, 199, 22, 235, 67, 108, 175, 97, 186, 96, 53, 146, 60, 232, 12, 54, 163, 124, 219, 23, 144, 219, 103, 214, 244, 40, 157, 38, 47, 242, 98, 26, 87, 123, 113, 111, 36, 36, 236, 66, 31, 197, 70, 135, 168, 25, 9, 201, 251, 42, 127, 149, 179, 206, 36, 242, 156, 37, 54, 154, 151, 246, 54, 54, 48, 46, 188, 18, 151, 86, 56, 12, 129, 66, 49, 208, 28, 113, 112, 250, 125, 90, 190, 137, 223, 180, 47, 215, 156, 112, 171, 181, 159, 177, 170, 105, 159, 55, 221, 66, 98, 193, 73, 40, 253, 57, 228, 77, 146, 67, 222, 244, 228, 144, 1, 69, 62, 3, 18, 3, 40, 150, 167, 161, 93, 230, 161, 240, 81, 81, 71, 34, 119, 54, 35, 232, 54, 67, 246, 20, 106, 123, 134, 100, 131, 150, 133, 235, 195, 199, 116, 236, 40, 127, 38, 235, 29, 136, 178, 182, 124, 214, 137, 90, 113, 182, 104, 193, 159, 201, 164, 101, 89, 180, 38, 237, 51, 215, 164, 154, 188, 127, 70, 74, 85, 183, 59, 138, 2, 199, 2, 184, 216, 25, 166, 229, 205, 73, 153, 182, 131, 227, 219, 244, 243, 54, 253, 33, 177, 46, 175, 39, 224, 85, 109, 107, 0, 59, 145, 117, 198, 135, 104, 17, 252, 248, 143, 38, 125, 98, 77, 186, 59, 45, 2, 108, 45, 245, 183, 161, 30, 125, 66, 29, 186, 3, 253, 9, 245, 240, 139, 209, 32, 128, 251, 143, 6, 125, 66, 13, 170, 238, 116, 30, 2, 108, 45, 245, 183, 129, 6, 85, 159, 110, 22, 170, 238, 102, 6, 10, 245, 240, 115, 106, 16, 15, 102, 244, 93, 25, 107, 252, 50, 34, 46, 136, 167, 103, 40, 240, 165, 193, 17, 36, 95, 133, 82, 224, 29, 4, 172, 99, 252, 27, 239, 161, 68, 8, 223, 185, 150, 137, 238, 242, 248, 37, 158, 130, 99, 9, 247, 101, 192, 87, 40, 150, 69, 109, 241, 151, 74, 210, 15, 10, 43, 95, 233, 144, 100, 2, 160, 70, 56, 45, 135, 18, 223, 52, 41, 203, 120, 200, 81, 138, 194, 54, 255, 227, 32, 36, 184, 24, 200, 154, 234, 162, 38, 160, 213, 90, 195, 124, 101, 48, 10, 68, 97, 180, 32, 84, 87, 208, 207, 104, 61, 106, 237, 68, 45, 246, 135, 87, 199, 183, 140, 84, 113, 241, 50, 201, 80, 50, 149, 241, 200, 202, 167, 30, 206, 203, 89, 194, 168, 238, 19, 96, 55, 139, 26, 78, 191, 146, 138, 248, 185, 55, 157, 239, 164, 97, 73, 117, 127, 26, 22, 50, 147, 108, 128, 157, 238, 96, 217, 230, 43, 12, 46, 37, 44, 62, 155, 228, 101, 210, 175, 161, 40, 156, 248, 132, 48, 95, 229, 61, 85, 27, 26, 255, 126, 249, 110, 158, 101, 48, 178, 30, 212, 72, 94, 183, 191, 33, 26, 84, 21, 11, 55, 55, 13, 73, 182, 105, 66, 185, 47, 123, 125, 31, 66, 99, 181, 51, 230, 154, 177, 175, 234, 141, 162, 118, 226, 12, 205, 138, 125, 215, 160, 135, 73, 133, 161, 76, 103, 43, 232, 66, 164, 87, 203, 66, 98, 197, 123, 250, 23, 146, 43, 222, 159, 91, 10, 150, 96, 29, 67, 254, 153, 69, 135, 211, 241, 5, 203, 206, 59, 222, 133, 191, 144, 240, 136, 14, 221, 177, 89, 226, 237, 124, 42, 131, 196, 151, 132, 206, 249, 122, 126, 19, 2, 106, 133, 176, 16, 82, 216, 131, 158, 126, 89, 82, 248, 42, 25, 198, 189, 197, 178, 169, 209, 35, 86, 86, 69, 191, 96, 133, 132, 202, 170, 92, 43, 86, 30, 10, 91, 40, 114, 240, 210, 17, 226, 22, 31, 123, 114, 105, 100, 27, 161, 90, 46, 63, 62, 217, 185, 139, 105, 204, 35, 3, 43, 245, 235, 142, 165, 32, 52, 147, 125, 65, 98, 64, 38, 167, 149, 229, 192, 154, 150, 62, 225, 224, 251, 38, 162, 47, 108, 244, 121, 117, 216, 86, 83, 100, 241, 68, 199, 245, 143, 156, 255, 111, 206, 53, 63, 131, 144, 211, 73, 45, 229, 234, 28, 225, 165, 240, 31, 35, 168, 70, 66, 31, 45, 227, 66, 131, 45, 23, 159, 64, 1, 108, 217, 231, 164, 221, 88, 246, 5, 45, 162, 50, 33, 168, 78, 12, 116, 194, 6, 95, 153, 9, 207, 102, 12, 29, 154, 244, 97, 92, 219, 195, 36, 147, 226, 142, 55, 207, 12, 242, 162, 151, 232, 58, 73, 129, 45, 239, 88, 237, 19, 104, 101, 220, 188, 127, 69, 249, 102, 203, 139, 105, 64, 194, 251, 48, 85, 194, 66, 52, 71, 85, 97, 147, 66, 86, 108, 106, 42, 245, 212, 88, 32, 198, 84, 249, 211, 195, 189, 253, 253, 95, 216, 24, 22, 123, 49, 147, 162, 210, 189, 40, 85, 111, 54, 105, 181, 172, 101, 176, 84, 44, 131, 165, 209, 227, 72, 37, 162, 216, 47, 119, 45, 12, 238, 89, 45, 143, 83, 235, 64, 69, 43, 110, 193, 21, 178, 61, 200, 50, 245, 224, 91, 235, 170, 69, 45, 69, 207, 151, 156, 147, 183, 49, 87, 185, 38, 186, 19, 245, 144, 241, 144, 228, 174, 179, 170, 158, 36, 136, 217, 25, 33, 246, 236, 8, 153, 97, 99, 61, 121, 151, 12, 159, 95, 206, 202, 168, 230, 122, 101, 15, 248, 198, 60, 75, 123, 121, 63, 17, 73, 198, 231, 151, 85, 146, 149, 140, 210, 195, 228, 143, 121, 146, 245, 146, 119, 207, 35, 113, 87, 172, 191, 242, 36, 206, 134, 243, 120, 152, 28, 197, 195, 37, 144, 253, 249, 108, 194, 79, 37, 252, 26, 23, 105, 156, 85, 77, 193, 15, 25, 175, 38, 73, 149, 103, 75, 42, 164, 229, 111, 9, 99, 116, 94, 76, 147, 254, 222, 188, 40, 24, 241, 139, 61, 214, 175, 37, 213, 122, 18, 244, 89, 58, 76, 171, 18, 1, 107, 9, 27, 38, 213, 47, 75, 88, 100, 111, 222, 184, 17, 155, 63, 124, 192, 162, 211, 190, 217, 80, 89, 46, 253, 48, 185, 156, 189, 215, 247, 182, 190, 207, 242, 247, 37, 63, 20, 87, 182, 91, 221, 121, 187, 123, 28, 119, 175, 54, 187, 223, 159, 252, 185, 221, 249, 238, 122, 109, 189, 181, 70, 117, 171, 72, 166, 249, 121, 34, 59, 175, 155, 43, 101, 174, 14, 39, 161, 19, 253, 208, 104, 93, 3, 185, 106, 196, 90, 129, 81, 110, 129, 104, 155, 150, 248, 222, 95, 222, 193, 228, 50, 233, 225, 190, 45, 107, 185, 35, 83, 217, 44, 168, 187, 167, 110, 66, 70, 10, 47, 30, 214, 104, 110, 145, 204, 38, 49, 63, 125, 204, 33, 59, 168, 251, 199, 155, 39, 29, 72, 158, 25, 109, 94, 70, 140, 185, 58, 208, 222, 215, 42, 133, 73, 52, 66, 6, 134, 241, 237, 21, 86, 60, 50, 117, 46, 85, 83, 103, 170, 92, 90, 195, 12, 234, 211, 87, 111, 127, 130, 13, 8, 45, 16, 157, 167, 221, 127, 158, 160, 245, 129, 103, 251, 47, 247, 143, 248, 51, 144, 41, 244, 32, 158, 204, 70, 113, 54, 135, 147, 16, 173, 246, 15, 59, 144, 51, 20, 120, 214, 163, 214, 7, 248, 37, 106, 178, 95, 107, 45, 108, 181, 135, 243, 73, 92, 200, 74, 113, 81, 117, 39, 249, 217, 105, 156, 125, 232, 37, 147, 238, 48, 158, 79, 210, 114, 244, 33, 203, 187, 167, 249, 24, 254, 100, 139, 236, 195, 213, 168, 59, 156, 231, 139, 57, 124, 25, 197, 227, 113, 12, 95, 166, 105, 38, 255, 116, 179, 152, 127, 189, 100, 246, 103, 136, 155, 74, 11, 218, 88, 146, 117, 95, 254, 216, 205, 147, 254, 135, 180, 27, 79, 83, 246, 255, 105, 150, 177, 255, 229, 85, 198, 236, 91, 146, 229, 189, 17, 67, 195, 190, 178, 134, 216, 255, 227, 9, 147, 140, 28, 126, 79, 230, 151, 236, 255, 41, 252, 100, 127, 179, 248, 60, 62, 131, 47, 179, 11, 120, 88, 197, 57, 255, 127, 1, 255, 151, 243, 15, 229, 48, 235, 254, 248, 188, 251, 226, 157, 250, 246, 230, 21, 255, 182, 247, 83, 247, 217, 115, 76, 226, 176, 136, 179, 254, 32, 174, 70, 73, 145, 244, 17, 35, 13, 233, 138, 153, 232, 55, 70, 48, 43, 210, 115, 102, 78, 231, 101, 34, 107, 95, 178, 127, 198, 2, 108, 113, 11, 128, 43, 148, 202, 240, 162, 214, 244, 64, 125, 56, 126, 218, 253, 237, 247, 238, 63, 227, 238, 197, 162, 123, 117, 130, 235, 105, 101, 64, 245, 12, 174, 117, 94, 214, 133, 66, 45, 22, 172, 76, 90, 32, 140, 231, 92, 204, 19, 8, 11, 169, 240, 13, 171, 240, 193, 34, 139, 0, 60, 184, 94, 179, 196, 137, 210, 164, 69, 240, 207, 237, 107, 42, 133, 80, 21, 51, 162, 87, 164, 179, 202, 91, 243, 225, 181, 213, 115, 80, 27, 47, 228, 131, 107, 213, 109, 92, 182, 246, 231, 102, 103, 155, 224, 80, 122, 231, 39, 180, 99, 208, 168, 198, 96, 152, 127, 248, 96, 147, 101, 21, 0, 179, 236, 86, 170, 24, 40, 213, 237, 161, 129, 145, 29, 230, 152, 85, 153, 100, 31, 46, 35, 115, 141, 246, 149, 248, 152, 1, 216, 223, 16, 165, 82, 32, 72, 49, 18, 72, 142, 213, 195, 132, 35, 78, 98, 235, 127, 37, 35, 20, 213, 74, 210, 41, 6, 94, 68, 213, 4, 240, 254, 159, 22, 113, 47, 151, 155, 199, 21, 38, 81, 84, 143, 89, 253, 212, 158, 63, 153, 153, 126, 230, 120, 61, 75, 109, 181, 235, 40, 53, 50, 216, 110, 181, 207, 98, 181, 239, 88, 107, 237, 94, 161, 193, 241, 226, 7, 51, 210, 101, 158, 12, 146, 206, 86, 27, 160, 176, 100, 118, 233, 227, 32, 154, 191, 17, 184, 255, 247, 255, 182, 172, 122, 247, 236, 122, 107, 13, 100, 205, 231, 7, 55, 22, 56, 187, 114, 141, 192, 33, 191, 185, 185, 200, 161, 74, 171, 9, 29, 170, 248, 89, 196, 238, 166, 147, 149, 219, 1, 44, 97, 221, 182, 51, 121, 45, 23, 158, 245, 143, 45, 56, 36, 34, 90, 93, 116, 116, 117, 75, 120, 96, 245, 2, 42, 205, 11, 230, 246, 194, 230, 248, 180, 143, 156, 76, 203, 233, 247, 218, 106, 229, 211, 91, 190, 41, 74, 44, 73, 2, 171, 164, 12, 217, 81, 236, 157, 219, 18, 136, 206, 164, 57, 135, 222, 241, 155, 143, 224, 184, 97, 9, 103, 213, 141, 135, 222, 186, 236, 182, 92, 132, 230, 160, 7, 247, 144, 242, 178, 198, 233, 79, 245, 53, 215, 10, 101, 23, 112, 26, 114, 160, 58, 132, 19, 93, 116, 95, 248, 210, 80, 66, 191, 146, 201, 224, 133, 139, 4, 114, 188, 53, 216, 27, 57, 7, 230, 150, 58, 201, 118, 234, 17, 19, 65, 4, 48, 52, 80, 46, 41, 122, 180, 162, 175, 191, 198, 178, 184, 20, 3, 106, 92, 227, 160, 167, 196, 226, 44, 103, 177, 19, 147, 195, 171, 100, 153, 32, 46, 229, 51, 108, 162, 186, 80, 201, 22, 221, 90, 80, 146, 100, 242, 100, 20, 151, 7, 23, 153, 218, 140, 220, 225, 178, 206, 72, 120, 45, 54, 59, 151, 97, 73, 181, 0, 143, 5, 220, 9, 178, 87, 243, 83, 230, 201, 212, 201, 91, 201, 248, 84, 97, 105, 107, 161, 193, 77, 77, 172, 40, 3, 99, 158, 194, 18, 72, 245, 217, 40, 154, 195, 18, 79, 33, 59, 36, 192, 156, 116, 150, 40, 39, 47, 190, 129, 204, 22, 67, 13, 219, 187, 62, 124, 208, 24, 32, 227, 119, 217, 178, 146, 226, 214, 53, 68, 126, 132, 15, 237, 189, 83, 154, 168, 70, 89, 50, 1, 205, 98, 236, 53, 175, 247, 25, 69, 77, 244, 76, 32, 49, 71, 217, 224, 115, 109, 94, 201, 144, 114, 37, 222, 132, 158, 187, 244, 111, 223, 5, 253, 222, 141, 90, 171, 138, 225, 33, 71, 101, 36, 81, 162, 14, 210, 231, 214, 57, 22, 207, 220, 235, 178, 86, 33, 69, 70, 33, 205, 233, 176, 42, 40, 34, 54, 102, 69, 50, 72, 88, 44, 219, 127, 68, 234, 137, 17, 209, 178, 88, 87, 59, 21, 239, 214, 82, 210, 13, 66, 242, 167, 35, 36, 78, 191, 228, 91, 220, 70, 233, 160, 82, 196, 227, 133, 28, 77, 135, 125, 213, 193, 181, 103, 12, 141, 98, 105, 45, 67, 10, 65, 143, 16, 243, 189, 98, 176, 103, 109, 226, 19, 30, 243, 26, 187, 78, 144, 100, 120, 57, 157, 38, 89, 156, 240, 93, 179, 237, 4, 201, 150, 193, 97, 7, 241, 178, 190, 37, 150, 195, 200, 62, 239, 203, 61, 169, 239, 150, 61, 209, 184, 213, 101, 3, 169, 167, 179, 43, 180, 131, 111, 154, 163, 50, 128, 145, 217, 25, 134, 187, 96, 29, 237, 137, 205, 71, 248, 24, 62, 242, 55, 9, 154, 44, 36, 217, 211, 128, 160, 224, 181, 128, 120, 54, 150, 161, 234, 47, 34, 251, 210, 50, 83, 143, 199, 110, 235, 86, 73, 165, 187, 232, 168, 43, 32, 61, 53, 201, 130, 124, 112, 113, 52, 164, 161, 238, 152, 109, 190, 134, 58, 83, 117, 125, 23, 176, 226, 72, 191, 228, 239, 100, 108, 183, 168, 131, 164, 187, 131, 241, 57, 55, 25, 73, 172, 222, 54, 3, 237, 26, 204, 104, 200, 236, 123, 64, 112, 87, 237, 10, 214, 107, 78, 36, 160, 90, 100, 201, 39, 253, 195, 106, 49, 193, 94, 136, 178, 64, 176, 97, 133, 215, 109, 205, 226, 238, 219, 159, 97, 71, 36, 251, 242, 180, 136, 79, 225, 151, 56, 201, 221, 186, 26, 117, 247, 222, 192, 35, 246, 229, 167, 56, 43, 225, 151, 121, 244, 211, 207, 230, 81, 5, 191, 204, 163, 195, 151, 184, 22, 251, 101, 30, 29, 253, 134, 107, 177, 95, 95, 93, 227, 3, 76, 34, 81, 41, 146, 206, 100, 187, 184, 240, 39, 14, 6, 240, 230, 150, 119, 137, 124, 157, 101, 139, 39, 60, 157, 173, 222, 236, 201, 187, 121, 6, 219, 121, 45, 124, 200, 163, 111, 22, 53, 184, 14, 146, 67, 134, 229, 200, 45, 241, 254, 86, 247, 221, 106, 6, 209, 227, 199, 105, 58, 106, 170, 81, 175, 142, 115, 163, 221, 203, 39, 252, 54, 66, 181, 150, 43, 219, 79, 147, 114, 35, 62, 143, 83, 126, 70, 86, 112, 177, 108, 175, 41, 4, 212, 117, 206, 248, 246, 124, 177, 159, 249, 22, 104, 232, 198, 232, 149, 16, 73, 62, 120, 86, 68, 66, 195, 214, 96, 193, 98, 63, 184, 62, 215, 100, 245, 172, 102, 117, 175, 193, 186, 89, 221, 218, 96, 227, 0, 186, 117, 252, 191, 60, 135, 225, 132, 205, 126, 220, 109, 181, 178, 136, 181, 79, 188, 175, 83, 110, 205, 214, 0, 70, 239, 132, 9, 102, 34, 15, 41, 126, 103, 105, 187, 135, 196, 222, 64, 42, 67, 9, 211, 244, 131, 186, 152, 121, 73, 108, 183, 100, 140, 58, 136, 56, 202, 129, 184, 223, 63, 112, 181, 164, 108, 219, 2, 134, 25, 161, 213, 74, 159, 41, 65, 170, 203, 40, 65, 170, 251, 38, 158, 50, 225, 172, 209, 67, 251, 108, 43, 90, 212, 183, 154, 169, 91, 226, 167, 160, 88, 243, 69, 93, 39, 92, 178, 187, 119, 92, 103, 42, 40, 178, 147, 19, 58, 165, 133, 81, 73, 205, 198, 119, 82, 208, 11, 80, 173, 154, 225, 8, 154, 63, 231, 238, 223, 132, 14, 71, 240, 240, 140, 35, 75, 182, 3, 201, 163, 202, 132, 31, 92, 182, 31, 1, 82, 249, 246, 86, 140, 91, 189, 171, 213, 54, 180, 32, 0, 210, 250, 148, 40, 74, 166, 47, 54, 147, 100, 163, 89, 170, 246, 106, 102, 235, 28, 125, 248, 150, 32, 14, 104, 174, 224, 197, 111, 80, 62, 128, 163, 202, 116, 216, 245, 181, 185, 158, 131, 248, 129, 203, 55, 85, 213, 123, 138, 37, 10, 206, 122, 123, 173, 222, 243, 36, 158, 175, 213, 159, 208, 127, 176, 253, 173, 21, 168, 0, 9, 34, 202, 210, 54, 102, 108, 191, 26, 67, 211, 178, 100, 226, 230, 65, 92, 125, 243, 223, 116, 34, 26, 205, 194, 71, 180, 31, 154, 189, 93, 120, 158, 39, 96, 50, 164, 222, 89, 201, 65, 56, 103, 186, 91, 110, 251, 28, 146, 123, 224, 161, 56, 218, 127, 83, 19, 212, 35, 170, 241, 35, 51, 126, 79, 169, 238, 56, 10, 109, 82, 91, 56, 213, 132, 255, 151, 30, 116, 63, 133, 201, 86, 79, 143, 68, 222, 236, 59, 27, 188, 150, 67, 227, 176, 236, 130, 113, 72, 197, 99, 26, 6, 213, 231, 43, 39, 248, 213, 124, 26, 131, 29, 216, 168, 196, 165, 143, 225, 254, 119, 66, 227, 90, 79, 118, 163, 109, 190, 249, 73, 161, 63, 134, 210, 110, 180, 45, 222, 234, 8, 77, 17, 140, 252, 41, 171, 99, 48, 97, 238, 53, 200, 221, 160, 126, 144, 52, 41, 221, 98, 153, 231, 227, 249, 236, 53, 236, 246, 132, 183, 119, 58, 163, 90, 192, 222, 139, 178, 74, 250, 158, 121, 9, 37, 214, 140, 133, 177, 43, 168, 105, 68, 131, 89, 109, 216, 126, 52, 76, 157, 102, 251, 204, 43, 87, 72, 82, 97, 148, 128, 149, 22, 170, 240, 201, 67, 237, 147, 217, 212, 145, 137, 202, 109, 152, 215, 168, 221, 215, 99, 106, 59, 196, 52, 212, 27, 183, 85, 44, 114, 110, 34, 68, 111, 208, 163, 247, 49, 4, 244, 37, 124, 75, 139, 188, 86, 72, 179, 198, 59, 46, 10, 235, 196, 32, 243, 208, 235, 177, 238, 119, 187, 161, 9, 99, 214, 113, 51, 23, 218, 59, 216, 216, 84, 211, 132, 200, 193, 161, 230, 142, 55, 79, 106, 192, 213, 206, 36, 90, 103, 195, 186, 39, 10, 49, 223, 219, 80, 61, 164, 211, 198, 62, 197, 30, 184, 69, 212, 30, 109, 111, 144, 26, 216, 59, 73, 38, 130, 23, 105, 181, 186, 181, 144, 40, 87, 54, 54, 164, 117, 136, 161, 38, 231, 97, 69, 178, 107, 235, 227, 177, 240, 104, 146, 156, 179, 216, 95, 139, 1, 220, 34, 165, 4, 0, 238, 80, 193, 102, 109, 42, 232, 3, 175, 86, 212, 151, 92, 147, 116, 27, 131, 85, 240, 99, 165, 10, 26, 108, 248, 132, 119, 208, 50, 228, 63, 172, 222, 111, 82, 127, 231, 6, 124, 55, 68, 14, 114, 166, 253, 198, 154, 109, 224, 105, 215, 150, 245, 98, 195, 18, 66, 87, 186, 113, 2, 208, 77, 9, 138, 146, 87, 200, 226, 147, 220, 88, 216, 24, 57, 226, 93, 108, 132, 4, 220, 110, 109, 249, 178, 144, 174, 17, 204, 84, 18, 194, 177, 114, 29, 146, 108, 227, 10, 166, 88, 93, 146, 199, 100, 75, 51, 31, 13, 5, 94, 220, 154, 205, 242, 130, 13, 221, 115, 52, 16, 173, 238, 28, 239, 184, 243, 207, 179, 30, 145, 38, 116, 210, 249, 210, 113, 226, 147, 69, 8, 11, 153, 20, 45, 25, 2, 101, 209, 206, 27, 252, 104, 163, 135, 246, 197, 92, 201, 130, 84, 178, 208, 28, 179, 231, 86, 67, 234, 142, 59, 82, 147, 216, 92, 63, 203, 158, 246, 251, 105, 37, 89, 215, 242, 160, 124, 155, 151, 116, 62, 115, 132, 40, 40, 153, 168, 51, 111, 253, 62, 164, 72, 31, 235, 37, 111, 87, 63, 88, 85, 143, 35, 47, 17, 202, 85, 112, 207, 242, 11, 2, 90, 143, 182, 216, 40, 6, 196, 149, 174, 37, 235, 143, 13, 125, 108, 112, 161, 229, 131, 109, 55, 110, 112, 41, 81, 60, 208, 102, 70, 133, 110, 117, 109, 184, 235, 66, 2, 141, 24, 142, 6, 140, 36, 66, 208, 177, 26, 247, 44, 59, 41, 158, 233, 38, 194, 172, 37, 212, 96, 183, 208, 115, 193, 25, 254, 212, 75, 30, 79, 175, 131, 94, 173, 203, 239, 158, 235, 216, 212, 231, 218, 41, 245, 190, 127, 231, 246, 108, 107, 65, 80, 101, 223, 161, 81, 199, 174, 165, 204, 18, 24, 93, 132, 161, 151, 232, 92, 19, 125, 148, 115, 170, 146, 32, 249, 211, 178, 5, 64, 23, 1, 36, 234, 105, 75, 252, 138, 28, 193, 136, 215, 156, 94, 255, 233, 40, 170, 67, 200, 185, 168, 233, 142, 212, 57, 237, 84, 64, 152, 26, 155, 175, 16, 27, 197, 220, 194, 89, 22, 237, 218, 66, 230, 153, 79, 214, 119, 107, 218, 12, 198, 26, 98, 83, 130, 93, 13, 91, 14, 251, 230, 212, 34, 193, 115, 88, 131, 8, 21, 77, 8, 60, 70, 165, 83, 190, 53, 155, 176, 0, 182, 186, 13, 254, 32, 114, 234, 33, 145, 94, 172, 123, 217, 73, 73, 161, 174, 51, 245, 178, 157, 25, 63, 236, 91, 11, 31, 241, 80, 53, 167, 214, 25, 86, 139, 200, 27, 134, 224, 192, 162, 68, 249, 45, 118, 146, 178, 113, 26, 48, 28, 84, 219, 215, 31, 222, 46, 176, 246, 36, 13, 238, 40, 184, 94, 26, 46, 19, 85, 20, 92, 19, 25, 53, 135, 200, 64, 42, 77, 84, 225, 75, 203, 109, 235, 74, 92, 233, 212, 223, 98, 196, 73, 84, 117, 19, 60, 244, 38, 154, 213, 17, 152, 11, 136, 220, 240, 201, 184, 255, 234, 66, 164, 160, 139, 133, 111, 76, 82, 215, 10, 57, 215, 52, 53, 138, 202, 84, 147, 10, 184, 214, 171, 51, 24, 205, 5, 64, 162, 200, 227, 173, 97, 140, 42, 204, 131, 20, 20, 41, 63, 101, 204, 137, 6, 105, 213, 90, 146, 12, 254, 123, 39, 114, 91, 194, 247, 161, 18, 37, 37, 33, 166, 153, 4, 63, 124, 136, 72, 236, 89, 211, 56, 11, 65, 111, 33, 109, 46, 186, 157, 91, 201, 28, 96, 240, 30, 42, 21, 202, 18, 88, 118, 242, 93, 120, 42, 106, 116, 162, 180, 35, 43, 179, 200, 165, 254, 66, 120, 125, 29, 242, 214, 38, 255, 179, 189, 73, 82, 31, 117, 141, 180, 4, 97, 173, 14, 37, 52, 220, 30, 107, 235, 59, 111, 91, 196, 56, 16, 29, 132, 27, 232, 185, 116, 155, 155, 194, 102, 122, 81, 30, 214, 67, 58, 98, 186, 103, 229, 3, 249, 246, 56, 172, 122, 150, 255, 113, 172, 234, 162, 85, 247, 115, 215, 157, 178, 51, 238, 208, 142, 20, 169, 60, 159, 36, 113, 102, 73, 148, 106, 229, 40, 255, 81, 60, 23, 72, 113, 92, 173, 118, 198, 25, 84, 206, 90, 22, 197, 36, 245, 207, 135, 136, 212, 160, 106, 46, 120, 225, 56, 135, 75, 29, 66, 197, 67, 233, 200, 249, 242, 249, 68, 95, 191, 195, 195, 96, 211, 40, 71, 51, 244, 222, 18, 53, 78, 246, 56, 139, 139, 5, 195, 163, 61, 77, 179, 116, 58, 159, 130, 165, 184, 20, 95, 236, 17, 247, 47, 177, 220, 94, 6, 204, 168, 8, 26, 157, 81, 113, 238, 69, 148, 124, 100, 22, 73, 212, 125, 172, 168, 55, 69, 79, 84, 63, 106, 216, 252, 48, 200, 92, 253, 130, 153, 1, 51, 189, 148, 160, 229, 204, 22, 93, 212, 250, 156, 51, 46, 179, 134, 84, 143, 125, 42, 47, 65, 58, 232, 246, 116, 121, 151, 250, 195, 53, 251, 48, 121, 249, 58, 158, 73, 79, 234, 183, 36, 30, 179, 95, 228, 118, 100, 198, 135, 42, 229, 75, 123, 176, 199, 2, 93, 154, 231, 156, 13, 208, 8, 163, 154, 51, 233, 156, 253, 10, 112, 67, 168, 23, 236, 162, 42, 160, 145, 150, 253, 124, 18, 95, 45, 100, 238, 69, 156, 211, 166, 143, 83, 180, 207, 165, 36, 48, 142, 2, 201, 174, 189, 23, 230, 22, 117, 189, 19, 241, 215, 139, 232, 34, 106, 226, 116, 49, 25, 144, 18, 78, 61, 8, 202, 244, 14, 148, 82, 217, 55, 69, 179, 184, 126, 208, 123, 189, 135, 175, 127, 234, 107, 128, 69, 240, 199, 166, 193, 221, 230, 131, 169, 33, 204, 105, 66, 140, 205, 77, 242, 123, 233, 200, 32, 210, 166, 241, 226, 52, 169, 35, 206, 71, 141, 56, 93, 162, 145, 186, 45, 25, 245, 55, 28, 182, 180, 204, 140, 191, 127, 16, 55, 172, 46, 33, 154, 211, 114, 95, 11, 122, 223, 47, 233, 190, 171, 70, 176, 104, 141, 226, 210, 35, 90, 206, 153, 45, 131, 92, 177, 168, 20, 111, 184, 233, 77, 226, 178, 132, 253, 47, 204, 100, 242, 235, 74, 224, 123, 189, 154, 5, 105, 25, 122, 197, 156, 220, 192, 129, 143, 135, 217, 23, 169, 50, 171, 103, 73, 32, 24, 91, 77, 160, 255, 173, 155, 219, 91, 129, 62, 160, 226, 38, 218, 53, 76, 40, 103, 168, 180, 220, 37, 11, 86, 144, 185, 123, 65, 161, 11, 74, 151, 77, 173, 85, 44, 21, 155, 242, 217, 52, 104, 60, 143, 61, 185, 203, 16, 249, 30, 182, 170, 22, 98, 101, 105, 207, 218, 143, 88, 90, 170, 226, 113, 107, 232, 197, 172, 203, 155, 120, 230, 221, 104, 232, 107, 104, 25, 166, 55, 158, 157, 143, 8, 143, 224, 196, 138, 86, 206, 43, 93, 90, 217, 249, 6, 14, 201, 162, 159, 147, 133, 179, 171, 119, 156, 237, 68, 127, 42, 119, 99, 39, 106, 101, 243, 105, 82, 164, 189, 150, 176, 236, 59, 198, 157, 188, 22, 62, 243, 120, 64, 225, 123, 113, 153, 188, 72, 139, 178, 50, 53, 74, 117, 115, 172, 240, 216, 118, 162, 227, 214, 28, 118, 248, 193, 93, 178, 19, 56, 163, 4, 95, 248, 174, 189, 214, 201, 53, 217, 212, 91, 59, 164, 170, 212, 157, 103, 28, 129, 107, 58, 25, 115, 224, 141, 121, 41, 78, 163, 219, 141, 136, 7, 166, 9, 245, 12, 20, 48, 184, 9, 214, 128, 107, 152, 242, 144, 133, 91, 226, 125, 51, 254, 38, 164, 179, 205, 160, 156, 13, 202, 82, 75, 29, 92, 254, 248, 229, 7, 77, 35, 223, 158, 111, 242, 167, 126, 240, 29, 4, 206, 223, 26, 103, 42, 24, 58, 188, 139, 77, 176, 1, 74, 85, 245, 2, 160, 250, 176, 199, 154, 44, 2, 235, 154, 238, 254, 220, 186, 56, 208, 225, 157, 147, 216, 88, 165, 50, 243, 180, 235, 224, 253, 107, 207, 117, 13, 152, 85, 105, 159, 140, 153, 180, 155, 179, 150, 59, 78, 22, 78, 164, 210, 81, 175, 27, 177, 214, 19, 59, 119, 176, 142, 216, 116, 13, 81, 174, 100, 9, 225, 236, 229, 45, 59, 19, 163, 72, 142, 248, 99, 96, 45, 195, 98, 37, 194, 245, 82, 205, 70, 47, 231, 136, 64, 9, 225, 6, 103, 121, 207, 8, 92, 228, 12, 207, 80, 66, 197, 183, 160, 98, 94, 216, 228, 49, 101, 214, 42, 132, 69, 154, 172, 186, 161, 138, 2, 244, 121, 144, 160, 87, 64, 109, 44, 139, 174, 49, 174, 115, 179, 243, 209, 90, 115, 65, 103, 225, 176, 112, 152, 104, 207, 93, 24, 240, 108, 37, 42, 125, 118, 164, 136, 47, 14, 97, 28, 171, 244, 60, 85, 157, 228, 107, 1, 245, 151, 194, 59, 118, 197, 57, 213, 6, 163, 43, 111, 22, 104, 45, 25, 37, 126, 38, 23, 47, 157, 23, 104, 37, 221, 230, 59, 6, 117, 151, 167, 205, 51, 43, 171, 7, 244, 208, 138, 204, 116, 89, 253, 54, 236, 186, 246, 104, 34, 2, 135, 105, 194, 167, 172, 233, 48, 203, 139, 228, 45, 204, 74, 243, 88, 46, 241, 56, 28, 119, 128, 124, 152, 78, 97, 41, 65, 223, 10, 111, 237, 51, 12, 77, 217, 216, 41, 116, 167, 66, 238, 54, 187, 190, 178, 237, 34, 214, 57, 221, 198, 187, 34, 152, 108, 47, 209, 158, 82, 151, 135, 56, 174, 147, 45, 3, 49, 53, 44, 254, 46, 59, 124, 171, 243, 238, 92, 135, 245, 99, 184, 77, 232, 90, 66, 21, 33, 41, 82, 180, 210, 168, 173, 72, 110, 70, 29, 241, 33, 16, 98, 41, 28, 254, 171, 45, 31, 108, 227, 55, 73, 226, 97, 245, 102, 38, 52, 50, 234, 225, 99, 113, 93, 230, 17, 145, 141, 2, 230, 200, 194, 146, 253, 245, 162, 238, 210, 137, 217, 179, 218, 228, 46, 52, 4, 54, 230, 155, 101, 134, 63, 229, 245, 200, 196, 207, 94, 186, 10, 193, 183, 53, 70, 187, 190, 36, 109, 139, 59, 95, 224, 140, 106, 143, 245, 88, 120, 97, 80, 196, 29, 162, 22, 220, 117, 198, 139, 66, 125, 85, 206, 227, 220, 52, 151, 207, 252, 187, 139, 124, 62, 136, 200, 249, 153, 186, 102, 121, 195, 71, 48, 89, 60, 177, 8, 151, 203, 27, 29, 180, 166, 112, 130, 127, 72, 34, 24, 50, 186, 6, 195, 21, 219, 218, 35, 39, 131, 0, 181, 220, 239, 35, 197, 196, 9, 122, 74, 36, 119, 174, 154, 161, 52, 67, 77, 208, 6, 214, 236, 172, 166, 201, 207, 31, 162, 251, 48, 163, 222, 103, 78, 195, 125, 30, 63, 220, 55, 125, 26, 103, 22, 52, 114, 196, 85, 136, 82, 215, 31, 28, 199, 96, 182, 6, 35, 23, 183, 139, 156, 140, 1, 15, 207, 113, 131, 134, 144, 50, 208, 54, 154, 146, 112, 235, 1, 111, 243, 184, 117, 202, 240, 243, 55, 97, 245, 122, 73, 198, 9, 238, 201, 18, 53, 83, 123, 168, 91, 226, 50, 152, 185, 144, 155, 156, 89, 128, 84, 103, 214, 11, 10, 0, 103, 83, 168, 109, 223, 12, 155, 206, 180, 29, 247, 37, 26, 241, 164, 229, 132, 131, 246, 77, 178, 147, 247, 234, 241, 251, 210, 90, 238, 58, 24, 232, 179, 64, 228, 184, 214, 237, 223, 219, 99, 135, 51, 181, 113, 162, 39, 248, 185, 189, 37, 150, 179, 217, 173, 22, 133, 237, 164, 129, 75, 186, 206, 29, 208, 24, 115, 71, 87, 57, 36, 229, 29, 57, 170, 52, 192, 68, 208, 214, 19, 1, 255, 222, 166, 121, 135, 199, 14, 226, 161, 251, 76, 13, 126, 219, 62, 216, 105, 66, 105, 160, 27, 174, 29, 222, 112, 80, 211, 152, 71, 177, 148, 168, 31, 57, 248, 137, 42, 168, 235, 140, 252, 136, 201, 217, 203, 208, 33, 194, 148, 136, 171, 43, 22, 234, 229, 177, 50, 1, 227, 13, 220, 32, 199, 194, 66, 52, 166, 37, 227, 172, 117, 66, 114, 42, 254, 65, 241, 220, 112, 170, 99, 56, 153, 7, 159, 24, 90, 246, 244, 35, 186, 135, 34, 176, 100, 55, 207, 228, 109, 19, 6, 101, 39, 66, 206, 134, 122, 17, 131, 102, 78, 47, 223, 193, 176, 38, 84, 205, 160, 99, 194, 224, 170, 77, 113, 39, 130, 25, 68, 227, 67, 162, 100, 245, 210, 219, 238, 49, 16, 118, 178, 164, 77, 243, 24, 217, 234, 29, 99, 111, 107, 104, 218, 203, 167, 179, 184, 72, 142, 242, 31, 211, 172, 223, 190, 236, 68, 228, 184, 236, 63, 240, 66, 234, 37, 50, 1, 191, 227, 7, 11, 199, 91, 5, 137, 225, 120, 5, 68, 41, 47, 61, 255, 71, 39, 250, 189, 206, 24, 246, 68, 37, 200, 19, 183, 107, 34, 7, 215, 235, 150, 87, 135, 203, 250, 45, 143, 167, 47, 131, 29, 73, 86, 56, 236, 132, 230, 94, 32, 211, 72, 248, 243, 136, 128, 157, 122, 47, 151, 86, 63, 222, 159, 178, 10, 157, 232, 5, 121, 251, 27, 167, 42, 64, 17, 195, 135, 99, 105, 39, 149, 141, 161, 107, 120, 40, 35, 147, 190, 122, 61, 216, 77, 248, 104, 225, 104, 17, 219, 175, 222, 27, 175, 59, 36, 164, 120, 7, 231, 234, 197, 246, 56, 13, 193, 189, 80, 12, 192, 11, 2, 82, 107, 160, 80, 177, 129, 117, 230, 102, 92, 195, 121, 168, 36, 223, 144, 239, 79, 23, 214, 204, 132, 53, 9, 68, 223, 142, 147, 218, 220, 151, 239, 220, 243, 42, 155, 230, 81, 230, 200, 74, 126, 253, 128, 179, 92, 209, 78, 48, 19, 229, 201, 52, 89, 107, 217, 98, 132, 113, 206, 79, 243, 215, 164, 130, 234, 15, 247, 72, 39, 189, 230, 22, 5, 45, 64, 19, 52, 229, 226, 42, 255, 230, 19, 46, 94, 137, 185, 197, 164, 155, 205, 173, 233, 118, 233, 66, 15, 126, 130, 15, 49, 221, 112, 41, 35, 224, 46, 250, 90, 114, 226, 114, 59, 100, 245, 86, 34, 225, 41, 126, 42, 226, 173, 26, 17, 12, 172, 101, 96, 28, 27, 230, 73, 221, 138, 1, 169, 178, 250, 170, 193, 82, 70, 172, 140, 96, 201, 234, 1, 161, 247, 19, 45, 37, 80, 32, 49, 50, 204, 40, 29, 46, 88, 63, 167, 28, 58, 67, 57, 138, 160, 140, 148, 160, 46, 62, 140, 252, 1, 202, 152, 234, 228, 114, 75, 221, 242, 65, 86, 8, 104, 101, 5, 18, 106, 85, 61, 119, 39, 113, 90, 255, 89, 90, 206, 38, 241, 82, 52, 18, 12, 155, 80, 138, 79, 238, 116, 2, 153, 29, 38, 133, 120, 51, 66, 8, 169, 15, 214, 199, 32, 9, 247, 162, 136, 185, 37, 104, 132, 148, 2, 123, 177, 138, 13, 88, 13, 177, 250, 128, 205, 120, 181, 100, 171, 135, 108, 22, 79, 7, 44, 152, 204, 42, 1, 210, 98, 141, 122, 49, 226, 17, 13, 141, 77, 8, 233, 146, 174, 59, 240, 65, 252, 162, 83, 205, 241, 7, 224, 195, 210, 48, 47, 147, 151, 69, 62, 159, 153, 53, 90, 7, 41, 2, 249, 40, 121, 126, 255, 124, 113, 23, 185, 126, 140, 249, 203, 207, 247, 251, 249, 240, 217, 114, 254, 152, 156, 54, 158, 237, 110, 147, 251, 199, 120, 238, 36, 255, 79, 16, 210, 53, 0, 91, 144, 63, 246, 58, 192, 210, 169, 246, 115, 175, 5, 212, 37, 231, 125, 211, 252, 151, 153, 160, 15, 102, 146, 97, 146, 182, 218, 237, 39, 189, 116, 26, 79, 160, 148, 41, 136, 206, 24, 171, 73, 27, 104, 80, 32, 117, 188, 176, 230, 127, 117, 215, 151, 55, 151, 174, 112, 99, 66, 150, 172, 13, 244, 220, 13, 225, 247, 194, 23, 143, 133, 212, 230, 129, 121, 207, 20, 39, 208, 158, 1, 107, 61, 22, 78, 69, 64, 244, 236, 150, 30, 108, 162, 132, 3, 240, 33, 120, 149, 89, 144, 161, 200, 39, 234, 33, 92, 122, 114, 219, 35, 239, 109, 210, 248, 204, 70, 128, 94, 127, 9, 255, 165, 27, 100, 201, 3, 156, 220, 226, 69, 139, 233, 105, 206, 197, 2, 94, 118, 45, 86, 186, 68, 209, 90, 13, 195, 154, 245, 201, 248, 105, 61, 148, 32, 159, 102, 169, 36, 217, 191, 143, 190, 229, 115, 181, 24, 129, 91, 157, 8, 246, 89, 110, 213, 137, 103, 192, 163, 131, 38, 49, 1, 131, 190, 188, 67, 3, 2, 124, 87, 26, 126, 208, 252, 223, 193, 135, 247, 161, 94, 19, 194, 169, 227, 213, 226, 199, 35, 183, 217, 63, 212, 110, 131, 62, 56, 190, 30, 84, 71, 196, 92, 106, 100, 77, 198, 9, 129, 75, 227, 207, 247, 228, 51, 47, 169, 13, 120, 59, 170, 199, 107, 198, 162, 98, 140, 202, 100, 172, 128, 112, 115, 205, 99, 156, 151, 214, 122, 176, 70, 187, 88, 207, 111, 159, 163, 219, 18, 140, 150, 44, 191, 108, 198, 242, 128, 123, 13, 213, 241, 248, 151, 253, 200, 156, 97, 171, 247, 96, 69, 7, 104, 133, 90, 151, 148, 111, 52, 130, 22, 238, 57, 71, 197, 46, 237, 82, 108, 176, 36, 85, 75, 165, 210, 117, 247, 29, 149, 18, 35, 180, 12, 93, 160, 23, 156, 239, 101, 95, 96, 220, 222, 90, 102, 247, 106, 226, 5, 64, 179, 172, 118, 56, 26, 128, 14, 216, 54, 114, 24, 92, 250, 215, 254, 124, 112, 5, 147, 95, 108, 86, 35, 59, 52, 108, 24, 214, 175, 94, 90, 110, 184, 55, 216, 210, 75, 111, 196, 242, 235, 76, 222, 143, 63, 61, 219, 137, 30, 136, 204, 193, 143, 251, 47, 152, 133, 146, 223, 127, 127, 167, 191, 239, 189, 122, 129, 190, 191, 213, 223, 159, 253, 143, 41, 223, 255, 191, 6, 207, 203, 55, 168, 252, 240, 103, 253, 253, 127, 14, 12, 204, 255, 188, 253, 93, 151, 255, 252, 218, 192, 255, 252, 238, 55, 243, 253, 55, 3, 255, 234, 119, 243, 253, 224, 245, 59, 253, 253, 237, 239, 47, 53, 252, 187, 223, 12, 158, 163, 55, 6, 254, 151, 151, 255, 208, 229, 191, 252, 190, 175, 191, 255, 10, 48, 234, 251, 47, 191, 234, 239, 255, 120, 106, 240, 252, 227, 0, 125, 127, 11, 223, 73, 102, 14, 242, 241, 214, 171, 16, 155, 220, 203, 234, 188, 62, 177, 193, 109, 172, 238, 43, 23, 173, 220, 102, 205, 29, 172, 255, 123, 12, 119, 176, 254, 249, 224, 250, 255, 216, 247, 176, 218, 14, 129, 117, 251, 42, 14, 231, 86, 185, 54, 152, 18, 107, 126, 123, 94, 56, 75, 32, 143, 213, 207, 19, 18, 86, 109, 187, 75, 39, 36, 215, 250, 89, 215, 227, 87, 190, 121, 248, 203, 89, 147, 119, 179, 5, 58, 187, 231, 123, 95, 165, 220, 96, 107, 65, 170, 149, 93, 43, 61, 72, 87, 117, 37, 197, 199, 122, 248, 253, 184, 76, 10, 179, 21, 23, 241, 41, 223, 144, 194, 254, 194, 27, 116, 193, 162, 50, 174, 136, 88, 71, 88, 216, 126, 114, 30, 163, 45, 45, 173, 1, 139, 57, 47, 82, 225, 137, 14, 231, 103, 133, 248, 91, 204, 225, 239, 40, 102, 94, 91, 194, 247, 22, 141, 71, 211, 2, 87, 27, 103, 253, 152, 239, 198, 137, 243, 92, 252, 173, 248, 22, 148, 73, 58, 229, 20, 76, 39, 139, 41, 134, 159, 230, 162, 249, 233, 98, 202, 155, 200, 139, 5, 175, 95, 197, 211, 137, 108, 162, 74, 38, 115, 92, 165, 26, 197, 156, 242, 42, 61, 149, 11, 187, 39, 132, 255, 254, 69, 148, 101, 75, 205, 217, 124, 167, 110, 228, 196, 180, 21, 108, 71, 252, 47, 215, 144, 201, 77, 38, 48, 214, 151, 81, 224, 220, 39, 94, 47, 22, 40, 36, 144, 88, 134, 188, 244, 44, 21, 19, 93, 29, 136, 63, 75, 86, 139, 253, 57, 27, 185, 210, 57, 144, 103, 108, 130, 11, 198, 42, 107, 182, 108, 189, 56, 196, 14, 107, 201, 120, 112, 23, 75, 198, 58, 147, 119, 58, 88, 190, 98, 44, 128, 151, 112, 114, 133, 69, 227, 90, 118, 222, 245, 194, 177, 101, 8, 48, 104, 22, 82, 119, 158, 9, 32, 203, 200, 80, 96, 158, 251, 66, 49, 12, 238, 123, 238, 212, 166, 30, 185, 167, 58, 5, 64, 245, 125, 30, 61, 169, 239, 3, 192, 75, 233, 218, 171, 163, 11, 234, 186, 88, 233, 170, 98, 188, 176, 210, 214, 226, 171, 177, 158, 40, 29, 226, 150, 233, 16, 29, 153, 172, 160, 215, 142, 96, 66, 174, 184, 178, 87, 240, 199, 123, 93, 184, 67, 107, 221, 202, 249, 76, 199, 48, 184, 134, 115, 74, 164, 169, 167, 129, 252, 225, 153, 117, 245, 114, 104, 137, 156, 172, 141, 47, 89, 20, 183, 215, 96, 151, 28, 219, 67, 207, 238, 104, 29, 150, 34, 21, 39, 21, 106, 223, 80, 16, 88, 33, 165, 120, 26, 174, 145, 90, 149, 86, 95, 37, 117, 89, 178, 242, 58, 169, 7, 133, 32, 254, 160, 126, 189, 212, 162, 253, 179, 172, 152, 66, 57, 115, 46, 10, 14, 214, 139, 125, 32, 203, 22, 85, 151, 28, 126, 161, 200, 224, 165, 35, 255, 204, 51, 117, 240, 208, 195, 58, 5, 97, 26, 16, 147, 233, 129, 94, 172, 247, 84, 210, 32, 166, 214, 44, 174, 160, 97, 158, 164, 132, 27, 101, 246, 247, 126, 121, 43, 138, 208, 57, 155, 142, 65, 238, 229, 161, 65, 34, 191, 125, 148, 101, 176, 144, 202, 222, 197, 66, 152, 117, 234, 247, 139, 95, 10, 11, 241, 226, 211, 44, 134, 73, 121, 230, 20, 192, 158, 56, 38, 135, 153, 216, 207, 110, 18, 18, 23, 73, 50, 238, 199, 11, 190, 15, 38, 46, 138, 252, 130, 167, 149, 71, 242, 4, 197, 4, 92, 111, 185, 115, 50, 41, 226, 6, 80, 139, 36, 46, 0, 108, 187, 219, 135, 217, 172, 133, 142, 25, 72, 8, 230, 206, 87, 163, 0, 8, 79, 100, 215, 226, 151, 180, 134, 209, 143, 242, 249, 50, 2, 210, 108, 94, 37, 245, 48, 101, 194, 34, 251, 126, 61, 140, 210, 109, 144, 62, 128, 180, 232, 133, 196, 133, 119, 4, 26, 188, 71, 36, 48, 106, 248, 126, 19, 179, 46, 73, 133, 172, 77, 39, 172, 219, 44, 114, 82, 76, 119, 178, 204, 105, 161, 164, 11, 157, 174, 41, 252, 216, 75, 157, 13, 230, 203, 224, 98, 39, 94, 170, 84, 120, 84, 104, 96, 146, 148, 113, 198, 215, 110, 160, 223, 216, 209, 215, 83, 106, 112, 33, 179, 110, 30, 142, 118, 13, 2, 27, 167, 94, 113, 212, 163, 117, 39, 139, 156, 158, 153, 219, 89, 246, 212, 52, 57, 139, 159, 228, 183, 161, 184, 186, 66, 9, 118, 58, 85, 242, 219, 30, 174, 194, 137, 115, 94, 213, 89, 176, 211, 187, 173, 171, 171, 53, 28, 26, 26, 100, 173, 95, 142, 246, 90, 53, 87, 33, 125, 207, 226, 202, 43, 226, 163, 46, 153, 217, 129, 140, 43, 255, 244, 222, 120, 96, 113, 37, 207, 228, 159, 138, 125, 166, 40, 34, 64, 209, 128, 107, 94, 66, 209, 0, 224, 64, 14, 173, 169, 224, 236, 160, 61, 15, 31, 178, 2, 36, 88, 92, 2, 150, 138, 111, 130, 245, 156, 96, 146, 61, 16, 61, 20, 64, 17, 57, 48, 125, 109, 113, 178, 153, 56, 19, 88, 75, 156, 79, 227, 82, 158, 113, 91, 77, 154, 129, 134, 81, 177, 181, 29, 88, 125, 128, 121, 102, 107, 123, 165, 179, 115, 28, 91, 224, 204, 156, 102, 202, 134, 64, 204, 90, 5, 240, 250, 5, 9, 199, 29, 10, 132, 69, 196, 69, 115, 28, 72, 153, 79, 240, 93, 41, 89, 142, 19, 253, 190, 114, 121, 116, 188, 188, 72, 225, 237, 15, 138, 19, 27, 210, 129, 80, 181, 224, 20, 155, 158, 196, 119, 76, 190, 65, 33, 130, 183, 33, 62, 135, 15, 58, 137, 142, 94, 202, 43, 170, 139, 153, 52, 84, 187, 174, 38, 159, 122, 107, 154, 109, 97, 41, 179, 187, 194, 60, 156, 85, 186, 241, 18, 62, 55, 238, 70, 109, 205, 186, 110, 132, 91, 245, 247, 10, 60, 50, 218, 45, 229, 210, 4, 26, 88, 44, 234, 72, 83, 78, 80, 168, 242, 42, 164, 113, 87, 112, 37, 218, 94, 191, 190, 5, 109, 245, 117, 107, 199, 250, 53, 124, 110, 60, 214, 75, 234, 214, 141, 118, 184, 93, 63, 75, 29, 85, 92, 194, 208, 126, 255, 22, 12, 13, 212, 53, 54, 28, 108, 217, 161, 172, 177, 55, 226, 129, 121, 235, 12, 189, 87, 86, 81, 45, 109, 94, 237, 237, 152, 20, 150, 78, 225, 190, 118, 70, 136, 56, 231, 58, 75, 95, 133, 159, 106, 141, 3, 212, 88, 133, 179, 78, 11, 235, 78, 209, 141, 25, 223, 8, 81, 64, 229, 120, 240, 179, 146, 136, 76, 167, 183, 16, 145, 64, 93, 63, 113, 34, 234, 90, 137, 184, 178, 188, 5, 113, 129, 186, 126, 226, 112, 184, 71, 73, 172, 213, 253, 171, 155, 106, 254, 21, 251, 212, 209, 135, 23, 134, 100, 62, 135, 77, 249, 74, 44, 244, 172, 174, 112, 82, 39, 160, 38, 100, 129, 104, 39, 45, 192, 135, 145, 107, 134, 101, 104, 117, 186, 241, 86, 79, 115, 191, 225, 106, 155, 61, 13, 128, 27, 5, 186, 219, 66, 179, 36, 81, 219, 133, 248, 177, 37, 253, 46, 74, 160, 181, 173, 186, 37, 182, 68, 241, 120, 12, 182, 231, 208, 98, 136, 214, 214, 240, 91, 14, 108, 55, 199, 221, 223, 131, 167, 89, 231, 169, 177, 57, 100, 202, 11, 99, 113, 154, 112, 222, 114, 107, 245, 18, 189, 57, 245, 58, 212, 89, 144, 220, 149, 58, 11, 246, 37, 76, 163, 48, 33, 13, 250, 42, 212, 249, 14, 250, 67, 96, 190, 254, 58, 106, 247, 117, 5, 50, 152, 180, 56, 158, 76, 90, 107, 120, 22, 177, 239, 163, 213, 190, 60, 140, 29, 78, 240, 160, 96, 37, 88, 135, 15, 228, 170, 149, 250, 98, 39, 167, 85, 165, 113, 71, 245, 64, 222, 180, 163, 48, 176, 43, 119, 148, 15, 247, 170, 181, 196, 216, 7, 59, 43, 141, 151, 132, 167, 215, 243, 67, 188, 246, 2, 7, 116, 6, 173, 136, 146, 72, 136, 194, 223, 102, 16, 79, 222, 38, 44, 38, 226, 103, 62, 183, 182, 55, 77, 124, 23, 203, 151, 109, 152, 199, 248, 41, 152, 223, 87, 73, 89, 154, 167, 223, 209, 135, 175, 225, 160, 172, 126, 248, 45, 90, 63, 5, 147, 79, 171, 218, 79, 105, 221, 7, 40, 151, 79, 214, 24, 91, 210, 178, 0, 171, 88, 0, 210, 66, 226, 168, 69, 172, 79, 87, 23, 213, 40, 154, 145, 49, 220, 198, 179, 84, 11, 189, 164, 246, 92, 165, 125, 87, 206, 187, 26, 28, 16, 33, 31, 246, 114, 126, 14, 187, 187, 159, 13, 32, 195, 183, 160, 143, 213, 66, 186, 42, 243, 190, 65, 75, 142, 226, 10, 239, 204, 26, 168, 245, 8, 89, 213, 201, 88, 148, 146, 172, 205, 71, 158, 122, 111, 117, 242, 4, 62, 122, 73, 245, 76, 36, 81, 206, 88, 147, 102, 72, 116, 242, 228, 140, 38, 79, 172, 193, 131, 33, 53, 149, 142, 207, 60, 47, 39, 148, 66, 251, 86, 100, 93, 114, 255, 173, 224, 136, 40, 73, 168, 187, 232, 162, 62, 171, 44, 211, 14, 100, 242, 87, 53, 231, 121, 75, 46, 105, 82, 252, 8, 81, 135, 38, 123, 1, 111, 111, 243, 71, 184, 106, 223, 252, 192, 149, 131, 15, 85, 119, 215, 214, 77, 218, 228, 181, 217, 197, 140, 91, 118, 14, 24, 224, 94, 52, 110, 153, 218, 12, 111, 195, 110, 109, 107, 76, 213, 251, 242, 2, 247, 103, 56, 87, 222, 163, 170, 214, 125, 120, 10, 185, 233, 202, 170, 184, 77, 205, 0, 234, 126, 50, 209, 201, 116, 189, 103, 219, 252, 72, 179, 182, 221, 120, 215, 233, 107, 39, 218, 94, 235, 68, 221, 109, 79, 19, 48, 74, 178, 9, 54, 6, 158, 55, 155, 17, 238, 91, 134, 213, 69, 167, 71, 222, 224, 12, 188, 29, 75, 227, 180, 45, 110, 35, 164, 161, 119, 110, 81, 172, 200, 202, 55, 195, 218, 164, 255, 65, 164, 190, 215, 78, 241, 173, 252, 188, 246, 19, 99, 131, 109, 1, 199, 198, 153, 3, 63, 114, 30, 191, 160, 86, 244, 145, 175, 41, 239, 219, 116, 176, 101, 119, 95, 164, 211, 108, 174, 150, 168, 154, 204, 238, 238, 62, 44, 154, 211, 252, 188, 251, 79, 151, 109, 8, 249, 162, 119, 160, 226, 5, 201, 80, 7, 34, 223, 181, 19, 180, 210, 191, 249, 197, 19, 150, 56, 222, 226, 190, 167, 88, 120, 97, 214, 5, 20, 33, 102, 47, 189, 11, 41, 222, 177, 111, 125, 146, 59, 98, 244, 38, 85, 188, 109, 112, 213, 221, 172, 148, 44, 178, 159, 149, 108, 91, 22, 47, 74, 247, 232, 220, 38, 209, 185, 205, 128, 206, 193, 204, 215, 22, 72, 232, 140, 254, 3, 159, 191, 128, 249, 239, 179, 252, 130, 181, 185, 99, 182, 204, 2, 124, 112, 199, 172, 26, 175, 154, 61, 179, 214, 144, 54, 219, 53, 27, 218, 222, 113, 215, 251, 102, 195, 140, 167, 238, 238, 23, 180, 115, 214, 226, 231, 10, 123, 103, 151, 48, 245, 174, 119, 207, 170, 61, 99, 24, 70, 149, 221, 110, 143, 173, 10, 197, 48, 176, 42, 83, 218, 37, 24, 203, 123, 132, 86, 228, 12, 188, 76, 236, 117, 100, 223, 168, 132, 211, 187, 132, 210, 222, 92, 214, 135, 148, 240, 81, 174, 87, 100, 53, 67, 158, 239, 68, 38, 218, 228, 37, 47, 119, 100, 208, 201, 127, 193, 109, 104, 34, 246, 228, 63, 95, 239, 232, 16, 148, 255, 126, 101, 253, 238, 239, 224, 200, 116, 180, 163, 2, 83, 254, 243, 39, 250, 115, 76, 127, 254, 76, 127, 78, 119, 76, 48, 203, 11, 202, 29, 19, 211, 242, 130, 171, 29, 43, 180, 21, 197, 231, 254, 226, 95, 237, 98, 223, 245, 62, 136, 223, 54, 151, 221, 109, 164, 145, 253, 106, 70, 8, 84, 101, 53, 105, 223, 108, 197, 237, 153, 125, 118, 199, 204, 173, 178, 247, 219, 138, 212, 205, 125, 231, 74, 246, 32, 126, 136, 118, 52, 190, 19, 177, 153, 225, 190, 231, 14, 115, 237, 194, 121, 11, 130, 183, 126, 247, 152, 54, 131, 168, 108, 61, 186, 17, 57, 208, 155, 158, 247, 101, 202, 54, 57, 240, 225, 141, 217, 15, 174, 29, 154, 60, 175, 241, 85, 89, 121, 167, 45, 145, 80, 127, 137, 178, 233, 166, 244, 185, 183, 244, 202, 91, 122, 238, 45, 253, 213, 42, 229, 61, 227, 151, 175, 3, 215, 30, 239, 70, 15, 252, 78, 191, 126, 117, 175, 72, 170, 212, 4, 16, 146, 255, 140, 141, 15, 151, 160, 226, 105, 25, 63, 166, 250, 138, 50, 197, 227, 86, 69, 107, 11, 180, 219, 11, 47, 51, 250, 222, 210, 145, 183, 244, 39, 111, 233, 212, 91, 90, 122, 75, 199, 222, 210, 159, 107, 7, 164, 38, 12, 213, 220, 80, 105, 176, 27, 113, 82, 230, 205, 86, 96, 229, 107, 111, 55, 94, 45, 237, 70, 32, 70, 109, 64, 138, 71, 180, 110, 199, 19, 11, 217, 95, 77, 228, 101, 82, 157, 14, 8, 53, 74, 43, 157, 148, 8, 205, 199, 112, 191, 133, 219, 223, 192, 233, 137, 32, 150, 227, 222, 137, 251, 214, 62, 77, 163, 152, 93, 70, 124, 177, 64, 254, 248, 217, 51, 87, 4, 26, 53, 155, 155, 208, 249, 105, 204, 116, 220, 200, 79, 184, 145, 241, 77, 26, 193, 215, 76, 27, 158, 95, 75, 191, 102, 194, 194, 47, 22, 122, 57, 111, 61, 106, 197, 69, 213, 157, 228, 103, 167, 113, 214, 98, 243, 253, 217, 105, 46, 103, 255, 86, 47, 153, 116, 135, 108, 40, 211, 114, 4, 79, 240, 79, 9, 145, 100, 221, 225, 105, 55, 79, 250, 240, 156, 253, 120, 249, 35, 255, 33, 159, 166, 221, 120, 154, 194, 19, 248, 163, 203, 78, 51, 222, 14, 252, 209, 101, 234, 21, 40, 59, 248, 135, 126, 154, 100, 121, 111, 148, 10, 242, 208, 47, 253, 124, 20, 143, 225, 17, 252, 209, 101, 227, 9, 235, 100, 206, 171, 84, 147, 145, 41, 159, 204, 47, 161, 108, 114, 106, 138, 166, 0, 41, 112, 139, 175, 250, 73, 198, 226, 191, 51, 254, 40, 59, 55, 165, 179, 11, 142, 22, 254, 232, 178, 42, 230, 96, 240, 7, 149, 45, 68, 217, 2, 149, 149, 115, 94, 86, 170, 99, 150, 173, 179, 184, 11, 7, 55, 187, 163, 100, 118, 202, 156, 81, 248, 203, 92, 107, 62, 18, 113, 247, 21, 60, 137, 39, 241, 164, 247, 253, 223, 85, 133, 44, 239, 158, 230, 188, 195, 217, 41, 42, 203, 22, 156, 42, 195, 213, 114, 152, 117, 79, 147, 238, 160, 128, 226, 114, 112, 106, 149, 103, 19, 40, 63, 31, 86, 164, 156, 3, 159, 94, 149, 184, 176, 55, 98, 67, 194, 145, 12, 135, 164, 156, 119, 185, 87, 146, 22, 5, 228, 176, 36, 144, 125, 78, 110, 191, 156, 224, 194, 164, 228, 56, 203, 25, 46, 20, 212, 14, 40, 228, 240, 148, 83, 53, 72, 73, 97, 33, 26, 34, 164, 166, 188, 245, 148, 182, 158, 10, 185, 42, 19, 92, 120, 54, 227, 44, 166, 13, 77, 185, 108, 76, 7, 4, 103, 198, 69, 56, 235, 209, 194, 137, 232, 17, 193, 153, 9, 81, 161, 56, 103, 188, 245, 89, 89, 224, 194, 82, 240, 243, 130, 64, 206, 57, 67, 98, 138, 243, 42, 22, 195, 167, 91, 191, 26, 117, 123, 83, 62, 212, 240, 135, 20, 50, 69, 200, 74, 249, 164, 251, 19, 124, 119, 30, 87, 232, 113, 133, 30, 15, 133, 118, 13, 99, 140, 114, 56, 207, 23, 115, 183, 37, 166, 103, 227, 216, 82, 56, 86, 204, 116, 7, 10, 229, 55, 82, 206, 20, 73, 8, 39, 65, 127, 49, 231, 200, 225, 143, 41, 188, 100, 154, 61, 228, 200, 75, 12, 187, 152, 115, 142, 193, 159, 142, 58, 16, 0, 6, 237, 112, 126, 90, 249, 108, 218, 143, 191, 0, 248, 235, 215, 10, 197, 179, 103, 240, 251, 217, 115, 245, 251, 197, 63, 224, 247, 139, 119, 234, 247, 209, 91, 248, 125, 244, 74, 253, 254, 157, 195, 255, 174, 225, 255, 249, 14, 126, 239, 61, 83, 191, 227, 5, 23, 149, 108, 174, 137, 60, 61, 227, 70, 176, 95, 232, 49, 237, 245, 254, 128, 146, 98, 172, 229, 182, 119, 198, 133, 118, 154, 79, 117, 137, 224, 100, 79, 159, 236, 102, 95, 185, 178, 92, 246, 180, 209, 234, 23, 220, 252, 142, 71, 99, 83, 114, 193, 133, 170, 208, 35, 60, 140, 207, 121, 235, 137, 182, 83, 163, 162, 16, 70, 68, 211, 147, 158, 114, 73, 206, 103, 177, 46, 17, 38, 213, 24, 108, 142, 118, 164, 5, 240, 140, 87, 88, 104, 250, 207, 248, 243, 51, 221, 198, 120, 40, 40, 155, 234, 54, 38, 162, 207, 179, 153, 238, 207, 148, 43, 69, 161, 237, 226, 180, 228, 50, 56, 45, 180, 85, 156, 46, 236, 146, 178, 39, 228, 107, 162, 41, 169, 132, 102, 23, 177, 238, 113, 53, 230, 124, 170, 46, 52, 47, 171, 9, 39, 239, 34, 209, 109, 85, 217, 192, 226, 211, 226, 180, 111, 141, 201, 98, 202, 219, 154, 20, 133, 145, 172, 228, 178, 2, 225, 242, 76, 149, 220, 204, 255, 57, 43, 146, 65, 82, 20, 9, 68, 238, 80, 4, 235, 131, 108, 114, 190, 132, 159, 133, 122, 59, 96, 43, 62, 29, 57, 192, 167, 163, 32, 240, 185, 11, 124, 30, 2, 238, 77, 29, 96, 86, 20, 2, 254, 195, 5, 254, 35, 8, 124, 225, 2, 95, 4, 129, 47, 93, 224, 203, 32, 240, 194, 5, 94, 132, 128, 251, 3, 7, 152, 21, 133, 128, 75, 23, 184, 196, 192, 204, 144, 26, 232, 228, 212, 129, 78, 78, 67, 168, 147, 158, 11, 220, 11, 2, 247, 93, 224, 126, 152, 142, 204, 133, 206, 130, 208, 3, 151, 234, 65, 144, 234, 193, 208, 5, 30, 6, 81, 243, 169, 144, 66, 159, 205, 66, 168, 103, 46, 67, 102, 65, 134, 204, 92, 134, 204, 250, 33, 224, 194, 237, 97, 17, 236, 97, 225, 202, 116, 17, 148, 233, 194, 149, 143, 162, 12, 2, 187, 98, 90, 4, 197, 180, 184, 114, 129, 175, 66, 192, 124, 210, 167, 192, 252, 189, 10, 254, 81, 41, 93, 13, 40, 7, 97, 104, 119, 12, 203, 89, 24, 218, 229, 94, 249, 71, 24, 218, 53, 9, 229, 69, 16, 122, 238, 178, 100, 30, 100, 201, 249, 196, 1, 62, 159, 132, 128, 23, 174, 57, 93, 4, 205, 233, 194, 197, 188, 8, 99, 118, 85, 113, 145, 5, 129, 93, 86, 47, 66, 234, 114, 122, 234, 112, 3, 138, 2, 192, 131, 212, 1, 6, 159, 215, 207, 232, 211, 193, 216, 133, 30, 7, 161, 207, 156, 46, 66, 17, 130, 158, 150, 6, 56, 119, 236, 7, 20, 133, 80, 255, 225, 162, 254, 35, 104, 200, 78, 255, 112, 244, 11, 138, 66, 208, 213, 153, 3, 205, 138, 2, 100, 159, 59, 10, 6, 69, 33, 96, 71, 64, 78, 169, 232, 17, 58, 206, 231, 46, 244, 60, 132, 250, 202, 177, 55, 60, 162, 242, 163, 238, 245, 29, 151, 2, 138, 16, 244, 213, 8, 1, 59, 168, 123, 225, 169, 174, 119, 230, 240, 26, 138, 2, 168, 167, 206, 48, 114, 183, 223, 15, 156, 199, 14, 112, 30, 7, 248, 209, 155, 57, 110, 2, 20, 5, 48, 151, 206, 4, 3, 69, 161, 30, 150, 206, 12, 3, 69, 65, 104, 71, 64, 122, 97, 11, 220, 115, 45, 112, 47, 108, 129, 123, 165, 163, 51, 80, 20, 132, 118, 164, 15, 138, 130, 208, 238, 208, 148, 65, 13, 235, 185, 214, 189, 23, 182, 238, 189, 178, 112, 161, 139, 32, 244, 149, 99, 131, 161, 40, 48, 148, 87, 174, 100, 95, 133, 36, 187, 159, 59, 84, 67, 81, 128, 142, 190, 59, 146, 253, 240, 72, 246, 93, 110, 247, 195, 220, 238, 207, 29, 3, 223, 159, 207, 2, 178, 157, 244, 28, 133, 132, 162, 0, 234, 196, 37, 36, 9, 19, 146, 184, 195, 158, 132, 135, 61, 41, 29, 110, 67, 81, 8, 186, 114, 6, 50, 169, 70, 33, 232, 129, 219, 203, 65, 184, 151, 3, 119, 112, 6, 225, 193, 25, 184, 60, 25, 132, 121, 50, 40, 93, 74, 202, 32, 37, 34, 171, 65, 160, 121, 134, 195, 43, 129, 67, 215, 182, 14, 195, 182, 117, 152, 59, 33, 24, 20, 33, 232, 113, 62, 54, 208, 46, 75, 134, 97, 150, 12, 93, 91, 50, 12, 219, 146, 97, 233, 82, 82, 78, 195, 208, 110, 47, 107, 24, 56, 119, 161, 231, 65, 232, 81, 236, 184, 240, 163, 248, 52, 12, 237, 216, 87, 40, 10, 66, 59, 238, 14, 79, 63, 121, 135, 114, 228, 14, 229, 40, 60, 148, 163, 51, 199, 239, 130, 34, 191, 194, 143, 198, 46, 234, 113, 24, 117, 238, 66, 231, 97, 232, 153, 11, 61, 11, 67, 151, 142, 10, 67, 81, 16, 218, 81, 179, 81, 88, 205, 70, 174, 233, 25, 149, 33, 197, 73, 123, 14, 106, 40, 10, 160, 78, 39, 78, 39, 161, 40, 4, 157, 185, 184, 179, 48, 238, 204, 197, 157, 133, 113, 187, 74, 153, 134, 149, 50, 117, 149, 50, 13, 43, 101, 234, 78, 171, 105, 120, 90, 61, 115, 133, 251, 140, 10, 55, 146, 192, 179, 216, 113, 167, 160, 40, 0, 236, 90, 238, 179, 176, 229, 62, 27, 185, 208, 163, 48, 180, 59, 146, 103, 225, 145, 60, 115, 85, 225, 44, 172, 10, 103, 174, 184, 158, 133, 197, 245, 204, 53, 83, 103, 97, 51, 53, 30, 58, 250, 14, 69, 33, 232, 204, 81, 5, 40, 10, 24, 250, 241, 185, 99, 2, 161, 200, 63, 56, 227, 115, 103, 216, 161, 40, 68, 200, 185, 35, 82, 80, 20, 64, 125, 233, 56, 198, 80, 228, 7, 158, 156, 58, 236, 131, 162, 0, 29, 147, 158, 163, 54, 80, 20, 64, 221, 115, 140, 60, 20, 5, 128, 83, 39, 211, 0, 69, 1, 96, 87, 250, 38, 97, 233, 155, 184, 218, 59, 9, 107, 239, 196, 149, 190, 73, 88, 250, 38, 174, 231, 53, 9, 123, 94, 19, 55, 85, 51, 9, 167, 106, 38, 60, 91, 110, 65, 87, 97, 104, 39, 224, 131, 162, 16, 116, 229, 242, 164, 34, 60, 153, 156, 27, 224, 115, 151, 221, 231, 101, 8, 216, 13, 19, 38, 193, 48, 97, 234, 218, 179, 105, 208, 158, 77, 251, 206, 200, 64, 81, 160, 135, 211, 196, 25, 25, 40, 10, 160, 30, 56, 49, 45, 20, 133, 128, 29, 61, 159, 14, 66, 122, 14, 11, 138, 14, 112, 80, 84, 197, 58, 26, 133, 78, 67, 169, 154, 105, 230, 72, 19, 20, 5, 56, 253, 135, 51, 224, 80, 20, 192, 92, 56, 106, 14, 69, 33, 162, 221, 64, 124, 26, 14, 196, 167, 165, 99, 129, 161, 40, 64, 136, 59, 149, 78, 195, 83, 233, 116, 238, 162, 158, 7, 81, 95, 57, 201, 6, 40, 10, 161, 190, 114, 217, 119, 21, 180, 33, 211, 43, 71, 23, 161, 40, 0, 157, 185, 145, 74, 22, 140, 84, 50, 215, 94, 103, 97, 123, 157, 185, 46, 64, 22, 118, 1, 50, 119, 104, 178, 50, 56, 57, 102, 174, 161, 204, 194, 134, 50, 115, 77, 95, 22, 54, 125, 153, 59, 238, 89, 120, 220, 51, 55, 233, 150, 133, 147, 110, 249, 216, 161, 27, 138, 66, 208, 133, 51, 56, 80, 228, 151, 169, 220, 93, 109, 200, 233, 106, 3, 2, 158, 37, 14, 29, 80, 20, 0, 30, 58, 6, 10, 138, 252, 201, 228, 153, 27, 167, 204, 194, 113, 202, 172, 112, 233, 40, 130, 252, 152, 185, 203, 30, 51, 186, 236, 65, 160, 221, 140, 222, 44, 156, 209, 155, 185, 134, 100, 22, 54, 36, 51, 215, 133, 159, 149, 33, 95, 100, 230, 250, 0, 179, 176, 15, 48, 115, 69, 123, 22, 22, 237, 153, 235, 3, 204, 194, 62, 192, 204, 85, 132, 89, 88, 17, 102, 174, 34, 204, 194, 138, 48, 91, 184, 3, 191, 8, 14, 124, 49, 117, 160, 161, 40, 4, 237, 26, 135, 34, 108, 28, 10, 151, 131, 69, 152, 131, 165, 59, 179, 151, 225, 153, 189, 116, 231, 223, 114, 16, 76, 53, 148, 238, 4, 92, 134, 39, 96, 216, 230, 100, 67, 15, 131, 114, 82, 14, 29, 247, 5, 138, 66, 208, 35, 103, 33, 1, 138, 252, 58, 92, 78, 28, 231, 25, 138, 66, 168, 93, 135, 184, 12, 59, 196, 229, 31, 78, 236, 1, 69, 65, 104, 23, 247, 31, 97, 220, 110, 102, 162, 44, 67, 43, 117, 165, 171, 11, 101, 88, 23, 74, 87, 23, 202, 176, 46, 148, 110, 128, 85, 134, 3, 172, 242, 194, 49, 84, 80, 132, 161, 47, 16, 176, 219, 199, 139, 81, 16, 216, 149, 237, 139, 176, 108, 47, 28, 207, 1, 138, 2, 208, 213, 212, 9, 154, 160, 200, 111, 2, 43, 215, 94, 86, 225, 148, 71, 229, 102, 22, 171, 112, 102, 177, 114, 215, 29, 170, 240, 186, 67, 229, 230, 33, 171, 112, 30, 178, 114, 3, 155, 42, 28, 216, 84, 87, 206, 68, 9, 69, 1, 232, 249, 208, 153, 221, 161, 40, 8, 237, 80, 2, 69, 33, 104, 215, 207, 152, 135, 253, 140, 185, 59, 103, 207, 195, 115, 246, 188, 112, 132, 27, 138, 252, 3, 63, 191, 114, 59, 121, 69, 58, 57, 191, 66, 192, 46, 29, 212, 151, 66, 192, 176, 131, 212, 2, 230, 155, 74, 253, 68, 159, 143, 29, 162, 161, 200, 79, 244, 249, 216, 69, 61, 174, 66, 192, 238, 220, 116, 30, 158, 155, 206, 221, 185, 233, 60, 60, 55, 157, 151, 206, 230, 44, 40, 10, 64, 139, 125, 141, 4, 154, 239, 113, 244, 250, 242, 151, 99, 135, 108, 40, 10, 160, 190, 156, 58, 100, 67, 81, 16, 218, 209, 95, 40, 242, 243, 239, 210, 245, 4, 46, 195, 158, 192, 194, 205, 176, 47, 194, 25, 246, 133, 203, 237, 69, 152, 219, 98, 175, 39, 133, 158, 39, 1, 254, 93, 165, 142, 35, 0, 69, 1, 212, 87, 19, 135, 35, 80, 228, 231, 200, 213, 212, 25, 26, 40, 10, 0, 187, 93, 188, 10, 119, 241, 202, 181, 172, 87, 165, 135, 14, 124, 220, 110, 191, 74, 138, 184, 202, 139, 253, 126, 146, 85, 105, 181, 176, 95, 216, 5, 231, 58, 213, 25, 198, 87, 201, 48, 238, 45, 84, 141, 223, 138, 24, 110, 246, 125, 29, 207, 228, 101, 186, 112, 218, 226, 183, 36, 30, 179, 146, 54, 190, 11, 155, 86, 123, 147, 92, 86, 237, 184, 24, 146, 51, 166, 21, 191, 135, 216, 57, 187, 33, 177, 193, 65, 223, 78, 176, 117, 114, 96, 182, 42, 22, 232, 228, 153, 58, 130, 45, 78, 98, 236, 240, 118, 54, 50, 69, 64, 39, 234, 243, 243, 159, 252, 128, 131, 58, 238, 121, 205, 168, 224, 71, 216, 18, 251, 134, 195, 36, 74, 179, 178, 138, 179, 94, 146, 15, 120, 95, 15, 171, 124, 38, 200, 97, 244, 122, 143, 184, 235, 134, 209, 213, 179, 162, 77, 56, 185, 161, 154, 228, 116, 195, 45, 199, 145, 190, 90, 235, 58, 196, 61, 126, 27, 114, 59, 185, 204, 62, 27, 251, 42, 77, 193, 191, 34, 255, 218, 208, 7, 115, 63, 96, 128, 97, 229, 114, 134, 117, 56, 55, 232, 81, 117, 81, 227, 101, 146, 73, 149, 250, 136, 109, 186, 234, 136, 111, 137, 175, 187, 152, 157, 92, 81, 111, 161, 32, 7, 191, 103, 77, 94, 202, 50, 19, 175, 237, 96, 42, 213, 28, 114, 67, 157, 255, 114, 13, 131, 3, 154, 240, 3, 116, 112, 87, 2, 189, 125, 14, 129, 244, 242, 108, 144, 14, 231, 10, 200, 92, 44, 136, 96, 46, 138, 180, 242, 63, 63, 6, 162, 83, 73, 194, 73, 163, 78, 208, 42, 186, 55, 182, 37, 13, 215, 168, 237, 148, 5, 91, 223, 59, 11, 216, 219, 77, 87, 84, 222, 22, 121, 149, 123, 187, 250, 50, 169, 8, 80, 181, 152, 37, 237, 181, 142, 104, 76, 242, 224, 117, 60, 78, 246, 212, 139, 21, 83, 214, 90, 155, 98, 183, 165, 152, 35, 34, 66, 32, 148, 181, 145, 188, 112, 208, 128, 192, 112, 91, 232, 2, 215, 139, 140, 128, 89, 34, 51, 2, 104, 9, 55, 29, 61, 255, 56, 108, 117, 154, 233, 212, 182, 47, 177, 133, 109, 195, 70, 74, 236, 132, 232, 25, 50, 98, 111, 146, 11, 143, 201, 236, 68, 23, 194, 24, 41, 11, 1, 22, 253, 94, 179, 86, 140, 105, 247, 154, 28, 114, 105, 2, 248, 17, 178, 165, 118, 200, 192, 170, 186, 135, 163, 116, 106, 187, 43, 46, 245, 250, 242, 82, 62, 205, 217, 194, 217, 200, 128, 223, 166, 33, 7, 25, 109, 145, 217, 246, 23, 121, 241, 60, 238, 141, 248, 201, 213, 211, 184, 55, 30, 100, 194, 218, 63, 45, 134, 145, 231, 230, 17, 16, 192, 215, 242, 250, 29, 115, 117, 201, 189, 253, 82, 182, 254, 58, 240, 38, 144, 78, 212, 98, 141, 181, 196, 197, 39, 208, 32, 251, 10, 114, 200, 38, 227, 215, 222, 217, 223, 153, 169, 96, 150, 26, 197, 108, 22, 82, 240, 254, 185, 126, 197, 70, 175, 113, 31, 246, 88, 155, 160, 114, 136, 25, 254, 238, 124, 223, 137, 158, 37, 189, 124, 58, 75, 39, 9, 99, 84, 123, 179, 19, 225, 42, 70, 87, 153, 45, 46, 196, 173, 73, 222, 254, 164, 90, 169, 20, 61, 242, 74, 5, 126, 180, 214, 186, 51, 66, 223, 96, 82, 139, 234, 61, 204, 56, 29, 213, 176, 245, 158, 11, 129, 98, 3, 156, 24, 234, 241, 88, 39, 158, 21, 233, 11, 241, 182, 34, 168, 99, 93, 183, 64, 104, 240, 200, 142, 160, 96, 113, 188, 117, 162, 190, 109, 158, 160, 161, 147, 62, 4, 126, 231, 227, 30, 27, 159, 196, 121, 49, 134, 49, 12, 28, 244, 125, 149, 139, 27, 143, 228, 171, 60, 236, 183, 100, 58, 172, 145, 213, 56, 245, 112, 43, 30, 242, 54, 233, 125, 82, 55, 190, 145, 232, 14, 46, 18, 195, 108, 48, 74, 165, 105, 179, 175, 53, 244, 95, 173, 108, 249, 187, 14, 107, 55, 112, 73, 232, 90, 102, 248, 44, 171, 41, 46, 79, 194, 111, 177, 84, 164, 226, 235, 196, 124, 216, 162, 221, 37, 200, 149, 98, 91, 215, 144, 220, 180, 125, 247, 46, 108, 242, 146, 84, 140, 214, 185, 248, 73, 10, 78, 90, 190, 128, 185, 4, 124, 131, 41, 158, 124, 164, 25, 97, 165, 226, 178, 21, 129, 11, 29, 112, 151, 173, 34, 15, 64, 205, 49, 172, 74, 151, 255, 15, 99, 176, 233, 111, 244, 77, 252, 230, 110, 91, 132, 42, 236, 175, 191, 181, 195, 120, 144, 200, 183, 117, 74, 166, 248, 27, 102, 15, 56, 162, 251, 226, 251, 253, 186, 182, 185, 73, 245, 48, 17, 144, 215, 213, 83, 151, 62, 13, 121, 80, 121, 148, 91, 116, 209, 235, 178, 134, 146, 32, 249, 112, 9, 57, 250, 78, 203, 248, 180, 84, 213, 215, 224, 126, 150, 239, 55, 55, 255, 190, 245, 253, 247, 219, 223, 60, 252, 251, 195, 205, 239, 191, 223, 114, 16, 25, 7, 140, 98, 70, 204, 124, 57, 201, 79, 227, 137, 25, 58, 196, 68, 53, 103, 227, 193, 213, 119, 147, 41, 150, 248, 145, 81, 190, 249, 209, 73, 152, 122, 140, 98, 122, 62, 228, 47, 33, 127, 90, 150, 233, 48, 107, 87, 204, 48, 65, 212, 55, 72, 139, 178, 58, 204, 231, 69, 143, 188, 119, 143, 59, 147, 198, 171, 224, 192, 136, 253, 142, 177, 123, 140, 47, 16, 81, 124, 203, 233, 53, 73, 242, 254, 160, 126, 110, 77, 109, 48, 103, 9, 10, 176, 25, 117, 222, 67, 138, 193, 118, 197, 205, 241, 112, 189, 132, 85, 28, 48, 107, 204, 233, 174, 210, 12, 207, 97, 252, 94, 213, 34, 159, 226, 126, 26, 92, 107, 20, 112, 156, 44, 74, 126, 195, 42, 131, 70, 87, 121, 192, 101, 125, 109, 64, 98, 129, 139, 203, 140, 117, 37, 114, 157, 49, 238, 181, 186, 205, 21, 221, 77, 204, 95, 27, 31, 189, 222, 63, 60, 220, 127, 243, 18, 142, 233, 94, 211, 106, 179, 36, 235, 179, 185, 239, 249, 101, 47, 225, 214, 142, 129, 72, 96, 3, 39, 221, 8, 211, 130, 115, 115, 50, 38, 130, 117, 2, 19, 123, 172, 171, 89, 87, 253, 82, 15, 13, 99, 233, 39, 101, 47, 170, 121, 115, 221, 51, 246, 188, 72, 103, 224, 168, 2, 179, 58, 170, 217, 224, 125, 177, 12, 157, 115, 157, 47, 148, 162, 248, 202, 127, 45, 76, 149, 31, 75, 220, 16, 78, 67, 99, 250, 183, 117, 163, 148, 215, 125, 196, 84, 184, 140, 222, 213, 172, 246, 55, 238, 25, 26, 235, 138, 42, 122, 101, 140, 102, 52, 185, 133, 139, 200, 188, 131, 242, 94, 136, 8, 17, 215, 218, 240, 106, 74, 149, 18, 177, 190, 14, 183, 117, 217, 202, 75, 163, 32, 208, 90, 199, 116, 60, 227, 227, 112, 152, 192, 117, 51, 237, 44, 158, 194, 219, 35, 248, 15, 114, 169, 42, 7, 53, 70, 2, 188, 45, 162, 170, 186, 192, 247, 74, 8, 94, 25, 238, 249, 227, 214, 79, 96, 161, 239, 132, 80, 16, 52, 216, 33, 113, 136, 246, 225, 37, 117, 94, 255, 125, 235, 59, 126, 47, 58, 0, 224, 59, 2, 199, 92, 15, 142, 114, 164, 220, 188, 171, 8, 68, 10, 186, 17, 152, 247, 239, 103, 16, 248, 190, 127, 143, 111, 53, 229, 148, 107, 65, 21, 249, 194, 14, 86, 112, 29, 254, 219, 207, 24, 81, 59, 146, 177, 98, 224, 68, 211, 72, 177, 236, 91, 245, 121, 105, 7, 104, 239, 112, 234, 124, 118, 95, 12, 222, 75, 60, 120, 195, 47, 122, 240, 134, 203, 6, 111, 248, 37, 14, 222, 16, 6, 111, 120, 215, 131, 247, 138, 191, 114, 17, 105, 30, 30, 180, 6, 189, 174, 101, 188, 51, 21, 175, 96, 201, 17, 245, 86, 156, 201, 187, 98, 89, 213, 85, 174, 199, 130, 250, 66, 67, 91, 158, 219, 176, 164, 153, 226, 211, 1, 3, 161, 6, 86, 62, 244, 92, 147, 127, 237, 138, 34, 237, 165, 78, 142, 29, 12, 100, 223, 214, 44, 235, 41, 43, 222, 147, 74, 17, 30, 173, 151, 255, 134, 163, 53, 92, 62, 90, 195, 47, 103, 180, 68, 2, 225, 125, 57, 63, 21, 175, 131, 100, 29, 140, 11, 158, 56, 209, 161, 244, 222, 40, 233, 141, 69, 131, 123, 121, 82, 244, 120, 90, 212, 74, 32, 176, 202, 124, 172, 212, 171, 69, 173, 244, 2, 247, 2, 25, 12, 241, 255, 100, 136, 115, 8, 13, 146, 24, 135, 147, 64, 223, 152, 251, 60, 131, 52, 105, 59, 201, 250, 238, 133, 199, 128, 125, 7, 85, 7, 210, 77, 101, 6, 22, 79, 84, 27, 228, 58, 127, 114, 209, 191, 34, 164, 19, 109, 174, 117, 184, 147, 104, 161, 16, 20, 212, 34, 120, 14, 87, 7, 123, 170, 115, 79, 79, 197, 0, 32, 81, 136, 166, 199, 26, 59, 22, 61, 233, 142, 27, 56, 35, 16, 60, 30, 81, 117, 2, 137, 2, 92, 253, 57, 190, 0, 217, 84, 70, 88, 73, 122, 224, 80, 9, 194, 207, 73, 145, 37, 19, 214, 199, 162, 35, 240, 125, 136, 54, 59, 108, 54, 204, 89, 212, 206, 29, 125, 40, 240, 203, 18, 143, 174, 136, 72, 193, 31, 230, 40, 57, 162, 21, 136, 160, 182, 252, 201, 83, 54, 225, 49, 18, 238, 139, 102, 54, 52, 254, 251, 212, 109, 35, 234, 106, 139, 55, 16, 225, 16, 19, 86, 8, 165, 13, 244, 246, 218, 219, 43, 68, 35, 177, 47, 211, 171, 36, 160, 54, 137, 80, 7, 201, 48, 247, 10, 112, 168, 137, 85, 130, 184, 183, 50, 107, 160, 228, 111, 211, 48, 27, 145, 229, 85, 142, 104, 157, 163, 238, 240, 129, 87, 180, 136, 196, 232, 43, 73, 74, 141, 134, 36, 74, 61, 56, 121, 93, 221, 218, 154, 166, 138, 160, 122, 188, 139, 73, 147, 131, 171, 222, 176, 90, 39, 173, 154, 90, 46, 177, 4, 103, 35, 145, 181, 229, 149, 14, 254, 237, 69, 182, 177, 188, 58, 194, 170, 135, 209, 211, 131, 73, 218, 75, 254, 122, 198, 91, 24, 50, 42, 174, 242, 142, 123, 45, 87, 128, 98, 61, 194, 6, 156, 97, 36, 98, 104, 158, 81, 203, 204, 205, 161, 180, 220, 245, 168, 165, 105, 247, 32, 230, 79, 40, 218, 114, 22, 103, 182, 10, 105, 187, 105, 148, 167, 153, 197, 229, 200, 150, 201, 173, 28, 252, 143, 98, 102, 1, 119, 19, 145, 5, 184, 198, 230, 181, 151, 247, 147, 183, 57, 99, 223, 211, 170, 61, 203, 203, 85, 164, 245, 176, 70, 86, 25, 170, 84, 38, 28, 140, 72, 1, 126, 199, 170, 30, 18, 105, 230, 25, 6, 85, 151, 9, 2, 132, 121, 250, 247, 147, 93, 94, 201, 177, 69, 158, 21, 12, 158, 67, 244, 173, 191, 168, 110, 143, 226, 98, 143, 117, 253, 41, 99, 205, 97, 71, 55, 129, 12, 179, 192, 192, 72, 184, 124, 246, 221, 38, 167, 67, 148, 60, 129, 146, 31, 95, 188, 32, 148, 173, 71, 91, 92, 163, 188, 228, 241, 122, 168, 223, 226, 69, 131, 43, 211, 6, 141, 32, 250, 36, 26, 78, 224, 158, 32, 80, 22, 113, 10, 95, 188, 120, 81, 71, 136, 122, 185, 136, 232, 83, 87, 246, 114, 45, 250, 27, 251, 246, 144, 97, 91, 215, 13, 116, 101, 3, 107, 172, 108, 243, 114, 107, 147, 125, 212, 142, 155, 94, 206, 162, 99, 230, 202, 47, 89, 41, 147, 221, 42, 146, 89, 18, 87, 226, 30, 229, 187, 18, 51, 42, 95, 2, 181, 97, 81, 70, 39, 85, 164, 90, 223, 126, 143, 227, 124, 6, 248, 55, 45, 135, 172, 78, 123, 43, 122, 252, 56, 218, 254, 110, 205, 31, 239, 255, 93, 13, 3, 180, 158, 69, 95, 71, 109, 93, 129, 113, 107, 11, 145, 119, 20, 153, 87, 145, 243, 55, 185, 61, 122, 228, 44, 139, 177, 250, 214, 173, 217, 71, 240, 70, 213, 67, 227, 43, 50, 193, 127, 66, 174, 186, 231, 245, 104, 157, 67, 171, 142, 115, 217, 116, 232, 141, 172, 71, 62, 163, 160, 214, 110, 219, 119, 53, 78, 10, 33, 3, 121, 147, 92, 8, 24, 189, 83, 64, 194, 253, 146, 149, 241, 0, 18, 122, 239, 146, 50, 41, 206, 147, 254, 225, 36, 175, 218, 102, 69, 154, 153, 194, 195, 134, 160, 91, 142, 109, 87, 207, 60, 189, 109, 186, 229, 51, 80, 141, 111, 224, 36, 219, 64, 244, 222, 3, 206, 4, 80, 76, 86, 100, 245, 153, 63, 194, 178, 224, 51, 232, 144, 125, 162, 245, 94, 39, 213, 40, 239, 239, 15, 196, 86, 184, 190, 220, 7, 231, 132, 184, 248, 211, 114, 201, 165, 111, 183, 84, 99, 39, 152, 250, 50, 169, 68, 133, 23, 108, 226, 35, 236, 21, 91, 238, 54, 137, 71, 33, 214, 10, 116, 77, 166, 136, 15, 182, 3, 21, 183, 150, 153, 126, 250, 82, 148, 224, 86, 71, 178, 189, 82, 120, 208, 64, 133, 154, 27, 8, 75, 245, 230, 2, 178, 103, 9, 177, 91, 189, 148, 132, 114, 67, 24, 96, 242, 134, 137, 27, 204, 41, 156, 46, 103, 66, 97, 116, 202, 25, 5, 222, 7, 40, 230, 152, 93, 53, 165, 176, 34, 209, 27, 152, 79, 30, 59, 29, 186, 201, 236, 161, 241, 89, 57, 22, 53, 87, 236, 170, 249, 131, 181, 173, 166, 148, 93, 53, 129, 216, 111, 173, 64, 124, 217, 182, 51, 35, 215, 53, 154, 41, 37, 192, 80, 163, 49, 105, 45, 53, 123, 58, 106, 250, 134, 162, 71, 213, 181, 58, 156, 214, 123, 103, 108, 43, 39, 22, 233, 225, 142, 250, 184, 0, 107, 22, 223, 217, 180, 116, 52, 138, 43, 250, 56, 198, 158, 120, 237, 38, 143, 102, 59, 52, 106, 55, 121, 108, 19, 20, 219, 1, 20, 106, 246, 54, 18, 122, 187, 13, 30, 196, 27, 216, 80, 191, 234, 87, 64, 253, 53, 196, 214, 136, 61, 249, 187, 110, 51, 7, 170, 227, 71, 22, 200, 203, 172, 218, 150, 187, 113, 67, 138, 141, 24, 224, 82, 119, 158, 11, 230, 145, 24, 238, 96, 148, 0, 49, 197, 158, 242, 187, 219, 218, 3, 39, 111, 10, 86, 25, 5, 103, 124, 81, 238, 97, 146, 240, 7, 210, 243, 226, 47, 226, 91, 67, 206, 134, 187, 164, 27, 89, 235, 175, 252, 221, 177, 104, 205, 207, 54, 54, 114, 35, 179, 145, 37, 223, 66, 172, 2, 220, 123, 203, 69, 94, 45, 250, 179, 18, 203, 228, 72, 32, 72, 127, 162, 109, 20, 188, 144, 207, 146, 116, 179, 137, 40, 183, 60, 35, 228, 132, 49, 119, 64, 235, 151, 2, 246, 182, 39, 163, 8, 249, 235, 9, 119, 98, 95, 16, 247, 248, 230, 168, 193, 82, 190, 240, 88, 74, 53, 52, 27, 179, 121, 57, 82, 56, 30, 89, 194, 111, 47, 255, 155, 12, 51, 173, 206, 62, 178, 189, 174, 242, 192, 215, 162, 255, 22, 142, 186, 72, 254, 113, 215, 156, 59, 240, 143, 66, 56, 92, 12, 255, 165, 93, 125, 233, 224, 123, 68, 61, 252, 122, 54, 230, 127, 76, 22, 157, 8, 25, 103, 46, 213, 114, 242, 233, 136, 229, 43, 77, 68, 157, 54, 20, 241, 5, 207, 236, 31, 50, 31, 173, 19, 109, 108, 136, 228, 76, 90, 205, 133, 2, 34, 157, 16, 27, 73, 14, 6, 135, 24, 0, 226, 123, 252, 219, 81, 147, 94, 158, 143, 249, 222, 94, 189, 128, 161, 90, 195, 185, 179, 248, 130, 64, 240, 58, 27, 172, 20, 155, 109, 238, 68, 78, 14, 147, 161, 82, 187, 163, 92, 164, 179, 218, 12, 112, 195, 73, 237, 217, 240, 181, 121, 52, 227, 253, 8, 238, 160, 200, 33, 176, 53, 163, 102, 103, 38, 223, 52, 146, 12, 241, 20, 196, 40, 68, 218, 139, 228, 196, 106, 148, 252, 92, 87, 152, 92, 233, 223, 215, 126, 10, 24, 120, 171, 171, 222, 211, 43, 24, 179, 107, 61, 216, 152, 134, 26, 121, 236, 31, 120, 107, 147, 160, 192, 129, 187, 76, 196, 194, 223, 121, 39, 76, 50, 104, 20, 247, 27, 242, 8, 147, 239, 236, 162, 184, 14, 75, 63, 117, 66, 152, 119, 179, 197, 83, 55, 219, 183, 201, 24, 73, 151, 127, 131, 224, 110, 53, 117, 64, 154, 121, 15, 181, 14, 200, 3, 130, 226, 129, 7, 69, 77, 254, 138, 80, 205, 153, 33, 89, 210, 137, 60, 179, 179, 203, 216, 211, 116, 216, 56, 110, 85, 74, 248, 152, 85, 122, 210, 98, 99, 73, 61, 58, 86, 208, 122, 252, 223, 252, 153, 183, 165, 73, 154, 141, 111, 208, 22, 84, 11, 183, 38, 158, 122, 219, 203, 39, 253, 27, 52, 23, 110, 202, 223, 204, 32, 189, 76, 110, 208, 78, 85, 5, 27, 130, 71, 254, 60, 67, 60, 73, 123, 229, 234, 109, 165, 193, 166, 82, 127, 75, 229, 148, 73, 218, 234, 237, 240, 106, 193, 182, 228, 83, 255, 220, 86, 164, 227, 228, 6, 13, 242, 122, 225, 22, 229, 227, 192, 66, 217, 13, 218, 155, 135, 197, 131, 63, 243, 183, 52, 187, 73, 75, 179, 154, 150, 102, 86, 75, 207, 203, 94, 60, 99, 49, 44, 3, 61, 157, 87, 201, 175, 16, 24, 182, 207, 233, 139, 88, 103, 115, 176, 193, 216, 226, 159, 147, 228, 4, 123, 254, 138, 47, 153, 40, 80, 199, 53, 200, 231, 149, 70, 130, 103, 219, 222, 104, 158, 141, 213, 138, 202, 166, 229, 82, 139, 215, 107, 70, 176, 109, 77, 181, 241, 8, 94, 28, 105, 199, 67, 170, 85, 245, 194, 201, 251, 173, 251, 182, 155, 104, 218, 95, 111, 20, 244, 42, 148, 29, 68, 33, 43, 5, 46, 222, 255, 250, 143, 121, 94, 61, 186, 111, 57, 153, 184, 35, 41, 76, 215, 254, 176, 157, 199, 111, 8, 118, 215, 231, 170, 168, 214, 31, 249, 170, 24, 102, 160, 61, 88, 119, 208, 61, 42, 70, 26, 163, 79, 44, 227, 172, 55, 202, 233, 30, 151, 219, 4, 242, 178, 201, 251, 143, 227, 8, 80, 238, 182, 238, 51, 254, 121, 229, 82, 180, 200, 198, 160, 245, 4, 96, 14, 133, 80, 199, 1, 235, 202, 252, 126, 22, 45, 230, 144, 174, 102, 255, 223, 41, 165, 128, 59, 226, 104, 107, 168, 149, 205, 218, 228, 66, 221, 48, 197, 144, 138, 106, 227, 124, 212, 221, 209, 11, 88, 107, 200, 21, 141, 54, 167, 150, 207, 201, 243, 98, 114, 199, 34, 48, 42, 146, 65, 13, 149, 188, 193, 165, 18, 144, 84, 55, 56, 221, 118, 88, 115, 186, 237, 48, 124, 186, 141, 53, 230, 61, 104, 166, 114, 232, 75, 78, 183, 177, 234, 226, 116, 155, 130, 111, 116, 186, 109, 89, 163, 31, 249, 116, 27, 207, 31, 122, 15, 183, 65, 119, 204, 250, 128, 34, 231, 70, 135, 219, 48, 42, 121, 184, 77, 180, 107, 229, 8, 86, 57, 219, 166, 18, 159, 55, 61, 219, 198, 43, 232, 63, 134, 219, 72, 248, 142, 216, 72, 244, 249, 118, 250, 231, 226, 40, 30, 57, 171, 118, 80, 35, 100, 7, 106, 13, 195, 160, 96, 101, 77, 22, 47, 76, 5, 103, 225, 226, 128, 137, 137, 67, 18, 93, 152, 144, 88, 247, 248, 161, 97, 14, 166, 151, 79, 14, 196, 122, 130, 191, 123, 231, 73, 177, 104, 170, 96, 225, 5, 27, 212, 217, 198, 139, 53, 53, 253, 21, 41, 112, 223, 224, 17, 217, 176, 63, 45, 171, 95, 238, 210, 13, 29, 57, 179, 79, 196, 84, 148, 57, 138, 131, 186, 195, 50, 214, 140, 239, 137, 41, 255, 203, 32, 252, 175, 141, 153, 218, 235, 184, 145, 96, 170, 62, 154, 98, 31, 201, 78, 50, 32, 203, 27, 27, 11, 111, 108, 44, 18, 154, 236, 139, 155, 197, 28, 255, 42, 181, 235, 224, 120, 108, 229, 45, 171, 164, 132, 149, 10, 143, 166, 227, 145, 58, 234, 72, 28, 236, 47, 147, 92, 75, 207, 239, 25, 36, 222, 188, 7, 58, 244, 69, 228, 218, 57, 106, 110, 24, 252, 34, 101, 193, 138, 84, 230, 82, 185, 161, 29, 185, 217, 236, 115, 203, 48, 166, 139, 211, 212, 88, 130, 161, 91, 119, 32, 192, 194, 60, 79, 226, 42, 61, 79, 150, 111, 123, 26, 115, 179, 138, 161, 31, 71, 155, 132, 98, 239, 86, 35, 82, 165, 131, 117, 3, 62, 214, 238, 35, 11, 152, 110, 66, 82, 15, 197, 230, 43, 103, 239, 85, 179, 125, 179, 168, 19, 114, 139, 148, 195, 244, 218, 110, 200, 109, 83, 78, 165, 64, 71, 172, 189, 84, 98, 239, 2, 87, 50, 78, 141, 163, 102, 160, 89, 140, 68, 52, 111, 17, 81, 63, 8, 203, 57, 236, 74, 95, 193, 25, 250, 228, 243, 212, 205, 236, 118, 203, 238, 163, 43, 246, 159, 212, 74, 15, 8, 17, 159, 206, 76, 139, 35, 1, 252, 77, 211, 250, 50, 15, 190, 234, 246, 30, 253, 100, 62, 49, 25, 49, 83, 117, 15, 54, 9, 204, 146, 30, 115, 13, 16, 60, 204, 254, 46, 82, 172, 242, 201, 172, 242, 44, 138, 241, 200, 62, 158, 49, 17, 224, 203, 1, 118, 92, 127, 187, 153, 164, 76, 38, 76, 6, 229, 58, 195, 143, 121, 62, 73, 226, 172, 221, 120, 70, 113, 22, 231, 5, 50, 59, 95, 0, 29, 19, 235, 57, 162, 182, 189, 164, 36, 59, 231, 30, 222, 51, 98, 247, 84, 50, 102, 175, 173, 160, 157, 37, 67, 185, 84, 200, 248, 160, 64, 216, 47, 202, 140, 167, 199, 25, 232, 59, 208, 195, 190, 121, 52, 254, 105, 80, 227, 179, 126, 123, 198, 80, 166, 44, 154, 72, 190, 36, 215, 204, 37, 106, 133, 121, 45, 235, 127, 113, 142, 217, 0, 17, 101, 43, 188, 238, 107, 99, 125, 71, 53, 62, 174, 87, 230, 156, 243, 65, 227, 226, 104, 141, 207, 221, 18, 16, 30, 145, 68, 43, 16, 65, 209, 228, 171, 54, 127, 73, 249, 228, 61, 251, 34, 133, 148, 80, 246, 165, 73, 234, 18, 113, 4, 217, 173, 21, 70, 143, 28, 118, 183, 66, 2, 184, 90, 110, 104, 69, 249, 251, 40, 190, 79, 203, 161, 254, 99, 139, 152, 103, 127, 188, 65, 132, 37, 75, 144, 115, 191, 145, 203, 243, 105, 66, 210, 58, 111, 0, 9, 210, 13, 76, 23, 87, 161, 131, 1, 243, 27, 226, 162, 55, 122, 46, 54, 93, 136, 179, 11, 122, 211, 192, 103, 50, 91, 33, 146, 26, 27, 47, 217, 183, 59, 146, 43, 14, 226, 93, 221, 232, 162, 157, 158, 116, 91, 185, 166, 153, 108, 45, 127, 178, 203, 3, 164, 58, 52, 99, 11, 126, 19, 115, 20, 164, 36, 195, 125, 178, 182, 168, 193, 115, 17, 195, 101, 116, 118, 28, 211, 93, 237, 10, 118, 19, 227, 66, 1, 91, 200, 170, 137, 144, 13, 206, 44, 224, 17, 186, 189, 21, 251, 159, 28, 142, 155, 37, 179, 88, 92, 239, 246, 185, 164, 78, 182, 79, 178, 140, 64, 218, 29, 229, 31, 88, 3, 176, 241, 72, 53, 227, 132, 244, 173, 78, 139, 135, 244, 106, 43, 138, 230, 71, 19, 81, 196, 75, 145, 114, 7, 213, 38, 119, 148, 54, 79, 204, 131, 119, 120, 205, 66, 65, 217, 190, 252, 152, 239, 96, 174, 245, 195, 96, 121, 225, 29, 156, 39, 76, 102, 52, 162, 145, 56, 189, 209, 142, 220, 24, 104, 183, 143, 226, 17, 32, 239, 80, 110, 142, 241, 136, 207, 187, 128, 244, 240, 235, 107, 190, 160, 196, 52, 208, 179, 74, 86, 122, 59, 148, 149, 126, 21, 151, 85, 3, 51, 253, 185, 189, 204, 91, 155, 107, 212, 207, 79, 110, 178, 17, 35, 61, 25, 182, 110, 180, 69, 178, 108, 182, 89, 87, 250, 34, 76, 53, 73, 167, 49, 139, 150, 117, 20, 18, 56, 123, 72, 12, 179, 54, 182, 79, 196, 188, 223, 237, 126, 42, 91, 11, 87, 139, 255, 245, 146, 103, 88, 156, 224, 186, 204, 79, 147, 58, 187, 31, 8, 81, 166, 241, 172, 153, 19, 233, 98, 254, 87, 203, 155, 153, 220, 144, 149, 250, 109, 24, 210, 79, 249, 176, 255, 234, 59, 196, 177, 204, 221, 133, 207, 83, 145, 68, 70, 88, 86, 72, 44, 189, 75, 250, 243, 30, 30, 146, 232, 6, 22, 244, 110, 101, 189, 101, 83, 135, 183, 96, 126, 230, 192, 187, 32, 4, 125, 20, 193, 38, 182, 27, 78, 145, 120, 169, 15, 108, 100, 253, 214, 50, 202, 155, 230, 103, 220, 235, 205, 167, 115, 117, 124, 163, 217, 9, 26, 38, 111, 235, 235, 39, 206, 98, 134, 79, 144, 41, 250, 160, 16, 163, 115, 105, 168, 198, 178, 64, 14, 129, 214, 202, 241, 187, 116, 56, 170, 190, 108, 97, 230, 36, 126, 113, 18, 77, 168, 250, 242, 197, 90, 186, 20, 183, 22, 238, 110, 215, 22, 110, 175, 47, 242, 217, 101, 251, 60, 41, 202, 164, 193, 105, 217, 143, 226, 219, 182, 28, 74, 238, 40, 24, 156, 166, 253, 254, 68, 220, 16, 35, 61, 198, 193, 36, 207, 249, 101, 47, 209, 127, 243, 144, 64, 15, 141, 216, 125, 127, 193, 239, 146, 101, 227, 35, 190, 194, 137, 40, 129, 67, 150, 184, 83, 235, 28, 174, 128, 215, 2, 35, 170, 117, 241, 110, 78, 141, 216, 164, 213, 249, 79, 43, 112, 227, 120, 12, 8, 255, 137, 64, 84, 37, 240, 32, 53, 36, 126, 44, 42, 0, 33, 186, 45, 143, 40, 132, 214, 123, 15, 3, 119, 163, 124, 174, 64, 15, 237, 97, 192, 210, 193, 201, 92, 217, 237, 252, 207, 182, 132, 79, 189, 45, 65, 156, 238, 18, 231, 145, 73, 35, 130, 182, 46, 152, 42, 124, 94, 253, 115, 122, 212, 248, 110, 10, 179, 202, 202, 127, 202, 93, 136, 122, 119, 133, 103, 161, 149, 102, 96, 198, 120, 129, 55, 51, 135, 140, 144, 34, 132, 220, 228, 195, 124, 154, 252, 5, 247, 198, 65, 183, 190, 184, 197, 173, 18, 17, 245, 111, 185, 51, 110, 201, 198, 56, 115, 35, 3, 17, 93, 247, 42, 115, 195, 94, 222, 214, 23, 149, 32, 20, 20, 173, 146, 34, 180, 46, 144, 194, 43, 29, 189, 201, 188, 207, 186, 247, 197, 47, 227, 212, 170, 165, 201, 25, 210, 101, 28, 209, 185, 143, 158, 20, 180, 174, 238, 175, 95, 202, 249, 188, 171, 52, 200, 240, 91, 151, 161, 3, 134, 195, 120, 42, 14, 23, 252, 51, 41, 114, 91, 38, 64, 97, 253, 75, 206, 244, 158, 147, 177, 119, 118, 168, 83, 49, 113, 37, 63, 92, 226, 210, 46, 249, 189, 243, 29, 200, 200, 12, 50, 60, 67, 44, 205, 52, 238, 249, 148, 18, 79, 156, 123, 129, 171, 148, 55, 45, 187, 183, 213, 137, 246, 176, 185, 27, 136, 4, 209, 32, 51, 152, 7, 238, 197, 236, 216, 210, 14, 154, 25, 88, 214, 208, 64, 53, 164, 174, 70, 50, 107, 219, 192, 141, 61, 18, 24, 41, 222, 12, 52, 47, 66, 106, 205, 43, 247, 76, 223, 89, 5, 249, 162, 131, 180, 74, 166, 165, 100, 175, 65, 99, 177, 17, 213, 124, 84, 171, 119, 77, 62, 183, 199, 112, 123, 36, 183, 169, 142, 98, 46, 102, 158, 196, 105, 108, 33, 157, 68, 6, 150, 100, 86, 231, 37, 186, 16, 137, 1, 188, 76, 42, 97, 246, 218, 114, 68, 240, 107, 228, 144, 209, 161, 245, 238, 133, 46, 67, 225, 135, 4, 9, 118, 252, 238, 48, 192, 79, 16, 161, 73, 147, 156, 97, 177, 183, 47, 194, 39, 112, 96, 69, 213, 149, 203, 114, 170, 113, 241, 110, 79, 107, 147, 32, 157, 56, 248, 53, 29, 174, 255, 141, 20, 229, 155, 45, 15, 2, 254, 22, 65, 247, 100, 11, 124, 172, 211, 45, 130, 52, 232, 146, 185, 11, 67, 156, 111, 65, 104, 175, 9, 7, 132, 249, 151, 149, 124, 239, 169, 16, 250, 195, 122, 224, 201, 87, 195, 167, 169, 11, 164, 240, 105, 55, 72, 180, 73, 124, 33, 5, 67, 115, 219, 74, 250, 126, 160, 83, 169, 212, 100, 236, 21, 193, 106, 209, 216, 10, 163, 225, 163, 251, 16, 72, 122, 83, 182, 40, 219, 174, 42, 217, 115, 104, 204, 167, 201, 116, 156, 68, 232, 18, 11, 46, 108, 200, 118, 202, 105, 85, 221, 89, 161, 235, 208, 155, 43, 150, 49, 248, 70, 254, 165, 110, 203, 241, 51, 239, 146, 175, 203, 120, 74, 143, 95, 24, 94, 214, 204, 131, 7, 131, 118, 132, 45, 178, 224, 97, 248, 78, 30, 206, 115, 12, 65, 67, 200, 155, 206, 137, 242, 24, 221, 30, 14, 32, 147, 139, 27, 12, 144, 9, 27, 121, 117, 193, 40, 78, 52, 251, 138, 217, 35, 158, 187, 188, 17, 178, 197, 108, 26, 191, 165, 189, 16, 164, 65, 105, 39, 202, 7, 131, 50, 169, 108, 5, 91, 246, 1, 162, 57, 51, 16, 250, 253, 242, 105, 85, 193, 61, 78, 253, 182, 66, 47, 121, 131, 163, 175, 135, 127, 231, 61, 46, 47, 82, 126, 46, 113, 201, 45, 116, 146, 220, 77, 233, 15, 247, 98, 230, 199, 109, 237, 56, 243, 189, 238, 223, 33, 147, 192, 184, 8, 246, 242, 145, 194, 177, 93, 131, 227, 29, 188, 121, 58, 201, 122, 201, 114, 52, 223, 212, 145, 146, 78, 251, 203, 49, 60, 220, 81, 223, 30, 212, 224, 122, 150, 20, 41, 99, 73, 45, 58, 208, 19, 248, 187, 124, 247, 89, 99, 180, 43, 10, 5, 105, 255, 77, 114, 33, 209, 163, 54, 107, 58, 176, 140, 200, 253, 65, 168, 114, 7, 188, 45, 57, 151, 171, 27, 82, 249, 149, 120, 63, 172, 218, 93, 116, 225, 73, 29, 69, 7, 179, 248, 143, 121, 88, 58, 110, 196, 53, 80, 253, 156, 227, 85, 109, 69, 252, 214, 205, 3, 82, 166, 89, 200, 71, 92, 168, 27, 126, 70, 49, 4, 4, 69, 157, 202, 39, 176, 75, 187, 123, 7, 252, 95, 198, 183, 134, 236, 95, 162, 227, 156, 14, 62, 31, 178, 39, 209, 210, 123, 46, 37, 154, 239, 136, 85, 130, 170, 200, 226, 108, 18, 213, 124, 149, 199, 204, 209, 204, 170, 239, 218, 97, 181, 222, 210, 106, 253, 157, 91, 119, 190, 164, 242, 182, 183, 185, 173, 111, 107, 170, 60, 240, 183, 82, 91, 231, 161, 183, 153, 7, 219, 53, 85, 190, 241, 55, 83, 91, 231, 91, 183, 206, 96, 146, 199, 245, 149, 254, 30, 168, 244, 237, 67, 127, 165, 27, 24, 191, 229, 102, 254, 211, 72, 210, 211, 108, 177, 76, 144, 104, 5, 149, 61, 90, 69, 126, 74, 121, 75, 194, 29, 241, 174, 118, 110, 251, 8, 108, 147, 92, 0, 164, 144, 79, 171, 21, 34, 184, 26, 15, 121, 92, 139, 6, 240, 15, 17, 252, 85, 3, 248, 239, 16, 252, 69, 3, 248, 45, 181, 154, 40, 89, 204, 120, 241, 66, 64, 95, 62, 4, 198, 242, 247, 235, 180, 215, 218, 151, 157, 104, 209, 137, 174, 58, 209, 133, 51, 152, 110, 215, 61, 170, 80, 219, 117, 63, 252, 119, 126, 202, 190, 125, 120, 185, 109, 83, 102, 104, 218, 244, 208, 228, 55, 3, 33, 138, 130, 208, 222, 161, 8, 66, 123, 7, 34, 8, 237, 25, 6, 46, 146, 181, 131, 176, 170, 118, 28, 134, 157, 236, 78, 36, 94, 154, 6, 249, 76, 113, 18, 144, 75, 248, 103, 246, 163, 41, 237, 245, 147, 43, 166, 29, 179, 210, 150, 86, 138, 115, 169, 165, 117, 25, 227, 69, 254, 141, 151, 224, 26, 83, 180, 20, 227, 67, 35, 200, 250, 234, 214, 118, 67, 94, 126, 191, 166, 222, 225, 3, 40, 96, 16, 41, 97, 34, 153, 47, 129, 5, 246, 122, 10, 85, 152, 103, 104, 68, 121, 25, 60, 203, 211, 100, 144, 139, 0, 37, 115, 196, 154, 111, 50, 233, 191, 97, 12, 46, 177, 49, 22, 181, 235, 173, 49, 170, 207, 181, 99, 5, 4, 223, 219, 8, 14, 120, 127, 87, 192, 176, 229, 236, 139, 71, 119, 120, 153, 62, 169, 96, 158, 94, 230, 69, 250, 13, 155, 140, 53, 60, 122, 29, 44, 237, 156, 130, 18, 61, 245, 129, 137, 46, 40, 56, 217, 33, 31, 160, 74, 140, 232, 161, 57, 214, 237, 107, 96, 203, 100, 152, 166, 93, 33, 97, 166, 11, 181, 216, 49, 157, 233, 160, 230, 60, 107, 89, 202, 124, 225, 180, 196, 182, 181, 12, 36, 114, 20, 72, 136, 172, 231, 203, 238, 151, 87, 131, 101, 189, 40, 248, 150, 170, 96, 236, 98, 80, 204, 237, 133, 35, 190, 168, 160, 96, 212, 42, 48, 100, 123, 101, 158, 44, 88, 73, 111, 210, 218, 196, 41, 35, 56, 27, 161, 196, 226, 6, 58, 3, 213, 15, 197, 125, 249, 75, 172, 137, 110, 168, 19, 61, 176, 16, 104, 121, 19, 12, 10, 107, 131, 87, 5, 44, 1, 67, 237, 24, 206, 155, 86, 212, 221, 191, 70, 100, 83, 115, 229, 42, 34, 102, 125, 87, 247, 205, 8, 220, 181, 153, 31, 253, 247, 163, 220, 96, 122, 89, 45, 138, 86, 33, 244, 199, 242, 212, 15, 171, 188, 72, 252, 65, 95, 96, 93, 181, 141, 36, 86, 92, 247, 252, 53, 191, 118, 58, 228, 218, 139, 22, 252, 161, 97, 104, 233, 150, 180, 241, 228, 201, 19, 95, 43, 223, 153, 41, 238, 156, 113, 102, 111, 18, 79, 103, 71, 249, 47, 188, 157, 117, 255, 244, 88, 219, 219, 232, 60, 20, 104, 232, 90, 158, 168, 179, 81, 15, 16, 151, 112, 15, 30, 4, 248, 180, 66, 51, 53, 156, 194, 45, 61, 244, 247, 199, 227, 77, 54, 238, 79, 40, 118, 54, 221, 88, 1, 187, 175, 27, 161, 64, 91, 52, 16, 10, 74, 58, 145, 53, 246, 158, 168, 27, 97, 240, 196, 14, 14, 134, 27, 56, 202, 55, 112, 12, 63, 122, 76, 46, 122, 237, 11, 202, 253, 94, 170, 29, 159, 170, 75, 225, 204, 72, 153, 183, 31, 255, 96, 234, 242, 61, 122, 238, 212, 230, 81, 195, 64, 184, 175, 168, 57, 167, 148, 248, 84, 50, 16, 252, 43, 12, 250, 60, 160, 69, 199, 13, 6, 180, 169, 51, 94, 59, 191, 243, 29, 49, 42, 20, 194, 217, 77, 219, 5, 168, 243, 107, 150, 123, 54, 171, 249, 54, 146, 92, 225, 28, 62, 255, 99, 158, 158, 171, 199, 136, 66, 19, 70, 34, 82, 63, 7, 173, 119, 166, 32, 84, 190, 163, 37, 86, 5, 162, 253, 14, 205, 134, 232, 46, 117, 148, 173, 106, 132, 230, 97, 24, 205, 195, 21, 208, 124, 23, 70, 243, 221, 10, 104, 216, 136, 5, 241, 108, 109, 27, 68, 40, 240, 194, 202, 24, 45, 49, 165, 54, 219, 24, 200, 82, 182, 5, 178, 57, 97, 52, 223, 133, 201, 220, 164, 100, 134, 51, 57, 29, 156, 96, 169, 35, 49, 156, 222, 9, 161, 112, 70, 53, 156, 243, 9, 161, 112, 70, 180, 38, 17, 20, 194, 17, 24, 77, 185, 93, 88, 241, 73, 88, 71, 100, 5, 247, 242, 236, 60, 41, 170, 167, 89, 127, 47, 159, 45, 142, 114, 80, 171, 74, 120, 219, 53, 94, 44, 0, 233, 181, 154, 37, 112, 7, 245, 142, 1, 251, 152, 168, 176, 14, 72, 91, 152, 175, 86, 241, 175, 31, 173, 0, 171, 43, 212, 228, 181, 112, 207, 195, 185, 45, 55, 171, 38, 121, 74, 57, 135, 249, 67, 98, 99, 52, 79, 163, 177, 122, 151, 164, 131, 133, 220, 37, 231, 31, 34, 241, 44, 56, 50, 226, 241, 65, 211, 197, 190, 102, 188, 91, 194, 48, 74, 82, 152, 101, 222, 213, 99, 210, 89, 187, 119, 180, 59, 206, 220, 206, 43, 241, 249, 46, 102, 81, 102, 213, 206, 171, 81, 82, 240, 66, 223, 36, 110, 118, 180, 226, 249, 91, 76, 137, 232, 124, 27, 33, 252, 27, 51, 187, 106, 60, 184, 21, 63, 54, 4, 17, 196, 41, 153, 129, 231, 108, 177, 47, 22, 85, 14, 57, 51, 234, 204, 92, 58, 77, 178, 18, 118, 179, 100, 201, 5, 47, 59, 210, 115, 99, 147, 174, 247, 241, 211, 134, 157, 167, 45, 249, 187, 79, 96, 130, 120, 249, 162, 242, 164, 175, 33, 213, 254, 92, 219, 91, 209, 151, 103, 203, 10, 114, 195, 170, 172, 130, 49, 184, 64, 228, 149, 123, 208, 145, 37, 14, 7, 198, 198, 223, 9, 124, 111, 151, 25, 255, 80, 15, 228, 214, 185, 229, 56, 17, 201, 28, 43, 248, 220, 15, 149, 118, 218, 212, 254, 109, 215, 234, 37, 217, 17, 228, 99, 0, 134, 77, 204, 119, 229, 38, 203, 173, 61, 180, 26, 30, 163, 71, 14, 16, 225, 91, 179, 110, 210, 22, 100, 55, 77, 47, 109, 220, 127, 219, 181, 104, 34, 189, 244, 145, 187, 164, 151, 48, 186, 54, 43, 33, 195, 102, 53, 172, 232, 241, 12, 167, 194, 66, 252, 104, 123, 244, 40, 33, 107, 75, 208, 53, 53, 174, 181, 219, 82, 240, 80, 169, 205, 243, 246, 43, 163, 89, 104, 243, 22, 78, 137, 28, 154, 104, 74, 144, 214, 40, 172, 108, 165, 98, 165, 169, 21, 72, 255, 180, 6, 106, 65, 208, 64, 108, 123, 32, 96, 97, 174, 213, 56, 58, 3, 162, 161, 79, 122, 251, 96, 51, 170, 61, 244, 61, 12, 80, 181, 189, 26, 45, 249, 33, 159, 109, 218, 119, 102, 65, 225, 250, 250, 253, 215, 207, 224, 254, 250, 74, 34, 55, 23, 216, 19, 187, 214, 151, 217, 220, 122, 11, 216, 192, 130, 233, 181, 52, 48, 93, 223, 220, 134, 164, 27, 132, 104, 80, 109, 102, 196, 16, 14, 184, 249, 4, 83, 129, 234, 85, 45, 191, 36, 88, 57, 240, 93, 109, 138, 229, 72, 162, 150, 214, 91, 237, 214, 58, 80, 191, 113, 185, 14, 93, 19, 223, 23, 232, 251, 21, 250, 126, 177, 222, 90, 227, 98, 202, 79, 94, 208, 38, 182, 87, 110, 98, 141, 190, 145, 193, 9, 194, 183, 58, 98, 116, 229, 251, 183, 30, 81, 113, 108, 20, 58, 111, 65, 236, 204, 103, 141, 70, 224, 219, 206, 85, 246, 156, 168, 27, 200, 119, 173, 115, 36, 15, 227, 88, 162, 212, 226, 57, 159, 214, 218, 10, 125, 20, 38, 205, 34, 89, 236, 152, 29, 229, 69, 53, 138, 179, 126, 123, 99, 99, 131, 185, 59, 229, 221, 250, 117, 106, 59, 63, 35, 13, 41, 222, 235, 188, 63, 159, 36, 109, 45, 127, 208, 174, 57, 185, 167, 142, 224, 4, 208, 241, 35, 235, 104, 91, 46, 89, 17, 193, 152, 224, 168, 53, 43, 20, 135, 234, 211, 110, 87, 96, 85, 213, 97, 7, 238, 209, 134, 54, 251, 109, 94, 222, 225, 8, 212, 218, 7, 62, 35, 111, 189, 181, 36, 47, 226, 33, 115, 154, 177, 133, 202, 153, 99, 174, 121, 103, 92, 89, 112, 215, 209, 241, 47, 197, 64, 119, 99, 95, 174, 29, 123, 179, 159, 119, 62, 153, 60, 114, 170, 30, 21, 113, 86, 194, 123, 214, 178, 202, 169, 175, 227, 145, 154, 96, 2, 55, 20, 136, 35, 224, 179, 212, 104, 2, 30, 12, 124, 186, 168, 146, 87, 202, 224, 52, 51, 106, 15, 52, 2, 217, 227, 63, 163, 211, 249, 96, 144, 20, 59, 184, 201, 31, 121, 17, 111, 207, 137, 198, 76, 163, 59, 232, 187, 23, 76, 196, 56, 20, 179, 46, 22, 195, 119, 77, 23, 181, 28, 9, 180, 130, 133, 166, 2, 0, 199, 155, 124, 147, 154, 25, 9, 55, 112, 115, 249, 188, 92, 151, 240, 164, 206, 38, 27, 77, 144, 156, 186, 91, 57, 175, 209, 218, 209, 205, 109, 8, 28, 40, 25, 212, 82, 29, 172, 135, 18, 57, 101, 12, 131, 223, 17, 41, 96, 196, 11, 32, 49, 140, 116, 97, 48, 144, 118, 22, 48, 220, 211, 108, 65, 19, 47, 206, 35, 247, 45, 41, 40, 110, 131, 223, 63, 206, 211, 73, 191, 29, 119, 78, 59, 189, 187, 183, 104, 227, 148, 159, 191, 111, 244, 10, 245, 77, 50, 50, 80, 19, 185, 91, 15, 205, 142, 14, 57, 102, 49, 159, 132, 204, 40, 88, 22, 129, 119, 139, 247, 247, 48, 249, 99, 127, 58, 155, 200, 86, 248, 84, 169, 150, 214, 217, 100, 22, 75, 197, 210, 243, 47, 69, 47, 135, 6, 228, 82, 62, 56, 189, 131, 118, 227, 78, 116, 218, 168, 93, 133, 220, 195, 94, 242, 122, 79, 207, 115, 146, 138, 243, 198, 4, 75, 4, 131, 159, 236, 3, 74, 59, 209, 71, 144, 12, 22, 255, 64, 155, 251, 240, 82, 52, 38, 33, 6, 109, 28, 118, 109, 99, 61, 15, 226, 218, 100, 55, 194, 229, 108, 146, 246, 210, 234, 89, 50, 19, 123, 148, 78, 229, 125, 53, 102, 114, 160, 32, 108, 92, 241, 185, 117, 123, 94, 121, 29, 207, 126, 17, 77, 169, 209, 140, 245, 129, 126, 123, 0, 17, 158, 211, 21, 240, 156, 54, 26, 77, 242, 142, 111, 111, 79, 169, 120, 42, 249, 185, 121, 191, 143, 44, 106, 79, 111, 218, 113, 27, 209, 150, 175, 231, 46, 137, 183, 16, 253, 90, 209, 134, 91, 5, 185, 254, 221, 113, 110, 138, 199, 41, 12, 162, 81, 254, 72, 59, 29, 188, 156, 185, 28, 218, 187, 82, 40, 234, 147, 96, 55, 51, 82, 129, 49, 17, 163, 33, 5, 82, 68, 246, 98, 104, 2, 182, 105, 21, 172, 91, 24, 171, 208, 223, 101, 190, 130, 149, 88, 132, 123, 239, 254, 229, 199, 43, 118, 79, 113, 107, 158, 222, 243, 240, 52, 148, 163, 228, 47, 136, 148, 20, 171, 150, 237, 12, 148, 186, 131, 150, 179, 206, 51, 36, 6, 137, 158, 136, 106, 70, 64, 190, 196, 6, 158, 255, 75, 143, 128, 100, 55, 244, 163, 41, 199, 37, 35, 5, 7, 60, 140, 228, 60, 33, 204, 123, 243, 203, 235, 247, 63, 254, 126, 244, 252, 176, 125, 154, 86, 37, 61, 233, 3, 37, 209, 122, 244, 119, 216, 143, 18, 61, 160, 81, 210, 243, 163, 247, 63, 238, 31, 181, 251, 113, 21, 195, 123, 66, 225, 6, 7, 189, 127, 227, 34, 47, 192, 133, 226, 165, 178, 170, 58, 91, 90, 242, 11, 144, 163, 199, 143, 225, 117, 172, 240, 24, 118, 234, 136, 184, 4, 80, 29, 67, 213, 147, 232, 195, 46, 7, 37, 45, 190, 188, 251, 22, 85, 63, 81, 203, 95, 243, 26, 60, 253, 179, 73, 154, 119, 157, 164, 216, 100, 21, 153, 147, 4, 105, 138, 25, 56, 74, 70, 236, 116, 4, 204, 167, 47, 120, 217, 60, 72, 93, 91, 252, 150, 91, 162, 96, 92, 251, 102, 238, 8, 136, 220, 159, 226, 0, 57, 163, 227, 112, 22, 195, 37, 7, 195, 34, 78, 51, 153, 209, 204, 171, 120, 34, 34, 163, 235, 104, 151, 99, 129, 87, 117, 207, 171, 100, 159, 84, 194, 244, 74, 82, 213, 17, 85, 113, 251, 148, 188, 90, 6, 98, 103, 13, 218, 214, 207, 25, 239, 210, 158, 231, 24, 188, 44, 215, 17, 190, 192, 237, 196, 238, 114, 55, 163, 120, 136, 55, 51, 202, 250, 252, 157, 188, 226, 50, 14, 149, 116, 215, 125, 132, 67, 185, 140, 225, 252, 186, 21, 172, 79, 135, 41, 148, 114, 50, 53, 172, 166, 183, 108, 180, 67, 19, 241, 241, 1, 54, 89, 122, 123, 230, 166, 238, 9, 234, 5, 98, 57, 233, 139, 184, 202, 32, 41, 14, 230, 213, 219, 60, 205, 170, 196, 10, 175, 241, 129, 59, 212, 178, 96, 125, 199, 180, 91, 127, 157, 139, 239, 227, 242, 106, 77, 5, 98, 214, 203, 50, 185, 17, 122, 15, 39, 255, 59, 138, 247, 29, 139, 104, 180, 85, 213, 185, 149, 16, 80, 168, 31, 239, 227, 217, 108, 178, 16, 18, 79, 174, 226, 144, 120, 151, 144, 144, 207, 108, 72, 254, 146, 78, 235, 26, 7, 229, 50, 25, 46, 194, 122, 113, 45, 247, 228, 118, 87, 213, 143, 253, 172, 87, 240, 217, 198, 210, 7, 221, 123, 170, 92, 178, 150, 17, 130, 245, 93, 46, 76, 74, 52, 165, 209, 16, 205, 62, 162, 59, 5, 154, 171, 29, 73, 102, 106, 29, 35, 53, 93, 85, 35, 143, 61, 26, 71, 1, 142, 55, 79, 196, 93, 56, 122, 226, 50, 98, 75, 31, 104, 110, 170, 139, 1, 124, 115, 52, 81, 231, 173, 160, 58, 203, 179, 18, 206, 148, 103, 52, 148, 238, 228, 231, 229, 175, 248, 33, 126, 13, 98, 221, 49, 97, 117, 139, 91, 10, 85, 77, 193, 224, 190, 253, 205, 125, 140, 123, 104, 154, 177, 122, 104, 133, 44, 150, 53, 86, 167, 151, 172, 77, 209, 127, 90, 228, 237, 88, 191, 137, 46, 235, 166, 119, 144, 5, 199, 0, 168, 27, 59, 164, 79, 215, 68, 210, 86, 21, 233, 149, 143, 45, 251, 62, 74, 90, 196, 11, 220, 137, 213, 239, 146, 165, 78, 124, 25, 138, 50, 239, 217, 9, 232, 209, 150, 209, 115, 244, 224, 177, 61, 194, 217, 137, 239, 188, 204, 157, 116, 128, 80, 164, 111, 127, 18, 87, 77, 225, 219, 171, 112, 195, 89, 212, 149, 164, 147, 40, 205, 13, 144, 211, 140, 203, 59, 245, 87, 167, 241, 226, 52, 121, 65, 156, 2, 49, 111, 114, 88, 114, 61, 136, 40, 90, 211, 32, 214, 21, 117, 6, 147, 227, 44, 120, 110, 167, 67, 208, 106, 228, 184, 39, 185, 107, 158, 160, 9, 79, 155, 5, 67, 58, 210, 66, 9, 244, 18, 105, 17, 130, 179, 212, 200, 241, 38, 12, 104, 27, 207, 177, 191, 100, 105, 213, 236, 28, 5, 110, 218, 154, 168, 95, 174, 224, 36, 96, 240, 37, 147, 189, 229, 182, 104, 6, 57, 182, 46, 101, 3, 169, 198, 146, 26, 54, 243, 182, 14, 249, 216, 58, 77, 148, 243, 36, 82, 192, 59, 160, 61, 190, 185, 131, 224, 229, 16, 73, 236, 243, 179, 81, 76, 22, 212, 27, 67, 152, 237, 232, 68, 88, 144, 53, 116, 205, 236, 236, 204, 207, 203, 232, 183, 166, 232, 107, 119, 202, 69, 2, 178, 124, 230, 181, 3, 121, 77, 191, 156, 113, 177, 62, 14, 28, 85, 188, 59, 63, 61, 205, 76, 172, 235, 174, 33, 96, 253, 246, 120, 245, 59, 105, 182, 31, 114, 243, 217, 51, 205, 207, 122, 47, 95, 147, 160, 59, 47, 155, 197, 151, 10, 209, 128, 24, 225, 14, 37, 71, 49, 200, 13, 35, 149, 29, 42, 19, 141, 195, 22, 60, 118, 206, 112, 226, 78, 45, 137, 55, 208, 204, 179, 239, 56, 20, 48, 188, 142, 155, 225, 77, 231, 61, 244, 71, 76, 62, 27, 135, 152, 214, 192, 58, 97, 22, 223, 165, 129, 75, 51, 106, 223, 188, 70, 143, 3, 150, 60, 137, 185, 37, 16, 42, 191, 18, 102, 197, 45, 126, 133, 95, 219, 219, 31, 38, 47, 97, 35, 35, 218, 71, 134, 222, 106, 227, 7, 166, 106, 59, 203, 172, 63, 106, 214, 50, 254, 183, 68, 236, 155, 86, 204, 154, 127, 142, 140, 138, 192, 173, 111, 57, 245, 73, 90, 40, 38, 180, 39, 3, 106, 240, 105, 231, 180, 217, 82, 99, 182, 246, 159, 9, 195, 63, 97, 68, 70, 172, 215, 119, 145, 136, 169, 167, 117, 83, 137, 117, 237, 152, 154, 78, 68, 113, 221, 248, 11, 217, 50, 147, 11, 26, 114, 21, 4, 11, 100, 55, 151, 19, 205, 92, 227, 51, 80, 254, 226, 91, 116, 26, 112, 89, 208, 35, 66, 166, 226, 54, 157, 127, 67, 174, 248, 165, 201, 160, 95, 248, 254, 55, 117, 16, 120, 166, 47, 174, 251, 183, 86, 147, 184, 24, 2, 127, 142, 101, 151, 180, 235, 183, 52, 155, 98, 39, 105, 0, 145, 157, 40, 89, 146, 215, 81, 85, 60, 58, 185, 98, 82, 231, 60, 79, 251, 112, 54, 4, 48, 126, 249, 122, 125, 211, 92, 79, 141, 74, 112, 135, 129, 204, 45, 100, 5, 233, 86, 86, 130, 250, 157, 230, 29, 37, 97, 220, 109, 178, 247, 221, 167, 164, 214, 113, 8, 103, 61, 37, 118, 3, 84, 65, 83, 202, 56, 153, 170, 171, 238, 113, 82, 74, 190, 146, 225, 92, 221, 218, 40, 92, 40, 14, 108, 189, 160, 128, 191, 122, 197, 120, 154, 76, 219, 183, 60, 190, 226, 67, 221, 239, 160, 19, 35, 73, 35, 123, 171, 188, 77, 26, 235, 81, 202, 247, 59, 108, 25, 65, 71, 23, 93, 234, 247, 52, 90, 9, 30, 85, 107, 211, 174, 133, 220, 57, 197, 24, 44, 42, 196, 178, 113, 36, 194, 186, 225, 238, 35, 175, 211, 139, 151, 155, 249, 115, 113, 52, 41, 150, 182, 126, 205, 93, 56, 191, 77, 223, 45, 199, 84, 177, 98, 109, 53, 94, 200, 214, 239, 164, 255, 222, 78, 91, 241, 157, 148, 52, 36, 199, 158, 229, 44, 41, 199, 110, 48, 167, 47, 95, 13, 134, 97, 177, 14, 194, 194, 11, 116, 26, 67, 125, 180, 147, 144, 93, 246, 193, 212, 41, 207, 189, 76, 226, 33, 158, 48, 191, 19, 243, 165, 89, 126, 195, 188, 51, 41, 27, 245, 54, 13, 237, 165, 55, 91, 215, 72, 240, 166, 251, 7, 193, 88, 192, 55, 67, 187, 99, 136, 38, 254, 115, 119, 106, 38, 77, 197, 222, 137, 153, 223, 223, 193, 71, 158, 59, 51, 98, 0, 140, 252, 170, 181, 68, 206, 34, 6, 97, 38, 57, 232, 60, 189, 47, 29, 79, 15, 56, 43, 191, 124, 255, 156, 9, 217, 228, 112, 217, 155, 68, 73, 71, 204, 219, 65, 156, 80, 207, 32, 105, 123, 226, 206, 78, 116, 214, 128, 151, 192, 145, 151, 118, 183, 77, 238, 145, 59, 109, 103, 235, 235, 39, 218, 128, 137, 220, 81, 125, 22, 228, 183, 36, 30, 179, 57, 247, 125, 220, 239, 11, 29, 195, 35, 119, 24, 133, 223, 202, 112, 168, 222, 84, 32, 49, 176, 2, 255, 235, 53, 58, 81, 75, 130, 192, 86, 101, 214, 142, 217, 252, 126, 40, 71, 109, 194, 198, 134, 113, 177, 72, 233, 149, 68, 193, 205, 115, 146, 22, 89, 229, 198, 205, 210, 14, 157, 147, 59, 147, 40, 50, 117, 123, 145, 227, 1, 65, 19, 108, 70, 125, 207, 221, 18, 73, 143, 156, 247, 58, 252, 238, 19, 89, 81, 157, 191, 247, 242, 190, 55, 73, 226, 162, 253, 145, 25, 207, 27, 249, 28, 172, 15, 52, 28, 100, 37, 135, 215, 204, 164, 252, 243, 159, 36, 81, 124, 236, 51, 109, 172, 146, 79, 34, 198, 162, 169, 207, 193, 206, 80, 203, 245, 194, 236, 190, 86, 195, 247, 18, 17, 60, 12, 162, 29, 91, 168, 215, 188, 140, 31, 197, 229, 39, 225, 58, 107, 231, 115, 176, 220, 219, 236, 93, 243, 155, 53, 226, 99, 246, 255, 7, 98, 234, 46, 220 }; + static const unsigned char data[] = { 120, 156, 237, 125, 105, 119, 219, 70, 178, 232, 247, 252, 10, 152, 231, 221, 152, 26, 145, 186, 146, 236, 108, 178, 229, 28, 71, 94, 162, 27, 111, 207, 114, 38, 147, 209, 243, 245, 129, 72, 144, 132, 72, 2, 12, 0, 74, 162, 98, 253, 247, 215, 213, 107, 85, 47, 32, 40, 81, 182, 39, 51, 60, 199, 22, 217, 168, 174, 174, 174, 174, 170, 174, 170, 94, 112, 22, 23, 81, 89, 245, 63, 60, 46, 138, 120, 241, 33, 205, 250, 201, 197, 235, 65, 180, 31, 241, 223, 135, 226, 231, 131, 175, 206, 36, 212, 81, 85, 164, 217, 240, 67, 57, 63, 41, 249, 55, 6, 104, 23, 25, 224, 223, 146, 120, 252, 50, 158, 49, 24, 249, 13, 227, 201, 103, 135, 85, 82, 196, 85, 154, 103, 28, 9, 250, 253, 224, 171, 193, 60, 235, 241, 39, 47, 210, 178, 106, 111, 68, 127, 126, 21, 177, 79, 53, 74, 203, 173, 73, 146, 13, 171, 17, 171, 178, 253, 224, 171, 171, 175, 224, 193, 36, 169, 56, 220, 155, 34, 175, 114, 246, 0, 240, 191, 62, 57, 77, 122, 213, 135, 94, 145, 196, 85, 210, 206, 230, 147, 201, 198, 3, 6, 170, 193, 182, 76, 71, 157, 206, 83, 192, 211, 60, 205, 8, 20, 20, 80, 144, 217, 188, 28, 17, 16, 40, 160, 32, 229, 36, 237, 37, 4, 134, 151, 88, 64, 121, 81, 81, 24, 86, 0, 32, 47, 227, 113, 114, 144, 103, 140, 191, 115, 198, 150, 147, 73, 210, 134, 74, 29, 83, 117, 3, 120, 161, 153, 246, 54, 233, 229, 69, 95, 179, 173, 72, 170, 121, 145, 133, 217, 114, 245, 149, 139, 95, 160, 232, 68, 127, 94, 109, 160, 209, 248, 57, 46, 89, 115, 179, 164, 168, 22, 237, 188, 19, 205, 172, 22, 102, 17, 99, 85, 78, 72, 121, 151, 255, 148, 231, 147, 36, 206, 218, 103, 22, 240, 157, 59, 103, 22, 228, 171, 249, 244, 36, 41, 28, 192, 77, 10, 119, 48, 74, 122, 99, 209, 143, 131, 60, 41, 122, 156, 92, 93, 39, 29, 68, 237, 179, 104, 127, 127, 63, 154, 179, 193, 28, 164, 89, 210, 143, 62, 126, 140, 68, 17, 239, 47, 7, 131, 207, 187, 81, 145, 159, 63, 45, 138, 188, 104, 239, 236, 118, 88, 251, 66, 144, 25, 174, 78, 212, 202, 121, 3, 173, 13, 139, 196, 23, 92, 248, 76, 115, 12, 47, 43, 61, 204, 170, 100, 200, 41, 127, 128, 136, 120, 200, 68, 212, 180, 38, 59, 179, 253, 192, 30, 146, 151, 113, 53, 250, 48, 77, 25, 131, 58, 209, 246, 197, 206, 0, 127, 104, 243, 71, 241, 52, 249, 123, 60, 153, 39, 255, 76, 138, 188, 125, 209, 137, 22, 22, 167, 46, 120, 47, 23, 208, 225, 246, 69, 116, 135, 125, 191, 136, 190, 254, 154, 21, 192, 215, 5, 69, 246, 60, 169, 94, 38, 213, 40, 239, 183, 95, 119, 162, 55, 10, 209, 170, 31, 209, 27, 80, 106, 64, 204, 153, 33, 134, 166, 253, 122, 227, 248, 205, 123, 195, 14, 241, 216, 30, 22, 93, 74, 71, 70, 118, 71, 131, 26, 52, 119, 14, 203, 131, 120, 50, 137, 97, 204, 161, 242, 134, 119, 56, 127, 232, 68, 213, 98, 150, 228, 3, 222, 192, 6, 97, 57, 148, 16, 70, 28, 106, 145, 254, 37, 89, 180, 227, 98, 56, 159, 38, 89, 165, 7, 152, 117, 13, 112, 177, 174, 73, 148, 10, 130, 96, 21, 32, 172, 35, 45, 97, 3, 91, 208, 59, 84, 184, 152, 158, 228, 147, 150, 61, 0, 194, 224, 49, 130, 153, 180, 117, 162, 41, 31, 14, 44, 200, 170, 41, 99, 244, 24, 178, 29, 211, 101, 81, 131, 145, 102, 6, 147, 99, 2, 177, 74, 37, 238, 13, 51, 68, 170, 136, 85, 232, 49, 38, 62, 147, 148, 180, 5, 154, 78, 196, 234, 110, 16, 86, 203, 161, 212, 168, 188, 204, 190, 255, 3, 210, 29, 3, 74, 216, 163, 138, 169, 56, 207, 146, 94, 154, 148, 218, 238, 40, 62, 176, 65, 143, 231, 147, 10, 149, 227, 193, 56, 96, 212, 51, 176, 173, 158, 121, 108, 104, 62, 160, 34, 134, 165, 90, 82, 226, 34, 23, 181, 175, 136, 136, 33, 146, 14, 54, 48, 22, 108, 51, 182, 59, 209, 19, 102, 37, 167, 179, 116, 146, 60, 46, 134, 237, 157, 78, 116, 160, 250, 125, 133, 123, 127, 64, 186, 141, 231, 213, 118, 153, 196, 69, 111, 244, 116, 146, 192, 48, 71, 184, 159, 175, 177, 50, 193, 196, 135, 6, 146, 73, 3, 123, 218, 110, 191, 150, 114, 177, 17, 61, 122, 244, 136, 25, 27, 195, 8, 14, 177, 239, 51, 64, 221, 29, 131, 7, 176, 56, 50, 246, 40, 218, 137, 126, 68, 70, 77, 3, 28, 239, 188, 223, 136, 246, 148, 5, 131, 86, 178, 232, 209, 62, 16, 83, 219, 200, 216, 130, 71, 36, 141, 89, 251, 153, 120, 156, 76, 202, 4, 241, 25, 158, 64, 31, 54, 213, 115, 133, 98, 28, 61, 196, 24, 20, 236, 182, 61, 138, 76, 171, 227, 222, 56, 233, 115, 110, 51, 99, 132, 7, 113, 192, 84, 160, 253, 32, 2, 84, 172, 13, 246, 101, 115, 211, 182, 127, 128, 226, 245, 241, 248, 61, 103, 34, 25, 36, 218, 54, 234, 243, 216, 16, 42, 200, 184, 178, 59, 213, 176, 221, 49, 204, 162, 175, 193, 112, 223, 152, 2, 44, 133, 48, 38, 182, 24, 30, 85, 204, 223, 234, 41, 97, 156, 112, 159, 194, 47, 146, 94, 115, 244, 16, 27, 35, 164, 26, 247, 190, 103, 83, 89, 39, 186, 203, 219, 80, 158, 214, 93, 36, 192, 131, 34, 159, 242, 86, 253, 2, 184, 203, 4, 208, 136, 221, 238, 123, 35, 117, 178, 47, 196, 124, 97, 133, 234, 68, 158, 78, 116, 76, 123, 27, 46, 15, 94, 196, 101, 245, 47, 172, 142, 128, 186, 27, 5, 245, 77, 61, 223, 112, 84, 75, 87, 226, 82, 42, 192, 31, 218, 218, 73, 116, 16, 0, 125, 186, 219, 88, 223, 64, 251, 217, 223, 110, 247, 211, 170, 91, 109, 179, 159, 92, 219, 176, 188, 221, 146, 198, 77, 76, 19, 94, 173, 123, 16, 110, 128, 41, 31, 25, 58, 159, 162, 50, 141, 124, 224, 103, 185, 163, 41, 208, 193, 141, 7, 228, 121, 189, 182, 216, 141, 18, 73, 37, 28, 118, 109, 0, 226, 236, 53, 236, 192, 211, 179, 164, 88, 180, 1, 235, 9, 147, 228, 65, 182, 70, 245, 247, 250, 113, 219, 13, 6, 114, 6, 177, 29, 184, 145, 91, 9, 80, 119, 151, 186, 103, 218, 19, 54, 68, 7, 253, 97, 226, 169, 176, 6, 112, 21, 211, 149, 119, 97, 11, 132, 237, 14, 51, 59, 103, 121, 218, 87, 70, 153, 43, 26, 55, 61, 145, 208, 180, 192, 4, 103, 212, 205, 167, 136, 119, 200, 128, 26, 250, 152, 119, 217, 225, 186, 217, 137, 198, 236, 203, 70, 80, 33, 7, 49, 19, 198, 90, 165, 100, 62, 93, 18, 82, 75, 49, 252, 66, 106, 60, 66, 16, 208, 198, 221, 6, 131, 184, 214, 161, 219, 89, 113, 232, 60, 83, 41, 30, 186, 160, 46, 113, 118, 40, 45, 34, 131, 225, 81, 157, 163, 124, 154, 124, 177, 154, 83, 50, 226, 254, 210, 138, 115, 83, 189, 17, 90, 81, 163, 54, 82, 177, 252, 122, 195, 199, 254, 54, 212, 102, 157, 227, 246, 169, 180, 6, 152, 209, 84, 105, 158, 229, 197, 211, 184, 55, 250, 98, 245, 102, 32, 232, 251, 203, 170, 78, 67, 181, 169, 213, 12, 69, 145, 61, 182, 47, 227, 217, 23, 59, 174, 211, 120, 246, 133, 142, 41, 212, 124, 204, 106, 190, 74, 206, 159, 36, 89, 153, 136, 88, 2, 242, 11, 235, 26, 115, 168, 204, 250, 63, 75, 250, 60, 143, 106, 39, 194, 154, 72, 1, 124, 126, 205, 202, 120, 144, 188, 153, 87, 210, 177, 44, 219, 143, 57, 28, 66, 93, 47, 55, 143, 67, 214, 20, 4, 231, 54, 140, 233, 26, 7, 253, 83, 217, 82, 198, 138, 198, 254, 7, 231, 157, 50, 168, 183, 193, 191, 53, 27, 195, 53, 243, 208, 101, 158, 228, 69, 83, 6, 190, 77, 250, 243, 222, 151, 235, 194, 21, 156, 188, 91, 183, 90, 40, 145, 8, 63, 227, 94, 111, 62, 157, 79, 98, 146, 97, 246, 89, 53, 108, 101, 80, 37, 18, 50, 239, 132, 66, 230, 64, 122, 200, 230, 204, 183, 200, 160, 44, 203, 183, 184, 116, 48, 75, 182, 185, 249, 30, 153, 36, 155, 10, 205, 130, 55, 69, 82, 66, 22, 98, 223, 14, 167, 224, 211, 32, 129, 169, 232, 243, 219, 223, 32, 117, 239, 31, 120, 225, 16, 57, 212, 75, 37, 64, 155, 155, 254, 7, 39, 69, 18, 143, 221, 71, 87, 95, 133, 127, 113, 217, 82, 173, 218, 30, 115, 104, 60, 176, 129, 95, 194, 162, 48, 107, 40, 75, 140, 128, 182, 209, 131, 230, 142, 9, 145, 93, 191, 185, 148, 58, 111, 27, 136, 53, 24, 203, 245, 170, 171, 215, 86, 6, 115, 87, 26, 99, 112, 94, 17, 253, 246, 89, 70, 108, 97, 55, 60, 41, 207, 149, 81, 6, 45, 237, 219, 116, 56, 170, 254, 99, 110, 247, 221, 236, 245, 95, 203, 232, 118, 187, 107, 52, 186, 193, 52, 182, 34, 239, 83, 218, 220, 110, 247, 211, 216, 220, 21, 173, 174, 151, 71, 95, 156, 209, 21, 234, 127, 107, 150, 151, 163, 255, 178, 205, 47, 39, 241, 22, 108, 112, 0, 175, 47, 253, 146, 102, 253, 246, 140, 49, 44, 237, 197, 85, 178, 162, 1, 54, 203, 113, 218, 16, 175, 55, 247, 194, 136, 11, 13, 160, 166, 185, 177, 249, 69, 53, 174, 19, 162, 91, 219, 112, 154, 70, 225, 28, 68, 133, 217, 212, 220, 56, 89, 75, 77, 33, 143, 187, 69, 45, 111, 218, 82, 173, 189, 113, 8, 207, 162, 16, 162, 213, 55, 224, 124, 237, 231, 63, 163, 126, 139, 163, 190, 100, 104, 131, 249, 104, 186, 168, 186, 116, 41, 245, 32, 159, 45, 126, 75, 217, 40, 101, 237, 138, 17, 158, 192, 82, 31, 251, 194, 254, 36, 25, 108, 136, 114, 182, 0, 221, 112, 132, 1, 164, 72, 152, 105, 79, 207, 146, 119, 188, 65, 2, 45, 104, 64, 176, 124, 31, 170, 85, 225, 97, 180, 205, 216, 108, 182, 252, 197, 23, 109, 177, 188, 78, 225, 58, 182, 71, 226, 251, 236, 209, 173, 131, 54, 6, 147, 51, 195, 132, 31, 1, 135, 8, 221, 156, 103, 214, 10, 49, 34, 92, 84, 88, 74, 247, 145, 224, 124, 3, 178, 67, 116, 75, 12, 126, 178, 159, 242, 17, 229, 227, 74, 246, 15, 254, 8, 240, 203, 155, 212, 45, 155, 110, 51, 92, 184, 211, 105, 22, 79, 80, 175, 161, 189, 165, 125, 102, 64, 55, 25, 40, 94, 157, 118, 183, 151, 207, 51, 181, 251, 87, 195, 11, 218, 186, 124, 96, 58, 210, 91, 230, 187, 125, 161, 22, 223, 87, 9, 35, 246, 16, 228, 237, 235, 175, 225, 255, 135, 178, 104, 83, 224, 163, 251, 48, 196, 232, 226, 231, 198, 249, 134, 15, 151, 90, 246, 159, 247, 225, 249, 136, 217, 22, 166, 220, 252, 9, 196, 28, 158, 165, 40, 142, 154, 59, 90, 14, 99, 94, 31, 87, 249, 123, 62, 19, 0, 144, 229, 123, 146, 41, 94, 125, 250, 201, 36, 97, 54, 154, 87, 180, 92, 98, 134, 193, 118, 64, 171, 220, 46, 225, 148, 226, 194, 192, 62, 145, 47, 172, 99, 118, 54, 163, 202, 237, 146, 80, 199, 176, 209, 124, 237, 155, 0, 39, 19, 102, 194, 249, 212, 90, 74, 83, 176, 253, 41, 12, 230, 114, 187, 51, 246, 25, 29, 210, 231, 107, 24, 160, 181, 219, 154, 213, 109, 136, 51, 244, 215, 178, 41, 205, 108, 8, 202, 60, 113, 106, 156, 169, 89, 236, 111, 138, 206, 2, 174, 147, 71, 98, 14, 179, 222, 100, 222, 79, 202, 118, 104, 27, 13, 72, 208, 170, 2, 35, 183, 208, 123, 92, 168, 240, 46, 57, 20, 22, 171, 141, 114, 102, 36, 240, 166, 30, 45, 80, 6, 171, 220, 116, 106, 109, 46, 205, 48, 7, 214, 185, 249, 84, 90, 19, 233, 31, 217, 174, 17, 221, 194, 111, 177, 21, 6, 200, 239, 22, 209, 64, 92, 39, 60, 151, 172, 210, 31, 240, 83, 30, 98, 36, 229, 6, 236, 199, 149, 216, 99, 61, 78, 65, 120, 50, 60, 118, 98, 143, 118, 210, 23, 163, 135, 7, 210, 236, 11, 183, 118, 145, 191, 74, 206, 9, 246, 182, 4, 19, 235, 99, 71, 73, 245, 150, 197, 244, 197, 89, 210, 63, 154, 228, 102, 27, 57, 119, 126, 105, 107, 13, 235, 237, 0, 201, 205, 64, 119, 69, 31, 27, 108, 67, 247, 112, 201, 240, 200, 58, 96, 177, 148, 163, 219, 158, 72, 83, 65, 30, 246, 217, 40, 167, 213, 194, 62, 151, 3, 122, 18, 174, 245, 42, 185, 48, 7, 160, 232, 190, 124, 174, 96, 112, 208, 128, 21, 81, 250, 249, 19, 207, 238, 119, 226, 153, 67, 108, 65, 170, 137, 35, 4, 135, 131, 223, 10, 190, 142, 217, 225, 148, 117, 106, 93, 156, 150, 67, 107, 139, 236, 128, 231, 41, 61, 38, 41, 98, 184, 158, 39, 149, 32, 253, 25, 83, 89, 50, 112, 188, 37, 157, 199, 228, 98, 38, 237, 139, 174, 233, 129, 223, 161, 98, 57, 253, 37, 229, 230, 91, 87, 97, 22, 226, 222, 110, 160, 173, 93, 98, 253, 203, 249, 4, 68, 254, 79, 97, 29, 81, 244, 211, 137, 250, 121, 198, 10, 184, 118, 69, 87, 198, 172, 8, 250, 30, 65, 50, 54, 182, 146, 177, 152, 241, 126, 57, 149, 228, 195, 129, 31, 116, 212, 199, 12, 22, 144, 179, 5, 13, 59, 73, 56, 57, 144, 2, 4, 115, 122, 73, 67, 130, 220, 77, 205, 50, 222, 5, 205, 178, 125, 203, 66, 74, 10, 206, 100, 244, 30, 31, 243, 234, 239, 27, 144, 225, 224, 221, 177, 115, 2, 179, 56, 45, 156, 69, 254, 93, 212, 125, 0, 56, 222, 134, 169, 42, 53, 155, 85, 245, 131, 157, 247, 1, 130, 8, 197, 0, 218, 128, 218, 85, 62, 202, 146, 144, 118, 16, 133, 86, 35, 182, 62, 115, 171, 95, 50, 107, 145, 53, 178, 41, 82, 33, 132, 193, 243, 227, 178, 237, 136, 207, 144, 25, 189, 114, 182, 187, 102, 85, 145, 174, 130, 100, 199, 131, 228, 151, 100, 177, 2, 134, 93, 95, 26, 142, 105, 39, 23, 152, 146, 111, 159, 24, 100, 251, 72, 245, 160, 218, 227, 98, 184, 239, 117, 78, 225, 32, 144, 48, 157, 170, 4, 182, 95, 136, 35, 169, 28, 19, 63, 245, 230, 57, 67, 166, 192, 88, 192, 132, 51, 44, 188, 78, 227, 156, 168, 132, 166, 153, 21, 73, 175, 41, 140, 171, 170, 40, 193, 67, 184, 216, 222, 141, 62, 194, 159, 29, 241, 231, 190, 129, 153, 151, 140, 152, 67, 51, 169, 154, 179, 92, 146, 45, 158, 211, 92, 208, 13, 90, 239, 78, 232, 220, 147, 218, 84, 99, 159, 106, 98, 174, 104, 150, 156, 71, 7, 109, 56, 68, 112, 140, 52, 201, 154, 227, 241, 49, 53, 73, 16, 105, 217, 218, 221, 141, 252, 33, 248, 72, 111, 8, 108, 152, 111, 47, 78, 198, 38, 12, 208, 34, 137, 107, 43, 227, 147, 29, 141, 118, 232, 156, 7, 16, 158, 173, 140, 104, 176, 190, 217, 241, 32, 128, 106, 220, 160, 250, 86, 71, 30, 155, 51, 197, 158, 165, 11, 179, 101, 7, 151, 94, 121, 251, 162, 210, 157, 188, 57, 228, 104, 99, 56, 186, 255, 72, 9, 227, 143, 214, 145, 60, 16, 47, 173, 0, 29, 131, 156, 57, 26, 48, 96, 250, 55, 197, 255, 225, 9, 23, 128, 39, 113, 21, 235, 115, 186, 176, 53, 105, 115, 147, 108, 78, 234, 8, 193, 12, 172, 102, 220, 202, 7, 233, 4, 223, 170, 159, 142, 19, 236, 98, 114, 201, 10, 7, 11, 186, 142, 27, 100, 214, 202, 54, 247, 192, 247, 110, 188, 161, 140, 164, 178, 57, 169, 36, 157, 125, 237, 81, 29, 227, 33, 29, 91, 227, 25, 26, 203, 154, 145, 20, 227, 135, 196, 25, 58, 131, 109, 51, 223, 119, 6, 228, 246, 153, 137, 126, 151, 78, 147, 103, 121, 49, 141, 171, 131, 184, 55, 18, 98, 123, 174, 207, 142, 163, 83, 223, 140, 132, 228, 67, 149, 191, 200, 89, 103, 18, 121, 208, 19, 91, 226, 11, 123, 31, 29, 88, 44, 94, 137, 171, 0, 156, 198, 64, 177, 32, 168, 163, 56, 233, 253, 33, 45, 95, 197, 175, 218, 23, 27, 78, 188, 215, 58, 204, 88, 213, 180, 207, 155, 110, 33, 177, 224, 36, 148, 254, 36, 246, 54, 73, 98, 111, 123, 146, 216, 128, 34, 159, 1, 137, 1, 20, 203, 243, 224, 46, 243, 80, 252, 170, 168, 35, 169, 3, 54, 205, 232, 54, 67, 70, 26, 106, 123, 134, 100, 139, 150, 133, 235, 195, 199, 116, 236, 93, 254, 68, 214, 123, 45, 202, 218, 242, 89, 39, 106, 197, 217, 162, 5, 127, 38, 147, 150, 101, 38, 155, 180, 207, 252, 157, 106, 242, 225, 9, 41, 85, 221, 238, 40, 10, 28, 179, 226, 98, 103, 152, 150, 55, 39, 101, 218, 142, 206, 111, 210, 207, 155, 244, 135, 4, 219, 188, 158, 128, 87, 181, 173, 1, 236, 68, 214, 33, 35, 162, 69, 240, 227, 63, 154, 244, 137, 53, 105, 125, 90, 4, 216, 90, 234, 111, 67, 61, 250, 132, 58, 180, 6, 253, 9, 245, 240, 139, 209, 32, 128, 251, 143, 6, 125, 66, 13, 170, 214, 58, 15, 1, 182, 150, 250, 219, 64, 131, 170, 79, 55, 11, 85, 235, 153, 129, 66, 61, 252, 156, 26, 196, 35, 36, 125, 89, 199, 6, 191, 61, 137, 11, 226, 201, 41, 138, 166, 105, 196, 5, 217, 95, 161, 20, 120, 11, 3, 235, 24, 255, 198, 123, 40, 17, 194, 119, 174, 101, 162, 187, 60, 40, 138, 167, 224, 88, 194, 133, 29, 240, 21, 138, 101, 81, 91, 252, 165, 146, 244, 163, 194, 202, 151, 90, 36, 153, 0, 168, 17, 78, 203, 161, 196, 55, 77, 202, 50, 30, 114, 148, 162, 176, 205, 255, 56, 8, 9, 46, 6, 178, 161, 186, 168, 9, 104, 181, 54, 48, 95, 25, 140, 2, 81, 24, 45, 8, 213, 21, 244, 51, 218, 140, 90, 123, 81, 139, 253, 225, 213, 241, 53, 39, 85, 92, 60, 79, 50, 148, 205, 101, 60, 178, 18, 186, 71, 243, 114, 150, 48, 170, 251, 4, 216, 77, 227, 134, 243, 191, 164, 34, 126, 238, 93, 79, 112, 242, 192, 164, 186, 63, 15, 12, 169, 81, 54, 192, 78, 119, 176, 108, 243, 37, 14, 151, 18, 22, 159, 77, 242, 50, 233, 215, 80, 20, 206, 188, 66, 238, 64, 37, 94, 85, 27, 26, 255, 97, 249, 118, 158, 101, 48, 178, 30, 212, 72, 94, 119, 191, 33, 26, 84, 21, 11, 55, 57, 14, 153, 187, 105, 66, 185, 47, 123, 125, 23, 226, 109, 181, 53, 231, 138, 177, 175, 234, 141, 162, 118, 226, 12, 205, 138, 125, 215, 160, 71, 73, 133, 161, 76, 103, 43, 232, 66, 164, 151, 235, 66, 98, 197, 123, 250, 23, 146, 43, 222, 159, 27, 10, 150, 96, 29, 67, 254, 153, 69, 135, 211, 241, 5, 203, 206, 91, 222, 133, 191, 144, 240, 136, 14, 173, 217, 44, 241, 118, 62, 149, 65, 226, 107, 82, 103, 124, 67, 65, 19, 2, 106, 133, 176, 16, 82, 216, 131, 158, 126, 89, 82, 248, 34, 25, 198, 189, 197, 178, 169, 209, 35, 86, 86, 69, 191, 96, 133, 132, 202, 170, 92, 43, 86, 30, 10, 91, 40, 114, 240, 210, 17, 226, 22, 31, 123, 114, 203, 101, 27, 161, 90, 46, 63, 62, 217, 89, 199, 52, 230, 145, 129, 149, 250, 181, 102, 41, 8, 205, 100, 95, 144, 24, 144, 201, 105, 101, 57, 176, 166, 165, 79, 56, 248, 190, 137, 232, 11, 27, 125, 94, 29, 246, 245, 20, 89, 60, 105, 251, 214, 106, 86, 251, 184, 171, 57, 107, 253, 220, 18, 250, 155, 74, 134, 95, 8, 144, 99, 77, 103, 131, 213, 71, 157, 151, 194, 127, 140, 160, 26, 45, 124, 176, 108, 164, 27, 236, 107, 249, 4, 74, 110, 235, 55, 39, 237, 218, 250, 45, 104, 17, 149, 9, 65, 117, 162, 174, 147, 82, 248, 94, 82, 120, 54, 99, 232, 144, 99, 3, 227, 218, 30, 38, 153, 84, 105, 188, 67, 105, 144, 23, 189, 68, 215, 73, 10, 60, 187, 140, 213, 102, 140, 86, 198, 167, 176, 175, 40, 223, 108, 121, 49, 13, 72, 120, 31, 166, 74, 88, 193, 230, 168, 42, 219, 167, 95, 89, 35, 150, 183, 209, 83, 67, 135, 248, 88, 229, 143, 143, 14, 14, 15, 127, 101, 67, 94, 28, 196, 76, 232, 202, 107, 219, 21, 147, 118, 211, 59, 134, 90, 45, 107, 41, 49, 21, 75, 137, 105, 244, 48, 82, 201, 60, 246, 203, 93, 79, 132, 203, 114, 203, 227, 212, 58, 21, 211, 138, 91, 112, 111, 112, 15, 50, 117, 61, 248, 214, 186, 108, 81, 107, 219, 243, 37, 56, 229, 21, 220, 85, 174, 123, 217, 137, 122, 136, 213, 146, 220, 77, 86, 213, 147, 72, 50, 91, 86, 196, 198, 43, 33, 147, 76, 150, 38, 111, 147, 225, 211, 139, 89, 25, 213, 220, 169, 237, 1, 223, 154, 103, 105, 47, 239, 39, 34, 81, 251, 244, 162, 74, 178, 146, 81, 122, 148, 252, 49, 79, 178, 94, 242, 246, 105, 36, 46, 252, 245, 87, 158, 196, 217, 112, 30, 15, 147, 119, 241, 112, 9, 100, 127, 62, 155, 240, 163, 37, 127, 143, 139, 52, 206, 170, 166, 224, 71, 140, 87, 147, 164, 202, 179, 37, 21, 210, 242, 183, 132, 49, 58, 47, 166, 73, 255, 96, 94, 20, 140, 248, 197, 1, 235, 215, 146, 106, 61, 9, 250, 36, 29, 166, 85, 137, 128, 181, 72, 14, 147, 234, 215, 37, 44, 178, 119, 213, 92, 139, 205, 31, 63, 98, 209, 105, 95, 111, 168, 172, 176, 104, 152, 92, 204, 62, 232, 203, 119, 63, 100, 249, 135, 146, 159, 108, 44, 219, 173, 238, 188, 221, 61, 142, 187, 151, 219, 221, 31, 222, 255, 185, 219, 249, 254, 106, 99, 179, 181, 65, 149, 177, 72, 166, 249, 89, 34, 59, 175, 155, 43, 101, 190, 19, 39, 242, 19, 253, 208, 104, 93, 3, 185, 106, 196, 90, 129, 81, 238, 77, 105, 155, 150, 248, 6, 110, 222, 193, 228, 34, 233, 225, 190, 45, 107, 185, 35, 151, 3, 88, 96, 124, 71, 93, 103, 141, 20, 94, 60, 172, 209, 220, 34, 153, 77, 98, 126, 132, 156, 67, 118, 80, 247, 143, 183, 223, 119, 32, 1, 105, 180, 121, 25, 49, 230, 254, 71, 123, 115, 178, 20, 38, 209, 8, 25, 24, 198, 183, 23, 88, 241, 200, 212, 188, 84, 77, 157, 169, 120, 105, 13, 51, 168, 143, 95, 188, 249, 25, 54, 113, 180, 64, 116, 30, 119, 255, 249, 30, 173, 177, 60, 57, 124, 126, 248, 142, 63, 3, 153, 66, 15, 226, 201, 108, 20, 103, 115, 56, 206, 210, 106, 255, 184, 7, 121, 87, 129, 103, 51, 106, 125, 132, 95, 162, 38, 251, 181, 209, 194, 86, 123, 56, 159, 196, 133, 172, 20, 23, 85, 119, 146, 159, 158, 196, 217, 199, 94, 50, 233, 14, 227, 249, 36, 45, 71, 31, 179, 188, 123, 146, 143, 225, 79, 182, 200, 62, 94, 142, 186, 195, 121, 190, 152, 195, 151, 81, 60, 30, 199, 240, 101, 154, 102, 242, 79, 55, 139, 249, 215, 11, 102, 127, 134, 184, 169, 180, 160, 141, 37, 89, 247, 249, 79, 221, 60, 233, 127, 76, 187, 241, 52, 101, 255, 159, 100, 25, 251, 95, 222, 71, 205, 190, 37, 89, 222, 27, 49, 52, 236, 43, 107, 136, 253, 63, 158, 48, 201, 200, 225, 247, 100, 126, 193, 254, 159, 194, 79, 246, 55, 139, 207, 226, 83, 248, 50, 59, 135, 135, 85, 156, 243, 255, 23, 240, 127, 57, 255, 88, 14, 179, 238, 79, 79, 187, 207, 222, 170, 111, 175, 94, 240, 111, 7, 63, 119, 159, 60, 197, 36, 14, 139, 56, 235, 15, 226, 106, 148, 20, 73, 31, 49, 210, 144, 174, 152, 137, 126, 99, 4, 179, 34, 61, 99, 230, 116, 94, 38, 178, 246, 5, 251, 103, 44, 192, 14, 183, 0, 184, 66, 169, 12, 47, 106, 77, 15, 212, 199, 227, 199, 221, 223, 126, 239, 254, 51, 238, 158, 47, 186, 151, 239, 113, 61, 173, 12, 168, 158, 193, 181, 201, 203, 186, 80, 168, 197, 130, 149, 73, 11, 132, 241, 156, 137, 121, 2, 97, 33, 21, 190, 97, 21, 62, 90, 100, 17, 128, 123, 87, 27, 150, 56, 81, 154, 180, 8, 254, 185, 123, 69, 165, 16, 170, 98, 70, 244, 138, 116, 86, 121, 107, 222, 191, 178, 122, 14, 106, 227, 133, 188, 119, 165, 186, 141, 203, 54, 254, 220, 238, 236, 18, 28, 74, 239, 252, 132, 118, 12, 26, 213, 24, 12, 243, 143, 31, 109, 178, 172, 2, 96, 150, 221, 74, 21, 3, 165, 186, 61, 52, 48, 178, 195, 28, 179, 42, 147, 236, 195, 101, 30, 183, 76, 141, 25, 128, 253, 13, 81, 42, 5, 130, 20, 35, 129, 228, 88, 61, 76, 120, 199, 73, 108, 253, 175, 100, 132, 162, 90, 73, 58, 197, 192, 139, 168, 154, 0, 222, 255, 211, 34, 254, 232, 114, 243, 184, 194, 36, 138, 234, 49, 171, 159, 218, 243, 39, 51, 211, 79, 28, 175, 103, 169, 173, 118, 29, 165, 70, 6, 219, 173, 246, 89, 172, 246, 154, 181, 214, 238, 21, 26, 28, 47, 126, 48, 35, 93, 230, 201, 32, 233, 108, 181, 1, 10, 75, 102, 151, 62, 14, 162, 249, 27, 129, 251, 127, 255, 111, 199, 170, 119, 199, 174, 183, 209, 64, 214, 124, 126, 112, 99, 129, 179, 43, 215, 8, 28, 242, 155, 155, 139, 28, 170, 180, 154, 208, 161, 138, 159, 69, 236, 174, 59, 89, 185, 29, 192, 18, 214, 109, 59, 147, 215, 114, 225, 217, 188, 109, 193, 33, 17, 209, 234, 162, 163, 171, 91, 194, 3, 43, 64, 80, 105, 94, 48, 183, 23, 78, 45, 164, 125, 228, 100, 90, 78, 255, 58, 62, 94, 123, 175, 226, 2, 203, 191, 69, 201, 47, 217, 201, 42, 41, 67, 182, 24, 123, 248, 182, 20, 163, 195, 137, 206, 237, 7, 248, 149, 89, 112, 238, 180, 132, 75, 11, 140, 151, 223, 186, 232, 182, 92, 132, 230, 196, 15, 247, 178, 242, 178, 38, 112, 72, 245, 125, 231, 10, 101, 23, 112, 26, 114, 160, 58, 132, 36, 93, 116, 113, 252, 210, 112, 68, 191, 203, 203, 224, 133, 27, 37, 114, 188, 69, 219, 27, 125, 7, 230, 167, 58, 237, 112, 234, 17, 51, 67, 132, 56, 52, 80, 46, 41, 122, 180, 162, 175, 191, 198, 210, 181, 20, 3, 106, 92, 227, 160, 199, 5, 227, 44, 103, 241, 23, 147, 229, 203, 228, 118, 133, 25, 73, 244, 210, 1, 131, 93, 113, 231, 42, 243, 163, 201, 14, 138, 164, 204, 228, 140, 226, 242, 245, 121, 166, 118, 151, 119, 184, 210, 176, 190, 188, 20, 187, 215, 203, 176, 200, 91, 128, 199, 2, 238, 61, 50, 158, 243, 19, 230, 86, 213, 9, 110, 201, 24, 94, 97, 177, 109, 33, 41, 73, 77, 224, 42, 163, 116, 158, 79, 19, 72, 245, 105, 59, 154, 80, 19, 79, 33, 85, 37, 192, 156, 220, 154, 40, 39, 175, 82, 130, 52, 27, 67, 13, 251, 245, 62, 126, 212, 24, 32, 189, 121, 209, 178, 86, 57, 172, 139, 173, 252, 8, 239, 219, 227, 175, 137, 106, 148, 178, 19, 208, 44, 224, 223, 240, 186, 194, 81, 212, 68, 97, 5, 18, 115, 56, 18, 62, 87, 230, 37, 31, 41, 183, 6, 219, 208, 115, 151, 254, 221, 117, 208, 239, 221, 121, 183, 170, 24, 30, 113, 84, 70, 18, 37, 234, 32, 125, 110, 157, 99, 241, 204, 189, 128, 109, 21, 82, 100, 72, 212, 156, 14, 171, 130, 34, 98, 107, 86, 36, 131, 132, 5, 214, 125, 154, 255, 22, 35, 162, 101, 177, 174, 118, 42, 222, 238, 166, 164, 27, 132, 196, 181, 53, 78, 191, 228, 123, 4, 71, 233, 160, 82, 196, 151, 158, 28, 124, 106, 95, 158, 113, 229, 25, 67, 163, 88, 90, 203, 144, 66, 208, 67, 233, 124, 243, 31, 108, 66, 156, 248, 132, 199, 188, 72, 177, 19, 36, 25, 94, 143, 168, 73, 22, 103, 198, 55, 108, 59, 65, 82, 119, 112, 122, 69, 188, 46, 114, 137, 229, 48, 178, 207, 251, 114, 71, 234, 187, 101, 79, 52, 110, 117, 125, 69, 234, 233, 236, 10, 237, 224, 187, 11, 169, 12, 96, 100, 118, 186, 99, 29, 172, 163, 61, 177, 249, 8, 31, 195, 71, 254, 46, 75, 147, 18, 37, 155, 84, 16, 20, 188, 152, 18, 79, 235, 50, 110, 254, 85, 164, 130, 90, 102, 234, 241, 216, 109, 221, 42, 169, 180, 142, 142, 186, 2, 210, 83, 179, 53, 200, 7, 23, 71, 67, 26, 234, 142, 217, 183, 109, 168, 51, 85, 55, 247, 1, 43, 78, 59, 148, 252, 173, 160, 237, 22, 245, 180, 116, 119, 48, 62, 231, 110, 44, 137, 213, 219, 102, 160, 93, 131, 25, 13, 153, 125, 179, 12, 238, 170, 93, 193, 122, 113, 142, 4, 84, 43, 62, 249, 164, 127, 84, 45, 38, 216, 157, 81, 22, 8, 118, 32, 241, 186, 173, 89, 220, 125, 243, 11, 108, 113, 101, 95, 30, 23, 241, 9, 252, 18, 119, 3, 180, 46, 71, 221, 131, 87, 240, 136, 125, 249, 57, 206, 74, 248, 101, 30, 253, 252, 139, 121, 84, 193, 47, 243, 232, 232, 57, 174, 197, 126, 153, 71, 239, 126, 195, 181, 216, 175, 175, 174, 240, 137, 52, 145, 53, 21, 25, 112, 178, 255, 95, 248, 19, 175, 7, 240, 46, 160, 183, 137, 124, 161, 106, 139, 103, 95, 157, 189, 251, 236, 201, 219, 121, 6, 251, 179, 45, 124, 40, 52, 104, 22, 194, 184, 14, 146, 67, 134, 229, 200, 45, 113, 35, 87, 247, 221, 106, 6, 209, 227, 199, 105, 58, 106, 170, 81, 175, 142, 115, 163, 221, 203, 39, 252, 126, 75, 181, 112, 45, 219, 79, 147, 114, 43, 62, 139, 83, 126, 146, 90, 112, 177, 108, 111, 40, 4, 212, 7, 207, 248, 121, 11, 177, 65, 253, 6, 104, 232, 78, 247, 149, 16, 73, 62, 120, 150, 103, 66, 195, 214, 96, 245, 228, 48, 184, 88, 216, 100, 41, 175, 102, 169, 177, 193, 34, 94, 221, 66, 101, 227, 104, 190, 117, 252, 191, 60, 161, 226, 196, 240, 126, 220, 109, 181, 204, 137, 181, 79, 188, 1, 86, 238, 181, 215, 0, 70, 239, 132, 9, 102, 34, 15, 235, 13, 206, 194, 124, 15, 137, 189, 129, 84, 134, 18, 166, 233, 123, 117, 193, 247, 146, 32, 113, 201, 24, 117, 16, 113, 148, 3, 113, 191, 255, 218, 213, 146, 178, 109, 11, 24, 102, 132, 86, 43, 125, 72, 8, 169, 46, 163, 4, 169, 238, 171, 120, 202, 132, 179, 70, 15, 237, 195, 202, 104, 135, 129, 213, 76, 221, 126, 3, 10, 138, 53, 95, 212, 117, 194, 37, 187, 123, 199, 117, 166, 130, 34, 123, 255, 158, 78, 105, 97, 84, 82, 179, 241, 45, 39, 244, 74, 93, 171, 102, 56, 20, 231, 207, 185, 251, 55, 161, 195, 17, 60, 13, 229, 200, 146, 237, 64, 242, 168, 50, 225, 39, 209, 237, 71, 128, 84, 190, 15, 24, 227, 86, 111, 255, 181, 13, 45, 8, 128, 180, 62, 37, 138, 146, 233, 171, 242, 36, 217, 104, 150, 170, 189, 236, 219, 186, 109, 33, 124, 239, 20, 7, 52, 151, 58, 227, 119, 120, 191, 134, 179, 231, 116, 216, 245, 69, 204, 158, 235, 26, 2, 215, 185, 170, 170, 119, 20, 75, 20, 156, 245, 62, 100, 189, 193, 75, 60, 223, 168, 191, 199, 225, 222, 238, 183, 86, 160, 2, 36, 136, 40, 75, 219, 152, 177, 253, 178, 21, 77, 203, 146, 137, 155, 7, 113, 245, 205, 127, 211, 137, 104, 52, 11, 31, 209, 126, 104, 246, 118, 225, 121, 158, 128, 201, 144, 122, 11, 42, 7, 225, 156, 233, 238, 184, 237, 115, 72, 238, 129, 135, 226, 104, 255, 221, 95, 80, 143, 168, 198, 79, 204, 248, 61, 166, 186, 227, 40, 180, 201, 145, 173, 57, 103, 229, 253, 184, 49, 231, 191, 114, 27, 50, 182, 232, 167, 224, 134, 104, 199, 129, 104, 162, 125, 231, 137, 215, 166, 106, 28, 150, 197, 52, 174, 186, 120, 76, 3, 196, 250, 148, 240, 4, 191, 6, 83, 99, 176, 67, 62, 149, 27, 246, 137, 162, 255, 253, 235, 184, 214, 163, 253, 104, 151, 239, 81, 83, 232, 143, 161, 180, 27, 237, 138, 55, 168, 66, 83, 4, 35, 127, 202, 234, 24, 76, 152, 123, 13, 178, 90, 168, 31, 36, 19, 77, 119, 19, 231, 249, 120, 62, 123, 9, 27, 155, 225, 77, 185, 142, 188, 23, 176, 69, 166, 172, 146, 190, 103, 198, 70, 41, 71, 99, 123, 237, 10, 106, 130, 213, 96, 86, 27, 118, 132, 1, 78, 133, 217, 229, 244, 194, 21, 146, 84, 152, 107, 96, 165, 133, 42, 124, 200, 86, 123, 171, 54, 117, 100, 10, 119, 27, 230, 53, 106, 183, 95, 153, 218, 14, 49, 13, 45, 138, 219, 42, 22, 57, 55, 69, 164, 247, 81, 210, 171, 71, 2, 250, 18, 190, 229, 72, 94, 203, 165, 89, 227, 29, 23, 133, 117, 98, 144, 121, 232, 245, 204, 123, 235, 221, 119, 134, 49, 235, 140, 2, 23, 218, 53, 236, 63, 171, 105, 66, 100, 39, 81, 115, 199, 219, 239, 107, 192, 213, 6, 50, 90, 103, 203, 186, 147, 13, 49, 223, 219, 80, 61, 164, 211, 198, 33, 197, 30, 184, 177, 215, 30, 109, 111, 248, 30, 216, 226, 74, 166, 200, 103, 105, 181, 186, 181, 144, 40, 87, 54, 54, 164, 117, 136, 46, 39, 103, 97, 69, 178, 107, 235, 147, 224, 240, 104, 146, 156, 197, 89, 165, 197, 0, 110, 97, 83, 2, 0, 215, 5, 97, 179, 54, 21, 244, 129, 191, 47, 234, 75, 174, 73, 186, 141, 193, 42, 248, 9, 106, 5, 13, 54, 124, 194, 59, 104, 25, 242, 31, 87, 239, 55, 169, 191, 119, 13, 190, 27, 34, 7, 57, 211, 126, 99, 205, 182, 240, 180, 107, 203, 122, 177, 101, 9, 161, 43, 221, 56, 53, 234, 38, 75, 69, 201, 11, 100, 241, 73, 214, 48, 108, 140, 28, 241, 46, 182, 66, 2, 110, 183, 182, 124, 193, 76, 215, 8, 230, 112, 9, 225, 88, 185, 142, 72, 30, 118, 5, 83, 172, 46, 164, 100, 178, 165, 153, 143, 134, 2, 47, 251, 205, 102, 121, 193, 134, 238, 41, 26, 136, 86, 119, 142, 55, 70, 250, 231, 89, 143, 72, 19, 58, 233, 124, 233, 132, 55, 201, 34, 132, 133, 76, 138, 150, 12, 129, 178, 104, 231, 13, 126, 180, 209, 67, 251, 98, 187, 100, 65, 42, 89, 104, 142, 217, 115, 171, 33, 117, 71, 36, 169, 73, 108, 174, 159, 101, 143, 251, 253, 180, 146, 172, 107, 121, 80, 190, 201, 75, 58, 159, 57, 66, 20, 148, 76, 212, 153, 55, 126, 31, 82, 36, 214, 245, 174, 2, 87, 63, 88, 85, 79, 136, 35, 17, 202, 141, 6, 158, 128, 2, 1, 109, 70, 59, 108, 20, 3, 226, 74, 151, 235, 245, 199, 134, 62, 54, 184, 208, 194, 202, 174, 27, 81, 249, 67, 27, 33, 239, 210, 204, 168, 160, 182, 174, 13, 127, 100, 161, 134, 163, 1, 35, 137, 16, 116, 172, 198, 3, 135, 98, 128, 103, 186, 137, 48, 107, 9, 53, 216, 45, 244, 92, 16, 136, 63, 245, 146, 199, 23, 30, 64, 175, 54, 229, 119, 207, 117, 134, 234, 227, 222, 27, 232, 125, 215, 213, 205, 217, 214, 130, 160, 42, 116, 134, 200, 199, 174, 165, 204, 18, 24, 93, 132, 161, 23, 86, 93, 17, 125, 148, 115, 170, 146, 32, 249, 211, 178, 5, 64, 23, 1, 36, 234, 105, 75, 252, 138, 28, 193, 136, 55, 156, 94, 255, 233, 40, 170, 67, 200, 153, 168, 233, 142, 212, 25, 237, 84, 64, 152, 26, 155, 175, 16, 27, 197, 220, 194, 89, 22, 237, 219, 66, 230, 153, 79, 54, 247, 107, 218, 12, 198, 26, 98, 187, 134, 93, 13, 91, 14, 251, 150, 226, 34, 193, 115, 88, 131, 8, 21, 77, 8, 60, 70, 165, 83, 190, 53, 155, 176, 0, 182, 186, 9, 254, 32, 114, 234, 33, 145, 94, 108, 122, 217, 73, 73, 161, 174, 51, 245, 178, 157, 25, 63, 236, 91, 11, 31, 241, 72, 53, 167, 86, 96, 86, 139, 200, 27, 134, 224, 192, 162, 68, 249, 45, 118, 250, 182, 113, 130, 52, 28, 84, 219, 55, 125, 222, 44, 176, 246, 36, 13, 214, 20, 92, 47, 13, 151, 137, 42, 10, 174, 137, 92, 163, 67, 100, 32, 201, 40, 170, 240, 69, 247, 182, 117, 165, 180, 116, 234, 111, 48, 226, 36, 170, 186, 14, 30, 122, 233, 210, 234, 8, 204, 93, 91, 110, 248, 100, 220, 127, 117, 247, 87, 208, 197, 194, 151, 131, 169, 27, 180, 156, 27, 201, 26, 69, 101, 170, 73, 5, 92, 235, 213, 25, 140, 230, 174, 43, 81, 228, 241, 214, 48, 70, 21, 230, 65, 10, 138, 148, 159, 48, 230, 68, 131, 180, 106, 45, 73, 147, 127, 215, 137, 220, 150, 240, 125, 194, 68, 73, 73, 136, 105, 38, 193, 143, 31, 35, 18, 123, 214, 52, 206, 66, 208, 27, 72, 155, 139, 110, 239, 70, 50, 7, 24, 188, 103, 127, 133, 178, 4, 22, 228, 124, 119, 251, 138, 26, 157, 40, 237, 200, 202, 44, 114, 169, 127, 249, 130, 190, 78, 124, 103, 155, 255, 217, 221, 38, 169, 143, 186, 70, 90, 130, 176, 86, 135, 18, 26, 110, 143, 181, 245, 189, 183, 45, 98, 28, 136, 14, 194, 219, 30, 184, 116, 155, 75, 241, 102, 122, 187, 2, 172, 20, 117, 196, 116, 207, 202, 7, 242, 77, 141, 88, 245, 44, 255, 227, 88, 213, 69, 251, 17, 206, 92, 119, 202, 206, 184, 67, 59, 82, 164, 242, 124, 146, 196, 153, 37, 81, 170, 149, 119, 249, 79, 226, 185, 64, 138, 227, 106, 181, 103, 208, 160, 114, 86, 249, 40, 38, 169, 127, 62, 68, 181, 227, 105, 62, 212, 0, 8, 46, 57, 110, 227, 82, 87, 81, 113, 87, 186, 120, 190, 76, 63, 209, 228, 239, 241, 0, 217, 212, 203, 113, 14, 189, 61, 72, 141, 160, 45, 1, 226, 118, 205, 176, 28, 76, 211, 44, 157, 206, 167, 96, 67, 46, 196, 23, 91, 22, 174, 247, 185, 201, 82, 208, 167, 88, 70, 138, 240, 74, 210, 205, 69, 221, 8, 159, 96, 184, 35, 124, 206, 77, 167, 82, 40, 152, 225, 21, 117, 31, 170, 161, 48, 69, 143, 212, 160, 212, 200, 204, 253, 160, 164, 232, 119, 86, 13, 216, 12, 67, 9, 90, 46, 57, 162, 139, 218, 108, 229, 76, 100, 88, 67, 170, 199, 62, 203, 38, 65, 58, 232, 37, 11, 242, 149, 11, 247, 55, 236, 171, 13, 202, 151, 241, 76, 58, 140, 191, 37, 241, 152, 253, 34, 247, 157, 51, 62, 84, 41, 95, 219, 133, 77, 54, 232, 26, 204, 181, 45, 148, 162, 116, 152, 34, 41, 170, 185, 99, 129, 15, 160, 2, 220, 18, 118, 8, 54, 226, 21, 64, 102, 203, 126, 62, 137, 47, 23, 50, 73, 37, 238, 29, 160, 143, 83, 180, 85, 170, 36, 48, 142, 61, 145, 204, 249, 32, 230, 37, 196, 188, 78, 196, 223, 121, 164, 139, 232, 92, 160, 139, 201, 144, 150, 112, 2, 71, 80, 166, 55, 49, 149, 106, 34, 80, 52, 139, 43, 73, 111, 235, 243, 233, 214, 135, 175, 91, 207, 59, 138, 234, 107, 64, 16, 224, 143, 205, 105, 119, 63, 28, 230, 57, 17, 129, 219, 101, 249, 26, 121, 126, 11, 163, 183, 76, 55, 200, 239, 165, 122, 134, 134, 96, 26, 47, 78, 146, 186, 65, 184, 5, 174, 127, 162, 105, 107, 57, 9, 226, 252, 157, 102, 142, 203, 49, 51, 173, 25, 189, 87, 40, 212, 94, 44, 109, 149, 110, 145, 80, 219, 94, 109, 89, 227, 141, 6, 52, 45, 15, 245, 172, 208, 247, 79, 11, 190, 91, 168, 176, 21, 29, 197, 165, 199, 138, 58, 199, 109, 13, 114, 37, 63, 165, 120, 195, 92, 111, 18, 151, 37, 236, 22, 100, 206, 18, 191, 201, 10, 190, 223, 218, 230, 29, 255, 20, 21, 236, 220, 208, 59, 69, 92, 239, 182, 167, 107, 81, 138, 15, 34, 219, 215, 158, 51, 143, 198, 178, 154, 224, 72, 105, 126, 250, 95, 210, 189, 187, 19, 96, 57, 42, 110, 50, 239, 13, 19, 58, 144, 183, 100, 111, 191, 224, 241, 90, 217, 42, 220, 9, 154, 133, 160, 170, 218, 157, 183, 138, 229, 84, 73, 165, 192, 52, 104, 66, 187, 3, 185, 193, 29, 5, 119, 246, 164, 80, 136, 165, 251, 3, 107, 43, 124, 105, 25, 51, 79, 220, 72, 47, 121, 95, 222, 196, 19, 239, 30, 119, 95, 67, 203, 48, 189, 242, 108, 186, 71, 120, 4, 39, 86, 244, 27, 188, 178, 175, 45, 39, 223, 33, 39, 89, 244, 75, 178, 112, 14, 148, 140, 179, 189, 232, 79, 21, 232, 236, 69, 173, 108, 62, 77, 138, 180, 215, 18, 30, 225, 158, 137, 215, 175, 68, 82, 98, 60, 160, 240, 189, 184, 76, 158, 165, 69, 89, 153, 26, 165, 186, 133, 94, 4, 190, 123, 209, 113, 107, 14, 155, 203, 225, 94, 250, 9, 28, 143, 133, 47, 124, 195, 120, 235, 253, 21, 57, 79, 82, 59, 164, 170, 116, 125, 254, 105, 64, 92, 155, 134, 0, 28, 120, 107, 94, 138, 59, 93, 108, 18, 197, 3, 211, 132, 122, 6, 214, 32, 120, 122, 195, 128, 107, 152, 242, 40, 103, 143, 178, 97, 176, 9, 153, 11, 97, 80, 206, 201, 26, 169, 227, 14, 46, 63, 239, 126, 212, 52, 242, 115, 101, 102, 121, 203, 15, 190, 135, 192, 249, 11, 116, 77, 5, 67, 135, 119, 47, 0, 236, 220, 85, 85, 189, 0, 168, 62, 28, 14, 34, 123, 116, 116, 77, 247, 96, 73, 93, 154, 206, 225, 157, 147, 119, 94, 165, 114, 62, 171, 234, 224, 253, 91, 131, 234, 26, 48, 155, 134, 124, 50, 102, 86, 69, 156, 173, 54, 227, 100, 225, 164, 139, 58, 234, 197, 103, 214, 118, 143, 206, 26, 182, 121, 52, 221, 226, 33, 55, 26, 8, 225, 236, 229, 45, 219, 65, 82, 36, 71, 252, 49, 176, 150, 97, 177, 214, 41, 245, 74, 250, 86, 47, 231, 136, 64, 9, 225, 93, 18, 242, 182, 46, 120, 165, 4, 60, 67, 249, 110, 223, 122, 183, 121, 31, 165, 199, 16, 90, 139, 196, 22, 105, 178, 234, 150, 42, 10, 208, 231, 65, 130, 222, 112, 185, 181, 44, 249, 137, 113, 157, 153, 45, 251, 214, 146, 56, 58, 196, 141, 133, 195, 100, 169, 220, 117, 91, 207, 78, 207, 210, 103, 71, 138, 248, 252, 8, 198, 177, 74, 207, 82, 213, 73, 190, 84, 91, 255, 122, 26, 199, 174, 56, 199, 177, 97, 116, 229, 253, 60, 173, 37, 163, 196, 111, 165, 192, 59, 155, 10, 180, 209, 201, 230, 59, 6, 117, 119, 15, 153, 103, 150, 35, 5, 244, 208, 138, 204, 116, 89, 253, 54, 236, 186, 242, 104, 34, 2, 135, 105, 194, 167, 172, 233, 48, 203, 139, 228, 13, 204, 105, 243, 88, 174, 192, 59, 28, 119, 128, 124, 152, 78, 96, 165, 87, 191, 159, 198, 218, 6, 30, 154, 240, 177, 195, 235, 78, 164, 60, 130, 113, 195, 22, 219, 95, 173, 139, 127, 140, 111, 70, 48, 221, 78, 136, 17, 158, 162, 151, 135, 246, 110, 64, 34, 19, 45, 10, 185, 159, 133, 206, 56, 212, 249, 154, 174, 251, 124, 27, 78, 28, 186, 140, 88, 5, 191, 138, 20, 173, 132, 106, 231, 169, 187, 128, 122, 61, 158, 115, 190, 5, 162, 109, 213, 166, 255, 2, 236, 123, 187, 248, 165, 222, 88, 172, 188, 25, 93, 141, 140, 198, 39, 88, 93, 150, 121, 100, 100, 31, 153, 57, 235, 183, 228, 96, 154, 168, 187, 212, 49, 240, 108, 70, 112, 215, 161, 3, 39, 218, 204, 42, 244, 159, 242, 69, 17, 36, 74, 88, 186, 72, 205, 119, 189, 71, 251, 190, 53, 188, 22, 119, 254, 192, 149, 214, 254, 246, 177, 240, 2, 161, 136, 59, 100, 45, 184, 177, 148, 23, 133, 250, 170, 156, 215, 185, 105, 46, 159, 249, 55, 159, 250, 124, 32, 177, 86, 98, 234, 154, 213, 111, 31, 193, 100, 109, 221, 34, 92, 174, 126, 119, 208, 146, 243, 123, 252, 67, 18, 193, 144, 209, 37, 122, 110, 8, 172, 45, 212, 50, 132, 81, 187, 193, 124, 164, 152, 40, 71, 79, 201, 228, 102, 118, 51, 148, 102, 168, 9, 218, 192, 150, 14, 171, 105, 242, 243, 199, 232, 46, 204, 232, 119, 153, 211, 114, 151, 71, 63, 119, 77, 159, 198, 153, 5, 141, 2, 1, 21, 96, 213, 245, 7, 71, 97, 152, 173, 193, 184, 203, 237, 34, 39, 99, 192, 115, 21, 184, 65, 67, 72, 25, 104, 27, 77, 137, 184, 245, 128, 233, 57, 110, 157, 48, 252, 252, 157, 160, 189, 94, 146, 113, 130, 123, 178, 68, 121, 10, 30, 234, 150, 184, 44, 102, 46, 230, 38, 103, 22, 32, 213, 153, 117, 131, 2, 192, 217, 20, 106, 219, 55, 195, 167, 51, 109, 247, 125, 203, 43, 120, 210, 116, 130, 89, 251, 190, 249, 201, 7, 245, 248, 67, 105, 237, 134, 120, 61, 208, 135, 104, 201, 57, 231, 155, 191, 193, 208, 14, 167, 106, 227, 84, 79, 240, 117, 115, 75, 44, 103, 191, 27, 237, 25, 178, 83, 30, 46, 233, 58, 243, 65, 99, 220, 61, 93, 229, 136, 148, 119, 228, 168, 210, 0, 23, 65, 91, 79, 4, 252, 7, 155, 230, 61, 30, 187, 136, 135, 238, 51, 53, 248, 109, 251, 70, 4, 19, 202, 3, 221, 240, 114, 130, 45, 7, 53, 141, 185, 20, 75, 137, 250, 145, 27, 19, 80, 5, 117, 41, 161, 31, 49, 185, 180, 32, 116, 250, 62, 37, 226, 234, 138, 197, 134, 90, 107, 238, 200, 49, 246, 4, 142, 144, 33, 98, 33, 34, 211, 146, 113, 214, 122, 79, 50, 66, 254, 65, 241, 220, 83, 174, 99, 72, 185, 94, 52, 49, 180, 28, 232, 71, 116, 139, 93, 96, 223, 198, 60, 147, 215, 52, 25, 148, 157, 8, 57, 27, 234, 149, 84, 154, 57, 189, 124, 15, 195, 154, 80, 57, 131, 142, 9, 131, 171, 246, 76, 191, 23, 204, 32, 26, 31, 18, 37, 171, 151, 222, 118, 143, 129, 176, 247, 75, 218, 52, 143, 145, 173, 222, 51, 246, 182, 134, 166, 131, 124, 58, 139, 139, 228, 93, 254, 83, 154, 245, 219, 23, 157, 136, 220, 51, 241, 15, 188, 207, 230, 2, 153, 128, 223, 241, 131, 133, 227, 221, 130, 196, 112, 188, 2, 162, 148, 175, 70, 249, 71, 39, 250, 189, 206, 24, 246, 68, 37, 72, 154, 183, 107, 34, 23, 215, 75, 151, 47, 24, 145, 245, 91, 158, 200, 64, 6, 91, 146, 172, 112, 216, 11, 205, 61, 67, 166, 145, 240, 231, 1, 1, 59, 241, 190, 34, 66, 253, 248, 112, 194, 42, 116, 162, 103, 228, 61, 184, 156, 170, 0, 69, 12, 31, 142, 229, 157, 68, 60, 134, 174, 225, 161, 140, 100, 250, 234, 69, 169, 215, 225, 163, 133, 163, 69, 108, 191, 60, 189, 100, 88, 38, 164, 120, 15, 175, 52, 136, 221, 211, 26, 130, 123, 161, 24, 128, 23, 4, 164, 214, 64, 161, 98, 3, 235, 204, 205, 184, 134, 243, 80, 73, 190, 33, 223, 159, 174, 172, 153, 9, 107, 18, 152, 190, 13, 137, 181, 185, 55, 223, 133, 33, 171, 156, 169, 66, 153, 43, 43, 249, 246, 35, 206, 178, 69, 123, 193, 76, 152, 39, 211, 101, 237, 1, 18, 35, 140, 115, 142, 154, 191, 38, 21, 85, 127, 246, 83, 58, 233, 53, 215, 15, 105, 1, 154, 160, 41, 23, 87, 249, 55, 159, 112, 241, 58, 210, 13, 38, 221, 108, 110, 77, 183, 75, 151, 169, 240, 19, 124, 198, 245, 230, 159, 107, 46, 198, 4, 28, 78, 31, 173, 78, 100, 111, 7, 189, 222, 74, 36, 192, 197, 79, 69, 196, 86, 35, 196, 129, 213, 24, 140, 99, 203, 60, 169, 91, 243, 32, 85, 86, 95, 247, 88, 202, 136, 149, 17, 44, 89, 255, 32, 244, 126, 162, 197, 16, 10, 36, 70, 134, 153, 181, 163, 5, 235, 231, 148, 67, 103, 40, 203, 17, 148, 145, 18, 20, 206, 135, 145, 63, 64, 57, 95, 157, 30, 111, 169, 11, 182, 200, 26, 7, 173, 172, 64, 66, 173, 170, 231, 174, 27, 64, 235, 63, 73, 203, 217, 36, 94, 138, 70, 130, 97, 35, 76, 241, 201, 61, 166, 32, 179, 195, 164, 16, 111, 72, 10, 33, 245, 193, 250, 24, 36, 225, 158, 21, 49, 183, 37, 141, 144, 82, 96, 47, 86, 177, 245, 181, 33, 86, 31, 176, 25, 175, 150, 108, 245, 136, 249, 1, 233, 128, 133, 163, 89, 37, 64, 90, 172, 81, 47, 198, 79, 114, 39, 79, 104, 208, 67, 212, 46, 225, 169, 3, 31, 196, 47, 184, 213, 28, 127, 0, 62, 44, 102, 243, 50, 121, 94, 228, 243, 153, 89, 190, 118, 144, 34, 144, 91, 89, 2, 241, 79, 101, 235, 88, 6, 193, 152, 63, 197, 82, 136, 17, 149, 47, 101, 57, 196, 207, 219, 207, 182, 36, 130, 201, 105, 227, 169, 121, 125, 75, 35, 116, 28, 234, 150, 72, 112, 251, 107, 89, 38, 33, 8, 233, 82, 137, 173, 84, 183, 189, 92, 178, 212, 159, 248, 220, 75, 38, 117, 107, 24, 62, 95, 230, 203, 92, 199, 8, 38, 220, 193, 19, 177, 218, 237, 39, 189, 116, 26, 79, 160, 148, 41, 150, 78, 172, 43, 207, 4, 104, 80, 32, 117, 188, 176, 156, 28, 117, 151, 168, 119, 201, 65, 225, 198, 132, 44, 89, 66, 233, 185, 135, 167, 238, 132, 47, 54, 13, 169, 205, 61, 243, 82, 77, 78, 160, 61, 205, 215, 186, 101, 156, 138, 128, 232, 217, 45, 221, 219, 70, 166, 28, 248, 16, 188, 42, 53, 200, 80, 228, 248, 245, 16, 46, 61, 209, 30, 144, 151, 84, 106, 124, 102, 191, 70, 175, 191, 132, 255, 210, 215, 179, 228, 1, 206, 63, 243, 162, 197, 244, 36, 231, 98, 145, 177, 153, 73, 44, 8, 138, 162, 141, 26, 134, 53, 235, 147, 113, 70, 123, 104, 29, 97, 154, 165, 146, 100, 255, 153, 179, 150, 207, 159, 100, 4, 238, 116, 34, 216, 234, 187, 83, 39, 158, 1, 183, 21, 154, 196, 4, 12, 250, 242, 38, 42, 200, 131, 184, 210, 240, 163, 230, 255, 30, 190, 2, 7, 234, 53, 33, 156, 122, 151, 45, 126, 201, 192, 46, 251, 135, 218, 109, 208, 7, 199, 161, 133, 234, 136, 152, 11, 141, 172, 201, 56, 33, 112, 105, 252, 249, 145, 47, 230, 177, 181, 1, 111, 71, 245, 120, 195, 88, 84, 140, 81, 153, 140, 21, 16, 110, 111, 120, 140, 243, 210, 90, 247, 54, 104, 23, 235, 249, 237, 243, 230, 91, 130, 209, 146, 229, 23, 205, 88, 30, 136, 33, 160, 58, 30, 255, 178, 31, 153, 147, 224, 245, 222, 180, 232, 0, 173, 80, 235, 30, 243, 253, 96, 208, 194, 29, 231, 192, 245, 133, 93, 138, 13, 150, 164, 106, 169, 84, 186, 49, 141, 163, 82, 98, 132, 150, 161, 11, 244, 130, 243, 189, 236, 11, 140, 187, 59, 203, 236, 94, 77, 236, 2, 104, 150, 213, 14, 71, 38, 208, 1, 219, 70, 14, 131, 59, 36, 116, 108, 17, 92, 232, 229, 215, 131, 214, 200, 14, 13, 97, 134, 245, 139, 188, 86, 72, 224, 141, 40, 245, 10, 37, 177, 252, 58, 225, 249, 211, 207, 79, 246, 162, 123, 34, 61, 242, 211, 225, 51, 102, 161, 228, 247, 223, 223, 234, 239, 7, 47, 158, 161, 239, 111, 244, 247, 39, 255, 99, 202, 15, 255, 175, 193, 243, 252, 21, 42, 63, 250, 69, 127, 255, 159, 215, 6, 230, 127, 222, 252, 174, 203, 127, 121, 105, 224, 127, 121, 251, 155, 249, 254, 155, 129, 127, 241, 187, 249, 254, 250, 229, 91, 253, 253, 205, 239, 207, 53, 252, 219, 223, 12, 158, 119, 175, 12, 252, 175, 207, 255, 161, 203, 127, 253, 253, 80, 127, 255, 59, 192, 168, 239, 191, 254, 93, 127, 255, 199, 99, 131, 231, 31, 175, 209, 247, 55, 240, 157, 36, 48, 97, 217, 194, 122, 239, 115, 147, 123, 223, 157, 119, 69, 55, 184, 237, 221, 125, 191, 180, 149, 2, 174, 185, 227, 253, 127, 143, 225, 142, 247, 63, 239, 93, 253, 31, 251, 158, 119, 219, 33, 176, 110, 119, 191, 222, 231, 150, 2, 82, 228, 216, 53, 124, 59, 2, 229, 153, 249, 237, 190, 185, 129, 66, 30, 171, 159, 239, 73, 84, 184, 235, 46, 116, 145, 204, 248, 103, 221, 61, 177, 242, 11, 22, 190, 156, 29, 20, 110, 2, 69, 103, 82, 125, 239, 8, 151, 219, 177, 45, 72, 181, 14, 111, 165, 98, 233, 26, 188, 164, 248, 88, 15, 191, 31, 151, 73, 23, 183, 226, 34, 62, 225, 219, 135, 216, 223, 228, 130, 135, 57, 39, 140, 43, 34, 228, 18, 134, 190, 159, 156, 197, 104, 3, 82, 107, 192, 66, 223, 243, 84, 56, 196, 195, 249, 105, 33, 254, 22, 115, 248, 59, 138, 153, 243, 152, 240, 157, 96, 227, 209, 180, 192, 213, 198, 89, 63, 230, 123, 167, 226, 60, 23, 127, 43, 190, 97, 104, 146, 78, 57, 5, 211, 201, 98, 138, 225, 167, 185, 104, 126, 186, 152, 242, 38, 242, 98, 193, 235, 87, 241, 116, 34, 155, 168, 146, 201, 28, 87, 169, 70, 49, 167, 188, 74, 79, 228, 50, 252, 123, 194, 127, 255, 146, 215, 178, 141, 1, 217, 124, 175, 110, 228, 196, 236, 25, 108, 71, 252, 47, 87, 252, 201, 181, 100, 48, 214, 23, 81, 224, 118, 3, 188, 186, 47, 80, 72, 32, 177, 104, 124, 225, 89, 216, 39, 186, 58, 16, 127, 150, 172, 237, 251, 83, 78, 114, 93, 122, 32, 207, 115, 5, 151, 247, 85, 34, 113, 217, 234, 126, 136, 29, 214, 2, 255, 96, 29, 11, 252, 58, 185, 121, 50, 88, 190, 190, 47, 128, 151, 112, 114, 133, 37, 254, 90, 118, 174, 123, 153, 223, 50, 4, 24, 52, 11, 169, 59, 79, 72, 144, 69, 127, 40, 48, 207, 125, 17, 33, 6, 247, 61, 119, 106, 211, 192, 192, 83, 157, 2, 160, 250, 190, 192, 130, 212, 247, 1, 224, 141, 15, 218, 185, 164, 219, 31, 116, 177, 210, 85, 197, 120, 97, 165, 173, 165, 114, 99, 61, 81, 86, 198, 45, 211, 153, 2, 100, 178, 130, 193, 3, 130, 9, 69, 4, 202, 94, 193, 31, 239, 91, 81, 28, 90, 235, 246, 57, 204, 116, 40, 133, 107, 56, 103, 138, 154, 122, 26, 200, 45, 159, 89, 111, 152, 8, 109, 104, 32, 59, 25, 150, 108, 97, 176, 87, 204, 151, 28, 17, 69, 207, 214, 189, 172, 116, 131, 117, 115, 74, 152, 56, 27, 83, 251, 50, 167, 192, 138, 54, 197, 211, 112, 77, 219, 170, 180, 250, 170, 182, 203, 214, 149, 215, 181, 61, 40, 4, 241, 175, 235, 215, 183, 45, 218, 63, 203, 10, 55, 148, 51, 7, 165, 224, 96, 189, 216, 7, 178, 108, 17, 124, 201, 113, 43, 138, 12, 222, 207, 246, 207, 60, 83, 71, 93, 61, 172, 83, 16, 166, 1, 49, 33, 191, 214, 155, 43, 60, 149, 52, 136, 169, 53, 139, 43, 104, 152, 231, 91, 225, 138, 185, 195, 131, 95, 223, 136, 34, 116, 178, 171, 99, 144, 123, 121, 104, 144, 200, 111, 183, 178, 186, 24, 82, 251, 117, 172, 47, 90, 167, 212, 63, 209, 10, 227, 151, 182, 198, 24, 226, 240, 167, 89, 101, 148, 90, 194, 41, 128, 189, 149, 76, 186, 51, 113, 46, 194, 100, 108, 206, 147, 100, 220, 143, 23, 124, 63, 85, 92, 20, 249, 57, 207, 187, 143, 228, 73, 156, 9, 4, 5, 114, 7, 110, 82, 196, 13, 160, 22, 73, 92, 0, 216, 110, 183, 15, 243, 108, 11, 29, 87, 145, 16, 44, 208, 168, 70, 1, 16, 158, 233, 175, 197, 47, 105, 13, 163, 31, 229, 243, 101, 4, 164, 217, 188, 74, 234, 97, 202, 164, 151, 103, 253, 122, 24, 101, 49, 64, 166, 1, 210, 162, 23, 50, 59, 222, 17, 104, 240, 34, 183, 192, 168, 225, 251, 197, 204, 194, 45, 21, 178, 54, 157, 6, 215, 191, 122, 108, 180, 172, 110, 5, 153, 82, 177, 150, 53, 100, 11, 37, 93, 69, 118, 141, 243, 109, 175, 35, 55, 152, 193, 131, 43, 201, 120, 29, 88, 225, 81, 1, 143, 201, 0, 199, 25, 95, 24, 131, 126, 227, 240, 69, 79, 242, 193, 85, 226, 58, 207, 32, 218, 55, 8, 108, 156, 122, 57, 87, 143, 214, 90, 86, 144, 61, 226, 230, 172, 41, 107, 154, 156, 149, 101, 242, 219, 80, 92, 93, 162, 213, 11, 58, 121, 243, 251, 82, 46, 195, 171, 18, 188, 170, 179, 26, 170, 119, 252, 87, 151, 27, 56, 224, 53, 200, 90, 191, 190, 59, 104, 213, 92, 99, 248, 3, 139, 150, 47, 137, 231, 189, 196, 215, 0, 50, 46, 253, 14, 71, 227, 129, 197, 149, 60, 238, 72, 42, 246, 58, 163, 56, 7, 197, 56, 174, 105, 10, 197, 56, 128, 3, 185, 216, 166, 130, 179, 139, 251, 44, 124, 208, 15, 144, 96, 113, 9, 88, 57, 190, 17, 219, 115, 138, 78, 246, 64, 244, 80, 0, 69, 228, 210, 128, 43, 139, 147, 205, 196, 153, 192, 90, 226, 124, 18, 151, 242, 156, 229, 106, 210, 12, 52, 140, 138, 157, 221, 192, 210, 14, 204, 81, 59, 187, 43, 157, 223, 228, 216, 2, 231, 54, 53, 83, 182, 4, 98, 214, 42, 128, 215, 175, 246, 56, 14, 90, 32, 216, 35, 78, 163, 227, 210, 202, 44, 137, 239, 214, 235, 114, 204, 2, 139, 138, 92, 225, 95, 158, 167, 240, 130, 42, 197, 137, 45, 233, 124, 168, 90, 112, 146, 82, 59, 0, 123, 38, 139, 162, 16, 193, 171, 172, 159, 194, 7, 221, 198, 112, 194, 236, 250, 248, 1, 170, 46, 102, 225, 80, 237, 186, 154, 124, 218, 174, 105, 182, 133, 165, 204, 238, 10, 243, 142, 86, 233, 198, 115, 248, 92, 187, 27, 181, 53, 235, 186, 17, 110, 213, 223, 43, 240, 230, 104, 183, 148, 59, 20, 104, 96, 177, 168, 35, 77, 57, 80, 161, 202, 171, 144, 198, 221, 200, 149, 104, 123, 249, 242, 6, 180, 213, 215, 173, 29, 235, 151, 240, 185, 246, 88, 47, 169, 91, 55, 218, 225, 118, 253, 44, 117, 84, 113, 9, 67, 251, 253, 27, 48, 52, 80, 215, 216, 112, 176, 101, 71, 178, 198, 193, 136, 167, 10, 90, 167, 234, 158, 90, 179, 19, 79, 217, 188, 218, 11, 188, 41, 44, 157, 194, 125, 237, 140, 16, 113, 206, 141, 219, 190, 10, 63, 215, 26, 7, 168, 177, 10, 103, 157, 22, 54, 157, 162, 107, 51, 190, 17, 162, 128, 202, 241, 192, 105, 37, 17, 153, 78, 111, 32, 34, 129, 186, 126, 226, 68, 196, 182, 18, 113, 101, 121, 3, 226, 2, 117, 253, 196, 225, 80, 145, 146, 88, 171, 251, 151, 215, 213, 252, 75, 246, 169, 163, 15, 47, 119, 201, 12, 19, 155, 242, 149, 88, 232, 89, 93, 225, 164, 78, 64, 77, 200, 2, 209, 78, 90, 128, 15, 35, 87, 66, 111, 30, 109, 222, 48, 89, 228, 216, 139, 229, 59, 119, 205, 173, 177, 171, 237, 221, 53, 0, 110, 220, 233, 238, 242, 205, 146, 68, 237, 254, 226, 135, 245, 244, 171, 203, 129, 214, 182, 98, 164, 216, 225, 198, 35, 64, 216, 109, 69, 139, 33, 62, 220, 192, 175, 126, 178, 29, 43, 119, 187, 22, 158, 216, 157, 167, 134, 211, 100, 146, 13, 99, 113, 154, 216, 176, 79, 36, 90, 189, 228, 71, 187, 177, 32, 122, 58, 11, 186, 178, 82, 103, 193, 162, 133, 105, 20, 70, 171, 65, 95, 133, 1, 89, 67, 127, 8, 204, 215, 95, 71, 237, 190, 174, 64, 6, 147, 22, 199, 147, 73, 107, 3, 107, 139, 125, 123, 189, 142, 30, 96, 236, 112, 58, 10, 133, 71, 193, 58, 124, 32, 87, 173, 212, 23, 27, 115, 173, 42, 141, 59, 170, 7, 242, 186, 29, 133, 129, 93, 185, 163, 124, 184, 87, 173, 37, 198, 62, 216, 89, 105, 46, 37, 60, 125, 103, 17, 68, 136, 207, 112, 8, 105, 208, 138, 184, 140, 4, 69, 252, 21, 79, 241, 228, 77, 194, 162, 48, 126, 210, 121, 103, 119, 219, 68, 148, 177, 124, 3, 153, 121, 140, 159, 130, 193, 127, 145, 148, 165, 121, 250, 61, 125, 248, 18, 142, 135, 235, 135, 223, 162, 117, 104, 152, 100, 104, 85, 251, 41, 173, 123, 15, 173, 103, 144, 181, 218, 150, 180, 44, 192, 42, 22, 242, 180, 144, 56, 106, 17, 235, 211, 85, 90, 53, 138, 102, 100, 12, 183, 241, 188, 216, 122, 111, 26, 61, 83, 73, 234, 149, 179, 196, 6, 7, 196, 228, 71, 189, 156, 223, 62, 208, 61, 204, 6, 144, 83, 92, 208, 199, 106, 67, 130, 42, 243, 190, 86, 84, 142, 226, 10, 47, 18, 29, 168, 53, 25, 89, 213, 201, 145, 148, 146, 172, 237, 7, 158, 122, 111, 116, 186, 6, 62, 122, 105, 250, 84, 164, 109, 78, 89, 147, 102, 72, 116, 186, 230, 148, 166, 107, 172, 193, 131, 33, 53, 149, 142, 79, 61, 111, 108, 150, 66, 251, 70, 228, 121, 114, 255, 59, 68, 16, 81, 146, 80, 119, 225, 73, 125, 86, 89, 238, 30, 200, 84, 181, 106, 110, 195, 189, 97, 146, 52, 41, 126, 132, 168, 67, 147, 189, 128, 183, 79, 109, 32, 92, 181, 175, 195, 226, 202, 193, 135, 170, 187, 111, 235, 38, 109, 242, 202, 108, 74, 199, 45, 59, 231, 69, 112, 47, 26, 183, 76, 109, 134, 183, 97, 183, 182, 53, 166, 234, 37, 194, 129, 91, 99, 156, 183, 253, 160, 170, 158, 229, 58, 42, 172, 171, 226, 54, 53, 3, 168, 251, 201, 68, 167, 239, 245, 22, 124, 243, 35, 205, 218, 118, 227, 93, 167, 175, 157, 104, 119, 163, 19, 117, 119, 61, 77, 192, 40, 201, 38, 216, 24, 120, 94, 247, 74, 184, 111, 25, 86, 23, 157, 30, 121, 131, 51, 240, 202, 80, 141, 211, 182, 184, 141, 144, 134, 94, 68, 74, 177, 34, 43, 223, 12, 107, 147, 254, 7, 145, 250, 222, 197, 201, 79, 102, 240, 218, 143, 140, 13, 182, 5, 28, 27, 103, 14, 252, 192, 121, 252, 140, 90, 209, 7, 190, 166, 188, 175, 24, 196, 150, 221, 125, 187, 96, 179, 185, 90, 162, 106, 50, 187, 187, 251, 217, 104, 22, 245, 243, 238, 227, 93, 182, 41, 230, 139, 222, 201, 139, 151, 79, 67, 29, 136, 124, 151, 173, 208, 74, 255, 230, 215, 173, 88, 226, 120, 131, 91, 206, 98, 225, 133, 89, 215, 174, 132, 152, 189, 244, 6, 176, 120, 207, 190, 235, 76, 238, 10, 210, 155, 125, 241, 246, 203, 85, 119, 5, 83, 178, 200, 190, 96, 178, 253, 155, 129, 249, 117, 110, 155, 232, 220, 118, 64, 231, 96, 230, 107, 11, 36, 116, 70, 255, 145, 207, 95, 192, 252, 15, 89, 126, 206, 218, 220, 51, 91, 143, 1, 62, 184, 243, 88, 141, 87, 205, 222, 99, 107, 72, 155, 237, 62, 14, 109, 70, 89, 247, 254, 227, 48, 227, 169, 187, 251, 5, 237, 64, 182, 248, 185, 194, 30, 228, 37, 76, 93, 247, 46, 100, 181, 111, 14, 195, 168, 178, 155, 237, 85, 86, 161, 24, 6, 86, 101, 74, 187, 4, 99, 121, 143, 208, 26, 160, 129, 151, 169, 196, 142, 236, 27, 149, 112, 122, 131, 86, 218, 155, 203, 250, 144, 132, 126, 151, 235, 53, 96, 205, 144, 167, 123, 145, 137, 54, 121, 201, 243, 61, 25, 116, 242, 95, 112, 7, 160, 136, 61, 249, 207, 151, 123, 58, 4, 229, 191, 95, 88, 191, 251, 123, 56, 50, 29, 237, 169, 192, 148, 255, 252, 153, 254, 28, 211, 159, 191, 208, 159, 211, 61, 19, 204, 242, 130, 114, 207, 196, 180, 188, 224, 114, 207, 10, 109, 69, 241, 153, 191, 248, 239, 118, 177, 239, 82, 43, 196, 111, 155, 203, 215, 201, 176, 122, 99, 94, 20, 213, 202, 54, 164, 49, 180, 181, 188, 103, 54, 38, 30, 51, 31, 204, 222, 228, 44, 242, 60, 119, 157, 183, 38, 4, 241, 67, 104, 164, 241, 189, 23, 123, 45, 238, 122, 94, 51, 160, 253, 61, 111, 65, 240, 98, 254, 30, 83, 125, 144, 171, 157, 7, 215, 34, 7, 122, 211, 243, 113, 217, 33, 7, 62, 188, 49, 251, 193, 149, 67, 147, 245, 174, 3, 248, 168, 69, 3, 167, 45, 145, 239, 127, 142, 146, 253, 166, 244, 169, 183, 244, 210, 91, 122, 230, 45, 253, 187, 85, 202, 123, 198, 223, 143, 0, 92, 123, 184, 31, 221, 243, 71, 8, 106, 163, 135, 204, 192, 212, 68, 27, 146, 255, 140, 141, 247, 151, 160, 226, 57, 28, 63, 166, 250, 138, 50, 31, 228, 86, 69, 75, 31, 180, 219, 11, 47, 51, 250, 222, 210, 145, 183, 244, 103, 111, 233, 212, 91, 90, 122, 75, 199, 222, 210, 95, 106, 7, 164, 38, 102, 213, 220, 80, 57, 179, 107, 113, 82, 38, 217, 86, 96, 229, 75, 111, 55, 94, 44, 237, 70, 32, 160, 109, 64, 138, 71, 180, 110, 198, 19, 11, 217, 95, 77, 228, 101, 6, 158, 14, 8, 53, 74, 43, 29, 79, 9, 77, 222, 112, 183, 137, 219, 223, 192, 145, 149, 32, 150, 227, 222, 123, 247, 133, 192, 154, 70, 49, 187, 140, 248, 202, 130, 252, 241, 139, 103, 174, 8, 52, 106, 246, 94, 161, 179, 243, 152, 233, 184, 145, 159, 113, 35, 227, 235, 52, 130, 111, 98, 55, 60, 191, 146, 78, 208, 132, 197, 106, 44, 78, 115, 94, 107, 214, 138, 139, 170, 59, 201, 79, 79, 226, 172, 197, 156, 131, 211, 147, 92, 186, 10, 173, 94, 50, 233, 14, 217, 80, 166, 229, 8, 158, 224, 159, 18, 34, 201, 186, 195, 147, 110, 158, 244, 225, 57, 251, 241, 252, 39, 254, 67, 62, 77, 187, 241, 52, 133, 39, 240, 71, 151, 157, 100, 188, 29, 248, 163, 203, 212, 91, 138, 246, 240, 15, 253, 52, 201, 242, 222, 40, 21, 228, 161, 95, 250, 249, 40, 30, 195, 35, 248, 163, 203, 198, 19, 214, 201, 156, 87, 169, 38, 35, 83, 62, 153, 95, 64, 217, 228, 196, 20, 77, 1, 82, 224, 22, 95, 245, 147, 140, 5, 139, 167, 252, 81, 118, 102, 74, 103, 231, 28, 45, 252, 209, 101, 85, 204, 193, 224, 15, 42, 91, 136, 178, 5, 42, 43, 231, 188, 172, 84, 103, 91, 91, 167, 113, 23, 78, 203, 118, 71, 201, 236, 132, 121, 174, 240, 151, 249, 225, 124, 36, 226, 238, 11, 120, 18, 79, 226, 73, 239, 135, 239, 84, 133, 44, 239, 158, 228, 188, 195, 217, 9, 42, 203, 22, 156, 42, 195, 213, 114, 152, 117, 79, 146, 238, 160, 128, 226, 114, 112, 98, 149, 103, 19, 40, 63, 27, 86, 164, 156, 3, 159, 92, 150, 184, 176, 55, 98, 67, 194, 145, 12, 135, 164, 156, 119, 185, 87, 146, 22, 5, 228, 176, 36, 144, 125, 78, 110, 191, 156, 224, 194, 164, 228, 56, 203, 25, 46, 20, 212, 14, 40, 228, 240, 132, 83, 53, 72, 73, 97, 33, 26, 34, 164, 166, 188, 245, 148, 182, 158, 10, 185, 42, 19, 92, 120, 58, 227, 44, 166, 13, 77, 185, 108, 76, 7, 4, 103, 198, 69, 56, 235, 209, 194, 137, 232, 17, 193, 153, 9, 81, 161, 56, 103, 188, 245, 89, 89, 224, 194, 82, 240, 243, 156, 64, 206, 57, 67, 98, 138, 243, 50, 22, 195, 167, 91, 191, 28, 117, 123, 83, 62, 212, 240, 135, 20, 50, 69, 200, 74, 249, 164, 251, 51, 124, 119, 30, 87, 232, 113, 133, 30, 15, 133, 118, 13, 99, 140, 114, 56, 207, 23, 115, 183, 37, 166, 103, 227, 216, 82, 56, 86, 204, 116, 7, 10, 229, 55, 82, 206, 20, 73, 8, 39, 65, 127, 62, 231, 200, 225, 143, 41, 188, 96, 154, 61, 228, 200, 75, 12, 187, 152, 115, 142, 193, 159, 142, 58, 235, 0, 6, 237, 104, 126, 82, 249, 108, 218, 79, 191, 2, 248, 203, 151, 10, 197, 147, 39, 240, 251, 201, 83, 245, 251, 217, 63, 224, 247, 179, 183, 234, 247, 187, 55, 240, 251, 221, 11, 245, 251, 119, 14, 255, 187, 134, 255, 231, 91, 248, 125, 240, 68, 253, 142, 23, 92, 84, 178, 185, 38, 242, 228, 148, 27, 193, 126, 161, 199, 180, 215, 251, 3, 74, 138, 177, 150, 219, 222, 41, 23, 218, 105, 62, 213, 37, 130, 147, 61, 125, 156, 158, 125, 229, 202, 114, 209, 211, 70, 171, 95, 112, 243, 59, 30, 141, 77, 201, 57, 23, 170, 66, 143, 240, 48, 62, 227, 173, 39, 218, 78, 141, 138, 66, 24, 17, 77, 79, 122, 194, 37, 57, 159, 197, 186, 68, 152, 84, 99, 176, 57, 218, 145, 22, 192, 83, 94, 97, 161, 233, 63, 229, 207, 79, 117, 27, 227, 161, 160, 108, 170, 219, 152, 136, 62, 207, 102, 186, 63, 83, 174, 20, 133, 182, 139, 211, 146, 203, 224, 180, 208, 86, 113, 186, 176, 75, 202, 158, 144, 175, 137, 166, 164, 18, 154, 93, 196, 186, 199, 213, 152, 243, 169, 58, 215, 188, 172, 38, 156, 188, 243, 68, 183, 85, 101, 3, 139, 79, 139, 147, 190, 53, 38, 139, 41, 111, 107, 82, 20, 70, 178, 146, 139, 10, 132, 203, 51, 85, 114, 51, 255, 231, 172, 72, 6, 73, 81, 36, 16, 230, 67, 17, 44, 38, 178, 201, 249, 2, 126, 22, 234, 245, 159, 173, 248, 100, 228, 0, 159, 140, 130, 192, 103, 46, 240, 89, 8, 184, 55, 117, 128, 89, 81, 8, 248, 15, 23, 248, 143, 32, 240, 185, 11, 124, 30, 4, 190, 112, 129, 47, 130, 192, 11, 23, 120, 17, 2, 238, 15, 28, 96, 86, 20, 2, 46, 93, 224, 18, 3, 51, 67, 106, 160, 147, 19, 7, 58, 57, 9, 161, 78, 122, 46, 112, 47, 8, 220, 119, 129, 251, 97, 58, 50, 23, 58, 11, 66, 15, 92, 170, 7, 65, 170, 7, 67, 23, 120, 24, 68, 205, 167, 66, 10, 125, 58, 11, 161, 158, 185, 12, 153, 5, 25, 50, 115, 25, 50, 235, 135, 128, 11, 183, 135, 69, 176, 135, 133, 43, 211, 69, 80, 166, 11, 87, 62, 138, 50, 8, 236, 138, 105, 17, 20, 211, 226, 210, 5, 190, 12, 1, 243, 73, 159, 2, 243, 87, 143, 248, 71, 165, 116, 53, 160, 28, 132, 161, 221, 49, 44, 103, 97, 104, 151, 123, 229, 31, 97, 104, 215, 36, 148, 231, 65, 232, 185, 203, 146, 121, 144, 37, 103, 19, 7, 248, 108, 18, 2, 94, 184, 230, 116, 17, 52, 167, 11, 23, 243, 34, 140, 217, 85, 197, 69, 22, 4, 118, 89, 189, 8, 169, 203, 201, 137, 195, 13, 40, 10, 0, 15, 82, 7, 24, 124, 94, 63, 163, 79, 6, 99, 23, 122, 28, 132, 62, 117, 186, 8, 69, 8, 122, 90, 26, 224, 220, 177, 31, 80, 20, 66, 253, 135, 139, 250, 143, 160, 33, 59, 249, 195, 209, 47, 40, 10, 65, 87, 167, 14, 52, 43, 10, 144, 125, 230, 40, 24, 20, 133, 128, 29, 1, 57, 161, 162, 71, 232, 56, 155, 187, 208, 243, 16, 234, 75, 199, 222, 240, 136, 202, 143, 186, 215, 119, 92, 10, 40, 66, 208, 151, 35, 4, 236, 160, 238, 133, 167, 186, 222, 169, 195, 107, 40, 10, 160, 158, 58, 195, 200, 221, 126, 63, 112, 30, 59, 192, 121, 28, 224, 71, 111, 230, 184, 9, 80, 20, 192, 92, 58, 19, 12, 20, 133, 122, 88, 58, 51, 12, 20, 5, 161, 29, 1, 233, 133, 45, 112, 207, 181, 192, 189, 176, 5, 238, 149, 142, 206, 64, 81, 16, 218, 145, 62, 40, 10, 66, 187, 67, 83, 6, 53, 172, 231, 90, 247, 94, 216, 186, 247, 202, 194, 133, 46, 130, 208, 151, 142, 13, 134, 162, 192, 80, 94, 186, 146, 125, 25, 146, 236, 126, 238, 80, 13, 69, 1, 58, 250, 238, 72, 246, 195, 35, 217, 119, 185, 221, 15, 115, 187, 63, 119, 12, 124, 127, 62, 11, 200, 118, 210, 115, 20, 18, 138, 2, 168, 19, 151, 144, 36, 76, 72, 226, 14, 123, 18, 30, 246, 164, 116, 184, 13, 69, 33, 232, 202, 25, 200, 164, 26, 133, 160, 7, 110, 47, 7, 225, 94, 14, 220, 193, 25, 132, 7, 103, 224, 242, 100, 16, 230, 201, 160, 116, 41, 41, 131, 148, 136, 172, 6, 129, 230, 25, 14, 175, 4, 14, 93, 219, 58, 12, 219, 214, 97, 238, 132, 96, 80, 132, 160, 199, 249, 216, 64, 187, 44, 25, 134, 89, 50, 116, 109, 201, 48, 108, 75, 134, 165, 75, 73, 57, 13, 67, 187, 189, 172, 97, 224, 220, 133, 158, 7, 161, 71, 177, 227, 194, 143, 226, 147, 48, 180, 99, 95, 161, 40, 8, 237, 184, 59, 60, 253, 228, 29, 202, 145, 59, 148, 163, 240, 80, 142, 78, 29, 191, 11, 138, 252, 10, 63, 26, 187, 168, 199, 97, 212, 185, 11, 157, 135, 161, 103, 46, 244, 44, 12, 93, 58, 42, 12, 69, 65, 104, 71, 205, 70, 97, 53, 27, 185, 166, 103, 84, 134, 20, 39, 237, 57, 168, 161, 40, 128, 58, 157, 56, 157, 132, 162, 16, 116, 230, 226, 206, 194, 184, 51, 23, 119, 22, 198, 237, 42, 101, 26, 86, 202, 212, 85, 202, 52, 172, 148, 169, 59, 173, 166, 225, 105, 245, 212, 21, 238, 83, 42, 220, 72, 2, 79, 99, 199, 157, 130, 162, 0, 176, 107, 185, 79, 195, 150, 251, 116, 228, 66, 143, 194, 208, 238, 72, 158, 134, 71, 242, 212, 85, 133, 211, 176, 42, 156, 186, 226, 122, 26, 22, 215, 83, 215, 76, 157, 134, 205, 212, 120, 232, 232, 59, 20, 133, 160, 51, 71, 21, 160, 40, 96, 232, 199, 103, 142, 9, 132, 34, 255, 224, 140, 207, 156, 97, 135, 162, 16, 33, 103, 142, 72, 65, 81, 0, 245, 133, 227, 24, 67, 145, 31, 120, 114, 226, 176, 15, 138, 2, 116, 76, 122, 142, 218, 64, 81, 0, 117, 207, 49, 242, 80, 20, 0, 78, 157, 76, 3, 20, 5, 128, 93, 233, 155, 132, 165, 111, 226, 106, 239, 36, 172, 189, 19, 87, 250, 38, 97, 233, 155, 184, 158, 215, 36, 236, 121, 77, 220, 84, 205, 36, 156, 170, 153, 240, 108, 185, 5, 93, 133, 161, 157, 128, 15, 138, 66, 208, 149, 203, 147, 138, 240, 100, 114, 102, 128, 207, 92, 118, 159, 149, 33, 96, 55, 76, 152, 4, 195, 132, 169, 107, 207, 166, 65, 123, 54, 237, 59, 35, 3, 69, 129, 30, 78, 19, 103, 100, 160, 40, 128, 122, 224, 196, 180, 80, 20, 2, 118, 244, 124, 58, 8, 233, 57, 44, 40, 58, 192, 65, 81, 21, 235, 104, 20, 58, 13, 165, 106, 166, 153, 35, 77, 80, 20, 224, 244, 31, 206, 128, 67, 81, 0, 115, 225, 168, 57, 20, 133, 136, 118, 3, 241, 105, 56, 16, 159, 150, 142, 5, 134, 162, 0, 33, 238, 84, 58, 13, 79, 165, 211, 185, 139, 122, 30, 68, 125, 233, 36, 27, 160, 40, 132, 250, 210, 101, 223, 101, 208, 134, 76, 47, 29, 93, 132, 162, 0, 116, 230, 70, 42, 89, 48, 82, 201, 92, 123, 157, 133, 237, 117, 230, 186, 0, 89, 216, 5, 200, 220, 161, 201, 202, 224, 228, 152, 185, 134, 50, 11, 27, 202, 204, 53, 125, 89, 216, 244, 101, 238, 184, 103, 225, 113, 207, 220, 164, 91, 22, 78, 186, 229, 99, 135, 110, 40, 10, 65, 23, 206, 224, 64, 145, 95, 166, 114, 119, 181, 33, 167, 171, 13, 8, 120, 150, 56, 116, 64, 81, 0, 120, 232, 24, 40, 40, 242, 39, 147, 103, 110, 156, 50, 11, 199, 41, 179, 194, 165, 163, 8, 242, 99, 230, 46, 123, 204, 232, 178, 7, 129, 118, 51, 122, 179, 112, 70, 111, 230, 26, 146, 89, 216, 144, 204, 92, 23, 126, 86, 134, 124, 145, 153, 235, 3, 204, 194, 62, 192, 204, 21, 237, 89, 88, 180, 103, 174, 15, 48, 11, 251, 0, 51, 87, 17, 102, 97, 69, 152, 185, 138, 48, 11, 43, 194, 108, 225, 14, 252, 34, 56, 240, 197, 212, 129, 134, 162, 16, 180, 107, 28, 138, 176, 113, 40, 92, 14, 22, 97, 14, 150, 238, 204, 94, 134, 103, 246, 210, 157, 127, 203, 65, 48, 213, 80, 186, 19, 112, 25, 158, 128, 97, 155, 147, 13, 61, 12, 202, 73, 57, 116, 220, 23, 40, 10, 65, 143, 156, 133, 4, 40, 242, 235, 112, 57, 113, 156, 103, 40, 10, 161, 118, 29, 226, 50, 236, 16, 151, 127, 56, 177, 7, 20, 5, 161, 93, 220, 127, 132, 113, 187, 153, 137, 178, 12, 173, 212, 149, 174, 46, 148, 97, 93, 40, 93, 93, 40, 195, 186, 80, 186, 1, 86, 25, 14, 176, 202, 115, 199, 80, 65, 17, 134, 62, 71, 192, 110, 31, 207, 71, 65, 96, 87, 182, 207, 195, 178, 189, 112, 60, 7, 40, 10, 64, 87, 83, 39, 104, 130, 34, 191, 9, 172, 92, 123, 89, 133, 83, 30, 149, 155, 89, 172, 194, 153, 197, 202, 93, 119, 168, 194, 235, 14, 149, 155, 135, 172, 194, 121, 200, 202, 13, 108, 170, 112, 96, 83, 93, 58, 19, 37, 20, 5, 160, 231, 67, 103, 118, 135, 162, 32, 180, 67, 9, 20, 133, 160, 93, 63, 99, 30, 246, 51, 230, 238, 156, 61, 15, 207, 217, 243, 194, 17, 110, 40, 242, 15, 252, 252, 210, 237, 228, 37, 233, 228, 252, 18, 1, 187, 116, 80, 95, 10, 1, 195, 14, 82, 11, 152, 111, 42, 245, 19, 125, 54, 118, 136, 134, 34, 63, 209, 103, 99, 23, 245, 184, 10, 1, 187, 115, 211, 89, 120, 110, 58, 115, 231, 166, 179, 240, 220, 116, 86, 58, 155, 179, 160, 40, 0, 45, 246, 53, 18, 104, 190, 199, 209, 235, 203, 95, 140, 29, 178, 161, 40, 128, 250, 98, 234, 144, 13, 69, 65, 104, 71, 127, 161, 200, 207, 191, 11, 215, 19, 184, 8, 123, 2, 11, 55, 195, 190, 8, 103, 216, 23, 46, 183, 23, 97, 110, 139, 189, 158, 20, 122, 158, 4, 248, 119, 153, 58, 142, 0, 20, 5, 80, 95, 78, 28, 142, 64, 145, 159, 35, 151, 83, 103, 104, 160, 40, 0, 236, 118, 241, 50, 220, 197, 75, 215, 178, 94, 150, 30, 58, 240, 217, 188, 195, 42, 41, 226, 42, 47, 14, 251, 73, 86, 165, 213, 194, 126, 89, 27, 28, 2, 85, 7, 30, 95, 36, 195, 184, 183, 80, 53, 126, 43, 98, 184, 120, 248, 101, 60, 147, 119, 253, 194, 105, 139, 223, 146, 120, 204, 74, 218, 248, 154, 111, 90, 237, 85, 114, 81, 181, 227, 98, 72, 14, 164, 86, 252, 154, 100, 231, 236, 134, 196, 6, 167, 130, 59, 193, 214, 201, 233, 218, 170, 88, 160, 147, 103, 234, 188, 182, 56, 137, 177, 199, 219, 217, 202, 20, 1, 157, 168, 207, 15, 139, 242, 3, 14, 234, 108, 232, 21, 163, 130, 31, 97, 75, 236, 11, 24, 147, 40, 205, 202, 42, 206, 122, 73, 62, 224, 125, 61, 170, 242, 153, 32, 135, 209, 235, 61, 15, 175, 27, 70, 55, 227, 138, 54, 225, 228, 134, 106, 146, 211, 13, 151, 48, 71, 250, 30, 174, 171, 16, 247, 248, 101, 205, 237, 228, 34, 251, 108, 236, 171, 52, 5, 255, 138, 252, 107, 67, 31, 204, 245, 133, 1, 134, 149, 203, 25, 214, 225, 220, 160, 231, 218, 69, 141, 231, 73, 38, 85, 234, 22, 219, 116, 213, 17, 95, 98, 95, 119, 111, 60, 185, 125, 223, 66, 65, 78, 137, 207, 154, 188, 197, 102, 38, 222, 115, 194, 84, 170, 57, 228, 150, 58, 255, 229, 26, 6, 7, 52, 225, 7, 232, 224, 98, 5, 122, 85, 29, 2, 233, 229, 217, 32, 29, 206, 21, 144, 185, 133, 16, 193, 156, 23, 105, 229, 127, 126, 12, 68, 167, 146, 132, 247, 141, 58, 65, 171, 232, 222, 216, 150, 52, 92, 163, 182, 83, 22, 108, 125, 239, 44, 96, 111, 55, 93, 81, 121, 83, 228, 85, 238, 237, 234, 243, 164, 34, 64, 213, 98, 150, 180, 55, 58, 162, 49, 201, 131, 151, 241, 56, 57, 80, 47, 213, 76, 89, 107, 109, 138, 221, 150, 98, 142, 136, 8, 129, 80, 214, 70, 242, 194, 65, 3, 2, 195, 109, 161, 11, 92, 47, 50, 2, 102, 137, 204, 8, 160, 37, 220, 116, 244, 252, 118, 216, 234, 52, 211, 169, 109, 95, 98, 11, 219, 134, 173, 148, 216, 9, 209, 51, 100, 196, 94, 37, 231, 30, 147, 217, 137, 206, 133, 49, 82, 22, 2, 44, 250, 157, 102, 173, 24, 211, 238, 53, 57, 228, 134, 5, 240, 35, 100, 75, 237, 144, 129, 85, 117, 143, 70, 233, 212, 118, 87, 92, 234, 245, 77, 167, 124, 154, 179, 133, 179, 145, 1, 191, 73, 67, 14, 50, 218, 34, 179, 237, 207, 242, 226, 105, 220, 27, 241, 147, 171, 39, 113, 111, 60, 200, 132, 181, 127, 92, 12, 35, 207, 53, 37, 32, 128, 47, 229, 93, 61, 230, 158, 147, 59, 135, 165, 108, 253, 101, 224, 69, 37, 157, 168, 197, 26, 107, 137, 91, 82, 160, 65, 246, 21, 228, 144, 77, 198, 47, 189, 179, 191, 51, 83, 193, 44, 53, 138, 217, 44, 164, 224, 253, 115, 253, 138, 141, 94, 225, 62, 28, 176, 54, 65, 229, 16, 51, 252, 221, 249, 161, 19, 61, 73, 122, 249, 116, 150, 78, 18, 198, 168, 246, 118, 39, 194, 85, 140, 174, 50, 91, 92, 136, 43, 150, 188, 253, 73, 181, 82, 41, 122, 228, 149, 10, 252, 104, 173, 117, 103, 132, 190, 238, 164, 22, 213, 7, 152, 113, 58, 170, 97, 235, 53, 28, 2, 197, 22, 56, 49, 212, 227, 177, 78, 60, 43, 210, 23, 226, 69, 76, 80, 199, 186, 110, 129, 208, 224, 145, 29, 65, 193, 226, 120, 231, 189, 250, 182, 253, 30, 13, 157, 244, 33, 240, 139, 54, 15, 216, 248, 36, 206, 123, 59, 140, 97, 224, 160, 31, 170, 92, 92, 143, 36, 223, 52, 98, 191, 154, 212, 97, 141, 172, 198, 169, 135, 43, 244, 144, 183, 73, 47, 159, 186, 246, 245, 69, 107, 184, 117, 12, 179, 193, 40, 149, 166, 205, 190, 3, 209, 127, 15, 179, 229, 239, 58, 172, 221, 194, 37, 161, 59, 156, 225, 179, 172, 166, 184, 105, 9, 191, 58, 84, 145, 138, 239, 30, 243, 97, 139, 246, 151, 32, 87, 138, 109, 93, 67, 114, 221, 246, 221, 171, 186, 201, 155, 105, 49, 90, 231, 150, 40, 41, 56, 105, 249, 12, 230, 18, 240, 13, 166, 120, 242, 145, 102, 132, 149, 138, 203, 86, 4, 46, 116, 192, 93, 182, 138, 60, 0, 53, 199, 176, 42, 93, 254, 63, 140, 193, 182, 191, 209, 87, 241, 171, 245, 182, 8, 85, 216, 95, 127, 107, 71, 241, 32, 145, 175, 72, 149, 76, 241, 55, 204, 30, 112, 68, 119, 197, 247, 187, 117, 109, 115, 147, 234, 97, 34, 32, 175, 171, 167, 110, 136, 26, 242, 160, 242, 93, 110, 209, 69, 239, 214, 26, 74, 130, 228, 195, 37, 228, 232, 11, 48, 227, 147, 82, 85, 223, 128, 251, 89, 126, 216, 222, 254, 110, 231, 135, 31, 118, 191, 185, 255, 221, 253, 237, 31, 126, 216, 113, 16, 25, 7, 140, 98, 70, 204, 124, 62, 201, 79, 226, 137, 25, 58, 196, 68, 53, 103, 227, 193, 213, 23, 153, 41, 150, 248, 145, 81, 190, 249, 209, 73, 152, 122, 140, 98, 122, 62, 226, 47, 160, 127, 92, 150, 233, 48, 107, 87, 204, 48, 65, 212, 55, 72, 139, 178, 58, 202, 231, 69, 143, 188, 168, 144, 59, 147, 198, 171, 224, 192, 136, 253, 142, 177, 123, 136, 47, 16, 81, 124, 203, 233, 53, 73, 242, 254, 160, 126, 110, 77, 109, 48, 103, 9, 10, 176, 25, 117, 94, 254, 138, 193, 246, 197, 53, 243, 112, 189, 132, 85, 28, 48, 107, 204, 233, 174, 210, 12, 207, 97, 252, 18, 214, 34, 159, 226, 126, 26, 92, 27, 20, 112, 156, 44, 74, 126, 29, 43, 131, 70, 87, 121, 192, 205, 126, 109, 64, 98, 129, 139, 155, 143, 117, 37, 114, 247, 49, 238, 181, 186, 250, 21, 93, 100, 220, 3, 55, 60, 122, 121, 120, 116, 116, 248, 234, 57, 28, 211, 189, 162, 213, 102, 73, 214, 103, 115, 223, 211, 139, 94, 194, 173, 29, 3, 145, 192, 6, 78, 186, 17, 166, 5, 231, 154, 101, 76, 4, 235, 4, 38, 246, 88, 87, 179, 238, 5, 166, 30, 26, 198, 210, 79, 202, 94, 84, 243, 82, 190, 39, 236, 121, 145, 206, 192, 81, 5, 102, 117, 84, 179, 193, 203, 101, 25, 58, 231, 238, 95, 40, 69, 241, 149, 255, 90, 152, 42, 63, 150, 184, 33, 156, 134, 198, 244, 111, 235, 70, 41, 175, 251, 136, 169, 112, 25, 189, 175, 89, 237, 111, 220, 51, 52, 214, 21, 85, 244, 202, 24, 205, 104, 114, 11, 23, 145, 121, 7, 229, 157, 16, 17, 34, 174, 181, 225, 213, 148, 42, 37, 98, 115, 19, 110, 235, 178, 149, 151, 70, 65, 160, 181, 142, 233, 120, 194, 199, 225, 40, 129, 235, 102, 218, 89, 60, 133, 151, 91, 240, 31, 228, 6, 86, 14, 106, 140, 4, 120, 91, 68, 85, 117, 129, 239, 253, 17, 188, 50, 92, 10, 200, 173, 159, 192, 66, 95, 32, 161, 32, 104, 176, 67, 226, 16, 237, 195, 75, 234, 188, 254, 251, 206, 247, 252, 18, 117, 0, 192, 23, 10, 142, 185, 30, 188, 203, 145, 114, 243, 174, 34, 16, 41, 232, 70, 96, 62, 124, 152, 65, 224, 251, 225, 3, 190, 2, 149, 83, 174, 5, 85, 228, 11, 59, 88, 193, 117, 248, 111, 63, 99, 68, 237, 73, 198, 138, 129, 19, 77, 35, 197, 178, 175, 224, 231, 165, 29, 160, 189, 195, 169, 243, 217, 125, 49, 120, 207, 241, 224, 13, 191, 232, 193, 27, 46, 27, 188, 225, 151, 56, 120, 67, 24, 188, 225, 186, 7, 239, 5, 127, 35, 36, 210, 60, 60, 104, 13, 122, 93, 203, 120, 103, 42, 94, 193, 146, 35, 234, 173, 56, 147, 119, 197, 178, 170, 171, 92, 143, 5, 245, 133, 134, 182, 60, 183, 97, 73, 51, 197, 167, 3, 6, 66, 13, 172, 124, 232, 185, 83, 255, 202, 21, 69, 218, 75, 157, 28, 123, 61, 144, 125, 219, 176, 172, 167, 172, 120, 71, 42, 69, 120, 180, 158, 255, 27, 142, 214, 112, 249, 104, 13, 191, 156, 209, 18, 9, 132, 15, 229, 252, 68, 188, 173, 146, 117, 48, 46, 120, 226, 68, 135, 210, 7, 163, 164, 55, 22, 13, 30, 228, 73, 209, 227, 105, 81, 43, 129, 192, 42, 243, 177, 82, 111, 62, 181, 210, 11, 220, 11, 100, 48, 196, 255, 147, 33, 206, 17, 52, 72, 98, 28, 78, 2, 125, 161, 239, 211, 12, 210, 164, 237, 36, 235, 187, 183, 35, 3, 246, 61, 84, 29, 72, 55, 149, 25, 88, 60, 81, 109, 144, 187, 255, 201, 91, 1, 20, 33, 157, 104, 123, 163, 195, 157, 68, 11, 133, 160, 160, 22, 193, 83, 184, 103, 216, 83, 157, 123, 122, 42, 6, 0, 137, 66, 52, 61, 212, 216, 177, 232, 73, 119, 220, 192, 25, 129, 224, 241, 136, 170, 19, 72, 20, 224, 234, 79, 241, 109, 201, 166, 50, 194, 74, 210, 3, 71, 74, 16, 126, 73, 138, 44, 153, 176, 62, 22, 29, 129, 239, 99, 180, 221, 97, 179, 97, 206, 162, 118, 238, 232, 67, 129, 95, 150, 120, 116, 69, 68, 10, 254, 48, 71, 201, 17, 173, 64, 4, 181, 227, 79, 158, 178, 9, 143, 145, 112, 87, 52, 179, 165, 241, 223, 165, 110, 27, 81, 87, 91, 188, 129, 8, 135, 152, 176, 66, 40, 109, 160, 183, 215, 222, 92, 33, 26, 137, 125, 153, 94, 38, 1, 181, 73, 132, 58, 72, 134, 185, 247, 133, 67, 77, 172, 18, 196, 189, 149, 89, 3, 37, 127, 219, 134, 217, 136, 44, 175, 114, 68, 155, 28, 117, 135, 15, 188, 162, 69, 36, 70, 95, 72, 82, 106, 52, 36, 81, 234, 193, 201, 235, 234, 214, 54, 52, 85, 4, 213, 195, 125, 76, 154, 28, 92, 245, 2, 216, 58, 105, 213, 212, 114, 137, 37, 56, 27, 137, 172, 45, 175, 116, 240, 111, 46, 178, 141, 229, 213, 17, 86, 61, 140, 158, 30, 76, 210, 94, 242, 215, 51, 222, 194, 144, 81, 113, 149, 23, 226, 107, 185, 2, 20, 155, 17, 54, 224, 12, 35, 17, 67, 243, 140, 90, 102, 110, 14, 165, 229, 174, 71, 45, 77, 187, 7, 49, 127, 66, 209, 150, 179, 56, 179, 85, 72, 219, 77, 163, 60, 205, 44, 46, 71, 182, 76, 110, 229, 224, 223, 138, 153, 5, 220, 77, 68, 22, 224, 26, 155, 215, 94, 222, 79, 222, 228, 140, 125, 143, 171, 246, 44, 47, 87, 145, 214, 163, 26, 89, 101, 168, 82, 153, 112, 48, 34, 5, 248, 29, 171, 122, 68, 164, 153, 103, 24, 84, 93, 38, 8, 16, 230, 233, 223, 143, 246, 121, 37, 199, 22, 121, 86, 48, 120, 14, 209, 183, 254, 162, 186, 61, 138, 139, 3, 214, 245, 199, 140, 53, 71, 29, 221, 4, 50, 204, 2, 3, 35, 225, 226, 201, 247, 219, 156, 14, 81, 242, 8, 74, 126, 122, 246, 140, 80, 182, 25, 237, 112, 141, 242, 146, 199, 235, 161, 126, 139, 183, 18, 174, 76, 27, 52, 130, 232, 147, 104, 56, 129, 7, 130, 64, 89, 196, 41, 124, 246, 236, 89, 29, 33, 234, 77, 36, 162, 79, 93, 217, 203, 141, 232, 111, 236, 219, 125, 134, 109, 83, 55, 208, 149, 13, 108, 176, 178, 237, 139, 157, 109, 246, 81, 59, 110, 122, 57, 139, 142, 153, 43, 191, 100, 165, 76, 118, 171, 72, 102, 73, 92, 137, 123, 148, 215, 37, 102, 84, 190, 4, 106, 195, 162, 140, 78, 170, 72, 181, 190, 253, 1, 199, 249, 12, 240, 111, 90, 14, 89, 157, 246, 78, 244, 240, 97, 180, 251, 253, 134, 63, 222, 255, 78, 13, 3, 180, 158, 69, 95, 71, 109, 93, 129, 113, 107, 7, 145, 247, 46, 50, 111, 74, 231, 175, 125, 123, 240, 192, 89, 22, 99, 245, 173, 91, 179, 223, 193, 11, 95, 143, 140, 175, 200, 4, 255, 17, 185, 234, 158, 215, 163, 117, 142, 172, 58, 206, 101, 211, 161, 23, 198, 190, 243, 25, 5, 181, 118, 219, 94, 215, 56, 41, 132, 12, 228, 85, 114, 46, 96, 244, 78, 1, 9, 247, 107, 86, 198, 3, 72, 232, 189, 77, 202, 164, 56, 75, 250, 71, 147, 188, 106, 155, 21, 105, 102, 10, 143, 26, 130, 238, 56, 182, 93, 61, 243, 244, 182, 233, 150, 207, 64, 53, 190, 129, 147, 108, 3, 209, 123, 15, 56, 19, 64, 49, 89, 145, 213, 103, 254, 8, 203, 130, 207, 160, 67, 246, 137, 214, 123, 153, 84, 163, 188, 127, 56, 16, 91, 225, 250, 114, 31, 156, 19, 226, 226, 79, 203, 37, 151, 190, 10, 83, 141, 157, 96, 234, 243, 164, 18, 21, 158, 177, 137, 143, 176, 87, 108, 185, 219, 38, 30, 133, 88, 43, 208, 53, 153, 34, 222, 219, 13, 84, 220, 89, 102, 250, 233, 27, 84, 130, 91, 29, 201, 246, 74, 225, 65, 3, 21, 106, 110, 32, 44, 213, 155, 11, 200, 158, 37, 196, 110, 245, 6, 19, 202, 13, 97, 128, 201, 27, 38, 174, 49, 167, 112, 186, 156, 9, 133, 209, 41, 103, 20, 120, 121, 160, 152, 99, 246, 213, 148, 194, 138, 68, 111, 96, 62, 121, 232, 116, 232, 58, 179, 135, 198, 103, 229, 88, 212, 92, 177, 175, 230, 15, 214, 182, 154, 82, 246, 213, 4, 98, 191, 181, 2, 241, 101, 215, 206, 140, 92, 213, 104, 166, 148, 0, 67, 141, 198, 164, 181, 212, 236, 233, 168, 233, 27, 138, 30, 85, 215, 234, 112, 90, 47, 169, 177, 173, 156, 88, 164, 135, 59, 234, 227, 2, 172, 89, 188, 182, 105, 233, 221, 40, 174, 232, 227, 24, 123, 226, 181, 155, 60, 154, 237, 208, 168, 221, 228, 177, 75, 80, 236, 6, 80, 168, 217, 219, 72, 232, 205, 54, 120, 16, 111, 96, 75, 253, 170, 95, 1, 245, 215, 16, 91, 35, 14, 228, 239, 186, 205, 28, 168, 142, 31, 89, 32, 47, 179, 106, 91, 238, 198, 13, 41, 54, 98, 128, 75, 221, 121, 46, 152, 239, 196, 112, 7, 163, 4, 136, 41, 14, 148, 223, 221, 214, 30, 56, 121, 173, 176, 202, 40, 56, 227, 139, 114, 15, 147, 132, 63, 144, 158, 23, 127, 107, 223, 6, 114, 54, 220, 37, 221, 200, 90, 127, 229, 47, 154, 69, 107, 126, 182, 177, 145, 27, 153, 141, 44, 249, 22, 98, 21, 224, 193, 27, 46, 242, 106, 209, 159, 149, 88, 38, 71, 2, 65, 250, 19, 109, 163, 224, 133, 124, 150, 164, 155, 77, 68, 185, 229, 25, 33, 39, 140, 185, 3, 90, 191, 20, 176, 183, 61, 25, 69, 200, 95, 143, 184, 19, 251, 140, 184, 199, 215, 71, 13, 150, 242, 153, 199, 82, 170, 161, 217, 154, 205, 203, 145, 194, 241, 192, 18, 126, 123, 249, 223, 100, 152, 105, 117, 246, 145, 237, 117, 149, 7, 190, 17, 253, 183, 112, 212, 69, 242, 143, 187, 230, 220, 129, 127, 16, 194, 225, 98, 248, 47, 237, 234, 75, 7, 223, 35, 234, 225, 119, 185, 49, 255, 99, 178, 232, 68, 200, 56, 115, 169, 150, 147, 79, 71, 44, 95, 105, 34, 234, 180, 161, 136, 207, 121, 102, 255, 136, 249, 104, 157, 104, 107, 75, 36, 103, 210, 106, 46, 20, 16, 233, 132, 216, 72, 242, 122, 112, 132, 1, 32, 190, 199, 191, 29, 53, 233, 229, 249, 152, 239, 237, 213, 11, 24, 170, 53, 156, 59, 139, 207, 9, 4, 175, 179, 197, 74, 177, 217, 230, 78, 228, 228, 40, 25, 42, 181, 123, 151, 139, 116, 86, 155, 1, 110, 57, 169, 61, 27, 190, 54, 143, 102, 188, 31, 193, 29, 20, 57, 4, 182, 102, 212, 236, 204, 228, 155, 70, 146, 33, 158, 130, 24, 133, 72, 123, 145, 156, 88, 141, 146, 159, 155, 10, 147, 43, 253, 135, 218, 79, 1, 3, 111, 117, 213, 123, 122, 5, 99, 118, 173, 7, 27, 211, 80, 35, 15, 253, 3, 111, 109, 18, 20, 56, 112, 151, 137, 88, 248, 59, 239, 132, 73, 6, 141, 226, 126, 67, 30, 97, 242, 157, 93, 20, 87, 97, 233, 167, 78, 8, 243, 110, 118, 120, 234, 102, 247, 38, 25, 35, 233, 242, 111, 17, 220, 173, 166, 14, 72, 51, 239, 161, 214, 1, 185, 71, 80, 220, 243, 160, 168, 201, 95, 17, 170, 57, 51, 36, 75, 58, 145, 103, 118, 118, 25, 123, 146, 14, 27, 199, 173, 74, 9, 31, 178, 74, 143, 90, 108, 44, 169, 71, 199, 10, 90, 15, 255, 155, 63, 243, 182, 52, 73, 179, 241, 53, 218, 130, 106, 225, 214, 196, 83, 111, 123, 249, 164, 127, 141, 230, 194, 77, 249, 155, 25, 164, 23, 201, 53, 218, 169, 170, 96, 67, 240, 200, 159, 103, 136, 39, 105, 175, 92, 189, 173, 52, 216, 84, 234, 111, 169, 156, 50, 73, 91, 189, 29, 94, 45, 216, 150, 124, 234, 159, 219, 138, 116, 156, 92, 163, 65, 94, 47, 220, 162, 124, 28, 88, 40, 187, 70, 123, 243, 176, 120, 240, 103, 254, 150, 102, 215, 105, 105, 86, 211, 210, 204, 106, 233, 105, 217, 139, 103, 44, 134, 101, 160, 39, 243, 42, 249, 59, 4, 134, 237, 51, 250, 214, 214, 217, 28, 108, 48, 182, 248, 103, 36, 57, 193, 158, 191, 224, 75, 38, 10, 212, 113, 13, 242, 121, 165, 145, 224, 217, 182, 55, 154, 103, 99, 181, 162, 178, 109, 185, 212, 226, 245, 154, 17, 108, 91, 83, 109, 60, 128, 23, 71, 218, 241, 144, 106, 85, 189, 112, 242, 110, 235, 174, 237, 38, 154, 246, 55, 27, 5, 189, 10, 101, 7, 81, 200, 74, 129, 139, 119, 191, 254, 99, 158, 87, 15, 238, 90, 78, 38, 238, 72, 10, 211, 181, 63, 108, 231, 241, 27, 130, 221, 247, 185, 42, 170, 245, 7, 190, 42, 134, 25, 104, 15, 214, 26, 186, 71, 197, 72, 99, 244, 137, 101, 156, 245, 70, 57, 221, 227, 114, 147, 64, 94, 54, 121, 247, 97, 28, 1, 202, 253, 214, 93, 198, 63, 175, 92, 138, 22, 217, 24, 180, 30, 1, 204, 145, 16, 234, 56, 96, 93, 153, 223, 207, 162, 197, 28, 210, 213, 236, 255, 181, 82, 10, 184, 35, 142, 182, 134, 90, 217, 172, 77, 46, 212, 13, 83, 12, 169, 168, 54, 206, 71, 173, 143, 94, 192, 90, 67, 174, 104, 180, 57, 181, 124, 78, 158, 23, 147, 53, 139, 192, 168, 72, 6, 53, 84, 242, 6, 151, 74, 64, 82, 93, 227, 116, 219, 81, 205, 233, 182, 163, 240, 233, 54, 214, 152, 247, 160, 153, 202, 161, 47, 57, 221, 198, 170, 139, 211, 109, 10, 190, 209, 233, 182, 101, 141, 222, 242, 233, 54, 158, 63, 244, 30, 110, 131, 238, 152, 245, 1, 69, 206, 181, 14, 183, 97, 84, 242, 112, 155, 104, 215, 202, 17, 172, 114, 182, 77, 37, 62, 175, 123, 182, 141, 87, 208, 127, 12, 183, 145, 240, 189, 99, 35, 209, 231, 219, 233, 159, 138, 163, 120, 228, 172, 218, 235, 26, 33, 123, 173, 214, 48, 12, 10, 86, 214, 100, 241, 194, 84, 112, 22, 46, 94, 51, 49, 113, 72, 162, 11, 19, 18, 235, 1, 63, 52, 204, 193, 244, 242, 201, 107, 177, 158, 224, 239, 222, 89, 82, 44, 154, 42, 88, 120, 193, 6, 117, 182, 241, 98, 77, 77, 127, 69, 10, 220, 55, 120, 68, 54, 236, 79, 203, 234, 151, 187, 116, 67, 71, 206, 236, 19, 49, 21, 101, 142, 226, 117, 221, 97, 25, 107, 198, 247, 196, 148, 255, 101, 16, 254, 215, 214, 76, 237, 117, 220, 74, 48, 85, 183, 166, 216, 239, 100, 39, 25, 144, 229, 141, 141, 133, 55, 54, 22, 9, 77, 246, 197, 205, 98, 142, 255, 46, 181, 235, 245, 241, 216, 202, 91, 86, 73, 9, 43, 21, 30, 77, 199, 35, 245, 174, 35, 113, 176, 191, 76, 114, 45, 61, 191, 99, 144, 120, 243, 30, 232, 208, 23, 145, 107, 231, 168, 185, 97, 240, 179, 148, 5, 43, 82, 153, 75, 229, 134, 118, 228, 102, 179, 207, 45, 195, 152, 46, 78, 83, 99, 9, 134, 110, 173, 65, 128, 133, 121, 158, 196, 85, 122, 150, 44, 223, 246, 52, 230, 102, 21, 67, 63, 140, 182, 9, 197, 222, 173, 70, 164, 74, 7, 235, 6, 124, 172, 221, 71, 22, 48, 221, 132, 164, 30, 138, 205, 87, 206, 222, 171, 102, 251, 102, 81, 39, 228, 22, 41, 135, 233, 181, 221, 144, 219, 166, 156, 74, 129, 142, 88, 123, 169, 196, 222, 5, 174, 100, 156, 26, 71, 205, 64, 179, 24, 137, 104, 222, 34, 162, 254, 58, 44, 231, 176, 43, 125, 5, 103, 232, 147, 207, 83, 215, 179, 219, 45, 187, 143, 174, 216, 127, 82, 43, 61, 32, 68, 124, 58, 51, 45, 142, 4, 240, 55, 77, 235, 203, 60, 248, 170, 219, 7, 244, 147, 249, 196, 100, 196, 76, 213, 3, 216, 36, 48, 75, 122, 204, 53, 64, 240, 48, 251, 187, 72, 177, 202, 39, 179, 202, 179, 40, 198, 35, 251, 120, 198, 68, 128, 47, 7, 216, 113, 253, 205, 102, 146, 50, 153, 48, 25, 148, 235, 12, 63, 229, 249, 36, 137, 179, 118, 227, 25, 197, 89, 156, 23, 200, 236, 124, 1, 116, 76, 172, 231, 136, 218, 246, 146, 146, 236, 156, 123, 120, 207, 136, 221, 99, 201, 152, 131, 182, 130, 118, 150, 12, 229, 82, 33, 227, 131, 2, 97, 191, 40, 51, 30, 31, 103, 160, 239, 64, 15, 251, 230, 209, 248, 199, 65, 141, 207, 250, 237, 25, 67, 153, 178, 104, 34, 249, 146, 92, 51, 151, 168, 21, 230, 181, 172, 255, 197, 57, 102, 3, 68, 148, 173, 240, 186, 175, 141, 245, 29, 213, 184, 93, 175, 204, 57, 231, 131, 198, 197, 209, 26, 159, 187, 37, 32, 60, 34, 137, 86, 32, 130, 162, 201, 87, 109, 254, 146, 242, 201, 123, 246, 69, 10, 41, 161, 236, 75, 147, 212, 37, 226, 8, 178, 91, 43, 140, 30, 57, 236, 238, 132, 4, 112, 181, 220, 208, 138, 242, 119, 43, 190, 79, 203, 161, 254, 182, 69, 204, 179, 63, 222, 32, 194, 146, 37, 200, 185, 219, 200, 229, 249, 52, 33, 105, 157, 55, 128, 4, 233, 26, 166, 139, 171, 208, 235, 1, 243, 27, 226, 162, 55, 122, 42, 54, 93, 136, 179, 11, 122, 211, 192, 103, 50, 91, 33, 146, 26, 27, 47, 217, 183, 53, 201, 21, 7, 241, 174, 110, 116, 209, 78, 79, 186, 173, 92, 211, 76, 182, 150, 63, 218, 231, 1, 82, 29, 154, 177, 5, 191, 141, 57, 10, 82, 146, 225, 62, 89, 91, 212, 224, 185, 136, 225, 50, 58, 59, 142, 233, 174, 118, 5, 187, 141, 113, 161, 128, 45, 100, 213, 68, 200, 6, 103, 22, 240, 8, 221, 220, 138, 253, 79, 14, 199, 205, 146, 89, 44, 174, 119, 251, 92, 82, 39, 219, 39, 89, 70, 32, 109, 77, 249, 7, 214, 0, 108, 60, 82, 205, 56, 33, 125, 171, 211, 226, 33, 189, 218, 138, 162, 249, 209, 68, 20, 241, 82, 164, 220, 65, 181, 205, 29, 165, 237, 247, 230, 193, 91, 188, 102, 161, 160, 108, 95, 126, 204, 119, 48, 215, 250, 97, 176, 188, 240, 22, 206, 19, 38, 51, 26, 209, 72, 156, 222, 104, 71, 110, 12, 180, 219, 71, 241, 8, 144, 119, 36, 55, 199, 120, 196, 231, 109, 64, 122, 248, 245, 53, 95, 80, 98, 26, 232, 89, 37, 43, 189, 27, 202, 74, 191, 136, 203, 170, 129, 153, 254, 220, 94, 230, 141, 205, 53, 234, 231, 39, 55, 217, 136, 145, 158, 12, 91, 55, 218, 33, 89, 54, 219, 172, 43, 125, 17, 166, 154, 164, 211, 152, 69, 203, 58, 10, 9, 156, 61, 36, 134, 89, 27, 219, 71, 98, 222, 239, 118, 63, 149, 173, 133, 171, 197, 255, 122, 201, 51, 44, 78, 112, 93, 230, 167, 73, 157, 221, 13, 132, 40, 211, 120, 214, 204, 137, 116, 49, 255, 171, 229, 205, 76, 110, 200, 74, 253, 54, 12, 233, 167, 124, 216, 255, 238, 59, 196, 177, 204, 221, 133, 207, 99, 145, 68, 70, 88, 86, 72, 44, 189, 77, 250, 243, 30, 30, 146, 232, 26, 22, 116, 189, 178, 222, 178, 169, 195, 91, 48, 63, 115, 224, 93, 16, 130, 110, 69, 176, 137, 237, 134, 83, 36, 94, 234, 3, 27, 89, 191, 181, 140, 242, 182, 249, 25, 247, 122, 243, 233, 92, 29, 223, 104, 118, 130, 134, 201, 219, 230, 230, 123, 103, 49, 195, 39, 200, 20, 125, 80, 136, 209, 185, 52, 84, 99, 89, 32, 135, 64, 107, 229, 248, 109, 58, 28, 85, 95, 182, 48, 115, 18, 191, 56, 137, 38, 84, 125, 249, 98, 45, 93, 138, 27, 11, 119, 183, 107, 11, 183, 215, 23, 249, 236, 178, 125, 150, 20, 101, 210, 224, 180, 236, 173, 248, 182, 45, 135, 146, 53, 5, 131, 211, 180, 223, 159, 136, 27, 98, 164, 199, 56, 152, 228, 57, 191, 236, 37, 250, 111, 30, 18, 232, 161, 17, 187, 239, 207, 249, 93, 178, 108, 124, 196, 87, 56, 17, 37, 112, 200, 18, 119, 106, 157, 195, 21, 240, 90, 96, 68, 181, 46, 222, 205, 169, 17, 155, 180, 58, 255, 105, 5, 110, 28, 143, 1, 225, 63, 17, 136, 170, 4, 30, 164, 134, 196, 143, 69, 5, 32, 68, 183, 229, 17, 133, 208, 122, 239, 81, 224, 110, 148, 207, 21, 232, 161, 61, 12, 88, 58, 56, 153, 43, 187, 157, 255, 217, 150, 240, 169, 183, 37, 136, 211, 93, 226, 60, 50, 105, 68, 208, 214, 5, 83, 133, 207, 171, 127, 78, 143, 26, 223, 77, 97, 86, 89, 249, 79, 185, 11, 81, 239, 174, 240, 44, 180, 210, 12, 204, 24, 47, 240, 102, 230, 144, 17, 82, 132, 144, 155, 124, 148, 79, 147, 191, 224, 222, 56, 232, 214, 23, 183, 184, 85, 34, 162, 254, 45, 119, 198, 45, 217, 24, 103, 110, 100, 32, 162, 235, 94, 101, 110, 216, 203, 219, 250, 162, 18, 132, 130, 162, 85, 82, 132, 214, 5, 82, 120, 165, 163, 55, 153, 247, 89, 247, 190, 248, 101, 156, 90, 181, 52, 57, 67, 186, 140, 35, 58, 119, 235, 73, 65, 235, 234, 254, 250, 165, 156, 207, 187, 74, 131, 12, 191, 117, 25, 58, 96, 56, 138, 167, 226, 112, 193, 63, 147, 34, 183, 101, 2, 20, 214, 191, 228, 76, 239, 57, 25, 123, 103, 135, 58, 21, 19, 87, 242, 195, 37, 46, 237, 146, 223, 59, 223, 129, 140, 204, 32, 195, 51, 196, 210, 76, 227, 129, 79, 41, 241, 196, 121, 16, 184, 74, 121, 219, 178, 123, 59, 157, 232, 0, 155, 187, 129, 72, 16, 13, 50, 131, 121, 224, 94, 204, 142, 45, 237, 160, 153, 129, 101, 13, 13, 84, 67, 234, 106, 36, 179, 182, 13, 220, 56, 32, 129, 145, 226, 205, 64, 243, 34, 164, 214, 188, 114, 207, 244, 157, 85, 144, 47, 58, 72, 171, 100, 90, 74, 246, 26, 52, 22, 27, 81, 205, 7, 181, 122, 215, 228, 115, 115, 12, 55, 71, 114, 147, 234, 40, 230, 98, 230, 73, 156, 198, 22, 210, 73, 100, 96, 73, 102, 117, 94, 162, 11, 145, 24, 192, 243, 164, 18, 102, 175, 45, 71, 4, 191, 70, 14, 25, 29, 90, 239, 78, 232, 50, 20, 126, 72, 144, 96, 199, 239, 14, 3, 252, 4, 17, 154, 52, 201, 25, 22, 123, 251, 34, 124, 2, 7, 86, 84, 93, 185, 44, 167, 26, 23, 239, 246, 180, 54, 9, 210, 137, 131, 95, 211, 225, 250, 223, 72, 81, 190, 217, 241, 32, 224, 111, 17, 116, 79, 182, 192, 199, 58, 221, 34, 72, 131, 46, 153, 187, 48, 196, 249, 22, 132, 246, 138, 112, 64, 152, 127, 89, 201, 247, 158, 10, 161, 63, 172, 7, 158, 124, 53, 124, 154, 186, 64, 10, 159, 118, 131, 68, 155, 196, 23, 82, 48, 52, 183, 173, 164, 239, 71, 58, 149, 74, 77, 198, 94, 17, 172, 22, 141, 173, 48, 26, 62, 186, 15, 129, 164, 55, 101, 139, 178, 237, 170, 146, 61, 135, 198, 124, 154, 76, 199, 73, 132, 46, 177, 224, 194, 134, 108, 167, 156, 86, 213, 157, 21, 186, 14, 189, 185, 98, 25, 131, 175, 229, 95, 234, 182, 28, 63, 115, 157, 124, 93, 198, 83, 122, 252, 194, 240, 178, 102, 30, 124, 61, 104, 71, 216, 34, 11, 30, 134, 239, 228, 225, 60, 199, 16, 52, 132, 188, 238, 156, 40, 143, 209, 29, 224, 0, 50, 57, 191, 198, 0, 153, 176, 145, 87, 23, 140, 226, 68, 179, 175, 152, 61, 226, 185, 203, 27, 33, 91, 204, 166, 241, 91, 218, 11, 65, 26, 148, 118, 162, 124, 48, 40, 147, 202, 86, 176, 101, 31, 32, 154, 51, 3, 161, 63, 44, 31, 87, 21, 220, 227, 212, 111, 43, 244, 146, 55, 56, 250, 186, 255, 29, 239, 113, 121, 158, 242, 115, 137, 75, 110, 161, 147, 228, 110, 75, 127, 184, 23, 51, 63, 110, 103, 207, 153, 239, 117, 255, 142, 152, 4, 198, 69, 176, 151, 15, 20, 142, 221, 26, 28, 111, 225, 205, 211, 73, 214, 75, 150, 163, 249, 166, 142, 148, 116, 218, 95, 142, 225, 254, 158, 250, 118, 175, 6, 215, 147, 164, 72, 25, 75, 106, 209, 93, 173, 52, 128, 214, 88, 46, 223, 178, 214, 152, 150, 21, 37, 137, 180, 255, 42, 57, 151, 232, 81, 155, 53, 189, 94, 70, 228, 225, 32, 84, 185, 3, 46, 154, 116, 0, 212, 181, 170, 252, 30, 189, 31, 87, 237, 46, 186, 37, 165, 142, 162, 215, 179, 248, 143, 121, 88, 164, 174, 197, 53, 176, 23, 57, 199, 171, 218, 138, 248, 85, 157, 175, 73, 153, 102, 33, 23, 19, 161, 163, 248, 25, 197, 16, 144, 46, 117, 148, 159, 192, 46, 237, 238, 26, 248, 191, 140, 111, 13, 217, 191, 196, 48, 112, 58, 248, 36, 202, 158, 68, 75, 47, 199, 148, 104, 190, 39, 166, 12, 170, 34, 51, 181, 77, 244, 249, 69, 30, 51, 239, 52, 171, 190, 111, 135, 109, 193, 142, 182, 5, 223, 187, 117, 231, 75, 42, 239, 122, 155, 219, 249, 182, 166, 202, 61, 127, 43, 181, 117, 238, 123, 155, 185, 183, 91, 83, 229, 27, 127, 51, 181, 117, 190, 117, 235, 12, 38, 121, 92, 95, 233, 187, 64, 165, 111, 239, 251, 43, 93, 195, 98, 174, 104, 43, 151, 79, 37, 159, 70, 240, 30, 103, 139, 101, 114, 71, 43, 168, 12, 213, 42, 226, 86, 202, 155, 24, 62, 15, 171, 107, 167, 219, 91, 224, 178, 100, 26, 32, 133, 20, 95, 173, 136, 194, 109, 125, 200, 9, 92, 52, 128, 191, 143, 224, 47, 27, 192, 127, 143, 224, 207, 27, 192, 239, 168, 5, 78, 201, 98, 198, 139, 103, 2, 250, 226, 62, 48, 150, 191, 242, 167, 189, 209, 190, 232, 68, 139, 78, 116, 217, 137, 206, 157, 177, 119, 187, 238, 81, 180, 218, 174, 251, 225, 191, 247, 83, 246, 237, 253, 139, 93, 155, 50, 67, 211, 182, 135, 38, 191, 145, 9, 81, 20, 132, 246, 14, 69, 16, 218, 59, 16, 65, 104, 207, 48, 112, 145, 172, 29, 132, 149, 149, 105, 21, 85, 58, 10, 7, 9, 157, 72, 188, 244, 13, 242, 177, 226, 36, 35, 87, 135, 207, 28, 7, 80, 218, 235, 231, 121, 76, 59, 230, 187, 45, 218, 20, 231, 82, 43, 238, 50, 198, 139, 252, 27, 47, 193, 53, 118, 107, 41, 198, 251, 70, 234, 245, 213, 179, 237, 134, 188, 252, 97, 67, 189, 131, 8, 80, 192, 32, 82, 194, 196, 98, 132, 4, 22, 216, 235, 41, 84, 97, 170, 161, 17, 229, 149, 176, 195, 65, 147, 89, 46, 2, 148, 140, 18, 107, 214, 201, 164, 255, 138, 49, 184, 196, 150, 91, 212, 174, 55, 221, 168, 62, 87, 165, 21, 16, 252, 96, 35, 120, 205, 251, 187, 2, 134, 29, 103, 95, 63, 186, 131, 204, 244, 73, 37, 35, 232, 101, 100, 164, 223, 176, 73, 90, 195, 163, 215, 217, 210, 206, 41, 40, 209, 83, 31, 152, 232, 130, 130, 147, 29, 242, 1, 170, 196, 142, 30, 154, 99, 221, 190, 6, 182, 76, 134, 105, 218, 21, 18, 102, 231, 80, 139, 29, 211, 153, 14, 106, 206, 179, 22, 167, 108, 29, 78, 171, 236, 90, 203, 88, 34, 199, 130, 132, 200, 122, 190, 236, 126, 124, 53, 88, 214, 139, 142, 111, 168, 10, 198, 46, 6, 197, 220, 94, 248, 226, 139, 34, 10, 70, 173, 98, 67, 182, 90, 230, 249, 130, 149, 244, 38, 179, 109, 156, 242, 130, 179, 29, 74, 44, 174, 161, 51, 80, 253, 72, 220, 247, 191, 196, 154, 232, 134, 58, 209, 61, 11, 129, 150, 55, 193, 160, 176, 54, 120, 85, 192, 18, 48, 212, 142, 225, 188, 105, 69, 221, 93, 108, 68, 54, 53, 87, 198, 34, 98, 54, 247, 117, 223, 140, 192, 93, 153, 249, 209, 127, 191, 203, 53, 166, 151, 213, 2, 122, 21, 205, 223, 86, 20, 112, 84, 229, 69, 226, 143, 63, 3, 235, 194, 109, 36, 177, 226, 186, 234, 175, 249, 181, 217, 161, 176, 65, 180, 224, 143, 82, 67, 75, 207, 164, 141, 71, 143, 30, 249, 90, 249, 222, 76, 113, 103, 140, 51, 7, 147, 120, 58, 123, 151, 255, 202, 219, 217, 244, 79, 143, 181, 189, 141, 206, 66, 65, 140, 174, 229, 9, 128, 27, 245, 0, 113, 9, 247, 224, 94, 128, 79, 43, 52, 83, 195, 41, 220, 210, 125, 127, 127, 60, 174, 103, 227, 254, 132, 194, 120, 211, 141, 21, 176, 251, 186, 17, 138, 249, 69, 3, 161, 8, 166, 19, 89, 99, 239, 73, 0, 32, 12, 158, 64, 195, 193, 112, 203, 33, 234, 245, 252, 200, 91, 79, 15, 8, 38, 249, 242, 3, 126, 167, 214, 142, 125, 213, 29, 120, 102, 96, 205, 203, 158, 127, 52, 117, 249, 150, 68, 119, 38, 244, 104, 109, 32, 243, 160, 168, 57, 163, 148, 248, 52, 56, 144, 135, 80, 24, 244, 241, 71, 139, 142, 219, 31, 255, 166, 174, 126, 173, 247, 192, 247, 11, 169, 64, 11, 167, 113, 109, 7, 163, 206, 107, 90, 238, 55, 173, 230, 57, 73, 114, 133, 235, 249, 244, 143, 121, 122, 166, 30, 35, 10, 77, 68, 139, 72, 253, 28, 180, 174, 77, 159, 168, 58, 68, 75, 108, 22, 36, 30, 58, 52, 49, 163, 187, 212, 81, 150, 176, 17, 154, 251, 97, 52, 247, 87, 64, 243, 125, 24, 205, 247, 43, 160, 97, 35, 22, 196, 179, 179, 107, 16, 161, 176, 14, 235, 110, 180, 196, 80, 219, 108, 99, 32, 75, 217, 22, 72, 44, 133, 209, 124, 31, 38, 115, 155, 146, 25, 78, 42, 117, 112, 174, 167, 142, 196, 112, 166, 41, 132, 194, 25, 213, 112, 250, 41, 132, 194, 25, 209, 154, 156, 84, 8, 71, 96, 52, 229, 102, 234, 189, 213, 13, 168, 48, 161, 228, 198, 211, 131, 60, 59, 75, 138, 234, 113, 214, 63, 200, 103, 139, 119, 57, 232, 96, 37, 28, 255, 26, 252, 0, 164, 87, 176, 150, 192, 189, 174, 247, 81, 216, 199, 4, 168, 117, 64, 218, 28, 125, 181, 138, 171, 255, 96, 85, 46, 105, 235, 234, 79, 177, 225, 158, 135, 211, 108, 110, 130, 79, 242, 148, 114, 14, 243, 135, 132, 233, 200, 7, 64, 99, 245, 54, 73, 7, 11, 185, 225, 208, 63, 68, 226, 89, 112, 100, 196, 227, 215, 77, 151, 64, 155, 241, 110, 9, 195, 40, 73, 97, 150, 121, 23, 226, 73, 103, 237, 222, 209, 238, 56, 142, 0, 175, 196, 39, 199, 152, 5, 188, 85, 59, 175, 70, 73, 193, 11, 125, 51, 190, 217, 28, 140, 39, 123, 49, 127, 162, 163, 130, 132, 240, 111, 204, 84, 172, 241, 224, 86, 252, 216, 16, 68, 16, 167, 100, 6, 158, 224, 197, 22, 99, 84, 57, 228, 249, 168, 227, 135, 233, 52, 201, 74, 216, 24, 148, 37, 231, 188, 236, 157, 158, 72, 155, 116, 189, 143, 159, 54, 236, 60, 109, 201, 223, 125, 2, 19, 196, 203, 151, 218, 39, 125, 13, 169, 182, 58, 219, 174, 141, 190, 135, 92, 86, 144, 123, 127, 101, 21, 140, 193, 5, 34, 111, 47, 132, 142, 44, 241, 78, 48, 54, 254, 122, 229, 59, 251, 108, 166, 8, 245, 64, 238, 66, 92, 142, 19, 145, 204, 177, 130, 63, 127, 95, 105, 167, 77, 237, 223, 246, 173, 94, 146, 205, 85, 62, 6, 96, 216, 196, 124, 87, 46, 184, 220, 37, 69, 171, 225, 49, 122, 224, 0, 17, 190, 53, 235, 38, 109, 65, 118, 211, 244, 210, 198, 253, 183, 125, 139, 38, 210, 75, 31, 185, 75, 122, 9, 163, 107, 179, 18, 146, 125, 86, 195, 138, 30, 207, 112, 42, 44, 196, 233, 182, 71, 143, 18, 178, 177, 4, 93, 83, 227, 90, 187, 89, 7, 15, 149, 58, 135, 96, 191, 125, 155, 197, 65, 111, 224, 192, 205, 145, 137, 212, 4, 105, 141, 66, 214, 86, 42, 86, 200, 90, 129, 76, 84, 107, 160, 22, 50, 13, 196, 174, 7, 2, 22, 20, 91, 171, 70, 126, 75, 67, 62, 232, 28, 244, 93, 239, 216, 108, 214, 59, 79, 63, 238, 7, 168, 223, 189, 29, 154, 243, 35, 62, 123, 181, 215, 102, 145, 225, 205, 2, 135, 47, 159, 192, 171, 5, 42, 137, 220, 188, 91, 128, 216, 201, 190, 76, 84, 215, 91, 212, 6, 22, 81, 47, 19, 130, 41, 252, 230, 38, 36, 93, 35, 62, 132, 106, 51, 35, 214, 112, 246, 208, 39, 232, 10, 84, 47, 216, 249, 37, 198, 74, 239, 239, 107, 211, 46, 71, 18, 181, 180, 217, 106, 183, 54, 129, 250, 173, 139, 77, 232, 154, 248, 190, 64, 223, 47, 209, 247, 243, 205, 214, 6, 23, 123, 126, 40, 134, 54, 177, 187, 114, 19, 27, 244, 101, 25, 78, 6, 96, 167, 35, 70, 119, 119, 197, 132, 120, 19, 153, 94, 43, 50, 217, 231, 70, 153, 132, 29, 72, 37, 240, 121, 177, 17, 248, 174, 243, 222, 3, 206, 166, 107, 104, 92, 173, 251, 39, 79, 110, 89, 194, 221, 226, 25, 179, 214, 198, 10, 125, 20, 70, 219, 34, 89, 108, 175, 30, 229, 69, 53, 138, 179, 126, 123, 107, 107, 139, 57, 116, 229, 122, 61, 87, 117, 246, 131, 145, 134, 76, 193, 203, 188, 63, 159, 36, 109, 173, 17, 208, 174, 57, 230, 169, 206, 107, 5, 208, 241, 251, 13, 208, 30, 110, 178, 252, 132, 49, 193, 185, 124, 86, 40, 110, 96, 72, 187, 93, 129, 85, 85, 135, 237, 218, 239, 182, 244, 196, 214, 230, 229, 29, 142, 64, 45, 52, 225, 11, 21, 172, 87, 220, 228, 69, 60, 100, 97, 1, 182, 153, 57, 11, 61, 52, 239, 140, 179, 14, 1, 9, 58, 43, 168, 24, 232, 110, 232, 204, 117, 232, 98, 54, 127, 207, 39, 147, 7, 78, 213, 119, 69, 156, 149, 240, 82, 190, 172, 114, 234, 107, 149, 172, 9, 151, 112, 67, 14, 159, 191, 211, 233, 128, 165, 102, 28, 240, 96, 224, 147, 69, 149, 188, 80, 38, 176, 153, 153, 189, 167, 17, 200, 30, 255, 25, 157, 204, 7, 131, 164, 216, 195, 77, 254, 196, 139, 120, 123, 78, 188, 105, 26, 221, 67, 223, 189, 96, 34, 138, 163, 152, 117, 177, 24, 190, 43, 186, 130, 232, 72, 160, 21, 14, 53, 21, 0, 56, 11, 231, 155, 102, 205, 72, 184, 161, 169, 203, 231, 229, 186, 132, 221, 17, 54, 253, 105, 130, 164, 211, 209, 202, 121, 141, 214, 158, 110, 110, 75, 224, 64, 185, 177, 150, 234, 96, 61, 148, 200, 200, 99, 24, 252, 66, 81, 1, 35, 222, 22, 138, 97, 164, 147, 134, 129, 180, 251, 130, 225, 30, 103, 11, 154, 135, 114, 30, 185, 175, 212, 65, 145, 41, 252, 254, 105, 158, 78, 250, 237, 184, 115, 210, 233, 173, 223, 162, 141, 83, 126, 89, 195, 18, 25, 55, 158, 178, 25, 25, 168, 137, 28, 197, 251, 102, 251, 140, 28, 179, 152, 79, 66, 102, 20, 44, 139, 192, 187, 197, 251, 123, 148, 252, 113, 56, 157, 77, 100, 43, 124, 242, 86, 251, 24, 216, 100, 22, 75, 197, 210, 30, 1, 69, 47, 135, 6, 228, 82, 62, 56, 89, 67, 187, 113, 39, 58, 105, 212, 174, 66, 238, 97, 47, 121, 23, 172, 231, 57, 201, 76, 122, 163, 158, 37, 130, 193, 143, 129, 2, 165, 157, 232, 22, 36, 131, 69, 120, 208, 230, 33, 188, 65, 143, 73, 136, 65, 27, 135, 157, 237, 88, 207, 131, 184, 54, 217, 250, 113, 49, 155, 164, 189, 180, 122, 146, 204, 196, 134, 176, 19, 121, 185, 145, 153, 28, 40, 8, 27, 87, 124, 201, 129, 61, 175, 188, 140, 103, 191, 138, 166, 212, 104, 198, 250, 246, 7, 123, 0, 17, 158, 147, 21, 240, 156, 52, 26, 77, 242, 66, 120, 111, 79, 169, 120, 42, 249, 185, 126, 191, 223, 89, 212, 158, 92, 183, 227, 54, 162, 29, 95, 207, 93, 18, 111, 32, 250, 181, 162, 13, 87, 80, 114, 253, 91, 115, 246, 141, 71, 78, 12, 162, 81, 134, 76, 59, 29, 188, 156, 185, 28, 218, 187, 82, 40, 234, 211, 124, 215, 51, 82, 129, 49, 17, 163, 33, 5, 82, 228, 46, 196, 208, 4, 108, 211, 42, 88, 119, 48, 86, 161, 191, 203, 124, 5, 43, 117, 10, 151, 36, 254, 203, 143, 87, 236, 30, 249, 215, 60, 189, 227, 225, 105, 40, 11, 203, 223, 38, 42, 41, 86, 45, 219, 57, 54, 117, 97, 49, 103, 157, 103, 72, 12, 18, 61, 17, 213, 140, 128, 124, 227, 17, 60, 255, 151, 30, 1, 201, 110, 232, 71, 83, 142, 75, 70, 10, 14, 120, 24, 201, 121, 66, 152, 247, 234, 215, 151, 31, 126, 250, 253, 221, 211, 163, 246, 73, 90, 149, 244, 132, 23, 148, 68, 155, 209, 119, 176, 249, 39, 186, 71, 163, 164, 167, 239, 62, 252, 116, 248, 174, 221, 143, 171, 24, 94, 42, 11, 215, 125, 232, 221, 47, 231, 121, 1, 46, 20, 47, 149, 85, 213, 65, 228, 146, 223, 150, 29, 61, 124, 8, 239, 238, 133, 199, 176, 45, 74, 196, 37, 128, 234, 24, 170, 190, 143, 62, 238, 115, 80, 210, 226, 243, 245, 183, 168, 250, 137, 90, 254, 154, 215, 224, 9, 169, 109, 210, 188, 235, 36, 197, 38, 111, 202, 156, 36, 72, 156, 204, 192, 81, 50, 98, 119, 187, 31, 29, 95, 243, 201, 241, 33, 92, 148, 200, 100, 186, 45, 126, 203, 221, 109, 32, 53, 125, 51, 51, 5, 4, 250, 79, 113, 151, 1, 235, 229, 209, 44, 134, 251, 54, 134, 69, 156, 102, 50, 35, 156, 87, 241, 68, 196, 93, 87, 209, 62, 199, 2, 111, 141, 159, 87, 201, 33, 169, 132, 185, 33, 25, 161, 78, 75, 139, 139, 208, 228, 45, 71, 16, 153, 107, 208, 182, 126, 206, 70, 38, 237, 121, 110, 100, 144, 229, 58, 127, 32, 112, 59, 153, 1, 185, 49, 85, 60, 196, 251, 82, 101, 125, 254, 122, 104, 113, 47, 140, 90, 180, 208, 125, 132, 243, 225, 108, 56, 249, 205, 63, 88, 91, 143, 82, 40, 229, 100, 106, 88, 77, 111, 217, 104, 179, 45, 226, 227, 61, 108, 16, 245, 78, 219, 109, 221, 19, 212, 11, 196, 114, 210, 23, 113, 171, 70, 82, 188, 158, 87, 111, 242, 52, 171, 18, 43, 120, 199, 199, 56, 81, 203, 130, 245, 29, 211, 110, 253, 205, 66, 190, 143, 203, 171, 13, 21, 230, 89, 239, 109, 229, 38, 238, 3, 92, 66, 209, 81, 188, 239, 88, 68, 163, 93, 199, 206, 5, 153, 128, 66, 253, 248, 16, 207, 102, 147, 133, 208, 39, 114, 43, 140, 196, 187, 132, 132, 124, 102, 67, 242, 247, 197, 90, 55, 138, 40, 135, 204, 112, 17, 214, 219, 107, 185, 39, 119, 46, 171, 126, 28, 102, 189, 130, 207, 101, 150, 62, 232, 222, 83, 229, 146, 181, 140, 16, 108, 238, 115, 97, 82, 162, 41, 77, 146, 104, 246, 1, 221, 105, 209, 92, 237, 62, 129, 249, 177, 63, 107, 205, 240, 222, 34, 114, 109, 113, 8, 31, 93, 195, 67, 30, 123, 236, 15, 5, 56, 222, 126, 47, 46, 169, 210, 78, 130, 81, 98, 250, 64, 203, 150, 186, 177, 195, 231, 15, 17, 227, 182, 19, 52, 110, 242, 16, 144, 227, 94, 24, 123, 69, 143, 168, 240, 242, 23, 252, 118, 13, 13, 98, 93, 254, 98, 117, 139, 219, 77, 85, 77, 193, 224, 190, 253, 205, 125, 140, 123, 104, 154, 177, 122, 104, 133, 135, 214, 220, 164, 206, 240, 89, 187, 253, 255, 180, 200, 219, 179, 126, 19, 203, 166, 155, 222, 67, 243, 25, 6, 64, 221, 216, 35, 125, 186, 34, 122, 183, 170, 130, 175, 69, 247, 148, 180, 100, 194, 169, 193, 115, 96, 151, 44, 156, 227, 91, 138, 212, 100, 151, 189, 7, 171, 178, 99, 172, 30, 122, 240, 208, 30, 225, 236, 189, 239, 32, 216, 90, 58, 64, 40, 210, 215, 178, 137, 59, 224, 240, 181, 114, 184, 225, 44, 234, 74, 210, 73, 68, 236, 38, 35, 210, 140, 203, 59, 141, 13, 166, 241, 226, 36, 121, 118, 27, 14, 216, 173, 155, 55, 217, 31, 114, 183, 144, 40, 218, 184, 205, 182, 117, 156, 130, 50, 34, 134, 139, 142, 219, 232, 185, 50, 19, 65, 43, 169, 229, 17, 203, 190, 121, 130, 92, 31, 109, 18, 205, 176, 33, 11, 36, 129, 158, 35, 11, 130, 224, 44, 19, 226, 248, 149, 6, 180, 141, 189, 173, 95, 179, 180, 106, 118, 56, 10, 55, 109, 185, 108, 207, 87, 112, 23, 49, 248, 18, 183, 207, 114, 96, 53, 131, 28, 59, 159, 50, 1, 81, 50, 66, 141, 186, 121, 133, 144, 124, 108, 29, 17, 204, 121, 178, 50, 224, 39, 210, 30, 95, 223, 85, 244, 114, 136, 44, 32, 241, 3, 143, 76, 22, 212, 107, 140, 152, 221, 236, 68, 88, 137, 53, 116, 141, 159, 230, 120, 106, 203, 232, 183, 156, 181, 43, 215, 249, 66, 2, 178, 220, 7, 179, 19, 70, 154, 126, 233, 123, 97, 91, 244, 169, 226, 64, 207, 231, 86, 141, 213, 103, 105, 117, 125, 193, 110, 154, 153, 116, 148, 187, 204, 135, 77, 174, 39, 52, 222, 75, 179, 195, 80, 172, 204, 158, 105, 81, 172, 15, 149, 53, 9, 90, 110, 100, 179, 248, 146, 56, 154, 179, 66, 184, 67, 235, 23, 24, 228, 154, 225, 254, 30, 85, 167, 198, 177, 63, 22, 123, 71, 19, 112, 167, 150, 4, 237, 200, 97, 57, 116, 252, 80, 24, 94, 199, 59, 245, 102, 220, 239, 251, 211, 14, 190, 233, 1, 49, 173, 129, 97, 199, 44, 94, 231, 220, 144, 102, 116, 106, 240, 206, 23, 28, 176, 228, 235, 12, 59, 2, 161, 10, 71, 192, 153, 218, 225, 87, 178, 182, 189, 253, 97, 242, 18, 182, 207, 162, 125, 52, 71, 90, 109, 252, 200, 84, 109, 111, 217, 196, 137, 154, 181, 230, 205, 27, 34, 246, 205, 200, 102, 91, 78, 142, 236, 177, 192, 173, 111, 173, 246, 73, 90, 40, 177, 98, 207, 163, 116, 174, 164, 157, 211, 22, 95, 141, 217, 198, 127, 230, 90, 255, 92, 27, 25, 177, 222, 220, 71, 34, 166, 158, 214, 205, 194, 214, 53, 146, 106, 38, 22, 197, 117, 227, 47, 100, 203, 204, 203, 104, 200, 85, 38, 73, 32, 187, 190, 156, 104, 230, 26, 119, 139, 242, 23, 95, 112, 214, 128, 203, 130, 30, 17, 105, 23, 55, 233, 252, 43, 114, 101, 59, 205, 168, 254, 202, 55, 225, 170, 139, 17, 102, 250, 34, 210, 127, 107, 53, 137, 139, 33, 240, 231, 88, 118, 73, 123, 205, 75, 83, 146, 118, 166, 19, 16, 217, 217, 198, 37, 201, 81, 85, 197, 163, 147, 43, 102, 70, 207, 242, 180, 15, 167, 217, 0, 227, 151, 175, 215, 215, 77, 152, 214, 168, 4, 119, 24, 200, 220, 66, 22, 121, 111, 100, 37, 168, 223, 105, 222, 57, 21, 198, 221, 38, 7, 112, 124, 74, 106, 157, 201, 114, 150, 60, 99, 55, 175, 33, 104, 74, 25, 39, 83, 250, 234, 146, 79, 243, 249, 212, 241, 132, 94, 101, 17, 239, 212, 57, 83, 183, 11, 11, 215, 144, 51, 193, 122, 145, 14, 127, 69, 152, 241, 160, 153, 21, 219, 241, 248, 192, 247, 245, 120, 6, 157, 51, 201, 114, 178, 173, 211, 219, 164, 25, 133, 82, 190, 135, 104, 199, 40, 48, 186, 144, 89, 191, 79, 216, 202, 119, 170, 90, 219, 118, 45, 228, 166, 170, 1, 199, 42, 64, 44, 54, 71, 34, 172, 54, 238, 62, 242, 166, 189, 120, 249, 244, 117, 38, 14, 137, 198, 114, 14, 219, 112, 247, 236, 220, 164, 239, 150, 195, 173, 88, 177, 177, 26, 47, 100, 235, 107, 233, 191, 183, 211, 86, 200, 47, 37, 13, 233, 167, 103, 37, 93, 234, 231, 231, 137, 239, 111, 65, 21, 149, 182, 233, 213, 8, 111, 100, 28, 235, 184, 56, 188, 173, 65, 99, 168, 15, 64, 19, 114, 250, 42, 184, 8, 194, 51, 137, 147, 120, 136, 125, 152, 239, 133, 11, 99, 54, 45, 224, 97, 55, 9, 72, 245, 194, 42, 29, 56, 53, 91, 175, 77, 240, 97, 172, 123, 193, 240, 204, 231, 52, 185, 226, 135, 124, 177, 51, 215, 91, 34, 77, 197, 94, 95, 137, 95, 49, 197, 133, 150, 251, 151, 98, 0, 140, 196, 169, 29, 24, 156, 69, 12, 194, 248, 29, 208, 121, 250, 74, 18, 60, 99, 227, 213, 198, 229, 187, 142, 77, 20, 45, 135, 203, 222, 90, 79, 58, 98, 94, 192, 229, 68, 223, 6, 73, 219, 147, 10, 232, 68, 167, 13, 120, 9, 28, 121, 110, 119, 219, 172, 34, 112, 63, 250, 116, 115, 243, 189, 182, 189, 34, 19, 90, 159, 211, 251, 45, 137, 199, 204, 13, 250, 16, 247, 251, 194, 60, 224, 145, 59, 138, 194, 47, 62, 58, 82, 47, 3, 146, 24, 88, 129, 255, 13, 86, 157, 168, 37, 65, 224, 128, 7, 107, 199, 28, 98, 58, 146, 163, 54, 97, 99, 195, 184, 88, 164, 244, 214, 188, 224, 150, 99, 73, 139, 172, 114, 237, 102, 105, 135, 206, 200, 181, 126, 20, 153, 186, 96, 207, 113, 74, 161, 9, 230, 228, 124, 224, 158, 162, 164, 71, 78, 217, 29, 126, 61, 151, 172, 168, 238, 124, 241, 242, 190, 55, 73, 226, 162, 125, 203, 140, 231, 141, 124, 14, 214, 7, 26, 14, 178, 146, 195, 107, 102, 82, 254, 249, 79, 4, 42, 62, 246, 153, 54, 86, 201, 39, 17, 99, 209, 212, 231, 96, 103, 168, 229, 122, 97, 118, 223, 92, 229, 123, 79, 23, 30, 6, 209, 142, 45, 212, 27, 94, 198, 143, 226, 242, 147, 112, 157, 181, 243, 57, 88, 238, 109, 118, 221, 252, 102, 141, 248, 152, 253, 255, 1, 71, 136, 183, 26 }; static const unsigned char * const compressedSources = reinterpret_cast(data); uint32_t GetCompressedSize() { - return 22509; + return 22722; } uint32_t GetRawScriptsSize() { - return 134371; + return 139556; } } // selfhosted } // js diff --git a/src/third_party/mozjs-38/extract/js/src/vm/Shape.cpp b/src/third_party/mozjs-38/extract/js/src/vm/Shape.cpp index d5c9ef1df7a..cd3a648c03f 100644 --- a/src/third_party/mozjs-38/extract/js/src/vm/Shape.cpp +++ b/src/third_party/mozjs-38/extract/js/src/vm/Shape.cpp @@ -387,8 +387,10 @@ NativeObject::getChildPropertyOnDictionary(ExclusiveContext* cx, HandleNativeObj if (!shape) return nullptr; if (childRoot->hasSlot() && childRoot->slot() >= obj->lastProperty()->base()->slotSpan()) { - if (!obj->setSlotSpan(cx, childRoot->slot() + 1)) + if (!obj->setSlotSpan(cx, childRoot->slot() + 1)) { + new (shape) Shape(obj->lastProperty()->base()->unowned(), 0); return nullptr; + } } shape->initDictionaryShape(*childRoot, obj->numFixedSlots(), &obj->shape_); } diff --git a/src/third_party/mozjs-38/extract/js/src/vm/StructuredClone.cpp b/src/third_party/mozjs-38/extract/js/src/vm/StructuredClone.cpp index ef5501faed8..9564cc6a95a 100644 --- a/src/third_party/mozjs-38/extract/js/src/vm/StructuredClone.cpp +++ b/src/third_party/mozjs-38/extract/js/src/vm/StructuredClone.cpp @@ -384,10 +384,11 @@ static void Discard(uint64_t* buffer, size_t nbytes, const JSStructuredCloneCallbacks* cb, void* cbClosure) { MOZ_ASSERT(nbytes % sizeof(uint64_t) == 0); - if (nbytes < sizeof(uint64_t)) + uint64_t* end = buffer + nbytes / sizeof(uint64_t); + uint64_t* point = buffer; + if (point == end) return; // Empty buffer - uint64_t* point = buffer; uint32_t tag, data; SCInput::getPair(point++, &tag, &data); if (tag != SCTAG_TRANSFER_MAP_HEADER) @@ -399,14 +400,24 @@ Discard(uint64_t* buffer, size_t nbytes, const JSStructuredCloneCallbacks* cb, v // freeTransfer should not GC JS::AutoSuppressGCAnalysis nogc; + if (point == end) + return; + uint64_t numTransferables = LittleEndian::readUint64(point++); while (numTransferables--) { + if (point == end) + return; + uint32_t ownership; SCInput::getPair(point++, &tag, &ownership); MOZ_ASSERT(tag >= SCTAG_TRANSFER_MAP_PENDING_ENTRY); + if (point == end) + return; void* content; SCInput::getPtr(point++, &content); + if (point == end) + return; uint64_t extraData = LittleEndian::readUint64(point++); diff --git a/src/third_party/mozjs-38/extract/js/src/vm/TypeInference.cpp b/src/third_party/mozjs-38/extract/js/src/vm/TypeInference.cpp index f03c4b50798..5c4471378f1 100644 --- a/src/third_party/mozjs-38/extract/js/src/vm/TypeInference.cpp +++ b/src/third_party/mozjs-38/extract/js/src/vm/TypeInference.cpp @@ -3966,6 +3966,12 @@ JSScript::maybeSweepTypes(AutoClearTypeInferenceStateOnOOM* oom) for (unsigned i = 0; i < num; i++) typeArray[i].sweep(zone(), *oom); + if (oom->hadOOM()) { + // It's possible we OOM'd while copying freeze constraints, so they + // need to be regenerated. + hasFreezeConstraints_ = false; + } + // Update the recompile indexes in any IonScripts still on the script. if (hasIonScript()) ionScript()->recompileInfoRef().shouldSweep(types); diff --git a/src/third_party/mozjs-38/extract/js/src/vm/TypeInference.h b/src/third_party/mozjs-38/extract/js/src/vm/TypeInference.h index ec63d159706..b8797955fa4 100644 --- a/src/third_party/mozjs-38/extract/js/src/vm/TypeInference.h +++ b/src/third_party/mozjs-38/extract/js/src/vm/TypeInference.h @@ -571,6 +571,9 @@ class AutoClearTypeInferenceStateOnOOM void setOOM() { oom = true; } + bool hadOOM() const { + return oom; + } }; /* Superclass common to stack and heap type sets. */ diff --git a/src/third_party/mozjs-38/extract/mfbt/Opaque.h b/src/third_party/mozjs-38/extract/mfbt/Opaque.h new file mode 100644 index 00000000000..d7239ee7c23 --- /dev/null +++ b/src/third_party/mozjs-38/extract/mfbt/Opaque.h @@ -0,0 +1,44 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* An opaque integral type supporting only comparison operators. */ + +#ifndef mozilla_Opaque_h +#define mozilla_Opaque_h + +#include "mozilla/TypeTraits.h" + +namespace mozilla { + +/** + * Opaque is a replacement for integral T in cases where only comparisons + * must be supported, and it's desirable to prevent accidental dependency on + * exact values. + */ +template +class Opaque final +{ + static_assert(mozilla::IsIntegral::value, + "mozilla::Opaque only supports integral types"); + + T mValue; + +public: + Opaque() {} + explicit Opaque(T aValue) : mValue(aValue) {} + + bool operator==(const Opaque& aOther) const { + return mValue == aOther.mValue; + } + + bool operator!=(const Opaque& aOther) const { + return !(*this == aOther); + } +}; + +} // namespace mozilla + +#endif /* mozilla_Opaque_h */ diff --git a/src/third_party/mozjs-38/extract/mfbt/moz.build b/src/third_party/mozjs-38/extract/mfbt/moz.build index 82b7e7e7fbe..51736cf7ed5 100644 --- a/src/third_party/mozjs-38/extract/mfbt/moz.build +++ b/src/third_party/mozjs-38/extract/mfbt/moz.build @@ -53,6 +53,7 @@ EXPORTS.mozilla = [ 'Move.h', 'NullPtr.h', 'NumericLimits.h', + 'Opaque.h', 'Pair.h', 'PodOperations.h', 'Poison.h', diff --git a/src/third_party/mozjs-38/get_sources.sh b/src/third_party/mozjs-38/get_sources.sh index 054eb941902..a589216380b 100755 --- a/src/third_party/mozjs-38/get_sources.sh +++ b/src/third_party/mozjs-38/get_sources.sh @@ -2,8 +2,8 @@ # how we got the last firefox sources -wget "https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/38.6.1esr/source/firefox-38.6.1esr.source.tar.bz2" +wget "https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/38.8.0esr/source/firefox-38.8.0esr.source.tar.bz2" -tar -jxf firefox-38.6.1esr.source.tar.bz2 +tar -jxf firefox-38.8.0esr.source.tar.bz2 mv mozilla-esr38 mozilla-release diff --git a/src/third_party/mozjs-38/include/js/HashTable.h b/src/third_party/mozjs-38/include/js/HashTable.h index 5fb5012c4a8..36f37d6852a 100644 --- a/src/third_party/mozjs-38/include/js/HashTable.h +++ b/src/third_party/mozjs-38/include/js/HashTable.h @@ -13,6 +13,7 @@ #include "mozilla/Casting.h" #include "mozilla/MemoryReporting.h" #include "mozilla/Move.h" +#include "mozilla/Opaque.h" #include "mozilla/PodOperations.h" #include "mozilla/ReentrancyGuard.h" #include "mozilla/TemplateLib.h" @@ -32,6 +33,8 @@ namespace detail { /*****************************************************************************/ +using Generation = mozilla::Opaque; + // A JS-friendly, STL-like container providing a hash-based map from keys to // values. In particular, HashMap calls constructors and destructors of all // objects added so non-PODs may be used safely. @@ -205,7 +208,9 @@ class HashMap // If |generation()| is the same before and after a HashMap operation, // pointers into the table remain valid. - uint32_t generation() const { return impl.generation(); } + Generation generation() const { + return impl.generation(); + } /************************************************** Shorthand operations */ @@ -436,7 +441,9 @@ class HashSet // If |generation()| is the same before and after a HashSet operation, // pointers into the table remain valid. - uint32_t generation() const { return impl.generation(); } + Generation generation() const { + return impl.generation(); + } /************************************************** Shorthand operations */ @@ -771,7 +778,7 @@ class HashTable : private AllocPolicy Entry* entry_; #ifdef JS_DEBUG const HashTable* table_; - uint32_t generation; + Generation generation; #endif protected: @@ -878,7 +885,7 @@ class HashTable : private AllocPolicy #ifdef JS_DEBUG const HashTable* table_; uint64_t mutationCount; - uint32_t generation; + Generation generation; bool validEntry; #endif @@ -1017,8 +1024,8 @@ class HashTable : private AllocPolicy static const size_t CAP_BITS = 24; public: - Entry* table; // entry storage - uint32_t gen; // entry storage generation number + uint64_t gen; // entry storage generation number + Entry* table; // entry storage uint32_t entryCount; // number of entries in table uint32_t removedCount:CAP_BITS; // removed entry sentinels in table uint32_t hashShift:8; // multiplicative hash shift @@ -1102,8 +1109,8 @@ class HashTable : private AllocPolicy public: explicit HashTable(AllocPolicy ap) : AllocPolicy(ap) - , table(nullptr) , gen(0) + , table(nullptr) , entryCount(0) , removedCount(0) , hashShift(sHashBits) @@ -1529,10 +1536,10 @@ class HashTable : private AllocPolicy return JS_BIT(sHashBits - hashShift); } - uint32_t generation() const + Generation generation() const { MOZ_ASSERT(table); - return gen; + return Generation(gen); } size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const diff --git a/src/third_party/mozjs-38/include/jsapi.h b/src/third_party/mozjs-38/include/jsapi.h index 2b279b52092..0d8cd52b12b 100644 --- a/src/third_party/mozjs-38/include/jsapi.h +++ b/src/third_party/mozjs-38/include/jsapi.h @@ -275,10 +275,6 @@ class AutoHashMapRooter : protected AutoGCRooter return map.sizeOfIncludingThis(mallocSizeOf); } - uint32_t generation() const { - return map.generation(); - } - /************************************************** Shorthand operations */ bool has(const Lookup& l) const { @@ -390,10 +386,6 @@ class AutoHashSetRooter : protected AutoGCRooter return set.sizeOfIncludingThis(mallocSizeOf); } - uint32_t generation() const { - return set.generation(); - } - /************************************************** Shorthand operations */ bool has(const Lookup& l) const { diff --git a/src/third_party/mozjs-38/include/mozilla/Opaque.h b/src/third_party/mozjs-38/include/mozilla/Opaque.h new file mode 100644 index 00000000000..d7239ee7c23 --- /dev/null +++ b/src/third_party/mozjs-38/include/mozilla/Opaque.h @@ -0,0 +1,44 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* An opaque integral type supporting only comparison operators. */ + +#ifndef mozilla_Opaque_h +#define mozilla_Opaque_h + +#include "mozilla/TypeTraits.h" + +namespace mozilla { + +/** + * Opaque is a replacement for integral T in cases where only comparisons + * must be supported, and it's desirable to prevent accidental dependency on + * exact values. + */ +template +class Opaque final +{ + static_assert(mozilla::IsIntegral::value, + "mozilla::Opaque only supports integral types"); + + T mValue; + +public: + Opaque() {} + explicit Opaque(T aValue) : mValue(aValue) {} + + bool operator==(const Opaque& aOther) const { + return mValue == aOther.mValue; + } + + bool operator!=(const Opaque& aOther) const { + return !(*this == aOther); + } +}; + +} // namespace mozilla + +#endif /* mozilla_Opaque_h */ diff --git a/src/third_party/mozjs-38/mongodb.patch b/src/third_party/mozjs-38/mongodb.patch index 15967bd70d9..bfaae03dc3c 100644 --- a/src/third_party/mozjs-38/mongodb.patch +++ b/src/third_party/mozjs-38/mongodb.patch @@ -24,3 +24,27 @@ index e05665f..042a220 100644 const BYTE* ref; const BYTE* const iend = ip + inputSize; +diff --git a/src/third_party/mozjs-38/include/mozilla/Types.h b/src/third_party/mozjs-38/include/mozilla/Types.h +index 2697ec1..e7e18ab 100644 +--- a/src/third_party/mozjs-38/include/mozilla/Types.h ++++ b/src/third_party/mozjs-38/include/mozilla/Types.h +@@ -37,10 +37,7 @@ + * These macros are designed for use by library interfaces -- not for normal + * methods or data used cross-file. + */ +-// MongoDB Modification - SERVER-20311 +-// Disable all visibility hints since we are statically linking on Windows. +-// This prevents mongod.exe and other binaries from exporting these functions. +-#if 0 //defined(WIN32) ++#if defined(WIN32) + # define MOZ_EXPORT __declspec(dllexport) + #else /* Unix */ + # ifdef HAVE_VISIBILITY_ATTRIBUTE +@@ -52,6 +49,7 @@ + # endif + #endif + ++ + /* + * Whereas implementers use MOZ_EXPORT to declare and define library symbols, + * users use MOZ_IMPORT_API and MOZ_IMPORT_DATA to access them. Most often the diff --git a/src/third_party/mozjs-38/platform/arm64/linux/build/js-confdefs.h b/src/third_party/mozjs-38/platform/arm64/linux/build/js-confdefs.h index 999c91763ed..bee7d546412 100644 --- a/src/third_party/mozjs-38/platform/arm64/linux/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/arm64/linux/build/js-confdefs.h @@ -67,10 +67,10 @@ #define JS_STANDALONE 1 #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so" #define MOZ_UPDATE_CHANNEL default diff --git a/src/third_party/mozjs-38/platform/arm64/linux/include/js-config.h b/src/third_party/mozjs-38/platform/arm64/linux/include/js-config.h index dd3535ced05..c075812b991 100644 --- a/src/third_party/mozjs-38/platform/arm64/linux/include/js-config.h +++ b/src/third_party/mozjs-38/platform/arm64/linux/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-38/platform/i386/linux/build/js-confdefs.h b/src/third_party/mozjs-38/platform/i386/linux/build/js-confdefs.h index bebdfeab17b..68e2f9d5f58 100644 --- a/src/third_party/mozjs-38/platform/i386/linux/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/i386/linux/build/js-confdefs.h @@ -70,10 +70,10 @@ #define JS_STANDALONE 1 #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so" #define MOZ_UPDATE_CHANNEL default diff --git a/src/third_party/mozjs-38/platform/i386/linux/include/js-config.h b/src/third_party/mozjs-38/platform/i386/linux/include/js-config.h index 0b97dc2d48f..4d140eecf2e 100644 --- a/src/third_party/mozjs-38/platform/i386/linux/include/js-config.h +++ b/src/third_party/mozjs-38/platform/i386/linux/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-38/platform/i386/openbsd/build/js-confdefs.h b/src/third_party/mozjs-38/platform/i386/openbsd/build/js-confdefs.h index a9a75e9958d..297ba39d1f5 100644 --- a/src/third_party/mozjs-38/platform/i386/openbsd/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/i386/openbsd/build/js-confdefs.h @@ -59,10 +59,10 @@ #define JS_STANDALONE 1 #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so.1.0" #define MOZ_UPDATE_CHANNEL default diff --git a/src/third_party/mozjs-38/platform/i386/openbsd/include/js-config.h b/src/third_party/mozjs-38/platform/i386/openbsd/include/js-config.h index abe437abb7a..d2e2bcb0c91 100644 --- a/src/third_party/mozjs-38/platform/i386/openbsd/include/js-config.h +++ b/src/third_party/mozjs-38/platform/i386/openbsd/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-38/platform/i386/windows/build/js-confdefs.h b/src/third_party/mozjs-38/platform/i386/windows/build/js-confdefs.h index e46fd83c737..e027c288af3 100644 --- a/src/third_party/mozjs-38/platform/i386/windows/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/i386/windows/build/js-confdefs.h @@ -32,10 +32,10 @@ #define MALLOC_H #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".dll" #define MOZ_MEMORY 1 diff --git a/src/third_party/mozjs-38/platform/i386/windows/include/js-config.h b/src/third_party/mozjs-38/platform/i386/windows/include/js-config.h index a78d3daf9fe..0020a59223a 100644 --- a/src/third_party/mozjs-38/platform/i386/windows/include/js-config.h +++ b/src/third_party/mozjs-38/platform/i386/windows/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-38/platform/x86_64/freebsd/build/js-confdefs.h b/src/third_party/mozjs-38/platform/x86_64/freebsd/build/js-confdefs.h index a64b834f7c0..2a108fe363a 100644 --- a/src/third_party/mozjs-38/platform/x86_64/freebsd/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/x86_64/freebsd/build/js-confdefs.h @@ -63,10 +63,10 @@ #define JS_STANDALONE 1 #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so" #define MOZ_UPDATE_CHANNEL default diff --git a/src/third_party/mozjs-38/platform/x86_64/freebsd/include/js-config.h b/src/third_party/mozjs-38/platform/x86_64/freebsd/include/js-config.h index 544278fee5e..290b4a1fd45 100644 --- a/src/third_party/mozjs-38/platform/x86_64/freebsd/include/js-config.h +++ b/src/third_party/mozjs-38/platform/x86_64/freebsd/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-38/platform/x86_64/linux/build/js-confdefs.h b/src/third_party/mozjs-38/platform/x86_64/linux/build/js-confdefs.h index f5069d6d644..6451dc43eff 100644 --- a/src/third_party/mozjs-38/platform/x86_64/linux/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/x86_64/linux/build/js-confdefs.h @@ -71,10 +71,10 @@ #define JS_STANDALONE 1 #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so" #define MOZ_UPDATE_CHANNEL default diff --git a/src/third_party/mozjs-38/platform/x86_64/linux/include/js-config.h b/src/third_party/mozjs-38/platform/x86_64/linux/include/js-config.h index dd3535ced05..c075812b991 100644 --- a/src/third_party/mozjs-38/platform/x86_64/linux/include/js-config.h +++ b/src/third_party/mozjs-38/platform/x86_64/linux/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-38/platform/x86_64/openbsd/build/js-confdefs.h b/src/third_party/mozjs-38/platform/x86_64/openbsd/build/js-confdefs.h index 890078476f8..1180a068dc2 100644 --- a/src/third_party/mozjs-38/platform/x86_64/openbsd/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/x86_64/openbsd/build/js-confdefs.h @@ -62,10 +62,10 @@ #define JS_STANDALONE 1 #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so.1.0" #define MOZ_UPDATE_CHANNEL default diff --git a/src/third_party/mozjs-38/platform/x86_64/openbsd/include/js-config.h b/src/third_party/mozjs-38/platform/x86_64/openbsd/include/js-config.h index e6cf8c4a69b..59b145407e1 100644 --- a/src/third_party/mozjs-38/platform/x86_64/openbsd/include/js-config.h +++ b/src/third_party/mozjs-38/platform/x86_64/openbsd/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-38/platform/x86_64/osx/build/js-confdefs.h b/src/third_party/mozjs-38/platform/x86_64/osx/build/js-confdefs.h index c66542b7bf7..ae99c33ecca 100644 --- a/src/third_party/mozjs-38/platform/x86_64/osx/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/x86_64/osx/build/js-confdefs.h @@ -58,10 +58,10 @@ #define JS_STANDALONE 1 #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".dylib" #define MOZ_UPDATE_CHANNEL default diff --git a/src/third_party/mozjs-38/platform/x86_64/osx/include/js-config.h b/src/third_party/mozjs-38/platform/x86_64/osx/include/js-config.h index 544278fee5e..290b4a1fd45 100644 --- a/src/third_party/mozjs-38/platform/x86_64/osx/include/js-config.h +++ b/src/third_party/mozjs-38/platform/x86_64/osx/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-38/platform/x86_64/solaris/build/js-confdefs.h b/src/third_party/mozjs-38/platform/x86_64/solaris/build/js-confdefs.h index 58af7c43c83..d0357db4012 100644 --- a/src/third_party/mozjs-38/platform/x86_64/solaris/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/x86_64/solaris/build/js-confdefs.h @@ -62,10 +62,10 @@ #define JS_STANDALONE 1 #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".so" #define MOZ_UPDATE_CHANNEL default diff --git a/src/third_party/mozjs-38/platform/x86_64/solaris/include/js-config.h b/src/third_party/mozjs-38/platform/x86_64/solaris/include/js-config.h index 2008beb3d54..bba3ba9ab55 100644 --- a/src/third_party/mozjs-38/platform/x86_64/solaris/include/js-config.h +++ b/src/third_party/mozjs-38/platform/x86_64/solaris/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/mozjs-38/platform/x86_64/windows/build/js-confdefs.h b/src/third_party/mozjs-38/platform/x86_64/windows/build/js-confdefs.h index 0e856775ed6..3daabba2fd7 100644 --- a/src/third_party/mozjs-38/platform/x86_64/windows/build/js-confdefs.h +++ b/src/third_party/mozjs-38/platform/x86_64/windows/build/js-confdefs.h @@ -33,10 +33,10 @@ #define MALLOC_H #define MALLOC_H #define MOZILLA_UAVERSION "38.0" -#define MOZILLA_VERSION "38.6.1" -#define MOZILLA_VERSION_U 38.6.1 +#define MOZILLA_VERSION "38.8.0" +#define MOZILLA_VERSION_U 38.8.0 #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #define MOZ_DEBUG_SYMBOLS 1 #define MOZ_DLL_SUFFIX ".dll" #define MOZ_MEMORY 1 diff --git a/src/third_party/mozjs-38/platform/x86_64/windows/include/js-config.h b/src/third_party/mozjs-38/platform/x86_64/windows/include/js-config.h index df5d20e3fe7..db0de4128f1 100644 --- a/src/third_party/mozjs-38/platform/x86_64/windows/include/js-config.h +++ b/src/third_party/mozjs-38/platform/x86_64/windows/include/js-config.h @@ -54,6 +54,6 @@ /* MOZILLA JSAPI version number components */ #define MOZJS_MAJOR_VERSION 38 -#define MOZJS_MINOR_VERSION 6 +#define MOZJS_MINOR_VERSION 8 #endif /* js_config_h */ diff --git a/src/third_party/wiredtiger/.gitignore b/src/third_party/wiredtiger/.gitignore index bef47daabf9..4611f2aa98c 100644 --- a/src/third_party/wiredtiger/.gitignore +++ b/src/third_party/wiredtiger/.gitignore @@ -16,11 +16,12 @@ *.obj *.pdb *.pyc +.dirstamp .sconf_temp .sconsign.dblite -.dirstamp /Makefile.am /aclocal.m4 +/autom4te.cache/ /build_posix/aclocal/libtool.m4 /build_posix/aclocal/ltoptions.m4 /build_posix/aclocal/ltsugar.m4 @@ -32,7 +33,7 @@ /config.status /configure /configure.ac -/libtool/ +/libtool /stamp-h1 /wiredtiger.h /wiredtiger.pc @@ -63,9 +64,9 @@ WT_TEST/ # Python /lang/python/_wiredtiger.so +/lang/python/wiredtiger.py /lang/python/wiredtiger/__init__.py /lang/python/wiredtiger/_wiredtiger.pyd -/lang/python/wiredtiger.py /lang/python/wiredtiger_wrap.c _wiredtiger.pyd @@ -89,23 +90,28 @@ _wiredtiger.pyd **/examples/c/ex_pack **/examples/c/ex_process **/examples/c/ex_schema -**/examples/c/ex_scope **/examples/c/ex_stat **/examples/c/ex_sync **/examples/c/ex_thread **/test/bloom/t **/test/checkpoint/t -**/test/csuite/test_wt1965_col_efficiency -**/test/csuite/test_wt2246_col_append -**/test/csuite/test_wt2323_join_visibility -**/test/csuite/test_wt2403_lsm_workload -**/test/csuite/test_wt2447_join_main_table -**/test/csuite/test_wt2535_insert_race -**/test/csuite/test_wt2592_join_schema -**/test/csuite/test_wt2695_checksum -**/test/csuite/test_wt2719_reconfig -**/test/csuite/test_wt2834_join_bloom_fix -**/test/csuite/test_wt2853_perf +**/test_scope +**/test_wt1965_col_efficiency +**/test_wt2246_col_append +**/test_wt2323_join_visibility +**/test_wt2403_lsm_workload +**/test_wt2447_join_main_table +**/test_wt2535_insert_race +**/test_wt2592_join_schema +**/test_wt2695_checksum +**/test_wt2719_reconfig +**/test_wt2834_join_bloom_fix +**/test_wt2853_perf +**/test_wt2909_checkpoint_integrity +**/test_wt2999_join_extractor +**/test_wt3120_filesys +**/test_wt3135_search_near_collator +**/test_wt3184_dup_index_collator **/test/cursor_order/cursor_order **/test/fops/t **/test/format/s_dumpcmp @@ -120,4 +126,5 @@ _wiredtiger.pyd **/test/recovery/random-abort **/test/recovery/truncated-log **/test/salvage/t +**/test/syscall/test_wt2336_base **/test/thread/t diff --git a/src/third_party/wiredtiger/NEWS b/src/third_party/wiredtiger/NEWS index af8b15488cc..268949b119f 100644 --- a/src/third_party/wiredtiger/NEWS +++ b/src/third_party/wiredtiger/NEWS @@ -1,3 +1,418 @@ +WiredTiger release 2.9.1, 2016-12-22 +------------------------------------ + +New features and API changes; refer to the API documentation for full details: + +* SERVER-26545 Remove fixed-size limitation on WiredTiger hazard pointers. See the upgrading documentation for details +* WT-283 Add a new WT_SESSION::alter method that can be used to reconfigure table metadata +* WT-2670 Change the default file system access pattern advice for data files from random to no advice. Add access_pattern_hint configuration option for WT_SESSION::create API that can be used to advise the file system of expected access semantics. See the upgrading documentation for details. +* WT-3034 Add support for including updates when reading from named snapshots + +Significant changes and bug fixes: + +* WT-2960 Reduce likelihood of using the lookaside file, especially when inserting multi-megabyte values +* WT-3056 Allow projected table and join cursors to use primary keys +* WT-3070 Fix a bug in search_near on indexes + +Other noteworthy changes since the previous release: + +* WT-2336 Add a test validating schema operations via file system call monitoring +* WT-2402 Pad structures to avoid cache line sharing +* WT-2771 Add a statistic to track per-btree dirty cache usage +* WT-2833 Add projections to wt dump utility +* WT-2969 Possible snapshot corruption during compaction +* WT-3014 Add GCC/clang support for ELF symbol visibility +* WT-3021 Fixes for java log example, raw mode in java, and raw mode in log cursors +* WT-3025 Fix error path in log_force_sync +* WT-3028 Don't check for blocked eviction with in-memory workloads +* WT-3030 Fix a race between scans and splits reading the index hint +* WT-3037 Clean up some log slot comments +* WT-3048 WiredTiger maximum size warning uses the wrong format +* WT-3051 Remove external __wt_hex symbol +* WT-3052 Improve search if index hint is wrong +* WT-3053 Make Python use internal memory allocation again +* WT-3054 Make a PackOutputStream constructor that is compatible with the previous interface. +* WT-3055 When an AsyncOp is created, cache the whether the cursor is "raw" +* WT-3057 WiredTiger hazard pointers should use the WT_REF, not the WT_PAGE +* WT-3061 Syscall testing should support pwrite64 on Linux +* WT-3064 Minor tree cleanups: .gitignore, NEWS misspelling +* WT-3066 Minor code cleanups +* WT-3068 Copy artifacts of test runs in wtperf_run script +* WT-3068 Have Jenkins include specific files for copy rather than exclude +* WT-3069 Fix LevelDB APIs build failures +* WT-3071 Fixed sign-conversion compiler errors in Java and Python SWIG code +* WT-3075 Document and enforce that WiredTiger now depends on Python 2.7 +* WT-3078 Test reconfiguration hang in the statlog server +* WT-3080 Python test suite: add elapsed time for tests +* WT-3082 Python test suite: shorten default run to avoid timeouts +* WT-3084 Fix Coverity resource leak complaint +* WT-3091 Add stats to test_perf001 test, so we can investigate what happened when it failed + +WiredTiger release 2.9.0, 2016-09-06 +------------------------------------ + +New features and API changes; refer to the API documentation for full details: + +* WT-2360 Add nesting and disjunction functionality to cursor joins +* WT-2552 Add a public API to allow custom filesystem implementations +* WT-2738 Remove the ability to change the default checkpoint name. See the upgrading documentation for details. +* WT-2711 Change the statistics log configuration options. Restrict where statistics log files can be created to remove security vulnerability. See upgrading documentation for details. +* WT-2764 Enhance the checkpoint implementation, which led to a change in default eviction settings. See the upgrading documentation for details. +* WT-2880 Add Zstandard compression support +* See the upgrading documentation for information about renamed and removed statistics. + +Significant changes and bug fixes: + +* WT-2026 Fix a bug where in-memory pages were allowed to grow too large +* WT-2343 Ensure we cannot rename/drop when a backup cursor is open +* WT-2520 Ensure WT_SESSION::verify does not alter tables +* WT-2576 Ensure variable-length column-store cannot out-of-order return +* WT-2664 Change eviction so any eviction thread can find candidates +* WT-2702 Resolve an issue where under high thread load, WiredTiger exceeds cache size +* WT-2731 Resolve an issue where raw compression can create pages that are larger than expected +* WT-2737 Allow reconciliation to scrub dirty pages rather than evicting them +* WT-2757 Fix a bug in column stores where it was not possible to retrieve the new record number after an append +* WT-2816 Improve eviction performance + +Issues fixed in MongoDB: + +* SERVER-23504 Resolve a resource leak +* SERVER-23588 Stop using _open_osfhandle on Windows +* SERVER-23659 Improve log error message at startup +* SERVER-23661 Fix an issue where $sample takes disproportionately long time on newly created collection +* SERVER-24306 Fix stall in log_flush switching to new files +* SERVER-24580 Add more eviction stats to track efficiency +* SERVER-24580 Enhance eviction when application threads are contributing +* SERVER-24971 Don't cache buffers after application eviction +* SERVER-25843 Remove a redundant test +* SERVER-25845 Resolve an explicit null dereferenced +* SERVER-25846 Resolve a dereference after null with a check +* SERVER-26753 Don't spin on a read-lock in a tight loop + +Other noteworthy changes since the previous release: + +* WT-2103 Add incremental backup testing to test/format +* WT-2330 Ensure In-memory configurations do not create on-disk collection files +* WT-2450 Resolve an issue where salvage releases pages, then explicitly evicts them. Which can race with the eviction server +* WT-2453 Add multiple eviction queues +* WT-2504 Remove READONLY conditional for base config +* WT-2505 Resolve clang analyzer warnings +* WT-2508 Ensure test programs should remove test directories on the "clean" target +* WT-2512 Change how wtperf throttle values per thread are calculated +* WT-2513 Fix a type conversion warning +* WT-2517 Resolve incorrect uses of setvbuf on Windows +* WT-2518 Optimization of LSM checkpoint handle acquisition +* WT-2522 Resolve incorrect format code in message +* WT-2525 Miscellaneous cleanups of In-memory configurations +* WT-2526 Prevent mixing and matching readonly file-handles and read/write data-handles +* WT-2528 Fix a style error in WiredTiger build +* WT-2529 Be less aggressive asserting in readonly connections +* WT-2531 Resolve a case where in-memory tables are wasting space in truncation +* WT-2532 Change the WT_STREAM_APPEND and WT_STREAM_LINE_BUFFER flags as they overlap +* WT-2533 Don't let in-memory tables return a zero size +* WT-2534 Use atomic add when allocating transaction IDs +* WT-2535 Extend test/format to test for transactions reading their writes +* WT-2537 Check for an old and new uninitialized LSN for recovery +* WT-2539 Make streams a separate handle type from files +* WT-2540 Separate stream and file handle methods +* WT-2542 Resolve fixed-length column store reconciliation overwriting original values +* WT-2544 Fix eviction stats when clear is used +* WT-2546 Change evict server to be smarter when it decides whether or not to evict pages +* WT-2547 Add eviction testing configurations for 1 thread +* WT-2548 Cap the amount of data handed to raw compression +* WT-2550 Add support for raw mode cursors in java +* WT-2553 Add some In-memory documentation and stress testing +* WT-2556 Fix a typo in the Java example code +* WT-2557 Ensure that we truncate logs before closing backup cursor +* WT-2558 Reorganize WT_PAGE structure +* WT-2559 Resolve a potential segfault in Windows logging code +* WT-2560 Add a rwlock to protect transaction state, don't spin on this lock +* WT-2562 Add sleep loop to adapt for slow machines like PPC +* WT-2565 Further limit the amount of data passed to raw compression +* WT-2566 Add explicit memory barriers to Lock/unlock operations +* WT-2567 Add a check to see if logging is enabled for truncate rather than assuming it is +* WT-2568 Add a backward compatible constructor for PackInputStream +* WT-2569 Ensure __win_handle_read is followed by a call to GetLastError() +* WT-2570 Minor lint and cleanups to code +* WT-2571 Join code clean up +* WT-2572 Ensure test/format doesn't select an incompatible set of options for in-memory options configured +* WT-2573 Resolve a potential case where a double free can occur +* WT-2574 Ensure test/format frees all allocated configure memory +* WT-2576 Ensure variable-length column-store cannot out-of-order return +* WT-2577 Resolve a potential double-free following an in-memory split +* WT-2579 Change so diagnostic only code can use file streams directly +* WT-2580 Fix potential SWIG naming conflict in Java +* WT-2581 Resolve a case where we fail to track saved updates during a page split +* WT-2582 Fix a case where eviction can return WT_RESTART when running in-memory +* WT-2584 Change to not use periods at the end of error messages +* WT-2586 Fix examples/ex_config +* WT-2589 Ensure we clear the cursor statistics for LAS when 'clear' is set +* WT-2592 Fix joins for the non-recno, non-raw case +* WT-2593 Free the dirlist at the end of the function +* WT-2595 Fix compiler warning in packing test +* WT-2598 Change so that in-memory has faster lookup on file names +* WT-2599 Split out the checksum code from the support directory +* WT-2600 Clean up test program includes +* WT-2602 In LSM, use the chunk size to set maximum page size in memory +* WT-2605 Add C tests used in testing joins +* WT-2609 Resolve incorrect "skips API_END call" error +* WT-2610 Reduce hazard pointer array size +* WT-2611 Modify wtperf to handle escaped quotes +* WT-2612 Ensure the dist/s_prototypes script does not create a debugging file +* WT-2613 Add WT_UNUSED to a variable to fix Windows compilation +* WT-2615 Enabling checkpoints in test/format leads to reduced concurrency +* WT-2616 Fix a deadlock with in-memory size lookups +* WT-2617 Don't use u_int in the example code. It's not Windows native +* WT-2621 Fix missing parentheses on call to __wt_errno +* WT-2622 Clear old position for all random lookups +* WT-2624 Fix example program build with MSVC 2013 +* WT-2626 Fix MSVC 2015 snprintf redefinition +* WT-2627 Resolve Coverity complaints +* WT-2628 Ensure reconciliation doesn't return without unlocking the page lock +* WT-2629 Ensure stacks are not executable in assembly source +* WT-2630 Rename pluggable filesystem methods to avoid reserved names +* WT-2631 Ensure nullptr is not passed for parameters marked with attribute non-null +* WT-2632 Tolerate EBUSY when a checkpoint cursor is open +* WT-2637 Update file-extension documentation to cover not-supported cases +* WT-2638 Add extension to the Windows file system implementation +* WT-2644 Fix 'wt load -r' (rename) with LSM +* WT-2645 Move the complexity of dump from the dump utility to the metadata cursor +* WT-2646 Add checkpoint_wait configuration option to drop +* WT-2648 Change cache-line alignment for new ports +* WT-2651 Fix a Coverity found resource leak +* WT-2652 Remove unnecessary wt_ftruncate call +* WT-2653 Add display of device configuration to custom file-system example +* WT-2656 Fix GCC 4.7 compiler warnings +* WT-2658 Ensure we only include PPC-specific files in PPC builds +* WT-2659 Assorted lint and cleanup to csuite tests +* WT-2660 Fix a hang between eviction and connection close +* WT-2662 For spell check, strip out double quote literals, they confuse aspell +* WT-2664 Add ability for eviction workers to populate eviction queues +* WT-2665 Limit allocator fragmentation from the WiredTiger cache +* WT-2667 Add fops to Evergreen testing for Windows and Linux +* WT-2668 Create join statistics that are useful and are easy to understand +* WT-2671 Dump more information about the file layout in verify debug mode +* WT-2672 Handle system calls that don't set errno +* WT-2673 Stop automatically increasing memory page max +* WT-2674 Simplify metadata file check +* WT-2676 Don't use key size in column store in-memory splits +* WT-2677 Fix JSON output so only printable ASCII is produced +* WT-2678 Fix cases where metadata implies that an empty value is true +* WT-2682 Add option to configure WiredTiger with strict compiler flags +* WT-2683 Allow in memory storage engine to report zero disk usage +* WT-2685 Fix clear walk hazard pointer failure +* WT-2686 Report an error any time we fail to scan the log +* WT-2687 Provide ability for test suite to verify the exit status of the wt utility +* WT-2688 Improve build error messages when SWIG is unavailable +* WT-2689 Fix heap-use-after-free on cursor error path +* WT-2691 Use wrappers for ctype functions to avoid sign extension errors +* WT-2692 Fix race in file system example +* WT-2693 Ensure open_cursor error paths have consistent error handling +* WT-2695 Integrate s390x accelerated crc32c support +* WT-2696 Wait if we find an unbuffered flag without the size set yet +* WT-2698 Change flag into atomically manipulated field to avoid deadlock +* WT-2702 Block operations when the cache is 100% full +* WT-2704 Fix bug in diagnostic code tracking duration of stuck cache +* WT-2706 Fix lost log writes when switching files +* WT-2707 Add stricter checks in dist/s_label, and add some WT_ERR calls +* WT-2708 Fix a split child-update race with reconciliation/eviction +* WT-2709 Resolve a connection reconfigure segfault in __wt_conn_cache_pool_destroy +* WT-2710 WT_FILE_HANDLE_INMEM no longer needs an off field +* WT-2713 Document WT_PANIC so pluggable filesystems can panic +* WT-2715 Resolve random abort test failure with partial write +* WT-2719 Allow make check without verbose configured +* WT-2719 Fix memory leak in reconfig test +* WT-2720 Change the return codes in run.py to error on failures +* WT-2722 Escape regular expression meta-character so egrep works on all systems +* WT-2724 Pass in session address, not pointer to it +* WT-2728 Only verify the log file header during recovery +* WT-2729 Focus eviction on the largest trees in cache +* WT-2730 Btree can incorrectly match key slots on new pages +* WT-2731 Finer adjustment for page size with raw compression +* WT-2732 Coverity analysis defect 99665, Remove redundant test +* WT-2734 Improve documentation of eviction configuration settings +* WT-2739 pluggable file systems documentation cleanups +* WT-2743 Fixup new I/O thread count statistics +* WT-2744 Detect and ignore partial line +* WT-2746 Add a new statistic tracking I/O for checkpoints +* WT-2751 column-store statistics incorrectly calculates the number of entries +* WT-2752 Fix errors in wtperf config +* WT-2755 Fix for cases where tools treats size_t as 4B type +* WT-2756 Upgrade the autoconf archive package to check for swig 3.0 +* WT-2757 Column tables behave differently when column names are provided +* WT-2759 Releasing the hot-backup lock doesn't require the schema lock +* WT-2760 Fix a bug in backup related to directory sync. Change the filesystem API to make durable the default +* WT-2762 Handle 'Y' as an options from wtperf monitor file +* WT-2763 Change test ID's in test_intpack to fix OS X test errors +* WT-2764 Fix a bug calculating the dirty percentage of cache +* WT-2765 Ensure indices are shown in the dump output +* WT-2766 Don't sweep LAS cache when aren't making progress in eviction +* WT-2767 In test/suite/run.py, add -s N option to run an individual scenario +* WT-2769 Update documentation to reflect correct limits of memory_page_max +* WT-2770 Add statistics tracking schema operations +* WT-2772 Adjust log.wtperf config. Remove unneeded config entries +* WT-2773 Ensure that search_near in an index finds exact matches +* WT-2774 Minor cleanups/improvements +* WT-2778 Enhance output formatting when running Python test suite +* WT-2779 Fix large pages getting generated with raw compression +* WT-2781 Don't take checkpoint lock if checkpoint_wait=0 for bulk cursor open +* WT-2782 Minor text changes for file system functions +* WT-2783 Clean up wtperf configuration object management +* WT-2785 Scrub dirty pages rather than evicting them single-page reconciliation +* WT-2787 Include src/include/wiredtiger_ext.h is problematic +* WT-2788 Do not touch memory already freed during a close API call +* WT-2791 Have evergreen upload artifacts for each build. Rename current artifacts to "Binaries" +* WT-2793 Enhance statistics related to overflow values +* WT-2795 Update documentation about read-only mode +* WT-2796 Fix a memory leak when using the lookaside table +* WT-2798 Fix data consistency bug with table creates during a checkpoint +* WT-2800 Fix incorrect error message +* WT-2801 Prevent eviction of metadata updates by a running checkpoint +* WT-2802 Copy values during commit before releasing snapshot +* WT-2803 Add VERBOSE=1 to all make check jobs +* WT-2804 Don't read values in a tree without a snapshot +* WT-2805 Avoid infinite recursion on error stream failure +* WT-2806 Fix an off-by-one allocation in wtperf +* WT-2807 Change the memory allocator for wtperf performance tests +* WT-2811 The checkpoint session should not ignore it's own transaction ID +* WT-2812 Verify cache_size before dividing to avoid division by 0 +* WT-2813 Configure eviction dirty settings explicitly for LSM in test/format +* WT-2814 Add new single op truncate mode to wtperf +* WT-2816 General improvements to WiredTiger eviction performance +* WT-2817 Add wtperf conf to stress test checkpoints with updates +* WT-2818 Change the page visibility check when queuing pages for eviction +* WT-2820 Add gcc warn_unused_result attribute +* WT-2822 Simplify error handling by using a panic mutex in functions that cannot fail +* WT-2823 Support file handles without a truncate method +* WT-2824 Fix double concatenation the config strings +* WT-2826 Fix a clang38 false positive on uninitialized variable +* WT-2827 Set a reasonable minimum for log_size +* WT-2828 Change wtperf long tests to better match MongoDB workloads +* WT-2829 Fix missing define for BerkeleyDB +* WT-2831 Skip checkpointing if there have been no modifications +* WT-2832 Python test uses hard-coded temporary directory +* WT-2834 Fix sign-change warning with auto conversion from uint to int +* WT-2834 Shared bloom filters allow only a partial shortcut +* WT-2835 Stop WT_CONNECTION.leak-memory skipping memory map and cache cleanup +* WT-2838 Don't free session handles on close if leak memory is configured +* WT-2839 Remove cases ignoring return value of function +* WT-2840 Fix garbage values found with clang analysis +* WT-2841 Initialize verbose checkpoint timer at start of checkpoint +* WT-2842 Add explicit include in wtperf to resolve build warning +* WT-2843 If there is no truncate available, manually zero the log file +* WT-2846 Ensure that all resources are released when destroying a thread group +* WT-2846 Fixes for new thread group code +* WT-2847 Merge fair locks into read/write locks +* WT-2850 Fix clang 4.1 attribute warnings +* WT-2853 Don't force eviction if multiple cursors are pinning the page +* WT-2857 POSIX ftruncate calls should be use HAVE_FTRUNCATE define +* WT-2858 Rename wtperf's CONFIG structure +* WT-2859 Restructure statistics gathering macros +* WT-2863 Support UTF-8 paths on Windows +* WT-2864 Update reconfigure test to detect hangs +* WT-2865 Fix a bug where the eviction server could panic after a WT_NOTFOUND +* WT-2866 Don't set eviction stuck unless the cache is full +* WT-2867 Review and fix barrier usage in __lsm_tree_close +* WT-2868 Add sample_interval to checkpoint-stress.wtperf +* WT-2869 Fix a performance regression on secondaries +* WT-2870 Rename wtperf jobs for consistency +* WT-2871 Make verbose formats and argument types match +* WT-2872 Ensure tests with tiny caches don't get stuck due to the dirty trigger +* WT-2873 Refactor CRC32 code +* WT-2874 Change test_compact01 to avoid eviction +* WT-2875 Add environment variable to disable long csuite tests +* WT-2876 Add an oplog-like ability to wtperf utility +* WT-2878 Fix an issue where verbose changes affected performance +* WT-2881 Add -Wpedantic to clang compiler warning flags +* WT-2883 Remove potentially recursive call for verbose handleops +* WT-2885 Lint __wt_checkpoint_signal function +* WT-2886 Ignore dirty eviction settings when in_memory is true +* WT-2888 Switch functions to return void where possible +* WT-2892 Fix case where hot backup can race with block truncate +* WT-2894 Add wtperf stress workload that tries to induce negative scaling +* WT-2895 Reduce the runtime of make check testing with disable long +* WT-2896 Fix a resource leak +* WT-2897 Fix an issue where checkpoints can become corrupted on failure +* WT-2900 Add ARM8 build support to WiredTiger and fix ARM CRC assembler tags +* WT-2901 Make checkpoint scrubbing configurable +* WT-2902 Track per-tree the efficiency of eviction walks +* WT-2903 Change the eviction_scrub_target default to 15% +* WT-2904 Fix a bug where the reported checkpoint size could be many times data size +* WT-2904 Revert overly strong assertion +* WT-2905 Remove dead code +* WT-2907 Fixed java concurrent close test to have both insert threads and scan threads +* WT-2908 Add a dry-run option to python suite +* WT-2910 When running in-memory, only evict dirty pages +* WT-2911 Add support for gcc6 +* WT-2912 Make --enable-strict run on the zseries test box +* WT-2913 Re-sort eviction queues if we find no new candidates +* WT-2914 In test/csuite program, added explicit call to wiredtiger_open to satisfy some linkers +* WT-2916 Fix and simplify s_whitespace +* WT-2917 Split wtperf's configuration into per-database and per-run parts +* WT-2918 The dist scripts create C files s_whitespace complains about +* WT-2919 Don't mask error returns from style checking scripts +* WT-2920 New eviction statistics +* WT-2921 Reset the count when the last hazard pointer is cleared +* WT-2923 Fix heap-use-after-free on address in compaction +* WT-2924 Ensure we are doing eviction when threads are waiting for it +* WT-2925 Change where the WT_THREAD_PANIC_FAIL flag lives +* WT-2926 WT_CONNECTION.reconfigure can attempt unlock of not-locked lock +* WT-2928 Don't give up when the urgent queue is half empty +* WT-2928 Eviction failing to switch queues can lead to starvation +* WT-2931 Enforce limits on dirty data for in-memory +* WT-2932 Add a configuration option allowing tables to ignore cache limits +* WT-2933 Fix a race between named snapshots and checkpoints +* WT-2937 Only do an eviction walk after a page has been taken from the current queue +* WT-2938 Change assembly file extensions from .S to .sx +* WT-2941 Improve test/format to use faster key-generation functions +* WT-2942 Verbose messages should not have newlines +* WT-2945 Occasional hang running reconfigure fuzz test +* WT-2946 Fix dist/s_docs being incompatible with OS X Xcode installation +* WT-2947 Replace test suite populate functions with *DataSet classes +* WT-2948 Simplify error handling by making __wt_epoch return never fail +* WT-2949 Add option to skip closing the connection after test runs +* WT-2950 Free all reconciliation memory between calls to eviction +* WT-2953 Only test for checkpoint/LAS collision if LAS was used +* WT-2954 Evict pages that exceed memory_page_max even if the transaction ID is stuck +* WT-2955 Add statistics tracking the amount of time threads spend waiting for high level locks +* WT-2959 Ensure WT_SESSION_IMPL is never used before it's initialized +* WT-2961 Add a version drop-down to the web version of the docs +* WT-2962 Allow configuration of builtin extensions +* WT-2963 Fix a race bounding eviction in small caches +* WT-2964 Alter evict walk to not fill all its slots with internal pages when running in aggressive mode +* WT-2965 Remove sleep and retry loop from __evict_exclusive +* WT-2968 Don't open file handles on backup +* WT-2971 Add details on raw-compression into WT documentation +* WT-2975 Fix a leak of statistics data when reopening handles +* WT-2976 Add a statistic tracking how long app threads spend reading and writing +* WT-2977 Remove stash and check code from LSM test +* WT-2984 Keep sufficient history in the metadata for queries +* WT-2985 Fix a race during checkpoint that can cause a core dump +* WT-2987 Fix a bug where opening a cursor on an incomplete table drops core +* WT-2988 Fix __wt_epoch potentially returning garbage values +* WT-2991 Fix Coverity REVERSE_INULL +* WT-2998 Add error messages to error returns that might be confusing +* WT-2999 Added contributed test case that demonstrated the leak +* WT-3000 Wait for previous writes on first write to new log file +* WT-3001 Fix WT_EXTENSION_API references that are named inconsistently +* WT-3002 Allow applications to exempt threads from eviction +* WT-3003 Don't generate log record and op types +* WT-3003 Fix doxygen comment blocks +* WT-3004 Lint, declare functions that don't return a value as void +* WT-3005 Add top-level .gitignore file +* WT-3008 Move wtperf stress jobs to new stress runner folder +* WT-3009 Remove the eviction_dirty_target from test/format runs with less than 20MB cache +* WT-3011 Fix __wt_curjoin_open() saving the wrong URI in the cursor +* WT-3012 Check a btree is LSM primary before switching +* WT-3012 Don't track the LSM Primary as part of dirty bytes in cache +* WT-3015 Change when we will evict internal pages +* WT-3016 Change how file size is determined during compact tests to allow +* WT-3017 Don't set NULL hazard pointers +* WT-3020 Always make LSM chunks evictable when they are switched out +* WT-3022 Change lsm_tree flags to fields to prevent race conditions +* WT-3023 Don't treat splits as eviction making progress +* WT-3024 Fix a hang on close caused by leaving a transaction ID pinned + WiredTiger release 2.8.0, 2015-03-24 ------------------------------------ diff --git a/src/third_party/wiredtiger/README b/src/third_party/wiredtiger/README index 32549d099d5..f7edae2835d 100644 --- a/src/third_party/wiredtiger/README +++ b/src/third_party/wiredtiger/README @@ -1,6 +1,6 @@ -WiredTiger 2.8.1: (March 24, 2016) +WiredTiger 2.9.2: (December 23, 2016) -This is version 2.8.1 of WiredTiger. +This is version 2.9.2 of WiredTiger. WiredTiger release packages and documentation can be found at: @@ -8,7 +8,7 @@ WiredTiger release packages and documentation can be found at: The documentation for this specific release can be found at: - http://source.wiredtiger.com/2.8.1/index.html + http://source.wiredtiger.com/2.9.2/index.html The WiredTiger source code can be found at: diff --git a/src/third_party/wiredtiger/RELEASE_INFO b/src/third_party/wiredtiger/RELEASE_INFO index d29c29b554c..b7145aa2cb3 100644 --- a/src/third_party/wiredtiger/RELEASE_INFO +++ b/src/third_party/wiredtiger/RELEASE_INFO @@ -1,6 +1,6 @@ WIREDTIGER_VERSION_MAJOR=2 -WIREDTIGER_VERSION_MINOR=8 -WIREDTIGER_VERSION_PATCH=1 +WIREDTIGER_VERSION_MINOR=9 +WIREDTIGER_VERSION_PATCH=2 WIREDTIGER_VERSION="$WIREDTIGER_VERSION_MAJOR.$WIREDTIGER_VERSION_MINOR.$WIREDTIGER_VERSION_PATCH" WIREDTIGER_RELEASE_DATE=`date "+%B %e, %Y"` diff --git a/src/third_party/wiredtiger/SConstruct b/src/third_party/wiredtiger/SConstruct index 0ccdf59babc..b397f662be7 100644 --- a/src/third_party/wiredtiger/SConstruct +++ b/src/third_party/wiredtiger/SConstruct @@ -294,6 +294,7 @@ env.Depends(wtdll, [filelistfile, version_file]) Default(wtlib, wtdll) wtbin = env.Program("wt", [ + "src/utilities/util_alter.c", "src/utilities/util_backup.c", "src/utilities/util_cpyright.c", "src/utilities/util_compact.c", @@ -312,6 +313,7 @@ wtbin = env.Program("wt", [ "src/utilities/util_rename.c", "src/utilities/util_salvage.c", "src/utilities/util_stat.c", + "src/utilities/util_truncate.c", "src/utilities/util_upgrade.c", "src/utilities/util_verbose.c", "src/utilities/util_verify.c", @@ -374,7 +376,6 @@ examples = [ "ex_pack", "ex_process", "ex_schema", - "ex_scope", "ex_stat", "ex_thread", ] diff --git a/src/third_party/wiredtiger/api/leveldb/basho/perf_count.cc b/src/third_party/wiredtiger/api/leveldb/basho/perf_count.cc index 0e666ac1dc0..97b8cd6dbdf 100644 --- a/src/third_party/wiredtiger/api/leveldb/basho/perf_count.cc +++ b/src/third_party/wiredtiger/api/leveldb/basho/perf_count.cc @@ -19,16 +19,19 @@ // under the License. // // ------------------------------------------------------------------- +#define __STDC_FORMAT_MACROS +#include +#include #include +#include #include + #include #include #include #include #include -#include -#include #ifndef STORAGE_LEVELDB_INCLUDE_PERF_COUNT_H_ #include "perf_count.h" @@ -36,8 +39,6 @@ #include "util/coding.h" -#define __STDC_FORMAT_MACROS -#include #ifdef OS_SOLARIS # include diff --git a/src/third_party/wiredtiger/api/leveldb/hyper_wt.cc b/src/third_party/wiredtiger/api/leveldb/hyper_wt.cc index 95c82289e18..c12aedc7bad 100644 --- a/src/third_party/wiredtiger/api/leveldb/hyper_wt.cc +++ b/src/third_party/wiredtiger/api/leveldb/hyper_wt.cc @@ -27,6 +27,7 @@ #include "leveldb_wt.h" #include +#include #include #include #include @@ -143,7 +144,14 @@ class ReplayIteratorImpl : public ReplayIterator { } private: - void SeekTo(WT_LSN *lsn); + /* + * A log sequence number, representing a position in the transaction log. + */ + typedef struct { + uint32_t file; /* Log file number */ + unsigned long long offset; /* Log file offset */ + } LSN; + void SeekTo(LSN *lsn); // No copying allowed ReplayIteratorImpl(const ReplayIterator&) { } void operator=(const ReplayIterator&) { } @@ -151,7 +159,7 @@ class ReplayIteratorImpl : public ReplayIterator { Status status_; WT_CURSOR *cursor_; WT_ITEM key_, value_; - WT_LSN lsn_; + LSN lsn_; bool valid_; uint64_t txnid; uint32_t fileid, opcount, optype, rectype; @@ -201,7 +209,7 @@ ReplayIteratorImpl::Next() { void ReplayIteratorImpl::SeekToLast() { int ret = 0; - WT_LSN last_lsn; + LSN last_lsn; last_lsn.file = 0; if (cursor_ != NULL) { @@ -235,7 +243,7 @@ ReplayIteratorImpl::SeekToLast() { void ReplayIteratorImpl::SeekTo(const std::string& timestamp) { - WT_LSN target_lsn; + LSN target_lsn; int ret = 0; if (timestamp == "all") { @@ -260,7 +268,7 @@ ReplayIteratorImpl::SeekTo(const std::string& timestamp) { // Set the cursor on the first modification record at or after the // given LSN. void -ReplayIteratorImpl::SeekTo(WT_LSN *target_lsn) { +ReplayIteratorImpl::SeekTo(LSN *target_lsn) { int ret = 0; valid_ = false; diff --git a/src/third_party/wiredtiger/api/leveldb/leveldb_wt.h b/src/third_party/wiredtiger/api/leveldb/leveldb_wt.h index dc185183034..351eb9f3dda 100644 --- a/src/third_party/wiredtiger/api/leveldb/leveldb_wt.h +++ b/src/third_party/wiredtiger/api/leveldb/leveldb_wt.h @@ -43,6 +43,7 @@ #include "basho/perf_count.h" #endif +#include #include "wiredtiger.h" #define WT_URI "table:data" diff --git a/src/third_party/wiredtiger/bench/wtperf/config.c b/src/third_party/wiredtiger/bench/wtperf/config.c index 5b14a4cdf68..e4eee66e4cb 100644 --- a/src/third_party/wiredtiger/bench/wtperf/config.c +++ b/src/third_party/wiredtiger/bench/wtperf/config.c @@ -215,6 +215,7 @@ config_threads(WTPERF *wtperf, const char *config, size_t len) return (EINVAL); } workp = &wtperf->workload[wtperf->workload_cnt++]; + workp->table_index = INT32_MAX; while ((ret = scan->next(scan, &k, &v)) == 0) { if (STRING_MATCH("count", k.str, k.len)) { @@ -233,12 +234,28 @@ config_threads(WTPERF *wtperf, const char *config, size_t len) goto err; continue; } + if (STRING_MATCH("pause", k.str, k.len)) { + if ((workp->pause = v.val) < 0) + goto err; + continue; + } if (STRING_MATCH("read", k.str, k.len) || STRING_MATCH("reads", k.str, k.len)) { if ((workp->read = v.val) < 0) goto err; continue; } + if (STRING_MATCH("read_range", k.str, k.len)) { + if ((workp->read_range = v.val) < 0) + goto err; + continue; + } + if (STRING_MATCH("table", k.str, k.len)) { + if (v.val <= 0) + goto err; + workp->table_index = (int32_t)v.val - 1; + continue; + } if (STRING_MATCH("throttle", k.str, k.len)) { workp->throttle = (uint64_t)v.val; continue; @@ -421,14 +438,13 @@ config_opt(WTPERF *wtperf, WT_CONFIG_ITEM *k, WT_CONFIG_ITEM *v) return (EINVAL); } strp = (char **)valueloc; - newlen = v->len + 1; if (*strp == NULL) begin = newstr = dstrdup(v->str); else { - newlen += strlen(*strp) + 1; - newstr = dcalloc(newlen, sizeof(char)); - snprintf(newstr, newlen, - "%s,%*s", *strp, (int)v->len, v->str); + newlen = strlen(*strp) + v->len + strlen(",") + 1; + newstr = dmalloc(newlen); + testutil_check(__wt_snprintf(newstr, newlen, + "%s,%.*s", *strp, (int)v->len, v->str)); /* Free the old value now we've copied it. */ free(*strp); begin = &newstr[(newlen - 1) - v->len]; @@ -622,17 +638,9 @@ config_opt_str(WTPERF *wtperf, const char *optstr) return (ret); } - /* - * Append the current line to our copy of the config. The config is - * stored in the order it is processed, so added options will be after - * any parsed from the original config. We allocate len + 1 to allow for - * a null byte to be added. - */ - config_line = dcalloc(sizeof(CONFIG_QUEUE_ENTRY), 1); - config_line->string = dstrdup(optstr); - TAILQ_INSERT_TAIL(&opts->config_head, config_line, q); - while (ret == 0) { + size_t pos; + if ((ret = scan->next(scan, &k, &v)) != 0) { /* Any parse error has already been reported. */ if (ret == WT_NOTFOUND) @@ -640,6 +648,46 @@ config_opt_str(WTPERF *wtperf, const char *optstr) break; } ret = config_opt(wtperf, &k, &v); + + /* + * Append the key-value pair to our copy of the config. + * The config is stored in the order it is processed, so added + * options will be after any parsed from the original config. + */ + config_line = dcalloc(sizeof(CONFIG_QUEUE_ENTRY), 1); + /* + * If key or value is a string, consider extra space for the + * quotes. Add 2 to the required space for '=' and the ending + * null character in "key=value". + */ + config_line->string = dcalloc( + k.len + (k.type == WT_CONFIG_ITEM_STRING ? 2 : 0) + + v.len + (v.type == WT_CONFIG_ITEM_STRING ? 2 : 0) + 2, 1); + pos = 0; + if (k.type == WT_CONFIG_ITEM_STRING) { + config_line->string[pos] = '"'; + pos++; + } + strncpy(config_line->string + pos, k.str, k.len); + pos += k.len; + if (k.type == WT_CONFIG_ITEM_STRING) { + config_line->string[pos] = '"'; + pos++; + } + config_line->string[pos] = '='; + pos++; + if (v.type == WT_CONFIG_ITEM_STRING) { + config_line->string[pos] = '"'; + pos++; + } + strncpy(config_line->string + pos, v.str, v.len); + pos += v.len; + if (v.type == WT_CONFIG_ITEM_STRING) { + config_line->string[pos] = '"'; + pos++; + } + config_line->string[pos] = '\0'; + TAILQ_INSERT_TAIL(&opts->config_head, config_line, q); } if ((t_ret = scan->close(scan)) != 0) { lprintf(wtperf, ret, 0, "Error in config_scan_end"); @@ -663,7 +711,7 @@ config_opt_name_value(WTPERF *wtperf, const char *name, const char *value) /* name="value" */ len = strlen(name) + strlen(value) + 4; optstr = dmalloc(len); - snprintf(optstr, len, "%s=\"%s\"", name, value); + testutil_check(__wt_snprintf(optstr, len, "%s=\"%s\"", name, value)); ret = config_opt_str(wtperf, optstr); free(optstr); return (ret); @@ -728,16 +776,33 @@ config_sanity(WTPERF *wtperf) opts->value_sz_min = opts->value_sz; } - if (opts->readonly && wtperf->workload != NULL) + if (wtperf->workload != NULL) for (i = 0, workp = wtperf->workload; - i < wtperf->workload_cnt; ++i, ++workp) - if (workp->insert != 0 || workp->update != 0 || - workp->truncate != 0) { + i < wtperf->workload_cnt; ++i, ++workp) { + if (opts->readonly && + (workp->insert != 0 || workp->update != 0 || + workp->truncate != 0)) { fprintf(stderr, "Invalid workload: insert, update or " "truncate specified with readonly\n"); return (EINVAL); } + if (workp->insert != 0 && + workp->table_index != INT32_MAX) { + fprintf(stderr, + "Invalid workload: Cannot insert into " + "specific table only\n"); + return (EINVAL); + } + if (workp->table_index != INT32_MAX && + workp->table_index >= (int32_t)opts->table_count) { + fprintf(stderr, + "Workload table index %" PRId32 + " is larger than table count %" PRId32, + workp->table_index, opts->table_count); + return (EINVAL); + } + } return (0); } @@ -754,8 +819,11 @@ config_consolidate(CONFIG_OPTS *opts) /* * This loop iterates over the config queue and for each entry checks if - * a later queue entry has the same key. If there's a match, the current - * queue entry is removed and we continue. + * a later queue entry has the same key. If there's a match, and key is + * "conn_config" or "table_config", the later queue entry is replaced + * with a concatenated entry of the two queue entries, the current queue + * entry is removed. For any other key, if there is a match, the current + * queue entry is removed. */ conf_line = TAILQ_FIRST(&opts->config_head); while (conf_line != NULL) { @@ -771,6 +839,34 @@ config_consolidate(CONFIG_OPTS *opts) if (strncmp(conf_line->string, test_line->string, (size_t)((string_key - conf_line->string) + 1)) == 0) { + if ((strncmp("conn_config=", conf_line->string, + (size_t)((string_key - conf_line->string) + + 1)) == 0) || + (strncmp("table_config=", conf_line->string, + (size_t)((string_key - conf_line->string) + + 1)) == 0)) { + char *concat_str, *val_pointer; + + /* + * To concatenate the two config + * strings, copy the first string to a + * new one, replace the ending '"' with + * a ',' and then concatenate the second + * string's value after its starting '"' + */ + val_pointer = + strchr(test_line->string, '=') + 2; + concat_str = + dmalloc(strlen(conf_line->string) + + strlen(val_pointer) + 1); + strcpy(concat_str, conf_line->string); + concat_str[strlen(concat_str) - 1] = + ','; + strcat(concat_str, val_pointer); + free(test_line->string); + test_line->string = concat_str; + } + TAILQ_REMOVE(&opts->config_head, conf_line, q); free(conf_line->string); free(conf_line); diff --git a/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c b/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c index 13fa55e86f5..4387860cfb2 100644 --- a/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c +++ b/src/third_party/wiredtiger/bench/wtperf/idle_table_cycle.c @@ -80,8 +80,8 @@ cycle_idle_tables(void *arg) } for (cycle_count = 0; wtperf->idle_cycle_run; ++cycle_count) { - snprintf(uri, sizeof(uri), - "%s_cycle%07d", wtperf->uris[0], cycle_count); + testutil_check(__wt_snprintf(uri, sizeof(uri), + "%s_cycle%07d", wtperf->uris[0], cycle_count)); /* Don't busy cycle in this loop. */ __wt_sleep(1, 0); @@ -120,6 +120,7 @@ cycle_idle_tables(void *arg) return (NULL); start = stop; +#if 1 /* * Drop the table. Keep retrying on EBUSY failure - it is an * expected return when checkpoints are happening. @@ -136,6 +137,7 @@ cycle_idle_tables(void *arg) } if (check_timing(wtperf, "drop", start, &stop) != 0) return (NULL); +#endif } return (NULL); diff --git a/src/third_party/wiredtiger/bench/wtperf/misc.c b/src/third_party/wiredtiger/bench/wtperf/misc.c index 24b3323a49a..0874794e01e 100644 --- a/src/third_party/wiredtiger/bench/wtperf/misc.c +++ b/src/third_party/wiredtiger/bench/wtperf/misc.c @@ -46,8 +46,8 @@ setup_log_file(WTPERF *wtperf) len = strlen(wtperf->monitor_dir) + strlen(opts->table_name) + strlen(".stat") + 2; fname = dmalloc(len); - snprintf(fname, len, - "%s/%s.stat", wtperf->monitor_dir, opts->table_name); + testutil_check(__wt_snprintf(fname, len, + "%s/%s.stat", wtperf->monitor_dir, opts->table_name)); if ((wtperf->logf = fopen(fname, "w")) == NULL) { ret = errno; fprintf(stderr, "%s: %s\n", fname, strerror(ret)); diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u.wtperf index 536127f0dd8..4d2a70f1107 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u.wtperf @@ -5,7 +5,7 @@ # # Set cache to half of memory of AWS perf instance. Enable logging and # checkpoints. Collect wiredtiger stats for ftdc. -conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=4)" +conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=8)" create=false compression="snappy" sess_config="isolation=snapshot" diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-80r20u.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-80r20u.wtperf index d6218c44af0..6645df835df 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-80r20u.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-80r20u.wtperf @@ -5,7 +5,7 @@ # # Set cache to half of memory of AWS perf instance. Enable logging and # checkpoints. Collect wiredtiger stats for ftdc. -conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=4)" +conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=8)" create=false compression="snappy" # close_conn as false allows this test to close/finish faster, but if running diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-populate.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-populate.wtperf index f9aed094aa1..ab7b17ca683 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-populate.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-populate.wtperf @@ -9,7 +9,7 @@ # # This generates about 80 Gb of uncompressed data. But it should compress # well and be small on disk. -conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=4)" +conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=8)" compact=true compression="snappy" sess_config="isolation=snapshot" diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-rdonly.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-rdonly.wtperf index 2c9540ff589..e8958d20e2c 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-rdonly.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-rdonly.wtperf @@ -5,7 +5,7 @@ # # Set cache to half of memory of AWS perf instance. Enable logging and # checkpoints. Collect wiredtiger stats for ftdc. -conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=4)" +conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=8)" create=false compression="snappy" sess_config="isolation=snapshot" diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/checkpoint-stress.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/checkpoint-stress.wtperf index bbd3a3ba5ed..5daa276e622 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/checkpoint-stress.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/checkpoint-stress.wtperf @@ -1,6 +1,6 @@ # A stress configuration to create long running checkpoints while doing a lot # of updates. -conn_config="cache_size=16GB,eviction=(threads_max=4),log=(enabled=false)" +conn_config="cache_size=16GB,eviction=(threads_max=8),log=(enabled=false)" table_config="leaf_page_max=32k,internal_page_max=16k,allocation_size=4k,split_pct=90,type=file" # Enough data to fill the cache. 150 million 1k records results in two ~11GB # tables diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-1.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-1.wtperf index 24da4dd7902..741101d083f 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-1.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-1.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict btree configuration -conn_config="cache_size=50M" +conn_config="cache_size=50M,eviction=(threads_max=1)" table_config="type=file" icount=10000000 report_interval=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-readonly.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-readonly.wtperf index 25599fadd8d..972bc371f2d 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-readonly.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-readonly.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict btree configuration -conn_config="cache_size=50M,eviction=(threads_max=4),mmap=false" +conn_config="cache_size=50M,eviction=(threads_max=8),mmap=false" table_config="type=file" icount=10000000 report_interval=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-stress-multi.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-stress-multi.wtperf index a5a29f66fa0..5a2cad6d78e 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-stress-multi.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-stress-multi.wtperf @@ -1,4 +1,4 @@ -conn_config="cache_size=1G,eviction=(threads_max=4),session_max=2000" +conn_config="cache_size=1G,eviction=(threads_max=8),session_max=2000" table_config="type=file" table_count=100 close_conn=false diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-stress.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-stress.wtperf index 740fb88c050..96e3f01b325 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-stress.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree-stress.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict btree configuration -conn_config="cache_size=50M,eviction=(threads_max=4)" +conn_config="cache_size=50M,eviction=(threads_max=8)" table_config="type=file" icount=10000000 report_interval=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree.wtperf index e7d967e5c63..3810e6a8294 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-btree.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict btree configuration -conn_config="cache_size=50M,eviction=(threads_max=4)" +conn_config="cache_size=50M,eviction=(threads_max=8)" table_config="type=file" icount=10000000 report_interval=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm-1.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm-1.wtperf index ad885d98eb7..641a85dc889 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm-1.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm-1.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict lsm configuration -conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6)" +conn_config="cache_size=50M,eviction=(threads_max=1),lsm_manager=(worker_thread_max=6)" table_config="type=lsm,lsm=(chunk_size=2M),os_cache_dirty_max=16MB" compact=true icount=10000000 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm-readonly.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm-readonly.wtperf index 661b8e21924..470dca695dd 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm-readonly.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm-readonly.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict lsm configuration -conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6),eviction=(threads_max=4)" +conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6),eviction=(threads_max=8)" table_config="type=lsm,lsm=(chunk_size=2M),os_cache_dirty_max=16MB" compact=true icount=10000000 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm.wtperf index b872d429046..a0f2a78d013 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/evict-lsm.wtperf @@ -1,5 +1,5 @@ # wtperf options file: evict lsm configuration -conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6),eviction=(threads_max=4)" +conn_config="cache_size=50M,lsm_manager=(worker_thread_max=6),eviction=(threads_max=8)" table_config="type=lsm,lsm=(chunk_size=2M),os_cache_dirty_max=16MB" compact=true icount=10000000 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/log.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/log.wtperf index 6cf50dfb5a5..4379ba22373 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/log.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/log.wtperf @@ -16,7 +16,7 @@ # - Config + "-C "checkpoint=(wait=0)": no checkpoints # - Config + "-C "log=(enabled,prealloc=false,file_max=1M)": no pre-allocation # -conn_config="cache_size=5G,log=(enabled=true),checkpoint=(log_size=500M),eviction=(threads_max=4)" +conn_config="cache_size=5G,log=(enabled=true),checkpoint=(log_size=500M),eviction=(threads_max=8)" table_config="type=file" icount=1000000 report_interval=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/many-table-stress.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/many-table-stress.wtperf new file mode 100644 index 00000000000..6cf1d5d2696 --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/many-table-stress.wtperf @@ -0,0 +1,19 @@ +# Create a set of tables with uneven distribution of data +conn_config="cache_size=1G,eviction=(threads_max=8),file_manager=(close_idle_time=100000),checkpoint=(wait=20,log_size=2GB),statistics=(fast),statistics_log=(wait=5,json),session_max=1000" +table_config="type=file" +table_count=2000 +icount=0 +random_range=1000000000 +pareto=10 +range_partition=true +report_interval=5 + +run_ops=1000000 +populate_threads=0 +icount=0 +threads=((count=60,inserts=1)) + +# Warn if a latency over 1 second is seen +max_latency=1000 +sample_interval=5 +sample_rate=1 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/mongodb-secondary-apply.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/mongodb-secondary-apply.wtperf index f9e41184f95..58bd1a76b97 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/mongodb-secondary-apply.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/mongodb-secondary-apply.wtperf @@ -1,5 +1,5 @@ # Simulate the MongoDB oplog apply threads on a secondary. -conn_config="cache_size=10GB,session_max=1000,eviction=(threads_min=4,threads_max=4),log=(enabled=false),transaction_sync=(enabled=false),checkpoint_sync=true,checkpoint=(wait=60),statistics=(fast),statistics_log=(json,wait=1)" +conn_config="cache_size=10GB,session_max=1000,eviction=(threads_min=4,threads_max=8),log=(enabled=false),transaction_sync=(enabled=false),checkpoint_sync=true,checkpoint=(wait=60),statistics=(fast),statistics_log=(json,wait=1)" table_config="allocation_size=4k,memory_page_max=5MB,prefix_compression=false,split_pct=75,leaf_page_max=32k,internal_page_max=16k,type=file" # Spread the workload out over several tables. table_count=4 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-read-heavy-stress.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-read-heavy-stress.wtperf index d7b27f8fda4..f07e6c80b39 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-read-heavy-stress.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-read-heavy-stress.wtperf @@ -2,7 +2,7 @@ # up by dividing the workload across a lot of threads. This needs to be # tuned to the particular machine so the workload is close to capacity in the # steady state, but not overwhelming. -conn_config="cache_size=20GB,session_max=1000,eviction=(threads_min=4,threads_max=4),log=(enabled=false),transaction_sync=(enabled=false),checkpoint_sync=true,checkpoint=(wait=60),statistics=(fast),statistics_log=(json,wait=1)" +conn_config="cache_size=20GB,session_max=1000,eviction=(threads_min=4,threads_max=8),log=(enabled=false),transaction_sync=(enabled=false),checkpoint_sync=true,checkpoint=(wait=60),statistics=(fast),statistics_log=(json,wait=1)" table_config="allocation_size=4k,memory_page_max=10MB,prefix_compression=false,split_pct=90,leaf_page_max=32k,internal_page_max=16k,type=file" # Divide original icount by database_count. table_count=8 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-stress.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-stress.wtperf index b10b08f6035..bee1f431043 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-stress.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-stress.wtperf @@ -1,7 +1,7 @@ # wtperf options file: multi-database configuration attempting to # trigger slow operations by overloading CPU and disk. # References Jira WT-2131 -conn_config="cache_size=2GB,eviction=(threads_min=2,threads_max=2),log=(enabled=false),direct_io=(data,checkpoint),buffer_alignment=4096,checkpoint_sync=true,checkpoint=(wait=60)" +conn_config="cache_size=2GB,eviction=(threads_min=2,threads_max=8),log=(enabled=false),direct_io=(data,checkpoint),buffer_alignment=4096,checkpoint_sync=true,checkpoint=(wait=60)" table_config="allocation_size=4k,prefix_compression=false,split_pct=75,leaf_page_max=4k,internal_page_max=16k,leaf_item_max=1433,internal_item_max=3100,type=file" # Divide original icount by database_count. database_count=5 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-zipfian-populate.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-zipfian-populate.wtperf index ddd9c055eac..1fdba049779 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-zipfian-populate.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-zipfian-populate.wtperf @@ -1,5 +1,5 @@ # Create a set of tables with uneven distribution of data -conn_config="cache_size=1G,eviction=(threads_max=4),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics=(fast),statistics_log=(wait=5,json),session_max=1000" +conn_config="cache_size=1G,eviction=(threads_max=8),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics=(fast),statistics_log=(wait=5,json),session_max=1000" table_config="type=file" table_count=100 icount=0 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-zipfian-workload.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-zipfian-workload.wtperf index 380350c88c8..dfb3306a7a5 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-zipfian-workload.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/multi-btree-zipfian-workload.wtperf @@ -1,5 +1,5 @@ # Read from a set of tables with uneven distribution of data -conn_config="cache_size=1G,eviction=(threads_max=4),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics=(fast),statistics_log=(wait=5,json),session_max=1000" +conn_config="cache_size=1G,eviction=(threads_max=8),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics=(fast),statistics_log=(wait=5,json),session_max=1000" table_config="type=file" table_count=100 icount=0 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh b/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh index 9968edc468a..fdf3b14e991 100755 --- a/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh +++ b/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh @@ -99,6 +99,11 @@ while test "$run" -le "$runmax"; do if test "$?" -ne "0"; then exit 1 fi + + # Copy the artifacts from the run + backup_dir=${home}_$(basename $wttest)_${run}_$(date +"%s") + rsync -r -m --include="*Stat*" --include="CONFIG.wtperf" --include="*monitor" --include="latency*" --include="test.stat" --exclude="*" $home/ $backup_dir + # Load is always using floating point, so handle separately l=`grep "^Load time:" ./WT_TEST/test.stat` if test "$?" -eq "0"; then diff --git a/src/third_party/wiredtiger/bench/wtperf/stress/btree-split-stress.wtperf b/src/third_party/wiredtiger/bench/wtperf/stress/btree-split-stress.wtperf index deb8c70d12f..eb6ca1cfddc 100644 --- a/src/third_party/wiredtiger/bench/wtperf/stress/btree-split-stress.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/stress/btree-split-stress.wtperf @@ -1,4 +1,4 @@ -conn_config="cache_size=2GB,statistics=[fast,clear],statistics_log=(wait=10),eviction=(threads_max=4,threads_min=4)" +conn_config="cache_size=2GB,statistics=[fast,clear],statistics_log=(wait=10),eviction=(threads_max=8,threads_min=4)" table_config="type=file,leaf_page_max=8k,internal_page_max=8k,memory_page_max=2MB,split_deepen_min_child=250" icount=200000 report_interval=5 @@ -6,5 +6,4 @@ run_time=300 reopen_connection=false populate_threads=2 value_sz=256 -read_range=100 -threads=((count=4,inserts=1,throttle=100000),(count=8,reads=1)) +threads=((count=4,inserts=1,throttle=100000),(count=8,reads=1,read_range=100)) diff --git a/src/third_party/wiredtiger/bench/wtperf/track.c b/src/third_party/wiredtiger/bench/wtperf/track.c index 822bdaa4b4a..86a26120a6a 100644 --- a/src/third_party/wiredtiger/bench/wtperf/track.c +++ b/src/third_party/wiredtiger/bench/wtperf/track.c @@ -288,8 +288,8 @@ latency_print_single(WTPERF *wtperf, TRACK *total, const char *name) uint64_t cumops; char path[1024]; - snprintf(path, sizeof(path), - "%s/latency.%s", wtperf->monitor_dir, name); + testutil_check(__wt_snprintf(path, sizeof(path), + "%s/latency.%s", wtperf->monitor_dir, name)); if ((fp = fopen(path, "w")) == NULL) { lprintf(wtperf, errno, 0, "%s", path); return; diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.c b/src/third_party/wiredtiger/bench/wtperf/wtperf.c index 8c7f0053388..6d79eebe8b2 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.c +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.c @@ -32,9 +32,6 @@ #define DEFAULT_HOME "WT_TEST" #define DEFAULT_MONITOR_DIR "WT_TEST" -static const char * const debug_cconfig = ""; -static const char * const debug_tconfig = ""; - static void *checkpoint_worker(void *); static int drop_all_tables(WTPERF *); static int execute_populate(WTPERF *); @@ -432,19 +429,17 @@ err: wtperf->error = wtperf->stop = true; * search do them. Ensuring the keys we see are always in order. */ static int -do_range_reads(WTPERF *wtperf, WT_CURSOR *cursor) +do_range_reads(WTPERF *wtperf, WT_CURSOR *cursor, int64_t read_range) { - CONFIG_OPTS *opts; - size_t range; uint64_t next_val, prev_val; + int64_t range; char *range_key_buf; char buf[512]; int ret; - opts = wtperf->opts; ret = 0; - if (opts->read_range == 0) + if (read_range == 0) return (0); memset(&buf[0], 0, 512 * sizeof(char)); @@ -454,7 +449,7 @@ do_range_reads(WTPERF *wtperf, WT_CURSOR *cursor) testutil_check(cursor->get_key(cursor, &range_key_buf)); extract_key(range_key_buf, &next_val); - for (range = 0; range < opts->read_range; ++range) { + for (range = 0; range < read_range; ++range) { prev_val = next_val; ret = cursor->next(cursor); /* We are done if we reach the end. */ @@ -475,12 +470,56 @@ do_range_reads(WTPERF *wtperf, WT_CURSOR *cursor) return (0); } +/* pre_load_data -- + * Pull everything into cache before starting the workload phase. + */ +static int +pre_load_data(WTPERF *wtperf) +{ + CONFIG_OPTS *opts; + WT_CONNECTION *conn; + WT_CURSOR *cursor; + WT_SESSION *session; + char *key; + int ret; + size_t i; + + opts = wtperf->opts; + conn = wtperf->conn; + + if ((ret = conn->open_session( + conn, NULL, opts->sess_config, &session)) != 0) { + lprintf(wtperf, ret, 0, "worker: WT_CONNECTION.open_session"); + goto err; + } + for (i = 0; i < opts->table_count; i++) { + if ((ret = session->open_cursor(session, + wtperf->uris[i], NULL, NULL, &cursor)) != 0) { + lprintf(wtperf, ret, 0, + "worker: WT_SESSION.open_cursor: %s", + wtperf->uris[i]); + goto err; + } + while (cursor->next(cursor) == 0) + if ((ret = cursor->get_key(cursor, &key)) != 0) + goto err; + if ((ret = cursor->close(cursor)) != 0) + goto err; + } + if ((ret = session->close(session, NULL)) != 0) + goto err; + if (ret != 0) +err: lprintf(wtperf, ret, 0, "Pre-workload traverse error"); + return (ret); +} + static void * worker(void *arg) { struct timespec start, stop; CONFIG_OPTS *opts; TRACK *trk; + WORKLOAD *workload; WTPERF *wtperf; WTPERF_THREAD *thread; WT_CONNECTION *conn; @@ -495,13 +534,14 @@ worker(void *arg) char buf[512]; thread = (WTPERF_THREAD *)arg; + workload = thread->workload; wtperf = thread->wtperf; opts = wtperf->opts; conn = wtperf->conn; cursors = NULL; - log_table_cursor = NULL; /* -Wconditional-initialized */ + cursor = log_table_cursor = NULL; /* -Wconditional-initialized */ ops = 0; - ops_per_txn = thread->workload->ops_per_txn; + ops_per_txn = workload->ops_per_txn; session = NULL; trk = NULL; @@ -510,9 +550,9 @@ worker(void *arg) lprintf(wtperf, ret, 0, "worker: WT_CONNECTION.open_session"); goto err; } - cursors = dcalloc(opts->table_count, sizeof(WT_CURSOR *)); for (i = 0; i < opts->table_count_idle; i++) { - snprintf(buf, 512, "%s_idle%05d", wtperf->uris[0], (int)i); + testutil_check(__wt_snprintf( + buf, 512, "%s_idle%05d", wtperf->uris[0], (int)i)); if ((ret = session->open_cursor( session, buf, NULL, NULL, &tmp_cursor)) != 0) { lprintf(wtperf, ret, 0, @@ -525,14 +565,34 @@ worker(void *arg) goto err; } } - for (i = 0; i < opts->table_count; i++) { + if (workload->table_index != INT32_MAX) { if ((ret = session->open_cursor(session, - wtperf->uris[i], NULL, NULL, &cursors[i])) != 0) { + wtperf->uris[workload->table_index], + NULL, NULL, &cursor)) != 0) { lprintf(wtperf, ret, 0, "worker: WT_SESSION.open_cursor: %s", - wtperf->uris[i]); + wtperf->uris[workload->table_index]); + goto err; + } + if ((ret = session->open_cursor(session, + wtperf->uris[workload->table_index], + NULL, "next_random=true", &thread->rand_cursor)) != 0) { + lprintf(wtperf, ret, 0, + "worker: WT_SESSION.open_cursor: random %s", + wtperf->uris[workload->table_index]); goto err; } + } else { + cursors = dcalloc(opts->table_count, sizeof(WT_CURSOR *)); + for (i = 0; i < opts->table_count; i++) { + if ((ret = session->open_cursor(session, + wtperf->uris[i], NULL, NULL, &cursors[i])) != 0) { + lprintf(wtperf, ret, 0, + "worker: WT_SESSION.open_cursor: %s", + wtperf->uris[i]); + goto err; + } + } } if (opts->log_like_table && (ret = session->open_cursor(session, wtperf->log_table_uri, NULL, NULL, &log_table_cursor)) != 0) { @@ -543,19 +603,19 @@ worker(void *arg) } /* Setup the timer for throttling. */ - if (thread->workload->throttle != 0) + if (workload->throttle != 0) setup_throttle(thread); /* Setup for truncate */ - if (thread->workload->truncate != 0) + if (workload->truncate != 0) if ((ret = setup_truncate(wtperf, thread, session)) != 0) goto err; key_buf = thread->key_buf; value_buf = thread->value_buf; - op = thread->workload->ops; - op_end = op + sizeof(thread->workload->ops); + op = workload->ops; + op_end = op + sizeof(workload->ops); if ((ops_per_txn != 0 || opts->log_like_table) && (ret = session->begin_transaction(session, NULL)) != 0) { @@ -564,6 +624,8 @@ worker(void *arg) } while (!wtperf->stop) { + if (workload->pause != 0) + (void)sleep((unsigned int)workload->pause); /* * Generate the next key and setup operation specific * statistics tracking objects. @@ -603,10 +665,12 @@ worker(void *arg) generate_key(opts, key_buf, next_val); - /* - * Spread the data out around the multiple databases. - */ - cursor = cursors[map_key_to_table(wtperf->opts, next_val)]; + if (workload->table_index == INT32_MAX) + /* + * Spread the data out around the multiple databases. + */ + cursor = cursors[ + map_key_to_table(wtperf->opts, next_val)]; /* * Skip the first time we do an operation, when trk->ops @@ -642,7 +706,8 @@ worker(void *arg) * for several operations, confirming that the * next key is in the correct order. */ - ret = do_range_reads(wtperf, cursor); + ret = do_range_reads(wtperf, + cursor, workload->read_range); } if (ret == 0 || ret == WT_NOTFOUND) @@ -689,7 +754,7 @@ worker(void *arg) */ strncpy(value_buf, value, opts->value_sz_max - 1); - if (thread->workload->update_delta != 0) + if (workload->update_delta != 0) update_value_delta(thread); if (value_buf[0] == 'a') value_buf[0] = 'b'; @@ -806,7 +871,7 @@ op_err: if (ret == WT_ROLLBACK && ops_per_txn != 0) { /* Schedule the next operation */ if (++op == op_end) - op = thread->workload->ops; + op = workload->ops; /* * Decrement throttle ops and check if we should sleep @@ -843,7 +908,7 @@ run_mix_schedule_op(WORKLOAD *workp, int op, int64_t op_cnt) uint8_t *p, *end; /* Jump around the array to roughly spread out the operations. */ - jump = 100 / op_cnt; + jump = (int)(100 / op_cnt); /* * Find a read operation and replace it with another operation. This @@ -884,17 +949,6 @@ run_mix_schedule(WTPERF *wtperf, WORKLOAD *workp) opts = wtperf->opts; - /* Confirm reads, inserts, truncates and updates cannot all be zero. */ - if (workp->insert == 0 && workp->read == 0 && - workp->truncate == 0 && workp->update == 0) { - lprintf(wtperf, EINVAL, 0, "no operations scheduled"); - return (EINVAL); - } - - /* - * Handle truncate first - it's a special case that can't be used in - * a mixed workload. - */ if (workp->truncate != 0) { if (workp->insert != 0 || workp->read != 0 || workp->update != 0) { @@ -906,6 +960,12 @@ run_mix_schedule(WTPERF *wtperf, WORKLOAD *workp) return (0); } + /* Confirm reads, inserts and updates cannot all be zero. */ + if (workp->insert == 0 && workp->read == 0 && workp->update == 0) { + lprintf(wtperf, EINVAL, 0, "no operations scheduled"); + return (EINVAL); + } + /* * Check for a simple case where the thread is only doing insert or * update operations (because the default operation for a @@ -1238,7 +1298,8 @@ monitor(void *arg) /* Open the logging file. */ len = strlen(wtperf->monitor_dir) + 100; path = dmalloc(len); - snprintf(path, len, "%s/monitor", wtperf->monitor_dir); + testutil_check(__wt_snprintf( + path, len, "%s/monitor", wtperf->monitor_dir)); if ((fp = fopen(path, "w")) == NULL) { lprintf(wtperf, errno, 0, "%s", path); goto err; @@ -1593,6 +1654,9 @@ close_reopen(WTPERF *wtperf) opts = wtperf->opts; + if (opts->in_memory) + return (0); + if (!opts->readonly && !opts->reopen_connection) return (0); /* @@ -1875,19 +1939,19 @@ create_uris(WTPERF *wtperf) /* If there is only one table, just use the base name. */ wtperf->uris[i] = dmalloc(len); if (opts->table_count == 1) - snprintf(wtperf->uris[i], - len, "table:%s", opts->table_name); + testutil_check(__wt_snprintf(wtperf->uris[i], + len, "table:%s", opts->table_name)); else - snprintf(wtperf->uris[i], - len, "table:%s%05d", opts->table_name, i); + testutil_check(__wt_snprintf(wtperf->uris[i], + len, "table:%s%05d", opts->table_name, i)); } /* Create the log-like-table URI. */ len = strlen("table:") + strlen(opts->table_name) + strlen("_log_table") + 1; wtperf->log_table_uri = dmalloc(len); - snprintf( - wtperf->log_table_uri, len, "table:%s_log_table", opts->table_name); + testutil_check(__wt_snprintf(wtperf->log_table_uri, + len, "table:%s_log_table", opts->table_name)); } static int @@ -1909,7 +1973,8 @@ create_tables(WTPERF *wtperf) } for (i = 0; i < opts->table_count_idle; i++) { - snprintf(buf, 512, "%s_idle%05d", wtperf->uris[0], (int)i); + testutil_check(__wt_snprintf( + buf, 512, "%s_idle%05d", wtperf->uris[0], (int)i)); if ((ret = session->create( session, buf, opts->table_config)) != 0) { lprintf(wtperf, ret, 0, @@ -1938,8 +2003,9 @@ create_tables(WTPERF *wtperf) return (ret); } if (opts->index) { - snprintf(buf, 512, "index:%s:val_idx", - wtperf->uris[i] + strlen("table:")); + testutil_check(__wt_snprintf(buf, 512, + "index:%s:val_idx", + wtperf->uris[i] + strlen("table:"))); if ((ret = session->create( session, buf, "columns=(val)")) != 0) { lprintf(wtperf, ret, 0, @@ -2124,15 +2190,15 @@ start_all_runs(WTPERF *wtperf) */ len = strlen(wtperf->home) + 5; next_wtperf->home = dmalloc(len); - snprintf( - next_wtperf->home, len, "%s/D%02d", wtperf->home, (int)i); + testutil_check(__wt_snprintf( + next_wtperf->home, len, "%s/D%02d", wtperf->home, (int)i)); if (opts->create != 0) recreate_dir(next_wtperf->home); len = strlen(wtperf->monitor_dir) + 5; next_wtperf->monitor_dir = dmalloc(len); - snprintf(next_wtperf->monitor_dir, - len, "%s/D%02d", wtperf->monitor_dir, (int)i); + testutil_check(__wt_snprintf(next_wtperf->monitor_dir, + len, "%s/D%02d", wtperf->monitor_dir, (int)i)); if (opts->create != 0 && strcmp(next_wtperf->home, next_wtperf->monitor_dir) != 0) recreate_dir(next_wtperf->monitor_dir); @@ -2244,6 +2310,8 @@ start_run(WTPERF *wtperf) opts->checkpoint_threads, checkpoint_worker) != 0) goto err; } + if (opts->pre_load_data && (ret = pre_load_data(wtperf)) != 0) + goto err; /* Execute the workload. */ if ((ret = execute_workload(wtperf)) != 0) goto err; @@ -2361,11 +2429,11 @@ main(int argc, char *argv[]) { CONFIG_OPTS *opts; WTPERF *wtperf, _wtperf; - size_t req_len, sreq_len; + size_t pos, req_len, sreq_len; bool monitor_set; int ch, ret; const char *cmdflags = "C:h:m:O:o:T:"; - const char *config_opts; + const char *append_comma, *config_opts; char *cc_buf, *path, *sess_cfg, *tc_buf, *user_cconfig, *user_tconfig; /* The first WTPERF structure (from which all others are derived). */ @@ -2479,9 +2547,9 @@ main(int argc, char *argv[]) */ req_len = strlen(",async=(enabled=true,threads=)") + 4; wtperf->async_config = dmalloc(req_len); - snprintf(wtperf->async_config, req_len, + testutil_check(__wt_snprintf(wtperf->async_config, req_len, ",async=(enabled=true,threads=%" PRIu32 ")", - opts->async_threads); + opts->async_threads)); } if ((ret = config_compress(wtperf)) != 0) goto err; @@ -2502,53 +2570,96 @@ main(int argc, char *argv[]) __wt_stream_set_line_buffer(stdout); /* Concatenate non-default configuration strings. */ - if (opts->verbose > 1 || user_cconfig != NULL || - opts->session_count_idle > 0 || wtperf->compress_ext != NULL || - wtperf->async_config != NULL) { - req_len = strlen(debug_cconfig) + 20; - if (user_cconfig != NULL) - req_len += strlen(user_cconfig); - if (wtperf->async_config != NULL) - req_len += strlen(wtperf->async_config); - if (wtperf->compress_ext != NULL) - req_len += strlen(wtperf->compress_ext); + if (user_cconfig != NULL || opts->session_count_idle > 0 || + wtperf->compress_ext != NULL || wtperf->async_config != NULL || + opts->in_memory) { + req_len = 20; + req_len += wtperf->async_config != NULL ? + strlen(wtperf->async_config) : 0; + req_len += wtperf->compress_ext != NULL ? + strlen(wtperf->compress_ext) : 0; if (opts->session_count_idle > 0) { - sreq_len = strlen(",session_max=") + 6; + sreq_len = strlen("session_max=") + 6; req_len += sreq_len; sess_cfg = dmalloc(sreq_len); - snprintf(sess_cfg, sreq_len, - ",session_max=%" PRIu32, + testutil_check(__wt_snprintf(sess_cfg, sreq_len, + "session_max=%" PRIu32, opts->session_count_idle + - wtperf->workers_cnt + opts->populate_threads + 10); + wtperf->workers_cnt + opts->populate_threads + 10)); } + req_len += opts->in_memory ? strlen("in_memory=true") : 0; + req_len += user_cconfig != NULL ? strlen(user_cconfig) : 0; cc_buf = dmalloc(req_len); - snprintf(cc_buf, req_len, "%s,%s,%s,%s,%s", - wtperf->async_config ? wtperf->async_config : "", - wtperf->compress_ext ? wtperf->compress_ext : "", - opts->verbose > 1 ? debug_cconfig : "", - sess_cfg != NULL ? sess_cfg : "", - user_cconfig != NULL ? user_cconfig : ""); - if (strlen(cc_buf) && (ret = + + pos = 0; + append_comma = ""; + if (wtperf->async_config != NULL && + strlen(wtperf->async_config) != 0) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, wtperf->async_config)); + append_comma = ","; + } + if (wtperf->compress_ext != NULL && + strlen(wtperf->compress_ext) != 0) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, wtperf->compress_ext)); + append_comma = ","; + } + if (opts->in_memory) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, "in_memory=true")); + append_comma = ","; + } + if (sess_cfg != NULL && strlen(sess_cfg) != 0) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, sess_cfg)); + append_comma = ","; + } + if (user_cconfig != NULL && strlen(user_cconfig) != 0) { + testutil_check(__wt_snprintf_len_incr( + cc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, user_cconfig)); + } + + if (strlen(cc_buf) != 0 && (ret = config_opt_name_value(wtperf, "conn_config", cc_buf)) != 0) goto err; } - if (opts->verbose > 1 || opts->index || + if (opts->index || user_tconfig != NULL || wtperf->compress_table != NULL) { - req_len = strlen(debug_tconfig) + 20; - if (user_tconfig != NULL) - req_len += strlen(user_tconfig); - if (wtperf->compress_table != NULL) - req_len += strlen(wtperf->compress_table); - if (opts->index) - req_len += strlen(INDEX_COL_NAMES); + req_len = 20; + req_len += wtperf->compress_table != NULL ? + strlen(wtperf->compress_table) : 0; + req_len += opts->index ? strlen(INDEX_COL_NAMES) : 0; + req_len += user_tconfig != NULL ? strlen(user_tconfig) : 0; tc_buf = dmalloc(req_len); - snprintf(tc_buf, req_len, "%s,%s,%s,%s", - opts->index ? INDEX_COL_NAMES : "", - wtperf->compress_table != NULL ? - wtperf->compress_table : "", - opts->verbose > 1 ? debug_tconfig : "", - user_tconfig ? user_tconfig : ""); - if (strlen(tc_buf) && (ret = + + pos = 0; + append_comma = ""; + if (wtperf->compress_table != NULL && + strlen(wtperf->compress_table) != 0) { + testutil_check(__wt_snprintf_len_incr( + tc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, wtperf->compress_table)); + append_comma = ","; + } + if (opts->index) { + testutil_check(__wt_snprintf_len_incr( + tc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, INDEX_COL_NAMES)); + append_comma = ","; + } + if (user_tconfig != NULL && strlen(user_tconfig) != 0) { + testutil_check(__wt_snprintf_len_incr( + tc_buf + pos, req_len - pos, &pos, "%s%s", + append_comma, user_tconfig)); + } + + if (strlen(tc_buf) != 0 && (ret = config_opt_name_value(wtperf, "table_config", tc_buf)) != 0) goto err; } @@ -2556,8 +2667,9 @@ main(int argc, char *argv[]) req_len = strlen(opts->table_config) + strlen(LOG_PARTIAL_CONFIG) + 1; wtperf->partial_config = dmalloc(req_len); - snprintf(wtperf->partial_config, req_len, "%s%s", - opts->table_config, LOG_PARTIAL_CONFIG); + testutil_check(__wt_snprintf( + wtperf->partial_config, req_len, "%s%s", + opts->table_config, LOG_PARTIAL_CONFIG)); } /* * Set the config for reopen. If readonly add in that string. @@ -2570,11 +2682,12 @@ main(int argc, char *argv[]) req_len = strlen(opts->conn_config) + 1; wtperf->reopen_config = dmalloc(req_len); if (opts->readonly) - snprintf(wtperf->reopen_config, req_len, "%s%s", - opts->conn_config, READONLY_CONFIG); + testutil_check(__wt_snprintf( + wtperf->reopen_config, req_len, "%s%s", + opts->conn_config, READONLY_CONFIG)); else - snprintf(wtperf->reopen_config, - req_len, "%s", opts->conn_config); + testutil_check(__wt_snprintf( + wtperf->reopen_config, req_len, "%s", opts->conn_config)); /* Sanity-check the configuration. */ if ((ret = config_sanity(wtperf)) != 0) @@ -2587,7 +2700,8 @@ main(int argc, char *argv[]) /* Write a copy of the config. */ req_len = strlen(wtperf->home) + strlen("/CONFIG.wtperf") + 1; path = dmalloc(req_len); - snprintf(path, req_len, "%s/CONFIG.wtperf", wtperf->home); + testutil_check(__wt_snprintf( + path, req_len, "%s/CONFIG.wtperf", wtperf->home)); config_opt_log(opts, path); free(path); @@ -2712,7 +2826,8 @@ recreate_dir(const char *name) len = strlen(name) * 2 + 100; buf = dmalloc(len); - (void)snprintf(buf, len, "rm -rf %s && mkdir %s", name, name); + testutil_check(__wt_snprintf( + buf, len, "rm -rf %s && mkdir %s", name, name)); testutil_checkfmt(system(buf), "system: %s", buf); free(buf); } @@ -2779,13 +2894,42 @@ static uint64_t wtperf_rand(WTPERF_THREAD *thread) { CONFIG_OPTS *opts; + WT_CURSOR *rnd_cursor; WTPERF *wtperf; double S1, S2, U; uint64_t rval; + int ret; + char *key_buf; wtperf = thread->wtperf; opts = wtperf->opts; + /* + * If we have a random cursor set up then use it. + */ + if ((rnd_cursor = thread->rand_cursor) != NULL) { + if ((ret = rnd_cursor->next(rnd_cursor)) != 0) { + lprintf(wtperf, ret, 0, "worker: rand next failed"); + /* 0 is outside the expected range. */ + return (0); + } + if ((ret = rnd_cursor->get_key(rnd_cursor, &key_buf)) != 0) { + lprintf(wtperf, ret, 0, + "worker: rand next key retrieval"); + return (0); + } + /* + * Resetting the cursor is not fatal. We still return the + * value we retrieved above. We do it so that we don't + * leave a cursor positioned. + */ + if ((ret = rnd_cursor->reset(rnd_cursor)) != 0) + lprintf(wtperf, ret, 0, + "worker: rand cursor reset failed"); + extract_key(key_buf, &rval); + return (rval); + } + /* * Use WiredTiger's random number routine: it's lock-free and fairly * good. diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.h b/src/third_party/wiredtiger/bench/wtperf/wtperf.h index 81d74e134f6..3efb8ab700e 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.h +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.h @@ -66,6 +66,9 @@ typedef struct { uint64_t throttle; /* Maximum operations/second */ /* Number of operations per transaction. Zero for autocommit */ int64_t ops_per_txn; + int64_t pause; /* Time between scans */ + int64_t read_range; /* Range of reads */ + int32_t table_index; /* Table to focus ops on */ int64_t truncate; /* Truncate ratio */ uint64_t truncate_pct; /* Truncate Percent */ uint64_t truncate_count; /* Truncate Count */ @@ -225,6 +228,7 @@ typedef struct { struct __wtperf_thread { /* Per-thread structure */ WTPERF *wtperf; /* Enclosing configuration */ + WT_CURSOR *rand_cursor; /* Random key cursor */ WT_RAND_STATE rnd; /* Random number generation state */ diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i b/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i index 680eb53a90e..90f70457407 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i @@ -110,6 +110,8 @@ DEF_OPT_AS_UINT32(database_count, 1, DEF_OPT_AS_BOOL(drop_tables, 0, "Whether to drop all tables at the end of the run, and report time taken" " to do the drop.") +DEF_OPT_AS_BOOL(in_memory, 0, + "Whether to create the database in-memory.") DEF_OPT_AS_UINT32(icount, 5000, "number of records to initially populate. If multiple tables are " "configured the count is spread evenly across all tables.") @@ -145,12 +147,13 @@ DEF_OPT_AS_UINT32(populate_ops_per_txn, 0, "phase, zero for auto-commit") DEF_OPT_AS_UINT32(populate_threads, 1, "number of populate threads, 1 for bulk load") +DEF_OPT_AS_BOOL(pre_load_data, 0, + "Scan all data prior to starting the workload phase to warm the cache") DEF_OPT_AS_UINT32(random_range, 0, "if non zero choose a value from within this range as the key for " "insert operations") DEF_OPT_AS_BOOL(random_value, 0, "generate random content for the value") DEF_OPT_AS_BOOL(range_partition, 0, "partition data by range (vs hash)") -DEF_OPT_AS_UINT32(read_range, 0, "scan a range of keys after each search") DEF_OPT_AS_BOOL(readonly, 0, "reopen the connection between populate and workload phases in readonly " "mode. Requires reopen_connection turned on (default). Requires that " @@ -192,9 +195,10 @@ DEF_OPT_AS_STRING(threads, "", "workload configuration: each 'count' " "'threads=((count=2,reads=1)(count=8,reads=1,inserts=2,updates=1))' " "which would create 2 threads doing nothing but reads and 8 threads " "each doing 50% inserts and 25% reads and updates. Allowed configuration " - "values are 'count', 'throttle', 'update_delta', 'reads', 'inserts', " - "'updates', 'truncate', 'truncate_pct' and 'truncate_count'. There are " - "also behavior modifiers, supported modifiers are 'ops_per_txn'") + "values are 'count', 'throttle', 'update_delta', 'reads', 'read_range', " + "'inserts', 'updates', 'truncate', 'truncate_pct' and 'truncate_count'. " + "There are also behavior modifiers, supported modifiers are " + "'ops_per_txn'") DEF_OPT_AS_CONFIG_STRING(transaction_config, "", "WT_SESSION.begin_transaction configuration string, applied during the " "populate phase when populate_ops_per_txn is nonzero") diff --git a/src/third_party/wiredtiger/build_posix/Make.base b/src/third_party/wiredtiger/build_posix/Make.base index 5b945aca5e0..e5228fac885 100644 --- a/src/third_party/wiredtiger/build_posix/Make.base +++ b/src/third_party/wiredtiger/build_posix/Make.base @@ -17,6 +17,7 @@ endif bin_PROGRAMS = wt wt_SOURCES =\ + src/utilities/util_alter.c \ src/utilities/util_backup.c \ src/utilities/util_cpyright.c \ src/utilities/util_compact.c \ @@ -35,6 +36,7 @@ wt_SOURCES =\ src/utilities/util_rename.c \ src/utilities/util_salvage.c \ src/utilities/util_stat.c \ + src/utilities/util_truncate.c \ src/utilities/util_upgrade.c \ src/utilities/util_verbose.c \ src/utilities/util_verify.c \ diff --git a/src/third_party/wiredtiger/build_posix/Make.subdirs b/src/third_party/wiredtiger/build_posix/Make.subdirs index 55941837249..4ecec37ca6c 100644 --- a/src/third_party/wiredtiger/build_posix/Make.subdirs +++ b/src/third_party/wiredtiger/build_posix/Make.subdirs @@ -17,6 +17,7 @@ ext/encryptors/nop ext/encryptors/rotn ext/extractors/csv ext/test/kvs_bdb HAVE_BERKELEY_DB +ext/test/fail_fs . api/leveldb LEVELDB examples/c @@ -40,6 +41,7 @@ test/packing test/readonly test/recovery test/salvage +test/syscall test/thread # Benchmark programs. diff --git a/src/third_party/wiredtiger/build_posix/aclocal/ax_check_compile_flag.m4 b/src/third_party/wiredtiger/build_posix/aclocal/ax_check_compile_flag.m4 new file mode 100644 index 00000000000..ca3639715e7 --- /dev/null +++ b/src/third_party/wiredtiger/build_posix/aclocal/ax_check_compile_flag.m4 @@ -0,0 +1,74 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_COMPILE_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS], [INPUT]) +# +# DESCRIPTION +# +# Check whether the given FLAG works with the current language's compiler +# or gives an error. (Warnings, however, are ignored) +# +# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on +# success/failure. +# +# If EXTRA-FLAGS is defined, it is added to the current language's default +# flags (e.g. CFLAGS) when the check is done. The check is thus made with +# the flags: "CFLAGS EXTRA-FLAGS FLAG". This can for example be used to +# force the compiler to issue an error when a bad flag is given. +# +# INPUT gives an alternative input source to AC_COMPILE_IFELSE. +# +# NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this +# macro in sync with AX_CHECK_{PREPROC,LINK}_FLAG. +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim +# Copyright (c) 2011 Maarten Bosmans +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 4 + +AC_DEFUN([AX_CHECK_COMPILE_FLAG], +[AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF +AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]flags_$4_$1])dnl +AC_CACHE_CHECK([whether _AC_LANG compiler accepts $1], CACHEVAR, [ + ax_check_save_flags=$[]_AC_LANG_PREFIX[]FLAGS + _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $4 $1" + AC_COMPILE_IFELSE([m4_default([$5],[AC_LANG_PROGRAM()])], + [AS_VAR_SET(CACHEVAR,[yes])], + [AS_VAR_SET(CACHEVAR,[no])]) + _AC_LANG_PREFIX[]FLAGS=$ax_check_save_flags]) +AS_VAR_IF(CACHEVAR,yes, + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_COMPILE_FLAGS diff --git a/src/third_party/wiredtiger/build_posix/aclocal/options.m4 b/src/third_party/wiredtiger/build_posix/aclocal/options.m4 index 7043430a6d6..bc4b31dfee3 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/options.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/options.m4 @@ -57,7 +57,7 @@ AH_TEMPLATE( HAVE_CRC32_HARDWARE, [Define to 1 to configure CRC32 hardware support.]) AC_MSG_CHECKING(if --enable-crc32-hardware option specified) AC_ARG_ENABLE(crc32-hardware, - AC_HELP_STRING([--enable-crc32-hardware], + AS_HELP_STRING([--enable-crc32-hardware], [Enable CRC32 hardware support.]), r=$enableval, r=yes) case "$r" in no) wt_cv_enable_crc32_hardware=no;; diff --git a/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 b/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 index c107dd017d7..659867fa69e 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/strict.m4 @@ -31,6 +31,7 @@ AC_DEFUN([AM_GCC_WARNINGS], [ w="$w -Wstrict-prototypes" w="$w -Wswitch-enum" w="$w -Wundef" + w="$w -Wuninitialized" w="$w -Wunreachable-code" w="$w -Wunsafe-loop-optimizations" w="$w -Wunused" @@ -66,6 +67,10 @@ AC_DEFUN([AM_CLANG_WARNINGS], [ # w="$w -Wno-error=cast-qual" w="$w -Wno-cast-qual" + # On Centos 7.3.1611, system header files aren't compatible with + # -Wdisabled-macro-expansion. + w="$w -Wno-disabled-macro-expansion" + case "$1" in *Apple*clang*version*4.1*) # Apple clang has its own numbering system, and older OS X diff --git a/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 b/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 index 997f571782c..c677ce41192 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/version-set.m4 @@ -1,14 +1,14 @@ dnl build by dist/s_version VERSION_MAJOR=2 -VERSION_MINOR=8 -VERSION_PATCH=1 -VERSION_STRING='"WiredTiger 2.8.1: (March 24, 2016)"' +VERSION_MINOR=9 +VERSION_PATCH=2 +VERSION_STRING='"WiredTiger 2.9.2: (December 23, 2016)"' AC_SUBST(VERSION_MAJOR) AC_SUBST(VERSION_MINOR) AC_SUBST(VERSION_PATCH) AC_SUBST(VERSION_STRING) -VERSION_NOPATCH=2.8 +VERSION_NOPATCH=2.9 AC_SUBST(VERSION_NOPATCH) diff --git a/src/third_party/wiredtiger/build_posix/aclocal/version.m4 b/src/third_party/wiredtiger/build_posix/aclocal/version.m4 index 0e199cb9546..29782a22f82 100644 --- a/src/third_party/wiredtiger/build_posix/aclocal/version.m4 +++ b/src/third_party/wiredtiger/build_posix/aclocal/version.m4 @@ -1,2 +1,2 @@ dnl WiredTiger product version for AC_INIT. Maintained by dist/s_version -2.8.1 +2.9.2 diff --git a/src/third_party/wiredtiger/build_posix/configure.ac.in b/src/third_party/wiredtiger/build_posix/configure.ac.in index ad00b19a3bb..0fef587b4b8 100644 --- a/src/third_party/wiredtiger/build_posix/configure.ac.in +++ b/src/third_party/wiredtiger/build_posix/configure.ac.in @@ -22,33 +22,12 @@ AC_PROG_CC(cc gcc) AC_PROG_CXX(c++ g++) AM_PROG_AS(as gas) -# Configure options. -AM_OPTIONS - define([AC_LIBTOOL_LANG_CXX_CONFIG], [:])dnl define([AC_LIBTOOL_LANG_F77_CONFIG], [:])dnl + LT_PREREQ(2.2.6) LT_INIT([pic-only]) AC_SUBST([LIBTOOL_DEPS]) - -# If enable-strict is configured, turn on as much error checking as we can for -# this compiler. Intended for developers, and only works for gcc/clang, but it -# fills a need. -if test "$wt_cv_enable_strict" = "yes"; then - wt_cv_cc_version="`$CC --version | sed -eq`" - case "$wt_cv_cc_version" in - *clang*) - AM_CLANG_WARNINGS($wt_cv_cc_version);; - *cc*|*CC*) # cc, CC, gcc, GCC - AM_GCC_WARNINGS($wt_cv_cc_version);; - *) - AC_MSG_ERROR( - [--enable-strict does not support "$wt_cv_cc_version".]);; - esac - - AM_CFLAGS="$AM_CFLAGS $wt_cv_strict_warnings" -fi - AM_CONDITIONAL([POSIX_HOST], [true]) AM_CONDITIONAL([WINDOWS_HOST], [false]) @@ -69,6 +48,7 @@ AS_CASE([$host_cpu], [aarch64*], [wt_cv_arm64="yes"], [wt_cv_arm64="no"]) AM_CONDITIONAL([ARM64_HOST], [test "$wt_cv_arm64" = "yes"]) +AS_CASE([$host_os], [*solaris*], [wt_cv_solaris="yes"], [wt_cv_solaris="no"]) # This is a workaround as part of WT-2459. Currently, clang (v3.7) does not # support compiling the ASM code we have to perform the CRC checks on PowerPC. @@ -82,9 +62,17 @@ if test "$wt_cv_powerpc" = "yes" -a "$CC" != "$CCAS"; then fi AC_SUBST(AM_LIBTOOLFLAGS) +# WiredTiger uses anonymous unions to pad structures. It's part of C11, but +# some compilers require -std=c11 to support them. Turn on that flag for any +# compiler that supports it, except for Solaris, where gcc -std=c11 makes +# some none-C11 prototypes unavailable. +if test "$wt_cv_solaris" = "no"; then + AX_CHECK_COMPILE_FLAG([-std=c11], [AM_CFLAGS="$AM_CFLAGS -std=c11"]) +fi + if test "$GCC" = "yes"; then # The Solaris gcc compiler gets the additional -pthreads flag. - if test "`uname -s`" = "SunOS"; then + if test "$wt_cv_solaris" = "yes"; then AM_CFLAGS="$AM_CFLAGS -pthreads" fi @@ -95,11 +83,35 @@ if test "$GCC" = "yes"; then fi else # The Solaris native compiler gets the additional -mt flag. - if test "`uname -s`" = "SunOS"; then + if test "$wt_cv_solaris" = "yes"; then AM_CFLAGS="$AM_CFLAGS -mt" fi fi +# Linux requires _GNU_SOURCE to be defined +AS_CASE([$host_os], [linux*], [AM_CFLAGS="$AM_CFLAGS -D_GNU_SOURCE"]) + +# Configure options. +AM_OPTIONS + +# If enable-strict is configured, turn on as much error checking as we can for +# this compiler. Intended for developers, and only works for gcc/clang, but it +# fills a need. +if test "$wt_cv_enable_strict" = "yes"; then + wt_cv_cc_version="`$CC --version | sed -eq`" + case "$wt_cv_cc_version" in + *clang*) + AM_CLANG_WARNINGS($wt_cv_cc_version);; + *cc*|*CC*) # cc, CC, gcc, GCC + AM_GCC_WARNINGS($wt_cv_cc_version);; + *) + AC_MSG_ERROR( + [--enable-strict does not support "$wt_cv_cc_version".]);; + esac + + AM_CFLAGS="$AM_CFLAGS $wt_cv_strict_warnings" +fi + # Java and Python APIs if test "$wt_cv_enable_java" = "yes" -o "$wt_cv_enable_python" = "yes"; then # Only a warning, we need to build release packages without SWIG. @@ -159,11 +171,6 @@ if test "$ac_cv_sizeof_void_p" != "8" ; then fi AC_MSG_RESULT(yes) -# Linux requires _GNU_SOURCE to be defined -case "$host_os" in -linux*) AM_CFLAGS="$AM_CFLAGS -D_GNU_SOURCE" ;; -esac - # Linux requires buffers aligned to 4KB boundaries for O_DIRECT to work. BUFFER_ALIGNMENT=0 if test "$ax_cv_func_posix_memalign_works" = "yes" ; then diff --git a/src/third_party/wiredtiger/dist/api_config.py b/src/third_party/wiredtiger/dist/api_config.py index 1069a7037ce..d83a632321e 100644 --- a/src/third_party/wiredtiger/dist/api_config.py +++ b/src/third_party/wiredtiger/dist/api_config.py @@ -198,7 +198,8 @@ def get_default(c): return 'false' elif c.default == 'true': return 'true' - elif t == 'string' and c.default == 'none': + elif t == 'string' and c.default == 'none' and \ + not c.flags.get('choices', []): return '' elif t == 'category': return '(%s)' % (','.join('%s=%s' % (subc.name, get_default(subc)) diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 22d06c380ae..22600dd5e29 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -118,8 +118,23 @@ lsm_config = [ ]), ] +file_runtime_config = [ + Config('access_pattern_hint', 'none', r''' + It is recommended that workloads that consist primarily of + updates and/or point queries specify \c random. Workloads that + do many cursor scans through large ranges of data specify + \c sequential and other workloads specify \c none. The + option leads to an advisory call to an appropriate operating + system API where available''', + choices=['none', 'random', 'sequential']), + Config('cache_resident', 'false', r''' + do not ever evict the object's pages from cache. Not compatible with + LSM tables; see @ref tuning_cache_resident for more information''', + type='boolean'), +] + # Per-file configuration -file_config = format_meta + [ +file_config = format_meta + file_runtime_config + [ Config('block_allocation', 'best', r''' configure block allocation. Permitted values are \c "first" or \c "best"; the \c "first" configuration uses a first-available @@ -138,10 +153,6 @@ file_config = format_meta + [ WT_CONNECTION::add_compressor. If WiredTiger has builtin support for \c "lz4", \c "snappy", \c "zlib" or \c "zstd" compression, these names are also available. See @ref compression for more information'''), - Config('cache_resident', 'false', r''' - do not ever evict the object's pages from cache. Not compatible with - LSM tables; see @ref tuning_cache_resident for more information''', - type='boolean'), Config('checksum', 'uncompressed', r''' configure block checksums; permitted values are on (checksum all blocks), off (checksum no blocks) and @@ -190,9 +201,8 @@ file_config = format_meta + [ WiredTiger to consume memory over the configured cache limit''', type='boolean'), Config('internal_key_truncate', 'true', r''' - configure internal key truncation, discarding unnecessary - trailing bytes on internal keys (ignored for custom - collators)''', + configure internal key truncation, discarding unnecessary trailing + bytes on internal keys (ignored for custom collators)''', type='boolean'), Config('internal_page_max', '4KB', r''' the maximum page size for internal nodes, in bytes; the size @@ -285,12 +295,12 @@ file_config = format_meta + [ Config('split_deepen_per_child', '0', r''' entries allocated per child when deepening the tree''', type='int', undoc=True), - Config('split_pct', '75', r''' + Config('split_pct', '90', r''' the Btree page split size as a percentage of the maximum Btree page size, that is, when a Btree page is split, it will be split into smaller pages, where each page is the specified percentage of the maximum Btree page size''', - min='25', max='100'), + min='50', max='100'), ] # File metadata, including both configurable and non-configurable (internal) @@ -396,7 +406,7 @@ connection_runtime_config = [ Config('eviction', '', r''' eviction configuration options''', type='category', subconfig=[ - Config('threads_max', '1', r''' + Config('threads_max', '8', r''' maximum number of threads WiredTiger will start to help evict pages from cache. The number of threads started will vary depending on the current eviction load. Each eviction worker @@ -514,6 +524,7 @@ connection_runtime_config = [ 'checkpoint', 'compact', 'evict', + 'evict_stuck', 'evictserver', 'fileops', 'handleops', @@ -527,6 +538,7 @@ connection_runtime_config = [ 'rebalance', 'reconcile', 'recovery', + 'recovery_progress', 'salvage', 'shared_cache', 'split', @@ -637,6 +649,12 @@ wiredtiger_open_statistics_log_configuration = [ ] session_config = [ + Config('ignore_cache_size', 'false', r''' + when set, operations performed by this session ignore the cache size + and are not blocked when the cache is full. Note that use of this + option for operations that create cache pressure can starve ordinary + sessions that obey the cache size.''', + type='boolean'), Config('isolation', 'read-committed', r''' the default isolation level for operations in this session''', choices=['read-uncommitted', 'read-committed', 'snapshot']), @@ -652,6 +670,11 @@ wiredtiger_open_common =\ should be used (4KB on Linux systems when direct I/O is configured, zero elsewhere)''', min='-1', max='1MB'), + Config('builtin_extension_config', '', r''' + A structure where the keys are the names of builtin extensions and the + values are passed to WT_CONNECTION::load_extension as the \c config + parameter (for example, + builtin_extension_config={zlib={compression_level=3}})'''), Config('checkpoint_sync', 'true', r''' flush files to stable storage when closing or writing checkpoints''', @@ -695,7 +718,7 @@ wiredtiger_open_common =\ ]), Config('extensions', '', r''' list of shared library extensions to load (using dlopen). - Any values specified to an library extension are passed to + Any values specified to a library extension are passed to WT_CONNECTION::load_extension as the \c config parameter (for example, extensions=(/path/ext.so={entry=my_entry}))''', @@ -709,7 +732,7 @@ wiredtiger_open_common =\ Config('hazard_max', '1000', r''' maximum number of simultaneous hazard pointers per session handle''', - min='15'), + min=15, undoc=True), Config('mmap', 'true', r''' Use memory mapping to access files when possible''', type='boolean'), @@ -816,6 +839,8 @@ methods = { 'WT_CURSOR.reconfigure' : Method(cursor_runtime_config), +'WT_SESSION.alter' : Method(file_runtime_config), + 'WT_SESSION.close' : Method([]), 'WT_SESSION.compact' : Method([ @@ -837,9 +862,10 @@ methods = { 'WT_SESSION.drop' : Method([ Config('checkpoint_wait', 'true', r''' - wait for the checkpoint lock, if \c checkpoint_wait=false, perform - the drop operation without taking a lock, returning EBUSY if the - operation conflicts with a running checkpoint''', + wait for concurrent checkpoints to complete before attempting the drop + operation. If \c checkpoint_wait=false, attempt the drop operation + without waiting, returning EBUSY if the operation conflicts with a + running checkpoint''', type='boolean', undoc=True), Config('force', 'false', r''' return success if the object does not exist''', @@ -1109,6 +1135,10 @@ methods = { Config('to', '', r''' drop all snapshots up to and including the specified name'''), ]), + Config('include_updates', 'false', r''' + make updates from the current transaction visible to users of the + named snapshot. Transactions started with such a named snapshot are + restricted to being read-only''', type='boolean'), Config('name', '', r'''specify a name for the snapshot'''), ]), diff --git a/src/third_party/wiredtiger/dist/api_err.py b/src/third_party/wiredtiger/dist/api_err.py index 82f961a4ac9..bd379ac8d70 100644 --- a/src/third_party/wiredtiger/dist/api_err.py +++ b/src/third_party/wiredtiger/dist/api_err.py @@ -82,7 +82,7 @@ for line in open('../src/include/wiredtiger.in', 'r'): ''.join('\n * ' + l for l in textwrap.wrap( textwrap.dedent(err.long_desc).strip(), 77)) + '\n' if err.long_desc else '')) - tfile.write('#define\t%s\t%d\n' % (err.name, err.value)) + tfile.write('#define\t%s\t(%d)\n' % (err.name, err.value)) if 'undoc' in err.flags: tfile.write('/*! @endcond */\n') tfile.write('/*\n') diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist index fe9a17b7799..5a3348b940a 100644 --- a/src/third_party/wiredtiger/dist/filelist +++ b/src/third_party/wiredtiger/dist/filelist @@ -30,6 +30,7 @@ src/btree/bt_io.c src/btree/bt_misc.c src/btree/bt_ovfl.c src/btree/bt_page.c +src/btree/bt_random.c src/btree/bt_read.c src/btree/bt_rebalance.c src/btree/bt_ret.c @@ -132,6 +133,7 @@ src/os_posix/os_path.c POSIX_HOST src/os_posix/os_priv.c POSIX_HOST src/os_posix/os_setvbuf.c POSIX_HOST src/os_posix/os_sleep.c POSIX_HOST +src/os_posix/os_snprintf.c POSIX_HOST src/os_posix/os_thread.c POSIX_HOST src/os_posix/os_time.c POSIX_HOST src/os_posix/os_yield.c POSIX_HOST @@ -151,7 +153,6 @@ src/os_win/os_snprintf.c WINDOWS_HOST src/os_win/os_thread.c WINDOWS_HOST src/os_win/os_time.c WINDOWS_HOST src/os_win/os_utf8.c WINDOWS_HOST -src/os_win/os_vsnprintf.c WINDOWS_HOST src/os_win/os_winerr.c WINDOWS_HOST src/os_win/os_yield.c WINDOWS_HOST src/packing/pack_api.c @@ -159,6 +160,7 @@ src/packing/pack_impl.c src/packing/pack_stream.c src/reconcile/rec_track.c src/reconcile/rec_write.c +src/schema/schema_alter.c src/schema/schema_create.c src/schema/schema_drop.c src/schema/schema_list.c diff --git a/src/third_party/wiredtiger/dist/flags.py b/src/third_party/wiredtiger/dist/flags.py index e200f95fba6..64b5d789e72 100644 --- a/src/third_party/wiredtiger/dist/flags.py +++ b/src/third_party/wiredtiger/dist/flags.py @@ -64,6 +64,7 @@ flags = { 'VERB_COMPACT', 'VERB_EVICT', 'VERB_EVICTSERVER', + 'VERB_EVICT_STUCK', 'VERB_FILEOPS', 'VERB_HANDLEOPS', 'VERB_LOG', @@ -76,6 +77,7 @@ flags = { 'VERB_REBALANCE', 'VERB_RECONCILE', 'VERB_RECOVERY', + 'VERB_RECOVERY_PROGRESS', 'VERB_SALVAGE', 'VERB_SHARED_CACHE', 'VERB_SPLIT', @@ -94,19 +96,19 @@ flags = { 'CONN_CACHE_POOL', 'CONN_CKPT_SYNC', 'CONN_CLOSING', + 'CONN_CLOSING_NO_MORE_OPENS', 'CONN_EVICTION_RUN', 'CONN_IN_MEMORY', 'CONN_LAS_OPEN', 'CONN_LEAK_MEMORY', - 'CONN_LOG_SERVER_RUN', 'CONN_LSM_MERGE', 'CONN_PANIC', 'CONN_READONLY', 'CONN_RECOVERING', 'CONN_SERVER_ASYNC', 'CONN_SERVER_CHECKPOINT', + 'CONN_SERVER_LOG', 'CONN_SERVER_LSM', - 'CONN_SERVER_RUN', 'CONN_SERVER_STATISTICS', 'CONN_SERVER_SWEEP', 'CONN_WAS_BACKUP', @@ -114,14 +116,15 @@ flags = { 'session' : [ 'SESSION_CAN_WAIT', 'SESSION_INTERNAL', - 'SESSION_LOCK_NO_WAIT', 'SESSION_LOCKED_CHECKPOINT', - 'SESSION_LOCKED_HANDLE_LIST', + 'SESSION_LOCKED_HANDLE_LIST_READ', + 'SESSION_LOCKED_HANDLE_LIST_WRITE', 'SESSION_LOCKED_METADATA', 'SESSION_LOCKED_PASS', 'SESSION_LOCKED_SCHEMA', 'SESSION_LOCKED_SLOT', - 'SESSION_LOCKED_TABLE', + 'SESSION_LOCKED_TABLE_READ', + 'SESSION_LOCKED_TABLE_WRITE', 'SESSION_LOCKED_TURTLE', 'SESSION_LOGGING_INMEM', 'SESSION_LOOKASIDE_CURSOR', diff --git a/src/third_party/wiredtiger/dist/package/wiredtiger.spec b/src/third_party/wiredtiger/dist/package/wiredtiger.spec index 365c330d86a..aacdf327c98 100644 --- a/src/third_party/wiredtiger/dist/package/wiredtiger.spec +++ b/src/third_party/wiredtiger/dist/package/wiredtiger.spec @@ -1,5 +1,5 @@ Name: wiredtiger -Version: 2.8.1 +Version: 2.9.2 Release: 1%{?dist} Summary: WiredTiger data storage engine diff --git a/src/third_party/wiredtiger/dist/s_all b/src/third_party/wiredtiger/dist/s_all index 31abab28910..be33657e640 100755 --- a/src/third_party/wiredtiger/dist/s_all +++ b/src/third_party/wiredtiger/dist/s_all @@ -57,7 +57,7 @@ errchk() # Some tests shouldn't return an error, we exclude them here. case "$1" in *s_export|*s_tags) - break;; + ;; *) errfound=1;; esac @@ -84,22 +84,23 @@ run "sh ./s_copyright" run "sh ./s_style" COMMANDS=" -2>&1 ./s_tags > ${t_pfx}tags 2>&1 ./s_define > ${t_pfx}s_define -2>&1 ./s_typedef -c > ${t_pfx}s_typedef_c -2>&1 ./s_funcs > ${t_pfx}s_funcs +2>&1 ./s_docs > ${t_pfx}s_docs 2>&1 ./s_export > ${t_pfx}s_export +2>&1 ./s_funcs > ${t_pfx}s_funcs 2>&1 ./s_getopt > ${t_pfx}s_getopt 2>&1 ./s_label > ${t_pfx}s_label 2>&1 ./s_lang > ${t_pfx}s_lang 2>&1 ./s_longlines > ${t_pfx}s_longlines +2>&1 ./s_python > ${t_pfx}s_python 2>&1 ./s_stat > ${t_pfx}_stat 2>&1 ./s_string > ${t_pfx}s_string -2>&1 python style.py > ${t_pfx}py_style -2>&1 ./s_python > ${t_pfx}s_python +2>&1 ./s_tags > ${t_pfx}tags +2>&1 ./s_typedef -c > ${t_pfx}s_typedef_c +2>&1 ./s_void > ${t_pfx}s_void 2>&1 ./s_whitespace > ${t_pfx}s_whitespace 2>&1 ./s_win > ${t_pfx}s_win -2>&1 ./s_docs > ${t_pfx}s_docs" +2>&1 python style.py > ${t_pfx}py_style" # Parallelize if possible. xp="" @@ -110,14 +111,13 @@ fi echo "$COMMANDS" | xargs $xp -I{} /bin/sh -c {} for f in `find . -name ${t_pfx}\*`; do - if ! `test -s $f`; then - continue + if `test -s $f`; then + LOCAL_NAME=`basename $f` + # Find original command and trim redirect garbage + FAILED_CMD=`echo "$COMMANDS" | grep $LOCAL_NAME | \ + sed -e 's/ >.*//' -e 's/.* //'` + errchk "$FAILED_CMD" $f fi - LOCAL_NAME=`basename $f` - # Find original command and trim redirect garbage - FAILED_CMD=`echo "$COMMANDS" | grep $LOCAL_NAME | \ - sed -e 's/ >.*//' -e 's/.* //'` - errchk "$FAILED_CMD" $f done echo 'dist/s_all run finished' diff --git a/src/third_party/wiredtiger/dist/s_define.list b/src/third_party/wiredtiger/dist/s_define.list index 7b11d665de5..8911d888077 100644 --- a/src/third_party/wiredtiger/dist/s_define.list +++ b/src/third_party/wiredtiger/dist/s_define.list @@ -1,5 +1,4 @@ # List of WiredTiger #defines that are "unused", but it's OK. -ALIGN_CHECK API_CALL API_CALL_NOCONF API_SESSION_INIT @@ -8,17 +7,18 @@ JOINABLE_CURSOR_CALL_CHECK LF_MASK LLONG_MAX LLONG_MIN -SIZE_CHECK TXN_API_CALL TXN_API_CALL_NOCONF TXN_API_END WIN32_LEAN_AND_MEAN +WT_ALIGN_CHECK WT_ATOMIC_CAS WT_ATOMIC_FUNC WT_BLOCK_DESC_SIZE WT_BLOCK_HEADER_SIZE WT_CACHE_LINE_ALIGNMENT -WT_COMPILER_TYPE_ALIGN +WT_CACHE_LINE_PAD_BEGIN +WT_CACHE_LINE_PAD_END WT_CONN_CHECK_PANIC WT_DEADLOCK WT_DEBUG_BYTE @@ -35,10 +35,14 @@ WT_LOG_SLOT_MAXBITS WT_LOG_SLOT_UNBUFFERED_ISSET WT_PACKED_STRUCT_BEGIN WT_PACKED_STRUCT_END +WT_PADDING_CHECK WT_READ_BARRIER WT_REF_SIZE WT_SESSION_LOCKED_CHECKPOINT +WT_SESSION_LOCKED_TABLE_READ +WT_SESSION_LOCKED_TABLE_WRITE WT_SESSION_LOCKED_TURTLE +WT_SIZE_CHECK WT_STATS_FIELD_TO_OFFSET WT_STATS_SLOT_ID WT_STAT_CONN_DECRV @@ -54,7 +58,7 @@ WT_STAT_INCRV_BASE WT_STAT_WRITE WT_TIMEDIFF_US WT_TRET_ERROR_OK -WT_WITH_LOCK +WT_WITH_LOCK_NOWAIT WT_WITH_LOCK_WAIT __F __WIREDTIGER_EXT_H_ diff --git a/src/third_party/wiredtiger/dist/s_docs b/src/third_party/wiredtiger/dist/s_docs index f4332257193..6ebffb947ec 100755 --- a/src/third_party/wiredtiger/dist/s_docs +++ b/src/third_party/wiredtiger/dist/s_docs @@ -96,7 +96,8 @@ spellchk() type aspell > /dev/null 2>&1 || return (cd ../src/docs && - cat *.dox | aspell --lang=en --personal=./spell.ok list) | + cat *.dox | + aspell --encoding=iso-8859-1 --lang=en --personal=./spell.ok list) | sort -u > $t test -s $t && { echo "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=" diff --git a/src/third_party/wiredtiger/dist/s_funcs.list b/src/third_party/wiredtiger/dist/s_funcs.list index 01835390997..b73767cad13 100644 --- a/src/third_party/wiredtiger/dist/s_funcs.list +++ b/src/third_party/wiredtiger/dist/s_funcs.list @@ -13,7 +13,6 @@ __wt_bloom_get __wt_bulk_insert_fix __wt_bulk_insert_row __wt_bulk_insert_var -__wt_cache_dump __wt_config_getone __wt_cursor_get_raw_value __wt_debug_addr diff --git a/src/third_party/wiredtiger/dist/s_prototypes b/src/third_party/wiredtiger/dist/s_prototypes index 4c07b9aa160..9675cd5a843 100755 --- a/src/third_party/wiredtiger/dist/s_prototypes +++ b/src/third_party/wiredtiger/dist/s_prototypes @@ -39,7 +39,7 @@ proto() -e H \ -e x \ -e '}' \ - -e '# Add the gcc warn_unused_result attribute to any external' \ + -e '# Add the warn_unused_result attribute to any external' \ -e '# functions that return an int.' \ -e '/^extern int /s/$/ WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result))/' \ -e 's/$/;/' \ diff --git a/src/third_party/wiredtiger/dist/s_python b/src/third_party/wiredtiger/dist/s_python index b8aa5848637..bfe3ba57783 100755 --- a/src/third_party/wiredtiger/dist/s_python +++ b/src/third_party/wiredtiger/dist/s_python @@ -17,10 +17,10 @@ test -s $t && { echo '[tab] characters appear in Python scripts:' cat $t } -# Check Python coding standards: check for trailing semi-colons. +# Check Python coding standards: check for uncommented trailing semi-colons. # Don't check too widely, there are third-party tools that fail this test as # well as scripts in this directory that output C code, and so fail the test. -egrep ';$' `find lang test -name '*.py'`> $t +egrep '^[^#]*;$' `find lang test -name '*.py'`> $t test -s $t && { echo 'trailing semi-colons in selected Python code:' cat $t diff --git a/src/third_party/wiredtiger/dist/s_stat b/src/third_party/wiredtiger/dist/s_stat index 5d5937e1833..6aeeca6faa6 100755 --- a/src/third_party/wiredtiger/dist/s_stat +++ b/src/third_party/wiredtiger/dist/s_stat @@ -25,9 +25,6 @@ cat << UNUSED_STAT_FIELDS lock_checkpoint_count lock_checkpoint_wait_application lock_checkpoint_wait_internal -lock_handle_list_count -lock_handle_list_wait_application -lock_handle_list_wait_internal lock_metadata_count lock_metadata_wait_application lock_metadata_wait_internal diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index d19c05e802b..1f7f7d9fd3a 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -21,6 +21,7 @@ Alakuijala Alexandrescu's Alloc Async +AsyncOp Athanassoulis Athlon BBBBB @@ -38,6 +39,7 @@ BTREE BUF BZ Barack +BerkeleyDB Bitfield Bitwise Brueckner @@ -61,10 +63,12 @@ CPUs CRC CSV CStream +CURFILE CURSORs CURSTD CallsCustDate Castagnoli +CentOS Checkpointing Checksum Checksums @@ -88,6 +92,7 @@ DNE DOI DONTNEED DUPLICATEV +DataSet DbCursor DbEnv Decrement @@ -126,6 +131,7 @@ FNV FORALL FOREACH FS +FTRUNCATE FULLFSYNC Facebook FindClose @@ -156,6 +162,7 @@ HHHHLL HHHLL Hendrik HyperLevelDB +ID's IEC IEEE IKEY @@ -167,6 +174,7 @@ INITIALIZER INMEM INSN INTL +INULL ISA ITEMs ITER @@ -217,8 +225,10 @@ MEMALIGN MERCHANTABILITY METADATA MONGODB +MOVEFILE MRXB MRXBOPC +MSDN MSVC MULTI MULTIBLOCK @@ -232,8 +242,7 @@ Metadata Mewhort Mitzenmacher MongoDB -MoveFile -MoveFileW +MoveFileExW Multi MultiByteToWideChar Multithreaded @@ -247,6 +256,7 @@ NONINFRINGEMENT NOTFOUND NOTREACHED NOVALUE +NOWAIT NUL NUM NetBSD @@ -269,6 +279,8 @@ PRIu PRNG PTHREAD PTR +PackInputStream +PackOutputStream Pandis Phong PlatformSDK @@ -300,6 +312,7 @@ Readonly Rebalance RedHat Redistributions +Refactor Resize RocksDB SIMD @@ -331,6 +344,7 @@ Split's Stoica StoreLoad StoreStore +Syscall TAILQ TCMalloc TESTUTIL @@ -397,14 +411,18 @@ WiredTigerPreplog WiredTigerTmplog WiredTigerTxn WithSeeds +Wpedantic WriteFile Wuninitialized Wunused XP +Xcode +Xcode/ Yann ZSTD Zlib Zlib's +Zstandard Zstd Zstd's abcdef @@ -428,6 +446,7 @@ argc args argv asm +aspell assertfmt async asyncopp @@ -547,6 +566,7 @@ create's createCStream crypto cryptobad +csuite csv ctime ctype @@ -598,6 +618,7 @@ dequeue dequeued der dereference +dereferenced desc designator dest @@ -610,6 +631,7 @@ difftime dir dirlist disjunction +disjunctions dlclose dlh dll @@ -633,6 +655,7 @@ dumpcmp dumpfile dup eg +egrep emp encodings encryptor @@ -723,6 +746,7 @@ gettime gettimeofday getv github +gitignore gobare goesc gostring @@ -749,6 +773,7 @@ idx ifdef ifdef's iiSii +iiiS iiii iiu ikey @@ -834,6 +859,7 @@ libdatasource libs libtool libwiredtiger +linkers llll llu loadtext @@ -935,6 +961,7 @@ nowait nset nsnap nul +nullptr nuls numSymbols numbare @@ -948,6 +975,7 @@ oo opcode opendir openfile +oplog optimizations optype ori @@ -969,6 +997,7 @@ pclose pcpu perf pfx +pluggable popen poptable popthreads @@ -1113,12 +1142,15 @@ subgetraw subgets subinit sublicense +subtest subtree sunique superset sw +sx sy sys +syscall sz t's tV @@ -1156,6 +1188,7 @@ txt typedef uB uS +ui uint uintmax unbare @@ -1191,6 +1224,7 @@ upg uri uri's uris +usec usecs usedp userbad @@ -1221,10 +1255,14 @@ vunpack vw vxr waitpid +waker +wakeup +wakeups walk's warmup wb wcursor +whitespace wiredTiger wiredtiger workFactor @@ -1238,6 +1276,7 @@ ws wtbench wti wtperf +wtperf's wts wtstats xF @@ -1250,6 +1289,7 @@ zf zfree zlib zlib's +zseries zstd zstd's zu diff --git a/src/third_party/wiredtiger/dist/s_style b/src/third_party/wiredtiger/dist/s_style index 8e755224ee2..69cf1f667fa 100755 --- a/src/third_party/wiredtiger/dist/s_style +++ b/src/third_party/wiredtiger/dist/s_style @@ -93,10 +93,21 @@ else cat $t fi - # Alignment directive before "struct". - egrep 'WT_COMPILER_TYPE_ALIGN.*struct' $f > $t - test -s $t && { - echo "$f: compiler alignment direction must precede \"struct\"" + if ! expr "$f" : 'examples/c/*' > /dev/null && + ! expr "$f" : 'ext/*' > /dev/null && + ! expr "$f" : 'src/os_posix/os_snprintf.c' > /dev/null && + egrep '[^a-z_]snprintf\(|[^a-z_]vsnprintf\(' $f > $t; then + echo "$f: snprintf call, use WiredTiger library replacements" + cat $t + fi + + # If we don't have matching pack-begin and pack-end calls, we don't get + # an error, we just get a Windows performance regression. Using awk and + # not wc to ensure there's no whitespace in the assignment. + egrep WT_PACKED_STRUCT $f > $t + cnt=`awk 'BEGIN { line = 0 } { ++line } END { print line }' < $t` + test `expr "$cnt" % 2` -ne 0 && { + echo "$f: mismatched WT_PACKED_STRUCT_BEGIN/END lines" cat $t } diff --git a/src/third_party/wiredtiger/dist/s_void b/src/third_party/wiredtiger/dist/s_void old mode 100644 new mode 100755 index b9c3f5e0ef7..249f043d029 --- a/src/third_party/wiredtiger/dist/s_void +++ b/src/third_party/wiredtiger/dist/s_void @@ -5,12 +5,14 @@ trap 'rm -f $t' 0 1 2 3 13 15 cd .. -# Turn a C file into a line per function that returns an int. +# Parse a C file, discarding functions that don't return an int, and formatting +# the remaining functions as a single line. file_parse() { sed -n \ -e '/^int$/b loop' \ -e '/^static int$/b loop' \ + -e '/^static inline int$/b loop' \ -e 'd' \ -e ': loop' \ -e 'H' \ @@ -34,6 +36,7 @@ func_ok() -e '/int __compact_uri_analyze$/d' \ -e '/int __config_parser_close$/d' \ -e '/int __curlog_reset$/d' \ + -e '/int __cursor_fix_implicit$/d' \ -e '/int __handle_close_default$/d' \ -e '/int __handle_progress_default$/d' \ -e '/int __im_file_close$/d' \ @@ -42,7 +45,7 @@ func_ok() -e '/int __im_file_sync$/d' \ -e '/int __im_fs_directory_list_free$/d' \ -e '/int __im_fs_exist$/d' \ - -e '/int __posix_file_close$/d' \ + -e '/int __page_write_gen_wrapped_check$/d' \ -e '/int __posix_terminate$/d' \ -e '/int __rec_destroy_session$/d' \ -e '/int __win_terminate$/d' \ @@ -59,11 +62,14 @@ func_ok() -e '/int __wt_lsm_manager_pop_entry$/d' \ -e '/int __wt_once$/d' \ -e '/int __wt_posix_directory_list_free$/d' \ + -e '/int __wt_spin_init$/d' \ + -e '/int __wt_spin_trylock$/d' \ -e '/int __wt_stat_connection_desc$/d' \ -e '/int __wt_stat_dsrc_desc$/d' \ -e '/int __wt_stat_join_desc$/d' \ -e '/int __wt_win_directory_list_free$/d' \ -e '/int bdb_compare_reverse$/d' \ + -e '/int copyout_val$/d' \ -e '/int csv_error$/d' \ -e '/int csv_terminate$/d' \ -e '/int demo_file_close$/d' \ @@ -72,10 +78,20 @@ func_ok() -e '/int demo_file_sync$/d' \ -e '/int demo_fs_directory_list_free$/d' \ -e '/int demo_fs_exist$/d' \ + -e '/int fail_file_lock$/d' \ + -e '/int fail_file_sync$/d' \ + -e '/int fail_fs_directory_list_free$/d' \ + -e '/int fail_fs_exist$/d' \ + -e '/int fail_fs_simulate_fail$/d' \ + -e '/int fail_fs_terminate$/d' \ -e '/int handle_message$/d' \ -e '/int handle_progress$/d' \ -e '/int helium_cursor_reset$/d' \ -e '/int helium_session_verify$/d' \ + -e '/int index_compare_primary$/d' \ + -e '/int index_compare_S$/d' \ + -e '/int index_compare_u$/d' \ + -e '/int index_extractor_u$/d' \ -e '/int log_print_err$/d' \ -e '/int lz4_error$/d' \ -e '/int lz4_pre_size$/d' \ @@ -108,25 +124,29 @@ func_ok() -e '/int zstd_terminate$/d' } -# Complain about functions which return an "int" but which don't return except -# at the end of the function. This script is a kluge and isn't run by default. for f in `find bench ext src test -name '*.[ci]'`; do if expr "$f" : '.*/windows_shim.c' > /dev/null; then continue fi + # Complain about functions which return an "int" but which don't return + # except at the end of the function. + # # Turn each function into a single line, then discard the function's # final "return" call, then discard any function that still has some # form of return assignment or call. file_parse $f | sed -e 's/return ([^)]*); }$//' \ - -e '/[A-Z]*_API_CALL[A-Z_]*(/d' \ + -e '/[_A-Z]*_API_CALL[_A-Z]*(/d' \ -e '/WT_CURSOR_NEEDKEY(/d' \ -e '/WT_CURSOR_NEEDVALUE(/d' \ -e '/WT_ERR[A-Z_]*(/d' \ -e '/WT_ILLEGAL_VALUE[A-Z_]*(/d' \ -e '/WT_PANIC[A-Z_]*(/d' \ -e '/WT_RET[A-Z_]*(/d' \ + -e '/WT_SIZE_CHECK_PACK(/d' \ + -e '/WT_SIZE_CHECK_UNPACK(/d' \ + -e '/WT_SYSCALL(/d' \ -e '/WT_TRET(/d' \ -e '/[^a-z_]ret = /d' \ -e '/[^a-z_]return (/d' \ @@ -134,8 +154,34 @@ for f in `find bench ext src test -name '*.[ci]'`; do -e 's/^ *//' | func_ok > $t test -s $t && { - echo "========== $f" - cat $t + echo "==============================================" + echo "$f:" + cat $t | sed 's/^/ /' + echo "Function could return void instead of int." + echo "Add false positives to the list in dist/s_void." + echo "==============================================" + } + + # Complain about functions which declare a "ret" value but never use it. + file_parse $f | + grep 'WT_DECL_RET' | + sed -e '/ret =/d' \ + -e '/[_A-Z]*_API_CALL[_A-Z]*(/d' \ + -e '/WT_CURSOR_NEEDKEY/d' \ + -e '/WT_CURSOR_NEEDVALUE/d' \ + -e '/WT_ERR/d' \ + -e '/WT_ILLEGAL_VALUE_ERR/d' \ + -e '/WT_SYSCALL.*ret/d' \ + -e '/WT_TRET/d' \ + -e 's/^\([^(]*\).*/\1/' \ + -e 's/^ *//' > $t + test -s $t && { + echo "==============================================" + echo "$f:" + cat $t | sed 's/^/ /' + echo "Function declares ret without using it." + echo "Add false positives to the list in dist/s_void." + echo "==============================================" } done diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index bcf5201bd90..ac79ffd029a 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -193,6 +193,7 @@ connection_stats = [ CacheStat('cache_bytes_other', 'bytes not belonging to page images in the cache', 'no_clear,no_scale,size'), CacheStat('cache_bytes_read', 'bytes read into cache', 'size'), CacheStat('cache_bytes_write', 'bytes written from cache', 'size'), + CacheStat('cache_eviction_active_workers', 'eviction worker thread active', 'no_clear'), CacheStat('cache_eviction_aggressive_set', 'eviction currently operating in aggressive mode', 'no_clear,no_scale'), CacheStat('cache_eviction_app', 'pages evicted by application threads'), CacheStat('cache_eviction_app_dirty', 'modified pages evicted by application threads'), @@ -205,6 +206,7 @@ connection_stats = [ CacheStat('cache_eviction_force', 'pages evicted because they exceeded the in-memory maximum'), CacheStat('cache_eviction_force_delete', 'pages evicted because they had chains of deleted items'), CacheStat('cache_eviction_force_fail', 'failed eviction of pages that exceeded the in-memory maximum'), + CacheStat('cache_eviction_force_retune', 'force re-tuning of eviction workers once in a while'), CacheStat('cache_eviction_get_ref', 'eviction calls to get a page'), CacheStat('cache_eviction_get_ref_empty', 'eviction calls to get a page found queue empty'), CacheStat('cache_eviction_get_ref_empty2', 'eviction calls to get a page found queue empty after locking'), @@ -222,12 +224,15 @@ connection_stats = [ CacheStat('cache_eviction_slow', 'eviction server unable to reach eviction goal'), CacheStat('cache_eviction_split_internal', 'internal pages split during eviction'), CacheStat('cache_eviction_split_leaf', 'leaf pages split during eviction'), + CacheStat('cache_eviction_stable_state_workers', 'eviction worker thread stable number', 'no_clear'), CacheStat('cache_eviction_state', 'eviction state', 'no_clear,no_scale'), CacheStat('cache_eviction_walk', 'pages walked for eviction'), CacheStat('cache_eviction_walks_abandoned', 'eviction walks abandoned'), CacheStat('cache_eviction_walks_active', 'files with active eviction walks', 'no_clear,no_scale'), CacheStat('cache_eviction_walks_started', 'files with new eviction walks started'), CacheStat('cache_eviction_worker_evicting', 'eviction worker thread evicting pages'), + CacheStat('cache_eviction_worker_created', 'eviction worker thread created'), + CacheStat('cache_eviction_worker_removed', 'eviction worker thread removed'), CacheStat('cache_hazard_checks', 'hazard pointer check calls'), CacheStat('cache_hazard_max', 'hazard pointer maximum array length', 'max_aggregate,no_scale'), CacheStat('cache_hazard_walks', 'hazard pointer check entries walked'), @@ -284,9 +289,7 @@ connection_stats = [ LockStat('lock_checkpoint_count', 'checkpoint lock acquisitions'), LockStat('lock_checkpoint_wait_application', 'checkpoint lock application thread wait time (usecs)'), LockStat('lock_checkpoint_wait_internal', 'checkpoint lock internal thread wait time (usecs)'), - LockStat('lock_handle_list_count', 'handle-list lock acquisitions'), - LockStat('lock_handle_list_wait_application', 'handle-list lock application thread wait time (usecs)'), - LockStat('lock_handle_list_wait_internal', 'handle-list lock internal thread wait time (usecs)'), + LockStat('lock_handle_list_wait_eviction', 'handle-list lock eviction thread wait time (usecs)'), LockStat('lock_metadata_count', 'metadata lock acquisitions'), LockStat('lock_metadata_wait_application', 'metadata lock application thread wait time (usecs)'), LockStat('lock_metadata_wait_internal', 'metadata lock internal thread wait time (usecs)'), @@ -321,10 +324,12 @@ connection_stats = [ LogStat('log_scan_records', 'records processed by log scan'), LogStat('log_scan_rereads', 'log scan records requiring two reads'), LogStat('log_scans', 'log scan operations'), + LogStat('log_slot_active_closed', 'consolidated slot join active slot closed'), LogStat('log_slot_closes', 'consolidated slot closures'), LogStat('log_slot_coalesced', 'written slots coalesced'), LogStat('log_slot_consolidated', 'logging bytes consolidated', 'size'), LogStat('log_slot_joins', 'consolidated slot joins'), + LogStat('log_slot_no_free_slots', 'consolidated slot transitions unable to find free slot'), LogStat('log_slot_races', 'consolidated slot join races'), LogStat('log_slot_switch_busy', 'busy returns attempting to switch slots'), LogStat('log_slot_transitions', 'consolidated slot join transitions'), @@ -367,6 +372,9 @@ connection_stats = [ ########################################## SessionStat('session_cursor_open', 'open cursor count', 'no_clear,no_scale'), SessionStat('session_open', 'open session count', 'no_clear,no_scale'), + SessionStat('session_table_alter_fail', 'table alter failed calls', 'no_clear,no_scale'), + SessionStat('session_table_alter_skip', 'table alter unchanged and skipped', 'no_clear,no_scale'), + SessionStat('session_table_alter_success', 'table alter successful calls', 'no_clear,no_scale'), SessionStat('session_table_compact_fail', 'table compact failed calls', 'no_clear,no_scale'), SessionStat('session_table_compact_success', 'table compact successful calls', 'no_clear,no_scale'), SessionStat('session_table_create_fail', 'table create failed calls', 'no_clear,no_scale'), @@ -474,6 +482,7 @@ dsrc_stats = [ ########################################## # Cache and eviction statistics ########################################## + CacheStat('cache_bytes_dirty', 'tracked dirty bytes in the cache', 'no_clear,no_scale,size'), CacheStat('cache_bytes_inuse', 'bytes currently in the cache', 'no_clear,no_scale,size'), CacheStat('cache_bytes_read', 'bytes read into cache', 'size'), CacheStat('cache_bytes_write', 'bytes written from cache', 'size'), diff --git a/src/third_party/wiredtiger/examples/c/Makefile.am b/src/third_party/wiredtiger/examples/c/Makefile.am index d5305eec5c8..20936661e06 100644 --- a/src/third_party/wiredtiger/examples/c/Makefile.am +++ b/src/third_party/wiredtiger/examples/c/Makefile.am @@ -20,7 +20,6 @@ noinst_PROGRAMS = \ ex_pack \ ex_process \ ex_schema \ - ex_scope \ ex_stat \ ex_sync \ ex_thread diff --git a/src/third_party/wiredtiger/examples/c/ex_all.c b/src/third_party/wiredtiger/examples/c/ex_all.c index ea646604a76..82620673fe1 100644 --- a/src/third_party/wiredtiger/examples/c/ex_all.c +++ b/src/third_party/wiredtiger/examples/c/ex_all.c @@ -557,6 +557,12 @@ session_ops(WT_SESSION *session) /*! [Create a column-store table] */ ret = session->create(session, "table:mytable", "key_format=r,value_format=S"); + + /*! [Alter a table] */ + ret = session->alter(session, + "table:mytable", "access_pattern_hint=random"); + /*! [Alter a table] */ + /*! [Create a column-store table] */ ret = session->drop(session, "table:mytable", NULL); @@ -842,8 +848,8 @@ my_compare(WT_COLLATOR *collator, WT_SESSION *session, p1 = (const char *)value1->data; p2 = (const char *)value2->data; - while (*p1 != '\0' && *p1 == *p2) - p1++, p2++; + for (; *p1 != '\0' && *p1 == *p2; ++p1, ++p2) + ; *cmp = (int)*p2 - (int)*p1; return (0); diff --git a/src/third_party/wiredtiger/examples/c/ex_async.c b/src/third_party/wiredtiger/examples/c/ex_async.c index f7531a5c3d8..5cfafca0418 100644 --- a/src/third_party/wiredtiger/examples/c/ex_async.c +++ b/src/third_party/wiredtiger/examples/c/ex_async.c @@ -170,12 +170,12 @@ main(void) * an asynchronous insert. */ /*! [async set the operation's string key] */ - snprintf(k[i], sizeof(k), "key%d", i); + (void)snprintf(k[i], sizeof(k), "key%d", i); op->set_key(op, k[i]); /*! [async set the operation's string key] */ /*! [async set the operation's string value] */ - snprintf(v[i], sizeof(v), "value%d", i); + (void)snprintf(v[i], sizeof(v), "value%d", i); op->set_value(op, v[i]); /*! [async set the operation's string value] */ @@ -218,7 +218,7 @@ main(void) * Set the operation's string key and value, and then do * an asynchronous search. */ - snprintf(k[i], sizeof(k), "key%d", i); + (void)snprintf(k[i], sizeof(k), "key%d", i); op->set_key(op, k[i]); ret = op->search(op); /*! [async search] */ diff --git a/src/third_party/wiredtiger/examples/c/ex_backup.c b/src/third_party/wiredtiger/examples/c/ex_backup.c index 0697cbb3458..83cc9b22ecc 100644 --- a/src/third_party/wiredtiger/examples/c/ex_backup.c +++ b/src/third_party/wiredtiger/examples/c/ex_backup.c @@ -96,7 +96,7 @@ compare_backups(int i) if (i == 0) (void)strncpy(msg, "MAIN", sizeof(msg)); else - snprintf(msg, sizeof(msg), "%d", i); + (void)snprintf(msg, sizeof(msg), "%d", i); printf( "Iteration %s: Tables %s.%d and %s.%d %s\n", msg, full_out, i, incr_out, i, ret == 0 ? "identical" : "differ"); @@ -131,8 +131,8 @@ setup_directories(void) * For incremental backups we need 0-N. The 0 incremental * directory will compare with the original at the end. */ - snprintf(buf, sizeof(buf), "rm -rf %s.%d && mkdir %s.%d", - home_incr, i, home_incr, i); + (void)snprintf(buf, sizeof(buf), + "rm -rf %s.%d && mkdir %s.%d", home_incr, i, home_incr, i); if ((ret = system(buf)) != 0) { fprintf(stderr, "%s: failed ret %d\n", buf, ret); return (ret); @@ -142,8 +142,8 @@ setup_directories(void) /* * For full backups we need 1-N. */ - snprintf(buf, sizeof(buf), "rm -rf %s.%d && mkdir %s.%d", - home_full, i, home_full, i); + (void)snprintf(buf, sizeof(buf), + "rm -rf %s.%d && mkdir %s.%d", home_full, i, home_full, i); if ((ret = system(buf)) != 0) { fprintf(stderr, "%s: failed ret %d\n", buf, ret); return (ret); @@ -164,8 +164,8 @@ add_work(WT_SESSION *session, int iter) * Perform some operations with individual auto-commit transactions. */ for (i = 0; i < MAX_KEYS; i++) { - snprintf(k, sizeof(k), "key.%d.%d", iter, i); - snprintf(v, sizeof(v), "value.%d.%d", iter, i); + (void)snprintf(k, sizeof(k), "key.%d.%d", iter, i); + (void)snprintf(v, sizeof(v), "value.%d.%d", iter, i); cursor->set_key(cursor, k); cursor->set_value(cursor, v); ret = cursor->insert(cursor); @@ -187,7 +187,7 @@ take_full_backup(WT_SESSION *session, int i) * directories. Otherwise only into the appropriate full directory. */ if (i != 0) { - snprintf(h, sizeof(h), "%s.%d", home_full, i); + (void)snprintf(h, sizeof(h), "%s.%d", home_full, i); hdir = h; } else hdir = home_incr; @@ -200,14 +200,15 @@ take_full_backup(WT_SESSION *session, int i) * Take a full backup into each incremental directory. */ for (j = 0; j < MAX_ITERATIONS; j++) { - snprintf(h, sizeof(h), "%s.%d", home_incr, j); + (void)snprintf(h, sizeof(h), + "%s.%d", home_incr, j); (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); ret = system(buf); } else { - snprintf(h, sizeof(h), "%s.%d", home_full, i); + (void)snprintf(h, sizeof(h), "%s.%d", home_full, i); (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, hdir, filename); ret = system(buf); @@ -237,12 +238,12 @@ take_incr_backup(WT_SESSION *session, int i) * Copy into the 0 incremental directory and then each of the * incremental directories for this iteration and later. */ - snprintf(h, sizeof(h), "%s.0", home_incr); + (void)snprintf(h, sizeof(h), "%s.0", home_incr); (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); ret = system(buf); for (j = i; j < MAX_ITERATIONS; j++) { - snprintf(h, sizeof(h), "%s.%d", home_incr, j); + (void)snprintf(h, sizeof(h), "%s.%d", home_incr, j); (void)snprintf(buf, sizeof(buf), "cp %s/%s %s/%s", home, filename, h, filename); ret = system(buf); @@ -270,7 +271,8 @@ main(void) int i, ret; char cmd_buf[256]; - snprintf(cmd_buf, sizeof(cmd_buf), "rm -rf %s && mkdir %s", home, home); + (void)snprintf(cmd_buf, sizeof(cmd_buf), + "rm -rf %s && mkdir %s", home, home); if ((ret = system(cmd_buf)) != 0) { fprintf(stderr, "%s: failed ret %d\n", cmd_buf, ret); return (EXIT_FAILURE); diff --git a/src/third_party/wiredtiger/examples/c/ex_data_source.c b/src/third_party/wiredtiger/examples/c/ex_data_source.c index 6ed80dfcf19..387248f6ae2 100644 --- a/src/third_party/wiredtiger/examples/c/ex_data_source.c +++ b/src/third_party/wiredtiger/examples/c/ex_data_source.c @@ -46,6 +46,21 @@ my_data_source_init(WT_CONNECTION *connection) } /*! [WT_EXTENSION_API declaration] */ +/*! [WT_DATA_SOURCE alter] */ +static int +my_alter(WT_DATA_SOURCE *dsrc, WT_SESSION *session, + const char *uri, WT_CONFIG_ARG *config) +/*! [WT_DATA_SOURCE alter] */ +{ + /* Unused parameters */ + (void)dsrc; + (void)session; + (void)uri; + (void)config; + + return (0); +} + /*! [WT_DATA_SOURCE create] */ static int my_create(WT_DATA_SOURCE *dsrc, WT_SESSION *session, @@ -604,6 +619,7 @@ main(void) { /*! [WT_DATA_SOURCE register] */ static WT_DATA_SOURCE my_dsrc = { + my_alter, my_create, my_compact, my_drop, diff --git a/src/third_party/wiredtiger/examples/c/ex_encrypt.c b/src/third_party/wiredtiger/examples/c/ex_encrypt.c index 00dc66fc24d..1520bd286cd 100644 --- a/src/third_party/wiredtiger/examples/c/ex_encrypt.c +++ b/src/third_party/wiredtiger/examples/c/ex_encrypt.c @@ -507,12 +507,12 @@ main(void) * we decrypt on read. */ for (i = 0; i < MAX_KEYS; i++) { - snprintf(keybuf, sizeof(keybuf), "key%d", i); + (void)snprintf(keybuf, sizeof(keybuf), "key%d", i); c1->set_key(c1, keybuf); c2->set_key(c2, keybuf); nc->set_key(nc, keybuf); - snprintf(valbuf, sizeof(valbuf), "value%d", i); + (void)snprintf(valbuf, sizeof(valbuf), "value%d", i); c1->set_value(c1, valbuf); c2->set_value(c2, valbuf); nc->set_value(nc, valbuf); diff --git a/src/third_party/wiredtiger/examples/c/ex_file_system.c b/src/third_party/wiredtiger/examples/c/ex_file_system.c index 524a5d03f89..e807ac54d3b 100644 --- a/src/third_party/wiredtiger/examples/c/ex_file_system.c +++ b/src/third_party/wiredtiger/examples/c/ex_file_system.c @@ -29,35 +29,49 @@ * demonstrates how to use the custom file system interface */ -/* - * Include WiredTiger internal functions: we need architecture portable locking - * in this example, and we use the TAILQ_XXX functions to keep the code simple. - * - * Application-writers SHOULD NOT INCLUDE "wt_internal.h", the public WiredTiger - * include files should be used instead: - * - * #include - * #include - */ -#include "wt_internal.h" +#include +#include +#include +#include +#include +#include + +#ifndef _WIN32 +#include +#else +#include "windows_shim.h" +#endif + +#include +#include /* - * This example code uses internal WiredTiger functions for portable locking. - * We use #defines to clarify the meaning and ignore errors to simplify the - * code. - * - * Application writers SHOULD NOT COPY THIS LOCKING CODE, it's special-case code - * to make this example portable across platforms. + * This example code uses pthread functions for portable locking, we ignore + * errors for simplicity. */ -#define ALLOCATE_FILE_SYSTEM_LOCK(demo_fs) \ - (void)__wt_spin_init(NULL, &(demo_fs)->lock, "demo file handle lock") -#define DESTROY_FILE_SYSTEM_LOCK(wt_session, demo_fs) \ - __wt_spin_destroy((WT_SESSION_IMPL *)(wt_session), &(demo_fs)->lock) -#define LOCK_FILE_SYSTEM(wt_session, demo_fs) \ - __wt_spin_lock((WT_SESSION_IMPL *)(wt_session), &(demo_fs)->lock) -#define UNLOCK_FILE_SYSTEM(wt_session, demo_fs) \ - __wt_spin_unlock( \ - (WT_SESSION_IMPL *)(wt_session), &(demo_fs)->lock) +static void +allocate_file_system_lock(pthread_rwlock_t *lockp) +{ + assert(pthread_rwlock_init(lockp, NULL) == 0); +} + +static void +destroy_file_system_lock(pthread_rwlock_t *lockp) +{ + assert(pthread_rwlock_destroy(lockp) == 0); +} + +static void +lock_file_system(pthread_rwlock_t *lockp) +{ + assert(pthread_rwlock_wrlock(lockp) == 0); +} + +static void +unlock_file_system(pthread_rwlock_t *lockp) +{ + assert(pthread_rwlock_unlock(lockp) == 0); +} /* * Example file system implementation, using memory buffers to represent files. @@ -72,7 +86,7 @@ typedef struct { * might require finer granularity, for example, a single lock for the * file system handle list and per-handle locks serializing I/O. */ - WT_SPINLOCK lock; /* Lock */ + pthread_rwlock_t lock; /* Lock */ int opened_file_count; int opened_unique_file_count; @@ -194,19 +208,9 @@ demo_file_system_create(WT_CONNECTION *conn, WT_CONFIG_ARG *config) * the underlying filesystem implementation. See the main function for * the setup of those configuration strings; here we parse configuration * information as passed in by main, through WiredTiger. - * - * Retrieve our configuration information, the "config" value. */ - if ((ret = wtext->config_get(wtext, NULL, config, "config", &v)) != 0) { - (void)wtext->err_printf(wtext, NULL, - "WT_EXTENSION_API.config_get: config: %s", - wtext->strerror(wtext, NULL, ret)); - goto err; - } - - /* Open a WiredTiger parser on the "config" value. */ - if ((ret = wtext->config_parser_open( - wtext, NULL, v.str, v.len, &config_parser)) != 0) { + if ((ret = wtext->config_parser_open_arg( + wtext, NULL, config, &config_parser)) != 0) { (void)wtext->err_printf(wtext, NULL, "WT_EXTENSION_API.config_parser_open: config: %s", wtext->strerror(wtext, NULL, ret)); @@ -249,7 +253,7 @@ demo_file_system_create(WT_CONNECTION *conn, WT_CONFIG_ARG *config) goto err; } - ALLOCATE_FILE_SYSTEM_LOCK(demo_fs); + allocate_file_system_lock(&demo_fs->lock); /* Initialize the in-memory jump table. */ file_system->fs_directory_list = demo_fs_directory_list; @@ -298,7 +302,7 @@ demo_fs_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, demo_fh = NULL; wtext = demo_fs->wtext; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); ++demo_fs->opened_file_count; /* @@ -318,7 +322,7 @@ demo_fs_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, *file_handlep = (WT_FILE_HANDLE *)demo_fh; - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); return (0); } @@ -376,7 +380,7 @@ err: free(demo_fh->buf); free(demo_fh); } - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); return (ret); } @@ -395,6 +399,9 @@ demo_fs_directory_list(WT_FILE_SYSTEM *file_system, uint32_t allocated, count; int ret = 0; char *name, **entries; + void *p; + + (void)session; /* Unused */ demo_fs = (DEMO_FILE_SYSTEM *)file_system; @@ -406,7 +413,7 @@ demo_fs_directory_list(WT_FILE_SYSTEM *file_system, len = strlen(directory); prefix_len = prefix == NULL ? 0 : strlen(prefix); - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); TAILQ_FOREACH(demo_fh, &demo_fs->fileq, q) { name = demo_fh->iface.name; if (strncmp(name, directory, len) != 0 || @@ -418,14 +425,16 @@ demo_fs_directory_list(WT_FILE_SYSTEM *file_system, * matter if the list is a bit longer than necessary. */ if (count >= allocated) { - entries = realloc( - entries, (allocated + 10) * sizeof(char *)); - if (entries == NULL) { + p = realloc( + entries, (allocated + 10) * sizeof(*entries)); + if (p == NULL) { ret = ENOMEM; goto err; } - memset(entries + allocated * sizeof(char *), - 0, 10 * sizeof(char *)); + + entries = p; + memset(entries + allocated * sizeof(*entries), + 0, 10 * sizeof(*entries)); allocated += 10; } entries[count++] = strdup(name); @@ -434,7 +443,7 @@ demo_fs_directory_list(WT_FILE_SYSTEM *file_system, *dirlistp = entries; *countp = count; -err: UNLOCK_FILE_SYSTEM(session, demo_fs); +err: unlock_file_system(&demo_fs->lock); if (ret == 0) return (0); @@ -476,11 +485,13 @@ demo_fs_exist(WT_FILE_SYSTEM *file_system, { DEMO_FILE_SYSTEM *demo_fs; + (void)session; /* Unused */ + demo_fs = (DEMO_FILE_SYSTEM *)file_system; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); *existp = demo_handle_search(file_system, name) != NULL; - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); return (0); } @@ -497,15 +508,16 @@ demo_fs_remove(WT_FILE_SYSTEM *file_system, DEMO_FILE_HANDLE *demo_fh; int ret = 0; - (void)flags; /* Unused */ + (void)session; /* Unused */ + (void)flags; /* Unused */ demo_fs = (DEMO_FILE_SYSTEM *)file_system; ret = ENOENT; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); if ((demo_fh = demo_handle_search(file_system, name)) != NULL) ret = demo_handle_remove(session, demo_fh); - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); return (ret); } @@ -523,11 +535,12 @@ demo_fs_rename(WT_FILE_SYSTEM *file_system, char *copy; int ret = 0; - (void)flags; /* Unused */ + (void)session; /* Unused */ + (void)flags; /* Unused */ demo_fs = (DEMO_FILE_SYSTEM *)file_system; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); if ((demo_fh = demo_handle_search(file_system, from)) == NULL) ret = ENOENT; else if ((copy = strdup(to)) == NULL) @@ -536,7 +549,7 @@ demo_fs_rename(WT_FILE_SYSTEM *file_system, free(demo_fh->iface.name); demo_fh->iface.name = copy; } - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); return (ret); } @@ -555,10 +568,10 @@ demo_fs_size(WT_FILE_SYSTEM *file_system, demo_fs = (DEMO_FILE_SYSTEM *)file_system; ret = ENOENT; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); if ((demo_fh = demo_handle_search(file_system, name)) != NULL) ret = demo_file_size((WT_FILE_HANDLE *)demo_fh, session, sizep); - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); return (ret); } @@ -588,7 +601,7 @@ demo_fs_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *session) printf("\t%d reads, %d writes\n", demo_fs->read_ops, demo_fs->write_ops); - DESTROY_FILE_SYSTEM_LOCK(session, demo_fs); + destroy_file_system_lock(&demo_fs->lock); free(demo_fs); return (ret); @@ -604,13 +617,15 @@ demo_file_close(WT_FILE_HANDLE *file_handle, WT_SESSION *session) DEMO_FILE_HANDLE *demo_fh; DEMO_FILE_SYSTEM *demo_fs; + (void)session; /* Unused */ + demo_fh = (DEMO_FILE_HANDLE *)file_handle; demo_fs = demo_fh->demo_fs; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); if (--demo_fh->ref == 0) ++demo_fs->closed_file_count; - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); return (0); } @@ -648,7 +663,7 @@ demo_file_read(WT_FILE_HANDLE *file_handle, wtext = demo_fs->wtext; off = (size_t)offset; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); ++demo_fs->read_ops; if (off < demo_fh->size) { if (len > demo_fh->size - off) @@ -656,7 +671,7 @@ demo_file_read(WT_FILE_HANDLE *file_handle, memcpy(buf, (uint8_t *)demo_fh->buf + off, len); } else ret = EIO; /* EOF */ - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); if (ret == 0) return (0); @@ -677,12 +692,14 @@ demo_file_size( DEMO_FILE_HANDLE *demo_fh; DEMO_FILE_SYSTEM *demo_fs; + (void)session; /* Unused */ + demo_fh = (DEMO_FILE_HANDLE *)file_handle; demo_fs = demo_fh->demo_fs; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); *sizep = (wt_off_t)demo_fh->size; - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); return (0); } @@ -749,10 +766,10 @@ demo_file_truncate( demo_fh = (DEMO_FILE_HANDLE *)file_handle; demo_fs = demo_fh->demo_fs; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); if ((ret = demo_buffer_resize(session, demo_fh, offset)) == 0) demo_fh->size = (size_t)offset; - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); return (ret); } @@ -775,7 +792,7 @@ demo_file_write(WT_FILE_HANDLE *file_handle, WT_SESSION *session, wtext = demo_fs->wtext; off = (size_t)offset; - LOCK_FILE_SYSTEM(session, demo_fs); + lock_file_system(&demo_fs->lock); ++demo_fs->write_ops; if ((ret = demo_buffer_resize(session, demo_fh, offset + (wt_off_t)(len + DEMO_FILE_SIZE_INCREMENT))) == 0) { @@ -783,7 +800,7 @@ demo_file_write(WT_FILE_HANDLE *file_handle, WT_SESSION *session, if (off + len > demo_fh->size) demo_fh->size = off + len; } - UNLOCK_FILE_SYSTEM(session, demo_fs); + unlock_file_system(&demo_fs->lock); if (ret == 0) return (0); diff --git a/src/third_party/wiredtiger/examples/c/ex_log.c b/src/third_party/wiredtiger/examples/c/ex_log.c index fdbc39412ae..0d8fbf97233 100644 --- a/src/third_party/wiredtiger/examples/c/ex_log.c +++ b/src/third_party/wiredtiger/examples/c/ex_log.c @@ -291,8 +291,8 @@ main(void) char cmd_buf[256], k[16], v[16]; count_min = 0; - snprintf(cmd_buf, sizeof(cmd_buf), "rm -rf %s %s && mkdir %s %s", - home1, home2, home1, home2); + (void)snprintf(cmd_buf, sizeof(cmd_buf), + "rm -rf %s %s && mkdir %s %s", home1, home2, home1, home2); if ((ret = system(cmd_buf)) != 0) { fprintf(stderr, "%s: failed ret %d\n", cmd_buf, ret); return (EXIT_FAILURE); @@ -312,8 +312,8 @@ main(void) * Perform some operations with individual auto-commit transactions. */ for (record_count = 0, i = 0; i < MAX_KEYS; i++, record_count++) { - snprintf(k, sizeof(k), "key%d", i); - snprintf(v, sizeof(v), "value%d", i); + (void)snprintf(k, sizeof(k), "key%d", i); + (void)snprintf(v, sizeof(v), "value%d", i); cursor->set_key(cursor, k); cursor->set_value(cursor, v); ret = cursor->insert(cursor); @@ -324,8 +324,8 @@ main(void) * Perform some operations within a single transaction. */ for (i = MAX_KEYS; i < MAX_KEYS+5; i++, record_count++) { - snprintf(k, sizeof(k), "key%d", i); - snprintf(v, sizeof(v), "value%d", i); + (void)snprintf(k, sizeof(k), "key%d", i); + (void)snprintf(v, sizeof(v), "value%d", i); cursor->set_key(cursor, k); cursor->set_value(cursor, v); ret = cursor->insert(cursor); diff --git a/src/third_party/wiredtiger/examples/c/ex_scope.c b/src/third_party/wiredtiger/examples/c/ex_scope.c deleted file mode 100644 index 795ad85d57b..00000000000 --- a/src/third_party/wiredtiger/examples/c/ex_scope.c +++ /dev/null @@ -1,217 +0,0 @@ -/*- - * Public Domain 2014-2016 MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. - * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * ex_scope.c - * demonstrates the scope of buffers holding cursor keys and values. - */ -#include -#include -#include -#include - -#include - -#ifdef _WIN32 -/* snprintf is not supported on <= VS2013 */ -#define snprintf _snprintf -#endif - -static const char *home; - -static int -cursor_scope_ops(WT_CURSOR *cursor) -{ - struct { - const char *op; - const char *key; - const char *value; - int (*apply)(WT_CURSOR *); - } *op, ops[] = { - { "insert", "key1", "value1", cursor->insert, }, - { "update", "key1", "value2", cursor->update, }, - { "search", "key1", "value2", cursor->search, }, - { "remove", "key1", "value2", cursor->remove, }, - { NULL, NULL, NULL, NULL } - }; - WT_SESSION *session; - const char *key, *value; - char keybuf[10], valuebuf[10]; - int ret; - - session = cursor->session; - - for (op = ops; op->key != NULL; op++) { - key = value = NULL; - - /*! [cursor scope operation] */ - (void)snprintf(keybuf, sizeof(keybuf), "%s", op->key); - cursor->set_key(cursor, keybuf); - (void)snprintf(valuebuf, sizeof(valuebuf), "%s", op->value); - cursor->set_value(cursor, valuebuf); - - /* - * The application must keep key and value memory valid until - * the next operation that positions the cursor, modifies the - * data, or resets or closes the cursor. - * - * Modifying either the key or value buffers is not permitted. - */ - - /* Apply the operation (insert, update, search or remove). */ - if ((ret = op->apply(cursor)) != 0) { - fprintf(stderr, - "%s: error performing the operation: %s\n", - op->op, session->strerror(session, ret)); - return (ret); - } - - /* - * The cursor no longer references application memory, so - * application buffers can be safely overwritten. - */ - strcpy(keybuf, "no key"); - strcpy(valuebuf, "no value"); - - /* - * Check that get_key/value behave as expected after the - * operation. - */ - if (op->apply == cursor->insert) { - /* - * WT_CURSOR::insert no longer references application - * memory, but as it does not position the cursor, it - * doesn't reference memory owned by the cursor, either. - */ - printf("ex_scope: " - "expect two WiredTiger error messages:\n"); - if ((ret = cursor->get_key(cursor, &key)) == 0 || - (ret = cursor->get_value(cursor, &value)) == 0) { - fprintf(stderr, - "%s: error in get_key/value: %s\n", - op->op, session->strerror(session, ret)); - return (ret); - } - continue; - } - if (op->apply == cursor->remove) { - /* - * WT_CURSOR::remove no longer references application - * memory; as it does not position the cursor, it will - * reference key memory owned by the cursor, but has no - * value. - */ - printf("ex_scope: " - "expect one WiredTiger error message:\n"); - if ((ret = cursor->get_key(cursor, &key)) != 0 || - (ret = cursor->get_value(cursor, &value)) == 0) { - fprintf(stderr, - "%s: error in get_key/value: %s\n", - op->op, session->strerror(session, ret)); - return (ret); - } - } else /* search, update */{ - /* - * WT_CURSOR::search and WT_CURSOR::update no longer - * reference application memory; as they position the - * cursor, they will reference key/value memory owned - * by the cursor. - */ - if ((ret = cursor->get_key(cursor, &key)) != 0 || - (ret = cursor->get_value(cursor, &value)) != 0) { - fprintf(stderr, - "%s: error in get_key/value: %s\n", - op->op, session->strerror(session, ret)); - return (ret); - } - } - - /* - * Modifying the memory referenced by either key or value is - * not permitted. - * - * Check that the cursor's key and value are what we expect. - */ - if (key == keybuf || - (op->apply != cursor->remove && value == valuebuf)) { - fprintf(stderr, - "%s: cursor points at application memory!\n", - op->op); - return (EINVAL); - } - - if (strcmp(key, op->key) != 0 || - (op->apply != cursor->remove && - strcmp(value, op->value) != 0)) { - fprintf(stderr, - "%s: unexpected key / value!\n", op->op); - return (EINVAL); - } - /*! [cursor scope operation] */ - } - - return (0); -} - -int -main(void) -{ - WT_CONNECTION *conn; - WT_CURSOR *cursor; - WT_SESSION *session; - int ret; - - /* - * Create a clean test directory for this run of the test program if the - * environment variable isn't already set (as is done by make check). - */ - if (getenv("WIREDTIGER_HOME") == NULL) { - home = "WT_HOME"; - ret = system("rm -rf WT_HOME && mkdir WT_HOME"); - } else - home = NULL; - - /* Open a connection, create a simple table, open a cursor. */ - if ((ret = wiredtiger_open(home, NULL, "create", &conn)) != 0 || - (ret = conn->open_session(conn, NULL, NULL, &session)) != 0) { - fprintf(stderr, "Error connecting to %s: %s\n", - home == NULL ? "." : home, wiredtiger_strerror(ret)); - return (EXIT_FAILURE); - } - - ret = session->create(session, - "table:scope", "key_format=S,value_format=S,columns=(k,v)"); - - ret = session->open_cursor(session, - "table:scope", NULL, NULL, &cursor); - - ret = cursor_scope_ops(cursor); - - /* Close the connection and clean up. */ - ret = conn->close(conn, NULL); - - return (ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE); -} diff --git a/src/third_party/wiredtiger/examples/c/ex_sync.c b/src/third_party/wiredtiger/examples/c/ex_sync.c index 2c610b1e570..b2d74b52f7f 100644 --- a/src/third_party/wiredtiger/examples/c/ex_sync.c +++ b/src/third_party/wiredtiger/examples/c/ex_sync.c @@ -59,8 +59,8 @@ main(void) char cmd_buf[256], k[16], v[16]; const char *conf; - snprintf(cmd_buf, sizeof(cmd_buf), "rm -rf %s && mkdir %s", - home, home); + (void)snprintf(cmd_buf, sizeof(cmd_buf), + "rm -rf %s && mkdir %s", home, home); if ((ret = system(cmd_buf)) != 0) { fprintf(stderr, "%s: failed ret %d\n", cmd_buf, ret); return (EXIT_FAILURE); @@ -98,8 +98,8 @@ main(void) ret = session->commit_transaction(session, conf); ret = session->begin_transaction(session, NULL); } - snprintf(k, sizeof(k), "key%d", i); - snprintf(v, sizeof(v), "value%d", i); + (void)snprintf(k, sizeof(k), "key%d", i); + (void)snprintf(v, sizeof(v), "value%d", i); cursor->set_key(cursor, k); cursor->set_value(cursor, v); ret = cursor->insert(cursor); @@ -113,8 +113,8 @@ main(void) * Perform some operations within a single transaction. */ for (i = MAX_KEYS; i < MAX_KEYS+5; i++, record_count++) { - snprintf(k, sizeof(k), "key%d", i); - snprintf(v, sizeof(v), "value%d", i); + (void)snprintf(k, sizeof(k), "key%d", i); + (void)snprintf(v, sizeof(v), "value%d", i); cursor->set_key(cursor, k); cursor->set_value(cursor, v); ret = cursor->insert(cursor); @@ -129,8 +129,8 @@ main(void) * Demonstrate using log_flush to force the log to disk. */ for (i = 0; i < MAX_KEYS; i++, record_count++) { - snprintf(k, sizeof(k), "key%d", record_count); - snprintf(v, sizeof(v), "value%d", record_count); + (void)snprintf(k, sizeof(k), "key%d", record_count); + (void)snprintf(v, sizeof(v), "value%d", record_count); cursor->set_key(cursor, k); cursor->set_value(cursor, v); ret = cursor->insert(cursor); @@ -138,8 +138,8 @@ main(void) ret = session->log_flush(session, "sync=on"); for (i = 0; i < MAX_KEYS; i++, record_count++) { - snprintf(k, sizeof(k), "key%d", record_count); - snprintf(v, sizeof(v), "value%d", record_count); + (void)snprintf(k, sizeof(k), "key%d", record_count); + (void)snprintf(v, sizeof(v), "value%d", record_count); cursor->set_key(cursor, k); cursor->set_value(cursor, v); ret = cursor->insert(cursor); diff --git a/src/third_party/wiredtiger/examples/c/ex_thread.c b/src/third_party/wiredtiger/examples/c/ex_thread.c index 7c52d3b8189..fa82bd5f113 100644 --- a/src/third_party/wiredtiger/examples/c/ex_thread.c +++ b/src/third_party/wiredtiger/examples/c/ex_thread.c @@ -30,14 +30,13 @@ * table from multiple threads. */ -#ifndef _WIN32 -#include -#endif #include #include #include -#ifdef _WIN32 +#ifndef _WIN32 +#include +#else #include "windows_shim.h" #endif diff --git a/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_log.java b/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_log.java index 03eab6b38b1..233ad1361d8 100644 --- a/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_log.java +++ b/src/third_party/wiredtiger/examples/java/com/wiredtiger/examples/ex_log.java @@ -35,7 +35,7 @@ import java.util.*; class Lsn { int file; - long offset; + int offset; } public class ex_log { @@ -57,7 +57,7 @@ public class ex_log { conn = wiredtiger.open(home2, CONN_CONFIG); Session session = conn.open_session(null); - session.create(uri, "key_format=U,value_format=U"); + session.create(uri, "key_format=S,value_format=S"); return (session); } @@ -72,17 +72,17 @@ public class ex_log { while ((ret = cursor.next()) == 0) { ret = curs_copy.next(); - byte[] key = cursor.getKeyByteArray(); - byte[] value = cursor.getValueByteArray(); - byte[] key_copy = curs_copy.getKeyByteArray(); - byte[] value_copy = curs_copy.getValueByteArray(); - if (!Arrays.equals(key, key_copy) || - !Arrays.equals(value, value_copy)) { + String key = cursor.getKeyString(); + String value = cursor.getValueString(); + String key_copy = curs_copy.getKeyString(); + String value_copy = curs_copy.getValueString(); + if (!key.equals(key_copy) || + !value.equals(value_copy)) { System.err.println( - "Mismatched: key " + new String(key) + - ", key_copy " + new String(key_copy) + - ", value " + new String(value) + - ", value_copy " + new String(value_copy)); + "Mismatched: key " + key + ", key_copy " + key_copy + + ", value " + value + ", value_copy " + value_copy); + ret = cursor.close(); + ret = curs_copy.close(); return (1); } } @@ -121,7 +121,7 @@ public class ex_log { * A simple walk of the log. */ static int - simple_walk_log(Session session) + simple_walk_log(Session session, int count_min) throws WiredTigerException { Cursor cursor; @@ -129,16 +129,18 @@ public class ex_log { byte[] logrec_key, logrec_value; long txnid; int fileid, opcount, optype, rectype; - int ret; + int count, ret; /*! [log cursor open] */ cursor = session.open_cursor("log:", null, null); /*! [log cursor open] */ + count = 0; while ((ret = cursor.next()) == 0) { + count++; /*! [log cursor get_key] */ lsn.file = cursor.getKeyInt(); - lsn.offset = cursor.getKeyLong(); + lsn.offset = cursor.getKeyInt(); opcount = cursor.getKeyInt(); /*! [log cursor get_key] */ /*! [log cursor get_value] */ @@ -156,6 +158,11 @@ public class ex_log { if (ret == wiredtiger.WT_NOTFOUND) ret = 0; ret = cursor.close(); + if (count < count_min) { + System.err.println("Expected minimum " + count_min + + " records, found " + count); + return (1); + } return (ret); } /*! [log cursor walk] */ @@ -185,7 +192,7 @@ public class ex_log { lsnsave = new Lsn(); while ((ret = cursor.next()) == 0) { lsn.file = cursor.getKeyInt(); - lsn.offset = cursor.getKeyLong(); + lsn.offset = cursor.getKeyInt(); opcount = cursor.getKeyInt(); /* @@ -194,8 +201,10 @@ public class ex_log { * that LSN to the end (where the multi-step transaction * was performed). Just choose the record that is MAX_KEYS. */ - if (++i == MAX_KEYS) - lsnsave = lsn; + if (++i == MAX_KEYS) { + lsnsave.file = lsn.file; + lsnsave.offset = lsn.offset; + } txnid = cursor.getValueLong(); rectype = cursor.getValueInt(); optype = cursor.getValueInt(); @@ -217,10 +226,10 @@ public class ex_log { /* * If the operation is a put, replay it here on the backup - * connection. Note, we cheat by looking only for fileid 1 - * in this example. The metadata is fileid 0. + * connection. Note, we cheat by looking at the fileid. + * The metadata is fileid 0, skip its records. */ - if (fileid == 1 && rectype == wiredtiger.WT_LOGREC_COMMIT && + if (fileid != 0 && rectype == wiredtiger.WT_LOGREC_COMMIT && optype == wiredtiger.WT_LOGOP_ROW_PUT) { if (!in_txn) { ret = session2.begin_transaction(null); @@ -238,15 +247,20 @@ public class ex_log { /* * Compare the tables after replay. They should be identical. */ - if (compare_tables(session, session2) != 0) - System.out.println("compare failed"); + if (compare_tables(session, session2) != 0) { + cursor.close(); + session2.close(null); + wt_conn2.close(null); + return (ret); + } ret = session2.close(null); ret = wt_conn2.close(null); ret = cursor.reset(); /*! [log cursor set_key] */ cursor.putKeyInt(lsnsave.file); - cursor.putKeyLong(lsnsave.offset); + cursor.putKeyInt(lsnsave.offset); + cursor.putKeyInt(0); /*! [log cursor set_key] */ /*! [log cursor search] */ ret = cursor.search(); @@ -256,9 +270,9 @@ public class ex_log { * Walk all records starting with this key. */ first = true; - while (ret == 0) { /*TODO: not quite right*/ + while (ret == 0) { lsn.file = cursor.getKeyInt(); - lsn.offset = cursor.getKeyLong(); + lsn.offset = cursor.getKeyInt(); opcount = cursor.getKeyInt(); if (first) { first = false; @@ -293,8 +307,9 @@ public class ex_log { Connection wt_conn; Cursor cursor; Session session; - int i, record_count, ret; + int count_min, i, record_count, ret; + count_min = 0; try { String command = "/bin/rm -rf " + home1 + " " + home2; Process proc = Runtime.getRuntime().exec(command); @@ -317,6 +332,7 @@ public class ex_log { session = wt_conn.open_session(null); ret = session.create(uri, "key_format=S,value_format=S"); + count_min++; cursor = session.open_cursor(uri, null, null); /* @@ -328,6 +344,7 @@ public class ex_log { cursor.putKeyString(k); cursor.putValueString(v); ret = cursor.insert(); + count_min++; } ret = session.begin_transaction(null); /* @@ -341,10 +358,12 @@ public class ex_log { ret = cursor.insert(); } ret = session.commit_transaction(null); + count_min++; ret = cursor.close(); /*! [log cursor printf] */ ret = session.log_printf("Wrote " + record_count + " records"); + count_min++; /*! [log cursor printf] */ session.close(null); @@ -360,7 +379,7 @@ public class ex_log { } session = wt_conn.open_session(null); - ret = simple_walk_log(session); + ret = simple_walk_log(session, count_min); ret = walk_log(session); ret = session.close(null); ret = wt_conn.close(null); diff --git a/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c b/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c index 3665ec48b9a..09a793646e7 100644 --- a/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/zlib/zlib_compress.c @@ -483,8 +483,7 @@ static int zlib_init_config( WT_CONNECTION *connection, WT_CONFIG_ARG *config, int *zlib_levelp) { - WT_CONFIG_ITEM k, v; - WT_CONFIG_PARSER *config_parser; + WT_CONFIG_ITEM v; WT_EXTENSION_API *wt_api; int ret, zlib_level; @@ -497,49 +496,27 @@ zlib_init_config( * level; review the configuration. */ wt_api = connection->get_extension_api(connection); - if ((ret = - wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_EXTENSION_API.config_get: zlib configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - if ((ret = wt_api->config_parser_open( - wt_api, NULL, v.str, v.len, &config_parser)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_EXTENSION_API.config_parser_open: zlib configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - while ((ret = config_parser->next(config_parser, &k, &v)) == 0) - if (strlen("compression_level") == k.len && - strncmp("compression_level", k.str, k.len) == 0) { - /* - * Between 0-9: level: see zlib manual. - */ - zlib_level = (int)v.val; - if (zlib_level < 0 || zlib_level > 9) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.next: zlib configure: " - "unsupported compression level %d", - zlib_level); - return (EINVAL); - } - *zlib_levelp = zlib_level; - continue; + if ((ret = wt_api->config_get( + wt_api, NULL, config, "compression_level", &v)) == 0) { + /* + * Between 0-9: level: see zlib manual. + */ + zlib_level = (int)v.val; + if (zlib_level < 0 || zlib_level > 9) { + (void)wt_api->err_printf(wt_api, NULL, + "zlib_init_config: " + "unsupported compression level %d", + zlib_level); + return (EINVAL); } - if (ret != WT_NOTFOUND) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.next: zlib configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - if ((ret = config_parser->close(config_parser)) != 0) { + *zlib_levelp = zlib_level; + } else if (ret != WT_NOTFOUND) { (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.close: zlib configure: %s", + "zlib_init_config: %s", wt_api->strerror(wt_api, NULL, ret)); return (ret); } + return (0); } diff --git a/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c b/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c index a459b01d60a..ea8ec97602f 100644 --- a/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c +++ b/src/third_party/wiredtiger/ext/compressors/zstd/zstd_compress.c @@ -232,8 +232,7 @@ static int zstd_init_config( WT_CONNECTION *connection, WT_CONFIG_ARG *config, int *compression_levelp) { - WT_CONFIG_ITEM k, v; - WT_CONFIG_PARSER *config_parser; + WT_CONFIG_ITEM v; WT_EXTENSION_API *wt_api; int ret; @@ -246,38 +245,16 @@ zstd_init_config( * level; review the configuration. */ wt_api = connection->get_extension_api(connection); - if ((ret = - wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_EXTENSION_API.config_get: zstd configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - if ((ret = wt_api->config_parser_open( - wt_api, NULL, v.str, v.len, &config_parser)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_EXTENSION_API.config_parser_open: zstd configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - while ((ret = config_parser->next(config_parser, &k, &v)) == 0) - if (strlen("compression_level") == k.len && - strncmp("compression_level", k.str, k.len) == 0) { - *compression_levelp = (int)v.val; - continue; - } - if (ret != WT_NOTFOUND) { + if ((ret = wt_api->config_get( + wt_api, NULL, config, "compression_level", &v)) == 0) + *compression_levelp = (int)v.val; + else if (ret != WT_NOTFOUND) { (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.next: zstd configure: %s", - wt_api->strerror(wt_api, NULL, ret)); - return (ret); - } - if ((ret = config_parser->close(config_parser)) != 0) { - (void)wt_api->err_printf(wt_api, NULL, - "WT_CONFIG_PARSER.close: zstd configure: %s", + "zstd_init_config: %s", wt_api->strerror(wt_api, NULL, ret)); return (ret); } + return (0); } diff --git a/src/third_party/wiredtiger/ext/datasources/helium/helium.c b/src/third_party/wiredtiger/ext/datasources/helium/helium.c index 473c569f0cc..c584141b00d 100644 --- a/src/third_party/wiredtiger/ext/datasources/helium/helium.c +++ b/src/third_party/wiredtiger/ext/datasources/helium/helium.c @@ -3329,6 +3329,7 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) * compile-time should the structure change underneath us. */ static const WT_DATA_SOURCE wtds = { + NULL, /* No session.alter */ helium_session_create, /* session.create */ NULL, /* No session.compaction */ helium_session_drop, /* session.drop */ @@ -3380,15 +3381,9 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) goto err; ds->lockinit = 1; - /* Get the configuration string. */ - if ((ret = wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) - EMSG_ERR(wt_api, NULL, ret, - "WT_EXTENSION_API.config_get: config: %s", - wt_api->strerror(wt_api, NULL, ret)); - /* Step through the list of Helium sources, opening each one. */ - if ((ret = wt_api->config_parser_open( - wt_api, NULL, v.str, v.len, &config_parser)) != 0) + if ((ret = wt_api->config_parser_open_arg( + wt_api, NULL, config, &config_parser)) != 0) EMSG_ERR(wt_api, NULL, ret, "WT_EXTENSION_API.config_parser_open: config: %s", wt_api->strerror(wt_api, NULL, ret)); diff --git a/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c b/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c index 559c8e6e33a..0b905a0540d 100644 --- a/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c +++ b/src/third_party/wiredtiger/ext/encryptors/rotn/rotn_encrypt.c @@ -76,7 +76,7 @@ typedef struct { u_char *shift_forw; /* Encrypt shift data from secretkey */ u_char *shift_back; /* Decrypt shift data from secretkey */ size_t shift_len; /* Length of shift* byte arrays */ - int force_error; /* Force a decrypt error for testing */ + bool force_error; /* Force a decrypt error for testing */ } ROTN_ENCRYPTOR; /*! [WT_ENCRYPTOR initialization structure] */ @@ -429,43 +429,19 @@ rotn_terminate(WT_ENCRYPTOR *encryptor, WT_SESSION *session) static int rotn_configure(ROTN_ENCRYPTOR *rotn_encryptor, WT_CONFIG_ARG *config) { - WT_CONFIG_ITEM k, v; - WT_CONFIG_PARSER *config_parser; + WT_CONFIG_ITEM v; WT_EXTENSION_API *wt_api; /* Extension API */ - int ret, t_ret; + int ret; wt_api = rotn_encryptor->wt_api; /* Get the configuration string. */ - if ((ret = wt_api->config_get(wt_api, NULL, config, "config", &v)) != 0) - return (rotn_error(rotn_encryptor, NULL, ret, - "WT_EXTENSION_API.config_get")); - - /* Step through the list of configuration options. */ - if ((ret = wt_api->config_parser_open( - wt_api, NULL, v.str, v.len, &config_parser)) != 0) - return (rotn_error(rotn_encryptor, NULL, ret, - "WT_EXTENSION_API.config_parser_open")); - - while ((ret = config_parser->next(config_parser, &k, &v)) == 0) { - if (strncmp("rotn_force_error", k.str, k.len) == 0 && - strlen("rotn_force_error") == k.len) { - rotn_encryptor->force_error = v.val == 0 ? 0 : 1; - continue; - } else { - if ((ret = config_parser->close(config_parser)) != 0) - return (rotn_error(rotn_encryptor, - NULL, ret, "WT_CONFIG_PARSER.close")); - return (rotn_error(rotn_encryptor, NULL, EINVAL, - "unknown config key")); - } - } - if ((t_ret = config_parser->close(config_parser)) != 0) - return (rotn_error(rotn_encryptor, NULL, t_ret, - "WT_CONFIG_PARSER.close")); - if (ret != WT_NOTFOUND) - return (rotn_error(rotn_encryptor, NULL, ret, - "WT_CONFIG_PARSER.next")); + if ((ret = wt_api->config_get( + wt_api, NULL, config, "rotn_force_error", &v)) == 0) + rotn_encryptor->force_error = v.val != 0; + else if (ret != WT_NOTFOUND) + return (rotn_error(rotn_encryptor, NULL, EINVAL, + "error parsing config")); return (0); } diff --git a/src/third_party/wiredtiger/ext/test/fail_fs/Makefile.am b/src/third_party/wiredtiger/ext/test/fail_fs/Makefile.am new file mode 100644 index 00000000000..f31f5395cd1 --- /dev/null +++ b/src/third_party/wiredtiger/ext/test/fail_fs/Makefile.am @@ -0,0 +1,9 @@ +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include + +noinst_LTLIBRARIES = libwiredtiger_fail_fs.la +libwiredtiger_fail_fs_la_SOURCES = fail_fs.c + +# libtool hack: noinst_LTLIBRARIES turns off building shared libraries as well +# as installation, it will only build static libraries. As far as I can tell, +# the "approved" libtool way to turn them back on is by adding -rpath. +libwiredtiger_fail_fs_la_LDFLAGS = -avoid-version -module -rpath /nowhere diff --git a/src/third_party/wiredtiger/ext/test/fail_fs/fail_fs.c b/src/third_party/wiredtiger/ext/test/fail_fs/fail_fs.c new file mode 100644 index 00000000000..d0d8a14c8c2 --- /dev/null +++ b/src/third_party/wiredtiger/ext/test/fail_fs/fail_fs.c @@ -0,0 +1,847 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "queue.h" + +#define FAIL_FS_GIGABYTE (1024 * 1024 * 1024) + +#define FAIL_FS_ENV_ENABLE "WT_FAIL_FS_ENABLE" +#define FAIL_FS_ENV_WRITE_ALLOW "WT_FAIL_FS_WRITE_ALLOW" +#define FAIL_FS_ENV_READ_ALLOW "WT_FAIL_FS_READ_ALLOW" + +/* + * A "fail file system", that is, a file system extension that fails when we + * want it to. This is only used in test frameworks, this fact allows us to + * simplify some error paths. This code is not portable to Windows, as it has + * direct knowledge of file descriptors, environment variables and stack + * traces. + * + * When the filesystem extension is configured, parameters can set how many + * reads or writes can be allowed before failure. If this is not fine-grained + * enough, an 'environment' configuration parameter can be specified. If that + * is used, then on every file system read or write, environment variables are + * checked that control when reading or writing should fail. + */ +typedef struct { + WT_FILE_SYSTEM iface; + /* + * WiredTiger performs schema and I/O operations in parallel, all file + * system and file handle access must be thread-safe. This extension + * uses a single, global file system lock. + */ + pthread_rwlock_t lock; /* Lock */ + bool fail_enabled; + bool use_environment; + bool verbose; + int64_t read_ops; + int64_t write_ops; + int64_t allow_reads; + int64_t allow_writes; + /* Queue of file handles */ + TAILQ_HEAD(fail_file_handle_qh, fail_file_handle) fileq; + WT_EXTENSION_API *wtext; /* Extension functions */ +} FAIL_FILE_SYSTEM; + +typedef struct fail_file_handle { + WT_FILE_HANDLE iface; + + /* + * Track the system file descriptor for each file. + */ + FAIL_FILE_SYSTEM *fail_fs; /* Enclosing file system */ + TAILQ_ENTRY(fail_file_handle) q; /* Queue of handles */ + int fd; /* System file descriptor */ +} FAIL_FILE_HANDLE; + +static int fail_file_close(WT_FILE_HANDLE *, WT_SESSION *); +static void fail_file_handle_remove(WT_SESSION *, FAIL_FILE_HANDLE *); +static int fail_file_lock(WT_FILE_HANDLE *, WT_SESSION *, bool); +static int fail_file_read( + WT_FILE_HANDLE *, WT_SESSION *, wt_off_t, size_t, void *); +static int fail_file_size(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t *); +static int fail_file_sync(WT_FILE_HANDLE *, WT_SESSION *); +static int fail_file_truncate(WT_FILE_HANDLE *, WT_SESSION *, wt_off_t); +static int fail_file_write( + WT_FILE_HANDLE *, WT_SESSION *, wt_off_t, size_t, const void *); +static bool fail_fs_arg( + const char *, WT_CONFIG_ITEM *, WT_CONFIG_ITEM *, int64_t *); +static int fail_fs_directory_list(WT_FILE_SYSTEM *, WT_SESSION *, + const char *, const char *, char ***, uint32_t *); +static int fail_fs_directory_list_free( + WT_FILE_SYSTEM *, WT_SESSION *, char **, uint32_t); +static void fail_fs_env(const char *, int64_t *); +static int fail_fs_exist(WT_FILE_SYSTEM *, WT_SESSION *, const char *, bool *); +static int fail_fs_open(WT_FILE_SYSTEM *, WT_SESSION *, + const char *, WT_FS_OPEN_FILE_TYPE, uint32_t, WT_FILE_HANDLE **); +static int fail_fs_remove( + WT_FILE_SYSTEM *, WT_SESSION *, const char *, uint32_t); +static int fail_fs_rename( + WT_FILE_SYSTEM *, WT_SESSION *, const char *, const char *, uint32_t); +static int fail_fs_simulate_fail( + FAIL_FILE_HANDLE *, WT_SESSION *, int64_t, const char *); +static int fail_fs_size( + WT_FILE_SYSTEM *, WT_SESSION *, const char *, wt_off_t *); +static int fail_fs_terminate(WT_FILE_SYSTEM *, WT_SESSION *); + +/* + * We use pthread functions for portable locking. + * Assert on errors for simplicity. + */ +static void +fail_fs_allocate_lock(pthread_rwlock_t *lockp) +{ + assert(pthread_rwlock_init(lockp, NULL) == 0); +} + +static void +fail_fs_destroy_lock(pthread_rwlock_t *lockp) +{ + assert(pthread_rwlock_destroy(lockp) == 0); +} + +static void +fail_fs_lock(pthread_rwlock_t *lockp) +{ + assert(pthread_rwlock_wrlock(lockp) == 0); +} + +static void +fail_fs_unlock(pthread_rwlock_t *lockp) +{ + assert(pthread_rwlock_unlock(lockp) == 0); +} + +/* + * fail_file_close -- + * ANSI C close. + */ +static int +fail_file_close(WT_FILE_HANDLE *file_handle, WT_SESSION *session) +{ + FAIL_FILE_HANDLE *fail_fh; + FAIL_FILE_SYSTEM *fail_fs; + int ret; + + (void)session; /* Unused */ + + fail_fh = (FAIL_FILE_HANDLE *)file_handle; + fail_fs = fail_fh->fail_fs; + + /* + * We don't actually open an fd when opening directories for flushing, + * so ignore that case here. + */ + if (fail_fh->fd < 0) + return (0); + ret = close(fail_fh->fd); + fail_fh->fd = -1; + fail_fs_lock(&fail_fs->lock); + fail_file_handle_remove(session, fail_fh); + fail_fs_unlock(&fail_fs->lock); + return (ret); +} + +/* + * fail_file_handle_remove -- + * Destroy an in-memory file handle. Should only happen on remove or + * shutdown. The file system lock must be held during this call. + */ +static void +fail_file_handle_remove(WT_SESSION *session, FAIL_FILE_HANDLE *fail_fh) +{ + FAIL_FILE_SYSTEM *fail_fs; + + (void)session; /* Unused */ + fail_fs = fail_fh->fail_fs; + + TAILQ_REMOVE(&fail_fs->fileq, fail_fh, q); + + free(fail_fh->iface.name); + free(fail_fh); +} + +/* + * fail_file_lock -- + * Lock/unlock a file. + */ +static int +fail_file_lock(WT_FILE_HANDLE *file_handle, WT_SESSION *session, bool lock) +{ + /* Locks are always granted. */ + (void)file_handle; /* Unused */ + (void)session; /* Unused */ + (void)lock; /* Unused */ + + return (0); +} + +/* + * fail_file_read -- + * POSIX pread. + */ +static int +fail_file_read(WT_FILE_HANDLE *file_handle, + WT_SESSION *session, wt_off_t offset, size_t len, void *buf) +{ + FAIL_FILE_HANDLE *fail_fh; + FAIL_FILE_SYSTEM *fail_fs; + WT_EXTENSION_API *wtext; + int64_t envint, read_ops; + int ret; + size_t chunk; + ssize_t nr; + uint8_t *addr; + + fail_fh = (FAIL_FILE_HANDLE *)file_handle; + fail_fs = fail_fh->fail_fs; + wtext = fail_fs->wtext; + read_ops = 0; + ret = 0; + + fail_fs_lock(&fail_fs->lock); + + if (fail_fs->use_environment) { + fail_fs_env(FAIL_FS_ENV_ENABLE, &envint); + if (envint != 0) { + if (!fail_fs->fail_enabled) { + fail_fs->fail_enabled = true; + fail_fs_env(FAIL_FS_ENV_READ_ALLOW, + &fail_fs->allow_reads); + fail_fs->read_ops = 0; + } + read_ops = ++fail_fs->read_ops; + } else + fail_fs->fail_enabled = false; + } else + read_ops = ++fail_fs->read_ops; + + fail_fs_unlock(&fail_fs->lock); + + if (fail_fs->fail_enabled && fail_fs->allow_reads != 0 && + read_ops % fail_fs->allow_reads == 0) + return (fail_fs_simulate_fail( + fail_fh, session, read_ops, "read")); + + /* Break reads larger than 1GB into 1GB chunks. */ + for (addr = buf; len > 0; addr += nr, len -= (size_t)nr, offset += nr) { + chunk = (len < FAIL_FS_GIGABYTE) ? len : FAIL_FS_GIGABYTE; + if ((nr = pread(fail_fh->fd, addr, chunk, offset)) <= 0) { + (void)wtext->err_printf(wtext, session, + "%s: handle-read: failed to read %" PRIuMAX + " bytes at offset %" PRIuMAX ": %s", + fail_fh->iface.name, + (uintmax_t)len, (uintmax_t)offset, + wtext->strerror(wtext, NULL, errno)); + ret = (nr == 0 ? WT_ERROR : errno); + break; + } + } + return (ret); +} + +/* + * fail_file_size -- + * Get the size of a file in bytes, by file handle. + */ +static int +fail_file_size( + WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t *sizep) +{ + FAIL_FILE_HANDLE *fail_fh; + struct stat statbuf; + int ret; + + (void)session; /* Unused */ + + fail_fh = (FAIL_FILE_HANDLE *)file_handle; + ret = 0; + + if ((ret = fstat(fail_fh->fd, &statbuf)) != 0) + return (ret); + *sizep = statbuf.st_size; + return (0); +} + +/* + * fail_file_sync -- + * Ensure the content of the file is stable. This is a no-op in our + * file system. + */ +static int +fail_file_sync(WT_FILE_HANDLE *file_handle, WT_SESSION *session) +{ + (void)file_handle; /* Unused */ + (void)session; /* Unused */ + + return (0); +} + +/* + * fail_file_truncate -- + * POSIX ftruncate. + */ +static int +fail_file_truncate( + WT_FILE_HANDLE *file_handle, WT_SESSION *session, wt_off_t offset) +{ + FAIL_FILE_HANDLE *fail_fh; + + (void)session; /* Unused */ + + fail_fh = (FAIL_FILE_HANDLE *)file_handle; + return (ftruncate(fail_fh->fd, offset)); +} + +/* + * fail_file_write -- + * POSIX pwrite. + */ +static int +fail_file_write(WT_FILE_HANDLE *file_handle, WT_SESSION *session, + wt_off_t offset, size_t len, const void *buf) +{ + FAIL_FILE_HANDLE *fail_fh; + FAIL_FILE_SYSTEM *fail_fs; + WT_EXTENSION_API *wtext; + int64_t envint, write_ops; + int ret; + size_t chunk; + ssize_t nr; + const uint8_t *addr; + + fail_fh = (FAIL_FILE_HANDLE *)file_handle; + fail_fs = fail_fh->fail_fs; + wtext = fail_fs->wtext; + write_ops = 0; + ret = 0; + + fail_fs_lock(&fail_fs->lock); + + if (fail_fs->use_environment) { + fail_fs_env(FAIL_FS_ENV_ENABLE, &envint); + if (envint != 0) { + if (!fail_fs->fail_enabled) { + fail_fs->fail_enabled = true; + fail_fs_env(FAIL_FS_ENV_WRITE_ALLOW, + &fail_fs->allow_writes); + fail_fs->write_ops = 0; + } + write_ops = ++fail_fs->write_ops; + } else + fail_fs->fail_enabled = false; + } else + write_ops = ++fail_fs->write_ops; + + fail_fs_unlock(&fail_fs->lock); + + if (fail_fs->fail_enabled && fail_fs->allow_writes != 0 && + write_ops % fail_fs->allow_writes == 0) + return (fail_fs_simulate_fail( + fail_fh, session, write_ops, "write")); + + /* Break writes larger than 1GB into 1GB chunks. */ + for (addr = buf; len > 0; addr += nr, len -= (size_t)nr, offset += nr) { + chunk = (len < FAIL_FS_GIGABYTE) ? len : FAIL_FS_GIGABYTE; + if ((nr = pwrite(fail_fh->fd, addr, chunk, offset)) <= 0) { + (void)wtext->err_printf(wtext, session, + "%s: handle-write: failed to write %" PRIuMAX + " bytes at offset %" PRIuMAX ": %s", + fail_fh->iface.name, + (uintmax_t)len, (uintmax_t)offset, + wtext->strerror(wtext, NULL, errno)); + ret = (nr == 0 ? WT_ERROR : errno); + break; + } + } + return (ret); +} + +/* + * fail_fs_arg -- + * If the key matches, return the value interpreted as an integer. + */ +static bool +fail_fs_arg(const char *match, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value, + int64_t *argp) +{ + if (strncmp(match, key->str, key->len) == 0 && + match[key->len] == '\0' && + (value->type == WT_CONFIG_ITEM_BOOL || + value->type == WT_CONFIG_ITEM_NUM)) { + *argp = value->val; + return (true); + } + return (false); +} + +/* + * fail_fs_directory_list -- + * Return a list of files in a given sub-directory. + */ +static int +fail_fs_directory_list(WT_FILE_SYSTEM *file_system, + WT_SESSION *session, const char *directory, + const char *prefix, char ***dirlistp, uint32_t *countp) +{ + FAIL_FILE_HANDLE *fail_fh; + FAIL_FILE_SYSTEM *fail_fs; + size_t len, prefix_len; + uint32_t allocated, count; + int ret; + char *name, **entries; + void *p; + + (void)session; /* Unused */ + + fail_fs = (FAIL_FILE_SYSTEM *)file_system; + ret = 0; + *dirlistp = NULL; + *countp = 0; + + entries = NULL; + allocated = count = 0; + len = strlen(directory); + prefix_len = prefix == NULL ? 0 : strlen(prefix); + + fail_fs_lock(&fail_fs->lock); + TAILQ_FOREACH(fail_fh, &fail_fs->fileq, q) { + name = fail_fh->iface.name; + if (strncmp(name, directory, len) != 0 || + (prefix != NULL && strncmp(name, prefix, prefix_len) != 0)) + continue; + + /* + * Increase the list size in groups of 10, it doesn't + * matter if the list is a bit longer than necessary. + */ + if (count >= allocated) { + p = realloc( + entries, (allocated + 10) * sizeof(*entries)); + if (p == NULL) { + ret = ENOMEM; + goto err; + } + entries = p; + memset(entries + allocated * sizeof(*entries), + 0, 10 * sizeof(*entries)); + allocated += 10; + } + entries[count++] = strdup(name); + } + + *dirlistp = entries; + *countp = count; + +err: fail_fs_unlock(&fail_fs->lock); + if (ret == 0) + return (0); + + if (entries != NULL) { + while (count > 0) + free(entries[--count]); + free(entries); + } + + return (ret); +} + +/* + * fail_fs_directory_list_free -- + * Free memory allocated by fail_fs_directory_list. + */ +static int +fail_fs_directory_list_free(WT_FILE_SYSTEM *file_system, + WT_SESSION *session, char **dirlist, uint32_t count) +{ + (void)file_system; /* Unused */ + (void)session; /* Unused */ + + if (dirlist != NULL) { + while (count > 0) + free(dirlist[--count]); + free(dirlist); + } + return (0); +} + +/* + * fail_fs_env -- + * If the name is in the environment, return its integral value. + */ +static void +fail_fs_env(const char *name, int64_t *valp) +{ + int64_t result; + char *s, *value; + + result = 0; + if ((value = getenv(name)) != NULL) { + s = value; + if (strcmp(value, "true") == 0) + result = 1; + else if (strcmp(value, "false") != 0) { + result = strtoll(value, &s, 10); + if (*s != '\0') + result = 0; + } + } + *valp = result; +} + +/* + * fail_fs_exist -- + * Return if the file exists. + */ +static int +fail_fs_exist(WT_FILE_SYSTEM *file_system, + WT_SESSION *session, const char *name, bool *existp) +{ + (void)file_system; /* Unused */ + (void)session; /* Unused */ + + *existp = (access(name, F_OK) == 0); + return (0); +} + +/* + * fail_fs_open -- + * fopen for the fail file system. + */ +static int +fail_fs_open(WT_FILE_SYSTEM *file_system, WT_SESSION *session, + const char *name, WT_FS_OPEN_FILE_TYPE file_type, uint32_t flags, + WT_FILE_HANDLE **file_handlep) +{ + FAIL_FILE_HANDLE *fail_fh; + FAIL_FILE_SYSTEM *fail_fs; + WT_EXTENSION_API *wtext; + WT_FILE_HANDLE *file_handle; + int fd, open_flags, ret; + + (void)session; /* Unused */ + + *file_handlep = NULL; + + fail_fh = NULL; + fail_fs = (FAIL_FILE_SYSTEM *)file_system; + fd = -1; + ret = 0; + + if (fail_fs->verbose) { + wtext = fail_fs->wtext; + (void)wtext->msg_printf(wtext, session, "fail_fs: open: %s", + name); + } + + fail_fs_lock(&fail_fs->lock); + + open_flags = 0; + if ((flags & WT_FS_OPEN_CREATE) != 0) + open_flags |= O_CREAT; + if ((flags & WT_FS_OPEN_EXCLUSIVE) != 0) + open_flags |= O_EXCL; + if ((flags & WT_FS_OPEN_READONLY) != 0) + open_flags |= O_RDONLY; + else + open_flags |= O_RDWR; + + /* + * Opening a file handle on a directory is only to support filesystems + * that require a directory sync for durability. This is a no-op + * for this file system. + */ + if (file_type == WT_FS_OPEN_FILE_TYPE_DIRECTORY) + fd = -1; + else if ((fd = open(name, open_flags, 0666)) < 0) { + ret = errno; + goto err; + } + + /* We create a handle structure for each open. */ + if ((fail_fh = calloc(1, sizeof(FAIL_FILE_HANDLE))) == NULL) { + ret = ENOMEM; + goto err; + } + + /* Initialize private information. */ + fail_fh->fail_fs = fail_fs; + fail_fh->fd = fd; + + /* Initialize public information. */ + file_handle = (WT_FILE_HANDLE *)fail_fh; + if ((file_handle->name = strdup(name)) == NULL) { + ret = ENOMEM; + goto err; + } + + /* Setup the function call table. */ + file_handle->close = fail_file_close; + file_handle->fh_advise = NULL; + file_handle->fh_extend = NULL; + file_handle->fh_extend_nolock = NULL; + file_handle->fh_lock = fail_file_lock; + file_handle->fh_map = NULL; + file_handle->fh_map_discard = NULL; + file_handle->fh_map_preload = NULL; + file_handle->fh_unmap = NULL; + file_handle->fh_read = fail_file_read; + file_handle->fh_size = fail_file_size; + file_handle->fh_sync = fail_file_sync; + file_handle->fh_sync_nowait = NULL; + file_handle->fh_truncate = fail_file_truncate; + file_handle->fh_write = fail_file_write; + + TAILQ_INSERT_HEAD(&fail_fs->fileq, fail_fh, q); + + *file_handlep = file_handle; + + if (0) { +err: if (fd != -1) + (void)close(fd); + free(fail_fh); + } + + fail_fs_unlock(&fail_fs->lock); + return (ret); +} + +/* + * fail_fs_remove -- + * POSIX remove. + */ +static int +fail_fs_remove(WT_FILE_SYSTEM *file_system, + WT_SESSION *session, const char *name, uint32_t flags) +{ + (void)file_system; /* Unused */ + (void)session; /* Unused */ + (void)flags; /* Unused */ + + return (unlink(name)); +} + +/* + * fail_fs_rename -- + * POSIX rename. + */ +static int +fail_fs_rename(WT_FILE_SYSTEM *file_system, + WT_SESSION *session, const char *from, const char *to, uint32_t flags) +{ + (void)file_system; /* Unused */ + (void)session; /* Unused */ + (void)flags; /* Unused */ + + return (rename(from, to)); +} + +/* + * fail_fs_simulate_fail -- + * Simulate a failure from this file system by reporting it + * and returning a non-zero return code. + */ +static int +fail_fs_simulate_fail(FAIL_FILE_HANDLE *fail_fh, WT_SESSION *session, + int64_t nops, const char *opkind) +{ + FAIL_FILE_SYSTEM *fail_fs; + WT_EXTENSION_API *wtext; +#ifdef __FreeBSD__ + size_t btret, i; +#else + int btret, i; +#endif + void *bt[100]; + char **btstr; + + fail_fs = fail_fh->fail_fs; + if (fail_fs->verbose) { + wtext = fail_fs->wtext; + (void)wtext->msg_printf(wtext, session, + "fail_fs: %s: simulated failure after %" PRId64 + " %s operations", fail_fh->iface.name, nops, opkind); +#ifdef __FreeBSD__ + btret = backtrace(bt, sizeof(bt) / sizeof(bt[0])); +#else + btret = backtrace(bt, (int)(sizeof(bt) / sizeof(bt[0]))); +#endif + if ((btstr = backtrace_symbols(bt, btret)) != NULL) { + for (i = 0; i < btret; i++) + (void)wtext->msg_printf(wtext, session, " %s", + btstr[i]); + free(btstr); + } + } + return (EIO); +} + +/* + * fail_fs_size -- + * Get the size of a file in bytes, by file name. + */ +static int +fail_fs_size(WT_FILE_SYSTEM *file_system, + WT_SESSION *session, const char *name, wt_off_t *sizep) +{ + struct stat statbuf; + int ret; + + (void)file_system; /* Unused */ + (void)session; /* Unused */ + + ret = 0; + if ((ret = stat(name, &statbuf)) != 0) + return (ret); + *sizep = statbuf.st_size; + return (0); +} + +/* + * fail_fs_terminate -- + * Discard any resources on termination + */ +static int +fail_fs_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *session) +{ + FAIL_FILE_HANDLE *fail_fh; + FAIL_FILE_SYSTEM *fail_fs; + + fail_fs = (FAIL_FILE_SYSTEM *)file_system; + + while ((fail_fh = TAILQ_FIRST(&fail_fs->fileq)) != NULL) + fail_file_handle_remove(session, fail_fh); + + fail_fs_destroy_lock(&fail_fs->lock); + free(fail_fs); + + return (0); +} + +/* + * wiredtiger_extension_init -- + * WiredTiger fail filesystem extension. + */ +int +wiredtiger_extension_init(WT_CONNECTION *conn, WT_CONFIG_ARG *config) +{ + FAIL_FILE_SYSTEM *fail_fs; + WT_CONFIG_ITEM k, v; + WT_CONFIG_PARSER *config_parser; + WT_EXTENSION_API *wtext; + WT_FILE_SYSTEM *file_system; + int64_t argval; + int ret; + + ret = 0; + wtext = conn->get_extension_api(conn); + if ((fail_fs = calloc(1, sizeof(FAIL_FILE_SYSTEM))) == NULL) { + (void)wtext->err_printf(wtext, NULL, + "fail_file_system extension_init: %s", + wtext->strerror(wtext, NULL, ENOMEM)); + return (ENOMEM); + } + fail_fs->wtext = wtext; + file_system = (WT_FILE_SYSTEM *)fail_fs; + + /* Get any configuration values. */ + if ((ret = wtext->config_parser_open_arg( + wtext, NULL, config, &config_parser)) != 0) { + (void)wtext->err_printf(wtext, NULL, + "WT_EXTENSION_API.config_parser_open: config: %s", + wtext->strerror(wtext, NULL, ret)); + goto err; + } + while ((ret = config_parser->next(config_parser, &k, &v)) == 0) { + if (fail_fs_arg("environment", &k, &v, &argval)) { + fail_fs->use_environment = (argval != 0); + continue; + } else if (fail_fs_arg("verbose", &k, &v, &argval)) { + fail_fs->verbose = (argval != 0); + continue; + } else if (fail_fs_arg("allow_writes", &k, &v, + &fail_fs->allow_writes)) + continue; + else if (fail_fs_arg("allow_reads", &k, &v, + &fail_fs->allow_reads)) + continue; + + (void)wtext->err_printf(wtext, NULL, + "WT_CONFIG_PARSER.next: unexpected configuration " + "information: %.*s=%.*s: %s", + (int)k.len, k.str, (int)v.len, v.str, + wtext->strerror(wtext, NULL, ret)); + goto err; + } + if (ret != WT_NOTFOUND) { + (void)wtext->err_printf(wtext, NULL, + "WT_CONFIG_PARSER.next: config: %s", + wtext->strerror(wtext, NULL, ret)); + goto err; + } + if ((ret = config_parser->close(config_parser)) != 0) { + (void)wtext->err_printf(wtext, NULL, + "WT_CONFIG_PARSER.close: config: %s", + wtext->strerror(wtext, NULL, ret)); + goto err; + } + if (fail_fs->allow_writes != 0 || fail_fs->allow_reads != 0) + fail_fs->fail_enabled = true; + + fail_fs_allocate_lock(&fail_fs->lock); + /* Initialize the in-memory jump table. */ + file_system->fs_directory_list = fail_fs_directory_list; + file_system->fs_directory_list_free = fail_fs_directory_list_free; + file_system->fs_exist = fail_fs_exist; + file_system->fs_open_file = fail_fs_open; + file_system->fs_remove = fail_fs_remove; + file_system->fs_rename = fail_fs_rename; + file_system->fs_size = fail_fs_size; + file_system->terminate = fail_fs_terminate; + if ((ret = conn->set_file_system(conn, file_system, NULL)) != 0) { + (void)wtext->err_printf(wtext, NULL, + "WT_CONNECTION.set_file_system: %s", + wtext->strerror(wtext, NULL, ret)); + goto err; + } + return (0); + +err: free(fail_fs); + return (ret); +} diff --git a/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c b/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c index 866cd0663ce..0791b077750 100644 --- a/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c +++ b/src/third_party/wiredtiger/ext/test/kvs_bdb/kvs_bdb.c @@ -1016,9 +1016,10 @@ wiredtiger_extension_init(WT_CONNECTION *connection, WT_CONFIG_ARG *config) { /* * List of the WT_DATA_SOURCE methods -- it's static so it breaks at - * compile-time should the structure changes underneath us. + * compile-time should the structure change underneath us. */ static WT_DATA_SOURCE wtds = { + NULL, /* No session.alter */ kvs_session_create, /* session.create */ NULL, /* No session.compaction */ kvs_session_drop, /* session.drop */ diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data new file mode 100644 index 00000000000..abf3fe5cb9c --- /dev/null +++ b/src/third_party/wiredtiger/import.data @@ -0,0 +1,6 @@ +{ + "commit": "b8f590dea0400666ef26e21adf11c5997bb5ef1b", + "github": "wiredtiger/wiredtiger.git", + "vendor": "wiredtiger", + "branch": "mongodb-3.2" +} diff --git a/src/third_party/wiredtiger/lang/java/Makefile.am b/src/third_party/wiredtiger/lang/java/Makefile.am index 7184fe610dc..2ff822a5d08 100644 --- a/src/third_party/wiredtiger/lang/java/Makefile.am +++ b/src/third_party/wiredtiger/lang/java/Makefile.am @@ -49,6 +49,7 @@ JAVA_JUNIT = \ $(JAVATEST)/ConcurrentCloseTest.java \ $(JAVATEST)/CursorTest.java \ $(JAVATEST)/CursorTest02.java \ + $(JAVATEST)/CursorTest03.java \ $(JAVATEST)/ExceptionTest.java \ $(JAVATEST)/PackTest.java \ $(JAVATEST)/PackTest02.java \ diff --git a/src/third_party/wiredtiger/lang/java/java_doc.i b/src/third_party/wiredtiger/lang/java/java_doc.i index 2264cb31ef1..3606bed1d69 100644 --- a/src/third_party/wiredtiger/lang/java/java_doc.i +++ b/src/third_party/wiredtiger/lang/java/java_doc.i @@ -27,6 +27,7 @@ COPYDOC(__wt_async_op, WT_ASYNC_OP, remove) COPYDOC(__wt_async_op, WT_ASYNC_OP, compact) COPYDOC(__wt_async_op, WT_ASYNC_OP, get_id) COPYDOC(__wt_async_op, WT_ASYNC_OP, get_type) +COPYDOC(__wt_session, WT_SESSION, alter) COPYDOC(__wt_session, WT_SESSION, close) COPYDOC(__wt_session, WT_SESSION, reconfigure) COPYDOC(__wt_session, WT_SESSION, open_cursor) diff --git a/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackOutputStream.java b/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackOutputStream.java index 46b3aef0974..b6804a2992f 100644 --- a/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackOutputStream.java +++ b/src/third_party/wiredtiger/lang/java/src/com/wiredtiger/db/PackOutputStream.java @@ -42,6 +42,7 @@ public class PackOutputStream { protected PackFormatInputStream format; protected ByteArrayOutputStream packed; protected byte[] intBuf; + protected boolean isRaw; /** * Constructor. @@ -50,9 +51,21 @@ public class PackOutputStream { * defines the layout of this packed value. */ public PackOutputStream(String format) { - this.format = new PackFormatInputStream(format, false); - intBuf = new byte[MAX_INT_BYTES]; - packed = new ByteArrayOutputStream(100); + this(format, false); + } + + /** + * Constructor. + * + * \param format A String that contains the WiredTiger format that + * defines the layout of this packed value. + * \param isRaw The stream is opened raw. + */ + public PackOutputStream(String format, boolean isRaw) { + this.format = new PackFormatInputStream(format, isRaw); + this.intBuf = new byte[MAX_INT_BYTES]; + this.packed = new ByteArrayOutputStream(100); + this.isRaw = isRaw; } /** @@ -111,7 +124,9 @@ public class PackOutputStream { throws WiredTigerPackingException { int padBytes = 0; - format.checkType('U', false); + if (!isRaw) { + format.checkType('U', false); + } boolean havesize = format.hasLength(); char type = format.getType(); if (havesize) { diff --git a/src/third_party/wiredtiger/lang/java/wiredtiger.i b/src/third_party/wiredtiger/lang/java/wiredtiger.i index d6e0660dc2c..275b708090c 100644 --- a/src/third_party/wiredtiger/lang/java/wiredtiger.i +++ b/src/third_party/wiredtiger/lang/java/wiredtiger.i @@ -127,7 +127,7 @@ static void throwWiredTigerException(JNIEnv *jenv, int err) { %typemap(javain) uint64_t "$javainput" %typemap(javaout) uint64_t { - return $jnicall; + return ($jnicall); } /* Return byte[] from cursor.get_value */ @@ -137,7 +137,7 @@ static void throwWiredTigerException(JNIEnv *jenv, int err) { %typemap(javain) WT_ITEM, WT_ITEM * "$javainput" %typemap(javaout) WT_ITEM, WT_ITEM * { - return $jnicall; + return ($jnicall); } %typemap(in) WT_ITEM * (WT_ITEM item) %{ @@ -165,7 +165,7 @@ static void throwWiredTigerException(JNIEnv *jenv, int err) { %typemap(out) int %{ if ($1 != 0 && $1 != WT_NOTFOUND) { throwWiredTigerException(jenv, $1); - return $null; + return ($null); } $result = $1; %} @@ -174,7 +174,7 @@ static void throwWiredTigerException(JNIEnv *jenv, int err) { if (!val) { SWIG_JavaThrowException(jenv, SWIG_JavaNullPointerException, #name " is null"); - return $null; + return ($null); } %enddef @@ -319,6 +319,15 @@ WT_CLASS(struct __wt_async_op, WT_ASYNC_OP, op) %rename (getValueFormat) __wt_async_op::getValue_format; %rename (getType) __wt_async_op::get_type; +/* + * Special cases: override the out typemap, return checking is done in the + * wrapper. + */ +%typemap(out) int __wt_cursor::compare_wrap, + int __wt_cursor::equals_wrap %{ + $result = $1; +%} + /* SWIG magic to turn Java byte strings into data / size. */ %apply (char *STRING, int LENGTH) { (char *data, int size) }; @@ -529,7 +538,6 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; %} %extend __wt_async_op { - %javamethodmodifiers get_key_wrap "protected"; WT_ITEM get_key_wrap(JNIEnv *jenv) { WT_ITEM k; @@ -537,7 +545,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; k.data = NULL; if ((ret = $self->get_key($self, &k)) != 0) throwWiredTigerException(jenv, ret); - return k; + return (k); } %javamethodmodifiers get_value_wrap "protected"; @@ -547,40 +555,40 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; v.data = NULL; if ((ret = $self->get_value($self, &v)) != 0) throwWiredTigerException(jenv, ret); - return v; + return (v); } %javamethodmodifiers insert_wrap "protected"; int insert_wrap(WT_ITEM *k, WT_ITEM *v) { $self->set_key($self, k); $self->set_value($self, v); - return $self->insert($self); + return ($self->insert($self)); } %javamethodmodifiers remove_wrap "protected"; int remove_wrap(WT_ITEM *k) { $self->set_key($self, k); - return $self->remove($self); + return ($self->remove($self)); } %javamethodmodifiers search_wrap "protected"; int search_wrap(WT_ITEM *k) { $self->set_key($self, k); - return $self->search($self); + return ($self->search($self)); } %javamethodmodifiers update_wrap "protected"; int update_wrap(WT_ITEM *k, WT_ITEM *v) { $self->set_key($self, k); $self->set_value($self, v); - return $self->update($self); + return ($self->update($self)); } %javamethodmodifiers _java_raw "protected"; bool _java_raw(JNIEnv *jenv) { (void)jenv; JAVA_CALLBACK *jcb = (JAVA_CALLBACK *)$self->c.lang_private; - return jcb->cursor_raw; + return (jcb->cursor_raw); } %javamethodmodifiers _java_init "protected"; @@ -597,6 +605,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; %typemap(javabody) struct __wt_async_op %{ private long swigCPtr; protected boolean swigCMemOwn; + protected boolean javaRaw; protected String keyFormat; protected String valueFormat; protected PackOutputStream keyPacker; @@ -609,13 +618,14 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; swigCPtr = cPtr; keyFormat = getKey_format(); valueFormat = getValue_format(); - keyPacker = new PackOutputStream(keyFormat); - valuePacker = new PackOutputStream(valueFormat); + javaRaw = _java_raw(); + keyPacker = new PackOutputStream(keyFormat, javaRaw); + valuePacker = new PackOutputStream(valueFormat, javaRaw); wiredtigerJNI.AsyncOp__java_init(swigCPtr, this, this); } protected static long getCPtr($javaclassname obj) { - return (obj == null) ? 0 : obj.swigCPtr; + return ((obj == null) ? 0 : obj.swigCPtr); } %} @@ -1098,7 +1108,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; if (keyUnpacker == null) keyUnpacker = new PackInputStream(keyFormat, get_key_wrap(), - _java_raw()); + javaRaw); return keyUnpacker; } @@ -1112,7 +1122,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; if (valueUnpacker == null) valueUnpacker = new PackInputStream(valueFormat, get_value_wrap(), - _java_raw()); + javaRaw); return valueUnpacker; } @@ -1127,7 +1137,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; k.data = NULL; if ((ret = $self->get_key($self, &k)) != 0) throwWiredTigerException(jenv, ret); - return k; + return (k); } %javamethodmodifiers get_value_wrap "protected"; @@ -1137,20 +1147,20 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; v.data = NULL; if ((ret = $self->get_value($self, &v)) != 0) throwWiredTigerException(jenv, ret); - return v; + return (v); } %javamethodmodifiers insert_wrap "protected"; int insert_wrap(WT_ITEM *k, WT_ITEM *v) { $self->set_key($self, k); $self->set_value($self, v); - return $self->insert($self); + return ($self->insert($self)); } %javamethodmodifiers remove_wrap "protected"; int remove_wrap(WT_ITEM *k) { $self->set_key($self, k); - return $self->remove($self); + return ($self->remove($self)); } %javamethodmodifiers reset_wrap "protected"; @@ -1161,7 +1171,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; %javamethodmodifiers search_wrap "protected"; int search_wrap(WT_ITEM *k) { $self->set_key($self, k); - return $self->search($self); + return ($self->search($self)); } %javamethodmodifiers search_near_wrap "protected"; @@ -1181,7 +1191,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; int update_wrap(WT_ITEM *k, WT_ITEM *v) { $self->set_key($self, k); $self->set_value($self, v); - return $self->update($self); + return ($self->update($self)); } %javamethodmodifiers compare_wrap "protected"; @@ -1189,7 +1199,7 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; int cmp, ret = $self->compare($self, other, &cmp); if (ret != 0) throwWiredTigerException(jenv, ret); - return cmp; + return (cmp); } %javamethodmodifiers equals_wrap "protected"; @@ -1197,14 +1207,14 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; int cmp, ret = $self->equals($self, other, &cmp); if (ret != 0) throwWiredTigerException(jenv, ret); - return cmp; + return (cmp); } %javamethodmodifiers _java_raw "protected"; bool _java_raw(JNIEnv *jenv) { (void)jenv; JAVA_CALLBACK *jcb = (JAVA_CALLBACK *)$self->lang_private; - return jcb->cursor_raw; + return (jcb->cursor_raw); } %javamethodmodifiers _java_init "protected"; @@ -1232,8 +1242,8 @@ WT_ASYNC_CALLBACK javaApiAsyncHandler = {javaAsyncHandler}; swigCPtr = cPtr; keyFormat = getKey_format(); valueFormat = getValue_format(); - keyPacker = new PackOutputStream(keyFormat); - valuePacker = new PackOutputStream(valueFormat); + keyPacker = new PackOutputStream(keyFormat, _java_raw()); + valuePacker = new PackOutputStream(valueFormat, _java_raw()); wiredtigerJNI.Cursor__java_init(swigCPtr, this, this); } @@ -1953,7 +1963,7 @@ WT_CONNECTION *wiredtiger_open_wrap(JNIEnv *jenv, const char *home, const char * err: if (ret != 0) throwWiredTigerException(jenv, ret); - return conn; + return (conn); } } @@ -1983,7 +1993,7 @@ err: if (ret != 0) err: if (ret != 0) throwWiredTigerException(jenv, ret); - return asyncop; + return (asyncop); } } @@ -2007,7 +2017,7 @@ err: if (ret != 0) err: if (ret != 0) throwWiredTigerException(jenv, ret); - return session; + return (session); } } @@ -2035,14 +2045,14 @@ err: if (ret != 0) err: if (ret != 0) throwWiredTigerException(jenv, ret); - return cursor; + return (cursor); } } %extend __wt_async_op { long get_id_wrap(JNIEnv *jenv) { WT_UNUSED(jenv); - return (self->get_id(self)); + return ((long)self->get_id(self)); } } @@ -2053,6 +2063,6 @@ err: if (ret != 0) ret = self->transaction_pinned_range(self, &range); if (ret != 0) throwWiredTigerException(jenv, ret); - return range; + return ((long)range); } } diff --git a/src/third_party/wiredtiger/lang/python/Makefile.am b/src/third_party/wiredtiger/lang/python/Makefile.am index 03c65a57028..b32d0321194 100644 --- a/src/third_party/wiredtiger/lang/python/Makefile.am +++ b/src/third_party/wiredtiger/lang/python/Makefile.am @@ -17,7 +17,8 @@ install-exec-local: (cd $(PYSRC) && \ $(PYTHON) setup.py build_py -d $(abs_builddir)/build && \ $(PYTHON) setup.py build_ext -f -b $(abs_builddir)/build $(PYDIRS) && \ - $(PYTHON) setup.py install_lib -b $(abs_builddir)/build --skip-build $(PYTHON_INSTALL_ARG)) + $(PYTHON) setup.py install_lib -b $(abs_builddir)/build --skip-build $(PYTHON_INSTALL_ARG) && \ + rm -rf $(abs_builddir)/build) # We build in different places for an install vs running from the tree: # clean up both. Don't rely on "setup.py clean" -- everything that should diff --git a/src/third_party/wiredtiger/lang/python/setup.py b/src/third_party/wiredtiger/lang/python/setup.py index cf5b0c5cab1..9063a891fb9 100644 --- a/src/third_party/wiredtiger/lang/python/setup.py +++ b/src/third_party/wiredtiger/lang/python/setup.py @@ -35,8 +35,9 @@ from distutils.core import setup, Extension if not 'ARCHFLAGS' in os.environ: os.environ['ARCHFLAGS'] = '' -# Suppress warnings building SWIG generated code -extra_cflags = [ '-w', '-I../../src/include'] +# Suppress warnings building SWIG generated code. SWIG boiler plate +# functions have sign conversion warnings, so those warnings must be disabled. +extra_cflags = [ '-w', '-I../../src/include', '-Wno-sign-conversion'] dir = os.path.dirname(__file__) diff --git a/src/third_party/wiredtiger/lang/python/setup_pip.py b/src/third_party/wiredtiger/lang/python/setup_pip.py new file mode 100644 index 00000000000..636eecab80a --- /dev/null +++ b/src/third_party/wiredtiger/lang/python/setup_pip.py @@ -0,0 +1,408 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +# This script builds a Python source distribution that can built be installed +# via pip install. This must be run in a git repository to determine the files +# to package. Also as a prerequisite, SWIG must be run as the generated files +# are part of the package. To create the distribution, in this directory, run +# "python setup_pip.py sdist", this creates a tar.gz file under ./dist . +from __future__ import print_function +import os, os.path, re, shutil, site, sys +from setuptools import setup, Distribution +from distutils.extension import Extension +import distutils.sysconfig +import distutils.ccompiler +from distutils.errors import CompileError, LinkError +import subprocess +from subprocess import call +import setuptools.command.install +import setuptools.command.build_ext + +# msg -- +# Print a message to stderr. +def msg(s): + print(os.path.basename(__file__) + ": " + s, file=sys.stderr) + +# die -- +# For failures, show a message and exit. +def die(s): + msg(s) + sys.exit(1) + +# build_commands -- +# Run a sequence of commands, and die if any fail. +def build_commands(commands, build_dir, build_env): + for command in commands: + callargs = [ 'sh', '-c', command ] + verbose_command = '"' + '" "'.join(callargs) + '"' + print('running: ' + verbose_command) + if call(callargs, cwd=build_dir, env=build_env) != 0: + die('build command failed: ' + verbose_command) + +# check_needed_dependencies -- +# Make a quick check of any needed library dependencies, and +# add to the library path and include path as needed. If a library +# is not found, it is not definitive. +def check_needed_dependencies(builtins, inc_paths, lib_paths): + library_dirs = get_library_dirs() + compiler = distutils.ccompiler.new_compiler() + distutils.sysconfig.customize_compiler(compiler) + compiler.set_library_dirs(library_dirs) + missing = [] + for name, libname, instructions in builtins: + found = compiler.find_library_file(library_dirs, libname) + if found is None: + msg(libname + ": missing") + msg(instructions) + msg("after installing it, set LD_LIBRARY_PATH or DYLD_LIBRARY_PATH") + missing.append(libname) + else: + package_top = os.path.dirname(os.path.dirname(found)) + inc_paths.append(os.path.join(package_top, 'include')) + lib_paths.append(os.path.join(package_top, 'lib')) + + # XXX: we are not accounting for other directories that might be + # discoverable via /sbin/ldconfig. It might be better to write a tiny + # compile using -lsnappy, -lz... + # + #if len(missing) > 0: + # die("install packages for: " + str(missing)) + +# find_executable -- +# Locate an executable in the PATH. +def find_executable(exename, path): + p = subprocess.Popen(['which', exename ], stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = p.communicate('') + out = str(out) # needed for Python3 + if out == '': + if err != '': + err = ': "' + err + '"' + die('"' + exename + '": not found in path' + err) + dirname = os.path.dirname(out) + if not dirname in path: + path.append(dirname) + +# get_build_path -- +# Create a PATH that can be used for installation. Apparently, +# installation commands are run with a restricted PATH, and +# autoreconf/aclocal will not normally be found. +def get_build_path(): + build_paths = [] + find_executable('autoreconf', build_paths) + find_executable('aclocal', build_paths) + build_path = os.environ['PATH'] + ':' + ':'.join(build_paths) + return build_path + +# get_compile_flags -- +# Get system specific compile flags. Return a triple: C preprocessor +# flags, C compilation flags and linker flags. +def get_compile_flags(inc_paths, lib_paths): + # Suppress warnings building SWIG generated code + if sys.platform == 'win32' and cc == 'msvc': + cflags = ['/arch:SSE2', '/EHsc'] + cppflags = [] + ldflags = [] + # Windows untested and incomplete, don't claim that it works. + die('Windows is not supported by this setup script') + else: + cflags = [ '-w', '-Wno-sign-conversion', '-std=c11' ] + cppflags = ['-I' + path for path in inc_paths] + cppflags.append('-DHAVE_CONFIG_H') + ldflags = ['-L' + path for path in lib_paths] + if sys.platform == 'darwin': + cflags.extend([ '-arch', 'x86_64' ]) + return (cppflags, cflags, ldflags) + +# get_sources_curdir -- +# Get a list of sources from the current directory +def get_sources_curdir(): + DEVNULL = open(os.devnull, 'w') + gitproc = subprocess.Popen( + ['git', 'ls-tree', '-r', '--name-only', 'HEAD^{tree}'], + stdin=DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + sources = [line.rstrip() for line in gitproc.stdout.readlines()] + err = gitproc.stderr.read() + gitproc.wait() + subret = gitproc.returncode + if subret != 0 or err: + msg("git command to get sources returned " + str(subret) + + ", error=" + str(err)) + die("this command must be run in a git repository") + return sources + +# get_wiredtiger_versions -- +# Read the version information from the RELEASE_INFO file. +def get_wiredtiger_versions(wt_dir): + v = {} + for l in open(os.path.join(wt_dir, 'RELEASE_INFO')): + if re.match(r'WIREDTIGER_VERSION_(?:MAJOR|MINOR|PATCH)=', l): + exec(l, v) + wt_ver = '%d.%d' % (v['WIREDTIGER_VERSION_MAJOR'], + v['WIREDTIGER_VERSION_MINOR']) + wt_full_ver = wt_ver + '.%d' % (v['WIREDTIGER_VERSION_PATCH']) + return (wt_ver, wt_full_ver) + +# get_library_dirs +# Build a plausible set of library directories. +def get_library_dirs(): + dirs = [] + dirs.append("/usr/local/lib") + dirs.append("/usr/local/lib64") + dirs.append("/lib/x86_64-linux-gnu") + dirs.append("/opt/local/lib") + dirs.append("/usr/lib") + dirs.append("/usr/lib64") + for path in ['LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH', 'LIBRARY_PATH']: + if path in os.environ: + dirs.extend(os.environ[path].split(':')) + dirs = list(set(filter(os.path.isdir, dirs))) + return dirs + +# source_filter +# Make any needed changes to the sources list. Any entry that +# needs to be moved is returned in a dictionary. +def source_filter(sources): + result = [] + movers = dict() + py_dir = os.path.join('lang', 'python') + pywt_dir = os.path.join(py_dir, 'wiredtiger') + pywt_prefix = pywt_dir + os.path.sep + for f in sources: + if not re.match(source_regex, f): + continue + src = f + dest = f + # move all lang/python files to the top level. + if dest.startswith(pywt_prefix): + dest = os.path.basename(dest) + if dest == 'pip_init.py': + dest = '__init__.py' + if dest != src: + movers[dest] = src + result.append(dest) + # Add SWIG generated files + result.append('wiredtiger.py') + movers['wiredtiger.py'] = os.path.join(pywt_dir, '__init__.py') + result.append(os.path.join(py_dir, 'wiredtiger_wrap.c')) + return result, movers + +################################################################ +# Do some initial setup and checks. +this_abs_script = os.path.abspath(__file__) +this_dir = os.path.dirname(this_abs_script) +pip_command = None +for arg in sys.argv[1:]: + if arg[0] != '-' and pip_command == None: + pip_command = arg + break + +if this_dir.endswith(os.sep + os.path.join('lang', 'python')): + wt_dir = os.path.dirname(os.path.dirname(this_dir)) + os.chdir(wt_dir) +elif os.path.isfile(os.path.join(this_dir, 'LICENSE')): + wt_dir = this_dir +else: + die('running from an unknown directory') + +python3 = (sys.version_info[0] > 2) +if python3: + die('Python3 is not yet supported') + +# Ensure that Extensions won't be built for 32 bit, +# that won't work with WiredTiger. +if sys.maxsize < 2**32: + die('need to be running on a 64 bit system, and have a 64 bit Python') + +python_rel_dir = os.path.join('lang', 'python') +build_dir = os.path.join(wt_dir, 'build_posix') +makefile = os.path.join(build_dir, 'Makefile') +built_sentinal = os.path.join(build_dir, 'built.txt') +conf_make_dir = 'build_posix' +wt_swig_lib_name = os.path.join(python_rel_dir, '_wiredtiger.so') + +################################################################ +# Put together build options for the WiredTiger extension. +short_description = 'high performance, scalable, production quality, ' + \ + 'NoSQL, Open Source extensible platform for data management' +long_description = 'WiredTiger is a ' + short_description + '.\n\n' + \ + open(os.path.join(wt_dir, 'README')).read() + +wt_ver, wt_full_ver = get_wiredtiger_versions(wt_dir) +build_path = get_build_path() + +# We only need a small set of directories to build a WT library, +# we also include any files at the top level. +source_regex = r'^(?:(?:api|build_posix|ext|lang/python|src|dist)/|[^/]*$)' + +# The builtins that we include in this distribution. +builtins = [ + # [ name, libname, instructions ] + [ 'snappy', 'snappy', + 'Note: a suitable version of snappy can be found at\n' + \ + ' https://github.com/google/snappy/releases/download/' + \ + '1.1.3/snappy-1.1.3.tar.gz\n' + \ + 'It can be installed via: yum install snappy snappy-devel' + \ + 'or via: apt-get install libsnappy-dev' ], + [ 'zlib', 'z', + 'Need to install zlib\n' + \ + 'It can be installed via: apt-get install zlib1g' ] +] +builtin_names = [b[0] for b in builtins] +builtin_libraries = [b[1] for b in builtins] + +# Here's the configure/make operations we perform before the python extension +# is linked. +configure_cmds = [ + './makemake --clean-and-make', + './reconf', + # force building a position independent library; it will be linked + # into a single shared library with the SWIG interface code. + 'CFLAGS="${CFLAGS:-} -fPIC -DPIC" ' + \ + '../configure --enable-python --with-builtins=' + ','.join(builtin_names) +] + +# build all the builtins, at the moment they are all compressors. +make_cmds = [] +for name in builtin_names: + make_cmds.append('(cd ext/compressors/' + name + '/; make)') +make_cmds.append('make libwiredtiger.la') + +inc_paths = [ os.path.join(build_dir, 'src', 'include'), build_dir, '.' ] +lib_paths = [ '.' ] # wiredtiger.so is moved into the top level directory + +check_needed_dependencies(builtins, inc_paths, lib_paths) + +cppflags, cflags, ldflags = get_compile_flags(inc_paths, lib_paths) + +# If we are creating a source distribution, create a staging directory +# with just the right sources. Put the result in the python dist directory. +if pip_command == 'sdist': + sources, movers = source_filter(get_sources_curdir()) + stage_dir = os.path.join(python_rel_dir, 'stage') + shutil.rmtree(stage_dir, True) + os.makedirs(stage_dir) + shutil.copy2(this_abs_script, os.path.join(stage_dir, 'setup.py')) + for f in sources: + d = os.path.join(stage_dir, os.path.dirname(f)) + if not os.path.isdir(d): + os.makedirs(d) + if f in movers: + src = movers[f] + else: + src = f + # Symlinks are not followed in setup, we need to use real files. + shutil.copy2(src, os.path.join(stage_dir, f)) + os.chdir(stage_dir) + sys.argv.append('--dist-dir=' + os.path.join('..', 'dist')) +else: + sources = [ os.path.join(python_rel_dir, 'wiredtiger_wrap.c') ] + +wt_ext = Extension('_wiredtiger', + sources = sources, + extra_compile_args = cflags + cppflags, + extra_link_args = ldflags, + libraries = builtin_libraries, + extra_objects = [ os.path.join(build_dir, '.libs', 'libwiredtiger.a') ], + include_dirs = inc_paths, + library_dirs = lib_paths, +) +extensions = [ wt_ext ] +env = { "CFLAGS" : ' '.join(cflags), + "CPPFLAGS" : ' '.join(cppflags), + "LDFLAGS" : ' '.join(ldflags), + "PATH" : build_path } + +class BinaryDistribution(Distribution): + def is_pure(self): + return False + +class WTInstall(setuptools.command.install.install): + def run(self): + self.run_command("build_ext") + return setuptools.command.install.install.run(self) + +class WTBuildExt(setuptools.command.build_ext.build_ext): + def __init__(self, *args, **kwargs): + setuptools.command.build_ext.build_ext.__init__(self, *args, **kwargs) + + def run(self): + # only run this once + if not os.path.isfile(built_sentinal): + try: + os.remove(makefile) + except OSError: + pass + self.execute( + lambda: build_commands(configure_cmds, conf_make_dir, env), [], + 'wiredtiger configure') + if not os.path.isfile(makefile): + die('configure failed, file does not exist: ' + makefile) + self.execute( + lambda: build_commands(make_cmds, conf_make_dir, env), [], + 'wiredtiger make') + open(built_sentinal, 'a').close() + return setuptools.command.build_ext.build_ext.run(self) + +setup( + name = 'wiredtiger', + version = wt_full_ver, + author = 'The WiredTiger Development Team, part of MongoDB', + author_email = 'info@wiredtiger.com', + description = short_description, + license='GPL2,GPL3,Commercial', + long_description = long_description, + url = 'http://source.wiredtiger.com/', + keywords = 'scalable NoSQL database datastore engine open source', + packages = ['wiredtiger'], + ext_package = 'wiredtiger', + ext_modules = extensions, + include_package_data = True, + distclass = BinaryDistribution, + package_dir = { 'wiredtiger' : '.' }, + cmdclass = { 'install': WTInstall, 'build_ext': WTBuildExt }, + package_data = { + 'wiredtiger' : [ wt_swig_lib_name, '*.py' ] + }, + classifiers=[ + 'Intended Audience :: Developers', + 'Programming Language :: C', + 'Programming Language :: C++', + 'Programming Language :: Python', + 'Programming Language :: Java', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: POSIX', + 'Operating System :: POSIX :: BSD', + 'Operating System :: POSIX :: Linux', + 'Operating System :: POSIX :: SunOS/Solaris', + ] +) + +if pip_command == 'sdist': + shutil.rmtree(os.path.join(this_dir, 'stage')) diff --git a/src/third_party/wiredtiger/lang/python/wiredtiger/pip_init.py b/src/third_party/wiredtiger/lang/python/wiredtiger/pip_init.py new file mode 100644 index 00000000000..d59c8218976 --- /dev/null +++ b/src/third_party/wiredtiger/lang/python/wiredtiger/pip_init.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +# pip_init.py +# This is installed as __init__.py, and imports the file created by SWIG. +# This is needed because SWIG's import helper code created by certain SWIG +# versions may be broken, see: https://github.com/swig/swig/issues/769 . +# Importing indirectly seems to avoid these issues. +import os, sys +fname = os.path.basename(__file__) +if fname != '__init__.py' and fname != '__init__.pyc': + print(__file__ + ': this file is not yet installed') + sys.exit(1) + +# After importing the SWIG-generated file, copy all symbols from from it +# to this module so they will appear in the wiredtiger namespace. +me = sys.modules[__name__] +sys.path.append(os.path.dirname(__file__)) # needed for Python3 +import wiredtiger +for name in dir(wiredtiger): + value = getattr(wiredtiger, name) + setattr(me, name, value) diff --git a/src/third_party/wiredtiger/src/async/async_api.c b/src/third_party/wiredtiger/src/async/async_api.c index 54bcb7cd26c..3319332aa04 100644 --- a/src/third_party/wiredtiger/src/async/async_api.c +++ b/src/third_party/wiredtiger/src/async/async_api.c @@ -240,8 +240,7 @@ __async_start(WT_SESSION_IMPL *session) async = conn->async; TAILQ_INIT(&async->formatqh); WT_RET(__wt_spin_init(session, &async->ops_lock, "ops")); - WT_RET(__wt_cond_alloc( - session, "async flush", false, &async->flush_cond)); + WT_RET(__wt_cond_alloc(session, "async flush", &async->flush_cond)); WT_RET(__wt_async_op_init(session)); /* @@ -339,17 +338,15 @@ __wt_async_reconfig(WT_SESSION_IMPL *session, const char *cfg[]) * 2. If async is off, and the user wants it on, start it. * 3. If not a toggle and async is off, we're done. */ - if (conn->async_cfg && !run) { - /* Case 1 */ + if (conn->async_cfg && !run) { /* Case 1 */ WT_TRET(__wt_async_flush(session)); ret = __wt_async_destroy(session); conn->async_cfg = false; return (ret); - } else if (!conn->async_cfg && run) - /* Case 2 */ + } + if (!conn->async_cfg && run) /* Case 2 */ return (__async_start(session)); - else if (!conn->async_cfg) - /* Case 3 */ + if (!conn->async_cfg) /* Case 3 */ return (0); /* @@ -443,7 +440,7 @@ __wt_async_destroy(WT_SESSION_IMPL *session) session, async->worker_tids[i])); async->worker_tids[i] = 0; } - WT_TRET(__wt_cond_destroy(session, &async->flush_cond)); + __wt_cond_destroy(session, &async->flush_cond); /* Close the server threads' sessions. */ for (i = 0; i < conn->async_workers; i++) @@ -541,7 +538,7 @@ retry: async->flush_op.state = WT_ASYNCOP_READY; WT_RET(__wt_async_op_enqueue(session, &async->flush_op)); while (async->flush_state != WT_ASYNC_FLUSH_COMPLETE) - __wt_cond_wait(session, async->flush_cond, 100000); + __wt_cond_wait(session, async->flush_cond, 100000, NULL); /* * Flush is done. Clear the flags. */ diff --git a/src/third_party/wiredtiger/src/async/async_op.c b/src/third_party/wiredtiger/src/async/async_op.c index 8c074e503d9..6908802dbff 100644 --- a/src/third_party/wiredtiger/src/async/async_op.c +++ b/src/third_party/wiredtiger/src/async/async_op.c @@ -254,7 +254,6 @@ __wt_async_op_enqueue(WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL *op) { WT_ASYNC *async; WT_CONNECTION_IMPL *conn; - WT_DECL_RET; uint64_t cur_head, cur_tail, my_alloc, my_slot; #ifdef HAVE_DIAGNOSTIC WT_ASYNC_OP_IMPL *my_op; @@ -307,7 +306,7 @@ __wt_async_op_enqueue(WT_SESSION_IMPL *session, WT_ASYNC_OP_IMPL *op) WT_ORDERED_READ(cur_head, async->head); } WT_PUBLISH(async->head, my_alloc); - return (ret); + return (0); } /* diff --git a/src/third_party/wiredtiger/src/async/async_worker.c b/src/third_party/wiredtiger/src/async/async_worker.c index b1bc3902f7c..11f59ed14f1 100644 --- a/src/third_party/wiredtiger/src/async/async_worker.c +++ b/src/third_party/wiredtiger/src/async/async_worker.c @@ -107,7 +107,7 @@ __async_flush_wait(WT_SESSION_IMPL *session, WT_ASYNC *async, uint64_t my_gen) { while (async->flush_state == WT_ASYNC_FLUSHING && async->flush_gen == my_gen) - __wt_cond_wait(session, async->flush_cond, 10000); + __wt_cond_wait(session, async->flush_cond, 10000, NULL); } /* diff --git a/src/third_party/wiredtiger/src/block/block_addr.c b/src/third_party/wiredtiger/src/block/block_addr.c index dadd17de4a6..a67efca62a3 100644 --- a/src/third_party/wiredtiger/src/block/block_addr.c +++ b/src/third_party/wiredtiger/src/block/block_addr.c @@ -192,6 +192,7 @@ __wt_block_buffer_to_ckpt(WT_SESSION_IMPL *session, int __wt_block_ckpt_decode(WT_SESSION *wt_session, size_t allocsize, const uint8_t *p, WT_BLOCK_CKPT *ci) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_SESSION_IMPL *session; @@ -225,7 +226,7 @@ __wt_block_ckpt_to_buffer(WT_SESSION_IMPL *session, ci->discard.offset, ci->discard.size, ci->discard.checksum)); a = (uint64_t)ci->file_size; WT_RET(__wt_vpack_uint(pp, 0, a)); - a = (uint64_t)ci->ckpt_size; + a = ci->ckpt_size; WT_RET(__wt_vpack_uint(pp, 0, a)); return (0); diff --git a/src/third_party/wiredtiger/src/block/block_ckpt.c b/src/third_party/wiredtiger/src/block/block_ckpt.c index 48522768dc9..05e4dcc098e 100644 --- a/src/third_party/wiredtiger/src/block/block_ckpt.c +++ b/src/third_party/wiredtiger/src/block/block_ckpt.c @@ -53,7 +53,6 @@ __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_DECL_RET; uint8_t *endp; - WT_UNUSED(addr_size); ci = NULL; /* diff --git a/src/third_party/wiredtiger/src/block/block_compact.c b/src/third_party/wiredtiger/src/block/block_compact.c index 6d5d48bdb22..eb6647dd03c 100644 --- a/src/third_party/wiredtiger/src/block/block_compact.c +++ b/src/third_party/wiredtiger/src/block/block_compact.c @@ -22,6 +22,12 @@ __wt_block_compact_start(WT_SESSION_IMPL *session, WT_BLOCK *block) /* Switch to first-fit allocation. */ __wt_block_configure_first_fit(block, true); + /* Reset the compaction state information. */ + block->compact_pct_tenths = 0; + block->compact_pages_reviewed = 0; + block->compact_pages_skipped = 0; + block->compact_pages_written = 0; + return (0); } @@ -55,7 +61,6 @@ __wt_block_compact_end(WT_SESSION_IMPL *session, WT_BLOCK *block) int __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) { - WT_DECL_RET; WT_EXT *ext; WT_EXTLIST *el; wt_off_t avail_eighty, avail_ninety, eighty, ninety; @@ -71,16 +76,6 @@ __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) if (block->size <= WT_MEGABYTE) return (0); - /* - * Reset the compaction state information. This is done here, not in the - * compaction "start" routine, because this function is called first to - * determine if compaction is useful. - */ - block->compact_pct_tenths = 0; - block->compact_pages_reviewed = 0; - block->compact_pages_skipped = 0; - block->compact_pages_written = 0; - __wt_spin_lock(session, &block->live_lock); /* Dump the current state of the file. */ @@ -142,7 +137,7 @@ __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp) __wt_spin_unlock(session, &block->live_lock); - return (ret); + return (0); } /* @@ -153,7 +148,6 @@ int __wt_block_compact_page_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, bool *skipp) { - WT_DECL_RET; WT_EXT *ext; WT_EXTLIST *el; wt_off_t limit, offset; @@ -198,7 +192,7 @@ __wt_block_compact_page_skip(WT_SESSION_IMPL *session, } #endif - return (ret); + return (0); } /* diff --git a/src/third_party/wiredtiger/src/block/block_ext.c b/src/third_party/wiredtiger/src/block/block_ext.c index 26acc8c560f..da7a06d873d 100644 --- a/src/third_party/wiredtiger/src/block/block_ext.c +++ b/src/third_party/wiredtiger/src/block/block_ext.c @@ -634,11 +634,11 @@ __wt_block_off_free( */ if ((ret = __wt_block_off_remove_overlap( session, block, &block->live.alloc, offset, size)) == 0) - ret = __block_merge(session, block, - &block->live.avail, offset, (wt_off_t)size); + ret = __block_merge( + session, block, &block->live.avail, offset, size); else if (ret == WT_NOTFOUND) - ret = __block_merge(session, block, - &block->live.discard, offset, (wt_off_t)size); + ret = __block_merge( + session, block, &block->live.discard, offset, size); return (ret); } @@ -1247,7 +1247,8 @@ __wt_block_extlist_write(WT_SESSION_IMPL *session, WT_DECL_RET; WT_EXT *ext; WT_PAGE_HEADER *dsk; - size_t entries, size; + uint32_t entries; + size_t size; uint8_t *p; WT_RET(__block_extlist_dump(session, block, el, "write")); @@ -1377,8 +1378,8 @@ __wt_block_extlist_init(WT_SESSION_IMPL *session, size = (name == NULL ? 0 : strlen(name)) + strlen(".") + (extname == NULL ? 0 : strlen(extname) + 1); WT_RET(__wt_calloc_def(session, size, &el->name)); - (void)snprintf(el->name, size, "%s.%s", - name == NULL ? "" : name, extname == NULL ? "" : extname); + WT_RET(__wt_snprintf(el->name, size, "%s.%s", + name == NULL ? "" : name, extname == NULL ? "" : extname)); el->offset = WT_BLOCK_INVALID_OFFSET; el->track_size = track_size; diff --git a/src/third_party/wiredtiger/src/block/block_open.c b/src/third_party/wiredtiger/src/block/block_open.c index 4d15942709a..07ceb4c8159 100644 --- a/src/third_party/wiredtiger/src/block/block_open.c +++ b/src/third_party/wiredtiger/src/block/block_open.c @@ -201,6 +201,12 @@ __wt_block_open(WT_SESSION_IMPL *session, * "direct_io=checkpoint" configures direct I/O for readonly data files. */ flags = 0; + WT_ERR(__wt_config_gets(session, cfg, "access_pattern_hint", &cval)); + if (WT_STRING_MATCH("random", cval.str, cval.len)) + LF_SET(WT_FS_OPEN_ACCESS_RAND); + else if (WT_STRING_MATCH("sequential", cval.str, cval.len)) + LF_SET(WT_FS_OPEN_ACCESS_SEQ); + if (readonly && FLD_ISSET(conn->direct_io, WT_DIRECT_IO_CHECKPOINT)) LF_SET(WT_FS_OPEN_DIRECTIO); if (!readonly && FLD_ISSET(conn->direct_io, WT_DIRECT_IO_DATA)) diff --git a/src/third_party/wiredtiger/src/block/block_read.c b/src/third_party/wiredtiger/src/block/block_read.c index 1eeabd63d92..8d4aec7df75 100644 --- a/src/third_party/wiredtiger/src/block/block_read.c +++ b/src/third_party/wiredtiger/src/block/block_read.c @@ -24,8 +24,6 @@ __wt_bm_preload( uint32_t checksum, size; bool mapped; - WT_UNUSED(addr_size); - block = bm->block; WT_STAT_CONN_INCR(session, block_preload); @@ -41,7 +39,7 @@ __wt_bm_preload( (uint8_t *)bm->map + offset, size, bm->mapped_cookie); if (!mapped && handle->fh_advise != NULL) ret = handle->fh_advise(handle, (WT_SESSION *)session, - (wt_off_t)offset, (wt_off_t)size, WT_FILE_HANDLE_WILLNEED); + offset, (wt_off_t)size, WT_FILE_HANDLE_WILLNEED); if (ret != EBUSY && ret != ENOTSUP) return (ret); diff --git a/src/third_party/wiredtiger/src/block/block_slvg.c b/src/third_party/wiredtiger/src/block/block_slvg.c index 5ba95bb598e..b06a5062f50 100644 --- a/src/third_party/wiredtiger/src/block/block_slvg.c +++ b/src/third_party/wiredtiger/src/block/block_slvg.c @@ -168,7 +168,6 @@ __wt_block_salvage_valid(WT_SESSION_IMPL *session, wt_off_t offset; uint32_t size, checksum; - WT_UNUSED(session); WT_UNUSED(addr_size); /* diff --git a/src/third_party/wiredtiger/src/block/block_vrfy.c b/src/third_party/wiredtiger/src/block/block_vrfy.c index 94824ad19f8..154765ed079 100644 --- a/src/third_party/wiredtiger/src/block/block_vrfy.c +++ b/src/third_party/wiredtiger/src/block/block_vrfy.c @@ -22,7 +22,7 @@ static int __verify_set_file_size(WT_SESSION_IMPL *, WT_BLOCK *, WT_CKPT *); ((off) / (block)->allocsize - 1) #ifdef HAVE_VERBOSE #define WT_FRAG_TO_OFF(block, frag) \ - (((wt_off_t)(frag + 1)) * (block)->allocsize) + (((wt_off_t)((frag) + 1)) * (block)->allocsize) #endif /* diff --git a/src/third_party/wiredtiger/src/block/block_write.c b/src/third_party/wiredtiger/src/block/block_write.c index d08aba45920..ea7859d6a38 100644 --- a/src/third_party/wiredtiger/src/block/block_write.c +++ b/src/third_party/wiredtiger/src/block/block_write.c @@ -43,10 +43,10 @@ __wt_block_truncate(WT_SESSION_IMPL *session, WT_BLOCK *block, wt_off_t len) * more targeted solution at some point. */ if (!conn->hot_backup) { - __wt_readlock(session, conn->hot_backup_lock); + __wt_readlock(session, &conn->hot_backup_lock); if (!conn->hot_backup) ret = __wt_ftruncate(session, block->fh, len); - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); } /* diff --git a/src/third_party/wiredtiger/src/bloom/bloom.c b/src/third_party/wiredtiger/src/bloom/bloom.c index dde08d8a982..b8d75678835 100644 --- a/src/third_party/wiredtiger/src/bloom/bloom.c +++ b/src/third_party/wiredtiger/src/bloom/bloom.c @@ -37,8 +37,8 @@ __bloom_init(WT_SESSION_IMPL *session, len += strlen(config); WT_ERR(__wt_calloc_def(session, len, &bloom->config)); /* Add the standard config at the end, so it overrides user settings. */ - (void)snprintf(bloom->config, len, - "%s,%s", config == NULL ? "" : config, WT_BLOOM_TABLE_CONFIG); + WT_ERR(__wt_snprintf(bloom->config, len, + "%s,%s", config == NULL ? "" : config, WT_BLOOM_TABLE_CONFIG)); bloom->session = session; @@ -95,6 +95,7 @@ int __wt_bloom_create( WT_SESSION_IMPL *session, const char *uri, const char *config, uint64_t count, uint32_t factor, uint32_t k, WT_BLOOM **bloomp) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_BLOOM *bloom; WT_DECL_RET; @@ -148,6 +149,7 @@ int __wt_bloom_open(WT_SESSION_IMPL *session, const char *uri, uint32_t factor, uint32_t k, WT_CURSOR *owner, WT_BLOOM **bloomp) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_BLOOM *bloom; WT_CURSOR *c; @@ -178,6 +180,7 @@ err: WT_TRET(__wt_bloom_close(bloom)); */ void __wt_bloom_insert(WT_BLOOM *bloom, WT_ITEM *key) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { uint64_t h1, h2; uint32_t i; @@ -195,6 +198,7 @@ __wt_bloom_insert(WT_BLOOM *bloom, WT_ITEM *key) */ int __wt_bloom_finalize(WT_BLOOM *bloom) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_CURSOR *c; WT_DECL_RET; @@ -304,6 +308,7 @@ err: /* Don't return WT_NOTFOUND from a failed search. */ */ int __wt_bloom_get(WT_BLOOM *bloom, WT_ITEM *key) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_BLOOM_HASH bhash; @@ -363,6 +368,7 @@ __wt_bloom_intersection(WT_BLOOM *bloom, WT_BLOOM *other) */ int __wt_bloom_close(WT_BLOOM *bloom) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_DECL_RET; WT_SESSION_IMPL *session; @@ -385,6 +391,7 @@ __wt_bloom_close(WT_BLOOM *bloom) */ int __wt_bloom_drop(WT_BLOOM *bloom, const char *config) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_DECL_RET; WT_SESSION *wt_session; diff --git a/src/third_party/wiredtiger/src/btree/bt_compact.c b/src/third_party/wiredtiger/src/btree/bt_compact.c index e005674762d..4b40d0aeed0 100644 --- a/src/third_party/wiredtiger/src/btree/bt_compact.c +++ b/src/third_party/wiredtiger/src/btree/bt_compact.c @@ -60,7 +60,7 @@ __compact_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) */ if (mod->rec_result == WT_PM_REC_REPLACE || mod->rec_result == WT_PM_REC_MULTIBLOCK) - __wt_writelock(session, &page->page_lock); + WT_PAGE_LOCK(session, page); if (mod->rec_result == WT_PM_REC_REPLACE) ret = bm->compact_page_skip(bm, session, @@ -80,7 +80,7 @@ __compact_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) if (mod->rec_result == WT_PM_REC_REPLACE || mod->rec_result == WT_PM_REC_MULTIBLOCK) - __wt_writeunlock(session, &page->page_lock); + WT_PAGE_UNLOCK(session, page); return (ret); } @@ -90,16 +90,15 @@ __compact_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) * Compact a file. */ int -__wt_compact(WT_SESSION_IMPL *session, const char *cfg[]) +__wt_compact(WT_SESSION_IMPL *session) { WT_BM *bm; WT_BTREE *btree; WT_DECL_RET; WT_REF *ref; + u_int i; bool skip; - WT_UNUSED(cfg); - btree = S2BT(session); bm = btree->bm; ref = NULL; @@ -129,7 +128,13 @@ __wt_compact(WT_SESSION_IMPL *session, const char *cfg[]) __wt_spin_lock(session, &btree->flush_lock); /* Walk the tree reviewing pages to see if they should be re-written. */ - for (;;) { + for (i = 0;;) { + /* Periodically check if we've run out of time. */ + if (++i > 100) { + WT_ERR(__wt_session_compact_check_timeout(session)); + i = 0; + } + /* * Pages read for compaction aren't "useful"; don't update the * read generation of pages already in memory, and if a page is diff --git a/src/third_party/wiredtiger/src/btree/bt_curnext.c b/src/third_party/wiredtiger/src/btree/bt_curnext.c index 4d3976f9647..21e575ffca9 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curnext.c +++ b/src/third_party/wiredtiger/src/btree/bt_curnext.c @@ -338,7 +338,7 @@ new_insert: if ((ins = cbt->ins) != NULL) { } /* Check for the end of the page. */ - if (cbt->row_iteration_slot >= page->pg_row_entries * 2 + 1) + if (cbt->row_iteration_slot >= page->entries * 2 + 1) return (WT_NOTFOUND); ++cbt->row_iteration_slot; @@ -356,7 +356,7 @@ new_insert: if ((ins = cbt->ins) != NULL) { cbt->ins = NULL; cbt->slot = cbt->row_iteration_slot / 2 - 1; - rip = &page->pg_row_d[cbt->slot]; + rip = &page->pg_row[cbt->slot]; upd = __wt_txn_read(session, WT_ROW_UPDATE(page, rip)); if (upd != NULL && WT_UPDATE_DELETED_ISSET(upd)) { if (__wt_txn_visible_all(session, upd->txnid)) @@ -579,20 +579,20 @@ __wt_btcur_iterate_setup(WT_CURSOR_BTREE *cbt) int __wt_btcur_next(WT_CURSOR_BTREE *cbt, bool truncating) { + WT_CURSOR *cursor; WT_DECL_RET; WT_PAGE *page; WT_SESSION_IMPL *session; uint32_t flags; bool newpage; + cursor = &cbt->iface; session = (WT_SESSION_IMPL *)cbt->iface.session; WT_STAT_CONN_INCR(session, cursor_next); WT_STAT_DATA_INCR(session, cursor_next); - flags = WT_READ_SKIP_INTL; /* Tree walk flags. */ - if (truncating) - LF_SET(WT_READ_TRUNCATE); + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); WT_RET(__cursor_func_init(cbt, false)); @@ -608,6 +608,9 @@ __wt_btcur_next(WT_CURSOR_BTREE *cbt, bool truncating) * found. Then, move to the next page, until we reach the end of the * file. */ + flags = WT_READ_SKIP_INTL; /* tree walk flags */ + if (truncating) + LF_SET(WT_READ_TRUNCATE); for (newpage = false;; newpage = true) { page = cbt->ref == NULL ? NULL : cbt->ref->page; @@ -676,6 +679,8 @@ __wt_btcur_next(WT_CURSOR_BTREE *cbt, bool truncating) if (ret == 0) WT_ERR(__wt_cursor_key_order_check(session, cbt, true)); #endif + if (ret == 0) + F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); err: if (ret != 0) WT_TRET(__cursor_reset(cbt)); diff --git a/src/third_party/wiredtiger/src/btree/bt_curprev.c b/src/third_party/wiredtiger/src/btree/bt_curprev.c index 2dd443ffac1..bf4bdad6529 100644 --- a/src/third_party/wiredtiger/src/btree/bt_curprev.c +++ b/src/third_party/wiredtiger/src/btree/bt_curprev.c @@ -458,13 +458,13 @@ __cursor_row_prev(WT_CURSOR_BTREE *cbt, bool newpage) if (!F_ISSET_ATOMIC(page, WT_PAGE_BUILD_KEYS)) WT_RET(__wt_row_leaf_keys(session, page)); - if (page->pg_row_entries == 0) + if (page->entries == 0) cbt->ins_head = WT_ROW_INSERT_SMALLEST(page); else cbt->ins_head = - WT_ROW_INSERT_SLOT(page, page->pg_row_entries - 1); + WT_ROW_INSERT_SLOT(page, page->entries - 1); cbt->ins = WT_SKIP_LAST(cbt->ins_head); - cbt->row_iteration_slot = page->pg_row_entries * 2 + 1; + cbt->row_iteration_slot = page->entries * 2 + 1; cbt->rip_saved = NULL; goto new_insert; } @@ -515,7 +515,7 @@ new_insert: if ((ins = cbt->ins) != NULL) { cbt->ins = NULL; cbt->slot = cbt->row_iteration_slot / 2 - 1; - rip = &page->pg_row_d[cbt->slot]; + rip = &page->pg_row[cbt->slot]; upd = __wt_txn_read(session, WT_ROW_UPDATE(page, rip)); if (upd != NULL && WT_UPDATE_DELETED_ISSET(upd)) { if (__wt_txn_visible_all(session, upd->txnid)) @@ -535,20 +535,20 @@ new_insert: if ((ins = cbt->ins) != NULL) { int __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) { + WT_CURSOR *cursor; WT_DECL_RET; WT_PAGE *page; WT_SESSION_IMPL *session; uint32_t flags; bool newpage; + cursor = &cbt->iface; session = (WT_SESSION_IMPL *)cbt->iface.session; WT_STAT_CONN_INCR(session, cursor_prev); WT_STAT_DATA_INCR(session, cursor_prev); - flags = WT_READ_PREV | WT_READ_SKIP_INTL; /* Tree walk flags. */ - if (truncating) - LF_SET(WT_READ_TRUNCATE); + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); WT_RET(__cursor_func_init(cbt, false)); @@ -564,6 +564,9 @@ __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) * found. Then, move to the previous page, until we reach the start * of the file. */ + flags = WT_READ_PREV | WT_READ_SKIP_INTL; /* tree walk flags */ + if (truncating) + LF_SET(WT_READ_TRUNCATE); for (newpage = false;; newpage = true) { page = cbt->ref == NULL ? NULL : cbt->ref->page; @@ -631,6 +634,8 @@ __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) if (ret == 0) WT_ERR(__wt_cursor_key_order_check(session, cbt, false)); #endif + if (ret == 0) + F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); err: if (ret != 0) WT_TRET(__cursor_reset(cbt)); diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index 0454ea7c20e..f0aa632551b 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -8,6 +8,84 @@ #include "wt_internal.h" +/* + * When returning an error, we need to restore the cursor to a valid state, the + * upper-level cursor code is likely to retry. This structure and the associated + * functions are used save and restore the cursor state. + */ +typedef struct { + WT_ITEM key; + WT_ITEM value; + uint64_t recno; + uint32_t flags; +} WT_CURFILE_STATE; + +/* + * __cursor_state_save -- + * Save the cursor's external state. + */ +static inline void +__cursor_state_save(WT_CURSOR *cursor, WT_CURFILE_STATE *state) +{ + WT_ITEM_SET(state->key, cursor->key); + WT_ITEM_SET(state->value, cursor->value); + state->recno = cursor->recno; + state->flags = cursor->flags; +} + +/* + * __cursor_state_restore -- + * Restore the cursor's external state. + */ +static inline void +__cursor_state_restore(WT_CURSOR *cursor, WT_CURFILE_STATE *state) +{ + if (F_ISSET(state, WT_CURSTD_KEY_EXT)) + WT_ITEM_SET(cursor->key, state->key); + if (F_ISSET(state, WT_CURSTD_VALUE_EXT)) + WT_ITEM_SET(cursor->value, state->value); + cursor->recno = state->recno; + F_CLR(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); + F_SET(cursor, F_MASK(state, WT_CURSTD_KEY_EXT | WT_CURSTD_VALUE_EXT)); + +} + +/* + * __cursor_page_pinned -- + * Return if we have a page pinned and it's not been flagged for forced + * eviction (the forced eviction test is so we periodically release pages + * grown too large). + */ +static inline bool +__cursor_page_pinned(WT_CURSOR_BTREE *cbt) +{ + return (F_ISSET(cbt, WT_CBT_ACTIVE) && + cbt->ref->page->read_gen != WT_READGEN_OLDEST); +} + +/* + * __cursor_copy_int_key -- + * If we're pointing into the tree, save the key into local memory. + */ +static inline int +__cursor_copy_int_key(WT_CURSOR *cursor) +{ + /* + * We're about to discard the cursor's position and the cursor layer + * might retry the operation. We discard pinned pages on error, which + * will invalidate pinned keys. Clear WT_CURSTD_KEY_INT in all cases, + * the underlying page is gone whether we can allocate memory or not. + */ + if (F_ISSET(cursor, WT_CURSTD_KEY_INT)) { + F_CLR(cursor, WT_CURSTD_KEY_INT); + if (!WT_DATA_IN_ITEM(&cursor->key)) + WT_RET(__wt_buf_set((WT_SESSION_IMPL *)cursor->session, + &cursor->key, cursor->key.data, cursor->key.size)); + F_SET(cursor, WT_CURSTD_KEY_EXT); + } + return (0); +} + /* * __cursor_size_chk -- * Return if an inserted item is too large. @@ -41,7 +119,7 @@ __cursor_size_chk(WT_SESSION_IMPL *session, WT_ITEM *kv) if (kv->size > WT_BTREE_MAX_OBJECT_SIZE) WT_RET_MSG(session, EINVAL, "item size of %" WT_SIZET_FMT " exceeds the maximum " - "supported WiredTiger size of %d", + "supported WiredTiger size of %" PRIu32, kv->size, WT_BTREE_MAX_OBJECT_SIZE); /* Check what the block manager can actually write. */ @@ -54,6 +132,34 @@ __cursor_size_chk(WT_SESSION_IMPL *session, WT_ITEM *kv) return (0); } +/* + * __cursor_disable_bulk -- + * Disable bulk loads into a tree. + */ +static inline void +__cursor_disable_bulk(WT_SESSION_IMPL *session, WT_BTREE *btree) +{ + /* + * Once a tree (other than the LSM primary) is no longer empty, eviction + * should pay attention to it, and it's no longer possible to bulk-load + * into it. + */ + if (!btree->original) + return; + if (btree->lsm_primary) { + btree->original = 0; /* Make the next test faster. */ + return; + } + + /* + * We use a compare-and-swap here to avoid races among the first inserts + * into a tree. Eviction is disabled when an empty tree is opened, and + * it must only be enabled once. + */ + if (__wt_atomic_cas8(&btree->original, 1, 0)) + __wt_evict_file_exclusive_off(session); +} + /* * __cursor_fix_implicit -- * Return if search went past the end of the tree. @@ -76,11 +182,11 @@ __cursor_fix_implicit(WT_BTREE *btree, WT_CURSOR_BTREE *cbt) } /* - * __cursor_valid -- + * __wt_cursor_valid -- * Return if the cursor references an valid key/value pair. */ -static inline bool -__cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp) +bool +__wt_cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp) { WT_BTREE *btree; WT_CELL *cell; @@ -163,7 +269,7 @@ __cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp) * column-store pages don't have slots, but map one-to-one to * keys, check for retrieval past the end of the page. */ - if (cbt->recno >= cbt->ref->ref_recno + page->pg_fix_entries) + if (cbt->recno >= cbt->ref->ref_recno + page->entries) return (false); /* @@ -173,9 +279,9 @@ __cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp) break; case BTREE_COL_VAR: /* The search function doesn't check for empty pages. */ - if (page->pg_var_entries == 0) + if (page->entries == 0) return (false); - WT_ASSERT(session, cbt->slot < page->pg_var_entries); + WT_ASSERT(session, cbt->slot < page->entries); /* * Column-store updates are stored as "insert" objects. If @@ -191,16 +297,16 @@ __cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp) * backing store; check the cell for a record already deleted * when read. */ - cip = &page->pg_var_d[cbt->slot]; + cip = &page->pg_var[cbt->slot]; if ((cell = WT_COL_PTR(page, cip)) == NULL || __wt_cell_type(cell) == WT_CELL_DEL) return (false); break; case BTREE_ROW: /* The search function doesn't check for empty pages. */ - if (page->pg_row_entries == 0) + if (page->entries == 0) return (false); - WT_ASSERT(session, cbt->slot < page->pg_row_entries); + WT_ASSERT(session, cbt->slot < page->entries); /* * See above: for row-store, no insert object can have the same @@ -263,7 +369,7 @@ __cursor_col_modify( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, bool is_remove) { return (__wt_col_modify(session, - cbt, cbt->iface.recno, &cbt->iface.value, NULL, is_remove)); + cbt, cbt->iface.recno, &cbt->iface.value, NULL, is_remove, false)); } /* @@ -275,7 +381,7 @@ __cursor_row_modify( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, bool is_remove) { return (__wt_row_modify(session, - cbt, &cbt->iface.key, &cbt->iface.value, NULL, is_remove)); + cbt, &cbt->iface.key, &cbt->iface.value, NULL, is_remove, false)); } /* @@ -285,13 +391,17 @@ __cursor_row_modify( int __wt_btcur_reset(WT_CURSOR_BTREE *cbt) { + WT_CURSOR *cursor; WT_SESSION_IMPL *session; + cursor = &cbt->iface; session = (WT_SESSION_IMPL *)cbt->iface.session; WT_STAT_CONN_INCR(session, cursor_reset); WT_STAT_DATA_INCR(session, cursor_reset); + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + return (__cursor_reset(cbt)); } @@ -303,6 +413,7 @@ int __wt_btcur_search(WT_CURSOR_BTREE *cbt) { WT_BTREE *btree; + WT_CURFILE_STATE state; WT_CURSOR *cursor; WT_DECL_RET; WT_SESSION_IMPL *session; @@ -317,20 +428,28 @@ __wt_btcur_search(WT_CURSOR_BTREE *cbt) WT_STAT_CONN_INCR(session, cursor_search); WT_STAT_DATA_INCR(session, cursor_search); + __cursor_state_save(cursor, &state); + + /* + * The pinned page goes away if we do a search, make sure there's a + * local copy of any key, then re-save the cursor state. + */ + WT_ERR(__cursor_copy_int_key(cursor)); + __cursor_state_save(cursor, &state); + /* * If we have a page pinned, search it; if we don't have a page pinned, * or the search of the pinned page doesn't find an exact match, search * from the root. */ valid = false; - if (F_ISSET(cbt, WT_CBT_ACTIVE) && - cbt->ref->page->read_gen != WT_READGEN_OLDEST) { - WT_ERR(__wt_txn_cursor_op(session)); + if (__cursor_page_pinned(cbt)) { + __wt_txn_cursor_op(session); WT_ERR(btree->type == BTREE_ROW ? __cursor_row_search(session, cbt, cbt->ref, false) : __cursor_col_search(session, cbt, cbt->ref)); - valid = cbt->compare == 0 && __cursor_valid(cbt, &upd); + valid = cbt->compare == 0 && __wt_cursor_valid(cbt, &upd); } if (!valid) { WT_ERR(__cursor_func_init(cbt, true)); @@ -338,7 +457,7 @@ __wt_btcur_search(WT_CURSOR_BTREE *cbt) WT_ERR(btree->type == BTREE_ROW ? __cursor_row_search(session, cbt, NULL, false) : __cursor_col_search(session, cbt, NULL)); - valid = cbt->compare == 0 && __cursor_valid(cbt, &upd); + valid = cbt->compare == 0 && __wt_cursor_valid(cbt, &upd); } if (valid) @@ -352,6 +471,8 @@ __wt_btcur_search(WT_CURSOR_BTREE *cbt) cbt->v = 0; cursor->value.data = &cbt->v; cursor->value.size = 1; + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); } else ret = WT_NOTFOUND; @@ -360,8 +481,10 @@ __wt_btcur_search(WT_CURSOR_BTREE *cbt) WT_ERR(__wt_cursor_key_order_init(session, cbt)); #endif -err: if (ret != 0) +err: if (ret != 0) { WT_TRET(__cursor_reset(cbt)); + __cursor_state_restore(cursor, &state); + } return (ret); } @@ -373,6 +496,7 @@ int __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) { WT_BTREE *btree; + WT_CURFILE_STATE state; WT_CURSOR *cursor; WT_DECL_RET; WT_SESSION_IMPL *session; @@ -389,6 +513,15 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) WT_STAT_CONN_INCR(session, cursor_search_near); WT_STAT_DATA_INCR(session, cursor_search_near); + __cursor_state_save(cursor, &state); + + /* + * The pinned page goes away if we do a search, make sure there's a + * local copy of any key, then re-save the cursor state. + */ + WT_ERR(__cursor_copy_int_key(cursor)); + __cursor_state_save(cursor, &state); + /* * If we have a row-store page pinned, search it; if we don't have a * page pinned, or the search of the pinned page doesn't find an exact @@ -402,10 +535,8 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) * existing record. */ valid = false; - if (btree->type == BTREE_ROW && - F_ISSET(cbt, WT_CBT_ACTIVE) && - cbt->ref->page->read_gen != WT_READGEN_OLDEST) { - WT_ERR(__wt_txn_cursor_op(session)); + if (btree->type == BTREE_ROW && __cursor_page_pinned(cbt)) { + __wt_txn_cursor_op(session); WT_ERR(__cursor_row_search(session, cbt, cbt->ref, true)); @@ -418,16 +549,15 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) * might be legitimately positioned after the last page slot). * Ignore those cases, it makes things too complicated. */ - if (cbt->slot != 0 && - cbt->slot != cbt->ref->page->pg_row_entries - 1) - valid = __cursor_valid(cbt, &upd); + if (cbt->slot != 0 && cbt->slot != cbt->ref->page->entries - 1) + valid = __wt_cursor_valid(cbt, &upd); } if (!valid) { WT_ERR(__cursor_func_init(cbt, true)); WT_ERR(btree->type == BTREE_ROW ? __cursor_row_search(session, cbt, NULL, true) : __cursor_col_search(session, cbt, NULL)); - valid = __cursor_valid(cbt, &upd); + valid = __wt_cursor_valid(cbt, &upd); } /* @@ -456,6 +586,8 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) cursor->value.data = &cbt->v; cursor->value.size = 1; exact = 0; + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); } else if ((ret = __wt_btcur_next(cbt, false)) != WT_NOTFOUND) exact = 1; else { @@ -463,22 +595,25 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) WT_ERR(btree->type == BTREE_ROW ? __cursor_row_search(session, cbt, NULL, true) : __cursor_col_search(session, cbt, NULL)); - if (__cursor_valid(cbt, &upd)) { + if (__wt_cursor_valid(cbt, &upd)) { exact = cbt->compare; ret = __wt_kv_return(session, cbt, upd); } else if ((ret = __wt_btcur_prev(cbt, false)) != WT_NOTFOUND) exact = -1; } +err: if (ret == 0 && exactp != NULL) + *exactp = exact; + #ifdef HAVE_DIAGNOSTIC if (ret == 0) - WT_ERR(__wt_cursor_key_order_init(session, cbt)); + WT_TRET(__wt_cursor_key_order_init(session, cbt)); #endif -err: if (ret != 0) + if (ret != 0) { WT_TRET(__cursor_reset(cbt)); - if (exactp != NULL && (ret == 0 || ret == WT_NOTFOUND)) - *exactp = exact; + __cursor_state_restore(cursor, &state); + } return (ret); } @@ -490,9 +625,11 @@ int __wt_btcur_insert(WT_CURSOR_BTREE *cbt) { WT_BTREE *btree; + WT_CURFILE_STATE state; WT_CURSOR *cursor; WT_DECL_RET; WT_SESSION_IMPL *session; + bool append_key; btree = cbt->btree; cursor = &cbt->iface; @@ -503,30 +640,86 @@ __wt_btcur_insert(WT_CURSOR_BTREE *cbt) WT_STAT_DATA_INCRV(session, cursor_insert_bytes, cursor->key.size + cursor->value.size); + __cursor_state_save(cursor, &state); + if (btree->type == BTREE_ROW) WT_RET(__cursor_size_chk(session, &cursor->key)); WT_RET(__cursor_size_chk(session, &cursor->value)); + /* It's no longer possible to bulk-load into the tree. */ + __cursor_disable_bulk(session, btree); + + /* + * Insert a new record if WT_CURSTD_APPEND configured, (ignoring any + * application set record number). Although append can't be configured + * for a row-store, this code would break if it were, and that's owned + * by the upper cursor layer, be cautious. + */ + append_key = + F_ISSET(cursor, WT_CURSTD_APPEND) && btree->type != BTREE_ROW; + /* - * The tree is no longer empty: eviction should pay attention to it, - * and it's no longer possible to bulk-load into it. + * If inserting with overwrite configured, and positioned to an on-page + * key, the update doesn't require another search. The cursor won't be + * positioned on a page with an external key set, but be sure. Cursors + * configured for append aren't included, regardless of whether or not + * they meet all other criteria. */ - if (btree->bulk_load_ok) { - btree->bulk_load_ok = false; - __wt_btree_evictable(session, true); + if (__cursor_page_pinned(cbt) && + F_ISSET_ALL(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_OVERWRITE) && + !append_key) { + WT_ERR(__wt_txn_autocommit_check(session)); + /* + * The cursor position may not be exact (the cursor's comparison + * value not equal to zero). Correct to an exact match so we can + * update whatever we're pointing at. + */ + cbt->compare = 0; + ret = btree->type == BTREE_ROW ? + __cursor_row_modify(session, cbt, false) : + __cursor_col_modify(session, cbt, false); + if (ret == 0) + goto done; + + /* + * The pinned page goes away if we fail for any reason, make + * sure there's a local copy of any key. (Restart could still + * use the pinned page, but that's an unlikely path.) Re-save + * the cursor state: we may retry but eventually fail. + */ + WT_TRET(__cursor_copy_int_key(cursor)); + __cursor_state_save(cursor, &state); + goto err; } -retry: WT_RET(__cursor_func_init(cbt, true)); + /* + * The pinned page goes away if we do a search, make sure there's a + * local copy of any key. Re-save the cursor state: we may retry but + * eventually fail. + */ + WT_ERR(__cursor_copy_int_key(cursor)); + __cursor_state_save(cursor, &state); - switch (btree->type) { - case BTREE_COL_FIX: - case BTREE_COL_VAR: +retry: WT_ERR(__cursor_func_init(cbt, true)); + + if (btree->type == BTREE_ROW) { + WT_ERR(__cursor_row_search(session, cbt, NULL, true)); + /* + * If not overwriting, fail if the key exists, else insert the + * key/value pair. + */ + if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE) && + cbt->compare == 0 && __wt_cursor_valid(cbt, NULL)) + WT_ERR(WT_DUPLICATE_KEY); + + ret = __cursor_row_modify(session, cbt, false); + } else { /* - * If WT_CURSTD_APPEND is set, insert a new record (ignoring - * the application's record number). The real record number - * is assigned by the serialized append operation. + * Optionally insert a new record (ignoring the application's + * record number). The real record number is allocated by the + * serialized append operation. */ - if (F_ISSET(cursor, WT_CURSTD_APPEND)) + if (append_key) cbt->iface.recno = WT_RECNO_OOB; WT_ERR(__cursor_col_search(session, cbt, NULL)); @@ -538,26 +731,14 @@ retry: WT_RET(__cursor_func_init(cbt, true)); * Fail in that case, the record exists. */ if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE) && - ((cbt->compare == 0 && __cursor_valid(cbt, NULL)) || + ((cbt->compare == 0 && __wt_cursor_valid(cbt, NULL)) || (cbt->compare != 0 && __cursor_fix_implicit(btree, cbt)))) WT_ERR(WT_DUPLICATE_KEY); WT_ERR(__cursor_col_modify(session, cbt, false)); - if (F_ISSET(cursor, WT_CURSTD_APPEND)) - cbt->iface.recno = cbt->recno; - break; - case BTREE_ROW: - WT_ERR(__cursor_row_search(session, cbt, NULL, true)); - /* - * If not overwriting, fail if the key exists, else insert the - * key/value pair. - */ - if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE) && - cbt->compare == 0 && __cursor_valid(cbt, NULL)) - WT_ERR(WT_DUPLICATE_KEY); - ret = __cursor_row_modify(session, cbt, false); - break; + if (append_key) + cbt->iface.recno = cbt->recno; } err: if (ret == WT_RESTART) { @@ -565,11 +746,17 @@ err: if (ret == WT_RESTART) { WT_STAT_DATA_INCR(session, cursor_restart); goto retry; } - /* Insert doesn't maintain a position across calls, clear resources. */ - if (ret == 0) - WT_TRET(__curfile_leave(cbt)); + +done: /* Insert doesn't maintain a position across calls, clear resources. */ + if (ret == 0) { + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + if (append_key) + F_SET(cursor, WT_CURSTD_KEY_INT); + } + WT_TRET(__cursor_reset(cbt)); if (ret != 0) - WT_TRET(__cursor_reset(cbt)); + __cursor_state_restore(cursor, &state); + return (ret); } @@ -605,16 +792,15 @@ __curfile_update_check(WT_CURSOR_BTREE *cbt) } /* - * __wt_btcur_update_check -- + * __wt_btcur_insert_check -- * Check whether an update would conflict. * - * This can be used to replace WT_CURSOR::insert or WT_CURSOR::update, so - * they only check for conflicts without updating the tree. It is used to - * maintain snapshot isolation for transactions that span multiple chunks - * in an LSM tree. + * This can replace WT_CURSOR::insert, so it only checks for conflicts without + * updating the tree. It is used to maintain snapshot isolation for transactions + * that span multiple chunks in an LSM tree. */ int -__wt_btcur_update_check(WT_CURSOR_BTREE *cbt) +__wt_btcur_insert_check(WT_CURSOR_BTREE *cbt) { WT_BTREE *btree; WT_CURSOR *cursor; @@ -625,31 +811,35 @@ __wt_btcur_update_check(WT_CURSOR_BTREE *cbt) btree = cbt->btree; session = (WT_SESSION_IMPL *)cursor->session; -retry: WT_RET(__cursor_func_init(cbt, true)); + /* + * The pinned page goes away if we do a search, make sure there's a + * local copy of any key. Unlike most of the btree cursor routines, + * we don't have to save/restore the cursor key state, none of the + * work done here changes the key state. + */ + WT_ERR(__cursor_copy_int_key(cursor)); - switch (btree->type) { - case BTREE_ROW: +retry: WT_ERR(__cursor_func_init(cbt, true)); + + if (btree->type == BTREE_ROW) { WT_ERR(__cursor_row_search(session, cbt, NULL, true)); - /* - * Just check for conflicts. - */ + /* Just check for conflicts. */ ret = __curfile_update_check(cbt); - break; - case BTREE_COL_FIX: - case BTREE_COL_VAR: + } else WT_ERR(__wt_illegal_value(session, NULL)); - break; - } err: if (ret == WT_RESTART) { WT_STAT_CONN_INCR(session, cursor_restart); WT_STAT_DATA_INCR(session, cursor_restart); goto retry; } - WT_TRET(__curfile_leave(cbt)); - if (ret != 0) - WT_TRET(__cursor_reset(cbt)); + + /* Insert doesn't maintain a position across calls, clear resources. */ + if (ret == 0) + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + WT_TRET(__cursor_reset(cbt)); + return (ret); } @@ -661,9 +851,11 @@ int __wt_btcur_remove(WT_CURSOR_BTREE *cbt) { WT_BTREE *btree; + WT_CURFILE_STATE state; WT_CURSOR *cursor; WT_DECL_RET; WT_SESSION_IMPL *session; + bool positioned; btree = cbt->btree; cursor = &cbt->iface; @@ -673,22 +865,80 @@ __wt_btcur_remove(WT_CURSOR_BTREE *cbt) WT_STAT_DATA_INCR(session, cursor_remove); WT_STAT_DATA_INCRV(session, cursor_remove_bytes, cursor->key.size); -retry: WT_RET(__cursor_func_init(cbt, true)); + __cursor_state_save(cursor, &state); - switch (btree->type) { - case BTREE_COL_FIX: - case BTREE_COL_VAR: + /* + * WT_CURSOR.remove has a unique semantic, the cursor stays positioned + * if it starts positioned, otherwise clear the cursor on completion. + */ + positioned = F_ISSET(cursor, WT_CURSTD_KEY_INT); + + /* + * If remove positioned to an on-page key, the remove doesn't require + * another search. We don't care about the "overwrite" configuration + * because regardless of the overwrite setting, any existing record is + * removed, and the record must exist with a positioned cursor. The + * cursor won't be positioned on a page with an external key set, but + * be sure. + */ + if (__cursor_page_pinned(cbt) && F_ISSET(cursor, WT_CURSTD_KEY_INT)) { + WT_ERR(__wt_txn_autocommit_check(session)); + + /* + * The cursor position may not be exact (the cursor's comparison + * value not equal to zero). Correct to an exact match so we can + * remove whatever we're pointing at. + */ + cbt->compare = 0; + ret = btree->type == BTREE_ROW ? + __cursor_row_modify(session, cbt, true) : + __cursor_col_modify(session, cbt, true); + if (ret == 0) + goto done; + + /* + * The pinned page goes away if we fail for any reason, make + * sure there's a local copy of any key. (Restart could still + * use the pinned page, but that's an unlikely path.) Re-save + * the cursor state: we may retry but eventually fail. + */ + WT_TRET(__cursor_copy_int_key(cursor)); + __cursor_state_save(cursor, &state); + goto err; + } + + /* + * The pinned page goes away if we do a search, make sure there's a + * local copy of any key. Re-save the cursor state: we may retry but + * eventually fail. + */ + WT_ERR(__cursor_copy_int_key(cursor)); + __cursor_state_save(cursor, &state); + +retry: WT_ERR(__cursor_func_init(cbt, true)); + + if (btree->type == BTREE_ROW) { + WT_ERR(__cursor_row_search(session, cbt, NULL, false)); + + /* Check whether an update would conflict. */ + WT_ERR(__curfile_update_check(cbt)); + + if (cbt->compare != 0 || !__wt_cursor_valid(cbt, NULL)) + WT_ERR(WT_NOTFOUND); + + ret = __cursor_row_modify(session, cbt, true); + } else { WT_ERR(__cursor_col_search(session, cbt, NULL)); /* * If we find a matching record, check whether an update would * conflict. Do this before checking if the update is visible - * in __cursor_valid, or we can miss conflict. + * in __wt_cursor_valid, or we can miss conflict. */ WT_ERR(__curfile_update_check(cbt)); /* Remove the record if it exists. */ - if (cbt->compare != 0 || !__cursor_valid(cbt, NULL)) { + if (cbt->compare != 0 || !__wt_cursor_valid(cbt, NULL)) { if (!__cursor_fix_implicit(btree, cbt)) WT_ERR(WT_NOTFOUND); /* @@ -704,19 +954,6 @@ retry: WT_RET(__cursor_func_init(cbt, true)); cbt->recno = cursor->recno; } else ret = __cursor_col_modify(session, cbt, true); - break; - case BTREE_ROW: - /* Remove the record if it exists. */ - WT_ERR(__cursor_row_search(session, cbt, NULL, false)); - - /* Check whether an update would conflict. */ - WT_ERR(__curfile_update_check(cbt)); - - if (cbt->compare != 0 || !__cursor_valid(cbt, NULL)) - WT_ERR(WT_NOTFOUND); - - ret = __cursor_row_modify(session, cbt, true); - break; } err: if (ret == WT_RESTART) { @@ -724,15 +961,27 @@ err: if (ret == WT_RESTART) { WT_STAT_DATA_INCR(session, cursor_restart); goto retry; } + /* - * If the cursor is configured to overwrite and the record is not - * found, that is exactly what we want. + * If the cursor is configured to overwrite and the record is not found, + * that is exactly what we want, return success. */ if (F_ISSET(cursor, WT_CURSTD_OVERWRITE) && ret == WT_NOTFOUND) ret = 0; - if (ret != 0) +done: /* + * If the cursor was positioned, it stays positioned, point the cursor + * at an internal copy of the key. Otherwise, there's no position or + * key/value. + */ + if (ret == 0) + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + if (ret == 0 && positioned) + WT_TRET(__wt_key_return(session, cbt)); + else WT_TRET(__cursor_reset(cbt)); + if (ret != 0) + __cursor_state_restore(cursor, &state); return (ret); } @@ -745,6 +994,7 @@ int __wt_btcur_update(WT_CURSOR_BTREE *cbt) { WT_BTREE *btree; + WT_CURFILE_STATE state; WT_CURSOR *cursor; WT_DECL_RET; WT_SESSION_IMPL *session; @@ -757,24 +1007,71 @@ __wt_btcur_update(WT_CURSOR_BTREE *cbt) WT_STAT_DATA_INCR(session, cursor_update); WT_STAT_DATA_INCRV(session, cursor_update_bytes, cursor->value.size); + __cursor_state_save(cursor, &state); + if (btree->type == BTREE_ROW) WT_RET(__cursor_size_chk(session, &cursor->key)); WT_RET(__cursor_size_chk(session, &cursor->value)); + /* It's no longer possible to bulk-load into the tree. */ + __cursor_disable_bulk(session, btree); + /* - * The tree is no longer empty: eviction should pay attention to it, - * and it's no longer possible to bulk-load into it. + * If update positioned to an on-page key, the update doesn't require + * another search. We don't care about the "overwrite" configuration + * because regardless of the overwrite setting, any existing record is + * updated, and the record must exist with a positioned cursor. The + * cursor won't be positioned on a page with an external key set, but + * be sure. */ - if (btree->bulk_load_ok) { - btree->bulk_load_ok = false; - __wt_btree_evictable(session, true); + if (__cursor_page_pinned(cbt) && F_ISSET(cursor, WT_CURSTD_KEY_INT)) { + WT_ERR(__wt_txn_autocommit_check(session)); + /* + * The cursor position may not be exact (the cursor's comparison + * value not equal to zero). Correct to an exact match so we can + * update whatever we're pointing at. + */ + cbt->compare = 0; + ret = btree->type == BTREE_ROW ? + __cursor_row_modify(session, cbt, false) : + __cursor_col_modify(session, cbt, false); + if (ret == 0) + goto done; + + /* + * The pinned page goes away if we fail for any reason, make + * sure there's a local copy of any key. (Restart could still + * use the pinned page, but that's an unlikely path.) Re-save + * the cursor state: we may retry but eventually fail. + */ + WT_TRET(__cursor_copy_int_key(cursor)); + __cursor_state_save(cursor, &state); + goto err; } -retry: WT_RET(__cursor_func_init(cbt, true)); + /* + * The pinned page goes away if we do a search, make sure there's a + * local copy of any key. Re-save the cursor state: we may retry but + * eventually fail. + */ + WT_ERR(__cursor_copy_int_key(cursor)); + __cursor_state_save(cursor, &state); - switch (btree->type) { - case BTREE_COL_FIX: - case BTREE_COL_VAR: +retry: WT_ERR(__cursor_func_init(cbt, true)); + + if (btree->type == BTREE_ROW) { + WT_ERR(__cursor_row_search(session, cbt, NULL, true)); + /* + * If not overwriting, check for conflicts and fail if the key + * does not exist. + */ + if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE)) { + WT_ERR(__curfile_update_check(cbt)); + if (cbt->compare != 0 || !__wt_cursor_valid(cbt, NULL)) + WT_ERR(WT_NOTFOUND); + } + ret = __cursor_row_modify(session, cbt, false); + } else { WT_ERR(__cursor_col_search(session, cbt, NULL)); /* @@ -787,25 +1084,12 @@ retry: WT_RET(__cursor_func_init(cbt, true)); */ if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE)) { WT_ERR(__curfile_update_check(cbt)); - if ((cbt->compare != 0 || !__cursor_valid(cbt, NULL)) && + if ((cbt->compare != 0 || + !__wt_cursor_valid(cbt, NULL)) && !__cursor_fix_implicit(btree, cbt)) WT_ERR(WT_NOTFOUND); } ret = __cursor_col_modify(session, cbt, false); - break; - case BTREE_ROW: - WT_ERR(__cursor_row_search(session, cbt, NULL, true)); - /* - * If not overwriting, check for conflicts and fail if the key - * does not exist. - */ - if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE)) { - WT_ERR(__curfile_update_check(cbt)); - if (cbt->compare != 0 || !__cursor_valid(cbt, NULL)) - WT_ERR(WT_NOTFOUND); - } - ret = __cursor_row_modify(session, cbt, false); - break; } err: if (ret == WT_RESTART) { @@ -822,116 +1106,14 @@ err: if (ret == WT_RESTART) { * To make this work, we add a field to the btree cursor to pass back a * pointer to the modify function's allocated update structure. */ - if (ret == 0) +done: if (ret == 0) WT_TRET(__wt_kv_return(session, cbt, cbt->modify_update)); - if (ret != 0) + if (ret != 0) { WT_TRET(__cursor_reset(cbt)); - return (ret); -} - -/* - * __wt_btcur_next_random -- - * Move to a random record in the tree. There are two algorithms, one - * where we select a record at random from the whole tree on each - * retrieval and one where we first select a record at random from the - * whole tree, and then subsequently sample forward from that location. - * The sampling approach allows us to select reasonably uniform random - * points from unbalanced trees. - */ -int -__wt_btcur_next_random(WT_CURSOR_BTREE *cbt) -{ - WT_BTREE *btree; - WT_DECL_RET; - WT_SESSION_IMPL *session; - WT_UPDATE *upd; - wt_off_t size; - uint64_t skip; - - session = (WT_SESSION_IMPL *)cbt->iface.session; - btree = cbt->btree; - - /* - * Only supports row-store: applications can trivially select a random - * value from a column-store, if there were any reason to do so. - */ - if (btree->type != BTREE_ROW) - WT_RET_MSG(session, ENOTSUP, - "WT_CURSOR.next_random only supported by row-store tables"); - - WT_STAT_CONN_INCR(session, cursor_next); - WT_STAT_DATA_INCR(session, cursor_next); - - /* - * If retrieving random values without sampling, or we don't have a - * page reference, pick a roughly random leaf page in the tree. - */ - if (cbt->ref == NULL || cbt->next_random_sample_size == 0) { - /* - * Skip past the sample size of the leaf pages in the tree - * between each random key return to compensate for unbalanced - * trees. - * - * Use the underlying file size divided by its block allocation - * size as our guess of leaf pages in the file (this can be - * entirely wrong, as it depends on how many pages are in this - * particular checkpoint, how large the leaf and internal pages - * really are, and other factors). Then, divide that value by - * the configured sample size and increment the final result to - * make sure tiny files don't leave us with a skip value of 0. - * - * !!! - * Ideally, the number would be prime to avoid restart issues. - */ - if (cbt->next_random_sample_size != 0) { - WT_ERR(btree->bm->size(btree->bm, session, &size)); - cbt->next_random_leaf_skip = (uint64_t) - ((size / btree->allocsize) / - cbt->next_random_sample_size) + 1; - } - - /* - * Choose a leaf page from the tree. - */ - WT_ERR(__cursor_func_init(cbt, true)); - WT_WITH_PAGE_INDEX( - session, ret = __wt_row_random_descent(session, cbt)); - WT_ERR(ret); - } else { - /* - * Read through the tree, skipping leaf pages. Be cautious about - * the skip count: if the last leaf page skipped was also the - * last leaf page in the tree, it may be set to zero on return - * with the end-of-walk condition. - * - * Pages read for data sampling aren't "useful"; don't update - * the read generation of pages already in memory, and if a page - * is read, set its generation to a low value so it is evicted - * quickly. - */ - for (skip = - cbt->next_random_leaf_skip; cbt->ref == NULL || skip > 0;) - WT_ERR(__wt_tree_walk_skip(session, &cbt->ref, &skip, - WT_READ_NO_GEN | - WT_READ_SKIP_INTL | WT_READ_WONT_NEED)); + __cursor_state_restore(cursor, &state); } - /* - * Select a random entry from the leaf page. If it's not valid, move to - * the next entry, if that doesn't work, move to the previous entry. - */ - WT_ERR(__wt_row_random_leaf(session, cbt)); - if (__cursor_valid(cbt, &upd)) - WT_ERR(__wt_kv_return(session, cbt, upd)); - else { - if ((ret = __wt_btcur_next(cbt, false)) == WT_NOTFOUND) - ret = __wt_btcur_prev(cbt, false); - WT_ERR(ret); - } - return (0); - -err: WT_TRET(__cursor_reset(cbt)); return (ret); } @@ -1060,9 +1242,12 @@ __cursor_truncate(WT_SESSION_IMPL *session, WT_DECL_RET; /* - * First, call the standard cursor remove method to do a full search and - * re-position the cursor because we don't have a saved copy of the - * page's write generation information, which we need to remove records. + * First, call the cursor search method to re-position the cursor: we + * may not have a cursor position (if the higher-level truncate code + * switched the cursors to have an "external" cursor key, and because + * we don't save a copy of the page's write generation information, + * which we need to remove records. + * * Once that's done, we can delete records without a full search, unless * we encounter a restart error because the page was modified by some * other thread of control; in that case, repeat the full search to @@ -1075,20 +1260,19 @@ __cursor_truncate(WT_SESSION_IMPL *session, * instantiated the end cursor, so we know that page is pinned in memory * and we can proceed without concern. */ -retry: WT_RET(__wt_btcur_remove(start)); +retry: WT_RET(__wt_btcur_search(start)); + WT_ASSERT(session, + F_MASK((WT_CURSOR *)start, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT); + + for (;;) { + if ((ret = rmfunc(session, start, 1)) != 0) + break; - /* - * Reset ret each time through so that we don't loop forever in - * the cursor equals case. - */ - for (ret = 0;;) { if (stop != NULL && __cursor_equals(start, stop)) break; if ((ret = __wt_btcur_next(start, true)) != 0) break; - start->compare = 0; /* Exact match */ - if ((ret = rmfunc(session, start, 1)) != 0) - break; + start->compare = 0; /* Exact match */ } if (ret == WT_RESTART) { @@ -1121,29 +1305,32 @@ __cursor_truncate_fix(WT_SESSION_IMPL *session, * record 37, records 1-36 magically appear. Those records can't be * deleted, which means we have to ignore already "deleted" records. * - * First, call the standard cursor remove method to do a full search and - * re-position the cursor because we don't have a saved copy of the - * page's write generation information, which we need to remove records. + * First, call the cursor search method to re-position the cursor: we + * may not have a cursor position (if the higher-level truncate code + * switched the cursors to have an "external" cursor key, and because + * we don't save a copy of the page's write generation information, + * which we need to remove records. + * * Once that's done, we can delete records without a full search, unless * we encounter a restart error because the page was modified by some * other thread of control; in that case, repeat the full search to * refresh the page's modification information. */ -retry: WT_RET(__wt_btcur_remove(start)); - /* - * Reset ret each time through so that we don't loop forever in - * the cursor equals case. - */ - for (ret = 0;;) { +retry: WT_RET(__wt_btcur_search(start)); + WT_ASSERT(session, + F_MASK((WT_CURSOR *)start, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT); + + for (;;) { + value = (const uint8_t *)start->iface.value.data; + if (*value != 0 && + (ret = rmfunc(session, start, 1)) != 0) + break; + if (stop != NULL && __cursor_equals(start, stop)) break; if ((ret = __wt_btcur_next(start, true)) != 0) break; start->compare = 0; /* Exact match */ - value = (const uint8_t *)start->iface.value.data; - if (*value != 0 && - (ret = rmfunc(session, start, 1)) != 0) - break; } if (ret == WT_RESTART) { @@ -1263,7 +1450,7 @@ __wt_btcur_close(WT_CURSOR_BTREE *cbt, bool lowlevel) * Skip the usual cursor tear-down in that case. */ if (!lowlevel) - ret = __curfile_leave(cbt); + ret = __cursor_reset(cbt); __wt_buf_free(session, &cbt->_row_key); __wt_buf_free(session, &cbt->_tmp); diff --git a/src/third_party/wiredtiger/src/btree/bt_debug.c b/src/third_party/wiredtiger/src/btree/bt_debug.c index bcc7d27a569..fdc33b608ec 100644 --- a/src/third_party/wiredtiger/src/btree/bt_debug.c +++ b/src/third_party/wiredtiger/src/btree/bt_debug.c @@ -34,7 +34,7 @@ static const /* Output separator */ static int __debug_cell(WT_DBG *, const WT_PAGE_HEADER *, WT_CELL_UNPACK *); static int __debug_cell_data( - WT_DBG *, WT_PAGE *, int type, const char *, WT_CELL_UNPACK *); + WT_DBG *, WT_PAGE *, int, const char *, WT_CELL_UNPACK *); static int __debug_col_skip(WT_DBG *, WT_INSERT_HEAD *, const char *, bool); static int __debug_config(WT_SESSION_IMPL *, WT_DBG *, const char *); static int __debug_dsk_cell(WT_DBG *, const WT_PAGE_HEADER *); @@ -64,7 +64,7 @@ __wt_debug_set_verbose(WT_SESSION_IMPL *session, const char *v) const char *cfg[2] = { NULL, NULL }; char buf[256]; - snprintf(buf, sizeof(buf), "verbose=[%s]", v); + WT_RET(__wt_snprintf(buf, sizeof(buf), "verbose=[%s]", v)); cfg[0] = buf; return (__wt_verbose_config(session, cfg)); } @@ -77,7 +77,7 @@ static inline int __debug_hex_byte(WT_DBG *ds, uint8_t v) { return (ds->f( - ds, "#%c%c", __wt_hex[(v & 0xf0) >> 4], __wt_hex[v & 0x0f])); + ds, "#%c%c", __wt_hex((v & 0xf0) >> 4), __wt_hex(v & 0x0f))); } /* @@ -87,6 +87,7 @@ __debug_hex_byte(WT_DBG *ds, uint8_t v) static int __dmsg_event(WT_DBG *ds, const char *fmt, ...) { + WT_DECL_RET; WT_ITEM *msg; WT_SESSION_IMPL *session; size_t len, space; @@ -107,8 +108,9 @@ __dmsg_event(WT_DBG *ds, const char *fmt, ...) p = (char *)msg->mem + msg->size; space = msg->memsize - msg->size; va_start(ap, fmt); - len = (size_t)vsnprintf(p, space, fmt, ap); + ret = __wt_vsnprintf_len_set(p, space, &len, fmt, ap); va_end(ap); + WT_RET(ret); /* Check if there was enough space. */ if (len < space) { @@ -447,13 +449,14 @@ __debug_tree_shape_info(WT_PAGE *page) v = page->memory_footprint; if (v >= WT_GIGABYTE) - snprintf(buf, sizeof(buf), + (void)__wt_snprintf(buf, sizeof(buf), "(%p %" PRIu64 "G)", (void *)page, v / WT_GIGABYTE); else if (v >= WT_MEGABYTE) - snprintf(buf, sizeof(buf), + (void)__wt_snprintf(buf, sizeof(buf), "(%p %" PRIu64 "M)", (void *)page, v / WT_MEGABYTE); else - snprintf(buf, sizeof(buf), "(%p %" PRIu64 ")", (void *)page, v); + (void)__wt_snprintf(buf, sizeof(buf), + "(%p %" PRIu64 ")", (void *)page, v); return (buf); } @@ -652,7 +655,7 @@ __debug_page_metadata(WT_DBG *ds, WT_REF *ref) page = ref->page; mod = page->modify; - WT_RET(ds->f(ds, "%p", (void *)page)); + WT_RET(ds->f(ds, "%p", (void *)ref)); switch (page->type) { case WT_PAGE_COL_INT: @@ -662,29 +665,30 @@ __debug_page_metadata(WT_DBG *ds, WT_REF *ref) break; case WT_PAGE_COL_FIX: WT_RET(ds->f(ds, " recno %" PRIu64, ref->ref_recno)); - entries = page->pg_fix_entries; + entries = page->entries; break; case WT_PAGE_COL_VAR: WT_RET(ds->f(ds, " recno %" PRIu64, ref->ref_recno)); - entries = page->pg_var_entries; + entries = page->entries; break; case WT_PAGE_ROW_INT: WT_INTL_INDEX_GET(session, page, pindex); entries = pindex->entries; break; case WT_PAGE_ROW_LEAF: - entries = page->pg_row_entries; + entries = page->entries; break; WT_ILLEGAL_VALUE(session); } WT_RET(ds->f(ds, ": %s\n", __wt_page_type_string(page->type))); - WT_RET(ds->f(ds, - "\t" "disk %p, entries %" PRIu32, (void *)page->dsk, entries)); + WT_RET(ds->f(ds, "\t" "disk %p", (void *)page->dsk)); + if (page->dsk != NULL) + WT_RET(ds->f( + ds, ", dsk_mem_size %" PRIu32, page->dsk->mem_size)); + WT_RET(ds->f(ds, ", entries %" PRIu32, entries)); WT_RET(ds->f(ds, ", %s", __wt_page_is_modified(page) ? "dirty" : "clean")); - WT_RET(ds->f(ds, ", %s", __wt_rwlock_islocked( - session, &page->page_lock) ? "locked" : "unlocked")); if (F_ISSET_ATOMIC(page, WT_PAGE_BUILD_KEYS)) WT_RET(ds->f(ds, ", keys-built")); @@ -696,8 +700,6 @@ __debug_page_metadata(WT_DBG *ds, WT_REF *ref) WT_RET(ds->f(ds, ", evict-lru")); if (F_ISSET_ATOMIC(page, WT_PAGE_OVERFLOW_KEYS)) WT_RET(ds->f(ds, ", overflow-keys")); - if (F_ISSET_ATOMIC(page, WT_PAGE_SPLIT_BLOCK)) - WT_RET(ds->f(ds, ", split-block")); if (F_ISSET_ATOMIC(page, WT_PAGE_SPLIT_INSERT)) WT_RET(ds->f(ds, ", split-insert")); if (F_ISSET_ATOMIC(page, WT_PAGE_UPDATE_IGNORE)) @@ -837,7 +839,8 @@ __debug_page_col_var(WT_DBG *ds, WT_REF *ref) __wt_cell_unpack(cell, unpack); rle = __wt_cell_rle(unpack); } - snprintf(tag, sizeof(tag), "%" PRIu64 " %" PRIu64, recno, rle); + WT_RET(__wt_snprintf( + tag, sizeof(tag), "%" PRIu64 " %" PRIu64, recno, rle)); WT_RET( __debug_cell_data(ds, page, WT_PAGE_COL_VAR, tag, unpack)); @@ -1003,37 +1006,37 @@ __debug_ref(WT_DBG *ds, WT_REF *ref) WT_SESSION_IMPL *session; size_t addr_size; const uint8_t *addr; + const char *state; session = ds->session; - WT_RET(ds->f(ds, "\t")); switch (ref->state) { case WT_REF_DISK: - WT_RET(ds->f(ds, "disk")); + state = "disk"; break; case WT_REF_DELETED: - WT_RET(ds->f(ds, "deleted")); + state = "deleted"; break; case WT_REF_LOCKED: - WT_RET(ds->f(ds, "locked %p", (void *)ref->page)); + state = "locked"; break; case WT_REF_MEM: - WT_RET(ds->f(ds, "memory %p", (void *)ref->page)); + state = "memory"; break; case WT_REF_READING: - WT_RET(ds->f(ds, "reading")); + state = "reading"; break; case WT_REF_SPLIT: - WT_RET(ds->f(ds, "split")); + state = "split"; break; default: - WT_RET(ds->f(ds, "INVALID")); + state = "INVALID"; break; } __wt_ref_info(ref, &addr, &addr_size, NULL); - return (ds->f(ds, " %s\n", - __wt_addr_string(session, addr, addr_size, ds->tmp))); + return (ds->f(ds, "\t" "%p %s %s\n", (void *)ref, + state, __wt_addr_string(session, addr, addr_size, ds->tmp))); } /* @@ -1104,9 +1107,9 @@ __debug_cell(WT_DBG *ds, const WT_PAGE_HEADER *dsk, WT_CELL_UNPACK *unpack) case WT_CELL_VALUE_OVFL_RM: type = "ovfl"; addr: WT_RET(__wt_scr_alloc(session, 128, &buf)); - WT_RET(ds->f(ds, ", %s %s", type, + ret = ds->f(ds, ", %s %s", type, __wt_addr_string( - session, unpack->data, unpack->size, buf))); + session, unpack->data, unpack->size, buf)); __wt_scr_free(session, &buf); WT_RET(ret); break; diff --git a/src/third_party/wiredtiger/src/btree/bt_delete.c b/src/third_party/wiredtiger/src/btree/bt_delete.c index 00e41475de9..b55ad291c5e 100644 --- a/src/third_party/wiredtiger/src/btree/bt_delete.c +++ b/src/third_party/wiredtiger/src/btree/bt_delete.c @@ -318,13 +318,12 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) * hard case is if a page splits: the update structures might be moved * to different pages, and we still have to find them all for an abort. */ - if (page_del != NULL) WT_RET(__wt_calloc_def( - session, page->pg_row_entries + 1, &page_del->update_list)); + session, page->entries + 1, &page_del->update_list)); /* Allocate the per-page update array. */ - WT_ERR(__wt_calloc_def(session, page->pg_row_entries, &upd_array)); + WT_ERR(__wt_calloc_def(session, page->entries, &upd_array)); page->modify->mod_row_update = upd_array; /* @@ -332,7 +331,7 @@ __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) * structures, fill in the per-page update array with references to * deleted items. */ - for (i = 0, size = 0; i < page->pg_row_entries; ++i) { + for (i = 0, size = 0; i < page->entries; ++i) { WT_ERR(__wt_calloc_one(session, &upd)); WT_UPDATE_DELETED_SET(upd); diff --git a/src/third_party/wiredtiger/src/btree/bt_discard.c b/src/third_party/wiredtiger/src/btree/bt_discard.c index 162bc07a1c2..04c0a5d410d 100644 --- a/src/third_party/wiredtiger/src/btree/bt_discard.c +++ b/src/third_party/wiredtiger/src/btree/bt_discard.c @@ -16,29 +16,66 @@ static void __free_skip_array( WT_SESSION_IMPL *, WT_INSERT_HEAD **, uint32_t, bool); static void __free_skip_list(WT_SESSION_IMPL *, WT_INSERT *, bool); static void __free_update(WT_SESSION_IMPL *, WT_UPDATE **, uint32_t, bool); +static void __page_out_int(WT_SESSION_IMPL *, WT_PAGE **, bool); /* - * __wt_ref_out -- + * __wt_ref_out_int -- * Discard an in-memory page, freeing all memory associated with it. */ void -__wt_ref_out(WT_SESSION_IMPL *session, WT_REF *ref) +__wt_ref_out_int(WT_SESSION_IMPL *session, WT_REF *ref, bool rewrite) { /* * A version of the page-out function that allows us to make additional * diagnostic checks. + * + * The WT_REF cannot be the eviction thread's location. */ WT_ASSERT(session, S2BT(session)->evict_ref != ref); - __wt_page_out(session, &ref->page); +#ifdef HAVE_DIAGNOSTIC + { + WT_HAZARD *hp; + int i; + /* + * Make sure no other thread has a hazard pointer on the page we are + * about to discard. This is complicated by the fact that readers + * publish their hazard pointer before re-checking the page state, so + * our check can race with readers without indicating a real problem. + * Wait for up to a second for hazard pointers to be cleared. + */ + for (hp = NULL, i = 0; i < 100; i++) { + if ((hp = __wt_hazard_check(session, ref)) == NULL) + break; + __wt_sleep(0, 10000); + } + if (hp != NULL) + __wt_errx(session, + "discarded page has hazard pointer: (%p: %s, line %d)", + (void *)hp->ref, hp->file, hp->line); + WT_ASSERT(session, hp == NULL); + } +#endif + + __page_out_int(session, &ref->page, rewrite); } /* - * __wt_page_out -- + * __wt_ref_out -- * Discard an in-memory page, freeing all memory associated with it. */ void -__wt_page_out(WT_SESSION_IMPL *session, WT_PAGE **pagep) +__wt_ref_out(WT_SESSION_IMPL *session, WT_REF *ref) +{ + __wt_ref_out_int(session, ref, false); +} + +/* + * __page_out_int -- + * Discard an in-memory page, freeing all memory associated with it. + */ +static void +__page_out_int(WT_SESSION_IMPL *session, WT_PAGE **pagep, bool rewrite) { WT_PAGE *page; WT_PAGE_HEADER *dsk; @@ -61,31 +98,6 @@ __wt_page_out(WT_SESSION_IMPL *session, WT_PAGE **pagep) */ WT_ASSERT(session, !__wt_page_is_modified(page)); WT_ASSERT(session, !F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU)); - WT_ASSERT(session, !__wt_rwlock_islocked(session, &page->page_lock)); - -#ifdef HAVE_DIAGNOSTIC - { - WT_HAZARD *hp; - int i; - /* - * Make sure no other thread has a hazard pointer on the page we are - * about to discard. This is complicated by the fact that readers - * publish their hazard pointer before re-checking the page state, so - * our check can race with readers without indicating a real problem. - * Wait for up to a second for hazard pointers to be cleared. - */ - for (hp = NULL, i = 0; i < 100; i++) { - if ((hp = __wt_page_hazard_check(session, page)) == NULL) - break; - __wt_sleep(0, 10000); - } - if (hp != NULL) - __wt_errx(session, - "discarded page has hazard pointer: (%p: %s, line %d)", - (void *)hp->page, hp->file, hp->line); - WT_ASSERT(session, hp == NULL); - } -#endif /* * If a root page split, there may be one or more pages linked from the @@ -101,7 +113,7 @@ __wt_page_out(WT_SESSION_IMPL *session, WT_PAGE **pagep) } /* Update the cache's information. */ - __wt_cache_page_evict(session, page); + __wt_cache_page_evict(session, page, rewrite); dsk = (WT_PAGE_HEADER *)page->dsk; if (F_ISSET_ATOMIC(page, WT_PAGE_DISK_ALLOC)) @@ -145,6 +157,16 @@ __wt_page_out(WT_SESSION_IMPL *session, WT_PAGE **pagep) __wt_overwrite_and_free(session, page); } +/* + * __wt_page_out -- + * Discard an in-memory page, freeing all memory associated with it. + */ +void +__wt_page_out(WT_SESSION_IMPL *session, WT_PAGE **pagep) +{ + __page_out_int(session, pagep, false); +} + /* * __free_page_modify -- * Discard the page's associated modification structures. @@ -204,8 +226,7 @@ __free_page_modify(WT_SESSION_IMPL *session, WT_PAGE *page) if (mod->mod_col_update != NULL) __free_skip_array(session, mod->mod_col_update, page->type == - WT_PAGE_COL_FIX ? 1 : page->pg_var_entries, - update_ignore); + WT_PAGE_COL_FIX ? 1 : page->entries, update_ignore); break; case WT_PAGE_ROW_LEAF: /* @@ -217,12 +238,12 @@ __free_page_modify(WT_SESSION_IMPL *session, WT_PAGE *page) */ if (mod->mod_row_insert != NULL) __free_skip_array(session, mod->mod_row_insert, - page->pg_row_entries + 1, update_ignore); + page->entries + 1, update_ignore); /* Free the update array. */ if (mod->mod_row_update != NULL) __free_update(session, mod->mod_row_update, - page->pg_row_entries, update_ignore); + page->entries, update_ignore); break; } @@ -232,6 +253,7 @@ __free_page_modify(WT_SESSION_IMPL *session, WT_PAGE *page) __wt_ovfl_discard_free(session, page); __wt_free(session, page->modify->ovfl_track); + __wt_spin_destroy(session, &page->modify->page_lock); __wt_free(session, page->modify); } @@ -330,7 +352,7 @@ static void __free_page_col_var(WT_SESSION_IMPL *session, WT_PAGE *page) { /* Free the RLE lookup array. */ - __wt_free(session, page->pg_var_repeats); + __wt_free(session, page->u.col_var.repeats); } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index 337a3ea036f..a0da7df0998 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -14,6 +14,44 @@ static int __btree_page_sizes(WT_SESSION_IMPL *); static int __btree_preload(WT_SESSION_IMPL *); static int __btree_tree_open_empty(WT_SESSION_IMPL *, bool); +/* + * __btree_clear -- + * Clear a Btree, either on handle discard or re-open. + */ +static int +__btree_clear(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + WT_DECL_RET; + + btree = S2BT(session); + + /* + * If the tree hasn't gone through an open/close cycle, there's no + * cleanup to be done. + */ + if (!F_ISSET(btree, WT_BTREE_CLOSED)) + return (0); + + /* Close the Huffman tree. */ + __wt_btree_huffman_close(session); + + /* Terminate any associated collator. */ + if (btree->collator_owned && btree->collator->terminate != NULL) + WT_TRET(btree->collator->terminate( + btree->collator, &session->iface)); + + /* Destroy locks. */ + __wt_rwlock_destroy(session, &btree->ovfl_lock); + __wt_spin_destroy(session, &btree->flush_lock); + + /* Free allocated memory. */ + __wt_free(session, btree->key_format); + __wt_free(session, btree->value_format); + + return (ret); +} + /* * __wt_btree_open -- * Open a Btree. @@ -28,12 +66,27 @@ __wt_btree_open(WT_SESSION_IMPL *session, const char *op_cfg[]) WT_DATA_HANDLE *dhandle; WT_DECL_RET; size_t root_addr_size; + uint32_t mask; uint8_t root_addr[WT_BTREE_MAX_ADDR_COOKIE]; const char *filename; bool creation, forced_salvage, readonly; - dhandle = session->dhandle; btree = S2BT(session); + dhandle = session->dhandle; + + /* + * This may be a re-open of an underlying object and we have to clean + * up. We can't clear the operation flags, however, they're set by the + * connection handle software that called us. + */ + WT_RET(__btree_clear(session)); + + mask = F_MASK(btree, WT_BTREE_SPECIAL_FLAGS); + memset(btree, 0, sizeof(*btree)); + btree->flags = mask; + + /* Set the data handle first, our called functions reasonably use it. */ + btree->dhandle = dhandle; /* Checkpoint files are readonly. */ readonly = dhandle->checkpoint != NULL || @@ -126,6 +179,20 @@ __wt_btree_open(WT_SESSION_IMPL *session, const char *op_cfg[]) } } + /* + * Eviction ignores trees until the handle's open flag is set, configure + * eviction before that happens. + * + * Files that can still be bulk-loaded cannot be evicted. + * Permanently cache-resident files can never be evicted. + * Special operations don't enable eviction. (The underlying commands + * may turn on eviction, but it's their decision.) + */ + if (btree->original || + F_ISSET(btree, WT_BTREE_IN_MEMORY | WT_BTREE_REBALANCE | + WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY)) + WT_ERR(__wt_evict_file_exclusive_on(session)); + if (0) { err: WT_TRET(__wt_btree_close(session)); } @@ -147,7 +214,24 @@ __wt_btree_close(WT_SESSION_IMPL *session) btree = S2BT(session); + /* + * The close process isn't the same as discarding the handle: we might + * re-open the handle, which isn't a big deal, but the backing blocks + * for the handle may not yet have been discarded from the cache, and + * eviction uses WT_BTREE structure elements. Free backing resources + * but leave the rest alone, and we'll discard the structure when we + * discard the data handle. + * + * Handles can be closed multiple times, ignore all but the first. + */ + if (F_ISSET(btree, WT_BTREE_CLOSED)) + return (0); + F_SET(btree, WT_BTREE_CLOSED); + + /* Discard any underlying block manager resources. */ if ((bm = btree->bm) != NULL) { + btree->bm = NULL; + /* Unload the checkpoint, unless it's a special command. */ if (!F_ISSET(btree, WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY)) @@ -155,33 +239,26 @@ __wt_btree_close(WT_SESSION_IMPL *session) /* Close the underlying block manager reference. */ WT_TRET(bm->close(bm, session)); - - btree->bm = NULL; } - /* Close the Huffman tree. */ - __wt_btree_huffman_close(session); - - /* Destroy locks. */ - __wt_rwlock_destroy(session, &btree->ovfl_lock); - __wt_spin_destroy(session, &btree->flush_lock); - - /* Free allocated memory. */ - __wt_free(session, btree->key_format); - __wt_free(session, btree->value_format); + return (ret); +} - if (btree->collator_owned) { - if (btree->collator->terminate != NULL) - WT_TRET(btree->collator->terminate( - btree->collator, &session->iface)); - btree->collator_owned = 0; - } - btree->collator = NULL; - btree->kencryptor = NULL; +/* + * __wt_btree_discard -- + * Discard a Btree. + */ +int +__wt_btree_discard(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + WT_DECL_RET; - btree->bulk_load_ok = false; + ret = __btree_clear(session); - F_CLR(btree, WT_BTREE_SPECIAL_FLAGS); + btree = S2BT(session); + __wt_overwrite_and_free(session, btree); + session->dhandle->handle = NULL; return (ret); } @@ -267,9 +344,9 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) WT_RET(__wt_config_gets(session, cfg, "cache_resident", &cval)); if (cval.val) - F_SET(btree, WT_BTREE_IN_MEMORY | WT_BTREE_NO_EVICTION); + F_SET(btree, WT_BTREE_IN_MEMORY); else - F_CLR(btree, WT_BTREE_IN_MEMORY | WT_BTREE_NO_EVICTION); + F_CLR(btree, WT_BTREE_IN_MEMORY); WT_RET(__wt_config_gets(session, cfg, "ignore_in_memory_cache_size", &cval)); @@ -282,6 +359,14 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) } else F_CLR(btree, WT_BTREE_IGNORE_CACHE); + /* + * The metadata isn't blocked by in-memory cache limits because metadata + * "unroll" is performed by updates that are potentially blocked by the + * cache-full checks. + */ + if (WT_IS_METADATA(btree->dhandle)) + F_SET(btree, WT_BTREE_IGNORE_CACHE); + WT_RET(__wt_config_gets(session, cfg, "log.enabled", &cval)); if (cval.val) F_CLR(btree, WT_BTREE_NO_LOGGING); @@ -341,7 +426,7 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) * always inherit from the connection. */ WT_RET(__wt_config_gets(session, cfg, "encryption.name", &cval)); - if (WT_IS_METADATA(session, btree->dhandle) || cval.len == 0) + if (WT_IS_METADATA(btree->dhandle) || cval.len == 0) btree->kencryptor = conn->kencryptor; else if (WT_STRING_MATCH("none", cval.str, cval.len)) btree->kencryptor = NULL; @@ -359,8 +444,7 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) } /* Initialize locks. */ - WT_RET(__wt_rwlock_alloc( - session, &btree->ovfl_lock, "btree overflow lock")); + WT_RET(__wt_rwlock_init(session, &btree->ovfl_lock)); WT_RET(__wt_spin_init(session, &btree->flush_lock, "btree flush")); btree->checkpointing = WT_CKPT_OFF; /* Not checkpointing */ @@ -432,7 +516,7 @@ __wt_btree_tree_open( * Failure to open metadata means that the database is unavailable. * Try to provide a helpful failure message. */ - if (ret != 0 && WT_IS_METADATA(session, session->dhandle)) { + if (ret != 0 && WT_IS_METADATA(session->dhandle)) { __wt_errx(session, "WiredTiger has failed to open its metadata"); __wt_errx(session, "This may be due to the database" @@ -483,13 +567,10 @@ __btree_tree_open_empty(WT_SESSION_IMPL *session, bool creation) /* * Newly created objects can be used for cursor inserts or for bulk * loads; set a flag that's cleared when a row is inserted into the - * tree. Objects being bulk-loaded cannot be evicted, we set it - * globally, there's no point in searching empty trees for eviction. + * tree. */ - if (creation) { - btree->bulk_load_ok = true; - __wt_btree_evictable(session, false); - } + if (creation) + btree->original = 1; /* * A note about empty trees: the initial tree is a single root page. @@ -581,27 +662,6 @@ __wt_btree_new_leaf_page(WT_SESSION_IMPL *session, WT_PAGE **pagep) return (0); } -/* - * __wt_btree_evictable -- - * Setup or release a cache-resident tree. - */ -void -__wt_btree_evictable(WT_SESSION_IMPL *session, bool on) -{ - WT_BTREE *btree; - - btree = S2BT(session); - - /* Permanently cache-resident files can never be evicted. */ - if (F_ISSET(btree, WT_BTREE_IN_MEMORY)) - return; - - if (on) - F_CLR(btree, WT_BTREE_NO_EVICTION); - else - F_SET(btree, WT_BTREE_NO_EVICTION); -} - /* * __btree_preload -- * Pre-load internal pages. @@ -728,9 +788,16 @@ __btree_page_sizes(WT_SESSION_IMPL *session) * Get the split percentage (reconciliation splits pages into smaller * than the maximum page size chunks so we don't split every time a * new entry is added). Determine how large newly split pages will be. + * Set to the minimum, if the read value is less than that. */ WT_RET(__wt_config_gets(session, cfg, "split_pct", &cval)); - btree->split_pct = (int)cval.val; + if (cval.val < WT_BTREE_MIN_SPLIT_PCT) { + btree->split_pct = WT_BTREE_MIN_SPLIT_PCT; + WT_RET(__wt_msg(session, + "Re-setting split_pct for %s to the minimum allowed of " + "%d%%.", session->dhandle->name, WT_BTREE_MIN_SPLIT_PCT)); + } else + btree->split_pct = (int)cval.val; intl_split_size = __wt_split_page_size(btree, btree->maxintlpage); leaf_split_size = __wt_split_page_size(btree, btree->maxleafpage); diff --git a/src/third_party/wiredtiger/src/btree/bt_io.c b/src/third_party/wiredtiger/src/btree/bt_io.c index a8645f79dbe..b5e4d52394a 100644 --- a/src/third_party/wiredtiger/src/btree/bt_io.c +++ b/src/third_party/wiredtiger/src/btree/bt_io.c @@ -183,7 +183,7 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t dst_len, len, result_len, size, src_len; int compression_failed; /* Extension API, so not a bool. */ uint8_t *dst, *src; - bool data_checksum, encrypted; + bool data_checksum, encrypted, timer; btree = S2BT(session); bm = btree->bm; @@ -216,7 +216,7 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, &result_len)); WT_ASSERT(session, dsk->mem_size == result_len + WT_BLOCK_COMPRESS_SKIP); - ctmp->size = (uint32_t)result_len + WT_BLOCK_COMPRESS_SKIP; + ctmp->size = result_len + WT_BLOCK_COMPRESS_SKIP; ip = ctmp; } else { WT_ASSERT(session, dsk->mem_size == buf->size); @@ -357,7 +357,8 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, data_checksum = !compressed; break; } - if (!F_ISSET(session, WT_SESSION_INTERNAL)) + timer = !F_ISSET(session, WT_SESSION_INTERNAL); + if (timer) __wt_epoch(session, &start); /* Call the block manager to write the block. */ @@ -367,7 +368,7 @@ __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, bm, session, ip, addr, addr_sizep, data_checksum, checkpoint_io)); /* Update some statistics now that the write is done */ - if (!F_ISSET(session, WT_SESSION_INTERNAL)) { + if (timer) { __wt_epoch(session, &stop); WT_STAT_CONN_INCR(session, cache_write_app_count); WT_STAT_CONN_INCRV(session, cache_write_app_time, diff --git a/src/third_party/wiredtiger/src/btree/bt_misc.c b/src/third_party/wiredtiger/src/btree/bt_misc.c index b6e2cc07f5a..3bec65c2567 100644 --- a/src/third_party/wiredtiger/src/btree/bt_misc.c +++ b/src/third_party/wiredtiger/src/btree/bt_misc.c @@ -14,6 +14,7 @@ */ const char * __wt_page_type_string(u_int type) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { switch (type) { case WT_PAGE_INVALID: diff --git a/src/third_party/wiredtiger/src/btree/bt_ovfl.c b/src/third_party/wiredtiger/src/btree/bt_ovfl.c index 29ea561db3a..ae0da62af57 100644 --- a/src/third_party/wiredtiger/src/btree/bt_ovfl.c +++ b/src/third_party/wiredtiger/src/btree/bt_ovfl.c @@ -67,11 +67,11 @@ __wt_ovfl_read(WT_SESSION_IMPL *session, * Acquire the overflow lock, and retest the on-page cell's value inside * the lock. */ - __wt_readlock(session, S2BT(session)->ovfl_lock); + __wt_readlock(session, &S2BT(session)->ovfl_lock); ret = __wt_cell_type_raw(unpack->cell) == WT_CELL_VALUE_OVFL_RM ? __wt_ovfl_txnc_search(page, unpack->data, unpack->size, store) : __ovfl_read(session, unpack->data, unpack->size, store); - __wt_readunlock(session, S2BT(session)->ovfl_lock); + __wt_readunlock(session, &S2BT(session)->ovfl_lock); return (ret); } @@ -249,7 +249,7 @@ __wt_ovfl_discard(WT_SESSION_IMPL *session, WT_CELL *cell) * Acquire the overflow lock to avoid racing with a thread reading the * backing overflow blocks. */ - __wt_writelock(session, btree->ovfl_lock); + __wt_writelock(session, &btree->ovfl_lock); switch (unpack->raw) { case WT_CELL_KEY_OVFL: @@ -263,7 +263,7 @@ __wt_ovfl_discard(WT_SESSION_IMPL *session, WT_CELL *cell) WT_ILLEGAL_VALUE(session); } - __wt_writeunlock(session, btree->ovfl_lock); + __wt_writeunlock(session, &btree->ovfl_lock); /* Free the backing disk blocks. */ return (bm->free(bm, session, unpack->data, unpack->size)); diff --git a/src/third_party/wiredtiger/src/btree/bt_page.c b/src/third_party/wiredtiger/src/btree/bt_page.c index 7bac7079fe8..f20f6398e37 100644 --- a/src/third_party/wiredtiger/src/btree/bt_page.c +++ b/src/third_party/wiredtiger/src/btree/bt_page.c @@ -67,7 +67,7 @@ __wt_page_alloc(WT_SESSION_IMPL *session, switch (type) { case WT_PAGE_COL_FIX: - page->pg_fix_entries = alloc_entries; + page->entries = alloc_entries; break; case WT_PAGE_COL_INT: case WT_PAGE_ROW_INT: @@ -102,12 +102,12 @@ err: if ((pindex = WT_INTL_INDEX_GET_SAFE(page)) != NULL) { } break; case WT_PAGE_COL_VAR: - page->pg_var_d = (WT_COL *)((uint8_t *)page + sizeof(WT_PAGE)); - page->pg_var_entries = alloc_entries; + page->pg_var = (WT_COL *)((uint8_t *)page + sizeof(WT_PAGE)); + page->entries = alloc_entries; break; case WT_PAGE_ROW_LEAF: - page->pg_row_d = (WT_ROW *)((uint8_t *)page + sizeof(WT_PAGE)); - page->pg_row_entries = alloc_entries; + page->pg_row = (WT_ROW *)((uint8_t *)page + sizeof(WT_PAGE)); + page->entries = alloc_entries; break; WT_ILLEGAL_VALUE(session); } @@ -333,9 +333,10 @@ __inmem_col_var( WT_CELL *cell; WT_CELL_UNPACK *unpack, _unpack; const WT_PAGE_HEADER *dsk; + size_t size; uint64_t rle; - size_t bytes_allocated; uint32_t i, indx, n, repeat_off; + void *p; btree = S2BT(session); dsk = page->dsk; @@ -343,7 +344,6 @@ __inmem_col_var( repeats = NULL; repeat_off = 0; unpack = &_unpack; - bytes_allocated = 0; /* * Walk the page, building references: the page contains unsorted value @@ -351,7 +351,7 @@ __inmem_col_var( * (WT_CELL_VALUE_OVFL) or deleted items (WT_CELL_DEL). */ indx = 0; - cip = page->pg_var_d; + cip = page->pg_var; WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { __wt_cell_unpack(cell, unpack); WT_COL_PTR_SET(cip, WT_PAGE_DISK_OFFSET(page, cell)); @@ -367,12 +367,14 @@ __inmem_col_var( if (rle > 1) { if (repeats == NULL) { __inmem_col_var_repeats(session, page, &n); - WT_RET(__wt_realloc_def(session, - &bytes_allocated, n + 1, &repeats)); + size = sizeof(WT_COL_VAR_REPEAT) + + (n + 1) * sizeof(WT_COL_RLE); + WT_RET(__wt_calloc(session, 1, size, &p)); + *sizep += size; - page->pg_var_repeats = repeats; + page->u.col_var.repeats = p; page->pg_var_nrepeats = n; - *sizep += bytes_allocated; + repeats = page->pg_var_repeats; } repeats[repeat_off].indx = indx; repeats[repeat_off].recno = recno; @@ -569,7 +571,7 @@ __inmem_row_leaf(WT_SESSION_IMPL *session, WT_PAGE *page) unpack = &_unpack; /* Walk the page, building indices. */ - rip = page->pg_row_d; + rip = page->pg_row; WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { __wt_cell_unpack(cell, unpack); switch (unpack->type) { diff --git a/src/third_party/wiredtiger/src/btree/bt_random.c b/src/third_party/wiredtiger/src/btree/bt_random.c new file mode 100644 index 00000000000..c5948ec4ab5 --- /dev/null +++ b/src/third_party/wiredtiger/src/btree/bt_random.c @@ -0,0 +1,432 @@ +/*- + * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_row_random_leaf -- + * Return a random key from a row-store leaf page. + */ +int +__wt_row_random_leaf(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) +{ + WT_INSERT *ins, **start, **stop; + WT_INSERT_HEAD *ins_head; + WT_PAGE *page; + uint64_t samples; + uint32_t choice, entries, i; + int level; + + page = cbt->ref->page; + start = stop = NULL; /* [-Wconditional-uninitialized] */ + entries = 0; /* [-Wconditional-uninitialized] */ + + __cursor_pos_clear(cbt); + + /* If the page has disk-based entries, select from them. */ + if (page->entries != 0) { + cbt->compare = 0; + cbt->slot = __wt_random(&session->rnd) % page->entries; + + /* + * The real row-store search function builds the key, so we + * have to as well. + */ + return (__wt_row_leaf_key(session, + page, page->pg_row + cbt->slot, cbt->tmp, false)); + } + + /* + * If the tree is new (and not empty), it might have a large insert + * list. + * + * Walk down the list until we find a level with at least 50 entries, + * that's where we'll start rolling random numbers. The value 50 is + * used to ignore levels with only a few entries, that is, levels which + * are potentially badly skewed. + */ + F_SET(cbt, WT_CBT_SEARCH_SMALLEST); + if ((ins_head = WT_ROW_INSERT_SMALLEST(page)) == NULL) + return (WT_NOTFOUND); + for (level = WT_SKIP_MAXDEPTH - 1; level >= 0; --level) { + start = &ins_head->head[level]; + for (entries = 0, stop = start; + *stop != NULL; stop = &(*stop)->next[level]) + ++entries; + + if (entries > 50) + break; + } + + /* + * If it's a tiny list and we went all the way to level 0, correct the + * level; entries is correctly set. + */ + if (level < 0) + level = 0; + + /* + * Step down the skip list levels, selecting a random chunk of the name + * space at each level. + */ + for (samples = entries; level > 0; samples += entries) { + /* + * There are (entries) or (entries + 1) chunks of the name space + * considered at each level. They are: between start and the 1st + * element, between the 1st and 2nd elements, and so on to the + * last chunk which is the name space after the stop element on + * the current level. This last chunk of name space may or may + * not be there: as we descend the levels of the skip list, this + * chunk may appear, depending if the next level down has + * entries logically after the stop point in the current level. + * We can't ignore those entries: because of the algorithm used + * to determine the depth of a skiplist, there may be a large + * number of entries "revealed" by descending a level. + * + * If the next level down has more items after the current stop + * point, there are (entries + 1) chunks to consider, else there + * are (entries) chunks. + */ + if (*(stop - 1) == NULL) + choice = __wt_random(&session->rnd) % entries; + else + choice = __wt_random(&session->rnd) % (entries + 1); + + if (choice == entries) { + /* + * We selected the name space after the stop element on + * this level. Set the start point to the current stop + * point, descend a level and move the stop element to + * the end of the list, that is, the end of the newly + * discovered name space, counting entries as we go. + */ + start = stop; + --start; + --level; + for (entries = 0, stop = start; + *stop != NULL; stop = &(*stop)->next[level]) + ++entries; + } else { + /* + * We selected another name space on the level. Move the + * start pointer the selected number of entries forward + * to the start of the selected chunk (if the selected + * number is 0, start won't move). Set the stop pointer + * to the next element in the list and drop both start + * and stop down a level. + */ + for (i = 0; i < choice; ++i) + start = &(*start)->next[level]; + stop = &(*start)->next[level]; + + --start; + --stop; + --level; + + /* Count the entries in the selected name space. */ + for (entries = 0, + ins = *start; ins != *stop; ins = ins->next[level]) + ++entries; + } + } + + /* + * When we reach the bottom level, entries will already be set. Select + * a random entry from the name space and return it. + * + * It should be impossible for the entries count to be 0 at this point, + * but check for it out of paranoia and to quiet static testing tools. + */ + if (entries > 0) + entries = __wt_random(&session->rnd) % entries; + for (ins = *start; entries > 0; --entries) + ins = ins->next[0]; + + cbt->ins = ins; + cbt->ins_head = ins_head; + cbt->compare = 0; + + /* + * Random lookups in newly created collections can be slow if a page + * consists of a large skiplist. Schedule the page for eviction if we + * encounter a large skiplist. This worthwhile because applications + * that take a sample often take many samples, so the overhead of + * traversing the skip list each time accumulates to real time. + */ + if (samples > 5000) + __wt_page_evict_soon(session, cbt->ref); + + return (0); +} + +/* + * __wt_random_descent -- + * Find a random page in a tree for either sampling or eviction. + */ +int +__wt_random_descent(WT_SESSION_IMPL *session, WT_REF **refp, bool eviction) +{ + WT_BTREE *btree; + WT_DECL_RET; + WT_PAGE *page; + WT_PAGE_INDEX *pindex; + WT_REF *current, *descent; + uint32_t flags, i, entries, retry; + + *refp = NULL; + + btree = S2BT(session); + current = NULL; + retry = 100; + + /* Eviction should not be tapped to do eviction. */ + if (eviction) + flags = WT_READ_CACHE | WT_READ_NO_EVICT | WT_READ_NO_GEN | + WT_READ_NO_WAIT | WT_READ_NOTFOUND_OK | WT_READ_RESTART_OK; + else + flags = WT_READ_RESTART_OK; + + if (0) { +restart: /* + * Discard the currently held page and restart the search from + * the root. + */ + WT_RET(__wt_page_release(session, current, flags)); + } + + /* Search the internal pages of the tree. */ + current = &btree->root; + for (;;) { + page = current->page; + if (!WT_PAGE_IS_INTERNAL(page)) + break; + + WT_INTL_INDEX_GET(session, page, pindex); + entries = pindex->entries; + + /* Eviction just wants any random child. */ + if (eviction) { + descent = pindex->index[ + __wt_random(&session->rnd) % entries]; + goto descend; + } + + /* + * There may be empty pages in the tree, and they're useless to + * us. If we don't find a non-empty page in "entries" random + * guesses, take the first non-empty page in the tree. If the + * search page contains nothing other than empty pages, restart + * from the root some number of times before giving up. + * + * Random sampling is looking for a key/value pair on a random + * leaf page, and so will accept any page that contains a valid + * key/value pair, so on-disk is fine, but deleted is not. + */ + descent = NULL; + for (i = 0; i < entries; ++i) { + descent = + pindex->index[__wt_random(&session->rnd) % entries]; + if (descent->state == WT_REF_MEM || + descent->state == WT_REF_DISK) + break; + } + if (i == entries) + for (i = 0; i < entries; ++i) { + descent = pindex->index[i]; + if (descent->state == WT_REF_MEM || + descent->state == WT_REF_DISK) + break; + } + if (i == entries || descent == NULL) { + if (--retry > 0) + goto restart; + + WT_RET(__wt_page_release(session, current, flags)); + return (WT_NOTFOUND); + } + + /* + * Swap the current page for the child page. If the page splits + * while we're retrieving it, restart the search at the root. + * + * On other error, simply return, the swap call ensures we're + * holding nothing on failure. + */ +descend: if ((ret = + __wt_page_swap(session, current, descent, flags)) == 0) { + current = descent; + continue; + } + if (eviction && (ret == WT_NOTFOUND || ret == WT_RESTART)) + break; + if (ret == WT_RESTART) + goto restart; + return (ret); + } + + /* + * There is no point starting with the root page: the walk will exit + * immediately. In that case we aren't holding a hazard pointer so + * there is nothing to release. + */ + if (!eviction || !__wt_ref_is_root(current)) + *refp = current; + return (0); +} + +/* + * __wt_btcur_next_random -- + * Move to a random record in the tree. There are two algorithms, one + * where we select a record at random from the whole tree on each + * retrieval and one where we first select a record at random from the + * whole tree, and then subsequently sample forward from that location. + * The sampling approach allows us to select reasonably uniform random + * points from unbalanced trees. + */ +int +__wt_btcur_next_random(WT_CURSOR_BTREE *cbt) +{ + WT_BTREE *btree; + WT_CURSOR *cursor; + WT_DECL_RET; + WT_SESSION_IMPL *session; + WT_UPDATE *upd; + wt_off_t size; + uint64_t n, skip; + + btree = cbt->btree; + cursor = &cbt->iface; + session = (WT_SESSION_IMPL *)cbt->iface.session; + + /* + * Only supports row-store: applications can trivially select a random + * value from a column-store, if there were any reason to do so. + */ + if (btree->type != BTREE_ROW) + WT_RET_MSG(session, ENOTSUP, + "WT_CURSOR.next_random only supported by row-store tables"); + + WT_STAT_CONN_INCR(session, cursor_next); + WT_STAT_DATA_INCR(session, cursor_next); + + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + +#ifdef HAVE_DIAGNOSTIC + /* + * Under some conditions we end up using the underlying cursor.next to + * walk through the object. Since there are multiple calls, we can hit + * the cursor-order checks, turn them off. + */ + __wt_cursor_key_order_reset(cbt); +#endif + /* + * If we don't have a current position in the tree, or if retrieving + * random values without sampling, pick a roughly random leaf page in + * the tree and return an entry from it. + */ + if (cbt->ref == NULL || cbt->next_random_sample_size == 0) { + WT_ERR(__cursor_func_init(cbt, true)); + WT_WITH_PAGE_INDEX(session, + ret = __wt_random_descent(session, &cbt->ref, false)); + if (ret == 0) + goto random_page_entry; + + /* + * Random descent may return not-found: the tree might be empty + * or have so many deleted items we didn't find any valid pages. + * We can't return WT_NOTFOUND to the application unless a tree + * is really empty, fallback to skipping through tree pages. + */ + WT_ERR_NOTFOUND_OK(ret); + } + + /* + * Cursor through the tree, skipping past the sample size of the leaf + * pages in the tree between each random key return to compensate for + * unbalanced trees. + * + * If the random descent attempt failed, we don't have a configured + * sample size, use 100 for no particular reason. + */ + if (cbt->next_random_sample_size == 0) + cbt->next_random_sample_size = 100; + + /* + * If the random descent attempt failed, or it's our first skip attempt, + * we haven't yet set the pages to skip, do it now. + * + * Use the underlying file size divided by its block allocation size as + * our guess of leaf pages in the file (this can be entirely wrong, as + * it depends on how many pages are in this particular checkpoint, how + * large the leaf and internal pages really are, and other factors). + * Then, divide that value by the configured sample size and increment + * the final result to make sure tiny files don't leave us with a skip + * value of 0. + * + * !!! + * Ideally, the number would be prime to avoid restart issues. + */ + if (cbt->next_random_leaf_skip == 0) { + WT_ERR(btree->bm->size(btree->bm, session, &size)); + cbt->next_random_leaf_skip = (uint64_t) + ((size / btree->allocsize) / + cbt->next_random_sample_size) + 1; + } + + /* + * Be paranoid about loop termination: first, if the last leaf page + * skipped was also the last leaf page in the tree, skip may be set to + * zero on return along with the NULL WT_REF end-of-walk condition. + * Second, if a tree has no valid pages at all (the condition after + * initial creation), we might make no progress at all, or finally, if + * a tree has only deleted pages, we'll make progress, but never get a + * useful WT_REF. And, of course, the tree can switch from one of these + * states to another without warning. Decrement skip regardless of what + * is happening in the search, guarantee we eventually quit. + * + * Pages read for data sampling aren't "useful"; don't update the read + * generation of pages already in memory, and if a page is read, set + * its generation to a low value so it is evicted quickly. + */ + for (skip = cbt->next_random_leaf_skip; cbt->ref == NULL || skip > 0;) { + n = skip; + WT_ERR(__wt_tree_walk_skip(session, &cbt->ref, &skip, + WT_READ_NO_GEN | WT_READ_SKIP_INTL | WT_READ_WONT_NEED)); + if (n == skip) { + if (skip == 0) + break; + --skip; + } + } + + /* + * We can't return WT_NOTFOUND to the application unless a tree is + * really empty, fallback to a random entry from the first page in the + * tree that has anything at all. + */ + if (cbt->ref == NULL) + WT_ERR(__wt_btcur_next(cbt, false)); + +random_page_entry: + /* + * Select a random entry from the leaf page. If it's not valid, move to + * the next entry, if that doesn't work, move to the previous entry. + */ + WT_ERR(__wt_row_random_leaf(session, cbt)); + if (__wt_cursor_valid(cbt, &upd)) + WT_ERR(__wt_kv_return(session, cbt, upd)); + else { + if ((ret = __wt_btcur_next(cbt, false)) == WT_NOTFOUND) + ret = __wt_btcur_prev(cbt, false); + WT_ERR(ret); + } + return (0); + +err: WT_TRET(__cursor_reset(cbt)); + return (ret); +} diff --git a/src/third_party/wiredtiger/src/btree/bt_read.c b/src/third_party/wiredtiger/src/btree/bt_read.c index 90188498535..ae1f8427b25 100644 --- a/src/third_party/wiredtiger/src/btree/bt_read.c +++ b/src/third_party/wiredtiger/src/btree/bt_read.c @@ -90,7 +90,7 @@ __col_instantiate(WT_SESSION_IMPL *session, { /* Search the page and add updates. */ WT_RET(__wt_col_search(session, recno, ref, cbt)); - WT_RET(__wt_col_modify(session, cbt, recno, NULL, upd, false)); + WT_RET(__wt_col_modify(session, cbt, recno, NULL, upd, false, false)); return (0); } @@ -104,7 +104,7 @@ __row_instantiate(WT_SESSION_IMPL *session, { /* Search the page and add updates. */ WT_RET(__wt_row_search(session, key, ref, cbt, true)); - WT_RET(__wt_row_modify(session, cbt, key, NULL, upd, false)); + WT_RET(__wt_row_modify(session, cbt, key, NULL, upd, false, false)); return (0); } @@ -324,7 +324,7 @@ __evict_force_check(WT_SESSION_IMPL *session, WT_REF *ref) * If this session has more than one hazard pointer, eviction will fail * and there is no point trying. */ - if (__wt_hazard_count(session, page) > 1) + if (__wt_hazard_count(session, ref) > 1) return (false); /* If we can do an in-memory split, do it. */ @@ -369,6 +369,7 @@ __page_read(WT_SESSION_IMPL *session, WT_REF *ref) size_t addr_size; uint32_t previous_state; const uint8_t *addr; + bool timer; btree = S2BT(session); page = NULL; @@ -408,10 +409,11 @@ __page_read(WT_SESSION_IMPL *session, WT_REF *ref) * There's an address, read or map the backing disk page and build an * in-memory version of the page. */ - if (!F_ISSET(session, WT_SESSION_INTERNAL)) + timer = !F_ISSET(session, WT_SESSION_INTERNAL); + if (timer) __wt_epoch(session, &start); WT_ERR(__wt_bt_read(session, &tmp, addr, addr_size)); - if (!F_ISSET(session, WT_SESSION_INTERNAL)) { + if (timer) { __wt_epoch(session, &stop); WT_STAT_CONN_INCR(session, cache_read_app_count); WT_STAT_CONN_INCRV(session, cache_read_app_time, @@ -590,8 +592,9 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags */ if (LF_ISSET(WT_READ_NO_EVICT) || F_ISSET(session, WT_SESSION_NO_EVICTION) || - (F_ISSET(btree, WT_BTREE_NO_EVICTION) && - !F_ISSET(btree, WT_BTREE_NO_RECONCILE))) + btree->lsm_primary || + (btree->evict_disabled > 0 && + !F_ISSET(btree, WT_BTREE_ALLOW_SPLITS))) goto skip_evict; /* diff --git a/src/third_party/wiredtiger/src/btree/bt_rebalance.c b/src/third_party/wiredtiger/src/btree/bt_rebalance.c index 29380459b94..68848c7c8f5 100644 --- a/src/third_party/wiredtiger/src/btree/bt_rebalance.c +++ b/src/third_party/wiredtiger/src/btree/bt_rebalance.c @@ -265,7 +265,7 @@ __rebalance_row_leaf_key(WT_SESSION_IMPL *session, */ WT_RET(__wt_bt_read(session, rs->tmp1, addr, addr_len)); WT_RET(__wt_page_inmem(session, NULL, rs->tmp1->data, 0, 0, &page)); - ret = __wt_row_leaf_key_copy(session, page, &page->pg_row_d[0], key); + ret = __wt_row_leaf_key_copy(session, page, &page->pg_row[0], key); __wt_page_out(session, &page); return (ret); } @@ -406,12 +406,10 @@ __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) WT_BTREE *btree; WT_DECL_RET; WT_REBALANCE_STUFF *rs, _rstuff; - bool evict_reset; WT_UNUSED(cfg); btree = S2BT(session); - evict_reset = false; /* * If the tree has never been written to disk, we're done, rebalance @@ -433,14 +431,6 @@ __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) /* Set the internal page tree type. */ rs->type = btree->root.page->type; - /* - * Get exclusive access to the file. (Not required, the only page in the - * cache is the root page, and that cannot be evicted; however, this way - * eviction ignores the tree entirely.) - */ - WT_ERR(__wt_evict_file_exclusive_on(session)); - evict_reset = true; - /* Recursively walk the tree. */ switch (rs->type) { case WT_PAGE_ROW_INT: @@ -471,10 +461,7 @@ __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) btree->root.page = rs->root; rs->root = NULL; -err: if (evict_reset) - __wt_evict_file_exclusive_off(session); - - /* Discard any leftover root page we created. */ +err: /* Discard any leftover root page we created. */ if (rs->root != NULL) { __wt_page_modify_clear(session, rs->root); __wt_page_out(session, &rs->root); diff --git a/src/third_party/wiredtiger/src/btree/bt_ret.c b/src/third_party/wiredtiger/src/btree/bt_ret.c index 8ef2db67e7b..f17fa1b85d1 100644 --- a/src/third_party/wiredtiger/src/btree/bt_ret.c +++ b/src/third_party/wiredtiger/src/btree/bt_ret.c @@ -9,65 +9,22 @@ #include "wt_internal.h" /* - * __wt_kv_return -- - * Return a page referenced key/value pair to the application. + * __key_return -- + * Change the cursor to reference an internal return key. */ -int -__wt_kv_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) +static inline int +__key_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) { - WT_BTREE *btree; - WT_CELL *cell; - WT_CELL_UNPACK unpack; WT_CURSOR *cursor; WT_ITEM *tmp; WT_PAGE *page; WT_ROW *rip; - uint8_t v; - - btree = S2BT(session); page = cbt->ref->page; cursor = &cbt->iface; - switch (page->type) { - case WT_PAGE_COL_FIX: - /* - * The interface cursor's record has usually been set, but that - * isn't universally true, specifically, cursor.search_near may - * call here without first setting the interface cursor. - */ - cursor->recno = cbt->recno; - - /* If the cursor references a WT_UPDATE item, return it. */ - if (upd != NULL) { - cursor->value.data = WT_UPDATE_DATA(upd); - cursor->value.size = upd->size; - return (0); - } - - /* Take the value from the original page. */ - v = __bit_getv_recno(cbt->ref, cursor->recno, btree->bitcnt); - return (__wt_buf_set(session, &cursor->value, &v, 1)); - case WT_PAGE_COL_VAR: - /* - * The interface cursor's record has usually been set, but that - * isn't universally true, specifically, cursor.search_near may - * call here without first setting the interface cursor. - */ - cursor->recno = cbt->recno; - - /* If the cursor references a WT_UPDATE item, return it. */ - if (upd != NULL) { - cursor->value.data = WT_UPDATE_DATA(upd); - cursor->value.size = upd->size; - return (0); - } - - /* Take the value from the original page cell. */ - cell = WT_COL_PTR(page, &page->pg_var_d[cbt->slot]); - break; - case WT_PAGE_ROW_LEAF: - rip = &page->pg_row_d[cbt->slot]; + if (page->type == WT_PAGE_ROW_LEAF) { + rip = &page->pg_row[cbt->slot]; /* * If the cursor references a WT_INSERT item, take its key. @@ -79,7 +36,10 @@ __wt_kv_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) if (cbt->ins != NULL) { cursor->key.data = WT_INSERT_KEY(cbt->ins); cursor->key.size = WT_INSERT_KEY_SIZE(cbt->ins); - } else if (cbt->compare == 0) { + return (0); + } + + if (cbt->compare == 0) { /* * If not in an insert list and there's an exact match, * the row-store search function built the key we want @@ -97,16 +57,51 @@ __wt_kv_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) cursor->key.data = cbt->row_key->data; cursor->key.size = cbt->row_key->size; - } else - WT_RET(__wt_row_leaf_key( - session, page, rip, &cursor->key, false)); - - /* If the cursor references a WT_UPDATE item, return it. */ - if (upd != NULL) { - cursor->value.data = WT_UPDATE_DATA(upd); - cursor->value.size = upd->size; return (0); } + return (__wt_row_leaf_key( + session, page, rip, &cursor->key, false)); + } + + /* + * WT_PAGE_COL_FIX, WT_PAGE_COL_VAR: + * The interface cursor's record has usually been set, but that + * isn't universally true, specifically, cursor.search_near may call + * here without first setting the interface cursor. + */ + cursor->recno = cbt->recno; + return (0); +} + +/* + * __value_return -- + * Change the cursor to reference an internal return value. + */ +static inline int +__value_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) +{ + WT_BTREE *btree; + WT_CELL *cell; + WT_CELL_UNPACK unpack; + WT_CURSOR *cursor; + WT_PAGE *page; + WT_ROW *rip; + uint8_t v; + + btree = S2BT(session); + + page = cbt->ref->page; + cursor = &cbt->iface; + + /* If the cursor references a WT_UPDATE item, return it. */ + if (upd != NULL) { + cursor->value.data = WT_UPDATE_DATA(upd); + cursor->value.size = upd->size; + return (0); + } + + if (page->type == WT_PAGE_ROW_LEAF) { + rip = &page->pg_row[cbt->slot]; /* Simple values have their location encoded in the WT_ROW. */ if (__wt_row_leaf_value(page, rip, &cursor->value)) @@ -121,13 +116,65 @@ __wt_kv_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) cursor->value.size = 0; return (0); } - break; - WT_ILLEGAL_VALUE(session); + __wt_cell_unpack(cell, &unpack); + return (__wt_page_cell_data_ref( + session, page, &unpack, &cursor->value)); + } - /* The value is an on-page cell, unpack and expand it as necessary. */ - __wt_cell_unpack(cell, &unpack); - WT_RET(__wt_page_cell_data_ref(session, page, &unpack, &cursor->value)); + if (page->type == WT_PAGE_COL_VAR) { + /* Take the value from the original page cell. */ + cell = WT_COL_PTR(page, &page->pg_var[cbt->slot]); + __wt_cell_unpack(cell, &unpack); + return (__wt_page_cell_data_ref( + session, page, &unpack, &cursor->value)); + } + + /* WT_PAGE_COL_FIX: Take the value from the original page. */ + v = __bit_getv_recno(cbt->ref, cursor->recno, btree->bitcnt); + return (__wt_buf_set(session, &cursor->value, &v, 1)); +} + +/* + * __wt_key_return -- + * Change the cursor to reference an internal return key. + */ +int +__wt_key_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) +{ + WT_CURSOR *cursor; + + cursor = &cbt->iface; + + /* + * We may already have an internal key, in which case the cursor may + * not be set up to get another copy (for example, when we rely on a + * search-function result). + */ + F_CLR(cursor, WT_CURSTD_KEY_EXT); + if (!F_ISSET(cursor, WT_CURSTD_KEY_INT)) { + WT_RET(__key_return(session, cbt)); + F_SET(cursor, WT_CURSTD_KEY_INT); + } + return (0); +} + +/* + * __wt_kv_return -- + * Return a page referenced key/value pair to the application. + */ +int +__wt_kv_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) +{ + WT_CURSOR *cursor; + + cursor = &cbt->iface; + + WT_RET(__wt_key_return(session, cbt)); + + F_CLR(cursor, WT_CURSTD_VALUE_EXT); + WT_RET(__value_return(session, cbt, upd)); + F_SET(cursor, WT_CURSTD_VALUE_INT); return (0); } diff --git a/src/third_party/wiredtiger/src/btree/bt_slvg.c b/src/third_party/wiredtiger/src/btree/bt_slvg.c index f269c2d7f43..165f932afb2 100644 --- a/src/third_party/wiredtiger/src/btree/bt_slvg.c +++ b/src/third_party/wiredtiger/src/btree/bt_slvg.c @@ -166,13 +166,11 @@ __wt_bt_salvage(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, const char *cfg[]) WT_DECL_RET; WT_STUFF *ss, stuff; uint32_t i, leaf_cnt; - bool evict_reset; WT_UNUSED(cfg); btree = S2BT(session); bm = btree->bm; - evict_reset = false; WT_CLEAR(stuff); ss = &stuff; @@ -183,13 +181,6 @@ __wt_bt_salvage(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, const char *cfg[]) WT_ERR(__wt_scr_alloc(session, 0, &ss->tmp1)); WT_ERR(__wt_scr_alloc(session, 0, &ss->tmp2)); - /* - * Salvage handles its own page eviction; get exclusive access to the - * file, have eviction ignore the tree entirely. - */ - WT_ERR(__wt_evict_file_exclusive_on(session)); - evict_reset = true; - /* * Step 1: * Inform the underlying block manager that we're salvaging the file. @@ -350,9 +341,6 @@ err: WT_TRET(bm->salvage_end(bm, session)); if (ss->root_ref.page != NULL) __wt_ref_out(session, &ss->root_ref); - if (evict_reset) - __wt_evict_file_exclusive_off(session); - /* Discard the leaf and overflow page memory. */ WT_TRET(__slvg_cleanup(session, ss)); @@ -603,9 +591,9 @@ __slvg_trk_leaf(WT_SESSION_IMPL *session, */ WT_ERR(__wt_page_inmem(session, NULL, dsk, 0, 0, &page)); WT_ERR(__wt_row_leaf_key_copy(session, - page, &page->pg_row_d[0], &trk->row_start)); - WT_ERR(__wt_row_leaf_key_copy(session, page, - &page->pg_row_d[page->pg_row_entries - 1], &trk->row_stop)); + page, &page->pg_row[0], &trk->row_start)); + WT_ERR(__wt_row_leaf_key_copy(session, + page, &page->pg_row[page->entries - 1], &trk->row_stop)); __wt_verbose(session, WT_VERB_SALVAGE, "%s start key %s", @@ -1235,7 +1223,7 @@ __slvg_col_build_leaf(WT_SESSION_IMPL *session, WT_TRACK *trk, WT_REF *ref) WT_PAGE *page; WT_SALVAGE_COOKIE *cookie, _cookie; uint64_t recno, skip, take; - uint32_t *entriesp, save_entries; + uint32_t save_entries; cookie = &_cookie; WT_CLEAR(*cookie); @@ -1244,11 +1232,8 @@ __slvg_col_build_leaf(WT_SESSION_IMPL *session, WT_TRACK *trk, WT_REF *ref) WT_RET(__wt_page_in(session, ref, 0)); page = ref->page; - entriesp = page->type == WT_PAGE_COL_VAR ? - &page->pg_var_entries : &page->pg_fix_entries; - - save_col_var = page->pg_var_d; - save_entries = *entriesp; + save_col_var = page->pg_var; + save_entries = page->entries; /* * Calculate the number of K/V entries we are going to skip, and @@ -1300,11 +1285,11 @@ __slvg_col_build_leaf(WT_SESSION_IMPL *session, WT_TRACK *trk, WT_REF *ref) /* Write the new version of the leaf page to disk. */ WT_ERR(__slvg_modify_init(session, page)); - WT_ERR(__wt_reconcile(session, ref, cookie, WT_VISIBILITY_ERR)); + WT_ERR(__wt_reconcile(session, ref, cookie, WT_VISIBILITY_ERR, NULL)); /* Reset the page. */ - page->pg_var_d = save_col_var; - *entriesp = save_entries; + page->pg_var = save_col_var; + page->entries = save_entries; ret = __wt_page_release(session, ref, 0); if (ret == 0) @@ -1973,14 +1958,14 @@ __slvg_row_build_leaf( /* We should have selected some entries, but not the entire page. */ WT_ASSERT(session, skip_start + skip_stop > 0 && - skip_start + skip_stop < page->pg_row_entries); + skip_start + skip_stop < page->entries); /* * Take a copy of this page's first key to define the start of * its range. The key may require processing, otherwise, it's * a copy from the page. */ - rip = page->pg_row_d + skip_start; + rip = page->pg_row + skip_start; WT_ERR(__wt_row_leaf_key(session, page, rip, key, false)); WT_ERR(__wt_row_ikey_incr( session, ref->home, 0, key->data, key->size, ref)); @@ -1988,14 +1973,14 @@ __slvg_row_build_leaf( /* Set the referenced flag on overflow pages we're using. */ if (trk->trk_ovfl_cnt != 0) WT_ERR(__slvg_row_ovfl(session, - trk, page, skip_start, page->pg_row_entries - skip_stop)); + trk, page, skip_start, page->entries - skip_stop)); /* * Change the page to reflect the correct record count: there is no * need to copy anything on the page itself, the entries value limits * the number of page items. */ - page->pg_row_entries -= skip_stop; + page->entries -= skip_stop; cookie->skip = skip_start; /* @@ -2011,10 +1996,10 @@ __slvg_row_build_leaf( /* Write the new version of the leaf page to disk. */ WT_ERR(__slvg_modify_init(session, page)); - WT_ERR(__wt_reconcile(session, ref, cookie, WT_VISIBILITY_ERR)); + WT_ERR(__wt_reconcile(session, ref, cookie, WT_VISIBILITY_ERR, NULL)); /* Reset the page. */ - page->pg_row_entries += skip_stop; + page->entries += skip_stop; /* * Discard our hazard pointer and evict the page, updating the @@ -2081,7 +2066,7 @@ __slvg_row_ovfl(WT_SESSION_IMPL *session, * We're merging a row-store page, and we took some number of records, * figure out which (if any) overflow records we used. */ - for (rip = page->pg_row_d + start; start < stop; ++start, ++rip) { + for (rip = page->pg_row + start; start < stop; ++start, ++rip) { copy = WT_ROW_KEY_COPY(rip); (void)__wt_row_leaf_key_info( page, copy, NULL, &cell, NULL, NULL); diff --git a/src/third_party/wiredtiger/src/btree/bt_split.c b/src/third_party/wiredtiger/src/btree/bt_split.c index 017c820ea29..92be2125b88 100644 --- a/src/third_party/wiredtiger/src/btree/bt_split.c +++ b/src/third_party/wiredtiger/src/btree/bt_split.c @@ -10,8 +10,8 @@ #define WT_MEM_TRANSFER(from_decr, to_incr, len) do { \ size_t __len = (len); \ - from_decr += __len; \ - to_incr += __len; \ + (from_decr) += __len; \ + (to_incr) += __len; \ } while (0) /* @@ -53,6 +53,16 @@ __split_oldest_gen(WT_SESSION_IMPL *session) return (oldest); } +/* + * __wt_split_obsolete -- + * Check if it is safe to free / evict based on split generation. + */ +bool +__wt_split_obsolete(WT_SESSION_IMPL *session, uint64_t split_gen) +{ + return (split_gen < __split_oldest_gen(session)); +} + /* * __split_stash_add -- * Add a new entry into the session's split stash list. @@ -109,7 +119,7 @@ __wt_split_stash_discard(WT_SESSION_IMPL *session) ++i, ++stash) { if (stash->p == NULL) continue; - else if (stash->split_gen >= oldest) + if (stash->split_gen >= oldest) break; /* * It's a bad thing if another thread is in this memory after @@ -177,7 +187,7 @@ __split_safe_free(WT_SESSION_IMPL *session, exclusive = true; if (exclusive) { - __wt_free(session, p); + __wt_overwrite_and_free_len(session, p, s); return (0); } @@ -187,7 +197,7 @@ __split_safe_free(WT_SESSION_IMPL *session, #ifdef HAVE_DIAGNOSTIC /* * __split_verify_intl_key_order -- - * Verify the key order on an internal page after a split, diagnostic only. + * Verify the key order on an internal page after a split. */ static void __split_verify_intl_key_order(WT_SESSION_IMPL *session, WT_PAGE *page) @@ -239,6 +249,46 @@ __split_verify_intl_key_order(WT_SESSION_IMPL *session, WT_PAGE *page) break; } } + +/* + * __split_verify_root -- + * Verify a root page involved in a split. + */ +static int +__split_verify_root(WT_SESSION_IMPL *session, WT_PAGE *page) +{ + WT_DECL_RET; + WT_REF *ref; + + /* The split is complete and live, verify all of the pages involved. */ + __split_verify_intl_key_order(session, page); + + WT_INTL_FOREACH_BEGIN(session, page, ref) { + /* + * An eviction thread might be attempting to evict the page + * (the WT_REF may be WT_REF_LOCKED), or it may be a disk based + * page (the WT_REF may be WT_REF_READING), or it may be in + * some other state. Acquire a hazard pointer for any + * in-memory pages so we know the state of the page. + * + * Ignore pages not in-memory (deleted, on-disk, being read), + * there's no in-memory structure to check. + */ + if ((ret = __wt_page_in(session, + ref, WT_READ_CACHE | WT_READ_NO_EVICT)) == WT_NOTFOUND) + continue; + WT_ERR(ret); + + __split_verify_intl_key_order(session, ref->page); + + WT_ERR(__wt_page_release(session, ref, WT_READ_NO_EVICT)); + } WT_INTL_FOREACH_END; + + return (0); + +err: /* Something really bad just happened. */ + WT_PANIC_RET(session, ret, "fatal error during page split"); +} #endif /* @@ -390,12 +440,12 @@ __split_ref_move(WT_SESSION_IMPL *session, WT_PAGE *from_home, } /* - * __split_ref_step1 -- + * __split_ref_prepare -- * Prepare a set of WT_REFs for a move. */ static void -__split_ref_step1( - WT_SESSION_IMPL *session, WT_PAGE_INDEX *pindex, bool skip_first) +__split_ref_prepare(WT_SESSION_IMPL *session, + WT_PAGE_INDEX *pindex, uint64_t split_gen, bool skip_first) { WT_PAGE *child; WT_REF *child_ref, *ref; @@ -418,30 +468,25 @@ __split_ref_step1( child = ref->page; /* - * Block eviction and splits in newly created pages. + * Block eviction in newly created pages. * * Once the split is live, newly created internal pages might be * evicted and their WT_REF structures freed. If that happened * before all threads exit the index of the page that previously * "owned" the WT_REF, a thread might see a freed WT_REF. To - * ensure that doesn't happen, the newly created page's modify - * structure has a field with a transaction ID that's checked - * before any internal page is evicted. Unfortunately, we don't - * know the correct value until we update the original page's - * index (we need a transaction ID from after that update), but - * the act of updating the original page's index is what allows - * the eviction to happen. + * ensure that doesn't happen, the newly created page contains + * the current split generation and can't be evicted until + * all readers have left the old generation. * - * Split blocking was because historic versions of the split - * code didn't update the WT_REF.home field until after the - * split was live, so the WT_REF.home fields being updated could - * split again before the update, there's a race between splits - * as to which would update them first. The current code updates - * the WT_REF.home fields before going live (in this function), - * this shouldn't be an issue, but for now splits remain turned - * off. + * Historic, we also blocked splits in newly created pages + * because we didn't update the WT_REF.home field until after + * the split was live, so the WT_REF.home fields being updated + * could split again before the update, there's a race between + * splits as to which would update them first. The current code + * updates the WT_REF.home fields before going live (in this + * function), this isn't an issue. */ - F_SET_ATOMIC(child, WT_PAGE_SPLIT_BLOCK); + child->pg_intl_split_gen = split_gen; /* * We use a page flag to prevent the child from splitting from @@ -464,64 +509,6 @@ __split_ref_step1( } } -/* - * __split_ref_step2 -- - * Allow the newly created children to be evicted or split. - */ -static int -__split_ref_step2( - WT_SESSION_IMPL *session, WT_PAGE_INDEX *pindex, bool skip_first) -{ - WT_DECL_RET; - WT_PAGE *child; - WT_REF *ref; - uint32_t i; - - /* - * The split has gone live, enable eviction and splits on the newly - * created internal pages. - */ - WT_WRITE_BARRIER(); - - for (i = skip_first ? 1 : 0; i < pindex->entries; ++i) { - ref = pindex->index[i]; - - /* - * We don't hold hazard pointers on created pages, they cannot - * be evicted because the page-modify transaction value set as - * they were created prevents eviction. (See above, we reset - * that value as part of fixing up the page.) But, an eviction - * thread might be attempting to evict the page (the WT_REF may - * be WT_REF_LOCKED), or it may be a disk based page (the WT_REF - * may be WT_REF_READING), or it may be in some other state. - * Acquire a hazard pointer for any in-memory pages so we know - * the state of the page. Ignore pages not in-memory (deleted, - * on-disk, being read), there's no in-memory structure to fix. - */ - if ((ret = __wt_page_in(session, - ref, WT_READ_CACHE | WT_READ_NO_EVICT)) == WT_NOTFOUND) - continue; - WT_ERR(ret); - - child = ref->page; - - /* The child can now be evicted or split. */ - F_CLR_ATOMIC(child, WT_PAGE_SPLIT_BLOCK); - -#ifdef HAVE_DIAGNOSTIC - WT_WITH_PAGE_INDEX(session, - __split_verify_intl_key_order(session, child)); -#endif - - WT_ERR(__wt_hazard_clear(session, child)); - } - - return (0); - -err: /* Something really bad just happened. */ - WT_PANIC_RET(session, ret, "fatal error resolving a split"); -} - /* * __split_root -- * Split the root page in-memory, deepening the tree. @@ -653,8 +640,12 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) /* Start making real changes to the tree, errors are fatal. */ complete = WT_ERR_PANIC; - /* Prepare the WT_REFs for the move. */ - __split_ref_step1(session, alloc_index, false); + /* + * Prepare the WT_REFs for the move: this requires a stable split + * generation to block splits in newly created pages, so get one. + */ + WT_ENTER_PAGE_INDEX(session); + __split_ref_prepare(session, alloc_index, session->split_gen, false); /* * Confirm the root page's index hasn't moved, then update it, which @@ -662,20 +653,27 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) */ WT_ASSERT(session, WT_INTL_INDEX_GET_SAFE(root) == pindex); WT_INTL_INDEX_SET(root, alloc_index); + alloc_index = NULL; + + WT_LEAVE_PAGE_INDEX(session); + + /* + * Get a generation for this split, mark the root page. This must be + * after the new index is swapped into place in order to know that no + * readers are looking at the old index. + */ + split_gen = __wt_atomic_addv64(&S2C(session)->split_gen, 1); + root->pg_intl_split_gen = split_gen; #ifdef HAVE_DIAGNOSTIC WT_WITH_PAGE_INDEX(session, - __split_verify_intl_key_order(session, root)); + ret = __split_verify_root(session, root)); + WT_ERR(ret); #endif - /* Finalize the WT_REFs we moved. */ - WT_ERR(__split_ref_step2(session, alloc_index, false)); - /* The split is complete and correct, ignore benign errors. */ + /* The split is complete and verified, ignore benign errors. */ complete = WT_ERR_IGNORE; - /* We've installed the allocated page-index, ensure error handling. */ - alloc_index = NULL; - /* * We can't free the previous root's index, there may be threads using * it. Add to the session's discard list, to be freed once we know no @@ -686,7 +684,6 @@ __split_root(WT_SESSION_IMPL *session, WT_PAGE *root) * fails, we don't roll back that change, because threads may already * be using the new index. */ - split_gen = __wt_atomic_addv64(&S2C(session)->split_gen, 1); size = sizeof(WT_PAGE_INDEX) + pindex->entries * sizeof(WT_REF *); WT_TRET(__split_safe_free(session, split_gen, false, pindex, size)); root_decr += size; @@ -846,10 +843,13 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, WT_INTL_INDEX_SET(parent, alloc_index); alloc_index = NULL; -#ifdef HAVE_DIAGNOSTIC - WT_WITH_PAGE_INDEX(session, - __split_verify_intl_key_order(session, parent)); -#endif + /* + * Get a generation for this split, mark the page. This must be after + * the new index is swapped into place in order to know that no readers + * are looking at the old index. + */ + split_gen = __wt_atomic_addv64(&S2C(session)->split_gen, 1); + parent->pg_intl_split_gen = split_gen; /* * If discarding the page's original WT_REF field, reset it to split. @@ -869,16 +869,25 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, __wt_free(session, ref->page_del); } + /* + * Set the discarded WT_REF state to split, ensuring we don't + * race with any discard of the WT_REF deleted fields. + */ WT_PUBLISH(ref->state, WT_REF_SPLIT); + + /* + * Push out the change: not required for correctness, but stops + * threads spinning on incorrect page references. + */ + WT_FULL_BARRIER(); } - /* - * Push out the changes: not required for correctness, but don't let - * threads spin on incorrect page references longer than necessary. - */ - WT_FULL_BARRIER(); +#ifdef HAVE_DIAGNOSTIC + WT_WITH_PAGE_INDEX(session, + __split_verify_intl_key_order(session, parent)); +#endif - /* The split is complete and correct, ignore benign errors. */ + /* The split is complete and verified, ignore benign errors. */ complete = WT_ERR_IGNORE; /* @@ -908,7 +917,6 @@ __split_parent(WT_SESSION_IMPL *session, WT_REF *ref, WT_REF **ref_new, * * Acquire a new split generation. */ - split_gen = __wt_atomic_addv64(&S2C(session)->split_gen, 1); for (i = 0, deleted_refs = scr->mem; i < deleted_entries; ++i) { next_ref = pindex->index[deleted_refs[i]]; WT_ASSERT(session, next_ref->state == WT_REF_SPLIT); @@ -1160,16 +1168,34 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) /* Start making real changes to the tree, errors are fatal. */ complete = WT_ERR_PANIC; - /* Prepare the WT_REFs for the move. */ - __split_ref_step1(session, alloc_index, true); + /* + * Prepare the WT_REFs for the move: this requires a stable split + * generation to block splits in newly created pages, so get one. + */ + WT_ENTER_PAGE_INDEX(session); + __split_ref_prepare(session, alloc_index, session->split_gen, true); /* Split into the parent. */ - WT_ERR(__split_parent(session, page_ref, alloc_index->index, - alloc_index->entries, parent_incr, false, false)); + if ((ret = __split_parent(session, page_ref, alloc_index->index, + alloc_index->entries, parent_incr, false, false)) == 0) { + /* + * Confirm the page's index hasn't moved, then update it, which + * makes the split visible to threads descending the tree. + */ + WT_ASSERT(session, WT_INTL_INDEX_GET_SAFE(page) == pindex); + WT_INTL_INDEX_SET(page, replace_index); + } - /* Confirm the page's index hasn't moved, then update it. */ - WT_ASSERT(session, WT_INTL_INDEX_GET_SAFE(page) == pindex); - WT_INTL_INDEX_SET(page, replace_index); + WT_LEAVE_PAGE_INDEX(session); + WT_ERR(ret); + + /* + * Get a generation for this split, mark the parent page. This must be + * after the new index is swapped into place in order to know that no + * readers are looking at the old index. + */ + split_gen = __wt_atomic_addv64(&S2C(session)->split_gen, 1); + page->pg_intl_split_gen = split_gen; #ifdef HAVE_DIAGNOSTIC WT_WITH_PAGE_INDEX(session, @@ -1178,18 +1204,9 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) __split_verify_intl_key_order(session, page)); #endif - /* Finalize the WT_REFs we moved. */ - WT_ERR(__split_ref_step2(session, alloc_index, true)); - - /* The split is complete and correct, ignore benign errors. */ + /* The split is complete and verified, ignore benign errors. */ complete = WT_ERR_IGNORE; - /* - * Push out the changes: not required for correctness, but no reason - * to wait. - */ - WT_FULL_BARRIER(); - /* * We don't care about the page-index we allocated, all we needed was * the array of WT_REF structures, which has now been split into the @@ -1207,7 +1224,6 @@ __split_internal(WT_SESSION_IMPL *session, WT_PAGE *parent, WT_PAGE *page) * back that change, because threads may already be using the new parent * page. */ - split_gen = __wt_atomic_addv64(&S2C(session)->split_gen, 1); size = sizeof(WT_PAGE_INDEX) + pindex->entries * sizeof(WT_REF *); WT_TRET(__split_safe_free(session, split_gen, false, pindex, size)); page_decr += size; @@ -1240,12 +1256,12 @@ err: switch (complete) { } /* - * __split_internal_lock -- + * __split_internal_lock_worker -- * Lock an internal page. */ static int -__split_internal_lock(WT_SESSION_IMPL *session, WT_REF *ref, bool trylock, - WT_PAGE **parentp, bool *hazardp) +__split_internal_lock_worker(WT_SESSION_IMPL *session, + WT_REF *ref, bool trylock, WT_PAGE **parentp, bool *hazardp) { WT_DECL_RET; WT_PAGE *parent; @@ -1284,17 +1300,19 @@ __split_internal_lock(WT_SESSION_IMPL *session, WT_REF *ref, bool trylock, for (;;) { parent = ref->home; - /* Skip pages that aren't ready to split. */ - if (F_ISSET_ATOMIC(parent, WT_PAGE_SPLIT_BLOCK)) - return (EBUSY); + /* + * The page will be marked dirty, and we can only lock a page + * with a modify structure. + */ + WT_RET(__wt_page_modify_init(session, parent)); if (trylock) - WT_RET(__wt_try_writelock(session, &parent->page_lock)); + WT_RET(WT_PAGE_TRYLOCK(session, parent)); else - __wt_writelock(session, &parent->page_lock); + WT_PAGE_LOCK(session, parent); if (parent == ref->home) break; - __wt_writeunlock(session, &parent->page_lock); + WT_PAGE_UNLOCK(session, parent); } /* @@ -1317,7 +1335,33 @@ __split_internal_lock(WT_SESSION_IMPL *session, WT_REF *ref, bool trylock, *parentp = parent; return (0); -err: __wt_writeunlock(session, &parent->page_lock); +err: WT_PAGE_UNLOCK(session, parent); + return (ret); +} + +/* + * __split_internal_lock -- + * Lock an internal page. + */ +static int +__split_internal_lock(WT_SESSION_IMPL *session, + WT_REF *ref, bool trylock, WT_PAGE **parentp, bool *hazardp) +{ + WT_DECL_RET; + + /* + * There's no lock on our parent page and we're about to acquire one, + * which implies using the WT_REF.home field to reference our parent + * page. As a child of the parent page, we prevent its eviction, but + * that's a weak guarantee. If the parent page splits, and our WT_REF + * were to move with the split, the WT_REF.home field might change + * underneath us and we could race, and end up attempting to access + * an evicted page. Set the session page-index generation so if the + * parent splits, it still can't be evicted. + */ + WT_WITH_PAGE_INDEX(session, + ret = __split_internal_lock_worker( + session, ref, trylock, parentp, hazardp)); return (ret); } @@ -1331,9 +1375,9 @@ __split_internal_unlock(WT_SESSION_IMPL *session, WT_PAGE *parent, bool hazard) WT_DECL_RET; if (hazard) - ret = __wt_hazard_clear(session, parent); + ret = __wt_hazard_clear(session, parent->pg_intl_parent_ref); - __wt_writeunlock(session, &parent->page_lock); + WT_PAGE_UNLOCK(session, parent); return (ret); } @@ -1547,7 +1591,7 @@ __split_multi_inmem( /* Apply the modification. */ WT_ERR(__wt_col_modify( - session, &cbt, recno, NULL, upd, false)); + session, &cbt, recno, NULL, upd, false, true)); break; case WT_PAGE_ROW_LEAF: /* Build a key. */ @@ -1569,7 +1613,7 @@ __split_multi_inmem( /* Apply the modification. */ WT_ERR(__wt_row_modify( - session, &cbt, key, NULL, upd, false)); + session, &cbt, key, NULL, upd, false, true)); break; WT_ILLEGAL_VALUE_ERR(session); } @@ -1770,9 +1814,9 @@ __split_insert(WT_SESSION_IMPL *session, WT_REF *ref) /* Find the last item on the page. */ if (type == WT_PAGE_ROW_LEAF) - ins_head = page->pg_row_entries == 0 ? + ins_head = page->entries == 0 ? WT_ROW_INSERT_SMALLEST(page) : - WT_ROW_INSERT_SLOT(page, page->pg_row_entries - 1); + WT_ROW_INSERT_SLOT(page, page->entries - 1); else ins_head = WT_COL_APPEND(page); moved_ins = WT_SKIP_LAST(ins_head); @@ -1822,7 +1866,7 @@ __split_insert(WT_SESSION_IMPL *session, WT_REF *ref) key->size = WT_INSERT_KEY_SIZE(ins); } else WT_ERR(__wt_row_leaf_key( - session, page, &page->pg_row_d[0], key, true)); + session, page, &page->pg_row[0], key, true)); WT_ERR(__wt_row_ikey(session, 0, key->data, key->size, child)); parent_incr += sizeof(WT_IKEY) + key->size; __wt_scr_free(session, &key); @@ -2086,8 +2130,7 @@ __wt_split_insert(WT_SESSION_IMPL *session, WT_REF *ref) WT_PAGE *parent; bool hazard; - __wt_verbose( - session, WT_VERB_SPLIT, "%p: split-insert", (void *)ref->page); + __wt_verbose(session, WT_VERB_SPLIT, "%p: split-insert", (void *)ref); WT_RET(__split_internal_lock(session, ref, true, &parent, &hazard)); if ((ret = __split_insert(session, ref)) != 0) { @@ -2178,8 +2221,7 @@ __wt_split_multi(WT_SESSION_IMPL *session, WT_REF *ref, int closing) WT_PAGE *parent; bool hazard; - __wt_verbose( - session, WT_VERB_SPLIT, "%p: split-multi", (void *)ref->page); + __wt_verbose(session, WT_VERB_SPLIT, "%p: split-multi", (void *)ref); WT_RET(__split_internal_lock(session, ref, false, &parent, &hazard)); if ((ret = __split_multi(session, ref, closing)) != 0 || closing) { @@ -2207,8 +2249,7 @@ __wt_split_reverse(WT_SESSION_IMPL *session, WT_REF *ref) WT_PAGE *parent; bool hazard; - __wt_verbose( - session, WT_VERB_SPLIT, "%p: reverse-split", (void *)ref->page); + __wt_verbose(session, WT_VERB_SPLIT, "%p: reverse-split", (void *)ref); WT_RET(__split_internal_lock(session, ref, false, &parent, &hazard)); ret = __split_parent(session, ref, NULL, 0, 0, false, true); @@ -2229,8 +2270,7 @@ __wt_split_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, WT_MULTI *multi) page = ref->page; - __wt_verbose( - session, WT_VERB_SPLIT, "%p: split-rewrite", (void *)ref->page); + __wt_verbose(session, WT_VERB_SPLIT, "%p: split-rewrite", (void *)ref); /* * This isn't a split: a reconciliation failed because we couldn't write @@ -2266,7 +2306,7 @@ __wt_split_rewrite(WT_SESSION_IMPL *session, WT_REF *ref, WT_MULTI *multi) * reconciliation, do it now. */ __wt_page_modify_clear(session, page); - __wt_ref_out(session, ref); + __wt_ref_out_int(session, ref, true); /* Swap the new page into place. */ ref->page = new->page; diff --git a/src/third_party/wiredtiger/src/btree/bt_stat.c b/src/third_party/wiredtiger/src/btree/bt_stat.c index 06428b87f6e..0da0e0807bd 100644 --- a/src/third_party/wiredtiger/src/btree/bt_stat.c +++ b/src/third_party/wiredtiger/src/btree/bt_stat.c @@ -40,6 +40,8 @@ __wt_btree_stat_init(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst) WT_STAT_SET(session, stats, btree_maxleafpage, btree->maxleafpage); WT_STAT_SET(session, stats, btree_maxleafvalue, btree->maxleafvalue); + WT_STAT_SET(session, stats, cache_bytes_dirty, + __wt_btree_dirty_inuse(session)); WT_STAT_SET(session, stats, cache_bytes_inuse, __wt_btree_bytes_inuse(session)); @@ -104,8 +106,7 @@ __stat_page(WT_SESSION_IMPL *session, WT_PAGE *page, WT_DSRC_STATS **stats) switch (page->type) { case WT_PAGE_COL_FIX: WT_STAT_INCR(session, stats, btree_column_fix); - WT_STAT_INCRV( - session, stats, btree_entries, page->pg_fix_entries); + WT_STAT_INCRV(session, stats, btree_entries, page->entries); break; case WT_PAGE_COL_INT: WT_STAT_INCR(session, stats, btree_column_internal); diff --git a/src/third_party/wiredtiger/src/btree/bt_sync.c b/src/third_party/wiredtiger/src/btree/bt_sync.c index 6d4ad9d0d0f..ead6ccc4ac0 100644 --- a/src/third_party/wiredtiger/src/btree/bt_sync.c +++ b/src/third_party/wiredtiger/src/btree/bt_sync.c @@ -78,6 +78,7 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) uint64_t internal_bytes, internal_pages, leaf_bytes, leaf_pages; uint64_t oldest_id, saved_pinned_id; uint32_t flags; + bool timer; conn = S2C(session); btree = S2BT(session); @@ -88,7 +89,8 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) internal_bytes = leaf_bytes = 0; internal_pages = leaf_pages = 0; - if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) + timer = WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT); + if (timer) __wt_epoch(session, &start); switch (syncop) { @@ -133,11 +135,11 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) if (__wt_page_is_modified(page) && WT_TXNID_LT(page->modify->update_txn, oldest_id)) { if (txn->isolation == WT_ISO_READ_COMMITTED) - WT_ERR(__wt_txn_get_snapshot(session)); + __wt_txn_get_snapshot(session); leaf_bytes += page->memory_footprint; ++leaf_pages; - WT_ERR(__wt_reconcile( - session, walk, NULL, WT_CHECKPOINTING)); + WT_ERR(__wt_reconcile(session, + walk, NULL, WT_CHECKPOINTING, NULL)); } } break; @@ -155,7 +157,7 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) * the checkpoint are included. */ if (txn->isolation == WT_ISO_READ_COMMITTED) - WT_ERR(__wt_txn_get_snapshot(session)); + __wt_txn_get_snapshot(session); /* * We cannot check the tree modified flag in the case of a @@ -186,9 +188,9 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) * to grow significantly larger than the configured maximum * size. */ - F_SET(btree, WT_BTREE_NO_RECONCILE); + F_SET(btree, WT_BTREE_ALLOW_SPLITS); ret = __wt_evict_file_exclusive_on(session); - F_CLR(btree, WT_BTREE_NO_RECONCILE); + F_CLR(btree, WT_BTREE_ALLOW_SPLITS); WT_ERR(ret); __wt_evict_file_exclusive_off(session); @@ -233,7 +235,7 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) ++leaf_pages; } WT_ERR(__wt_reconcile( - session, walk, NULL, WT_CHECKPOINTING)); + session, walk, NULL, WT_CHECKPOINTING, NULL)); } break; case WT_SYNC_CLOSE: @@ -242,7 +244,7 @@ __sync_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) break; } - if (WT_VERBOSE_ISSET(session, WT_VERB_CHECKPOINT)) { + if (timer) { __wt_epoch(session, &end); __wt_verbose(session, WT_VERB_CHECKPOINT, "__sync_file WT_SYNC_%s wrote: %" PRIu64 diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy.c b/src/third_party/wiredtiger/src/btree/bt_vrfy.c index 340f9bb6f0e..7475811adc5 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy.c @@ -216,13 +216,11 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) ckpt->raw.data, ckpt->raw.size, root_addr, &root_addr_size, true)); - /* - * Ignore trees with no root page. - * Verify, then discard the checkpoint from the cache. - */ - if (root_addr_size != 0 && - (ret = __wt_btree_tree_open( - session, root_addr, root_addr_size)) == 0) { + /* Skip trees with no root page. */ + if (root_addr_size != 0) { + WT_ERR(__wt_btree_tree_open( + session, root_addr, root_addr_size)); + if (WT_VRFY_DUMP(vs)) WT_ERR(__wt_msg(session, "Root: %s %s", __wt_addr_string(session, @@ -230,14 +228,38 @@ __wt_verify(WT_SESSION_IMPL *session, const char *cfg[]) __wt_page_type_string( btree->root.page->type))); + __wt_evict_file_exclusive_off(session); + + /* Verify the tree. */ WT_WITH_PAGE_INDEX(session, ret = __verify_tree(session, &btree->root, vs)); + /* + * We have an exclusive lock on the handle, but we're + * swapping root pages in-and-out of that handle, and + * there's a race with eviction entering the tree and + * seeing an invalid root page. Eviction must work on + * trees being verified (else we'd have to do our own + * eviction), lock eviction out whenever we're loading + * a new root page. This loops works because we are + * called with eviction locked out, so we release the + * lock at the top of the loop and re-acquire it here. + */ + WT_TRET(__wt_evict_file_exclusive_on(session)); WT_TRET(__wt_cache_op(session, WT_SYNC_DISCARD)); } /* Unload the checkpoint. */ WT_TRET(bm->checkpoint_unload(bm, session)); + + /* + * We've finished one checkpoint's verification (verification, + * then cache eviction and checkpoint unload): if any errors + * occurred, quit. Done this way because otherwise we'd need + * at least two more state variables on error, one to know if + * we need to discard the tree from the cache and one to know + * if we need to unload the checkpoint. + */ WT_ERR(ret); /* Display the tree shape. */ @@ -252,7 +274,7 @@ err: /* Inform the underlying block manager we're done. */ /* Discard the list of checkpoints. */ if (ckptbase != NULL) - __wt_meta_ckptlist_free(session, ckptbase); + __wt_meta_ckptlist_free(session, &ckptbase); /* Free allocated memory. */ __wt_scr_free(session, &vs->max_key); @@ -386,7 +408,7 @@ recno_chk: if (recno != vs->record_total + 1) } switch (page->type) { case WT_PAGE_COL_FIX: - vs->record_total += page->pg_fix_entries; + vs->record_total += page->entries; break; case WT_PAGE_COL_VAR: recno = 0; @@ -614,7 +636,7 @@ __verify_row_leaf_key_order( * If a tree is empty (just created), it won't have keys; if there * are no keys, we're done. */ - if (page->pg_row_entries == 0) + if (page->entries == 0) return (0); /* @@ -624,7 +646,7 @@ __verify_row_leaf_key_order( */ if (vs->max_addr->size != 0) { WT_RET(__wt_row_leaf_key_copy( - session, page, page->pg_row_d, vs->tmp1)); + session, page, page->pg_row, vs->tmp1)); /* * Compare the key against the largest key we've seen so far. @@ -653,7 +675,7 @@ __verify_row_leaf_key_order( /* Update the largest key we've seen to the last key on this page. */ WT_RET(__wt_row_leaf_key_copy(session, page, - page->pg_row_d + (page->pg_row_entries - 1), vs->max_key)); + page->pg_row + (page->entries - 1), vs->max_key)); (void)__wt_page_addr_string(session, ref, vs->max_addr); return (0); diff --git a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c index 3a6fd8261ba..a4071c44aee 100644 --- a/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c +++ b/src/third_party/wiredtiger/src/btree/bt_vrfy_dsk.c @@ -203,7 +203,8 @@ __verify_dsk_row( WT_ITEM *last; enum { FIRST, WAS_KEY, WAS_VALUE } last_cell_type; void *huffman; - uint32_t cell_num, cell_type, i, key_cnt, prefix; + size_t prefix; + uint32_t cell_num, cell_type, i, key_cnt; uint8_t *end; int cmp; @@ -343,8 +344,9 @@ __verify_dsk_row( if (cell_num > 1 && prefix > last->size) WT_ERR_VRFY(session, "key %" PRIu32 " on page at %s has a prefix " - "compression count of %" PRIu32 ", larger than " - "the length of the previous key, %" WT_SIZET_FMT, + "compression count of %" WT_SIZET_FMT + ", larger than the length of the previous key, %" + WT_SIZET_FMT, cell_num, tag, prefix, last->size); /* diff --git a/src/third_party/wiredtiger/src/btree/bt_walk.c b/src/third_party/wiredtiger/src/btree/bt_walk.c index fb0d2296823..86484feb7c9 100644 --- a/src/third_party/wiredtiger/src/btree/bt_walk.c +++ b/src/third_party/wiredtiger/src/btree/bt_walk.c @@ -17,54 +17,60 @@ __ref_index_slot(WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE_INDEX **pindexp, uint32_t *slotp) { WT_PAGE_INDEX *pindex; - uint32_t i; + WT_REF **start, **stop, **p, **t; + uint32_t entries, slot; - /* - * Copy the parent page's index value: the page can split at any time, - * but the index's value is always valid, even if it's not up-to-date. - */ -retry: WT_INTL_INDEX_GET(session, ref->home, pindex); + for (;;) { + /* + * Copy the parent page's index value: the page can split at + * any time, but the index's value is always valid, even if + * it's not up-to-date. + */ + WT_INTL_INDEX_GET(session, ref->home, pindex); + entries = pindex->entries; - /* - * Use the page's reference hint: it should be correct unless the page - * split before our slot. If the page splits after our slot, the hint - * will point earlier in the array than our actual slot, so the first - * loop is from the hint to the end of the list, and the second loop - * is from the start of the list to the end of the list. (The second - * loop overlaps the first, but that only happen in cases where we've - * split the tree and aren't going to find our slot at all, that's not - * worth optimizing.) - * - * It's not an error for the reference hint to be wrong, it just means - * the first retrieval (which sets the hint for subsequent retrievals), - * is slower. - */ - i = ref->pindex_hint; - if (i < pindex->entries && pindex->index[i] == ref) { - *pindexp = pindex; - *slotp = i; - return; - } - while (++i < pindex->entries) - if (pindex->index[i] == ref) { - *pindexp = pindex; - *slotp = ref->pindex_hint = i; - return; - } - for (i = 0; i < pindex->entries; ++i) - if (pindex->index[i] == ref) { - *pindexp = pindex; - *slotp = ref->pindex_hint = i; - return; + /* + * Use the page's reference hint: it should be correct unless + * there was a split or delete in the parent before our slot. + * If the hint is wrong, it can be either too big or too small, + * but often only by a small amount. Search up and down the + * index starting from the hint. + * + * It's not an error for the reference hint to be wrong, it + * just means the first retrieval (which sets the hint for + * subsequent retrievals), is slower. + */ + slot = ref->pindex_hint; + if (slot >= entries) + slot = entries - 1; + if (pindex->index[slot] == ref) + goto found; + for (start = &pindex->index[0], + stop = &pindex->index[entries - 1], + p = t = &pindex->index[slot]; + p > start || t < stop;) { + if (p > start && *--p == ref) { + slot = (uint32_t)(p - start); + goto found; + } + if (t < stop && *++t == ref) { + slot = (uint32_t)(t - start); + goto found; + } } - /* - * If we don't find our reference, the page split and our home pointer - * references the wrong page. When internal pages split, their WT_REF - * structure home values are updated; yield and wait for that to happen. - */ - __wt_yield(); - goto retry; + /* + * If we don't find our reference, the page split and our home + * pointer references the wrong page. When internal pages + * split, their WT_REF structure home values are updated; yield + * and wait for that to happen. + */ + __wt_yield(); + } + +found: WT_ASSERT(session, pindex->index[slot] == ref); + *pindexp = pindex; + *slotp = slot; } /* @@ -334,9 +340,7 @@ __tree_walk_internal(WT_SESSION_IMPL *session, * Take a copy of any held page and clear the return value. Remember * the hazard pointer we're currently holding. * - * We may be passed a pointer to btree->evict_page that we are clearing - * here. We check when discarding pages that we're not discarding that - * page, so this clear must be done before the page is released. + * Clear the returned value, it makes future error handling easier. */ couple = couple_orig = ref = *refp; *refp = NULL; @@ -344,16 +348,19 @@ __tree_walk_internal(WT_SESSION_IMPL *session, /* If no page is active, begin a walk from the start/end of the tree. */ if (ref == NULL) { restart: /* - * We can reach here with a NULL or root reference; the release + * We can be here with a NULL or root WT_REF; the page release * function handles them internally, don't complicate this code * by calling them out. */ WT_ERR(__wt_page_release(session, couple, flags)); - couple = couple_orig = ref = &btree->root; - if (ref->page == NULL) - goto done; + /* + * We're not supposed to walk trees without root pages. As this + * has not always been the case, assert to debug that change. + */ + WT_ASSERT(session, btree->root.page != NULL); + couple = couple_orig = ref = &btree->root; initial_descent = true; goto descend; } @@ -431,8 +438,8 @@ restart: /* /* * Move to the next slot, and set the reference hint if * it's wrong (used when we continue the walk). We don't - * update those hints when splitting, so it's common for - * them to be incorrect in some workloads. + * always update the hints when splitting, it's expected + * for them to be incorrect in some workloads. */ ref = pindex->index[slot]; if (ref->pindex_hint != slot) diff --git a/src/third_party/wiredtiger/src/btree/col_modify.c b/src/third_party/wiredtiger/src/btree/col_modify.c index a7920da5267..8b758453288 100644 --- a/src/third_party/wiredtiger/src/btree/col_modify.c +++ b/src/third_party/wiredtiger/src/btree/col_modify.c @@ -17,7 +17,8 @@ static int __col_insert_alloc( */ int __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, - uint64_t recno, WT_ITEM *value, WT_UPDATE *upd_arg, bool is_remove) + uint64_t recno, WT_ITEM *value, + WT_UPDATE *upd_arg, bool is_remove, bool exclusive) { WT_BTREE *btree; WT_DECL_RET; @@ -103,7 +104,7 @@ __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, /* Serialize the update. */ WT_ERR(__wt_update_serial( - session, page, &cbt->ins->upd, &upd, upd_size)); + session, page, &cbt->ins->upd, &upd, upd_size, false)); } else { /* Allocate the append/update list reference as necessary. */ if (append) { @@ -115,9 +116,8 @@ __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, page, mod->mod_col_update, ins_headp, 1); ins_headp = &mod->mod_col_update[0]; } else { - WT_PAGE_ALLOC_AND_SWAP(session, - page, mod->mod_col_update, ins_headp, - page->pg_var_entries); + WT_PAGE_ALLOC_AND_SWAP(session, page, + mod->mod_col_update, ins_headp, page->entries); ins_headp = &mod->mod_col_update[cbt->slot]; } @@ -186,11 +186,11 @@ __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, if (append) WT_ERR(__wt_col_append_serial( session, page, cbt->ins_head, cbt->ins_stack, - &ins, ins_size, &cbt->recno, skipdepth)); + &ins, ins_size, &cbt->recno, skipdepth, exclusive)); else WT_ERR(__wt_insert_serial( session, page, cbt->ins_head, cbt->ins_stack, - &ins, ins_size, skipdepth)); + &ins, ins_size, skipdepth, exclusive)); } /* If the update was successful, add it to the in-memory log. */ diff --git a/src/third_party/wiredtiger/src/btree/col_srch.c b/src/third_party/wiredtiger/src/btree/col_srch.c index 64ee9e94f4c..c72d66f8796 100644 --- a/src/third_party/wiredtiger/src/btree/col_srch.c +++ b/src/third_party/wiredtiger/src/btree/col_srch.c @@ -240,8 +240,8 @@ leaf_only: cbt->compare = 1; return (0); } - if (recno >= current->ref_recno + page->pg_fix_entries) { - cbt->recno = current->ref_recno + page->pg_fix_entries; + if (recno >= current->ref_recno + page->entries) { + cbt->recno = current->ref_recno + page->entries; goto past_end; } else { cbt->recno = recno; @@ -257,8 +257,7 @@ leaf_only: } if ((cip = __col_var_search(current, recno, NULL)) == NULL) { cbt->recno = __col_var_last_recno(current); - cbt->slot = page->pg_var_entries == 0 ? - 0 : page->pg_var_entries - 1; + cbt->slot = page->entries == 0 ? 0 : page->entries - 1; goto past_end; } else { cbt->recno = recno; diff --git a/src/third_party/wiredtiger/src/btree/row_key.c b/src/third_party/wiredtiger/src/btree/row_key.c index 99ee34a6c5d..032fdf7d897 100644 --- a/src/third_party/wiredtiger/src/btree/row_key.c +++ b/src/third_party/wiredtiger/src/btree/row_key.c @@ -26,7 +26,7 @@ __wt_row_leaf_keys(WT_SESSION_IMPL *session, WT_PAGE *page) btree = S2BT(session); - if (page->pg_row_entries == 0) { /* Just checking... */ + if (page->entries == 0) { /* Just checking... */ F_SET_ATOMIC(page, WT_PAGE_BUILD_KEYS); return (0); } @@ -51,15 +51,15 @@ __wt_row_leaf_keys(WT_SESSION_IMPL *session, WT_PAGE *page) */ WT_RET(__wt_scr_alloc(session, 0, &key)); WT_RET(__wt_scr_alloc(session, - (uint32_t)__bitstr_size(page->pg_row_entries), &tmp)); + (uint32_t)__bitstr_size(page->entries), &tmp)); memset(tmp->mem, 0, tmp->memsize); if ((gap = btree->key_gap) == 0) gap = 1; - __inmem_row_leaf_slots(tmp->mem, 0, page->pg_row_entries, gap); + __inmem_row_leaf_slots(tmp->mem, 0, page->entries, gap); /* Instantiate the keys. */ - for (rip = page->pg_row_d, i = 0; i < page->pg_row_entries; ++rip, ++i) + for (rip = page->pg_row, i = 0; i < page->entries; ++rip, ++i) if (__bit_test(tmp->mem, i)) WT_ERR(__wt_row_leaf_key_work( session, page, rip, key, true)); @@ -282,7 +282,7 @@ switch_and_jump: /* Switching to a forward roll. */ * the tracking cache. */ if (slot_offset == 0) { - __wt_readlock(session, btree->ovfl_lock); + __wt_readlock(session, &btree->ovfl_lock); copy = WT_ROW_KEY_COPY(rip); if (!__wt_row_leaf_key_info(page, copy, NULL, &cell, &keyb->data, &keyb->size)) { @@ -290,7 +290,7 @@ switch_and_jump: /* Switching to a forward roll. */ ret = __wt_dsk_cell_data_ref(session, WT_PAGE_ROW_LEAF, unpack, keyb); } - __wt_readunlock(session, btree->ovfl_lock); + __wt_readunlock(session, &btree->ovfl_lock); WT_ERR(ret); break; } diff --git a/src/third_party/wiredtiger/src/btree/row_modify.c b/src/third_party/wiredtiger/src/btree/row_modify.c index a1c214e5b8b..0dd562ed1aa 100644 --- a/src/third_party/wiredtiger/src/btree/row_modify.c +++ b/src/third_party/wiredtiger/src/btree/row_modify.c @@ -15,18 +15,13 @@ int __wt_page_modify_alloc(WT_SESSION_IMPL *session, WT_PAGE *page) { - WT_CONNECTION_IMPL *conn; + WT_DECL_RET; WT_PAGE_MODIFY *modify; - conn = S2C(session); - WT_RET(__wt_calloc_one(session, &modify)); - /* - * Select a spinlock for the page; let the barrier immediately below - * keep things from racing too badly. - */ - modify->page_lock = ++conn->page_lock_cnt % WT_PAGE_LOCKS; + /* Initialize the spinlock for the page. */ + WT_ERR(__wt_spin_init(session, &modify->page_lock, "btree page")); /* * Multiple threads of control may be searching and deciding to modify @@ -37,8 +32,8 @@ __wt_page_modify_alloc(WT_SESSION_IMPL *session, WT_PAGE *page) if (__wt_atomic_cas_ptr(&page->modify, NULL, modify)) __wt_cache_page_inmem_incr(session, page, sizeof(*modify)); else - __wt_free(session, modify); - return (0); +err: __wt_free(session, modify); + return (ret); } /* @@ -47,7 +42,8 @@ __wt_page_modify_alloc(WT_SESSION_IMPL *session, WT_PAGE *page) */ int __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, - WT_ITEM *key, WT_ITEM *value, WT_UPDATE *upd_arg, bool is_remove) + WT_ITEM *key, WT_ITEM *value, + WT_UPDATE *upd_arg, bool is_remove, bool exclusive) { WT_DECL_RET; WT_INSERT *ins; @@ -85,9 +81,8 @@ __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, if (cbt->compare == 0) { if (cbt->ins == NULL) { /* Allocate an update array as necessary. */ - WT_PAGE_ALLOC_AND_SWAP(session, - page, mod->mod_row_update, - upd_entry, page->pg_row_entries); + WT_PAGE_ALLOC_AND_SWAP(session, page, + mod->mod_row_update, upd_entry, page->entries); /* Set the WT_UPDATE array reference. */ upd_entry = &mod->mod_row_update[cbt->slot]; @@ -133,7 +128,7 @@ __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, /* Serialize the update. */ WT_ERR(__wt_update_serial( - session, page, upd_entry, &upd, upd_size)); + session, page, upd_entry, &upd, upd_size, exclusive)); } else { /* * Allocate the insert array as necessary. @@ -147,10 +142,10 @@ __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, * slot. That's hard, so we set a flag. */ WT_PAGE_ALLOC_AND_SWAP(session, page, - mod->mod_row_insert, ins_headp, page->pg_row_entries + 1); + mod->mod_row_insert, ins_headp, page->entries + 1); ins_slot = F_ISSET(cbt, WT_CBT_SEARCH_SMALLEST) ? - page->pg_row_entries: cbt->slot; + page->entries: cbt->slot; ins_headp = &mod->mod_row_insert[ins_slot]; /* Allocate the WT_INSERT_HEAD structure as necessary. */ @@ -208,7 +203,7 @@ __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, /* Insert the WT_INSERT structure. */ WT_ERR(__wt_insert_serial( session, page, cbt->ins_head, cbt->ins_stack, - &ins, ins_size, skipdepth)); + &ins, ins_size, skipdepth, exclusive)); } if (logged) diff --git a/src/third_party/wiredtiger/src/btree/row_srch.c b/src/third_party/wiredtiger/src/btree/row_srch.c index d4e82c458d4..9c3d467340e 100644 --- a/src/third_party/wiredtiger/src/btree/row_srch.c +++ b/src/third_party/wiredtiger/src/btree/row_srch.c @@ -486,14 +486,14 @@ leaf_only: if (insert && descend_right) { cbt->append_tree = 1; - if (page->pg_row_entries == 0) { - cbt->slot = WT_ROW_SLOT(page, page->pg_row_d); + if (page->entries == 0) { + cbt->slot = WT_ROW_SLOT(page, page->pg_row); F_SET(cbt, WT_CBT_SEARCH_SMALLEST); ins_head = WT_ROW_INSERT_SMALLEST(page); } else { cbt->slot = WT_ROW_SLOT(page, - page->pg_row_d + (page->pg_row_entries - 1)); + page->pg_row + (page->entries - 1)); ins_head = WT_ROW_INSERT_SLOT(page, cbt->slot); } @@ -511,11 +511,11 @@ leaf_only: * doing the tests and error handling inside the loop costs about 5%. */ base = 0; - limit = page->pg_row_entries; + limit = page->entries; if (collator == NULL && srch_key->size <= WT_COMPARE_SHORT_MAXLEN) for (; limit != 0; limit >>= 1) { indx = base + (limit >> 1); - rip = page->pg_row_d + indx; + rip = page->pg_row + indx; WT_ERR( __wt_row_leaf_key(session, page, rip, item, true)); @@ -529,7 +529,7 @@ leaf_only: else if (collator == NULL) for (; limit != 0; limit >>= 1) { indx = base + (limit >> 1); - rip = page->pg_row_d + indx; + rip = page->pg_row + indx; WT_ERR( __wt_row_leaf_key(session, page, rip, item, true)); @@ -547,7 +547,7 @@ leaf_only: else for (; limit != 0; limit >>= 1) { indx = base + (limit >> 1); - rip = page->pg_row_d + indx; + rip = page->pg_row + indx; WT_ERR( __wt_row_leaf_key(session, page, rip, item, true)); @@ -591,13 +591,13 @@ leaf_match: cbt->compare = 0; */ if (base == 0) { cbt->compare = 1; - cbt->slot = WT_ROW_SLOT(page, page->pg_row_d); + cbt->slot = WT_ROW_SLOT(page, page->pg_row); F_SET(cbt, WT_CBT_SEARCH_SMALLEST); ins_head = WT_ROW_INSERT_SMALLEST(page); } else { cbt->compare = -1; - cbt->slot = WT_ROW_SLOT(page, page->pg_row_d + (base - 1)); + cbt->slot = WT_ROW_SLOT(page, page->pg_row + (base - 1)); ins_head = WT_ROW_INSERT_SLOT(page, cbt->slot); } @@ -623,215 +623,3 @@ leaf_match: cbt->compare = 0; err: WT_TRET(__wt_page_release(session, current, 0)); return (ret); } - -/* - * __wt_row_random_leaf -- - * Return a random key from a row-store leaf page. - */ -int -__wt_row_random_leaf(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) -{ - WT_INSERT *ins, **start, **stop; - WT_INSERT_HEAD *ins_head; - WT_PAGE *page; - uint64_t samples; - uint32_t choice, entries, i; - int level; - - page = cbt->ref->page; - start = stop = NULL; /* [-Wconditional-uninitialized] */ - entries = 0; /* [-Wconditional-uninitialized] */ - - __cursor_pos_clear(cbt); - - /* If the page has disk-based entries, select from them. */ - if (page->pg_row_entries != 0) { - cbt->compare = 0; - cbt->slot = __wt_random(&session->rnd) % page->pg_row_entries; - - /* - * The real row-store search function builds the key, so we - * have to as well. - */ - return (__wt_row_leaf_key(session, - page, page->pg_row_d + cbt->slot, cbt->tmp, false)); - } - - /* - * If the tree is new (and not empty), it might have a large insert - * list. - * - * Walk down the list until we find a level with at least 50 entries, - * that's where we'll start rolling random numbers. The value 50 is - * used to ignore levels with only a few entries, that is, levels which - * are potentially badly skewed. - */ - F_SET(cbt, WT_CBT_SEARCH_SMALLEST); - if ((ins_head = WT_ROW_INSERT_SMALLEST(page)) == NULL) - return (WT_NOTFOUND); - for (level = WT_SKIP_MAXDEPTH - 1; level >= 0; --level) { - start = &ins_head->head[level]; - for (entries = 0, stop = start; - *stop != NULL; stop = &(*stop)->next[level]) - ++entries; - - if (entries > 50) - break; - } - - /* - * If it's a tiny list and we went all the way to level 0, correct the - * level; entries is correctly set. - */ - if (level < 0) - level = 0; - - /* - * Step down the skip list levels, selecting a random chunk of the name - * space at each level. - */ - for (samples = entries; level > 0; samples += entries) { - /* - * There are (entries) or (entries + 1) chunks of the name space - * considered at each level. They are: between start and the 1st - * element, between the 1st and 2nd elements, and so on to the - * last chunk which is the name space after the stop element on - * the current level. This last chunk of name space may or may - * not be there: as we descend the levels of the skip list, this - * chunk may appear, depending if the next level down has - * entries logically after the stop point in the current level. - * We can't ignore those entries: because of the algorithm used - * to determine the depth of a skiplist, there may be a large - * number of entries "revealed" by descending a level. - * - * If the next level down has more items after the current stop - * point, there are (entries + 1) chunks to consider, else there - * are (entries) chunks. - */ - if (*(stop - 1) == NULL) - choice = __wt_random(&session->rnd) % entries; - else - choice = __wt_random(&session->rnd) % (entries + 1); - - if (choice == entries) { - /* - * We selected the name space after the stop element on - * this level. Set the start point to the current stop - * point, descend a level and move the stop element to - * the end of the list, that is, the end of the newly - * discovered name space, counting entries as we go. - */ - start = stop; - --start; - --level; - for (entries = 0, stop = start; - *stop != NULL; stop = &(*stop)->next[level]) - ++entries; - } else { - /* - * We selected another name space on the level. Move the - * start pointer the selected number of entries forward - * to the start of the selected chunk (if the selected - * number is 0, start won't move). Set the stop pointer - * to the next element in the list and drop both start - * and stop down a level. - */ - for (i = 0; i < choice; ++i) - start = &(*start)->next[level]; - stop = &(*start)->next[level]; - - --start; - --stop; - --level; - - /* Count the entries in the selected name space. */ - for (entries = 0, - ins = *start; ins != *stop; ins = ins->next[level]) - ++entries; - } - } - - /* - * When we reach the bottom level, entries will already be set. Select - * a random entry from the name space and return it. - * - * It should be impossible for the entries count to be 0 at this point, - * but check for it out of paranoia and to quiet static testing tools. - */ - if (entries > 0) - entries = __wt_random(&session->rnd) % entries; - for (ins = *start; entries > 0; --entries) - ins = ins->next[0]; - - cbt->ins = ins; - cbt->ins_head = ins_head; - cbt->compare = 0; - - /* - * Random lookups in newly created collections can be slow if a page - * consists of a large skiplist. Schedule the page for eviction if we - * encounter a large skiplist. This worthwhile because applications - * that take a sample often take many samples, so the overhead of - * traversing the skip list each time accumulates to real time. - */ - if (samples > 5000) - __wt_page_evict_soon(session, cbt->ref); - - return (0); -} - -/* - * __wt_row_random_descent -- - * Find a random leaf page in a row-store tree. - */ -int -__wt_row_random_descent(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) -{ - WT_BTREE *btree; - WT_DECL_RET; - WT_PAGE *page; - WT_PAGE_INDEX *pindex; - WT_REF *current, *descent; - - btree = S2BT(session); - current = NULL; - - if (0) { -restart: /* - * Discard the currently held page and restart the search from - * the root. - */ - WT_RET(__wt_page_release(session, current, 0)); - } - - /* Search the internal pages of the tree. */ - current = &btree->root; - for (;;) { - page = current->page; - if (page->type != WT_PAGE_ROW_INT) - break; - - WT_INTL_INDEX_GET(session, page, pindex); - descent = pindex->index[ - __wt_random(&session->rnd) % pindex->entries]; - - /* - * Swap the current page for the child page. If the page splits - * while we're retrieving it, restart the search at the root. - * - * On other error, simply return, the swap call ensures we're - * holding nothing on failure. - */ - if ((ret = __wt_page_swap( - session, current, descent, WT_READ_RESTART_OK)) == 0) { - current = descent; - continue; - } - if (ret == WT_RESTART) - goto restart; - return (ret); - } - - cbt->ref = current; - return (0); -} diff --git a/src/third_party/wiredtiger/src/cache/cache_las.c b/src/third_party/wiredtiger/src/cache/cache_las.c index 41f50957809..41da4225f3a 100644 --- a/src/third_party/wiredtiger/src/cache/cache_las.c +++ b/src/third_party/wiredtiger/src/cache/cache_las.c @@ -63,6 +63,7 @@ int __wt_las_create(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; + WT_DECL_RET; uint32_t session_flags; const char *drop_cfg[] = { WT_CONFIG_BASE(session, WT_SESSION_drop), "force=true", NULL }; @@ -80,7 +81,9 @@ __wt_las_create(WT_SESSION_IMPL *session) * * Discard any previous incarnation of the table. */ - WT_RET(__wt_session_drop(session, WT_LAS_URI, drop_cfg)); + WT_WITH_SCHEMA_LOCK(session, + ret = __wt_schema_drop(session, WT_LAS_URI, drop_cfg)); + WT_RET(ret); /* Re-create the table. */ WT_RET(__wt_session_create(session, WT_LAS_URI, WT_LAS_FORMAT)); diff --git a/src/third_party/wiredtiger/src/checksum/power8/crc32_wrapper.c b/src/third_party/wiredtiger/src/checksum/power8/crc32_wrapper.c index ddfa2bdaeb8..a9be9ced1c6 100644 --- a/src/third_party/wiredtiger/src/checksum/power8/crc32_wrapper.c +++ b/src/third_party/wiredtiger/src/checksum/power8/crc32_wrapper.c @@ -1,4 +1,6 @@ #if defined(__powerpc64__) +#include "wt_internal.h" + #define CRC_TABLE #include "crc32_constants.h" @@ -68,8 +70,6 @@ out: } #endif -#include "wt_internal.h" - /* * __wt_checksum_hw -- * WiredTiger: return a checksum for a chunk of memory. diff --git a/src/third_party/wiredtiger/src/checksum/software/checksum.c b/src/third_party/wiredtiger/src/checksum/software/checksum.c index 30362584a3e..65ed74bbe06 100644 --- a/src/third_party/wiredtiger/src/checksum/software/checksum.c +++ b/src/third_party/wiredtiger/src/checksum/software/checksum.c @@ -1101,6 +1101,7 @@ static const uint32_t g_crc_slicing[8][256] = { */ uint32_t __wt_checksum_sw(const void *chunk, size_t len) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { uint32_t crc, next; size_t nqwords; diff --git a/src/third_party/wiredtiger/src/checksum/zseries/crc32-s390x.c b/src/third_party/wiredtiger/src/checksum/zseries/crc32-s390x.c index f77d6768d42..28b46594220 100644 --- a/src/third_party/wiredtiger/src/checksum/zseries/crc32-s390x.c +++ b/src/third_party/wiredtiger/src/checksum/zseries/crc32-s390x.c @@ -6,8 +6,20 @@ * Author(s): Hendrik Brueckner * */ +#include "wt_internal.h" + #include #include + +#if defined(HAVE_CRC32_HARDWARE) + +#include + +/* RHEL 7 has kernel support, but does not define this constant in the lib c headers. */ +#ifndef HWCAP_S390_VX +#define HWCAP_S390_VX 2048 +#endif + #include "crc32-s390x.h" #include "slicing-consts.h" @@ -69,8 +81,6 @@ unsigned int __wt_crc32c_le(unsigned int crc, const unsigned char *buf, size_t l /* Main CRC-32 functions */ DEFINE_CRC32_VX(__wt_crc32c_le_vx, __wt_crc32c_le_vgfm_16, __wt_crc32c_le) -#include "wt_internal.h" - /* * __wt_checksum_hw -- * WiredTiger: return a checksum for a chunk of memory. @@ -81,6 +91,8 @@ __wt_checksum_hw(const void *chunk, size_t len) return (~__wt_crc32c_le_vx(0xffffffff, chunk, len)); } +#endif + /* * __wt_checksum_init -- * WiredTiger: detect CRC hardware and set the checksum function. @@ -89,8 +101,14 @@ void __wt_checksum_init(void) { #if defined(HAVE_CRC32_HARDWARE) - __wt_process.checksum = __wt_checksum_hw; -#else + unsigned long caps = getauxval(AT_HWCAP); + + if (caps & HWCAP_S390_VX) + __wt_process.checksum = __wt_checksum_hw; + else + __wt_process.checksum = __wt_checksum_sw; + +#else /* !HAVE_CRC32_HARDWARE */ __wt_process.checksum = __wt_checksum_sw; #endif } diff --git a/src/third_party/wiredtiger/src/config/config.c b/src/third_party/wiredtiger/src/config/config.c index 3416153d160..a47dfe76aec 100644 --- a/src/third_party/wiredtiger/src/config/config.c +++ b/src/third_party/wiredtiger/src/config/config.c @@ -711,7 +711,7 @@ __wt_config_getones(WT_SESSION_IMPL *session, /* * __wt_config_getones_none -- * Get the value for a given string key from a single config string. - * Treat "none" as empty. + * Treat "none" as empty. */ int __wt_config_getones_none(WT_SESSION_IMPL *session, diff --git a/src/third_party/wiredtiger/src/config/config_api.c b/src/third_party/wiredtiger/src/config/config_api.c index 76a51903588..c1299baaafe 100644 --- a/src/third_party/wiredtiger/src/config/config_api.c +++ b/src/third_party/wiredtiger/src/config/config_api.c @@ -158,24 +158,25 @@ wiredtiger_config_validate(WT_SESSION *wt_session, } /* - * __conn_foc_add -- + * __wt_conn_foc_add -- * Add a new entry into the connection's free-on-close list. */ -static int -__conn_foc_add(WT_SESSION_IMPL *session, const void *p) +void +__wt_conn_foc_add(WT_SESSION_IMPL *session, const void *p) { WT_CONNECTION_IMPL *conn; conn = S2C(session); /* - * Our caller is expected to be holding any locks we need. + * Callers of this function are expected to be holding the connection's + * api_lock. + * + * All callers of this function currently ignore errors. */ - WT_RET(__wt_realloc_def( - session, &conn->foc_size, conn->foc_cnt + 1, &conn->foc)); - - conn->foc[conn->foc_cnt++] = (void *)p; - return (0); + if (__wt_realloc_def( + session, &conn->foc_size, conn->foc_cnt + 1, &conn->foc) == 0) + conn->foc[conn->foc_cnt++] = (void *)p; } /* @@ -214,7 +215,7 @@ __wt_configure_method(WT_SESSION_IMPL *session, WT_CONFIG_ENTRY *entry; WT_CONNECTION_IMPL *conn; WT_DECL_RET; - size_t cnt; + size_t cnt, len; char *newcheck_name, *p; /* @@ -275,12 +276,10 @@ __wt_configure_method(WT_SESSION_IMPL *session, */ WT_ERR(__wt_calloc_one(session, &entry)); entry->method = (*epp)->method; - WT_ERR(__wt_calloc_def(session, - strlen((*epp)->base) + strlen(",") + strlen(config) + 1, &p)); - (void)strcpy(p, (*epp)->base); - (void)strcat(p, ","); - (void)strcat(p, config); + len = strlen((*epp)->base) + strlen(",") + strlen(config) + 1; + WT_ERR(__wt_calloc_def(session, len, &p)); entry->base = p; + WT_ERR(__wt_snprintf(p, len, "%s,%s", (*epp)->base, config)); /* * There may be a default value in the config argument passed in (for @@ -328,12 +327,12 @@ __wt_configure_method(WT_SESSION_IMPL *session, * order to avoid freeing chunks of memory twice. Again, this isn't a * commonly used API and it shouldn't ever happen, just leak it. */ - (void)__conn_foc_add(session, entry->base); - (void)__conn_foc_add(session, entry); - (void)__conn_foc_add(session, checks); - (void)__conn_foc_add(session, newcheck->type); - (void)__conn_foc_add(session, newcheck->checks); - (void)__conn_foc_add(session, newcheck_name); + __wt_conn_foc_add(session, entry->base); + __wt_conn_foc_add(session, entry); + __wt_conn_foc_add(session, checks); + __wt_conn_foc_add(session, newcheck->type); + __wt_conn_foc_add(session, newcheck->checks); + __wt_conn_foc_add(session, newcheck_name); /* * Instead of using locks to protect configuration information, assume diff --git a/src/third_party/wiredtiger/src/config/config_collapse.c b/src/third_party/wiredtiger/src/config/config_collapse.c index 7fe78d06ba7..5abe7556a03 100644 --- a/src/third_party/wiredtiger/src/config/config_collapse.c +++ b/src/third_party/wiredtiger/src/config/config_collapse.c @@ -372,6 +372,7 @@ __config_merge_cmp(const void *a, const void *b) int __wt_config_merge(WT_SESSION_IMPL *session, const char **cfg, const char *cfg_strip, const char **config_ret) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_CONFIG_MERGE merge; WT_DECL_RET; diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index d57bc418c93..f152fbacad4 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -24,6 +24,7 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_load_extension[] = { }; static const WT_CONFIG_CHECK confchk_WT_CONNECTION_open_session[] = { + { "ignore_cache_size", "boolean", NULL, NULL, NULL, 0 }, { "isolation", "string", NULL, "choices=[\"read-uncommitted\",\"read-committed\"," "\"snapshot\"]", @@ -146,11 +147,12 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { confchk_WT_CONNECTION_reconfigure_statistics_log_subconfigs, 5 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," - "\"evict\",\"evictserver\",\"fileops\",\"handleops\",\"log\"," - "\"lsm\",\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\"," - "\"read\",\"rebalance\",\"reconcile\",\"recovery\",\"salvage\"," - "\"shared_cache\",\"split\",\"temporary\",\"thread_group\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," + "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," + "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," + "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," + "\"split\",\"temporary\",\"thread_group\",\"transaction\"," + "\"verify\",\"version\",\"write\"]", NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } }; @@ -161,6 +163,14 @@ static const WT_CONFIG_CHECK confchk_WT_CURSOR_reconfigure[] = { { NULL, NULL, NULL, NULL, NULL, 0 } }; +static const WT_CONFIG_CHECK confchk_WT_SESSION_alter[] = { + { "access_pattern_hint", "string", + NULL, "choices=[\"none\",\"random\",\"sequential\"]", + NULL, 0 }, + { "cache_resident", "boolean", NULL, NULL, NULL, 0 }, + { NULL, NULL, NULL, NULL, NULL, 0 } +}; + static const WT_CONFIG_CHECK confchk_WT_SESSION_begin_transaction[] = { { "isolation", "string", NULL, "choices=[\"read-uncommitted\",\"read-committed\"," @@ -223,6 +233,9 @@ static const WT_CONFIG_CHECK }; static const WT_CONFIG_CHECK confchk_WT_SESSION_create[] = { + { "access_pattern_hint", "string", + NULL, "choices=[\"none\",\"random\",\"sequential\"]", + NULL, 0 }, { "allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0 }, @@ -281,7 +294,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create[] = { { "source", "string", NULL, NULL, NULL, 0 }, { "split_deepen_min_child", "int", NULL, NULL, NULL, 0 }, { "split_deepen_per_child", "int", NULL, NULL, NULL, 0 }, - { "split_pct", "int", NULL, "min=25,max=100", NULL, 0 }, + { "split_pct", "int", NULL, "min=50,max=100", NULL, 0 }, { "type", "string", NULL, NULL, NULL, 0 }, { "value_format", "format", __wt_struct_confchk, NULL, @@ -344,6 +357,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_open_cursor[] = { }; static const WT_CONFIG_CHECK confchk_WT_SESSION_reconfigure[] = { + { "ignore_cache_size", "boolean", NULL, NULL, NULL, 0 }, { "isolation", "string", NULL, "choices=[\"read-uncommitted\",\"read-committed\"," "\"snapshot\"]", @@ -369,6 +383,7 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_snapshot[] = { { "drop", "category", NULL, NULL, confchk_WT_SESSION_snapshot_drop_subconfigs, 4 }, + { "include_updates", "boolean", NULL, NULL, NULL, 0 }, { "name", "string", NULL, NULL, NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } }; @@ -398,6 +413,9 @@ static const WT_CONFIG_CHECK confchk_colgroup_meta[] = { }; static const WT_CONFIG_CHECK confchk_file_config[] = { + { "access_pattern_hint", "string", + NULL, "choices=[\"none\",\"random\",\"sequential\"]", + NULL, 0 }, { "allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0 }, @@ -448,7 +466,7 @@ static const WT_CONFIG_CHECK confchk_file_config[] = { { "prefix_compression_min", "int", NULL, "min=0", NULL, 0 }, { "split_deepen_min_child", "int", NULL, NULL, NULL, 0 }, { "split_deepen_per_child", "int", NULL, NULL, NULL, 0 }, - { "split_pct", "int", NULL, "min=25,max=100", NULL, 0 }, + { "split_pct", "int", NULL, "min=50,max=100", NULL, 0 }, { "value_format", "format", __wt_struct_confchk, NULL, NULL, 0 }, @@ -456,6 +474,9 @@ static const WT_CONFIG_CHECK confchk_file_config[] = { }; static const WT_CONFIG_CHECK confchk_file_meta[] = { + { "access_pattern_hint", "string", + NULL, "choices=[\"none\",\"random\",\"sequential\"]", + NULL, 0 }, { "allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0 }, @@ -509,7 +530,7 @@ static const WT_CONFIG_CHECK confchk_file_meta[] = { { "prefix_compression_min", "int", NULL, "min=0", NULL, 0 }, { "split_deepen_min_child", "int", NULL, NULL, NULL, 0 }, { "split_deepen_per_child", "int", NULL, NULL, NULL, 0 }, - { "split_pct", "int", NULL, "min=25,max=100", NULL, 0 }, + { "split_pct", "int", NULL, "min=50,max=100", NULL, 0 }, { "value_format", "format", __wt_struct_confchk, NULL, NULL, 0 }, @@ -534,6 +555,9 @@ static const WT_CONFIG_CHECK confchk_index_meta[] = { }; static const WT_CONFIG_CHECK confchk_lsm_meta[] = { + { "access_pattern_hint", "string", + NULL, "choices=[\"none\",\"random\",\"sequential\"]", + NULL, 0 }, { "allocation_size", "int", NULL, "min=512B,max=128MB", NULL, 0 }, @@ -590,7 +614,7 @@ static const WT_CONFIG_CHECK confchk_lsm_meta[] = { { "prefix_compression_min", "int", NULL, "min=0", NULL, 0 }, { "split_deepen_min_child", "int", NULL, NULL, NULL, 0 }, { "split_deepen_per_child", "int", NULL, NULL, NULL, 0 }, - { "split_pct", "int", NULL, "min=25,max=100", NULL, 0 }, + { "split_pct", "int", NULL, "min=50,max=100", NULL, 0 }, { "value_format", "format", __wt_struct_confchk, NULL, NULL, 0 }, @@ -657,6 +681,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3 }, { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0 }, + { "builtin_extension_config", "string", NULL, NULL, NULL, 0 }, { "cache_overhead", "int", NULL, "min=0,max=30", NULL, 0 }, { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0 }, { "checkpoint", "category", @@ -725,11 +750,12 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { { "use_environment_priv", "boolean", NULL, NULL, NULL, 0 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," - "\"evict\",\"evictserver\",\"fileops\",\"handleops\",\"log\"," - "\"lsm\",\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\"," - "\"read\",\"rebalance\",\"reconcile\",\"recovery\",\"salvage\"," - "\"shared_cache\",\"split\",\"temporary\",\"thread_group\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," + "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," + "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," + "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," + "\"split\",\"temporary\",\"thread_group\",\"transaction\"," + "\"verify\",\"version\",\"write\"]", NULL, 0 }, { "write_through", "list", NULL, "choices=[\"data\",\"log\"]", @@ -742,6 +768,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3 }, { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0 }, + { "builtin_extension_config", "string", NULL, NULL, NULL, 0 }, { "cache_overhead", "int", NULL, "min=0,max=30", NULL, 0 }, { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0 }, { "checkpoint", "category", @@ -810,11 +837,12 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { { "use_environment_priv", "boolean", NULL, NULL, NULL, 0 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," - "\"evict\",\"evictserver\",\"fileops\",\"handleops\",\"log\"," - "\"lsm\",\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\"," - "\"read\",\"rebalance\",\"reconcile\",\"recovery\",\"salvage\"," - "\"shared_cache\",\"split\",\"temporary\",\"thread_group\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," + "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," + "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," + "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," + "\"split\",\"temporary\",\"thread_group\",\"transaction\"," + "\"verify\",\"version\",\"write\"]", NULL, 0 }, { "version", "string", NULL, NULL, NULL, 0 }, { "write_through", "list", @@ -828,6 +856,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3 }, { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0 }, + { "builtin_extension_config", "string", NULL, NULL, NULL, 0 }, { "cache_overhead", "int", NULL, "min=0,max=30", NULL, 0 }, { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0 }, { "checkpoint", "category", @@ -890,11 +919,12 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { confchk_wiredtiger_open_transaction_sync_subconfigs, 2 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," - "\"evict\",\"evictserver\",\"fileops\",\"handleops\",\"log\"," - "\"lsm\",\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\"," - "\"read\",\"rebalance\",\"reconcile\",\"recovery\",\"salvage\"," - "\"shared_cache\",\"split\",\"temporary\",\"thread_group\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," + "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," + "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," + "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," + "\"split\",\"temporary\",\"thread_group\",\"transaction\"," + "\"verify\",\"version\",\"write\"]", NULL, 0 }, { "version", "string", NULL, NULL, NULL, 0 }, { "write_through", "list", @@ -908,6 +938,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { NULL, NULL, confchk_wiredtiger_open_async_subconfigs, 3 }, { "buffer_alignment", "int", NULL, "min=-1,max=1MB", NULL, 0 }, + { "builtin_extension_config", "string", NULL, NULL, NULL, 0 }, { "cache_overhead", "int", NULL, "min=0,max=30", NULL, 0 }, { "cache_size", "int", NULL, "min=1MB,max=10TB", NULL, 0 }, { "checkpoint", "category", @@ -970,11 +1001,12 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { confchk_wiredtiger_open_transaction_sync_subconfigs, 2 }, { "verbose", "list", NULL, "choices=[\"api\",\"block\",\"checkpoint\",\"compact\"," - "\"evict\",\"evictserver\",\"fileops\",\"handleops\",\"log\"," - "\"lsm\",\"lsm_manager\",\"metadata\",\"mutex\",\"overflow\"," - "\"read\",\"rebalance\",\"reconcile\",\"recovery\",\"salvage\"," - "\"shared_cache\",\"split\",\"temporary\",\"thread_group\"," - "\"transaction\",\"verify\",\"version\",\"write\"]", + "\"evict\",\"evict_stuck\",\"evictserver\",\"fileops\"," + "\"handleops\",\"log\",\"lsm\",\"lsm_manager\",\"metadata\"," + "\"mutex\",\"overflow\",\"read\",\"rebalance\",\"reconcile\"," + "\"recovery\",\"recovery_progress\",\"salvage\",\"shared_cache\"," + "\"split\",\"temporary\",\"thread_group\",\"transaction\"," + "\"verify\",\"version\",\"write\"]", NULL, 0 }, { "write_through", "list", NULL, "choices=[\"data\",\"log\"]", @@ -1017,13 +1049,13 @@ static const WT_CONFIG_ENTRY config_entries[] = { confchk_WT_CONNECTION_load_extension, 4 }, { "WT_CONNECTION.open_session", - "isolation=read-committed", - confchk_WT_CONNECTION_open_session, 1 + "ignore_cache_size=false,isolation=read-committed", + confchk_WT_CONNECTION_open_session, 2 }, { "WT_CONNECTION.reconfigure", "async=(enabled=false,ops_max=1024,threads=2),cache_overhead=8," "cache_size=100MB,checkpoint=(log_size=0,wait=0),error_prefix=," - "eviction=(threads_max=1,threads_min=1)," + "eviction=(threads_max=8,threads_min=1)," "eviction_checkpoint_target=5,eviction_dirty_target=5," "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" ",file_manager=(close_handle_minimum=250,close_idle_time=30," @@ -1047,6 +1079,10 @@ static const WT_CONFIG_ENTRY config_entries[] = { "append=false,overwrite=true", confchk_WT_CURSOR_reconfigure, 2 }, + { "WT_SESSION.alter", + "access_pattern_hint=none,cache_resident=false", + confchk_WT_SESSION_alter, 2 + }, { "WT_SESSION.begin_transaction", "isolation=,name=,priority=0,snapshot=,sync=", confchk_WT_SESSION_begin_transaction, 5 @@ -1068,12 +1104,13 @@ static const WT_CONFIG_ENTRY config_entries[] = { confchk_WT_SESSION_compact, 1 }, { "WT_SESSION.create", - "allocation_size=4KB,app_metadata=,block_allocation=best," - "block_compressor=,cache_resident=false,checksum=uncompressed," - "colgroups=,collator=,columns=,dictionary=0,encryption=(keyid=," - "name=),exclusive=false,extractor=,format=btree,huffman_key=," - "huffman_value=,ignore_in_memory_cache_size=false,immutable=false" - ",internal_item_max=0,internal_key_max=0," + "access_pattern_hint=none,allocation_size=4KB,app_metadata=," + "block_allocation=best,block_compressor=,cache_resident=false," + "checksum=uncompressed,colgroups=,collator=,columns=,dictionary=0" + ",encryption=(keyid=,name=),exclusive=false,extractor=," + "format=btree,huffman_key=,huffman_value=," + "ignore_in_memory_cache_size=false,immutable=false," + "internal_item_max=0,internal_key_max=0," "internal_key_truncate=true,internal_page_max=4KB,key_format=u," "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," "leaf_value_max=0,log=(enabled=true),lsm=(auto_throttle=true," @@ -1082,8 +1119,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { "chunk_size=10MB,merge_max=15,merge_min=0),memory_page_max=5MB," "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," "prefix_compression_min=4,source=,split_deepen_min_child=0," - "split_deepen_per_child=0,split_pct=75,type=file,value_format=u", - confchk_WT_SESSION_create, 41 + "split_deepen_per_child=0,split_pct=90,type=file,value_format=u", + confchk_WT_SESSION_create, 42 }, { "WT_SESSION.drop", "checkpoint_wait=true,force=false,lock_wait=true," @@ -1116,8 +1153,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { NULL, 0 }, { "WT_SESSION.reconfigure", - "isolation=read-committed", - confchk_WT_SESSION_reconfigure, 1 + "ignore_cache_size=false,isolation=read-committed", + confchk_WT_SESSION_reconfigure, 2 }, { "WT_SESSION.rename", "", @@ -1136,8 +1173,8 @@ static const WT_CONFIG_ENTRY config_entries[] = { confchk_WT_SESSION_salvage, 1 }, { "WT_SESSION.snapshot", - "drop=(all=false,before=,names=,to=),name=", - confchk_WT_SESSION_snapshot, 2 + "drop=(all=false,before=,names=,to=),include_updates=false,name=", + confchk_WT_SESSION_snapshot, 3 }, { "WT_SESSION.strerror", "", @@ -1165,35 +1202,35 @@ static const WT_CONFIG_ENTRY config_entries[] = { confchk_colgroup_meta, 5 }, { "file.config", - "allocation_size=4KB,app_metadata=,block_allocation=best," - "block_compressor=,cache_resident=false,checksum=uncompressed," - "collator=,columns=,dictionary=0,encryption=(keyid=,name=)," - "format=btree,huffman_key=,huffman_value=," - "ignore_in_memory_cache_size=false,internal_item_max=0," - "internal_key_max=0,internal_key_truncate=true," - "internal_page_max=4KB,key_format=u,key_gap=10,leaf_item_max=0," - "leaf_key_max=0,leaf_page_max=32KB,leaf_value_max=0," - "log=(enabled=true),memory_page_max=5MB,os_cache_dirty_max=0," - "os_cache_max=0,prefix_compression=false,prefix_compression_min=4" - ",split_deepen_min_child=0,split_deepen_per_child=0,split_pct=75," - "value_format=u", - confchk_file_config, 34 - }, - { "file.meta", - "allocation_size=4KB,app_metadata=,block_allocation=best," - "block_compressor=,cache_resident=false,checkpoint=," - "checkpoint_lsn=,checksum=uncompressed,collator=,columns=," - "dictionary=0,encryption=(keyid=,name=),format=btree,huffman_key=" - ",huffman_value=,id=,ignore_in_memory_cache_size=false," + "access_pattern_hint=none,allocation_size=4KB,app_metadata=," + "block_allocation=best,block_compressor=,cache_resident=false," + "checksum=uncompressed,collator=,columns=,dictionary=0," + "encryption=(keyid=,name=),format=btree,huffman_key=," + "huffman_value=,ignore_in_memory_cache_size=false," "internal_item_max=0,internal_key_max=0," "internal_key_truncate=true,internal_page_max=4KB,key_format=u," "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," "leaf_value_max=0,log=(enabled=true),memory_page_max=5MB," "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," "prefix_compression_min=4,split_deepen_min_child=0," - "split_deepen_per_child=0,split_pct=75,value_format=u," - "version=(major=0,minor=0)", - confchk_file_meta, 38 + "split_deepen_per_child=0,split_pct=90,value_format=u", + confchk_file_config, 35 + }, + { "file.meta", + "access_pattern_hint=none,allocation_size=4KB,app_metadata=," + "block_allocation=best,block_compressor=,cache_resident=false," + "checkpoint=,checkpoint_lsn=,checksum=uncompressed,collator=," + "columns=,dictionary=0,encryption=(keyid=,name=),format=btree," + "huffman_key=,huffman_value=,id=," + "ignore_in_memory_cache_size=false,internal_item_max=0," + "internal_key_max=0,internal_key_truncate=true," + "internal_page_max=4KB,key_format=u,key_gap=10,leaf_item_max=0," + "leaf_key_max=0,leaf_page_max=32KB,leaf_value_max=0," + "log=(enabled=true),memory_page_max=5MB,os_cache_dirty_max=0," + "os_cache_max=0,prefix_compression=false,prefix_compression_min=4" + ",split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90," + "value_format=u,version=(major=0,minor=0)", + confchk_file_meta, 39 }, { "index.meta", "app_metadata=,collator=,columns=,extractor=,immutable=false," @@ -1201,23 +1238,23 @@ static const WT_CONFIG_ENTRY config_entries[] = { confchk_index_meta, 10 }, { "lsm.meta", - "allocation_size=4KB,app_metadata=,block_allocation=best," - "block_compressor=,cache_resident=false,checksum=uncompressed," - "chunks=,collator=,columns=,dictionary=0,encryption=(keyid=," - "name=),format=btree,huffman_key=,huffman_value=," - "ignore_in_memory_cache_size=false,internal_item_max=0," - "internal_key_max=0,internal_key_truncate=true," - "internal_page_max=4KB,key_format=u,key_gap=10,last=," - "leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB," - "leaf_value_max=0,log=(enabled=true),lsm=(auto_throttle=true," - "bloom=true,bloom_bit_count=16,bloom_config=,bloom_hash_count=8," - "bloom_oldest=false,chunk_count_limit=0,chunk_max=5GB," - "chunk_size=10MB,merge_max=15,merge_min=0),memory_page_max=5MB," - "old_chunks=,os_cache_dirty_max=0,os_cache_max=0," - "prefix_compression=false,prefix_compression_min=4," - "split_deepen_min_child=0,split_deepen_per_child=0,split_pct=75," - "value_format=u", - confchk_lsm_meta, 38 + "access_pattern_hint=none,allocation_size=4KB,app_metadata=," + "block_allocation=best,block_compressor=,cache_resident=false," + "checksum=uncompressed,chunks=,collator=,columns=,dictionary=0," + "encryption=(keyid=,name=),format=btree,huffman_key=," + "huffman_value=,ignore_in_memory_cache_size=false," + "internal_item_max=0,internal_key_max=0," + "internal_key_truncate=true,internal_page_max=4KB,key_format=u," + "key_gap=10,last=,leaf_item_max=0,leaf_key_max=0," + "leaf_page_max=32KB,leaf_value_max=0,log=(enabled=true)," + "lsm=(auto_throttle=true,bloom=true,bloom_bit_count=16," + "bloom_config=,bloom_hash_count=8,bloom_oldest=false," + "chunk_count_limit=0,chunk_max=5GB,chunk_size=10MB,merge_max=15," + "merge_min=0),memory_page_max=5MB,old_chunks=," + "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false," + "prefix_compression_min=4,split_deepen_min_child=0," + "split_deepen_per_child=0,split_pct=90,value_format=u", + confchk_lsm_meta, 39 }, { "table.meta", "app_metadata=,colgroups=,collator=,columns=,key_format=u," @@ -1226,62 +1263,64 @@ static const WT_CONFIG_ENTRY config_entries[] = { }, { "wiredtiger_open", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",cache_overhead=8,cache_size=100MB,checkpoint=(log_size=0," - "wait=0),checkpoint_sync=true,config_base=true,create=false," - "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=1,threads_min=1)," - "eviction_checkpoint_target=5,eviction_dirty_target=5," - "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" - ",exclusive=false,extensions=,file_extend=," - "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000,in_memory=false," - "log=(archive=true,compressor=,enabled=false,file_max=100MB," - "path=\".\",prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,readonly=false,session_max=100," - "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," - "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" - ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" - ",wait=0),transaction_sync=(enabled=false,method=fsync)," + ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," + "config_base=true,create=false,direct_io=,encryption=(keyid=," + "name=,secretkey=),error_prefix=,eviction=(threads_max=8," + "threads_min=1),eviction_checkpoint_target=5," + "eviction_dirty_target=5,eviction_dirty_trigger=20," + "eviction_target=80,eviction_trigger=95,exclusive=false," + "extensions=,file_extend=,file_manager=(close_handle_minimum=250," + "close_idle_time=30,close_scan_interval=10),hazard_max=1000," + "in_memory=false,log=(archive=true,compressor=,enabled=false," + "file_max=100MB,path=\".\",prealloc=true,recover=on," + "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," + "lsm_merge=true,mmap=true,multiprocess=false,readonly=false," + "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" + ",name=,quota=0,reserve=0,size=500MB),statistics=none," + "statistics_log=(json=false,on_close=false,path=\".\",sources=," + "timestamp=\"%b %d %H:%M:%S\",wait=0)," + "transaction_sync=(enabled=false,method=fsync)," "use_environment=true,use_environment_priv=false,verbose=," "write_through=", - confchk_wiredtiger_open, 39 + confchk_wiredtiger_open, 40 }, { "wiredtiger_open_all", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",cache_overhead=8,cache_size=100MB,checkpoint=(log_size=0," - "wait=0),checkpoint_sync=true,config_base=true,create=false," - "direct_io=,encryption=(keyid=,name=,secretkey=),error_prefix=," - "eviction=(threads_max=1,threads_min=1)," - "eviction_checkpoint_target=5,eviction_dirty_target=5," - "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" - ",exclusive=false,extensions=,file_extend=," - "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000,in_memory=false," - "log=(archive=true,compressor=,enabled=false,file_max=100MB," - "path=\".\",prealloc=true,recover=on,zero_fill=false)," - "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," - "mmap=true,multiprocess=false,readonly=false,session_max=100," - "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," - "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" - ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" - ",wait=0),transaction_sync=(enabled=false,method=fsync)," + ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true," + "config_base=true,create=false,direct_io=,encryption=(keyid=," + "name=,secretkey=),error_prefix=,eviction=(threads_max=8," + "threads_min=1),eviction_checkpoint_target=5," + "eviction_dirty_target=5,eviction_dirty_trigger=20," + "eviction_target=80,eviction_trigger=95,exclusive=false," + "extensions=,file_extend=,file_manager=(close_handle_minimum=250," + "close_idle_time=30,close_scan_interval=10),hazard_max=1000," + "in_memory=false,log=(archive=true,compressor=,enabled=false," + "file_max=100MB,path=\".\",prealloc=true,recover=on," + "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," + "lsm_merge=true,mmap=true,multiprocess=false,readonly=false," + "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" + ",name=,quota=0,reserve=0,size=500MB),statistics=none," + "statistics_log=(json=false,on_close=false,path=\".\",sources=," + "timestamp=\"%b %d %H:%M:%S\",wait=0)," + "transaction_sync=(enabled=false,method=fsync)," "use_environment=true,use_environment_priv=false,verbose=," "version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_all, 40 + confchk_wiredtiger_open_all, 41 }, { "wiredtiger_open_basecfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",cache_overhead=8,cache_size=100MB,checkpoint=(log_size=0," - "wait=0),checkpoint_sync=true,direct_io=,encryption=(keyid=,name=" - ",secretkey=),error_prefix=,eviction=(threads_max=1," - "threads_min=1),eviction_checkpoint_target=5," - "eviction_dirty_target=5,eviction_dirty_trigger=20," - "eviction_target=80,eviction_trigger=95,extensions=,file_extend=," - "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000,log=(archive=true," - "compressor=,enabled=false,file_max=100MB,path=\".\"," - "prealloc=true,recover=on,zero_fill=false)," + ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true,direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=5,eviction_dirty_target=5," + "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" + ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" + ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," + "log=(archive=true,compressor=,enabled=false,file_max=100MB," + "path=\".\",prealloc=true,recover=on,zero_fill=false)," "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," "mmap=true,multiprocess=false,readonly=false,session_max=100," "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," @@ -1289,20 +1328,20 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),transaction_sync=(enabled=false,method=fsync),verbose=," "version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_basecfg, 34 + confchk_wiredtiger_open_basecfg, 35 }, { "wiredtiger_open_usercfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" - ",cache_overhead=8,cache_size=100MB,checkpoint=(log_size=0," - "wait=0),checkpoint_sync=true,direct_io=,encryption=(keyid=,name=" - ",secretkey=),error_prefix=,eviction=(threads_max=1," - "threads_min=1),eviction_checkpoint_target=5," - "eviction_dirty_target=5,eviction_dirty_trigger=20," - "eviction_target=80,eviction_trigger=95,extensions=,file_extend=," - "file_manager=(close_handle_minimum=250,close_idle_time=30," - "close_scan_interval=10),hazard_max=1000,log=(archive=true," - "compressor=,enabled=false,file_max=100MB,path=\".\"," - "prealloc=true,recover=on,zero_fill=false)," + ",builtin_extension_config=,cache_overhead=8,cache_size=100MB," + "checkpoint=(log_size=0,wait=0),checkpoint_sync=true,direct_io=," + "encryption=(keyid=,name=,secretkey=),error_prefix=," + "eviction=(threads_max=8,threads_min=1)," + "eviction_checkpoint_target=5,eviction_dirty_target=5," + "eviction_dirty_trigger=20,eviction_target=80,eviction_trigger=95" + ",extensions=,file_extend=,file_manager=(close_handle_minimum=250" + ",close_idle_time=30,close_scan_interval=10),hazard_max=1000," + "log=(archive=true,compressor=,enabled=false,file_max=100MB," + "path=\".\",prealloc=true,recover=on,zero_fill=false)," "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," "mmap=true,multiprocess=false,readonly=false,session_max=100," "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," @@ -1310,7 +1349,7 @@ static const WT_CONFIG_ENTRY config_entries[] = { ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" ",wait=0),transaction_sync=(enabled=false,method=fsync),verbose=," "write_through=", - confchk_wiredtiger_open_usercfg, 33 + confchk_wiredtiger_open_usercfg, 34 }, { NULL, NULL, NULL, 0 } }; diff --git a/src/third_party/wiredtiger/src/config/config_ext.c b/src/third_party/wiredtiger/src/config/config_ext.c index 56c0018f8c3..88f1390843a 100644 --- a/src/third_party/wiredtiger/src/config/config_ext.c +++ b/src/third_party/wiredtiger/src/config/config_ext.c @@ -8,23 +8,10 @@ #include "wt_internal.h" -/* - * __wt_ext_config_parser_open -- - * WT_EXTENSION_API->config_parser_open implementation - */ -int -__wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, - const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) -{ - WT_UNUSED(wt_ext); - return (wiredtiger_config_parser_open( - wt_session, config, len, config_parserp)); -} - /* * __wt_ext_config_get -- * Given a NULL-terminated list of configuration strings, find the final - * value for a given string key (external API version). + * value for a given string key (external API version). */ int __wt_ext_config_get(WT_EXTENSION_API *wt_api, @@ -43,3 +30,65 @@ __wt_ext_config_get(WT_EXTENSION_API *wt_api, return (WT_NOTFOUND); return (__wt_config_gets(session, cfg, key, cval)); } + +/* + * __wt_ext_config_get_string -- + * Given a configuration string, find the value for a given string key + * (external API version). + */ +int +__wt_ext_config_get_string(WT_EXTENSION_API *wt_api, + WT_SESSION *wt_session, const char *config, const char *key, + WT_CONFIG_ITEM *cval) +{ + WT_CONNECTION_IMPL *conn; + WT_SESSION_IMPL *session; + + conn = (WT_CONNECTION_IMPL *)wt_api->conn; + if ((session = (WT_SESSION_IMPL *)wt_session) == NULL) + session = conn->default_session; + + return (__wt_config_getones(session, config, key, cval)); +} + +/* + * __wt_ext_config_parser_open -- + * WT_EXTENSION_API->config_parser_open implementation + */ +int +__wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, + const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) +{ + WT_UNUSED(wt_ext); + return (wiredtiger_config_parser_open( + wt_session, config, len, config_parserp)); +} + +/* + * __wt_ext_config_parser_open_arg -- + * WT_EXTENSION_API->config_parser_open_arg implementation + */ +int +__wt_ext_config_parser_open_arg(WT_EXTENSION_API *wt_ext, + WT_SESSION *wt_session, WT_CONFIG_ARG *cfg_arg, + WT_CONFIG_PARSER **config_parserp) +{ + const char **cfg, *p; + size_t len; + + WT_UNUSED(wt_ext); + + /* Find the last non-NULL entry in the configuration stack. */ + if ((cfg = (const char **)cfg_arg) == NULL || *cfg == NULL) { + p = NULL; + len = 0; + } else { + while (cfg[1] != NULL) + ++cfg; + p = *cfg; + len = strlen(p); + } + + return (wiredtiger_config_parser_open( + wt_session, p, len, config_parserp)); +} diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index a6e0f57f02e..68d45678965 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -239,8 +239,6 @@ __conn_add_compressor(WT_CONNECTION *wt_conn, WT_NAMED_COMPRESSOR *ncomp; WT_SESSION_IMPL *session; - WT_UNUSED(name); - WT_UNUSED(compressor); ncomp = NULL; conn = (WT_CONNECTION_IMPL *)wt_conn; @@ -756,8 +754,11 @@ __conn_get_extension_api(WT_CONNECTION *wt_conn) conn->extension_api.scr_free = __wt_ext_scr_free; conn->extension_api.collator_config = ext_collator_config; conn->extension_api.collate = ext_collate; - conn->extension_api.config_parser_open = __wt_ext_config_parser_open; conn->extension_api.config_get = __wt_ext_config_get; + conn->extension_api.config_get_string = __wt_ext_config_get_string; + conn->extension_api.config_parser_open = __wt_ext_config_parser_open; + conn->extension_api.config_parser_open_arg = + __wt_ext_config_parser_open_arg; conn->extension_api.metadata_insert = __wt_ext_metadata_insert; conn->extension_api.metadata_remove = __wt_ext_metadata_remove; conn->extension_api.metadata_search = __wt_ext_metadata_search; @@ -789,40 +790,75 @@ __conn_get_extension_api(WT_CONNECTION *wt_conn) return (&conn->extension_api); } +/* + * __conn_builtin_init -- + * Initialize and configure a builtin extension. + */ +static int +__conn_builtin_init(WT_CONNECTION_IMPL *conn, const char *name, + int (*extension_init)(WT_CONNECTION *, WT_CONFIG_ARG *), + const char *cfg[]) +{ + WT_CONFIG_ITEM all_configs, cval; + WT_DECL_RET; + WT_SESSION_IMPL *session; + char *config; + const char *ext_cfg[] = { NULL, NULL }; + + session = conn->default_session; + + WT_RET(__wt_config_gets( + session, cfg, "builtin_extension_config", &all_configs)); + WT_CLEAR(cval); + WT_RET_NOTFOUND_OK(__wt_config_subgets( + session, &all_configs, name, &cval)); + WT_RET(__wt_strndup(session, cval.str, cval.len, &config)); + ext_cfg[0] = config; + + ret = extension_init(&conn->iface, (WT_CONFIG_ARG *)ext_cfg); + __wt_free(session, config); + + return (ret); +} + #ifdef HAVE_BUILTIN_EXTENSION_LZ4 - extern int lz4_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); +extern int lz4_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); #endif #ifdef HAVE_BUILTIN_EXTENSION_SNAPPY - extern int snappy_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); +extern int snappy_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); #endif #ifdef HAVE_BUILTIN_EXTENSION_ZLIB - extern int zlib_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); +extern int zlib_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); #endif #ifdef HAVE_BUILTIN_EXTENSION_ZSTD - extern int zstd_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); +extern int zstd_extension_init(WT_CONNECTION *, WT_CONFIG_ARG *); #endif /* - * __conn_load_default_extensions -- + * __conn_builtin_extensions -- * Load extensions that are enabled via --with-builtins */ static int -__conn_load_default_extensions(WT_CONNECTION_IMPL *conn) +__conn_builtin_extensions(WT_CONNECTION_IMPL *conn, const char *cfg[]) { - WT_UNUSED(conn); - #ifdef HAVE_BUILTIN_EXTENSION_LZ4 - WT_RET(lz4_extension_init(&conn->iface, NULL)); + WT_RET(__conn_builtin_init(conn, "lz4", lz4_extension_init, cfg)); #endif #ifdef HAVE_BUILTIN_EXTENSION_SNAPPY - WT_RET(snappy_extension_init(&conn->iface, NULL)); + WT_RET(__conn_builtin_init(conn, "snappy", snappy_extension_init, cfg)); #endif #ifdef HAVE_BUILTIN_EXTENSION_ZLIB - WT_RET(zlib_extension_init(&conn->iface, NULL)); + WT_RET(__conn_builtin_init(conn, "zlib", zlib_extension_init, cfg)); #endif #ifdef HAVE_BUILTIN_EXTENSION_ZSTD - WT_RET(zstd_extension_init(&conn->iface, NULL)); + WT_RET(__conn_builtin_init(conn, "zstd", zstd_extension_init, cfg)); #endif + + /* Avoid warnings if no builtin extensions are configured. */ + WT_UNUSED(conn); + WT_UNUSED(cfg); + WT_UNUSED(__conn_builtin_init); + return (0); } @@ -839,10 +875,11 @@ __conn_load_extension_int(WT_SESSION_IMPL *session, WT_DLH *dlh; int (*load)(WT_CONNECTION *, WT_CONFIG_ARG *); bool is_local; - const char *init_name, *terminate_name; + const char *ext_config, *init_name, *terminate_name; + const char *ext_cfg[2]; dlh = NULL; - init_name = terminate_name = NULL; + ext_config = init_name = terminate_name = NULL; is_local = strcmp(path, "local") == 0; /* Ensure that the load matches the phase of startup we are in. */ @@ -872,8 +909,14 @@ __conn_load_extension_int(WT_SESSION_IMPL *session, WT_ERR( __wt_dlsym(session, dlh, terminate_name, false, &dlh->terminate)); + WT_CLEAR(cval); + WT_ERR_NOTFOUND_OK(__wt_config_gets(session, cfg, "config", &cval)); + WT_ERR(__wt_strndup(session, cval.str, cval.len, &ext_config)); + ext_cfg[0] = ext_config; + ext_cfg[1] = NULL; + /* Call the load function last, it simplifies error handling. */ - WT_ERR(load(&S2C(session)->iface, (WT_CONFIG_ARG *)cfg)); + WT_ERR(load(&S2C(session)->iface, (WT_CONFIG_ARG *)ext_cfg)); /* Link onto the environment's list of open libraries. */ __wt_spin_lock(session, &S2C(session)->api_lock); @@ -883,6 +926,7 @@ __conn_load_extension_int(WT_SESSION_IMPL *session, err: if (dlh != NULL) WT_TRET(__wt_dlclose(session, dlh)); + __wt_free(session, ext_config); __wt_free(session, init_name); __wt_free(session, terminate_name); return (ret); @@ -1618,8 +1662,8 @@ __conn_single(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR_MSG(session, EINVAL, "Creating a new database is incompatible with " "read-only configuration"); - len = (size_t)snprintf(buf, sizeof(buf), - "%s\n%s\n", WT_WIREDTIGER, WIREDTIGER_VERSION_STRING); + WT_ERR(__wt_snprintf_len_set(buf, sizeof(buf), &len, + "%s\n%s\n", WT_WIREDTIGER, WIREDTIGER_VERSION_STRING)); WT_ERR(__wt_write(session, fh, (wt_off_t)0, len, buf)); WT_ERR(__wt_fsync(session, fh, true)); } else { @@ -1754,6 +1798,7 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) { "checkpoint", WT_VERB_CHECKPOINT }, { "compact", WT_VERB_COMPACT }, { "evict", WT_VERB_EVICT }, + { "evict_stuck", WT_VERB_EVICT_STUCK }, { "evictserver", WT_VERB_EVICTSERVER }, { "fileops", WT_VERB_FILEOPS }, { "handleops", WT_VERB_HANDLEOPS }, @@ -1767,6 +1812,7 @@ __wt_verbose_config(WT_SESSION_IMPL *session, const char *cfg[]) { "rebalance", WT_VERB_REBALANCE }, { "reconcile", WT_VERB_RECONCILE }, { "recovery", WT_VERB_RECOVERY }, + { "recovery_progress", WT_VERB_RECOVERY_PROGRESS }, { "salvage", WT_VERB_SALVAGE }, { "shared_cache", WT_VERB_SHARED_CACHE }, { "split", WT_VERB_SPLIT }, @@ -1942,6 +1988,16 @@ __conn_set_file_system( CONNECTION_API_CALL(conn, session, set_file_system, config, cfg); WT_UNUSED(cfg); + /* + * You can only configure a file system once, and attempting to do it + * again probably means the extension argument didn't have early-load + * set and we've already configured the default file system. + */ + if (conn->file_system != NULL) + WT_ERR_MSG(session, EPERM, + "filesystem already configured; custom filesystems should " + "enable \"early_load\" configuration"); + conn->file_system = file_system; err: API_END_RET(session, ret); @@ -2130,6 +2186,15 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, if (cval.val) F_SET(conn, WT_CONN_READONLY); + /* Configure error messages so we get them right early. */ + WT_ERR(__wt_config_gets(session, cfg, "error_prefix", &cval)); + if (cval.len != 0) + WT_ERR(__wt_strndup( + session, cval.str, cval.len, &conn->error_prefix)); + + /* Set the database home so extensions have access to it. */ + WT_ERR(__conn_home(session, home, cfg)); + /* * Load early extensions before doing further initialization (one early * extension is to configure a file system). @@ -2153,6 +2218,9 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_ERR( __conn_chk_file_system(session, F_ISSET(conn, WT_CONN_READONLY))); + /* Make sure no other thread of control already owns this database. */ + WT_ERR(__conn_single(session, cfg)); + /* * Capture the config_base setting file for later use. Again, if the * application doesn't want us to read the base configuration file, @@ -2162,18 +2230,6 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_ERR(__wt_config_gets(session, cfg, "config_base", &cval)); config_base_set = cval.val != 0; - /* Configure error messages so we get them right early. */ - WT_ERR(__wt_config_gets(session, cfg, "error_prefix", &cval)); - if (cval.len != 0) - WT_ERR(__wt_strndup( - session, cval.str, cval.len, &conn->error_prefix)); - - /* Get the database home. */ - WT_ERR(__conn_home(session, home, cfg)); - - /* Make sure no other thread of control already owns this database. */ - WT_ERR(__conn_single(session, cfg)); - /* * Build the real configuration stack, in the following order (where * later entries override earlier entries): @@ -2194,10 +2250,9 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_ERR(__wt_scr_alloc(session, 0, &i3)); cfg[0] = WT_CONFIG_BASE(session, wiredtiger_open_all); cfg[1] = NULL; - WT_ERR_TEST(snprintf(version, sizeof(version), + WT_ERR(__wt_snprintf(version, sizeof(version), "version=(major=%d,minor=%d)", - WIREDTIGER_VERSION_MAJOR, WIREDTIGER_VERSION_MINOR) >= - (int)sizeof(version), ENOMEM); + WIREDTIGER_VERSION_MAJOR, WIREDTIGER_VERSION_MINOR)); __conn_config_append(cfg, version); /* Ignore the base_config file if config_base_set is false. */ @@ -2265,9 +2320,6 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, } WT_ERR(__wt_verbose_config(session, cfg)); - WT_ERR(__wt_config_gets(session, cfg, "hazard_max", &cval)); - conn->hazard_max = (uint32_t)cval.val; - WT_ERR(__wt_config_gets(session, cfg, "session_max", &cval)); conn->session_size = (uint32_t)cval.val + WT_EXTRA_INTERNAL_SESSIONS; @@ -2355,7 +2407,7 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, * everything else to be in place, and the extensions call back into the * library. */ - WT_ERR(__conn_load_default_extensions(conn)); + WT_ERR(__conn_builtin_extensions(conn, cfg)); WT_ERR(__conn_load_extensions(session, cfg, false)); /* diff --git a/src/third_party/wiredtiger/src/conn/conn_cache.c b/src/third_party/wiredtiger/src/conn/conn_cache.c index fe5f94ea03d..12279709191 100644 --- a/src/third_party/wiredtiger/src/conn/conn_cache.c +++ b/src/third_party/wiredtiger/src/conn/conn_cache.c @@ -143,7 +143,8 @@ __wt_cache_config(WT_SESSION_IMPL *session, bool reconfigure, const char *cfg[]) if (reconfigure) WT_RET(__wt_thread_group_resize( session, &conn->evict_threads, - conn->evict_threads_min, conn->evict_threads_max, + conn->evict_threads_min, + conn->evict_threads_max, WT_THREAD_CAN_WAIT | WT_THREAD_PANIC_FAIL)); return (0); @@ -186,8 +187,8 @@ __wt_cache_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET_MSG(session, EINVAL, "eviction target must be lower than the eviction trigger"); - WT_RET(__wt_cond_auto_alloc(session, "cache eviction server", - false, 10000, WT_MILLION, &cache->evict_cond)); + WT_RET(__wt_cond_auto_alloc(session, + "cache eviction server", 10000, WT_MILLION, &cache->evict_cond)); WT_RET(__wt_spin_init(session, &cache->evict_pass_lock, "evict pass")); WT_RET(__wt_spin_init(session, &cache->evict_queue_lock, "cache eviction queue")); @@ -311,7 +312,7 @@ __wt_cache_destroy(WT_SESSION_IMPL *session) cache->bytes_dirty_intl + cache->bytes_dirty_leaf, cache->pages_dirty_intl + cache->pages_dirty_leaf); - WT_TRET(__wt_cond_auto_destroy(session, &cache->evict_cond)); + __wt_cond_destroy(session, &cache->evict_cond); __wt_spin_destroy(session, &cache->evict_pass_lock); __wt_spin_destroy(session, &cache->evict_queue_lock); __wt_spin_destroy(session, &cache->evict_walk_lock); diff --git a/src/third_party/wiredtiger/src/conn/conn_cache_pool.c b/src/third_party/wiredtiger/src/conn/conn_cache_pool.c index 79c2fc23da5..3f7a770b762 100644 --- a/src/third_party/wiredtiger/src/conn/conn_cache_pool.c +++ b/src/third_party/wiredtiger/src/conn/conn_cache_pool.c @@ -32,7 +32,7 @@ */ #define WT_CACHE_POOL_APP_EVICT_MULTIPLIER 3 #define WT_CACHE_POOL_APP_WAIT_MULTIPLIER 6 -#define WT_CACHE_POOL_READ_MULTIPLIER 1 +#define WT_CACHE_POOL_READ_MULTIPLIER 1 static void __cache_pool_adjust( WT_SESSION_IMPL *, uint64_t, uint64_t, bool, bool *); @@ -104,8 +104,8 @@ __wt_cache_pool_config(WT_SESSION_IMPL *session, const char **cfg) TAILQ_INIT(&cp->cache_pool_qh); WT_ERR(__wt_spin_init( session, &cp->cache_pool_lock, "cache shared pool")); - WT_ERR(__wt_cond_alloc(session, - "cache pool server", false, &cp->cache_pool_cond)); + WT_ERR(__wt_cond_alloc( + session, "cache pool server", &cp->cache_pool_cond)); __wt_process.cache_pool = cp; __wt_verbose(session, @@ -225,7 +225,7 @@ err: __wt_spin_unlock(session, &__wt_process.spinlock); __wt_free(session, pool_name); if (ret != 0 && created) { __wt_free(session, cp->name); - WT_TRET(__wt_cond_destroy(session, &cp->cache_pool_cond)); + __wt_cond_destroy(session, &cp->cache_pool_cond); __wt_free(session, cp); } return (ret); @@ -391,7 +391,7 @@ __wt_conn_cache_pool_destroy(WT_SESSION_IMPL *session) __wt_free(session, cp->name); __wt_spin_destroy(session, &cp->cache_pool_lock); - WT_TRET(__wt_cond_destroy(session, &cp->cache_pool_cond)); + __wt_cond_destroy(session, &cp->cache_pool_cond); __wt_free(session, cp); } @@ -418,8 +418,9 @@ static void __cache_pool_balance(WT_SESSION_IMPL *session, bool forward) { WT_CACHE_POOL *cp; - bool adjusted; uint64_t bump_threshold, highest; + int i; + bool adjusted; cp = __wt_process.cache_pool; adjusted = false; @@ -438,11 +439,17 @@ __cache_pool_balance(WT_SESSION_IMPL *session, bool forward) /* * Actively attempt to: - * - Reduce the amount allocated, if we are over the budget + * - Reduce the amount allocated, if we are over the budget. * - Increase the amount used if there is capacity and any pressure. + * Don't keep trying indefinitely, if we aren't succeeding in reducing + * the cache in use re-assessing the participants' states is necessary. + * We are also holding a lock across this process, which can slow + * participant shutdown if we spend a long time balancing. */ - while (F_ISSET(cp, WT_CACHE_POOL_ACTIVE) && - F_ISSET(S2C(session)->cache, WT_CACHE_POOL_RUN)) { + for (i = 0; + i < 2 * WT_CACHE_POOL_BUMP_THRESHOLD && + F_ISSET(cp, WT_CACHE_POOL_ACTIVE) && + F_ISSET(S2C(session)->cache, WT_CACHE_POOL_RUN); i++) { __cache_pool_adjust( session, highest, bump_threshold, forward, &adjusted); /* @@ -565,7 +572,7 @@ __cache_pool_adjust(WT_SESSION_IMPL *session, WT_CONNECTION_IMPL *entry; uint64_t adjustment, highest_percentile, pressure, reserved, smallest; u_int pct_full; - bool busy, pool_full, grow; + bool busy, decrease_ok, grow, pool_full; *adjustedp = false; cp = __wt_process.cache_pool; @@ -611,6 +618,34 @@ __cache_pool_adjust(WT_SESSION_IMPL *session, if (cache->cp_skip_count > 0 && --cache->cp_skip_count > 0) continue; + /* + * The bump threshold decreases as we try longer to balance + * the pool. Adjust how aggressively we free space from + * participants depending on how long we have been trying. + */ + decrease_ok = false; + /* + * Any participant is a candidate if we have been trying + * for long enough. + */ + if (bump_threshold == 0) + decrease_ok = true; + /* + * Participants that aren't doing application eviction and + * are showing a reasonable amount of usage are excluded + * even if we have been trying for a while. + */ + else if (bump_threshold < WT_CACHE_POOL_BUMP_THRESHOLD / 3 && + (!busy && highest > 1)) + decrease_ok = true; + /* + * Any participant that is proportionally less busy is a + * candidate from the first attempt. + */ + else if (highest > 1 && + pressure < WT_CACHE_POOL_REDUCE_THRESHOLD) + decrease_ok = true; + /* * If the entry is currently allocated less than the reserved * size, increase its allocation. This should only happen if: @@ -624,17 +659,12 @@ __cache_pool_adjust(WT_SESSION_IMPL *session, * Conditions for reducing the amount of resources for an * entry: * - the pool is full, - * - application threads are not busy doing eviction already, * - this entry has more than the minimum amount of space in * use, - * - the read pressure in this entry is below the threshold, - * other entries need more cache, the entry has more than - * the minimum space and there is no available space in the - * pool. + * - it was determined that this slot is a good candidate */ - } else if (pool_full && !busy && - entry->cache_size > reserved && - pressure < WT_CACHE_POOL_REDUCE_THRESHOLD && highest > 1) { + } else if (pool_full && + entry->cache_size > reserved && decrease_ok) { grow = false; /* * Don't drop the size down too much - or it can @@ -733,7 +763,7 @@ __wt_cache_pool_server(void *arg) F_ISSET(cache, WT_CACHE_POOL_RUN)) { if (cp->currently_used <= cp->size) __wt_cond_wait( - session, cp->cache_pool_cond, WT_MILLION); + session, cp->cache_pool_cond, WT_MILLION, NULL); /* * Re-check pool run flag - since we want to avoid getting the diff --git a/src/third_party/wiredtiger/src/conn/conn_ckpt.c b/src/third_party/wiredtiger/src/conn/conn_ckpt.c index 1d18c128c5b..dea261162c2 100644 --- a/src/third_party/wiredtiger/src/conn/conn_ckpt.c +++ b/src/third_party/wiredtiger/src/conn/conn_ckpt.c @@ -64,6 +64,16 @@ __ckpt_server_config(WT_SESSION_IMPL *session, const char **cfg, bool *startp) return (0); } +/* + * __ckpt_server_run_chk -- + * Check to decide if the checkpoint server should continue running. + */ +static bool +__ckpt_server_run_chk(WT_SESSION_IMPL *session) +{ + return (F_ISSET(S2C(session), WT_CONN_SERVER_CHECKPOINT)); +} + /* * __ckpt_server -- * The checkpoint server thread. @@ -80,14 +90,18 @@ __ckpt_server(void *arg) conn = S2C(session); wt_session = (WT_SESSION *)session; - while (F_ISSET(conn, WT_CONN_SERVER_RUN) && - F_ISSET(conn, WT_CONN_SERVER_CHECKPOINT)) { + for (;;) { /* * Wait... * NOTE: If the user only configured logsize, then usecs * will be 0 and this wait won't return until signalled. */ - __wt_cond_wait(session, conn->ckpt_cond, conn->ckpt_usecs); + __wt_cond_wait(session, + conn->ckpt_cond, conn->ckpt_usecs, __ckpt_server_run_chk); + + /* Check if we're quitting or being reconfigured. */ + if (!__ckpt_server_run_chk(session)) + break; /* * Checkpoint the database if the connection is marked dirty. @@ -115,7 +129,8 @@ __ckpt_server(void *arg) * it so we don't do another checkpoint * immediately. */ - __wt_cond_wait(session, conn->ckpt_cond, 1); + __wt_cond_wait( + session, conn->ckpt_cond, 1, NULL); } } else WT_STAT_CONN_INCR(session, txn_checkpoint_skipped); @@ -154,8 +169,7 @@ __ckpt_server_start(WT_CONNECTION_IMPL *conn) "checkpoint-server", true, session_flags, &conn->ckpt_session)); session = conn->ckpt_session; - WT_RET(__wt_cond_alloc( - session, "checkpoint server", false, &conn->ckpt_cond)); + WT_RET(__wt_cond_alloc(session, "checkpoint server", &conn->ckpt_cond)); /* * Start the thread. @@ -219,7 +233,7 @@ __wt_checkpoint_server_destroy(WT_SESSION_IMPL *session) WT_TRET(__wt_thread_join(session, conn->ckpt_tid)); conn->ckpt_tid_set = false; } - WT_TRET(__wt_cond_destroy(session, &conn->ckpt_cond)); + __wt_cond_destroy(session, &conn->ckpt_cond); /* Close the server thread's session. */ if (conn->ckpt_session != NULL) { diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index ac72e330b67..1816e66b0b7 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -12,37 +12,47 @@ * __conn_dhandle_destroy -- * Destroy a data handle. */ -static void +static int __conn_dhandle_destroy(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle) { + WT_DECL_RET; + + WT_WITH_DHANDLE(session, dhandle, ret = __wt_btree_discard(session)); + __wt_rwlock_destroy(session, &dhandle->rwlock); __wt_free(session, dhandle->name); __wt_free(session, dhandle->checkpoint); - __wt_free(session, dhandle->handle); __wt_spin_destroy(session, &dhandle->close_lock); __wt_stat_dsrc_discard(session, dhandle); __wt_overwrite_and_free(session, dhandle); + return (ret); } /* - * __conn_dhandle_alloc -- + * __wt_conn_dhandle_alloc -- * Allocate a new data handle and return it linked into the connection's * list. */ -static int -__conn_dhandle_alloc(WT_SESSION_IMPL *session, - const char *uri, const char *checkpoint, WT_DATA_HANDLE **dhandlep) +int +__wt_conn_dhandle_alloc( + WT_SESSION_IMPL *session, const char *uri, const char *checkpoint) { WT_BTREE *btree; WT_DATA_HANDLE *dhandle; WT_DECL_RET; uint64_t bucket; - *dhandlep = NULL; + /* + * Ensure no one beat us to creating the handle now that we hold the + * write lock. + */ + if ((ret = + __wt_conn_dhandle_find(session, uri, checkpoint)) != WT_NOTFOUND) + return (ret); WT_RET(__wt_calloc_one(session, &dhandle)); - WT_ERR(__wt_rwlock_alloc(session, &dhandle->rwlock, "data handle")); + WT_ERR(__wt_rwlock_init(session, &dhandle->rwlock)); dhandle->name_hash = __wt_hash_city64(uri, strlen(uri)); WT_ERR(__wt_strdup(session, uri, &dhandle->name)); WT_ERR(__wt_strdup(session, checkpoint, &dhandle->checkpoint)); @@ -75,10 +85,10 @@ __conn_dhandle_alloc(WT_SESSION_IMPL *session, bucket = dhandle->name_hash % WT_HASH_ARRAY_SIZE; WT_CONN_DHANDLE_INSERT(S2C(session), dhandle, bucket); - *dhandlep = dhandle; + session->dhandle = dhandle; return (0); -err: __conn_dhandle_destroy(session, dhandle); +err: WT_TRET(__conn_dhandle_destroy(session, dhandle)); return (ret); } @@ -122,10 +132,7 @@ __wt_conn_dhandle_find( } } - WT_RET(__conn_dhandle_alloc(session, uri, checkpoint, &dhandle)); - - session->dhandle = dhandle; - return (0); + return (WT_NOTFOUND); } /* @@ -153,11 +160,11 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) WT_RET(__wt_evict_file_exclusive_on(session)); /* - * If we don't already have the schema lock, make it an error to try - * to acquire it. The problem is that we are holding an exclusive - * lock on the handle, and if we attempt to acquire the schema lock - * we might deadlock with a thread that has the schema lock and wants - * a handle lock (specifically, checkpoint). + * If we don't already have the schema lock, make it an error to try to + * acquire it. The problem is that we are holding an exclusive lock on + * the handle, and if we attempt to acquire the schema lock we might + * deadlock with a thread that has the schema lock and wants a handle + * lock. */ no_schema_lock = false; if (!F_ISSET(session, WT_SESSION_LOCKED_SCHEMA)) { @@ -197,6 +204,7 @@ __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) } WT_TRET(__wt_btree_close(session)); + F_CLR(btree, WT_BTREE_SPECIAL_FLAGS); /* * If we marked a handle dead it will be closed by sweep, via @@ -306,7 +314,8 @@ __wt_conn_btree_open( F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE) && !LF_ISSET(WT_DHANDLE_LOCK_ONLY)); - WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_CLOSING)); + WT_ASSERT(session, + !F_ISSET(S2C(session), WT_CONN_CLOSING_NO_MORE_OPENS)); /* * If the handle is already open, it has to be closed so it can be @@ -400,10 +409,7 @@ __conn_btree_apply_internal(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle, return (ret == EBUSY ? 0 : ret); WT_SAVE_DHANDLE(session, ret = file_func(session, cfg)); - if (WT_META_TRACKING(session)) - WT_TRET(__wt_meta_track_handle_lock(session, false)); - else - WT_TRET(__wt_session_release_btree(session)); + WT_TRET(__wt_session_release_btree(session)); return (ret); } @@ -419,12 +425,11 @@ __wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, { WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *dhandle; + WT_DECL_RET; uint64_t bucket; conn = S2C(session); - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); - /* * If we're given a URI, then we walk only the hash list for that * name. If we don't have a URI we walk the entire dhandle list. @@ -432,29 +437,85 @@ __wt_conn_btree_apply(WT_SESSION_IMPL *session, const char *uri, if (uri != NULL) { bucket = __wt_hash_city64(uri, strlen(uri)) % WT_HASH_ARRAY_SIZE; - TAILQ_FOREACH(dhandle, &conn->dhhash[bucket], hashq) { + + for (dhandle = NULL;;) { + WT_WITH_HANDLE_LIST_READ_LOCK(session, + WT_DHANDLE_NEXT(session, dhandle, + &conn->dhhash[bucket], hashq)); + if (dhandle == NULL) + return (0); + if (!F_ISSET(dhandle, WT_DHANDLE_OPEN) || F_ISSET(dhandle, WT_DHANDLE_DEAD) || dhandle->checkpoint != NULL || strcmp(uri, dhandle->name) != 0) continue; - WT_RET(__conn_btree_apply_internal( - session, dhandle, file_func, name_func, cfg)); + WT_ERR(__conn_btree_apply_internal(session, + dhandle, file_func, name_func, cfg)); } } else { - TAILQ_FOREACH(dhandle, &conn->dhqh, q) { + for (dhandle = NULL;;) { + WT_WITH_HANDLE_LIST_READ_LOCK(session, + WT_DHANDLE_NEXT(session, dhandle, &conn->dhqh, q)); + if (dhandle == NULL) + return (0); + if (!F_ISSET(dhandle, WT_DHANDLE_OPEN) || F_ISSET(dhandle, WT_DHANDLE_DEAD) || dhandle->checkpoint != NULL || !WT_PREFIX_MATCH(dhandle->name, "file:") || - WT_IS_METADATA(session, dhandle)) + WT_IS_METADATA(dhandle)) continue; - WT_RET(__conn_btree_apply_internal( - session, dhandle, file_func, name_func, cfg)); + WT_ERR(__conn_btree_apply_internal(session, + dhandle, file_func, name_func, cfg)); } } - return (0); +err: WT_DHANDLE_RELEASE(dhandle); + return (ret); +} + +/* + * __conn_dhandle_close_one -- + * Lock and, if necessary, close a data handle. + */ +static int +__conn_dhandle_close_one(WT_SESSION_IMPL *session, + const char *uri, const char *checkpoint, bool force) +{ + WT_DECL_RET; + + /* + * Lock the handle exclusively. If this is part of schema-changing + * operation (indicated by metadata tracking being enabled), hold the + * lock for the duration of the operation. + */ + WT_RET(__wt_session_get_btree(session, uri, checkpoint, + NULL, WT_DHANDLE_EXCLUSIVE | WT_DHANDLE_LOCK_ONLY)); + if (WT_META_TRACKING(session)) + WT_RET(__wt_meta_track_handle_lock(session, false)); + + /* + * We have an exclusive lock, which means there are no cursors open at + * this point. Close the handle, if necessary. + */ + if (F_ISSET(session->dhandle, WT_DHANDLE_OPEN)) { + __wt_meta_track_sub_on(session); + ret = __wt_conn_btree_sync_and_close(session, false, force); + + /* + * If the close succeeded, drop any locks it acquired. If + * there was a failure, this function will fail and the whole + * transaction will be rolled back. + */ + if (ret == 0) + ret = __wt_meta_track_sub_off(session); + } + + if (!WT_META_TRACKING(session)) + WT_TRET(__wt_session_release_btree(session)); + + return (ret); } /* @@ -473,46 +534,26 @@ __wt_conn_dhandle_close_all( conn = S2C(session); - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); + WT_ASSERT(session, + F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE)); WT_ASSERT(session, session->dhandle == NULL); + /* + * Lock the live handle first. This ordering is important: we rely on + * locking the live handle to fail fast if the tree is busy (e.g., with + * cursors open or in a checkpoint). + */ + WT_ERR(__conn_dhandle_close_one(session, uri, NULL, force)); + bucket = __wt_hash_city64(uri, strlen(uri)) % WT_HASH_ARRAY_SIZE; TAILQ_FOREACH(dhandle, &conn->dhhash[bucket], hashq) { if (strcmp(dhandle->name, uri) != 0 || + dhandle->checkpoint == NULL || F_ISSET(dhandle, WT_DHANDLE_DEAD)) continue; - session->dhandle = dhandle; - - /* Lock the handle exclusively. */ - WT_ERR(__wt_session_get_btree(session, - dhandle->name, dhandle->checkpoint, - NULL, WT_DHANDLE_EXCLUSIVE | WT_DHANDLE_LOCK_ONLY)); - if (WT_META_TRACKING(session)) - WT_ERR(__wt_meta_track_handle_lock(session, false)); - - /* - * We have an exclusive lock, which means there are no cursors - * open at this point. Close the handle, if necessary. - */ - if (F_ISSET(dhandle, WT_DHANDLE_OPEN)) { - __wt_meta_track_sub_on(session); - ret = __wt_conn_btree_sync_and_close( - session, false, force); - - /* - * If the close succeeded, drop any locks it acquired. - * If there was a failure, this function will fail and - * the whole transaction will be rolled back. - */ - if (ret == 0) - ret = __wt_meta_track_sub_off(session); - } - - if (!WT_META_TRACKING(session)) - WT_TRET(__wt_session_release_btree(session)); - - WT_ERR(ret); + WT_ERR(__conn_dhandle_close_one( + session, dhandle->name, dhandle->checkpoint, force)); } err: session->dhandle = NULL; @@ -534,7 +575,8 @@ __conn_dhandle_remove(WT_SESSION_IMPL *session, bool final) dhandle = session->dhandle; bucket = dhandle->name_hash % WT_HASH_ARRAY_SIZE; - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); + WT_ASSERT(session, + F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE)); WT_ASSERT(session, dhandle != conn->cache->evict_file_next); /* Check if the handle was reacquired by a session while we waited. */ @@ -583,7 +625,7 @@ __wt_conn_dhandle_discard_single( } /* Try to remove the handle, protected by the data handle lock. */ - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, tret = __conn_dhandle_remove(session, final)); if (set_pass_intr) (void)__wt_atomic_subv32(&S2C(session)->cache->pass_intr, 1); @@ -594,7 +636,7 @@ __wt_conn_dhandle_discard_single( */ if (ret == 0 || final) { __conn_btree_config_clear(session); - __conn_dhandle_destroy(session, dhandle); + WT_TRET(__conn_dhandle_destroy(session, dhandle)); session->dhandle = NULL; } @@ -627,7 +669,7 @@ __wt_conn_dhandle_discard(WT_SESSION_IMPL *session) */ restart: TAILQ_FOREACH(dhandle, &conn->dhqh, q) { - if (WT_IS_METADATA(session, dhandle)) + if (WT_IS_METADATA(dhandle)) continue; WT_WITH_DHANDLE(session, dhandle, diff --git a/src/third_party/wiredtiger/src/conn/conn_handle.c b/src/third_party/wiredtiger/src/conn/conn_handle.c index 3571cc60115..5e4a8c29adc 100644 --- a/src/third_party/wiredtiger/src/conn/conn_handle.c +++ b/src/third_party/wiredtiger/src/conn/conn_handle.c @@ -53,25 +53,18 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) /* Spinlocks. */ WT_RET(__wt_spin_init(session, &conn->api_lock, "api")); WT_SPIN_INIT_TRACKED(session, &conn->checkpoint_lock, checkpoint); - WT_SPIN_INIT_TRACKED(session, &conn->dhandle_lock, handle_list); WT_RET(__wt_spin_init(session, &conn->encryptor_lock, "encryptor")); WT_RET(__wt_spin_init(session, &conn->fh_lock, "file list")); WT_RET(__wt_spin_init(session, &conn->las_lock, "lookaside table")); WT_SPIN_INIT_TRACKED(session, &conn->metadata_lock, metadata); WT_RET(__wt_spin_init(session, &conn->reconfig_lock, "reconfigure")); WT_SPIN_INIT_TRACKED(session, &conn->schema_lock, schema); - WT_SPIN_INIT_TRACKED(session, &conn->table_lock, table); WT_RET(__wt_spin_init(session, &conn->turtle_lock, "turtle file")); /* Read-write locks */ - WT_RET(__wt_rwlock_alloc( - session, &conn->hot_backup_lock, "hot backup")); - - WT_RET(__wt_calloc_def(session, WT_PAGE_LOCKS, &conn->page_lock)); - WT_CACHE_LINE_ALIGNMENT_VERIFY(session, conn->page_lock); - for (i = 0; i < WT_PAGE_LOCKS; ++i) - WT_RET( - __wt_spin_init(session, &conn->page_lock[i], "btree page")); + WT_RET(__wt_rwlock_init(session, &conn->dhandle_lock)); + WT_RET(__wt_rwlock_init(session, &conn->hot_backup_lock)); + WT_RET(__wt_rwlock_init(session, &conn->table_lock)); /* Setup the spin locks for the LSM manager queues. */ WT_RET(__wt_spin_init(session, @@ -81,7 +74,7 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_RET(__wt_spin_init( session, &conn->lsm_manager.switch_lock, "LSM switch queue lock")); WT_RET(__wt_cond_alloc( - session, "LSM worker cond", false, &conn->lsm_manager.work_cond)); + session, "LSM worker cond", &conn->lsm_manager.work_cond)); /* * Generation numbers. @@ -111,16 +104,14 @@ __wt_connection_init(WT_CONNECTION_IMPL *conn) * __wt_connection_destroy -- * Destroy the connection's underlying WT_CONNECTION_IMPL structure. */ -int +void __wt_connection_destroy(WT_CONNECTION_IMPL *conn) { - WT_DECL_RET; WT_SESSION_IMPL *session; - u_int i; /* Check there's something to destroy. */ if (conn == NULL) - return (0); + return; session = conn->default_session; @@ -137,7 +128,7 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn) __wt_spin_destroy(session, &conn->api_lock); __wt_spin_destroy(session, &conn->block_lock); __wt_spin_destroy(session, &conn->checkpoint_lock); - __wt_spin_destroy(session, &conn->dhandle_lock); + __wt_rwlock_destroy(session, &conn->dhandle_lock); __wt_spin_destroy(session, &conn->encryptor_lock); __wt_spin_destroy(session, &conn->fh_lock); __wt_rwlock_destroy(session, &conn->hot_backup_lock); @@ -145,16 +136,8 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn) __wt_spin_destroy(session, &conn->metadata_lock); __wt_spin_destroy(session, &conn->reconfig_lock); __wt_spin_destroy(session, &conn->schema_lock); - __wt_spin_destroy(session, &conn->table_lock); + __wt_rwlock_destroy(session, &conn->table_lock); __wt_spin_destroy(session, &conn->turtle_lock); - for (i = 0; i < WT_PAGE_LOCKS; ++i) - __wt_spin_destroy(session, &conn->page_lock[i]); - __wt_free(session, conn->page_lock); - - /* Destroy the file-system configuration. */ - if (conn->file_system != NULL && conn->file_system->terminate != NULL) - WT_TRET(conn->file_system->terminate( - conn->file_system, (WT_SESSION *)session)); /* Free allocated memory. */ __wt_free(session, conn->cfg); @@ -164,5 +147,4 @@ __wt_connection_destroy(WT_CONNECTION_IMPL *conn) __wt_stat_connection_discard(session, conn); __wt_free(NULL, conn); - return (ret); } diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index 8198b3a1a02..d2ed314fd2e 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -174,7 +174,7 @@ __logmgr_config( WT_RET(__logmgr_sync_cfg(session, cfg)); if (conn->log_cond != NULL) - __wt_cond_auto_signal(session, conn->log_cond); + __wt_cond_signal(session, conn->log_cond); return (0); } @@ -237,7 +237,7 @@ __log_archive_once(WT_SESSION_IMPL *session, uint32_t backup_file) * We can only archive files if a hot backup is not in progress or * if we are the backup. */ - __wt_readlock(session, conn->hot_backup_lock); + __wt_readlock(session, &conn->hot_backup_lock); locked = true; if (!conn->hot_backup || backup_file != 0) { for (i = 0; i < logcount; i++) { @@ -248,7 +248,7 @@ __log_archive_once(WT_SESSION_IMPL *session, uint32_t backup_file) session, WT_LOG_FILENAME, lognum)); } } - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); locked = false; /* @@ -260,7 +260,7 @@ __log_archive_once(WT_SESSION_IMPL *session, uint32_t backup_file) if (0) err: __wt_err(session, ret, "log archive server error"); if (locked) - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); WT_TRET(__wt_fs_directory_list_free(session, &logfiles, logcount)); return (ret); } @@ -341,7 +341,7 @@ __wt_log_truncate_files( conn = S2C(session); if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) return (0); - if (F_ISSET(conn, WT_CONN_SERVER_RUN) && + if (F_ISSET(conn, WT_CONN_SERVER_LOG) && FLD_ISSET(conn->log_flags, WT_CONN_LOG_ARCHIVE)) WT_RET_MSG(session, EINVAL, "Attempt to archive manually while a server is running"); @@ -355,9 +355,9 @@ __wt_log_truncate_files( __wt_verbose(session, WT_VERB_LOG, "log_truncate_files: Archive once up to %" PRIu32, backup_file); - __wt_writelock(session, log->log_archive_lock); + __wt_writelock(session, &log->log_archive_lock); ret = __log_archive_once(session, backup_file); - __wt_writeunlock(session, log->log_archive_lock); + __wt_writeunlock(session, &log->log_archive_lock); return (ret); } @@ -382,7 +382,7 @@ __log_file_server(void *arg) conn = S2C(session); log = conn->log; locked = false; - while (F_ISSET(conn, WT_CONN_LOG_SERVER_RUN)) { + while (F_ISSET(conn, WT_CONN_SERVER_LOG)) { /* * If there is a log file to close, make sure any outstanding * write operations have completed, then fsync and close it. @@ -433,7 +433,7 @@ __log_file_server(void *arg) */ if (!conn->hot_backup) { __wt_readlock( - session, conn->hot_backup_lock); + session, &conn->hot_backup_lock); if (!conn->hot_backup) WT_ERR_ERROR_OK( __wt_ftruncate(session, @@ -441,7 +441,7 @@ __log_file_server(void *arg) close_end_lsn.l.offset), ENOTSUP); __wt_readunlock( - session, conn->hot_backup_lock); + session, &conn->hot_backup_lock); } WT_SET_LSN(&close_end_lsn, close_end_lsn.l.file + 1, 0); @@ -505,8 +505,7 @@ __log_file_server(void *arg) locked = false; __wt_spin_unlock(session, &log->log_sync_lock); } else { - __wt_cond_auto_signal( - session, conn->log_wrlsn_cond); + __wt_cond_signal(session, conn->log_wrlsn_cond); /* * We do not want to wait potentially a second * to process this. Yield to give the wrlsn @@ -517,8 +516,9 @@ __log_file_server(void *arg) continue; } } + /* Wait until the next event. */ - __wt_cond_wait(session, conn->log_file_cond, WT_MILLION / 10); + __wt_cond_wait(session, conn->log_file_cond, 100000, NULL); } if (0) { @@ -708,7 +708,7 @@ __log_wrlsn_server(void *arg) log = conn->log; yield = 0; WT_INIT_LSN(&prev); - while (F_ISSET(conn, WT_CONN_LOG_SERVER_RUN)) { + while (F_ISSET(conn, WT_CONN_SERVER_LOG)) { /* * Write out any log record buffers if anything was done * since last time. Only call the function to walk the @@ -730,12 +730,8 @@ __log_wrlsn_server(void *arg) if (yield++ < WT_THOUSAND) __wt_yield(); else - /* - * Send in false because if we did any work we would - * not be on this path. - */ __wt_cond_auto_wait( - session, conn->log_wrlsn_cond, did_work); + session, conn->log_wrlsn_cond, did_work, NULL); } /* * On close we need to do this one more time because there could @@ -787,7 +783,7 @@ __log_server(void *arg) * takes to sync out an earlier file. */ did_work = true; - while (F_ISSET(conn, WT_CONN_LOG_SERVER_RUN)) { + while (F_ISSET(conn, WT_CONN_SERVER_LOG)) { /* * Slots depend on future activity. Force out buffered * writes in case we are idle. This cannot be part of the @@ -814,10 +810,11 @@ __log_server(void *arg) * agreed not to rename or remove any files in * the database directory. */ - __wt_readlock(session, conn->hot_backup_lock); + __wt_readlock(session, &conn->hot_backup_lock); if (!conn->hot_backup) ret = __log_prealloc_once(session); - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock( + session, &conn->hot_backup_lock); WT_ERR(ret); } @@ -826,10 +823,10 @@ __log_server(void *arg) */ if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_ARCHIVE)) { if (__wt_try_writelock( - session, log->log_archive_lock) == 0) { + session, &log->log_archive_lock) == 0) { ret = __log_archive_once(session, 0); __wt_writeunlock( - session, log->log_archive_lock); + session, &log->log_archive_lock); WT_ERR(ret); } else __wt_verbose(session, WT_VERB_LOG, @@ -839,10 +836,9 @@ __log_server(void *arg) } /* Wait until the next event. */ - __wt_epoch(session, &start); - __wt_cond_auto_wait_signal(session, - conn->log_cond, did_work, &signalled); + __wt_cond_auto_wait_signal( + session, conn->log_cond, did_work, NULL, &signalled); __wt_epoch(session, &now); timediff = WT_TIMEDIFF_MS(now, start); } @@ -884,8 +880,7 @@ __wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_spin_init(session, &log->log_sync_lock, "log sync")); WT_RET(__wt_spin_init(session, &log->log_writelsn_lock, "log write LSN")); - WT_RET(__wt_rwlock_alloc(session, - &log->log_archive_lock, "log archive lock")); + WT_RET(__wt_rwlock_init(session, &log->log_archive_lock)); if (FLD_ISSET(conn->direct_io, WT_DIRECT_IO_LOG)) log->allocsize = (uint32_t) WT_MAX(conn->buffer_alignment, WT_LOG_ALIGN); @@ -904,10 +899,8 @@ __wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_INIT_LSN(&log->write_lsn); WT_INIT_LSN(&log->write_start_lsn); log->fileid = 0; - WT_RET(__wt_cond_alloc( - session, "log sync", false, &log->log_sync_cond)); - WT_RET(__wt_cond_alloc( - session, "log write", false, &log->log_write_cond)); + WT_RET(__wt_cond_alloc(session, "log sync", &log->log_sync_cond)); + WT_RET(__wt_cond_alloc(session, "log write", &log->log_write_cond)); WT_RET(__wt_log_open(session)); WT_RET(__wt_log_slot_init(session)); @@ -930,6 +923,8 @@ __wt_logmgr_open(WT_SESSION_IMPL *session) if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) return (0); + F_SET(conn, WT_CONN_SERVER_LOG); + /* * Start the log close thread. It is not configurable. * If logging is enabled, this thread runs. @@ -937,8 +932,8 @@ __wt_logmgr_open(WT_SESSION_IMPL *session) session_flags = WT_SESSION_NO_DATA_HANDLES; WT_RET(__wt_open_internal_session(conn, "log-close-server", false, session_flags, &conn->log_file_session)); - WT_RET(__wt_cond_alloc(conn->log_file_session, - "log close server", false, &conn->log_file_cond)); + WT_RET(__wt_cond_alloc( + conn->log_file_session, "log close server", &conn->log_file_cond)); /* * Start the log file close thread. @@ -954,8 +949,7 @@ __wt_logmgr_open(WT_SESSION_IMPL *session) WT_RET(__wt_open_internal_session(conn, "log-wrlsn-server", false, session_flags, &conn->log_wrlsn_session)); WT_RET(__wt_cond_auto_alloc(conn->log_wrlsn_session, - "log write lsn server", false, 10000, WT_MILLION, - &conn->log_wrlsn_cond)); + "log write lsn server", 10000, WT_MILLION, &conn->log_wrlsn_cond)); WT_RET(__wt_thread_create(conn->log_wrlsn_session, &conn->log_wrlsn_tid, __log_wrlsn_server, conn->log_wrlsn_session)); conn->log_wrlsn_tid_set = true; @@ -969,13 +963,13 @@ __wt_logmgr_open(WT_SESSION_IMPL *session) if (conn->log_session != NULL) { WT_ASSERT(session, conn->log_cond != NULL); WT_ASSERT(session, conn->log_tid_set == true); - __wt_cond_auto_signal(session, conn->log_cond); + __wt_cond_signal(session, conn->log_cond); } else { /* The log server gets its own session. */ WT_RET(__wt_open_internal_session(conn, "log-server", false, session_flags, &conn->log_session)); WT_RET(__wt_cond_auto_alloc(conn->log_session, - "log server", false, 50000, WT_MILLION, &conn->log_cond)); + "log server", 50000, WT_MILLION, &conn->log_cond)); /* * Start the thread. @@ -1001,6 +995,8 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) conn = S2C(session); + F_CLR(conn, WT_CONN_SERVER_LOG); + if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) { /* * We always set up the log_path so printlog can work without @@ -1011,7 +1007,7 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) return (0); } if (conn->log_tid_set) { - __wt_cond_auto_signal(session, conn->log_cond); + __wt_cond_signal(session, conn->log_cond); WT_TRET(__wt_thread_join(session, conn->log_tid)); conn->log_tid_set = false; } @@ -1026,7 +1022,7 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) conn->log_file_session = NULL; } if (conn->log_wrlsn_tid_set) { - __wt_cond_auto_signal(session, conn->log_wrlsn_cond); + __wt_cond_signal(session, conn->log_wrlsn_cond); WT_TRET(__wt_thread_join(session, conn->log_wrlsn_tid)); conn->log_wrlsn_tid_set = false; } @@ -1047,12 +1043,12 @@ __wt_logmgr_destroy(WT_SESSION_IMPL *session) } /* Destroy the condition variables now that all threads are stopped */ - WT_TRET(__wt_cond_auto_destroy(session, &conn->log_cond)); - WT_TRET(__wt_cond_destroy(session, &conn->log_file_cond)); - WT_TRET(__wt_cond_auto_destroy(session, &conn->log_wrlsn_cond)); + __wt_cond_destroy(session, &conn->log_cond); + __wt_cond_destroy(session, &conn->log_file_cond); + __wt_cond_destroy(session, &conn->log_wrlsn_cond); - WT_TRET(__wt_cond_destroy(session, &conn->log->log_sync_cond)); - WT_TRET(__wt_cond_destroy(session, &conn->log->log_write_cond)); + __wt_cond_destroy(session, &conn->log->log_sync_cond); + __wt_cond_destroy(session, &conn->log->log_write_cond); __wt_rwlock_destroy(session, &conn->log->log_archive_lock); __wt_spin_destroy(session, &conn->log->log_lock); __wt_spin_destroy(session, &conn->log->log_slot_lock); diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index 69b50147bf5..eb3c79422a0 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -21,16 +21,9 @@ __wt_connection_open(WT_CONNECTION_IMPL *conn, const char *cfg[]) session = conn->default_session; WT_ASSERT(session, session->iface.connection == &conn->iface); - /* - * Tell internal server threads to run: this must be set before opening - * any sessions. - */ - F_SET(conn, WT_CONN_SERVER_RUN | WT_CONN_LOG_SERVER_RUN); - /* WT_SESSION_IMPL array. */ WT_RET(__wt_calloc(session, conn->session_size, sizeof(WT_SESSION_IMPL), &conn->sessions)); - WT_CACHE_LINE_ALIGNMENT_VERIFY(session, conn->sessions); /* * Open the default session. We open this before starting service @@ -95,13 +88,22 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) for (;;) { WT_TRET(__wt_txn_update_oldest(session, WT_TXN_OLDEST_STRICT | WT_TXN_OLDEST_WAIT)); - if (txn_global->oldest_id == txn_global->current) + if (txn_global->oldest_id == txn_global->current && + txn_global->metadata_pinned == txn_global->current) break; __wt_yield(); } - /* Clear any pending async ops. */ + /* Shut down the subsystems, ensuring workers see the state change. */ + F_SET(conn, WT_CONN_CLOSING); + WT_FULL_BARRIER(); + + /* + * Clear any pending async operations and shut down the async worker + * threads and system before closing LSM. + */ WT_TRET(__wt_async_flush(session)); + WT_TRET(__wt_async_destroy(session)); /* * Shut down server threads other than the eviction server, which is @@ -109,15 +111,20 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) * btree handles, so take care in ordering shutdown to make sure they * exit before files are closed. */ - F_CLR(conn, WT_CONN_SERVER_RUN); - WT_TRET(__wt_async_destroy(session)); WT_TRET(__wt_lsm_manager_destroy(session)); - WT_TRET(__wt_sweep_destroy(session)); - F_SET(conn, WT_CONN_CLOSING); + /* + * Once the async and LSM threads exit, we shouldn't be opening any + * more files. + */ + F_SET(conn, WT_CONN_CLOSING_NO_MORE_OPENS); + WT_FULL_BARRIER(); WT_TRET(__wt_checkpoint_server_destroy(session)); WT_TRET(__wt_statlog_destroy(session, true)); + WT_TRET(__wt_sweep_destroy(session)); + + /* The eviction server is shut down last. */ WT_TRET(__wt_evict_destroy(session)); /* Shut down the lookaside table, after all eviction is complete. */ @@ -126,7 +133,7 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) /* Close open data handles. */ WT_TRET(__wt_conn_dhandle_discard(session)); - /* Shut down metadata tracking, required before creating tables. */ + /* Shut down metadata tracking. */ WT_TRET(__wt_meta_track_destroy(session)); /* @@ -140,7 +147,6 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_DONE)) WT_TRET(__wt_txn_checkpoint_log( session, true, WT_TXN_LOG_CKPT_STOP, NULL)); - F_CLR(conn, WT_CONN_LOG_SERVER_RUN); WT_TRET(__wt_logmgr_destroy(session)); /* Free memory for collators, compressors, data sources. */ @@ -159,15 +165,6 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) /* Discard transaction state. */ __wt_txn_global_destroy(session); - /* Close extensions, first calling any unload entry point. */ - while ((dlh = TAILQ_FIRST(&conn->dlhqh)) != NULL) { - TAILQ_REMOVE(&conn->dlhqh, dlh, q); - - if (dlh->terminate != NULL) - WT_TRET(dlh->terminate(wt_conn)); - WT_TRET(__wt_dlclose(session, dlh)); - } - /* Close the lock file, opening up the database to other connections. */ if (conn->lock_fh != NULL) WT_TRET(__wt_close(session, &conn->lock_fh)); @@ -199,8 +196,22 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) __wt_free(session, s->hazard); } + /* Destroy the file-system configuration. */ + if (conn->file_system != NULL && conn->file_system->terminate != NULL) + WT_TRET(conn->file_system->terminate( + conn->file_system, (WT_SESSION *)session)); + + /* Close extensions, first calling any unload entry point. */ + while ((dlh = TAILQ_FIRST(&conn->dlhqh)) != NULL) { + TAILQ_REMOVE(&conn->dlhqh, dlh, q); + + if (dlh->terminate != NULL) + WT_TRET(dlh->terminate(wt_conn)); + WT_TRET(__wt_dlclose(session, dlh)); + } + /* Destroy the handle. */ - WT_TRET(__wt_connection_destroy(conn)); + __wt_connection_destroy(conn); return (ret); } diff --git a/src/third_party/wiredtiger/src/conn/conn_stat.c b/src/third_party/wiredtiger/src/conn/conn_stat.c index 3bcdfd7ecb1..3234d51cd4c 100644 --- a/src/third_party/wiredtiger/src/conn/conn_stat.c +++ b/src/third_party/wiredtiger/src/conn/conn_stat.c @@ -409,7 +409,6 @@ __statlog_log_one(WT_SESSION_IMPL *session, WT_ITEM *path, WT_ITEM *tmp) struct timespec ts; struct tm *tm, _tm; WT_CONNECTION_IMPL *conn; - WT_DECL_RET; WT_FSTREAM *log_stream; conn = S2C(session); @@ -446,12 +445,9 @@ __statlog_log_one(WT_SESSION_IMPL *session, WT_ITEM *path, WT_ITEM *tmp) * Lock the schema and walk the list of open handles, dumping * any that match the list of object sources. */ - if (conn->stat_sources != NULL) { - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_conn_btree_apply( + if (conn->stat_sources != NULL) + WT_RET(__wt_conn_btree_apply( session, NULL, __statlog_apply, NULL, NULL)); - WT_RET(ret); - } /* * Walk the list of open LSM trees, dumping any that match the @@ -485,8 +481,7 @@ __statlog_on_close(WT_SESSION_IMPL *session) if (!FLD_ISSET(conn->stat_flags, WT_STAT_ON_CLOSE)) return (0); - if (F_ISSET(conn, WT_CONN_SERVER_RUN) && - F_ISSET(conn, WT_CONN_SERVER_STATISTICS)) + if (F_ISSET(conn, WT_CONN_SERVER_STATISTICS)) WT_RET_MSG(session, EINVAL, "Attempt to log statistics while a server is running"); @@ -497,6 +492,16 @@ err: __wt_scr_free(session, &tmp); return (ret); } +/* + * __statlog_server_run_chk -- + * Check to decide if the statistics log server should continue running. + */ +static bool +__statlog_server_run_chk(WT_SESSION_IMPL *session) +{ + return (F_ISSET(S2C(session), WT_CONN_SERVER_STATISTICS)); +} + /* * __statlog_server -- * The statistics server thread. @@ -525,10 +530,14 @@ __statlog_server(void *arg) WT_ERR(__wt_buf_init(session, &path, strlen(conn->stat_path) + 128)); WT_ERR(__wt_buf_init(session, &tmp, strlen(conn->stat_path) + 128)); - while (F_ISSET(conn, WT_CONN_SERVER_RUN) && - F_ISSET(conn, WT_CONN_SERVER_STATISTICS)) { + for (;;) { /* Wait until the next event. */ - __wt_cond_wait(session, conn->stat_cond, conn->stat_usecs); + __wt_cond_wait(session, conn->stat_cond, + conn->stat_usecs, __statlog_server_run_chk); + + /* Check if we're quitting or being reconfigured. */ + if (!__statlog_server_run_chk(session)) + break; if (WT_STAT_ENABLED(session)) WT_ERR(__statlog_log_one(session, &path, &tmp)); @@ -563,7 +572,7 @@ __statlog_start(WT_CONNECTION_IMPL *conn) session = conn->stat_session; WT_RET(__wt_cond_alloc( - session, "statistics log server", false, &conn->stat_cond)); + session, "statistics log server", &conn->stat_cond)); /* * Start the thread. @@ -639,7 +648,7 @@ __wt_statlog_destroy(WT_SESSION_IMPL *session, bool is_close) WT_TRET(__wt_thread_join(session, conn->stat_tid)); conn->stat_tid_set = false; } - WT_TRET(__wt_cond_destroy(session, &conn->stat_cond)); + __wt_cond_destroy(session, &conn->stat_cond); /* Log a set of statistics on shutdown if configured. */ if (is_close) diff --git a/src/third_party/wiredtiger/src/conn/conn_sweep.c b/src/third_party/wiredtiger/src/conn/conn_sweep.c index dba37fa2eb0..e273f1d08e5 100644 --- a/src/third_party/wiredtiger/src/conn/conn_sweep.c +++ b/src/third_party/wiredtiger/src/conn/conn_sweep.c @@ -10,7 +10,7 @@ #define WT_DHANDLE_CAN_DISCARD(dhandle) \ (!F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE | WT_DHANDLE_OPEN) && \ - dhandle->session_inuse == 0 && dhandle->session_ref == 0) + (dhandle)->session_inuse == 0 && (dhandle)->session_ref == 0) /* * __sweep_mark -- @@ -26,7 +26,7 @@ __sweep_mark(WT_SESSION_IMPL *session, time_t now) conn = S2C(session); TAILQ_FOREACH(dhandle, &conn->dhqh, q) { - if (WT_IS_METADATA(session, dhandle)) + if (WT_IS_METADATA(dhandle)) continue; /* @@ -81,7 +81,7 @@ __sweep_expire_one(WT_SESSION_IMPL *session) * handle list lock so that connection-level handle searches * never need to retry. */ - WT_RET(__wt_try_writelock(session, dhandle->rwlock)); + WT_RET(__wt_try_writelock(session, &dhandle->rwlock)); /* Only sweep clean trees where all updates are visible. */ if (btree->modified || @@ -95,7 +95,7 @@ __sweep_expire_one(WT_SESSION_IMPL *session) */ ret = __wt_conn_btree_sync_and_close(session, false, true); -err: __wt_writeunlock(session, dhandle->rwlock); +err: __wt_writeunlock(session, &dhandle->rwlock); return (ret); } @@ -122,7 +122,7 @@ __sweep_expire(WT_SESSION_IMPL *session, time_t now) if (conn->open_btree_count < conn->sweep_handles_min) break; - if (WT_IS_METADATA(session, dhandle) || + if (WT_IS_METADATA(dhandle) || !F_ISSET(dhandle, WT_DHANDLE_OPEN) || dhandle->session_inuse != 0 || dhandle->timeofdeath == 0 || @@ -188,7 +188,7 @@ __sweep_remove_one(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle) WT_DECL_RET; /* Try to get exclusive access. */ - WT_RET(__wt_try_writelock(session, dhandle->rwlock)); + WT_RET(__wt_try_writelock(session, &dhandle->rwlock)); /* * If there are no longer any references to the handle in any @@ -205,7 +205,7 @@ __sweep_remove_one(WT_SESSION_IMPL *session, WT_DATA_HANDLE *dhandle) * don't retry the discard until it times out again. */ if (ret != 0) { -err: __wt_writeunlock(session, dhandle->rwlock); +err: __wt_writeunlock(session, &dhandle->rwlock); } return (ret); @@ -228,12 +228,12 @@ __sweep_remove_handles(WT_SESSION_IMPL *session) dhandle != NULL; dhandle = dhandle_next) { dhandle_next = TAILQ_NEXT(dhandle, q); - if (WT_IS_METADATA(session, dhandle)) + if (WT_IS_METADATA(dhandle)) continue; if (!WT_DHANDLE_CAN_DISCARD(dhandle)) continue; - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, ret = __sweep_remove_one(session, dhandle)); if (ret == 0) WT_STAT_CONN_INCR(session, dh_sweep_remove); @@ -245,6 +245,16 @@ __sweep_remove_handles(WT_SESSION_IMPL *session) return (ret == EBUSY ? 0 : ret); } +/* + * __sweep_server_run_chk -- + * Check to decide if the checkpoint server should continue running. + */ +static bool +__sweep_server_run_chk(WT_SESSION_IMPL *session) +{ + return (F_ISSET(S2C(session), WT_CONN_SERVER_SWEEP)); +} + /* * __sweep_server -- * The handle sweep server thread. @@ -266,11 +276,15 @@ __sweep_server(void *arg) /* * Sweep for dead and excess handles. */ - while (F_ISSET(conn, WT_CONN_SERVER_RUN) && - F_ISSET(conn, WT_CONN_SERVER_SWEEP)) { + for (;;) { /* Wait until the next event. */ - __wt_cond_wait(session, - conn->sweep_cond, conn->sweep_interval * WT_MILLION); + __wt_cond_wait(session, conn->sweep_cond, + conn->sweep_interval * WT_MILLION, __sweep_server_run_chk); + + /* Check if we're quitting or being reconfigured. */ + if (!__sweep_server_run_chk(session)) + break; + __wt_seconds(session, &now); WT_STAT_CONN_INCR(session, dh_sweeps); @@ -390,7 +404,7 @@ __wt_sweep_create(WT_SESSION_IMPL *session) session = conn->sweep_session; WT_RET(__wt_cond_alloc( - session, "handle sweep server", false, &conn->sweep_cond)); + session, "handle sweep server", &conn->sweep_cond)); WT_RET(__wt_thread_create( session, &conn->sweep_tid, __sweep_server, session)); @@ -418,7 +432,7 @@ __wt_sweep_destroy(WT_SESSION_IMPL *session) WT_TRET(__wt_thread_join(session, conn->sweep_tid)); conn->sweep_tid_set = 0; } - WT_TRET(__wt_cond_destroy(session, &conn->sweep_cond)); + __wt_cond_destroy(session, &conn->sweep_cond); if (conn->sweep_session != NULL) { wt_session = &conn->sweep_session->iface; diff --git a/src/third_party/wiredtiger/src/cursor/cur_backup.c b/src/third_party/wiredtiger/src/cursor/cur_backup.c index 3585082644f..61ced8d11e7 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_backup.c +++ b/src/third_party/wiredtiger/src/cursor/cur_backup.c @@ -144,8 +144,8 @@ __wt_curbackup_open(WT_SESSION_IMPL *session, * Start the backup and fill in the cursor's list. Acquire the schema * lock, we need a consistent view when creating a copy. */ - WT_WITH_CHECKPOINT_LOCK(session, ret, - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, ret = __backup_start(session, cb, cfg))); WT_ERR(ret); @@ -230,10 +230,10 @@ __backup_start( * We are holding the checkpoint and schema locks so schema operations * will not see the backup file list until it is complete and valid. */ - __wt_writelock(session, conn->hot_backup_lock); + __wt_writelock(session, &conn->hot_backup_lock); conn->hot_backup = true; conn->hot_backup_list = NULL; - __wt_writeunlock(session, conn->hot_backup_lock); + __wt_writeunlock(session, &conn->hot_backup_lock); /* We're the lock holder, we own cleanup. */ F_SET(cb, WT_CURBACKUP_LOCKER); @@ -297,9 +297,9 @@ err: /* Close the hot backup file. */ if (ret == 0) { WT_ASSERT(session, dest != NULL); WT_TRET(__wt_fs_rename(session, WT_BACKUP_TMP, dest, false)); - __wt_writelock(session, conn->hot_backup_lock); + __wt_writelock(session, &conn->hot_backup_lock); conn->hot_backup_list = cb->list; - __wt_writeunlock(session, conn->hot_backup_lock); + __wt_writeunlock(session, &conn->hot_backup_lock); } return (ret); @@ -319,9 +319,9 @@ __backup_stop(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) conn = S2C(session); /* Release all btree names held by the backup. */ - __wt_writelock(session, conn->hot_backup_lock); + __wt_writelock(session, &conn->hot_backup_lock); conn->hot_backup_list = NULL; - __wt_writeunlock(session, conn->hot_backup_lock); + __wt_writeunlock(session, &conn->hot_backup_lock); if (cb->list != NULL) { for (i = 0; cb->list[i] != NULL; ++i) __wt_free(session, cb->list[i]); @@ -332,9 +332,9 @@ __backup_stop(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) WT_TRET(__wt_backup_file_remove(session)); /* Checkpoint deletion can proceed, as can the next hot backup. */ - __wt_writelock(session, conn->hot_backup_lock); + __wt_writelock(session, &conn->hot_backup_lock); conn->hot_backup = false; - __wt_writeunlock(session, conn->hot_backup_lock); + __wt_writeunlock(session, &conn->hot_backup_lock); return (ret); } @@ -346,13 +346,9 @@ __backup_stop(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) static int __backup_all(WT_SESSION_IMPL *session) { - WT_DECL_RET; - /* Build a list of the file objects that need to be copied. */ - WT_WITH_HANDLE_LIST_LOCK(session, ret = - __wt_meta_apply_all(session, NULL, __backup_list_uri_append, NULL)); - - return (ret); + return (__wt_meta_apply_all( + session, NULL, __backup_list_uri_append, NULL)); } /* diff --git a/src/third_party/wiredtiger/src/cursor/cur_config.c b/src/third_party/wiredtiger/src/cursor/cur_config.c index 2d3f3ffd176..4001188e21c 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_config.c +++ b/src/third_party/wiredtiger/src/cursor/cur_config.c @@ -49,8 +49,6 @@ __wt_curconfig_open(WT_SESSION_IMPL *session, WT_STATIC_ASSERT(offsetof(WT_CURSOR_CONFIG, iface) == 0); - WT_UNUSED(uri); - WT_RET(__wt_calloc_one(session, &cconfig)); cursor = &cconfig->iface; diff --git a/src/third_party/wiredtiger/src/cursor/cur_ds.c b/src/third_party/wiredtiger/src/cursor/cur_ds.c index 458e0f1b1f0..131d1ffa930 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_ds.c +++ b/src/third_party/wiredtiger/src/cursor/cur_ds.c @@ -12,13 +12,11 @@ * __curds_txn_enter -- * Do transactional initialization when starting an operation. */ -static int +static void __curds_txn_enter(WT_SESSION_IMPL *session) { session->ncursors++; /* XXX */ - WT_RET(__wt_txn_cursor_op(session)); - - return (0); + __wt_txn_cursor_op(session); } /* @@ -187,7 +185,7 @@ __curds_next(WT_CURSOR *cursor) WT_STAT_CONN_INCR(session, cursor_next); WT_STAT_DATA_INCR(session, cursor_next); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); ret = __curds_cursor_resolve(cursor, source->next(source)); @@ -215,7 +213,7 @@ __curds_prev(WT_CURSOR *cursor) WT_STAT_CONN_INCR(session, cursor_prev); WT_STAT_DATA_INCR(session, cursor_prev); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); ret = __curds_cursor_resolve(cursor, source->prev(source)); @@ -267,7 +265,7 @@ __curds_search(WT_CURSOR *cursor) WT_STAT_CONN_INCR(session, cursor_search); WT_STAT_DATA_INCR(session, cursor_search); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); WT_ERR(__curds_key_set(cursor)); ret = __curds_cursor_resolve(cursor, source->search(source)); @@ -295,7 +293,7 @@ __curds_search_near(WT_CURSOR *cursor, int *exact) WT_STAT_CONN_INCR(session, cursor_search_near); WT_STAT_DATA_INCR(session, cursor_search_near); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); WT_ERR(__curds_key_set(cursor)); ret = @@ -321,7 +319,7 @@ __curds_insert(WT_CURSOR *cursor) CURSOR_UPDATE_API_CALL(cursor, session, insert, NULL); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); WT_STAT_CONN_INCR(session, cursor_insert); WT_STAT_DATA_INCR(session, cursor_insert); @@ -358,7 +356,7 @@ __curds_update(WT_CURSOR *cursor) WT_STAT_DATA_INCR(session, cursor_update); WT_STAT_DATA_INCRV(session, cursor_update_bytes, cursor->value.size); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); WT_ERR(__curds_key_set(cursor)); WT_ERR(__curds_value_set(cursor)); @@ -389,7 +387,7 @@ __curds_remove(WT_CURSOR *cursor) WT_STAT_DATA_INCR(session, cursor_remove); WT_STAT_DATA_INCRV(session, cursor_remove_bytes, cursor->key.size); - WT_ERR(__curds_txn_enter(session)); + __curds_txn_enter(session); WT_ERR(__curds_key_set(cursor)); ret = __curds_cursor_resolve(cursor, source->remove(source)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_file.c b/src/third_party/wiredtiger/src/cursor/cur_file.c index 9fc466f4c76..bc54f10f2d6 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_file.c +++ b/src/third_party/wiredtiger/src/cursor/cur_file.c @@ -8,29 +8,6 @@ #include "wt_internal.h" -/* - * WT_BTREE_CURSOR_SAVE_AND_RESTORE - * Save the cursor's key/value data/size fields, call an underlying btree - * function, and then consistently handle failure and success. - */ -#define WT_BTREE_CURSOR_SAVE_AND_RESTORE(cursor, f, ret) do { \ - WT_ITEM __key_copy = (cursor)->key; \ - uint64_t __recno = (cursor)->recno; \ - WT_ITEM __value_copy = (cursor)->value; \ - if (((ret) = (f)) == 0) { \ - F_CLR(cursor, WT_CURSTD_KEY_EXT | WT_CURSTD_VALUE_EXT); \ - F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); \ - } else { \ - if (F_ISSET(cursor, WT_CURSTD_KEY_EXT)) { \ - (cursor)->recno = __recno; \ - WT_ITEM_SET((cursor)->key, __key_copy); \ - } \ - if (F_ISSET(cursor, WT_CURSTD_VALUE_EXT)) \ - WT_ITEM_SET((cursor)->value, __value_copy); \ - F_CLR(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); \ - } \ -} while (0) - /* * __curfile_compare -- * WT_CURSOR->compare method for the btree cursor type. @@ -109,9 +86,12 @@ __curfile_next(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL(cursor, session, next, cbt->btree); - F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); - if ((ret = __wt_btcur_next(cbt, false)) == 0) - F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); + WT_ERR(__wt_btcur_next(cbt, false)); + + /* Next maintains a position, key and value. */ + WT_ASSERT(session, + F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && + F_MASK(cursor, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); err: API_END_RET(session, ret); } @@ -131,9 +111,12 @@ __wt_curfile_next_random(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL(cursor, session, next, cbt->btree); - F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); - if ((ret = __wt_btcur_next_random(cbt)) == 0) - F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); + WT_ERR(__wt_btcur_next_random(cbt)); + + /* Next-random maintains a position, key and value. */ + WT_ASSERT(session, + F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && + F_MASK(cursor, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); err: API_END_RET(session, ret); } @@ -152,9 +135,12 @@ __curfile_prev(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL(cursor, session, prev, cbt->btree); - F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); - if ((ret = __wt_btcur_prev(cbt, false)) == 0) - F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); + WT_ERR(__wt_btcur_prev(cbt, false)); + + /* Prev maintains a position, key and value. */ + WT_ASSERT(session, + F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && + F_MASK(cursor, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); err: API_END_RET(session, ret); } @@ -175,7 +161,10 @@ __curfile_reset(WT_CURSOR *cursor) ret = __wt_btcur_reset(cbt); - F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + /* Reset maintains no position, key or value. */ + WT_ASSERT(session, + F_MASK(cursor, WT_CURSTD_KEY_SET) == 0 && + F_MASK(cursor, WT_CURSTD_VALUE_SET) == 0); err: API_END_RET(session, ret); } @@ -194,10 +183,15 @@ __curfile_search(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL(cursor, session, search, cbt->btree); - WT_CURSOR_NEEDKEY(cursor); + WT_CURSOR_CHECKKEY(cursor); WT_CURSOR_NOVALUE(cursor); - WT_BTREE_CURSOR_SAVE_AND_RESTORE(cursor, __wt_btcur_search(cbt), ret); + WT_ERR(__wt_btcur_search(cbt)); + + /* Search maintains a position, key and value. */ + WT_ASSERT(session, + F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && + F_MASK(cursor, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); err: API_END_RET(session, ret); } @@ -216,11 +210,15 @@ __curfile_search_near(WT_CURSOR *cursor, int *exact) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_API_CALL(cursor, session, search_near, cbt->btree); - WT_CURSOR_NEEDKEY(cursor); + WT_CURSOR_CHECKKEY(cursor); WT_CURSOR_NOVALUE(cursor); - WT_BTREE_CURSOR_SAVE_AND_RESTORE( - cursor, __wt_btcur_search_near(cbt, exact), ret); + WT_ERR(__wt_btcur_search_near(cbt, exact)); + + /* Search-near maintains a position, key and value. */ + WT_ASSERT(session, + F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && + F_MASK(cursor, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); err: API_END_RET(session, ret); } @@ -238,38 +236,33 @@ __curfile_insert(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_UPDATE_API_CALL(cursor, session, insert, cbt->btree); + if (!F_ISSET(cursor, WT_CURSTD_APPEND)) - WT_CURSOR_NEEDKEY(cursor); - WT_CURSOR_NEEDVALUE(cursor); + WT_CURSOR_CHECKKEY(cursor); + WT_CURSOR_CHECKVALUE(cursor); - WT_BTREE_CURSOR_SAVE_AND_RESTORE(cursor, __wt_btcur_insert(cbt), ret); + WT_ERR(__wt_btcur_insert(cbt)); /* - * Insert is the one cursor operation that doesn't end with the cursor - * pointing to an on-page item (except for column-store appends, where - * we are returning a key). That is, the application's cursor continues - * to reference the application's memory after a successful cursor call, - * which isn't true anywhere else. We don't want to have to explain that - * scoping corner case, so we reset the application's cursor so it can - * free the referenced memory and continue on without risking subsequent - * core dumps. + * Insert maintains no position, key or value (except for column-store + * appends, where we are returning a key). */ - if (ret == 0) { - if (!F_ISSET(cursor, WT_CURSTD_APPEND)) - F_CLR(cursor, WT_CURSTD_KEY_INT); - F_CLR(cursor, WT_CURSTD_VALUE_INT); - } + WT_ASSERT(session, + (F_ISSET(cursor, WT_CURSTD_APPEND) && + F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT) || + (!F_ISSET(cursor, WT_CURSTD_APPEND) && + F_MASK(cursor, WT_CURSTD_KEY_SET) == 0)); err: CURSOR_UPDATE_API_END(session, ret); return (ret); } /* - * __curfile_update -- - * WT_CURSOR->update method for the btree cursor type. + * __wt_curfile_insert_check -- + * WT_CURSOR->insert_check method for the btree cursor type. */ -static int -__curfile_update(WT_CURSOR *cursor) +int +__wt_curfile_insert_check(WT_CURSOR *cursor) { WT_CURSOR_BTREE *cbt; WT_DECL_RET; @@ -278,21 +271,21 @@ __curfile_update(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_UPDATE_API_CALL(cursor, session, update, cbt->btree); - WT_CURSOR_NEEDKEY(cursor); - WT_CURSOR_NEEDVALUE(cursor); + WT_CURSOR_CHECKKEY(cursor); + WT_CURSOR_NOVALUE(cursor); - WT_BTREE_CURSOR_SAVE_AND_RESTORE(cursor, __wt_btcur_update(cbt), ret); + ret = __wt_btcur_insert_check(cbt); err: CURSOR_UPDATE_API_END(session, ret); return (ret); } /* - * __wt_curfile_update_check -- - * WT_CURSOR->update_check method for the btree cursor type. + * __curfile_update -- + * WT_CURSOR->update method for the btree cursor type. */ -int -__wt_curfile_update_check(WT_CURSOR *cursor) +static int +__curfile_update(WT_CURSOR *cursor) { WT_CURSOR_BTREE *cbt; WT_DECL_RET; @@ -301,11 +294,15 @@ __wt_curfile_update_check(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_UPDATE_API_CALL(cursor, session, update, cbt->btree); - WT_CURSOR_NEEDKEY(cursor); - WT_CURSOR_NOVALUE(cursor); + WT_CURSOR_CHECKKEY(cursor); + WT_CURSOR_CHECKVALUE(cursor); + + WT_ERR(__wt_btcur_update(cbt)); - WT_BTREE_CURSOR_SAVE_AND_RESTORE( - cursor, __wt_btcur_update_check(cbt), ret); + /* Update maintains a position, key and value. */ + WT_ASSERT(session, + F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT && + F_MASK(cursor, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); err: CURSOR_UPDATE_API_END(session, ret); return (ret); @@ -325,24 +322,21 @@ __curfile_remove(WT_CURSOR *cursor) cbt = (WT_CURSOR_BTREE *)cursor; CURSOR_REMOVE_API_CALL(cursor, session, cbt->btree); - WT_CURSOR_NEEDKEY(cursor); + WT_CURSOR_CHECKKEY(cursor); WT_CURSOR_NOVALUE(cursor); - WT_BTREE_CURSOR_SAVE_AND_RESTORE(cursor, __wt_btcur_remove(cbt), ret); + WT_ERR(__wt_btcur_remove(cbt)); /* - * After a successful remove, copy the key: the value is not available. + * Remove with a search-key is fire-and-forget, no position and no key. + * Remove starting from a position maintains the position and a key. + * We don't know which it was at this layer, so can only assert the key + * is not set at all, or internal. There's never a value. */ - if (ret == 0) { - if (F_ISSET(cursor, WT_CURSTD_KEY_INT) && - !WT_DATA_IN_ITEM(&(cursor)->key)) { - WT_ERR(__wt_buf_set(session, &cursor->key, - cursor->key.data, cursor->key.size)); - F_CLR(cursor, WT_CURSTD_KEY_INT); - F_SET(cursor, WT_CURSTD_KEY_EXT); - } - F_CLR(cursor, WT_CURSTD_VALUE_SET); - } + WT_ASSERT(session, + F_MASK(cursor, WT_CURSTD_KEY_SET) == 0 || + F_MASK(cursor, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT); + WT_ASSERT(session, F_MASK(cursor, WT_CURSTD_VALUE_SET) == 0); err: CURSOR_UPDATE_API_END(session, ret); return (ret); @@ -492,7 +486,14 @@ __curfile_create(WT_SESSION_IMPL *session, WT_STAT_DATA_INCR(session, cursor_create); if (0) { -err: WT_TRET(__curfile_close(cursor)); +err: /* + * Our caller expects to release the data handle if we fail. + * Disconnect it from the cursor before closing. + */ + if (session->dhandle != NULL) + __wt_cursor_dhandle_decr_use(session); + cbt->btree = NULL; + WT_TRET(__curfile_close(cursor)); *cursorp = NULL; } @@ -559,7 +560,7 @@ __wt_curfile_open(WT_SESSION_IMPL *session, const char *uri, * get the handle while holding the checkpoint lock. */ if (LF_ISSET(WT_DHANDLE_EXCLUSIVE) && checkpoint_wait) - WT_WITH_CHECKPOINT_LOCK(session, ret, + WT_WITH_CHECKPOINT_LOCK(session, ret = __wt_session_get_btree_ckpt( session, uri, cfg, flags)); else diff --git a/src/third_party/wiredtiger/src/cursor/cur_index.c b/src/third_party/wiredtiger/src/cursor/cur_index.c index eb5e15ae5c3..6fc01c0421f 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_index.c +++ b/src/third_party/wiredtiger/src/cursor/cur_index.c @@ -240,7 +240,17 @@ __curindex_search(WT_CURSOR *cursor) found_key = child->key; if (found_key.size < cursor->key.size) WT_ERR(WT_NOTFOUND); - found_key.size = cursor->key.size; + + /* + * Custom collators expect to see complete keys, pass an item containing + * all the visible fields so it unpacks correctly. + */ + if (cindex->index->collator != NULL && + !F_ISSET(cursor, WT_CURSTD_RAW_SEARCH)) + WT_ERR(__wt_struct_repack(session, child->key_format, + cindex->iface.key_format, &child->key, &found_key)); + else + found_key.size = cursor->key.size; WT_ERR(__wt_compare( session, cindex->index->collator, &cursor->key, &found_key, &cmp)); @@ -281,33 +291,48 @@ __curindex_search_near(WT_CURSOR *cursor, int *exact) * (usually) doesn't contain the primary key, so it is just a prefix of * any matching index key. That said, if there is an exact match, we * want to find the first matching index entry and set exact equal to - * zero. Do a search_near, step to the next entry if we land on one - * that is too small, then check that the prefix matches. + * zero. + * + * Do a search_near, and if we find an entry that is too small, step to + * the next one. In the unlikely event of a search past the end of the + * tree, go back to the last key. */ __wt_cursor_set_raw_key(child, &cursor->key); WT_ERR(child->search_near(child, &cmp)); - if (cmp < 0) - WT_ERR(child->next(child)); + if (cmp < 0) { + if ((ret = child->next(child)) == WT_NOTFOUND) + ret = child->prev(child); + WT_ERR(ret); + } /* * We expect partial matches, and want the smallest record with a key * greater than or equal to the search key. * - * If the key we find is shorter than the search key, it can't possibly - * match. + * If the found key starts with the search key, we indicate a match by + * setting exact equal to zero. * - * The only way for the key to be exactly equal is if there is an index - * on the primary key, because otherwise the primary key columns will - * be appended to the index key, but we don't disallow that (odd) case. + * The compare function expects application-supplied keys to come first + * so we flip the sign of the result to match what callers expect. */ found_key = child->key; - if (found_key.size < cursor->key.size) - WT_ERR(WT_NOTFOUND); - found_key.size = cursor->key.size; + if (found_key.size > cursor->key.size) { + /* + * Custom collators expect to see complete keys, pass an item + * containing all the visible fields so it unpacks correctly. + */ + if (cindex->index->collator != NULL) + WT_ERR(__wt_struct_repack(session, + cindex->child->key_format, cindex->iface.key_format, + &child->key, &found_key)); + else + found_key.size = cursor->key.size; + } WT_ERR(__wt_compare( session, cindex->index->collator, &cursor->key, &found_key, exact)); + *exact = -*exact; WT_ERR(__curindex_move(cindex)); @@ -515,8 +540,8 @@ __wt_curindex_open(WT_SESSION_IMPL *session, WT_ERR(__curindex_open_colgroups(session, cindex, cfg)); if (F_ISSET(cursor, WT_CURSTD_DUMP_JSON)) - __wt_json_column_init( - cursor, table->key_format, &idx->colconf, &table->colconf); + __wt_json_column_init(cursor, uri, table->key_format, + &idx->colconf, &table->colconf); if (0) { err: WT_TRET(__curindex_close(cursor)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_join.c b/src/third_party/wiredtiger/src/cursor/cur_join.c index 806436ebf38..80afaf798dc 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_join.c +++ b/src/third_party/wiredtiger/src/cursor/cur_join.c @@ -185,7 +185,7 @@ __curjoin_iter_set_entry(WT_CURSOR_JOIN_ITER *iter, u_int entry_pos) size = strlen(to_dup->internal_uri) + 3; WT_ERR(__wt_calloc(session, size, 1, &uri)); - snprintf(uri, size, "%s()", to_dup->internal_uri); + WT_ERR(__wt_snprintf(uri, size, "%s()", to_dup->internal_uri)); if ((c = iter->cursor) == NULL || !WT_STREQ(c->uri, uri)) { iter->cursor = NULL; if (c != NULL) @@ -270,7 +270,7 @@ again: iter->positioned = true; return (ret); } - else if (ret == WT_NOTFOUND) { + if (ret == WT_NOTFOUND) { WT_RET(__curjoin_iter_close_all(iter->child)); entry->subjoin->iter = NULL; iter->child = NULL; @@ -326,8 +326,7 @@ __curjoin_close(WT_CURSOR *cursor) JOINABLE_CURSOR_API_CALL(cursor, session, close, NULL); __wt_schema_release_table(session, cjoin->table); - /* These are owned by the table */ - cursor->internal_uri = NULL; + /* This is owned by the table */ cursor->key_format = NULL; if (cjoin->projection != NULL) { __wt_free(session, cjoin->projection); @@ -519,8 +518,7 @@ __curjoin_entry_in_range(WT_SESSION_IMPL *session, WT_CURSOR_JOIN_ENTRY *entry, } if (disjunction && end == endmax) return (WT_NOTFOUND); - else - return (0); + return (0); } typedef struct { @@ -921,7 +919,7 @@ __curjoin_init_next(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, "cursors"); /* Get a consistent view of our subordinate cursors if appropriate. */ - WT_RET(__wt_txn_cursor_op(session)); + __wt_txn_cursor_op(session); if (F_ISSET((WT_CURSOR *)cjoin, WT_CURSTD_RAW)) config = &raw_cfg[0]; @@ -931,7 +929,7 @@ __curjoin_init_next(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, if ((proj = cjoin->projection) != NULL) { size = strlen(urimain) + strlen(proj) + 1; WT_ERR(__wt_calloc(session, size, 1, &mainbuf)); - snprintf(mainbuf, size, "%s%s", urimain, proj); + WT_ERR(__wt_snprintf(mainbuf, size, "%s%s", urimain, proj)); urimain = mainbuf; } WT_ERR(__wt_open_cursor(session, urimain, (WT_CURSOR *)cjoin, config, @@ -976,8 +974,8 @@ __curjoin_init_next(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, if (!iterable && F_ISSET(je, WT_CURJOIN_ENTRY_BLOOM)) { if (session->txn.isolation == WT_ISO_READ_UNCOMMITTED) WT_ERR_MSG(session, EINVAL, - "join cursors with Bloom filters cannot be " - "used with read-uncommitted isolation"); + "join cursors with Bloom filters cannot be " + "used with read-uncommitted isolation"); if (je->bloom == NULL) { /* * Look for compatible filters to be shared, @@ -1150,8 +1148,8 @@ __curjoin_open_main(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, newsize = strlen(cjoin->table->name) + idx->colconf.len + 1; WT_ERR(__wt_calloc(session, 1, newsize, &main_uri)); - snprintf(main_uri, newsize, "%s%.*s", - cjoin->table->name, (int)idx->colconf.len, idx->colconf.str); + WT_ERR(__wt_snprintf(main_uri, newsize, "%s%.*s", + cjoin->table->name, (int)idx->colconf.len, idx->colconf.str)); WT_ERR(__wt_open_cursor(session, main_uri, (WT_CURSOR *)cjoin, raw_cfg, &entry->main)); if (idx->extractor == NULL) { @@ -1164,7 +1162,8 @@ __curjoin_open_main(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, */ len = strlen(entry->main->value_format) + 3; WT_ERR(__wt_calloc(session, len, 1, &newformat)); - snprintf(newformat, len, "%s0x", entry->main->value_format); + WT_ERR(__wt_snprintf( + newformat, len, "%s0x", entry->main->value_format)); __wt_free(session, entry->main->value_format); entry->main->value_format = newformat; } @@ -1310,11 +1309,10 @@ __wt_curjoin_open(WT_SESSION_IMPL *session, WT_RET_MSG(session, EINVAL, "unable to initialize a join cursor with existing owner"); - if (!WT_PREFIX_SKIP(uri, "join:")) - return (__wt_unexpected_object_type(session, uri, "join:")); tablename = uri; - if (!WT_PREFIX_SKIP(tablename, "table:")) - return (__wt_unexpected_object_type(session, uri, "table:")); + if (!WT_PREFIX_SKIP(tablename, "join:table:")) + return ( + __wt_unexpected_object_type(session, uri, "join:table:")); columns = strchr(tablename, '('); if (columns == NULL) @@ -1327,7 +1325,6 @@ __wt_curjoin_open(WT_SESSION_IMPL *session, cursor = &cjoin->iface; *cursor = iface; cursor->session = &session->iface; - cursor->internal_uri = table->name; cursor->key_format = table->key_format; cursor->value_format = table->value_format; cjoin->table = table; @@ -1336,7 +1333,7 @@ __wt_curjoin_open(WT_SESSION_IMPL *session, WT_ERR(__wt_scr_alloc(session, 0, &tmp)); if (columns != NULL) { WT_ERR(__wt_struct_reformat(session, table, - columns, strlen(columns), NULL, 1, tmp)); + columns, strlen(columns), NULL, false, tmp)); WT_ERR(__wt_strndup( session, tmp->data, tmp->size, &cursor->value_format)); WT_ERR(__wt_strdup(session, columns, &cjoin->projection)); @@ -1535,8 +1532,8 @@ __wt_curjoin_join(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, len = strlen(cindex->iface.key_format) + 3; WT_RET(__wt_calloc(session, len, 1, &entry->repack_format)); - snprintf(entry->repack_format, len, "%s0x", - cindex->iface.key_format); + WT_RET(__wt_snprintf(entry->repack_format, + len, "%s0x", cindex->iface.key_format)); } } return (0); diff --git a/src/third_party/wiredtiger/src/cursor/cur_json.c b/src/third_party/wiredtiger/src/cursor/cur_json.c index 093ec3c59ac..e8ddb767863 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_json.c +++ b/src/third_party/wiredtiger/src/cursor/cur_json.c @@ -8,8 +8,8 @@ #include "wt_internal.h" -static size_t __json_unpack_put(WT_SESSION_IMPL *, void *, u_char *, size_t, - WT_CONFIG_ITEM *); +static int __json_unpack_put( + WT_SESSION_IMPL *, void *, u_char *, size_t, WT_CONFIG_ITEM *, size_t *); static inline int __json_struct_size(WT_SESSION_IMPL *, const void *, size_t, const char *, WT_CONFIG_ITEM *, bool, size_t *); static inline int __json_struct_unpackv(WT_SESSION_IMPL *, const void *, size_t, @@ -23,20 +23,20 @@ static int __json_pack_size(WT_SESSION_IMPL *, const char *, WT_CONFIG_ITEM *, bool, const char *, size_t *); #define WT_PACK_JSON_GET(session, pv, jstr) do { \ - switch (pv.type) { \ + switch ((pv).type) { \ case 'x': \ break; \ case 's': \ case 'S': \ - WT_RET(json_string_arg(session, &jstr, &pv.u.item)); \ - pv.type = pv.type == 's' ? 'j' : 'J'; \ + WT_RET(json_string_arg(session, &(jstr), &(pv).u.item));\ + (pv).type = (pv).type == 's' ? 'j' : 'J'; \ break; \ case 'b': \ case 'h': \ case 'i': \ case 'l': \ case 'q': \ - WT_RET(json_int_arg(session, &jstr, &pv.u.i)); \ + WT_RET(json_int_arg(session, &(jstr), &(pv).u.i)); \ break; \ case 'B': \ case 'H': \ @@ -46,11 +46,11 @@ static int __json_pack_size(WT_SESSION_IMPL *, const char *, WT_CONFIG_ITEM *, case 'r': \ case 'R': \ case 't': \ - WT_RET(json_uint_arg(session, &jstr, &pv.u.u)); \ + WT_RET(json_uint_arg(session, &(jstr), &(pv).u.u)); \ break; \ case 'u': \ - WT_RET(json_string_arg(session, &jstr, &pv.u.item)); \ - pv.type = 'K'; \ + WT_RET(json_string_arg(session, &(jstr), &(pv).u.item));\ + (pv).type = 'K'; \ break; \ /* User format strings have already been validated. */ \ WT_ILLEGAL_VALUE(session); \ @@ -61,22 +61,22 @@ static int __json_pack_size(WT_SESSION_IMPL *, const char *, WT_CONFIG_ITEM *, * __json_unpack_put -- * Calculate the size of a packed byte string as formatted for JSON. */ -static size_t +static int __json_unpack_put(WT_SESSION_IMPL *session, void *voidpv, - u_char *buf, size_t bufsz, WT_CONFIG_ITEM *name) + u_char *buf, size_t bufsz, WT_CONFIG_ITEM *name, size_t *retsizep) { WT_PACK_VALUE *pv; const u_char *p, *end; size_t s, n; pv = (WT_PACK_VALUE *)voidpv; - s = (size_t)snprintf((char *)buf, bufsz, "\"%.*s\" : ", - (int)name->len, name->str); + + WT_RET(__wt_snprintf_len_set( + (char *)buf, bufsz, &s, "\"%.*s\" : ", (int)name->len, name->str)); if (s <= bufsz) { bufsz -= s; buf += s; - } - else + } else bufsz = 0; switch (pv->type) { @@ -118,7 +118,8 @@ __json_unpack_put(WT_SESSION_IMPL *session, void *voidpv, } if (bufsz > 0) *buf++ = '"'; - return (s); + *retsizep += s; + return (0); case 'U': case 'u': s += 2; @@ -140,14 +141,17 @@ __json_unpack_put(WT_SESSION_IMPL *session, void *voidpv, } if (bufsz > 0) *buf++ = '"'; - return (s); + *retsizep += s; + return (0); case 'b': case 'h': case 'i': case 'l': case 'q': - return (s + - (size_t)snprintf((char *)buf, bufsz, "%" PRId64, pv->u.i)); + WT_RET(__wt_snprintf_len_incr( + (char *)buf, bufsz, &s, "%" PRId64, pv->u.i)); + *retsizep += s; + return (0); case 'B': case 't': case 'H': @@ -156,11 +160,14 @@ __json_unpack_put(WT_SESSION_IMPL *session, void *voidpv, case 'Q': case 'r': case 'R': - return (s + - (size_t)snprintf((char *)buf, bufsz, "%" PRId64, pv->u.u)); + WT_RET(__wt_snprintf_len_incr( + (char *)buf, bufsz, &s, "%" PRId64, pv->u.u)); + *retsizep += s; + return (0); } - __wt_err(session, EINVAL, "unknown pack-value type: %c", (int)pv->type); - return ((size_t)-1); + + WT_RET_MSG(session, EINVAL, + "unknown pack-value type: %c", (int)pv->type); } /* @@ -194,7 +201,8 @@ __json_struct_size(WT_SESSION_IMPL *session, const void *buffer, needcr = true; WT_RET(__unpack_read(session, &pv, &p, (size_t)(end - p))); WT_RET(__pack_name_next(&packname, &name)); - result += __json_unpack_put(session, &pv, NULL, 0, &name); + WT_RET( + __json_unpack_put(session, &pv, NULL, 0, &name, &result)); } if (ret == WT_NOTFOUND) ret = 0; @@ -243,8 +251,9 @@ __json_struct_unpackv(WT_SESSION_IMPL *session, needcr = true; WT_RET(__unpack_read(session, &pv, &p, (size_t)(end - p))); WT_RET(__pack_name_next(&packname, &name)); - jsize = __json_unpack_put(session, - (u_char *)&pv, jbuf, jbufsize, &name); + jsize = 0; + WT_RET(__json_unpack_put(session, + (u_char *)&pv, jbuf, jbufsize, &name, &jsize)); WT_ASSERT(session, jsize <= jbufsize); jbuf += jsize; jbufsize -= jsize; @@ -270,7 +279,6 @@ __wt_json_alloc_unpack(WT_SESSION_IMPL *session, const void *buffer, bool iskey, va_list ap) { WT_CONFIG_ITEM *names; - WT_DECL_RET; size_t needed; char **json_bufp; @@ -288,7 +296,7 @@ __wt_json_alloc_unpack(WT_SESSION_IMPL *session, const void *buffer, WT_RET(__json_struct_unpackv(session, buffer, size, fmt, names, (u_char *)*json_bufp, needed + 1, iskey, ap)); - return (ret); + return (0); } /* @@ -305,7 +313,6 @@ __wt_json_close(WT_SESSION_IMPL *session, WT_CURSOR *cursor) __wt_free(session, json->value_buf); __wt_free(session, json); } - return; } /* @@ -315,6 +322,7 @@ __wt_json_close(WT_SESSION_IMPL *session, WT_CURSOR *cursor) */ size_t __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { u_char abbrev; @@ -323,33 +331,32 @@ __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode) if (bufsz >= 1) *buf = ch; return (1); - } else { - abbrev = '\0'; - switch (ch) { - case '\\': - case '"': - abbrev = ch; - break; - case '\f': - abbrev = 'f'; - break; - case '\n': - abbrev = 'n'; - break; - case '\r': - abbrev = 'r'; - break; - case '\t': - abbrev = 't'; - break; - } - if (abbrev != '\0') { - if (bufsz >= 2) { - *buf++ = '\\'; - *buf = abbrev; - } - return (2); + } + abbrev = '\0'; + switch (ch) { + case '\\': + case '"': + abbrev = ch; + break; + case '\f': + abbrev = 'f'; + break; + case '\n': + abbrev = 'n'; + break; + case '\r': + abbrev = 'r'; + break; + case '\t': + abbrev = 't'; + break; + } + if (abbrev != '\0') { + if (bufsz >= 2) { + *buf++ = '\\'; + *buf = abbrev; } + return (2); } } if (bufsz >= 6) { @@ -357,8 +364,8 @@ __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode) *buf++ = 'u'; *buf++ = '0'; *buf++ = '0'; - *buf++ = __wt_hex[(ch & 0xf0) >> 4]; - *buf++ = __wt_hex[ch & 0x0f]; + *buf++ = __wt_hex((ch & 0xf0) >> 4); + *buf++ = __wt_hex(ch & 0x0f); } return (6); } @@ -369,11 +376,11 @@ __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode) * of column names. */ void -__wt_json_column_init(WT_CURSOR *cursor, const char *keyformat, +__wt_json_column_init(WT_CURSOR *cursor, const char *uri, const char *keyformat, const WT_CONFIG_ITEM *idxconf, const WT_CONFIG_ITEM *colconf) { WT_CURSOR_JSON *json; - const char *p, *end, *beginkey; + const char *beginkey, *end, *lparen, *p; uint32_t keycnt, nkeys; json = (WT_CURSOR_JSON *)cursor->json_private; @@ -400,8 +407,16 @@ __wt_json_column_init(WT_CURSOR *cursor, const char *keyformat, keycnt++; p++; } - json->value_names.str = p; - json->value_names.len = WT_PTRDIFF(end, p); + if ((lparen = strchr(uri, '(')) != NULL) { + /* This cursor is a projection. */ + json->value_names.str = lparen; + json->value_names.len = strlen(lparen) - 1; + WT_ASSERT((WT_SESSION_IMPL *)cursor->session, + json->value_names.str[json->value_names.len] == ')'); + } else { + json->value_names.str = p; + json->value_names.len = WT_PTRDIFF(end, p); + } if (idxconf == NULL) { if (p > beginkey) p--; @@ -413,15 +428,16 @@ __wt_json_column_init(WT_CURSOR *cursor, const char *keyformat, #define MATCH_KEYWORD(session, in, result, keyword, matchval) do { \ size_t _kwlen = strlen(keyword); \ if (strncmp(in, keyword, _kwlen) == 0 && \ - !__wt_isalnum((u_char)in[_kwlen])) { \ - in += _kwlen; \ - result = matchval; \ + !__wt_isalnum((u_char)(in)[_kwlen])) { \ + (in) += _kwlen; \ + (result) = matchval; \ } else { \ - const char *_bad = in; \ - while (__wt_isalnum((u_char)*in)) \ - in++; \ - __wt_errx(session, "unknown keyword \"%.*s\" in JSON", \ - (int)(in - _bad), _bad); \ + const char *_bad = (in); \ + while (__wt_isalnum((u_char)*(in))) \ + (in)++; \ + WT_RET_MSG(session, EINVAL, \ + "unknown keyword \"%.*s\" in JSON", \ + (int)((in) - _bad), _bad); \ } \ } while (0) @@ -451,6 +467,7 @@ __wt_json_column_init(WT_CURSOR *cursor, const char *keyformat, int __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, const char **tokstart, size_t *toklen) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_SESSION_IMPL *session; int result; @@ -501,9 +518,9 @@ __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, } src++; } - if (result != 's') - __wt_errx(session, "unterminated string in JSON"); - break; + if (result == 's') + break; + WT_RET_MSG(session, EINVAL, "unterminated string in JSON"); case '-': case '0': case '1': @@ -561,15 +578,14 @@ __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, if (isalph) while (*src != '\0' && __wt_isalnum((u_char)*src)) src++; - __wt_errx(session, "unknown token \"%.*s\" in JSON", - (int)(src - bad), bad); - break; + WT_RET_MSG(session, EINVAL, + "unknown token \"%.*s\" in JSON", (int)(src - bad), bad); + /* NOTREACHED */ } + WT_ASSERT(session, result != -1); + *toklen = (size_t)(src - *tokstart); *toktype = result; - - if (result < 0) - WT_RET_MSG(session, EINVAL, "illegal token in JSON"); return (0); } @@ -580,6 +596,7 @@ __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, */ const char * __wt_json_tokname(int toktype) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { switch (toktype) { case 0: return (""); @@ -682,12 +699,13 @@ json_uint_arg(WT_SESSION_IMPL *session, const char **jstr, uint64_t *up) #define JSON_EXPECT_TOKEN_GET(session, jstr, tokval, start, sz) do { \ int __tok; \ - WT_RET(__wt_json_token((WT_SESSION *)session, jstr, &__tok, &start, &sz));\ - if (__tok != tokval) \ + WT_RET(__wt_json_token( \ + (WT_SESSION *)(session), jstr, &__tok, &(start), &(sz))); \ + if (__tok != (tokval)) \ WT_RET_MSG(session, EINVAL, \ "expected JSON %s, got %s", \ __wt_json_tokname(tokval), __wt_json_tokname(__tok)); \ - jstr = start + sz; \ + (jstr) = (start) + (sz); \ } while (0) #define JSON_EXPECT_TOKEN(session, jstr, tokval) do { \ @@ -817,6 +835,7 @@ __wt_json_to_item(WT_SESSION_IMPL *session, const char *jstr, */ ssize_t __wt_json_strlen(const char *src, size_t srclen) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { const char *srcend; size_t dstlen; @@ -857,8 +876,9 @@ __wt_json_strlen(const char *src, size_t srclen) * bytes. If dstlen is greater than the needed size, the result if zero padded. */ int -__wt_json_strncpy(WT_SESSION *wt_session, char **pdst, size_t dstlen, - const char *src, size_t srclen) +__wt_json_strncpy(WT_SESSION *wt_session, + char **pdst, size_t dstlen, const char *src, size_t srclen) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_SESSION_IMPL *session; char ch, *dst; diff --git a/src/third_party/wiredtiger/src/cursor/cur_log.c b/src/third_party/wiredtiger/src/cursor/cur_log.c index 21a7f674c68..e5b56aa406f 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_log.c +++ b/src/third_party/wiredtiger/src/cursor/cur_log.c @@ -150,18 +150,22 @@ static int __curlog_kv(WT_SESSION_IMPL *session, WT_CURSOR *cursor) { WT_CURSOR_LOG *cl; - WT_ITEM item; - uint32_t fileid, key_count, opsize, optype; + WT_DECL_RET; + uint32_t fileid, key_count, opsize, optype, raw; cl = (WT_CURSOR_LOG *)cursor; + /* Temporarily turn off raw so we can do direct cursor operations. */ + raw = F_MASK(cursor, WT_CURSTD_RAW); + F_CLR(cursor, WT_CURSTD_RAW); + /* * If it is a commit and we have stepped over the header, peek to get * the size and optype and read out any key/value from this operation. */ if ((key_count = cl->step_count++) > 0) { - WT_RET(__wt_logop_read(session, + WT_ERR(__wt_logop_read(session, &cl->stepp, cl->stepp_end, &optype, &opsize)); - WT_RET(__curlog_op_read(session, cl, optype, opsize, &fileid)); + WT_ERR(__curlog_op_read(session, cl, optype, opsize, &fileid)); /* Position on the beginning of the next record part. */ cl->stepp += opsize; } else { @@ -181,39 +185,14 @@ __curlog_kv(WT_SESSION_IMPL *session, WT_CURSOR *cursor) * The log cursor sets the LSN and step count as the cursor key and * and log record related data in the value. The data in the value * contains any operation key/value that was in the log record. - * For the special case that the caller needs the result in raw form, - * we create packed versions of the key/value. */ - if (FLD_ISSET(cursor->flags, WT_CURSTD_RAW)) { - memset(&item, 0, sizeof(item)); - WT_RET(wiredtiger_struct_size((WT_SESSION *)session, - &item.size, WT_LOGC_KEY_FORMAT, cl->cur_lsn->l.file, - cl->cur_lsn->l.offset, key_count)); - WT_RET(__wt_realloc(session, NULL, item.size, &cl->packed_key)); - item.data = cl->packed_key; - WT_RET(wiredtiger_struct_pack((WT_SESSION *)session, - cl->packed_key, item.size, WT_LOGC_KEY_FORMAT, - cl->cur_lsn->l.file, cl->cur_lsn->l.offset, key_count)); - __wt_cursor_set_key(cursor, &item); - - WT_RET(wiredtiger_struct_size((WT_SESSION *)session, - &item.size, WT_LOGC_VALUE_FORMAT, cl->txnid, cl->rectype, - optype, fileid, cl->opkey, cl->opvalue)); - WT_RET(__wt_realloc(session, NULL, item.size, - &cl->packed_value)); - item.data = cl->packed_value; - WT_RET(wiredtiger_struct_pack((WT_SESSION *)session, - cl->packed_value, item.size, WT_LOGC_VALUE_FORMAT, - cl->txnid, cl->rectype, optype, fileid, cl->opkey, - cl->opvalue)); - __wt_cursor_set_value(cursor, &item); - } else { - __wt_cursor_set_key(cursor, cl->cur_lsn->l.file, - cl->cur_lsn->l.offset, key_count); - __wt_cursor_set_value(cursor, cl->txnid, cl->rectype, optype, - fileid, cl->opkey, cl->opvalue); - } - return (0); + __wt_cursor_set_key(cursor, cl->cur_lsn->l.file, cl->cur_lsn->l.offset, + key_count); + __wt_cursor_set_value(cursor, cl->txnid, cl->rectype, optype, fileid, + cl->opkey, cl->opvalue); + +err: F_SET(cursor, raw); + return (ret); } /* @@ -264,17 +243,19 @@ __curlog_search(WT_CURSOR *cursor) WT_DECL_RET; WT_LSN key; WT_SESSION_IMPL *session; - uint32_t counter, key_file, key_offset; + uint32_t counter, key_file, key_offset, raw; cl = (WT_CURSOR_LOG *)cursor; + /* Temporarily turn off raw so we can do direct cursor operations. */ + raw = F_MASK(cursor, WT_CURSTD_RAW); + F_CLR(cursor, WT_CURSTD_RAW); CURSOR_API_CALL(cursor, session, search, NULL); /* * !!! We are ignoring the counter and only searching based on the LSN. */ - WT_ERR(__wt_cursor_get_key((WT_CURSOR *)cl, - &key_file, &key_offset, &counter)); + WT_ERR(__wt_cursor_get_key(cursor, &key_file, &key_offset, &counter)); WT_SET_LSN(&key, key_file, key_offset); ret = __wt_log_scan(session, &key, WT_LOGSCAN_ONE, __curlog_logrec, cl); @@ -285,7 +266,8 @@ __curlog_search(WT_CURSOR *cursor) WT_STAT_CONN_INCR(session, cursor_search); WT_STAT_DATA_INCR(session, cursor_search); -err: API_END_RET(session, ret); +err: F_SET(cursor, raw); + API_END_RET(session, ret); } /* @@ -323,7 +305,7 @@ __curlog_close(WT_CURSOR *cursor) WT_ASSERT(session, FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)); if (F_ISSET(cl, WT_CURLOG_ARCHIVE_LOCK)) - __wt_readunlock(session, conn->log->log_archive_lock); + __wt_readunlock(session, &conn->log->log_archive_lock); __wt_free(session, cl->cur_lsn); __wt_free(session, cl->next_lsn); @@ -401,7 +383,7 @@ __wt_curlog_open(WT_SESSION_IMPL *session, WT_ERR(__wt_log_force_write(session, 1, NULL)); /* Log cursors block archiving. */ - __wt_readlock(session, log->log_archive_lock); + __wt_readlock(session, &log->log_archive_lock); F_SET(cl, WT_CURLOG_ARCHIVE_LOCK); if (0) { diff --git a/src/third_party/wiredtiger/src/cursor/cur_metadata.c b/src/third_party/wiredtiger/src/cursor/cur_metadata.c index 10e2fdf28be..fbfc73956e2 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_metadata.c +++ b/src/third_party/wiredtiger/src/cursor/cur_metadata.c @@ -16,7 +16,7 @@ WT_CURSOR_NEEDKEY(cursor); \ WT_ERR(__wt_buf_set(session, \ &((WT_CURSOR_METADATA *)(cursor))->file_cursor->key, \ - cursor->key.data, cursor->key.size)); \ + (cursor)->key.data, (cursor)->key.size)); \ F_SET(((WT_CURSOR_METADATA *)(cursor))->file_cursor, \ WT_CURSTD_KEY_EXT); \ } while (0) @@ -25,7 +25,7 @@ WT_CURSOR_NEEDVALUE(cursor); \ WT_ERR(__wt_buf_set(session, \ &((WT_CURSOR_METADATA *)(cursor))->file_cursor->value, \ - cursor->value.data, cursor->value.size)); \ + (cursor)->value.data, (cursor)->value.size)); \ F_SET(((WT_CURSOR_METADATA *)(cursor))->file_cursor, \ WT_CURSTD_VALUE_EXT); \ } while (0) diff --git a/src/third_party/wiredtiger/src/cursor/cur_stat.c b/src/third_party/wiredtiger/src/cursor/cur_stat.c index 5fde64c74ca..0bff642370d 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_stat.c +++ b/src/third_party/wiredtiger/src/cursor/cur_stat.c @@ -163,7 +163,6 @@ static void __curstat_set_value(WT_CURSOR *cursor, ...) { WT_UNUSED(cursor); - return; } /* @@ -478,8 +477,8 @@ __curstat_join_desc(WT_CURSOR_STAT *cst, int slot, const char **resultp) len = strlen("join: ") + strlen(sgrp->desc_prefix) + strlen(static_desc) + 1; WT_RET(__wt_realloc(session, NULL, len, &cst->desc_buf)); - snprintf(cst->desc_buf, len, "join: %s%s", sgrp->desc_prefix, - static_desc); + WT_RET(__wt_snprintf( + cst->desc_buf, len, "join: %s%s", sgrp->desc_prefix, static_desc)); *resultp = cst->desc_buf; return (0); } diff --git a/src/third_party/wiredtiger/src/cursor/cur_std.c b/src/third_party/wiredtiger/src/cursor/cur_std.c index 6264de89df9..99a9e373354 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_std.c +++ b/src/third_party/wiredtiger/src/cursor/cur_std.c @@ -144,6 +144,7 @@ __wt_cursor_set_notsup(WT_CURSOR *cursor) */ int __wt_cursor_kv_not_set(WT_CURSOR *cursor, bool key) + WT_GCC_FUNC_ATTRIBUTE((cold)) { WT_SESSION_IMPL *session; @@ -632,6 +633,7 @@ __wt_cursor_reconfigure(WT_CURSOR *cursor, const char *config) int __wt_cursor_dup_position(WT_CURSOR *to_dup, WT_CURSOR *cursor) { + WT_DECL_RET; WT_ITEM key; /* @@ -661,9 +663,11 @@ __wt_cursor_dup_position(WT_CURSOR *to_dup, WT_CURSOR *cursor) * cursors cannot reference application memory after cursor operations * and that requirement will save the day. */ - WT_RET(cursor->search(cursor)); + F_SET(cursor, WT_CURSTD_RAW_SEARCH); + ret = cursor->search(cursor); + F_CLR(cursor, WT_CURSTD_RAW_SEARCH); - return (0); + return (ret); } /* diff --git a/src/third_party/wiredtiger/src/cursor/cur_table.c b/src/third_party/wiredtiger/src/cursor/cur_table.c index 502d93639a8..f6855172e90 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_table.c +++ b/src/third_party/wiredtiger/src/cursor/cur_table.c @@ -14,8 +14,8 @@ static int __curtable_update(WT_CURSOR *cursor); #define APPLY_CG(ctable, f) do { \ WT_CURSOR **__cp; \ u_int __i; \ - for (__i = 0, __cp = ctable->cg_cursors; \ - __i < WT_COLGROUPS(ctable->table); \ + for (__i = 0, __cp = (ctable)->cg_cursors; \ + __i < WT_COLGROUPS((ctable)->table); \ __i++, __cp++) \ WT_TRET((*__cp)->f(*__cp)); \ } while (0) @@ -511,9 +511,16 @@ __curtable_insert(WT_CURSOR *cursor) */ F_SET(primary, flag_orig | WT_CURSTD_KEY_EXT | WT_CURSTD_VALUE_EXT); - if (ret == WT_DUPLICATE_KEY && F_ISSET(cursor, WT_CURSTD_OVERWRITE)) + if (ret == WT_DUPLICATE_KEY && F_ISSET(cursor, WT_CURSTD_OVERWRITE)) { WT_ERR(__curtable_update(cursor)); - else { + + /* + * The cursor is no longer positioned. This isn't just cosmetic, + * without a reset, iteration on this cursor won't start at the + * beginning/end of the table. + */ + APPLY_CG(ctable, reset); + } else { WT_ERR(ret); for (i = 1; i < WT_COLGROUPS(ctable->table); i++, cp++) { @@ -601,22 +608,53 @@ err: CURSOR_UPDATE_API_END(session, ret); static int __curtable_remove(WT_CURSOR *cursor) { + WT_CURSOR *primary; WT_CURSOR_TABLE *ctable; WT_DECL_RET; WT_SESSION_IMPL *session; + bool positioned; ctable = (WT_CURSOR_TABLE *)cursor; JOINABLE_CURSOR_REMOVE_API_CALL(cursor, session, NULL); WT_ERR(__curtable_open_indices(ctable)); + /* Check if the cursor was positioned. */ + primary = *ctable->cg_cursors; + positioned = F_ISSET(primary, WT_CURSTD_KEY_INT); + /* Find the old record so it can be removed from indices */ if (ctable->table->nindices > 0) { APPLY_CG(ctable, search); + if (ret == WT_NOTFOUND) + goto notfound; WT_ERR(ret); WT_ERR(__apply_idx(ctable, offsetof(WT_CURSOR, remove), false)); } APPLY_CG(ctable, remove); + if (ret == WT_NOTFOUND) + goto notfound; + WT_ERR(ret); + +notfound: + /* + * If the cursor is configured to overwrite and the record is not found, + * that is exactly what we want. + */ + if (ret == WT_NOTFOUND && F_ISSET(primary, WT_CURSTD_OVERWRITE)) + ret = 0; + + /* + * If the cursor was positioned, it stays positioned with a key but no + * no value, otherwise, there's no position, key or value. This isn't + * just cosmetic, without a reset, iteration on this cursor won't start + * at the beginning/end of the table. + */ + F_CLR(primary, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + if (positioned) + F_SET(primary, WT_CURSTD_KEY_INT); + else + APPLY_CG(ctable, reset); err: CURSOR_UPDATE_API_END(session, ret); return (ret); @@ -763,16 +801,13 @@ err: API_END_RET(session, ret); static int __curtable_complete(WT_SESSION_IMPL *session, WT_TABLE *table) { - WT_DECL_RET; bool complete; if (table->cg_complete) return (0); /* If the table is incomplete, wait on the table lock and recheck. */ - complete = false; - WT_WITH_TABLE_LOCK(session, ret, complete = table->cg_complete); - WT_RET(ret); + WT_WITH_TABLE_READ_LOCK(session, complete = table->cg_complete); if (!complete) WT_RET_MSG(session, EINVAL, "'%s' not available until all column groups are created", @@ -908,6 +943,12 @@ __wt_curtable_open(WT_SESSION_IMPL *session, table->cgroups[0]->source, NULL, cfg, cursorp); __wt_schema_release_table(session, table); + if (ret == 0) { + /* Fix up the public URI to match what was passed in. */ + cursor = *cursorp; + __wt_free(session, cursor->uri); + WT_TRET(__wt_strdup(session, uri, &cursor->uri)); + } return (ret); } @@ -927,7 +968,7 @@ __wt_curtable_open(WT_SESSION_IMPL *session, WT_ERR(__wt_scr_alloc(session, 0, &tmp)); if (columns != NULL) { WT_ERR(__wt_struct_reformat(session, table, - columns, strlen(columns), NULL, true, tmp)); + columns, strlen(columns), NULL, false, tmp)); WT_ERR(__wt_strndup( session, tmp->data, tmp->size, &cursor->value_format)); @@ -954,7 +995,7 @@ __wt_curtable_open(WT_SESSION_IMPL *session, if (F_ISSET(cursor, WT_CURSTD_DUMP_JSON)) __wt_json_column_init( - cursor, table->key_format, NULL, &table->colconf); + cursor, uri, table->key_format, NULL, &table->colconf); /* * Open the colgroup cursors immediately: we're going to need them for @@ -992,11 +1033,15 @@ __wt_curtable_open(WT_SESSION_IMPL *session, if (0) { err: if (*cursorp != NULL) { - if (*cursorp != cursor) - WT_TRET(__wt_cursor_close(*cursorp)); + /* + * When a dump cursor is opened, then *cursorp, not + * cursor, is the dump cursor. Close the dump cursor, + * and the table cursor will be closed as its child. + */ + cursor = *cursorp; *cursorp = NULL; } - WT_TRET(__curtable_close(cursor)); + WT_TRET(cursor->close(cursor)); } __wt_scr_free(session, &tmp); diff --git a/src/third_party/wiredtiger/src/docs/Doxyfile b/src/third_party/wiredtiger/src/docs/Doxyfile index 69e9716b425..3d8c46962f1 100644 --- a/src/third_party/wiredtiger/src/docs/Doxyfile +++ b/src/third_party/wiredtiger/src/docs/Doxyfile @@ -216,11 +216,19 @@ ALIASES = "notyet{1}=Note: "\1" not yet supported in Wired "hrow{3}=\1\2\3" \ "hrow{4}=\1\2\3\4" \ "hrow{5}=\1\2\3\4\5" \ + "hrow{6}=\1\2\3\4\5\6" \ + "hrow{7}=\1\2\3\4\5\6\7" \ + "hrow{8}=\1\2\3\4\5\6\7\8" \ + "hrow{9}=\1\2\3\4\5\6\7\8\9" \ "row{1}=\1" \ "row{2}=\1\2" \ "row{3}=\1\2\3" \ "row{4}=\1\2\3\4" \ "row{5}=\1\2\3\4\5" \ + "row{6}=\1\2\3\4\5\6" \ + "row{7}=\1\2\3\4\5\6\7" \ + "row{8}=\1\2\3\4\5\6\7\8" \ + "row{9}=\1\2\3\4\5\6\7\8\9" \ "configstart{2}=@param config\n Configuration string, see @ref config_strings. Permitted values:\n @hrow{Name,Effect,Values}" \ "config{3}= @row{\1,\2,\3}" \ "configend=
" \ diff --git a/src/third_party/wiredtiger/src/docs/command-line.dox b/src/third_party/wiredtiger/src/docs/command-line.dox index 0f5c56d25ce..df52324f8f8 100644 --- a/src/third_party/wiredtiger/src/docs/command-line.dox +++ b/src/third_party/wiredtiger/src/docs/command-line.dox @@ -36,6 +36,39 @@ the user wants to force recovery on any command, use the \c -R option. In general, commands that modify the database or tables will run recovery by default and commands that only read data will not run recovery. +
+@section util_alter wt alter +Alter a table. + +@subsection util_alter_synopsis Synopsis +wt [-RVv] [-C config] [-E secretkey ] [-h directory] alter uri configuration ... + +The \c uri and \c configuration pairs may be specified to the +\c alter command. These configuration pairs can be used to modify the +configuration values from those passed to the WT_SESSION::create +call. + +The \c uri part of the configuration pair should match only one of the +objects being altered, but may be a prefix of the object being matched. +For example, the following two sets of configuration pairs are +equivalent in the case of altering a single table named \c xxx. + +@code +table access_pattern_hint=sequential +table:xxx access_pattern_hint=sequential +@endcode + +It's an error, however, to specify a matching prefix that matches more +than a single object being altered. + +Multiple \c configuration arguments may be specified. For example, the +following two sets of configuration pairs are equivalent: + +@code +table:xxx access_pattern_hint=random,cache_resident=false +table:xxx access_pattern_hint=random table:xxx cache_resident=false +@endcode +
@section util_backup wt backup Perform a backup of a database or set of data sources. @@ -336,6 +369,19 @@ The following are command-specific options for the \c stat command: Include only "fast" statistics in the output (equivalent to passing statistics=(fast)) to WT_SESSION::open_cursor. +
+@section util_truncate wt truncate +Truncate a table, removing all data. + +The \c truncate command truncates the specified \c uri. It is equivalent to a +call to WT_SESSION::truncate with no start or stop specified. + +@subsection util_truncate_synopsis Synopsis +wt [-RVv] [-C config] [-E secretkey ] [-h directory] truncate uri + +@subsection util_truncate_options Options +The \c truncate command has no command-specific options. +
@section util_upgrade wt upgrade Upgrade a table. diff --git a/src/third_party/wiredtiger/src/docs/cursor-ops.dox b/src/third_party/wiredtiger/src/docs/cursor-ops.dox index b743d81db57..e479ff29191 100644 --- a/src/third_party/wiredtiger/src/docs/cursor-ops.dox +++ b/src/third_party/wiredtiger/src/docs/cursor-ops.dox @@ -145,9 +145,5 @@ that may not be modified or freed by the application. If a longer scope is required, the application must make a copy of the memory before the cursor is re-used, closed or reset. -The comments in this example code explain when the application can safely -modify memory passed to WT_CURSOR::set_key or WT_CURSOR::set_value: - -@snippet ex_scope.c cursor scope operation @m_endif */ diff --git a/src/third_party/wiredtiger/src/docs/cursor-random.dox b/src/third_party/wiredtiger/src/docs/cursor-random.dox index a0a3212be6d..b6434e3d161 100644 --- a/src/third_party/wiredtiger/src/docs/cursor-random.dox +++ b/src/third_party/wiredtiger/src/docs/cursor-random.dox @@ -20,9 +20,4 @@ cursor configured using \c next_random_sample_size divides the object into \c next_random_sample_size pieces, and each subsequent retrieval returns a record from the next one of those pieces. -For example, setting \c next_random_sample_percent to \c 10 would cause -the cursor to sequentially return records from each tenth part of the -object. Setting \c next_random_sample_percent to \c 1000 would cause the -cursor to sequentially return records from each .1% of the object. - */ diff --git a/src/third_party/wiredtiger/src/docs/file-formats.dox b/src/third_party/wiredtiger/src/docs/file-formats.dox index d8990aca7a6..21dc4580bc2 100644 --- a/src/third_party/wiredtiger/src/docs/file-formats.dox +++ b/src/third_party/wiredtiger/src/docs/file-formats.dox @@ -110,7 +110,7 @@ considered. (See @subpage_single huffman for details.) compressing blocks of the backing object's file. The cost is additional CPU and memory use when reading and writing pages to disk. Note the additional CPU cost of block compression can be high, and should be -considered. (See @x_ref compression_formats for details.) +considered. (See @x_ref compression_considerations for details.) Block compression is disabled by default. @@ -146,7 +146,7 @@ Huffman encoding can be high, and should be considered. compressing blocks of the backing object's file. The cost is additional CPU and memory use when reading and writing pages to disk. Note the additional CPU cost of block compression can be high, and should be -considered. (See @x_ref compression_formats for details.) +considered. (See @x_ref compression_considerations for details.) Block compression is disabled by default. @@ -157,7 +157,7 @@ compression: block compression. compressing blocks of the backing object's file. The cost is additional CPU and memory use when reading and writing pages to disk. Note the additional CPU cost of block compression can be high, and should be -considered. (See @x_ref compression_formats for details.) +considered. (See @x_ref compression_considerations for details.) Block compression is disabled by default. diff --git a/src/third_party/wiredtiger/src/docs/programming.dox b/src/third_party/wiredtiger/src/docs/programming.dox index 81e612e8ee8..aa76bef4614 100644 --- a/src/third_party/wiredtiger/src/docs/programming.dox +++ b/src/third_party/wiredtiger/src/docs/programming.dox @@ -66,14 +66,13 @@ each of which is ordered by one or more columns. - @subpage_single wtstats

- @subpage_single tune_memory_allocator -- @subpage_single tune_page_sizes +- @subpage_single tune_page_size_and_comp - @subpage_single tune_cache - @subpage_single tune_bulk_load - @subpage_single tune_cursor_persist - @subpage_single tune_read_only - @subpage_single tune_durability - @subpage_single tune_checksum -- @subpage_single tune_compression - @subpage_single tune_file_alloc - @subpage_single tune_system_buffer_cache - @subpage_single tune_transparent_huge_pages diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index 2413cbc93fb..bc2e16b1122 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -50,6 +50,7 @@ LDFLAGS LIBS LLVM LOGREC +LRU LRVv LSB LSM @@ -167,6 +168,7 @@ dNLen dNOff dT dataN +database's dataitem dataset datasets diff --git a/src/third_party/wiredtiger/src/docs/top/main.dox b/src/third_party/wiredtiger/src/docs/top/main.dox index 08f7536d540..84487c13174 100644 --- a/src/third_party/wiredtiger/src/docs/top/main.dox +++ b/src/third_party/wiredtiger/src/docs/top/main.dox @@ -6,12 +6,12 @@ WiredTiger is an high performance, scalable, production quality, NoSQL, @section releases Releases -@row{WiredTiger 2.8.0 (current), - [Release package], - [Documentation]} -@row{WiredTiger 2.7.0 (current), - [Release package], - [Documentation]} +@row{WiredTiger 2.9.1 (current), + [Release package], + [Documentation]} +@row{WiredTiger 2.9.0 (previous), + [Release package], + [Documentation]} @row{Development branch, [Source code], [Documentation]} diff --git a/src/third_party/wiredtiger/src/docs/transactions.dox b/src/third_party/wiredtiger/src/docs/transactions.dox index bbbd2d52296..3b438eda366 100644 --- a/src/third_party/wiredtiger/src/docs/transactions.dox +++ b/src/third_party/wiredtiger/src/docs/transactions.dox @@ -141,7 +141,7 @@ as if the transaction started at the time of the WT_SESSION::snapshot call that created the snapshot. Named snapshots keep data pinned in cache as if a real transaction were -running for the time that the named transaction is active. The resources +running for the time that the named snapshot is active. The resources associated with named snapshots should be released by calling WT_SESSION::snapshot with a configuration that includes "drop=". See WT_SESSION::snapshot documentation for details of diff --git a/src/third_party/wiredtiger/src/docs/tune-compression.dox b/src/third_party/wiredtiger/src/docs/tune-compression.dox deleted file mode 100644 index 8db2151aa76..00000000000 --- a/src/third_party/wiredtiger/src/docs/tune-compression.dox +++ /dev/null @@ -1,62 +0,0 @@ -/*! @page tune_compression Compression - -WiredTiger includes a number of optional compression techniques. Configuring -compression generally decreases on-disk and in-memory resource requirements -and the amount of I/O, and increases CPU cost when data are read and written. - -Configuring compression may change application throughput. For example, -in applications using solid-state drives (where I/O is less expensive), -turning off compression may increase application performance by reducing -CPU costs; in applications where I/O costs are more expensive, turning on -compression may increase application performance by reducing the overall -number of I/O operations. - -An example of turning on row-store key prefix compression: - -@snippet ex_all.c Configure key prefix compression on - -An example of turning on row-store or column-store dictionary compression: - -@snippet ex_all.c Configure dictionary compression on - -@section compression_formats Block Compression Formats -WiredTiger provides two methods of compressing your data when using block -compression: the raw and noraw methods. These methods change how WiredTiger -works to fit data into the blocks that are stored on disk. - -@subsection noraw_compression Noraw Compression -Noraw compression is the traditional compression model where a fixed -amount of data is given to the compression system, then turned into a -compressed block of data. The amount of data chosen to compress is the -data needed to fill the uncompressed block. Thus when compressed, the block will -be smaller than the normal data size and the sizes written to disk will often -vary depending on how compressible the data being stored is. Algorithms -using noraw compression include zlib-noraw, lz4-noraw and snappy. - -@subsection raw_compression Raw Compression -WiredTiger's raw compression takes advantage of compressors that provide a -streaming compression API. Using the streaming API WiredTiger will try to fit -as much data as possible into one block. This means that blocks created -with raw compression should be of similar size. Using a streaming compression -method should also make for less overhead in compression, as the setup and -initial work for compressing is done fewer times compared to the amount of -data stored. Algorithms using raw compression include zlib, lz4. - -@subsection to_raw_or_noraw Choosing between Raw and Noraw Compression -When looking at which compression method to use the biggest consideration is -that raw compression will normally provide higher compression levels while -using more CPU for compression. - -An additional consideration is that raw compression may provide a performance -advantage in workloads where data is accessed sequentially. That is because -more data is generally packed into each block on disk. Conversely, noraw -compression may perform better for workloads with random access patterns -because each block will tend to be smaller and require less work to read and -decompress. - -See @ref file_formats_compression for more information on available -compression techniques. - -See @ref compression for information on how to configure and enable compression. - - */ diff --git a/src/third_party/wiredtiger/src/docs/tune-page-size-and-comp.dox b/src/third_party/wiredtiger/src/docs/tune-page-size-and-comp.dox new file mode 100644 index 00000000000..96b0fda2333 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/tune-page-size-and-comp.dox @@ -0,0 +1,426 @@ +/*! @page tune_page_size_and_comp Tuning page size and compression + +This document aims to explain the role played by different page sizes in +WiredTiger. It also details motivation behind an application wanting to modify +these page sizes from their default values and the procedure to do so. +Applications commonly configure page sizes based on their workload's typical key +and value size. Once a page size has been chosen, appropriate defaults for the +other configuration values are derived by WiredTiger from the page sizes, and +relatively few applications will need to modify the other page and key/value +size configuration options. WiredTiger also offers several compression options +that have an impact on the size of the data both in-memory and on-disk. Hence +while selecting page sizes, an application must also look at its desired +compression needs. Since the data and workload for a table differs from one +table to another in the database, an application can choose to set page sizes +and compression options on a per-table basis. + +@section data_life_cycle Data life cycle +Before detailing each page size, here is a review of how data gets stored inside +WiredTiger: + - WiredTiger uses the physical disks to store data durably, creating on-disk +files for the tables in the database directory. It also caches the portion of +the table being currently accessed by the application for reading or writing in +main memory. + - WiredTiger maintains a table's data in memory using a data structure called a +B-Tree ( +B+ Tree to be specific), +referring to the nodes of a B-Tree as pages. Internal pages carry only keys. The +leaf pages store both keys and values. + - The format of the in-memory pages is not the same as the format of the +on-disk pages. Therefore, the in-memory pages regularly go through a process +called reconciliation to create data structures appropriate for storage on the +disk. These data structures are referred to as on-disk pages. An application can +set a maximum size separately for the internal and leaf on-disk pages otherwise +WiredTiger uses a default value. If reconciliation of an in-memory page is +leading to an on-disk page size greater than this maximum, WiredTiger creates +multiple smaller on-disk pages. + - A component of WiredTiger called the Block Manager divides the on-disk pages +into smaller chunks called blocks, which then get written to the disk. The size +of these blocks is defined by a parameter called allocation_size, which is the +underlying unit of allocation for the file the data gets stored in. An +application might choose to have data compressed before it gets stored to disk +by enabling block compression. + - A database's tables are usually much larger than the main memory available. +Not all of the data can be kept in memory at any given time. A process called +eviction takes care of making space for new data by freeing the memory of data +infrequently accessed. An eviction server regularly finds in-memory pages that +have not been accessed in a while (following an LRU algorithm). Several +background eviction threads continuously process these pages, reconcile them to +disk and remove them from the main memory. + - When an application does an insert or an update of a key/value pair, the +associated key is used to refer to an in-memory page. In the case of this page +not being in memory, appropriate on-disk page(s) are read and an in-memory page +constructed (the opposite of reconciliation). A data structure is maintained on +every in-memory page to store any insertions or modifications to the data done +on that page. As more and more data gets written to this page, the page's memory +footprint keeps growing. + - An application can choose to set the maximum size a page is allowed to grow +in-memory. A default size is set by WiredTiger if the application doesn't +specify one. To keep page management efficient, as a page grows larger in-memory +and approaches this maximum size, if possible, it is split into smaller +in-memory pages. + - When doing an insert or an update, if a page grows larger than the maximum, +the application thread is used to forcefully evict this page. This is done to +split the growing page into smaller in-memory pages and reconcile them into +on-disk pages. Once written to the disk they are removed from the main memory, +making space for more data to be written. When an application gets involved in +forced eviction, it might take longer than usual to do these inserts and +updates. It is not always possible to (force) evict a page from memory and this +page can temporarily grow larger in size than the configured maximum. This page +then remains marked to be evicted and reattempts are made as the application +puts more data in it. + +@section configurable_page_struct Configurable page structures in WiredTiger +There are three page sizes that the user can configure: + 1. The maximum page size of any type of in-memory page in the WiredTiger cache, +memory_page_max. + 2. The maximum size of the on-disk page for an internal page, internal_page_max. + 3. The maximum size of the on-disk leaf page, leaf_page_max. + +There are additional configuration settings that tune more esoteric and +specialized data. Those are included for completeness but are rarely changed. + +@subsection memory_page_max memory_page_max +The maximum size a table's page is allowed to grow to in memory before being +reconciled to disk. + - An integer, with acceptable values between 512B and 10TB + - Default size: 5 MB + - Additionally constrained by the condition: + leaf_page_max <= memory_page_max <= cache_size/10 + - Motivation to tune the value: +\n memory_page_max is significant for applications wanting to tune for +consistency in write intensive workloads. + - This is the parameter to start with for tuning and trying different values +to find the correct balance between overall throughput and individual operation +latency for each table. + - Splitting a growing in-memory page into smaller pages and reconciliation +both require exclusive access to the page which makes an application's write +operations wait. Having a large memory_page_max means that the pages will need +to be split and reconciled less often. But when that happens, the duration that +an exclusive access to the page is required is longer, increasing the latency of +an application's insert or update operations. Conversely, having a smaller +memory_page_max reduces the time taken for splitting and reconciling the pages, +but causes it to happen more frequently, forcing more frequent but shorter +exclusive accesses to the pages. + - Applications should choose the memory_page_max value considering the +trade-off between frequency of exclusive access to the pages (for reconciliation +or splitting pages into smaller pages) versus the duration that the exclusive +access is required. + - Configuration: +\n Specified as memory_page_max configuration option to WT_SESSION::create(). An +example of such a configuration string is as follows: + +
+     "key_format=S,value_format=S,memory_page_max=10MB"
+
+ +@subsection internal_page_max internal_page_max +The maximum page size for the reconciled on-disk internal pages of the B-Tree, +in bytes. When an internal page grows past this size, it splits into multiple +pages. + - An integer, with acceptable values between 512B and 512MB + - Default size: 4 KB (*appropriate for applications with relatively small keys) + - Additionally constrained by the condition: the size must be a multiple of the +allocation size + - Motivation to tune the value: +\n internal_page_max is significant for applications wanting to avoid excessive +L2 cache misses while searching the tree. + - Recall that only keys are stored on internal pages, so the type and size of +the key values for a table help drive the setting for this parameter. + - Should be sized to fit into on-chip caches. + - Applications doing full-table scans with out-of-memory workloads might +increase internal_page_max to transfer more data per I/O. + - Influences the shape of the B-Tree, i.e. depth and the number of children +each page in B-Tree has. To iterate to the desired key/value pair in the B-Tree, +WiredTiger has to binary search the key-range in a page to determine the child +page to proceed to and continue down the depth until it reaches the correct leaf +page. Having an unusually deep B-Tree, or having too many children per page can +negatively impact time taken to iterate the B-Tree, slowing down the application. +The number of children per page and, hence, the tree depth depends upon the +number of keys that can be stored in an internal page, which is +internal_page_max divided by key size. Applications should choose an appropriate +internal_page_max size that avoids the B-Tree from getting too deep. + - Configuration: +\n Specified as internal_page_max configuration option to WT_SESSION::create(). +An example of such a configuration string is as follows: + +
+     "key_format=S,value_format=S,internal_page_max=16KB,leaf_page_max=1MB"
+
+ +@subsection leaf_page_max leaf_page_max +The maximum page size for the reconciled on-disk leaf pages of the B-Tree, in +bytes. When a leaf page grows past this size, it splits into multiple pages. + - An integer, with acceptable values between 512B and 512MB + - Default size: 32 KB (*appropriate for applications with relatively small keys +and values) + - Additionally constrained by the condition: must be a multiple of the +allocation size + - Motivation to tune the value: +\n leaf_page_max is significant for applications wanting to maximize sequential +data transfer from a storage device. + - Should be sized to maximize I/O performance (when reading from disk, it is +usually desirable to read a large amount of data, assuming some locality of +reference in the application's access pattern). + - Applications doing full-table scans through out-of-cache workloads might +increase leaf_page_max to transfer more data per I/O. + - Applications focused on read/write amplification might decrease the page +size to better match the underlying storage block size. + - Configuration: +\n Specified as leaf_page_max configuration option to WT_SESSION::create(). An +example of such a configuration string is as follows: + +
+     "key_format=S,value_format=S,internal_page_max=16KB,leaf_page_max=1MB"
+
+ +The following configuration items following are rarely used. They are described +for completeness: + +@subsection allocation_size allocation_size +This is the underlying unit of allocation for the file. As the unit of file +allocation, it sets the minimum page size and how much space is wasted when +storing small amounts of data and overflow items. + - an integer between 512B and 128 MB + - must a power-of-two + - default : 4 KB + - Motivation to tune the value: +\n Most applications should not need to tune the allocation size. + - To be compatible with virtual memory page sizes and direct I/O requirements +on the platform (4KB for most common server platforms) + - Smaller values decrease the file space required by overflow items. + - For example, if the allocation size is set to 4KB, an overflow item of +18,000 bytes requires 5 allocation units and wastes about 2KB of space. If the +allocation size is 16KB, the same overflow item would waste more than 10KB. + - Configuration: +\n Specified as allocation_size configuration option to WT_SESSION::create(). An +example of such a configuration string is as follows: + +
+     "key_format=S,value_format=S,allocation_size=4KB"
+
+ +@subsection key_val_max internal/leaf key/value max + - Overflow items +\n Overflow items are keys and values too large to easily store on a page. Overflow +items are stored separately in the file from the page where the item logically +appears, and so reading or writing an overflow item is more expensive than an +on-page item, normally requiring additional I/O. Additionally, overflow values +are not cached in memory. This means overflow items won't affect the caching +behavior of the application. It also means that each time an overflow value is +read, it is re-read from disk. + - internal_key_max +\n The largest key stored in an internal page, in bytes. If set, keys larger than +the specified size are stored as overflow items. + - The default and the maximum allowed value are both one-tenth the size of a +newly split internal page. + - leaf_key_max +\n The largest key stored in a leaf page, in bytes. If set, keys larger than the +specified size are stored as overflow items. + - The default value is one-tenth the size of a newly split leaf page. + - leaf_value_max +\n The largest value stored in a leaf page, in bytes. If set, values larger than +the specified size are stored as overflow items + - The default is one-half the size of a newly split leaf page. + - If the size is larger than the maximum leaf page size, the page size is +temporarily ignored when large values are written. + - Motivation to tune the values: +\n Most applications should not need to tune the maximum key and value sizes. +Applications requiring a small page size, but also having latency concerns such +that the additional work to retrieve an overflow item may find modifying these +values useful. +\n Since overflow items are separately stored in the on-disk file, aren't cached +and require additional I/O to access (read or write), applications should avoid +creating overflow items. + - Since page sizes also determine the default size of overflow items, i.e., +keys and values too large to easily store on a page, they can be configured to +avoid performance penalties working with overflow items: + - Applications with large keys and values, and concerned with latency, +might increase the page size to avoid creating overflow items, in order to avoid +the additional cost of retrieving them. + - Applications with large keys and values, doing random searches, might +decrease the page size to avoid wasting cache space on overflow items that +aren't likely to be needed. + - Applications with large keys and values, doing table scans, might +increase the page size to avoid creating overflow items, as the overflow items +must be read into memory in all cases, anyway. + - internal_key_max, leaf_key_max and leaf_value_max configuration values +allow applications to change the size at which a key or value will be treated +as an overflow item. + - Most applications should not need to tune the maximum key and value +sizes. + - The value of internal_key_max is relative to the maximum internal page +size. Because the number of keys on an internal page determines the depth of the +tree, the internal_key_max value can only be adjusted within a certain range, +and the configured value will be automatically adjusted by WiredTiger, if +necessary, to ensure a reasonable number of keys fit on an internal page. + - The values of leaf_key_max and leaf_value_max are not relative to the +maximum leaf page size. If either is larger than the maximum page size, the page +size will be ignored when the larger keys and values are being written, and a +larger page will be created as necessary. + - Configuration: +\n Specified as internal_key_max, leaf_key_max and leaf_value_max configuration +options to WT_SESSION::create(). An example of configuration string for a large +leaf overflow value: + +
+     "key_format=S,value_format=S,leaf_page_max=16KB,leaf_value_max=256KB"
+
+ +@subsection split_pct split_pct (split percentage) +The size (specified as percentage of internal/leaf page_max) at which the +reconciled page must be split into multiple smaller pages before being sent for +compression and then be written to the disk. If the reconciled page can fit into +a single on-disk page without the page growing beyond it's set max size, +split_pct is ignored and the page isn't split. + - an integer between 25 and 100 + - default : 75 + - Motivation to tune the value: +\n Most applications should not need to tune the split percentage size. + - This value should be selected to avoid creating a large number of tiny +pages or repeatedly splitting whenever new entries are inserted. +\n For example, if the maximum page size is 1MB, a split_pct value of 10% +would potentially result in creating a large number of 100KB pages, which may +not be optimal for future I/O. Or, if the maximum page size is 1MB, a split_pct +value of 90% would potentially result in repeatedly splitting pages as the split +pages grow to 1MB over and over. The default value for split_pct is 75%, +intended to keep large pages relatively large, while still giving split pages +room to grow. + - Configuration: +\n Specified as split_pct configuration option to WT_SESSION::create(). An +example of such a configuration string is as follows: + +
+     "key_format=S,value_format=S,split_pct=60"
+
+ +@section compression_considerations Compression considerations +WiredTiger compresses data at several stages to preserve memory and disk space. +Applications can configure these different compression algorithms to tailor +their requirements between memory, disk and CPU consumption. Compression +algorithms other than block compression work by modifying how the keys and +values are represented, and hence reduce data size in-memory and on-disk. Block +compression on the other hand compress the data in its binary representation +while saving it on the disk. + +Configuring compression may change application throughput. For example, in +applications using solid-state drives (where I/O is less expensive), turning +off compression may increase application performance by reducing CPU costs; in +applications where I/O costs are more expensive, turning on compression may +increase application performance by reducing the overall number of I/O +operations. + +WiredTiger uses some internal algorithms to compress the amount of data stored +that are not configurable, but always on. For example, run-length reduces the +size requirement by storing sequential, duplicate values in the store only a +single time (with an associated count). + +Different compression options available with WiredTiger: + - Key-prefix + - Reduces the size requirement by storing any identical key prefix only once +per page. The cost is additional CPU and memory when operating on the in-memory +tree. Specifically, reverse sequential cursor movement (but not forward) through +a prefix-compressed page or the random lookup of a key/value pair will allocate +sufficient memory to hold some number of uncompressed keys. So, for example, if +key prefix compression only saves a small number of bytes per key, the +additional memory cost of instantiating the uncompressed key may mean prefix +compression is not worthwhile. Further, in cases where the on-disk cost is the +primary concern, block compression may mean prefix compression is less useful. + - Configuration: +\n Specified as prefix_compression configuration option to +WT_SESSION::create(). Applications may limit the use of prefix compression by +configuring the minimum number of bytes that must be gained before prefix +compression is used with prefix_compression_min configuration option. An example +of such a configuration string is as follows: + +
+          "key_format=S,value_format=S,prefix_compression=true,prefix_compression_min=7"
+
+ + - Dictionary + - Reduces the size requirement by storing any identical value only once per +page. + - Configuration: +\n Specified as dictionary configuration configuration option to +WT_SESSION::create(), which specifies the maximum number of unique values +remembered in the B-Tree row-store leaf page value dictionary. An example of +such a configuration string is as follows: + +
+          "key_format=S,value_format=S,dictionary=1000"
+
+ + - Huffman + - Reduces the size requirement by compressing individual key/value items, and +can be separately configured either or both keys and values. The additional CPU +cost of Huffman encoding can be high, and should be considered. (See Huffman +Encoding for details.) + - Configuration: +\n Specified as huffman_key and/or huffman_value configuration option to +WT_SESSION::create(). These options can take values of "english" (to use a +built-in English language frequency table), "utf8" or "utf16" (to +use a custom utf8 or utf16 symbol frequency table file). An example of such a +configuration string is as follows: + +
+          "key_format=S,value_format=S,huffman_key=english,huffman_value=english"
+
+ + - Block Compression + - Reduces the size requirement of on-disk objects by compressing blocks of +the backing object's file. The additional CPU cost of block compression can be +high, and should be considered. When block compression has been configured, +configured page sizes will not match the actual size of the page on disk. + - WiredTiger provides two methods of compressing your data when using block +compression: the raw and noraw methods. These methods change how WiredTiger +works to fit data into the blocks that are stored on disk. Applications needing +to write specific sized blocks may want to consider implementing a +WT_COMPRESSOR::compress_raw function. + - Noraw compression: +\n A fixed amount of data is given to the compression system, then turned into +a compressed block of data. The amount of data chosen to compress is the data +needed to fill the uncompressed block. Thus when compressed, the block will be +smaller than the normal data size and the sizes written to disk will often vary +depending on how compressible the data being stored is. Algorithms using noraw +compression include zlib-noraw, lz4-noraw and snappy. +Noraw compression is better suited for workloads with random access patterns +because each block will tend to be smaller and require less work to read and +decompress. + - Raw compression: +\n WiredTiger's raw compression takes advantage of compressors that provide a +streaming compression API. Using the streaming API WiredTiger will try to fit as +much data as possible into one block. This means that blocks created with raw +compression should be of similar size. Using a streaming compression method +should also make for less overhead in compression, as the setup and initial work +for compressing is done fewer times compared to the amount of data stored. +Algorithms using raw compression include zlib, lz4. +Compared to noraw, raw compression provides more compression while using more +CPU. Raw compression may provide a performance advantage in workloads where data +is accessed sequentially. That is because more data is generally packed into +each block on disk. + - Configuration: +\n Specified as the block_compressor configuration option to +WT_SESSION::create(). If WiredTiger has builtin support for "lz4", "snappy", +"zlib" or "zstd" compression, these names are available as the value to the +option. An example of such a configuration string is as follows: + +
+          "key_format=S,value_format=S,block_compressor=snappy"
+
+ +See @ref compression for further information on how to configure and enable +different compression options. + +@subsection table_compress Table summarizing compression in WiredTiger + +
+@hrow{Compression Type, Supported by row-store, Supported by variable col-store, + Supported by fixed col-store, Default config, Reduces in-mem size, + Reduces on-disk size, CPU and Memory cost} +@row{Key-prefix, yes, no, no, disabled, yes, yes, minor} +@row{Dictionary, yes, yes, no, disabled, yes, yes, minor} +@row{Huffman, yes, yes, no, disabled, yes, yes, can be high} +@row{Block, yes, yes, yes, disabled, no, yes, can be high} +
+ +*/ diff --git a/src/third_party/wiredtiger/src/docs/tune-page-sizes.dox b/src/third_party/wiredtiger/src/docs/tune-page-sizes.dox deleted file mode 100644 index 130e047a02d..00000000000 --- a/src/third_party/wiredtiger/src/docs/tune-page-sizes.dox +++ /dev/null @@ -1,142 +0,0 @@ -/*! @page tune_page_sizes Page and overflow key/value sizes - -There are seven page and key/value size configuration strings: - -- allocation size (\c allocation_size), -- page sizes (\c internal_page_max and \c leaf_page_max), -- key and value sizes (\c internal_key_max, \c leaf_key_max and \c leaf_value_max), and the -- page-split percentage (\c split_pct). - -All seven are specified to the WT_SESSION::create method, in other -words, they are configurable on a per-file basis. - -Applications commonly configure page sizes, based on their workload's -typical key and value size. Once the correct page size has been chosen, -appropriate defaults for the other configuration values are derived from -the page sizes, and relatively few applications will need to modify the -other page and key/value size configuration options. - -An example of configuring page and key/value sizes: - -@snippet ex_all.c Create a table and configure the page size - -@section tune_page_sizes_sizes Page, key and value sizes - -The \c internal_page_max and \c leaf_page_max configuration values -specify a maximum size for Btree internal and leaf pages. That is, when -an internal or leaf page grows past that size, it splits into multiple -pages. Generally, internal pages should be sized to fit into on-chip -caches in order to minimize cache misses when searching the tree, while -leaf pages should be sized to maximize I/O performance (if reading from -disk is necessary, it is usually desirable to read a large amount of -data, assuming some locality of reference in the application's access -pattern). - -The default page size configurations (2KB for \c internal_page_max, 32KB -for \c leaf_page_max), are appropriate for applications with relatively -small keys and values. - -- Applications doing full-table scans through out-of-memory workloads -might increase both internal and leaf page sizes to transfer more data -per I/O. -- Applications focused on read/write amplification might decrease the page -size to better match the underlying storage block size. - -When block compression has been configured, configured page sizes will -not match the actual size of the page on disk. Block compression in -WiredTiger happens within the I/O subsystem, and so a page might split -even if subsequent compression would result in a resulting page size -small enough to leave as a single page. In other words, page sizes are -based on in-memory sizes, not on-disk sizes. Applications needing to -write specific sized blocks may want to consider implementing a -WT_COMPRESSOR::compress_raw function. - -The page sizes also determine the default size of overflow items, that -is, keys and values too large to easily store on a page. Overflow items -are stored separately in the file from the page where the item logically -appears, and so reading or writing an overflow item is more expensive -than an on-page item, normally requiring additional I/O. Additionally, -overflow values are not cached in memory. This means overflow items -won't affect the caching behavior of the application, but it also means -that each time an overflow value is read, it is re-read from disk. - -For both of these reasons, applications should avoid creating large -numbers of commonly referenced overflow items. This is especially -important for keys, as keys on internal pages are referenced during -random searches, not just during data retrieval. Generally, -applications should make every attempt to avoid creating overflow keys. - -- Applications with large keys and values, and concerned with latency, -might increase the page size to avoid creating overflow items, in order -to avoid the additional cost of retrieving them. - -- Applications with large keys and values, doing random searches, might -decrease the page size to avoid wasting cache space on overflow items -that aren't likely to be needed. - -- Applications with large keys and values, doing table scans, might -increase the page size to avoid creating overflow items, as the overflow -items must be read into memory in all cases, anyway. - -The \c internal_key_max, \c leaf_key_max and \c leaf_value_max -configuration values allow applications to change the size at which a -key or value will be treated as an overflow item. - -The value of \c internal_key_max is relative to the maximum internal -page size. Because the number of keys on an internal page determines -the depth of the tree, the \c internal_key_max value can only be -adjusted within a certain range, and the configured value will be -automatically adjusted by WiredTiger, if necessary to ensure a -reasonable number of keys fit on an internal page. - -The values of \c leaf_key_max and \c leaf_value_max are not relative to -the maximum leaf page size. If either is larger than the maximum page -size, the page size will be ignored when the larger keys and values are -being written, and a larger page will be created as necessary. - -Most applications should not need to tune the maximum key and value -sizes. Applications requiring a small page size, but also having -latency concerns such that the additional work to retrieve an overflow -item is an issue, may find them useful. - -An example of configuring a large leaf overflow value: - -@snippet ex_all.c Create a table and configure a large leaf value max - -@section tune_page_sizes_split_percentage Split percentage - -The \c split_pct configuration string configures the size of a split -page. When a page grows sufficiently large that it must be written as -multiple disk blocks, the newly written block size is \c split_pct -percent of the maximum page size. This value should be selected to -avoid creating a large number of tiny pages or repeatedly splitting -whenever new entries are inserted. For example, if the maximum page -size is 1MB, a \c split_pct value of 10% would potentially result in -creating a large number of 100KB pages, which may not be optimal for -future I/O. Or, if the maximum page size is 1MB, a \c split_pct value -of 90% would potentially result in repeatedly splitting pages as the -split pages grow to 1MB over and over. The default value for \c -split_pct is 75%, intended to keep large pages relatively large, while -still giving split pages room to grow. - -Most applications should not need to tune the split percentage size. - -@section tune_page_sizes_allocation_size Allocation size - -The \c allocation_size configuration value is the underlying unit of -allocation for the file. As the unit of file allocation, it sets the -minimum page size and how much space is wasted when storing small -amounts of data and overflow items. For example, if the allocation size -is set to 4KB, an overflow item of 18,000 bytes requires 5 allocation -units and wastes about 2KB of space. If the allocation size is 16KB, -the same overflow item would waste more than 10KB. - -The default allocation size is 4KB, chosen for compatibility with -virtual memory page sizes and direct I/O requirements on common server -platforms. - -Most applications should not need to tune the allocation size; it is -primarily intended for applications coping with the specific -requirements some file systems make to support features like direct I/O. - -*/ diff --git a/src/third_party/wiredtiger/src/docs/upgrading.dox b/src/third_party/wiredtiger/src/docs/upgrading.dox index caa6b036892..e5fce3d0d5d 100644 --- a/src/third_party/wiredtiger/src/docs/upgrading.dox +++ b/src/third_party/wiredtiger/src/docs/upgrading.dox @@ -1,7 +1,75 @@ /*! @page upgrading Upgrading WiredTiger applications -@section version_281 Upgrading to Version 2.8.1 +@section version_292 Upgrading to Version 2.9.2

+ +
WiredTiger utility now supports truncate
+
+The WiredTiger utility \c wt can now \c truncate objects, removing all +contents from the specified object. +
+ +
Handle list lock statistics
+
+In the 2.9.1 release we added statistics tracking handle list lock timing, we +have switched that lock from a spin lock to a read-write lock, and consequently +changed the statistics tracking lock related wait time. +
+ +
Forced and named checkpoint error conditions changed
+
+There are new cases where checkpoints created with an explicit name or the +"force" configuration option can return an EBUSY error. This can happen if +the checkpoint overlaps with other schema operations, for example table create. +
+ +
WT_CURSOR::remove may not return a positioned cursor
+
+The WT_CURSOR::remove method was previously documented to always return a +positioned cursor on success, which is not possible when \c overwrite=true +and the record does not exist. + +The documentation has been updated, and the method has been changed to +never return a cursor position unless called with an existing cursor +position. In other words, if the cursor is positioned and the +WT_CURSOR::remove is called, the cursor will remain positioned; if the +cursor is not positioned and the WT_CURSOR::remove method is called, the +cursor will not be positioned on return. +
+ +
+@section version_291 Upgrading to Version 2.9.1 +
+ +
Changes to hazard pointer configuration
+
+The \c hazard_max parameter to ::wiredtiger_open is now ignored. Memory is +allocated for hazard pointers as required by each session. +
+ +
Change to the default fadvise behavior for data files
+
+The old default behavior was to advise the file system that access would be +random for data files, and there was no way to alter that. We no longer +call advise the file system of expected access patterns by default, and +have added a new \c access_pattern_hint configuration option available for +WT_SESSION::create that can be used to restore the old default by setting +the value to "random". +
+ +
+@section version_290 Upgrading to Version 2.9.0 +
+ +
Changes to cursor behavior after WT_CURSOR::insert
+
+After a successful call to WT_CURSOR::insert, unless a cursor has record +number keys and was configured with "append=true", the cursor's key is +cleared and a subsequent call to the WT_CURSOR::get_key method will fail. +In all cursors, the value is cleared and a subsequent call to +WT_CURSOR::get_value will fail. +
+
Cache management defaults
The default values for the \c eviction_dirty_target and \c @@ -17,6 +85,15 @@ how much work is done at the beginning of a checkpoint to make the critical section of checkpoints complete more quickly.
+
Change to default fadvise setting for data files
+
+The default behavior for data files was to advise the file system to optimize +for random access on POSIX and Windows platforms. The default is now to not +advise about access patterns. There is a new \c access_pattern_hint +configuration string available to WT_SESSION::create that can be used +to configure the old default behavior. +
+
Checkpoint server created checkpoint names
The ::wiredtiger_open checkpoint configuration no longer supports the @@ -34,13 +111,13 @@ of the path may no longer be specified. Applications depending on the ability to set statistics log file names will require modification.
-
Deprecated statistics field
+
Removed and renamed statistics fields
-The connection statistic \c WT_STAT_CONN_CACHE_BYTES_OVERFLOW has been -removed. Overflow information is now available in the -\c WT_STAT_CONN_CACHE_BYTES_OVERFLOW and \c WT_STAT_CONN_CACHE_OVERFLOW_VALUE. -Applications specifically looking for that statistic will require -modification. +WT_STAT_JOIN_ACTUAL_COUNT is renamed to WT_STAT_JOIN_ITERATED + +WT_STAT_CONN_CACHE_BYTES_OVERFLOW removed since overflow pages are not cached + +WT_STAT_CONN_CACHE_EVICTION_SERVER_NOT_EVICTING removed since all eviction threads can now be the server
@@ -281,7 +358,7 @@ be updated. The WT_SESSION::create \c internal_item_max and \c leaf_item_max configuration strings are now deprecated in favor of the \c internal_key_max, \c leaf_key_max, and \c leaf_value_max -configuration strings. See @ref tune_page_sizes for more information. +configuration strings. See @ref tune_page_size_and_comp for more information.
diff --git a/src/third_party/wiredtiger/src/docs/wtperf.dox b/src/third_party/wiredtiger/src/docs/wtperf.dox index 83aadf8a776..6bdcf5f4f8d 100644 --- a/src/third_party/wiredtiger/src/docs/wtperf.dox +++ b/src/third_party/wiredtiger/src/docs/wtperf.dox @@ -167,6 +167,8 @@ do population phase; false to use existing database number of WiredTiger databases to use. Each database will execute the workload using a separate home directory and complete set of worker threads @par drop_tables (boolean, default=false) Whether to drop all tables at the end of the run, and report time taken to do the drop. +@par in_memory (boolean, default=false) +Whether to create the database in-memory. @par icount (unsigned int, default=5000) number of records to initially populate. If multiple tables are configured the count is spread evenly across all tables. @par idle_table_cycle (unsigned int, default=0) @@ -195,14 +197,14 @@ use pareto distribution for random numbers. Zero to disable, otherwise a percen number of operations to group into each transaction in the populate phase, zero for auto-commit @par populate_threads (unsigned int, default=1) number of populate threads, 1 for bulk load +@par pre_load_data (boolean, default=false) +Scan all data prior to starting the workload phase to warm the cache @par random_range (unsigned int, default=0) if non zero choose a value from within this range as the key for insert operations @par random_value (boolean, default=false) generate random content for the value @par range_partition (boolean, default=false) partition data by range (vs hash) -@par read_range (unsigned int, default=0) -scan a range of keys after each search @par readonly (boolean, default=false) reopen the connection between populate and workload phases in readonly mode. Requires reopen_connection turned on (default). Requires that read be the only workload specified @par reopen_connection (boolean, default=true) @@ -228,7 +230,7 @@ number of tables to run operations over. Keys are divided evenly over the table @par table_count_idle (unsigned int, default=0) number of tables to create, that won't be populated. Default 0. @par threads (string, default="") -workload configuration: each 'count' entry is the total number of threads, and the 'insert', 'read' and 'update' entries are the ratios of insert, read and update operations done by each worker thread; If a throttle value is provided each thread will do a maximum of that number of operations per second; multiple workload configurations may be specified per threads configuration; for example, a more complex threads configuration might be 'threads=((count=2,reads=1)(count=8,reads=1,inserts=2,updates=1))' which would create 2 threads doing nothing but reads and 8 threads each doing 50% inserts and 25% reads and updates. Allowed configuration values are 'count', 'throttle', 'update_delta', 'reads', 'inserts', 'updates', 'truncate', 'truncate_pct' and 'truncate_count'. There are also behavior modifiers, supported modifiers are 'ops_per_txn' +workload configuration: each 'count' entry is the total number of threads, and the 'insert', 'read' and 'update' entries are the ratios of insert, read and update operations done by each worker thread; If a throttle value is provided each thread will do a maximum of that number of operations per second; multiple workload configurations may be specified per threads configuration; for example, a more complex threads configuration might be 'threads=((count=2,reads=1)(count=8,reads=1,inserts=2,updates=1))' which would create 2 threads doing nothing but reads and 8 threads each doing 50% inserts and 25% reads and updates. Allowed configuration values are 'count', 'throttle', 'update_delta', 'reads', 'read_range', 'inserts', 'updates', 'truncate', 'truncate_pct' and 'truncate_count'. There are also behavior modifiers, supported modifiers are 'ops_per_txn' @par transaction_config (string, default="") WT_SESSION.begin_transaction configuration string, applied during the populate phase when populate_ops_per_txn is nonzero @par table_name (string, default="test") diff --git a/src/third_party/wiredtiger/src/evict/evict_file.c b/src/third_party/wiredtiger/src/evict/evict_file.c index b0cd50cc655..3d8f4a61ca7 100644 --- a/src/third_party/wiredtiger/src/evict/evict_file.c +++ b/src/third_party/wiredtiger/src/evict/evict_file.c @@ -15,15 +15,27 @@ int __wt_evict_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) { + WT_BTREE *btree; WT_DECL_RET; WT_PAGE *page; WT_REF *next_ref, *ref; + btree = S2BT(session); + /* - * We need exclusive access to the file -- disable ordinary eviction - * and drain any blocks already queued. + * We need exclusive access to the file, we're about to discard the root + * page. Assert eviction has been locked out. */ - WT_RET(__wt_evict_file_exclusive_on(session)); + WT_ASSERT(session, + btree->evict_disabled > 0 || + !F_ISSET(session->dhandle, WT_DHANDLE_OPEN)); + + /* + * We do discard objects without pages in memory. If that's the case, + * we're done. + */ + if (btree->root.page == NULL) + return (0); /* Make sure the oldest transaction ID is up-to-date. */ WT_RET(__wt_txn_update_oldest( @@ -57,7 +69,8 @@ __wt_evict_file(WT_SESSION_IMPL *session, WT_CACHE_OP syncop) * error, retrying later. */ if (syncop == WT_SYNC_CLOSE && __wt_page_is_modified(page)) - WT_ERR(__wt_reconcile(session, ref, NULL, WT_EVICTING)); + WT_ERR(__wt_reconcile( + session, ref, NULL, WT_EVICTING, NULL)); /* * We can't evict the page just returned to us (it marks our @@ -101,7 +114,5 @@ err: /* On error, clear any left-over tree walk. */ session, next_ref, WT_READ_NO_EVICT)); } - __wt_evict_file_exclusive_off(session); - return (ret); } diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 909fe813f98..26bbf9f679b 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -15,6 +15,7 @@ static int __evict_lru_walk(WT_SESSION_IMPL *); static int __evict_page(WT_SESSION_IMPL *, bool); static int __evict_pass(WT_SESSION_IMPL *); static int __evict_server(WT_SESSION_IMPL *, bool *); +static int __evict_tune_workers(WT_SESSION_IMPL *session); static int __evict_walk(WT_SESSION_IMPL *, WT_EVICT_QUEUE *); static int __evict_walk_file( WT_SESSION_IMPL *, WT_EVICT_QUEUE *, u_int, u_int *); @@ -22,6 +23,60 @@ static int __evict_walk_file( #define WT_EVICT_HAS_WORKERS(s) \ (S2C(s)->evict_threads.current_threads > 1) +/* + * __evict_lock_handle_list -- + * Try to get the handle list lock, with yield and sleep back off. + * Keep timing statistics overall. + */ +static int +__evict_lock_handle_list(WT_SESSION_IMPL *session) +{ + struct timespec enter, leave; + WT_CACHE *cache; + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + WT_RWLOCK *dh_lock; + u_int spins; + bool dh_stats; + + conn = S2C(session); + cache = conn->cache; + dh_lock = &conn->dhandle_lock; + + /* + * Setup tracking of handle lock acquisition wait time if statistics + * are enabled. + */ + dh_stats = WT_STAT_ENABLED(session); + + if (dh_stats) + __wt_epoch(session, &enter); + + /* + * Use a custom lock acquisition back off loop so the eviction server + * notices any interrupt quickly. + */ + for (spins = 0; + (ret = __wt_try_readlock(session, dh_lock)) == EBUSY && + cache->pass_intr == 0; spins++) { + if (spins < WT_THOUSAND) + __wt_yield(); + else + __wt_sleep(0, WT_THOUSAND); + } + /* + * Only record statistics on success. + */ + WT_RET(ret); + if (dh_stats) { + __wt_epoch(session, &leave); + WT_STAT_CONN_INCRV( + session, lock_handle_list_wait_eviction, + (int64_t)WT_TIMEDIFF_US(leave, enter)); + } + return (0); +} + /* * __evict_entry_priority -- * Get the adjusted read generation for an eviction entry. @@ -143,8 +198,7 @@ __wt_evict_list_clear_page(WT_SESSION_IMPL *session, WT_REF *ref) } __wt_spin_unlock(session, &cache->evict_queues[q].evict_lock); } - WT_ASSERT(session, - !F_ISSET_ATOMIC(ref->page, WT_PAGE_EVICT_LRU)); + WT_ASSERT(session, !F_ISSET_ATOMIC(ref->page, WT_PAGE_EVICT_LRU)); __wt_spin_unlock(session, &cache->evict_queue_lock); } @@ -213,7 +267,7 @@ __wt_evict_server_wake(WT_SESSION_IMPL *session) } #endif - __wt_cond_auto_signal(session, cache->evict_cond); + __wt_cond_signal(session, cache->evict_cond); } /* @@ -226,12 +280,12 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread) WT_CACHE *cache; WT_CONNECTION_IMPL *conn; WT_DECL_RET; - bool did_work; + bool did_work, was_intr; conn = S2C(session); cache = conn->cache; -#ifdef HAVE_DIAGNOSTIC +#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) /* * Ensure the cache stuck timer is initialized when starting eviction. */ @@ -254,12 +308,28 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread) ret = __evict_server(session, &did_work); F_CLR(cache->walk_session, WT_SESSION_LOCKED_PASS); F_CLR(session, WT_SESSION_LOCKED_PASS); + was_intr = cache->pass_intr != 0; __wt_spin_unlock(session, &cache->evict_pass_lock); WT_ERR(ret); + + /* + * If the eviction server was interrupted, wait until + * requests have been processed: the system may + * otherwise be busy so don't go to sleep. + */ + if (was_intr) { + while (cache->pass_intr != 0 && + F_ISSET(conn, WT_CONN_EVICTION_RUN) && + F_ISSET(thread, WT_THREAD_RUN)) + __wt_yield(); + continue; + } + __wt_verbose(session, WT_VERB_EVICTSERVER, "sleeping"); + /* Don't rely on signals: check periodically. */ __wt_cond_auto_wait( - session, cache->evict_cond, did_work); + session, cache->evict_cond, did_work, NULL); __wt_verbose(session, WT_VERB_EVICTSERVER, "waking"); } else WT_ERR(__evict_lru_pages(session, false)); @@ -271,7 +341,7 @@ __wt_evict_thread_run(WT_SESSION_IMPL *session, WT_THREAD *thread) * can be closed. */ if (thread->id == 0) { - WT_WITH_PASS_LOCK(session, ret, + WT_WITH_PASS_LOCK(session, ret = __evict_clear_all_walks(session)); WT_ERR(ret); /* @@ -299,14 +369,13 @@ err: WT_PANIC_MSG(session, ret, "cache eviction thread error"); static int __evict_server(WT_SESSION_IMPL *session, bool *did_work) { +#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) + struct timespec now; +#endif WT_CACHE *cache; WT_CONNECTION_IMPL *conn; WT_DECL_RET; -#ifdef HAVE_DIAGNOSTIC - struct timespec now; -#endif uint64_t orig_pages_evicted; - u_int spins; conn = S2C(session); cache = conn->cache; @@ -317,7 +386,8 @@ __evict_server(WT_SESSION_IMPL *session, bool *did_work) /* Evict pages from the cache as needed. */ WT_RET(__evict_pass(session)); - if (!F_ISSET(conn, WT_CONN_EVICTION_RUN)) + if (!F_ISSET(conn, WT_CONN_EVICTION_RUN) || + cache->pass_intr != 0) return (0); /* @@ -325,41 +395,55 @@ __evict_server(WT_SESSION_IMPL *session, bool *did_work) * otherwise we can block applications evicting large pages. */ if (!__wt_cache_stuck(session)) { - for (spins = 0; (ret = __wt_spin_trylock( - session, &conn->dhandle_lock)) == EBUSY && - cache->pass_intr == 0; spins++) { - if (spins < WT_THOUSAND) - __wt_yield(); - else - __wt_sleep(0, WT_THOUSAND); - } /* - * If we gave up acquiring the lock, that indicates a - * session is waiting for us to clear walks. Do that - * as part of a normal pass (without the handle list + * Try to get the handle list lock: if we give up, that + * indicates a session is waiting for us to clear walks. Do + * that as part of a normal pass (without the handle list * lock) to avoid deadlock. */ - if (ret == EBUSY) + if ((ret = __evict_lock_handle_list(session)) == EBUSY) return (0); WT_RET(ret); ret = __evict_clear_all_walks(session); - __wt_spin_unlock(session, &conn->dhandle_lock); + __wt_readunlock(session, &conn->dhandle_lock); WT_RET(ret); cache->pages_evicted = 0; } else if (cache->pages_evicted != cache->pages_evict) { cache->pages_evicted = cache->pages_evict; -#ifdef HAVE_DIAGNOSTIC +#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) __wt_epoch(session, &cache->stuck_ts); - } else { - /* After being stuck for 5 minutes, give up. */ + } else if (!F_ISSET(conn, WT_CONN_IN_MEMORY)) { + /* + * If we're stuck for 5 minutes in diagnostic mode, or the + * verbose evict_stuck flag is configured, log the cache + * and transaction state. + * + * If we're stuck for 5 minutes in diagnostic mode, give up. + * + * We don't do this check for in-memory workloads because + * application threads are not blocked by the cache being full. + * If the cache becomes full of clean pages, we can be + * servicing reads while the cache appears stuck to eviction. + */ __wt_epoch(session, &now); if (WT_TIMEDIFF_SEC(now, cache->stuck_ts) > 300) { - ret = ETIMEDOUT; - __wt_err(session, ret, +#if defined(HAVE_DIAGNOSTIC) + __wt_err(session, ETIMEDOUT, "Cache stuck for too long, giving up"); - WT_TRET(__wt_cache_dump(session, NULL)); + ret = ETIMEDOUT; + WT_TRET(__wt_verbose_dump_txn(session)); + WT_TRET(__wt_verbose_dump_cache(session)); return (ret); +#elif defined(HAVE_VERBOSE) + if (WT_VERBOSE_ISSET(session, WT_VERB_EVICT_STUCK)) { + WT_RET(__wt_verbose_dump_txn(session)); + WT_RET(__wt_verbose_dump_cache(session)); + + /* Reset the timer. */ + __wt_epoch(session, &cache->stuck_ts); + } +#endif } #endif } @@ -382,11 +466,13 @@ __wt_evict_create(WT_SESSION_IMPL *session) /* Set first, the thread might run before we finish up. */ F_SET(conn, WT_CONN_EVICTION_RUN); - /* Create the eviction thread group */ + /* + * Create the eviction thread group. + * Set the group size to the maximum allowed sessions. + */ WT_RET(__wt_thread_group_create(session, &conn->evict_threads, - "eviction-server", conn->evict_threads_min, - conn->evict_threads_max, WT_THREAD_CAN_WAIT | WT_THREAD_PANIC_FAIL, - __wt_evict_thread_run)); + "eviction-server", conn->evict_threads_min, conn->evict_threads_max, + WT_THREAD_CAN_WAIT | WT_THREAD_PANIC_FAIL, __wt_evict_thread_run)); /* * Allow queues to be populated now that the eviction threads @@ -413,7 +499,7 @@ __wt_evict_destroy(WT_SESSION_IMPL *session) return (0); /* Wait for any eviction thread group changes to stabilize. */ - __wt_writelock(session, conn->evict_threads.lock); + __wt_writelock(session, &conn->evict_threads.lock); /* * Signal the threads to finish and stop populating the queue. @@ -541,6 +627,8 @@ __evict_pass(WT_SESSION_IMPL *session) if (loop == 0) prev = now; + if (conn->evict_threads.threads[0]->session == session) + WT_RET(__evict_tune_workers(session)); /* * Increment the shared read generation. Do this occasionally * even if eviction is not currently required, so that pages @@ -566,14 +654,6 @@ __evict_pass(WT_SESSION_IMPL *session) if (!__evict_update_work(session)) break; - /* - * Try to start a new thread if we have capacity and haven't - * reached the eviction targets. - */ - if (F_ISSET(cache, WT_CACHE_EVICT_ALL)) - WT_RET(__wt_thread_group_start_one( - session, &conn->evict_threads, false)); - __wt_verbose(session, WT_VERB_EVICTSERVER, "Eviction pass with: Max: %" PRIu64 " In use: %" PRIu64 " Dirty: %" PRIu64, @@ -648,8 +728,8 @@ __evict_pass(WT_SESSION_IMPL *session) */ WT_STAT_CONN_INCR(session, cache_eviction_server_slept); - __wt_cond_wait( - session, cache->evict_cond, WT_THOUSAND); + __wt_cond_wait(session, + cache->evict_cond, WT_THOUSAND, NULL); continue; } @@ -676,7 +756,7 @@ __evict_pass(WT_SESSION_IMPL *session) * Clear a single walk point. */ static int -__evict_clear_walk(WT_SESSION_IMPL *session, bool count_stat) +__evict_clear_walk(WT_SESSION_IMPL *session) { WT_BTREE *btree; WT_CACHE *cache; @@ -693,14 +773,14 @@ __evict_clear_walk(WT_SESSION_IMPL *session, bool count_stat) if ((ref = btree->evict_ref) == NULL) return (0); - if (count_stat) - WT_STAT_CONN_INCR(session, cache_eviction_walks_abandoned); + WT_STAT_CONN_INCR(session, cache_eviction_walks_abandoned); /* - * Clear evict_ref first, in case releasing it forces eviction (we - * assert we never try to evict the current eviction walk point). + * Clear evict_ref before releasing it in case that forces eviction (we + * assert that we never try to evict the current eviction walk point). */ btree->evict_ref = NULL; + WT_WITH_DHANDLE(cache->walk_session, session->dhandle, (ret = __wt_page_release(cache->walk_session, ref, WT_READ_NO_EVICT))); @@ -723,7 +803,7 @@ __evict_clear_all_walks(WT_SESSION_IMPL *session) TAILQ_FOREACH(dhandle, &conn->dhqh, q) if (WT_PREFIX_MATCH(dhandle->name, "file:")) WT_WITH_DHANDLE(session, dhandle, - WT_TRET(__evict_clear_walk(session, true))); + WT_TRET(__evict_clear_walk(session))); return (ret); } @@ -744,31 +824,19 @@ __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session) btree = S2BT(session); cache = S2C(session)->cache; - /* - * Hold the walk lock to set the no-eviction flag. - * - * The no-eviction flag can be set permanently, in which case we never - * increment the no-eviction count. - */ + /* Hold the walk lock to turn off eviction. */ __wt_spin_lock(session, &cache->evict_walk_lock); - if (F_ISSET(btree, WT_BTREE_NO_EVICTION)) { - if (btree->evict_disabled != 0) - ++btree->evict_disabled; + if (++btree->evict_disabled > 1) { __wt_spin_unlock(session, &cache->evict_walk_lock); return (0); } - ++btree->evict_disabled; /* * Ensure no new pages from the file will be queued for eviction after - * this point. + * this point, then clear any existing LRU eviction walk for the file. */ - F_SET(btree, WT_BTREE_NO_EVICTION); (void)__wt_atomic_addv32(&cache->pass_intr, 1); - - /* Clear any existing LRU eviction walk for the file. */ - WT_WITH_PASS_LOCK(session, ret, - ret = __evict_clear_walk(session, true)); + WT_WITH_PASS_LOCK(session, ret = __evict_clear_walk(session)); (void)__wt_atomic_subv32(&cache->pass_intr, 1); WT_ERR(ret); @@ -799,7 +867,6 @@ __wt_evict_file_exclusive_on(WT_SESSION_IMPL *session) if (0) { err: --btree->evict_disabled; - F_CLR(btree, WT_BTREE_NO_EVICTION); } __wt_spin_unlock(session, &cache->evict_walk_lock); return (ret); @@ -824,17 +891,233 @@ __wt_evict_file_exclusive_off(WT_SESSION_IMPL *session) */ WT_DIAGNOSTIC_YIELD; + /* Hold the walk lock to turn on eviction. */ + __wt_spin_lock(session, &cache->evict_walk_lock); WT_ASSERT(session, - btree->evict_ref == NULL && F_ISSET(btree, WT_BTREE_NO_EVICTION)); + btree->evict_ref == NULL && btree->evict_disabled > 0); + --btree->evict_disabled; + __wt_spin_unlock(session, &cache->evict_walk_lock); +} + +#define EVICT_TUNE_BATCH 1 /* Max workers to add each period */ +/* + * Data points needed before deciding if we should keep adding workers or settle + * on an earlier value. + */ +#define EVICT_TUNE_DATAPT_MIN 3 +#define EVICT_TUNE_PERIOD 1 /* Tune period in seconds */ + +/* + * We will do a fresh re-tune every that many seconds to adjust to + * significant phase changes. + */ +#define EVICT_FORCE_RETUNE 30 + +/* + * __evict_tune_workers -- + * Find the right number of eviction workers. Gradually ramp up the number of + * workers increasing the number in batches indicated by the setting above. + * Store the number of workers that gave us the best throughput so far and the + * number of data points we have tried. + * + * Every once in a while when we have the minimum number of data points we check + * whether the eviction throughput achieved with the current number of workers + * is the best we have seen so far. If so, we will keep increasing the number of + * workers. If not, we are past the infliction point on the eviction throughput + * curve. In that case, we will set the number of workers to the best observed + * so far and settle into a stable state. + */ +static int +__evict_tune_workers(WT_SESSION_IMPL *session) +{ + struct timespec current_time; + WT_CACHE *cache; + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + uint64_t delta_msec, delta_pages; + uint64_t pgs_evicted_cur, pgs_evicted_persec_cur, time_diff; + int32_t cur_threads, i, target_threads, thread_surplus; + + conn = S2C(session); + cache = conn->cache; + + WT_ASSERT(session, conn->evict_threads.threads[0]->session == session); + pgs_evicted_cur = pgs_evicted_persec_cur = 0; + + __wt_epoch(session, ¤t_time); + time_diff = WT_TIMEDIFF_SEC(current_time, conn->evict_tune_last_time); /* - * The no-eviction flag can be set permanently, in which case we never - * increment the no-eviction count. + * If we have reached the stable state and have not run long enough to + * surpass the forced re-tuning threshold, return. */ - __wt_spin_lock(session, &cache->evict_walk_lock); - if (btree->evict_disabled > 0 && --btree->evict_disabled == 0) - F_CLR(btree, WT_BTREE_NO_EVICTION); - __wt_spin_unlock(session, &cache->evict_walk_lock); + if (conn->evict_tune_stable) { + if (time_diff < EVICT_FORCE_RETUNE) + return (0); + + /* + * Stable state was reached a long time ago. Let's re-tune. + * Reset all the state. + */ + conn->evict_tune_stable = 0; + conn->evict_tune_last_action_time.tv_sec = 0; + conn->evict_tune_pgs_last = 0; + conn->evict_tune_num_points = 0; + conn->evict_tune_pg_sec_max = 0; + conn->evict_tune_workers_best = 0; + + /* Reduce the number of eviction workers to the minimum */ + thread_surplus = + (int32_t)conn->evict_threads.current_threads - + (int32_t)conn->evict_threads_min; + + for (i = 0; i < thread_surplus; i++) { + WT_ERR(__wt_thread_group_stop_one( + session, &conn->evict_threads, false)); + WT_STAT_CONN_INCR(session, + cache_eviction_worker_removed); + } + WT_STAT_CONN_INCR(session, cache_eviction_force_retune); + } else + if (time_diff < EVICT_TUNE_PERIOD) + /* + * If we have not reached stable state, don't do + * anything unless enough time has passed since the last + * time we have taken any action in this function. + */ + return (0); + + /* + * Measure the number of evicted pages so far. Eviction rate correlates + * to performance, so this is our metric of success. + */ + pgs_evicted_cur = cache->pages_evict; + + /* + * If we have recorded the number of pages evicted at the end of + * the previous measurement interval, we can compute the eviction + * rate in evicted pages per second achieved during the current + * measurement interval. + * Otherwise, we just record the number of evicted pages and return. + */ + if (conn->evict_tune_pgs_last == 0) + goto err; + + delta_msec = WT_TIMEDIFF_MS(current_time, conn->evict_tune_last_time); + delta_pages = pgs_evicted_cur - conn->evict_tune_pgs_last; + pgs_evicted_persec_cur = (delta_pages * WT_THOUSAND) / delta_msec; + conn->evict_tune_num_points++; + + /* + * Keep track of the maximum eviction throughput seen and the number + * of workers corresponding to that throughput. + */ + if (pgs_evicted_persec_cur > conn->evict_tune_pg_sec_max) { + conn->evict_tune_pg_sec_max = pgs_evicted_persec_cur; + conn->evict_tune_workers_best = + conn->evict_threads.current_threads; + } + + /* + * Compare the current number of data points with the number + * needed variable. If they are equal, we will check whether + * we are still going up on the performance curve, in which + * case we will continue increasing the number of workers, or + * we are past the inflection point on the curve, in which case + * we will go back to the best observed number of workers and + * settle into a stable state. + */ + if (conn->evict_tune_num_points >= conn->evict_tune_datapts_needed) { + if (conn->evict_tune_workers_best == + conn->evict_threads.current_threads && + conn->evict_threads.current_threads < + conn->evict_threads_max) { + /* + * Keep adding workers. We will check again + * at the next check point. + */ + conn->evict_tune_datapts_needed += WT_MIN( + EVICT_TUNE_DATAPT_MIN, + (conn->evict_threads_max - + conn->evict_threads.current_threads) / + EVICT_TUNE_BATCH); + } else { + /* + * We are past the inflection point. Choose the + * best number of eviction workers observed and + * settle into a stable state. + */ + thread_surplus = + (int32_t)conn->evict_threads.current_threads - + (int32_t)conn->evict_tune_workers_best; + + for (i = 0; i < thread_surplus; i++) { + /* + * If we get an error, it should be because we + * were unable to acquire the thread group lock. + * Break out of trying. + */ + WT_ERR(__wt_thread_group_stop_one( + session, &conn->evict_threads, false)); + WT_STAT_CONN_INCR(session, + cache_eviction_worker_removed); + } + WT_STAT_CONN_SET(session, + cache_eviction_stable_state_workers, + conn->evict_tune_workers_best); + conn->evict_tune_stable = true; + WT_STAT_CONN_SET(session, cache_eviction_active_workers, + conn->evict_threads.current_threads); + goto err; + } + } + + /* + * If we have not added any worker threads in the past, we set the + * number needed equal to the number of data points that we must + * accumulate before deciding if we should keep adding workers or settle + * on a previously tried value of workers. + */ + if (conn->evict_tune_last_action_time.tv_sec == 0) + conn->evict_tune_datapts_needed = WT_MIN(EVICT_TUNE_DATAPT_MIN, + (conn->evict_threads_max - + conn->evict_threads.current_threads) / EVICT_TUNE_BATCH); + + if (F_ISSET(cache, WT_CACHE_EVICT_ALL)) { + cur_threads = (int32_t)conn->evict_threads.current_threads; + target_threads = WT_MIN(cur_threads + EVICT_TUNE_BATCH, + (int32_t)conn->evict_threads_max); + /* + * Start the new threads. + */ + for (i = cur_threads; i < target_threads; ++i) { + /* + * If we get an error, it should be because we were + * unable to acquire the thread group lock. Break out + * of trying. + */ + WT_ERR(__wt_thread_group_start_one(session, + &conn->evict_threads, false)); + WT_STAT_CONN_INCR(session, + cache_eviction_worker_created); + __wt_verbose(session, WT_VERB_EVICTSERVER, + "added worker thread"); + } + conn->evict_tune_last_action_time = current_time; + } + + WT_STAT_CONN_SET(session, cache_eviction_active_workers, + conn->evict_threads.current_threads); + +err: conn->evict_tune_last_time = current_time; + conn->evict_tune_pgs_last = pgs_evicted_cur; + /* + * If we got an EBUSY trying to acquire the lock just return. + * We can try to tune the workers next time. + */ + if (ret == EBUSY) + ret = 0; + return (ret); } /* @@ -860,7 +1143,8 @@ __evict_lru_pages(WT_SESSION_IMPL *session, bool is_server) /* If a worker thread found the queue empty, pause. */ if (ret == WT_NOTFOUND && !is_server && F_ISSET(S2C(session), WT_CONN_EVICTION_RUN)) - __wt_cond_wait(session, conn->evict_threads.wait_cond, 10000); + __wt_cond_wait( + session, conn->evict_threads.wait_cond, 10000, NULL); return (ret == WT_NOTFOUND ? 0 : ret); } @@ -1039,7 +1323,7 @@ __evict_walk(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue) WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *dhandle; WT_DECL_RET; - u_int max_entries, retries, slot, spins, start_slot, total_candidates; + u_int max_entries, retries, slot, start_slot, total_candidates; bool dhandle_locked, incr; conn = S2C(session); @@ -1077,16 +1361,7 @@ retry: while (slot < max_entries) { * reference count to keep it alive while we sweep. */ if (!dhandle_locked) { - for (spins = 0; (ret = __wt_spin_trylock( - session, &conn->dhandle_lock)) == EBUSY && - cache->pass_intr == 0; - spins++) { - if (spins < WT_THOUSAND) - __wt_yield(); - else - __wt_sleep(0, WT_THOUSAND); - } - WT_ERR(ret); + WT_ERR(__evict_lock_handle_list(session)); dhandle_locked = true; } @@ -1122,7 +1397,7 @@ retry: while (slot < max_entries) { /* Skip files that don't allow eviction. */ btree = dhandle->handle; - if (F_ISSET(btree, WT_BTREE_NO_EVICTION)) + if (btree->evict_disabled > 0) continue; /* @@ -1141,9 +1416,17 @@ retry: while (slot < max_entries) { !__wt_cache_aggressive(session)) continue; - /* Skip files if we have used all available hazard pointers. */ - if (btree->evict_ref == NULL && session->nhazard >= - conn->hazard_max - WT_MIN(conn->hazard_max / 2, 10)) + /* + * Skip files if we have too many active walks. + * + * This used to be limited by the configured maximum number of + * hazard pointers per session. Even though that ceiling has + * been removed, we need to test eviction with huge numbers of + * active trees before allowing larger numbers of hazard + * pointers in the walk session. + */ + if (btree->evict_ref == NULL && + session->nhazard > WT_EVICT_MAX_TREES) continue; /* @@ -1157,7 +1440,7 @@ retry: while (slot < max_entries) { (void)__wt_atomic_addi32(&dhandle->session_inuse, 1); incr = true; - __wt_spin_unlock(session, &conn->dhandle_lock); + __wt_readunlock(session, &conn->dhandle_lock); dhandle_locked = false; /* @@ -1170,13 +1453,23 @@ retry: while (slot < max_entries) { * the tree's current eviction point, and part of the process is * waiting on this thread to acknowledge that action. */ - if (!F_ISSET(btree, WT_BTREE_NO_EVICTION) && + if (btree->evict_disabled == 0 && !__wt_spin_trylock(session, &cache->evict_walk_lock)) { - if (!F_ISSET(btree, WT_BTREE_NO_EVICTION)) { + if (btree->evict_disabled == 0) { + /* + * Assert the handle has a root page: eviction + * should have been locked out if the tree is + * being discarded or the root page is changing. + * As this has not always been the case, assert + * to debug that change. + */ + WT_ASSERT(session, btree->root.page != NULL); + cache->evict_file_next = dhandle; - WT_WITH_DHANDLE(session, dhandle, ret = - __evict_walk_file(session, queue, - max_entries, &slot)); + WT_WITH_DHANDLE(session, dhandle, + ret = __evict_walk_file( + session, queue, max_entries, &slot)); + WT_ASSERT(session, session->split_gen == 0); } __wt_spin_unlock(session, &cache->evict_walk_lock); @@ -1204,7 +1497,7 @@ retry: while (slot < max_entries) { } err: if (dhandle_locked) { - __wt_spin_unlock(session, &conn->dhandle_lock); + __wt_readunlock(session, &conn->dhandle_lock); dhandle_locked = false; } @@ -1267,8 +1560,8 @@ __evict_push_candidate(WT_SESSION_IMPL *session, * Get a few page eviction candidates from a single underlying file. */ static int -__evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, - u_int max_entries, u_int *slotp) +__evict_walk_file(WT_SESSION_IMPL *session, + WT_EVICT_QUEUE *queue, u_int max_entries, u_int *slotp) { WT_BTREE *btree; WT_CACHE *cache; @@ -1300,6 +1593,19 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, start = queue->evict_queue + *slotp; remaining_slots = max_entries - *slotp; total_slots = max_entries - queue->evict_entries; + btree_inuse = cache_inuse = 0; + target_pages_clean = target_pages_dirty = 0; + + /* + * The number of times we should fill the queue by the end of + * considering all trees. + */ +#define QUEUE_FILLS_PER_PASS 10 + + /* + * The minimum number of pages we should consider per tree. + */ +#define MIN_PAGES_PER_TREE 10 /* * The target number of pages for this tree is proportional to the @@ -1308,13 +1614,12 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, * cache (and only have to walk it once). */ if (F_ISSET(cache, WT_CACHE_EVICT_CLEAN)) { - btree_inuse = __wt_btree_bytes_inuse(session); + btree_inuse = __wt_btree_bytes_evictable(session); cache_inuse = __wt_cache_bytes_inuse(cache); bytes_per_slot = 1 + cache_inuse / total_slots; target_pages_clean = (uint32_t)( (btree_inuse + bytes_per_slot / 2) / bytes_per_slot); - } else - target_pages_clean = 0; + } if (F_ISSET(cache, WT_CACHE_EVICT_DIRTY)) { btree_inuse = __wt_btree_dirty_leaf_inuse(session); @@ -1322,35 +1627,58 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, bytes_per_slot = 1 + cache_inuse / total_slots; target_pages_dirty = (uint32_t)( (btree_inuse + bytes_per_slot / 2) / bytes_per_slot); - } else - target_pages_dirty = 0; + } - target_pages = WT_MAX(target_pages_clean, target_pages_dirty); + /* + * Weight the number of target pages by the number of times we want to + * fill the cache per pass through all the trees. Note that we don't + * build this into the calculation above because we don't want to favor + * small trees, so round to a whole number of slots (zero for small + * trees) before multiplying. + */ + target_pages = WT_MAX(target_pages_clean, target_pages_dirty) * + QUEUE_FILLS_PER_PASS; + /* + * Randomly walk trees with a small fraction of the cache in case there + * are so many trees that none of them use enough of the cache to be + * allocated slots. + * + * The chance of walking a tree is equal to the chance that a random + * byte in cache belongs to the tree, weighted by how many times we + * want to fill queues during a pass through all the trees in cache. + */ if (target_pages == 0) { - /* - * Randomly walk trees with a tiny fraction of the cache in - * case there are so many trees that none of them use enough of - * the cache to be allocated slots. Walk small trees 1% of the - * time. - */ - if (__wt_random(&session->rnd) > UINT32_MAX / 100) + if (F_ISSET(cache, WT_CACHE_EVICT_CLEAN)) { + btree_inuse = __wt_btree_bytes_evictable(session); + cache_inuse = __wt_cache_bytes_inuse(cache); + } else { + btree_inuse = __wt_btree_dirty_leaf_inuse(session); + cache_inuse = __wt_cache_dirty_leaf_inuse(cache); + } + if (btree_inuse == 0 || cache_inuse == 0) + return (0); + if (__wt_random64(&session->rnd) % cache_inuse > + btree_inuse * QUEUE_FILLS_PER_PASS) return (0); - target_pages = 10; } + /* + * There is some cost associated with walking a tree. If we're going + * to visit this tree, always look for a minimum number of pages. + */ + if (target_pages < MIN_PAGES_PER_TREE) + target_pages = MIN_PAGES_PER_TREE; + + /* + * If the tree is dead or we're near the end of the queue, fill the + * remaining slots. + */ if (F_ISSET(session->dhandle, WT_DHANDLE_DEAD) || target_pages > remaining_slots) target_pages = remaining_slots; end = start + target_pages; - walk_flags = - WT_READ_CACHE | WT_READ_NO_EVICT | WT_READ_NO_GEN | WT_READ_NO_WAIT; - - /* Randomize the walk direction. */ - if (btree->evict_walk_reverse) - FLD_SET(walk_flags, WT_READ_PREV); - /* * Examine at least a reasonable number of pages before deciding * whether to give up. When we are only looking for dirty pages, @@ -1361,9 +1689,44 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, !F_ISSET(cache, WT_CACHE_EVICT_CLEAN)) min_pages *= 10; + walk_flags = + WT_READ_CACHE | WT_READ_NO_EVICT | WT_READ_NO_GEN | WT_READ_NO_WAIT; + + /* + * Choose a random point in the tree if looking for candidates in a + * tree with no starting point set. This is mostly aimed at ensuring + * eviction fairly visits all pages in trees with a lot of in-cache + * content. + */ + switch ((WT_EVICT_WALK_START)btree->evict_start_type) { + case WT_EVICT_WALK_NEXT: + break; + case WT_EVICT_WALK_PREV: + FLD_SET(walk_flags, WT_READ_PREV); + break; + case WT_EVICT_WALK_RAND_PREV: + FLD_SET(walk_flags, WT_READ_PREV); + /* FALLTHROUGH */ + case WT_EVICT_WALK_RAND_NEXT: + if (btree->evict_ref == NULL) { + /* Ensure internal pages indexes remain valid */ + WT_WITH_PAGE_INDEX(session, ret = __wt_random_descent( + session, &btree->evict_ref, true)); + WT_RET_NOTFOUND_OK(ret); + } + break; + } + + /* + * Get some more eviction candidate pages, starting at the last saved + * point. Clear the saved point immediately, we assert when discarding + * pages we're not discarding an eviction point, so this clear must be + * complete before the page is released. + */ + ref = btree->evict_ref; + btree->evict_ref = NULL; + /* - * Get some more eviction candidate pages. - * * !!! Take care terminating this loop. * * Don't make an extra call to __wt_tree_walk after we hit the end of a @@ -1376,7 +1739,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, for (evict = start, pages_queued = pages_seen = refs_walked = 0; evict < end && (ret == 0 || ret == WT_NOTFOUND); ret = __wt_tree_walk_count( - session, &btree->evict_ref, &refs_walked, walk_flags)) { + session, &ref, &refs_walked, walk_flags)) { /* * Check whether we're finding a good ratio of candidates vs * pages seen. Some workloads create "deserts" in trees where @@ -1387,10 +1750,18 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, pages_seen > min_pages && (pages_queued == 0 || (pages_seen / pages_queued) > (min_pages / target_pages)); - if (give_up) + if (give_up) { + /* + * Try a different walk start point next time if a + * walk gave up. + */ + btree->evict_start_type = + (btree->evict_start_type + 1) % + WT_EVICT_WALK_START_NUM; break; + } - if ((ref = btree->evict_ref) == NULL) { + if (ref == NULL) { if (++restarts == 2) break; WT_STAT_CONN_INCR( @@ -1424,7 +1795,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, if (page->read_gen == WT_READGEN_NOTSET) __wt_cache_read_gen_new(session, page); - /* Pages we no longer need (clean or dirty), are found money. */ + /* Pages being forcibly evicted go on the urgent queue. */ if (page->read_gen == WT_READGEN_OLDEST || page->memory_footprint >= btree->splitmempage) { WT_STAT_CONN_INCR( @@ -1434,7 +1805,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, continue; } - /* Pages that are empty or from dead trees are also good. */ + /* Pages that are empty or from dead trees are fast-tracked. */ if (__wt_page_is_empty(page) || F_ISSET(session->dhandle, WT_DHANDLE_DEAD)) goto fast; @@ -1449,7 +1820,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, /* Limit internal pages to 50% of the total. */ if (WT_PAGE_IS_INTERNAL(page) && - internal_pages >= (int)(evict - start) / 2) + internal_pages > (int)(evict - start) / 2) continue; /* If eviction gets aggressive, anything else is fair game. */ @@ -1480,7 +1851,7 @@ fast: /* If the page can't be evicted, give up. */ ++pages_queued; if (WT_PAGE_IS_INTERNAL(page)) - ++internal_pages; + ++internal_pages; __wt_verbose(session, WT_VERB_EVICTSERVER, "select: %p, size %" WT_SIZET_FMT, @@ -1493,18 +1864,20 @@ fast: /* If the page can't be evicted, give up. */ session, cache_eviction_pages_queued, (u_int)(evict - start)); /* - * If we didn't find any candidates in the file, reverse the direction - * of the walk and skip it next time. + * If we couldn't find the number of pages we were looking for, skip + * the tree next time. */ - if (give_up) - btree->evict_walk_reverse = !btree->evict_walk_reverse; - if (give_up && !urgent_queued) + if (pages_queued < target_pages / 2 && !urgent_queued) btree->evict_walk_period = WT_MIN( WT_MAX(1, 2 * btree->evict_walk_period), 100); else if (pages_queued == target_pages) btree->evict_walk_period = 0; + else if (btree->evict_walk_period > 0) + btree->evict_walk_period /= 2; /* + * Give up the walk occasionally. + * * If we happen to end up on the root page or a page requiring urgent * eviction, clear it. We have to track hazard pointers, and the root * page complicates that calculation. @@ -1516,16 +1889,20 @@ fast: /* If the page can't be evicted, give up. */ * If we land on a page requiring forced eviction, move on to the next * page: we want this page evicted as quickly as possible. */ - if ((ref = btree->evict_ref) != NULL) { - /* Give up the walk occasionally. */ + if (ref != NULL) { if (__wt_ref_is_root(ref) || evict == start || give_up || ref->page->read_gen == WT_READGEN_OLDEST || - ref->page->memory_footprint >= btree->splitmempage) - WT_RET(__evict_clear_walk(session, restarts == 0)); - else if (ref->page->read_gen == WT_READGEN_OLDEST) + ref->page->memory_footprint >= btree->splitmempage) { + if (restarts == 0) + WT_STAT_CONN_INCR( + session, cache_eviction_walks_abandoned); + WT_RET(__wt_page_release(cache->walk_session, + ref, WT_READ_NO_EVICT)); + ref = NULL; + } else if (ref->page->read_gen == WT_READGEN_OLDEST) WT_RET_NOTFOUND_OK(__wt_tree_walk_count( - session, &btree->evict_ref, - &refs_walked, walk_flags)); + session, &ref, &refs_walked, walk_flags)); + btree->evict_ref = ref; } WT_STAT_CONN_INCRV(session, cache_eviction_walk, refs_walked); @@ -1782,6 +2159,7 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, u_int pct_full) WT_TXN_GLOBAL *txn_global; WT_TXN_STATE *txn_state; uint64_t init_evict_count, max_pages_evicted; + bool timer; conn = S2C(session); cache = conn->cache; @@ -1802,7 +2180,9 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, u_int pct_full) __wt_evict_server_wake(session); /* Track how long application threads spend doing eviction. */ - if (WT_STAT_ENABLED(session) && !F_ISSET(session, WT_SESSION_INTERNAL)) + timer = + WT_STAT_ENABLED(session) && !F_ISSET(session, WT_SESSION_INTERNAL); + if (timer) __wt_epoch(session, &enter); for (init_evict_count = cache->pages_evict;; ret = 0) { @@ -1859,8 +2239,8 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, u_int pct_full) break; case WT_NOTFOUND: /* Allow the queue to re-populate before retrying. */ - __wt_cond_wait( - session, conn->evict_threads.wait_cond, 10000); + __wt_cond_wait(session, + conn->evict_threads.wait_cond, 10000, NULL); cache->app_waits++; break; default: @@ -1868,8 +2248,7 @@ __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, u_int pct_full) } } -err: if (WT_STAT_ENABLED(session) && - !F_ISSET(session, WT_SESSION_INTERNAL)) { +err: if (timer) { __wt_epoch(session, &leave); WT_STAT_CONN_INCRV(session, application_cache_time, WT_TIMEDIFF_US(leave, enter)); @@ -1897,7 +2276,7 @@ __wt_page_evict_urgent(WT_SESSION_IMPL *session, WT_REF *ref) page = ref->page; if (F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU) || - F_ISSET(S2BT(session), WT_BTREE_NO_EVICTION)) + S2BT(session)->evict_disabled > 0) return (false); /* Append to the urgent queue if we can. */ @@ -1907,7 +2286,7 @@ __wt_page_evict_urgent(WT_SESSION_IMPL *session, WT_REF *ref) __wt_spin_lock(session, &cache->evict_queue_lock); if (F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU) || - F_ISSET(S2BT(session), WT_BTREE_NO_EVICTION)) + S2BT(session)->evict_disabled > 0) goto done; __wt_spin_lock(session, &urgent_queue->evict_lock); @@ -1956,125 +2335,140 @@ __wt_evict_priority_clear(WT_SESSION_IMPL *session) S2BT(session)->evict_priority = 0; } -#ifdef HAVE_DIAGNOSTIC +#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) /* - * __wt_cache_dump -- - * Dump debugging information to a file (default stderr) about the size of - * the files in the cache. + * __verbose_dump_cache_single -- + * Output diagnostic information about a single file in the cache. */ -int -__wt_cache_dump(WT_SESSION_IMPL *session, const char *ofile) +static int +__verbose_dump_cache_single(WT_SESSION_IMPL *session, + uint64_t *total_bytesp, uint64_t *total_dirty_bytesp) { - FILE *fp; - WT_CONNECTION_IMPL *conn; - WT_DATA_HANDLE *dhandle, *saved_dhandle; + WT_DATA_HANDLE *dhandle; WT_PAGE *page; WT_REF *next_walk; + size_t size; uint64_t intl_bytes, intl_bytes_max, intl_dirty_bytes; uint64_t intl_dirty_bytes_max, intl_dirty_pages, intl_pages; uint64_t leaf_bytes, leaf_bytes_max, leaf_dirty_bytes; uint64_t leaf_dirty_bytes_max, leaf_dirty_pages, leaf_pages; + + intl_bytes = intl_bytes_max = intl_dirty_bytes = 0; + intl_dirty_bytes_max = intl_dirty_pages = intl_pages = 0; + leaf_bytes = leaf_bytes_max = leaf_dirty_bytes = 0; + leaf_dirty_bytes_max = leaf_dirty_pages = leaf_pages = 0; + + next_walk = NULL; + while (__wt_tree_walk(session, &next_walk, + WT_READ_CACHE | WT_READ_NO_EVICT | WT_READ_NO_WAIT) == 0 && + next_walk != NULL) { + page = next_walk->page; + size = page->memory_footprint; + + if (WT_PAGE_IS_INTERNAL(page)) { + ++intl_pages; + intl_bytes += size; + intl_bytes_max = WT_MAX(intl_bytes_max, size); + if (__wt_page_is_modified(page)) { + ++intl_dirty_pages; + intl_dirty_bytes += size; + intl_dirty_bytes_max = + WT_MAX(intl_dirty_bytes_max, size); + } + } else { + ++leaf_pages; + leaf_bytes += size; + leaf_bytes_max = WT_MAX(leaf_bytes_max, size); + if (__wt_page_is_modified(page)) { + ++leaf_dirty_pages; + leaf_dirty_bytes += size; + leaf_dirty_bytes_max = + WT_MAX(leaf_dirty_bytes_max, size); + } + } + } + + dhandle = session->dhandle; + if (dhandle->checkpoint == NULL) + WT_RET(__wt_msg(session, "%s():", dhandle->name)); + else + WT_RET(__wt_msg(session, "%s(checkpoint=%s):", + dhandle->name, dhandle->checkpoint)); + if (intl_pages != 0) + WT_RET(__wt_msg(session, + "internal: " + "%" PRIu64 " pages, " + "%" PRIu64 "MB, " + "%" PRIu64 "/%" PRIu64 " clean/dirty pages, " + "%" PRIu64 "/%" PRIu64 " clean/dirty MB, " + "%" PRIu64 "MB max page, " + "%" PRIu64 "MB max dirty page", + intl_pages, + intl_bytes / WT_MEGABYTE, + intl_pages - intl_dirty_pages, + intl_dirty_pages, + (intl_bytes - intl_dirty_bytes) / WT_MEGABYTE, + intl_dirty_bytes / WT_MEGABYTE, + intl_bytes_max / WT_MEGABYTE, + intl_dirty_bytes_max / WT_MEGABYTE)); + if (leaf_pages != 0) + WT_RET(__wt_msg(session, + "leaf: " + "%" PRIu64 " pages, " + "%" PRIu64 "MB, " + "%" PRIu64 "/%" PRIu64 " clean/dirty pages, " + "%" PRIu64 "/%" PRIu64 " clean/dirty MB, " + "%" PRIu64 "MB max page, " + "%" PRIu64 "MB max dirty page", + leaf_pages, + leaf_bytes / WT_MEGABYTE, + leaf_pages - leaf_dirty_pages, + leaf_dirty_pages, + (leaf_bytes - leaf_dirty_bytes) / WT_MEGABYTE, + leaf_dirty_bytes / WT_MEGABYTE, + leaf_bytes_max / WT_MEGABYTE, + leaf_dirty_bytes_max / WT_MEGABYTE)); + + *total_bytesp += intl_bytes + leaf_bytes; + *total_dirty_bytesp += intl_dirty_bytes + leaf_dirty_bytes; + + return (0); +} + +/* + * __wt_verbose_dump_cache -- + * Output diagnostic information about the cache. + */ +int +__wt_verbose_dump_cache(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_DATA_HANDLE *dhandle; + WT_DECL_RET; uint64_t total_bytes, total_dirty_bytes; - size_t size; conn = S2C(session); total_bytes = total_dirty_bytes = 0; - if (ofile == NULL) - fp = stderr; - else if ((fp = fopen(ofile, "w")) == NULL) - return (EIO); + WT_RET(__wt_msg(session, "%s", WT_DIVIDER)); + WT_RET(__wt_msg(session, "cache dump")); - /* Note: odd string concatenation avoids spelling errors. */ - (void)fprintf(fp, "==========\n" "cache dump\n"); - - saved_dhandle = session->dhandle; - TAILQ_FOREACH(dhandle, &conn->dhqh, q) { + for (dhandle = NULL;;) { + WT_WITH_HANDLE_LIST_READ_LOCK(session, + WT_DHANDLE_NEXT(session, dhandle, &conn->dhqh, q)); + if (dhandle == NULL) + break; if (!WT_PREFIX_MATCH(dhandle->name, "file:") || !F_ISSET(dhandle, WT_DHANDLE_OPEN)) continue; - intl_bytes = intl_bytes_max = intl_dirty_bytes = 0; - intl_dirty_bytes_max = intl_dirty_pages = intl_pages = 0; - leaf_bytes = leaf_bytes_max = leaf_dirty_bytes = 0; - leaf_dirty_bytes_max = leaf_dirty_pages = leaf_pages = 0; - - next_walk = NULL; - session->dhandle = dhandle; - while (__wt_tree_walk(session, &next_walk, - WT_READ_CACHE | WT_READ_NO_EVICT | WT_READ_NO_WAIT) == 0 && - next_walk != NULL) { - page = next_walk->page; - size = page->memory_footprint; - - if (WT_PAGE_IS_INTERNAL(page)) { - ++intl_pages; - intl_bytes += size; - intl_bytes_max = WT_MAX(intl_bytes_max, size); - if (__wt_page_is_modified(page)) { - ++intl_dirty_pages; - intl_dirty_bytes += size; - intl_dirty_bytes_max = - WT_MAX(intl_dirty_bytes_max, size); - } - } else { - ++leaf_pages; - leaf_bytes += size; - leaf_bytes_max = WT_MAX(leaf_bytes_max, size); - if (__wt_page_is_modified(page)) { - ++leaf_dirty_pages; - leaf_dirty_bytes += size; - leaf_dirty_bytes_max = - WT_MAX(leaf_dirty_bytes_max, size); - } - } - } - session->dhandle = NULL; - - if (dhandle->checkpoint == NULL) - (void)fprintf(fp, "%s(): \n", dhandle->name); - else - (void)fprintf(fp, "%s(checkpoint=%s): \n", - dhandle->name, dhandle->checkpoint); - if (intl_pages != 0) - (void)fprintf(fp, - "\t" "internal: " - "%" PRIu64 " pages, " - "%" PRIu64 "MB, " - "%" PRIu64 "/%" PRIu64 " clean/dirty pages, " - "%" PRIu64 "/%" PRIu64 " clean/dirty MB, " - "%" PRIu64 "MB max page, " - "%" PRIu64 "MB max dirty page\n", - intl_pages, - intl_bytes >> 20, - intl_pages - intl_dirty_pages, - intl_dirty_pages, - (intl_bytes - intl_dirty_bytes) >> 20, - intl_dirty_bytes >> 20, - intl_bytes_max >> 20, - intl_dirty_bytes_max >> 20); - if (leaf_pages != 0) - (void)fprintf(fp, - "\t" "leaf: " - "%" PRIu64 " pages, " - "%" PRIu64 "MB, " - "%" PRIu64 "/%" PRIu64 " clean/dirty pages, " - "%" PRIu64 "/%" PRIu64 " clean/dirty MB, " - "%" PRIu64 "MB max page, " - "%" PRIu64 "MB max dirty page\n", - leaf_pages, - leaf_bytes >> 20, - leaf_pages - leaf_dirty_pages, - leaf_dirty_pages, - (leaf_bytes - leaf_dirty_bytes) >> 20, - leaf_dirty_bytes >> 20, - leaf_bytes_max >> 20, - leaf_dirty_bytes_max >> 20); - - total_bytes += intl_bytes + leaf_bytes; - total_dirty_bytes += intl_dirty_bytes + leaf_dirty_bytes; + WT_WITH_DHANDLE(session, dhandle, + ret = __verbose_dump_cache_single( + session, &total_bytes, &total_dirty_bytes)); + if (ret != 0) + break; } - session->dhandle = saved_dhandle; + WT_RET(ret); /* * Apply the overhead percentage so our total bytes are comparable with @@ -2082,16 +2476,16 @@ __wt_cache_dump(WT_SESSION_IMPL *session, const char *ofile) */ total_bytes = __wt_cache_bytes_plus_overhead(conn->cache, total_bytes); - (void)fprintf(fp, + WT_RET(__wt_msg(session, "cache dump: " - "total found = %" PRIu64 "MB vs tracked inuse %" PRIu64 "MB\n" - "total dirty bytes = %" PRIu64 "MB\n", - total_bytes >> 20, __wt_cache_bytes_inuse(conn->cache) >> 20, - total_dirty_bytes >> 20); - (void)fprintf(fp, "==========\n"); - - if (ofile != NULL && fclose(fp) != 0) - return (EIO); + "total found: %" PRIu64 "MB vs tracked inuse %" PRIu64 "MB", + total_bytes / WT_MEGABYTE, + __wt_cache_bytes_inuse(conn->cache) / WT_MEGABYTE)); + WT_RET(__wt_msg(session, + "total dirty bytes: %" PRIu64 "MB", + total_dirty_bytes / WT_MEGABYTE)); + WT_RET(__wt_msg(session, "%s", WT_DIVIDER)); + return (0); } #endif diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index 3d1557e027e..85689efd0b1 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -37,7 +37,7 @@ __evict_exclusive(WT_SESSION_IMPL *session, WT_REF *ref) * Check for a hazard pointer indicating another thread is using the * page, meaning the page cannot be evicted. */ - if (__wt_page_hazard_check(session, ref->page) == NULL) + if (__wt_hazard_check(session, ref) == NULL) return (0); WT_STAT_DATA_INCR(session, cache_eviction_hazard); @@ -66,7 +66,7 @@ __wt_page_release_evict(WT_SESSION_IMPL *session, WT_REF *ref) * between. */ locked = __wt_atomic_casv32(&ref->state, WT_REF_MEM, WT_REF_LOCKED); - if ((ret = __wt_hazard_clear(session, page)) != 0 || !locked) { + if ((ret = __wt_hazard_clear(session, ref)) != 0 || !locked) { if (locked) ref->state = WT_REF_MEM; return (ret == 0 ? EBUSY : ret); @@ -396,7 +396,7 @@ __evict_review( WT_DECL_RET; WT_PAGE *page; uint32_t flags; - bool modified; + bool lookaside_retry, modified; flags = WT_EVICTING; *flagsp = flags; @@ -480,8 +480,8 @@ __evict_review( if (LF_ISSET(WT_EVICT_INMEM_SPLIT)) return (__wt_split_insert(session, ref)); - /* We are done if reconciliation is disabled. */ - if (F_ISSET(S2BT(session), WT_BTREE_NO_RECONCILE)) + /* If splits are the only permitted operation, we're done. */ + if (F_ISSET(S2BT(session), WT_BTREE_ALLOW_SPLITS)) return (EBUSY); } @@ -495,27 +495,29 @@ __evict_review( * If we have an exclusive lock (we're discarding the tree), assert * there are no updates we cannot read. * - * Don't set any other flags for internal pages: they don't have update - * lists to be saved and restored, nor can we re-create them in memory. + * Don't set any other flags for internal pages: there are no update + * lists to be saved and restored, changes can't be written into the + * lookaside table, nor can we re-create internal pages in memory. * * For leaf pages: * - * If an in-memory configuration or the page is being forcibly evicted, - * set the update-restore flag, so reconciliation will write blocks it + * In-memory pages are a known configuration. + * + * Set the update/restore flag, so reconciliation will write blocks it * can write and create a list of skipped updates for blocks it cannot - * write, along with disk images. This is how eviction of active, huge + * write, along with disk images. This is how eviction of active, huge * pages works: we take a big page and reconcile it into blocks, some of * which we write and discard, the rest of which we re-create as smaller * in-memory pages, (restoring the updates that stopped us from writing - * the block), and inserting the whole mess into the page's parent. - * - * Otherwise, if eviction is getting pressed, configure reconciliation - * to write not-yet-globally-visible updates to the lookaside table, - * allowing the eviction of pages we'd otherwise have to retain in cache - * to support older readers. + * the block), and inserting the whole mess into the page's parent. Set + * the flag in all cases because the incremental cost of update/restore + * in reconciliation is minimal, eviction shouldn't have picked a page + * where update/restore is necessary, absent some cache pressure. It's + * possible updates occurred after we selected this page for eviction, + * but it's unlikely and we don't try and manage that risk. * - * Finally, if we don't need to do eviction at the moment, create disk - * images of split pages in order to re-instantiate them. + * Additionally, if we aren't trying to free space in the cache, scrub + * the page and keep it in memory. */ cache = S2C(session)->cache; if (closing) @@ -524,25 +526,33 @@ __evict_review( if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) LF_SET(WT_EVICT_IN_MEMORY | WT_EVICT_SCRUB | WT_EVICT_UPDATE_RESTORE); - else if (__wt_cache_stuck(session)) - LF_SET(WT_EVICT_LOOKASIDE); - else if (!__wt_txn_visible_all( - session, page->modify->update_txn) || - page->read_gen == WT_READGEN_OLDEST || - page->memory_footprint >= S2BT(session)->splitmempage) + else { LF_SET(WT_EVICT_UPDATE_RESTORE); - /* - * If we aren't trying to free space in the cache, scrub the - * page and keep it around. - */ - if (!LF_ISSET(WT_EVICT_LOOKASIDE) && - F_ISSET(cache, WT_CACHE_EVICT_SCRUB)) - LF_SET(WT_EVICT_SCRUB); + if (F_ISSET(cache, WT_CACHE_EVICT_SCRUB)) + LF_SET(WT_EVICT_SCRUB); + } } - *flagsp = flags; - WT_RET(__wt_reconcile(session, ref, NULL, flags)); + /* Reconcile the page. */ + ret = __wt_reconcile(session, ref, NULL, flags, &lookaside_retry); + + /* + * If reconciliation fails, eviction is stuck and reconciliation reports + * it might succeed if we use the lookaside table (the page didn't have + * uncommitted updates, it was not-yet-globally visible updates causing + * the problem), configure reconciliation to write those updates to the + * lookaside table, allowing the eviction of pages we'd otherwise have + * to retain in cache to support older readers. + */ + if (ret == EBUSY && __wt_cache_stuck(session) && lookaside_retry) { + LF_CLR(WT_EVICT_SCRUB | WT_EVICT_UPDATE_RESTORE); + LF_SET(WT_EVICT_LOOKASIDE); + ret = __wt_reconcile(session, ref, NULL, flags, NULL); + } + + *flagsp = flags; + WT_RET(ret); /* * Success: assert the page is clean or reconciliation was configured diff --git a/src/third_party/wiredtiger/src/evict/evict_stat.c b/src/third_party/wiredtiger/src/evict/evict_stat.c index 2dd3b1e83a0..7c2d5722a63 100644 --- a/src/third_party/wiredtiger/src/evict/evict_stat.c +++ b/src/third_party/wiredtiger/src/evict/evict_stat.c @@ -134,5 +134,5 @@ __wt_curstat_cache_walk(WT_SESSION_IMPL *session) WT_STAT_DATA_SET(session, cache_state_root_size, btree->root.page->memory_footprint); - WT_WITH_HANDLE_LIST_LOCK(session, __evict_stat_walk(session)); + __evict_stat_walk(session); } diff --git a/src/third_party/wiredtiger/src/include/api.h b/src/third_party/wiredtiger/src/include/api.h index 2783d17f825..a3636eb8040 100644 --- a/src/third_party/wiredtiger/src/include/api.h +++ b/src/third_party/wiredtiger/src/include/api.h @@ -7,22 +7,21 @@ */ /* Standard entry points to the API: declares/initializes local variables. */ -#define API_SESSION_INIT(s, h, n, cur, dh) \ +#define API_SESSION_INIT(s, h, n, dh) \ WT_DATA_HANDLE *__olddh = (s)->dhandle; \ const char *__oldname = (s)->name; \ - (s)->cursor = (cur); \ (s)->dhandle = (dh); \ (s)->name = (s)->lastop = #h "." #n; \ -#define API_CALL_NOCONF(s, h, n, cur, dh) do { \ - API_SESSION_INIT(s, h, n, cur, dh); \ +#define API_CALL_NOCONF(s, h, n, dh) do { \ + API_SESSION_INIT(s, h, n, dh); \ WT_ERR(WT_SESSION_CHECK_PANIC(s)); \ __wt_verbose((s), WT_VERB_API, "CALL: " #h ":" #n) -#define API_CALL(s, h, n, cur, dh, config, cfg) do { \ - const char *cfg[] = \ +#define API_CALL(s, h, n, dh, config, cfg) do { \ + const char *(cfg)[] = \ { WT_CONFIG_BASE(s, h##_##n), config, NULL }; \ - API_SESSION_INIT(s, h, n, cur, dh); \ + API_SESSION_INIT(s, h, n, dh); \ WT_ERR(WT_SESSION_CHECK_PANIC(s)); \ if ((config) != NULL) \ WT_ERR(__wt_config_check((s), \ @@ -42,17 +41,17 @@ } while (0) /* An API call wrapped in a transaction if necessary. */ -#define TXN_API_CALL(s, h, n, cur, bt, config, cfg) do { \ +#define TXN_API_CALL(s, h, n, bt, config, cfg) do { \ bool __autotxn = false; \ - API_CALL(s, h, n, bt, cur, config, cfg); \ + API_CALL(s, h, n, bt, config, cfg); \ __autotxn = !F_ISSET(&(s)->txn, WT_TXN_AUTOCOMMIT | WT_TXN_RUNNING);\ if (__autotxn) \ F_SET(&(s)->txn, WT_TXN_AUTOCOMMIT) /* An API call wrapped in a transaction if necessary. */ -#define TXN_API_CALL_NOCONF(s, h, n, cur, bt) do { \ +#define TXN_API_CALL_NOCONF(s, h, n, bt) do { \ bool __autotxn = false; \ - API_CALL_NOCONF(s, h, n, cur, bt); \ + API_CALL_NOCONF(s, h, n, bt); \ __autotxn = !F_ISSET(&(s)->txn, WT_TXN_AUTOCOMMIT | WT_TXN_RUNNING);\ if (__autotxn) \ F_SET(&(s)->txn, WT_TXN_AUTOCOMMIT) @@ -63,15 +62,16 @@ if (__autotxn) { \ if (F_ISSET(&(s)->txn, WT_TXN_AUTOCOMMIT)) \ F_CLR(&(s)->txn, WT_TXN_AUTOCOMMIT); \ - else if (ret == 0 && !F_ISSET(&(s)->txn, WT_TXN_ERROR)) \ - ret = __wt_txn_commit((s), NULL); \ + else if ((ret) == 0 && \ + !F_ISSET(&(s)->txn, WT_TXN_ERROR)) \ + (ret) = __wt_txn_commit((s), NULL); \ else { \ if (retry) \ WT_TRET(__wt_session_copy_values(s)); \ WT_TRET(__wt_txn_rollback((s), NULL)); \ - if ((ret == 0 || ret == WT_ROLLBACK) && \ + if (((ret) == 0 || (ret) == WT_ROLLBACK) && \ (retry)) { \ - ret = 0; \ + (ret) = 0; \ continue; \ } \ WT_TRET(__wt_session_reset_cursors(s, false)); \ @@ -98,24 +98,24 @@ #define CONNECTION_API_CALL(conn, s, n, config, cfg) \ s = (conn)->default_session; \ - API_CALL(s, WT_CONNECTION, n, NULL, NULL, config, cfg) + API_CALL(s, WT_CONNECTION, n, NULL, config, cfg) #define CONNECTION_API_CALL_NOCONF(conn, s, n) \ s = (conn)->default_session; \ - API_CALL_NOCONF(s, WT_CONNECTION, n, NULL, NULL) + API_CALL_NOCONF(s, WT_CONNECTION, n, NULL) #define SESSION_API_CALL(s, n, config, cfg) \ - API_CALL(s, WT_SESSION, n, NULL, NULL, config, cfg) + API_CALL(s, WT_SESSION, n, NULL, config, cfg) #define SESSION_API_CALL_NOCONF(s, n) \ - API_CALL_NOCONF(s, WT_SESSION, n, NULL, NULL) + API_CALL_NOCONF(s, WT_SESSION, n, NULL) #define SESSION_TXN_API_CALL(s, n, config, cfg) \ - TXN_API_CALL(s, WT_SESSION, n, NULL, NULL, config, cfg) + TXN_API_CALL(s, WT_SESSION, n, NULL, config, cfg) #define CURSOR_API_CALL(cur, s, n, bt) \ (s) = (WT_SESSION_IMPL *)(cur)->session; \ - API_CALL_NOCONF(s, WT_CURSOR, n, cur, \ + API_CALL_NOCONF(s, WT_CURSOR, n, \ ((bt) == NULL) ? NULL : ((WT_BTREE *)(bt))->dhandle) #define JOINABLE_CURSOR_CALL_CHECK(cur) \ @@ -128,7 +128,7 @@ #define CURSOR_REMOVE_API_CALL(cur, s, bt) \ (s) = (WT_SESSION_IMPL *)(cur)->session; \ - TXN_API_CALL_NOCONF(s, WT_CURSOR, remove, cur, \ + TXN_API_CALL_NOCONF(s, WT_CURSOR, remove, \ ((bt) == NULL) ? NULL : ((WT_BTREE *)(bt))->dhandle); #define JOINABLE_CURSOR_REMOVE_API_CALL(cur, s, bt) \ @@ -137,7 +137,7 @@ #define CURSOR_UPDATE_API_CALL(cur, s, n, bt) \ (s) = (WT_SESSION_IMPL *)(cur)->session; \ - TXN_API_CALL_NOCONF(s, WT_CURSOR, n, cur, \ + TXN_API_CALL_NOCONF(s, WT_CURSOR, n, \ ((bt) == NULL) ? NULL : ((WT_BTREE *)(bt))->dhandle); \ if (F_ISSET(S2C(s), WT_CONN_IN_MEMORY) && \ !F_ISSET((WT_BTREE *)(bt), WT_BTREE_IGNORE_CACHE) && \ @@ -153,4 +153,4 @@ #define ASYNCOP_API_CALL(conn, s, n) \ s = (conn)->default_session; \ - API_CALL_NOCONF(s, asyncop, n, NULL, NULL) + API_CALL_NOCONF(s, asyncop, n, NULL) diff --git a/src/third_party/wiredtiger/src/include/bitstring.i b/src/third_party/wiredtiger/src/include/bitstring.i index 08746beb9b9..118dc0bba01 100644 --- a/src/third_party/wiredtiger/src/include/bitstring.i +++ b/src/third_party/wiredtiger/src/include/bitstring.i @@ -230,7 +230,7 @@ __bit_getv(uint8_t *bitf, uint64_t entry, uint8_t width) #define __BIT_GET(len, mask) \ case len: \ if (__bit_test(bitf, bit)) \ - value |= mask; \ + value |= (mask); \ ++bit \ /* FALLTHROUGH */ diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index 84c91097a99..d0b21b17965 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -414,18 +414,20 @@ struct __wt_page_modify { size_t discard_allocated; } *ovfl_track; +#define WT_PAGE_LOCK(s, p) \ + __wt_spin_lock((s), &(p)->modify->page_lock) +#define WT_PAGE_TRYLOCK(s, p) \ + __wt_spin_trylock((s), &(p)->modify->page_lock) +#define WT_PAGE_UNLOCK(s, p) \ + __wt_spin_unlock((s), &(p)->modify->page_lock) + WT_SPINLOCK page_lock; /* Page's spinlock */ + /* * The write generation is incremented when a page is modified, a page * is clean if the write generation is 0. */ uint32_t write_gen; -#define WT_PAGE_LOCK(s, p) \ - __wt_spin_lock((s), &S2C(s)->page_lock[(p)->modify->page_lock]) -#define WT_PAGE_UNLOCK(s, p) \ - __wt_spin_unlock((s), &S2C(s)->page_lock[(p)->modify->page_lock]) - uint8_t page_lock; /* Page's spinlock */ - #define WT_PM_REC_EMPTY 1 /* Reconciliation: no replacement */ #define WT_PM_REC_MULTIBLOCK 2 /* Reconciliation: multiple blocks */ #define WT_PM_REC_REPLACE 3 /* Reconciliation: single block */ @@ -434,6 +436,19 @@ struct __wt_page_modify { uint8_t update_restored; /* Page created by restoring updates */ }; +/* + * WT_COL_RLE -- + * Variable-length column-store pages have an array of page entries with RLE + * counts greater than 1 when reading the page, so it's not necessary to walk + * the page counting records to find a specific entry. We can do a binary search + * in this array, then an offset calculation to find the cell. + */ +WT_PACKED_STRUCT_BEGIN(__wt_col_rle) + uint64_t recno; /* Record number of first repeat. */ + uint64_t rle; /* Repeat count. */ + uint32_t indx; /* Slot of entry in col_var. */ +WT_PACKED_STRUCT_END + /* * WT_PAGE -- * The WT_PAGE structure describes the in-memory page information. @@ -444,9 +459,6 @@ struct __wt_page { /* * Internal pages (both column- and row-store). * - * The page record number is only used by column-store, but it's - * simpler having only one kind of internal page. - * * In-memory internal pages have an array of pointers to child * structures, maintained in collated order. When a page is * read into memory, the initial list of children is stored in @@ -473,6 +485,7 @@ struct __wt_page { */ struct { WT_REF *parent_ref; /* Parent reference */ + uint64_t split_gen; /* Generation of last split */ struct __wt_page_index { uint32_t entries; @@ -482,6 +495,8 @@ struct __wt_page { } intl; #undef pg_intl_parent_ref #define pg_intl_parent_ref u.intl.parent_ref +#undef pg_intl_split_gen +#define pg_intl_split_gen u.intl.split_gen /* * Macros to copy/set the index because the name is obscured to ensure @@ -494,7 +509,7 @@ struct __wt_page { #define WT_INTL_INDEX_GET_SAFE(page) \ ((page)->u.intl.__index) #define WT_INTL_INDEX_GET(session, page, pindex) do { \ - WT_ASSERT(session, session->split_gen != 0); \ + WT_ASSERT(session, (session)->split_gen != 0); \ (pindex) = WT_INTL_INDEX_GET_SAFE(page); \ } while (0) #define WT_INTL_INDEX_SET(page, v) do { \ @@ -518,53 +533,54 @@ struct __wt_page { } while (0) /* Row-store leaf page. */ - struct { - WT_ROW *d; /* Key/value pairs */ - uint32_t entries; /* Entries */ - } row; -#undef pg_row_d -#define pg_row_d u.row.d -#undef pg_row_entries -#define pg_row_entries u.row.entries + WT_ROW *row; /* Key/value pairs */ +#undef pg_row +#define pg_row u.row /* Fixed-length column-store leaf page. */ - struct { - uint8_t *bitf; /* Values */ - uint32_t entries; /* Entries */ - } col_fix; + uint8_t *fix_bitf; /* Values */ #undef pg_fix_bitf -#define pg_fix_bitf u.col_fix.bitf -#undef pg_fix_entries -#define pg_fix_entries u.col_fix.entries +#define pg_fix_bitf u.fix_bitf /* Variable-length column-store leaf page. */ struct { - WT_COL *d; /* Values */ + WT_COL *col_var; /* Values */ /* - * Variable-length column-store files maintain a list of - * RLE entries on the page so it's unnecessary to walk - * the page counting records to find a specific entry. + * Variable-length column-store pages have an array + * of page entries with RLE counts greater than 1 when + * reading the page, so it's not necessary to walk the + * page counting records to find a specific entry. We + * can do a binary search in this array, then an offset + * calculation to find the cell. + * + * It's a separate structure to keep the page structure + * as small as possible. */ - WT_COL_RLE *repeats; /* RLE array for lookups */ - uint32_t nrepeats; /* Number of repeat slots */ - - uint32_t entries; /* Entries */ + struct __wt_col_var_repeat { + uint32_t nrepeats; /* repeat slots */ + WT_COL_RLE repeats[0]; /* lookup RLE array */ + } *repeats; +#define WT_COL_VAR_REPEAT_SET(page) \ + ((page)->u.col_var.repeats != NULL) } col_var; -#undef pg_var_d -#define pg_var_d u.col_var.d +#undef pg_var +#define pg_var u.col_var.col_var #undef pg_var_repeats -#define pg_var_repeats u.col_var.repeats +#define pg_var_repeats u.col_var.repeats->repeats #undef pg_var_nrepeats -#define pg_var_nrepeats u.col_var.nrepeats -#undef pg_var_entries -#define pg_var_entries u.col_var.entries +#define pg_var_nrepeats u.col_var.repeats->nrepeats } u; /* - * The page's type and flags are positioned at the end of the WT_PAGE - * union, it reduces cache misses in the row-store search function. + * Page entries, type and flags are positioned at the end of the WT_PAGE + * union to reduce cache misses in the row-store search function. + * + * The entries field only applies to leaf pages, internal pages use the + * page-index entries instead. */ + uint32_t entries; /* Leaf page entries */ + #define WT_PAGE_IS_INTERNAL(page) \ ((page)->type == WT_PAGE_COL_INT || (page)->type == WT_PAGE_ROW_INT) #define WT_PAGE_INVALID 0 /* Invalid page */ @@ -582,20 +598,12 @@ struct __wt_page { #define WT_PAGE_DISK_MAPPED 0x04 /* Disk image in mapped memory */ #define WT_PAGE_EVICT_LRU 0x08 /* Page is on the LRU queue */ #define WT_PAGE_OVERFLOW_KEYS 0x10 /* Page has overflow keys */ -#define WT_PAGE_SPLIT_BLOCK 0x20 /* Split blocking eviction and splits */ -#define WT_PAGE_SPLIT_INSERT 0x40 /* A leaf page was split for append */ -#define WT_PAGE_UPDATE_IGNORE 0x80 /* Ignore updates on page discard */ +#define WT_PAGE_SPLIT_INSERT 0x20 /* A leaf page was split for append */ +#define WT_PAGE_UPDATE_IGNORE 0x40 /* Ignore updates on page discard */ uint8_t flags_atomic; /* Atomic flags, use F_*_ATOMIC */ uint8_t unused[2]; /* Unused padding */ - /* - * Used to protect and co-ordinate splits for internal pages and - * reconciliation for all pages. Only used to co-ordinate among the - * uncommon cases that require exclusive access to a page. - */ - WT_RWLOCK page_lock; - /* * The page's read generation acts as an LRU value for each page in the * tree; it is used by the eviction server thread to select pages to be @@ -621,8 +629,6 @@ struct __wt_page { #define WT_READGEN_START_VALUE 100 #define WT_READGEN_STEP 100 uint64_t read_gen; - /* The evict pass generation for the page */ - uint64_t evict_pass_gen; size_t memory_footprint; /* Memory attached to the page */ @@ -631,6 +637,10 @@ struct __wt_page { /* If/when the page is modified, we need lots more information. */ WT_PAGE_MODIFY *modify; + + /* This is the 64 byte boundary, try to keep hot fields above here. */ + + uint64_t evict_pass_gen; /* Eviction pass generation */ }; /* @@ -723,7 +733,7 @@ struct __wt_ref { * up our slot in the page's index structure. */ WT_PAGE * volatile home; /* Reference page */ - uint32_t pindex_hint; /* Reference page index hint */ + volatile uint32_t pindex_hint; /* Reference page index hint */ #define WT_REF_DISK 0 /* Page is on disk */ #define WT_REF_DELETED 1 /* Page is on disk, but deleted */ @@ -795,11 +805,11 @@ struct __wt_row { /* On-page key, on-page cell, or off-page WT_IKEY */ * Walk the entries of an in-memory row-store leaf page. */ #define WT_ROW_FOREACH(page, rip, i) \ - for ((i) = (page)->pg_row_entries, \ - (rip) = (page)->pg_row_d; (i) > 0; ++(rip), --(i)) + for ((i) = (page)->entries, \ + (rip) = (page)->pg_row; (i) > 0; ++(rip), --(i)) #define WT_ROW_FOREACH_REVERSE(page, rip, i) \ - for ((i) = (page)->pg_row_entries, \ - (rip) = (page)->pg_row_d + ((page)->pg_row_entries - 1); \ + for ((i) = (page)->entries, \ + (rip) = (page)->pg_row + ((page)->entries - 1); \ (i) > 0; --(rip), --(i)) /* @@ -807,7 +817,7 @@ struct __wt_row { /* On-page key, on-page cell, or off-page WT_IKEY */ * Return the 0-based array offset based on a WT_ROW reference. */ #define WT_ROW_SLOT(page, rip) \ - ((uint32_t)(((WT_ROW *)(rip)) - (page)->pg_row_d)) + ((uint32_t)(((WT_ROW *)(rip)) - (page)->pg_row)) /* * WT_COL -- @@ -831,18 +841,6 @@ struct __wt_col { uint32_t __col_value; }; -/* - * WT_COL_RLE -- - * In variable-length column store leaf pages, we build an array of entries - * with RLE counts greater than 1 when reading the page. We can do a binary - * search in this array, then an offset calculation to find the cell. - */ -WT_PACKED_STRUCT_BEGIN(__wt_col_rle) - uint64_t recno; /* Record number of first repeat. */ - uint64_t rle; /* Repeat count. */ - uint32_t indx; /* Slot of entry in col_var.d */ -WT_PACKED_STRUCT_END - /* * WT_COL_PTR, WT_COL_PTR_SET -- * Return/Set a pointer corresponding to the data offset. (If the item does @@ -859,15 +857,15 @@ WT_PACKED_STRUCT_END * Walk the entries of variable-length column-store leaf page. */ #define WT_COL_FOREACH(page, cip, i) \ - for ((i) = (page)->pg_var_entries, \ - (cip) = (page)->pg_var_d; (i) > 0; ++(cip), --(i)) + for ((i) = (page)->entries, \ + (cip) = (page)->pg_var; (i) > 0; ++(cip), --(i)) /* * WT_COL_SLOT -- * Return the 0-based array offset based on a WT_COL reference. */ #define WT_COL_SLOT(page, cip) \ - ((uint32_t)(((WT_COL *)cip) - (page)->pg_var_d)) + ((uint32_t)(((WT_COL *)(cip)) - (page)->pg_var)) /* * WT_IKEY -- @@ -932,7 +930,7 @@ WT_PACKED_STRUCT_BEGIN(__wt_update) #define WT_UPDATE_MEMSIZE(upd) \ WT_ALIGN(sizeof(WT_UPDATE) + \ (WT_UPDATE_DELETED_ISSET(upd) ? 0 : (upd)->size), 32) -}; +WT_PACKED_STRUCT_END /* * WT_INSERT -- @@ -976,10 +974,10 @@ struct __wt_insert { } key; } u; -#define WT_INSERT_KEY_SIZE(ins) (((WT_INSERT *)ins)->u.key.size) +#define WT_INSERT_KEY_SIZE(ins) (((WT_INSERT *)(ins))->u.key.size) #define WT_INSERT_KEY(ins) \ - ((void *)((uint8_t *)(ins) + ((WT_INSERT *)ins)->u.key.offset)) -#define WT_INSERT_RECNO(ins) (((WT_INSERT *)ins)->u.recno) + ((void *)((uint8_t *)(ins) + ((WT_INSERT *)(ins))->u.key.offset)) +#define WT_INSERT_RECNO(ins) (((WT_INSERT *)(ins))->u.recno) WT_INSERT *next[0]; /* forward-linked skip list */ }; @@ -988,9 +986,9 @@ struct __wt_insert { * Skiplist helper macros. */ #define WT_SKIP_FIRST(ins_head) \ - (((ins_head) == NULL) ? NULL : ((WT_INSERT_HEAD *)ins_head)->head[0]) + (((ins_head) == NULL) ? NULL : ((WT_INSERT_HEAD *)(ins_head))->head[0]) #define WT_SKIP_LAST(ins_head) \ - (((ins_head) == NULL) ? NULL : ((WT_INSERT_HEAD *)ins_head)->tail[0]) + (((ins_head) == NULL) ? NULL : ((WT_INSERT_HEAD *)(ins_head))->tail[0]) #define WT_SKIP_NEXT(ins) ((ins)->next[0]) #define WT_SKIP_FOREACH(ins, ins_head) \ for ((ins) = WT_SKIP_FIRST(ins_head); \ @@ -1003,7 +1001,7 @@ struct __wt_insert { #define WT_PAGE_ALLOC_AND_SWAP(s, page, dest, v, count) do { \ if (((v) = (dest)) == NULL) { \ WT_ERR(__wt_calloc_def(s, count, &(v))); \ - if (__wt_atomic_cas_ptr(&dest, NULL, v)) \ + if (__wt_atomic_cas_ptr(&(dest), NULL, v)) \ __wt_cache_page_inmem_incr( \ s, page, (count) * sizeof(*(v))); \ else \ @@ -1044,7 +1042,7 @@ struct __wt_insert_head { #define WT_ROW_INSERT_SMALLEST(page) \ ((page)->modify == NULL || \ (page)->modify->mod_row_insert == NULL ? \ - NULL : (page)->modify->mod_row_insert[(page)->pg_row_entries]) + NULL : (page)->modify->mod_row_insert[(page)->entries]) /* * The column-store leaf page update lists are arrays of pointers to structures, diff --git a/src/third_party/wiredtiger/src/include/btree.h b/src/third_party/wiredtiger/src/include/btree.h index 713d46ae85f..28fe1b94b23 100644 --- a/src/third_party/wiredtiger/src/include/btree.h +++ b/src/third_party/wiredtiger/src/include/btree.h @@ -44,7 +44,7 @@ * Record numbers are stored in 64-bit unsigned integers, meaning the largest * record number is "really, really big". */ -#define WT_BTREE_MAX_OBJECT_SIZE (UINT32_MAX - 1024) +#define WT_BTREE_MAX_OBJECT_SIZE ((uint32_t)(UINT32_MAX - 1024)) /* * A location in a file is a variable-length cookie, but it has a maximum size @@ -57,6 +57,12 @@ /* Evict pages if we see this many consecutive deleted records. */ #define WT_BTREE_DELETE_THRESHOLD 1000 +/* + * Minimum size of the chunks (in percentage of the page size) a page gets split + * into during reconciliation. + */ +#define WT_BTREE_MIN_SPLIT_PCT 50 + /* * WT_BTREE -- * A btree handle. @@ -114,23 +120,26 @@ struct __wt_btree { int split_pct; /* Split page percent */ WT_COMPRESSOR *compressor; /* Page compressor */ WT_KEYED_ENCRYPTOR *kencryptor; /* Page encryptor */ - WT_RWLOCK *ovfl_lock; /* Overflow lock */ + WT_RWLOCK ovfl_lock; /* Overflow lock */ uint64_t last_recno; /* Column-store last record number */ - WT_REF root; /* Root page reference */ - bool modified; /* If the tree ever modified */ - bool bulk_load_ok; /* Bulk-load is a possibility */ + WT_REF root; /* Root page reference */ + bool modified; /* If the tree ever modified */ + uint8_t original; /* Newly created: bulk-load possible + (want a bool but needs atomic cas) */ + + bool lsm_primary; /* Handle is/was the LSM primary */ WT_BM *bm; /* Block manager reference */ u_int block_header; /* WT_PAGE_HEADER_BYTE_SIZE */ uint64_t checkpoint_gen; /* Checkpoint generation */ - bool include_checkpoint_txn;/* ID checks include checkpoint */ uint64_t rec_max_txn; /* Maximum txn seen (clean trees) */ uint64_t write_gen; /* Write generation */ uint64_t bytes_inmem; /* Cache bytes in memory. */ + uint64_t bytes_dirty_intl; /* Bytes in dirty internal pages. */ uint64_t bytes_dirty_leaf; /* Bytes in dirty leaf pages. */ WT_REF *evict_ref; /* Eviction thread's location */ @@ -138,10 +147,10 @@ struct __wt_btree { u_int evict_walk_period; /* Skip this many LRU walks */ u_int evict_walk_saved; /* Saved walk skips for checkpoints */ u_int evict_walk_skips; /* Number of walks skipped */ - u_int evict_disabled; /* Eviction disabled count */ + int evict_disabled; /* Eviction disabled count */ volatile uint32_t evict_busy; /* Count of threads in eviction */ - bool evict_walk_reverse; /* Walk direction */ - + int evict_start_type; /* Start position for eviction walk + (see WT_EVICT_WALK_START). */ enum { WT_CKPT_OFF, WT_CKPT_PREPARE, WT_CKPT_RUNNING } checkpointing; /* Checkpoint in progress */ @@ -154,19 +163,19 @@ struct __wt_btree { WT_SPINLOCK flush_lock; /* Lock to flush the tree's pages */ /* Flags values up to 0xff are reserved for WT_DHANDLE_* */ -#define WT_BTREE_BULK 0x000100 /* Bulk-load handle */ -#define WT_BTREE_IGNORE_CACHE 0x000200 /* Cache-resident object */ -#define WT_BTREE_IN_MEMORY 0x000400 /* Cache-resident object */ -#define WT_BTREE_LOOKASIDE 0x000800 /* Look-aside table */ -#define WT_BTREE_NO_CHECKPOINT 0x001000 /* Disable checkpoints */ -#define WT_BTREE_NO_EVICTION 0x002000 /* Disable eviction */ -#define WT_BTREE_NO_LOGGING 0x004000 /* Disable logging */ -#define WT_BTREE_NO_RECONCILE 0x008000 /* Allow splits, even with no evict */ -#define WT_BTREE_REBALANCE 0x010000 /* Handle is for rebalance */ -#define WT_BTREE_SALVAGE 0x020000 /* Handle is for salvage */ -#define WT_BTREE_SKIP_CKPT 0x040000 /* Handle skipped checkpoint */ -#define WT_BTREE_UPGRADE 0x080000 /* Handle is for upgrade */ -#define WT_BTREE_VERIFY 0x100000 /* Handle is for verify */ +#define WT_BTREE_ALLOW_SPLITS 0x000100 /* Allow splits, even with no evict */ +#define WT_BTREE_BULK 0x000200 /* Bulk-load handle */ +#define WT_BTREE_CLOSED 0x000400 /* Handle closed */ +#define WT_BTREE_IGNORE_CACHE 0x000800 /* Cache-resident object */ +#define WT_BTREE_IN_MEMORY 0x001000 /* Cache-resident object */ +#define WT_BTREE_LOOKASIDE 0x002000 /* Look-aside table */ +#define WT_BTREE_NO_CHECKPOINT 0x004000 /* Disable checkpoints */ +#define WT_BTREE_NO_LOGGING 0x008000 /* Disable logging */ +#define WT_BTREE_REBALANCE 0x020000 /* Handle is for rebalance */ +#define WT_BTREE_SALVAGE 0x040000 /* Handle is for salvage */ +#define WT_BTREE_SKIP_CKPT 0x080000 /* Handle skipped checkpoint */ +#define WT_BTREE_UPGRADE 0x100000 /* Handle is for upgrade */ +#define WT_BTREE_VERIFY 0x200000 /* Handle is for verify */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/btree.i b/src/third_party/wiredtiger/src/include/btree.i index e48189d50ea..1d6fcd6272c 100644 --- a/src/third_party/wiredtiger/src/include/btree.i +++ b/src/third_party/wiredtiger/src/include/btree.i @@ -70,6 +70,47 @@ __wt_btree_bytes_inuse(WT_SESSION_IMPL *session) return (__wt_cache_bytes_plus_overhead(cache, btree->bytes_inmem)); } +/* + * __wt_btree_bytes_evictable -- + * Return the number of bytes that can be evicted (i.e. bytes apart from + * the pinned root page). + */ +static inline uint64_t +__wt_btree_bytes_evictable(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + WT_CACHE *cache; + WT_PAGE *root_page; + uint64_t bytes_inmem, bytes_root; + + btree = S2BT(session); + cache = S2C(session)->cache; + root_page = btree->root.page; + + bytes_inmem = btree->bytes_inmem; + bytes_root = root_page == NULL ? 0 : root_page->memory_footprint; + + return (bytes_inmem <= bytes_root ? 0 : + __wt_cache_bytes_plus_overhead(cache, bytes_inmem - bytes_root)); +} + +/* + * __wt_btree_dirty_inuse -- + * Return the number of dirty bytes in use. + */ +static inline uint64_t +__wt_btree_dirty_inuse(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + WT_CACHE *cache; + + btree = S2BT(session); + cache = S2C(session)->cache; + + return (__wt_cache_bytes_plus_overhead(cache, + btree->bytes_dirty_intl + btree->bytes_dirty_leaf)); +} + /* * __wt_btree_dirty_leaf_inuse -- * Return the number of bytes in use by dirty leaf pages. @@ -105,11 +146,12 @@ __wt_cache_page_inmem_incr(WT_SESSION_IMPL *session, WT_PAGE *page, size_t size) (void)__wt_atomic_addsize(&page->memory_footprint, size); if (__wt_page_is_modified(page)) { (void)__wt_atomic_addsize(&page->modify->bytes_dirty, size); - if (WT_PAGE_IS_INTERNAL(page)) + if (WT_PAGE_IS_INTERNAL(page)) { + (void)__wt_atomic_add64(&btree->bytes_dirty_intl, size); (void)__wt_atomic_add64(&cache->bytes_dirty_intl, size); - else { - (void)__wt_atomic_add64(&cache->bytes_dirty_leaf, size); + } else if (!btree->lsm_primary) { (void)__wt_atomic_add64(&btree->bytes_dirty_leaf, size); + (void)__wt_atomic_add64(&cache->bytes_dirty_leaf, size); } } /* Track internal size in cache. */ @@ -238,10 +280,12 @@ __wt_cache_page_byte_dirty_decr( if (i == 5) return; - if (WT_PAGE_IS_INTERNAL(page)) + if (WT_PAGE_IS_INTERNAL(page)) { + __wt_cache_decr_check_uint64(session, &btree->bytes_dirty_intl, + decr, "WT_BTREE.bytes_dirty_intl"); __wt_cache_decr_check_uint64(session, &cache->bytes_dirty_intl, decr, "WT_CACHE.bytes_dirty_intl"); - else { + } else if (!btree->lsm_primary) { __wt_cache_decr_check_uint64(session, &btree->bytes_dirty_leaf, decr, "WT_BTREE.bytes_dirty_leaf"); __wt_cache_decr_check_uint64(session, &cache->bytes_dirty_leaf, @@ -297,11 +341,14 @@ __wt_cache_dirty_incr(WT_SESSION_IMPL *session, WT_PAGE *page) */ size = page->memory_footprint; if (WT_PAGE_IS_INTERNAL(page)) { + (void)__wt_atomic_add64(&btree->bytes_dirty_intl, size); (void)__wt_atomic_add64(&cache->bytes_dirty_intl, size); (void)__wt_atomic_add64(&cache->pages_dirty_intl, 1); } else { - (void)__wt_atomic_add64(&btree->bytes_dirty_leaf, size); - (void)__wt_atomic_add64(&cache->bytes_dirty_leaf, size); + if (!btree->lsm_primary) { + (void)__wt_atomic_add64(&btree->bytes_dirty_leaf, size); + (void)__wt_atomic_add64(&cache->bytes_dirty_leaf, size); + } (void)__wt_atomic_add64(&cache->pages_dirty_leaf, 1); } (void)__wt_atomic_addsize(&page->modify->bytes_dirty, size); @@ -366,7 +413,7 @@ __wt_cache_page_image_incr(WT_SESSION_IMPL *session, uint32_t size) * Evict pages from the cache. */ static inline void -__wt_cache_page_evict(WT_SESSION_IMPL *session, WT_PAGE *page) +__wt_cache_page_evict(WT_SESSION_IMPL *session, WT_PAGE *page, bool rewrite) { WT_BTREE *btree; WT_CACHE *cache; @@ -390,23 +437,34 @@ __wt_cache_page_evict(WT_SESSION_IMPL *session, WT_PAGE *page) /* Update the cache's dirty-byte count. */ if (modify != NULL && modify->bytes_dirty != 0) { - if (WT_PAGE_IS_INTERNAL(page)) + if (WT_PAGE_IS_INTERNAL(page)) { + __wt_cache_decr_zero_uint64(session, + &btree->bytes_dirty_intl, + modify->bytes_dirty, "WT_BTREE.bytes_dirty_intl"); __wt_cache_decr_zero_uint64(session, &cache->bytes_dirty_intl, modify->bytes_dirty, "WT_CACHE.bytes_dirty_intl"); - else { - __wt_cache_decr_zero_uint64(session, - &cache->bytes_dirty_leaf, - modify->bytes_dirty, "WT_CACHE.bytes_dirty_leaf"); + } else if (!btree->lsm_primary) { __wt_cache_decr_zero_uint64(session, &btree->bytes_dirty_leaf, modify->bytes_dirty, "WT_BTREE.bytes_dirty_leaf"); + __wt_cache_decr_zero_uint64(session, + &cache->bytes_dirty_leaf, + modify->bytes_dirty, "WT_CACHE.bytes_dirty_leaf"); } } /* Update pages and bytes evicted. */ (void)__wt_atomic_add64(&cache->bytes_evict, page->memory_footprint); - (void)__wt_atomic_addv64(&cache->pages_evict, 1); + + /* + * Don't count rewrites as eviction: there's no guarantee we are making + * real progress. + */ + if (rewrite) + (void)__wt_atomic_subv64(&cache->pages_inmem, 1); + else + (void)__wt_atomic_addv64(&cache->pages_evict, 1); } /* @@ -982,7 +1040,7 @@ __wt_cursor_row_leaf_key(WT_CURSOR_BTREE *cbt, WT_ITEM *key) if (cbt->ins == NULL) { session = (WT_SESSION_IMPL *)cbt->iface.session; page = cbt->ref->page; - rip = &page->u.row.d[cbt->slot]; + rip = &page->pg_row[cbt->slot]; WT_RET(__wt_row_leaf_key(session, page, rip, key, false)); } else { key->data = WT_INSERT_KEY(cbt->ins); @@ -1179,11 +1237,10 @@ __wt_leaf_page_can_split(WT_SESSION_IMPL *session, WT_PAGE *page) * data in the last skiplist on the page. Split if there are enough * items and the skiplist does not fit within a single disk page. */ - ins_head = page->type == WT_PAGE_ROW_LEAF ? - (page->pg_row_entries == 0 ? + (page->entries == 0 ? WT_ROW_INSERT_SMALLEST(page) : - WT_ROW_INSERT_SLOT(page, page->pg_row_entries - 1)) : + WT_ROW_INSERT_SLOT(page, page->entries - 1)) : WT_COL_APPEND(page); if (ins_head == NULL) return (false); @@ -1297,9 +1354,14 @@ __wt_page_can_evict( * the original parent page's index, because evicting an internal page * discards its WT_REF array, and a thread traversing the original * parent page index might see a freed WT_REF. + * + * One special case where we know this is safe is if the handle is + * locked exclusive (e.g., when the whole tree is being evicted). In + * that case, no readers can be looking at an old index. */ - if (WT_PAGE_IS_INTERNAL(page) && - F_ISSET_ATOMIC(page, WT_PAGE_SPLIT_BLOCK)) + if (!F_ISSET(session->dhandle, WT_DHANDLE_EXCLUSIVE) && + WT_PAGE_IS_INTERNAL(page) && !__wt_split_obsolete( + session, page->pg_intl_split_gen)) return (false); /* @@ -1351,9 +1413,9 @@ __wt_page_release(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags) if (page->read_gen != WT_READGEN_OLDEST || LF_ISSET(WT_READ_NO_EVICT) || F_ISSET(session, WT_SESSION_NO_EVICTION) || - F_ISSET(btree, WT_BTREE_NO_EVICTION) || + btree->evict_disabled > 0 || !__wt_page_can_evict(session, ref, NULL)) - return (__wt_hazard_clear(session, page)); + return (__wt_hazard_clear(session, ref)); WT_RET_BUSY_OK(__wt_page_release_evict(session, ref)); return (0); @@ -1434,53 +1496,6 @@ __wt_page_swap_func( return (ret); } -/* - * __wt_page_hazard_check -- - * Return if there's a hazard pointer to the page in the system. - */ -static inline WT_HAZARD * -__wt_page_hazard_check(WT_SESSION_IMPL *session, WT_PAGE *page) -{ - WT_CONNECTION_IMPL *conn; - WT_HAZARD *hp; - WT_SESSION_IMPL *s; - uint32_t i, j, hazard_size, max, session_cnt; - - conn = S2C(session); - - /* - * No lock is required because the session array is fixed size, but it - * may contain inactive entries. We must review any active session - * that might contain a hazard pointer, so insert a barrier before - * reading the active session count. That way, no matter what sessions - * come or go, we'll check the slots for all of the sessions that could - * have been active when we started our check. - */ - WT_STAT_CONN_INCR(session, cache_hazard_checks); - WT_ORDERED_READ(session_cnt, conn->session_cnt); - for (s = conn->sessions, i = 0, j = 0, max = 0; - i < session_cnt; ++s, ++i) { - if (!s->active) - continue; - WT_ORDERED_READ(hazard_size, s->hazard_size); - if (s->hazard_size > max) { - max = s->hazard_size; - WT_STAT_CONN_SET(session, - cache_hazard_max, max); - } - for (hp = s->hazard; hp < s->hazard + hazard_size; ++hp) { - ++j; - if (hp->page == page) { - WT_STAT_CONN_INCRV(session, - cache_hazard_walks, j); - return (hp); - } - } - } - WT_STAT_CONN_INCRV(session, cache_hazard_walks, j); - return (NULL); -} - /* * __wt_skip_choose_depth -- * Randomly choose a depth for a skiplist insert. @@ -1518,7 +1533,7 @@ __wt_btree_lsm_over_size(WT_SESSION_IMPL *session, uint64_t maxsize) return (false); /* A tree that can be evicted always requires a switch. */ - if (!F_ISSET(btree, WT_BTREE_NO_EVICTION)) + if (btree->evict_disabled == 0) return (true); /* Check for a tree with a single leaf page. */ diff --git a/src/third_party/wiredtiger/src/include/buf.i b/src/third_party/wiredtiger/src/include/buf.i index ebbee6b4633..d192e292dcf 100644 --- a/src/third_party/wiredtiger/src/include/buf.i +++ b/src/third_party/wiredtiger/src/include/buf.i @@ -37,28 +37,30 @@ __wt_buf_extend(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) /* * __wt_buf_init -- - * Initialize a buffer at a specific size. + * Create an empty buffer at a specific size. */ static inline int __wt_buf_init(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) { + /* + * The buffer grow function does what we need, but anticipates data + * referenced by the buffer. Avoid any data copy by setting data to + * reference the buffer's allocated memory, and clearing it. + */ buf->data = buf->mem; - buf->size = 0; /* Clear existing data length */ - WT_RET(__wt_buf_grow(session, buf, size)); - - return (0); + buf->size = 0; + return (__wt_buf_grow(session, buf, size)); } /* * __wt_buf_initsize -- - * Initialize a buffer at a specific size, and set the data length. + * Create an empty buffer at a specific size, and set the data length. */ static inline int __wt_buf_initsize(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) { - buf->data = buf->mem; - buf->size = 0; /* Clear existing data length */ - WT_RET(__wt_buf_grow(session, buf, size)); + WT_RET(__wt_buf_init(session, buf, size)); + buf->size = size; /* Set the data length. */ return (0); @@ -72,14 +74,15 @@ static inline int __wt_buf_set( WT_SESSION_IMPL *session, WT_ITEM *buf, const void *data, size_t size) { - /* Ensure the buffer is large enough. */ - WT_RET(__wt_buf_initsize(session, buf, size)); - - /* Copy the data, allowing for overlapping strings. */ - if (size != 0) - memmove(buf->mem, data, size); - - return (0); + /* + * The buffer grow function does what we need, but expects the data to + * be referenced by the buffer. If we're copying data from outside the + * buffer, set it up so it makes sense to the buffer grow function. (No + * test needed, this works if WT_ITEM.data is already set to "data".) + */ + buf->data = data; + buf->size = size; + return (__wt_buf_grow(session, buf, size)); } /* diff --git a/src/third_party/wiredtiger/src/include/cache.h b/src/third_party/wiredtiger/src/include/cache.h index 9a2b83b5b57..04920c3585a 100644 --- a/src/third_party/wiredtiger/src/include/cache.h +++ b/src/third_party/wiredtiger/src/include/cache.h @@ -16,6 +16,17 @@ #define WT_EVICT_WALK_BASE 300 /* Pages tracked across file visits */ #define WT_EVICT_WALK_INCR 100 /* Pages added each walk */ +#define WT_EVICT_MAX_TREES 1000 /* Maximum walk points */ + +/* Ways to position when starting an eviction walk. */ +typedef enum { + WT_EVICT_WALK_NEXT, + WT_EVICT_WALK_PREV, + WT_EVICT_WALK_RAND_NEXT, + WT_EVICT_WALK_RAND_PREV +} WT_EVICT_WALK_START; +#define WT_EVICT_WALK_START_NUM (WT_EVICT_WALK_RAND_PREV + 1) + /* * WT_EVICT_ENTRY -- * Encapsulation of an eviction candidate. @@ -81,7 +92,7 @@ struct __wt_cache { uint64_t worker_evicts; /* Pages evicted by worker threads */ uint64_t evict_max_page_size; /* Largest page seen at eviction */ -#ifdef HAVE_DIAGNOSTIC +#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) struct timespec stuck_ts; /* Stuck timestamp */ #endif @@ -185,9 +196,9 @@ struct __wt_cache { uint32_t flags; }; -#define WT_WITH_PASS_LOCK(session, ret, op) do { \ +#define WT_WITH_PASS_LOCK(session, op) do { \ WT_ASSERT(session, !F_ISSET(session, WT_SESSION_LOCKED_PASS)); \ - WT_WITH_LOCK(session, ret, \ + WT_WITH_LOCK_WAIT(session, \ &cache->evict_pass_lock, WT_SESSION_LOCKED_PASS, op); \ } while (0) diff --git a/src/third_party/wiredtiger/src/include/cache.i b/src/third_party/wiredtiger/src/include/cache.i index 17ab39e97d2..90dd1bcdda8 100644 --- a/src/third_party/wiredtiger/src/include/cache.i +++ b/src/third_party/wiredtiger/src/include/cache.i @@ -360,11 +360,13 @@ __wt_cache_eviction_check(WT_SESSION_IMPL *session, bool busy, bool *didworkp) /* * LSM sets the no-cache-check flag when holding the LSM tree lock, in - * that case, or when holding the schema or handle list locks (which - * block eviction), we don't want to highjack the thread for eviction. + * that case, or when holding the handle list, schema or table locks + * (which can block checkpoints and eviction), don't block the thread + * for eviction. */ if (F_ISSET(session, WT_SESSION_NO_EVICTION | - WT_SESSION_LOCKED_HANDLE_LIST | WT_SESSION_LOCKED_SCHEMA)) + WT_SESSION_LOCKED_HANDLE_LIST | WT_SESSION_LOCKED_SCHEMA | + WT_SESSION_LOCKED_TABLE)) return (0); /* In memory configurations don't block when the cache is full. */ @@ -372,11 +374,14 @@ __wt_cache_eviction_check(WT_SESSION_IMPL *session, bool busy, bool *didworkp) return (0); /* - * Threads operating on cache-resident trees are ignored because they're - * not contributing to the problem. + * Threads operating on cache-resident trees are ignored because + * they're not contributing to the problem. We also don't block while + * reading metadata because we're likely to be holding some other + * resources that could block checkpoints or eviction. */ btree = S2BT_SAFE(session); - if (btree != NULL && F_ISSET(btree, WT_BTREE_IN_MEMORY)) + if (btree != NULL && (F_ISSET(btree, WT_BTREE_IN_MEMORY) || + WT_IS_METADATA(session->dhandle))) return (0); /* Check if eviction is needed. */ diff --git a/src/third_party/wiredtiger/src/include/cell.i b/src/third_party/wiredtiger/src/include/cell.i index c130768e595..71c2515daf0 100644 --- a/src/third_party/wiredtiger/src/include/cell.i +++ b/src/third_party/wiredtiger/src/include/cell.i @@ -361,14 +361,12 @@ __wt_cell_pack_leaf_key(WT_CELL *cell, uint8_t prefix, size_t size) cell->__chunk[0] = (uint8_t) ((byte << WT_CELL_SHORT_SHIFT) | WT_CELL_KEY_SHORT); return (1); - } else { - byte = (uint8_t)size; /* Type + length */ - cell->__chunk[0] = (uint8_t) - ((byte << WT_CELL_SHORT_SHIFT) | - WT_CELL_KEY_SHORT_PFX); - cell->__chunk[1] = prefix; /* Prefix */ - return (2); } + byte = (uint8_t)size; /* Type + length */ + cell->__chunk[0] = (uint8_t) + ((byte << WT_CELL_SHORT_SHIFT) | WT_CELL_KEY_SHORT_PFX); + cell->__chunk[1] = prefix; /* Prefix */ + return (2); } if (prefix == 0) { @@ -569,8 +567,8 @@ __wt_cell_unpack_safe( */ #define WT_CELL_LEN_CHK(t, len) do { \ if (start != NULL && \ - ((uint8_t *)t < (uint8_t *)start || \ - (((uint8_t *)t) + (len)) > (uint8_t *)end)) \ + ((uint8_t *)(t) < (uint8_t *)start || \ + (((uint8_t *)(t)) + (len)) > (uint8_t *)end)) \ return (WT_ERROR); \ } while (0) diff --git a/src/third_party/wiredtiger/src/include/column.i b/src/third_party/wiredtiger/src/include/column.i index d15f874b281..07b627315e6 100644 --- a/src/third_party/wiredtiger/src/include/column.i +++ b/src/third_party/wiredtiger/src/include/column.i @@ -108,7 +108,7 @@ __col_insert_search_match(WT_INSERT_HEAD *ins_head, uint64_t recno) /* Fast path the check for values at the end of the skiplist. */ if (recno > WT_INSERT_RECNO(ret_ins)) return (NULL); - else if (recno == WT_INSERT_RECNO(ret_ins)) + if (recno == WT_INSERT_RECNO(ret_ins)) return (ret_ins); /* @@ -127,7 +127,7 @@ __col_insert_search_match(WT_INSERT_HEAD *ins_head, uint64_t recno) if (cmp == 0) /* Exact match: return */ return (*insp); - else if (cmp > 0) /* Keep going at this level */ + if (cmp > 0) /* Keep going at this level */ insp = &(*insp)->next[i]; else { /* Drop down a level */ --i; @@ -221,13 +221,13 @@ __col_var_last_recno(WT_REF *ref) * This function ignores those records, our callers must handle that * explicitly, if they care. */ - if (page->pg_var_nrepeats == 0) - return (page->pg_var_entries == 0 ? 0 : - ref->ref_recno + (page->pg_var_entries - 1)); + if (!WT_COL_VAR_REPEAT_SET(page)) + return (page->entries == 0 ? 0 : + ref->ref_recno + (page->entries - 1)); repeat = &page->pg_var_repeats[page->pg_var_nrepeats - 1]; return ((repeat->recno + repeat->rle) - 1 + - (page->pg_var_entries - (repeat->indx + 1))); + (page->entries - (repeat->indx + 1))); } /* @@ -246,8 +246,7 @@ __col_fix_last_recno(WT_REF *ref) * This function ignores those records, our callers must handle that * explicitly, if they care. */ - return (page->pg_fix_entries == 0 ? - 0 : ref->ref_recno + (page->pg_fix_entries - 1)); + return (page->entries == 0 ? 0 : ref->ref_recno + (page->entries - 1)); } /* @@ -273,7 +272,9 @@ __col_var_search(WT_REF *ref, uint64_t recno, uint64_t *start_recnop) * slot for this record number, because we know any intervening records * have repeat counts of 1. */ - for (base = 0, limit = page->pg_var_nrepeats; limit != 0; limit >>= 1) { + for (base = 0, + limit = WT_COL_VAR_REPEAT_SET(page) ? page->pg_var_nrepeats : 0; + limit != 0; limit >>= 1) { indx = base + (limit >> 1); repeat = page->pg_var_repeats + indx; @@ -281,7 +282,7 @@ __col_var_search(WT_REF *ref, uint64_t recno, uint64_t *start_recnop) recno < repeat->recno + repeat->rle) { if (start_recnop != NULL) *start_recnop = repeat->recno; - return (page->pg_var_d + repeat->indx); + return (page->pg_var + repeat->indx); } if (recno < repeat->recno) continue; @@ -306,14 +307,14 @@ __col_var_search(WT_REF *ref, uint64_t recno, uint64_t *start_recnop) * !!! * The test could be written more simply as: * - * (recno >= start_recno + (page->pg_var_entries - start_indx)) + * (recno >= start_recno + (page->entries - start_indx)) * * It's split into two parts because the simpler test will overflow if * searching for large record numbers. */ if (recno >= start_recno && - recno - start_recno >= page->pg_var_entries - start_indx) + recno - start_recno >= page->entries - start_indx) return (NULL); - return (page->pg_var_d + start_indx + (uint32_t)(recno - start_recno)); + return (page->pg_var + start_indx + (uint32_t)(recno - start_recno)); } diff --git a/src/third_party/wiredtiger/src/include/compact.h b/src/third_party/wiredtiger/src/include/compact.h index 2bba52e7173..96797f6b275 100644 --- a/src/third_party/wiredtiger/src/include/compact.h +++ b/src/third_party/wiredtiger/src/include/compact.h @@ -6,8 +6,10 @@ * See the file LICENSE for redistribution information. */ -struct __wt_compact { +struct __wt_compact_state { uint32_t lsm_count; /* Number of LSM trees seen */ uint32_t file_count; /* Number of files seen */ uint64_t max_time; /* Configured timeout */ + + struct timespec begin; /* Starting time */ }; diff --git a/src/third_party/wiredtiger/src/include/config.h b/src/third_party/wiredtiger/src/include/config.h index 486aa50e86c..f2746fc76d9 100644 --- a/src/third_party/wiredtiger/src/include/config.h +++ b/src/third_party/wiredtiger/src/include/config.h @@ -62,39 +62,40 @@ struct __wt_config_parser_impl { #define WT_CONFIG_ENTRY_WT_CONNECTION_set_file_system 10 #define WT_CONFIG_ENTRY_WT_CURSOR_close 11 #define WT_CONFIG_ENTRY_WT_CURSOR_reconfigure 12 -#define WT_CONFIG_ENTRY_WT_SESSION_begin_transaction 13 -#define WT_CONFIG_ENTRY_WT_SESSION_checkpoint 14 -#define WT_CONFIG_ENTRY_WT_SESSION_close 15 -#define WT_CONFIG_ENTRY_WT_SESSION_commit_transaction 16 -#define WT_CONFIG_ENTRY_WT_SESSION_compact 17 -#define WT_CONFIG_ENTRY_WT_SESSION_create 18 -#define WT_CONFIG_ENTRY_WT_SESSION_drop 19 -#define WT_CONFIG_ENTRY_WT_SESSION_join 20 -#define WT_CONFIG_ENTRY_WT_SESSION_log_flush 21 -#define WT_CONFIG_ENTRY_WT_SESSION_log_printf 22 -#define WT_CONFIG_ENTRY_WT_SESSION_open_cursor 23 -#define WT_CONFIG_ENTRY_WT_SESSION_rebalance 24 -#define WT_CONFIG_ENTRY_WT_SESSION_reconfigure 25 -#define WT_CONFIG_ENTRY_WT_SESSION_rename 26 -#define WT_CONFIG_ENTRY_WT_SESSION_reset 27 -#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 28 -#define WT_CONFIG_ENTRY_WT_SESSION_salvage 29 -#define WT_CONFIG_ENTRY_WT_SESSION_snapshot 30 -#define WT_CONFIG_ENTRY_WT_SESSION_strerror 31 -#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 32 -#define WT_CONFIG_ENTRY_WT_SESSION_truncate 33 -#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 34 -#define WT_CONFIG_ENTRY_WT_SESSION_verify 35 -#define WT_CONFIG_ENTRY_colgroup_meta 36 -#define WT_CONFIG_ENTRY_file_config 37 -#define WT_CONFIG_ENTRY_file_meta 38 -#define WT_CONFIG_ENTRY_index_meta 39 -#define WT_CONFIG_ENTRY_lsm_meta 40 -#define WT_CONFIG_ENTRY_table_meta 41 -#define WT_CONFIG_ENTRY_wiredtiger_open 42 -#define WT_CONFIG_ENTRY_wiredtiger_open_all 43 -#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 44 -#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 45 +#define WT_CONFIG_ENTRY_WT_SESSION_alter 13 +#define WT_CONFIG_ENTRY_WT_SESSION_begin_transaction 14 +#define WT_CONFIG_ENTRY_WT_SESSION_checkpoint 15 +#define WT_CONFIG_ENTRY_WT_SESSION_close 16 +#define WT_CONFIG_ENTRY_WT_SESSION_commit_transaction 17 +#define WT_CONFIG_ENTRY_WT_SESSION_compact 18 +#define WT_CONFIG_ENTRY_WT_SESSION_create 19 +#define WT_CONFIG_ENTRY_WT_SESSION_drop 20 +#define WT_CONFIG_ENTRY_WT_SESSION_join 21 +#define WT_CONFIG_ENTRY_WT_SESSION_log_flush 22 +#define WT_CONFIG_ENTRY_WT_SESSION_log_printf 23 +#define WT_CONFIG_ENTRY_WT_SESSION_open_cursor 24 +#define WT_CONFIG_ENTRY_WT_SESSION_rebalance 25 +#define WT_CONFIG_ENTRY_WT_SESSION_reconfigure 26 +#define WT_CONFIG_ENTRY_WT_SESSION_rename 27 +#define WT_CONFIG_ENTRY_WT_SESSION_reset 28 +#define WT_CONFIG_ENTRY_WT_SESSION_rollback_transaction 29 +#define WT_CONFIG_ENTRY_WT_SESSION_salvage 30 +#define WT_CONFIG_ENTRY_WT_SESSION_snapshot 31 +#define WT_CONFIG_ENTRY_WT_SESSION_strerror 32 +#define WT_CONFIG_ENTRY_WT_SESSION_transaction_sync 33 +#define WT_CONFIG_ENTRY_WT_SESSION_truncate 34 +#define WT_CONFIG_ENTRY_WT_SESSION_upgrade 35 +#define WT_CONFIG_ENTRY_WT_SESSION_verify 36 +#define WT_CONFIG_ENTRY_colgroup_meta 37 +#define WT_CONFIG_ENTRY_file_config 38 +#define WT_CONFIG_ENTRY_file_meta 39 +#define WT_CONFIG_ENTRY_index_meta 40 +#define WT_CONFIG_ENTRY_lsm_meta 41 +#define WT_CONFIG_ENTRY_table_meta 42 +#define WT_CONFIG_ENTRY_wiredtiger_open 43 +#define WT_CONFIG_ENTRY_wiredtiger_open_all 44 +#define WT_CONFIG_ENTRY_wiredtiger_open_basecfg 45 +#define WT_CONFIG_ENTRY_wiredtiger_open_usercfg 46 /* * configuration section: END * DO NOT EDIT: automatically built by dist/flags.py. diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index 7d3d07a6abc..f74732684f5 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -107,7 +107,7 @@ struct __wt_named_extractor { * Allocate some additional slots for internal sessions so the user cannot * configure too few sessions for us to run. */ -#define WT_EXTRA_INTERNAL_SESSIONS 10 +#define WT_EXTRA_INTERNAL_SESSIONS 20 /* * WT_CONN_CHECK_PANIC -- @@ -123,15 +123,19 @@ struct __wt_named_extractor { * main queue and the hashed queue. */ #define WT_CONN_DHANDLE_INSERT(conn, dhandle, bucket) do { \ + WT_ASSERT(session, \ + F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE)); \ TAILQ_INSERT_HEAD(&(conn)->dhqh, dhandle, q); \ TAILQ_INSERT_HEAD(&(conn)->dhhash[bucket], dhandle, hashq); \ - ++conn->dhandle_count; \ + ++(conn)->dhandle_count; \ } while (0) #define WT_CONN_DHANDLE_REMOVE(conn, dhandle, bucket) do { \ + WT_ASSERT(session, \ + F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE)); \ TAILQ_REMOVE(&(conn)->dhqh, dhandle, q); \ TAILQ_REMOVE(&(conn)->dhhash[bucket], dhandle, hashq); \ - --conn->dhandle_count; \ + --(conn)->dhandle_count; \ } while (0) /* @@ -163,28 +167,13 @@ struct __wt_connection_impl { WT_SPINLOCK api_lock; /* Connection API spinlock */ WT_SPINLOCK checkpoint_lock; /* Checkpoint spinlock */ - WT_SPINLOCK dhandle_lock; /* Data handle list spinlock */ WT_SPINLOCK fh_lock; /* File handle queue spinlock */ WT_SPINLOCK metadata_lock; /* Metadata update spinlock */ WT_SPINLOCK reconfig_lock; /* Single thread reconfigure */ WT_SPINLOCK schema_lock; /* Schema operation spinlock */ - WT_SPINLOCK table_lock; /* Table creation spinlock */ + WT_RWLOCK table_lock; /* Table list lock */ WT_SPINLOCK turtle_lock; /* Turtle file spinlock */ - - /* - * We distribute the btree page locks across a set of spin locks. Don't - * use too many: they are only held for very short operations, each one - * is 64 bytes, so 256 will fill the L1 cache on most CPUs. - * - * Use a prime number of buckets rather than assuming a good hash - * (Reference Sedgewick, Algorithms in C, "Hash Functions"). - * - * Note: this can't be an array, we impose cache-line alignment and gcc - * doesn't support that for arrays smaller than the alignment. - */ -#define WT_PAGE_LOCKS 17 - WT_SPINLOCK *page_lock; /* Btree page spinlocks */ - u_int page_lock_cnt; /* Next spinlock to use */ + WT_RWLOCK dhandle_lock; /* Data handle list lock */ /* Connection queue */ TAILQ_ENTRY(__wt_connection_impl) q; @@ -255,12 +244,6 @@ struct __wt_connection_impl { size_t session_scratch_max; /* Max scratch memory per session */ - /* - * WiredTiger allocates space for a fixed number of hazard pointers - * in each thread of control. - */ - uint32_t hazard_max; /* Hazard array size */ - WT_CACHE *cache; /* Page cache */ volatile uint64_t cache_size; /* Cache size (either statically configured or the current size @@ -268,7 +251,7 @@ struct __wt_connection_impl { WT_TXN_GLOBAL txn_global; /* Global transaction state */ - WT_RWLOCK *hot_backup_lock; /* Hot backup serialization */ + WT_RWLOCK hot_backup_lock; /* Hot backup serialization */ bool hot_backup; /* Hot backup in progress */ char **hot_backup_list; /* Hot backup file list */ @@ -307,6 +290,15 @@ struct __wt_connection_impl { uint32_t evict_threads_max;/* Max eviction threads */ uint32_t evict_threads_min;/* Min eviction threads */ + uint32_t evict_tune_datapts_needed;/* Data needed to tune */ + struct timespec evict_tune_last_action_time;/* Time of last action */ + struct timespec evict_tune_last_time; /* Time of last check */ + uint32_t evict_tune_num_points; /* Number of values tried */ + uint64_t evict_tune_pgs_last; /* Number of pages evicted */ + uint64_t evict_tune_pg_sec_max; /* Max throughput encountered */ + bool evict_tune_stable; /* Are we stable? */ + uint32_t evict_tune_workers_best;/* Best performing value */ + #define WT_STATLOG_FILENAME "WiredTigerStat.%d.%H" WT_SESSION_IMPL *stat_session; /* Statistics log session */ wt_thread_t stat_tid; /* Statistics log thread */ @@ -332,11 +324,11 @@ struct __wt_connection_impl { bool log_tid_set; /* Log server thread set */ WT_CONDVAR *log_file_cond; /* Log file thread wait mutex */ WT_SESSION_IMPL *log_file_session;/* Log file thread session */ - wt_thread_t log_file_tid; /* Log file thread thread */ + wt_thread_t log_file_tid; /* Log file thread */ bool log_file_tid_set;/* Log file thread set */ WT_CONDVAR *log_wrlsn_cond;/* Log write lsn thread wait mutex */ WT_SESSION_IMPL *log_wrlsn_session;/* Log write lsn thread session */ - wt_thread_t log_wrlsn_tid; /* Log write lsn thread thread */ + wt_thread_t log_wrlsn_tid; /* Log write lsn thread */ bool log_wrlsn_tid_set;/* Log write lsn thread set */ WT_LOG *log; /* Logging structure */ WT_COMPRESSOR *log_compressor;/* Logging compressor */ diff --git a/src/third_party/wiredtiger/src/include/cursor.h b/src/third_party/wiredtiger/src/include/cursor.h index d522abc2a56..f32b4250d30 100644 --- a/src/third_party/wiredtiger/src/include/cursor.h +++ b/src/third_party/wiredtiger/src/include/cursor.h @@ -52,8 +52,8 @@ { 0 }, /* recno raw buffer */ \ NULL, /* json_private */ \ NULL, /* lang_private */ \ - { NULL, 0, 0, NULL, 0 }, /* WT_ITEM key */ \ - { NULL, 0, 0, NULL, 0 }, /* WT_ITEM value */ \ + { NULL, 0, NULL, 0, 0 }, /* WT_ITEM key */ \ + { NULL, 0, NULL, 0, 0 }, /* WT_ITEM value */ \ 0, /* int saved_err */ \ NULL, /* internal_uri */ \ 0 /* uint32_t flags */ \ @@ -73,7 +73,7 @@ struct __wt_cursor_backup { #define WT_CURBACKUP_LOCKER 0x01 /* Hot-backup started */ uint8_t flags; }; -#define WT_CURSOR_BACKUP_ID(cursor) (((WT_CURSOR_BACKUP *)cursor)->maxid) +#define WT_CURSOR_BACKUP_ID(cursor) (((WT_CURSOR_BACKUP *)(cursor))->maxid) struct __wt_cursor_btree { WT_CURSOR iface; @@ -474,7 +474,7 @@ struct __wt_cursor_stat { * Return a reference to a statistic cursor's stats structures. */ #define WT_CURSOR_STATS(cursor) \ - (((WT_CURSOR_STAT *)cursor)->stats) + (((WT_CURSOR_STAT *)(cursor))->stats) struct __wt_cursor_table { WT_CURSOR iface; @@ -493,7 +493,7 @@ struct __wt_cursor_table { }; #define WT_CURSOR_PRIMARY(cursor) \ - (((WT_CURSOR_TABLE *)cursor)->cg_cursors[0]) + (((WT_CURSOR_TABLE *)(cursor))->cg_cursors[0]) #define WT_CURSOR_RECNO(cursor) WT_STREQ((cursor)->key_format, "r") @@ -550,4 +550,4 @@ struct __wt_cursor_table { } while (0) #define WT_CURSOR_RAW_OK \ - WT_CURSTD_DUMP_HEX | WT_CURSTD_DUMP_PRINT | WT_CURSTD_RAW + (WT_CURSTD_DUMP_HEX | WT_CURSTD_DUMP_PRINT | WT_CURSTD_RAW) diff --git a/src/third_party/wiredtiger/src/include/cursor.i b/src/third_party/wiredtiger/src/include/cursor.i index e142441e0a6..12044e0e228 100644 --- a/src/third_party/wiredtiger/src/include/cursor.i +++ b/src/third_party/wiredtiger/src/include/cursor.i @@ -76,34 +76,19 @@ __cursor_leave(WT_SESSION_IMPL *session) } /* - * __curfile_enter -- - * Activate a file cursor. - */ -static inline int -__curfile_enter(WT_CURSOR_BTREE *cbt) -{ - WT_SESSION_IMPL *session; - - session = (WT_SESSION_IMPL *)cbt->iface.session; - - if (!F_ISSET(cbt, WT_CBT_NO_TXN)) - WT_RET(__cursor_enter(session)); - F_SET(cbt, WT_CBT_ACTIVE); - return (0); -} - -/* - * __curfile_leave -- - * Clear a file cursor's position. + * __cursor_reset -- + * Reset the cursor, it no longer holds any position. */ static inline int -__curfile_leave(WT_CURSOR_BTREE *cbt) +__cursor_reset(WT_CURSOR_BTREE *cbt) { WT_DECL_RET; WT_SESSION_IMPL *session; session = (WT_SESSION_IMPL *)cbt->iface.session; + __cursor_pos_clear(cbt); + /* If the cursor was active, deactivate it. */ if (F_ISSET(cbt, WT_CBT_ACTIVE)) { if (!F_ISSET(cbt, WT_CBT_NO_TXN)) @@ -111,12 +96,15 @@ __curfile_leave(WT_CURSOR_BTREE *cbt) F_CLR(cbt, WT_CBT_ACTIVE); } + /* If we're not holding a cursor reference, we're done. */ + if (cbt->ref == NULL) + return (0); + /* * If we were scanning and saw a lot of deleted records on this page, * try to evict the page when we release it. */ - if (cbt->ref != NULL && - cbt->page_deleted_count > WT_BTREE_DELETE_THRESHOLD) + if (cbt->page_deleted_count > WT_BTREE_DELETE_THRESHOLD) __wt_page_evict_soon(session, cbt->ref); cbt->page_deleted_count = 0; @@ -247,7 +235,7 @@ __cursor_func_init(WT_CURSOR_BTREE *cbt, bool reenter) #ifdef HAVE_DIAGNOSTIC __wt_cursor_key_order_reset(cbt); #endif - WT_RET(__curfile_leave(cbt)); + WT_RET(__cursor_reset(cbt)); } /* @@ -259,36 +247,22 @@ __cursor_func_init(WT_CURSOR_BTREE *cbt, bool reenter) /* If the transaction is idle, check that the cache isn't full. */ WT_RET(__wt_txn_idle_cache_check(session)); - if (!F_ISSET(cbt, WT_CBT_ACTIVE)) - WT_RET(__curfile_enter(cbt)); + /* Activate the file cursor. */ + if (!F_ISSET(cbt, WT_CBT_ACTIVE)) { + if (!F_ISSET(cbt, WT_CBT_NO_TXN)) + WT_RET(__cursor_enter(session)); + F_SET(cbt, WT_CBT_ACTIVE); + } /* * If this is an ordinary transactional cursor, make sure we are set up * to read. */ if (!F_ISSET(cbt, WT_CBT_NO_TXN)) - WT_RET(__wt_txn_cursor_op(session)); + __wt_txn_cursor_op(session); return (0); } -/* - * __cursor_reset -- - * Reset the cursor. - */ -static inline int -__cursor_reset(WT_CURSOR_BTREE *cbt) -{ - WT_DECL_RET; - - /* - * The cursor is leaving the API, and no longer holds any position, - * generally called to clean up the cursor after an error. - */ - ret = __curfile_leave(cbt); - __cursor_pos_clear(cbt); - return (ret); -} - /* * __cursor_row_slot_return -- * Return a row-store leaf page slot's K/V pair. diff --git a/src/third_party/wiredtiger/src/include/dhandle.h b/src/third_party/wiredtiger/src/include/dhandle.h index 001eb268f86..8861e96112b 100644 --- a/src/third_party/wiredtiger/src/include/dhandle.h +++ b/src/third_party/wiredtiger/src/include/dhandle.h @@ -37,12 +37,30 @@ #define WT_SESSION_META_DHANDLE(s) \ (((WT_CURSOR_BTREE *)((s)->meta_cursor))->btree->dhandle) +#define WT_DHANDLE_ACQUIRE(dhandle) \ + (void)__wt_atomic_add32(&(dhandle)->session_ref, 1) + +#define WT_DHANDLE_RELEASE(dhandle) \ + (void)__wt_atomic_sub32(&(dhandle)->session_ref, 1) + +#define WT_DHANDLE_NEXT(session, dhandle, head, field) do { \ + WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST));\ + if ((dhandle) == NULL) \ + (dhandle) = TAILQ_FIRST(head); \ + else { \ + WT_DHANDLE_RELEASE(dhandle); \ + (dhandle) = TAILQ_NEXT(dhandle, field); \ + } \ + if ((dhandle) != NULL) \ + WT_DHANDLE_ACQUIRE(dhandle); \ +} while (0) + /* * WT_DATA_HANDLE -- * A handle for a generic named data source. */ struct __wt_data_handle { - WT_RWLOCK *rwlock; /* Lock for shared/exclusive ops */ + WT_RWLOCK rwlock; /* Lock for shared/exclusive ops */ TAILQ_ENTRY(__wt_data_handle) q; TAILQ_ENTRY(__wt_data_handle) hashq; @@ -62,6 +80,8 @@ struct __wt_data_handle { const char *checkpoint; /* Checkpoint name (or NULL) */ const char **cfg; /* Configuration information */ + bool compact_skip; /* If the handle failed to compact */ + WT_DATA_SOURCE *dsrc; /* Data source for this handle */ void *handle; /* Generic handle */ diff --git a/src/third_party/wiredtiger/src/include/error.h b/src/third_party/wiredtiger/src/include/error.h index bbb7f989332..c338acb370f 100644 --- a/src/third_party/wiredtiger/src/include/error.h +++ b/src/third_party/wiredtiger/src/include/error.h @@ -67,14 +67,16 @@ int __ret; \ if ((__ret = (a)) != 0 && \ (__ret == WT_PANIC || \ - ret == 0 || ret == WT_DUPLICATE_KEY || ret == WT_NOTFOUND)) \ + ret == 0 || ret == WT_DUPLICATE_KEY || \ + ret == WT_NOTFOUND || ret == WT_RESTART)) \ ret = __ret; \ } while (0) #define WT_TRET_ERROR_OK(a, e) do { \ int __ret; \ if ((__ret = (a)) != 0 && __ret != (e) && \ (__ret == WT_PANIC || \ - ret == 0 || ret == WT_DUPLICATE_KEY || ret == WT_NOTFOUND)) \ + ret == 0 || ret == WT_DUPLICATE_KEY || \ + ret == WT_NOTFOUND || ret == WT_RESTART)) \ ret = __ret; \ } while (0) #define WT_TRET_NOTFOUND_OK(a) WT_TRET_ERROR_OK(a, WT_NOTFOUND) diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index 6234f2f6bc5..bf3279d0f94 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -14,7 +14,7 @@ extern int __wt_block_buffer_to_addr(WT_BLOCK *block, const uint8_t *p, wt_off_t extern int __wt_block_addr_invalid(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, bool live) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_addr_string(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_buffer_to_ckpt(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *p, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_block_ckpt_decode(WT_SESSION *wt_session, size_t allocsize, const uint8_t *p, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_block_ckpt_decode(WT_SESSION *wt_session, size_t allocsize, const uint8_t *p, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_ckpt_to_buffer(WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t **pp, WT_BLOCK_CKPT *ci) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_ckpt_init( WT_SESSION_IMPL *session, WT_BLOCK_CKPT *ci, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, uint8_t *root_addr, size_t *root_addr_sizep, bool checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -79,18 +79,18 @@ extern int __wt_block_discard(WT_SESSION_IMPL *session, WT_BLOCK *block, size_t extern int __wt_block_write_size(WT_SESSION_IMPL *session, WT_BLOCK *block, size_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_write(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, uint8_t *addr, size_t *addr_sizep, bool data_checksum, bool checkpoint_io) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_block_write_off(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, wt_off_t *offsetp, uint32_t *sizep, uint32_t *checksump, bool data_checksum, bool checkpoint_io, bool caller_locked) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_bloom_create( WT_SESSION_IMPL *session, const char *uri, const char *config, uint64_t count, uint32_t factor, uint32_t k, WT_BLOOM **bloomp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_bloom_open(WT_SESSION_IMPL *session, const char *uri, uint32_t factor, uint32_t k, WT_CURSOR *owner, WT_BLOOM **bloomp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_bloom_insert(WT_BLOOM *bloom, WT_ITEM *key); -extern int __wt_bloom_finalize(WT_BLOOM *bloom) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bloom_create( WT_SESSION_IMPL *session, const char *uri, const char *config, uint64_t count, uint32_t factor, uint32_t k, WT_BLOOM **bloomp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bloom_open(WT_SESSION_IMPL *session, const char *uri, uint32_t factor, uint32_t k, WT_CURSOR *owner, WT_BLOOM **bloomp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_bloom_insert(WT_BLOOM *bloom, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern int __wt_bloom_finalize(WT_BLOOM *bloom) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_bloom_hash(WT_BLOOM *bloom, WT_ITEM *key, WT_BLOOM_HASH *bhash); extern int __wt_bloom_hash_get(WT_BLOOM *bloom, WT_BLOOM_HASH *bhash) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_bloom_get(WT_BLOOM *bloom, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bloom_get(WT_BLOOM *bloom, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bloom_inmem_get(WT_BLOOM *bloom, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bloom_intersection(WT_BLOOM *bloom, WT_BLOOM *other) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_bloom_close(WT_BLOOM *bloom) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_bloom_drop(WT_BLOOM *bloom, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_compact(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bloom_close(WT_BLOOM *bloom) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_bloom_drop(WT_BLOOM *bloom, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_compact(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_compact_page_skip(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_key_order_check( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, bool next) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_key_order_init(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -98,14 +98,14 @@ extern void __wt_cursor_key_order_reset(WT_CURSOR_BTREE *cbt); extern void __wt_btcur_iterate_setup(WT_CURSOR_BTREE *cbt); extern int __wt_btcur_next(WT_CURSOR_BTREE *cbt, bool truncating) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_prev(WT_CURSOR_BTREE *cbt, bool truncating) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern bool __wt_cursor_valid(WT_CURSOR_BTREE *cbt, WT_UPDATE **updp); extern int __wt_btcur_reset(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_search(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_insert(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_btcur_update_check(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btcur_insert_check(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_remove(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_update(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_btcur_next_random(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_compare(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *cmpp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_equals(WT_CURSOR_BTREE *a_arg, WT_CURSOR_BTREE *b_arg, int *equalp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btcur_range_truncate(WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -126,6 +126,7 @@ extern int __wt_delete_page(WT_SESSION_IMPL *session, WT_REF *ref, bool *skipp) extern void __wt_delete_page_rollback(WT_SESSION_IMPL *session, WT_REF *ref); extern bool __wt_delete_page_skip(WT_SESSION_IMPL *session, WT_REF *ref, bool visible_all); extern int __wt_delete_page_instantiate(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_ref_out_int(WT_SESSION_IMPL *session, WT_REF *ref, bool rewrite); extern void __wt_ref_out(WT_SESSION_IMPL *session, WT_REF *ref); extern void __wt_page_out(WT_SESSION_IMPL *session, WT_PAGE **pagep); extern void __wt_free_ref( WT_SESSION_IMPL *session, WT_REF *ref, int page_type, bool free_pages); @@ -133,15 +134,15 @@ extern void __wt_free_ref_index(WT_SESSION_IMPL *session, WT_PAGE *page, WT_PAGE extern void __wt_free_update_list(WT_SESSION_IMPL *session, WT_UPDATE *upd); extern int __wt_btree_open(WT_SESSION_IMPL *session, const char *op_cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btree_close(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btree_discard(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_root_ref_init(WT_REF *root_ref, WT_PAGE *root, bool is_recno); extern int __wt_btree_tree_open( WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_btree_new_leaf_page(WT_SESSION_IMPL *session, WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_btree_evictable(WT_SESSION_IMPL *session, bool on); extern int __wt_btree_huffman_open(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_btree_huffman_close(WT_SESSION_IMPL *session); extern int __wt_bt_read(WT_SESSION_IMPL *session, WT_ITEM *buf, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bt_write(WT_SESSION_IMPL *session, WT_ITEM *buf, uint8_t *addr, size_t *addr_sizep, bool checkpoint, bool checkpoint_io, bool compressed) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern const char *__wt_page_type_string(u_int type); +extern const char *__wt_page_type_string(u_int type) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern const char *__wt_cell_type_string(uint8_t type); extern const char *__wt_page_addr_string(WT_SESSION_IMPL *session, WT_REF *ref, WT_ITEM *buf); extern const char *__wt_addr_string(WT_SESSION_IMPL *session, const uint8_t *addr, size_t addr_size, WT_ITEM *buf); @@ -150,6 +151,9 @@ extern int __wt_ovfl_cache(WT_SESSION_IMPL *session, WT_PAGE *page, void *cookie extern int __wt_ovfl_discard(WT_SESSION_IMPL *session, WT_CELL *cell) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_alloc(WT_SESSION_IMPL *session, uint8_t type, uint32_t alloc_entries, bool alloc_refs, WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_inmem(WT_SESSION_IMPL *session, WT_REF *ref, const void *image, size_t memsize, uint32_t flags, WT_PAGE **pagep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_row_random_leaf(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_random_descent(WT_SESSION_IMPL *session, WT_REF **refp, bool eviction) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_btcur_next_random(WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_las_remove_block(WT_SESSION_IMPL *session, WT_CURSOR *cursor, uint32_t btree_id, const uint8_t *addr, size_t addr_size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags @@ -158,8 +162,10 @@ __wt_page_in_func(WT_SESSION_IMPL *session, WT_REF *ref, uint32_t flags #endif ); extern int __wt_bt_rebalance(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_key_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_kv_return(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_UPDATE *upd) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bt_salvage(WT_SESSION_IMPL *session, WT_CKPT *ckptbase, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern bool __wt_split_obsolete(WT_SESSION_IMPL *session, uint64_t split_gen); extern void __wt_split_stash_discard(WT_SESSION_IMPL *session); extern void __wt_split_stash_discard_all( WT_SESSION_IMPL *session_safe, WT_SESSION_IMPL *session); extern int __wt_multi_to_ref(WT_SESSION_IMPL *session, WT_PAGE *page, WT_MULTI *multi, WT_REF **refp, size_t *incrp, bool closing) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -176,7 +182,7 @@ extern int __wt_verify_dsk(WT_SESSION_IMPL *session, const char *tag, WT_ITEM *b extern int __wt_tree_walk(WT_SESSION_IMPL *session, WT_REF **refp, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_tree_walk_count(WT_SESSION_IMPL *session, WT_REF **refp, uint64_t *walkcntp, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_tree_walk_skip(WT_SESSION_IMPL *session, WT_REF **refp, uint64_t *skipleafcntp, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, uint64_t recno, WT_ITEM *value, WT_UPDATE *upd_arg, bool is_remove) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, uint64_t recno, WT_ITEM *value, WT_UPDATE *upd_arg, bool is_remove, bool exclusive) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_col_search(WT_SESSION_IMPL *session, uint64_t search_recno, WT_REF *leaf, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_row_leaf_keys(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_row_leaf_key_copy( WT_SESSION_IMPL *session, WT_PAGE *page, WT_ROW *rip, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -185,15 +191,13 @@ extern int __wt_row_ikey_alloc(WT_SESSION_IMPL *session, uint32_t cell_offset, c extern int __wt_row_ikey_incr(WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t cell_offset, const void *key, size_t size, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_row_ikey(WT_SESSION_IMPL *session, uint32_t cell_offset, const void *key, size_t size, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_modify_alloc(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *key, WT_ITEM *value, WT_UPDATE *upd_arg, bool is_remove) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_row_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_ITEM *key, WT_ITEM *value, WT_UPDATE *upd_arg, bool is_remove, bool exclusive) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_row_insert_alloc(WT_SESSION_IMPL *session, WT_ITEM *key, u_int skipdepth, WT_INSERT **insp, size_t *ins_sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_update_alloc( WT_SESSION_IMPL *session, WT_ITEM *value, WT_UPDATE **updp, size_t *sizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern WT_UPDATE *__wt_update_obsolete_check( WT_SESSION_IMPL *session, WT_PAGE *page, WT_UPDATE *upd); extern void __wt_update_obsolete_free( WT_SESSION_IMPL *session, WT_PAGE *page, WT_UPDATE *upd); extern int __wt_search_insert(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, WT_INSERT_HEAD *ins_head, WT_ITEM *srch_key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_row_search(WT_SESSION_IMPL *session, WT_ITEM *srch_key, WT_REF *leaf, WT_CURSOR_BTREE *cbt, bool insert) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_row_random_leaf(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_row_random_descent(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_las_stats_update(WT_SESSION_IMPL *session); extern int __wt_las_create(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_las_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -203,7 +207,7 @@ extern int __wt_las_cursor_open(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) W extern void __wt_las_cursor( WT_SESSION_IMPL *session, WT_CURSOR **cursorp, uint32_t *session_flags); extern int __wt_las_cursor_close( WT_SESSION_IMPL *session, WT_CURSOR **cursorp, uint32_t session_flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_las_sweep(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern uint32_t __wt_checksum_sw(const void *chunk, size_t len); +extern uint32_t __wt_checksum_sw(const void *chunk, size_t len) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern void __wt_checksum_init(void); extern void __wt_config_initn( WT_SESSION_IMPL *session, WT_CONFIG *conf, const char *str, size_t len); extern void __wt_config_init(WT_SESSION_IMPL *session, WT_CONFIG *conf, const char *str); @@ -218,16 +222,19 @@ extern int __wt_config_getones_none(WT_SESSION_IMPL *session, const char *config extern int __wt_config_gets_def(WT_SESSION_IMPL *session, const char **cfg, const char *key, int def, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_subgetraw(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cfg, WT_CONFIG_ITEM *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_subgets(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *cfg, const char *key, WT_CONFIG_ITEM *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_conn_foc_add(WT_SESSION_IMPL *session, const void *p); extern void __wt_conn_foc_discard(WT_SESSION_IMPL *session); extern int __wt_configure_method(WT_SESSION_IMPL *session, const char *method, const char *uri, const char *config, const char *type, const char *check) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_check(WT_SESSION_IMPL *session, const WT_CONFIG_ENTRY *entry, const char *config, size_t config_len) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_collapse( WT_SESSION_IMPL *session, const char **cfg, char **config_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_config_merge(WT_SESSION_IMPL *session, const char **cfg, const char *cfg_strip, const char **config_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_config_merge(WT_SESSION_IMPL *session, const char **cfg, const char *cfg_strip, const char **config_ret) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_config_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_conn_config_discard(WT_SESSION_IMPL *session); extern const WT_CONFIG_ENTRY *__wt_conn_config_match(const char *method); -extern int __wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_config_get(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, WT_CONFIG_ARG *cfg_arg, const char *key, WT_CONFIG_ITEM *cval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_config_get_string(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *config, const char *key, WT_CONFIG_ITEM *cval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_config_parser_open(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_ext_config_parser_open_arg(WT_EXTENSION_API *wt_ext, WT_SESSION *wt_session, WT_CONFIG_ARG *cfg_arg, WT_CONFIG_PARSER **config_parserp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_config_upgrade(WT_SESSION_IMPL *session, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_wiredtiger_error(int error); extern int __wt_collator_config(WT_SESSION_IMPL *session, const char *uri, WT_CONFIG_ITEM *cname, WT_CONFIG_ITEM *metadata, WT_COLLATOR **collatorp, int *ownp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -251,6 +258,7 @@ extern WT_THREAD_RET __wt_cache_pool_server(void *arg); extern int __wt_checkpoint_server_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_checkpoint_server_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_checkpoint_signal(WT_SESSION_IMPL *session, wt_off_t logsize); +extern int __wt_conn_dhandle_alloc( WT_SESSION_IMPL *session, const char *uri, const char *checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_dhandle_find( WT_SESSION_IMPL *session, const char *uri, const char *checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_btree_sync_and_close(WT_SESSION_IMPL *session, bool final, bool force) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_btree_open( WT_SESSION_IMPL *session, const char *cfg[], uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -259,7 +267,7 @@ extern int __wt_conn_dhandle_close_all( WT_SESSION_IMPL *session, const char *ur extern int __wt_conn_dhandle_discard_single( WT_SESSION_IMPL *session, bool final, bool force) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_conn_dhandle_discard(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_connection_init(WT_CONNECTION_IMPL *conn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_connection_destroy(WT_CONNECTION_IMPL *conn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_connection_destroy(WT_CONNECTION_IMPL *conn); extern int __wt_logmgr_reconfig(WT_SESSION_IMPL *session, const char **cfg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_truncate_files( WT_SESSION_IMPL *session, WT_CURSOR *cursor, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_log_wrlsn(WT_SESSION_IMPL *session, int *yield); @@ -282,7 +290,7 @@ extern int __wt_curconfig_open(WT_SESSION_IMPL *session, const char *uri, const extern int __wt_curds_open( WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_DATA_SOURCE *dsrc, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curdump_create(WT_CURSOR *child, WT_CURSOR *owner, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curfile_next_random(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_curfile_update_check(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curfile_insert_check(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curfile_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curindex_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curjoin_joined(WT_CURSOR *cursor) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -290,13 +298,13 @@ extern int __wt_curjoin_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSO extern int __wt_curjoin_join(WT_SESSION_IMPL *session, WT_CURSOR_JOIN *cjoin, WT_INDEX *idx, WT_CURSOR *ref_cursor, uint8_t flags, uint8_t range, uint64_t count, uint32_t bloom_bit_count, uint32_t bloom_hash_count) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_json_alloc_unpack(WT_SESSION_IMPL *session, const void *buffer, size_t size, const char *fmt, WT_CURSOR_JSON *json, bool iskey, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_json_close(WT_SESSION_IMPL *session, WT_CURSOR *cursor); -extern size_t __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode); -extern void __wt_json_column_init(WT_CURSOR *cursor, const char *keyformat, const WT_CONFIG_ITEM *idxconf, const WT_CONFIG_ITEM *colconf); -extern int __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, const char **tokstart, size_t *toklen) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern const char *__wt_json_tokname(int toktype); +extern size_t __wt_json_unpack_char(u_char ch, u_char *buf, size_t bufsz, bool force_unicode) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern void __wt_json_column_init(WT_CURSOR *cursor, const char *uri, const char *keyformat, const WT_CONFIG_ITEM *idxconf, const WT_CONFIG_ITEM *colconf); +extern int __wt_json_token(WT_SESSION *wt_session, const char *src, int *toktype, const char **tokstart, size_t *toklen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern const char *__wt_json_tokname(int toktype) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_json_to_item(WT_SESSION_IMPL *session, const char *jstr, const char *format, WT_CURSOR_JSON *json, bool iskey, WT_ITEM *item) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern ssize_t __wt_json_strlen(const char *src, size_t srclen); -extern int __wt_json_strncpy(WT_SESSION *wt_session, char **pdst, size_t dstlen, const char *src, size_t srclen) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern ssize_t __wt_json_strlen(const char *src, size_t srclen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern int __wt_json_strncpy(WT_SESSION *wt_session, char **pdst, size_t dstlen, const char *src, size_t srclen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curlog_open(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curmetadata_open(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_curstat_dsrc_final(WT_CURSOR_STAT *cst); @@ -312,7 +320,7 @@ extern int __wt_cursor_equals_notsup(WT_CURSOR *cursor, WT_CURSOR *other, int *e extern int __wt_cursor_search_near_notsup(WT_CURSOR *cursor, int *exact) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_reconfigure_notsup(WT_CURSOR *cursor, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_cursor_set_notsup(WT_CURSOR *cursor); -extern int __wt_cursor_kv_not_set(WT_CURSOR *cursor, bool key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cursor_kv_not_set(WT_CURSOR *cursor, bool key) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_cursor_get_key(WT_CURSOR *cursor, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_cursor_set_key(WT_CURSOR *cursor, ...); extern int __wt_cursor_get_raw_key(WT_CURSOR *cursor, WT_ITEM *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -349,7 +357,7 @@ extern int __wt_cache_eviction_worker(WT_SESSION_IMPL *session, bool busy, u_int extern bool __wt_page_evict_urgent(WT_SESSION_IMPL *session, WT_REF *ref); extern void __wt_evict_priority_set(WT_SESSION_IMPL *session, uint64_t v); extern void __wt_evict_priority_clear(WT_SESSION_IMPL *session); -extern int __wt_cache_dump(WT_SESSION_IMPL *session, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_verbose_dump_cache(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_release_evict(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, bool closing) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_curstat_cache_walk(WT_SESSION_IMPL *session); @@ -396,11 +404,10 @@ extern int __wt_logop_row_truncate_unpack( WT_SESSION_IMPL *session, const uint8 extern int __wt_logop_row_truncate_print(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_op_printlog(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_log_slot_activate(WT_SESSION_IMPL *session, WT_LOGSLOT *slot); -extern int __wt_log_slot_switch( WT_SESSION_IMPL *session, WT_MYSLOT *myslot, bool retry, bool forced) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_log_slot_new(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_slot_switch(WT_SESSION_IMPL *session, WT_MYSLOT *myslot, bool retry, bool forced, bool *did_work) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_slot_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_slot_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, uint32_t flags, WT_MYSLOT *myslot); +extern int __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, uint32_t flags, WT_MYSLOT *myslot) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int64_t __wt_log_slot_release(WT_SESSION_IMPL *session, WT_MYSLOT *myslot, int64_t size); extern void __wt_log_slot_free(WT_SESSION_IMPL *session, WT_LOGSLOT *slot); extern int __wt_clsm_request_switch(WT_CURSOR_LSM *clsm) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -420,7 +427,7 @@ extern int __wt_lsm_manager_push_entry(WT_SESSION_IMPL *session, uint32_t type, extern int __wt_lsm_merge_update_tree(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int start_chunk, u_int nchunks, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_merge(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int id) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_meta_read(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, const char *newconfig) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_curstat_lsm_init( WT_SESSION_IMPL *session, const char *uri, WT_CURSOR_STAT *cst) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_close_all(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_bloom_name(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, uint32_t id, const char **retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -434,6 +441,7 @@ extern void __wt_lsm_tree_release(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tre extern void __wt_lsm_tree_throttle( WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool decrease_only); extern int __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_retire_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int start_chunk, u_int nchunks) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_tree_alter( WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_drop( WT_SESSION_IMPL *session, const char *name, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_rename(WT_SESSION_IMPL *session, const char *olduri, const char *newuri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_tree_truncate( WT_SESSION_IMPL *session, const char *name, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -449,20 +457,21 @@ extern int __wt_lsm_work_bloom(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) extern int __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_free_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_lsm_worker_start(WT_SESSION_IMPL *session, WT_LSM_WORKER_ARGS *args) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_lsm_worker_stop(WT_SESSION_IMPL *session, WT_LSM_WORKER_ARGS *args) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_apply_all(WT_SESSION_IMPL *session, int (*file_func)(WT_SESSION_IMPL *, const char *[]), int (*name_func)(WT_SESSION_IMPL *, const char *, bool *), const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_checkpoint(WT_SESSION_IMPL *session, const char *fname, const char *checkpoint, WT_CKPT *ckpt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_checkpoint_last_name( WT_SESSION_IMPL *session, const char *fname, const char **namep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_checkpoint_clear(WT_SESSION_IMPL *session, const char *fname) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_ckptlist_get( WT_SESSION_IMPL *session, const char *fname, WT_CKPT **ckptbasep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_meta_ckptlist_set(WT_SESSION_IMPL *session, const char *fname, WT_CKPT *ckptbase, WT_LSN *ckptlsn) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_meta_ckptlist_free(WT_SESSION_IMPL *session, WT_CKPT *ckptbase); +extern void __wt_meta_ckptlist_free(WT_SESSION_IMPL *session, WT_CKPT **ckptbasep); extern void __wt_meta_checkpoint_free(WT_SESSION_IMPL *session, WT_CKPT *ckpt); extern int __wt_ext_metadata_insert(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key, const char *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_metadata_remove( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_metadata_search(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key, char **valuep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_metadata_update(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *key, const char *value) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_metadata_get_ckptlist( WT_SESSION *session, const char *name, WT_CKPT **ckptbasep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_metadata_free_ckptlist(WT_SESSION *session, WT_CKPT *ckptbase); +extern int __wt_metadata_get_ckptlist( WT_SESSION *session, const char *name, WT_CKPT **ckptbasep) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_metadata_free_ckptlist(WT_SESSION *session, WT_CKPT *ckptbase) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_metadata_cursor_open( WT_SESSION_IMPL *session, const char *config, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_metadata_cursor(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_metadata_cursor_release(WT_SESSION_IMPL *session, WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -489,15 +498,15 @@ extern int __wt_turtle_update(WT_SESSION_IMPL *session, const char *key, const c extern int __wt_filename(WT_SESSION_IMPL *session, const char *name, char **path) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_nfilename( WT_SESSION_IMPL *session, const char *name, size_t namelen, char **path) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_remove_if_exists(WT_SESSION_IMPL *session, const char *name, bool durable) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_copy_and_sync(WT_SESSION *wt_session, const char *from, const char *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_copy_and_sync(WT_SESSION *wt_session, const char *from, const char *to) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_abort(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); -extern int __wt_calloc(WT_SESSION_IMPL *session, size_t number, size_t size, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_calloc(WT_SESSION_IMPL *session, size_t number, size_t size, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_malloc(WT_SESSION_IMPL *session, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_realloc(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_realloc_noclear(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_realloc_aligned(WT_SESSION_IMPL *session, size_t *bytes_allocated_ret, size_t bytes_to_allocate, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_strndup(WT_SESSION_IMPL *session, const void *str, size_t len, void *retp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_free_int(WT_SESSION_IMPL *session, const void *p_arg); +extern void __wt_free_int(WT_SESSION_IMPL *session, const void *p_arg) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_errno(void) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_strerror(WT_SESSION_IMPL *session, int error, char *errbuf, size_t errlen); extern int __wt_ext_map_windows_error( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, uint32_t windows_error) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -508,8 +517,8 @@ extern int __wt_close_connection_close(WT_SESSION_IMPL *session) WT_GCC_FUNC_DEC extern int __wt_os_inmemory(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_fopen(WT_SESSION_IMPL *session, const char *name, uint32_t open_flags, uint32_t flags, WT_FSTREAM **fstrp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_os_stdio(WT_SESSION_IMPL *session); -extern int __wt_getopt( const char *progname, int nargc, char *const *nargv, const char *ostr) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern uint64_t __wt_strtouq(const char *nptr, char **endptr, int base); +extern int __wt_getopt( const char *progname, int nargc, char *const *nargv, const char *ostr) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern uint64_t __wt_strtouq(const char *nptr, char **endptr, int base) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_ext_struct_pack(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, void *buffer, size_t size, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_struct_size(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, size_t *sizep, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ext_struct_unpack(WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const void *buffer, size_t size, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -540,7 +549,7 @@ extern int __wt_ovfl_txnc_add(WT_SESSION_IMPL *session, WT_PAGE *page, const uin extern void __wt_ovfl_txnc_free(WT_SESSION_IMPL *session, WT_PAGE *page); extern int __wt_ovfl_track_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_ovfl_track_wrapup_err(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_SALVAGE_COOKIE *salvage, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, WT_SALVAGE_COOKIE *salvage, uint32_t flags, bool *lookaside_retryp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern uint32_t __wt_split_page_size(WT_BTREE *btree, uint32_t maxpagesize); extern int __wt_bulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bulk_wrapup(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -548,6 +557,7 @@ extern int __wt_bulk_insert_row(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) extern int __wt_bulk_insert_fix( WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool deleted) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bulk_insert_fix_bitmap(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bulk_insert_var( WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk, bool deleted) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_alter(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_direct_io_size_check(WT_SESSION_IMPL *session, const char **cfg, const char *config_name, uint32_t *allocsizep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_colgroup_source(WT_SESSION_IMPL *session, WT_TABLE *table, const char *cgname, const char *config, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_index_source(WT_SESSION_IMPL *session, WT_TABLE *table, const char *idxname, const char *config, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -594,11 +604,11 @@ extern int __wt_session_copy_values(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_A extern int __wt_session_release_resources(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_open_cursor(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *owner, const char *cfg[], WT_CURSOR **cursorp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_create( WT_SESSION_IMPL *session, const char *uri, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_session_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_range_truncate(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *start, WT_CURSOR *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern const char *__wt_session_strerror(WT_SESSION *wt_session, int error); extern int __wt_open_session(WT_CONNECTION_IMPL *conn, WT_EVENT_HANDLER *event_handler, const char *config, bool open_metadata, WT_SESSION_IMPL **sessionp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_open_internal_session(WT_CONNECTION_IMPL *conn, const char *name, bool open_metadata, uint32_t session_flags, WT_SESSION_IMPL **sessionp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_session_compact_check_timeout(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_compact( WT_SESSION *wt_session, const char *uri, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_compact_readonly( WT_SESSION *wt_session, const char *uri, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_lock_dhandle( WT_SESSION_IMPL *session, uint32_t flags, bool *is_deadp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -608,17 +618,15 @@ extern void __wt_session_close_cache(WT_SESSION_IMPL *session); extern int __wt_session_get_btree(WT_SESSION_IMPL *session, const char *uri, const char *checkpoint, const char *cfg[], uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_session_lock_checkpoint(WT_SESSION_IMPL *session, const char *checkpoint) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_salvage(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_cond_auto_alloc( WT_SESSION_IMPL *session, const char *name, bool is_signalled, uint64_t min, uint64_t max, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_cond_auto_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond); -extern void __wt_cond_auto_wait_signal( WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool *signalled); -extern void __wt_cond_auto_wait( WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress); -extern int __wt_cond_auto_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_cond_auto_alloc(WT_SESSION_IMPL *session, const char *name, uint64_t min, uint64_t max, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cond_auto_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled); +extern void __wt_cond_auto_wait(WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool (*run_func)(WT_SESSION_IMPL *)); extern int __wt_decrypt(WT_SESSION_IMPL *session, WT_ENCRYPTOR *encryptor, size_t skip, WT_ITEM *in, WT_ITEM *out) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_encrypt(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t skip, WT_ITEM *in, WT_ITEM *out) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_encrypt_size(WT_SESSION_IMPL *session, WT_KEYED_ENCRYPTOR *kencryptor, size_t incoming_size, size_t *sizep); extern void __wt_event_handler_set(WT_SESSION_IMPL *session, WT_EVENT_HANDLER *handler); extern int __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, const char *file_name, int line_number, const char *fmt, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_err(WT_SESSION_IMPL *session, int error, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))); +extern void __wt_err(WT_SESSION_IMPL *session, int error, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern void __wt_errx(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))); extern int __wt_ext_err_printf( WT_EXTENSION_API *wt_api, WT_SESSION *wt_session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_msg(WT_SESSION_IMPL *session, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 2, 3))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -633,9 +641,9 @@ __wt_assert(WT_SESSION_IMPL *session, #ifdef HAVE_DIAGNOSTIC WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)) #endif -; -extern int __wt_panic(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_illegal_value(WT_SESSION_IMPL *session, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); + WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern int __wt_panic(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_illegal_value(WT_SESSION_IMPL *session, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_object_unsupported(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_bad_object_type(WT_SESSION_IMPL *session, const char *uri) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_unexpected_object_type( WT_SESSION_IMPL *session, const char *uri, const char *expect) WT_GCC_FUNC_DECL_ATTRIBUTE((cold)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -650,9 +658,10 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp , const char *file, int line #endif ); -extern int __wt_hazard_clear(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_hazard_clear(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_hazard_close(WT_SESSION_IMPL *session); -extern u_int __wt_hazard_count(WT_SESSION_IMPL *session, WT_PAGE *page); +extern WT_HAZARD *__wt_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref); +extern u_int __wt_hazard_count(WT_SESSION_IMPL *session, WT_REF *ref); extern void __wt_fill_hex(const uint8_t *src, size_t src_max, uint8_t *dest, size_t dest_max, size_t *lenp); extern int __wt_raw_to_hex( WT_SESSION_IMPL *session, const uint8_t *from, size_t size, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_raw_to_esc_hex( WT_SESSION_IMPL *session, const uint8_t *from, size_t size, WT_ITEM *to) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -665,24 +674,24 @@ extern void __wt_huffman_close(WT_SESSION_IMPL *session, void *huffman_arg); extern void __wt_print_huffman_code(void *huffman_arg, uint16_t symbol); extern int __wt_huffman_encode(WT_SESSION_IMPL *session, void *huffman_arg, const uint8_t *from_arg, size_t from_len, WT_ITEM *to_buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_huffman_decode(WT_SESSION_IMPL *session, void *huffman_arg, const uint8_t *from_arg, size_t from_len, WT_ITEM *to_buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_rwlock_alloc( WT_SESSION_IMPL *session, WT_RWLOCK **rwlockp, const char *name) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_try_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_readlock_spin(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); -extern void __wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); -extern void __wt_readunlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); -extern int __wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); -extern void __wt_writeunlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); -extern void __wt_rwlock_destroy(WT_SESSION_IMPL *session, WT_RWLOCK **rwlockp); -extern bool __wt_rwlock_islocked(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock); +extern int __wt_rwlock_init(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_rwlock_destroy(WT_SESSION_IMPL *session, WT_RWLOCK *l); +extern int __wt_try_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l); +extern void __wt_readunlock(WT_SESSION_IMPL *session, WT_RWLOCK *l); +extern int __wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l); +extern void __wt_writeunlock(WT_SESSION_IMPL *session, WT_RWLOCK *l); +extern bool __wt_rwlock_islocked(WT_SESSION_IMPL *session, WT_RWLOCK *l); extern uint32_t __wt_nlpo2_round(uint32_t v); extern uint32_t __wt_nlpo2(uint32_t v); extern uint32_t __wt_log2_int(uint32_t n); extern bool __wt_ispo2(uint32_t v); extern uint32_t __wt_rduppo2(uint32_t n, uint32_t po2); -extern void __wt_random_init(WT_RAND_STATE volatile *rnd_state); -extern void __wt_random_init_seed( WT_SESSION_IMPL *session, WT_RAND_STATE volatile *rnd_state); -extern uint32_t __wt_random(WT_RAND_STATE volatile *rnd_state); +extern void __wt_random_init(WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern void __wt_random_init_seed( WT_SESSION_IMPL *session, WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern uint32_t __wt_random(WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern uint64_t __wt_random64(WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_buf_grow_worker(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_buf_fmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_buf_catfmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -722,8 +731,9 @@ extern int __wt_thread_group_resize( WT_SESSION_IMPL *session, WT_THREAD_GROUP * extern int __wt_thread_group_create( WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, const char *name, uint32_t min, uint32_t max, uint32_t flags, int (*run_func)(WT_SESSION_IMPL *session, WT_THREAD *context)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_group_destroy(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_group_start_one( WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, bool wait) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_thread_group_stop_one( WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, bool wait) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_txn_release_snapshot(WT_SESSION_IMPL *session); -extern int __wt_txn_get_snapshot(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_txn_get_snapshot(WT_SESSION_IMPL *session); extern int __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_config(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_txn_release(WT_SESSION_IMPL *session); @@ -734,8 +744,9 @@ extern void __wt_txn_stats_update(WT_SESSION_IMPL *session); extern void __wt_txn_destroy(WT_SESSION_IMPL *session); extern int __wt_txn_global_init(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_txn_global_destroy(WT_SESSION_IMPL *session); +extern int __wt_verbose_dump_txn(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[], bool waiting) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_checkpoint_sync(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -751,7 +762,7 @@ extern int __wt_txn_checkpoint_logread(WT_SESSION_IMPL *session, const uint8_t * extern int __wt_txn_checkpoint_log( WT_SESSION_IMPL *session, bool full, uint32_t flags, WT_LSN *lsnp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_truncate_log( WT_SESSION_IMPL *session, WT_CURSOR_BTREE *start, WT_CURSOR_BTREE *stop) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_txn_truncate_end(WT_SESSION_IMPL *session); -extern int __wt_txn_printlog(WT_SESSION *wt_session, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_txn_printlog(WT_SESSION *wt_session, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_named_snapshot_begin(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_named_snapshot_drop(WT_SESSION_IMPL *session, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_txn_named_snapshot_get(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *nameval) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/extern_posix.h b/src/third_party/wiredtiger/src/include/extern_posix.h index fd94ef0ddf2..c0ed056c7b6 100644 --- a/src/third_party/wiredtiger/src/include/extern_posix.h +++ b/src/third_party/wiredtiger/src/include/extern_posix.h @@ -7,25 +7,26 @@ extern int __wt_dlsym(WT_SESSION_IMPL *session, WT_DLH *dlh, const char *name, b extern int __wt_dlclose(WT_SESSION_IMPL *session, WT_DLH *dlh) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_posix_file_extend( WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, wt_off_t offset) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_os_posix(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_posix_map(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, void *mapped_regionp, size_t *lenp, void *mapped_cookiep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_posix_map_preload(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, const void *map, size_t length, void *mapped_cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_posix_map_discard(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, void *map, size_t length, void *mapped_cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_posix_unmap(WT_FILE_HANDLE *fh, WT_SESSION *wt_session, void *mapped_region, size_t len, void *mapped_cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, bool is_signalled, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_cond_wait_signal( WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool *signalled); +extern int __wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cond_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled); extern void __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond); -extern int __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp); extern int __wt_once(void (*init_routine)(void)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_get_vm_pagesize(void) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern bool __wt_absolute_path(const char *path); extern const char *__wt_path_separator(void); extern bool __wt_has_priv(void); -extern void __wt_stream_set_line_buffer(FILE *fp); -extern void __wt_stream_set_no_buffer(FILE *fp); -extern void __wt_sleep(uint64_t seconds, uint64_t micro_seconds); +extern void __wt_stream_set_line_buffer(FILE *fp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern void __wt_stream_set_no_buffer(FILE *fp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern void __wt_sleep(uint64_t seconds, uint64_t micro_seconds) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern int __wt_vsnprintf_len_incr( char *buf, size_t size, size_t *retsizep, const char *fmt, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_thread_id(char *buf, size_t buflen); -extern void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp); -extern void __wt_yield(void); +extern int __wt_thread_id(char *buf, size_t buflen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern void __wt_yield(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); diff --git a/src/third_party/wiredtiger/src/include/extern_win.h b/src/third_party/wiredtiger/src/include/extern_win.h index f06ee881ece..d548ee0b2ec 100644 --- a/src/third_party/wiredtiger/src/include/extern_win.h +++ b/src/third_party/wiredtiger/src/include/extern_win.h @@ -10,10 +10,10 @@ extern int __wt_os_win(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((war extern int __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_win_map(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, void *mapped_regionp, size_t *lenp, void *mapped_cookiep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_win_unmap(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session, void *mapped_region, size_t length, void *mapped_cookie) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, bool is_signalled, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_cond_wait_signal( WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool *signalled); +extern int __wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cond_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled); extern void __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond); -extern int __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp); extern int __wt_once(void (*init_routine)(void)) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_get_vm_pagesize(void) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern bool __wt_absolute_path(const char *path); @@ -22,9 +22,10 @@ extern bool __wt_has_priv(void); extern void __wt_stream_set_line_buffer(FILE *fp); extern void __wt_stream_set_no_buffer(FILE *fp); extern void __wt_sleep(uint64_t seconds, uint64_t micro_seconds); +extern int __wt_vsnprintf_len_incr( char *buf, size_t size, size_t *retsizep, const char *fmt, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_thread_id(char *buf, size_t buflen); +extern int __wt_thread_id(char *buf, size_t buflen) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp); extern int __wt_to_utf16_string( WT_SESSION_IMPL *session, const char*utf8, WT_ITEM **outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_to_utf8_string( WT_SESSION_IMPL *session, const wchar_t*wide, WT_ITEM **outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/flags.h b/src/third_party/wiredtiger/src/include/flags.h index b0d167525b2..f26a45c68f5 100644 --- a/src/third_party/wiredtiger/src/include/flags.h +++ b/src/third_party/wiredtiger/src/include/flags.h @@ -6,19 +6,19 @@ #define WT_CONN_CACHE_POOL 0x00000001 #define WT_CONN_CKPT_SYNC 0x00000002 #define WT_CONN_CLOSING 0x00000004 -#define WT_CONN_EVICTION_RUN 0x00000008 -#define WT_CONN_IN_MEMORY 0x00000010 -#define WT_CONN_LAS_OPEN 0x00000020 -#define WT_CONN_LEAK_MEMORY 0x00000040 -#define WT_CONN_LOG_SERVER_RUN 0x00000080 +#define WT_CONN_CLOSING_NO_MORE_OPENS 0x00000008 +#define WT_CONN_EVICTION_RUN 0x00000010 +#define WT_CONN_IN_MEMORY 0x00000020 +#define WT_CONN_LAS_OPEN 0x00000040 +#define WT_CONN_LEAK_MEMORY 0x00000080 #define WT_CONN_LSM_MERGE 0x00000100 #define WT_CONN_PANIC 0x00000200 #define WT_CONN_READONLY 0x00000400 #define WT_CONN_RECOVERING 0x00000800 #define WT_CONN_SERVER_ASYNC 0x00001000 #define WT_CONN_SERVER_CHECKPOINT 0x00002000 -#define WT_CONN_SERVER_LSM 0x00004000 -#define WT_CONN_SERVER_RUN 0x00008000 +#define WT_CONN_SERVER_LOG 0x00004000 +#define WT_CONN_SERVER_LSM 0x00008000 #define WT_CONN_SERVER_STATISTICS 0x00010000 #define WT_CONN_SERVER_SWEEP 0x00020000 #define WT_CONN_WAS_BACKUP 0x00040000 @@ -53,23 +53,24 @@ #define WT_SESSION_CAN_WAIT 0x00000001 #define WT_SESSION_INTERNAL 0x00000002 #define WT_SESSION_LOCKED_CHECKPOINT 0x00000004 -#define WT_SESSION_LOCKED_HANDLE_LIST 0x00000008 -#define WT_SESSION_LOCKED_METADATA 0x00000010 -#define WT_SESSION_LOCKED_PASS 0x00000020 -#define WT_SESSION_LOCKED_SCHEMA 0x00000040 -#define WT_SESSION_LOCKED_SLOT 0x00000080 -#define WT_SESSION_LOCKED_TABLE 0x00000100 -#define WT_SESSION_LOCKED_TURTLE 0x00000200 -#define WT_SESSION_LOCK_NO_WAIT 0x00000400 -#define WT_SESSION_LOGGING_INMEM 0x00000800 -#define WT_SESSION_LOOKASIDE_CURSOR 0x00001000 -#define WT_SESSION_NO_CACHE 0x00002000 -#define WT_SESSION_NO_DATA_HANDLES 0x00004000 -#define WT_SESSION_NO_EVICTION 0x00008000 -#define WT_SESSION_NO_LOGGING 0x00010000 -#define WT_SESSION_NO_SCHEMA_LOCK 0x00020000 -#define WT_SESSION_QUIET_CORRUPT_FILE 0x00040000 -#define WT_SESSION_SERVER_ASYNC 0x00080000 +#define WT_SESSION_LOCKED_HANDLE_LIST_READ 0x00000008 +#define WT_SESSION_LOCKED_HANDLE_LIST_WRITE 0x00000010 +#define WT_SESSION_LOCKED_METADATA 0x00000020 +#define WT_SESSION_LOCKED_PASS 0x00000040 +#define WT_SESSION_LOCKED_SCHEMA 0x00000080 +#define WT_SESSION_LOCKED_SLOT 0x00000100 +#define WT_SESSION_LOCKED_TABLE_READ 0x00000200 +#define WT_SESSION_LOCKED_TABLE_WRITE 0x00000400 +#define WT_SESSION_LOCKED_TURTLE 0x00000800 +#define WT_SESSION_LOGGING_INMEM 0x00001000 +#define WT_SESSION_LOOKASIDE_CURSOR 0x00002000 +#define WT_SESSION_NO_CACHE 0x00004000 +#define WT_SESSION_NO_DATA_HANDLES 0x00008000 +#define WT_SESSION_NO_EVICTION 0x00010000 +#define WT_SESSION_NO_LOGGING 0x00020000 +#define WT_SESSION_NO_SCHEMA_LOCK 0x00040000 +#define WT_SESSION_QUIET_CORRUPT_FILE 0x00080000 +#define WT_SESSION_SERVER_ASYNC 0x00100000 #define WT_STAT_CLEAR 0x00000001 #define WT_STAT_JSON 0x00000002 #define WT_STAT_ON_CLOSE 0x00000004 @@ -91,27 +92,29 @@ #define WT_VERB_COMPACT 0x00000008 #define WT_VERB_EVICT 0x00000010 #define WT_VERB_EVICTSERVER 0x00000020 -#define WT_VERB_FILEOPS 0x00000040 -#define WT_VERB_HANDLEOPS 0x00000080 -#define WT_VERB_LOG 0x00000100 -#define WT_VERB_LSM 0x00000200 -#define WT_VERB_LSM_MANAGER 0x00000400 -#define WT_VERB_METADATA 0x00000800 -#define WT_VERB_MUTEX 0x00001000 -#define WT_VERB_OVERFLOW 0x00002000 -#define WT_VERB_READ 0x00004000 -#define WT_VERB_REBALANCE 0x00008000 -#define WT_VERB_RECONCILE 0x00010000 -#define WT_VERB_RECOVERY 0x00020000 -#define WT_VERB_SALVAGE 0x00040000 -#define WT_VERB_SHARED_CACHE 0x00080000 -#define WT_VERB_SPLIT 0x00100000 -#define WT_VERB_TEMPORARY 0x00200000 -#define WT_VERB_THREAD_GROUP 0x00400000 -#define WT_VERB_TRANSACTION 0x00800000 -#define WT_VERB_VERIFY 0x01000000 -#define WT_VERB_VERSION 0x02000000 -#define WT_VERB_WRITE 0x04000000 +#define WT_VERB_EVICT_STUCK 0x00000040 +#define WT_VERB_FILEOPS 0x00000080 +#define WT_VERB_HANDLEOPS 0x00000100 +#define WT_VERB_LOG 0x00000200 +#define WT_VERB_LSM 0x00000400 +#define WT_VERB_LSM_MANAGER 0x00000800 +#define WT_VERB_METADATA 0x00001000 +#define WT_VERB_MUTEX 0x00002000 +#define WT_VERB_OVERFLOW 0x00004000 +#define WT_VERB_READ 0x00008000 +#define WT_VERB_REBALANCE 0x00010000 +#define WT_VERB_RECONCILE 0x00020000 +#define WT_VERB_RECOVERY 0x00040000 +#define WT_VERB_RECOVERY_PROGRESS 0x00080000 +#define WT_VERB_SALVAGE 0x00100000 +#define WT_VERB_SHARED_CACHE 0x00200000 +#define WT_VERB_SPLIT 0x00400000 +#define WT_VERB_TEMPORARY 0x00800000 +#define WT_VERB_THREAD_GROUP 0x01000000 +#define WT_VERB_TRANSACTION 0x02000000 +#define WT_VERB_VERIFY 0x04000000 +#define WT_VERB_VERSION 0x08000000 +#define WT_VERB_WRITE 0x10000000 #define WT_VISIBILITY_ERR 0x00000080 /* * flags section: END diff --git a/src/third_party/wiredtiger/src/include/gcc.h b/src/third_party/wiredtiger/src/include/gcc.h index ce6afdd6e9c..22d78fc165a 100644 --- a/src/third_party/wiredtiger/src/include/gcc.h +++ b/src/third_party/wiredtiger/src/include/gcc.h @@ -10,8 +10,6 @@ #define WT_SIZET_FMT "zu" /* size_t format string */ /* Add GCC-specific attributes to types and function declarations. */ -#define WT_COMPILER_TYPE_ALIGN(x) __attribute__((aligned(x))) - #define WT_PACKED_STRUCT_BEGIN(name) \ struct __attribute__ ((__packed__)) name { #define WT_PACKED_STRUCT_END \ diff --git a/src/third_party/wiredtiger/src/include/hardware.h b/src/third_party/wiredtiger/src/include/hardware.h index 0e52818ae05..2530659db21 100644 --- a/src/third_party/wiredtiger/src/include/hardware.h +++ b/src/third_party/wiredtiger/src/include/hardware.h @@ -55,7 +55,19 @@ #else #define WT_CACHE_LINE_ALIGNMENT 64 #endif -#define WT_CACHE_LINE_ALIGNMENT_VERIFY(session, a) \ - WT_ASSERT(session, \ - WT_PTRDIFF(&(a)[1], &(a)[0]) >= WT_CACHE_LINE_ALIGNMENT && \ - WT_PTRDIFF(&(a)[1], &(a)[0]) % WT_CACHE_LINE_ALIGNMENT == 0) + +/* + * Pad a structure so an array of structures get separate cache lines. + * + * Note that we avoid compiler structure alignment because that requires + * allocating aligned blocks of memory, and alignment pollutes any other type + * that contains an aligned field. It is possible that a hot field positioned + * before this one will be on the same cache line, but not if it is also + * padded. + * + * This alignment has a small impact on portability as well, as we are using an + * anonymous union here which is supported under C11, earlier versions of + * the GNU standard, and MSVC versions as early as 2003. + */ +#define WT_CACHE_LINE_PAD_BEGIN union { struct { +#define WT_CACHE_LINE_PAD_END }; char __padding[WT_CACHE_LINE_ALIGNMENT]; }; diff --git a/src/third_party/wiredtiger/src/include/intpack.i b/src/third_party/wiredtiger/src/include/intpack.i index e8bea58cede..a534de9d9a8 100644 --- a/src/third_party/wiredtiger/src/include/intpack.i +++ b/src/third_party/wiredtiger/src/include/intpack.i @@ -59,21 +59,21 @@ /* Count the leading zero bytes. */ #if defined(__GNUC__) #define WT_LEADING_ZEROS(x, i) \ - (i = (x == 0) ? (int)sizeof(x) : __builtin_clzll(x) >> 3) + ((i) = ((x) == 0) ? (int)sizeof(x) : __builtin_clzll(x) >> 3) #elif defined(_MSC_VER) #define WT_LEADING_ZEROS(x, i) do { \ - if (x == 0) i = (int)sizeof(x); \ + if ((x) == 0) (i) = (int)sizeof(x); \ else { \ unsigned long __index; \ _BitScanReverse64(&__index, x); \ __index = 63 ^ __index; \ - i = (int)(__index >> 3); } \ + (i) = (int)(__index >> 3); } \ } while (0) #else #define WT_LEADING_ZEROS(x, i) do { \ uint64_t __x = (x); \ uint64_t __m = (uint64_t)0xff << 56; \ - for (i = 0; !(__x & __m) && i != 8; i++) \ + for ((i) = 0; !(__x & __m) && (i) != 8; (i)++) \ __m >>= 8; \ } while (0) #endif @@ -231,7 +231,8 @@ __wt_vpack_int(uint8_t **pp, size_t maxlen, int64_t x) if (x < NEG_2BYTE_MIN) { *p = NEG_MULTI_MARKER; return (__wt_vpack_negint(pp, maxlen, (uint64_t)x)); - } else if (x < NEG_1BYTE_MIN) { + } + if (x < NEG_1BYTE_MIN) { WT_SIZE_CHECK_PACK(2, maxlen); x -= NEG_2BYTE_MIN; *p++ = NEG_2BYTE_MARKER | GET_BITS(x, 13, 8); @@ -358,12 +359,10 @@ __wt_vsize_uint(uint64_t x) { if (x <= POS_1BYTE_MAX) return (1); - else if (x <= POS_2BYTE_MAX + 1) { + if (x <= POS_2BYTE_MAX + 1) return (2); - } else { - x -= POS_2BYTE_MAX + 1; - return (__wt_vsize_posint(x)); - } + x -= POS_2BYTE_MAX + 1; + return (__wt_vsize_posint(x)); } /* @@ -373,13 +372,12 @@ __wt_vsize_uint(uint64_t x) static inline size_t __wt_vsize_int(int64_t x) { - if (x < NEG_2BYTE_MIN) { + if (x < NEG_2BYTE_MIN) return (__wt_vsize_negint((uint64_t)x)); - } else if (x < NEG_1BYTE_MIN) { + if (x < NEG_1BYTE_MIN) return (2); - } else if (x < 0) { + if (x < 0) return (1); - } else - /* For non-negative values, use the unsigned code above. */ - return (__wt_vsize_uint((uint64_t)x)); + /* For non-negative values, use the unsigned code above. */ + return (__wt_vsize_uint((uint64_t)x)); } diff --git a/src/third_party/wiredtiger/src/include/lint.h b/src/third_party/wiredtiger/src/include/lint.h index 1b64186cbab..2d0f47988b7 100644 --- a/src/third_party/wiredtiger/src/include/lint.h +++ b/src/third_party/wiredtiger/src/include/lint.h @@ -9,8 +9,6 @@ #define WT_PTRDIFFT_FMT "td" /* ptrdiff_t format string */ #define WT_SIZET_FMT "zu" /* size_t format string */ -#define WT_COMPILER_TYPE_ALIGN(x) - #define WT_PACKED_STRUCT_BEGIN(name) \ struct name { #define WT_PACKED_STRUCT_END \ @@ -31,9 +29,9 @@ __wt_atomic_fetch_add##name(type *vp, type v) \ { \ type orig; \ \ - old = *vp; \ + orig = *vp; \ *vp += v; \ - return (old); \ + return (orig); \ } \ static inline ret \ __wt_atomic_store##name(type *vp, type v) \ @@ -42,7 +40,7 @@ __wt_atomic_store##name(type *vp, type v) \ \ orig = *vp; \ *vp = v; \ - return (old); \ + return (orig); \ } \ static inline ret \ __wt_atomic_sub##name(type *vp, type v) \ @@ -51,9 +49,9 @@ __wt_atomic_sub##name(type *vp, type v) \ return (*vp); \ } \ static inline bool \ -__wt_atomic_cas##name(type *vp, type old, type new) \ +__wt_atomic_cas##name(type *vp, type orig, type new) \ { \ - if (*vp == old) { \ + if (*vp == orig) { \ *vp = new; \ return (true); \ } \ @@ -77,8 +75,8 @@ WT_ATOMIC_FUNC(size, size_t, size_t) * Pointer compare and swap. */ static inline bool -__wt_atomic_cas_ptr(void *vp, void *old, void *new) { - if (*(void **)vp == old) { +__wt_atomic_cas_ptr(void *vp, void *orig, void *new) { + if (*(void **)vp == orig) { *(void **)vp = new; return (true); } diff --git a/src/third_party/wiredtiger/src/include/log.h b/src/third_party/wiredtiger/src/include/log.h index b91c7dffe60..fb3c961417f 100644 --- a/src/third_party/wiredtiger/src/include/log.h +++ b/src/third_party/wiredtiger/src/include/log.h @@ -86,8 +86,8 @@ union __wt_lsn { * The high bit is reserved for the special states. If the high bit is * set (WT_LOG_SLOT_RESERVED) then we are guaranteed to be in a special state. */ -#define WT_LOG_SLOT_FREE -1 /* Not in use */ -#define WT_LOG_SLOT_WRITTEN -2 /* Slot data written, not processed */ +#define WT_LOG_SLOT_FREE (-1) /* Not in use */ +#define WT_LOG_SLOT_WRITTEN (-2) /* Slot data written, not processed */ /* * We allocate the buffer size, but trigger a slot switch when we cross @@ -144,8 +144,8 @@ union __wt_lsn { /* Slot is in use, but closed to new joins */ #define WT_LOG_SLOT_CLOSED(state) \ (WT_LOG_SLOT_ACTIVE(state) && \ - (FLD64_ISSET((uint64_t)state, WT_LOG_SLOT_CLOSE) && \ - !FLD64_ISSET((uint64_t)state, WT_LOG_SLOT_RESERVED))) + (FLD64_ISSET((uint64_t)(state), WT_LOG_SLOT_CLOSE) && \ + !FLD64_ISSET((uint64_t)(state), WT_LOG_SLOT_RESERVED))) /* Slot is in use, all data copied into buffer */ #define WT_LOG_SLOT_INPROGRESS(state) \ (WT_LOG_SLOT_RELEASED(state) != WT_LOG_SLOT_JOINED(state)) @@ -159,10 +159,11 @@ union __wt_lsn { !FLD64_ISSET((uint64_t)(state), WT_LOG_SLOT_CLOSE) && \ WT_LOG_SLOT_JOINED(state) < WT_LOG_SLOT_BUF_MAX) -struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_logslot { +struct __wt_logslot { + WT_CACHE_LINE_PAD_BEGIN volatile int64_t slot_state; /* Slot state */ int64_t slot_unbuffered; /* Unbuffered data in this slot */ - int32_t slot_error; /* Error value */ + int slot_error; /* Error value */ wt_off_t slot_start_offset; /* Starting file offset */ wt_off_t slot_last_offset; /* Last record offset */ WT_LSN slot_release_lsn; /* Slot release LSN */ @@ -176,14 +177,15 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_logslot { #define WT_SLOT_SYNC 0x04 /* Needs sync on release */ #define WT_SLOT_SYNC_DIR 0x08 /* Directory sync on release */ uint32_t flags; /* Flags */ + WT_CACHE_LINE_PAD_END }; #define WT_SLOT_INIT_FLAGS 0 -#define WT_WITH_SLOT_LOCK(session, log, ret, op) do { \ +#define WT_WITH_SLOT_LOCK(session, log, op) do { \ WT_ASSERT(session, !F_ISSET(session, WT_SESSION_LOCKED_SLOT)); \ - WT_WITH_LOCK(session, ret, \ - &log->log_slot_lock, WT_SESSION_LOCKED_SLOT, op); \ + WT_WITH_LOCK_WAIT(session, \ + &(log)->log_slot_lock, WT_SESSION_LOCKED_SLOT, op); \ } while (0) struct __wt_myslot { @@ -191,7 +193,8 @@ struct __wt_myslot { wt_off_t end_offset; /* My end offset in buffer */ wt_off_t offset; /* Slot buffer offset */ #define WT_MYSLOT_CLOSE 0x01 /* This thread is closing the slot */ -#define WT_MYSLOT_UNBUFFERED 0x02 /* Write directly */ +#define WT_MYSLOT_NEEDS_RELEASE 0x02 /* This thread is releasing the slot */ +#define WT_MYSLOT_UNBUFFERED 0x04 /* Write directly */ uint32_t flags; /* Flags */ }; @@ -233,7 +236,7 @@ struct __wt_log { WT_SPINLOCK log_sync_lock; /* Locked: Single-thread fsync */ WT_SPINLOCK log_writelsn_lock; /* Locked: write LSN */ - WT_RWLOCK *log_archive_lock; /* Archive and log cursors */ + WT_RWLOCK log_archive_lock;/* Archive and log cursors */ /* Notify any waiting threads when sync_lsn is updated. */ WT_CONDVAR *log_sync_cond; @@ -252,6 +255,7 @@ struct __wt_log { #define WT_SLOT_POOL 128 WT_LOGSLOT *active_slot; /* Active slot */ WT_LOGSLOT slot_pool[WT_SLOT_POOL]; /* Pool of all slots */ + int32_t pool_index; /* Index into slot pool */ size_t slot_buf_size; /* Buffer size for slots */ #ifdef HAVE_DIAGNOSTIC uint64_t write_calls; /* Calls to log_write */ diff --git a/src/third_party/wiredtiger/src/include/lsm.h b/src/third_party/wiredtiger/src/include/lsm.h index b433e4c3c44..e3f6897ef9d 100644 --- a/src/third_party/wiredtiger/src/include/lsm.h +++ b/src/third_party/wiredtiger/src/include/lsm.h @@ -23,11 +23,14 @@ struct __wt_lsm_worker_cookie { struct __wt_lsm_worker_args { WT_SESSION_IMPL *session; /* Session */ WT_CONDVAR *work_cond; /* Owned by the manager */ + wt_thread_t tid; /* Thread id */ + bool tid_set; /* Thread id set */ + u_int id; /* My manager slot id */ uint32_t type; /* Types of operations handled */ -#define WT_LSM_WORKER_RUN 0x01 - uint32_t flags; /* Worker flags */ + + volatile bool running; /* Worker is running */ }; /* @@ -162,6 +165,9 @@ struct __wt_lsm_manager { #define WT_LSM_MAX_WORKERS 20 #define WT_LSM_MIN_WORKERS 3 WT_LSM_WORKER_ARGS lsm_worker_cookies[WT_LSM_MAX_WORKERS]; + +#define WT_LSM_MANAGER_SHUTDOWN 0x01 /* Manager has shut down */ + uint32_t flags; }; /* @@ -189,7 +195,7 @@ struct __wt_lsm_tree { #define LSM_TREE_MAX_QUEUE 100 uint32_t queue_ref; - WT_RWLOCK *rwlock; + WT_RWLOCK rwlock; TAILQ_ENTRY(__wt_lsm_tree) q; uint64_t dsk_gen; @@ -249,17 +255,21 @@ struct __wt_lsm_tree { int64_t lsm_merge_throttle; /* - * The tree is open for business. This used to be a flag, but it is - * susceptible to races. + * Following fields used to be flags but are susceptible to races. + * Don't merge them with flags. + */ + bool active; /* The tree is open for business */ + bool aggressive_timer_enabled; /* Timer for merge aggression enabled */ + bool need_switch; /* New chunk needs creating */ + + /* + * flags here are not protected for concurrent access, don't put + * anything here that is susceptible to races. */ - bool active; - -#define WT_LSM_TREE_AGGRESSIVE_TIMER 0x01 /* Timer for merge aggression */ -#define WT_LSM_TREE_COMPACTING 0x02 /* Tree being compacted */ -#define WT_LSM_TREE_MERGES 0x04 /* Tree should run merges */ -#define WT_LSM_TREE_NEED_SWITCH 0x08 /* New chunk needs creating */ -#define WT_LSM_TREE_OPEN 0x10 /* The tree is open */ -#define WT_LSM_TREE_THROTTLE 0x20 /* Throttle updates */ +#define WT_LSM_TREE_COMPACTING 0x01 /* Tree being compacted */ +#define WT_LSM_TREE_MERGES 0x02 /* Tree should run merges */ +#define WT_LSM_TREE_OPEN 0x04 /* The tree is open */ +#define WT_LSM_TREE_THROTTLE 0x08 /* Throttle updates */ uint32_t flags; }; diff --git a/src/third_party/wiredtiger/src/include/meta.h b/src/third_party/wiredtiger/src/include/meta.h index 6d4a167a8e5..68ac2e339d0 100644 --- a/src/third_party/wiredtiger/src/include/meta.h +++ b/src/third_party/wiredtiger/src/include/meta.h @@ -32,8 +32,7 @@ * Optimize comparisons against the metafile URI, flag handles that reference * the metadata file. */ -#define WT_IS_METADATA(session, dh) \ - F_ISSET((dh), WT_DHANDLE_IS_METADATA) +#define WT_IS_METADATA(dh) F_ISSET((dh), WT_DHANDLE_IS_METADATA) #define WT_METAFILE_ID 0 /* Metadata file ID */ #define WT_METADATA_VERSION "WiredTiger version" /* Version keys */ @@ -43,9 +42,9 @@ * WT_WITH_TURTLE_LOCK -- * Acquire the turtle file lock, perform an operation, drop the lock. */ -#define WT_WITH_TURTLE_LOCK(session, ret, op) do { \ +#define WT_WITH_TURTLE_LOCK(session, op) do { \ WT_ASSERT(session, !F_ISSET(session, WT_SESSION_LOCKED_TURTLE));\ - WT_WITH_LOCK(session, ret, \ + WT_WITH_LOCK_WAIT(session, \ &S2C(session)->turtle_lock, WT_SESSION_LOCKED_TURTLE, op); \ } while (0) diff --git a/src/third_party/wiredtiger/src/include/misc.h b/src/third_party/wiredtiger/src/include/misc.h index 83e238b0529..9161a215fdc 100644 --- a/src/third_party/wiredtiger/src/include/misc.h +++ b/src/third_party/wiredtiger/src/include/misc.h @@ -63,7 +63,7 @@ #define WT_MAX(a, b) ((a) < (b) ? (b) : (a)) /* Elements in an array. */ -#define WT_ELEMENTS(a) (sizeof(a) / sizeof(a[0])) +#define WT_ELEMENTS(a) (sizeof(a) / sizeof((a)[0])) /* 10 level skip lists, 1/4 have a link to the next element. */ #define WT_SKIP_MAXDEPTH 10 @@ -140,6 +140,7 @@ #define F_CLR(p, mask) FLD_CLR((p)->flags, mask) #define F_ISSET(p, mask) FLD_ISSET((p)->flags, mask) +#define F_ISSET_ALL(p, mask) (FLD_MASK((p)->flags, mask) == (mask)) #define F_MASK(p, mask) FLD_MASK((p)->flags, mask) #define F_SET(p, mask) FLD_SET((p)->flags, mask) @@ -180,14 +181,14 @@ */ #define WT_BINARY_SEARCH(key, arrayp, n, found) do { \ uint32_t __base, __indx, __limit; \ - found = false; \ + (found) = false; \ for (__base = 0, __limit = (n); __limit != 0; __limit >>= 1) { \ __indx = __base + (__limit >> 1); \ - if ((arrayp)[__indx] < key) { \ + if ((arrayp)[__indx] < (key)) { \ __base = __indx + 1; \ --__limit; \ - } else if ((arrayp)[__indx] == key) { \ - found = true; \ + } else if ((arrayp)[__indx] == (key)) { \ + (found) = true; \ break; \ } \ } \ @@ -206,8 +207,8 @@ /* Check if a string matches a prefix. */ #define WT_PREFIX_MATCH(str, pfx) \ - (((const char *)(str))[0] == ((const char *)pfx)[0] && \ - strncmp((str), (pfx), strlen(pfx)) == 0) + (((const char *)(str))[0] == ((const char *)(pfx))[0] && \ + strncmp(str, pfx, strlen(pfx)) == 0) /* Check if a string matches a prefix, and move past it. */ #define WT_PREFIX_SKIP(str, pfx) \ @@ -224,8 +225,8 @@ /* Check if a string matches a byte string of len bytes. */ #define WT_STRING_MATCH(str, bytes, len) \ - (((const char *)str)[0] == ((const char *)bytes)[0] && \ - strncmp(str, bytes, len) == 0 && (str)[(len)] == '\0') + (((const char *)(str))[0] == ((const char *)(bytes))[0] && \ + strncmp(str, bytes, len) == 0 && (str)[len] == '\0') /* * Macro that produces a string literal that isn't wrapped in quotes, to avoid @@ -275,6 +276,3 @@ union __wt_rand_state { uint32_t w, z; } x; }; - -/* Shared array for converting to hex */ -extern const u_char __wt_hex[]; diff --git a/src/third_party/wiredtiger/src/include/misc.i b/src/third_party/wiredtiger/src/include/misc.i index befd480e085..7040886cf82 100644 --- a/src/third_party/wiredtiger/src/include/misc.i +++ b/src/third_party/wiredtiger/src/include/misc.i @@ -11,11 +11,22 @@ * Wait on a mutex, optionally timing out. */ static inline void -__wt_cond_wait(WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs) +__wt_cond_wait(WT_SESSION_IMPL *session, + WT_CONDVAR *cond, uint64_t usecs, bool (*run_func)(WT_SESSION_IMPL *)) { bool notused; - __wt_cond_wait_signal(session, cond, usecs, ¬used); + __wt_cond_wait_signal(session, cond, usecs, run_func, ¬used); +} + +/* + * __wt_hex -- + * Convert a byte to a hex character. + */ +static inline u_char +__wt_hex(int c) +{ + return ((u_char)"0123456789abcdef"[c]); } /* @@ -75,3 +86,94 @@ __wt_verbose(WT_SESSION_IMPL *session, int flag, const char *fmt, ...) WT_UNUSED(fmt); #endif } + +/* + * __wt_snprintf -- + * snprintf convenience function, ignoring the returned size. + */ +static inline int +__wt_snprintf(char *buf, size_t size, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) +{ + WT_DECL_RET; + size_t len; + va_list ap; + + len = 0; + + va_start(ap, fmt); + ret = __wt_vsnprintf_len_incr(buf, size, &len, fmt, ap); + va_end(ap); + WT_RET(ret); + + /* It's an error if the buffer couldn't hold everything. */ + return (len >= size ? ERANGE : 0); +} + +/* + * __wt_vsnprintf -- + * vsnprintf convenience function, ignoring the returned size. + */ +static inline int +__wt_vsnprintf(char *buf, size_t size, const char *fmt, va_list ap) +{ + size_t len; + + len = 0; + + WT_RET(__wt_vsnprintf_len_incr(buf, size, &len, fmt, ap)); + + /* It's an error if the buffer couldn't hold everything. */ + return (len >= size ? ERANGE : 0); +} + +/* + * __wt_snprintf_len_set -- + * snprintf convenience function, setting the returned size. + */ +static inline int +__wt_snprintf_len_set( + char *buf, size_t size, size_t *retsizep, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 4, 5))) +{ + WT_DECL_RET; + va_list ap; + + *retsizep = 0; + + va_start(ap, fmt); + ret = __wt_vsnprintf_len_incr(buf, size, retsizep, fmt, ap); + va_end(ap); + return (ret); +} + +/* + * __wt_vsnprintf_len_set -- + * vsnprintf convenience function, setting the returned size. + */ +static inline int +__wt_vsnprintf_len_set( + char *buf, size_t size, size_t *retsizep, const char *fmt, va_list ap) +{ + *retsizep = 0; + + return (__wt_vsnprintf_len_incr(buf, size, retsizep, fmt, ap)); +} + +/* + * __wt_snprintf_len_incr -- + * snprintf convenience function, incrementing the returned size. + */ +static inline int +__wt_snprintf_len_incr( + char *buf, size_t size, size_t *retsizep, const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 4, 5))) +{ + WT_DECL_RET; + va_list ap; + + va_start(ap, fmt); + ret = __wt_vsnprintf_len_incr(buf, size, retsizep, fmt, ap); + va_end(ap); + return (ret); +} diff --git a/src/third_party/wiredtiger/src/include/msvc.h b/src/third_party/wiredtiger/src/include/msvc.h index d5be5bd8c60..6c5c8b67647 100644 --- a/src/third_party/wiredtiger/src/include/msvc.h +++ b/src/third_party/wiredtiger/src/include/msvc.h @@ -19,8 +19,6 @@ /* * Add MSVC-specific attributes and pragmas to types and function declarations. */ -#define WT_COMPILER_TYPE_ALIGN(x) __declspec(align(x)) - #define WT_PACKED_STRUCT_BEGIN(name) \ __pragma(pack(push,1)) \ struct name { diff --git a/src/third_party/wiredtiger/src/include/mutex.h b/src/third_party/wiredtiger/src/include/mutex.h index 84f015d6b67..36acea810d9 100644 --- a/src/third_party/wiredtiger/src/include/mutex.h +++ b/src/third_party/wiredtiger/src/include/mutex.h @@ -21,41 +21,37 @@ struct __wt_condvar { int waiters; /* Numbers of waiters, or -1 if signalled with no waiters. */ /* - * The following fields are only used for automatically adjusting - * condition variables. They could be in a separate structure. + * The following fields are used for automatically adjusting condition + * variable wait times. */ uint64_t min_wait; /* Minimum wait duration */ uint64_t max_wait; /* Maximum wait duration */ uint64_t prev_wait; /* Wait duration used last time */ }; -/* - * !!! - * Don't modify this structure without understanding the read/write locking - * functions. - */ -typedef union { /* Read/write lock */ - uint64_t u; - struct { - uint32_t wr; /* Writers and readers */ - } i; - struct { - uint16_t writers; /* Now serving for writers */ - uint16_t readers; /* Now serving for readers */ - uint16_t next; /* Next available ticket number */ - uint16_t writers_active;/* Count of active writers */ - } s; -} wt_rwlock_t; - /* * Read/write locks: * * WiredTiger uses read/write locks for shared/exclusive access to resources. + * !!! + * Don't modify this structure without understanding the read/write locking + * functions. */ -struct __wt_rwlock { - const char *name; /* Lock name for debugging */ - - wt_rwlock_t rwlock; /* Read/write lock */ +struct __wt_rwlock { /* Read/write lock */ + volatile union { + uint64_t v; /* Full 64-bit value */ + struct { + uint8_t current; /* Current ticket */ + uint8_t next; /* Next available ticket */ + uint8_t reader; /* Read queue ticket */ + uint8_t __notused; /* Padding */ + uint16_t readers_active;/* Count of active readers */ + uint16_t readers_queued;/* Count of queued readers */ + } s; + } u; + + WT_CONDVAR *cond_readers; /* Blocking readers */ + WT_CONDVAR *cond_writers; /* Blocking writers */ }; /* @@ -70,28 +66,17 @@ struct __wt_rwlock { #define SPINLOCK_PTHREAD_MUTEX 2 #define SPINLOCK_PTHREAD_MUTEX_ADAPTIVE 3 +struct __wt_spinlock { #if SPINLOCK_TYPE == SPINLOCK_GCC - -struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_spinlock { + WT_CACHE_LINE_PAD_BEGIN volatile int lock; - - /* - * We track acquisitions and time spent waiting for some locks. For - * performance reasons and to make it possible to write generic code - * that tracks statistics for different locks, we store the offset - * of the statistics fields to be updated during lock acquisition. - */ - int16_t stat_count_off; /* acquisitions offset */ - int16_t stat_app_usecs_off; /* waiting application threads offset */ - int16_t stat_int_usecs_off; /* waiting server threads offset */ -}; - #elif SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX ||\ SPINLOCK_TYPE == SPINLOCK_PTHREAD_MUTEX_ADAPTIVE ||\ SPINLOCK_TYPE == SPINLOCK_MSVC - -struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_spinlock { wt_mutex_t lock; +#else +#error Unknown spinlock type +#endif const char *name; /* Mutex name */ @@ -106,10 +91,8 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_spinlock { int16_t stat_int_usecs_off; /* waiting server threads offset */ int8_t initialized; /* Lock initialized, for cleanup */ -}; - -#else - -#error Unknown spinlock type +#if SPINLOCK_TYPE == SPINLOCK_GCC + WT_CACHE_LINE_PAD_END #endif +}; diff --git a/src/third_party/wiredtiger/src/include/mutex.i b/src/third_party/wiredtiger/src/include/mutex.i index a6309e0976b..2d483972ed2 100644 --- a/src/third_party/wiredtiger/src/include/mutex.i +++ b/src/third_party/wiredtiger/src/include/mutex.i @@ -14,6 +14,18 @@ * of instructions. */ +/* + * __spin_init_internal -- + * Initialize the WT portion of a spinlock. + */ +static inline void +__spin_init_internal(WT_SPINLOCK *t, const char *name) +{ + t->name = name; + t->stat_count_off = t->stat_app_usecs_off = t->stat_int_usecs_off = -1; + t->initialized = 1; +} + #if SPINLOCK_TYPE == SPINLOCK_GCC /* Default to spinning 1000 times before yielding. */ @@ -29,10 +41,9 @@ static inline int __wt_spin_init(WT_SESSION_IMPL *session, WT_SPINLOCK *t, const char *name) { WT_UNUSED(session); - WT_UNUSED(name); t->lock = 0; - t->stat_count_off = t->stat_app_usecs_off = t->stat_int_usecs_off = -1; + __spin_init_internal(t, name); return (0); } @@ -110,10 +121,7 @@ __wt_spin_init(WT_SESSION_IMPL *session, WT_SPINLOCK *t, const char *name) #else WT_RET(pthread_mutex_init(&t->lock, NULL)); #endif - - t->name = name; - t->stat_count_off = t->stat_app_usecs_off = t->stat_int_usecs_off = -1; - t->initialized = 1; + __spin_init_internal(t, name); WT_UNUSED(session); return (0); @@ -195,8 +203,7 @@ __wt_spin_init(WT_SESSION_IMPL *session, WT_SPINLOCK *t, const char *name) return (__wt_map_windows_error(windows_error)); } - t->name = name; - t->initialized = 1; + __spin_init_internal(t, name); return (0); } @@ -300,3 +307,22 @@ __wt_spin_lock_track(WT_SESSION_IMPL *session, WT_SPINLOCK *t) } else __wt_spin_lock(session, t); } + +/* + * __wt_spin_trylock_track -- + * Try to lock a spinlock or fail immediately if it is busy. + * Track if successful. + */ +static inline int +__wt_spin_trylock_track(WT_SESSION_IMPL *session, WT_SPINLOCK *t) +{ + int64_t **stats; + + if (t->stat_count_off != -1 && WT_STAT_ENABLED(session)) { + WT_RET(__wt_spin_trylock(session, t)); + stats = (int64_t **)S2C(session)->stats; + stats[session->stat_bucket][t->stat_count_off]++; + return (0); + } + return (__wt_spin_trylock(session, t)); +} diff --git a/src/third_party/wiredtiger/src/include/os.h b/src/third_party/wiredtiger/src/include/os.h index 7a8e47ed81f..73d89268392 100644 --- a/src/third_party/wiredtiger/src/include/os.h +++ b/src/third_party/wiredtiger/src/include/os.h @@ -11,8 +11,14 @@ * A call returning 0 indicates success; any call where \ * 0 is not the only successful return must provide an \ * expression evaluating to 0 in all successful cases. \ + * \ + * XXX \ + * Casting the call's return to int is because CentOS 7.3.1611 \ + * complains about syscall returning a long and the loss of \ + * integer precision in the assignment to ret. The cast should \ + * be a no-op everywhere. \ */ \ - if (((ret) = (call)) == 0) \ + if (((ret) = (int)(call)) == 0) \ break; \ /* \ * The call's error was either returned by the call or \ @@ -61,7 +67,7 @@ #define WT_TIMECMP(t1, t2) \ ((t1).tv_sec < (t2).tv_sec ? -1 : \ - (t1).tv_sec == (t2.tv_sec) ? \ + (t1).tv_sec == (t2).tv_sec ? \ (t1).tv_nsec < (t2).tv_nsec ? -1 : \ (t1).tv_nsec == (t2).tv_nsec ? 0 : 1 : 1) diff --git a/src/third_party/wiredtiger/src/include/os_windows.h b/src/third_party/wiredtiger/src/include/os_windows.h index 65938ac9f17..c1e5f788dc6 100644 --- a/src/third_party/wiredtiger/src/include/os_windows.h +++ b/src/third_party/wiredtiger/src/include/os_windows.h @@ -43,16 +43,6 @@ typedef uint32_t u_int; typedef unsigned char u_char; typedef uint64_t u_long; -/* <= VS 2013 is not C99 compat */ -#if _MSC_VER < 1900 -#define snprintf _wt_snprintf - -_Check_return_opt_ int __cdecl _wt_snprintf( - _Out_writes_(_MaxCount) char * _DstBuf, - _In_ size_t _MaxCount, - _In_z_ _Printf_format_string_ const char * _Format, ...); -#endif - /* * Windows does have ssize_t * Python headers declare also though so we need to guard it @@ -61,18 +51,6 @@ _Check_return_opt_ int __cdecl _wt_snprintf( typedef int ssize_t; #endif -/* - * Provide a custom version of vsnprintf that returns the - * needed buffer length instead of -1 on truncation - */ -#define vsnprintf _wt_vsnprintf - -_Check_return_opt_ int __cdecl _wt_vsnprintf( - _Out_writes_(_MaxCount) char * _DstBuf, - _In_ size_t _MaxCount, - _In_z_ _Printf_format_string_ const char * _Format, - va_list _ArgList); - /* Provide a custom version of localtime_r */ struct tm *localtime_r(const time_t* timer, struct tm* result); diff --git a/src/third_party/wiredtiger/src/include/packing.i b/src/third_party/wiredtiger/src/include/packing.i index 17ca261bcfc..0eadb2f2027 100644 --- a/src/third_party/wiredtiger/src/include/packing.i +++ b/src/third_party/wiredtiger/src/include/packing.i @@ -104,8 +104,8 @@ __pack_name_next(WT_PACK_NAME *pn, WT_CONFIG_ITEM *name) WT_CONFIG_ITEM ignore; if (pn->genname) { - (void)snprintf(pn->buf, sizeof(pn->buf), - (pn->iskey ? "key%d" : "value%d"), pn->count); + WT_RET(__wt_snprintf(pn->buf, sizeof(pn->buf), + (pn->iskey ? "key%d" : "value%d"), pn->count)); WT_CLEAR(*name); name->str = pn->buf; name->len = strlen(pn->buf); @@ -168,10 +168,15 @@ next: if (pack->cur == pack->end) (int)(pack->end - pack->orig), pack->orig); return (0); case 'u': - case 'U': /* Special case for items with a size prefix. */ pv->type = (!pv->havesize && *pack->cur != '\0') ? 'U' : 'u'; return (0); + case 'U': + /* + * Don't change the type. 'U' is used internally, so this type + * was already changed to explicitly include the size. + */ + return (0); case 'b': case 'h': case 'i': @@ -193,7 +198,7 @@ next: if (pack->cur == pack->end) return (0); default: WT_RET_MSG(pack->session, EINVAL, - "Invalid type '%c' found in format '%.*s'", + "Invalid type '%c' found in format '%.*s'", pv->type, (int)(pack->end - pack->orig), pack->orig); } @@ -201,43 +206,43 @@ next: if (pack->cur == pack->end) #define WT_PACK_GET(session, pv, ap) do { \ WT_ITEM *__item; \ - switch (pv.type) { \ + switch ((pv).type) { \ case 'x': \ break; \ case 's': \ case 'S': \ - pv.u.s = va_arg(ap, const char *); \ + (pv).u.s = va_arg(ap, const char *); \ break; \ case 'U': \ case 'u': \ __item = va_arg(ap, WT_ITEM *); \ - pv.u.item.data = __item->data; \ - pv.u.item.size = __item->size; \ + (pv).u.item.data = __item->data; \ + (pv).u.item.size = __item->size; \ break; \ case 'b': \ case 'h': \ case 'i': \ - pv.u.i = va_arg(ap, int); \ + (pv).u.i = va_arg(ap, int); \ break; \ case 'B': \ case 'H': \ case 'I': \ case 't': \ - pv.u.u = va_arg(ap, unsigned int); \ + (pv).u.u = va_arg(ap, unsigned int); \ break; \ case 'l': \ - pv.u.i = va_arg(ap, long); \ + (pv).u.i = va_arg(ap, long); \ break; \ case 'L': \ - pv.u.u = va_arg(ap, unsigned long); \ + (pv).u.u = va_arg(ap, unsigned long); \ break; \ case 'q': \ - pv.u.i = va_arg(ap, int64_t); \ + (pv).u.i = va_arg(ap, int64_t); \ break; \ case 'Q': \ case 'r': \ case 'R': \ - pv.u.u = va_arg(ap, uint64_t); \ + (pv).u.u = va_arg(ap, uint64_t); \ break; \ /* User format strings have already been validated. */ \ WT_ILLEGAL_VALUE(session); \ @@ -551,47 +556,47 @@ __unpack_read(WT_SESSION_IMPL *session, #define WT_UNPACK_PUT(session, pv, ap) do { \ WT_ITEM *__item; \ - switch (pv.type) { \ + switch ((pv).type) { \ case 'x': \ break; \ case 's': \ case 'S': \ - *va_arg(ap, const char **) = pv.u.s; \ + *va_arg(ap, const char **) = (pv).u.s; \ break; \ case 'U': \ case 'u': \ __item = va_arg(ap, WT_ITEM *); \ - __item->data = pv.u.item.data; \ - __item->size = pv.u.item.size; \ + __item->data = (pv).u.item.data; \ + __item->size = (pv).u.item.size; \ break; \ case 'b': \ - *va_arg(ap, int8_t *) = (int8_t)pv.u.i; \ + *va_arg(ap, int8_t *) = (int8_t)(pv).u.i; \ break; \ case 'h': \ - *va_arg(ap, int16_t *) = (short)pv.u.i; \ + *va_arg(ap, int16_t *) = (short)(pv).u.i; \ break; \ case 'i': \ case 'l': \ - *va_arg(ap, int32_t *) = (int32_t)pv.u.i; \ + *va_arg(ap, int32_t *) = (int32_t)(pv).u.i; \ break; \ case 'q': \ - *va_arg(ap, int64_t *) = pv.u.i; \ + *va_arg(ap, int64_t *) = (pv).u.i; \ break; \ case 'B': \ case 't': \ - *va_arg(ap, uint8_t *) = (uint8_t)pv.u.u; \ + *va_arg(ap, uint8_t *) = (uint8_t)(pv).u.u; \ break; \ case 'H': \ - *va_arg(ap, uint16_t *) = (uint16_t)pv.u.u; \ + *va_arg(ap, uint16_t *) = (uint16_t)(pv).u.u; \ break; \ case 'I': \ case 'L': \ - *va_arg(ap, uint32_t *) = (uint32_t)pv.u.u; \ + *va_arg(ap, uint32_t *) = (uint32_t)(pv).u.u; \ break; \ case 'Q': \ case 'r': \ case 'R': \ - *va_arg(ap, uint64_t *) = pv.u.u; \ + *va_arg(ap, uint64_t *) = (pv).u.u; \ break; \ /* User format strings have already been validated. */ \ WT_ILLEGAL_VALUE(session); \ diff --git a/src/third_party/wiredtiger/src/include/schema.h b/src/third_party/wiredtiger/src/include/schema.h index 6a5ce67a867..50e141d9921 100644 --- a/src/third_party/wiredtiger/src/include/schema.h +++ b/src/third_party/wiredtiger/src/include/schema.h @@ -78,6 +78,14 @@ struct __wt_table { */ #define WT_COLGROUPS(t) WT_MAX((t)->ncolgroups, 1) +/* Helpers for the locked state of the handle list and table locks. */ +#define WT_SESSION_LOCKED_HANDLE_LIST \ + (WT_SESSION_LOCKED_HANDLE_LIST_READ | \ + WT_SESSION_LOCKED_HANDLE_LIST_WRITE) +#define WT_SESSION_LOCKED_TABLE \ + (WT_SESSION_LOCKED_TABLE_READ | \ + WT_SESSION_LOCKED_TABLE_WRITE) + /* * WT_WITH_LOCK_WAIT -- * Wait for a lock, perform an operation, drop the lock. @@ -85,7 +93,7 @@ struct __wt_table { #define WT_WITH_LOCK_WAIT(session, lock, flag, op) do { \ if (F_ISSET(session, (flag))) { \ op; \ - } else { \ + } else { \ __wt_spin_lock_track(session, lock); \ F_SET(session, (flag)); \ op; \ @@ -95,76 +103,159 @@ struct __wt_table { } while (0) /* - * WT_WITH_LOCK -- - * Acquire a lock, perform an operation, drop the lock. + * WT_WITH_LOCK_NOWAIT -- + * Acquire a lock if available, perform an operation, drop the lock. */ -#define WT_WITH_LOCK(session, ret, lock, flag, op) do { \ - ret = 0; \ - if (!F_ISSET(session, (flag)) && \ - F_ISSET(session, WT_SESSION_LOCK_NO_WAIT)) { \ - if ((ret = __wt_spin_trylock(session, lock)) == 0) { \ - F_SET(session, (flag)); \ - op; \ - F_CLR(session, (flag)); \ - __wt_spin_unlock(session, lock); \ - } \ - } else \ - WT_WITH_LOCK_WAIT(session, lock, flag, op); \ +#define WT_WITH_LOCK_NOWAIT(session, ret, lock, flag, op) do { \ + (ret) = 0; \ + if (F_ISSET(session, (flag))) { \ + op; \ + } else if (((ret) = \ + __wt_spin_trylock_track(session, lock)) == 0) { \ + F_SET(session, (flag)); \ + op; \ + F_CLR(session, (flag)); \ + __wt_spin_unlock(session, lock); \ + } \ } while (0) /* - * WT_WITH_CHECKPOINT_LOCK -- + * WT_WITH_CHECKPOINT_LOCK, WT_WITH_CHECKPOINT_LOCK_NOWAIT -- * Acquire the checkpoint lock, perform an operation, drop the lock. */ -#define WT_WITH_CHECKPOINT_LOCK(session, ret, op) \ - WT_WITH_LOCK(session, ret, \ +#define WT_WITH_CHECKPOINT_LOCK(session, op) \ + WT_WITH_LOCK_WAIT(session, \ + &S2C(session)->checkpoint_lock, WT_SESSION_LOCKED_CHECKPOINT, op) +#define WT_WITH_CHECKPOINT_LOCK_NOWAIT(session, ret, op) \ + WT_WITH_LOCK_NOWAIT(session, ret, \ &S2C(session)->checkpoint_lock, WT_SESSION_LOCKED_CHECKPOINT, op) /* - * WT_WITH_HANDLE_LIST_LOCK -- - * Acquire the data handle list lock, perform an operation, drop the lock. + * WT_WITH_HANDLE_LIST_READ_LOCK -- + * Acquire the data handle list lock in shared mode, perform an operation, + * drop the lock. The handle list lock is a read-write lock so the + * implementation is different to the other lock macros. * * Note: always waits because some operations need the handle list lock to * discard handles, and we only expect it to be held across short * operations. */ -#define WT_WITH_HANDLE_LIST_LOCK(session, op) \ - WT_WITH_LOCK_WAIT(session, \ - &S2C(session)->dhandle_lock, WT_SESSION_LOCKED_HANDLE_LIST, op) +#define WT_WITH_HANDLE_LIST_READ_LOCK(session, op) do { \ + if (F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)) { \ + op; \ + } else { \ + __wt_readlock(session, &S2C(session)->dhandle_lock); \ + F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST_READ); \ + op; \ + F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST_READ); \ + __wt_readunlock(session, &S2C(session)->dhandle_lock); \ + } \ +} while (0) + +/* + * WT_WITH_HANDLE_LIST_WRITE_LOCK -- + * Acquire the data handle list lock in exclusive mode, perform an + * operation, drop the lock. The handle list lock is a read-write lock so + * the implementation is different to the other lock macros. + */ +#define WT_WITH_HANDLE_LIST_WRITE_LOCK(session, op) do { \ + if (F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE)) { \ + op; \ + } else { \ + WT_ASSERT(session, \ + !F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_READ));\ + __wt_writelock(session, &S2C(session)->dhandle_lock); \ + F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE); \ + op; \ + F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE); \ + __wt_writeunlock(session, &S2C(session)->dhandle_lock); \ + } \ +} while (0) /* * WT_WITH_METADATA_LOCK -- * Acquire the metadata lock, perform an operation, drop the lock. */ -#define WT_WITH_METADATA_LOCK(session, ret, op) \ - WT_WITH_LOCK(session, ret, \ +#define WT_WITH_METADATA_LOCK(session, op) \ + WT_WITH_LOCK_WAIT(session, \ &S2C(session)->metadata_lock, WT_SESSION_LOCKED_METADATA, op) /* - * WT_WITH_SCHEMA_LOCK -- + * WT_WITH_SCHEMA_LOCK, WT_WITH_SCHEMA_LOCK_NOWAIT -- * Acquire the schema lock, perform an operation, drop the lock. * Check that we are not already holding some other lock: the schema lock * must be taken first. */ -#define WT_WITH_SCHEMA_LOCK(session, ret, op) do { \ +#define WT_WITH_SCHEMA_LOCK(session, op) do { \ + WT_ASSERT(session, \ + F_ISSET(session, WT_SESSION_LOCKED_SCHEMA) || \ + !F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST | \ + WT_SESSION_NO_SCHEMA_LOCK | WT_SESSION_LOCKED_TABLE)); \ + WT_WITH_LOCK_WAIT(session, \ + &S2C(session)->schema_lock, WT_SESSION_LOCKED_SCHEMA, op); \ +} while (0) +#define WT_WITH_SCHEMA_LOCK_NOWAIT(session, ret, op) do { \ WT_ASSERT(session, \ F_ISSET(session, WT_SESSION_LOCKED_SCHEMA) || \ !F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST | \ WT_SESSION_NO_SCHEMA_LOCK | WT_SESSION_LOCKED_TABLE)); \ - WT_WITH_LOCK(session, ret, \ + WT_WITH_LOCK_NOWAIT(session, ret, \ &S2C(session)->schema_lock, WT_SESSION_LOCKED_SCHEMA, op); \ } while (0) /* - * WT_WITH_TABLE_LOCK -- + * WT_WITH_TABLE_READ_LOCK, WT_WITH_TABLE_WRITE_LOCK, + * WT_WITH_TABLE_WRITE_LOCK_NOWAIT -- * Acquire the table lock, perform an operation, drop the lock. + * The table lock is a read-write lock so the implementation is different + * to most other lock macros. + * + * Note: readlock always waits because some operations need the table lock + * to discard handles, and we only expect it to be held across short + * operations. */ -#define WT_WITH_TABLE_LOCK(session, ret, op) do { \ +#define WT_WITH_TABLE_READ_LOCK(session, op) do { \ + if (F_ISSET(session, WT_SESSION_LOCKED_TABLE)) { \ + op; \ + } else { \ + WT_ASSERT(session, \ + !F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); \ + __wt_readlock(session, &S2C(session)->table_lock); \ + F_SET(session, WT_SESSION_LOCKED_TABLE_READ); \ + op; \ + F_CLR(session, WT_SESSION_LOCKED_TABLE_READ); \ + __wt_readunlock(session, &S2C(session)->table_lock); \ + } \ +} while (0) + +#define WT_WITH_TABLE_WRITE_LOCK(session, op) do { \ + if (F_ISSET(session, WT_SESSION_LOCKED_TABLE_WRITE)) { \ + op; \ + } else { \ + WT_ASSERT(session, \ + !F_ISSET(session, WT_SESSION_LOCKED_TABLE_READ | \ + WT_SESSION_LOCKED_HANDLE_LIST)); \ + __wt_writelock(session, &S2C(session)->table_lock); \ + F_SET(session, WT_SESSION_LOCKED_TABLE_WRITE); \ + op; \ + F_CLR(session, WT_SESSION_LOCKED_TABLE_WRITE); \ + __wt_writeunlock(session, &S2C(session)->table_lock); \ + } \ +} while (0) +#define WT_WITH_TABLE_WRITE_LOCK_NOWAIT(session, ret, op) do { \ WT_ASSERT(session, \ - F_ISSET(session, WT_SESSION_LOCKED_TABLE) || \ - !F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); \ - WT_WITH_LOCK(session, ret, \ - &S2C(session)->table_lock, WT_SESSION_LOCKED_TABLE, op); \ + F_ISSET(session, WT_SESSION_LOCKED_TABLE_WRITE) || \ + !F_ISSET(session, WT_SESSION_LOCKED_TABLE_READ | \ + WT_SESSION_LOCKED_HANDLE_LIST)); \ + if (F_ISSET(session, WT_SESSION_LOCKED_TABLE_WRITE)) { \ + op; \ + } else if (((ret) = __wt_try_writelock(session, \ + &S2C(session)->table_lock)) == 0) { \ + F_SET(session, WT_SESSION_LOCKED_TABLE_WRITE); \ + op; \ + F_CLR(session, WT_SESSION_LOCKED_TABLE_WRITE); \ + __wt_writeunlock(session, &S2C(session)->table_lock); \ + } \ } while (0) /* @@ -176,19 +267,31 @@ struct __wt_table { WT_CONNECTION_IMPL *__conn = S2C(session); \ bool __checkpoint_locked = \ F_ISSET(session, WT_SESSION_LOCKED_CHECKPOINT); \ - bool __handle_locked = \ - F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST); \ - bool __table_locked = \ - F_ISSET(session, WT_SESSION_LOCKED_TABLE); \ + bool __handle_read_locked = \ + F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_READ); \ + bool __handle_write_locked = \ + F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE); \ + bool __table_read_locked = \ + F_ISSET(session, WT_SESSION_LOCKED_TABLE_READ); \ + bool __table_write_locked = \ + F_ISSET(session, WT_SESSION_LOCKED_TABLE_WRITE); \ bool __schema_locked = \ F_ISSET(session, WT_SESSION_LOCKED_SCHEMA); \ - if (__handle_locked) { \ - F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST); \ - __wt_spin_unlock(session, &__conn->dhandle_lock); \ + if (__handle_read_locked) { \ + F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST_READ); \ + __wt_readunlock(session, &__conn->dhandle_lock); \ + } \ + if (__handle_write_locked) { \ + F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE); \ + __wt_writeunlock(session, &__conn->dhandle_lock); \ + } \ + if (__table_read_locked) { \ + F_CLR(session, WT_SESSION_LOCKED_TABLE_READ); \ + __wt_readunlock(session, &__conn->table_lock); \ } \ - if (__table_locked) { \ - F_CLR(session, WT_SESSION_LOCKED_TABLE); \ - __wt_spin_unlock(session, &__conn->table_lock); \ + if (__table_write_locked) { \ + F_CLR(session, WT_SESSION_LOCKED_TABLE_WRITE); \ + __wt_writeunlock(session, &__conn->table_lock); \ } \ if (__schema_locked) { \ F_CLR(session, WT_SESSION_LOCKED_SCHEMA); \ @@ -207,12 +310,20 @@ struct __wt_table { __wt_spin_lock(session, &__conn->schema_lock); \ F_SET(session, WT_SESSION_LOCKED_SCHEMA); \ } \ - if (__table_locked) { \ - __wt_spin_lock(session, &__conn->table_lock); \ - F_SET(session, WT_SESSION_LOCKED_TABLE); \ + if (__table_read_locked) { \ + __wt_readlock(session, &__conn->table_lock); \ + F_SET(session, WT_SESSION_LOCKED_TABLE_READ); \ + } \ + if (__table_write_locked) { \ + __wt_writelock(session, &__conn->table_lock); \ + F_SET(session, WT_SESSION_LOCKED_TABLE_WRITE); \ + } \ + if (__handle_read_locked) { \ + __wt_readlock(session, &__conn->dhandle_lock); \ + F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST_READ); \ } \ - if (__handle_locked) { \ - __wt_spin_lock(session, &__conn->dhandle_lock); \ - F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST); \ + if (__handle_write_locked) { \ + __wt_writelock(session, &__conn->dhandle_lock); \ + F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE); \ } \ } while (0) diff --git a/src/third_party/wiredtiger/src/include/serial.i b/src/third_party/wiredtiger/src/include/serial.i index 982f196b0b8..0134e1a9c20 100644 --- a/src/third_party/wiredtiger/src/include/serial.i +++ b/src/third_party/wiredtiger/src/include/serial.i @@ -154,7 +154,7 @@ __col_append_serial_func(WT_SESSION_IMPL *session, WT_INSERT_HEAD *ins_head, static inline int __wt_col_append_serial(WT_SESSION_IMPL *session, WT_PAGE *page, WT_INSERT_HEAD *ins_head, WT_INSERT ***ins_stack, WT_INSERT **new_insp, - size_t new_ins_size, uint64_t *recnop, u_int skipdepth) + size_t new_ins_size, uint64_t *recnop, u_int skipdepth, bool exclusive) { WT_INSERT *new_ins = *new_insp; WT_DECL_RET; @@ -165,11 +165,16 @@ __wt_col_append_serial(WT_SESSION_IMPL *session, WT_PAGE *page, /* Clear references to memory we now own and must free on error. */ *new_insp = NULL; - /* Acquire the page's spinlock, call the worker function. */ - WT_PAGE_LOCK(session, page); + /* + * Acquire the page's spinlock unless we already have exclusive access. + * Then call the worker function. + */ + if (!exclusive) + WT_PAGE_LOCK(session, page); ret = __col_append_serial_func( session, ins_head, ins_stack, new_ins, recnop, skipdepth); - WT_PAGE_UNLOCK(session, page); + if (!exclusive) + WT_PAGE_UNLOCK(session, page); if (ret != 0) { /* Free unused memory on error. */ @@ -198,7 +203,7 @@ __wt_col_append_serial(WT_SESSION_IMPL *session, WT_PAGE *page, static inline int __wt_insert_serial(WT_SESSION_IMPL *session, WT_PAGE *page, WT_INSERT_HEAD *ins_head, WT_INSERT ***ins_stack, WT_INSERT **new_insp, - size_t new_ins_size, u_int skipdepth) + size_t new_ins_size, u_int skipdepth, bool exclusive) { WT_INSERT *new_ins = *new_insp; WT_DECL_RET; @@ -220,10 +225,12 @@ __wt_insert_serial(WT_SESSION_IMPL *session, WT_PAGE *page, ret = __insert_simple_func( session, ins_stack, new_ins, skipdepth); else { - WT_PAGE_LOCK(session, page); + if (!exclusive) + WT_PAGE_LOCK(session, page); ret = __insert_serial_func( session, ins_head, ins_stack, new_ins, skipdepth); - WT_PAGE_UNLOCK(session, page); + if (!exclusive) + WT_PAGE_UNLOCK(session, page); } if (ret != 0) { @@ -252,7 +259,8 @@ __wt_insert_serial(WT_SESSION_IMPL *session, WT_PAGE *page, */ static inline int __wt_update_serial(WT_SESSION_IMPL *session, WT_PAGE *page, - WT_UPDATE **srch_upd, WT_UPDATE **updp, size_t upd_size) + WT_UPDATE **srch_upd, WT_UPDATE **updp, size_t upd_size, + bool exclusive) { WT_DECL_RET; WT_UPDATE *obsolete, *upd = *updp; @@ -295,7 +303,7 @@ __wt_update_serial(WT_SESSION_IMPL *session, WT_PAGE *page, /* * If there are no subsequent WT_UPDATE structures we are done here. */ - if (upd->next == NULL) + if (upd->next == NULL || exclusive) return (0); /* @@ -316,11 +324,11 @@ __wt_update_serial(WT_SESSION_IMPL *session, WT_PAGE *page, } /* If we can't lock it, don't scan, that's okay. */ - if (__wt_try_writelock(session, &page->page_lock) != 0) + if (WT_PAGE_TRYLOCK(session, page) != 0) return (0); obsolete = __wt_update_obsolete_check(session, page, upd->next); - __wt_writeunlock(session, &page->page_lock); + WT_PAGE_UNLOCK(session, page); if (obsolete != NULL) __wt_update_obsolete_free(session, page, obsolete); diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index 3f9f495c134..1b2dfd1ed2b 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -23,7 +23,7 @@ struct __wt_data_handle_cache { * A hazard pointer. */ struct __wt_hazard { - WT_PAGE *page; /* Page address */ + WT_REF *ref; /* Page reference */ #ifdef HAVE_DIAGNOSTIC const char *file; /* File/line where hazard acquired */ int line; @@ -41,7 +41,7 @@ struct __wt_hazard { * WT_SESSION_IMPL -- * Implementation of WT_SESSION. */ -struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { +struct __wt_session_impl { WT_SESSION iface; void *lang_private; /* Language specific private storage */ @@ -52,8 +52,6 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { const char *lastop; /* Last operation */ uint32_t id; /* UID, offset in session array */ - WT_CONDVAR *cond; /* Condition variable */ - WT_EVENT_HANDLER *event_handler;/* Application's event handlers */ WT_DATA_HANDLE *dhandle; /* Current data handle */ @@ -69,13 +67,12 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { TAILQ_HEAD(__dhandles, __wt_data_handle_cache) dhandles; time_t last_sweep; /* Last sweep for dead handles */ - WT_CURSOR *cursor; /* Current cursor */ /* Cursors closed with the session */ TAILQ_HEAD(__cursors, __wt_cursor) cursors; WT_CURSOR_BACKUP *bkp_cursor; /* Hot backup cursor */ - WT_COMPACT *compact; /* Compaction information */ + WT_COMPACT_STATE *compact; /* Compaction information */ enum { WT_COMPACT_NONE=0, WT_COMPACT_RUNNING, WT_COMPACT_SUCCESS } compact_state; @@ -90,7 +87,7 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { void *meta_track_sub; /* Child transaction / save point */ size_t meta_track_alloc; /* Currently allocated */ int meta_track_nest; /* Nesting level of meta transaction */ -#define WT_META_TRACKING(session) (session->meta_track_next != NULL) +#define WT_META_TRACKING(session) ((session)->meta_track_next != NULL) /* * Each session keeps a cache of table handles. The set of handles @@ -100,6 +97,10 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { */ TAILQ_HEAD(__tables, __wt_table) tables; + /* Current rwlock for callback. */ + WT_RWLOCK *current_rwlock; + uint8_t current_rwticket; + WT_ITEM **scratch; /* Temporary memory for any function */ u_int scratch_alloc; /* Currently allocated */ size_t scratch_cached; /* Scratch bytes cached */ @@ -153,20 +154,16 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { uint32_t flags; /* - * The split stash memory and hazard information persist past session - * close because they are accessed by threads of control other than the - * thread owning the session. - * + * All of the following fields live at the end of the structure so it's + * easier to clear everything but the fields that persist. + */ +#define WT_SESSION_CLEAR_SIZE (offsetof(WT_SESSION_IMPL, rnd)) + + /* * The random number state persists past session close because we don't - * want to repeatedly allocate repeated values for skiplist depth if the + * want to repeatedly use the same values for skiplist depth when the * application isn't caching sessions. - * - * All of these fields live at the end of the structure so it's easier - * to clear everything but the fields that persist. */ -#define WT_SESSION_CLEAR_SIZE(s) \ - (WT_PTRDIFF(&(s)->rnd, s)) - WT_RAND_STATE rnd; /* Random number generation state */ /* Hashed handle reference list array */ @@ -175,6 +172,9 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { TAILQ_HEAD(__tables_hash, __wt_table) *tablehash; /* + * Split stash memory persists past session close because it's accessed + * by threads of control other than the thread owning the session. + * * Splits can "free" memory that may still be in use, and we use a * split generation number to track it, that is, the session stores a * reference to the memory and allocates a split generation; when no @@ -194,15 +194,22 @@ struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_session_impl { /* * Hazard pointers. * + * Hazard information persists past session close because it's accessed + * by threads of control other than the thread owning the session. + * * Use the non-NULL state of the hazard field to know if the session has * previously been initialized. */ #define WT_SESSION_FIRST_USE(s) \ ((s)->hazard == NULL) - /* The number of hazard pointers grows dynamically. */ -#define WT_HAZARD_INCR 1 - uint32_t hazard_size; /* Allocated slots in hazard array. */ + /* + * The hazard pointer array grows as necessary, initialize with 250 + * slots. + */ +#define WT_SESSION_INITIAL_HAZARD_SLOTS 250 + uint32_t hazard_size; /* Hazard pointer array slots */ + uint32_t hazard_inuse; /* Hazard pointer array slots in-use */ uint32_t nhazard; /* Count of active hazard pointers */ WT_HAZARD *hazard; /* Hazard pointer array */ }; diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index e53414fc0c9..6c274484bcb 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -72,7 +72,7 @@ * and the session ID is a small, monotonically increasing number. */ #define WT_STATS_SLOT_ID(session) \ - ((session)->id) % WT_COUNTER_SLOTS + (((session)->id) % WT_COUNTER_SLOTS) /* * Statistic structures are arrays of int64_t's. We have functions to read/write @@ -310,10 +310,15 @@ struct __wt_connection_stats { int64_t cache_eviction_slow; int64_t cache_eviction_state; int64_t cache_eviction_walks_abandoned; + int64_t cache_eviction_active_workers; + int64_t cache_eviction_worker_created; int64_t cache_eviction_worker_evicting; + int64_t cache_eviction_worker_removed; + int64_t cache_eviction_stable_state_workers; int64_t cache_eviction_force_fail; int64_t cache_eviction_walks_active; int64_t cache_eviction_walks_started; + int64_t cache_eviction_force_retune; int64_t cache_eviction_hazard; int64_t cache_hazard_checks; int64_t cache_hazard_walks; @@ -388,9 +393,7 @@ struct __wt_connection_stats { int64_t lock_checkpoint_count; int64_t lock_checkpoint_wait_application; int64_t lock_checkpoint_wait_internal; - int64_t lock_handle_list_count; - int64_t lock_handle_list_wait_application; - int64_t lock_handle_list_wait_internal; + int64_t lock_handle_list_wait_eviction; int64_t lock_metadata_count; int64_t lock_metadata_wait_application; int64_t lock_metadata_wait_internal; @@ -402,9 +405,11 @@ struct __wt_connection_stats { int64_t lock_table_wait_internal; int64_t log_slot_switch_busy; int64_t log_slot_closes; + int64_t log_slot_active_closed; int64_t log_slot_races; int64_t log_slot_transitions; int64_t log_slot_joins; + int64_t log_slot_no_free_slots; int64_t log_slot_unbuffered; int64_t log_bytes_payload; int64_t log_bytes_written; @@ -445,6 +450,9 @@ struct __wt_connection_stats { int64_t rec_split_stashed_objects; int64_t session_cursor_open; int64_t session_open; + int64_t session_table_alter_fail; + int64_t session_table_alter_success; + int64_t session_table_alter_skip; int64_t session_table_compact_fail; int64_t session_table_compact_success; int64_t session_table_create_fail; @@ -561,6 +569,7 @@ struct __wt_dsrc_stats { int64_t cache_pages_requested; int64_t cache_write; int64_t cache_write_restore; + int64_t cache_bytes_dirty; int64_t cache_eviction_clean; int64_t cache_state_gen_avg_gap; int64_t cache_state_avg_written_size; diff --git a/src/third_party/wiredtiger/src/include/thread_group.h b/src/third_party/wiredtiger/src/include/thread_group.h index 76758a090c4..77cff00dc8d 100644 --- a/src/third_party/wiredtiger/src/include/thread_group.h +++ b/src/third_party/wiredtiger/src/include/thread_group.h @@ -40,7 +40,7 @@ struct __wt_thread_group { const char *name; /* Name */ - WT_RWLOCK *lock; /* Protects group changes */ + WT_RWLOCK lock; /* Protects group changes */ /* * Condition signalled when wanting to wake up threads that are diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index 8128e8e4cc2..7e802c188ab 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -49,8 +49,11 @@ WT_ASSERT((s), (s)->txn.forced_iso > 0); \ (s)->txn.forced_iso--; \ WT_ASSERT((s), txn_state->id == saved_state.id && \ + (txn_state->metadata_pinned == saved_state.metadata_pinned ||\ + saved_state.metadata_pinned == WT_TXN_NONE) && \ (txn_state->pinned_id == saved_state.pinned_id || \ saved_state.pinned_id == WT_TXN_NONE)); \ + txn_state->metadata_pinned = saved_state.metadata_pinned; \ txn_state->pinned_id = saved_state.pinned_id; \ } while (0) @@ -59,14 +62,17 @@ struct __wt_named_snapshot { TAILQ_ENTRY(__wt_named_snapshot) q; - uint64_t pinned_id, snap_min, snap_max; + uint64_t id, pinned_id, snap_min, snap_max; uint64_t *snapshot; uint32_t snapshot_count; }; -struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) __wt_txn_state { +struct __wt_txn_state { + WT_CACHE_LINE_PAD_BEGIN volatile uint64_t id; volatile uint64_t pinned_id; + volatile uint64_t metadata_pinned; + WT_CACHE_LINE_PAD_END }; struct __wt_txn_global { @@ -86,7 +92,7 @@ struct __wt_txn_global { * Prevents the oldest ID moving forwards while threads are scanning * the global transaction state. */ - WT_RWLOCK *scan_rwlock; + WT_RWLOCK scan_rwlock; /* * Track information about the running checkpoint. The transaction @@ -94,14 +100,21 @@ struct __wt_txn_global { * for a long time so we keep them out of regular visibility checks. * Eviction and checkpoint operations know when they need to be aware * of checkpoint transactions. + * + * We rely on the fact that (a) the only table a checkpoint updates is + * the metadata; and (b) once checkpoint has finished reading a table, + * it won't revisit it. */ + volatile bool checkpoint_running; /* Checkpoint running */ volatile uint32_t checkpoint_id; /* Checkpoint's session ID */ - volatile uint64_t checkpoint_gen; - volatile uint64_t checkpoint_pinned; + volatile uint64_t checkpoint_gen; /* Checkpoint generation */ + volatile uint64_t checkpoint_pinned; /* Oldest ID for checkpoint */ volatile uint64_t checkpoint_txnid; /* Checkpoint's txn ID */ + volatile uint64_t metadata_pinned; /* Oldest ID for metadata */ + /* Named snapshot state. */ - WT_RWLOCK *nsnap_rwlock; + WT_RWLOCK nsnap_rwlock; volatile uint64_t nsnap_oldest_id; TAILQ_HEAD(__wt_nsnap_qh, __wt_named_snapshot) nsnaph; diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index cf7e2eafc65..314c948e4d1 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -62,7 +62,6 @@ __wt_txn_unmodify(WT_SESSION_IMPL *session) static inline int __wt_txn_modify(WT_SESSION_IMPL *session, WT_UPDATE *upd) { - WT_DECL_RET; WT_TXN_OP *op; WT_TXN *txn; @@ -77,7 +76,7 @@ __wt_txn_modify(WT_SESSION_IMPL *session, WT_UPDATE *upd) WT_TXN_OP_INMEM : WT_TXN_OP_BASIC; op->u.upd = upd; upd->txnid = session->txn.id; - return (ret); + return (0); } /* @@ -111,6 +110,13 @@ __wt_txn_oldest_id(WT_SESSION_IMPL *session) txn_global = &S2C(session)->txn_global; btree = S2BT_SAFE(session); + /* + * The metadata is tracked specially because of optimizations for + * checkpoints. + */ + if (session->dhandle != NULL && WT_IS_METADATA(session->dhandle)) + return (txn_global->metadata_pinned); + /* * Take a local copy of these IDs in case they are updated while we are * checking visibility. The read of the transaction ID pinned by a @@ -119,7 +125,8 @@ __wt_txn_oldest_id(WT_SESSION_IMPL *session) * minimum of it with the oldest ID, which is what we want. */ oldest_id = txn_global->oldest_id; - include_checkpoint_txn = btree == NULL || btree->include_checkpoint_txn; + include_checkpoint_txn = btree == NULL || + btree->checkpoint_gen != txn_global->checkpoint_gen; WT_READ_BARRIER(); checkpoint_pinned = txn_global->checkpoint_pinned; @@ -262,7 +269,7 @@ __wt_txn_begin(WT_SESSION_IMPL *session, const char *cfg[]) * eviction, it's better to do it beforehand. */ WT_RET(__wt_cache_eviction_check(session, false, NULL)); - WT_RET(__wt_txn_get_snapshot(session)); + __wt_txn_get_snapshot(session); } F_SET(txn, WT_TXN_RUNNING); @@ -451,7 +458,7 @@ __wt_txn_read_last(WT_SESSION_IMPL *session) * __wt_txn_cursor_op -- * Called for each cursor operation. */ -static inline int +static inline void __wt_txn_cursor_op(WT_SESSION_IMPL *session) { WT_TXN *txn; @@ -482,10 +489,10 @@ __wt_txn_cursor_op(WT_SESSION_IMPL *session) if (txn->isolation == WT_ISO_READ_UNCOMMITTED) { if (txn_state->pinned_id == WT_TXN_NONE) txn_state->pinned_id = txn_global->last_running; + if (txn_state->metadata_pinned == WT_TXN_NONE) + txn_state->metadata_pinned = txn_state->pinned_id; } else if (!F_ISSET(txn, WT_TXN_HAS_SNAPSHOT)) - WT_RET(__wt_txn_get_snapshot(session)); - - return (0); + __wt_txn_get_snapshot(session); } /* diff --git a/src/third_party/wiredtiger/src/include/verify_build.h b/src/third_party/wiredtiger/src/include/verify_build.h index 477b9b7c134..a657b9ac460 100644 --- a/src/third_party/wiredtiger/src/include/verify_build.h +++ b/src/third_party/wiredtiger/src/include/verify_build.h @@ -6,9 +6,6 @@ * See the file LICENSE for redistribution information. */ -#undef ALIGN_CHECK -#undef SIZE_CHECK - /* * NOTE: If you see a compile failure in this file, your compiler is laying out * structs in memory in a way WiredTiger does not expect. Please refer to the @@ -36,12 +33,12 @@ */ #define WT_STATIC_ASSERT(cond) (void)sizeof(char[1 - 2 * !(cond)]) -#define SIZE_CHECK(type, e) do { \ +#define WT_SIZE_CHECK(type, e) do { \ char __check_##type[1 - 2 * !(sizeof(type) == (e))]; \ (void)__check_##type; \ } while (0) -#define ALIGN_CHECK(type, a) \ +#define WT_ALIGN_CHECK(type, a) \ WT_STATIC_ASSERT(WT_ALIGN(sizeof(type), (a)) == sizeof(type)) /* @@ -53,8 +50,16 @@ static inline void __wt_verify_build(void) { /* Check specific structures weren't padded. */ - SIZE_CHECK(WT_BLOCK_DESC, WT_BLOCK_DESC_SIZE); - SIZE_CHECK(WT_REF, WT_REF_SIZE); + WT_SIZE_CHECK(WT_BLOCK_DESC, WT_BLOCK_DESC_SIZE); + WT_SIZE_CHECK(WT_REF, WT_REF_SIZE); + + /* Check specific structures were padded. */ +#define WT_PADDING_CHECK(s) \ + WT_STATIC_ASSERT( \ + sizeof(s) > WT_CACHE_LINE_ALIGNMENT || \ + sizeof(s) % WT_CACHE_LINE_ALIGNMENT == 0) + WT_PADDING_CHECK(WT_LOGSLOT); + WT_PADDING_CHECK(WT_TXN_STATE); /* * The btree code encodes key/value pairs in size_t's, and requires at @@ -71,6 +76,3 @@ __wt_verify_build(void) */ WT_STATIC_ASSERT(sizeof(wt_off_t) == 8); } - -#undef ALIGN_CHECK -#undef SIZE_CHECK diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 665e8eaf4b0..ddecb2ac765 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -36,7 +36,7 @@ extern "C" { #if defined(DOXYGEN) || defined(SWIG) #define __F(func) func #else -#define __F(func) (*func) +#define __F(func) (*(func)) #endif #ifdef SWIG @@ -114,16 +114,16 @@ struct __wt_item { size_t size; #ifndef DOXYGEN -#define WT_ITEM_ALIGNED 0x00000001 -#define WT_ITEM_INUSE 0x00000002 - /* This appears in the middle of the struct to avoid padding. */ - /*! Object flags (internal use). */ - uint32_t flags; - /*! Managed memory chunk (internal use). */ void *mem; + /*! Managed memory size (internal use). */ size_t memsize; + +#define WT_ITEM_ALIGNED 0x00000001 +#define WT_ITEM_INUSE 0x00000002 + /*! Object flags (internal use). */ + uint32_t flags; #endif }; @@ -427,7 +427,7 @@ struct __wt_cursor { * * @param cursor the cursor handle * @errors - * In particular, if \c overwrite is not configured and a record with + * In particular, if \c overwrite=false is configured and a record with * the specified key already exists, ::WT_DUPLICATE_KEY is returned. * Also, if \c in_memory is configured for the database and the insert * requires more than the configured cache size to complete, @@ -452,7 +452,9 @@ struct __wt_cursor { * * On success, the cursor ends positioned at the modified record; to * minimize cursor resources, the WT_CURSOR::reset method should be - * called as soon as the cursor no longer needs that position. + * called as soon as the cursor no longer needs that position. (The + * WT_CURSOR::insert method never keeps a cursor position and may be + * more efficient for that reason.) * * The maximum length of a single column stored in a table is not fixed * (as it partially depends on the underlying file configuration), but @@ -460,7 +462,7 @@ struct __wt_cursor { * * @param cursor the cursor handle * @errors - * In particular, if \c overwrite is not configured and no record with + * In particular, if \c overwrite=false is configured and no record with * the specified key exists, ::WT_NOTFOUND is returned. * Also, if \c in_memory is configured for the database and the insert * requires more than the configured cache size to complete, @@ -477,8 +479,18 @@ struct __wt_cursor { * * @snippet ex_all.c Remove a record * - * If the cursor was not configured with "overwrite=true", the key must - * be set and the key's record must exist; the record will be removed. + * If the cursor was configured with "overwrite=false" (not the + * default), the key must be set and the key's record must exist; the + * record will be removed. + * + * Any cursor position does not change: if the cursor was positioned + * before the WT_CURSOR::remove call, the cursor remains positioned + * at the removed record; to minimize cursor resources, the + * WT_CURSOR::reset method should be called as soon as the cursor no + * longer needs that position. If the cursor was not positioned before + * the WT_CURSOR::remove call, the cursor ends with no position, and a + * subsequent call to the WT_CURSOR::next (WT_CURSOR::prev) method will + * iterate from the beginning (end) of the table. * * @snippet ex_all.c Remove a record and fail if DNE * @@ -486,14 +498,10 @@ struct __wt_cursor { * (that is, a store with an 'r' type key and 't' type value) is * identical to setting the record's value to 0. * - * On success, the cursor ends positioned at the removed record; to - * minimize cursor resources, the WT_CURSOR::reset method should be - * called as soon as the cursor no longer needs that position. - * * @param cursor the cursor handle * @errors - * In particular, if \c overwrite is not configured and no record with - * the specified key exists, ::WT_NOTFOUND is returned. + * In particular, if \c overwrite=false is configured and no record + * with the specified key exists, ::WT_NOTFOUND is returned. */ int __F(remove)(WT_CURSOR *cursor); /*! @} */ @@ -576,8 +584,9 @@ struct __wt_cursor { #define WT_CURSTD_OPEN 0x00200 #define WT_CURSTD_OVERWRITE 0x00400 #define WT_CURSTD_RAW 0x00800 -#define WT_CURSTD_VALUE_EXT 0x01000 /* Value points out of the tree. */ -#define WT_CURSTD_VALUE_INT 0x02000 /* Value points into the tree. */ +#define WT_CURSTD_RAW_SEARCH 0x01000 +#define WT_CURSTD_VALUE_EXT 0x02000 /* Value points out of the tree. */ +#define WT_CURSTD_VALUE_INT 0x04000 /* Value points into the tree. */ #define WT_CURSTD_VALUE_SET (WT_CURSTD_VALUE_EXT | WT_CURSTD_VALUE_INT) uint32_t flags; #endif @@ -799,6 +808,34 @@ struct __wt_session { void *app_private; #endif + /*! + * Alter a table. + * + * This will allow modification of some table settings after + * creation. + * + * @snippet ex_all.c Alter a table + * + * @param session the session handle + * @param name the URI of the object to alter, such as \c "table:stock" + * @configstart{WT_SESSION.alter, see dist/api_data.py} + * @config{access_pattern_hint, It is recommended that workloads that + * consist primarily of updates and/or point queries specify \c random. + * Workloads that do many cursor scans through large ranges of data + * specify \c sequential and other workloads specify \c none. The + * option leads to an advisory call to an appropriate operating system + * API where available., a string\, chosen from the following options: + * \c "none"\, \c "random"\, \c "sequential"; default \c none.} + * @config{cache_resident, do not ever evict the object's pages from + * cache. Not compatible with LSM tables; see @ref + * tuning_cache_resident for more information., a boolean flag; default + * \c false.} + * @configend + * @errors + */ + int __F(alter)(WT_HANDLE_CLOSED(WT_SESSION) *session, + const char *name, const char *config); + /*! * Close the session handle. * @@ -827,6 +864,11 @@ struct __wt_session { * * @param session the session handle * @configstart{WT_SESSION.reconfigure, see dist/api_data.py} + * @config{ignore_cache_size, when set\, operations performed by this + * session ignore the cache size and are not blocked when the cache is + * full. Note that use of this option for operations that create cache + * pressure can starve ordinary sessions that obey the cache size., a + * boolean flag; default \c false.} * @config{isolation, the default isolation level for operations in this * session., a string\, chosen from the following options: \c * "read-uncommitted"\, \c "read-committed"\, \c "snapshot"; default \c @@ -990,6 +1032,13 @@ struct __wt_session { * \c "table:stock". For a description of URI formats * see @ref data_sources. * @configstart{WT_SESSION.create, see dist/api_data.py} + * @config{access_pattern_hint, It is recommended that workloads that + * consist primarily of updates and/or point queries specify \c random. + * Workloads that do many cursor scans through large ranges of data + * specify \c sequential and other workloads specify \c none. The + * option leads to an advisory call to an appropriate operating system + * API where available., a string\, chosen from the following options: + * \c "none"\, \c "random"\, \c "sequential"; default \c none.} * @config{allocation_size, the file unit allocation size\, in bytes\, * must a power-of-two; smaller values decrease the file space required * by overflow items\, and the default value of 4KB is a good choice @@ -1193,8 +1242,8 @@ struct __wt_session { * @config{split_pct, the Btree page split size as a percentage of the * maximum Btree page size\, that is\, when a Btree page is split\, it * will be split into smaller pages\, where each page is the specified - * percentage of the maximum Btree page size., an integer between 25 and - * 100; default \c 75.} + * percentage of the maximum Btree page size., an integer between 50 and + * 100; default \c 90.} * @config{type, set the type of data source used to store a column * group\, index or simple table. By default\, a \c "file:" URI is * derived from the object name. The \c type configuration can be used @@ -1431,6 +1480,10 @@ struct __wt_session { * contains. * @snippet ex_all.c Truncate a range * + * Any specified cursors end with no position, and subsequent calls to + * the WT_CURSOR::next (WT_CURSOR::prev) method will iterate from the + * beginning (end) of the table. + * * @param session the session handle * @param name the URI of the file or table to truncate * @param start optional cursor marking the first record discarded; @@ -1636,6 +1689,10 @@ struct __wt_session { * including the specified name., a string; default empty.} * @config{ * ),,} + * @config{include_updates, make updates from the current transaction + * visible to users of the named snapshot. Transactions started with + * such a named snapshot are restricted to being read-only., a boolean + * flag; default \c false.} * @config{name, specify a name for the snapshot., a string; default * empty.} * @configend @@ -1811,7 +1868,7 @@ struct __wt_connection { * threads WiredTiger will start to help evict pages from cache. The * number of threads started will vary depending on the current eviction * load. Each eviction worker thread uses a session from the configured - * session_max., an integer between 1 and 20; default \c 1.} + * session_max., an integer between 1 and 20; default \c 8.} * @config{    threads_min, minimum number of * threads WiredTiger will start to help evict pages from cache. The * number of threads currently running will vary depending on the @@ -1938,12 +1995,13 @@ struct __wt_connection { * as a list\, such as "verbose=[evictserver\,read]"., a * list\, with values chosen from the following options: \c "api"\, \c * "block"\, \c "checkpoint"\, \c "compact"\, \c "evict"\, \c - * "evictserver"\, \c "fileops"\, \c "handleops"\, \c "log"\, \c "lsm"\, - * \c "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c "overflow"\, \c - * "read"\, \c "rebalance"\, \c "reconcile"\, \c "recovery"\, \c - * "salvage"\, \c "shared_cache"\, \c "split"\, \c "temporary"\, \c - * "thread_group"\, \c "transaction"\, \c "verify"\, \c "version"\, \c - * "write"; default empty.} + * "evict_stuck"\, \c "evictserver"\, \c "fileops"\, \c "handleops"\, \c + * "log"\, \c "lsm"\, \c "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c + * "overflow"\, \c "read"\, \c "rebalance"\, \c "reconcile"\, \c + * "recovery"\, \c "recovery_progress"\, \c "salvage"\, \c + * "shared_cache"\, \c "split"\, \c "temporary"\, \c "thread_group"\, \c + * "transaction"\, \c "verify"\, \c "version"\, \c "write"; default + * empty.} * @configend * @errors */ @@ -2005,6 +2063,11 @@ struct __wt_connection { * connection's error handler is used. See @ref error_handling_event * for more information. * @configstart{WT_CONNECTION.open_session, see dist/api_data.py} + * @config{ignore_cache_size, when set\, operations performed by this + * session ignore the cache size and are not blocked when the cache is + * full. Note that use of this option for operations that create cache + * pressure can starve ordinary sessions that obey the cache size., a + * boolean flag; default \c false.} * @config{isolation, the default isolation level for operations in this * session., a string\, chosen from the following options: \c * "read-uncommitted"\, \c "read-committed"\, \c "snapshot"; default \c @@ -2204,6 +2267,11 @@ struct __wt_connection { * I/O. The default value of -1 indicates a platform-specific alignment value * should be used (4KB on Linux systems when direct I/O is configured\, zero * elsewhere)., an integer between -1 and 1MB; default \c -1.} + * @config{builtin_extension_config, A structure where the keys are the names of + * builtin extensions and the values are passed to WT_CONNECTION::load_extension + * as the \c config parameter (for example\, + * builtin_extension_config={zlib={compression_level=3}})., a + * string; default empty.} * @config{cache_overhead, assume the heap allocator overhead is the specified * percentage\, and adjust the cache usage by that amount (for example\, if * there is 10GB of data in cache\, a percentage of 10 means WiredTiger treats @@ -2277,7 +2345,7 @@ struct __wt_connection { * WiredTiger will start to help evict pages from cache. The number of threads * started will vary depending on the current eviction load. Each eviction * worker thread uses a session from the configured session_max., an integer - * between 1 and 20; default \c 1.} + * between 1 and 20; default \c 8.} * @config{    threads_min, * minimum number of threads WiredTiger will start to help evict pages from * cache. The number of threads currently running will vary depending on the @@ -2307,7 +2375,7 @@ struct __wt_connection { * @config{exclusive, fail if the database already exists\, generally used with * the \c create option., a boolean flag; default \c false.} * @config{extensions, list of shared library extensions to load (using dlopen). - * Any values specified to an library extension are passed to + * Any values specified to a library extension are passed to * WT_CONNECTION::load_extension as the \c config parameter (for example\, * extensions=(/path/ext.so={entry=my_entry}))., a list of strings; * default empty.} @@ -2329,8 +2397,6 @@ struct __wt_connection { * in seconds at which to check for files that are inactive and close them., an * integer between 1 and 100000; default \c 10.} * @config{ ),,} - * @config{hazard_max, maximum number of simultaneous hazard pointers per - * session handle., an integer greater than or equal to 15; default \c 1000.} * @config{in_memory, keep data in-memory only. See @ref in_memory for more * information., a boolean flag; default \c false.} * @config{log = (, enable logging. Enabling logging uses three sessions from @@ -2461,12 +2527,13 @@ struct __wt_connection { * WiredTiger is configured with --enable-verbose. Options are given as a * list\, such as "verbose=[evictserver\,read]"., a list\, with * values chosen from the following options: \c "api"\, \c "block"\, \c - * "checkpoint"\, \c "compact"\, \c "evict"\, \c "evictserver"\, \c "fileops"\, - * \c "handleops"\, \c "log"\, \c "lsm"\, \c "lsm_manager"\, \c "metadata"\, \c - * "mutex"\, \c "overflow"\, \c "read"\, \c "rebalance"\, \c "reconcile"\, \c - * "recovery"\, \c "salvage"\, \c "shared_cache"\, \c "split"\, \c "temporary"\, - * \c "thread_group"\, \c "transaction"\, \c "verify"\, \c "version"\, \c - * "write"; default empty.} + * "checkpoint"\, \c "compact"\, \c "evict"\, \c "evict_stuck"\, \c + * "evictserver"\, \c "fileops"\, \c "handleops"\, \c "log"\, \c "lsm"\, \c + * "lsm_manager"\, \c "metadata"\, \c "mutex"\, \c "overflow"\, \c "read"\, \c + * "rebalance"\, \c "reconcile"\, \c "recovery"\, \c "recovery_progress"\, \c + * "salvage"\, \c "shared_cache"\, \c "split"\, \c "temporary"\, \c + * "thread_group"\, \c "transaction"\, \c "verify"\, \c "version"\, \c "write"; + * default empty.} * @config{write_through, Use \c FILE_FLAG_WRITE_THROUGH on Windows to write to * files. Ignored on non-Windows systems. Options are given as a list\, such * as "write_through=[data]". Configuring \c write_through requires @@ -3010,27 +3077,27 @@ const char *wiredtiger_version(int *majorp, int *minorp, int *patchp); * transaction is in progress, it should be rolled back and the operation * retried in a new transaction. */ -#define WT_ROLLBACK -31800 +#define WT_ROLLBACK (-31800) /*! * Attempt to insert an existing key. * This error is generated when the application attempts to insert a record with * the same key as an existing record without the 'overwrite' configuration to * WT_SESSION::open_cursor. */ -#define WT_DUPLICATE_KEY -31801 +#define WT_DUPLICATE_KEY (-31801) /*! * Non-specific WiredTiger error. * This error is returned when an error is not covered by a specific error * return. */ -#define WT_ERROR -31802 +#define WT_ERROR (-31802) /*! * Item not found. * This error indicates an operation did not find a value to return. This * includes cursor search and other operations where no record matched the * cursor's search key such as WT_CURSOR::update or WT_CURSOR::remove. */ -#define WT_NOTFOUND -31803 +#define WT_NOTFOUND (-31803) /*! * WiredTiger library panic. * This error indicates an underlying problem that requires the application exit @@ -3038,17 +3105,17 @@ const char *wiredtiger_version(int *majorp, int *minorp, int *patchp); * returned from a WiredTiger interface, no further WiredTiger calls are * required. */ -#define WT_PANIC -31804 +#define WT_PANIC (-31804) /*! @cond internal */ /*! Restart the operation (internal). */ -#define WT_RESTART -31805 +#define WT_RESTART (-31805) /*! @endcond */ /*! * Recovery must be run to continue. * This error is generated when wiredtiger_open is configured to return an error * if recovery is required to use the database. */ -#define WT_RUN_RECOVERY -31806 +#define WT_RUN_RECOVERY (-31806) /*! * Operation would overflow cache. * This error is only generated when wiredtiger_open is configured to run in- @@ -3057,7 +3124,7 @@ const char *wiredtiger_version(int *majorp, int *minorp, int *patchp); * progress, it should be rolled back and the operation retried in a new * transaction. */ -#define WT_CACHE_FULL -31807 +#define WT_CACHE_FULL (-31807) /* * Error return section: END * DO NOT EDIT: automatically built by dist/api_err.py. @@ -3412,6 +3479,14 @@ struct __wt_compressor { * @snippet ex_data_source.c WT_DATA_SOURCE register */ struct __wt_data_source { + /*! + * Callback to alter an object. + * + * @snippet ex_data_source.c WT_DATA_SOURCE alter + */ + int (*alter)(WT_DATA_SOURCE *dsrc, WT_SESSION *session, + const char *uri, WT_CONFIG_ARG *config); + /*! * Callback to create a new object. * @@ -3715,21 +3790,25 @@ typedef enum { WT_FS_OPEN_FILE_TYPE_REGULAR /*!< open a regular file */ } WT_FS_OPEN_FILE_TYPE; +/*! WT_FILE_SYSTEM::open_file flags: random access pattern */ +#define WT_FS_OPEN_ACCESS_RAND 0x001 +/*! WT_FILE_SYSTEM::open_file flags: sequential access pattern */ +#define WT_FS_OPEN_ACCESS_SEQ 0x002 /*! WT_FILE_SYSTEM::open_file flags: create if does not exist */ -#define WT_FS_OPEN_CREATE 0x001 +#define WT_FS_OPEN_CREATE 0x004 /*! WT_FILE_SYSTEM::open_file flags: direct I/O requested */ -#define WT_FS_OPEN_DIRECTIO 0x002 +#define WT_FS_OPEN_DIRECTIO 0x008 /*! WT_FILE_SYSTEM::open_file flags: file creation must be durable */ -#define WT_FS_OPEN_DURABLE 0x004 +#define WT_FS_OPEN_DURABLE 0x010 /*! * WT_FILE_SYSTEM::open_file flags: return EBUSY if exclusive use not available */ -#define WT_FS_OPEN_EXCLUSIVE 0x008 +#define WT_FS_OPEN_EXCLUSIVE 0x020 #ifndef DOXYGEN -#define WT_FS_OPEN_FIXED 0x010 /* Path not home relative (internal) */ +#define WT_FS_OPEN_FIXED 0x040 /* Path not home relative (internal) */ #endif /*! WT_FILE_SYSTEM::open_file flags: open is read-only */ -#define WT_FS_OPEN_READONLY 0x020 +#define WT_FS_OPEN_READONLY 0x080 /*! * WT_FILE_SYSTEM::remove or WT_FILE_SYSTEM::rename flags: the remove or rename @@ -4365,390 +4444,406 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_CACHE_EVICTION_STATE 1051 /*! cache: eviction walks abandoned */ #define WT_STAT_CONN_CACHE_EVICTION_WALKS_ABANDONED 1052 +/*! cache: eviction worker thread active */ +#define WT_STAT_CONN_CACHE_EVICTION_ACTIVE_WORKERS 1053 +/*! cache: eviction worker thread created */ +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_CREATED 1054 /*! cache: eviction worker thread evicting pages */ -#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1053 +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_EVICTING 1055 +/*! cache: eviction worker thread removed */ +#define WT_STAT_CONN_CACHE_EVICTION_WORKER_REMOVED 1056 +/*! cache: eviction worker thread stable number */ +#define WT_STAT_CONN_CACHE_EVICTION_STABLE_STATE_WORKERS 1057 /*! cache: failed eviction of pages that exceeded the in-memory maximum */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1054 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_FAIL 1058 /*! cache: files with active eviction walks */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ACTIVE 1055 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_ACTIVE 1059 /*! cache: files with new eviction walks started */ -#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STARTED 1056 +#define WT_STAT_CONN_CACHE_EVICTION_WALKS_STARTED 1060 +/*! cache: force re-tuning of eviction workers once in a while */ +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_RETUNE 1061 /*! cache: hazard pointer blocked page eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1057 +#define WT_STAT_CONN_CACHE_EVICTION_HAZARD 1062 /*! cache: hazard pointer check calls */ -#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1058 +#define WT_STAT_CONN_CACHE_HAZARD_CHECKS 1063 /*! cache: hazard pointer check entries walked */ -#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1059 +#define WT_STAT_CONN_CACHE_HAZARD_WALKS 1064 /*! cache: hazard pointer maximum array length */ -#define WT_STAT_CONN_CACHE_HAZARD_MAX 1060 +#define WT_STAT_CONN_CACHE_HAZARD_MAX 1065 /*! cache: in-memory page passed criteria to be split */ -#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1061 +#define WT_STAT_CONN_CACHE_INMEM_SPLITTABLE 1066 /*! cache: in-memory page splits */ -#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1062 +#define WT_STAT_CONN_CACHE_INMEM_SPLIT 1067 /*! cache: internal pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1063 +#define WT_STAT_CONN_CACHE_EVICTION_INTERNAL 1068 /*! cache: internal pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1064 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_INTERNAL 1069 /*! cache: leaf pages split during eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1065 +#define WT_STAT_CONN_CACHE_EVICTION_SPLIT_LEAF 1070 /*! cache: lookaside table insert calls */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_INSERT 1066 +#define WT_STAT_CONN_CACHE_LOOKASIDE_INSERT 1071 /*! cache: lookaside table remove calls */ -#define WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE 1067 +#define WT_STAT_CONN_CACHE_LOOKASIDE_REMOVE 1072 /*! cache: maximum bytes configured */ -#define WT_STAT_CONN_CACHE_BYTES_MAX 1068 +#define WT_STAT_CONN_CACHE_BYTES_MAX 1073 /*! cache: maximum page size at eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1069 +#define WT_STAT_CONN_CACHE_EVICTION_MAXIMUM_PAGE_SIZE 1074 /*! cache: modified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1070 +#define WT_STAT_CONN_CACHE_EVICTION_DIRTY 1075 /*! cache: modified pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1071 +#define WT_STAT_CONN_CACHE_EVICTION_APP_DIRTY 1076 /*! cache: overflow pages read into cache */ -#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1072 +#define WT_STAT_CONN_CACHE_READ_OVERFLOW 1077 /*! cache: overflow values cached in memory */ -#define WT_STAT_CONN_CACHE_OVERFLOW_VALUE 1073 +#define WT_STAT_CONN_CACHE_OVERFLOW_VALUE 1078 /*! cache: page split during eviction deepened the tree */ -#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1074 +#define WT_STAT_CONN_CACHE_EVICTION_DEEPEN 1079 /*! cache: page written requiring lookaside records */ -#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1075 +#define WT_STAT_CONN_CACHE_WRITE_LOOKASIDE 1080 /*! cache: pages currently held in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_INUSE 1076 +#define WT_STAT_CONN_CACHE_PAGES_INUSE 1081 /*! cache: pages evicted because they exceeded the in-memory maximum */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1077 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE 1082 /*! cache: pages evicted because they had chains of deleted items */ -#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1078 +#define WT_STAT_CONN_CACHE_EVICTION_FORCE_DELETE 1083 /*! cache: pages evicted by application threads */ -#define WT_STAT_CONN_CACHE_EVICTION_APP 1079 +#define WT_STAT_CONN_CACHE_EVICTION_APP 1084 /*! cache: pages queued for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1080 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED 1085 /*! cache: pages queued for urgent eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1081 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_URGENT 1086 /*! cache: pages queued for urgent eviction during walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1082 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_QUEUED_OLDEST 1087 /*! cache: pages read into cache */ -#define WT_STAT_CONN_CACHE_READ 1083 +#define WT_STAT_CONN_CACHE_READ 1088 /*! cache: pages read into cache requiring lookaside entries */ -#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1084 +#define WT_STAT_CONN_CACHE_READ_LOOKASIDE 1089 /*! cache: pages requested from the cache */ -#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1085 +#define WT_STAT_CONN_CACHE_PAGES_REQUESTED 1090 /*! cache: pages seen by eviction walk */ -#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1086 +#define WT_STAT_CONN_CACHE_EVICTION_PAGES_SEEN 1091 /*! cache: pages selected for eviction unable to be evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1087 +#define WT_STAT_CONN_CACHE_EVICTION_FAIL 1092 /*! cache: pages walked for eviction */ -#define WT_STAT_CONN_CACHE_EVICTION_WALK 1088 +#define WT_STAT_CONN_CACHE_EVICTION_WALK 1093 /*! cache: pages written from cache */ -#define WT_STAT_CONN_CACHE_WRITE 1089 +#define WT_STAT_CONN_CACHE_WRITE 1094 /*! cache: pages written requiring in-memory restoration */ -#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1090 +#define WT_STAT_CONN_CACHE_WRITE_RESTORE 1095 /*! cache: percentage overhead */ -#define WT_STAT_CONN_CACHE_OVERHEAD 1091 +#define WT_STAT_CONN_CACHE_OVERHEAD 1096 /*! cache: tracked bytes belonging to internal pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1092 +#define WT_STAT_CONN_CACHE_BYTES_INTERNAL 1097 /*! cache: tracked bytes belonging to leaf pages in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_LEAF 1093 +#define WT_STAT_CONN_CACHE_BYTES_LEAF 1098 /*! cache: tracked dirty bytes in the cache */ -#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1094 +#define WT_STAT_CONN_CACHE_BYTES_DIRTY 1099 /*! cache: tracked dirty pages in the cache */ -#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1095 +#define WT_STAT_CONN_CACHE_PAGES_DIRTY 1100 /*! cache: unmodified pages evicted */ -#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1096 +#define WT_STAT_CONN_CACHE_EVICTION_CLEAN 1101 /*! connection: auto adjusting condition resets */ -#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1097 +#define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1102 /*! connection: auto adjusting condition wait calls */ -#define WT_STAT_CONN_COND_AUTO_WAIT 1098 +#define WT_STAT_CONN_COND_AUTO_WAIT 1103 /*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1099 +#define WT_STAT_CONN_FILE_OPEN 1104 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1100 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1105 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1101 +#define WT_STAT_CONN_MEMORY_FREE 1106 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1102 +#define WT_STAT_CONN_MEMORY_GROW 1107 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1103 +#define WT_STAT_CONN_COND_WAIT 1108 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1104 +#define WT_STAT_CONN_RWLOCK_READ 1109 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1105 +#define WT_STAT_CONN_RWLOCK_WRITE 1110 /*! connection: total fsync I/Os */ -#define WT_STAT_CONN_FSYNC_IO 1106 +#define WT_STAT_CONN_FSYNC_IO 1111 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1107 +#define WT_STAT_CONN_READ_IO 1112 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1108 +#define WT_STAT_CONN_WRITE_IO 1113 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1109 +#define WT_STAT_CONN_CURSOR_CREATE 1114 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1110 +#define WT_STAT_CONN_CURSOR_INSERT 1115 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1111 +#define WT_STAT_CONN_CURSOR_NEXT 1116 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1112 +#define WT_STAT_CONN_CURSOR_PREV 1117 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1113 +#define WT_STAT_CONN_CURSOR_REMOVE 1118 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1114 +#define WT_STAT_CONN_CURSOR_RESET 1119 /*! cursor: cursor restarted searches */ -#define WT_STAT_CONN_CURSOR_RESTART 1115 +#define WT_STAT_CONN_CURSOR_RESTART 1120 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1116 +#define WT_STAT_CONN_CURSOR_SEARCH 1121 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1117 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1122 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1118 +#define WT_STAT_CONN_CURSOR_UPDATE 1123 /*! cursor: truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1119 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1124 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1120 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1125 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1121 +#define WT_STAT_CONN_DH_SWEEP_REF 1126 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1122 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1127 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1123 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1128 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1124 +#define WT_STAT_CONN_DH_SWEEP_TOD 1129 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1125 +#define WT_STAT_CONN_DH_SWEEPS 1130 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1126 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1131 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1127 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1132 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1128 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1133 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1129 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1134 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1130 -/*! lock: handle-list lock acquisitions */ -#define WT_STAT_CONN_LOCK_HANDLE_LIST_COUNT 1131 -/*! lock: handle-list lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_HANDLE_LIST_WAIT_APPLICATION 1132 -/*! lock: handle-list lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_HANDLE_LIST_WAIT_INTERNAL 1133 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1135 +/*! lock: handle-list lock eviction thread wait time (usecs) */ +#define WT_STAT_CONN_LOCK_HANDLE_LIST_WAIT_EVICTION 1136 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1134 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1137 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1135 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1138 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1136 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1139 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1137 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1140 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1138 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1141 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1139 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1142 /*! lock: table lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_COUNT 1140 +#define WT_STAT_CONN_LOCK_TABLE_COUNT 1143 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1141 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1144 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1142 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1145 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1143 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1146 /*! log: consolidated slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1144 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1147 +/*! log: consolidated slot join active slot closed */ +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1148 /*! log: consolidated slot join races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1145 +#define WT_STAT_CONN_LOG_SLOT_RACES 1149 /*! log: consolidated slot join transitions */ -#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1146 +#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1150 /*! log: consolidated slot joins */ -#define WT_STAT_CONN_LOG_SLOT_JOINS 1147 +#define WT_STAT_CONN_LOG_SLOT_JOINS 1151 +/*! log: consolidated slot transitions unable to find free slot */ +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1152 /*! log: consolidated slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1148 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1153 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1149 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1154 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1150 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1155 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1151 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1156 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1152 +#define WT_STAT_CONN_LOG_FLUSH 1157 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1153 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1158 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1154 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1159 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1155 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1160 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1156 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1161 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1157 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1162 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1158 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1163 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1159 +#define WT_STAT_CONN_LOG_SCANS 1164 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1160 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1165 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1161 +#define WT_STAT_CONN_LOG_WRITE_LSN 1166 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1162 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1167 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1163 +#define WT_STAT_CONN_LOG_SYNC 1168 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1164 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1169 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1165 +#define WT_STAT_CONN_LOG_SYNC_DIR 1170 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1166 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1171 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1167 +#define WT_STAT_CONN_LOG_WRITES 1172 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1168 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1173 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1169 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1174 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1170 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1175 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1171 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1176 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1172 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1177 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1173 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1178 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1174 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1179 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1175 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1180 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1176 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1181 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1177 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1182 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1178 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1183 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1179 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1184 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1180 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1185 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1181 +#define WT_STAT_CONN_REC_PAGES 1186 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1182 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1187 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1183 +#define WT_STAT_CONN_REC_PAGE_DELETE 1188 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1184 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1189 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1185 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1190 /*! session: open cursor count */ -#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1186 +#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1191 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1187 +#define WT_STAT_CONN_SESSION_OPEN 1192 +/*! session: table alter failed calls */ +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1193 +/*! session: table alter successful calls */ +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1194 +/*! session: table alter unchanged and skipped */ +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1195 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1188 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1196 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1189 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1197 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1190 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1198 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1191 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1199 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1192 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1200 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1193 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1201 /*! session: table rebalance failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1194 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1202 /*! session: table rebalance successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1195 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1203 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1196 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1204 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1197 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1205 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1198 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1206 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1199 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1207 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1200 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1208 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1201 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1209 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1202 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1210 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1203 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1211 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1204 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1212 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1205 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1213 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1206 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1214 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1207 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1215 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1208 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1216 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1209 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1217 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1210 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1218 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1211 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1219 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1212 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1220 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1213 +#define WT_STAT_CONN_PAGE_SLEEP 1221 /*! transaction: number of named snapshots created */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1214 +#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1222 /*! transaction: number of named snapshots dropped */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1215 +#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1223 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1216 +#define WT_STAT_CONN_TXN_BEGIN 1224 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1217 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1225 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1218 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1226 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1219 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1227 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1220 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1228 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1221 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1229 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1222 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1230 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1223 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1231 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1224 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1232 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1225 +#define WT_STAT_CONN_TXN_CHECKPOINT 1233 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1226 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1234 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1227 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1235 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1228 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1236 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1229 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1237 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1230 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1238 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1231 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1239 /*! * transaction: transaction range of IDs currently pinned by named * snapshots */ -#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1232 +#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1240 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1233 +#define WT_STAT_CONN_TXN_SYNC 1241 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1234 +#define WT_STAT_CONN_TXN_COMMIT 1242 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1235 +#define WT_STAT_CONN_TXN_ROLLBACK 1243 /*! * @} @@ -4908,181 +5003,183 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_DSRC_CACHE_WRITE 2059 /*! cache: pages written requiring in-memory restoration */ #define WT_STAT_DSRC_CACHE_WRITE_RESTORE 2060 +/*! cache: tracked dirty bytes in the cache */ +#define WT_STAT_DSRC_CACHE_BYTES_DIRTY 2061 /*! cache: unmodified pages evicted */ -#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2061 +#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2062 /*! * cache_walk: Average difference between current eviction generation * when the page was last considered, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_GEN_AVG_GAP 2062 +#define WT_STAT_DSRC_CACHE_STATE_GEN_AVG_GAP 2063 /*! * cache_walk: Average on-disk page image size seen, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_AVG_WRITTEN_SIZE 2063 +#define WT_STAT_DSRC_CACHE_STATE_AVG_WRITTEN_SIZE 2064 /*! * cache_walk: Clean pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_CLEAN 2064 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_CLEAN 2065 /*! * cache_walk: Current eviction generation, only reported if cache_walk * or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_GEN_CURRENT 2065 +#define WT_STAT_DSRC_CACHE_STATE_GEN_CURRENT 2066 /*! * cache_walk: Dirty pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_DIRTY 2066 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_DIRTY 2067 /*! * cache_walk: Entries in the root page, only reported if cache_walk or * all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_ROOT_ENTRIES 2067 +#define WT_STAT_DSRC_CACHE_STATE_ROOT_ENTRIES 2068 /*! * cache_walk: Internal pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_INTERNAL 2068 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_INTERNAL 2069 /*! * cache_walk: Leaf pages currently in cache, only reported if cache_walk * or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES_LEAF 2069 +#define WT_STAT_DSRC_CACHE_STATE_PAGES_LEAF 2070 /*! * cache_walk: Maximum difference between current eviction generation * when the page was last considered, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_GEN_MAX_GAP 2070 +#define WT_STAT_DSRC_CACHE_STATE_GEN_MAX_GAP 2071 /*! * cache_walk: Maximum page size seen, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_MAX_PAGESIZE 2071 +#define WT_STAT_DSRC_CACHE_STATE_MAX_PAGESIZE 2072 /*! * cache_walk: Minimum on-disk page image size seen, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_MIN_WRITTEN_SIZE 2072 +#define WT_STAT_DSRC_CACHE_STATE_MIN_WRITTEN_SIZE 2073 /*! * cache_walk: On-disk page image sizes smaller than a single allocation * unit, only reported if cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_SMALLER_ALLOC_SIZE 2073 +#define WT_STAT_DSRC_CACHE_STATE_SMALLER_ALLOC_SIZE 2074 /*! * cache_walk: Pages created in memory and never written, only reported * if cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_MEMORY 2074 +#define WT_STAT_DSRC_CACHE_STATE_MEMORY 2075 /*! * cache_walk: Pages currently queued for eviction, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_QUEUED 2075 +#define WT_STAT_DSRC_CACHE_STATE_QUEUED 2076 /*! * cache_walk: Pages that could not be queued for eviction, only reported * if cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_NOT_QUEUEABLE 2076 +#define WT_STAT_DSRC_CACHE_STATE_NOT_QUEUEABLE 2077 /*! * cache_walk: Refs skipped during cache traversal, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_REFS_SKIPPED 2077 +#define WT_STAT_DSRC_CACHE_STATE_REFS_SKIPPED 2078 /*! * cache_walk: Size of the root page, only reported if cache_walk or all * statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_ROOT_SIZE 2078 +#define WT_STAT_DSRC_CACHE_STATE_ROOT_SIZE 2079 /*! * cache_walk: Total number of pages currently in cache, only reported if * cache_walk or all statistics are enabled */ -#define WT_STAT_DSRC_CACHE_STATE_PAGES 2079 +#define WT_STAT_DSRC_CACHE_STATE_PAGES 2080 /*! compression: compressed pages read */ -#define WT_STAT_DSRC_COMPRESS_READ 2080 +#define WT_STAT_DSRC_COMPRESS_READ 2081 /*! compression: compressed pages written */ -#define WT_STAT_DSRC_COMPRESS_WRITE 2081 +#define WT_STAT_DSRC_COMPRESS_WRITE 2082 /*! compression: page written failed to compress */ -#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2082 +#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2083 /*! compression: page written was too small to compress */ -#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2083 +#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2084 /*! compression: raw compression call failed, additional data available */ -#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2084 +#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2085 /*! compression: raw compression call failed, no additional data available */ -#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2085 +#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2086 /*! compression: raw compression call succeeded */ -#define WT_STAT_DSRC_COMPRESS_RAW_OK 2086 +#define WT_STAT_DSRC_COMPRESS_RAW_OK 2087 /*! cursor: bulk-loaded cursor-insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2087 +#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2088 /*! cursor: create calls */ -#define WT_STAT_DSRC_CURSOR_CREATE 2088 +#define WT_STAT_DSRC_CURSOR_CREATE 2089 /*! cursor: cursor-insert key and value bytes inserted */ -#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2089 +#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2090 /*! cursor: cursor-remove key bytes removed */ -#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2090 +#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2091 /*! cursor: cursor-update value bytes updated */ -#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2091 +#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2092 /*! cursor: insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT 2092 +#define WT_STAT_DSRC_CURSOR_INSERT 2093 /*! cursor: next calls */ -#define WT_STAT_DSRC_CURSOR_NEXT 2093 +#define WT_STAT_DSRC_CURSOR_NEXT 2094 /*! cursor: prev calls */ -#define WT_STAT_DSRC_CURSOR_PREV 2094 +#define WT_STAT_DSRC_CURSOR_PREV 2095 /*! cursor: remove calls */ -#define WT_STAT_DSRC_CURSOR_REMOVE 2095 +#define WT_STAT_DSRC_CURSOR_REMOVE 2096 /*! cursor: reset calls */ -#define WT_STAT_DSRC_CURSOR_RESET 2096 +#define WT_STAT_DSRC_CURSOR_RESET 2097 /*! cursor: restarted searches */ -#define WT_STAT_DSRC_CURSOR_RESTART 2097 +#define WT_STAT_DSRC_CURSOR_RESTART 2098 /*! cursor: search calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH 2098 +#define WT_STAT_DSRC_CURSOR_SEARCH 2099 /*! cursor: search near calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2099 +#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2100 /*! cursor: truncate calls */ -#define WT_STAT_DSRC_CURSOR_TRUNCATE 2100 +#define WT_STAT_DSRC_CURSOR_TRUNCATE 2101 /*! cursor: update calls */ -#define WT_STAT_DSRC_CURSOR_UPDATE 2101 +#define WT_STAT_DSRC_CURSOR_UPDATE 2102 /*! reconciliation: dictionary matches */ -#define WT_STAT_DSRC_REC_DICTIONARY 2102 +#define WT_STAT_DSRC_REC_DICTIONARY 2103 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2103 +#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2104 /*! * reconciliation: internal page key bytes discarded using suffix * compression */ -#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2104 +#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2105 /*! reconciliation: internal page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2105 +#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2106 /*! reconciliation: internal-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2106 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2107 /*! reconciliation: leaf page key bytes discarded using prefix compression */ -#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2107 +#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2108 /*! reconciliation: leaf page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2108 +#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2109 /*! reconciliation: leaf-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2109 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2110 /*! reconciliation: maximum blocks required for a page */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2110 +#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2111 /*! reconciliation: overflow values written */ -#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2111 +#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2112 /*! reconciliation: page checksum matches */ -#define WT_STAT_DSRC_REC_PAGE_MATCH 2112 +#define WT_STAT_DSRC_REC_PAGE_MATCH 2113 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_DSRC_REC_PAGES 2113 +#define WT_STAT_DSRC_REC_PAGES 2114 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_DSRC_REC_PAGES_EVICTION 2114 +#define WT_STAT_DSRC_REC_PAGES_EVICTION 2115 /*! reconciliation: pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE 2115 +#define WT_STAT_DSRC_REC_PAGE_DELETE 2116 /*! session: object compaction */ -#define WT_STAT_DSRC_SESSION_COMPACT 2116 +#define WT_STAT_DSRC_SESSION_COMPACT 2117 /*! session: open cursor count */ -#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2117 +#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2118 /*! transaction: update conflicts */ -#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2118 +#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2119 /*! * @} diff --git a/src/third_party/wiredtiger/src/include/wiredtiger_ext.h b/src/third_party/wiredtiger/src/include/wiredtiger_ext.h index 3d65cd1fc24..236d4e07e67 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger_ext.h +++ b/src/third_party/wiredtiger/src/include/wiredtiger_ext.h @@ -204,25 +204,47 @@ struct __wt_extension_api { WT_COLLATOR *collator, WT_ITEM *first, WT_ITEM *second, int *cmp); /*! - * @copydoc wiredtiger_config_parser_open + * Return the value of a configuration key. + * + * @param wt_api the extension handle + * @param session the session handle (or NULL if none available) + * @param config the configuration information passed to an application + * @param key configuration key string + * @param value the returned value + * @errors + * + * @snippet ex_data_source.c WT_EXTENSION config_get */ - int (*config_parser_open)(WT_EXTENSION_API *wt_api, WT_SESSION *session, - const char *config, size_t len, WT_CONFIG_PARSER **config_parserp); + int (*config_get)(WT_EXTENSION_API *wt_api, WT_SESSION *session, + WT_CONFIG_ARG *config, const char *key, WT_CONFIG_ITEM *value); /*! - * Return the value of a configuration string. + * Return the value of a configuration key from a string. * * @param wt_api the extension handle * @param session the session handle (or NULL if none available) + * @param config the configuration string * @param key configuration key string - * @param config the configuration information passed to an application * @param value the returned value * @errors * * @snippet ex_data_source.c WT_EXTENSION config_get */ - int (*config_get)(WT_EXTENSION_API *wt_api, WT_SESSION *session, - WT_CONFIG_ARG *config, const char *key, WT_CONFIG_ITEM *value); + int (*config_get_string)(WT_EXTENSION_API *wt_api, WT_SESSION *session, + const char *config, const char *key, WT_CONFIG_ITEM *value); + + /*! + * @copydoc wiredtiger_config_parser_open + */ + int (*config_parser_open)(WT_EXTENSION_API *wt_api, WT_SESSION *session, + const char *config, size_t len, WT_CONFIG_PARSER **config_parserp); + + /*! + * @copydoc wiredtiger_config_parser_open + */ + int (*config_parser_open_arg)(WT_EXTENSION_API *wt_api, + WT_SESSION *session, WT_CONFIG_ARG *config, + WT_CONFIG_PARSER **config_parserp); /*! * Insert a row into the metadata if it does not already exist. diff --git a/src/third_party/wiredtiger/src/include/wt_internal.h b/src/third_party/wiredtiger/src/include/wt_internal.h index dc3e3a0b6c0..cf79578985b 100644 --- a/src/third_party/wiredtiger/src/include/wt_internal.h +++ b/src/third_party/wiredtiger/src/include/wt_internal.h @@ -106,10 +106,12 @@ struct __wt_col; typedef struct __wt_col WT_COL; struct __wt_col_rle; typedef struct __wt_col_rle WT_COL_RLE; +struct __wt_col_var_repeat; + typedef struct __wt_col_var_repeat WT_COL_VAR_REPEAT; struct __wt_colgroup; typedef struct __wt_colgroup WT_COLGROUP; -struct __wt_compact; - typedef struct __wt_compact WT_COMPACT; +struct __wt_compact_state; + typedef struct __wt_compact_state WT_COMPACT_STATE; struct __wt_condvar; typedef struct __wt_condvar WT_CONDVAR; struct __wt_config; diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index 96b593ec706..803d3e8dfab 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -24,7 +24,7 @@ static int __log_write_internal( * __log_wait_for_earlier_slot -- * Wait for write_lsn to catch up to this slot. */ -static void +static int __log_wait_for_earlier_slot(WT_SESSION_IMPL *session, WT_LOGSLOT *slot) { WT_CONNECTION_IMPL *conn; @@ -41,16 +41,18 @@ __log_wait_for_earlier_slot(WT_SESSION_IMPL *session, WT_LOGSLOT *slot) * unlock in case an earlier thread is trying to switch its * slot and complete its operation. */ + WT_RET(WT_SESSION_CHECK_PANIC(session)); if (F_ISSET(session, WT_SESSION_LOCKED_SLOT)) __wt_spin_unlock(session, &log->log_slot_lock); - __wt_cond_auto_signal(session, conn->log_wrlsn_cond); + __wt_cond_signal(session, conn->log_wrlsn_cond); if (++yield_count < WT_THOUSAND) __wt_yield(); else - __wt_cond_wait(session, log->log_write_cond, 200); + __wt_cond_wait(session, log->log_write_cond, 200, NULL); if (F_ISSET(session, WT_SESSION_LOCKED_SLOT)) __wt_spin_lock(session, &log->log_slot_lock); } + return (0); } /* @@ -62,16 +64,21 @@ static int __log_fs_write(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, wt_off_t offset, size_t len, const void *buf) { + WT_DECL_RET; + /* * If we're writing into a new log file, we have to wait for all * writes to the previous log file to complete otherwise there could * be a hole at the end of the previous log file that we cannot detect. */ if (slot->slot_release_lsn.l.file < slot->slot_start_lsn.l.file) { - __log_wait_for_earlier_slot(session, slot); + WT_RET(__log_wait_for_earlier_slot(session, slot)); WT_RET(__wt_log_force_sync(session, &slot->slot_release_lsn)); } - return (__wt_write(session, slot->slot_fh, offset, len, buf)); + if ((ret = __wt_write(session, slot->slot_fh, offset, len, buf)) != 0) + WT_PANIC_MSG(session, ret, + "%s: fatal log failure", slot->slot_fh->name); + return (ret); } /* @@ -89,7 +96,7 @@ __wt_log_ckpt(WT_SESSION_IMPL *session, WT_LSN *ckp_lsn) log = conn->log; log->ckpt_lsn = *ckp_lsn; if (conn->log_cond != NULL) - __wt_cond_auto_signal(session, conn->log_cond); + __wt_cond_signal(session, conn->log_cond); } /* @@ -105,6 +112,7 @@ __wt_log_flush_lsn(WT_SESSION_IMPL *session, WT_LSN *lsn, bool start) conn = S2C(session); log = conn->log; + WT_RET(WT_SESSION_CHECK_PANIC(session)); WT_RET(__wt_log_force_write(session, 1, NULL)); __wt_log_wrlsn(session, NULL); if (start) @@ -159,6 +167,7 @@ __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn) uint64_t fsync_duration_usecs; log = S2C(session)->log; + log_fh = NULL; /* * We need to wait for the previous log file to get written @@ -168,8 +177,9 @@ __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn) * log file ready to close. */ while (log->sync_lsn.l.file < min_lsn->l.file) { + WT_RET(WT_SESSION_CHECK_PANIC(session)); __wt_cond_signal(session, S2C(session)->log_file_cond); - __wt_cond_wait(session, log->log_sync_cond, 10000); + __wt_cond_wait(session, log->log_sync_cond, 10000, NULL); } __wt_spin_lock(session, &log->log_sync_lock); WT_ASSERT(session, log->log_dir_fh != NULL); @@ -214,11 +224,12 @@ __wt_log_force_sync(WT_SESSION_IMPL *session, WT_LSN *min_lsn) WT_STAT_CONN_INCR(session, log_sync); WT_STAT_CONN_INCRV(session, log_sync_duration, fsync_duration_usecs); - WT_ERR(__wt_close(session, &log_fh)); __wt_cond_signal(session, log->log_sync_cond); } err: __wt_spin_unlock(session, &log->log_sync_lock); + if (log_fh != NULL) + WT_TRET(__wt_close(session, &log_fh)); return (ret); } @@ -298,14 +309,11 @@ void __wt_log_written_reset(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; - WT_LOG *log; conn = S2C(session); - if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) - return; - log = conn->log; - log->log_written = 0; - return; + + if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED)) + conn->log->log_written = 0; } /* @@ -425,8 +433,6 @@ __wt_log_extract_lognum( { const char *p; - WT_UNUSED(session); - if (id == NULL || name == NULL) return (WT_ERROR); if ((p = strrchr(name, '.')) == NULL || @@ -777,8 +783,8 @@ __log_openfile(WT_SESSION_IMPL *session, __wt_log_desc_byteswap(desc); if (desc->log_magic != WT_LOG_MAGIC) WT_PANIC_RET(session, WT_ERROR, - "log file %s corrupted: Bad magic number %" PRIu32, - (*fhp)->name, desc->log_magic); + "log file %s corrupted: Bad magic number %" PRIu32, + (*fhp)->name, desc->log_magic); if (desc->majorv > WT_LOG_MAJOR_VERSION || (desc->majorv == WT_LOG_MAJOR_VERSION && desc->minorv > WT_LOG_MINOR_VERSION)) @@ -895,12 +901,12 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) */ create_log = true; if (conn->log_prealloc > 0 && !conn->hot_backup) { - __wt_readlock(session, conn->hot_backup_lock); + __wt_readlock(session, &conn->hot_backup_lock); if (conn->hot_backup) - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); else { ret = __log_alloc_prealloc(session, log->fileid); - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); /* * If ret is 0 it means we found a pre-allocated file. @@ -915,7 +921,7 @@ __log_newfile(WT_SESSION_IMPL *session, bool conn_open, bool *created) else { WT_STAT_CONN_INCR(session, log_prealloc_missed); if (conn->log_cond != NULL) - __wt_cond_auto_signal( + __wt_cond_signal( session, conn->log_cond); } } @@ -1029,12 +1035,12 @@ __log_truncate_file(WT_SESSION_IMPL *session, WT_FH *log_fh, wt_off_t offset) log = conn->log; if (!F_ISSET(log, WT_LOG_TRUNCATE_NOTSUP) && !conn->hot_backup) { - __wt_readlock(session, conn->hot_backup_lock); + __wt_readlock(session, &conn->hot_backup_lock); if (conn->hot_backup) - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); else { ret = __wt_ftruncate(session, log_fh, offset); - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); if (ret != ENOTSUP) return (ret); F_SET(log, WT_LOG_TRUNCATE_NOTSUP); @@ -1276,7 +1282,7 @@ __wt_log_open(WT_SESSION_IMPL *session) * where the previous log file ends. */ if (!F_ISSET(conn, WT_CONN_READONLY)) { - WT_WITH_SLOT_LOCK(session, log, ret, + WT_WITH_SLOT_LOCK(session, log, ret = __log_newfile(session, true, NULL)); WT_ERR(ret); } @@ -1462,7 +1468,7 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) * be holes in the log file. */ WT_STAT_CONN_INCR(session, log_release_write_lsn); - __log_wait_for_earlier_slot(session, slot); + WT_ERR(__log_wait_for_earlier_slot(session, slot)); log->write_start_lsn = slot->slot_start_lsn; log->write_lsn = slot->slot_end_lsn; @@ -1483,6 +1489,7 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) * current fsync completes and advance log->sync_lsn. */ while (F_ISSET(slot, WT_SLOT_SYNC | WT_SLOT_SYNC_DIR)) { + WT_ERR(WT_SESSION_CHECK_PANIC(session)); /* * We have to wait until earlier log files have finished their * sync operations. The most recent one will set the LSN to the @@ -1490,7 +1497,8 @@ __wt_log_release(WT_SESSION_IMPL *session, WT_LOGSLOT *slot, bool *freep) */ if (log->sync_lsn.l.file < slot->slot_end_lsn.l.file || __wt_spin_trylock(session, &log->log_sync_lock) != 0) { - __wt_cond_wait(session, log->log_sync_cond, 10000); + __wt_cond_wait( + session, log->log_sync_cond, 10000, NULL); continue; } locked = true; @@ -1655,10 +1663,7 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, WT_RET(__log_get_files(session, WT_LOG_FILENAME, &logfiles, &logcount)); if (logcount == 0) - /* - * Return it is not supported if none don't exist. - */ - return (ENOTSUP); + WT_RET_MSG(session, ENOTSUP, "no log files found"); for (i = 0; i < logcount; i++) { WT_ERR(__wt_log_extract_lognum(session, logfiles[i], &lognum)); @@ -1674,6 +1679,10 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, &log_fh, WT_LOG_FILENAME, start_lsn.l.file, WT_LOG_OPEN_VERIFY)); WT_ERR(__wt_filesize(session, log_fh, &log_size)); rd_lsn = start_lsn; + if (LF_ISSET(WT_LOGSCAN_RECOVER)) + __wt_verbose(session, WT_VERB_RECOVERY_PROGRESS, + "Recovering log %" PRIu32 " through %" PRIu32, + rd_lsn.l.file, end_lsn.l.file); WT_ERR(__wt_scr_alloc(session, WT_LOG_ALIGN, &buf)); WT_ERR(__wt_scr_alloc(session, 0, &decryptitem)); @@ -1722,6 +1731,11 @@ advance: WT_ERR(__log_openfile(session, &log_fh, WT_LOG_FILENAME, rd_lsn.l.file, WT_LOG_OPEN_VERIFY)); + if (LF_ISSET(WT_LOGSCAN_RECOVER)) + __wt_verbose(session, WT_VERB_RECOVERY_PROGRESS, + "Recovering log %" PRIu32 + " through %" PRIu32, + rd_lsn.l.file, end_lsn.l.file); WT_ERR(__wt_filesize(session, log_fh, &log_size)); eol = false; continue; @@ -1758,9 +1772,8 @@ advance: if (eol) /* Found a hole. This LSN is the end. */ break; - else - /* Last record in log. Look for more. */ - goto advance; + /* Last record in log. Look for more. */ + goto advance; } rdup_len = __wt_rduppo2(reclen, allocsize); if (reclen > allocsize) { @@ -1906,7 +1919,6 @@ __wt_log_force_write(WT_SESSION_IMPL *session, bool retry, bool *did_work) { WT_LOG *log; WT_MYSLOT myslot; - uint32_t joined; log = S2C(session)->log; memset(&myslot, 0, sizeof(myslot)); @@ -1914,14 +1926,7 @@ __wt_log_force_write(WT_SESSION_IMPL *session, bool retry, bool *did_work) if (did_work != NULL) *did_work = true; myslot.slot = log->active_slot; - joined = WT_LOG_SLOT_JOINED(log->active_slot->slot_state); - if (joined == 0) { - WT_STAT_CONN_INCR(session, log_force_write_skip); - if (did_work != NULL) - *did_work = false; - return (0); - } - return (__wt_log_slot_switch(session, &myslot, retry, true)); + return (__wt_log_slot_switch(session, &myslot, retry, true, did_work)); } /* @@ -2120,7 +2125,11 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, WT_STAT_CONN_INCR(session, log_writes); - __wt_log_slot_join(session, rdup_len, flags, &myslot); + /* + * The only time joining a slot should ever return an error is if it + * detects a panic. + */ + WT_ERR(__wt_log_slot_join(session, rdup_len, flags, &myslot)); /* * If the addition of this record crosses the buffer boundary, * switch in a new slot. @@ -2129,7 +2138,7 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, ret = 0; if (myslot.end_offset >= WT_LOG_SLOT_BUF_MAX || F_ISSET(&myslot, WT_MYSLOT_UNBUFFERED) || force) - ret = __wt_log_slot_switch(session, &myslot, true, false); + ret = __wt_log_slot_switch(session, &myslot, true, false, NULL); if (ret == 0) ret = __log_fill(session, &myslot, false, record, &lsn); release_size = __wt_log_slot_release( @@ -2154,7 +2163,7 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, * XXX I've seen times when conditions are NULL. */ if (conn->log_cond != NULL) { - __wt_cond_auto_signal(session, conn->log_cond); + __wt_cond_signal(session, conn->log_cond); __wt_yield(); } else WT_ERR(__wt_log_force_write(session, 1, NULL)); @@ -2162,13 +2171,19 @@ __log_write_internal(WT_SESSION_IMPL *session, WT_ITEM *record, WT_LSN *lsnp, if (LF_ISSET(WT_LOG_FLUSH)) { /* Wait for our writes to reach the OS */ while (__wt_log_cmp(&log->write_lsn, &lsn) <= 0 && - myslot.slot->slot_error == 0) - __wt_cond_wait(session, log->log_write_cond, 10000); + myslot.slot->slot_error == 0) { + WT_ERR(WT_SESSION_CHECK_PANIC(session)); + __wt_cond_wait( + session, log->log_write_cond, 10000, NULL); + } } else if (LF_ISSET(WT_LOG_FSYNC)) { /* Wait for our writes to reach disk */ while (__wt_log_cmp(&log->sync_lsn, &lsn) <= 0 && - myslot.slot->slot_error == 0) - __wt_cond_wait(session, log->log_sync_cond, 10000); + myslot.slot->slot_error == 0) { + WT_ERR(WT_SESSION_CHECK_PANIC(session)); + __wt_cond_wait( + session, log->log_sync_cond, 10000, NULL); + } } /* @@ -2193,12 +2208,12 @@ err: /* * If one of the sync flags is set, assert the proper LSN has moved to - * match. + * match on success. */ - WT_ASSERT(session, !LF_ISSET(WT_LOG_FLUSH) || + WT_ASSERT(session, ret != 0 || !LF_ISSET(WT_LOG_FLUSH) || __wt_log_cmp(&log->write_lsn, &lsn) >= 0); - WT_ASSERT(session, - !LF_ISSET(WT_LOG_FSYNC) || __wt_log_cmp(&log->sync_lsn, &lsn) >= 0); + WT_ASSERT(session, ret != 0 || !LF_ISSET(WT_LOG_FSYNC) || + __wt_log_cmp(&log->sync_lsn, &lsn) >= 0); return (ret); } @@ -2223,8 +2238,10 @@ __wt_log_vprintf(WT_SESSION_IMPL *session, const char *fmt, va_list ap) return (0); va_copy(ap_copy, ap); - len = (size_t)vsnprintf(NULL, 0, fmt, ap_copy) + 1; + len = 1; + ret = __wt_vsnprintf_len_incr(NULL, 0, &len, fmt, ap_copy); va_end(ap_copy); + WT_RET(ret); WT_RET( __wt_logrec_alloc(session, sizeof(WT_LOG_RECORD) + len, &logrec)); @@ -2241,7 +2258,8 @@ __wt_log_vprintf(WT_SESSION_IMPL *session, const char *fmt, va_list ap) rec_fmt, rectype)); logrec->size += (uint32_t)header_size; - (void)vsnprintf((char *)logrec->data + logrec->size, len, fmt, ap); + WT_ERR(__wt_vsnprintf( + (char *)logrec->data + logrec->size, len, fmt, ap)); __wt_verbose(session, WT_VERB_LOG, "log_printf: %s", (char *)logrec->data + logrec->size); diff --git a/src/third_party/wiredtiger/src/log/log_slot.c b/src/third_party/wiredtiger/src/log/log_slot.c index c79805f5371..97e317ce68c 100644 --- a/src/third_party/wiredtiger/src/log/log_slot.c +++ b/src/third_party/wiredtiger/src/log/log_slot.c @@ -8,6 +8,49 @@ #include "wt_internal.h" +#ifdef HAVE_DIAGNOSTIC +/* + * __log_slot_dump -- + * Dump the entire slot state. + */ +static void +__log_slot_dump(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_LOG *log; + WT_LOGSLOT *slot; + int earliest, i; + + conn = S2C(session); + log = conn->log; + earliest = 0; + for (i = 0; i < WT_SLOT_POOL; i++) { + slot = &log->slot_pool[i]; + if (__wt_log_cmp(&slot->slot_release_lsn, + &log->slot_pool[earliest].slot_release_lsn) < 0) + earliest = i; + __wt_errx(session, "Slot %d:", i); + __wt_errx(session, " State: %" PRIx64 " Flags: %" PRIx32, + slot->slot_state, slot->flags); + __wt_errx(session, " Start LSN: %" PRIu32 "/%" PRIu32, + slot->slot_start_lsn.l.file, slot->slot_start_lsn.l.offset); + __wt_errx(session, " End LSN: %" PRIu32 "/%" PRIu32, + slot->slot_end_lsn.l.file, slot->slot_end_lsn.l.offset); + __wt_errx(session, " Release LSN: %" PRIu32 "/%" PRIu32, + slot->slot_release_lsn.l.file, + slot->slot_release_lsn.l.offset); + __wt_errx(session, " Offset: start: %" PRIuMAX + " last:%" PRIuMAX, (uintmax_t)slot->slot_start_offset, + (uintmax_t)slot->slot_last_offset); + __wt_errx(session, " Unbuffered: %" PRId64 + " error: %" PRId32, slot->slot_unbuffered, + slot->slot_error); + } + __wt_errx(session, "Earliest slot: %d", earliest); + +} +#endif + /* * __wt_log_slot_activate -- * Initialize a slot to become active. @@ -21,7 +64,6 @@ __wt_log_slot_activate(WT_SESSION_IMPL *session, WT_LOGSLOT *slot) conn = S2C(session); log = conn->log; - slot->slot_state = 0; /* * !!! slot_release_lsn must be set outside this function because * this function may be called after a log file switch and the @@ -30,12 +72,19 @@ __wt_log_slot_activate(WT_SESSION_IMPL *session, WT_LOGSLOT *slot) * set for closing the file handle on a log file switch. The flags * are reset when the slot is freed. See log_slot_free. */ + slot->slot_unbuffered = 0; slot->slot_start_lsn = slot->slot_end_lsn = log->alloc_lsn; slot->slot_start_offset = log->alloc_lsn.l.offset; slot->slot_last_offset = log->alloc_lsn.l.offset; slot->slot_fh = log->log_fh; slot->slot_error = 0; - slot->slot_unbuffered = 0; + WT_DIAGNOSTIC_YIELD; + /* + * Set the slot state last. Other threads may have a stale pointer + * to this slot and could try to alter the state and other fields once + * they see the state cleared. + */ + WT_PUBLISH(slot->slot_state, 0); } /* @@ -50,6 +99,10 @@ __log_slot_close( WT_CONNECTION_IMPL *conn; WT_LOG *log; int64_t end_offset, new_state, old_state; +#ifdef HAVE_DIAGNOSTIC + struct timespec begin, now; + int count; +#endif WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_SLOT)); WT_ASSERT(session, releasep != NULL); @@ -101,9 +154,33 @@ retry: * that value. If the state is unbuffered, wait for the unbuffered * size to be set. */ - while (WT_LOG_SLOT_UNBUFFERED_ISSET(old_state) && - slot->slot_unbuffered == 0) - __wt_yield(); +#ifdef HAVE_DIAGNOSTIC + count = 0; + __wt_epoch(session, &begin); +#endif + if (WT_LOG_SLOT_UNBUFFERED_ISSET(old_state)) { + while (slot->slot_unbuffered == 0) { + WT_RET(WT_SESSION_CHECK_PANIC(session)); + __wt_yield(); +#ifdef HAVE_DIAGNOSTIC + ++count; + if (count > WT_MILLION) { + __wt_epoch(session, &now); + if (WT_TIMEDIFF_SEC(now, begin) > 10) { + __wt_errx(session, "SLOT_CLOSE: Slot %" + PRIu32 " Timeout unbuffered, state 0x%" + PRIx64 " unbuffered %" PRIu64, + (uint32_t)(slot - &log->slot_pool[0]), + slot->slot_state, + slot->slot_unbuffered); + __log_slot_dump(session); + __wt_abort(session); + } + count = 0; + } +#endif + } + } end_offset = WT_LOG_SLOT_JOINED_BUFFERED(old_state) + slot->slot_unbuffered; @@ -117,18 +194,105 @@ retry: return (0); } +/* + * __log_slot_new -- + * Find a free slot and switch it as the new active slot. + * Must be called holding the slot lock. + */ +static int +__log_slot_new(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_LOG *log; + WT_LOGSLOT *slot; + int32_t i, pool_i; +#ifdef HAVE_DIAGNOSTIC + struct timespec begin, now; + int count; +#endif + + WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_SLOT)); + conn = S2C(session); + log = conn->log; + /* + * Although this function is single threaded, multiple threads could + * be trying to set a new active slot sequentially. If we find an + * active slot that is valid, return. + */ + if ((slot = log->active_slot) != NULL && + WT_LOG_SLOT_OPEN(slot->slot_state)) + return (0); + +#ifdef HAVE_DIAGNOSTIC + count = 0; + __wt_epoch(session, &begin); +#endif + /* + * Keep trying until we can find a free slot. + */ + for (;;) { + /* + * Rotate among the slots to lessen collisions. + */ + for (i = 0, pool_i = log->pool_index; i < WT_SLOT_POOL; + i++, pool_i++) { + if (pool_i >= WT_SLOT_POOL) + pool_i = 0; + slot = &log->slot_pool[pool_i]; + if (slot->slot_state == WT_LOG_SLOT_FREE) { + /* + * Acquire our starting position in the + * log file. Assume the full buffer size. + */ + WT_RET(__wt_log_acquire(session, + log->slot_buf_size, slot)); + /* + * We have a new, initialized slot to use. + * Set it as the active slot. + */ + WT_STAT_CONN_INCR(session, + log_slot_transitions); + log->active_slot = slot; + log->pool_index = pool_i; + return (0); + } + } + /* + * If we didn't find any free slots signal the worker thread. + */ + WT_STAT_CONN_INCR(session, log_slot_no_free_slots); + __wt_cond_signal(session, conn->log_wrlsn_cond); + __wt_yield(); +#ifdef HAVE_DIAGNOSTIC + ++count; + if (count > WT_MILLION) { + __wt_epoch(session, &now); + if (WT_TIMEDIFF_SEC(now, begin) > 10) { + __wt_errx(session, + "SLOT_NEW: Timeout free slot"); + __log_slot_dump(session); + __wt_abort(session); + } + count = 0; + } +#endif + } + /* NOTREACHED */ +} + /* * __log_slot_switch_internal -- * Switch out the current slot and set up a new one. */ static int __log_slot_switch_internal( - WT_SESSION_IMPL *session, WT_MYSLOT *myslot, bool forced) + WT_SESSION_IMPL *session, WT_MYSLOT *myslot, bool forced, bool *did_work) { WT_DECL_RET; WT_LOG *log; WT_LOGSLOT *slot; bool free_slot, release; + uint32_t joined; log = S2C(session)->log; release = false; @@ -142,10 +306,23 @@ __log_slot_switch_internal( */ if (slot != log->active_slot) return (0); + /* + * If the current active slot is unused and this is a forced switch, + * we're done. If this is a non-forced switch we always switch + * because the slot could be part of an unbuffered operation. + */ + joined = WT_LOG_SLOT_JOINED(slot->slot_state); + if (joined == 0 && forced) { + WT_STAT_CONN_INCR(session, log_force_write_skip); + if (did_work != NULL) + *did_work = false; + return (0); + } + WT_RET(WT_SESSION_CHECK_PANIC(session)); /* - * We may come through here multiple times if we were able to close - * a slot but could not set up a new one. If we closed it already, + * We may come through here multiple times if we were not able to + * set up a new one. If we closed it already, * don't try to do it again but still set up the new slot. */ if (!F_ISSET(myslot, WT_MYSLOT_CLOSE)) { @@ -157,20 +334,30 @@ __log_slot_switch_internal( if (ret == WT_NOTFOUND) return (0); WT_RET(ret); - if (release) { - WT_RET(__wt_log_release(session, slot, &free_slot)); - if (free_slot) - __wt_log_slot_free(session, slot); - } + /* + * Set that we have closed this slot because we may call in here + * multiple times if we retry creating a new slot. Similarly + * set retain whether this slot needs releasing so that we don't + * lose that information if we retry. + */ + F_SET(myslot, WT_MYSLOT_CLOSE); + if (release) + F_SET(myslot, WT_MYSLOT_NEEDS_RELEASE); } /* - * Set that we have closed this slot because we may call in here - * multiple times if we retry creating a new slot. + * Now that the slot is closed, set up a new one so that joining + * threads don't have to wait on writing the previous slot if we + * release it. Release after setting a new one. */ - F_SET(myslot, WT_MYSLOT_CLOSE); - WT_RET(__wt_log_slot_new(session)); + WT_RET(__log_slot_new(session)); F_CLR(myslot, WT_MYSLOT_CLOSE); - return (0); + if (F_ISSET(myslot, WT_MYSLOT_NEEDS_RELEASE)) { + WT_RET(__wt_log_release(session, slot, &free_slot)); + F_CLR(myslot, WT_MYSLOT_NEEDS_RELEASE); + if (free_slot) + __wt_log_slot_free(session, slot); + } + return (ret); } /* @@ -178,13 +365,14 @@ __log_slot_switch_internal( * Switch out the current slot and set up a new one. */ int -__wt_log_slot_switch( - WT_SESSION_IMPL *session, WT_MYSLOT *myslot, bool retry, bool forced) +__wt_log_slot_switch(WT_SESSION_IMPL *session, + WT_MYSLOT *myslot, bool retry, bool forced, bool *did_work) { WT_DECL_RET; WT_LOG *log; log = S2C(session)->log; + /* * !!! Since the WT_WITH_SLOT_LOCK macro is a do-while loop, the * compiler does not like it combined directly with the while loop @@ -197,8 +385,9 @@ __wt_log_slot_switch( * because we are responsible for setting up the new slot. */ do { - WT_WITH_SLOT_LOCK(session, log, ret, - ret = __log_slot_switch_internal(session, myslot, forced)); + WT_WITH_SLOT_LOCK(session, log, + ret = __log_slot_switch_internal( + session, myslot, forced, did_work)); if (ret == EBUSY) { WT_STAT_CONN_INCR(session, log_slot_switch_busy); __wt_yield(); @@ -207,67 +396,6 @@ __wt_log_slot_switch( return (ret); } -/* - * __wt_log_slot_new -- - * Find a free slot and switch it as the new active slot. - * Must be called holding the slot lock. - */ -int -__wt_log_slot_new(WT_SESSION_IMPL *session) -{ - WT_CONNECTION_IMPL *conn; - WT_LOG *log; - WT_LOGSLOT *slot; - int32_t i; - - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_SLOT)); - conn = S2C(session); - log = conn->log; - /* - * Although this function is single threaded, multiple threads could - * be trying to set a new active slot sequentially. If we find an - * active slot that is valid, return. - */ - if ((slot = log->active_slot) != NULL && - WT_LOG_SLOT_OPEN(slot->slot_state)) - return (0); - - /* - * Keep trying until we can find a free slot. - */ - for (;;) { - /* - * For now just restart at 0. We could use log->pool_index - * if that is inefficient. - */ - for (i = 0; i < WT_SLOT_POOL; i++) { - slot = &log->slot_pool[i]; - if (slot->slot_state == WT_LOG_SLOT_FREE) { - /* - * Acquire our starting position in the - * log file. Assume the full buffer size. - */ - WT_RET(__wt_log_acquire(session, - log->slot_buf_size, slot)); - /* - * We have a new, initialized slot to use. - * Set it as the active slot. - */ - WT_STAT_CONN_INCR(session, - log_slot_transitions); - log->active_slot = slot; - return (0); - } - } - /* - * If we didn't find any free slots signal the worker thread. - */ - __wt_cond_auto_signal(session, conn->log_wrlsn_cond); - __wt_yield(); - } - /* NOTREACHED */ -} - /* * __wt_log_slot_init -- * Initialize the slot array. @@ -283,7 +411,6 @@ __wt_log_slot_init(WT_SESSION_IMPL *session) conn = S2C(session); log = conn->log; - WT_CACHE_LINE_ALIGNMENT_VERIFY(session, log->slot_pool); for (i = 0; i < WT_SLOT_POOL; i++) log->slot_pool[i].slot_state = WT_LOG_SLOT_FREE; @@ -312,10 +439,13 @@ __wt_log_slot_init(WT_SESSION_IMPL *session) /* * We cannot initialize the release LSN in the activate function * because that function can be called after a log file switch. + * The release LSN is usually the same as the slot_start_lsn except + * around a log file switch. */ slot->slot_release_lsn = log->alloc_lsn; __wt_log_slot_activate(session, slot); log->active_slot = slot; + log->pool_index = 0; if (0) { err: while (--i >= 0) @@ -360,10 +490,9 @@ __wt_log_slot_destroy(WT_SESSION_IMPL *session) /* * __wt_log_slot_join -- - * Join a consolidated logging slot. Must be called with - * the read lock held. + * Join a consolidated logging slot. */ -void +int __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, uint32_t flags, WT_MYSLOT *myslot) { @@ -372,71 +501,76 @@ __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, WT_LOGSLOT *slot; int64_t flag_state, new_state, old_state, released; int32_t join_offset, new_join; -#ifdef HAVE_DIAGNOSTIC - bool unbuf_force; -#endif + bool unbuffered, yld; conn = S2C(session); log = conn->log; - /* - * Make sure the length cannot overflow. The caller should not - * even call this function if it doesn't fit but use direct - * writes. - */ WT_ASSERT(session, !F_ISSET(session, WT_SESSION_LOCKED_SLOT)); + WT_ASSERT(session, mysize != 0); /* * There should almost always be a slot open. */ + unbuffered = false; #ifdef HAVE_DIAGNOSTIC - unbuf_force = (++log->write_calls % WT_THOUSAND) == 0; + yld = (++log->write_calls % 7) == 0; + if ((log->write_calls % WT_THOUSAND) == 0 || + mysize > WT_LOG_SLOT_BUF_MAX) { +#else + yld = false; + if (mysize > WT_LOG_SLOT_BUF_MAX) { #endif + unbuffered = true; + F_SET(myslot, WT_MYSLOT_UNBUFFERED); + } for (;;) { WT_BARRIER(); + WT_RET(WT_SESSION_CHECK_PANIC(session)); slot = log->active_slot; old_state = slot->slot_state; - /* - * Try to join our size into the existing size and - * atomically write it back into the state. - */ - flag_state = WT_LOG_SLOT_FLAGS(old_state); - released = WT_LOG_SLOT_RELEASED(old_state); - join_offset = WT_LOG_SLOT_JOINED(old_state); -#ifdef HAVE_DIAGNOSTIC - if (unbuf_force || mysize > WT_LOG_SLOT_BUF_MAX) { -#else - if (mysize > WT_LOG_SLOT_BUF_MAX) { -#endif - new_join = join_offset + WT_LOG_SLOT_UNBUFFERED; - F_SET(myslot, WT_MYSLOT_UNBUFFERED); - myslot->slot = slot; + if (WT_LOG_SLOT_OPEN(old_state)) { + /* + * Try to join our size into the existing size and + * atomically write it back into the state. + */ + flag_state = WT_LOG_SLOT_FLAGS(old_state); + released = WT_LOG_SLOT_RELEASED(old_state); + join_offset = WT_LOG_SLOT_JOINED(old_state); + if (unbuffered) + new_join = join_offset + WT_LOG_SLOT_UNBUFFERED; + else + new_join = join_offset + (int32_t)mysize; + new_state = (int64_t)WT_LOG_SLOT_JOIN_REL( + (int64_t)new_join, (int64_t)released, + (int64_t)flag_state); + + /* + * Braces used due to potential empty body warning. + */ + if (yld) { + WT_DIAGNOSTIC_YIELD; + } + /* + * Attempt to swap our size into the state. + */ + if (__wt_atomic_casiv64( + &slot->slot_state, old_state, new_state)) + break; + WT_STAT_CONN_INCR(session, log_slot_races); } else - new_join = join_offset + (int32_t)mysize; - new_state = (int64_t)WT_LOG_SLOT_JOIN_REL( - (int64_t)new_join, (int64_t)released, (int64_t)flag_state); - - /* - * Check if the slot is open for joining and we are able to - * swap in our size into the state. - */ - if (WT_LOG_SLOT_OPEN(old_state) && - __wt_atomic_casiv64( - &slot->slot_state, old_state, new_state)) - break; + WT_STAT_CONN_INCR(session, log_slot_active_closed); /* * The slot is no longer open or we lost the race to * update it. Yield and try again. */ - WT_STAT_CONN_INCR(session, log_slot_races); __wt_yield(); } /* * We joined this slot. Fill in our information to return to * the caller. */ - if (mysize != 0) - WT_STAT_CONN_INCR(session, log_slot_joins); + WT_STAT_CONN_INCR(session, log_slot_joins); if (LF_ISSET(WT_LOG_DSYNC | WT_LOG_FSYNC)) F_SET(slot, WT_SLOT_SYNC_DIR); if (LF_ISSET(WT_LOG_FLUSH)) @@ -451,6 +585,7 @@ __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, myslot->slot = slot; myslot->offset = join_offset; myslot->end_offset = (wt_off_t)((uint64_t)join_offset + mysize); + return (0); } /* @@ -466,10 +601,15 @@ __wt_log_slot_release(WT_SESSION_IMPL *session, WT_MYSLOT *myslot, int64_t size) wt_off_t cur_offset, my_start; int64_t my_size, rel_size; - WT_UNUSED(session); slot = myslot->slot; my_start = slot->slot_start_offset + myslot->offset; + /* + * We maintain the last starting offset within this slot. + * This is used to know the offset of the last record that + * was written rather than the beginning record of the slot. + */ while ((cur_offset = slot->slot_last_offset) < my_start) { + WT_RET(WT_SESSION_CHECK_PANIC(session)); /* * Set our offset if we are larger. */ diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c index c20673563d9..4d00143b28f 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor.c @@ -10,10 +10,10 @@ #define WT_FORALL_CURSORS(clsm, c, i) \ for ((i) = (clsm)->nchunks; (i) > 0;) \ - if (((c) = (clsm)->chunks[--i]->cursor) != NULL) + if (((c) = (clsm)->chunks[--(i)]->cursor) != NULL) #define WT_LSM_CURCMP(s, lsm_tree, c1, c2, cmp) \ - __wt_compare(s, (lsm_tree)->collator, &(c1)->key, &(c2)->key, &cmp) + __wt_compare(s, (lsm_tree)->collator, &(c1)->key, &(c2)->key, &(cmp)) static int __clsm_lookup(WT_CURSOR_LSM *, WT_ITEM *); static int __clsm_open_cursors(WT_CURSOR_LSM *, bool, u_int, uint32_t); @@ -34,7 +34,7 @@ __wt_clsm_request_switch(WT_CURSOR_LSM *clsm) lsm_tree = clsm->lsm_tree; session = (WT_SESSION_IMPL *)clsm->iface.session; - if (!F_ISSET(lsm_tree, WT_LSM_TREE_NEED_SWITCH)) { + if (!lsm_tree->need_switch) { /* * Check that we are up-to-date: don't set the switch if the * tree has changed since we last opened cursors: that can lead @@ -44,8 +44,8 @@ __wt_clsm_request_switch(WT_CURSOR_LSM *clsm) __wt_lsm_tree_readlock(session, lsm_tree); if (lsm_tree->nchunks == 0 || (clsm->dsk_gen == lsm_tree->dsk_gen && - !F_ISSET(lsm_tree, WT_LSM_TREE_NEED_SWITCH))) { - F_SET(lsm_tree, WT_LSM_TREE_NEED_SWITCH); + !lsm_tree->need_switch)) { + lsm_tree->need_switch = true; ret = __wt_lsm_manager_push_entry( session, WT_LSM_WORK_SWITCH, 0, lsm_tree); } @@ -129,7 +129,7 @@ __clsm_enter_update(WT_CURSOR_LSM *clsm) * chunk grows twice as large as the configured size, block until it * can be switched. */ - hard_limit = F_ISSET(lsm_tree, WT_LSM_TREE_NEED_SWITCH); + hard_limit = lsm_tree->need_switch; if (have_primary) { WT_ENTER_PAGE_INDEX(session); @@ -178,20 +178,12 @@ __clsm_enter(WT_CURSOR_LSM *clsm, bool reset, bool update) if (reset) { WT_ASSERT(session, !F_ISSET(&clsm->iface, - WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT)); + WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT)); WT_RET(__clsm_reset_cursors(clsm, NULL)); } for (;;) { - /* - * If the cursor looks up-to-date, check if the cache is full. - * In case this call blocks, the check will be repeated before - * proceeding. - */ - if (clsm->dsk_gen != lsm_tree->dsk_gen && - lsm_tree->nchunks != 0) - goto open; - + /* Check if the cursor looks up-to-date. */ if (clsm->dsk_gen != lsm_tree->dsk_gen && lsm_tree->nchunks != 0) goto open; @@ -215,7 +207,7 @@ __clsm_enter(WT_CURSOR_LSM *clsm, bool reset, bool update) goto open; if (txn->isolation == WT_ISO_SNAPSHOT) - WT_RET(__wt_txn_cursor_op(session)); + __wt_txn_cursor_op(session); /* * Figure out how many updates are required for @@ -267,7 +259,7 @@ __clsm_enter(WT_CURSOR_LSM *clsm, bool reset, bool update) (!update && F_ISSET(clsm, WT_CLSM_OPEN_READ)))) break; -open: WT_WITH_SCHEMA_LOCK(session, ret, +open: WT_WITH_SCHEMA_LOCK(session, ret = __clsm_open_cursors(clsm, update, 0, 0)); WT_RET(ret); } @@ -304,7 +296,7 @@ __clsm_leave(WT_CURSOR_LSM *clsm) * byte, if the application uses two leading DC4 byte for some reason, we'll do * a wasted data copy each time a new value is inserted into the object. */ -static const WT_ITEM __tombstone = { "\x14\x14", 2, 0, NULL, 0 }; +static const WT_ITEM __tombstone = { "\x14\x14", 2, NULL, 0, 0 }; /* * __clsm_deleted -- @@ -409,13 +401,11 @@ static int __clsm_resize_chunks( WT_SESSION_IMPL *session, WT_CURSOR_LSM *clsm, u_int nchunks) { - WT_DECL_RET; WT_LSM_CURSOR_CHUNK *chunk; /* Don't allocate more iterators if we don't need them. */ - if (clsm->chunks_count >= nchunks) { - return (ret); - } + if (clsm->chunks_count >= nchunks) + return (0); WT_RET(__wt_realloc_def(session, &clsm->chunks_alloc, nchunks, &clsm->chunks)); @@ -423,7 +413,7 @@ __clsm_resize_chunks( WT_RET(__wt_calloc_one(session, &chunk)); clsm->chunks[clsm->chunks_count] = chunk; } - return (ret); + return (0); } /* @@ -434,9 +424,10 @@ static void __clsm_free_chunks(WT_SESSION_IMPL *session, WT_CURSOR_LSM *clsm) { size_t i; - for (i = 0; i < clsm->chunks_count; i++) { + + for (i = 0; i < clsm->chunks_count; i++) __wt_free(session, clsm->chunks[i]); - } + __wt_free(session, clsm->chunks); } @@ -667,7 +658,7 @@ retry: if (F_ISSET(clsm, WT_CLSM_MERGE)) { */ if (i != nchunks - 1) clsm->chunks[i]->cursor->insert = - __wt_curfile_update_check; + __wt_curfile_insert_check; if (!F_ISSET(clsm, WT_CLSM_MERGE) && F_ISSET(chunk, WT_LSM_CHUNK_BLOOM)) @@ -689,19 +680,29 @@ retry: if (F_ISSET(clsm, WT_CLSM_MERGE)) { if (chunk != NULL && !F_ISSET(chunk, WT_LSM_CHUNK_ONDISK) && chunk->switch_txn == WT_TXN_NONE) { - clsm->primary_chunk = chunk; primary = clsm->chunks[clsm->nchunks - 1]->cursor; + btree = ((WT_CURSOR_BTREE *)primary)->btree; + /* - * Disable eviction for the in-memory chunk. Also clear the - * bulk load flag here, otherwise eviction will be enabled by - * the first update. + * If the primary is not yet set as the primary, do that now. + * Note that eviction was configured off when the underlying + * object was created, which is what we want, leave it alone. + * + * We don't have to worry about races here: every thread that + * modifies the tree will have to come through here, at worse + * we set the flag repeatedly. We don't use a WT_BTREE handle + * flag, however, we could race doing the read-modify-write of + * the flags field. + * + * If something caused the chunk to be closed and reopened + * since it was created, we can no longer use it as a primary + * chunk and we need to force a switch. We detect the tree was + * created when it was opened by checking the "original" flag. */ - btree = ((WT_CURSOR_BTREE *)(primary))->btree; - if (btree->bulk_load_ok) { - btree->bulk_load_ok = false; - WT_WITH_BTREE(session, btree, - __wt_btree_evictable(session, false)); - } + if (!btree->lsm_primary && btree->original) + btree->lsm_primary = true; + if (btree->lsm_primary) + clsm->primary_chunk = chunk; } clsm->dsk_gen = lsm_tree->dsk_gen; @@ -763,7 +764,7 @@ __wt_clsm_init_merge( F_SET(clsm, WT_CLSM_MINOR_MERGE); clsm->nchunks = nchunks; - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret = __clsm_open_cursors(clsm, false, start_chunk, start_id)); return (ret); } @@ -1214,7 +1215,8 @@ __clsm_lookup(WT_CURSOR_LSM *clsm, WT_ITEM *value) WT_LSM_TREE_STAT_INCR( session, clsm->lsm_tree->bloom_miss); continue; - } else if (ret == 0) + } + if (ret == 0) WT_LSM_TREE_STAT_INCR( session, clsm->lsm_tree->bloom_hit); WT_ERR(ret); @@ -1240,10 +1242,10 @@ __clsm_lookup(WT_CURSOR_LSM *clsm, WT_ITEM *value) WT_ERR(WT_NOTFOUND); done: -err: F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); - if (ret == 0) { - clsm->current = c; +err: if (ret == 0) { + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); F_SET(cursor, WT_CURSTD_KEY_INT); + clsm->current = c; if (value == &cursor->value) F_SET(cursor, WT_CURSTD_VALUE_INT); } else if (c != NULL) @@ -1319,7 +1321,8 @@ __clsm_search_near(WT_CURSOR *cursor, int *exactp) if ((ret = c->search_near(c, &cmp)) == WT_NOTFOUND) { ret = 0; continue; - } else if (ret != 0) + } + if (ret != 0) goto err; /* Do we have an exact match? */ @@ -1339,7 +1342,8 @@ __clsm_search_near(WT_CURSOR *cursor, int *exactp) if ((ret = c->next(c)) == WT_NOTFOUND) { ret = 0; continue; - } else if (ret != 0) + } + if (ret != 0) goto err; } @@ -1565,12 +1569,23 @@ __clsm_update(WT_CURSOR *cursor) WT_CURSOR_NEEDVALUE(cursor); WT_ERR(__clsm_enter(clsm, false, true)); - if (F_ISSET(cursor, WT_CURSTD_OVERWRITE) || - (ret = __clsm_lookup(clsm, &value)) == 0) { - WT_ERR(__clsm_deleted_encode( - session, &cursor->value, &value, &buf)); - ret = __clsm_put(session, clsm, &cursor->key, &value, true); - } + if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE)) + WT_ERR(__clsm_lookup(clsm, &value)); + WT_ERR(__clsm_deleted_encode(session, &cursor->value, &value, &buf)); + WT_ERR(__clsm_put(session, clsm, &cursor->key, &value, true)); + + /* + * Set the cursor to reference the internal key/value of the positioned + * cursor. + */ + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + WT_ITEM_SET(cursor->key, clsm->current->key); + WT_ITEM_SET(cursor->value, clsm->current->value); + WT_ASSERT(session, + F_MASK(clsm->current, WT_CURSTD_KEY_SET) == WT_CURSTD_KEY_INT); + WT_ASSERT(session, + F_MASK(clsm->current, WT_CURSTD_VALUE_SET) == WT_CURSTD_VALUE_INT); + F_SET(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_VALUE_INT); err: __wt_scr_free(session, &buf); __clsm_leave(clsm); @@ -1589,18 +1604,34 @@ __clsm_remove(WT_CURSOR *cursor) WT_DECL_RET; WT_ITEM value; WT_SESSION_IMPL *session; + bool positioned; clsm = (WT_CURSOR_LSM *)cursor; + /* Check if the cursor is positioned. */ + positioned = F_ISSET(cursor, WT_CURSTD_KEY_INT); + CURSOR_REMOVE_API_CALL(cursor, session, NULL); WT_CURSOR_NEEDKEY(cursor); WT_CURSOR_NOVALUE(cursor); WT_ERR(__clsm_enter(clsm, false, true)); - if (F_ISSET(cursor, WT_CURSTD_OVERWRITE) || - (ret = __clsm_lookup(clsm, &value)) == 0) - ret = __clsm_put( - session, clsm, &cursor->key, &__tombstone, true); + if (!F_ISSET(cursor, WT_CURSTD_OVERWRITE)) + WT_ERR(__clsm_lookup(clsm, &value)); + WT_ERR(__clsm_put( + session, clsm, &cursor->key, &__tombstone, positioned)); + + /* + * If the cursor was positioned, it stays positioned with a key but no + * no value, otherwise, there's no position, key or value. This isn't + * just cosmetic, without a reset, iteration on this cursor won't start + * at the beginning/end of the table. + */ + F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); + if (positioned) + F_SET(cursor, WT_CURSTD_KEY_INT); + else + WT_TRET(cursor->reset(cursor)); err: __clsm_leave(clsm); CURSOR_UPDATE_API_END(session, ret); @@ -1630,8 +1661,6 @@ __wt_clsm_close(WT_CURSOR *cursor) /* In case we were somehow left positioned, clear that. */ __clsm_leave(clsm); - /* The WT_LSM_TREE owns the URI. */ - cursor->uri = NULL; if (clsm->lsm_tree != NULL) __wt_lsm_tree_release(session, clsm->lsm_tree); WT_TRET(__wt_cursor_close(cursor)); @@ -1693,8 +1722,8 @@ __wt_clsm_open(WT_SESSION_IMPL *session, bulk = cval.val != 0; /* Get the LSM tree. */ - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_lsm_tree_get(session, uri, bulk, &lsm_tree)); + ret = __wt_lsm_tree_get(session, uri, bulk, &lsm_tree); + /* * Check whether the exclusive open for a bulk load succeeded, and * if it did ensure that it's safe to bulk load into the tree. @@ -1713,7 +1742,7 @@ __wt_clsm_open(WT_SESSION_IMPL *session, cursor = &clsm->iface; *cursor = iface; cursor->session = &session->iface; - cursor->uri = lsm_tree->name; + WT_ERR(__wt_strdup(session, lsm_tree->name, &cursor->uri)); cursor->key_format = lsm_tree->key_format; cursor->value_format = lsm_tree->value_format; diff --git a/src/third_party/wiredtiger/src/lsm/lsm_cursor_bulk.c b/src/third_party/wiredtiger/src/lsm/lsm_cursor_bulk.c index 319426de3f0..7a6a40e380f 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_cursor_bulk.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_cursor_bulk.c @@ -45,7 +45,7 @@ __clsm_close_bulk(WT_CURSOR *cursor) total_chunks /= avg_chunks) ++chunk->generation; - WT_RET(__wt_lsm_meta_write(session, lsm_tree)); + WT_RET(__wt_lsm_meta_write(session, lsm_tree, NULL)); ++lsm_tree->dsk_gen; /* Close the LSM cursor */ @@ -113,7 +113,7 @@ __wt_clsm_open_bulk(WT_CURSOR_LSM *clsm, const char *cfg[]) * switch inline, since switch needs a schema lock and online index * creation opens a bulk cursor while holding the schema lock. */ - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_lsm_tree_switch(session, lsm_tree)); WT_RET(ret); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_manager.c b/src/third_party/wiredtiger/src/lsm/lsm_manager.c index cbd83a5cd30..b7d9086d10e 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_manager.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_manager.c @@ -89,7 +89,6 @@ __lsm_general_worker_start(WT_SESSION_IMPL *session) if (manager->lsm_workers % 2 == 0) FLD_SET(worker_args->type, WT_LSM_WORK_MERGE); } - F_SET(worker_args, WT_LSM_WORKER_RUN); WT_RET(__wt_lsm_worker_start(session, worker_args)); } @@ -129,17 +128,13 @@ __lsm_stop_workers(WT_SESSION_IMPL *session) manager->lsm_workers--) { worker_args = &manager->lsm_worker_cookies[manager->lsm_workers - 1]; - /* - * Clear this worker's flag so it stops. - */ - F_CLR(worker_args, WT_LSM_WORKER_RUN); - WT_ASSERT(session, worker_args->tid != 0); - WT_RET(__wt_thread_join(session, worker_args->tid)); - worker_args->tid = 0; + WT_ASSERT(session, worker_args->tid_set); + + WT_RET(__wt_lsm_worker_stop(session, worker_args)); worker_args->type = 0; - worker_args->flags = 0; + /* - * We do not clear the session because they are allocated + * We do not clear the other fields because they are allocated * statically when the connection was opened. */ } @@ -237,12 +232,12 @@ __wt_lsm_manager_start(WT_SESSION_IMPL *session) manager->lsm_worker_cookies[i].session = worker_session; } + F_SET(conn, WT_CONN_SERVER_LSM); + /* Start the LSM manager thread. */ WT_ERR(__wt_thread_create(session, &manager->lsm_worker_cookies[0].tid, __lsm_worker_manager, &manager->lsm_worker_cookies[0])); - F_SET(conn, WT_CONN_SERVER_LSM); - if (0) { err: for (i = 0; (worker_session = @@ -289,13 +284,18 @@ __wt_lsm_manager_destroy(WT_SESSION_IMPL *session) manager = &conn->lsm_manager; removed = 0; + /* + * Clear the LSM server flag and flush to ensure running threads see + * the state change. + */ + F_CLR(conn, WT_CONN_SERVER_LSM); + WT_FULL_BARRIER(); + WT_ASSERT(session, !F_ISSET(conn, WT_CONN_READONLY) || manager->lsm_workers == 0); if (manager->lsm_workers > 0) { - /* - * Stop the main LSM manager thread first. - */ - while (F_ISSET(conn, WT_CONN_SERVER_LSM)) + /* Wait for the main LSM manager thread to finish. */ + while (!F_ISSET(manager, WT_LSM_MANAGER_SHUTDOWN)) __wt_yield(); /* Clean up open LSM handles. */ @@ -303,7 +303,6 @@ __wt_lsm_manager_destroy(WT_SESSION_IMPL *session) WT_TRET(__wt_thread_join( session, manager->lsm_worker_cookies[0].tid)); - manager->lsm_worker_cookies[0].tid = 0; /* Release memory from any operations left on the queue. */ while ((current = TAILQ_FIRST(&manager->switchqh)) != NULL) { @@ -335,14 +334,14 @@ __wt_lsm_manager_destroy(WT_SESSION_IMPL *session) __wt_spin_destroy(session, &manager->switch_lock); __wt_spin_destroy(session, &manager->app_lock); __wt_spin_destroy(session, &manager->manager_lock); - WT_TRET(__wt_cond_destroy(session, &manager->work_cond)); + __wt_cond_destroy(session, &manager->work_cond); return (ret); } /* * __lsm_manager_worker_shutdown -- - * Shutdown the LSM manager and worker threads. + * Shutdown the LSM worker threads. */ static int __lsm_manager_worker_shutdown(WT_SESSION_IMPL *session) @@ -354,14 +353,13 @@ __lsm_manager_worker_shutdown(WT_SESSION_IMPL *session) manager = &S2C(session)->lsm_manager; /* - * Wait for the rest of the LSM workers to shutdown. Stop at index + * Wait for the rest of the LSM workers to shutdown. Start at index * one - since we (the manager) are at index 0. */ for (i = 1; i < manager->lsm_workers; i++) { - WT_ASSERT(session, manager->lsm_worker_cookies[i].tid != 0); - __wt_cond_signal(session, manager->work_cond); - WT_TRET(__wt_thread_join( - session, manager->lsm_worker_cookies[i].tid)); + WT_ASSERT(session, manager->lsm_worker_cookies[i].tid_set); + WT_TRET(__wt_lsm_worker_stop( + session, &manager->lsm_worker_cookies[i])); } return (ret); } @@ -383,12 +381,12 @@ __lsm_manager_run_server(WT_SESSION_IMPL *session) conn = S2C(session); dhandle_locked = false; - while (F_ISSET(conn, WT_CONN_SERVER_RUN)) { + while (F_ISSET(conn, WT_CONN_SERVER_LSM)) { __wt_sleep(0, 10000); if (TAILQ_EMPTY(&conn->lsmqh)) continue; - __wt_spin_lock(session, &conn->dhandle_lock); - F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST); + __wt_readlock(session, &conn->dhandle_lock); + F_SET(session, WT_SESSION_LOCKED_HANDLE_LIST_READ); dhandle_locked = true; TAILQ_FOREACH(lsm_tree, &S2C(session)->lsmqh, q) { if (!lsm_tree->active) @@ -448,14 +446,14 @@ __lsm_manager_run_server(WT_SESSION_IMPL *session) session, WT_LSM_WORK_MERGE, 0, lsm_tree)); } } - __wt_spin_unlock(session, &conn->dhandle_lock); - F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST); + __wt_readunlock(session, &conn->dhandle_lock); + F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST_READ); dhandle_locked = false; } err: if (dhandle_locked) { - __wt_spin_unlock(session, &conn->dhandle_lock); - F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST); + __wt_readunlock(session, &conn->dhandle_lock); + F_CLR(session, WT_SESSION_LOCKED_HANDLE_LIST_READ); } return (ret); } @@ -469,11 +467,13 @@ static WT_THREAD_RET __lsm_worker_manager(void *arg) { WT_DECL_RET; + WT_LSM_MANAGER *manager; WT_LSM_WORKER_ARGS *cookie; WT_SESSION_IMPL *session; cookie = (WT_LSM_WORKER_ARGS *)arg; session = cookie->session; + manager = &S2C(session)->lsm_manager; WT_ERR(__lsm_general_worker_start(session)); WT_ERR(__lsm_manager_run_server(session)); @@ -482,7 +482,11 @@ __lsm_worker_manager(void *arg) if (ret != 0) { err: WT_PANIC_MSG(session, ret, "LSM worker manager thread error"); } - F_CLR(S2C(session), WT_CONN_SERVER_LSM); + + /* Connection close waits on us to shutdown, let it know we're done. */ + F_SET(manager, WT_LSM_MANAGER_SHUTDOWN); + WT_FULL_BARRIER(); + return (WT_THREAD_RET_VALUE); } diff --git a/src/third_party/wiredtiger/src/lsm/lsm_merge.c b/src/third_party/wiredtiger/src/lsm/lsm_merge.c index 493855d489a..8838638f388 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_merge.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_merge.c @@ -46,7 +46,7 @@ __wt_lsm_merge_update_tree(WT_SESSION_IMPL *session, static void __lsm_merge_aggressive_clear(WT_LSM_TREE *lsm_tree) { - F_CLR(lsm_tree, WT_LSM_TREE_AGGRESSIVE_TIMER); + lsm_tree->aggressive_timer_enabled = false; lsm_tree->merge_aggressiveness = 0; } @@ -85,12 +85,12 @@ __lsm_merge_aggressive_update(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) } /* - * Start the timer if it isn't running. Use a flag to define whether + * Start the timer if it isn't running. Use a bool to define whether * the timer is running - since clearing and checking a special * timer value isn't simple. */ - if (!F_ISSET(lsm_tree, WT_LSM_TREE_AGGRESSIVE_TIMER)) { - F_SET(lsm_tree, WT_LSM_TREE_AGGRESSIVE_TIMER); + if (!lsm_tree->aggressive_timer_enabled) { + lsm_tree->aggressive_timer_enabled = true; __wt_epoch(session, &lsm_tree->merge_aggressive_ts); } @@ -187,7 +187,7 @@ __lsm_merge_span(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, continue; if (F_ISSET(chunk, WT_LSM_CHUNK_BLOOM) || chunk->generation > 0) break; - else if (FLD_ISSET(lsm_tree->bloom, WT_LSM_BLOOM_OFF) && + if (FLD_ISSET(lsm_tree->bloom, WT_LSM_BLOOM_OFF) && F_ISSET(chunk, WT_LSM_CHUNK_ONDISK)) break; } @@ -434,7 +434,7 @@ __wt_lsm_merge(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int id) F_SET(src, WT_CURSTD_RAW); WT_ERR(__wt_clsm_init_merge(src, start_chunk, start_id, nchunks)); - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_lsm_tree_setup_chunk(session, lsm_tree, chunk)); WT_ERR(ret); if (create_bloom) { @@ -579,7 +579,7 @@ __wt_lsm_merge(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, u_int id) * Any errors that happened after the tree was locked are * fatal - we can't guarantee the state of the tree. */ - if ((ret = __wt_lsm_meta_write(session, lsm_tree)) != 0) + if ((ret = __wt_lsm_meta_write(session, lsm_tree, NULL)) != 0) WT_PANIC_ERR(session, ret, "Failed finalizing LSM merge"); lsm_tree->dsk_gen++; @@ -604,13 +604,13 @@ err: if (locked) if (ret != 0 && created_chunk) { /* Drop the newly-created files on error. */ if (chunk->uri != NULL) { - WT_WITH_SCHEMA_LOCK(session, tret, + WT_WITH_SCHEMA_LOCK(session, tret = __wt_schema_drop( session, chunk->uri, drop_cfg)); WT_TRET(tret); } if (create_bloom && chunk->bloom_uri != NULL) { - WT_WITH_SCHEMA_LOCK(session, tret, + WT_WITH_SCHEMA_LOCK(session, tret = __wt_schema_drop( session, chunk->bloom_uri, drop_cfg)); WT_TRET(tret); @@ -625,7 +625,7 @@ err: if (locked) else __wt_verbose(session, WT_VERB_LSM, "Merge failed with %s", - __wt_strerror(session, ret, NULL, 0)); + __wt_strerror(session, ret, NULL, 0)); } F_CLR(session, WT_SESSION_NO_CACHE | WT_SESSION_NO_EVICTION); return (ret); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_meta.c b/src/third_party/wiredtiger/src/lsm/lsm_meta.c index ec52af96231..fc4dde82470 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_meta.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_meta.c @@ -229,7 +229,7 @@ __lsm_meta_read_v1( cv.len -= 2; } WT_ERR(__wt_config_check(session, - WT_CONFIG_REF(session, WT_SESSION_create), cv.str, cv.len)); + WT_CONFIG_REF(session, WT_SESSION_create), cv.str, cv.len)); WT_ERR(__wt_strndup(session, cv.str, cv.len, &lsm_tree->bloom_config)); WT_ERR(__wt_config_getones( session, lsmconf, "lsm.bloom_hash_count", &cv)); @@ -454,13 +454,14 @@ __wt_lsm_meta_read(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) * Write the metadata for an LSM tree. */ int -__wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) +__wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, + const char *newconfig) { WT_DECL_ITEM(buf); WT_DECL_RET; WT_LSM_CHUNK *chunk; u_int i; - const char *new_cfg[] = { NULL, NULL, NULL }; + const char *new_cfg[] = { NULL, NULL, NULL, NULL, NULL }; char *new_metadata; bool first; @@ -504,8 +505,10 @@ __wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) WT_ERR(__wt_buf_catfmt(session, buf, "]")); /* Update the existing configuration with the new values. */ - new_cfg[0] = lsm_tree->config; - new_cfg[1] = buf->data; + new_cfg[0] = WT_CONFIG_BASE(session, lsm_meta); + new_cfg[1] = lsm_tree->config; + new_cfg[2] = buf->data; + new_cfg[3] = newconfig; WT_ERR(__wt_config_collapse(session, new_cfg, &new_metadata)); ret = __wt_metadata_update(session, lsm_tree->name, new_metadata); WT_ERR(ret); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_stat.c b/src/third_party/wiredtiger/src/lsm/lsm_stat.c index 3fe3ca1ba81..411655878af 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_stat.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_stat.c @@ -29,24 +29,22 @@ __curstat_lsm_init( const char *cfg[] = { WT_CONFIG_BASE(session, WT_SESSION_open_cursor), NULL, NULL }; const char *disk_cfg[] = { - WT_CONFIG_BASE(session, WT_SESSION_open_cursor), - "checkpoint=" WT_CHECKPOINT, NULL, NULL }; + WT_CONFIG_BASE(session, WT_SESSION_open_cursor), + "checkpoint=" WT_CHECKPOINT, NULL, NULL }; locked = false; - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_lsm_tree_get(session, uri, false, &lsm_tree)); - WT_RET(ret); + WT_RET(__wt_lsm_tree_get(session, uri, false, &lsm_tree)); WT_ERR(__wt_scr_alloc(session, 0, &uribuf)); /* Propagate all, fast and/or clear to the cursors we open. */ if (cst->flags != 0) { - (void)snprintf(config, sizeof(config), + WT_ERR(__wt_snprintf(config, sizeof(config), "statistics=(%s%s%s%s)", F_ISSET(cst, WT_STAT_TYPE_ALL) ? "all," : "", F_ISSET(cst, WT_STAT_CLEAR) ? "clear," : "", !F_ISSET(cst, WT_STAT_TYPE_ALL) && F_ISSET(cst, WT_STAT_TYPE_FAST) ? "fast," : "", - F_ISSET(cst, WT_STAT_TYPE_SIZE) ? "size," : ""); + F_ISSET(cst, WT_STAT_TYPE_SIZE) ? "size," : "")); cfg[1] = disk_cfg[1] = config; } @@ -178,7 +176,7 @@ __wt_curstat_lsm_init( * Grab the schema lock because we will be locking the LSM tree and we * may need to open some files. */ - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret = __curstat_lsm_init(session, uri, cst)); return (ret); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_tree.c b/src/third_party/wiredtiger/src/lsm/lsm_tree.c index 0054dcd1583..36b31358c0b 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_tree.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_tree.c @@ -38,7 +38,7 @@ __lsm_tree_discard(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, bool final) /* We may be destroying an lsm_tree before it was added. */ if (F_ISSET(lsm_tree, WT_LSM_TREE_OPEN)) { WT_ASSERT(session, final || - F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); + F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE)); TAILQ_REMOVE(&S2C(session)->lsmqh, lsm_tree, q); } @@ -251,7 +251,7 @@ __lsm_tree_cleanup_old(WT_SESSION_IMPL *session, const char *uri) WT_RET(__wt_fs_exist(session, uri + strlen("file:"), &exists)); if (exists) - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_drop(session, uri, cfg)); return (ret); } @@ -293,8 +293,6 @@ int __wt_lsm_tree_setup_bloom( WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_LSM_CHUNK *chunk) { - WT_DECL_RET; - /* * The Bloom URI can be populated when the chunk is created, but * it isn't set yet on open or merge. @@ -302,8 +300,8 @@ __wt_lsm_tree_setup_bloom( if (chunk->bloom_uri == NULL) WT_RET(__wt_lsm_tree_bloom_name( session, lsm_tree, chunk->id, &chunk->bloom_uri)); - WT_RET(__lsm_tree_cleanup_old(session, chunk->bloom_uri)); - return (ret); + + return (__lsm_tree_cleanup_old(session, chunk->bloom_uri)); } /* @@ -323,9 +321,7 @@ __wt_lsm_tree_create(WT_SESSION_IMPL *session, metadata = NULL; /* If the tree can be opened, it already exists. */ - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_lsm_tree_get(session, uri, false, &lsm_tree)); - if (ret == 0) { + if ((ret = __wt_lsm_tree_get(session, uri, false, &lsm_tree)) == 0) { __wt_lsm_tree_release(session, lsm_tree); return (exclusive ? EEXIST : 0); } @@ -341,7 +337,7 @@ __wt_lsm_tree_create(WT_SESSION_IMPL *session, * error: the returned handle is NULL on error, and the metadata * tracking macros handle cleaning up on failure. */ - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, ret = __lsm_tree_open(session, uri, true, &lsm_tree)); if (ret == 0) __wt_lsm_tree_release(session, lsm_tree); @@ -406,6 +402,9 @@ __lsm_tree_find(WT_SESSION_IMPL *session, } *treep = lsm_tree; + + WT_ASSERT(session, lsm_tree->excl_session == + (exclusive ? session : NULL)); return (0); } @@ -458,7 +457,8 @@ __lsm_tree_open(WT_SESSION_IMPL *session, conn = S2C(session); lsm_tree = NULL; - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); + WT_ASSERT(session, + F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE)); /* Start the LSM manager thread if it isn't running. */ if (__wt_atomic_cas32(&conn->lsm_manager.lsm_workers, 0, 1)) @@ -471,7 +471,7 @@ __lsm_tree_open(WT_SESSION_IMPL *session, /* Try to open the tree. */ WT_RET(__wt_calloc_one(session, &lsm_tree)); - WT_ERR(__wt_rwlock_alloc(session, &lsm_tree->rwlock, "lsm tree")); + WT_ERR(__wt_rwlock_init(session, &lsm_tree->rwlock)); WT_ERR(__lsm_tree_set_name(session, lsm_tree, uri)); @@ -522,14 +522,21 @@ __wt_lsm_tree_get(WT_SESSION_IMPL *session, { WT_DECL_RET; - WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)); - - ret = __lsm_tree_find(session, uri, exclusive, treep); + /* + * Dropping and re-acquiring the lock is safe here, since the tree open + * call checks to see if another thread beat it to opening the tree + * before proceeding. + */ + if (exclusive) + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, + ret = __lsm_tree_find(session, uri, exclusive, treep)); + else + WT_WITH_HANDLE_LIST_READ_LOCK(session, + ret = __lsm_tree_find(session, uri, exclusive, treep)); if (ret == WT_NOTFOUND) - ret = __lsm_tree_open(session, uri, exclusive, treep); + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, + ret = __lsm_tree_open(session, uri, exclusive, treep)); - WT_ASSERT(session, ret != 0 || - (*treep)->excl_session == (exclusive ? session : NULL)); return (ret); } @@ -736,7 +743,7 @@ __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) if (!first_switch && (last_chunk = lsm_tree->chunk[nchunks - 1]) != NULL && !F_ISSET(last_chunk, WT_LSM_CHUNK_ONDISK) && - !F_ISSET(lsm_tree, WT_LSM_TREE_NEED_SWITCH)) + !lsm_tree->need_switch) goto err; /* Update the throttle time. */ @@ -758,8 +765,8 @@ __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) lsm_tree->chunk[lsm_tree->nchunks++] = chunk; WT_ERR(__wt_lsm_tree_setup_chunk(session, lsm_tree, chunk)); - WT_ERR(__wt_lsm_meta_write(session, lsm_tree)); - F_CLR(lsm_tree, WT_LSM_TREE_NEED_SWITCH); + WT_ERR(__wt_lsm_meta_write(session, lsm_tree, NULL)); + lsm_tree->need_switch = false; ++lsm_tree->dsk_gen; lsm_tree->modified = true; @@ -842,6 +849,45 @@ __wt_lsm_tree_retire_chunks(WT_SESSION_IMPL *session, return (0); } +/* + * __wt_lsm_tree_alter -- + * Alter an LSM tree. + */ +int +__wt_lsm_tree_alter( + WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + WT_DECL_RET; + WT_LSM_CHUNK *chunk; + WT_LSM_TREE *lsm_tree; + u_int i; + bool locked; + + locked = false; + + /* Get the LSM tree. */ + WT_RET(__wt_lsm_tree_get(session, uri, false, &lsm_tree)); + + /* Prevent any new opens. */ + __wt_lsm_tree_writelock(session, lsm_tree); + locked = true; + + /* Alter the chunks. */ + for (i = 0; i < lsm_tree->nchunks; i++) { + chunk = lsm_tree->chunk[i]; + WT_ERR(__wt_schema_alter(session, chunk->uri, cfg)); + if (F_ISSET(chunk, WT_LSM_CHUNK_BLOOM)) + WT_ERR( + __wt_schema_alter(session, chunk->bloom_uri, cfg)); + } + WT_ERR(__wt_lsm_meta_write(session, lsm_tree, cfg[0])); + +err: if (locked) + __wt_lsm_tree_writeunlock(session, lsm_tree); + __wt_lsm_tree_release(session, lsm_tree); + return (ret); +} + /* * __wt_lsm_tree_drop -- * Drop an LSM tree. @@ -860,9 +906,7 @@ __wt_lsm_tree_drop( locked = false; /* Get the LSM tree. */ - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_lsm_tree_get(session, name, true, &lsm_tree)); - WT_RET(ret); + WT_RET(__wt_lsm_tree_get(session, name, true, &lsm_tree)); WT_ASSERT(session, !lsm_tree->active); /* Prevent any new opens. */ @@ -895,7 +939,7 @@ __wt_lsm_tree_drop( WT_ASSERT(session, !lsm_tree->active); err: if (locked) __wt_lsm_tree_writeunlock(session, lsm_tree); - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, tret = __lsm_tree_discard(session, lsm_tree, false)); WT_TRET(tret); return (ret); @@ -921,9 +965,7 @@ __wt_lsm_tree_rename(WT_SESSION_IMPL *session, locked = false; /* Get the LSM tree. */ - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_lsm_tree_get(session, olduri, true, &lsm_tree)); - WT_RET(ret); + WT_RET(__wt_lsm_tree_get(session, olduri, true, &lsm_tree)); /* Prevent any new opens. */ __wt_lsm_tree_writelock(session, lsm_tree); @@ -955,7 +997,7 @@ __wt_lsm_tree_rename(WT_SESSION_IMPL *session, } } - WT_ERR(__wt_lsm_meta_write(session, lsm_tree)); + WT_ERR(__wt_lsm_meta_write(session, lsm_tree, NULL)); locked = false; __wt_lsm_tree_writeunlock(session, lsm_tree); WT_ERR(__wt_metadata_remove(session, olduri)); @@ -968,7 +1010,7 @@ err: if (locked) * Discard this LSM tree structure. The first operation on the renamed * tree will create a new one. */ - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, tret = __lsm_tree_discard(session, lsm_tree, false)); WT_TRET(tret); return (ret); @@ -993,9 +1035,7 @@ __wt_lsm_tree_truncate( locked = false; /* Get the LSM tree. */ - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_lsm_tree_get(session, name, true, &lsm_tree)); - WT_RET(ret); + WT_RET(__wt_lsm_tree_get(session, name, true, &lsm_tree)); /* Prevent any new opens. */ __wt_lsm_tree_writelock(session, lsm_tree); @@ -1010,7 +1050,7 @@ __wt_lsm_tree_truncate( WT_ERR(__wt_lsm_merge_update_tree( session, lsm_tree, 0, lsm_tree->nchunks, chunk)); - WT_ERR(__wt_lsm_meta_write(session, lsm_tree)); + WT_ERR(__wt_lsm_meta_write(session, lsm_tree, NULL)); locked = false; __wt_lsm_tree_writeunlock(session, lsm_tree); @@ -1029,7 +1069,7 @@ err: if (locked) * the last good version of the metadata will be used, resulting * in a valid (not truncated) tree. */ - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, tret = __lsm_tree_discard(session, lsm_tree, false)); WT_TRET(tret); } @@ -1043,7 +1083,7 @@ err: if (locked) void __wt_lsm_tree_readlock(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) { - __wt_readlock(session, lsm_tree->rwlock); + __wt_readlock(session, &lsm_tree->rwlock); /* * Diagnostic: avoid deadlocks with the schema lock: if we need it for @@ -1061,7 +1101,7 @@ __wt_lsm_tree_readunlock(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) { F_CLR(session, WT_SESSION_NO_EVICTION | WT_SESSION_NO_SCHEMA_LOCK); - __wt_readunlock(session, lsm_tree->rwlock); + __wt_readunlock(session, &lsm_tree->rwlock); } /* @@ -1071,7 +1111,7 @@ __wt_lsm_tree_readunlock(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) void __wt_lsm_tree_writelock(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) { - __wt_writelock(session, lsm_tree->rwlock); + __wt_writelock(session, &lsm_tree->rwlock); /* * Diagnostic: avoid deadlocks with the schema lock: if we need it for @@ -1089,7 +1129,7 @@ __wt_lsm_tree_writeunlock(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) { F_CLR(session, WT_SESSION_NO_EVICTION | WT_SESSION_NO_SCHEMA_LOCK); - __wt_writeunlock(session, lsm_tree->rwlock); + __wt_writeunlock(session, &lsm_tree->rwlock); } /* @@ -1102,7 +1142,6 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp) WT_DECL_RET; WT_LSM_CHUNK *chunk; WT_LSM_TREE *lsm_tree; - time_t begin, end; uint64_t progress; uint32_t i; bool compacting, flushing, locked, ref; @@ -1119,9 +1158,7 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp) /* Tell __wt_schema_worker not to look inside the LSM tree. */ *skipp = true; - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_lsm_tree_get(session, name, false, &lsm_tree)); - WT_RET(ret); + WT_RET(__wt_lsm_tree_get(session, name, false, &lsm_tree)); if (!F_ISSET(S2C(session), WT_CONN_LSM_MERGE)) WT_ERR_MSG(session, EINVAL, @@ -1139,8 +1176,6 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp) return (0); } - __wt_seconds(session, &begin); - /* * Compacting has two distinct phases. * 1. All in-memory chunks up to and including the current @@ -1266,12 +1301,9 @@ __wt_lsm_compact(WT_SESSION_IMPL *session, const char *name, bool *skipp) } else break; } + WT_ERR(__wt_session_compact_check_timeout(session)); __wt_sleep(1, 0); - __wt_seconds(session, &end); - if (session->compact->max_time > 0 && - session->compact->max_time < (uint64_t)(end - begin)) { - WT_ERR(ETIMEDOUT); - } + /* * Push merge operations while they are still getting work * done. If we are pushing merges, make sure they are @@ -1323,9 +1355,7 @@ __wt_lsm_tree_worker(WT_SESSION_IMPL *session, locked = false; exclusive = FLD_ISSET(open_flags, WT_DHANDLE_EXCLUSIVE); - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_lsm_tree_get(session, uri, exclusive, &lsm_tree)); - WT_RET(ret); + WT_RET(__wt_lsm_tree_get(session, uri, exclusive, &lsm_tree)); /* * We mark that we're busy using the tree to coordinate diff --git a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c index 917104031fc..b55db012451 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_work_unit.c @@ -171,12 +171,12 @@ __wt_lsm_work_switch( *ran = false; *entryp = NULL; - if (F_ISSET(entry->lsm_tree, WT_LSM_TREE_NEED_SWITCH)) { - WT_WITH_SCHEMA_LOCK(session, ret, + if (entry->lsm_tree->need_switch) { + WT_WITH_SCHEMA_LOCK(session, ret = __wt_lsm_tree_switch(session, entry->lsm_tree)); /* Failing to complete the switch is fine */ if (ret == EBUSY) { - if (F_ISSET(entry->lsm_tree, WT_LSM_TREE_NEED_SWITCH)) + if (entry->lsm_tree->need_switch) WT_ERR(__wt_lsm_manager_push_entry(session, WT_LSM_WORK_SWITCH, 0, entry->lsm_tree)); ret = 0; @@ -230,7 +230,7 @@ __wt_lsm_work_bloom(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) if (__wt_atomic_cas32(&chunk->bloom_busy, 0, 1)) { if (!F_ISSET(chunk, WT_LSM_CHUNK_BLOOM)) { ret = __lsm_bloom_create( - session, lsm_tree, chunk, (u_int)i); + session, lsm_tree, chunk, i); /* * Record if we were successful so that we can * later push a merge work unit. @@ -265,9 +265,9 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, { WT_DECL_RET; WT_TXN_ISOLATION saved_isolation; - bool flush_set; + bool flush_set, release_btree; - flush_set = false; + flush_set = release_btree = false; /* * If the chunk is already checkpointed, make sure it is also evicted. @@ -276,7 +276,7 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, if (F_ISSET(chunk, WT_LSM_CHUNK_ONDISK) && !F_ISSET(chunk, WT_LSM_CHUNK_STABLE) && !chunk->evicted) { - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, ret = __lsm_discard_handle(session, chunk->uri, NULL)); if (ret == 0) chunk->evicted = 1; @@ -318,19 +318,18 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, * We can wait here for checkpoints and fsyncs to complete, which can * take a long time. */ - if ((ret = __wt_session_get_btree( - session, chunk->uri, NULL, NULL, 0)) == 0) { - /* - * Set read-uncommitted: we have already checked that all of the - * updates in this chunk are globally visible, use the cheapest - * possible check in reconciliation. - */ - saved_isolation = session->txn.isolation; - session->txn.isolation = WT_ISO_READ_UNCOMMITTED; - ret = __wt_cache_op(session, WT_SYNC_WRITE_LEAVES); - session->txn.isolation = saved_isolation; - WT_TRET(__wt_session_release_btree(session)); - } + WT_ERR(__wt_session_get_btree(session, chunk->uri, NULL, NULL, 0)); + release_btree = true; + + /* + * Set read-uncommitted: we have already checked that all of the updates + * in this chunk are globally visible, use the cheapest possible check + * in reconciliation. + */ + saved_isolation = session->txn.isolation; + session->txn.isolation = WT_ISO_READ_UNCOMMITTED; + ret = __wt_cache_op(session, WT_SYNC_WRITE_LEAVES); + session->txn.isolation = saved_isolation; WT_ERR(ret); __wt_verbose(session, WT_VERB_LSM, "LSM worker checkpointing %s", @@ -346,14 +345,16 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, * time, and our checkpoint operation should be very quick. */ WT_ERR(__wt_meta_track_on(session)); - WT_WITH_CHECKPOINT_LOCK(session, ret, - WT_WITH_SCHEMA_LOCK(session, ret, - ret = __wt_schema_worker( - session, chunk->uri, __wt_checkpoint, NULL, NULL, 0))); + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, + ret = __wt_checkpoint(session, NULL))); WT_TRET(__wt_meta_track_off(session, false, ret != 0)); if (ret != 0) WT_ERR_MSG(session, ret, "LSM checkpoint"); + release_btree = false; + WT_ERR(__wt_session_release_btree(session)); + /* Now the file is written, get the chunk size. */ WT_ERR(__wt_lsm_tree_set_chunk_size(session, chunk)); @@ -364,7 +365,7 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, /* Lock the tree, mark the chunk as on disk and update the metadata. */ __wt_lsm_tree_writelock(session, lsm_tree); F_SET(chunk, WT_LSM_CHUNK_ONDISK); - ret = __wt_lsm_meta_write(session, lsm_tree); + ret = __wt_lsm_meta_write(session, lsm_tree, NULL); ++lsm_tree->dsk_gen; /* Update the throttle time. */ @@ -376,16 +377,6 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, WT_PUBLISH(chunk->flushing, 0); flush_set = false; - /* - * Clear the no-eviction flag so the primary can be evicted and - * eventually closed. Only do this once the checkpoint has succeeded: - * otherwise, accessing the leaf page during the checkpoint can trigger - * forced eviction. - */ - WT_ERR(__wt_session_get_btree(session, chunk->uri, NULL, NULL, 0)); - __wt_btree_evictable(session, true); - WT_ERR(__wt_session_release_btree(session)); - /* Make sure we aren't pinning a transaction ID. */ __wt_txn_release_snapshot(session); @@ -402,6 +393,8 @@ __wt_lsm_checkpoint_chunk(WT_SESSION_IMPL *session, err: if (flush_set) WT_PUBLISH(chunk->flushing, 0); + if (release_btree) + WT_TRET(__wt_session_release_btree(session)); return (ret); } @@ -469,7 +462,7 @@ __lsm_bloom_create(WT_SESSION_IMPL *session, /* Ensure the bloom filter is in the metadata. */ __wt_lsm_tree_writelock(session, lsm_tree); F_SET(chunk, WT_LSM_CHUNK_BLOOM); - ret = __wt_lsm_meta_write(session, lsm_tree); + ret = __wt_lsm_meta_write(session, lsm_tree, NULL); ++lsm_tree->dsk_gen; __wt_lsm_tree_writeunlock(session, lsm_tree); @@ -517,8 +510,8 @@ __lsm_drop_file(WT_SESSION_IMPL *session, const char *uri) * * This will fail with EBUSY if the file is still in use. */ - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __lsm_discard_handle(session, uri, WT_CHECKPOINT)); + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, + ret = __lsm_discard_handle(session, uri, WT_CHECKPOINT)); WT_RET(ret); /* @@ -526,7 +519,7 @@ __lsm_drop_file(WT_SESSION_IMPL *session, const char *uri) * results in the hot backup lock being taken when it updates the * metadata (which would be too late to prevent our drop). */ - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_drop(session, uri, drop_cfg)); if (ret == 0) @@ -610,7 +603,8 @@ __wt_lsm_free_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) if (drop_ret == EBUSY) { ++skipped; continue; - } else if (drop_ret != ENOENT) + } + if (drop_ret != ENOENT) WT_ERR(drop_ret); flush_metadata = true; @@ -621,7 +615,8 @@ __wt_lsm_free_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) if (drop_ret == EBUSY) { ++skipped; continue; - } else if (drop_ret != ENOENT) + } + if (drop_ret != ENOENT) WT_ERR(drop_ret); flush_metadata = true; } @@ -659,7 +654,7 @@ __wt_lsm_free_chunks(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) err: /* Flush the metadata unless the system is in panic */ if (flush_metadata && ret != WT_PANIC) { __wt_lsm_tree_writelock(session, lsm_tree); - WT_TRET(__wt_lsm_meta_write(session, lsm_tree)); + WT_TRET(__wt_lsm_meta_write(session, lsm_tree, NULL)); __wt_lsm_tree_writeunlock(session, lsm_tree); } __lsm_unpin_chunks(session, &cookie); diff --git a/src/third_party/wiredtiger/src/lsm/lsm_worker.c b/src/third_party/wiredtiger/src/lsm/lsm_worker.c index b0d0758775d..1cabbd4888d 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_worker.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_worker.c @@ -21,7 +21,23 @@ __wt_lsm_worker_start(WT_SESSION_IMPL *session, WT_LSM_WORKER_ARGS *args) { __wt_verbose(session, WT_VERB_LSM_MANAGER, "Start LSM worker %u type %#" PRIx32, args->id, args->type); - return (__wt_thread_create(session, &args->tid, __lsm_worker, args)); + + args->running = true; + WT_RET(__wt_thread_create(session, &args->tid, __lsm_worker, args)); + args->tid_set = true; + return (0); +} + +/* + * __wt_lsm_worker_stop -- + * A wrapper around the LSM worker thread stop. + */ +int +__wt_lsm_worker_stop(WT_SESSION_IMPL *session, WT_LSM_WORKER_ARGS *args) +{ + args->running = false; + args->tid_set = false; + return (__wt_thread_join(session, args->tid)); } /* @@ -84,7 +100,6 @@ err: __wt_lsm_manager_free_work_unit(session, entry); static WT_THREAD_RET __lsm_worker(void *arg) { - WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_LSM_WORK_UNIT *entry; WT_LSM_WORKER_ARGS *cookie; @@ -93,11 +108,9 @@ __lsm_worker(void *arg) cookie = (WT_LSM_WORKER_ARGS *)arg; session = cookie->session; - conn = S2C(session); entry = NULL; - while (F_ISSET(conn, WT_CONN_SERVER_RUN) && - F_ISSET(cookie, WT_LSM_WORKER_RUN)) { + while (cookie->running) { progress = false; /* @@ -154,7 +167,7 @@ __lsm_worker(void *arg) /* Don't busy wait if there was any work to do. */ if (!progress) { - __wt_cond_wait(session, cookie->work_cond, 10000); + __wt_cond_wait(session, cookie->work_cond, 10000, NULL); continue; } } diff --git a/src/third_party/wiredtiger/src/meta/meta_apply.c b/src/third_party/wiredtiger/src/meta/meta_apply.c index fb483c21dd9..dc93180a5e5 100644 --- a/src/third_party/wiredtiger/src/meta/meta_apply.c +++ b/src/third_party/wiredtiger/src/meta/meta_apply.c @@ -45,11 +45,7 @@ __meta_btree_apply(WT_SESSION_IMPL *session, WT_CURSOR *cursor, session, uri, NULL, NULL, 0)) != 0) return (ret == EBUSY ? 0 : ret); WT_SAVE_DHANDLE(session, ret = file_func(session, cfg)); - if (WT_META_TRACKING(session)) - WT_TRET(__wt_meta_track_handle_lock( - session, false)); - else - WT_TRET(__wt_session_release_btree(session)); + WT_TRET(__wt_session_release_btree(session)); WT_RET(ret); } WT_RET_NOTFOUND_OK(ret); diff --git a/src/third_party/wiredtiger/src/meta/meta_ckpt.c b/src/third_party/wiredtiger/src/meta/meta_ckpt.c index b985104c2eb..151bbe0e081 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ckpt.c +++ b/src/third_party/wiredtiger/src/meta/meta_ckpt.c @@ -297,7 +297,7 @@ __wt_meta_ckptlist_get( *ckptbasep = ckptbase; if (0) { -err: __wt_meta_ckptlist_free(session, ckptbase); +err: __wt_meta_ckptlist_free(session, &ckptbase); } __wt_free(session, config); __wt_scr_free(session, &buf); @@ -463,16 +463,16 @@ err: __wt_scr_free(session, &buf); * Discard the checkpoint array. */ void -__wt_meta_ckptlist_free(WT_SESSION_IMPL *session, WT_CKPT *ckptbase) +__wt_meta_ckptlist_free(WT_SESSION_IMPL *session, WT_CKPT **ckptbasep) { - WT_CKPT *ckpt; + WT_CKPT *ckpt, *ckptbase; - if (ckptbase == NULL) + if ((ckptbase = *ckptbasep) == NULL) return; WT_CKPT_FOREACH(ckptbase, ckpt) __wt_meta_checkpoint_free(session, ckpt); - __wt_free(session, ckptbase); + __wt_free(session, *ckptbasep); } /* diff --git a/src/third_party/wiredtiger/src/meta/meta_ext.c b/src/third_party/wiredtiger/src/meta/meta_ext.c index b48f7205807..aa1ea8b974d 100644 --- a/src/third_party/wiredtiger/src/meta/meta_ext.c +++ b/src/third_party/wiredtiger/src/meta/meta_ext.c @@ -88,6 +88,7 @@ __wt_ext_metadata_update(WT_EXTENSION_API *wt_api, int __wt_metadata_get_ckptlist( WT_SESSION *session, const char *name, WT_CKPT **ckptbasep) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { return (__wt_meta_ckptlist_get( (WT_SESSION_IMPL *)session, name, ckptbasep)); @@ -99,6 +100,7 @@ __wt_metadata_get_ckptlist( */ void __wt_metadata_free_ckptlist(WT_SESSION *session, WT_CKPT *ckptbase) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { - __wt_meta_ckptlist_free((WT_SESSION_IMPL *)session, ckptbase); + __wt_meta_ckptlist_free((WT_SESSION_IMPL *)session, &ckptbase); } diff --git a/src/third_party/wiredtiger/src/meta/meta_table.c b/src/third_party/wiredtiger/src/meta/meta_table.c index 76f4ef0c25f..aca69d0e6a2 100644 --- a/src/third_party/wiredtiger/src/meta/meta_table.c +++ b/src/third_party/wiredtiger/src/meta/meta_table.c @@ -68,9 +68,6 @@ __wt_metadata_cursor_open( if (F_ISSET(btree, WT_BTREE_NO_LOGGING)) F_CLR(btree, WT_BTREE_NO_LOGGING); - /* The metadata file always uses checkpoint IDs in visibility checks. */ - btree->include_checkpoint_txn = true; - return (0); } @@ -195,7 +192,7 @@ __wt_metadata_update( __metadata_turtle(key) ? "" : "not "); if (__metadata_turtle(key)) { - WT_WITH_TURTLE_LOCK(session, ret, + WT_WITH_TURTLE_LOCK(session, ret = __wt_turtle_update(session, key, value)); return (ret); } @@ -262,8 +259,17 @@ __wt_metadata_search(WT_SESSION_IMPL *session, const char *key, char **valuep) key, WT_META_TRACKING(session) ? "true" : "false", __metadata_turtle(key) ? "" : "not "); - if (__metadata_turtle(key)) - return (__wt_turtle_read(session, key, valuep)); + if (__metadata_turtle(key)) { + /* + * The returned value should only be set if ret is non-zero, but + * Coverity is convinced otherwise. The code path is used enough + * that Coverity complains a lot, add an error check to get some + * peace and quiet. + */ + if ((ret = __wt_turtle_read(session, key, valuep)) != 0) + __wt_free(session, *valuep); + return (ret); + } /* * All metadata reads are at read-uncommitted isolation. That's diff --git a/src/third_party/wiredtiger/src/meta/meta_track.c b/src/third_party/wiredtiger/src/meta/meta_track.c index 9655a0b26a7..460b615b267 100644 --- a/src/third_party/wiredtiger/src/meta/meta_track.c +++ b/src/third_party/wiredtiger/src/meta/meta_track.c @@ -286,7 +286,7 @@ __wt_meta_track_off(WT_SESSION_IMPL *session, bool need_sync, bool unroll) */ ckpt_session->txn.id = session->txn.id; F_SET(ckpt_session, WT_SESSION_LOCKED_METADATA); - WT_WITH_METADATA_LOCK(session, ret, + WT_WITH_METADATA_LOCK(session, WT_WITH_DHANDLE(ckpt_session, WT_SESSION_META_DHANDLE(session), ret = __wt_checkpoint(ckpt_session, NULL))); diff --git a/src/third_party/wiredtiger/src/meta/meta_turtle.c b/src/third_party/wiredtiger/src/meta/meta_turtle.c index ace0fabab48..5a089471059 100644 --- a/src/third_party/wiredtiger/src/meta/meta_turtle.c +++ b/src/third_party/wiredtiger/src/meta/meta_turtle.c @@ -47,7 +47,7 @@ __metadata_init(WT_SESSION_IMPL *session) * We're single-threaded, but acquire the schema lock regardless: the * lower level code checks that it is appropriately synchronized. */ - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_create(session, WT_METAFILE_URI, NULL)); return (ret); @@ -220,9 +220,8 @@ __wt_turtle_init(WT_SESSION_IMPL *session) /* Create the turtle file. */ WT_RET(__metadata_config(session, &metaconf)); - WT_WITH_TURTLE_LOCK(session, ret, - ret = __wt_turtle_update( - session, WT_METAFILE_URI, metaconf)); + WT_WITH_TURTLE_LOCK(session, ret = + __wt_turtle_update(session, WT_METAFILE_URI, metaconf)); WT_ERR(ret); } @@ -243,7 +242,7 @@ __wt_turtle_read(WT_SESSION_IMPL *session, const char *key, char **valuep) WT_DECL_ITEM(buf); WT_DECL_RET; WT_FSTREAM *fs; - bool exist, match; + bool exist; *valuep = NULL; @@ -259,22 +258,19 @@ __wt_turtle_read(WT_SESSION_IMPL *session, const char *key, char **valuep) __metadata_config(session, valuep) : WT_NOTFOUND); WT_RET(__wt_fopen(session, WT_METADATA_TURTLE, 0, WT_STREAM_READ, &fs)); - /* Search for the key. */ WT_ERR(__wt_scr_alloc(session, 512, &buf)); - for (match = false;;) { + + /* Search for the key. */ + do { WT_ERR(__wt_getline(session, fs, buf)); if (buf->size == 0) WT_ERR(WT_NOTFOUND); - if (strcmp(key, buf->data) == 0) - match = true; + } while (strcmp(key, buf->data) != 0); - /* Key matched: read the subsequent line for the value. */ - WT_ERR(__wt_getline(session, fs, buf)); - if (buf->size == 0) - WT_ERR(__wt_illegal_value(session, WT_METADATA_TURTLE)); - if (match) - break; - } + /* Key matched: read the subsequent line for the value. */ + WT_ERR(__wt_getline(session, fs, buf)); + if (buf->size == 0) + WT_ERR(WT_NOTFOUND); /* Copy the value for the caller. */ WT_ERR(__wt_strdup(session, buf->data, valuep)); @@ -284,7 +280,12 @@ err: WT_TRET(__wt_fclose(session, &fs)); if (ret != 0) __wt_free(session, *valuep); - return (ret); + + /* + * A file error or a missing key/value pair in the turtle file means + * something has gone horribly wrong -- we're done. + */ + return (ret == 0 ? 0 : __wt_illegal_value(session, WT_METADATA_TURTLE)); } /* @@ -323,5 +324,9 @@ __wt_turtle_update(WT_SESSION_IMPL *session, const char *key, const char *value) err: WT_TRET(__wt_fclose(session, &fs)); WT_TRET(__wt_remove_if_exists(session, WT_METADATA_TURTLE_SET, false)); - return (ret); + /* + * An error updating the turtle file means something has gone horribly + * wrong -- we're done. + */ + return (ret == 0 ? 0 : __wt_illegal_value(session, WT_METADATA_TURTLE)); } diff --git a/src/third_party/wiredtiger/src/os_common/filename.c b/src/third_party/wiredtiger/src/os_common/filename.c index 8b6c1269829..d5695f63d91 100644 --- a/src/third_party/wiredtiger/src/os_common/filename.c +++ b/src/third_party/wiredtiger/src/os_common/filename.c @@ -29,6 +29,7 @@ int __wt_nfilename( WT_SESSION_IMPL *session, const char *name, size_t namelen, char **path) { + WT_DECL_RET; size_t len; char *buf; @@ -39,16 +40,17 @@ __wt_nfilename( * the exists API which is used by the test utilities. */ if (session == NULL || __wt_absolute_path(name)) - WT_RET(__wt_strndup(session, name, namelen, path)); - else { - len = strlen(S2C(session)->home) + 1 + namelen + 1; - WT_RET(__wt_calloc(session, 1, len, &buf)); - snprintf(buf, len, "%s%s%.*s", S2C(session)->home, - __wt_path_separator(), (int)namelen, name); - *path = buf; - } + return (__wt_strndup(session, name, namelen, path)); + len = strlen(S2C(session)->home) + 1 + namelen + 1; + WT_RET(__wt_calloc(session, 1, len, &buf)); + WT_ERR(__wt_snprintf(buf, len, "%s%s%.*s", + S2C(session)->home, __wt_path_separator(), (int)namelen, name)); + *path = buf; return (0); + +err: __wt_free(session, buf); + return (ret); } /* @@ -72,6 +74,7 @@ __wt_remove_if_exists(WT_SESSION_IMPL *session, const char *name, bool durable) */ int __wt_copy_and_sync(WT_SESSION *wt_session, const char *from, const char *to) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_DECL_ITEM(tmp); WT_DECL_RET; diff --git a/src/third_party/wiredtiger/src/os_common/os_alloc.c b/src/third_party/wiredtiger/src/os_common/os_alloc.c index bfe5efafc8b..ef96ed09ea7 100644 --- a/src/third_party/wiredtiger/src/os_common/os_alloc.c +++ b/src/third_party/wiredtiger/src/os_common/os_alloc.c @@ -30,6 +30,7 @@ */ int __wt_calloc(WT_SESSION_IMPL *session, size_t number, size_t size, void *retp) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { void *p; @@ -282,6 +283,7 @@ __wt_strndup(WT_SESSION_IMPL *session, const void *str, size_t len, void *retp) */ void __wt_free_int(WT_SESSION_IMPL *session, const void *p_arg) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { void *p; diff --git a/src/third_party/wiredtiger/src/os_common/os_errno.c b/src/third_party/wiredtiger/src/os_common/os_errno.c index a8e56b7f1aa..7ac89536e79 100644 --- a/src/third_party/wiredtiger/src/os_common/os_errno.c +++ b/src/third_party/wiredtiger/src/os_common/os_errno.c @@ -44,7 +44,7 @@ __wt_strerror(WT_SESSION_IMPL *session, int error, char *errbuf, size_t errlen) * Fallback to a generic message. */ if (session == NULL && - snprintf(errbuf, errlen, "error return: %d", error) > 0) + __wt_snprintf(errbuf, errlen, "error return: %d", error) == 0) return (errbuf); if (session != NULL && __wt_buf_fmt( session, &session->err, "error return: %d", error) == 0) diff --git a/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c b/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c index 70a82007300..1670e97be45 100644 --- a/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c +++ b/src/third_party/wiredtiger/src/os_common/os_fs_inmemory.c @@ -515,8 +515,6 @@ __im_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session) WT_FILE_SYSTEM_INMEM *im_fs; WT_SESSION_IMPL *session; - WT_UNUSED(file_system); - session = (WT_SESSION_IMPL *)wt_session; im_fs = (WT_FILE_SYSTEM_INMEM *)file_system; diff --git a/src/third_party/wiredtiger/src/os_common/os_fstream.c b/src/third_party/wiredtiger/src/os_common/os_fstream.c index 5a368ea75e6..744da732d84 100644 --- a/src/third_party/wiredtiger/src/os_common/os_fstream.c +++ b/src/third_party/wiredtiger/src/os_common/os_fstream.c @@ -144,7 +144,7 @@ __fstream_printf( p = (char *)((uint8_t *)buf->mem + buf->size); WT_ASSERT(session, buf->memsize >= buf->size); space = buf->memsize - buf->size; - len = (size_t)vsnprintf(p, space, fmt, ap_copy); + WT_RET(__wt_vsnprintf_len_set(p, space, &len, fmt, ap_copy)); va_end(ap_copy); if (len < space) { diff --git a/src/third_party/wiredtiger/src/os_common/os_getopt.c b/src/third_party/wiredtiger/src/os_common/os_getopt.c index 0306ad1d79d..960776c3999 100644 --- a/src/third_party/wiredtiger/src/os_common/os_getopt.c +++ b/src/third_party/wiredtiger/src/os_common/os_getopt.c @@ -79,6 +79,7 @@ char *__wt_optarg; /* argument associated with option */ int __wt_getopt( const char *progname, int nargc, char * const *nargv, const char *ostr) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { static const char *place = EMSG; /* option letter processing */ const char *oli; /* option letter list index */ diff --git a/src/third_party/wiredtiger/src/os_common/os_strtouq.c b/src/third_party/wiredtiger/src/os_common/os_strtouq.c index 0ae604fc761..cb4da0de058 100644 --- a/src/third_party/wiredtiger/src/os_common/os_strtouq.c +++ b/src/third_party/wiredtiger/src/os_common/os_strtouq.c @@ -14,6 +14,7 @@ */ uint64_t __wt_strtouq(const char *nptr, char **endptr, int base) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { #if defined(HAVE_STRTOUQ) return (strtouq(nptr, endptr, base)); diff --git a/src/third_party/wiredtiger/src/os_posix/os_dir.c b/src/third_party/wiredtiger/src/os_posix/os_dir.c index 768a1324cd8..627278540d1 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_dir.c +++ b/src/third_party/wiredtiger/src/os_posix/os_dir.c @@ -28,8 +28,6 @@ __wt_posix_directory_list(WT_FILE_SYSTEM *file_system, int tret; char **entries; - WT_UNUSED(file_system); - session = (WT_SESSION_IMPL *)wt_session; *dirlistp = NULL; diff --git a/src/third_party/wiredtiger/src/os_posix/os_fs.c b/src/third_party/wiredtiger/src/os_posix/os_fs.c index c6272b5da52..bc8cbf67025 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_fs.c +++ b/src/third_party/wiredtiger/src/os_posix/os_fs.c @@ -575,7 +575,7 @@ __posix_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, WT_FILE_HANDLE_POSIX *pfh; WT_SESSION_IMPL *session; mode_t mode; - int f; + int advise_flag, f; WT_UNUSED(file_system); @@ -676,17 +676,24 @@ __posix_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, #if defined(HAVE_POSIX_FADVISE) /* - * Disable read-ahead on trees: it slows down random read workloads. + * If the user set an access pattern hint, call fadvise now. * Ignore fadvise when doing direct I/O, the kernel cache isn't * interesting. */ - if (!pfh->direct_io && file_type == WT_FS_OPEN_FILE_TYPE_DATA) { - WT_SYSCALL( - posix_fadvise(pfh->fd, 0, 0, POSIX_FADV_RANDOM), ret); + if (!pfh->direct_io && file_type == WT_FS_OPEN_FILE_TYPE_DATA && + LF_ISSET(WT_FS_OPEN_ACCESS_RAND | WT_FS_OPEN_ACCESS_SEQ)) { + advise_flag = 0; + if (LF_ISSET(WT_FS_OPEN_ACCESS_RAND)) + advise_flag = POSIX_FADV_RANDOM; + if (LF_ISSET(WT_FS_OPEN_ACCESS_SEQ)) + advise_flag = POSIX_FADV_SEQUENTIAL; + WT_SYSCALL(posix_fadvise(pfh->fd, 0, 0, advise_flag), ret); if (ret != 0) WT_ERR_MSG(session, ret, "%s: handle-open: posix_fadvise", name); } +#else + WT_UNUSED(advise_flag); #endif directory_open: @@ -746,8 +753,6 @@ __posix_terminate(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session) { WT_SESSION_IMPL *session; - WT_UNUSED(file_system); - session = (WT_SESSION_IMPL *)wt_session; __wt_free(session, file_system); diff --git a/src/third_party/wiredtiger/src/os_posix/os_getenv.c b/src/third_party/wiredtiger/src/os_posix/os_getenv.c index 7a086145cee..f779f90acee 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_getenv.c +++ b/src/third_party/wiredtiger/src/os_posix/os_getenv.c @@ -14,6 +14,7 @@ */ int __wt_getenv(WT_SESSION_IMPL *session, const char *variable, const char **envp) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { const char *temp; diff --git a/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c b/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c index d4ee532a2af..fe010b62305 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c +++ b/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c @@ -13,8 +13,7 @@ * Allocate and initialize a condition variable. */ int -__wt_cond_alloc(WT_SESSION_IMPL *session, - const char *name, bool is_signalled, WT_CONDVAR **condp) +__wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, WT_CONDVAR **condp) { WT_CONDVAR *cond; WT_DECL_RET; @@ -27,7 +26,7 @@ __wt_cond_alloc(WT_SESSION_IMPL *session, WT_ERR(pthread_cond_init(&cond->cond, NULL)); cond->name = name; - cond->waiters = is_signalled ? -1 : 0; + cond->waiters = 0; *condp = cond; return (0); @@ -42,8 +41,8 @@ err: __wt_free(session, cond); * out period expires, let the caller know. */ void -__wt_cond_wait_signal( - WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool *signalled) +__wt_cond_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, + uint64_t usecs, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled) { struct timespec ts; WT_DECL_RET; @@ -62,6 +61,23 @@ __wt_cond_wait_signal( WT_ERR(pthread_mutex_lock(&cond->mtx)); locked = true; + /* + * It's possible to race with threads waking us up. That's not a problem + * if there are multiple wakeups because the next wakeup will get us, or + * if we're only pausing for a short period. It's a problem if there's + * only a single wakeup, our waker is likely waiting for us to exit. + * After acquiring the mutex (so we're guaranteed to be awakened by any + * future wakeup call), optionally check if we're OK to keep running. + * This won't ensure our caller won't just loop and call us again, but + * at least it's not our fault. + * + * Assert we're not waiting longer than a second if not checking the + * run status. + */ + WT_ASSERT(session, run_func != NULL || usecs <= WT_MILLION); + if (run_func != NULL && !run_func(session)) + goto skipping; + if (usecs > 0) { __wt_epoch(session, &ts); ts.tv_sec += (time_t) @@ -81,7 +97,7 @@ __wt_cond_wait_signal( ret == ETIME || #endif ret == ETIMEDOUT) { - *signalled = false; +skipping: *signalled = false; ret = 0; } @@ -107,10 +123,13 @@ __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond) __wt_verbose(session, WT_VERB_MUTEX, "signal %s", cond->name); /* - * Our callers are often setting flags to cause a thread to exit. Add - * a barrier to ensure the flags are seen by the threads. + * Our callers often set flags to cause a thread to exit. Add a barrier + * to ensure exit flags are seen by the sleeping threads, otherwise we + * can wake up a thread, it immediately goes back to sleep, and we'll + * hang. Use a full barrier (we may not write before waiting on thread + * join). */ - WT_WRITE_BARRIER(); + WT_FULL_BARRIER(); /* * Fast path if we are in (or can enter), a state where the next waiter @@ -134,7 +153,7 @@ err: * __wt_cond_destroy -- * Destroy a condition variable. */ -int +void __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) { WT_CONDVAR *cond; @@ -142,11 +161,15 @@ __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) cond = *condp; if (cond == NULL) - return (0); + return; - ret = pthread_cond_destroy(&cond->cond); - WT_TRET(pthread_mutex_destroy(&cond->mtx)); - __wt_free(session, *condp); + if ((ret = pthread_cond_destroy(&cond->cond)) != 0) + WT_PANIC_MSG( + session, ret, "pthread_cond_destroy: %s", cond->name); - return (ret); + if ((ret = pthread_mutex_destroy(&cond->mtx)) != 0) + WT_PANIC_MSG( + session, ret, "pthread_mutex_destroy: %s", cond->name); + + __wt_free(session, *condp); } diff --git a/src/third_party/wiredtiger/src/os_posix/os_setvbuf.c b/src/third_party/wiredtiger/src/os_posix/os_setvbuf.c index d6107115eb3..ac3958be22f 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_setvbuf.c +++ b/src/third_party/wiredtiger/src/os_posix/os_setvbuf.c @@ -14,6 +14,7 @@ */ void __wt_stream_set_line_buffer(FILE *fp) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { /* * This function exists because MSVC doesn't support buffer sizes of 0 @@ -29,6 +30,7 @@ __wt_stream_set_line_buffer(FILE *fp) */ void __wt_stream_set_no_buffer(FILE *fp) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { (void)setvbuf(fp, NULL, _IONBF, 0); } diff --git a/src/third_party/wiredtiger/src/os_posix/os_sleep.c b/src/third_party/wiredtiger/src/os_posix/os_sleep.c index 8633b8d1ec0..2c60987ced7 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_sleep.c +++ b/src/third_party/wiredtiger/src/os_posix/os_sleep.c @@ -14,6 +14,7 @@ */ void __wt_sleep(uint64_t seconds, uint64_t micro_seconds) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { struct timeval t; diff --git a/src/third_party/wiredtiger/src/os_posix/os_snprintf.c b/src/third_party/wiredtiger/src/os_posix/os_snprintf.c new file mode 100644 index 00000000000..390e2e0334a --- /dev/null +++ b/src/third_party/wiredtiger/src/os_posix/os_snprintf.c @@ -0,0 +1,27 @@ +/*- + * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_vsnprintf_len_incr -- + * POSIX vsnprintf convenience function, incrementing the returned size. + */ +int +__wt_vsnprintf_len_incr( + char *buf, size_t size, size_t *retsizep, const char *fmt, va_list ap) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) +{ + WT_DECL_RET; + + if ((ret = vsnprintf(buf, size, fmt, ap)) >= 0) { + *retsizep += (size_t)ret; + return (0); + } + return (__wt_errno()); +} diff --git a/src/third_party/wiredtiger/src/os_posix/os_thread.c b/src/third_party/wiredtiger/src/os_posix/os_thread.c index e57a308c9b0..18e4c347436 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_thread.c +++ b/src/third_party/wiredtiger/src/os_posix/os_thread.c @@ -18,6 +18,13 @@ __wt_thread_create(WT_SESSION_IMPL *session, { WT_DECL_RET; + /* + * Creating a thread isn't a memory barrier, but WiredTiger commonly + * sets flags and or state and then expects worker threads to start. + * Include a barrier to ensure safety in those cases. + */ + WT_FULL_BARRIER(); + /* Spawn a new thread of control. */ WT_SYSCALL_RETRY(pthread_create(tidret, NULL, func, arg), ret); if (ret == 0) @@ -34,6 +41,13 @@ __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) { WT_DECL_RET; + /* + * Joining a thread isn't a memory barrier, but WiredTiger commonly + * sets flags and or state and then expects worker threads to halt. + * Include a barrier to ensure safety in those cases. + */ + WT_FULL_BARRIER(); + WT_SYSCALL(pthread_join(tid, NULL), ret); if (ret == 0) return (0); @@ -45,8 +59,9 @@ __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) * __wt_thread_id -- * Fill in a printable version of the process and thread IDs. */ -void +int __wt_thread_id(char *buf, size_t buflen) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { pthread_t self; @@ -56,10 +71,10 @@ __wt_thread_id(char *buf, size_t buflen) */ self = pthread_self(); #ifdef __sun - (void)snprintf(buf, buflen, - "%" PRIuMAX ":%u", (uintmax_t)getpid(), self); + return (__wt_snprintf(buf, buflen, + "%" PRIuMAX ":%u", (uintmax_t)getpid(), self)); #else - (void)snprintf(buf, buflen, - "%" PRIuMAX ":%p", (uintmax_t)getpid(), (void *)self); + return (__wt_snprintf(buf, buflen, + "%" PRIuMAX ":%p", (uintmax_t)getpid(), (void *)self)); #endif } diff --git a/src/third_party/wiredtiger/src/os_posix/os_time.c b/src/third_party/wiredtiger/src/os_posix/os_time.c index 719e214696b..6f150ee8ffe 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_time.c +++ b/src/third_party/wiredtiger/src/os_posix/os_time.c @@ -14,6 +14,7 @@ */ void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_DECL_RET; diff --git a/src/third_party/wiredtiger/src/os_posix/os_yield.c b/src/third_party/wiredtiger/src/os_posix/os_yield.c index 052a46940b7..f7c43aae746 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_yield.c +++ b/src/third_party/wiredtiger/src/os_posix/os_yield.c @@ -14,6 +14,15 @@ */ void __wt_yield(void) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { + /* + * Yielding the processor isn't documented as a memory barrier, and it's + * a reasonable expectation to have. There's no reason not to explicitly + * include a barrier since we're giving up the CPU, and ensures callers + * aren't ever surprised. + */ + WT_FULL_BARRIER(); + sched_yield(); } diff --git a/src/third_party/wiredtiger/src/os_win/os_dir.c b/src/third_party/wiredtiger/src/os_win/os_dir.c index f024d131387..47d4f95b793 100644 --- a/src/third_party/wiredtiger/src/os_win/os_dir.c +++ b/src/third_party/wiredtiger/src/os_win/os_dir.c @@ -30,8 +30,6 @@ __wt_win_directory_list(WT_FILE_SYSTEM *file_system, uint32_t count; char *dir_copy, **entries; - WT_UNUSED(file_system); - session = (WT_SESSION_IMPL *)wt_session; *dirlistp = NULL; diff --git a/src/third_party/wiredtiger/src/os_win/os_fs.c b/src/third_party/wiredtiger/src/os_win/os_fs.c index 7ab7178114b..5cf47ea5763 100644 --- a/src/third_party/wiredtiger/src/os_win/os_fs.c +++ b/src/third_party/wiredtiger/src/os_win/os_fs.c @@ -16,7 +16,6 @@ static int __win_fs_exist(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *name, bool *existp) { - WT_DECL_RET; WT_DECL_ITEM(name_wide); WT_SESSION_IMPL *session; @@ -43,8 +42,8 @@ __win_fs_remove(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, const char *name, uint32_t flags) { DWORD windows_error; - WT_DECL_RET; WT_DECL_ITEM(name_wide); + WT_DECL_RET; WT_SESSION_IMPL *session; WT_UNUSED(file_system); @@ -88,22 +87,19 @@ __win_fs_rename(WT_FILE_SYSTEM *file_system, WT_ERR(__wt_to_utf16_string(session, to, &to_wide)); /* - * Check if file exists since Windows does not override the file if - * it exists. + * We want an atomic rename, but that's not guaranteed by MoveFileExW + * (or by any MSDN API). Don't set the MOVEFILE_COPY_ALLOWED flag to + * prevent the system from falling back to a copy and delete process. + * Do set the MOVEFILE_WRITE_THROUGH flag so the window is as small + * as possible, just in case. WiredTiger renames are done in a single + * directory and we expect that to be an atomic metadata update on any + * modern filesystem. */ - if (GetFileAttributesW(to_wide->data) != INVALID_FILE_ATTRIBUTES) - if (DeleteFileW(to_wide->data) == FALSE) { - windows_error = __wt_getlasterror(); - __wt_errx(session, - "%s: file-rename: DeleteFileW: %s", - to, __wt_formatmessage(session, windows_error)); - WT_ERR(__wt_map_windows_error(windows_error)); - } - - if (MoveFileW(from_wide->data, to_wide->data) == FALSE) { + if (MoveFileExW(from_wide->data, to_wide->data, + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH) == FALSE) { windows_error = __wt_getlasterror(); __wt_errx(session, - "%s to %s: file-rename: MoveFileW: %s", + "%s to %s: file-rename: MoveFileExW: %s", from, to, __wt_formatmessage(session, windows_error)); WT_ERR(__wt_map_windows_error(windows_error)); } @@ -521,10 +517,16 @@ __win_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session, FLD_ISSET(conn->txn_logsync, WT_LOG_DSYNC)) f |= FILE_FLAG_WRITE_THROUGH; - /* Disable read-ahead on trees: it slows down random read workloads. */ - if (file_type == WT_FS_OPEN_FILE_TYPE_DATA) + /* If the user indicated a random workload, disable read-ahead. */ + if (file_type == WT_FS_OPEN_FILE_TYPE_DATA && + LF_ISSET(WT_FS_OPEN_ACCESS_RAND)) f |= FILE_FLAG_RANDOM_ACCESS; + /* If the user indicated a sequential workload, set that. */ + if (file_type == WT_FS_OPEN_FILE_TYPE_DATA && + LF_ISSET(WT_FS_OPEN_ACCESS_SEQ)) + f |= FILE_FLAG_SEQUENTIAL_SCAN; + win_fh->filehandle = CreateFileW(name_wide->data, desired_access, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, dwCreationDisposition, f, NULL); diff --git a/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c b/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c index 79c62ccd7f2..c1b9f509d33 100644 --- a/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c +++ b/src/third_party/wiredtiger/src/os_win/os_mtx_cond.c @@ -13,8 +13,7 @@ * Allocate and initialize a condition variable. */ int -__wt_cond_alloc(WT_SESSION_IMPL *session, - const char *name, bool is_signalled, WT_CONDVAR **condp) +__wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, WT_CONDVAR **condp) { WT_CONDVAR *cond; @@ -26,7 +25,7 @@ __wt_cond_alloc(WT_SESSION_IMPL *session, InitializeConditionVariable(&cond->cond); cond->name = name; - cond->waiters = is_signalled ? -1 : 0; + cond->waiters = 0; *condp = cond; return (0); @@ -38,8 +37,8 @@ __wt_cond_alloc(WT_SESSION_IMPL *session, * out period expires, let the caller know. */ void -__wt_cond_wait_signal( - WT_SESSION_IMPL *session, WT_CONDVAR *cond, uint64_t usecs, bool *signalled) +__wt_cond_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, + uint64_t usecs, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled) { BOOL sleepret; DWORD milliseconds, windows_error; @@ -59,8 +58,26 @@ __wt_cond_wait_signal( EnterCriticalSection(&cond->mtx); locked = true; + /* + * It's possible to race with threads waking us up. That's not a problem + * if there are multiple wakeups because the next wakeup will get us, or + * if we're only pausing for a short period. It's a problem if there's + * only a single wakeup, our waker is likely waiting for us to exit. + * After acquiring the mutex (so we're guaranteed to be awakened by any + * future wakeup call), optionally check if we're OK to keep running. + * This won't ensure our caller won't just loop and call us again, but + * at least it's not our fault. + * + * Assert we're not waiting longer than a second if not checking the + * run status. + */ + WT_ASSERT(session, run_func != NULL || usecs <= WT_MILLION); + + if (run_func != NULL && !run_func(session)) + goto skipping; + if (usecs > 0) { - milliseconds64 = usecs / 1000; + milliseconds64 = usecs / WT_THOUSAND; /* * Check for 32-bit unsigned integer overflow @@ -90,7 +107,7 @@ __wt_cond_wait_signal( if (sleepret == 0) { windows_error = __wt_getlasterror(); if (windows_error == ERROR_TIMEOUT) { - *signalled = false; +skipping: *signalled = false; sleepret = 1; } } @@ -117,17 +134,17 @@ void __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond) { WT_DECL_RET; - bool locked; - - locked = false; __wt_verbose(session, WT_VERB_MUTEX, "signal %s", cond->name); /* - * Our callers are often setting flags to cause a thread to exit. Add - * a barrier to ensure the flags are seen by the threads. + * Our callers often set flags to cause a thread to exit. Add a barrier + * to ensure exit flags are seen by the sleeping threads, otherwise we + * can wake up a thread, it immediately goes back to sleep, and we'll + * hang. Use a full barrier (we may not write before waiting on thread + * join). */ - WT_WRITE_BARRIER(); + WT_FULL_BARRIER(); /* * Fast path if we are in (or can enter), a state where the next waiter @@ -146,18 +163,16 @@ __wt_cond_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond) * __wt_cond_destroy -- * Destroy a condition variable. */ -int +void __wt_cond_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) { WT_CONDVAR *cond; cond = *condp; if (cond == NULL) - return (0); + return; /* Do nothing to delete Condition Variable */ DeleteCriticalSection(&cond->mtx); __wt_free(session, *condp); - - return (0); } diff --git a/src/third_party/wiredtiger/src/os_win/os_snprintf.c b/src/third_party/wiredtiger/src/os_win/os_snprintf.c index a6056ff9342..f3025b12a60 100644 --- a/src/third_party/wiredtiger/src/os_win/os_snprintf.c +++ b/src/third_party/wiredtiger/src/os_win/os_snprintf.c @@ -8,17 +8,47 @@ #include "wt_internal.h" -_Check_return_opt_ int __cdecl _wt_snprintf( - _Out_writes_(_MaxCount) char * _DstBuf, - _In_ size_t _MaxCount, - _In_z_ _Printf_format_string_ const char * _Format, ...) +/* + * __wt_vsnprintf_len_incr -- + * POSIX vsnprintf convenience function, incrementing the returned size. + */ +int +__wt_vsnprintf_len_incr( + char *buf, size_t size, size_t *retsizep, const char *fmt, va_list ap) { - va_list args; - WT_DECL_RET; + int len; + + /* + * WiredTiger calls with length 0 to get the needed buffer size. Call + * the count only version in this case, _vsnprintf_s will invoke the + * invalid parameter handler if count is less than or equal to zero. + */ + if (size == 0) { + *retsizep += (size_t)_vscprintf(fmt, ap); + return (0); + } + + /* + * Additionally, the invalid parameter handler is invoked if buffer or + * format is a NULL pointer. + */ + if (buf == NULL || fmt == NULL) + return (EINVAL); + + /* + * If the storage required to store the data and a terminating null + * exceeds size, the invalid parameter handler is invoked, unless + * count is _TRUNCATE, in which case as much of the string as will + * fit in the buffer is written and -1 returned. + */ + if ((len = _vsnprintf_s(buf, size, _TRUNCATE, fmt, ap)) >= 0) { + *retsizep += (size_t)len; + return (0); + } - va_start(args, _Format); - ret = _wt_vsnprintf(_DstBuf, _MaxCount, _Format, args); - va_end(args); + /* Return the buffer size required. */ + if (len == -1) + *retsizep += (size_t)_vscprintf(fmt, ap); - return (ret); + return (0); } diff --git a/src/third_party/wiredtiger/src/os_win/os_thread.c b/src/third_party/wiredtiger/src/os_win/os_thread.c index a34dff776b6..4c8f212bb4f 100644 --- a/src/third_party/wiredtiger/src/os_win/os_thread.c +++ b/src/third_party/wiredtiger/src/os_win/os_thread.c @@ -16,6 +16,13 @@ int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) { + /* + * Creating a thread isn't a memory barrier, but WiredTiger commonly + * sets flags and or state and then expects worker threads to start. + * Include a barrier to ensure safety in those cases. + */ + WT_FULL_BARRIER(); + /* Spawn a new thread of control. */ *tidret = (HANDLE)_beginthreadex(NULL, 0, func, arg, 0, NULL); if (*tidret != 0) @@ -33,6 +40,13 @@ __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) { DWORD windows_error; + /* + * Joining a thread isn't a memory barrier, but WiredTiger commonly + * sets flags and or state and then expects worker threads to halt. + * Include a barrier to ensure safety in those cases. + */ + WT_FULL_BARRIER(); + if ((windows_error = WaitForSingleObject(tid, INFINITE)) != WAIT_OBJECT_0) { if (windows_error == WAIT_FAILED) @@ -58,10 +72,10 @@ __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) * __wt_thread_id -- * Fill in a printable version of the process and thread IDs. */ -void +int __wt_thread_id(char *buf, size_t buflen) { - (void)snprintf(buf, buflen, + return (__wt_snprintf(buf, buflen, "%" PRIu64 ":%" PRIu64, - (uint64_t)GetCurrentProcessId(), (uint64_t)GetCurrentThreadId); + (uint64_t)GetCurrentProcessId(), (uint64_t)GetCurrentThreadId)); } diff --git a/src/third_party/wiredtiger/src/os_win/os_utf8.c b/src/third_party/wiredtiger/src/os_win/os_utf8.c index f7d11c24f03..ccd8321aecf 100644 --- a/src/third_party/wiredtiger/src/os_win/os_utf8.c +++ b/src/third_party/wiredtiger/src/os_win/os_utf8.c @@ -18,7 +18,6 @@ __wt_to_utf16_string( { DWORD windows_error; int bufferSize; - WT_DECL_RET; bufferSize = MultiByteToWideChar(CP_UTF8, 0, utf8, -1, NULL, 0); windows_error = __wt_getlasterror(); @@ -55,7 +54,6 @@ __wt_to_utf8_string( { DWORD windows_error; int bufferSize; - WT_DECL_RET; bufferSize = WideCharToMultiByte( CP_UTF8, 0, wide, -1, NULL, 0, NULL, NULL); diff --git a/src/third_party/wiredtiger/src/os_win/os_vsnprintf.c b/src/third_party/wiredtiger/src/os_win/os_vsnprintf.c deleted file mode 100644 index 63f96e79d5b..00000000000 --- a/src/third_party/wiredtiger/src/os_win/os_vsnprintf.c +++ /dev/null @@ -1,41 +0,0 @@ -/*- - * Copyright (c) 2014-2016 MongoDB, Inc. - * Copyright (c) 2008-2014 WiredTiger, Inc. - * All rights reserved. - * - * See the file LICENSE for redistribution information. - */ - -#include "wt_internal.h" - -_Check_return_opt_ int __cdecl _wt_vsnprintf( - _Out_writes_(_MaxCount) char * _DstBuf, - _In_ size_t _MaxCount, - _In_z_ _Printf_format_string_ const char * _Format, - va_list _ArgList) -{ - int len; - - /* - * WiredTiger will call with length 0 to get the needed buffer size - * We call the count only version in this case since vsnprintf_s assumes - * length is greater than zero or else it triggers the invalid_parameter - * handler. - */ - if (_MaxCount == 0) { - return _vscprintf(_Format, _ArgList); - } - - len = (size_t)_vsnprintf_s( - _DstBuf, _MaxCount, _TRUNCATE, _Format, _ArgList); - - /* - * The MSVC implementation returns -1 on truncation instead of what - * it would have written. We could let callers iteratively grow the - * buffer, or just ask us how big a buffer they would like. - */ - if (len == -1) - len = _vscprintf(_Format, _ArgList) + 1; - - return (len); -} diff --git a/src/third_party/wiredtiger/src/os_win/os_yield.c b/src/third_party/wiredtiger/src/os_win/os_yield.c index aab1559e072..038f2efe162 100644 --- a/src/third_party/wiredtiger/src/os_win/os_yield.c +++ b/src/third_party/wiredtiger/src/os_win/os_yield.c @@ -15,5 +15,13 @@ void __wt_yield(void) { + /* + * Yielding the processor isn't documented as a memory barrier, and it's + * a reasonable expectation to have. There's no reason not to explicitly + * include a barrier since we're giving up the CPU, and ensures callers + * aren't ever surprised. + */ + WT_FULL_BARRIER(); + SwitchToThread(); } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_track.c b/src/third_party/wiredtiger/src/reconcile/rec_track.c index 10a64fdf116..5bf425b1b21 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_track.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_track.c @@ -35,7 +35,6 @@ __ovfl_discard_verbose( { WT_CELL_UNPACK *unpack, _unpack; WT_DECL_ITEM(tmp); - WT_DECL_RET; WT_RET(__wt_scr_alloc(session, 512, &tmp)); @@ -50,7 +49,7 @@ __ovfl_discard_verbose( __wt_addr_string(session, unpack->data, unpack->size, tmp)); __wt_scr_free(session, &tmp); - return (ret); + return (0); } #if 0 @@ -83,7 +82,6 @@ static int __ovfl_discard_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) { WT_CELL **cellp; - WT_DECL_RET; WT_OVFL_TRACK *track; uint32_t i; @@ -101,7 +99,7 @@ __ovfl_discard_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) __wt_free(session, track->discard); track->discard_entries = track->discard_allocated = 0; - return (ret); + return (0); } /* @@ -170,7 +168,6 @@ __ovfl_reuse_verbose(WT_SESSION_IMPL *session, WT_PAGE *page, WT_OVFL_REUSE *reuse, const char *tag) { WT_DECL_ITEM(tmp); - WT_DECL_RET; WT_RET(__wt_scr_alloc(session, 64, &tmp)); @@ -188,7 +185,7 @@ __ovfl_reuse_verbose(WT_SESSION_IMPL *session, WT_MIN(reuse->value_size, 40), (char *)WT_OVFL_REUSE_VALUE(reuse)); __wt_scr_free(session, &tmp); - return (ret); + return (0); } #if 0 @@ -568,7 +565,6 @@ __ovfl_txnc_verbose(WT_SESSION_IMPL *session, WT_PAGE *page, WT_OVFL_TXNC *txnc, const char *tag) { WT_DECL_ITEM(tmp); - WT_DECL_RET; WT_RET(__wt_scr_alloc(session, 64, &tmp)); @@ -583,7 +579,7 @@ __ovfl_txnc_verbose(WT_SESSION_IMPL *session, WT_MIN(txnc->value_size, 40), (char *)WT_OVFL_TXNC_VALUE(txnc)); __wt_scr_free(session, &tmp); - return (ret); + return (0); } #if 0 @@ -879,9 +875,9 @@ __wt_ovfl_track_wrapup(WT_SESSION_IMPL *session, WT_PAGE *page) WT_RET(__ovfl_reuse_wrapup(session, page)); if (track->ovfl_txnc[0] != NULL) { - __wt_writelock(session, S2BT(session)->ovfl_lock); + __wt_writelock(session, &S2BT(session)->ovfl_lock); ret = __ovfl_txnc_wrapup(session, page); - __wt_writeunlock(session, S2BT(session)->ovfl_lock); + __wt_writeunlock(session, &S2BT(session)->ovfl_lock); } return (ret); } @@ -907,9 +903,9 @@ __wt_ovfl_track_wrapup_err(WT_SESSION_IMPL *session, WT_PAGE *page) WT_RET(__ovfl_reuse_wrapup_err(session, page)); if (track->ovfl_txnc[0] != NULL) { - __wt_writelock(session, S2BT(session)->ovfl_lock); + __wt_writelock(session, &S2BT(session)->ovfl_lock); ret = __ovfl_txnc_wrapup(session, page); - __wt_writeunlock(session, S2BT(session)->ovfl_lock); + __wt_writeunlock(session, &S2BT(session)->ovfl_lock); } return (ret); } diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index fe288beed15..e59d9796352 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -26,6 +26,11 @@ typedef struct { uint32_t flags; /* Caller's configuration */ WT_ITEM disk_image; /* Temporary disk-image buffer */ + /* + * Temporary buffer used to write out a disk image when managing two + * chunks worth of data in memory + */ + WT_ITEM *interim_buf; /* * Track start/stop write generation to decide if all changes to the @@ -43,6 +48,10 @@ typedef struct { /* Track the page's maximum transaction ID. */ uint64_t max_txn; + /* Track if all updates were skipped. */ + uint64_t update_cnt; + uint64_t update_skip_cnt; + /* * When we can't mark the page clean (for example, checkpoint found some * uncommitted updates), there's a leave-dirty flag. @@ -123,6 +132,7 @@ typedef struct { * repeatedly split a packed page. */ uint32_t split_size; /* Split page size */ + uint32_t min_split_size; /* Minimum split page size */ /* * The problem with splits is we've done a lot of work by the time we @@ -147,16 +157,6 @@ typedef struct { */ size_t offset; /* Split's first byte */ - /* - * The recno and entries fields are the starting record number - * of the split chunk (for column-store splits), and the number - * of entries in the split chunk. These fields are used both - * to write the split chunk, and to create a new internal page - * to reference the split pages. - */ - uint64_t recno; /* Split's starting record */ - uint32_t entries; /* Split's entries */ - WT_ADDR addr; /* Split's written location */ uint32_t size; /* Split's size */ uint32_t checksum; /* Split's checksum */ @@ -178,39 +178,42 @@ typedef struct { size_t supd_allocated; /* + * While reconciling pages, at any given time, we maintain two + * split chunks in the memory to be written out as pages. As we + * get to the last two chunks, if the last one turns out to be + * smaller than the minimum split size, we go back into the + * penultimate chunk and split at this minimum split size + * boundary. This moves some data from the penultimate chunk to + * the last chunk, hence increasing the size of the last page + * written without decreasing the penultimate page size beyond + * the minimum split size. For this reason, we maintain both a + * maximum split percentage boundary and a minimum split + * percentage boundary. + * + * The recno and entries fields are the starting record number + * of the split chunk (for column-store splits), and the number + * of entries in the split chunk. These fields are used both to + * write the split chunk, and to create a new internal page to + * reference the split pages. + * * The key for a row-store page; no column-store key is needed * because the page's recno, stored in the recno field, is the * column-store key. */ - WT_ITEM key; /* Promoted row-store key */ + uint32_t max_bnd_entries; + uint64_t max_bnd_recno; + WT_ITEM max_bnd_key; + + size_t min_bnd_offset; + uint32_t min_bnd_entries; + uint64_t min_bnd_recno; + WT_ITEM min_bnd_key; } *bnd; /* Saved boundaries */ uint32_t bnd_next; /* Next boundary slot */ uint32_t bnd_next_max; /* Maximum boundary slots used */ size_t bnd_entries; /* Total boundary slots */ size_t bnd_allocated; /* Bytes allocated */ - /* - * We track the total number of page entries copied into split chunks - * so we can easily figure out how many entries in the current split - * chunk. - */ - uint32_t total_entries; /* Total entries in splits */ - - /* - * And there's state information as to where in this process we are: - * (1) tracking split boundaries because we can still fit more split - * chunks into the maximum page size, (2) tracking the maximum page - * size boundary because we can't fit any more split chunks into the - * maximum page size, (3) not performing boundary checks because it's - * either not useful with the current page size configuration, or - * because we've already been forced to split. - */ - enum { SPLIT_BOUNDARY=0, /* Next: a split page boundary */ - SPLIT_MAX=1, /* Next: the maximum page boundary */ - SPLIT_TRACKING_OFF=2, /* No boundary checks */ - SPLIT_TRACKING_RAW=3 } /* Underlying compression decides */ - bnd_state; - /* * We track current information about the current record number, the * number of entries copied into the temporary buffer, where we are @@ -222,6 +225,8 @@ typedef struct { uint32_t entries; /* Current number of entries */ uint8_t *first_free; /* Current first free byte */ size_t space_avail; /* Remaining space in this chunk */ + /* Remaining space in this chunk to put a minimum size boundary */ + size_t min_space_avail; /* * Saved update list, supporting the WT_EVICT_UPDATE_RESTORE and @@ -243,15 +248,14 @@ typedef struct { /* * WT_DICTIONARY -- - * We optionally build a dictionary of row-store values for leaf - * pages. Where two value cells are identical, only write the value - * once, the second and subsequent copies point to the original cell. - * The dictionary is fixed size, but organized in a skip-list to make - * searches faster. + * We optionally build a dictionary of values for leaf pages. Where + * two value cells are identical, only write the value once, the second + * and subsequent copies point to the original cell. The dictionary is + * fixed size, but organized in a skip-list to make searches faster. */ struct __rec_dictionary { uint64_t hash; /* Hash value */ - void *cell; /* Matching cell */ + uint32_t offset; /* Matching cell */ u_int depth; /* Skiplist */ WT_DICTIONARY *next[0]; @@ -289,6 +293,13 @@ typedef struct { uint32_t tested_ref_state; /* Debugging information */ } WT_RECONCILE; +#define WT_CROSSING_MIN_BND(r, next_len) \ + ((r)->bnd[(r)->bnd_next].min_bnd_offset == 0 && \ + (next_len) > (r)->min_space_avail) +#define WT_CROSSING_SPLIT_BND(r, next_len) ((next_len) > (r)->space_avail) +#define WT_CHECK_CROSSING_BND(r, next_len) \ + (WT_CROSSING_MIN_BND(r, next_len) || WT_CROSSING_SPLIT_BND(r, next_len)) + static void __rec_bnd_cleanup(WT_SESSION_IMPL *, WT_RECONCILE *, bool); static void __rec_cell_build_addr(WT_SESSION_IMPL *, WT_RECONCILE *, const void *, size_t, u_int, uint64_t); @@ -310,6 +321,7 @@ static int __rec_col_var(WT_SESSION_IMPL *, static int __rec_col_var_helper(WT_SESSION_IMPL *, WT_RECONCILE *, WT_SALVAGE_COOKIE *, WT_ITEM *, bool, uint8_t, uint64_t); static int __rec_destroy_session(WT_SESSION_IMPL *); +static uint32_t __rec_min_split_page_size(WT_BTREE *, uint32_t); static int __rec_root_write(WT_SESSION_IMPL *, WT_PAGE *, uint32_t); static int __rec_row_int(WT_SESSION_IMPL *, WT_RECONCILE *, WT_PAGE *); static int __rec_row_leaf(WT_SESSION_IMPL *, @@ -319,7 +331,6 @@ static int __rec_row_leaf_insert( static int __rec_row_merge(WT_SESSION_IMPL *, WT_RECONCILE *, WT_PAGE *); static int __rec_split_col(WT_SESSION_IMPL *, WT_RECONCILE *, WT_PAGE *); static int __rec_split_discard(WT_SESSION_IMPL *, WT_PAGE *); -static int __rec_split_fixup(WT_SESSION_IMPL *, WT_RECONCILE *); static int __rec_split_row(WT_SESSION_IMPL *, WT_RECONCILE *, WT_PAGE *); static int __rec_split_row_promote( WT_SESSION_IMPL *, WT_RECONCILE *, WT_ITEM *, uint8_t); @@ -327,9 +338,10 @@ static int __rec_split_write(WT_SESSION_IMPL *, WT_RECONCILE *, WT_BOUNDARY *, WT_ITEM *, bool); static int __rec_update_las( WT_SESSION_IMPL *, WT_RECONCILE *, uint32_t, WT_BOUNDARY *); +static int __rec_write_check_complete(WT_SESSION_IMPL *, WT_RECONCILE *); static int __rec_write_init(WT_SESSION_IMPL *, WT_REF *, uint32_t, WT_SALVAGE_COOKIE *, void *); -static int __rec_write_status(WT_SESSION_IMPL *, WT_RECONCILE *, WT_PAGE *); +static void __rec_write_page_status(WT_SESSION_IMPL *, WT_RECONCILE *); static int __rec_write_wrapup(WT_SESSION_IMPL *, WT_RECONCILE *, WT_PAGE *); static int __rec_write_wrapup_err( WT_SESSION_IMPL *, WT_RECONCILE *, WT_PAGE *); @@ -345,8 +357,8 @@ static void __rec_dictionary_reset(WT_RECONCILE *); * Reconcile an in-memory page into its on-disk format, and write it. */ int -__wt_reconcile(WT_SESSION_IMPL *session, - WT_REF *ref, WT_SALVAGE_COOKIE *salvage, uint32_t flags) +__wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, + WT_SALVAGE_COOKIE *salvage, uint32_t flags, bool *lookaside_retryp) { WT_DECL_RET; WT_PAGE *page; @@ -356,6 +368,8 @@ __wt_reconcile(WT_SESSION_IMPL *session, page = ref->page; mod = page->modify; + if (lookaside_retryp != NULL) + *lookaside_retryp = false; __wt_verbose(session, WT_VERB_RECONCILE, "%s", __wt_page_type_string(page->type)); @@ -372,7 +386,7 @@ __wt_reconcile(WT_SESSION_IMPL *session, * In-memory splits: reconciliation of an internal page cannot handle * a child page splitting during the reconciliation. */ - __wt_writelock(session, &page->page_lock); + WT_PAGE_LOCK(session, page); oldest_id = __wt_txn_oldest_id(session); if (LF_ISSET(WT_EVICTING)) @@ -391,7 +405,7 @@ __wt_reconcile(WT_SESSION_IMPL *session, /* Initialize the reconciliation structure for each new run. */ if ((ret = __rec_write_init( session, ref, flags, salvage, &session->reconcile)) != 0) { - __wt_writeunlock(session, &page->page_lock); + WT_PAGE_UNLOCK(session, page); return (ret); } r = session->reconcile; @@ -421,18 +435,26 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_ILLEGAL_VALUE_SET(session); } - /* Get the final status for the reconciliation. */ + /* Checks for a successful reconciliation. */ if (ret == 0) - ret = __rec_write_status(session, r, page); + ret = __rec_write_check_complete(session, r); /* Wrap up the page reconciliation. */ - if (ret == 0) - ret = __rec_write_wrapup(session, r, page); + if (ret == 0 && (ret = __rec_write_wrapup(session, r, page)) == 0) + __rec_write_page_status(session, r); else WT_TRET(__rec_write_wrapup_err(session, r, page)); /* Release the reconciliation lock. */ - __wt_writeunlock(session, &page->page_lock); + WT_PAGE_UNLOCK(session, page); + + /* + * If our caller can configure lookaside table reconciliation, flag if + * that's worth trying. The lookaside table doesn't help if we skipped + * updates, it can only help with older readers preventing eviction. + */ + if (lookaside_retryp != NULL && r->update_cnt == r->update_skip_cnt) + *lookaside_retryp = true; /* Update statistics. */ WT_STAT_CONN_INCR(session, rec_pages); @@ -535,17 +557,14 @@ __rec_las_checkpoint_test(WT_SESSION_IMPL *session, WT_RECONCILE *r) } /* - * __rec_write_status -- - * Return the final status for reconciliation. + * __rec_write_check_complete -- + * Check that reconciliation should complete */ static int -__rec_write_status(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) +__rec_write_check_complete(WT_SESSION_IMPL *session, WT_RECONCILE *r) { - WT_BTREE *btree; - WT_PAGE_MODIFY *mod; - - btree = S2BT(session); - mod = page->modify; + WT_BOUNDARY *bnd; + size_t i; /* * If we have used the lookaside table, check for a lookaside table and @@ -554,6 +573,37 @@ __rec_write_status(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) if (r->cache_write_lookaside && __rec_las_checkpoint_test(session, r)) return (EBUSY); + /* + * If we are doing update/restore based eviction, confirm part of the + * page is being discarded, or at least 10% of the updates won't have + * to be re-instantiated. Otherwise, it isn't progress, don't bother. + */ + if (F_ISSET(r, WT_EVICT_UPDATE_RESTORE)) { + for (bnd = r->bnd, i = 0; i < r->bnd_entries; ++bnd, ++i) + if (bnd->supd == NULL) + break; + if (i == r->bnd_entries && + r->update_cnt / 10 >= r->update_skip_cnt) + return (EBUSY); + } + return (0); +} + +/* + * __rec_write_page_status -- + * Set the page status after reconciliation. + */ +static void +__rec_write_page_status(WT_SESSION_IMPL *session, WT_RECONCILE *r) +{ + WT_BTREE *btree; + WT_PAGE *page; + WT_PAGE_MODIFY *mod; + + btree = S2BT(session); + page = r->page; + mod = page->modify; + /* * Set the page's status based on whether or not we cleaned the page. */ @@ -612,8 +662,6 @@ __rec_write_status(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) else WT_ASSERT(session, !F_ISSET(r, WT_EVICTING)); } - - return (0); } /* @@ -675,6 +723,7 @@ __rec_root_write(WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t flags) * pages in memory; it's not needed here, asserted for safety. */ WT_ASSERT(session, mod->mod_multi[i].supd == NULL); + WT_ASSERT(session, mod->mod_multi[i].disk_image == NULL); WT_ERR(__wt_multi_to_ref(session, next, &mod->mod_multi[i], &pindex->index[i], NULL, false)); @@ -700,7 +749,7 @@ __rec_root_write(WT_SESSION_IMPL *session, WT_PAGE *page, uint32_t flags) * Fake up a reference structure, and write the next root page. */ __wt_root_ref_init(&fake_ref, next, page->type == WT_PAGE_COL_INT); - return (__wt_reconcile(session, &fake_ref, NULL, flags)); + return (__wt_reconcile(session, &fake_ref, NULL, flags, NULL)); err: __wt_page_out(session, &next); return (ret); @@ -841,6 +890,9 @@ __rec_write_init(WT_SESSION_IMPL *session, /* Track the page's maximum transaction ID. */ r->max_txn = WT_TXN_NONE; + /* Track if all updates were skipped. */ + r->update_cnt = r->update_skip_cnt = 0; + /* Track if the page can be marked clean. */ r->leave_dirty = false; @@ -923,6 +975,7 @@ __rec_destroy(WT_SESSION_IMPL *session, void *reconcilep) *(WT_RECONCILE **)reconcilep = NULL; __wt_buf_free(session, &r->disk_image); + __wt_scr_free(session, &r->interim_buf); __wt_free(session, r->raw_entries); __wt_free(session, r->raw_offsets); @@ -987,7 +1040,8 @@ __rec_bnd_cleanup(WT_SESSION_IMPL *session, WT_RECONCILE *r, bool destroy) __wt_free(session, bnd->addr.addr); __wt_free(session, bnd->disk_image); __wt_free(session, bnd->supd); - __wt_buf_free(session, &bnd->key); + __wt_buf_free(session, &bnd->max_bnd_key); + __wt_buf_free(session, &bnd->min_bnd_key); } __wt_free(session, r->bnd); r->bnd_next = 0; @@ -1082,6 +1136,7 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, } else upd_list = ins->upd; + ++r->update_cnt; for (skipped = false, max_txn = WT_TXN_NONE, min_txn = UINT64_MAX, upd = upd_list; upd != NULL; upd = upd->next) { @@ -1172,6 +1227,12 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, txnid != S2C(session)->txn_global.checkpoint_txnid || WT_SESSION_IS_CHECKPOINT(session)); #endif + + /* + * Track how many update chains we saw vs. how many update + * chains had an entry we skipped. + */ + ++r->update_skip_cnt; return (0); } @@ -1343,7 +1404,7 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, */ #define WT_CHILD_RELEASE(session, hazard, ref) do { \ if (hazard) { \ - hazard = false; \ + (hazard) = false; \ WT_TRET( \ __wt_page_release(session, ref, WT_READ_NO_EVICT)); \ } \ @@ -1665,6 +1726,17 @@ __rec_incr(WT_SESSION_IMPL *session, WT_RECONCILE *r, uint32_t v, size_t size) r->entries += v; r->space_avail -= size; r->first_free += size; + + /* + * If offset for the minimum split size boundary is not set, we have not + * yet reached the minimum boundary, reduce the space available for it. + */ + if (r->bnd[r->bnd_next].min_bnd_offset == 0) { + if (r->min_space_avail >= size) + r->min_space_avail -= size; + else + r->min_space_avail = 0; + } } /* @@ -1685,7 +1757,7 @@ __rec_copy_incr(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_KV *kv) * WT_CELLs are typically small, 1 or 2 bytes -- don't call memcpy, do * the copy in-line. */ - for (p = (uint8_t *)r->first_free, + for (p = r->first_free, t = (uint8_t *)&kv->cell, len = kv->cell_len; len > 0; --len) *p++ = *t++; @@ -1729,16 +1801,22 @@ __rec_dict_replace( return (0); /* - * If the dictionary cell reference is not set, we're creating a new - * entry in the dictionary, update its location. + * If the dictionary offset isn't set, we're creating a new entry in the + * dictionary, set its location. * - * If the dictionary cell reference is set, we have a matching value. - * Create a copy cell instead. + * If the dictionary offset is set, we have a matching value. Create a + * copy cell instead. */ - if (dp->cell == NULL) - dp->cell = r->first_free; + if (dp->offset == 0) + dp->offset = WT_PTRDIFF32(r->first_free, r->disk_image.mem); else { - offset = WT_PTRDIFF(r->first_free, dp->cell); + /* + * The offset is the byte offset from this cell to the previous, + * matching cell, NOT the byte offset from the beginning of the + * page. + */ + offset = (uint64_t)WT_PTRDIFF(r->first_free, + (uint8_t *)r->disk_image.mem + dp->offset); val->len = val->cell_len = __wt_cell_pack_copy(&val->cell, rle, offset); val->buf.data = NULL; @@ -1875,8 +1953,8 @@ static void __rec_split_bnd_init(WT_SESSION_IMPL *session, WT_BOUNDARY *bnd) { bnd->offset = 0; - bnd->recno = WT_RECNO_OOB; - bnd->entries = 0; + bnd->max_bnd_recno = WT_RECNO_OOB; + bnd->max_bnd_entries = 0; __wt_free(session, bnd->addr.addr); WT_CLEAR(bnd->addr); @@ -1891,6 +1969,10 @@ __rec_split_bnd_init(WT_SESSION_IMPL *session, WT_BOUNDARY *bnd) bnd->already_compressed = false; + bnd->min_bnd_offset = 0; + bnd->min_bnd_entries = 0; + bnd->min_bnd_recno = WT_RECNO_OOB; + /* * Don't touch the key, we re-use that memory in each new * reconciliation. @@ -1922,39 +2004,63 @@ __rec_split_bnd_grow(WT_SESSION_IMPL *session, WT_RECONCILE *r) } /* - * __wt_split_page_size -- - * Split page size calculation: we don't want to repeatedly split every - * time a new entry is added, so we split to a smaller-than-maximum page size. + * __rec_split_page_size_from_pct -- + * Given a split percentage, calculate split page size in bytes. */ -uint32_t -__wt_split_page_size(WT_BTREE *btree, uint32_t maxpagesize) -{ +static uint32_t +__rec_split_page_size_from_pct( + int split_pct, uint32_t maxpagesize, uint32_t allocsize) { uintmax_t a; uint32_t split_size; /* * Ideally, the split page size is some percentage of the maximum page - * size rounded to an allocation unit (round to an allocation unit so - * we don't waste space when we write). + * size rounded to an allocation unit (round to an allocation unit so we + * don't waste space when we write). */ a = maxpagesize; /* Don't overflow. */ split_size = (uint32_t)WT_ALIGN_NEAREST( - (a * (u_int)btree->split_pct) / 100, btree->allocsize); + (a * (u_int)split_pct) / 100, allocsize); /* - * Respect the configured split percentage if the calculated split - * size is either zero or a full page. The user has either configured - * an allocation size that matches the page size, or a split - * percentage that is close to zero or one hundred. Rounding is going - * to provide a worse outcome than having a split point that doesn't - * fall on an allocation size boundary in those cases. + * Respect the configured split percentage if the calculated split size + * is either zero or a full page. The user has either configured an + * allocation size that matches the page size, or a split percentage + * that is close to zero or one hundred. Rounding is going to provide a + * worse outcome than having a split point that doesn't fall on an + * allocation size boundary in those cases. */ if (split_size == 0 || split_size == maxpagesize) - split_size = (uint32_t)((a * (u_int)btree->split_pct) / 100); + split_size = (uint32_t)((a * (u_int)split_pct) / 100); return (split_size); } +/* + * __wt_split_page_size -- + * Split page size calculation: we don't want to repeatedly split every + * time a new entry is added, so we split to a smaller-than-maximum page size. + */ +uint32_t +__wt_split_page_size(WT_BTREE *btree, uint32_t maxpagesize) +{ + return (__rec_split_page_size_from_pct( + btree->split_pct, maxpagesize, btree->allocsize)); +} + +/* + * __rec_min_split_page_size -- + * Minimum split size boundary calculation: To track a boundary at the + * minimum split size that we could have split at instead of splitting at + * the split page size. + */ +static uint32_t +__rec_min_split_page_size(WT_BTREE *btree, uint32_t maxpagesize) +{ + return (__rec_split_page_size_from_pct( + WT_BTREE_MIN_SPLIT_PCT, maxpagesize, btree->allocsize)); +} + /* * __rec_split_init -- * Initialization for the reconciliation split functions. @@ -1966,7 +2072,7 @@ __rec_split_init(WT_SESSION_IMPL *session, WT_BM *bm; WT_BTREE *btree; WT_PAGE_HEADER *dsk; - size_t corrected_page_size; + size_t corrected_page_size, disk_img_buf_size; btree = S2BT(session); bm = btree->bm; @@ -2001,33 +2107,6 @@ __rec_split_init(WT_SESSION_IMPL *session, r->max_raw_page_size = r->page_size = (uint32_t)WT_MIN(r->page_size * 10, WT_MAX(r->page_size, btree->maxmempage / 2)); - - /* - * Ensure the disk image buffer is large enough for the max object, as - * corrected by the underlying block manager. - */ - corrected_page_size = r->page_size; - WT_RET(bm->write_size(bm, session, &corrected_page_size)); - WT_RET(__wt_buf_init(session, &r->disk_image, corrected_page_size)); - - /* - * Clear the disk page header to ensure all of it is initialized, even - * the unused fields. - * - * In the case of fixed-length column-store, clear the entire buffer: - * fixed-length column-store sets bits in bytes, where the bytes are - * assumed to initially be 0. - */ - memset(r->disk_image.mem, 0, page->type == WT_PAGE_COL_FIX ? - corrected_page_size : WT_PAGE_HEADER_SIZE); - - /* - * Set the page type (the type doesn't change, and setting it later - * would require additional code in a few different places). - */ - dsk = r->disk_image.mem; - dsk->type = page->type; - /* * If we have to split, we want to choose a smaller page size for the * split pages, because otherwise we could end up splitting one large @@ -2047,22 +2126,28 @@ __rec_split_init(WT_SESSION_IMPL *session, * creating overflow items and compacted data, for example, as those * items have already been written to disk). So, the loop calls the * helper functions when approaching a split boundary, and we save the - * information at that point. That allows us to go back and split the - * page at the boundary points if we eventually overflow the maximum - * page size. + * information at that point. We also save the boundary information at + * the minimum split size. We maintain two chunks (each boundary + * represents a chunk that gets written as a page) in the memory, + * writing out the older one to the disk as a page when we need to make + * space for a new chunk. On reaching the last chunk, if it turns out to + * be smaller than the minimum split size, we go back into the + * penultimate chunk and split at this minimum split size boundary. This + * moves some data from the penultimate chunk to the last chunk, hence + * increasing the size of the last page written without decreasing the + * penultimate page size beyond the minimum split size. * * Finally, all this doesn't matter for fixed-size column-store pages, * raw compression, and salvage. Fixed-size column store pages can * split under (very) rare circumstances, but they're allocated at a * fixed page size, never anything smaller. In raw compression, the - * underlying compression routine decides when we split, so it's not - * our problem. In salvage, as noted above, we can't split at all. + * underlying compression routine decides when we split, so it's not our + * problem. In salvage, as noted above, we can't split at all. */ if (r->raw_compression || r->salvage != NULL) { r->split_size = 0; r->space_avail = r->page_size - WT_PAGE_HEADER_BYTE_SIZE(btree); - } - else if (page->type == WT_PAGE_COL_FIX) { + } else if (page->type == WT_PAGE_COL_FIX) { r->split_size = r->page_size; r->space_avail = r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); @@ -2070,32 +2155,55 @@ __rec_split_init(WT_SESSION_IMPL *session, r->split_size = __wt_split_page_size(btree, r->page_size); r->space_avail = r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); + r->min_split_size = + __rec_min_split_page_size(btree, r->page_size); + r->min_space_avail = + r->min_split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); } + + /* + * Ensure the disk image buffer is large enough for the max object, as + * corrected by the underlying block manager. + * + * The buffer that we build disk image in, needs to hold two chunks + * worth of data. Since we want to support split_size more than the page + * size (to allow for adjustments based on the compression), this buffer + * should be greater of twice of split_size and page_size. + */ + corrected_page_size = r->page_size; + disk_img_buf_size = 2 * WT_MAX(corrected_page_size, r->split_size); + WT_RET(bm->write_size(bm, session, &corrected_page_size)); + WT_RET(__wt_buf_init(session, &r->disk_image, disk_img_buf_size)); + + /* + * Clear the disk page header to ensure all of it is initialized, even + * the unused fields. + * + * In the case of fixed-length column-store, clear the entire buffer: + * fixed-length column-store sets bits in bytes, where the bytes are + * assumed to initially be 0. + */ + memset(r->disk_image.mem, 0, page->type == WT_PAGE_COL_FIX ? + disk_img_buf_size : WT_PAGE_HEADER_SIZE); + + /* + * Set the page type (the type doesn't change, and setting it later + * would require additional code in a few different places). + */ + dsk = r->disk_image.mem; + dsk->type = page->type; + r->first_free = WT_PAGE_HEADER_BYTE(btree, dsk); /* Initialize the first boundary. */ r->bnd_next = 0; WT_RET(__rec_split_bnd_grow(session, r)); __rec_split_bnd_init(session, &r->bnd[0]); - r->bnd[0].recno = recno; + r->bnd[0].max_bnd_recno = recno; r->bnd[0].offset = WT_PAGE_HEADER_BYTE_SIZE(btree); - /* - * If the maximum page size is the same as the split page size, either - * because of the object type or application configuration, there isn't - * any need to maintain split boundaries within a larger page. - * - * No configuration for salvage here, because salvage can't split. - */ - if (r->raw_compression) - r->bnd_state = SPLIT_TRACKING_RAW; - else if (max == r->split_size) - r->bnd_state = SPLIT_TRACKING_OFF; - else - r->bnd_state = SPLIT_BOUNDARY; - - /* Initialize the entry counters. */ - r->entries = r->total_entries = 0; + /* Initialize the entry counter. */ + r->entries = 0; /* Initialize the starting record number. */ r->recno = recno; @@ -2298,19 +2406,112 @@ __rec_split_grow(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t add_len) { WT_BM *bm; WT_BTREE *btree; - size_t corrected_page_size, len; + size_t corrected_page_size, inuse, len; btree = S2BT(session); bm = btree->bm; len = WT_PTRDIFF(r->first_free, r->disk_image.mem); - corrected_page_size = len + add_len; + inuse = (len - r->bnd[r->bnd_next].offset) + + WT_PAGE_HEADER_BYTE_SIZE(btree); + corrected_page_size = inuse + add_len; + WT_RET(bm->write_size(bm, session, &corrected_page_size)); - WT_RET(__wt_buf_grow(session, &r->disk_image, corrected_page_size)); + /* Need to account for buffer carrying two chunks worth of data */ + WT_RET(__wt_buf_grow(session, &r->disk_image, 2 * corrected_page_size)); + r->first_free = (uint8_t *)r->disk_image.mem + len; - WT_ASSERT(session, corrected_page_size >= len); - r->space_avail = corrected_page_size - len; + WT_ASSERT(session, corrected_page_size >= inuse); + r->space_avail = corrected_page_size - inuse; WT_ASSERT(session, r->space_avail >= add_len); + + return (0); +} + +/* + * __rec_split_write_prev_and_shift_cur -- + * Write the previous split chunk to the disk as a page. Shift the contents + * of the current chunk to the start of the buffer, making space for a new + * chunk to be written. + * If the caller asks for a chunk resizing, the boundary between the two + * chunks is readjusted to the minimum split size boundary details stored + * in the previous chunk, letting the current chunk grow at the cost of the + * previous chunk. + */ +static int +__rec_split_write_prev_and_shift_cur( + WT_SESSION_IMPL *session, WT_RECONCILE *r, bool resize_chunks) +{ + WT_BM *bm; + WT_BOUNDARY *bnd_cur, *bnd_prev; + WT_BTREE *btree; + WT_PAGE_HEADER *dsk, *dsk_tmp; + size_t cur_len, len; + uint8_t *dsk_start; + + WT_ASSERT(session, r->bnd_next != 0); + + btree = S2BT(session); + bm = btree->bm; + bnd_cur = &r->bnd[r->bnd_next]; + bnd_prev = bnd_cur - 1; + dsk = r->disk_image.mem; + cur_len = WT_PTRDIFF(r->first_free, dsk) - bnd_cur->offset; + + /* + * Resize chunks if the current is smaller than the minimum, and there + * are details on the minimum split size boundary available in the + * previous boundary details. + * + * There is a possibility that we do not have a minimum boundary set, in + * such a case we skip chunk resizing. Such a condition is possible for + * instance when we are building the image in the buffer and the first + * K/V pair is large enough that it surpasses both the minimum split + * size and the split size the application has set. In such a case we + * split the chunk without saving any minimum boundary. + */ + if (resize_chunks && + cur_len < r->min_split_size && bnd_prev->min_bnd_offset != 0) { + bnd_cur->offset = bnd_prev->min_bnd_offset; + bnd_cur->max_bnd_entries += + bnd_prev->max_bnd_entries - bnd_prev->min_bnd_entries; + bnd_prev->max_bnd_entries = bnd_prev->min_bnd_entries; + bnd_cur->max_bnd_recno = bnd_prev->min_bnd_recno; + + WT_RET(__wt_buf_set(session, &bnd_cur->max_bnd_key, + bnd_prev->min_bnd_key.data, bnd_prev->min_bnd_key.size)); + + /* Update current chunk's length */ + cur_len = WT_PTRDIFF(r->first_free, dsk) - bnd_cur->offset; + } + + /* + * Create an interim buffer if not already done to prepare the previous + * chunk's disk image. + */ + len = bnd_cur->offset; + WT_RET(bm->write_size(bm, session, &len)); + if (r->interim_buf == NULL) + WT_RET(__wt_scr_alloc(session, len, &r->interim_buf)); + else + WT_RET(__wt_buf_init(session, r->interim_buf, len)); + + dsk_tmp = r->interim_buf->mem; + memcpy(dsk_tmp, dsk, bnd_cur->offset); + dsk_tmp->recno = bnd_prev->max_bnd_recno; + dsk_tmp->u.entries = bnd_prev->max_bnd_entries; + dsk_tmp->mem_size = WT_STORE_SIZE(bnd_cur->offset); + r->interim_buf->size = dsk_tmp->mem_size; + WT_RET(__rec_split_write(session, r, bnd_prev, r->interim_buf, false)); + + /* Shift the current chunk to the start of the buffer */ + dsk_start = WT_PAGE_HEADER_BYTE(btree, dsk); + (void)memmove(dsk_start, (uint8_t *)dsk + bnd_cur->offset, cur_len); + + /* Fix boundary offset */ + bnd_cur->offset = WT_PAGE_HEADER_BYTE_SIZE(btree); + /* Fix where free points */ + r->first_free = dsk_start + cur_len; return (0); } @@ -2330,6 +2531,9 @@ __rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) btree = S2BT(session); dsk = r->disk_image.mem; + /* Fixed length col store can call with next_len 0 */ + WT_ASSERT(session, next_len == 0 || r->space_avail < next_len); + /* * We should never split during salvage, and we're about to drop core * because there's no parent page. @@ -2339,147 +2543,63 @@ __rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) "%s page too large, attempted split during salvage", __wt_page_type_string(r->page->type)); - /* Hitting a page boundary resets the dictionary, in all cases. */ - __rec_dictionary_reset(r); - - inuse = WT_PTRDIFF(r->first_free, dsk); - switch (r->bnd_state) { - case SPLIT_BOUNDARY: - /* - * We can get here if the first key/value pair won't fit. - * Additionally, grow the buffer to contain the current item if - * we haven't already consumed a reasonable portion of a split - * chunk. - */ - if (inuse < r->split_size / 2) - break; - - /* - * About to cross a split boundary but not yet forced to split - * into multiple pages. If we have to split, this is one of the - * split points, save information about where we are when the - * split would have happened. - */ - WT_RET(__rec_split_bnd_grow(session, r)); - last = &r->bnd[r->bnd_next++]; - next = last + 1; - - /* Set the number of entries for the just finished chunk. */ - last->entries = r->entries - r->total_entries; - r->total_entries = r->entries; - - /* Set the key for the next chunk. */ - next->recno = r->recno; - if (dsk->type == WT_PAGE_ROW_INT || - dsk->type == WT_PAGE_ROW_LEAF) - WT_RET(__rec_split_row_promote( - session, r, &next->key, dsk->type)); - - /* - * Set the starting buffer offset and clear the entries (the - * latter not required, but cleaner). - */ - next->offset = WT_PTRDIFF(r->first_free, dsk); - next->entries = 0; - - /* Set the space available to another split-size chunk. */ - r->space_avail = - r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); - - /* - * Adjust the space available to handle two cases: - * - We don't have enough room for another full split-size - * chunk on the page. - * - We chose to fill past a page boundary because of a - * large item. - */ - if (inuse + r->space_avail > r->page_size) { - r->space_avail = - r->page_size > inuse ? (r->page_size - inuse) : 0; - - /* There are no further boundary points. */ - r->bnd_state = SPLIT_MAX; - } - - /* - * Return if the next object fits into this page, else we have - * to split the page. - */ - if (r->space_avail >= next_len) - return (0); - - /* FALLTHROUGH */ - case SPLIT_MAX: - /* - * We're going to have to split and create multiple pages. - * - * Cycle through the saved split-point information, writing the - * split chunks we have tracked. The underlying fixup function - * sets the space available and other information, and copied - * any unwritten chunk of data to the beginning of the buffer. - */ - WT_RET(__rec_split_fixup(session, r)); - - /* We're done saving split chunks. */ - r->bnd_state = SPLIT_TRACKING_OFF; - break; - case SPLIT_TRACKING_OFF: - /* - * We can get here if the first key/value pair won't fit. - * Additionally, grow the buffer to contain the current item if - * we haven't already consumed a reasonable portion of a split - * chunk. - */ - if (inuse < r->split_size / 2) - break; + last = &r->bnd[r->bnd_next]; + inuse = (WT_PTRDIFF(r->first_free, dsk) - last->offset) + + WT_PAGE_HEADER_BYTE_SIZE(btree); - /* - * The key/value pairs didn't fit into a single page, but either - * we've already noticed that and are now processing the rest of - * the pairs at split size boundaries, or the split size was the - * same as the page size, and we never bothered with split point - * information at all. - */ - WT_RET(__rec_split_bnd_grow(session, r)); - last = &r->bnd[r->bnd_next++]; - next = last + 1; + /* + * We can get here if the first key/value pair won't fit. + * Additionally, grow the buffer to contain the current item if we + * haven't already consumed a reasonable portion of a split chunk. + */ + if (inuse < r->split_size / 2) + goto done; - /* - * Set the key for the next chunk (before writing the block, a - * key range is needed in that code). - */ - next->recno = r->recno; - if (dsk->type == WT_PAGE_ROW_INT || - dsk->type == WT_PAGE_ROW_LEAF) - WT_RET(__rec_split_row_promote( - session, r, &next->key, dsk->type)); + /* All page boundaries reset the dictionary. */ + __rec_dictionary_reset(r); - /* Clear the entries (not required, but cleaner). */ - next->entries = 0; + /* Set the number of entries for the just finished chunk. */ + last->max_bnd_entries = r->entries; - /* Finalize the header information and write the page. */ - dsk->recno = last->recno; - dsk->u.entries = r->entries; - dsk->mem_size = WT_PTRDIFF32(r->first_free, dsk); + /* + * In case of bulk load, write out chunks as we get them. Otherwise we + * keep two chunks in memory at a given time. So, if there is a previous + * chunk, write it out, making space in the buffer for the next chunk to + * be written. + */ + if (r->is_bulk_load) { + dsk->recno = last->max_bnd_recno; + dsk->u.entries = last->max_bnd_entries; + dsk->mem_size = (uint32_t)inuse; r->disk_image.size = dsk->mem_size; - WT_RET( - __rec_split_write(session, r, last, &r->disk_image, false)); - - /* - * Set the caller's entry count and buffer information for the - * next chunk. We only get here if we're not splitting or have - * already split, so it's split-size chunks from here on out. - */ - r->entries = 0; + WT_RET(__rec_split_write( + session, r, last, &r->disk_image, false)); + /* Fix where free points */ r->first_free = WT_PAGE_HEADER_BYTE(btree, dsk); - r->space_avail = - r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); - break; - case SPLIT_TRACKING_RAW: - return (__wt_illegal_value(session, NULL)); - } + } else if (r->bnd_next != 0) + WT_RET(__rec_split_write_prev_and_shift_cur(session, r, false)); - /* + /* Prepare the next boundary */ + WT_RET(__rec_split_bnd_grow(session, r)); + r->bnd_next++; + next = &r->bnd[r->bnd_next]; + next->offset = WT_PTRDIFF(r->first_free, dsk); + /* Set the key for the next chunk. */ + next->max_bnd_recno = r->recno; + if (dsk->type == WT_PAGE_ROW_INT || dsk->type == WT_PAGE_ROW_LEAF) + WT_RET(__rec_split_row_promote( + session, r, &next->max_bnd_key, dsk->type)); + + r->entries = 0; + /* + * Set the space available to another split-size and minimum split-size + * chunk. + */ + r->space_avail = r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); + r->min_space_avail = + r->min_split_size - WT_PAGE_HEADER_BYTE_SIZE(btree); + +done: /* * Overflow values can be larger than the maximum page size but still be * "on-page". If the next key/value pair is larger than space available * after a split has happened (in other words, larger than the maximum @@ -2496,6 +2616,64 @@ __rec_split(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) return (0); } +/* + * __rec_split_crossing_bnd -- + * Save the details for the minimum split size boundary or call for a + * split. + */ +static inline int +__rec_split_crossing_bnd( + WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) +{ + WT_BOUNDARY *bnd; + WT_BTREE *btree; + WT_PAGE_HEADER *dsk; + size_t min_bnd_offset; + + WT_ASSERT(session, WT_CHECK_CROSSING_BND(r, next_len)); + + /* + * If crossing the minimum split size boundary, store the boundary + * details at the current location in the buffer. If we are crossing the + * split boundary at the same time, possible when the next record is + * large enough, just split at this point. + */ + if (WT_CROSSING_MIN_BND(r, next_len) && + !WT_CROSSING_SPLIT_BND(r, next_len)) { + btree = S2BT(session); + bnd = &r->bnd[r->bnd_next]; + dsk = r->disk_image.mem; + min_bnd_offset = (WT_PTRDIFF(r->first_free, dsk) - + bnd->offset) + WT_PAGE_HEADER_BYTE_SIZE(btree); + if (min_bnd_offset == WT_PAGE_HEADER_BYTE_SIZE(btree)) + /* + * This is possible if the first record doesn't fit in + * the minimum split size, we write this record without + * setting up any boundary here. We will get the + * opportunity to setup a boundary before writing out + * the next record. + */ + return (0); + + WT_ASSERT(session, bnd->min_bnd_offset == 0); + + /* All page boundaries reset the dictionary. */ + __rec_dictionary_reset(r); + + bnd->min_bnd_offset = min_bnd_offset; + bnd->min_bnd_entries = r->entries; + bnd->min_bnd_recno = r->recno; + if (dsk->type == WT_PAGE_ROW_INT || + dsk->type == WT_PAGE_ROW_LEAF) + WT_RET(__rec_split_row_promote( + session, r, &bnd->min_bnd_key, dsk->type)); + return (0); + } + + /* We are crossing a split boundary */ + return (__rec_split(session, r, next_len)); +} + /* * __rec_split_raw_worker -- * Handle the raw compression page reconciliation bookkeeping. @@ -2574,7 +2752,7 @@ __rec_split_raw_worker(WT_SESSION_IMPL *session, */ recno = WT_RECNO_OOB; if (dsk->type == WT_PAGE_COL_VAR) - recno = last->recno; + recno = last->max_bnd_recno; entry = max_image_slot = slots = 0; WT_CELL_FOREACH(btree, dsk, cell, unpack, i) { @@ -2801,7 +2979,7 @@ no_slots: */ dst->size = result_len + WT_BLOCK_COMPRESS_SKIP; dsk_dst = dst->mem; - dsk_dst->recno = last->recno; + dsk_dst->recno = last->max_bnd_recno; dsk_dst->mem_size = r->raw_offsets[result_slots] + WT_BLOCK_COMPRESS_SKIP; dsk_dst->u.entries = r->raw_entries[result_slots - 1]; @@ -2821,7 +2999,7 @@ no_slots: WT_RET(__wt_strndup(session, dsk, dsk_dst->mem_size, &last->disk_image)); disk_image = last->disk_image; - disk_image->recno = last->recno; + disk_image->recno = last->max_bnd_recno; disk_image->mem_size = dsk_dst->mem_size; disk_image->u.entries = dsk_dst->u.entries; } @@ -2837,7 +3015,7 @@ no_slots: len = WT_PTRDIFF( r->first_free, (uint8_t *)dsk + dsk_dst->mem_size); dsk_start = WT_PAGE_HEADER_BYTE(btree, dsk); - (void)memmove(dsk_start, (uint8_t *)r->first_free - len, len); + (void)memmove(dsk_start, r->first_free - len, len); r->entries -= r->raw_entries[result_slots - 1]; r->first_free = dsk_start + len; @@ -2851,14 +3029,14 @@ no_slots: */ switch (dsk->type) { case WT_PAGE_COL_INT: - next->recno = r->raw_recnos[result_slots]; + next->max_bnd_recno = r->raw_recnos[result_slots]; break; case WT_PAGE_COL_VAR: - next->recno = r->raw_recnos[result_slots - 1]; + next->max_bnd_recno = r->raw_recnos[result_slots - 1]; break; case WT_PAGE_ROW_INT: case WT_PAGE_ROW_LEAF: - next->recno = WT_RECNO_OOB; + next->max_bnd_recno = WT_RECNO_OOB; if (!last_block) { /* * Confirm there was uncompressed data remaining @@ -2867,7 +3045,7 @@ no_slots: */ WT_ASSERT(session, len > 0); WT_RET(__rec_split_row_promote_cell( - session, dsk, &next->key)); + session, dsk, &next->max_bnd_key)); } break; } @@ -2879,7 +3057,7 @@ no_slots: */ WT_STAT_DATA_INCR(session, compress_raw_fail); - dsk->recno = last->recno; + dsk->recno = last->max_bnd_recno; dsk->mem_size = WT_PTRDIFF32(r->first_free, dsk); dsk->u.entries = r->entries; r->disk_image.size = dsk->mem_size; @@ -2956,35 +3134,9 @@ __rec_split_raw(WT_SESSION_IMPL *session, WT_RECONCILE *r, size_t next_len) static int __rec_split_finish_std(WT_SESSION_IMPL *session, WT_RECONCILE *r) { - WT_BOUNDARY *bnd; + WT_BOUNDARY *bnd_cur, *bnd_prev; WT_PAGE_HEADER *dsk; - - /* Adjust the boundary information based on our split status. */ - switch (r->bnd_state) { - case SPLIT_BOUNDARY: - case SPLIT_MAX: - /* - * We never split, the reconciled page fit into a maximum page - * size. Change the first boundary slot to represent the full - * page (the first boundary slot is largely correct, just update - * the number of entries). - */ - r->bnd_next = 0; - break; - case SPLIT_TRACKING_OFF: - /* - * If we have already split, or aren't tracking boundaries, put - * the remaining data in the next boundary slot. - */ - WT_RET(__rec_split_bnd_grow(session, r)); - break; - case SPLIT_TRACKING_RAW: - /* - * We were configured for raw compression, and either we never - * wrote anything, or there's a remaindered block of data. - */ - break; - } + bool grow_bnd; /* * We may arrive here with no entries to write if the page was entirely @@ -3011,20 +3163,66 @@ __rec_split_finish_std(WT_SESSION_IMPL *session, WT_RECONCILE *r) return (EBUSY); } - /* Set the boundary reference and increment the count. */ - bnd = &r->bnd[r->bnd_next++]; - bnd->entries = r->entries; - - /* Finalize the header information. */ dsk = r->disk_image.mem; - dsk->recno = bnd->recno; - dsk->u.entries = r->entries; + + /* Set the number of entries for the just finished chunk. */ + bnd_cur = &r->bnd[r->bnd_next]; + bnd_cur->max_bnd_entries = r->entries; + + grow_bnd = true; + /* + * We can reach here even with raw_compression when the last split chunk + * is too small to be sent for raw compression. + */ + if (!r->is_bulk_load && !r->raw_compression) { + if (WT_PTRDIFF(r->first_free, dsk) > r->page_size && + r->bnd_next != 0) { + /* + * We hold two boundaries worth of data in the buffer, + * and this data doesn't fit in a single page. If the + * last chunk is too small, readjust the boundary to a + * pre-computed minimum. + * Write out the penultimate chunk to the disk as a page + */ + WT_RET(__rec_split_write_prev_and_shift_cur( + session, r, true)); + } else + if (r->bnd_next != 0) { + /* + * We have two boundaries, but the data in the + * buffer can fit a single page. Merge the + * boundaries to create a single chunk. + */ + bnd_prev = bnd_cur - 1; + bnd_prev->max_bnd_entries += + bnd_cur->max_bnd_entries; + r->bnd_next--; + grow_bnd = false; + } + } + + /* + * We already have space for an extra boundary if we merged two + * boundaries above, in that case we do not need to grow the boundary + * structure. + */ + if (grow_bnd) + WT_RET(__rec_split_bnd_grow(session, r)); + bnd_cur = &r->bnd[r->bnd_next]; + r->bnd_next++; + + /* + * Current boundary now has all the remaining data/last page now. + * Let's write it to the disk + */ + dsk->recno = bnd_cur->max_bnd_recno; + dsk->u.entries = bnd_cur->max_bnd_entries; dsk->mem_size = WT_PTRDIFF32(r->first_free, dsk); r->disk_image.size = dsk->mem_size; /* If this is a checkpoint, we're done, otherwise write the page. */ - return (__rec_is_checkpoint(session, r, bnd) ? - 0 : __rec_split_write(session, r, bnd, &r->disk_image, true)); + return (__rec_is_checkpoint(session, r, bnd_cur) ? + 0 : __rec_split_write(session, r, bnd_cur, &r->disk_image, true)); } /* @@ -3057,98 +3255,6 @@ __rec_split_finish(WT_SESSION_IMPL *session, WT_RECONCILE *r) return (__rec_split_finish_std(session, r)); } -/* - * __rec_split_fixup -- - * Fix up after crossing the maximum page boundary. - */ -static int -__rec_split_fixup(WT_SESSION_IMPL *session, WT_RECONCILE *r) -{ - WT_BOUNDARY *bnd; - WT_BTREE *btree; - WT_DECL_ITEM(tmp); - WT_DECL_RET; - WT_PAGE_HEADER *dsk; - size_t i, len; - uint8_t *dsk_start, *p; - - /* - * When we overflow physical limits of the page, we walk the list of - * split chunks we've created and write those pages out, then update - * the caller's information. - */ - btree = S2BT(session); - - /* - * The data isn't laid out on a page boundary or nul padded; copy it to - * a clean, aligned, padded buffer before writing it. - * - * Allocate a scratch buffer to hold the new disk image. Copy the disk - * page's header and block-manager space into the scratch buffer, most - * of the header information remains unchanged between the pages. - */ - WT_RET(__wt_scr_alloc(session, r->disk_image.memsize, &tmp)); - dsk = tmp->mem; - memcpy(dsk, r->disk_image.mem, WT_PAGE_HEADER_BYTE_SIZE(btree)); - - /* - * For each split chunk we've created, update the disk image and copy - * it into place. - */ - dsk_start = WT_PAGE_HEADER_BYTE(btree, dsk); - for (i = 0, bnd = r->bnd; i < r->bnd_next; ++i, ++bnd) { - /* Copy the page contents to the temporary buffer. */ - len = (bnd + 1)->offset - bnd->offset; - memcpy(dsk_start, - (uint8_t *)r->disk_image.mem + bnd->offset, len); - - /* Finalize the header information and write the page. */ - dsk->recno = bnd->recno; - dsk->u.entries = bnd->entries; - tmp->size = WT_PAGE_HEADER_BYTE_SIZE(btree) + len; - dsk->mem_size = WT_STORE_SIZE(tmp->size); - WT_ERR(__rec_split_write(session, r, bnd, tmp, false)); - } - - /* - * There is probably a remnant in the working buffer that didn't get - * written, copy it down to the beginning of the working buffer. - * - * Confirm the remnant is no larger than a split-sized chunk, including - * header. We know that's the maximum sized remnant because we only have - * remnants if split switches from accumulating to a split boundary to - * accumulating to the end of the page (the other path here is when we - * hit a split boundary, there was room for another split chunk in the - * page, and the next item still wouldn't fit, in which case there is no - * remnant). So: we were accumulating to the end of the page and created - * a remnant. We know the remnant cannot be as large as a split-sized - * chunk, including header, because if there was room for that large a - * remnant, we wouldn't have switched from accumulating to a page end. - */ - p = (uint8_t *)r->disk_image.mem + bnd->offset; - len = WT_PTRDIFF(r->first_free, p); - if (len >= r->split_size - WT_PAGE_HEADER_BYTE_SIZE(btree)) - WT_PANIC_ERR(session, EINVAL, - "Reconciliation remnant too large for the split buffer"); - dsk = r->disk_image.mem; - dsk_start = WT_PAGE_HEADER_BYTE(btree, dsk); - (void)memmove(dsk_start, p, len); - - /* - * Fix up our caller's information, including updating the starting - * record number. - */ - r->entries -= r->total_entries; - r->first_free = dsk_start + len; - WT_ASSERT(session, - r->page_size >= (WT_PAGE_HEADER_BYTE_SIZE(btree) + len)); - r->space_avail = - r->split_size - (WT_PAGE_HEADER_BYTE_SIZE(btree) + len); - -err: __wt_scr_free(session, &tmp); - return (ret); -} - /* * __rec_split_write -- * Write a disk block out for the split helper functions. @@ -3170,11 +3276,17 @@ __rec_split_write(WT_SESSION_IMPL *session, int cmp; uint8_t addr[WT_BTREE_MAX_ADDR_COOKIE]; bool need_image; +#ifdef HAVE_DIAGNOSTIC + bool verify_image; +#endif btree = S2BT(session); dsk = buf->mem; page = r->page; mod = page->modify; +#ifdef HAVE_DIAGNOSTIC + verify_image = true; +#endif /* Set the zero-length value flag in the page header. */ if (dsk->type == WT_PAGE_ROW_LEAF) { @@ -3186,8 +3298,6 @@ __rec_split_write(WT_SESSION_IMPL *session, F_SET(dsk, WT_PAGE_EMPTY_V_NONE); } - bnd->entries = r->entries; - /* Initialize the address (set the page type for the parent). */ switch (dsk->type) { case WT_PAGE_COL_FIX: @@ -3233,7 +3343,8 @@ __rec_split_write(WT_SESSION_IMPL *session, switch (page->type) { case WT_PAGE_COL_FIX: case WT_PAGE_COL_VAR: - if (WT_INSERT_RECNO(supd->ins) >= (bnd + 1)->recno) + if (WT_INSERT_RECNO(supd->ins) >= + (bnd + 1)->max_bnd_recno) goto supd_check_complete; break; case WT_PAGE_ROW_LEAF: @@ -3244,8 +3355,8 @@ __rec_split_write(WT_SESSION_IMPL *session, key->data = WT_INSERT_KEY(supd->ins); key->size = WT_INSERT_KEY_SIZE(supd->ins); } - WT_ERR(__wt_compare(session, - btree->collator, key, &(bnd + 1)->key, &cmp)); + WT_ERR(__wt_compare(session, btree->collator, + key, &(bnd + 1)->max_bnd_key, &cmp)); if (cmp >= 0) goto supd_check_complete; break; @@ -3335,18 +3446,21 @@ supd_check_complete: #ifdef HAVE_VERBOSE /* Output a verbose message if we create a page without many entries */ - if (WT_VERBOSE_ISSET(session, WT_VERB_SPLIT) && r->entries < 6) + if (WT_VERBOSE_ISSET(session, WT_VERB_SPLIT) && + bnd->max_bnd_entries < 6) __wt_verbose(session, WT_VERB_SPLIT, "Reconciliation creating a page with %" PRIu32 " entries, memory footprint %" WT_SIZET_FMT - ", page count %" PRIu32 ", %s, split state: %d", - r->entries, r->page->memory_footprint, r->bnd_next, - F_ISSET(r, WT_EVICTING) ? "evict" : "checkpoint", - r->bnd_state); + ", page count %" PRIu32 ", %s", bnd->max_bnd_entries, + r->page->memory_footprint, r->bnd_next, + F_ISSET(r, WT_EVICTING) ? "evict" : "checkpoint"); #endif WT_ERR(__wt_bt_write(session, buf, addr, &addr_size, false, F_ISSET(r, WT_CHECKPOINTING), bnd->already_compressed)); +#ifdef HAVE_DIAGNOSTIC + verify_image = false; +#endif WT_ERR(__wt_strndup(session, addr, addr_size, &bnd->addr.addr)); bnd->addr.size = (uint8_t)addr_size; @@ -3373,9 +3487,20 @@ copy_image: */ need_image = F_ISSET(r, WT_EVICT_SCRUB) || (F_ISSET(r, WT_EVICT_UPDATE_RESTORE) && bnd->supd != NULL); - if (need_image && bnd->disk_image == NULL) + if (need_image && bnd->disk_image == NULL) { +#ifdef HAVE_DIAGNOSTIC + /* + * The I/O routines verify all disk images we write, but there + * are paths in reconciliation that don't do I/O. Verify those + * images, too. + */ + WT_ASSERT(session, verify_image == false || + __wt_verify_dsk_image( + session, "[reconcile-image]", buf->data, 0, true) == 0); +#endif WT_ERR(__wt_strndup( session, buf->data, buf->size, &bnd->disk_image)); + } if (!need_image) __wt_free(session, bnd->disk_image); @@ -3531,11 +3656,12 @@ __wt_bulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) uint64_t recno; btree = S2BT(session); + /* * Bulk-load is only permitted on newly created files, not any empty * file -- see the checkpoint code for a discussion. */ - if (!btree->bulk_load_ok) + if (!btree->original) WT_RET_MSG(session, EINVAL, "bulk-load is only possible for newly created trees"); @@ -3552,16 +3678,7 @@ __wt_bulk_init(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) r = cbulk->reconcile; r->is_bulk_load = true; - recno = WT_RECNO_OOB; /* -Werror=maybe-uninitialized */ - switch (btree->type) { - case BTREE_COL_FIX: - case BTREE_COL_VAR: - recno = 1; - break; - case BTREE_ROW: - recno = WT_RECNO_OOB; - break; - } + recno = btree->type == BTREE_ROW ? WT_RECNO_OOB : 1; return (__rec_split_init( session, r, cbulk->leaf, recno, btree->maxleafpage)); @@ -3599,7 +3716,7 @@ __wt_bulk_wrapup(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_RET(__rec_split_finish(session, r)); WT_RET(__rec_write_wrapup(session, r, r->page)); - WT_RET(__rec_write_status(session, r, r->page)); + __rec_write_page_status(session, r); /* Mark the page's parent and the tree dirty. */ parent = r->ref->home; @@ -3636,11 +3753,12 @@ __wt_bulk_insert_row(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) cursor->value.data, cursor->value.size, (uint64_t)0)); /* Boundary: split or write the page. */ - if (key->len + val->len > r->space_avail) { - if (r->raw_compression) - WT_RET( - __rec_split_raw(session, r, key->len + val->len)); - else { + if (r->raw_compression) { + if (key->len + val->len > r->space_avail) + WT_RET(__rec_split_raw( + session, r, key->len + val->len)); + } else + if (WT_CROSSING_SPLIT_BND(r, key->len + val->len)) { /* * Turn off prefix compression until a full key written * to the new page, and (unless already working with an @@ -3652,10 +3770,9 @@ __wt_bulk_insert_row(WT_SESSION_IMPL *session, WT_CURSOR_BULK *cbulk) WT_RET(__rec_cell_build_leaf_key( session, r, NULL, 0, &ovfl_key)); } - - WT_RET(__rec_split(session, r, key->len + val->len)); + WT_RET(__rec_split_crossing_bnd( + session, r, key->len + val->len)); } - } /* Copy the key/value pair onto the page. */ __rec_copy_incr(session, r, key); @@ -3696,6 +3813,10 @@ __rec_col_fix_bulk_insert_split_check(WT_CURSOR_BULK *cbulk) * split. * * Boundary: split or write the page. + * + * No need to have a minimum split size boundary, all + * pages are filled 100% except the last, allowing it to + * grow in the future. */ __rec_incr(session, r, cbulk->entry, __bitstr_size( @@ -3800,10 +3921,12 @@ __wt_bulk_insert_var( r, cbulk->last.data, cbulk->last.size, cbulk->rle)); /* Boundary: split or write the page. */ - if (val->len > r->space_avail) - WT_RET(r->raw_compression ? - __rec_split_raw(session, r, val->len) : - __rec_split(session, r, val->len)); + if (r->raw_compression) { + if (val->len > r->space_avail) + WT_RET(__rec_split_raw(session, r, val->len)); + } else + if (WT_CROSSING_SPLIT_BND(r, val->len)) + WT_RET(__rec_split_crossing_bnd(session, r, val->len)); /* Copy the value onto the page. */ if (btree->dictionary) @@ -3939,10 +4062,13 @@ __rec_col_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) WT_CHILD_RELEASE_ERR(session, hazard, ref); /* Boundary: split or write the page. */ - if (val->len > r->space_avail) - WT_ERR(r->raw_compression ? - __rec_split_raw(session, r, val->len) : - __rec_split(session, r, val->len)); + if (r->raw_compression) { + if (val->len > r->space_avail) + WT_ERR(__rec_split_raw(session, r, val->len)); + } else + if (WT_CHECK_CROSSING_BND(r, val->len)) + WT_ERR(__rec_split_crossing_bnd( + session, r, val->len)); /* Copy the value onto the page. */ __rec_copy_incr(session, r, val); @@ -3984,10 +4110,13 @@ __rec_col_merge(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) addr->addr, addr->size, __rec_vtype(addr), r->recno); /* Boundary: split or write the page. */ - if (val->len > r->space_avail) - WT_RET(r->raw_compression ? - __rec_split_raw(session, r, val->len) : - __rec_split(session, r, val->len)); + if (r->raw_compression) { + if (val->len > r->space_avail) + WT_RET(__rec_split_raw(session, r, val->len)); + } else + if (WT_CHECK_CROSSING_BND(r, val->len)) + WT_RET(__rec_split_crossing_bnd( + session, r, val->len)); /* Copy the value onto the page. */ __rec_copy_incr(session, r, val); @@ -4017,7 +4146,7 @@ __rec_col_fix(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) /* Copy the original, disk-image bytes into place. */ memcpy(r->first_free, page->pg_fix_bitf, - __bitstr_size((size_t)page->pg_fix_entries * btree->bitcnt)); + __bitstr_size((size_t)page->entries * btree->bitcnt)); /* Update any changes to the original on-page data items. */ WT_SKIP_FOREACH(ins, WT_COL_UPDATE_SINGLE(page)) { @@ -4029,9 +4158,8 @@ __rec_col_fix(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) } /* Calculate the number of entries per page remainder. */ - entry = page->pg_fix_entries; - nrecs = WT_FIX_BYTES_TO_ENTRIES( - btree, r->space_avail) - page->pg_fix_entries; + entry = page->entries; + nrecs = WT_FIX_BYTES_TO_ENTRIES(btree, r->space_avail) - page->entries; r->recno += entry; /* Walk any append list. */ @@ -4096,6 +4224,10 @@ __rec_col_fix(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_REF *pageref) * split. * * Boundary: split or write the page. + * + * No need to have a minimum split size boundary, all + * pages are filled 100% except the last, allowing it to + * grow in the future. */ __rec_incr(session, r, entry, __bitstr_size((size_t)entry * btree->bitcnt)); @@ -4154,7 +4286,7 @@ __rec_col_fix_slvg(WT_SESSION_IMPL *session, session, r, page, pageref->ref_recno, btree->maxleafpage)); /* We may not be taking all of the entries on the original page. */ - page_take = salvage->take == 0 ? page->pg_fix_entries : salvage->take; + page_take = salvage->take == 0 ? page->entries : salvage->take; page_start = salvage->skip == 0 ? 0 : salvage->skip; /* Calculate the number of entries per page. */ @@ -4252,10 +4384,13 @@ __rec_col_var_helper(WT_SESSION_IMPL *session, WT_RECONCILE *r, session, r, value->data, value->size, rle)); /* Boundary: split or write the page. */ - if (val->len > r->space_avail) - WT_RET(r->raw_compression ? - __rec_split_raw(session, r, val->len) : - __rec_split(session, r, val->len)); + if (r->raw_compression) { + if (val->len > r->space_avail) + WT_RET(__rec_split_raw(session, r, val->len)); + } else + if (WT_CHECK_CROSSING_BND(r, val->len)) + WT_RET(__rec_split_crossing_bnd( + session, r, val->len)); /* Copy the value onto the page. */ if (!deleted && !overflow_type && btree->dictionary) @@ -4450,8 +4585,8 @@ record_loop: /* * * Write a placeholder. */ - WT_ASSERT(session, - F_ISSET(r, WT_EVICT_UPDATE_RESTORE)); + WT_ASSERT(session, + F_ISSET(r, WT_EVICT_UPDATE_RESTORE)); data = "@"; size = 1; @@ -4918,11 +5053,12 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) r->cell_zero = false; /* Boundary: split or write the page. */ - if (key->len + val->len > r->space_avail) { - if (r->raw_compression) + if (r->raw_compression) { + if (key->len + val->len > r->space_avail) WT_ERR(__rec_split_raw( session, r, key->len + val->len)); - else { + } else + if (WT_CHECK_CROSSING_BND(r, key->len + val->len)) { /* * In one path above, we copied address blocks * from the page rather than building the actual @@ -4934,10 +5070,10 @@ __rec_row_int(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) WT_IKEY_DATA(ikey), ikey->size)); key_onpage_ovfl = false; } - WT_ERR(__rec_split( + + WT_ERR(__rec_split_crossing_bnd( session, r, key->len + val->len)); } - } /* Copy the key and value onto the page. */ __rec_copy_incr(session, r, key); @@ -4987,10 +5123,14 @@ __rec_row_merge(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) addr->addr, addr->size, __rec_vtype(addr), WT_RECNO_OOB); /* Boundary: split or write the page. */ - if (key->len + val->len > r->space_avail) - WT_RET(r->raw_compression ? - __rec_split_raw(session, r, key->len + val->len) : - __rec_split(session, r, key->len + val->len)); + if (r->raw_compression) { + if (key->len + val->len > r->space_avail) + WT_RET(__rec_split_raw( + session, r, key->len + val->len)); + } else + if (WT_CHECK_CROSSING_BND(r, key->len + val->len)) + WT_RET(__rec_split_crossing_bnd( + session, r, key->len + val->len)); /* Copy the key and value onto the page. */ __rec_copy_incr(session, r, key); @@ -5319,16 +5459,17 @@ build: } /* Boundary: split or write the page. */ - if (key->len + val->len > r->space_avail) { - if (r->raw_compression) + if (r->raw_compression) { + if (key->len + val->len > r->space_avail) WT_ERR(__rec_split_raw( session, r, key->len + val->len)); - else { + } else + if (WT_CHECK_CROSSING_BND(r, key->len + val->len)) { /* - * In one path above, we copied address blocks - * from the page rather than building the actual - * key. In that case, we have to build the key - * now because we are about to promote it. + * If we copied address blocks from the page + * rather than building the actual key, we have + * to build the key now because we are about to + * promote it. */ if (key_onpage_ovfl) { WT_ERR(__wt_dsk_cell_data_ref(session, @@ -5347,14 +5488,13 @@ build: if (!ovfl_key) WT_ERR( __rec_cell_build_leaf_key( - session, - r, NULL, 0, &ovfl_key)); + session, r, NULL, 0, + &ovfl_key)); } - WT_ERR(__rec_split( + WT_ERR(__rec_split_crossing_bnd( session, r, key->len + val->len)); } - } /* Copy the key/value pair onto the page. */ __rec_copy_incr(session, r, key); @@ -5417,11 +5557,12 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) WT_INSERT_KEY(ins), WT_INSERT_KEY_SIZE(ins), &ovfl_key)); /* Boundary: split or write the page. */ - if (key->len + val->len > r->space_avail) { - if (r->raw_compression) + if (r->raw_compression) { + if (key->len + val->len > r->space_avail) WT_RET(__rec_split_raw( session, r, key->len + val->len)); - else { + } else + if (WT_CHECK_CROSSING_BND(r, key->len + val->len)) { /* * Turn off prefix compression until a full key * written to the new page, and (unless already @@ -5433,14 +5574,13 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) if (!ovfl_key) WT_RET( __rec_cell_build_leaf_key( - session, - r, NULL, 0, &ovfl_key)); + session, r, NULL, 0, + &ovfl_key)); } - WT_RET(__rec_split( + WT_RET(__rec_split_crossing_bnd( session, r, key->len + val->len)); } - } /* Copy the key/value pair onto the page. */ __rec_copy_incr(session, r, key); @@ -5467,7 +5607,6 @@ __rec_row_leaf_insert(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_INSERT *ins) static int __rec_split_discard(WT_SESSION_IMPL *session, WT_PAGE *page) { - WT_DECL_RET; WT_PAGE_MODIFY *mod; WT_MULTI *multi; uint32_t i; @@ -5527,7 +5666,7 @@ __rec_split_discard(WT_SESSION_IMPL *session, WT_PAGE *page) break; } - return (ret); + return (0); } /* @@ -5553,13 +5692,14 @@ __rec_split_dump_keys(WT_SESSION_IMPL *session, WT_PAGE *page, WT_RECONCILE *r) __wt_verbose(session, WT_VERB_SPLIT, "starting key %s", __wt_buf_set_printable( - session, bnd->key.data, bnd->key.size, tkey)); + session, bnd->max_bnd_key.data, + bnd->max_bnd_key.size, tkey)); break; case WT_PAGE_COL_FIX: case WT_PAGE_COL_INT: case WT_PAGE_COL_VAR: __wt_verbose(session, WT_VERB_SPLIT, - "starting recno %" PRIu64, bnd->recno); + "starting recno %" PRIu64, bnd->max_bnd_recno); break; WT_ILLEGAL_VALUE_ERR(session); } @@ -5821,10 +5961,10 @@ __rec_split_row(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) /* We never set the first page's key, grab it from the original page. */ ref = r->ref; if (__wt_ref_is_root(ref)) - WT_RET(__wt_buf_set(session, &r->bnd[0].key, "", 1)); + WT_RET(__wt_buf_set(session, &r->bnd[0].max_bnd_key, "", 1)); else { __wt_ref_key(ref->home, ref, &p, &size); - WT_RET(__wt_buf_set(session, &r->bnd[0].key, p, size)); + WT_RET(__wt_buf_set(session, &r->bnd[0].max_bnd_key, p, size)); } /* Allocate, then initialize the array of replacement blocks. */ @@ -5832,8 +5972,8 @@ __rec_split_row(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) for (multi = mod->mod_multi, bnd = r->bnd, i = 0; i < r->bnd_next; ++multi, ++bnd, ++i) { - WT_RET(__wt_row_ikey_alloc(session, 0, - bnd->key.data, bnd->key.size, &multi->key.ikey)); + WT_RET(__wt_row_ikey_alloc(session, 0, bnd->max_bnd_key.data, + bnd->max_bnd_key.size, &multi->key.ikey)); /* * Copy any disk image. Don't take saved updates without a @@ -5880,7 +6020,7 @@ __rec_split_col(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_PAGE *page) for (multi = mod->mod_multi, bnd = r->bnd, i = 0; i < r->bnd_next; ++multi, ++bnd, ++i) { - multi->key.recno = bnd->recno; + multi->key.recno = bnd->max_bnd_recno; /* * Copy any disk image. Don't take saved updates without a @@ -6357,7 +6497,8 @@ __rec_dictionary_lookup( for (dp = __rec_dictionary_skip_search(r->dictionary_head, hash); dp != NULL && dp->hash == hash; dp = dp->next[0]) { WT_RET(__wt_cell_pack_data_match( - dp->cell, &val->cell, val->buf.data, &match)); + (WT_CELL *)((uint8_t *)r->disk_image.mem + dp->offset), + &val->cell, val->buf.data, &match)); if (match) { WT_STAT_DATA_INCR(session, rec_dictionary); *dpp = dp; @@ -6383,7 +6524,7 @@ __rec_dictionary_lookup( * know where on the page it will be written). */ next = r->dictionary[r->dictionary_next++]; - next->cell = NULL; /* Not necessary, just cautious. */ + next->offset = 0; /* Not necessary, just cautious. */ next->hash = hash; __rec_dictionary_skip_insert(r->dictionary_head, next, hash); *dpp = next; diff --git a/src/third_party/wiredtiger/src/schema/schema_alter.c b/src/third_party/wiredtiger/src/schema/schema_alter.c new file mode 100644 index 00000000000..26d800aa98e --- /dev/null +++ b/src/third_party/wiredtiger/src/schema/schema_alter.c @@ -0,0 +1,180 @@ +/*- + * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __alter_file -- + * Alter a file. + */ +static int +__alter_file(WT_SESSION_IMPL *session, const char *uri, const char *newcfg[]) +{ + WT_DECL_RET; + const char *cfg[4], *filename; + char *config, *newconfig; + + filename = uri; + newconfig = NULL; + if (!WT_PREFIX_SKIP(filename, "file:")) + return (__wt_unexpected_object_type(session, uri, "file:")); + + /* Find the URI */ + WT_RET(__wt_metadata_search(session, uri, &config)); + + WT_ASSERT(session, newcfg[0] != NULL); + /* + * Start with the base configuration because collapse is like + * a projection and if we are reading older metadata, it may not + * have all the components. + */ + cfg[0] = WT_CONFIG_BASE(session, file_meta); + cfg[1] = config; + cfg[2] = newcfg[0]; + cfg[3] = NULL; + WT_ERR(__wt_config_collapse(session, cfg, &newconfig)); + /* + * Only rewrite if there are changes. + */ + if (strcmp(config, newconfig) != 0) + WT_ERR(__wt_metadata_update(session, uri, newconfig)); + else + WT_STAT_CONN_INCR(session, session_table_alter_skip); + +err: __wt_free(session, config); + __wt_free(session, newconfig); + return (ret); +} + +/* + * __alter_colgroup -- + * WT_SESSION::alter for a colgroup. + */ +static int +__alter_colgroup( + WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + WT_COLGROUP *colgroup; + WT_DECL_RET; + + WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TABLE)); + + /* If we can get the colgroup, perform any potential alterations. */ + if ((ret = __wt_schema_get_colgroup( + session, uri, false, NULL, &colgroup)) == 0) + WT_TRET(__wt_schema_alter(session, colgroup->source, cfg)); + + return (ret); +} + +/* + * __alter_index -- + * WT_SESSION::alter for an index. + */ +static int +__alter_index( + WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + WT_INDEX *idx; + WT_DECL_RET; + + /* If we can get the index, perform any potential alterations. */ + if ((ret = __wt_schema_get_index( + session, uri, false, NULL, &idx)) == 0) + WT_TRET(__wt_schema_alter(session, idx->source, cfg)); + + return (ret); +} + +/* + * __alter_table -- + * WT_SESSION::alter for a table. + */ +static int +__alter_table(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + WT_COLGROUP *colgroup; + WT_DECL_RET; + WT_TABLE *table; + const char *name; + u_int i; + + name = uri; + (void)WT_PREFIX_SKIP(name, "table:"); + + WT_RET(__wt_schema_get_table( + session, name, strlen(name), true, &table)); + + /* + * Alter the column groups only if we are using the default + * column group. Otherwise the user should alter each + * index or column group explicitly. + */ + if (table->ncolgroups == 0) + for (i = 0; i < WT_COLGROUPS(table); i++) { + if ((colgroup = table->cgroups[i]) == NULL) + continue; + /* + * Alter the column group before updating the metadata + * to avoid the metadata for the table becoming + * inconsistent if we can't get exclusive access. + */ + WT_ERR(__wt_schema_alter( + session, colgroup->source, cfg)); + } +err: __wt_schema_release_table(session, table); + return (ret); +} + +/* + * __wt_schema_alter -- + * Process a WT_SESSION::alter operation for all supported types. + */ +int +__wt_schema_alter(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) +{ + WT_DATA_SOURCE *dsrc; + WT_DECL_RET; + + WT_RET(__wt_meta_track_on(session)); + + /* Paranoia: clear any handle from our caller. */ + session->dhandle = NULL; + + if (WT_PREFIX_MATCH(uri, "colgroup:")) + ret = __alter_colgroup(session, uri, cfg); + else if (WT_PREFIX_MATCH(uri, "file:")) + ret = __alter_file(session, uri, cfg); + else if (WT_PREFIX_MATCH(uri, "index:")) + ret = __alter_index(session, uri, cfg); + else if (WT_PREFIX_MATCH(uri, "lsm:")) + ret = __wt_lsm_tree_alter(session, uri, cfg); + else if (WT_PREFIX_MATCH(uri, "table:")) + ret = __alter_table(session, uri, cfg); + else if ((dsrc = __wt_schema_get_source(session, uri)) != NULL) + ret = dsrc->alter == NULL ? + __wt_object_unsupported(session, uri) : + dsrc->alter(dsrc, + &session->iface, uri, (WT_CONFIG_ARG *)cfg); + else + ret = __wt_bad_object_type(session, uri); + + /* + * Map WT_NOTFOUND to ENOENT, based on the assumption WT_NOTFOUND means + * there was no metadata entry. + */ + if (ret == WT_NOTFOUND) + ret = ENOENT; + + /* Bump the schema generation so that stale data is ignored. */ + ++S2C(session)->schema_gen; + + WT_TRET(__wt_meta_track_off(session, true, ret != 0)); + + return (ret); +} diff --git a/src/third_party/wiredtiger/src/schema/schema_create.c b/src/third_party/wiredtiger/src/schema/schema_create.c index 020d5e72c13..0677fa711a5 100644 --- a/src/third_party/wiredtiger/src/schema/schema_create.c +++ b/src/third_party/wiredtiger/src/schema/schema_create.c @@ -35,7 +35,7 @@ __wt_direct_io_size_check(WT_SESSION_IMPL *session, * units of its happy place. */ if (FLD_ISSET(conn->direct_io, - WT_DIRECT_IO_CHECKPOINT | WT_DIRECT_IO_DATA)) { + WT_DIRECT_IO_CHECKPOINT | WT_DIRECT_IO_DATA)) { align = (int64_t)conn->buffer_alignment; if (align != 0 && (cval.val < align || cval.val % align != 0)) WT_RET_MSG(session, EINVAL, @@ -601,7 +601,8 @@ __create_table(WT_SESSION_IMPL *session, if (ncolgroups == 0) { cgsize = strlen("colgroup:") + strlen(tablename) + 1; WT_ERR(__wt_calloc_def(session, cgsize, &cgname)); - snprintf(cgname, cgsize, "colgroup:%s", tablename); + WT_ERR(__wt_snprintf( + cgname, cgsize, "colgroup:%s", tablename)); WT_ERR(__create_colgroup( session, cgname, exclusive, config)); } diff --git a/src/third_party/wiredtiger/src/schema/schema_drop.c b/src/third_party/wiredtiger/src/schema/schema_drop.c index 65c955cf1e9..49801e4e5f9 100644 --- a/src/third_party/wiredtiger/src/schema/schema_drop.c +++ b/src/third_party/wiredtiger/src/schema/schema_drop.c @@ -30,7 +30,7 @@ __drop_file( WT_RET(__wt_schema_backup_check(session, filename)); /* Close all btree handles associated with this file. */ - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, ret = __wt_conn_dhandle_close_all(session, uri, force)); WT_RET(ret); @@ -75,7 +75,7 @@ __drop_colgroup( /* * __drop_index -- - * WT_SESSION::drop for a colgroup. + * WT_SESSION::drop for an index. */ static int __drop_index( @@ -85,7 +85,7 @@ __drop_index( WT_DECL_RET; WT_TABLE *table; - /* If we can get the colgroup, detach it from the table. */ + /* If we can get the index, detach it from the table. */ if ((ret = __wt_schema_get_index( session, uri, force, &table, &idx)) == 0) { table->idx_complete = false; @@ -136,7 +136,7 @@ __drop_table(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) if ((idx = table->indices[i]) == NULL) continue; /* - * Drop the column group before updating the metadata to avoid + * Drop the index before updating the metadata to avoid * the metadata for the table becoming inconsistent if we can't * get exclusive access. */ diff --git a/src/third_party/wiredtiger/src/schema/schema_list.c b/src/third_party/wiredtiger/src/schema/schema_list.c index 79e3ef1da7c..74ef5135a4a 100644 --- a/src/third_party/wiredtiger/src/schema/schema_list.c +++ b/src/third_party/wiredtiger/src/schema/schema_list.c @@ -25,7 +25,7 @@ __schema_add_table(WT_SESSION_IMPL *session, /* Make sure the metadata is open before getting other locks. */ WT_RET(__wt_metadata_cursor(session, NULL)); - WT_WITH_TABLE_LOCK(session, ret, + WT_WITH_TABLE_READ_LOCK(session, ret = __wt_schema_open_table( session, name, namelen, ok_incomplete, &table)); WT_RET(ret); diff --git a/src/third_party/wiredtiger/src/schema/schema_rename.c b/src/third_party/wiredtiger/src/schema/schema_rename.c index f512482c162..a374f4c2831 100644 --- a/src/third_party/wiredtiger/src/schema/schema_rename.c +++ b/src/third_party/wiredtiger/src/schema/schema_rename.c @@ -33,7 +33,7 @@ __rename_file( WT_RET(__wt_schema_backup_check(session, filename)); WT_RET(__wt_schema_backup_check(session, newfile)); /* Close any btree handles in the file. */ - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, ret = __wt_conn_dhandle_close_all(session, uri, false)); WT_ERR(ret); diff --git a/src/third_party/wiredtiger/src/schema/schema_util.c b/src/third_party/wiredtiger/src/schema/schema_util.c index 433224a868e..9de4b916a79 100644 --- a/src/third_party/wiredtiger/src/schema/schema_util.c +++ b/src/third_party/wiredtiger/src/schema/schema_util.c @@ -26,7 +26,7 @@ __wt_schema_backup_check(WT_SESSION_IMPL *session, const char *name) conn = S2C(session); if (!conn->hot_backup) return (0); - __wt_readlock(session, conn->hot_backup_lock); + __wt_readlock(session, &conn->hot_backup_lock); /* * There is a window at the end of a backup where the list has been * cleared from the connection but the flag is still set. It is safe @@ -34,7 +34,7 @@ __wt_schema_backup_check(WT_SESSION_IMPL *session, const char *name) */ if (!conn->hot_backup || (backup_list = conn->hot_backup_list) == NULL) { - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); return (0); } for (i = 0; backup_list[i] != NULL; ++i) { @@ -43,7 +43,7 @@ __wt_schema_backup_check(WT_SESSION_IMPL *session, const char *name) break; } } - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); return (ret); } diff --git a/src/third_party/wiredtiger/src/schema/schema_worker.c b/src/third_party/wiredtiger/src/schema/schema_worker.c index 52be76bb7a5..62cdd7d367b 100644 --- a/src/third_party/wiredtiger/src/schema/schema_worker.c +++ b/src/third_party/wiredtiger/src/schema/schema_worker.c @@ -49,7 +49,7 @@ __wt_schema_worker(WT_SESSION_IMPL *session, * any open file handles, including checkpoints. */ if (FLD_ISSET(open_flags, WT_DHANDLE_EXCLUSIVE)) { - WT_WITH_HANDLE_LIST_LOCK(session, + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, ret = __wt_conn_dhandle_close_all( session, uri, false)); WT_ERR(ret); @@ -73,13 +73,8 @@ __wt_schema_worker(WT_SESSION_IMPL *session, WT_ERR(__wt_schema_worker(session, idx->source, file_func, name_func, cfg, open_flags)); } else if (WT_PREFIX_MATCH(uri, "lsm:")) { - /* - * LSM compaction is handled elsewhere, but if we get here - * trying to compact files, don't descend into an LSM tree. - */ - if (file_func != __wt_compact) - WT_ERR(__wt_lsm_tree_worker(session, - uri, file_func, name_func, cfg, open_flags)); + WT_ERR(__wt_lsm_tree_worker(session, + uri, file_func, name_func, cfg, open_flags)); } else if (WT_PREFIX_SKIP(tablename, "table:")) { WT_ERR(__wt_schema_get_table(session, tablename, strlen(tablename), false, &table)); @@ -115,15 +110,12 @@ __wt_schema_worker(WT_SESSION_IMPL *session, } } else if ((dsrc = __wt_schema_get_source(session, uri)) != NULL) { wt_session = (WT_SESSION *)session; - if (file_func == __wt_compact && dsrc->compact != NULL) - WT_ERR(dsrc->compact( - dsrc, wt_session, uri, (WT_CONFIG_ARG *)cfg)); - else if (file_func == __wt_salvage && dsrc->salvage != NULL) + if (file_func == __wt_salvage && dsrc->salvage != NULL) WT_ERR(dsrc->salvage( - dsrc, wt_session, uri, (WT_CONFIG_ARG *)cfg)); + dsrc, wt_session, uri, (WT_CONFIG_ARG *)cfg)); else if (file_func == __wt_verify && dsrc->verify != NULL) WT_ERR(dsrc->verify( - dsrc, wt_session, uri, (WT_CONFIG_ARG *)cfg)); + dsrc, wt_session, uri, (WT_CONFIG_ARG *)cfg)); else if (file_func == __wt_checkpoint) ; else if (file_func == __wt_checkpoint_get_handles) diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 3afea383b08..b7daf0e2e02 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -58,7 +58,6 @@ int __wt_session_copy_values(WT_SESSION_IMPL *session) { WT_CURSOR *cursor; - WT_DECL_RET; TAILQ_FOREACH(cursor, &session->cursors, q) if (F_ISSET(cursor, WT_CURSTD_VALUE_INT)) { @@ -80,7 +79,7 @@ __wt_session_copy_values(WT_SESSION_IMPL *session) F_SET(cursor, WT_CURSTD_VALUE_EXT); } - return (ret); + return (0); } /* @@ -129,10 +128,48 @@ __session_clear(WT_SESSION_IMPL *session) * * For these reasons, be careful when clearing the session structure. */ - memset(session, 0, WT_SESSION_CLEAR_SIZE(session)); - session->hazard_size = 0; - session->nhazard = 0; + memset(session, 0, WT_SESSION_CLEAR_SIZE); + WT_INIT_LSN(&session->bg_sync_lsn); + + session->hazard_inuse = 0; + session->nhazard = 0; +} + +/* + * __session_alter -- + * Alter a table setting. + */ +static int +__session_alter(WT_SESSION *wt_session, const char *uri, const char *config) +{ + WT_DECL_RET; + WT_SESSION_IMPL *session; + + session = (WT_SESSION_IMPL *)wt_session; + + SESSION_API_CALL(session, alter, config, cfg); + + /* Disallow objects in the WiredTiger name space. */ + WT_ERR(__wt_str_name_check(session, uri)); + + /* + * We replace the default configuration listing with the current + * configuration. Otherwise the defaults for values that can be + * altered would override settings used by the user in create. + */ + cfg[0] = cfg[1]; + cfg[1] = NULL; + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, + WT_WITH_TABLE_WRITE_LOCK(session, + ret = __wt_schema_alter(session, uri, cfg)))); + +err: if (ret != 0) + WT_STAT_CONN_INCR(session, session_table_alter_fail); + else + WT_STAT_CONN_INCR(session, session_table_alter_success); + API_END_RET_NOTFOUND_MAP(session, ret); } /* @@ -197,9 +234,6 @@ __session_close(WT_SESSION *wt_session, const char *config) /* Release common session resources. */ WT_TRET(__wt_session_release_resources(session)); - /* Destroy the thread's mutex. */ - WT_TRET(__wt_cond_destroy(session, &session->cond)); - /* The API lock protects opening and closing of sessions. */ __wt_spin_lock(session, &conn->api_lock); @@ -249,18 +283,36 @@ __session_reconfigure(WT_SESSION *wt_session, const char *config) session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, reconfigure, config, cfg); + /* + * Note that this method only checks keys that are passed in by the + * application: we don't want to reset other session settings to their + * default values. + */ + WT_UNUSED(cfg); + if (F_ISSET(&session->txn, WT_TXN_RUNNING)) WT_ERR_MSG(session, EINVAL, "transaction in progress"); - WT_TRET(__wt_session_reset_cursors(session, false)); + WT_ERR(__wt_session_reset_cursors(session, false)); - WT_ERR(__wt_config_gets_def(session, cfg, "isolation", 0, &cval)); - if (cval.len != 0) + ret = __wt_config_getones(session, config, "isolation", &cval); + if (ret == 0 && cval.len != 0) { session->isolation = session->txn.isolation = WT_STRING_MATCH("snapshot", cval.str, cval.len) ? WT_ISO_SNAPSHOT : WT_STRING_MATCH("read-uncommitted", cval.str, cval.len) ? WT_ISO_READ_UNCOMMITTED : WT_ISO_READ_COMMITTED; + } + WT_ERR_NOTFOUND_OK(ret); + + ret = __wt_config_getones(session, config, "ignore_cache_size", &cval); + if (ret == 0) { + if (cval.val) + F_SET(session, WT_SESSION_NO_EVICTION); + else + F_CLR(session, WT_SESSION_NO_EVICTION); + } + WT_ERR_NOTFOUND_OK(ret); err: API_END_RET_NOTFOUND_MAP(session, ret); } @@ -465,8 +517,8 @@ __wt_session_create( { WT_DECL_RET; - WT_WITH_SCHEMA_LOCK(session, ret, - WT_WITH_TABLE_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, + WT_WITH_TABLE_WRITE_LOCK(session, ret = __wt_schema_create(session, uri, config))); return (ret); } @@ -659,8 +711,8 @@ __session_rebalance(WT_SESSION *wt_session, const char *uri, const char *config) SESSION_API_CALL(session, rebalance, config, cfg); /* Block out checkpoints to avoid spurious EBUSY errors. */ - WT_WITH_CHECKPOINT_LOCK(session, ret, - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_bt_rebalance, NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_REBALANCE))); @@ -712,9 +764,9 @@ __session_rename(WT_SESSION *wt_session, WT_ERR(__wt_str_name_check(session, uri)); WT_ERR(__wt_str_name_check(session, newuri)); - WT_WITH_CHECKPOINT_LOCK(session, ret, - WT_WITH_SCHEMA_LOCK(session, ret, - WT_WITH_TABLE_LOCK(session, ret, + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, + WT_WITH_TABLE_WRITE_LOCK(session, ret = __wt_schema_rename(session, uri, newuri, cfg)))); err: if (ret != 0) @@ -772,45 +824,6 @@ __session_reset(WT_SESSION *wt_session) err: API_END_RET_NOTFOUND_MAP(session, ret); } -/* - * __wt_session_drop -- - * Internal version of WT_SESSION::drop. - */ -int -__wt_session_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) -{ - WT_DECL_RET; - WT_CONFIG_ITEM cval; - bool checkpoint_wait, lock_wait; - - WT_RET(__wt_config_gets_def(session, cfg, "checkpoint_wait", 1, &cval)); - checkpoint_wait = cval.val != 0; - WT_RET(__wt_config_gets_def(session, cfg, "lock_wait", 1, &cval)); - lock_wait = cval.val != 0 || F_ISSET(session, WT_SESSION_LOCK_NO_WAIT); - - if (!lock_wait) - F_SET(session, WT_SESSION_LOCK_NO_WAIT); - - /* - * Take the checkpoint lock if there is a need to prevent the drop - * operation from failing with EBUSY due to an ongoing checkpoint. - */ - if (checkpoint_wait) - WT_WITH_CHECKPOINT_LOCK(session, ret, - WT_WITH_SCHEMA_LOCK(session, ret, - WT_WITH_TABLE_LOCK(session, ret, - ret = __wt_schema_drop(session, uri, cfg)))); - else - WT_WITH_SCHEMA_LOCK(session, ret, - WT_WITH_TABLE_LOCK(session, ret, - ret = __wt_schema_drop(session, uri, cfg))); - - if (!lock_wait) - F_CLR(session, WT_SESSION_LOCK_NO_WAIT); - - return (ret); -} - /* * __session_drop -- * WT_SESSION->drop method. @@ -818,8 +831,10 @@ __wt_session_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) static int __session_drop(WT_SESSION *wt_session, const char *uri, const char *config) { + WT_CONFIG_ITEM cval; WT_DECL_RET; WT_SESSION_IMPL *session; + bool checkpoint_wait, lock_wait; session = (WT_SESSION_IMPL *)wt_session; SESSION_API_CALL(session, drop, config, cfg); @@ -827,7 +842,37 @@ __session_drop(WT_SESSION *wt_session, const char *uri, const char *config) /* Disallow objects in the WiredTiger name space. */ WT_ERR(__wt_str_name_check(session, uri)); - ret = __wt_session_drop(session, uri, cfg); + WT_ERR(__wt_config_gets_def(session, cfg, "checkpoint_wait", 1, &cval)); + checkpoint_wait = cval.val != 0; + WT_ERR(__wt_config_gets_def(session, cfg, "lock_wait", 1, &cval)); + lock_wait = cval.val != 0; + + /* + * Take the checkpoint lock if there is a need to prevent the drop + * operation from failing with EBUSY due to an ongoing checkpoint. + */ + if (checkpoint_wait) { + if (lock_wait) + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, + WT_WITH_TABLE_WRITE_LOCK(session, ret = + __wt_schema_drop(session, uri, cfg)))); + else + WT_WITH_CHECKPOINT_LOCK_NOWAIT(session, ret, + WT_WITH_SCHEMA_LOCK_NOWAIT(session, ret, + WT_WITH_TABLE_WRITE_LOCK_NOWAIT(session, ret, + ret = + __wt_schema_drop(session, uri, cfg)))); + } else { + if (lock_wait) + WT_WITH_SCHEMA_LOCK(session, + WT_WITH_TABLE_WRITE_LOCK(session, + ret = __wt_schema_drop(session, uri, cfg))); + else + WT_WITH_SCHEMA_LOCK_NOWAIT(session, ret, + WT_WITH_TABLE_WRITE_LOCK_NOWAIT(session, ret, + ret = __wt_schema_drop(session, uri, cfg))); + } err: if (ret != 0) WT_STAT_CONN_INCR(session, session_table_drop_fail); @@ -1018,8 +1063,8 @@ __session_salvage(WT_SESSION *wt_session, const char *uri, const char *config) "configurations"); /* Block out checkpoints to avoid spurious EBUSY errors. */ - WT_WITH_CHECKPOINT_LOCK(session, ret, - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_salvage, NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_SALVAGE))); @@ -1161,10 +1206,15 @@ __wt_session_range_truncate(WT_SESSION_IMPL *session, done: err: /* - * Close any locally-opened start cursor. + * Close any locally-opened start cursor. Reset application cursors, + * they've possibly moved and the application cannot use them. */ if (local_start) WT_TRET(start->close(start)); + else + WT_TRET(start->reset(start)); + if (stop != NULL) + WT_TRET(stop->reset(stop)); return (ret); } @@ -1219,8 +1269,8 @@ __session_truncate(WT_SESSION *wt_session, session, uri, start, stop)); else /* Wait for checkpoints to avoid EBUSY errors. */ - WT_WITH_CHECKPOINT_LOCK(session, ret, - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_truncate(session, uri, cfg))); } else WT_ERR(__wt_session_range_truncate(session, uri, start, stop)); @@ -1275,8 +1325,8 @@ __session_upgrade(WT_SESSION *wt_session, const char *uri, const char *config) SESSION_API_CALL(session, upgrade, config, cfg); /* Block out checkpoints to avoid spurious EBUSY errors. */ - WT_WITH_CHECKPOINT_LOCK(session, ret, - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_upgrade, NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_UPGRADE))); @@ -1324,8 +1374,8 @@ __session_verify(WT_SESSION *wt_session, const char *uri, const char *config) "configurations"); /* Block out checkpoints to avoid spurious EBUSY errors. */ - WT_WITH_CHECKPOINT_LOCK(session, ret, - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_CHECKPOINT_LOCK(session, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __wt_verify, NULL, cfg, WT_DHANDLE_EXCLUSIVE | WT_BTREE_VERIFY))); @@ -1441,6 +1491,20 @@ __session_transaction_pinned_range(WT_SESSION *wt_session, uint64_t *prange) err: API_END_RET(session, ret); } +/* + * __transaction_sync_run_chk -- + * Check to decide if the transaction sync call should continue running. + */ +static bool +__transaction_sync_run_chk(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + + conn = S2C(session); + + return (FLD_ISSET(conn->flags, WT_CONN_SERVER_LOG)); +} + /* * __session_transaction_sync -- * WT_SESSION->transaction_sync method. @@ -1455,7 +1519,7 @@ __session_transaction_sync(WT_SESSION *wt_session, const char *config) WT_SESSION_IMPL *session; WT_TXN *txn; struct timespec now, start; - uint64_t timeout_ms, waited_ms; + uint64_t remaining_usec, timeout_ms, waited_ms; bool forever; session = (WT_SESSION_IMPL *)wt_session; @@ -1508,22 +1572,20 @@ __session_transaction_sync(WT_SESSION *wt_session, const char *config) __wt_epoch(session, &start); /* * Keep checking the LSNs until we find it is stable or we reach - * our timeout. + * our timeout, or there's some other reason to quit. */ while (__wt_log_cmp(&session->bg_sync_lsn, &log->sync_lsn) > 0) { + if (!__transaction_sync_run_chk(session)) + WT_ERR(ETIMEDOUT); + __wt_cond_signal(session, conn->log_file_cond); __wt_epoch(session, &now); waited_ms = WT_TIMEDIFF_MS(now, start); - if (forever || waited_ms < timeout_ms) - /* - * Note, we will wait an increasing amount of time - * each iteration, likely doubling. Also note that - * the function timeout value is in usecs (we are - * computing the wait time in msecs and passing that - * in, unchanged, as the usecs to wait). - */ - __wt_cond_wait(session, log->log_sync_cond, waited_ms); - else + if (forever || waited_ms < timeout_ms) { + remaining_usec = (timeout_ms - waited_ms) * WT_THOUSAND; + __wt_cond_wait(session, log->log_sync_cond, + remaining_usec, __transaction_sync_run_chk); + } else WT_ERR(ETIMEDOUT); } @@ -1588,7 +1650,7 @@ __session_checkpoint(WT_SESSION *wt_session, const char *config) WT_ERR_MSG(session, EINVAL, "Checkpoint not permitted in a transaction"); - ret = __wt_txn_checkpoint(session, cfg); + ret = __wt_txn_checkpoint(session, cfg, true); /* * Release common session resources (for example, checkpoint may acquire @@ -1639,7 +1701,7 @@ __session_snapshot(WT_SESSION *wt_session, const char *config) WT_ERR(__wt_txn_named_snapshot_config( session, cfg, &has_create, &has_drop)); - __wt_writelock(session, txn_global->nsnap_rwlock); + __wt_writelock(session, &txn_global->nsnap_rwlock); /* Drop any snapshots to be removed first. */ if (has_drop) @@ -1649,7 +1711,7 @@ __session_snapshot(WT_SESSION *wt_session, const char *config) if (has_create) WT_ERR(__wt_txn_named_snapshot_begin(session, cfg)); -err: __wt_writeunlock(session, txn_global->nsnap_rwlock); +err: __wt_writeunlock(session, &txn_global->nsnap_rwlock); API_END_RET_NOTFOUND_MAP(session, ret); } @@ -1680,6 +1742,7 @@ __open_session(WT_CONNECTION_IMPL *conn, static const WT_SESSION stds = { NULL, NULL, + __session_alter, __session_close, __session_reconfigure, __wt_session_strerror, @@ -1707,6 +1770,7 @@ __open_session(WT_CONNECTION_IMPL *conn, }, stds_readonly = { NULL, NULL, + __session_alter, __session_close, __session_reconfigure, __wt_session_strerror, @@ -1748,7 +1812,7 @@ __open_session(WT_CONNECTION_IMPL *conn, * closes the connection. This is particularly intended to catch * cases where server threads open sessions. */ - WT_ASSERT(session, F_ISSET(conn, WT_CONN_SERVER_RUN)); + WT_ASSERT(session, !F_ISSET(conn, WT_CONN_CLOSING)); /* Find the first inactive session slot. */ for (session_ret = conn->sessions, @@ -1776,8 +1840,6 @@ __open_session(WT_CONNECTION_IMPL *conn, session_ret->name = NULL; session_ret->id = i; - WT_ERR(__wt_cond_alloc(session, "session", false, &session_ret->cond)); - if (WT_SESSION_FIRST_USE(session_ret)) __wt_random_init(&session_ret->rnd); @@ -1810,17 +1872,13 @@ __open_session(WT_CONNECTION_IMPL *conn, * session close because access to it isn't serialized. Allocate the * first time we open this session. */ - if (WT_SESSION_FIRST_USE(session_ret)) - WT_ERR(__wt_calloc_def( - session, conn->hazard_max, &session_ret->hazard)); - - /* - * Set an initial size for the hazard array. It will be grown as - * required up to hazard_max. The hazard_size is reset on close, since - * __wt_hazard_close ensures the array is cleared - so it is safe to - * reset the starting size on each open. - */ - session_ret->hazard_size = 0; + if (WT_SESSION_FIRST_USE(session_ret)) { + WT_ERR(__wt_calloc_def(session, + WT_SESSION_INITIAL_HAZARD_SLOTS, &session_ret->hazard)); + session_ret->hazard_size = WT_SESSION_INITIAL_HAZARD_SLOTS; + session_ret->hazard_inuse = 0; + session_ret->nhazard = 0; + } /* Cache the offset of this session's statistics bucket. */ session_ret->stat_bucket = WT_STATS_SLOT_ID(session); diff --git a/src/third_party/wiredtiger/src/session/session_compact.c b/src/third_party/wiredtiger/src/session/session_compact.c index 66635007723..72c072e0fb8 100644 --- a/src/third_party/wiredtiger/src/session/session_compact.c +++ b/src/third_party/wiredtiger/src/session/session_compact.c @@ -53,11 +53,14 @@ * blocks, it can't easily know this is the case, and so we'll waste a lot of * effort trying to compact files that can't be compacted. * - * Now, to the actual process. First, we checkpoint the high-level object - * (which is potentially composed of multiple files): there are potentially - * many dirty blocks in the cache, and we want to write them out and then - * discard previous checkpoints so we have as many blocks as possible on the - * file's "available for reuse" list when we start compaction. + * Finally, compaction checkpoints are database-wide, otherwise we can corrupt + * file relationships, for example, an index checkpointed by compaction could + * be out of sync with the primary after a crash. + * + * Now, to the actual process. First, we checkpoint the database: there are + * potentially many dirty blocks in the cache, and we want to write them out + * and then discard previous checkpoints so we have as many blocks as possible + * on the file's "available for reuse" list when we start compaction. * * Then, we compact the high-level object. * @@ -96,29 +99,6 @@ * towards the beginning of the file, and then the file can be truncated. */ -/* - * __compact_uri_analyze -- - * Extract information relevant to deciding what work compact needs to - * do from a URI that is part of a table schema. - * Called via the schema_worker function. - */ -static int -__compact_uri_analyze(WT_SESSION_IMPL *session, const char *uri, bool *skipp) -{ - /* - * Add references to schema URI objects to the list of objects to be - * compacted. Skip over LSM trees or we will get false positives on - * the "file:" URIs for the chunks. - */ - if (WT_PREFIX_MATCH(uri, "lsm:")) { - session->compact->lsm_count++; - *skipp = true; - } else if (WT_PREFIX_MATCH(uri, "file:")) - session->compact->file_count++; - - return (0); -} - /* * __compact_start -- * Start object compaction. @@ -145,6 +125,29 @@ __compact_end(WT_SESSION_IMPL *session) return (bm->compact_end(bm, session)); } +/* + * __compact_uri_analyze -- + * Extract information relevant to deciding what work compact needs to + * do from a URI that is part of a table schema. + * Called via the schema_worker function. + */ +static int +__compact_uri_analyze(WT_SESSION_IMPL *session, const char *uri, bool *skipp) +{ + /* + * Add references to schema URI objects to the list of objects to be + * compacted. Skip over LSM trees or we will get false positives on + * the "file:" URIs for the chunks. + */ + if (WT_PREFIX_MATCH(uri, "lsm:")) { + session->compact->lsm_count++; + *skipp = true; + } else if (WT_PREFIX_MATCH(uri, "file:")) + session->compact->file_count++; + + return (0); +} + /* * __compact_handle_append -- * Gather a file handle to be compacted. @@ -157,10 +160,6 @@ __compact_handle_append(WT_SESSION_IMPL *session, const char *cfg[]) WT_UNUSED(cfg); - /* Make sure there is space for the next entry. */ - WT_RET(__wt_realloc_def(session, &session->op_handle_allocated, - session->op_handle_next + 1, &session->op_handle)); - WT_RET(__wt_session_get_btree( session, session->dhandle->name, NULL, NULL, 0)); @@ -170,16 +169,20 @@ __compact_handle_append(WT_SESSION_IMPL *session, const char *cfg[]) return (ret); } + /* Make sure there is space for the next entry. */ + WT_RET(__wt_realloc_def(session, &session->op_handle_allocated, + session->op_handle_next + 1, &session->op_handle)); + session->op_handle[session->op_handle_next++] = session->dhandle; return (0); } /* - * __session_compact_check_timeout -- + * __wt_session_compact_check_timeout -- * Check if the timeout has been exceeded. */ -static int -__session_compact_check_timeout(WT_SESSION_IMPL *session, struct timespec begin) +int +__wt_session_compact_check_timeout(WT_SESSION_IMPL *session) { struct timespec end; @@ -187,64 +190,116 @@ __session_compact_check_timeout(WT_SESSION_IMPL *session, struct timespec begin) return (0); __wt_epoch(session, &end); - if (session->compact->max_time < WT_TIMEDIFF_SEC(end, begin)) - return (ETIMEDOUT); - return (0); + return (session->compact->max_time > + WT_TIMEDIFF_SEC(end, session->compact->begin) ? 0 : ETIMEDOUT); } /* - * __compact_file -- - * Function to alternate between checkpoints and compaction calls. + * __compact_checkpoint -- + * Perform a checkpoint for compaction. */ static int -__compact_file(WT_SESSION_IMPL *session, const char *cfg[]) +__compact_checkpoint(WT_SESSION_IMPL *session) { - struct timespec start_time; - WT_DATA_HANDLE *dhandle; - WT_DECL_ITEM(t); + WT_TXN_GLOBAL *txn_global; WT_DECL_RET; - int i; + uint64_t txn_gen; + + /* + * Force compaction checkpoints: we don't want to skip it because the + * work we need to have done is done in the underlying block manager. + */ const char *checkpoint_cfg[] = { - WT_CONFIG_BASE(session, WT_SESSION_checkpoint), NULL, NULL }; + WT_CONFIG_BASE(session, WT_SESSION_checkpoint), "force=1", NULL }; + + /* Checkpoints take a lot of time, check if we've run out. */ + WT_RET(__wt_session_compact_check_timeout(session)); + + if ((ret = __wt_txn_checkpoint(session, checkpoint_cfg, false)) == 0) + return (0); + WT_RET_BUSY_OK(ret); + + /* + * If there's a checkpoint running, wait for it to complete, checking if + * we're out of time. If there's no checkpoint running or the checkpoint + * generation number changes, the checkpoint blocking us has completed. + */ + txn_global = &S2C(session)->txn_global; + for (txn_gen = txn_global->checkpoint_gen;;) { + WT_READ_BARRIER(); + if (!txn_global->checkpoint_running || + txn_gen != txn_global->checkpoint_gen) + break; + + WT_RET(__wt_session_compact_check_timeout(session)); + __wt_sleep(2, 0); + } + + return (0); +} - dhandle = session->dhandle; +/* + * __compact_worker -- + * Function to alternate between checkpoints and compaction calls. + */ +static int +__compact_worker(WT_SESSION_IMPL *session) +{ + WT_DECL_RET; + u_int i, loop; + bool didwork; /* - * Force the checkpoint: we don't want to skip it because the work we - * need to have done is done in the underlying block manager. + * Reset the handles' compaction skip flag (we don't bother setting + * or resetting it when we finish compaction, it's simpler to do it + * once, here). */ - WT_ERR(__wt_scr_alloc(session, 128, &t)); - WT_ERR(__wt_buf_fmt( - session, t, "target=(\"%s\"),force=1", dhandle->name)); - checkpoint_cfg[1] = t->data; + for (i = 0; i < session->op_handle_next; ++i) + session->op_handle[i]->compact_skip = false; - __wt_epoch(session, &start_time); + /* + * Perform an initial checkpoint (see this file's leading comment for + * details). + */ + WT_ERR(__compact_checkpoint(session)); /* - * We compact 10% of the file on each pass (but the overall size of the + * We compact 10% of a file on each pass (but the overall size of the * file is decreasing each time, so we're not compacting 10% of the * original file each time). Try 100 times (which is clearly more than - * we need); quit if we make no progress and check for a timeout each - * time through the loop. + * we need); quit if we make no progress. */ - for (i = 0; i < 100; ++i) { - WT_ERR(__wt_txn_checkpoint(session, checkpoint_cfg)); - - session->compact_state = WT_COMPACT_RUNNING; - WT_WITH_DHANDLE(session, dhandle, - ret = __wt_compact(session, cfg)); - WT_ERR(ret); - if (session->compact_state != WT_COMPACT_SUCCESS) + for (loop = 0; loop < 100; ++loop) { + /* Step through the list of files being compacted. */ + for (didwork = false, i = 0; i < session->op_handle_next; ++i) { + /* Skip objects where there's no more work. */ + if (session->op_handle[i]->compact_skip) + continue; + + session->compact_state = WT_COMPACT_RUNNING; + WT_WITH_DHANDLE(session, + session->op_handle[i], ret = __wt_compact(session)); + WT_ERR(ret); + + /* If we did no work, skip this file in the future. */ + if (session->compact_state == WT_COMPACT_SUCCESS) + didwork = true; + else + session->op_handle[i]->compact_skip = true; + } + if (!didwork) break; - WT_ERR(__wt_txn_checkpoint(session, checkpoint_cfg)); - WT_ERR(__wt_txn_checkpoint(session, checkpoint_cfg)); - WT_ERR(__session_compact_check_timeout(session, start_time)); + /* + * Perform two checkpoints (see this file's leading comment for + * details). + */ + WT_ERR(__compact_checkpoint(session)); + WT_ERR(__compact_checkpoint(session)); } err: session->compact_state = WT_COMPACT_NONE; - __wt_scr_free(session, &t); return (ret); } @@ -256,8 +311,9 @@ int __wt_session_compact( WT_SESSION *wt_session, const char *uri, const char *config) { - WT_COMPACT compact; + WT_COMPACT_STATE compact; WT_CONFIG_ITEM cval; + WT_DATA_SOURCE *dsrc; WT_DECL_RET; WT_SESSION_IMPL *session; WT_TXN *txn; @@ -270,6 +326,17 @@ __wt_session_compact( if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) goto err; + /* + * Non-LSM object compaction requires checkpoints, which are impossible + * in transactional contexts. Disallow in all contexts (there's no + * reason for LSM to allow this, possible or not), and check now so the + * error message isn't confusing. + */ + txn = &session->txn; + if (F_ISSET(txn, WT_TXN_RUNNING)) + WT_ERR_MSG(session, EINVAL, + "compaction not permitted in a transaction"); + /* Disallow objects in the WiredTiger name space. */ WT_ERR(__wt_str_name_check(session, uri)); @@ -277,18 +344,28 @@ __wt_session_compact( !WT_PREFIX_MATCH(uri, "file:") && !WT_PREFIX_MATCH(uri, "index:") && !WT_PREFIX_MATCH(uri, "lsm:") && - !WT_PREFIX_MATCH(uri, "table:")) - WT_ERR(__wt_bad_object_type(session, uri)); + !WT_PREFIX_MATCH(uri, "table:")) { + if ((dsrc = __wt_schema_get_source(session, uri)) != NULL) + ret = dsrc->compact == NULL ? + __wt_object_unsupported(session, uri) : + dsrc->compact( + dsrc, wt_session, uri, (WT_CONFIG_ARG *)cfg); + else + ret = __wt_bad_object_type(session, uri); + goto err; + } - /* Setup the structure in the session handle */ - memset(&compact, 0, sizeof(WT_COMPACT)); + /* Setup the session handle's compaction state structure. */ + memset(&compact, 0, sizeof(WT_COMPACT_STATE)); session->compact = &compact; + /* Compaction can be time-limited. */ WT_ERR(__wt_config_gets(session, cfg, "timeout", &cval)); session->compact->max_time = (uint64_t)cval.val; + __wt_epoch(session, &session->compact->begin); /* Find the types of data sources being compacted. */ - WT_WITH_SCHEMA_LOCK(session, ret, + WT_WITH_SCHEMA_LOCK(session, ret = __wt_schema_worker(session, uri, __compact_handle_append, __compact_uri_analyze, cfg, 0)); WT_ERR(ret); @@ -296,23 +373,8 @@ __wt_session_compact( if (session->compact->lsm_count != 0) WT_ERR(__wt_schema_worker( session, uri, NULL, __wt_lsm_compact, cfg, 0)); - if (session->compact->file_count != 0) { - /* - * File compaction requires checkpoints, which will fail in a - * transactional context. Check now so the error message isn't - * confusing. - */ - txn = &session->txn; - if (F_ISSET(txn, WT_TXN_RUNNING)) - WT_ERR_MSG(session, EINVAL, - " File compaction not permitted in a transaction"); - - for (i = 0; i < session->op_handle_next; ++i) { - WT_WITH_DHANDLE(session, session->op_handle[i], - ret = __compact_file(session, cfg)); - WT_ERR(ret); - } - } + if (session->compact->file_count != 0) + WT_ERR(__compact_worker(session)); err: session->compact = NULL; diff --git a/src/third_party/wiredtiger/src/session/session_dhandle.c b/src/third_party/wiredtiger/src/session/session_dhandle.c index 725854c6001..ffeb6137766 100644 --- a/src/third_party/wiredtiger/src/session/session_dhandle.c +++ b/src/third_party/wiredtiger/src/session/session_dhandle.c @@ -44,8 +44,7 @@ __session_discard_dhandle( TAILQ_REMOVE(&session->dhandles, dhandle_cache, q); TAILQ_REMOVE(&session->dhhash[bucket], dhandle_cache, hashq); - (void)__wt_atomic_sub32(&dhandle_cache->dhandle->session_ref, 1); - + WT_DHANDLE_RELEASE(dhandle_cache->dhandle); __wt_overwrite_and_free(session, dhandle_cache); } @@ -68,7 +67,7 @@ __session_find_dhandle(WT_SESSION_IMPL *session, retry: TAILQ_FOREACH(dhandle_cache, &session->dhhash[bucket], hashq) { dhandle = dhandle_cache->dhandle; if (WT_DHANDLE_INACTIVE(dhandle) && - !WT_IS_METADATA(session, dhandle)) { + !WT_IS_METADATA(dhandle)) { __session_discard_dhandle(session, dhandle_cache); /* We deleted our entry, retry from the start. */ goto retry; @@ -181,17 +180,17 @@ __wt_session_lock_dhandle( */ if (F_ISSET(dhandle, WT_DHANDLE_OPEN) && (!want_exclusive || lock_busy)) { - __wt_readlock(session, dhandle->rwlock); + __wt_readlock(session, &dhandle->rwlock); if (F_ISSET(dhandle, WT_DHANDLE_DEAD)) { *is_deadp = 1; - __wt_readunlock(session, dhandle->rwlock); + __wt_readunlock(session, &dhandle->rwlock); return (0); } is_open = F_ISSET(dhandle, WT_DHANDLE_OPEN); if (is_open && !want_exclusive) return (0); - __wt_readunlock(session, dhandle->rwlock); + __wt_readunlock(session, &dhandle->rwlock); } else is_open = false; @@ -201,10 +200,11 @@ __wt_session_lock_dhandle( * with another thread that successfully opens the file, we * don't want to block waiting to get exclusive access. */ - if ((ret = __wt_try_writelock(session, dhandle->rwlock)) == 0) { + if ((ret = + __wt_try_writelock(session, &dhandle->rwlock)) == 0) { if (F_ISSET(dhandle, WT_DHANDLE_DEAD)) { *is_deadp = 1; - __wt_writeunlock(session, dhandle->rwlock); + __wt_writeunlock(session, &dhandle->rwlock); return (0); } @@ -215,7 +215,7 @@ __wt_session_lock_dhandle( if (F_ISSET(dhandle, WT_DHANDLE_OPEN) && !want_exclusive) { lock_busy = false; - __wt_writeunlock(session, dhandle->rwlock); + __wt_writeunlock(session, &dhandle->rwlock); continue; } @@ -229,7 +229,8 @@ __wt_session_lock_dhandle( WT_ASSERT(session, !F_ISSET(dhandle, WT_DHANDLE_DEAD)); return (0); } - if (ret != EBUSY || (is_open && want_exclusive)) + if (ret != EBUSY || (is_open && want_exclusive) || + LF_ISSET(WT_DHANDLE_LOCK_ONLY)) return (ret); lock_busy = true; @@ -261,8 +262,8 @@ __wt_session_release_btree(WT_SESSION_IMPL *session) * can get a handle without special flags. */ if (F_ISSET(dhandle, WT_DHANDLE_DISCARD | WT_DHANDLE_DISCARD_FORCE)) { - __session_find_dhandle(session, - dhandle->name, dhandle->checkpoint, &dhandle_cache); + WT_SAVE_DHANDLE(session, __session_find_dhandle(session, + dhandle->name, dhandle->checkpoint, &dhandle_cache)); if (dhandle_cache != NULL) __session_discard_dhandle(session, dhandle_cache); } @@ -270,6 +271,16 @@ __wt_session_release_btree(WT_SESSION_IMPL *session) if (F_ISSET(dhandle, WT_DHANDLE_DISCARD_FORCE)) { ret = __wt_conn_btree_sync_and_close(session, false, true); F_CLR(dhandle, WT_DHANDLE_DISCARD_FORCE); + } else if (F_ISSET(btree, WT_BTREE_BULK)) { + WT_ASSERT(session, F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE) && + !F_ISSET(dhandle, WT_DHANDLE_DISCARD)); + /* + * Acquire the schema lock while completing a bulk load. This + * avoids racing with a checkpoint while it gathers a set + * of handles. + */ + WT_WITH_SCHEMA_LOCK(session, ret = + __wt_conn_btree_sync_and_close(session, false, false)); } else if (F_ISSET(dhandle, WT_DHANDLE_DISCARD) || F_ISSET(btree, WT_BTREE_SPECIAL_FLAGS)) { WT_ASSERT(session, F_ISSET(dhandle, WT_DHANDLE_EXCLUSIVE)); @@ -286,9 +297,9 @@ __wt_session_release_btree(WT_SESSION_IMPL *session) if (locked) { if (write_locked) { F_CLR(dhandle, WT_DHANDLE_EXCLUSIVE); - __wt_writeunlock(session, dhandle->rwlock); + __wt_writeunlock(session, &dhandle->rwlock); } else - __wt_readunlock(session, dhandle->rwlock); + __wt_readunlock(session, &dhandle->rwlock); } session->dhandle = NULL; @@ -401,7 +412,7 @@ __session_dhandle_sweep(WT_SESSION_IMPL *session) difftime(now, dhandle->timeofdeath) > conn->sweep_idle_time))) { WT_STAT_CONN_INCR(session, dh_session_handles); - WT_ASSERT(session, !WT_IS_METADATA(session, dhandle)); + WT_ASSERT(session, !WT_IS_METADATA(dhandle)); __session_discard_dhandle(session, dhandle_cache); } dhandle_cache = dhandle_cache_next; @@ -411,17 +422,27 @@ __session_dhandle_sweep(WT_SESSION_IMPL *session) /* * __session_find_shared_dhandle -- * Search for a data handle in the connection and add it to a session's - * cache. Since the data handle isn't locked, this must be called holding - * the handle list lock, and we must increment the handle's reference - * count before releasing it. + * cache. We must increment the handle's reference count while holding + * the handle list lock. */ static int __session_find_shared_dhandle( WT_SESSION_IMPL *session, const char *uri, const char *checkpoint) { - WT_RET(__wt_conn_dhandle_find(session, uri, checkpoint)); - (void)__wt_atomic_add32(&session->dhandle->session_ref, 1); - return (0); + WT_DECL_RET; + + WT_WITH_HANDLE_LIST_READ_LOCK(session, + if ((ret = __wt_conn_dhandle_find(session, uri, checkpoint)) == 0) + WT_DHANDLE_ACQUIRE(session->dhandle)); + + if (ret != WT_NOTFOUND) + return (ret); + + WT_WITH_HANDLE_LIST_WRITE_LOCK(session, + if ((ret = __wt_conn_dhandle_alloc(session, uri, checkpoint)) == 0) + WT_DHANDLE_ACQUIRE(session->dhandle)); + + return (ret); } /* @@ -449,16 +470,16 @@ __session_get_dhandle( * We didn't find a match in the session cache, search the shared * handle list and cache the handle we find. */ - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __session_find_shared_dhandle(session, uri, checkpoint)); - WT_RET(ret); + WT_RET(__session_find_shared_dhandle(session, uri, checkpoint)); /* * Fixup the reference count on failure (we incremented the reference * count while holding the handle-list lock). */ - if ((ret = __session_add_dhandle(session)) != 0) - (void)__wt_atomic_sub32(&session->dhandle->session_ref, 1); + if ((ret = __session_add_dhandle(session)) != 0) { + WT_DHANDLE_RELEASE(session->dhandle); + session->dhandle = NULL; + } return (ret); } @@ -504,17 +525,15 @@ __wt_session_get_btree(WT_SESSION_IMPL *session, * reopen handles in the meantime. A combination of the schema * and handle list locks are used to enforce this. */ - if (!F_ISSET(session, WT_SESSION_LOCKED_SCHEMA) || - !F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST)) { + if (!F_ISSET(session, WT_SESSION_LOCKED_SCHEMA)) { dhandle->excl_session = NULL; dhandle->excl_ref = 0; F_CLR(dhandle, WT_DHANDLE_EXCLUSIVE); - __wt_writeunlock(session, dhandle->rwlock); + __wt_writeunlock(session, &dhandle->rwlock); - WT_WITH_SCHEMA_LOCK(session, ret, - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __wt_session_get_btree( - session, uri, checkpoint, cfg, flags))); + WT_WITH_SCHEMA_LOCK(session, + ret = __wt_session_get_btree( + session, uri, checkpoint, cfg, flags)); return (ret); } @@ -531,7 +550,7 @@ __wt_session_get_btree(WT_SESSION_IMPL *session, dhandle->excl_session = NULL; dhandle->excl_ref = 0; F_CLR(dhandle, WT_DHANDLE_EXCLUSIVE); - __wt_writeunlock(session, dhandle->rwlock); + __wt_writeunlock(session, &dhandle->rwlock); WT_RET(ret); } @@ -552,7 +571,7 @@ __wt_session_get_btree(WT_SESSION_IMPL *session, int __wt_session_lock_checkpoint(WT_SESSION_IMPL *session, const char *checkpoint) { - WT_DATA_HANDLE *dhandle, *saved_dhandle; + WT_DATA_HANDLE *saved_dhandle; WT_DECL_RET; WT_ASSERT(session, WT_META_TRACKING(session)); @@ -560,31 +579,33 @@ __wt_session_lock_checkpoint(WT_SESSION_IMPL *session, const char *checkpoint) /* * Get the checkpoint handle exclusive, so no one else can access it - * while we are creating the new checkpoint. + * while we are creating the new checkpoint. Hold the lock until the + * checkpoint completes. */ WT_ERR(__wt_session_get_btree(session, saved_dhandle->name, checkpoint, NULL, WT_DHANDLE_EXCLUSIVE | WT_DHANDLE_LOCK_ONLY)); + if ((ret = __wt_meta_track_handle_lock(session, false)) != 0) { + WT_TRET(__wt_session_release_btree(session)); + goto err; + } /* - * Flush any pages in this checkpoint from the cache (we are about to - * re-write the checkpoint which will mean cached pages no longer have - * valid contents). This is especially noticeable with memory mapped - * files, since changes to the underlying file are visible to the in - * memory pages. + * Get exclusive access to the handle and then flush any pages in this + * checkpoint from the cache (we are about to re-write the checkpoint + * which will mean cached pages no longer have valid contents). This + * is especially noticeable with memory mapped files, since changes to + * the underlying file are visible to the in-memory pages. */ + WT_ERR(__wt_evict_file_exclusive_on(session)); WT_ERR(__wt_cache_op(session, WT_SYNC_DISCARD)); /* * We lock checkpoint handles that we are overwriting, so the handle * must be closed when we release it. */ - dhandle = session->dhandle; - F_SET(dhandle, WT_DHANDLE_DISCARD); - - WT_ERR(__wt_meta_track_handle_lock(session, false)); + F_SET(session->dhandle, WT_DHANDLE_DISCARD); - /* Restore the original btree in the session. */ + /* Restore the original data handle in the session. */ err: session->dhandle = saved_dhandle; - return (ret); } diff --git a/src/third_party/wiredtiger/src/session/session_salvage.c b/src/third_party/wiredtiger/src/session/session_salvage.c index 983b28dd8ea..12ce71cdbb0 100644 --- a/src/third_party/wiredtiger/src/session/session_salvage.c +++ b/src/third_party/wiredtiger/src/session/session_salvage.c @@ -54,6 +54,6 @@ __wt_salvage(WT_SESSION_IMPL *session, const char *cfg[]) WT_ERR(__wt_meta_ckptlist_set( session, dhandle->name, ckptbase, NULL)); -err: __wt_meta_ckptlist_free(session, ckptbase); +err: __wt_meta_ckptlist_free(session, &ckptbase); return (ret); } diff --git a/src/third_party/wiredtiger/src/support/cond_auto.c b/src/third_party/wiredtiger/src/support/cond_auto.c index a3ae67f5baa..600e5eab0ff 100644 --- a/src/third_party/wiredtiger/src/support/cond_auto.c +++ b/src/third_party/wiredtiger/src/support/cond_auto.c @@ -1,29 +1,9 @@ /*- - * Public Domain 2014-2016 MongoDB, Inc. - * Public Domain 2008-2014 WiredTiger, Inc. + * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. * - * This is free and unencumbered software released into the public domain. - * - * Anyone is free to copy, modify, publish, use, compile, sell, or - * distribute this software, either in source code form or as a compiled - * binary, for any purpose, commercial or non-commercial, and by any - * means. - * - * In jurisdictions that recognize copyright laws, the author or authors - * of this software dedicate any and all copyright interest in the - * software to the public domain. We make this dedication for the benefit - * of the public at large and to the detriment of our heirs and - * successors. We intend this dedication to be an overt act of - * relinquishment in perpetuity of all present and future rights to this - * software under copyright law. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. + * See the file LICENSE for redistribution information. */ #include "wt_internal.h" @@ -38,13 +18,12 @@ * Allocate and initialize an automatically adjusting condition variable. */ int -__wt_cond_auto_alloc( - WT_SESSION_IMPL *session, const char *name, - bool is_signalled, uint64_t min, uint64_t max, WT_CONDVAR **condp) +__wt_cond_auto_alloc(WT_SESSION_IMPL *session, + const char *name, uint64_t min, uint64_t max, WT_CONDVAR **condp) { WT_CONDVAR *cond; - WT_RET(__wt_cond_alloc(session, name, is_signalled, condp)); + WT_RET(__wt_cond_alloc(session, name, condp)); cond = *condp; cond->min_wait = min; @@ -54,34 +33,20 @@ __wt_cond_auto_alloc( return (0); } -/* - * __wt_cond_auto_signal -- - * Signal a condition variable. - */ -void -__wt_cond_auto_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond) -{ - - WT_ASSERT(session, cond->min_wait != 0); - __wt_cond_signal(session, cond); -} - /* * __wt_cond_auto_wait_signal -- * Wait on a mutex, optionally timing out. If we get it before the time * out period expires, let the caller know. - * TODO: Can this version of the API be removed, now that we have the - * auto adjusting condition variables? */ void -__wt_cond_auto_wait_signal( - WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress, bool *signalled) +__wt_cond_auto_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, + bool progress, bool (*run_func)(WT_SESSION_IMPL *), bool *signalled) { uint64_t delta; /* * Catch cases where this function is called with a condition variable - * that was initialized non-auto. + * that wasn't initialized to do automatic adjustments. */ WT_ASSERT(session, cond->min_wait != 0); @@ -94,7 +59,8 @@ __wt_cond_auto_wait_signal( cond->max_wait, cond->prev_wait + delta); } - __wt_cond_wait_signal(session, cond, cond->prev_wait, signalled); + __wt_cond_wait_signal( + session, cond, cond->prev_wait, run_func, signalled); if (progress || *signalled) WT_STAT_CONN_INCR(session, cond_auto_wait_reset); @@ -108,24 +74,10 @@ __wt_cond_auto_wait_signal( * out period expires, let the caller know. */ void -__wt_cond_auto_wait( - WT_SESSION_IMPL *session, WT_CONDVAR *cond, bool progress) +__wt_cond_auto_wait(WT_SESSION_IMPL *session, + WT_CONDVAR *cond, bool progress, bool (*run_func)(WT_SESSION_IMPL *)) { - bool signalled; - - /* - * Call the signal version so the wait period is reset if the - * condition is woken explicitly. - */ - __wt_cond_auto_wait_signal(session, cond, progress, &signalled); -} + bool notused; -/* - * __wt_cond_auto_destroy -- - * Destroy a condition variable. - */ -int -__wt_cond_auto_destroy(WT_SESSION_IMPL *session, WT_CONDVAR **condp) -{ - return (__wt_cond_destroy(session, condp)); + __wt_cond_auto_wait_signal(session, cond, progress, run_func, ¬used); } diff --git a/src/third_party/wiredtiger/src/support/crypto.c b/src/third_party/wiredtiger/src/support/crypto.c index ab94ec2c829..cce0d228832 100644 --- a/src/third_party/wiredtiger/src/support/crypto.c +++ b/src/third_party/wiredtiger/src/support/crypto.c @@ -133,5 +133,4 @@ __wt_encrypt_size(WT_SESSION_IMPL *session, return; *sizep = incoming_size + kencryptor->size_const + WT_ENCRYPT_LEN_SIZE; - return; } diff --git a/src/third_party/wiredtiger/src/support/err.c b/src/third_party/wiredtiger/src/support/err.c index e48225c2c13..57efde72b23 100644 --- a/src/third_party/wiredtiger/src/support/err.c +++ b/src/third_party/wiredtiger/src/support/err.c @@ -102,9 +102,10 @@ __handler_failure(WT_SESSION_IMPL *session, */ char s[256]; - (void)snprintf(s, sizeof(s), + if (__wt_snprintf(s, sizeof(s), "application %s event handler failed: %s", - which, __wt_strerror(session, error, NULL, 0)); + which, __wt_strerror(session, error, NULL, 0)) != 0) + return; /* * Use the error handler to report the failure, unless it was the error @@ -148,6 +149,23 @@ __wt_event_handler_set(WT_SESSION_IMPL *session, WT_EVENT_HANDLER *handler) session->event_handler = handler; } +#define WT_ERROR_APPEND(p, remain, ...) do { \ + size_t __len; \ + WT_ERR(__wt_snprintf_len_set(p, remain, &__len, __VA_ARGS__)); \ + if (__len > remain) \ + __len = remain; \ + p += __len; \ + remain -= __len; \ +} while (0) +#define WT_ERROR_APPEND_AP(p, remain, ...) do { \ + size_t __len; \ + WT_ERR(__wt_vsnprintf_len_set(p, remain, &__len, __VA_ARGS__)); \ + if (__len > remain) \ + __len = remain; \ + p += __len; \ + remain -= __len; \ +} while (0) + /* * __wt_eventv -- * Report a message to an event handler. @@ -161,9 +179,9 @@ __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, WT_DECL_RET; WT_SESSION *wt_session; struct timespec ts; - size_t len, remain, wlen; + size_t len, remain; const char *err, *prefix; - char *end, *p, tid[128]; + char *p, tid[128]; /* * We're using a stack buffer because we want error messages no matter @@ -174,6 +192,8 @@ __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, * Buffer placed at the end of the stack in case snprintf overflows. */ char s[2048]; + p = s; + remain = sizeof(s); /* * !!! @@ -185,24 +205,8 @@ __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, * first session, but if the allocation of the first session fails, for * example, we can end up here without a session.) */ - if (session == NULL) { - if (fprintf(stderr, - "WiredTiger Error%s%s: ", - error == 0 ? "" : ": ", - error == 0 ? "" : - __wt_strerror(session, error, NULL, 0)) < 0) - ret = EIO; - if (vfprintf(stderr, fmt, ap) < 0) - ret = EIO; - if (fprintf(stderr, "\n") < 0) - ret = EIO; - if (fflush(stderr) != 0) - ret = EIO; - return (ret); - } - - p = s; - end = s + sizeof(s); + if (session == NULL) + goto err; /* * We have several prefixes for the error message: a timestamp and the @@ -211,42 +215,24 @@ __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, * followed by a colon. */ __wt_epoch(session, &ts); - __wt_thread_id(tid, sizeof(tid)); - remain = WT_PTRDIFF(end, p); - wlen = (size_t)snprintf(p, remain, "[%" PRIuMAX ":%" PRIuMAX "][%s]", + WT_ERR(__wt_thread_id(tid, sizeof(tid))); + WT_ERROR_APPEND(p, remain, + "[%" PRIuMAX ":%" PRIuMAX "][%s]", (uintmax_t)ts.tv_sec, (uintmax_t)ts.tv_nsec / WT_THOUSAND, tid); - p = wlen >= remain ? end : p + wlen; - if ((prefix = S2C(session)->error_prefix) != NULL) { - remain = WT_PTRDIFF(end, p); - wlen = (size_t)snprintf(p, remain, ", %s", prefix); - p = wlen >= remain ? end : p + wlen; - } + if ((prefix = S2C(session)->error_prefix) != NULL) + WT_ERROR_APPEND(p, remain, ", %s", prefix); prefix = session->dhandle == NULL ? NULL : session->dhandle->name; - if (prefix != NULL) { - remain = WT_PTRDIFF(end, p); - wlen = (size_t)snprintf(p, remain, ", %s", prefix); - p = wlen >= remain ? end : p + wlen; - } - if ((prefix = session->name) != NULL) { - remain = WT_PTRDIFF(end, p); - wlen = (size_t)snprintf(p, remain, ", %s", prefix); - p = wlen >= remain ? end : p + wlen; - } - remain = WT_PTRDIFF(end, p); - wlen = (size_t)snprintf(p, remain, ": "); - p = wlen >= remain ? end : p + wlen; - - if (file_name != NULL) { - remain = WT_PTRDIFF(end, p); - wlen = (size_t) - snprintf(p, remain, "%s, %d: ", file_name, line_number); - p = wlen >= remain ? end : p + wlen; - } + if (prefix != NULL) + WT_ERROR_APPEND(p, remain, ", %s", prefix); + if ((prefix = session->name) != NULL) + WT_ERROR_APPEND(p, remain, ", %s", prefix); + WT_ERROR_APPEND(p, remain, ": "); + + if (file_name != NULL) + WT_ERROR_APPEND(p, remain, "%s, %d: ", file_name, line_number); - remain = WT_PTRDIFF(end, p); - wlen = (size_t)vsnprintf(p, remain, fmt, ap); - p = wlen >= remain ? end : p + wlen; + WT_ERROR_APPEND_AP(p, remain, fmt, ap); if (error != 0) { /* @@ -261,10 +247,8 @@ __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, */ err = __wt_strerror(session, error, NULL, 0); len = strlen(err); - if (WT_PTRDIFF(p, s) < len || strcmp(p - len, err) != 0) { - remain = WT_PTRDIFF(end, p); - (void)snprintf(p, remain, ": %s", err); - } + if (WT_PTRDIFF(p, s) < len || strcmp(p - len, err) != 0) + WT_ERROR_APPEND(p, remain, ": %s", err); } /* @@ -279,7 +263,7 @@ __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, * * If an application-specified error message handler fails, complain * using the default error handler. If the default error handler fails, - * there's nothing to do. + * fallback to stderr. */ wt_session = (WT_SESSION *)session; handler = session->event_handler; @@ -293,6 +277,21 @@ __wt_eventv(WT_SESSION_IMPL *session, bool msg_event, int error, __handler_failure(session, ret, "error", true); } + if (ret != 0) { +err: if (fprintf(stderr, + "WiredTiger Error%s%s: ", + error == 0 ? "" : ": ", + error == 0 ? "" : + __wt_strerror(session, error, NULL, 0)) < 0) + WT_TRET(EIO); + if (vfprintf(stderr, fmt, ap) < 0) + WT_TRET(EIO); + if (fprintf(stderr, "\n") < 0) + WT_TRET(EIO); + if (fflush(stderr) != 0) + WT_TRET(EIO); + } + return (ret); } @@ -304,6 +303,7 @@ void __wt_err(WT_SESSION_IMPL *session, int error, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((cold)) WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { va_list ap; @@ -375,7 +375,7 @@ info_msg(WT_SESSION_IMPL *session, const char *fmt, va_list ap) */ char s[2048]; - (void)vsnprintf(s, sizeof(s), fmt, ap); + WT_RET(__wt_vsnprintf(s, sizeof(s), fmt, ap)); wt_session = (WT_SESSION *)session; handler = session->event_handler; @@ -470,6 +470,7 @@ __wt_assert(WT_SESSION_IMPL *session, #ifdef HAVE_DIAGNOSTIC WT_GCC_FUNC_ATTRIBUTE((noreturn)) #endif + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { va_list ap; @@ -491,6 +492,7 @@ __wt_assert(WT_SESSION_IMPL *session, int __wt_panic(WT_SESSION_IMPL *session) WT_GCC_FUNC_ATTRIBUTE((cold)) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { F_SET(S2C(session), WT_CONN_PANIC); __wt_err(session, WT_PANIC, "the process must exit and restart"); @@ -515,6 +517,7 @@ __wt_panic(WT_SESSION_IMPL *session) int __wt_illegal_value(WT_SESSION_IMPL *session, const char *name) WT_GCC_FUNC_ATTRIBUTE((cold)) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { __wt_errx(session, "%s%s%s", name == NULL ? "" : name, name == NULL ? "" : ": ", diff --git a/src/third_party/wiredtiger/src/support/hazard.c b/src/third_party/wiredtiger/src/support/hazard.c index b1978dbe97a..7e88ad183fe 100644 --- a/src/third_party/wiredtiger/src/support/hazard.c +++ b/src/third_party/wiredtiger/src/support/hazard.c @@ -12,6 +12,48 @@ static void __hazard_dump(WT_SESSION_IMPL *); #endif +/* + * hazard_grow -- + * Grow a hazard pointer array. + */ +static int +hazard_grow(WT_SESSION_IMPL *session) +{ + WT_HAZARD *nhazard; + size_t size; + void *ohazard; + + /* + * Allocate a new, larger hazard pointer array and copy the contents of + * the original into place. + */ + size = session->hazard_size; + WT_RET(__wt_calloc_def(session, size * 2, &nhazard)); + memcpy(nhazard, session->hazard, size * sizeof(WT_HAZARD)); + + /* + * Swap the new hazard pointer array into place after initialization + * is complete (initialization must complete before eviction can see + * the new hazard pointer array), then schedule the original to be + * freed. + */ + ohazard = session->hazard; + WT_PUBLISH(session->hazard, nhazard); + + __wt_spin_lock(session, &S2C(session)->api_lock); + __wt_conn_foc_add(session, ohazard); + __wt_spin_unlock(session, &S2C(session)->api_lock); + + /* + * Increase the size of the session's pointer array after swapping it + * into place (the session's reference must be updated before eviction + * can see the new size). + */ + WT_PUBLISH(session->hazard_size, (uint32_t)(size * 2)); + + return (0); +} + /* * __wt_hazard_set -- * Set a hazard pointer. @@ -23,18 +65,62 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp #endif ) { - WT_BTREE *btree; - WT_CONNECTION_IMPL *conn; WT_HAZARD *hp; - int restarts = 0; - btree = S2BT(session); - conn = S2C(session); *busyp = false; /* If a file can never be evicted, hazard pointers aren't required. */ - if (F_ISSET(btree, WT_BTREE_IN_MEMORY)) + if (F_ISSET(S2BT(session), WT_BTREE_IN_MEMORY)) + return (0); + + /* + * If there isn't a valid page, we're done. This read can race with + * eviction and splits, we re-check it after a barrier to make sure + * we have a valid reference. + */ + if (ref->state != WT_REF_MEM) { + *busyp = true; return (0); + } + + /* If we have filled the current hazard pointer array, grow it. */ + if (session->nhazard >= session->hazard_size) { + WT_ASSERT(session, + session->nhazard == session->hazard_size && + session->hazard_inuse == session->hazard_size); + WT_RET(hazard_grow(session)); + } + + /* + * If there are no available hazard pointer slots, make another one + * visible. + */ + if (session->nhazard >= session->hazard_inuse) { + WT_ASSERT(session, + session->nhazard == session->hazard_inuse && + session->hazard_inuse < session->hazard_size); + hp = &session->hazard[session->hazard_inuse++]; + } else { + WT_ASSERT(session, + session->nhazard < session->hazard_inuse && + session->hazard_inuse <= session->hazard_size); + + /* + * There must be an empty slot in the array, find it. Skip most + * of the active slots by starting after the active count slot; + * there may be a free slot before there, but checking is + * expensive. If we reach the end of the array, continue the + * search from the beginning of the array. + */ + for (hp = session->hazard + session->nhazard;; ++hp) { + if (hp >= session->hazard + session->hazard_inuse) + hp = session->hazard; + if (hp->ref == NULL) + break; + } + } + + WT_ASSERT(session, hp->ref == NULL); /* * Do the dance: @@ -49,83 +135,43 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp * pointer before it discards the page (the eviction server sets the * state to WT_REF_LOCKED, then flushes memory and checks the hazard * pointers). - * - * For sessions with many active hazard pointers, skip most of the - * active slots: there may be a free slot in there, but checking is - * expensive. Most hazard pointers are released quickly: optimize - * for that case. */ - for (hp = session->hazard + session->nhazard;; ++hp) { - /* - * If we get to the end of the array, either: - * 1. If we know there are free slots somewhere, and this is - * the first time through, continue the search from the - * start. Don't actually continue the loop because that - * will skip the first slot. - * 2. If we have searched all the way through and we have - * allocated the maximum number of slots, give up. - * 3. Allocate another increment of slots, up to the maximum. - * The slot we are on should now be available. - */ - if (hp >= session->hazard + session->hazard_size) { - if (session->nhazard < session->hazard_size && - restarts++ == 0) - hp = session->hazard; - else if (session->hazard_size >= conn->hazard_max) - break; - else - WT_PUBLISH(session->hazard_size, WT_MIN( - session->hazard_size + WT_HAZARD_INCR, - conn->hazard_max)); - } - - if (hp->page != NULL) - continue; - - hp->page = ref->page; + hp->ref = ref; #ifdef HAVE_DIAGNOSTIC - hp->file = file; - hp->line = line; + hp->file = file; + hp->line = line; #endif - /* Publish the hazard pointer before reading page's state. */ - WT_FULL_BARRIER(); + /* Publish the hazard pointer before reading page's state. */ + WT_FULL_BARRIER(); - /* - * Check if the page state is still valid, where valid means a - * state of WT_REF_MEM and the pointer is unchanged. (The - * pointer can change, it means the page was evicted between - * the time we set our hazard pointer and the publication. It - * would theoretically be possible for the page to be evicted - * and a different page read into the same memory, so the - * pointer hasn't changed but the contents have. That's OK, we - * found this page using the tree's key space, whatever page we - * find here is the page for us to use.) - */ - if (ref->page == hp->page && ref->state == WT_REF_MEM) { - ++session->nhazard; - return (0); - } + /* + * Check if the page state is still valid, where valid means a + * state of WT_REF_MEM. + */ + if (ref->state == WT_REF_MEM) { + ++session->nhazard; /* - * The page isn't available, it's being considered for eviction - * (or being evicted, for all we know). If the eviction server - * sees our hazard pointer before evicting the page, it will - * return the page to use, no harm done, if it doesn't, it will - * go ahead and complete the eviction. - * - * We don't bother publishing this update: the worst case is we - * prevent some random page from being evicted. + * Callers require a barrier here so operations holding + * the hazard pointer see consistent data. */ - hp->page = NULL; - *busyp = true; + WT_READ_BARRIER(); return (0); } -#ifdef HAVE_DIAGNOSTIC - __hazard_dump(session); -#endif - WT_RET_MSG(session, ENOMEM, - "session %p: hazard pointer table full", (void *)session); + /* + * The page isn't available, it's being considered for eviction + * (or being evicted, for all we know). If the eviction server + * sees our hazard pointer before evicting the page, it will + * return the page to use, no harm done, if it doesn't, it will + * go ahead and complete the eviction. + * + * We don't bother publishing this update: the worst case is we + * prevent some random page from being evicted. + */ + hp->ref = NULL; + *busyp = true; + return (0); } /* @@ -133,25 +179,22 @@ __wt_hazard_set(WT_SESSION_IMPL *session, WT_REF *ref, bool *busyp * Clear a hazard pointer. */ int -__wt_hazard_clear(WT_SESSION_IMPL *session, WT_PAGE *page) +__wt_hazard_clear(WT_SESSION_IMPL *session, WT_REF *ref) { - WT_BTREE *btree; WT_HAZARD *hp; - btree = S2BT(session); - /* If a file can never be evicted, hazard pointers aren't required. */ - if (F_ISSET(btree, WT_BTREE_IN_MEMORY)) + if (F_ISSET(S2BT(session), WT_BTREE_IN_MEMORY)) return (0); /* * Clear the caller's hazard pointer. * The common pattern is LIFO, so do a reverse search. */ - for (hp = session->hazard + session->hazard_size - 1; + for (hp = session->hazard + session->hazard_inuse - 1; hp >= session->hazard; --hp) - if (hp->page == page) { + if (hp->ref == ref) { /* * We don't publish the hazard pointer clear in the * general case. It's not required for correctness; @@ -160,14 +203,18 @@ __wt_hazard_clear(WT_SESSION_IMPL *session, WT_PAGE *page) * generation number was just set, it's unlikely the * page will be selected for eviction. */ - hp->page = NULL; + hp->ref = NULL; /* * If this was the last hazard pointer in the session, * reset the size so that checks can skip this session. + * + * A write-barrier() is necessary before the change to + * the in-use value, the number of active references + * can never be less than the number of in-use slots. */ if (--session->nhazard == 0) - WT_PUBLISH(session->hazard_size, 0); + WT_PUBLISH(session->hazard_inuse, 0); return (0); } @@ -177,7 +224,7 @@ __wt_hazard_clear(WT_SESSION_IMPL *session, WT_PAGE *page) */ WT_PANIC_RET(session, EINVAL, "session %p: clear hazard pointer: %p: not found", - (void *)session, (void *)page); + (void *)session, (void *)ref); } /* @@ -196,8 +243,8 @@ __wt_hazard_close(WT_SESSION_IMPL *session) * diagnostic. */ for (found = false, hp = session->hazard; - hp < session->hazard + session->hazard_size; ++hp) - if (hp->page != NULL) { + hp < session->hazard + session->hazard_inuse; ++hp) + if (hp->ref != NULL) { found = true; break; } @@ -224,9 +271,9 @@ __wt_hazard_close(WT_SESSION_IMPL *session) * can't think of a reason it would be). */ for (hp = session->hazard; - hp < session->hazard + session->hazard_size; ++hp) - if (hp->page != NULL) { - hp->page = NULL; + hp < session->hazard + session->hazard_inuse; ++hp) + if (hp->ref != NULL) { + hp->ref = NULL; --session->nhazard; } @@ -237,20 +284,95 @@ __wt_hazard_close(WT_SESSION_IMPL *session) (void *)session); } +/* + * hazard_get_reference -- + * Return a consistent reference to a hazard pointer array. + */ +static inline void +hazard_get_reference( + WT_SESSION_IMPL *session, WT_HAZARD **hazardp, uint32_t *hazard_inusep) +{ + /* + * Hazard pointer arrays can be swapped out from under us if they grow. + * First, read the current in-use value. The read must precede the read + * of the hazard pointer itself (so the in-use value is pessimistic + * should the hazard array grow), and additionally ensure we only read + * the in-use value once. Then, read the hazard pointer, also ensuring + * we only read it once. + * + * Use a barrier instead of marking the fields volatile because we don't + * want to slow down the rest of the hazard pointer functions that don't + * need special treatment. + */ + WT_ORDERED_READ(*hazard_inusep, session->hazard_inuse); + WT_ORDERED_READ(*hazardp, session->hazard); +} + +/* + * __wt_hazard_check -- + * Return if there's a hazard pointer to the page in the system. + */ +WT_HAZARD * +__wt_hazard_check(WT_SESSION_IMPL *session, WT_REF *ref) +{ + WT_CONNECTION_IMPL *conn; + WT_HAZARD *hp; + WT_SESSION_IMPL *s; + uint32_t i, j, hazard_inuse, max, session_cnt, walk_cnt; + + conn = S2C(session); + + WT_STAT_CONN_INCR(session, cache_hazard_checks); + + /* + * No lock is required because the session array is fixed size, but it + * may contain inactive entries. We must review any active session + * that might contain a hazard pointer, so insert a read barrier after + * reading the active session count. That way, no matter what sessions + * come or go, we'll check the slots for all of the sessions that could + * have been active when we started our check. + */ + WT_ORDERED_READ(session_cnt, conn->session_cnt); + for (s = conn->sessions, + i = j = max = walk_cnt = 0; i < session_cnt; ++s, ++i) { + if (!s->active) + continue; + + hazard_get_reference(s, &hp, &hazard_inuse); + + if (hazard_inuse > max) { + max = hazard_inuse; + WT_STAT_CONN_SET(session, cache_hazard_max, max); + } + + for (j = 0; j < hazard_inuse; ++hp, ++j) { + ++walk_cnt; + if (hp->ref == ref) { + WT_STAT_CONN_INCRV(session, + cache_hazard_walks, walk_cnt); + return (hp); + } + } + } + WT_STAT_CONN_INCRV(session, cache_hazard_walks, walk_cnt); + return (NULL); +} + /* * __wt_hazard_count -- * Count how many hazard pointers this session has on the given page. */ u_int -__wt_hazard_count(WT_SESSION_IMPL *session, WT_PAGE *page) +__wt_hazard_count(WT_SESSION_IMPL *session, WT_REF *ref) { WT_HAZARD *hp; + uint32_t i, hazard_inuse; u_int count; - for (count = 0, hp = session->hazard + session->hazard_size - 1; - hp >= session->hazard; - --hp) - if (hp->page == page) + hazard_get_reference(session, &hp, &hazard_inuse); + + for (count = 0, i = 0; i < hazard_inuse; ++hp, ++i) + if (hp->ref == ref) ++count; return (count); @@ -267,11 +389,11 @@ __hazard_dump(WT_SESSION_IMPL *session) WT_HAZARD *hp; for (hp = session->hazard; - hp < session->hazard + session->hazard_size; ++hp) - if (hp->page != NULL) + hp < session->hazard + session->hazard_inuse; ++hp) + if (hp->ref != NULL) __wt_errx(session, "session %p: hazard pointer %p: %s, line %d", (void *)session, - (void *)hp->page, hp->file, hp->line); + (void *)hp->ref, hp->file, hp->line); } #endif diff --git a/src/third_party/wiredtiger/src/support/hex.c b/src/third_party/wiredtiger/src/support/hex.c index 5c48ce8b74a..b54a08dd8f3 100644 --- a/src/third_party/wiredtiger/src/support/hex.c +++ b/src/third_party/wiredtiger/src/support/hex.c @@ -8,8 +8,6 @@ #include "wt_internal.h" -const u_char __wt_hex[] = "0123456789abcdef"; - /* * __fill_hex -- * In-memory conversion of raw bytes to a hexadecimal representation. @@ -25,8 +23,8 @@ __fill_hex(const uint8_t *src, size_t src_max, --dest_max; for (; src_max > 0 && dest_max > 1; src_max -= 1, dest_max -= 2, ++src) { - *dest++ = __wt_hex[(*src & 0xf0) >> 4]; - *dest++ = __wt_hex[*src & 0x0f]; + *dest++ = __wt_hex((*src & 0xf0) >> 4); + *dest++ = __wt_hex(*src & 0x0f); } *dest++ = '\0'; if (lenp != NULL) @@ -90,8 +88,8 @@ __wt_raw_to_esc_hex( *t++ = *p; } else { *t++ = '\\'; - *t++ = __wt_hex[(*p & 0xf0) >> 4]; - *t++ = __wt_hex[*p & 0x0f]; + *t++ = __wt_hex((*p & 0xf0) >> 4); + *t++ = __wt_hex(*p & 0x0f); } *t++ = '\0'; to->size = WT_PTRDIFF(t, to->mem); diff --git a/src/third_party/wiredtiger/src/support/mtx_rw.c b/src/third_party/wiredtiger/src/support/mtx_rw.c index ea18f556257..7905241be0e 100644 --- a/src/third_party/wiredtiger/src/support/mtx_rw.c +++ b/src/third_party/wiredtiger/src/support/mtx_rw.c @@ -27,7 +27,7 @@ */ /* - * Based on "Spinlocks and Read-Write Locks" by Dr. Steven Fuerst: + * Inspired by "Spinlocks and Read-Write Locks" by Dr. Steven Fuerst: * http://locklessinc.com/articles/locks/ * * Dr. Fuerst further credits: @@ -39,99 +39,75 @@ * by John Mellor-Crummey and Michael Scott in their landmark paper "Scalable * Reader-Writer Synchronization for Shared-Memory Multiprocessors". * - * The following is an explanation of this code. First, the underlying lock - * structure. + * The following is an explanation of our interpretation and implementation. + * First, the underlying lock structure. * + * volatile union { + * uint64_t v; // Full 64-bit value * struct { - * uint16_t writers; Now serving for writers - * uint16_t readers; Now serving for readers - * uint16_t next; Next available ticket number - * uint16_t __notused; Padding - * } + * uint8_t current; // Current ticket + * uint8_t next; // Next available ticket + * uint8_t reader; // Read queue ticket + * uint8_t __notused; // Padding + * uint16_t readers_active; // Count of active readers + * uint16_t readers_queued; // Count of queued readers + * } s; + * } u; * * First, imagine a store's 'take a number' ticket algorithm. A customer takes * a unique ticket number and customers are served in ticket order. In the data - * structure, 'writers' is the next writer to be served, 'readers' is the next - * reader to be served, and 'next' is the next available ticket number. + * structure, 'next' is the ticket that will be allocated next, and 'current' + * is the ticket being served. * - * Next, consider exclusive (write) locks. The 'now serving' number for writers - * is 'writers'. To lock, 'take a number' and wait until that number is being - * served; more specifically, atomically copy and increment the current value of - * 'next', and then wait until 'writers' equals that copied number. + * Next, consider exclusive (write) locks. To lock, 'take a number' and wait + * until that number is being served; more specifically, atomically increment + * 'next', and then wait until 'current' equals that allocated ticket. * - * Shared (read) locks are similar. Like writers, readers atomically get the - * next number available. However, instead of waiting for 'writers' to equal - * their number, they wait for 'readers' to equal their number. + * Shared (read) locks are similar, except that readers can share a ticket + * (both with each other and with a single writer). Readers with a given + * ticket execute before the writer with that ticket. In other words, writers + * wait for both their ticket to become current and for all readers to exit + * the lock. * - * This has the effect of queuing lock requests in the order they arrive - * (incidentally avoiding starvation). + * If there are no active writers (indicated by 'current' == 'next'), readers + * can immediately enter the lock by atomically incrementing 'readers_active'. + * When there are writers active, readers form a new queue by first setting + * 'reader' to 'next' (i.e. readers are scheduled after any queued writers, + * avoiding starvation), then atomically incrementing 'readers_queued'. * - * Each lock/unlock pair requires incrementing both 'readers' and 'writers'. - * In the case of a reader, the 'readers' increment happens when the reader - * acquires the lock (to allow read-lock sharing), and the 'writers' increment - * happens when the reader releases the lock. In the case of a writer, both - * 'readers' and 'writers' are incremented when the writer releases the lock. - * - * For example, consider the following read (R) and write (W) lock requests: - * - * writers readers next - * 0 0 0 - * R: ticket 0, readers match OK 0 1 1 - * R: ticket 1, readers match OK 0 2 2 - * R: ticket 2, readers match OK 0 3 3 - * W: ticket 3, writers no match block 0 3 4 - * R: ticket 2, unlock 1 3 4 - * R: ticket 0, unlock 2 3 4 - * R: ticket 1, unlock 3 3 4 - * W: ticket 3, writers match OK 3 3 4 - * - * Note the writer blocks until 'writers' equals its ticket number and it does - * not matter if readers unlock in order or not. - * - * Readers or writers entering the system after the write lock is queued block, - * and the next ticket holder (reader or writer) will unblock when the writer - * unlocks. An example, continuing from the last line of the above example: - * - * writers readers next - * W: ticket 3, writers match OK 3 3 4 - * R: ticket 4, readers no match block 3 3 5 - * R: ticket 5, readers no match block 3 3 6 - * W: ticket 6, writers no match block 3 3 7 - * W: ticket 3, unlock 4 4 7 - * R: ticket 4, readers match OK 4 5 7 - * R: ticket 5, readers match OK 4 6 7 - * - * The 'next' field is a 2-byte value so the available ticket number wraps at - * 64K requests. If a thread's lock request is not granted until the 'next' - * field cycles and the same ticket is taken by another thread, we could grant - * a lock to two separate threads at the same time, and bad things happen: two - * writer threads or a reader thread and a writer thread would run in parallel, - * and lock waiters could be skipped if the unlocks race. This is unlikely, it - * only happens if a lock request is blocked by 64K other requests. The fix is - * to grow the lock structure fields, but the largest atomic instruction we have - * is 8 bytes, the structure has no room to grow. + * The 'next' field is a 1-byte value so the available ticket number wraps + * after 256 requests. If a thread's write lock request would cause the 'next' + * field to catch up with 'current', instead it waits to avoid the same ticket + * being allocated to multiple threads. */ #include "wt_internal.h" /* - * __wt_rwlock_alloc -- - * Allocate and initialize a read/write lock. + * __wt_rwlock_init -- + * Initialize a read/write lock. */ int -__wt_rwlock_alloc( - WT_SESSION_IMPL *session, WT_RWLOCK **rwlockp, const char *name) +__wt_rwlock_init(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - WT_RWLOCK *rwlock; + l->u.v = 0; - __wt_verbose(session, WT_VERB_MUTEX, "rwlock: alloc %s", name); - - WT_RET(__wt_calloc_one(session, &rwlock)); + WT_RET(__wt_cond_alloc(session, "rwlock wait", &l->cond_readers)); + WT_RET(__wt_cond_alloc(session, "rwlock wait", &l->cond_writers)); + return (0); +} - rwlock->name = name; +/* + * __wt_rwlock_destroy -- + * Destroy a read/write lock. + */ +void +__wt_rwlock_destroy(WT_SESSION_IMPL *session, WT_RWLOCK *l) +{ + l->u.v = 0; - *rwlockp = rwlock; - return (0); + __wt_cond_destroy(session, &l->cond_readers); + __wt_cond_destroy(session, &l->cond_writers); } /* @@ -139,57 +115,40 @@ __wt_rwlock_alloc( * Try to get a shared lock, fail immediately if unavailable. */ int -__wt_try_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +__wt_try_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - wt_rwlock_t *l, new, old; + WT_RWLOCK new, old; WT_STAT_CONN_INCR(session, rwlock_read); - l = &rwlock->rwlock; - new = old = *l; + old.u.v = l->u.v; - /* - * This read lock can only be granted if the lock was last granted to - * a reader and there are no readers or writers blocked on the lock, - * that is, if this thread's ticket would be the next ticket granted. - * Do the cheap test to see if this can possibly succeed (and confirm - * the lock is in the correct state to grant this read lock). - */ - if (old.s.readers != old.s.next) + /* This read lock can only be granted if there are no active writers. */ + if (old.u.s.current != old.u.s.next) return (EBUSY); /* - * The replacement lock value is a result of allocating a new ticket and - * incrementing the reader value to match it. + * The replacement lock value is a result of adding an active reader. + * Check for overflow: if the maximum number of readers are already + * active, no new readers can enter the lock. */ - new.s.readers = new.s.next = old.s.next + 1; - return (__wt_atomic_cas64(&l->u, old.u, new.u) ? 0 : EBUSY); + new.u.v = old.u.v; + if (++new.u.s.readers_active == 0) + return (EBUSY); + + /* We rely on this atomic operation to provide a barrier. */ + return (__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v) ? 0 : EBUSY); } /* - * __wt_readlock_spin -- - * Spin to get a read lock: only yield the CPU if the lock is held - * exclusive. + * __read_blocked -- + * Check whether the current read lock request should keep waiting. */ -void -__wt_readlock_spin(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +static bool +__read_blocked(WT_SESSION_IMPL *session) { - wt_rwlock_t *l; - - l = &rwlock->rwlock; - - /* - * Try to get the lock in a single operation if it is available to - * readers. This avoids the situation where multiple readers arrive - * concurrently and have to line up in order to enter the lock. For - * read-heavy workloads it can make a significant difference. - */ - while (__wt_try_readlock(session, rwlock) != 0) { - if (l->s.writers_active > 0) - __wt_yield(); - else - WT_PAUSE(); - } + return (session->current_rwticket != + session->current_rwlock->u.s.current); } /* @@ -197,48 +156,97 @@ __wt_readlock_spin(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) * Get a shared lock. */ void -__wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +__wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - wt_rwlock_t *l; - uint16_t ticket; + WT_RWLOCK new, old; int pause_cnt; + int16_t writers_active; + uint8_t ticket; WT_STAT_CONN_INCR(session, rwlock_read); WT_DIAGNOSTIC_YIELD; - l = &rwlock->rwlock; + for (;;) { + /* + * Fast path: if there is no active writer, join the current + * group. + */ + for (old.u.v = l->u.v; + old.u.s.current == old.u.s.next; + old.u.v = l->u.v) { + new.u.v = old.u.v; + /* + * Check for overflow: if the maximum number of readers + * are already active, no new readers can enter the + * lock. + */ + if (++new.u.s.readers_active == 0) + goto stall; + if (__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v)) + return; + WT_PAUSE(); + } - /* - * Possibly wrap: if we have more than 64K lockers waiting, the ticket - * value will wrap and two lockers will simultaneously be granted the - * lock. - */ - ticket = __wt_atomic_fetch_add16(&l->s.next, 1); - for (pause_cnt = 0; ticket != l->s.readers;) { /* - * We failed to get the lock; pause before retrying and if we've - * paused enough, yield so we don't burn CPU to no purpose. This - * situation happens if there are more threads than cores in the - * system and we're thrashing on shared resources. + * There is an active writer: join the next group. + * + * Limit how many readers can queue: don't allow more readers + * to queue than there are active writers (calculated as + * `next - current`): otherwise, in write-heavy workloads, + * readers can keep queuing up in front of writers and + * throughput is unstable. + * + * If the maximum number of readers are already queued, wait + * until we can get a valid ticket. */ - if (++pause_cnt < WT_THOUSAND) + writers_active = old.u.s.next - old.u.s.current; + if (old.u.s.readers_queued > writers_active) { +stall: __wt_cond_wait(session, + l->cond_readers, 10 * WT_THOUSAND, NULL); + continue; + } + + /* + * If we are the first reader to queue, set the next read + * group. Note: don't re-read from the lock or we could race + * with a writer unlocking. + */ + new.u.v = old.u.v; + if (new.u.s.readers_queued++ == 0) + new.u.s.reader = new.u.s.next; + ticket = new.u.s.reader; + + if (__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v)) + break; + } + + /* Wait for our group to start. */ + for (pause_cnt = 0; ticket != l->u.s.current; pause_cnt++) { + if (pause_cnt < 1000) WT_PAUSE(); - else + else if (pause_cnt < 1200) __wt_yield(); + else { + session->current_rwlock = l; + session->current_rwticket = ticket; + __wt_cond_wait(session, + l->cond_readers, 10 * WT_THOUSAND, __read_blocked); + } } - /* - * We're the only writer of the readers field, so the update does not - * need to be atomic. - */ - ++l->s.readers; - /* * Applications depend on a barrier here so that operations holding the - * lock see consistent data. + * lock see consistent data. The atomic operation above isn't + * sufficient here because we don't own the lock until our ticket comes + * up and whatever data we are protecting may have changed in the + * meantime. */ WT_READ_BARRIER(); + + /* Sanity check that we (still) have the lock. */ + WT_ASSERT(session, + ticket == l->u.s.current && l->u.s.readers_active > 0); } /* @@ -246,19 +254,24 @@ __wt_readlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) * Release a shared lock. */ void -__wt_readunlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +__wt_readunlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - wt_rwlock_t *l; + WT_RWLOCK new, old; - WT_UNUSED(session); + do { + old.u.v = l->u.v; + WT_ASSERT(session, old.u.s.readers_active > 0); - l = &rwlock->rwlock; + /* + * Decrement the active reader count (other readers are doing + * the same, make sure we don't race). + */ + new.u.v = old.u.v; + --new.u.s.readers_active; + } while (!__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v)); - /* - * Increment the writers value (other readers are doing the same, make - * sure we don't race). - */ - (void)__wt_atomic_add16(&l->s.writers, 1); + if (new.u.s.readers_active == 0 && new.u.s.current != new.u.s.next) + __wt_cond_signal(session, l->cond_writers); } /* @@ -266,29 +279,50 @@ __wt_readunlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) * Try to get an exclusive lock, fail immediately if unavailable. */ int -__wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +__wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - wt_rwlock_t *l, new, old; + WT_RWLOCK new, old; WT_STAT_CONN_INCR(session, rwlock_write); - l = &rwlock->rwlock; - old = new = *l; - /* - * This write lock can only be granted if the lock was last granted to - * a writer and there are no readers or writers blocked on the lock, - * that is, if this thread's ticket would be the next ticket granted. - * Do the cheap test to see if this can possibly succeed (and confirm - * the lock is in the correct state to grant this write lock). + * This write lock can only be granted if no readers or writers blocked + * on the lock, that is, if this thread's ticket would be the next + * ticket granted. Check if this can possibly succeed (and confirm the + * lock is in the correct state to grant this write lock). */ - if (old.s.writers != old.s.next) + old.u.v = l->u.v; + if (old.u.s.current != old.u.s.next || old.u.s.readers_active != 0) return (EBUSY); - /* The replacement lock value is a result of allocating a new ticket. */ - ++new.s.next; - ++new.s.writers_active; - return (__wt_atomic_cas64(&l->u, old.u, new.u) ? 0 : EBUSY); + /* + * We've checked above that there is no writer active (since + * `current == next`), so there should be no readers queued. + */ + WT_ASSERT(session, old.u.s.readers_queued == 0); + + /* + * The replacement lock value is a result of allocating a new ticket. + * + * We rely on this atomic operation to provide a barrier. + */ + new.u.v = old.u.v; + new.u.s.next++; + return (__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v) ? 0 : EBUSY); +} + +/* + * __write_blocked -- + * Check whether the current write lock request should keep waiting. + */ +static bool +__write_blocked(WT_SESSION_IMPL *session) +{ + WT_RWLOCK *l; + + l = session->current_rwlock; + return (session->current_rwticket != l->u.s.current || + l->u.s.readers_active != 0); } /* @@ -296,41 +330,70 @@ __wt_try_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) * Wait to get an exclusive lock. */ void -__wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +__wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - wt_rwlock_t *l; - uint16_t ticket; + WT_RWLOCK new, old; int pause_cnt; + uint8_t ticket; WT_STAT_CONN_INCR(session, rwlock_write); - l = &rwlock->rwlock; + for (;;) { + old.u.v = l->u.v; + + /* Allocate a ticket. */ + new.u.v = old.u.v; + ticket = new.u.s.next++; - /* - * Possibly wrap: if we have more than 64K lockers waiting, the ticket - * value will wrap and two lockers will simultaneously be granted the - * lock. - */ - ticket = __wt_atomic_fetch_add16(&l->s.next, 1); - (void)__wt_atomic_add16(&l->s.writers_active, 1); - for (pause_cnt = 0; ticket != l->s.writers;) { /* - * We failed to get the lock; pause before retrying and if we've - * paused enough, sleep so we don't burn CPU to no purpose. This - * situation happens if there are more threads than cores in the - * system and we're thrashing on shared resources. + * Check for overflow: if the next ticket is allowed to catch + * up with the current batch, two writers could be granted the + * lock simultaneously. */ - if (++pause_cnt < WT_THOUSAND) + if (new.u.s.current == new.u.s.next) { + __wt_cond_wait(session, + l->cond_writers, 10 * WT_THOUSAND, NULL); + continue; + } + if (__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v)) + break; + } + + /* + * Wait for our group to start and any readers to drain. + * + * We take care here to do an atomic read of the full 64-bit lock + * value. Otherwise, reads are not guaranteed to be ordered and we + * could see no readers active from a different batch and decide that + * we have the lock. + */ + for (pause_cnt = 0, old.u.v = l->u.v; + ticket != old.u.s.current || old.u.s.readers_active != 0; + pause_cnt++, old.u.v = l->u.v) { + if (pause_cnt < 1000) WT_PAUSE(); - else - __wt_sleep(0, 10); + else if (pause_cnt < 1200) + __wt_yield(); + else { + session->current_rwlock = l; + session->current_rwticket = ticket; + __wt_cond_wait(session, + l->cond_writers, 10 * WT_THOUSAND, __write_blocked); + } } /* * Applications depend on a barrier here so that operations holding the - * lock see consistent data. + * lock see consistent data. The atomic operation above isn't + * sufficient here because we don't own the lock until our ticket comes + * up and whatever data we are protecting may have changed in the + * meantime. */ WT_READ_BARRIER(); + + /* Sanity check that we (still) have the lock. */ + WT_ASSERT(session, + ticket == l->u.s.current && l->u.s.readers_active == 0); } /* @@ -338,70 +401,54 @@ __wt_writelock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) * Release an exclusive lock. */ void -__wt_writeunlock(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +__wt_writeunlock(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - wt_rwlock_t *l, new; + WT_RWLOCK new, old; - WT_UNUSED(session); - - l = &rwlock->rwlock; - (void)__wt_atomic_sub16(&l->s.writers_active, 1); - - /* - * Ensure that all updates made while the lock was held are visible to - * the next thread to acquire the lock. - */ - WT_WRITE_BARRIER(); + do { + old.u.v = l->u.v; - new = *l; + /* + * We're holding the lock exclusive, there shouldn't be any + * active readers. + */ + WT_ASSERT(session, old.u.s.readers_active == 0); - /* - * We're the only writer of the writers/readers fields, so the update - * does not need to be atomic; we have to update both values at the - * same time though, otherwise we'd potentially race with the thread - * next granted the lock. - */ - ++new.s.writers; - ++new.s.readers; - l->i.wr = new.i.wr; + /* + * Allow the next batch to start. + * + * If there are readers in the next group, swap queued readers + * to active: this could race with new readlock requests, so we + * have to spin. + */ + new.u.v = old.u.v; + if (++new.u.s.current == new.u.s.reader) { + new.u.s.readers_active = new.u.s.readers_queued; + new.u.s.readers_queued = 0; + } + } while (!__wt_atomic_casv64(&l->u.v, old.u.v, new.u.v)); + + if (new.u.s.readers_active != 0) + __wt_cond_signal(session, l->cond_readers); + else if (new.u.s.current != new.u.s.next) + __wt_cond_signal(session, l->cond_writers); WT_DIAGNOSTIC_YIELD; } -/* - * __wt_rwlock_destroy -- - * Destroy a read/write lock. - */ -void -__wt_rwlock_destroy(WT_SESSION_IMPL *session, WT_RWLOCK **rwlockp) -{ - WT_RWLOCK *rwlock; - - rwlock = *rwlockp; /* Clear our caller's reference. */ - if (rwlock == NULL) - return; - *rwlockp = NULL; - - __wt_verbose( - session, WT_VERB_MUTEX, "rwlock: destroy %s", rwlock->name); - - __wt_free(session, rwlock); -} - #ifdef HAVE_DIAGNOSTIC /* * __wt_rwlock_islocked -- * Return if a read/write lock is currently locked for reading or writing. */ bool -__wt_rwlock_islocked(WT_SESSION_IMPL *session, WT_RWLOCK *rwlock) +__wt_rwlock_islocked(WT_SESSION_IMPL *session, WT_RWLOCK *l) { - wt_rwlock_t *l; + WT_RWLOCK old; WT_UNUSED(session); - l = &rwlock->rwlock; - - return (l->s.writers != l->s.next || l->s.readers != l->s.next); + old.u.v = l->u.v; + return (old.u.s.current != old.u.s.next || old.u.s.readers_active != 0); } #endif diff --git a/src/third_party/wiredtiger/src/support/rand.c b/src/third_party/wiredtiger/src/support/rand.c index 025b18e4ed3..4fae43edc8e 100644 --- a/src/third_party/wiredtiger/src/support/rand.c +++ b/src/third_party/wiredtiger/src/support/rand.c @@ -51,6 +51,7 @@ */ void __wt_random_init(WT_RAND_STATE volatile * rnd_state) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_RAND_STATE rnd; @@ -69,6 +70,7 @@ __wt_random_init(WT_RAND_STATE volatile * rnd_state) void __wt_random_init_seed( WT_SESSION_IMPL *session, WT_RAND_STATE volatile * rnd_state) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { struct timespec ts; WT_RAND_STATE rnd; @@ -86,6 +88,7 @@ __wt_random_init_seed( */ uint32_t __wt_random(WT_RAND_STATE volatile * rnd_state) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_RAND_STATE rnd; uint32_t w, z; @@ -117,3 +120,15 @@ __wt_random(WT_RAND_STATE volatile * rnd_state) return ((z << 16) + (w & 65535)); } + +/* + * __wt_random64 -- + * Return a 64-bit pseudo-random number. + */ +uint64_t +__wt_random64(WT_RAND_STATE volatile * rnd_state) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) +{ + return (((uint64_t)__wt_random(rnd_state) << 32) + + __wt_random(rnd_state)); +} diff --git a/src/third_party/wiredtiger/src/support/scratch.c b/src/third_party/wiredtiger/src/support/scratch.c index 69987ebc852..485cea90e89 100644 --- a/src/third_party/wiredtiger/src/support/scratch.c +++ b/src/third_party/wiredtiger/src/support/scratch.c @@ -69,13 +69,16 @@ int __wt_buf_fmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) { + WT_DECL_RET; va_list ap; size_t len; for (;;) { va_start(ap, fmt); - len = (size_t)vsnprintf(buf->mem, buf->memsize, fmt, ap); + ret = __wt_vsnprintf_len_set( + buf->mem, buf->memsize, &len, fmt, ap); va_end(ap); + WT_RET(ret); /* Check if there was enough space. */ if (len < buf->memsize) { @@ -100,6 +103,7 @@ int __wt_buf_catfmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4))) { + WT_DECL_RET; va_list ap; size_t len, space; char *p; @@ -117,8 +121,9 @@ __wt_buf_catfmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) p = (char *)((uint8_t *)buf->mem + buf->size); WT_ASSERT(session, buf->memsize >= buf->size); space = buf->memsize - buf->size; - len = (size_t)vsnprintf(p, space, fmt, ap); + ret = __wt_vsnprintf_len_set(p, space, &len, fmt, ap); va_end(ap); + WT_RET(ret); /* Check if there was enough space. */ if (len < space) { diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 5acd9fc713f..2c2217f8c20 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -64,6 +64,7 @@ static const char * const __stats_dsrc_desc[] = { "cache: pages requested from the cache", "cache: pages written from cache", "cache: pages written requiring in-memory restoration", + "cache: tracked dirty bytes in the cache", "cache: unmodified pages evicted", "cache_walk: Average difference between current eviction generation when the page was last considered", "cache_walk: Average on-disk page image size seen", @@ -225,6 +226,7 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->cache_pages_requested = 0; stats->cache_write = 0; stats->cache_write_restore = 0; + /* not clearing cache_bytes_dirty */ stats->cache_eviction_clean = 0; /* not clearing cache_state_gen_avg_gap */ /* not clearing cache_state_avg_written_size */ @@ -372,6 +374,7 @@ __wt_stat_dsrc_aggregate_single( to->cache_pages_requested += from->cache_pages_requested; to->cache_write += from->cache_write; to->cache_write_restore += from->cache_write_restore; + to->cache_bytes_dirty += from->cache_bytes_dirty; to->cache_eviction_clean += from->cache_eviction_clean; to->cache_state_gen_avg_gap += from->cache_state_gen_avg_gap; to->cache_state_avg_written_size += @@ -535,6 +538,7 @@ __wt_stat_dsrc_aggregate( WT_STAT_READ(from, cache_pages_requested); to->cache_write += WT_STAT_READ(from, cache_write); to->cache_write_restore += WT_STAT_READ(from, cache_write_restore); + to->cache_bytes_dirty += WT_STAT_READ(from, cache_bytes_dirty); to->cache_eviction_clean += WT_STAT_READ(from, cache_eviction_clean); to->cache_state_gen_avg_gap += WT_STAT_READ(from, cache_state_gen_avg_gap); @@ -673,10 +677,15 @@ static const char * const __stats_connection_desc[] = { "cache: eviction server unable to reach eviction goal", "cache: eviction state", "cache: eviction walks abandoned", + "cache: eviction worker thread active", + "cache: eviction worker thread created", "cache: eviction worker thread evicting pages", + "cache: eviction worker thread removed", + "cache: eviction worker thread stable number", "cache: failed eviction of pages that exceeded the in-memory maximum", "cache: files with active eviction walks", "cache: files with new eviction walks started", + "cache: force re-tuning of eviction workers once in a while", "cache: hazard pointer blocked page eviction", "cache: hazard pointer check calls", "cache: hazard pointer check entries walked", @@ -751,9 +760,7 @@ static const char * const __stats_connection_desc[] = { "lock: checkpoint lock acquisitions", "lock: checkpoint lock application thread wait time (usecs)", "lock: checkpoint lock internal thread wait time (usecs)", - "lock: handle-list lock acquisitions", - "lock: handle-list lock application thread wait time (usecs)", - "lock: handle-list lock internal thread wait time (usecs)", + "lock: handle-list lock eviction thread wait time (usecs)", "lock: metadata lock acquisitions", "lock: metadata lock application thread wait time (usecs)", "lock: metadata lock internal thread wait time (usecs)", @@ -765,9 +772,11 @@ static const char * const __stats_connection_desc[] = { "lock: table lock internal thread time waiting for the table lock (usecs)", "log: busy returns attempting to switch slots", "log: consolidated slot closures", + "log: consolidated slot join active slot closed", "log: consolidated slot join races", "log: consolidated slot join transitions", "log: consolidated slot joins", + "log: consolidated slot transitions unable to find free slot", "log: consolidated slot unbuffered writes", "log: log bytes of payload data", "log: log bytes written", @@ -808,6 +817,9 @@ static const char * const __stats_connection_desc[] = { "reconciliation: split objects currently awaiting free", "session: open cursor count", "session: open session count", + "session: table alter failed calls", + "session: table alter successful calls", + "session: table alter unchanged and skipped", "session: table compact failed calls", "session: table compact successful calls", "session: table create failed calls", @@ -951,10 +963,15 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_eviction_slow = 0; /* not clearing cache_eviction_state */ stats->cache_eviction_walks_abandoned = 0; + /* not clearing cache_eviction_active_workers */ + stats->cache_eviction_worker_created = 0; stats->cache_eviction_worker_evicting = 0; + stats->cache_eviction_worker_removed = 0; + /* not clearing cache_eviction_stable_state_workers */ stats->cache_eviction_force_fail = 0; /* not clearing cache_eviction_walks_active */ stats->cache_eviction_walks_started = 0; + stats->cache_eviction_force_retune = 0; stats->cache_eviction_hazard = 0; stats->cache_hazard_checks = 0; stats->cache_hazard_walks = 0; @@ -1029,9 +1046,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->lock_checkpoint_count = 0; stats->lock_checkpoint_wait_application = 0; stats->lock_checkpoint_wait_internal = 0; - stats->lock_handle_list_count = 0; - stats->lock_handle_list_wait_application = 0; - stats->lock_handle_list_wait_internal = 0; + stats->lock_handle_list_wait_eviction = 0; stats->lock_metadata_count = 0; stats->lock_metadata_wait_application = 0; stats->lock_metadata_wait_internal = 0; @@ -1043,9 +1058,11 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->lock_table_wait_internal = 0; stats->log_slot_switch_busy = 0; stats->log_slot_closes = 0; + stats->log_slot_active_closed = 0; stats->log_slot_races = 0; stats->log_slot_transitions = 0; stats->log_slot_joins = 0; + stats->log_slot_no_free_slots = 0; stats->log_slot_unbuffered = 0; stats->log_bytes_payload = 0; stats->log_bytes_written = 0; @@ -1086,6 +1103,9 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) /* not clearing rec_split_stashed_objects */ /* not clearing session_cursor_open */ /* not clearing session_open */ + /* not clearing session_table_alter_fail */ + /* not clearing session_table_alter_success */ + /* not clearing session_table_alter_skip */ /* not clearing session_table_compact_fail */ /* not clearing session_table_compact_success */ /* not clearing session_table_create_fail */ @@ -1222,14 +1242,24 @@ __wt_stat_connection_aggregate( to->cache_eviction_state += WT_STAT_READ(from, cache_eviction_state); to->cache_eviction_walks_abandoned += WT_STAT_READ(from, cache_eviction_walks_abandoned); + to->cache_eviction_active_workers += + WT_STAT_READ(from, cache_eviction_active_workers); + to->cache_eviction_worker_created += + WT_STAT_READ(from, cache_eviction_worker_created); to->cache_eviction_worker_evicting += WT_STAT_READ(from, cache_eviction_worker_evicting); + to->cache_eviction_worker_removed += + WT_STAT_READ(from, cache_eviction_worker_removed); + to->cache_eviction_stable_state_workers += + WT_STAT_READ(from, cache_eviction_stable_state_workers); to->cache_eviction_force_fail += WT_STAT_READ(from, cache_eviction_force_fail); to->cache_eviction_walks_active += WT_STAT_READ(from, cache_eviction_walks_active); to->cache_eviction_walks_started += WT_STAT_READ(from, cache_eviction_walks_started); + to->cache_eviction_force_retune += + WT_STAT_READ(from, cache_eviction_force_retune); to->cache_eviction_hazard += WT_STAT_READ(from, cache_eviction_hazard); to->cache_hazard_checks += WT_STAT_READ(from, cache_hazard_checks); @@ -1325,12 +1355,8 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, lock_checkpoint_wait_application); to->lock_checkpoint_wait_internal += WT_STAT_READ(from, lock_checkpoint_wait_internal); - to->lock_handle_list_count += - WT_STAT_READ(from, lock_handle_list_count); - to->lock_handle_list_wait_application += - WT_STAT_READ(from, lock_handle_list_wait_application); - to->lock_handle_list_wait_internal += - WT_STAT_READ(from, lock_handle_list_wait_internal); + to->lock_handle_list_wait_eviction += + WT_STAT_READ(from, lock_handle_list_wait_eviction); to->lock_metadata_count += WT_STAT_READ(from, lock_metadata_count); to->lock_metadata_wait_application += WT_STAT_READ(from, lock_metadata_wait_application); @@ -1348,9 +1374,13 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, lock_table_wait_internal); to->log_slot_switch_busy += WT_STAT_READ(from, log_slot_switch_busy); to->log_slot_closes += WT_STAT_READ(from, log_slot_closes); + to->log_slot_active_closed += + WT_STAT_READ(from, log_slot_active_closed); to->log_slot_races += WT_STAT_READ(from, log_slot_races); to->log_slot_transitions += WT_STAT_READ(from, log_slot_transitions); to->log_slot_joins += WT_STAT_READ(from, log_slot_joins); + to->log_slot_no_free_slots += + WT_STAT_READ(from, log_slot_no_free_slots); to->log_slot_unbuffered += WT_STAT_READ(from, log_slot_unbuffered); to->log_bytes_payload += WT_STAT_READ(from, log_bytes_payload); to->log_bytes_written += WT_STAT_READ(from, log_bytes_written); @@ -1397,6 +1427,12 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, rec_split_stashed_objects); to->session_cursor_open += WT_STAT_READ(from, session_cursor_open); to->session_open += WT_STAT_READ(from, session_open); + to->session_table_alter_fail += + WT_STAT_READ(from, session_table_alter_fail); + to->session_table_alter_success += + WT_STAT_READ(from, session_table_alter_success); + to->session_table_alter_skip += + WT_STAT_READ(from, session_table_alter_skip); to->session_table_compact_fail += WT_STAT_READ(from, session_table_compact_fail); to->session_table_compact_success += diff --git a/src/third_party/wiredtiger/src/support/thread_group.c b/src/third_party/wiredtiger/src/support/thread_group.c index a866d2d01c5..090621fa138 100644 --- a/src/third_party/wiredtiger/src/support/thread_group.c +++ b/src/third_party/wiredtiger/src/support/thread_group.c @@ -50,8 +50,7 @@ __thread_group_grow( { WT_THREAD *thread; - WT_ASSERT(session, - __wt_rwlock_islocked(session, group->lock)); + WT_ASSERT(session, __wt_rwlock_islocked(session, &group->lock)); /* * Any bounds checking is done by the caller so we know that @@ -72,20 +71,19 @@ __thread_group_grow( /* * __thread_group_shrink -- - * Decrease the number of running threads in the group, and free any + * Decrease the number of running threads in the group. Optionally free any * memory associated with slots larger than the new count. */ static int __thread_group_shrink(WT_SESSION_IMPL *session, - WT_THREAD_GROUP *group, uint32_t new_count) + WT_THREAD_GROUP *group, uint32_t new_count, bool free_thread) { WT_DECL_RET; WT_SESSION *wt_session; WT_THREAD *thread; uint32_t current_slot; - WT_ASSERT(session, - __wt_rwlock_islocked(session, group->lock)); + WT_ASSERT(session, __wt_rwlock_islocked(session, &group->lock)); for (current_slot = group->alloc; current_slot > new_count; ) { /* @@ -107,14 +105,15 @@ __thread_group_shrink(WT_SESSION_IMPL *session, WT_TRET(__wt_thread_join(session, thread->tid)); thread->tid = 0; } - - if (thread->session != NULL) { - wt_session = (WT_SESSION *)thread->session; - WT_TRET(wt_session->close(wt_session, NULL)); - thread->session = NULL; + if (free_thread) { + if (thread->session != NULL) { + wt_session = (WT_SESSION *)thread->session; + WT_TRET(wt_session->close(wt_session, NULL)); + thread->session = NULL; + } + __wt_free(session, thread); + group->threads[current_slot] = NULL; } - __wt_free(session, thread); - group->threads[current_slot] = NULL; } /* Update the thread group state to match our changes */ @@ -142,16 +141,19 @@ __thread_group_resize( WT_ASSERT(session, group->current_threads <= group->alloc && - __wt_rwlock_islocked(session, group->lock)); + __wt_rwlock_islocked(session, &group->lock)); if (new_min == group->min && new_max == group->max) return (0); + if (new_min > new_max) + return (EINVAL); + /* - * Coll shrink to reduce the number of thread structures and running + * Call shrink to reduce the number of thread structures and running * threads if required by the change in group size. */ - WT_RET(__thread_group_shrink(session, group, new_max)); + WT_RET(__thread_group_shrink(session, group, new_max, true)); /* * Only reallocate the thread array if it is the largest ever, since @@ -227,9 +229,9 @@ __wt_thread_group_resize( " from max: %" PRIu32 " -> %" PRIu32, (void *)group, group->min, new_min, group->max, new_max); - __wt_writelock(session, group->lock); + __wt_writelock(session, &group->lock); WT_TRET(__thread_group_resize(session, group, new_min, new_max, flags)); - __wt_writeunlock(session, group->lock); + __wt_writeunlock(session, &group->lock); return (ret); } @@ -255,22 +257,22 @@ __wt_thread_group_create( __wt_verbose(session, WT_VERB_THREAD_GROUP, "Creating thread group: %p", (void *)group); - WT_RET(__wt_rwlock_alloc(session, &group->lock, "Thread group")); + WT_RET(__wt_rwlock_init(session, &group->lock)); WT_ERR(__wt_cond_alloc( - session, "Thread group cond", false, &group->wait_cond)); + session, "thread group cond", &group->wait_cond)); cond_alloced = true; - __wt_writelock(session, group->lock); + __wt_writelock(session, &group->lock); group->run_func = run_func; group->name = name; WT_TRET(__thread_group_resize(session, group, min, max, flags)); - __wt_writeunlock(session, group->lock); + __wt_writeunlock(session, &group->lock); /* Cleanup on error to avoid leaking resources */ err: if (ret != 0) { if (cond_alloced) - WT_TRET(__wt_cond_destroy(session, &group->wait_cond)); + __wt_cond_destroy(session, &group->wait_cond); __wt_rwlock_destroy(session, &group->lock); } return (ret); @@ -288,14 +290,14 @@ __wt_thread_group_destroy(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group) __wt_verbose(session, WT_VERB_THREAD_GROUP, "Destroying thread group: %p", (void *)group); - WT_ASSERT(session, __wt_rwlock_islocked(session, group->lock)); + WT_ASSERT(session, __wt_rwlock_islocked(session, &group->lock)); /* Shut down all threads and free associated resources. */ - WT_TRET(__thread_group_shrink(session, group, 0)); + WT_TRET(__thread_group_shrink(session, group, 0, true)); __wt_free(session, group->threads); - WT_TRET(__wt_cond_destroy(session, &group->wait_cond)); + __wt_cond_destroy(session, &group->wait_cond); __wt_rwlock_destroy(session, &group->lock); /* @@ -322,15 +324,42 @@ __wt_thread_group_start_one( return (0); if (wait) - __wt_writelock(session, group->lock); - else if (__wt_try_writelock(session, group->lock) != 0) - return (0); + __wt_writelock(session, &group->lock); + else + WT_RET(__wt_try_writelock(session, &group->lock)); /* Recheck the bounds now that we hold the lock */ if (group->current_threads < group->max) WT_TRET(__thread_group_grow( session, group, group->current_threads + 1)); - __wt_writeunlock(session, group->lock); + __wt_writeunlock(session, &group->lock); + + return (ret); +} + +/* + * __wt_thread_group_stop_one -- + * Stop one thread if possible. + */ +int +__wt_thread_group_stop_one( + WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, bool wait) +{ + WT_DECL_RET; + + if (group->current_threads <= group->min) + return (0); + + if (wait) + __wt_writelock(session, &group->lock); + else + WT_RET(__wt_try_writelock(session, &group->lock)); + + /* Recheck the bounds now that we hold the lock */ + if (group->current_threads > group->min) + WT_TRET(__thread_group_shrink( + session, group, group->current_threads - 1, false)); + __wt_writeunlock(session, &group->lock); return (ret); } diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index 1c47f94bb60..ea7faa2e966 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -100,7 +100,7 @@ __wt_txn_release_snapshot(WT_SESSION_IMPL *session) session->txn.isolation == WT_ISO_READ_UNCOMMITTED || !__wt_txn_visible_all(session, txn_state->pinned_id)); - txn_state->pinned_id = WT_TXN_NONE; + txn_state->metadata_pinned = txn_state->pinned_id = WT_TXN_NONE; F_CLR(txn, WT_TXN_HAS_SNAPSHOT); } @@ -108,7 +108,7 @@ __wt_txn_release_snapshot(WT_SESSION_IMPL *session) * __wt_txn_get_snapshot -- * Allocate a snapshot. */ -int +void __wt_txn_get_snapshot(WT_SESSION_IMPL *session) { WT_CONNECTION_IMPL *conn; @@ -126,7 +126,7 @@ __wt_txn_get_snapshot(WT_SESSION_IMPL *session) n = 0; /* We're going to scan the table: wait for the lock. */ - __wt_readlock_spin(session, txn_global->scan_rwlock); + __wt_readlock(session, &txn_global->scan_rwlock); current_id = pinned_id = txn_global->current; prev_oldest_id = txn_global->oldest_id; @@ -137,8 +137,10 @@ __wt_txn_get_snapshot(WT_SESSION_IMPL *session) * metadata. We don't have to keep the checkpoint's changes pinned so * don't including it in the published pinned ID. */ - if ((id = txn_global->checkpoint_txnid) != WT_TXN_NONE) + if ((id = txn_global->checkpoint_txnid) != WT_TXN_NONE) { txn->snapshot[n++] = id; + txn_state->metadata_pinned = id; + } /* For pure read-only workloads, avoid scanning. */ if (prev_oldest_id == current_id) { @@ -178,9 +180,8 @@ __wt_txn_get_snapshot(WT_SESSION_IMPL *session) WT_ASSERT(session, prev_oldest_id == txn_global->oldest_id); txn_state->pinned_id = pinned_id; -done: __wt_readunlock(session, txn_global->scan_rwlock); +done: __wt_readunlock(session, &txn_global->scan_rwlock); __txn_sort_snapshot(session, n, current_id); - return (0); } /* @@ -189,14 +190,14 @@ done: __wt_readunlock(session, txn_global->scan_rwlock); */ static void __txn_oldest_scan(WT_SESSION_IMPL *session, - uint64_t *oldest_idp, uint64_t *last_runningp, + uint64_t *oldest_idp, uint64_t *last_runningp, uint64_t *metadata_pinnedp, WT_SESSION_IMPL **oldest_sessionp) { WT_CONNECTION_IMPL *conn; WT_SESSION_IMPL *oldest_session; WT_TXN_GLOBAL *txn_global; WT_TXN_STATE *s; - uint64_t id, last_running, oldest_id, prev_oldest_id; + uint64_t id, last_running, metadata_pinned, oldest_id, prev_oldest_id; uint32_t i, session_cnt; conn = S2C(session); @@ -205,24 +206,24 @@ __txn_oldest_scan(WT_SESSION_IMPL *session, /* The oldest ID cannot change while we are holding the scan lock. */ prev_oldest_id = txn_global->oldest_id; - oldest_id = last_running = txn_global->current; + last_running = oldest_id = txn_global->current; + if ((metadata_pinned = txn_global->checkpoint_txnid) == WT_TXN_NONE) + metadata_pinned = oldest_id; /* Walk the array of concurrent transactions. */ WT_ORDERED_READ(session_cnt, conn->session_cnt); for (i = 0, s = txn_global->states; i < session_cnt; i++, s++) { - /* - * Update the oldest ID. - * - * Ignore: IDs older than the oldest ID we saw. This can happen - * if we race with a thread that is allocating an ID -- the ID - * will not be used because the thread will keep spinning until - * it gets a valid one. - */ + /* Update the last running transaction ID. */ if ((id = s->id) != WT_TXN_NONE && WT_TXNID_LE(prev_oldest_id, id) && WT_TXNID_LT(id, last_running)) last_running = id; + /* Update the metadata pinned ID. */ + if ((id = s->metadata_pinned) != WT_TXN_NONE && + WT_TXNID_LT(id, metadata_pinned)) + metadata_pinned = id; + /* * !!! * Note: Don't ignore pinned ID values older than the previous @@ -246,9 +247,14 @@ __txn_oldest_scan(WT_SESSION_IMPL *session, WT_TXNID_LT(id, oldest_id)) oldest_id = id; + /* The metadata pinned ID can't move past the oldest ID. */ + if (WT_TXNID_LT(oldest_id, metadata_pinned)) + metadata_pinned = oldest_id; + + *last_runningp = last_running; + *metadata_pinnedp = metadata_pinned; *oldest_idp = oldest_id; *oldest_sessionp = oldest_session; - *last_runningp = last_running; } /* @@ -262,8 +268,8 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) WT_DECL_RET; WT_SESSION_IMPL *oldest_session; WT_TXN_GLOBAL *txn_global; - uint64_t current_id, last_running, oldest_id; - uint64_t prev_last_running, prev_oldest_id; + uint64_t current_id, last_running, metadata_pinned, oldest_id; + uint64_t prev_last_running, prev_metadata_pinned, prev_oldest_id; bool strict, wait; conn = S2C(session); @@ -271,26 +277,29 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) strict = LF_ISSET(WT_TXN_OLDEST_STRICT); wait = LF_ISSET(WT_TXN_OLDEST_WAIT); - current_id = last_running = txn_global->current; + current_id = last_running = metadata_pinned = txn_global->current; prev_last_running = txn_global->last_running; + prev_metadata_pinned = txn_global->metadata_pinned; prev_oldest_id = txn_global->oldest_id; /* * For pure read-only workloads, or if the update isn't forced and the * oldest ID isn't too far behind, avoid scanning. */ - if (prev_oldest_id == current_id || + if ((prev_oldest_id == current_id && + prev_metadata_pinned == current_id) || (!strict && WT_TXNID_LT(current_id, prev_oldest_id + 100))) return (0); /* First do a read-only scan. */ if (wait) - __wt_readlock_spin(session, txn_global->scan_rwlock); + __wt_readlock(session, &txn_global->scan_rwlock); else if ((ret = - __wt_try_readlock(session, txn_global->scan_rwlock)) != 0) + __wt_try_readlock(session, &txn_global->scan_rwlock)) != 0) return (ret == EBUSY ? 0 : ret); - __txn_oldest_scan(session, &oldest_id, &last_running, &oldest_session); - __wt_readunlock(session, txn_global->scan_rwlock); + __txn_oldest_scan(session, + &oldest_id, &last_running, &metadata_pinned, &oldest_session); + __wt_readunlock(session, &txn_global->scan_rwlock); /* * If the state hasn't changed (or hasn't moved far enough for @@ -299,14 +308,15 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) if ((oldest_id == prev_oldest_id || (!strict && WT_TXNID_LT(oldest_id, prev_oldest_id + 100))) && ((last_running == prev_last_running) || - (!strict && WT_TXNID_LT(last_running, prev_last_running + 100)))) + (!strict && WT_TXNID_LT(last_running, prev_last_running + 100))) && + metadata_pinned == prev_metadata_pinned) return (0); /* It looks like an update is necessary, wait for exclusive access. */ if (wait) - __wt_writelock(session, txn_global->scan_rwlock); + __wt_writelock(session, &txn_global->scan_rwlock); else if ((ret = - __wt_try_writelock(session, txn_global->scan_rwlock)) != 0) + __wt_try_writelock(session, &txn_global->scan_rwlock)) != 0) return (ret == EBUSY ? 0 : ret); /* @@ -314,7 +324,8 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) * scanning. */ if (WT_TXNID_LE(oldest_id, txn_global->oldest_id) && - WT_TXNID_LE(last_running, txn_global->last_running)) + WT_TXNID_LE(last_running, txn_global->last_running) && + WT_TXNID_LE(metadata_pinned, txn_global->metadata_pinned)) goto done; /* @@ -323,7 +334,8 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) * sure that there isn't a thread that has got a snapshot locally but * not yet published its snap_min. */ - __txn_oldest_scan(session, &oldest_id, &last_running, &oldest_session); + __txn_oldest_scan(session, + &oldest_id, &last_running, &metadata_pinned, &oldest_session); #ifdef HAVE_DIAGNOSTIC { @@ -339,7 +351,9 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) id == WT_TXN_NONE || !WT_TXNID_LT(id, oldest_id)); } #endif - /* Update the oldest ID. */ + /* Update the public IDs. */ + if (WT_TXNID_LT(txn_global->metadata_pinned, metadata_pinned)) + txn_global->metadata_pinned = metadata_pinned; if (WT_TXNID_LT(txn_global->oldest_id, oldest_id)) txn_global->oldest_id = oldest_id; if (WT_TXNID_LT(txn_global->last_running, last_running)) { @@ -361,7 +375,7 @@ __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) #endif } -done: __wt_writeunlock(session, txn_global->scan_rwlock); +done: __wt_writeunlock(session, &txn_global->scan_rwlock); return (ret); } @@ -699,7 +713,7 @@ __wt_txn_stats_update(WT_SESSION_IMPL *session) snapshot_pinned = txn_global->nsnap_oldest_id; WT_STAT_SET(session, stats, txn_pinned_range, - txn_global->current - txn_global->oldest_id); + txn_global->current - txn_global->oldest_id); WT_STAT_SET(session, stats, txn_pinned_snapshot_range, snapshot_pinned == WT_TXN_NONE ? @@ -750,23 +764,20 @@ __wt_txn_global_init(WT_SESSION_IMPL *session, const char *cfg[]) txn_global = &conn->txn_global; txn_global->current = txn_global->last_running = - txn_global->oldest_id = WT_TXN_FIRST; + txn_global->metadata_pinned = txn_global->oldest_id = WT_TXN_FIRST; WT_RET(__wt_spin_init(session, &txn_global->id_lock, "transaction id lock")); - WT_RET(__wt_rwlock_alloc(session, - &txn_global->scan_rwlock, "transaction scan lock")); - WT_RET(__wt_rwlock_alloc(session, - &txn_global->nsnap_rwlock, "named snapshot lock")); + WT_RET(__wt_rwlock_init(session, &txn_global->scan_rwlock)); + WT_RET(__wt_rwlock_init(session, &txn_global->nsnap_rwlock)); txn_global->nsnap_oldest_id = WT_TXN_NONE; TAILQ_INIT(&txn_global->nsnaph); WT_RET(__wt_calloc_def( session, conn->session_size, &txn_global->states)); - WT_CACHE_LINE_ALIGNMENT_VERIFY(session, txn_global->states); for (i = 0, s = txn_global->states; i < conn->session_size; i++, s++) - s->id = s->pinned_id = WT_TXN_NONE; + s->id = s->metadata_pinned = s->pinned_id = WT_TXN_NONE; return (0); } @@ -792,3 +803,98 @@ __wt_txn_global_destroy(WT_SESSION_IMPL *session) __wt_rwlock_destroy(session, &txn_global->nsnap_rwlock); __wt_free(session, txn_global->states); } + +#if defined(HAVE_DIAGNOSTIC) || defined(HAVE_VERBOSE) +/* + * __wt_verbose_dump_txn -- + * Output diagnostic information about the global transaction state. + */ +int +__wt_verbose_dump_txn(WT_SESSION_IMPL *session) +{ + WT_CONNECTION_IMPL *conn; + WT_TXN_GLOBAL *txn_global; + WT_TXN *txn; + WT_TXN_STATE *s; + const char *iso_tag; + uint64_t id; + uint32_t i, session_cnt; + + conn = S2C(session); + txn_global = &conn->txn_global; + + WT_RET(__wt_msg(session, "%s", WT_DIVIDER)); + WT_RET(__wt_msg(session, "transaction state dump")); + + WT_RET(__wt_msg(session, "current ID: %" PRIu64, txn_global->current)); + WT_RET(__wt_msg(session, + "last running ID: %" PRIu64, txn_global->last_running)); + WT_RET(__wt_msg(session, "oldest ID: %" PRIu64, txn_global->oldest_id)); + WT_RET(__wt_msg(session, + "oldest named snapshot ID: %" PRIu64, txn_global->nsnap_oldest_id)); + + WT_RET(__wt_msg(session, "checkpoint running? %s", + txn_global->checkpoint_running ? "yes" : "no")); + WT_RET(__wt_msg(session, + "checkpoint generation: %" PRIu64, txn_global->checkpoint_gen)); + WT_RET(__wt_msg(session, + "checkpoint pinned ID: %" PRIu64, txn_global->checkpoint_pinned)); + WT_RET(__wt_msg(session, + "checkpoint txn ID: %" PRIu64, txn_global->checkpoint_txnid)); + + WT_ORDERED_READ(session_cnt, conn->session_cnt); + WT_RET(__wt_msg(session, "session count: %" PRIu32, session_cnt)); + + WT_RET(__wt_msg(session, "Transaction state of active sessions:")); + + /* + * Walk each session transaction state and dump information. Accessing + * the content of session handles is not thread safe, so some + * information may change while traversing if other threads are active + * at the same time, which is OK since this is diagnostic code. + */ + for (i = 0, s = txn_global->states; i < session_cnt; i++, s++) { + /* Skip sessions with no active transaction */ + if ((id = s->id) == WT_TXN_NONE && s->pinned_id == WT_TXN_NONE) + continue; + + txn = &conn->sessions[i].txn; + iso_tag = "INVALID"; + switch (txn->isolation) { + case WT_ISO_READ_COMMITTED: + iso_tag = "WT_ISO_READ_COMMITTED"; + break; + case WT_ISO_READ_UNCOMMITTED: + iso_tag = "WT_ISO_READ_UNCOMMITTED"; + break; + case WT_ISO_SNAPSHOT: + iso_tag = "WT_ISO_SNAPSHOT"; + break; + } + + WT_RET(__wt_msg(session, + "ID: %6" PRIu64 + ", mod count: %u" + ", pinned ID: %" PRIu64 + ", snap min: %" PRIu64 + ", snap max: %" PRIu64 + ", metadata pinned ID: %" PRIu64 + ", flags: 0x%08" PRIx32 + ", name: %s" + ", isolation: %s", + id, + txn->mod_count, + s->pinned_id, + txn->snap_min, + txn->snap_max, + s->metadata_pinned, + txn->flags, + conn->sessions[i].name == NULL ? + "EMPTY" : conn->sessions[i].name, + iso_tag)); + } + WT_RET(__wt_msg(session, "%s", WT_DIVIDER)); + + return (0); +} +#endif diff --git a/src/third_party/wiredtiger/src/txn/txn_ckpt.c b/src/third_party/wiredtiger/src/txn/txn_ckpt.c index 1efd0200a3d..f4ccf5eacd0 100644 --- a/src/third_party/wiredtiger/src/txn/txn_ckpt.c +++ b/src/third_party/wiredtiger/src/txn/txn_ckpt.c @@ -8,9 +8,9 @@ #include "wt_internal.h" -static int __checkpoint_lock_tree( - WT_SESSION_IMPL *, bool, bool, const char *[]); -static int __checkpoint_mark_deletes(WT_SESSION_IMPL *, const char *[]); +static int __checkpoint_lock_dirty_tree( + WT_SESSION_IMPL *, bool, bool, bool, const char *[]); +static int __checkpoint_mark_skip(WT_SESSION_IMPL *, WT_CKPT *, bool); static int __checkpoint_presync(WT_SESSION_IMPL *, const char *[]); static int __checkpoint_tree_helper(WT_SESSION_IMPL *, const char *[]); @@ -89,6 +89,33 @@ err: WT_TRET(__wt_metadata_cursor_release(session, &cursor)); return (ret); } +/* + * __checkpoint_update_generation -- + * Update the checkpoint generation of the current tree. + * + * This indicates that the tree will not be visited again by the current + * checkpoint. + */ +static void +__checkpoint_update_generation(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + + btree = S2BT(session); + + /* + * Updates to the metadata are made by the checkpoint transaction, so + * the metadata tree's checkpoint generation should never be updated. + */ + if (WT_IS_METADATA(session->dhandle)) + return; + + WT_PUBLISH(btree->checkpoint_gen, + S2C(session)->txn_global.checkpoint_gen); + WT_STAT_DATA_SET(session, + btree_checkpoint_generation, btree->checkpoint_gen); +} + /* * __checkpoint_apply_all -- * Apply an operation to all files involved in a checkpoint. @@ -239,24 +266,82 @@ int __wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]) { WT_BTREE *btree; + WT_CONFIG_ITEM cval; WT_DECL_RET; const char *name; + bool force; - WT_UNUSED(cfg); + btree = S2BT(session); + + /* Find out if we have to force a checkpoint. */ + WT_RET(__wt_config_gets_def(session, cfg, "force", 0, &cval)); + force = cval.val != 0; + if (!force) { + WT_RET(__wt_config_gets_def(session, cfg, "name", 0, &cval)); + force = cval.len != 0; + } /* Should not be called with anything other than a file object. */ WT_ASSERT(session, session->dhandle->checkpoint == NULL); WT_ASSERT(session, WT_PREFIX_MATCH(session->dhandle->name, "file:")); /* Skip files that are never involved in a checkpoint. */ - if (F_ISSET(S2BT(session), WT_BTREE_NO_CHECKPOINT)) + if (F_ISSET(btree, WT_BTREE_NO_CHECKPOINT)) + return (0); + +#ifdef HAVE_DIAGNOSTIC + /* + * We may have raced between starting the checkpoint transaction and + * some operation completing on the handle that updated the metadata + * (e.g., closing a bulk load cursor). All such operations either have + * exclusive access to the handle or hold the schema lock. We are now + * holding the schema lock and have an open btree handle, so if we + * can't update the metadata, then there has been some state change + * invisible to the checkpoint transaction. + */ + if (!WT_IS_METADATA(session->dhandle)) { + WT_CURSOR *meta_cursor; + bool metadata_race; + + WT_ASSERT(session, !F_ISSET(&session->txn, WT_TXN_ERROR)); + WT_RET(__wt_metadata_cursor(session, &meta_cursor)); + meta_cursor->set_key(meta_cursor, session->dhandle->name); + ret = __wt_curfile_insert_check(meta_cursor); + if (ret == WT_ROLLBACK) { + metadata_race = true; + ret = 0; + } else + metadata_race = false; + WT_TRET(__wt_metadata_cursor_release(session, &meta_cursor)); + WT_RET(ret); + WT_ASSERT(session, !metadata_race); + } +#endif + + /* + * Decide whether the tree needs to be included in the checkpoint and + * if so, acquire the necessary locks. + */ + WT_SAVE_DHANDLE(session, ret = __checkpoint_lock_dirty_tree( + session, true, force, true, cfg)); + WT_RET(ret); + if (F_ISSET(btree, WT_BTREE_SKIP_CKPT)) { + WT_ASSERT(session, btree->ckpt == NULL); + __checkpoint_update_generation(session); return (0); + } - /* Make sure there is space for the next entry. */ + /* + * Make sure there is space for the new entry: do this before getting + * the handle to avoid cleanup if we can't allocate the memory. + */ WT_RET(__wt_realloc_def(session, &session->ckpt_handle_allocated, session->ckpt_handle_next + 1, &session->ckpt_handle)); - /* Not strictly necessary, but cleaner to clear the current handle. */ + /* + * The current tree will be included: get it again because the handle + * we have is only valid for the duration of this function. + */ name = session->dhandle->name; session->dhandle = NULL; @@ -268,48 +353,12 @@ __wt_checkpoint_get_handles(WT_SESSION_IMPL *session, const char *cfg[]) * with eviction and we don't want to unfairly penalize (or promote) * eviction in trees due to checkpoints. */ - btree = S2BT(session); btree->evict_walk_saved = btree->evict_walk_period; - WT_SAVE_DHANDLE(session, - ret = __checkpoint_lock_tree(session, true, true, cfg)); - if (ret != 0) { - WT_TRET(__wt_session_release_btree(session)); - return (ret); - } - - /* - * Flag that the handle is part of a checkpoint for the purposes - * of transaction visibility checks. - */ - WT_PUBLISH(btree->include_checkpoint_txn, true); - session->ckpt_handle[session->ckpt_handle_next++] = session->dhandle; return (0); } -/* - * __checkpoint_update_generation -- - * Update the checkpoint generation of the current tree. - * - * This indicates that the tree will not be visited again by the current - * checkpoint. - */ -static void -__checkpoint_update_generation(WT_SESSION_IMPL *session) -{ - WT_BTREE *btree; - - btree = S2BT(session); - if (!WT_IS_METADATA(session, session->dhandle)) - WT_PUBLISH(btree->include_checkpoint_txn, false); - - WT_PUBLISH(btree->checkpoint_gen, - S2C(session)->txn_global.checkpoint_gen); - WT_STAT_DATA_SET(session, - btree_checkpoint_generation, btree->checkpoint_gen); -} - /* * __checkpoint_reduce_dirty_cache -- * Release clean trees from the list cached for checkpoints. @@ -373,7 +422,6 @@ __checkpoint_reduce_dirty_cache(WT_SESSION_IMPL *session) __wt_sleep(0, stepdown_us / 10); __wt_epoch(session, &stop); current_us = WT_TIMEDIFF_US(stop, last); - total_ms = WT_TIMEDIFF_MS(stop, start); bytes_written_total = cache->bytes_written - bytes_written_start; @@ -435,36 +483,6 @@ __checkpoint_reduce_dirty_cache(WT_SESSION_IMPL *session) WT_STAT_CONN_SET(session, txn_checkpoint_scrub_time, total_ms); } -/* - * __checkpoint_release_clean_trees -- - * Release clean trees from the list cached for checkpoints. - */ -static int -__checkpoint_release_clean_trees(WT_SESSION_IMPL *session) -{ - WT_BTREE *btree; - WT_DATA_HANDLE *dhandle; - WT_DECL_RET; - u_int i; - - for (i = 0; i < session->ckpt_handle_next; i++) { - dhandle = session->ckpt_handle[i]; - btree = dhandle->handle; - if (!F_ISSET(btree, WT_BTREE_SKIP_CKPT)) - continue; - __wt_meta_ckptlist_free(session, btree->ckpt); - btree->ckpt = NULL; - WT_WITH_DHANDLE(session, dhandle, - __checkpoint_update_generation(session)); - session->ckpt_handle[i] = NULL; - WT_WITH_DHANDLE(session, dhandle, - ret = __wt_session_release_btree(session)); - WT_RET(ret); - } - - return (0); -} - /* * __checkpoint_stats -- * Update checkpoint timer stats. @@ -526,6 +544,112 @@ __checkpoint_verbose_track(WT_SESSION_IMPL *session, #endif } +/* + * __checkpoint_fail_reset -- + * Reset fields when a failure occurs. + */ +static void +__checkpoint_fail_reset(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + + btree = S2BT(session); + btree->modified = true; + __wt_meta_ckptlist_free(session, &btree->ckpt); +} + +/* + * __checkpoint_prepare -- + * Start the transaction for a checkpoint and gather handles. + */ +static int +__checkpoint_prepare(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + WT_TXN *txn; + WT_TXN_GLOBAL *txn_global; + WT_TXN_STATE *txn_state; + const char *txn_cfg[] = { WT_CONFIG_BASE(session, + WT_SESSION_begin_transaction), "isolation=snapshot", NULL }; + + conn = S2C(session); + txn = &session->txn; + txn_global = &conn->txn_global; + txn_state = WT_SESSION_TXN_STATE(session); + + /* + * Start a snapshot transaction for the checkpoint. + * + * Note: we don't go through the public API calls because they have + * side effects on cursors, which applications can hold open across + * calls to checkpoint. + */ + WT_RET(__wt_txn_begin(session, txn_cfg)); + + WT_DIAGNOSTIC_YIELD; + + /* Ensure a transaction ID is allocated prior to sharing it globally */ + WT_RET(__wt_txn_id_check(session)); + + /* + * Mark the connection as clean. If some data gets modified after + * generating checkpoint transaction id, connection will be reset to + * dirty when reconciliation marks the btree dirty on encountering the + * dirty page. + */ + conn->modified = false; + + /* + * Save the checkpoint session ID. + * + * We never do checkpoints in the default session (with id zero). + */ + WT_ASSERT(session, session->id != 0 && txn_global->checkpoint_id == 0); + txn_global->checkpoint_id = session->id; + + /* + * Remove the checkpoint transaction from the global table. + * + * This allows ordinary visibility checks to move forward because + * checkpoints often take a long time and only write to the metadata. + */ + __wt_writelock(session, &txn_global->scan_rwlock); + txn_global->checkpoint_txnid = txn->id; + txn_global->checkpoint_pinned = WT_MIN(txn->id, txn->snap_min); + + /* + * Sanity check that the oldest ID hasn't moved on before we have + * cleared our entry. + */ + WT_ASSERT(session, + WT_TXNID_LE(txn_global->oldest_id, txn_state->id) && + WT_TXNID_LE(txn_global->oldest_id, txn_state->pinned_id)); + + /* + * Clear our entry from the global transaction session table. Any + * operation that needs to know about the ID for this checkpoint will + * consider the checkpoint ID in the global structure. Most operations + * can safely ignore the checkpoint ID (see the visible all check for + * details). + */ + txn_state->id = txn_state->pinned_id = + txn_state->metadata_pinned = WT_TXN_NONE; + __wt_writeunlock(session, &txn_global->scan_rwlock); + + /* + * Get a list of handles we want to flush; for named checkpoints this + * may pull closed objects into the session cache. + * + * First, gather all handles, then start the checkpoint transaction, + * then release any clean handles. + */ + WT_ASSERT(session, session->ckpt_handle_next == 0); + WT_WITH_TABLE_READ_LOCK(session, ret = __checkpoint_apply_all( + session, cfg, __wt_checkpoint_get_handles, NULL)); + return (ret); +} + /* * __txn_checkpoint -- * Checkpoint a database or a list of objects in the database. @@ -541,25 +665,18 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) WT_TXN *txn; WT_TXN_GLOBAL *txn_global; WT_TXN_ISOLATION saved_isolation; - WT_TXN_STATE *txn_state; void *saved_meta_next; u_int i; uint64_t fsync_duration_usecs; - bool full, idle, logging, tracking; - const char *txn_cfg[] = { WT_CONFIG_BASE(session, - WT_SESSION_begin_transaction), "isolation=snapshot", NULL }; + bool failed, full, idle, logging, tracking; conn = S2C(session); cache = conn->cache; txn = &session->txn; txn_global = &conn->txn_global; - txn_state = WT_SESSION_TXN_STATE(session); saved_isolation = session->isolation; full = idle = logging = tracking = false; - /* Ensure the metadata table is open before taking any locks. */ - WT_RET(__wt_metadata_cursor(session, NULL)); - /* * Do a pass over the configuration arguments and figure out what kind * of checkpoint this is. @@ -625,86 +742,24 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) tracking = true; /* - * Get a list of handles we want to flush; for named checkpoints this - * may pull closed objects into the session cache. - * * We want to skip checkpointing clean handles whenever possible. That * is, when the checkpoint is not named or forced. However, we need to * take care about ordering with respect to the checkpoint transaction. * - * If we skip clean handles before starting the transaction, the + * We can't skip clean handles before starting the transaction or the * checkpoint can miss updates in trees that become dirty as the * checkpoint is starting. If we wait until the transaction has * started before locking a handle, there could be a metadata-changing * operation in between (e.g., salvage) that will cause a write * conflict when the checkpoint goes to write the metadata. * - * First, gather all handles, then start the checkpoint transaction, - * then release any clean handles. + * Hold the schema lock while starting the transaction and gathering + * handles so the set we get is complete and correct. */ - WT_ASSERT(session, session->ckpt_handle_next == 0); - WT_WITH_SCHEMA_LOCK(session, ret, - WT_WITH_TABLE_LOCK(session, ret, - WT_WITH_HANDLE_LIST_LOCK(session, - ret = __checkpoint_apply_all( - session, cfg, __wt_checkpoint_get_handles, NULL)))); + WT_WITH_SCHEMA_LOCK(session, ret = __checkpoint_prepare(session, cfg)); WT_ERR(ret); - /* - * Start a snapshot transaction for the checkpoint. - * - * Note: we don't go through the public API calls because they have - * side effects on cursors, which applications can hold open across - * calls to checkpoint. - */ - WT_ERR(__wt_txn_begin(session, txn_cfg)); - - /* Ensure a transaction ID is allocated prior to sharing it globally */ - WT_ERR(__wt_txn_id_check(session)); - - /* - * Mark the connection as clean. If some data gets modified after - * generating checkpoint transaction id, connection will be reset to - * dirty when reconciliation marks the btree dirty on encountering the - * dirty page. - */ - conn->modified = false; - - /* - * Save the checkpoint session ID. - * - * We never do checkpoints in the default session (with id zero). - */ - WT_ASSERT(session, session->id != 0 && txn_global->checkpoint_id == 0); - txn_global->checkpoint_id = session->id; - - /* - * Remove the checkpoint transaction from the global table. - * - * This allows ordinary visibility checks to move forward because - * checkpoints often take a long time and only write to the metadata. - */ - __wt_writelock(session, txn_global->scan_rwlock); - txn_global->checkpoint_txnid = txn->id; - txn_global->checkpoint_pinned = WT_MIN(txn->id, txn->snap_min); - - /* - * Sanity check that the oldest ID hasn't moved on before we have - * cleared our entry. - */ - WT_ASSERT(session, - WT_TXNID_LE(txn_global->oldest_id, txn_state->id) && - WT_TXNID_LE(txn_global->oldest_id, txn_state->pinned_id)); - - /* - * Clear our entry from the global transaction session table. Any - * operation that needs to know about the ID for this checkpoint will - * consider the checkpoint ID in the global structure. Most operations - * can safely ignore the checkpoint ID (see the visible all check for - * details). - */ - txn_state->id = txn_state->pinned_id = WT_TXN_NONE; - __wt_writeunlock(session, txn_global->scan_rwlock); + WT_ASSERT(session, txn->isolation == WT_ISO_SNAPSHOT); /* * Unblock updates -- we can figure out that any updates to clean pages @@ -713,16 +768,6 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) cache->eviction_scrub_limit = 0.0; WT_STAT_CONN_SET(session, txn_checkpoint_scrub_target, 0); - /* - * Mark old checkpoints that are being deleted and figure out which - * trees we can skip in this checkpoint. - * - * Release clean trees. Any updates made after this point will not - * visible to the checkpoint transaction. - */ - WT_ERR(__checkpoint_apply(session, cfg, __checkpoint_mark_deletes)); - WT_ERR(__checkpoint_release_clean_trees(session)); - /* Tell logging that we have started a database checkpoint. */ if (full && logging) WT_ERR(__wt_txn_checkpoint_log( @@ -785,7 +830,7 @@ __txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) /* Disable metadata tracking during the metadata checkpoint. */ saved_meta_next = session->meta_track_next; session->meta_track_next = NULL; - WT_WITH_METADATA_LOCK(session, ret, + WT_WITH_METADATA_LOCK(session, WT_WITH_DHANDLE(session, WT_SESSION_META_DHANDLE(session), ret = __wt_checkpoint(session, cfg))); @@ -829,12 +874,13 @@ err: /* * overwritten the checkpoint, so what ends up on disk is not * consistent. */ - if (ret != 0 && !conn->modified) + failed = ret != 0; + if (failed) conn->modified = true; session->isolation = txn->isolation = WT_ISO_READ_UNCOMMITTED; if (tracking) - WT_TRET(__wt_meta_track_off(session, false, ret != 0)); + WT_TRET(__wt_meta_track_off(session, false, failed)); cache->eviction_scrub_limit = 0.0; WT_STAT_CONN_SET(session, txn_checkpoint_scrub_target, 0); @@ -867,6 +913,13 @@ err: /* for (i = 0; i < session->ckpt_handle_next; ++i) { if (session->ckpt_handle[i] == NULL) continue; + /* + * If the operation failed, mark all trees dirty so they are + * included if a future checkpoint can succeed. + */ + if (failed) + WT_WITH_DHANDLE(session, session->ckpt_handle[i], + __checkpoint_fail_reset(session)); WT_WITH_DHANDLE(session, session->ckpt_handle[i], WT_TRET(__wt_session_release_btree(session))); } @@ -878,14 +931,40 @@ err: /* return (ret); } +/* + * __txn_checkpoint_wrapper -- + * Checkpoint wrapper. + */ +static int +__txn_checkpoint_wrapper(WT_SESSION_IMPL *session, const char *cfg[]) +{ + WT_DECL_RET; + WT_TXN_GLOBAL *txn_global; + + txn_global = &S2C(session)->txn_global; + + WT_STAT_CONN_SET(session, txn_checkpoint_running, 1); + txn_global->checkpoint_running = true; + WT_FULL_BARRIER(); + + ret = __txn_checkpoint(session, cfg); + + WT_STAT_CONN_SET(session, txn_checkpoint_running, 0); + txn_global->checkpoint_running = false; + WT_FULL_BARRIER(); + + return (ret); +} + /* * __wt_txn_checkpoint -- * Checkpoint a database or a list of objects in the database. */ int -__wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) +__wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[], bool waiting) { WT_DECL_RET; + uint32_t mask; /* * Reset open cursors. Do this explicitly, even though it will happen @@ -895,13 +974,22 @@ __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) */ WT_RET(__wt_session_reset_cursors(session, false)); + /* Ensure the metadata table is open before taking any locks. */ + WT_RET(__wt_metadata_cursor(session, NULL)); + /* * Don't highjack the session checkpoint thread for eviction. * * Application threads are not generally available for potentially slow * operations, but checkpoint does enough I/O it may be called upon to * perform slow operations for the block manager. + * + * Application checkpoints wait until the checkpoint lock is available, + * compaction checkpoints don't. */ +#define WT_TXN_SESSION_MASK \ + (WT_SESSION_CAN_WAIT | WT_SESSION_NO_EVICTION) + mask = F_MASK(session, WT_TXN_SESSION_MASK); F_SET(session, WT_SESSION_CAN_WAIT | WT_SESSION_NO_EVICTION); /* @@ -911,14 +999,15 @@ __wt_txn_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) * calls checkpoint directly, it can be tough to avoid. Serialize here * to ensure we don't get into trouble. */ - WT_STAT_CONN_SET(session, txn_checkpoint_running, 1); - - WT_WITH_CHECKPOINT_LOCK(session, ret, - ret = __txn_checkpoint(session, cfg)); - - WT_STAT_CONN_SET(session, txn_checkpoint_running, 0); + if (waiting) + WT_WITH_CHECKPOINT_LOCK(session, + ret = __txn_checkpoint_wrapper(session, cfg)); + else + WT_WITH_CHECKPOINT_LOCK_NOWAIT(session, ret, + ret = __txn_checkpoint_wrapper(session, cfg)); - F_CLR(session, WT_SESSION_CAN_WAIT | WT_SESSION_NO_EVICTION); + F_CLR(session, WT_TXN_SESSION_MASK); + F_SET(session, mask); return (ret); } @@ -1015,12 +1104,13 @@ __drop_to(WT_CKPT *ckptbase, const char *name, size_t len) } /* - * __checkpoint_lock_tree -- - * Acquire the locks required to checkpoint a tree. + * __checkpoint_lock_dirty_tree -- + * Decide whether the tree needs to be included in the checkpoint and if + * so, acquire the necessary locks. */ static int -__checkpoint_lock_tree(WT_SESSION_IMPL *session, - bool is_checkpoint, bool need_tracking, const char *cfg[]) +__checkpoint_lock_dirty_tree(WT_SESSION_IMPL *session, + bool is_checkpoint, bool force, bool need_tracking, const char *cfg[]) { WT_BTREE *btree; WT_CKPT *ckpt, *ckptbase; @@ -1057,7 +1147,7 @@ __checkpoint_lock_tree(WT_SESSION_IMPL *session, * - On connection close when we know there can't be any races. */ WT_ASSERT(session, !need_tracking || - WT_IS_METADATA(session, dhandle) || WT_META_TRACKING(session)); + WT_IS_METADATA(dhandle) || WT_META_TRACKING(session)); /* Get the list of checkpoints for this file. */ WT_RET(__wt_meta_ckptlist_get(session, dhandle->name, &ckptbase)); @@ -1127,7 +1217,7 @@ __checkpoint_lock_tree(WT_SESSION_IMPL *session, * Hold the lock until we're done (blocking hot backups from starting), * we don't want to race with a future hot backup. */ - __wt_readlock(session, conn->hot_backup_lock); + __wt_readlock(session, &conn->hot_backup_lock); hot_backup_locked = true; if (conn->hot_backup) WT_CKPT_FOREACH(ckptbase, ckpt) { @@ -1144,6 +1234,14 @@ __checkpoint_lock_tree(WT_SESSION_IMPL *session, ckpt->name); } + /* + * Mark old checkpoints that are being deleted and figure out which + * trees we can skip in this checkpoint. + */ + WT_ERR(__checkpoint_mark_skip(session, ckptbase, force)); + if (F_ISSET(btree, WT_BTREE_SKIP_CKPT)) + goto err; + /* * Lock the checkpoints that will be deleted. * @@ -1177,64 +1275,47 @@ __checkpoint_lock_tree(WT_SESSION_IMPL *session, } /* - * There are special files: those being bulk-loaded, salvaged, upgraded - * or verified during the checkpoint. We have to do something for those - * objects because a checkpoint is an external name the application can - * reference and the name must exist no matter what's happening during - * the checkpoint. For bulk-loaded files, we could block until the load - * completes, checkpoint the partial load, or magic up an empty-file - * checkpoint. The first is too slow, the second is insane, so do the - * third. - * Salvage, upgrade and verify don't currently require any work, all - * three hold the schema lock, blocking checkpoints. If we ever want to - * fix that (and I bet we eventually will, at least for verify), we can - * copy the last checkpoint the file has. That works if we guarantee - * salvage, upgrade and verify act on objects with previous checkpoints - * (true if handles are closed/re-opened between object creation and a - * subsequent salvage, upgrade or verify operation). Presumably, - * salvage and upgrade will discard all previous checkpoints when they - * complete, which is fine with us. This change will require reference - * counting checkpoints, and once that's done, we should use checkpoint - * copy instead of forcing checkpoints on clean objects to associate - * names with checkpoints. + * There are special tree: those being bulk-loaded, salvaged, upgraded + * or verified during the checkpoint. They should never be part of a + * checkpoint: we will fail to lock them because the operations have + * exclusive access to the handles. Named checkpoints will fail in that + * case, ordinary checkpoints will skip files that cannot be opened + * normally. */ WT_ASSERT(session, !is_checkpoint || !F_ISSET(btree, WT_BTREE_SPECIAL_FLAGS)); - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); - WT_ASSERT(session, btree->ckpt == NULL); + WT_ASSERT(session, btree->ckpt == NULL && + !F_ISSET(btree, WT_BTREE_SKIP_CKPT)); btree->ckpt = ckptbase; return (0); err: if (hot_backup_locked) - __wt_readunlock(session, conn->hot_backup_lock); + __wt_readunlock(session, &conn->hot_backup_lock); - __wt_meta_ckptlist_free(session, ckptbase); + __wt_meta_ckptlist_free(session, &ckptbase); __wt_free(session, name_alloc); return (ret); } /* - * __checkpoint_mark_deletes -- - * Figure out what old checkpoints will be deleted, and whether the - * checkpoint can be skipped entirely. + * __checkpoint_mark_skip -- + * Figure out whether the checkpoint can be skipped for a tree. */ static int -__checkpoint_mark_deletes( - WT_SESSION_IMPL *session, const char *cfg[]) +__checkpoint_mark_skip( + WT_SESSION_IMPL *session, WT_CKPT *ckptbase, bool force) { WT_BTREE *btree; - WT_CKPT *ckpt, *ckptbase; - WT_CONFIG_ITEM cval; + WT_CKPT *ckpt; const char *name; int deleted; - bool force; btree = S2BT(session); - ckptbase = btree->ckpt; /* * Check for clean objects not requiring a checkpoint. @@ -1260,12 +1341,7 @@ __checkpoint_mark_deletes( * to open the checkpoint in a cursor after taking any checkpoint, which * means it must exist. */ - force = false; F_CLR(btree, WT_BTREE_SKIP_CKPT); - if (!btree->modified && cfg != NULL) { - WT_RET(__wt_config_gets(session, cfg, "force", &cval)); - force = cval.val != 0; - } if (!btree->modified && !force) { deleted = 0; WT_CKPT_FOREACH(ckptbase, ckpt) @@ -1309,7 +1385,6 @@ __checkpoint_tree( WT_DATA_HANDLE *dhandle; WT_DECL_RET; WT_LSN ckptlsn; - int was_modified; bool fake_ckpt; WT_UNUSED(cfg); @@ -1320,7 +1395,6 @@ __checkpoint_tree( conn = S2C(session); dhandle = session->dhandle; fake_ckpt = false; - was_modified = btree->modified; /* * Set the checkpoint LSN to the maximum LSN so that if logging is @@ -1345,7 +1419,7 @@ __checkpoint_tree( * delete a physical checkpoint, and that will end in tears. */ if (is_checkpoint) - if (btree->bulk_load_ok) { + if (btree->original) { fake_ckpt = true; goto fake; } @@ -1421,7 +1495,7 @@ fake: /* * sync the file here or we could roll forward the metadata in * recovery and open a checkpoint that isn't yet durable. */ - if (WT_IS_METADATA(session, dhandle) || + if (WT_IS_METADATA(dhandle) || !F_ISSET(&session->txn, WT_TXN_RUNNING)) WT_ERR(__wt_checkpoint_sync(session, NULL)); @@ -1451,14 +1525,12 @@ err: /* * If the checkpoint didn't complete successfully, make sure the * tree is marked dirty. */ - if (ret != 0 && !btree->modified && was_modified) { + if (ret != 0) { btree->modified = true; - if (!S2C(session)->modified) - S2C(session)->modified = true; + S2C(session)->modified = true; } - __wt_meta_ckptlist_free(session, ckptbase); - btree->ckpt = NULL; + __wt_meta_ckptlist_free(session, &btree->ckpt); return (ret); } @@ -1477,7 +1549,8 @@ __checkpoint_presync(WT_SESSION_IMPL *session, const char *cfg[]) WT_UNUSED(cfg); btree = S2BT(session); - WT_ASSERT(session, !btree->include_checkpoint_txn); + WT_ASSERT(session, btree->checkpoint_gen == + S2C(session)->txn_global.checkpoint_gen); btree->evict_walk_period = btree->evict_walk_saved; return (0); } @@ -1526,21 +1599,24 @@ __checkpoint_tree_helper(WT_SESSION_IMPL *session, const char *cfg[]) int __wt_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) { + WT_CONFIG_ITEM cval; WT_DECL_RET; + bool force; /* Should not be called with a checkpoint handle. */ WT_ASSERT(session, session->dhandle->checkpoint == NULL); /* We must hold the metadata lock if checkpointing the metadata. */ - WT_ASSERT(session, !WT_IS_METADATA(session, session->dhandle) || + WT_ASSERT(session, !WT_IS_METADATA(session->dhandle) || F_ISSET(session, WT_SESSION_LOCKED_METADATA)); - WT_SAVE_DHANDLE(session, - ret = __checkpoint_lock_tree(session, true, true, cfg)); - WT_RET(ret); - WT_SAVE_DHANDLE(session, - ret = __checkpoint_mark_deletes(session, cfg)); + WT_RET(__wt_config_gets_def(session, cfg, "force", 0, &cval)); + force = cval.val != 0; + WT_SAVE_DHANDLE(session, ret = __checkpoint_lock_dirty_tree( + session, true, force, true, cfg)); WT_RET(ret); + if (F_ISSET(S2BT(session), WT_BTREE_SKIP_CKPT)) + return (0); return (__checkpoint_tree(session, true, cfg)); } @@ -1615,15 +1691,10 @@ __wt_checkpoint_close(WT_SESSION_IMPL *session, bool final) if (need_tracking) WT_RET(__wt_meta_track_on(session)); - WT_SAVE_DHANDLE(session, - ret = __checkpoint_lock_tree(session, false, need_tracking, NULL)); + WT_SAVE_DHANDLE(session, ret = __checkpoint_lock_dirty_tree( + session, false, false, need_tracking, NULL)); WT_ASSERT(session, ret == 0); - if (ret == 0) { - WT_SAVE_DHANDLE(session, - ret = __checkpoint_mark_deletes(session, NULL)); - WT_ASSERT(session, ret == 0); - } - if (ret == 0) + if (ret == 0 && !F_ISSET(btree, WT_BTREE_SKIP_CKPT)) ret = __checkpoint_tree(session, false, NULL); if (need_tracking) diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c index f1b78879d76..2931dc1ce82 100644 --- a/src/third_party/wiredtiger/src/txn/txn_log.c +++ b/src/third_party/wiredtiger/src/txn/txn_log.c @@ -269,7 +269,7 @@ __wt_txn_checkpoint_logread(WT_SESSION_IMPL *session, WT_ITEM ckpt_snapshot_unused; uint32_t ckpt_file, ckpt_offset; u_int ckpt_nsnapshot_unused; - const char *fmt = WT_UNCHECKED_STRING(IIIU); + const char *fmt = WT_UNCHECKED_STRING(IIIu); if ((ret = __wt_struct_unpack(session, *pp, WT_PTRDIFF(end, *pp), fmt, &ckpt_file, &ckpt_offset, @@ -297,7 +297,7 @@ __wt_txn_checkpoint_log( uint8_t *end, *p; size_t recsize; uint32_t i, rectype = WT_LOGREC_CHECKPOINT; - const char *fmt = WT_UNCHECKED_STRING(IIIIU); + const char *fmt = WT_UNCHECKED_STRING(IIIIu); txn = &session->txn; ckpt_lsn = &txn->ckpt_lsn; @@ -368,14 +368,16 @@ __wt_txn_checkpoint_log( /* * If this full checkpoint completed successfully and there is - * no hot backup in progress, tell the logging subsystem the - * checkpoint LSN so that it can archive. Do not update the - * logging checkpoint LSN if this is during a clean connection - * close, only during a full checkpoint. A clean close may not - * update any metadata LSN and we do not want to archive in - * that case. + * no hot backup in progress and this is not recovery, tell + * the logging subsystem the checkpoint LSN so that it can + * archive. Do not update the logging checkpoint LSN if this + * is during a clean connection close, only during a full + * checkpoint. A clean close may not update any metadata LSN + * and we do not want to archive in that case. */ - if (!S2C(session)->hot_backup && txn->full_ckpt) + if (!S2C(session)->hot_backup && + !F_ISSET(S2C(session), WT_CONN_RECOVERING) && + txn->full_ckpt) __wt_log_ckpt(session, ckpt_lsn); /* FALLTHROUGH */ @@ -551,6 +553,7 @@ __txn_printlog(WT_SESSION_IMPL *session, */ int __wt_txn_printlog(WT_SESSION *wt_session, uint32_t flags) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { WT_SESSION_IMPL *session; WT_TXN_PRINTLOG_ARGS args; diff --git a/src/third_party/wiredtiger/src/txn/txn_nsnap.c b/src/third_party/wiredtiger/src/txn/txn_nsnap.c index 7ba0cc8700e..659570dbcd9 100644 --- a/src/third_party/wiredtiger/src/txn/txn_nsnap.c +++ b/src/third_party/wiredtiger/src/txn/txn_nsnap.c @@ -28,7 +28,6 @@ __nsnap_destroy(WT_SESSION_IMPL *session, WT_NAMED_SNAPSHOT *nsnap) static int __nsnap_drop_one(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *name) { - WT_DECL_RET; WT_NAMED_SNAPSHOT *found; WT_TXN_GLOBAL *txn_global; @@ -56,7 +55,7 @@ __nsnap_drop_one(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *name) __nsnap_destroy(session, found); WT_STAT_CONN_INCR(session, txn_snapshots_dropped); - return (ret); + return (0); } /* @@ -67,7 +66,6 @@ __nsnap_drop_one(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *name) static int __nsnap_drop_to(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *name, bool inclusive) { - WT_DECL_RET; WT_NAMED_SNAPSHOT *last, *nsnap, *prev; WT_TXN_GLOBAL *txn_global; uint64_t new_nsnap_oldest; @@ -134,7 +132,7 @@ __nsnap_drop_to(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *name, bool inclusive) new_nsnap_oldest == WT_TXN_NONE || !__wt_txn_visible_all(session, new_nsnap_oldest)); - return (ret); + return (0); } /* @@ -152,26 +150,45 @@ __wt_txn_named_snapshot_begin(WT_SESSION_IMPL *session, const char *cfg[]) const char *txn_cfg[] = { WT_CONFIG_BASE(session, WT_SESSION_begin_transaction), "isolation=snapshot", NULL }; - bool started_txn; + bool include_updates, started_txn; started_txn = false; nsnap_new = NULL; txn_global = &S2C(session)->txn_global; txn = &session->txn; + WT_RET(__wt_config_gets_def(session, cfg, "include_updates", 0, &cval)); + include_updates = cval.val != 0; + WT_RET(__wt_config_gets_def(session, cfg, "name", 0, &cval)); WT_ASSERT(session, cval.len != 0); if (!F_ISSET(txn, WT_TXN_RUNNING)) { + if (include_updates) + WT_RET_MSG(session, EINVAL, "A transaction must be " + "running to include updates in a named snapshot"); + WT_RET(__wt_txn_begin(session, txn_cfg)); started_txn = true; } - F_SET(txn, WT_TXN_READONLY); + if (!include_updates) + F_SET(txn, WT_TXN_READONLY); /* Save a copy of the transaction's snapshot. */ WT_ERR(__wt_calloc_one(session, &nsnap_new)); nsnap = nsnap_new; WT_ERR(__wt_strndup(session, cval.str, cval.len, &nsnap->name)); + + /* + * To include updates from a writing transaction, make sure a + * transaction ID has been allocated. + */ + if (include_updates) { + WT_ERR(__wt_txn_id_check(session)); + WT_ASSERT(session, txn->id != WT_TXN_NONE); + nsnap->id = txn->id; + } else + nsnap->id = WT_TXN_NONE; nsnap->pinned_id = WT_SESSION_TXN_STATE(session)->pinned_id; nsnap->snap_min = txn->snap_min; nsnap->snap_max = txn->snap_max; @@ -194,9 +211,9 @@ __wt_txn_named_snapshot_begin(WT_SESSION_IMPL *session, const char *cfg[]) if (TAILQ_EMPTY(&txn_global->nsnaph)) { WT_ASSERT(session, txn_global->nsnap_oldest_id == WT_TXN_NONE && !__wt_txn_visible_all(session, nsnap_new->pinned_id)); - __wt_readlock(session, txn_global->scan_rwlock); + __wt_readlock(session, &txn_global->scan_rwlock); txn_global->nsnap_oldest_id = nsnap_new->pinned_id; - __wt_readunlock(session, txn_global->scan_rwlock); + __wt_readunlock(session, &txn_global->scan_rwlock); } TAILQ_INSERT_TAIL(&txn_global->nsnaph, nsnap_new, q); WT_STAT_CONN_INCR(session, txn_snapshots_created); @@ -209,8 +226,7 @@ err: if (started_txn) { WT_TRET(__wt_txn_rollback(session, NULL)); WT_DIAGNOSTIC_YIELD; WT_ASSERT(session, !__wt_txn_visible_all(session, pinned_id)); - } else if (ret == 0) - F_SET(txn, WT_TXN_NAMED_SNAPSHOT); + } if (nsnap_new != NULL) __nsnap_destroy(session, nsnap_new); @@ -281,16 +297,16 @@ __wt_txn_named_snapshot_get(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *nameval) if (session->ncursors > 0) WT_RET(__wt_session_copy_values(session)); - __wt_readlock(session, txn_global->nsnap_rwlock); + __wt_readlock(session, &txn_global->nsnap_rwlock); TAILQ_FOREACH(nsnap, &txn_global->nsnaph, q) if (WT_STRING_MATCH(nsnap->name, nameval->str, nameval->len)) { /* * Acquire the scan lock so the oldest ID can't move * forward without seeing our pinned ID. */ - __wt_readlock(session, txn_global->scan_rwlock); + __wt_readlock(session, &txn_global->scan_rwlock); txn_state->pinned_id = nsnap->pinned_id; - __wt_readunlock(session, txn_global->scan_rwlock); + __wt_readunlock(session, &txn_global->scan_rwlock); WT_ASSERT(session, !__wt_txn_visible_all( session, txn_state->pinned_id) && @@ -303,10 +319,15 @@ __wt_txn_named_snapshot_get(WT_SESSION_IMPL *session, WT_CONFIG_ITEM *nameval) memcpy(txn->snapshot, nsnap->snapshot, nsnap->snapshot_count * sizeof(*nsnap->snapshot)); + if (nsnap->id != WT_TXN_NONE) { + WT_ASSERT(session, txn->id == WT_TXN_NONE); + txn->id = nsnap->id; + F_SET(txn, WT_TXN_READONLY); + } F_SET(txn, WT_TXN_HAS_SNAPSHOT); break; } - __wt_readunlock(session, txn_global->nsnap_rwlock); + __wt_readunlock(session, &txn_global->nsnap_rwlock); if (nsnap == NULL) WT_RET_MSG(session, EINVAL, diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index 65811aa3bf4..30932195b1e 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -93,7 +93,7 @@ __recovery_cursor(WT_SESSION_IMPL *session, WT_RECOVERY *r, "%s op %" PRIu32 " to file %" PRIu32 " at LSN %" PRIu32 \ "/%" PRIu32, \ cursor == NULL ? "Skipping" : "Applying", \ - optype, fileid, lsnp->l.file, lsnp->l.offset); \ + optype, fileid, (lsnp)->l.file, (lsnp)->l.offset); \ if (cursor == NULL) \ break @@ -248,8 +248,6 @@ static int __txn_commit_apply( WT_RECOVERY *r, WT_LSN *lsnp, const uint8_t **pp, const uint8_t *end) { - WT_UNUSED(lsnp); - /* The logging subsystem zero-pads records. */ while (*pp < end && **pp) WT_RET(__txn_op_apply(r, lsnp, pp, end)); @@ -503,7 +501,7 @@ __wt_txn_recover(WT_SESSION_IMPL *session) * Pass WT_LOGSCAN_RECOVER so that old logs get truncated. */ r.metadata_only = false; - __wt_verbose(session, WT_VERB_RECOVERY, + __wt_verbose(session, WT_VERB_RECOVERY | WT_VERB_RECOVERY_PROGRESS, "Main recovery loop: starting at %" PRIu32 "/%" PRIu32, r.ckpt_lsn.l.file, r.ckpt_lsn.l.offset); WT_ERR(__wt_log_needs_recovery(session, &r.ckpt_lsn, &needs_rec)); diff --git a/src/third_party/wiredtiger/src/utilities/util.h b/src/third_party/wiredtiger/src/utilities/util.h index c2cf6c22aa4..93a96d44219 100644 --- a/src/third_party/wiredtiger/src/utilities/util.h +++ b/src/third_party/wiredtiger/src/utilities/util.h @@ -26,6 +26,7 @@ extern int __wt_optopt; /* character checked for validity */ extern int __wt_optreset; /* reset getopt */ extern char *__wt_optarg; /* argument associated with option */ +int util_alter(WT_SESSION *, int, char *[]); int util_backup(WT_SESSION *, int, char *[]); int util_cerr(WT_CURSOR *, const char *, int); int util_compact(WT_SESSION *, int, char *[]); @@ -39,7 +40,6 @@ int util_flush(WT_SESSION *, const char *); int util_list(WT_SESSION *, int, char *[]); int util_load(WT_SESSION *, int, char *[]); int util_loadtext(WT_SESSION *, int, char *[]); -char *util_name(WT_SESSION *, const char *, const char *); int util_printlog(WT_SESSION *, int, char *[]); int util_read(WT_SESSION *, int, char *[]); int util_read_line(WT_SESSION *, ULINE *, bool, bool *); @@ -48,6 +48,8 @@ int util_rename(WT_SESSION *, int, char *[]); int util_salvage(WT_SESSION *, int, char *[]); int util_stat(WT_SESSION *, int, char *[]); int util_str2recno(WT_SESSION *, const char *p, uint64_t *recnop); +int util_truncate(WT_SESSION *, int, char *[]); int util_upgrade(WT_SESSION *, int, char *[]); +char *util_uri(WT_SESSION *, const char *, const char *); int util_verify(WT_SESSION *, int, char *[]); int util_write(WT_SESSION *, int, char *[]); diff --git a/src/third_party/wiredtiger/src/utilities/util_alter.c b/src/third_party/wiredtiger/src/utilities/util_alter.c new file mode 100644 index 00000000000..ef01a1ed826 --- /dev/null +++ b/src/third_party/wiredtiger/src/utilities/util_alter.c @@ -0,0 +1,53 @@ +/*- + * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int usage(void); + +int +util_alter(WT_SESSION *session, int argc, char *argv[]) +{ + WT_DECL_RET; + int ch; + char **configp; + + while ((ch = __wt_getopt(progname, argc, argv, "")) != EOF) + switch (ch) { + case '?': + default: + return (usage()); + } + + argc -= __wt_optind; + argv += __wt_optind; + + /* The remaining arguments are uri/string pairs. */ + if (argc % 2 != 0) + return (usage()); + + for (configp = argv; + configp != NULL && *configp != NULL; configp += 2) + if ((ret = session->alter( + session, configp[0], configp[1])) != 0) { + (void)util_err(session, ret, + "session.alter: %s, %s", configp[0], configp[1]); + return (1); + } + return (0); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "alter uri configuration ...\n", + progname, usage_prefix); + return (1); +} diff --git a/src/third_party/wiredtiger/src/utilities/util_backup.c b/src/third_party/wiredtiger/src/utilities/util_backup.c index 5dc9671fb45..f1b31f7621a 100644 --- a/src/third_party/wiredtiger/src/utilities/util_backup.c +++ b/src/third_party/wiredtiger/src/utilities/util_backup.c @@ -109,9 +109,14 @@ copy(WT_SESSION *session, const char *directory, const char *name) /* Build the target pathname. */ len = strlen(directory) + strlen(name) + 2; - if ((to = malloc(len)) == NULL) - goto memerr; - (void)snprintf(to, len, "%s/%s", directory, name); + if ((to = malloc(len)) == NULL) { + fprintf(stderr, "%s: %s\n", progname, strerror(errno)); + return (1); + } + if ((ret = __wt_snprintf(to, len, "%s/%s", directory, name)) != 0) { + fprintf(stderr, "%s: %s\n", progname, strerror(ret)); + goto err; + } if (verbose && printf("Backing up %s/%s to %s\n", home, name, to) < 0) { fprintf(stderr, "%s: %s\n", progname, strerror(EIO)); @@ -126,11 +131,7 @@ copy(WT_SESSION *session, const char *directory, const char *name) fprintf(stderr, "%s/%s to %s: backup copy: %s\n", home, name, to, session->strerror(session, ret)); - if (0) { -memerr: fprintf(stderr, "%s: %s\n", progname, strerror(errno)); - } err: free(to); - return (ret); } diff --git a/src/third_party/wiredtiger/src/utilities/util_compact.c b/src/third_party/wiredtiger/src/utilities/util_compact.c index c114eb207fa..e469b4dce6e 100644 --- a/src/third_party/wiredtiger/src/utilities/util_compact.c +++ b/src/third_party/wiredtiger/src/utilities/util_compact.c @@ -30,21 +30,13 @@ util_compact(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the table name. */ if (argc != 1) return (usage()); - if ((uri = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); - if ((ret = session->compact(session, uri, NULL)) != 0) { - fprintf(stderr, "%s: compact(%s): %s\n", - progname, uri, session->strerror(session, ret)); - goto err; - } - - if (0) { -err: ret = 1; - } + if ((ret = session->compact(session, uri, NULL)) != 0) + (void)util_err(session, ret, "session.compact: %s", uri); free(uri); - return (ret); } diff --git a/src/third_party/wiredtiger/src/utilities/util_create.c b/src/third_party/wiredtiger/src/utilities/util_create.c index 4e609736f2d..7c22a67792b 100644 --- a/src/third_party/wiredtiger/src/utilities/util_create.c +++ b/src/third_party/wiredtiger/src/utilities/util_create.c @@ -15,9 +15,9 @@ util_create(WT_SESSION *session, int argc, char *argv[]) { WT_DECL_RET; int ch; - const char *config, *uri; + char *config, *uri; - config = NULL; + config = uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "c:")) != EOF) switch (ch) { case 'c': /* command-line configuration */ @@ -35,12 +35,14 @@ util_create(WT_SESSION *session, int argc, char *argv[]) if (argc != 1) return (usage()); - if ((uri = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); if ((ret = session->create(session, uri, config)) != 0) - return (util_err(session, ret, "%s: session.create", uri)); - return (0); + (void)util_err(session, ret, "session.create: %s", uri); + + free(uri); + return (ret); } static int diff --git a/src/third_party/wiredtiger/src/utilities/util_drop.c b/src/third_party/wiredtiger/src/utilities/util_drop.c index ba41445dfb6..456005d445d 100644 --- a/src/third_party/wiredtiger/src/utilities/util_drop.c +++ b/src/third_party/wiredtiger/src/utilities/util_drop.c @@ -15,8 +15,9 @@ util_drop(WT_SESSION *session, int argc, char *argv[]) { WT_DECL_RET; int ch; - char *name; + char *uri; + uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "")) != EOF) switch (ch) { case '?': @@ -30,12 +31,13 @@ util_drop(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the uri. */ if (argc != 1) return (usage()); - if ((name = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); - ret = session->drop(session, name, "force"); + if ((ret = session->drop(session, uri, "force")) != 0) + (void)util_err(session, ret, "session.drop: %s", uri); - free(name); + free(uri); return (ret); } diff --git a/src/third_party/wiredtiger/src/utilities/util_dump.c b/src/third_party/wiredtiger/src/utilities/util_dump.c index 7dde13ee837..955148b7d46 100644 --- a/src/third_party/wiredtiger/src/utilities/util_dump.c +++ b/src/third_party/wiredtiger/src/utilities/util_dump.c @@ -6,10 +6,14 @@ * See the file LICENSE for redistribution information. */ +#include #include "util.h" #include "util_dump.h" -static int dump_config(WT_SESSION *, const char *, bool, bool); +#define STRING_MATCH_CONFIG(s, item) \ + (strncmp(s, (item).str, (item).len) == 0 && (s)[(item).len] == '\0') + +static int dump_config(WT_SESSION *, const char *, WT_CURSOR *, bool, bool); static int dump_json_begin(WT_SESSION *); static int dump_json_end(WT_SESSION *); static int dump_json_separator(WT_SESSION *); @@ -17,7 +21,8 @@ static int dump_json_table_end(WT_SESSION *); static int dump_prefix(WT_SESSION *, bool, bool); static int dump_record(WT_CURSOR *, bool, bool); static int dump_suffix(WT_SESSION *, bool); -static int dump_table_config(WT_SESSION *, WT_CURSOR *, const char *, bool); +static int dump_table_config( + WT_SESSION *, WT_CURSOR *, WT_CURSOR *, const char *, bool); static int dump_table_parts_config( WT_SESSION *, WT_CURSOR *, const char *, const char *, bool); static int dup_json_string(const char *, char **); @@ -32,10 +37,11 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) size_t len; int ch, i; bool hex, json, reverse; - char *checkpoint, *config, *name; + char *checkpoint, *config, *p, *simpleuri, *uri; hex = json = reverse = false; - checkpoint = config = name = NULL; + checkpoint = config = simpleuri = uri = NULL; + cursor = NULL; while ((ch = __wt_getopt(progname, argc, argv, "c:f:jrx")) != EOF) switch (ch) { case 'c': @@ -75,21 +81,19 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) return (usage()); if (json && - ((ret = dump_json_begin(session)) != 0 || - (ret = dump_prefix(session, hex, json)) != 0)) + (dump_json_begin(session) != 0 || + dump_prefix(session, hex, json) != 0)) goto err; for (i = 0; i < argc; i++) { if (json && i > 0) - if ((ret = dump_json_separator(session)) != 0) + if (dump_json_separator(session) != 0) goto err; - free(name); - name = NULL; - - if ((name = util_name(session, argv[i], "table")) == NULL) - goto err; + free(uri); + free(simpleuri); + uri = simpleuri = NULL; - if (dump_config(session, name, hex, json) != 0) + if ((uri = util_uri(session, argv[i], "table")) == NULL) goto err; len = @@ -109,18 +113,34 @@ util_dump(WT_SESSION *session, int argc, char *argv[]) (void)strcat(config, json ? "dump=json" : (hex ? "dump=hex" : "dump=print")); if ((ret = session->open_cursor( - session, name, NULL, config, &cursor)) != 0) { + session, uri, NULL, config, &cursor)) != 0) { fprintf(stderr, "%s: cursor open(%s) failed: %s\n", - progname, name, session->strerror(session, ret)); + progname, uri, session->strerror(session, ret)); + goto err; + } + + if ((simpleuri = strdup(uri)) == NULL) { + (void)util_err(session, errno, NULL); goto err; } + if ((p = strchr(simpleuri, '(')) != NULL) + *p = '\0'; + if (dump_config(session, simpleuri, cursor, hex, json) != 0) + goto err; - if ((ret = dump_record(cursor, reverse, json)) != 0) + if (dump_record(cursor, reverse, json) != 0) goto err; - if (json && (ret = dump_json_table_end(session)) != 0) + if (json && dump_json_table_end(session) != 0) goto err; + + ret = cursor->close(cursor); + cursor = NULL; + if (ret != 0) { + (void)util_err(session, ret, NULL); + goto err; + } } - if (json && ((ret = dump_json_end(session)) != 0)) + if (json && dump_json_end(session) != 0) goto err; if (0) { @@ -128,8 +148,12 @@ err: ret = 1; } free(config); - free(name); - + free(uri); + free(simpleuri); + if (cursor != NULL && (ret = cursor->close(cursor)) != 0) { + (void)util_err(session, ret, NULL); + ret = 1; + } return (ret); } @@ -138,15 +162,16 @@ err: ret = 1; * Dump the config for the uri. */ static int -dump_config(WT_SESSION *session, const char *uri, bool hex, bool json) +dump_config(WT_SESSION *session, const char *uri, WT_CURSOR *cursor, bool hex, + bool json) { - WT_CURSOR *cursor; + WT_CURSOR *mcursor; WT_DECL_RET; int tret; /* Open a metadata cursor. */ if ((ret = session->open_cursor( - session, "metadata:create", NULL, NULL, &cursor)) != 0) { + session, "metadata:create", NULL, NULL, &mcursor)) != 0) { fprintf(stderr, "%s: %s: session.open_cursor: %s\n", progname, "metadata:create", session->strerror(session, ret)); return (1); @@ -156,10 +181,11 @@ dump_config(WT_SESSION *session, const char *uri, bool hex, bool json) * want to output a header if the user entered the wrong name. This is * where we find out a table doesn't exist, use a simple error message. */ - cursor->set_key(cursor, uri); - if ((ret = cursor->search(cursor)) == 0) { + mcursor->set_key(mcursor, uri); + if ((ret = mcursor->search(mcursor)) == 0) { if ((!json && dump_prefix(session, hex, json) != 0) || - dump_table_config(session, cursor, uri, json) != 0 || + dump_table_config(session, mcursor, cursor, + uri, json) != 0 || dump_suffix(session, json) != 0) ret = 1; } else if (ret == WT_NOTFOUND) @@ -167,8 +193,8 @@ dump_config(WT_SESSION *session, const char *uri, bool hex, bool json) else ret = util_err(session, ret, "%s", uri); - if ((tret = cursor->close(cursor)) != 0) { - tret = util_cerr(cursor, "close", tret); + if ((tret = mcursor->close(mcursor)) != 0) { + tret = util_cerr(mcursor, "close", tret); if (ret == 0) ret = tret; } @@ -224,17 +250,127 @@ dump_json_table_end(WT_SESSION *session) return (0); } +/* + * dump_add_config + * Add a formatted config string to an output buffer. + */ +static int +dump_add_config(WT_SESSION *session, char **bufp, size_t *leftp, + const char *fmt, ...) + WT_GCC_FUNC_ATTRIBUTE((format (printf, 4, 5))) +{ + WT_DECL_RET; + size_t n; + va_list ap; + + va_start(ap, fmt); + ret = __wt_vsnprintf_len_set(*bufp, *leftp, &n, fmt, ap); + va_end(ap); + if (ret != 0) + return (util_err(session, ret, NULL)); + *bufp += n; + *leftp -= (size_t)n; + return (0); +} + +/* + * dump_projection -- + * Create a new config containing projection information. + */ +static int +dump_projection(WT_SESSION *session, const char *config, WT_CURSOR *cursor, + char **newconfigp) +{ + WT_DECL_RET; + WT_CONFIG_ITEM key, value; + WT_CONFIG_PARSER *parser; + WT_EXTENSION_API *wt_api; + size_t len, vallen; + int nkeys; + char *newconfig; + const char *keyformat, *p; + + len = strlen(config) + strlen(cursor->value_format) + + strlen(cursor->uri) + 20; + if ((newconfig = malloc(len)) == NULL) + return util_err(session, errno, NULL); + *newconfigp = newconfig; + wt_api = session->connection->get_extension_api(session->connection); + if ((ret = wt_api->config_parser_open(wt_api, session, config, + strlen(config), &parser)) != 0) + return (util_err( + session, ret, "WT_EXTENSION_API.config_parser_open")); + keyformat = cursor->key_format; + for (nkeys = 0; *keyformat; keyformat++) + if (!__wt_isdigit((u_char)*keyformat)) + nkeys++; + + /* + * Copy the configuration, replacing some fields to match the + * projection. + */ + while ((ret = parser->next(parser, &key, &value)) == 0) { + WT_RET(dump_add_config(session, &newconfig, &len, + "%.*s=", (int)key.len, key.str)); + if (STRING_MATCH_CONFIG("value_format", key)) + WT_RET(dump_add_config(session, &newconfig, &len, + "%s", cursor->value_format)); + else if (STRING_MATCH_CONFIG("columns", key)) { + /* copy names of keys */ + p = value.str; + vallen = value.len; + while (vallen > 0) { + if ((*p == ',' || *p == ')') && --nkeys == 0) + break; + p++; + vallen--; + } + WT_RET(dump_add_config(session, &newconfig, &len, + "%.*s", (int)(p - value.str), value.str)); + + /* copy names of projected values */ + p = strchr(cursor->uri, '('); + assert(p != NULL); + assert(p[strlen(p) - 1] == ')'); + p++; + if (*p != ')') + WT_RET(dump_add_config(session, &newconfig, + &len, "%s", ",")); + WT_RET(dump_add_config(session, &newconfig, &len, + "%.*s),", (int)(strlen(p) - 1), p)); + } else if (value.type == WT_CONFIG_ITEM_STRING && + value.len != 0) + WT_RET(dump_add_config(session, &newconfig, &len, + "\"%.*s\",", (int)value.len, value.str)); + else + WT_RET(dump_add_config(session, &newconfig, &len, + "%.*s,", (int)value.len, value.str)); + } + if (ret != WT_NOTFOUND) + return (util_err(session, ret, "WT_CONFIG_PARSER.next")); + + assert(len > 0); + if ((ret = parser->close(parser)) != 0) + return (util_err( + session, ret, "WT_CONFIG_PARSER.close")); + + return (0); +} + /* * dump_table_config -- * Dump the config for a table. */ static int dump_table_config( - WT_SESSION *session, WT_CURSOR *cursor, const char *uri, bool json) + WT_SESSION *session, WT_CURSOR *mcursor, WT_CURSOR *cursor, + const char *uri, bool json) { WT_DECL_RET; + char *proj_config; const char *name, *v; + proj_config = NULL; /* Get the table name. */ if ((name = strchr(uri, ':')) == NULL) { fprintf(stderr, "%s: %s: corrupted uri\n", progname, uri); @@ -246,20 +382,25 @@ dump_table_config( * Dump out the config information: first, dump the uri entry itself, * it overrides all subsequent configurations. */ - cursor->set_key(cursor, uri); - if ((ret = cursor->search(cursor)) != 0) - return (util_cerr(cursor, "search", ret)); - if ((ret = cursor->get_value(cursor, &v)) != 0) - return (util_cerr(cursor, "get_value", ret)); - - WT_RET(print_config(session, uri, v, json, true)); + mcursor->set_key(mcursor, uri); + if ((ret = mcursor->search(mcursor)) != 0) + return (util_cerr(mcursor, "search", ret)); + if ((ret = mcursor->get_value(mcursor, &v)) != 0) + return (util_cerr(mcursor, "get_value", ret)); + + if (strchr(cursor->uri, '(') != NULL) { + WT_ERR(dump_projection(session, v, cursor, &proj_config)); + v = proj_config; + } + WT_ERR(print_config(session, uri, v, json, true)); - WT_RET(dump_table_parts_config( - session, cursor, name, "colgroup:", json)); - WT_RET(dump_table_parts_config( - session, cursor, name, "index:", json)); + WT_ERR(dump_table_parts_config( + session, mcursor, name, "colgroup:", json)); + WT_ERR(dump_table_parts_config( + session, mcursor, name, "index:", json)); - return (0); +err: free(proj_config); + return (ret); } /* @@ -295,9 +436,11 @@ dump_table_parts_config(WT_SESSION *session, WT_CURSOR *cursor, len = strlen(entry) + strlen(name) + 1; if ((uriprefix = malloc(len)) == NULL) - return util_err(session, errno, NULL); - - snprintf(uriprefix, len, "%s%s", entry, name); + return (util_err(session, errno, NULL)); + if ((ret = __wt_snprintf(uriprefix, len, "%s%s", entry, name)) != 0) { + free(uriprefix); + return (util_err(session, ret, NULL)); + } /* * Search the file looking for column group and index key/value pairs: @@ -364,17 +507,18 @@ dump_prefix(WT_SESSION *session, bool hex, bool json) (void)wiredtiger_version(&vmajor, &vminor, &vpatch); + if (json && printf( + " \"%s\" : \"%d (%d.%d.%d)\",\n", + DUMP_JSON_VERSION_MARKER, DUMP_JSON_CURRENT_VERSION, + vmajor, vminor, vpatch) < 0) + return (util_err(session, EIO, NULL)); + if (!json && (printf( "WiredTiger Dump (WiredTiger Version %d.%d.%d)\n", vmajor, vminor, vpatch) < 0 || printf("Format=%s\n", hex ? "hex" : "print") < 0 || printf("Header\n") < 0)) return (util_err(session, EIO, NULL)); - else if (json && printf( - " \"%s\" : \"%d (%d.%d.%d)\",\n", - DUMP_JSON_VERSION_MARKER, DUMP_JSON_CURRENT_VERSION, - vmajor, vminor, vpatch) < 0) - return (util_err(session, EIO, NULL)); return (0); } diff --git a/src/third_party/wiredtiger/src/utilities/util_list.c b/src/third_party/wiredtiger/src/utilities/util_list.c index e91dbfce05b..f19ba4d1f97 100644 --- a/src/third_party/wiredtiger/src/utilities/util_list.c +++ b/src/third_party/wiredtiger/src/utilities/util_list.c @@ -19,10 +19,10 @@ util_list(WT_SESSION *session, int argc, char *argv[]) WT_DECL_RET; int ch; bool cflag, vflag; - char *name; + char *uri; cflag = vflag = false; - name = NULL; + uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "cv")) != EOF) switch (ch) { case 'c': @@ -42,17 +42,16 @@ util_list(WT_SESSION *session, int argc, char *argv[]) case 0: break; case 1: - if ((name = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); break; default: return (usage()); } - ret = list_print(session, name, cflag, vflag); - - free(name); + ret = list_print(session, uri, cflag, vflag); + free(uri); return (ret); } @@ -99,7 +98,7 @@ list_get_allocsize(WT_SESSION *session, const char *key, size_t *allocsize) * List the high-level objects in the database. */ static int -list_print(WT_SESSION *session, const char *name, bool cflag, bool vflag) +list_print(WT_SESSION *session, const char *uri, bool cflag, bool vflag) { WT_CURSOR *cursor; WT_DECL_RET; @@ -120,7 +119,7 @@ list_print(WT_SESSION *session, const char *name, bool cflag, bool vflag) ret, "%s: WT_SESSION.open_cursor", WT_METADATA_URI)); } - found = name == NULL; + found = uri == NULL; while ((ret = cursor->next(cursor)) == 0) { /* Get the key. */ if ((ret = cursor->get_key(cursor, &key)) != 0) @@ -129,8 +128,8 @@ list_print(WT_SESSION *session, const char *name, bool cflag, bool vflag) /* * If a name is specified, only show objects that match. */ - if (name != NULL) { - if (!WT_PREFIX_MATCH(key, name)) + if (uri != NULL) { + if (!WT_PREFIX_MATCH(key, uri)) continue; found = true; } @@ -161,7 +160,7 @@ list_print(WT_SESSION *session, const char *name, bool cflag, bool vflag) if (ret != WT_NOTFOUND) return (util_cerr(cursor, "next", ret)); if (!found) { - fprintf(stderr, "%s: %s: not found\n", progname, name); + fprintf(stderr, "%s: %s: not found\n", progname, uri); return (1); } diff --git a/src/third_party/wiredtiger/src/utilities/util_load.c b/src/third_party/wiredtiger/src/utilities/util_load.c index ac18df80851..d2f00402217 100644 --- a/src/third_party/wiredtiger/src/utilities/util_load.c +++ b/src/third_party/wiredtiger/src/utilities/util_load.c @@ -80,8 +80,8 @@ util_load(WT_SESSION *session, int argc, char *argv[]) if (no_overwrite) flags |= LOAD_JSON_NO_OVERWRITE; return (util_load_json(session, filename, flags)); - } else - return (load_dump(session)); + } + return (load_dump(session)); } /* @@ -120,13 +120,15 @@ load_dump(WT_SESSION *session) goto err; /* Open the insert cursor. */ - (void)snprintf(config, sizeof(config), + if ((ret = __wt_snprintf(config, sizeof(config), "dump=%s%s%s", hex ? "hex" : "print", - append ? ",append" : "", no_overwrite ? ",overwrite=false" : ""); + append ? ",append" : "", + no_overwrite ? ",overwrite=false" : "")) != 0) + return (util_err(session, ret, NULL)); if ((ret = session->open_cursor( session, uri, NULL, config, &cursor)) != 0) { - ret = util_err(session, ret, "%s: session.open", uri); + ret = util_err(session, ret, "%s: session.open_cursor", uri); goto err; } @@ -472,6 +474,7 @@ config_update(WT_SESSION *session, char **list) static int config_rename(WT_SESSION *session, char **urip, const char *name) { + WT_DECL_RET; size_t len; char *buf, *p; @@ -490,7 +493,9 @@ config_rename(WT_SESSION *session, char **urip, const char *name) } *p = '\0'; p = strchr(p + 1, ':'); - snprintf(buf, len, "%s:%s%s", *urip, name, p == NULL ? "" : p); + if ((ret = __wt_snprintf( + buf, len, "%s:%s%s", *urip, name, p == NULL ? "" : p)) != 0) + return (util_err(session, ret, NULL)); *urip = buf; return (0); diff --git a/src/third_party/wiredtiger/src/utilities/util_load_json.c b/src/third_party/wiredtiger/src/utilities/util_load_json.c index 020a4ed9ba9..c693e2b7651 100644 --- a/src/third_party/wiredtiger/src/utilities/util_load_json.c +++ b/src/third_party/wiredtiger/src/utilities/util_load_json.c @@ -145,6 +145,7 @@ static int json_kvraw_append(WT_SESSION *session, JSON_INPUT_STATE *ins, const char *str, size_t len) { + WT_DECL_RET; size_t needsize; char *tmp; @@ -152,11 +153,15 @@ json_kvraw_append(WT_SESSION *session, needsize = strlen(ins->kvraw) + len + 2; if ((tmp = malloc(needsize)) == NULL) return (util_err(session, errno, NULL)); - snprintf(tmp, needsize, "%s %.*s", ins->kvraw, (int)len, str); + WT_ERR(__wt_snprintf( + tmp, needsize, "%s %.*s", ins->kvraw, (int)len, str)); free(ins->kvraw); ins->kvraw = tmp; } return (0); + +err: free(tmp); + return (util_err(session, ret, NULL)); } /* @@ -181,7 +186,7 @@ json_strdup(WT_SESSION *session, JSON_INPUT_STATE *ins, char **resultp) goto err; } resultlen += 1; - if ((result = (char *)malloc((size_t)resultlen)) == NULL) { + if ((result = malloc((size_t)resultlen)) == NULL) { ret = util_err(session, errno, NULL); goto err; } @@ -236,13 +241,16 @@ json_data(WT_SESSION *session, goto err; uri = clp->list[0]; - (void)snprintf(config, sizeof(config), + if ((ret = __wt_snprintf(config, sizeof(config), "dump=json%s%s", LF_ISSET(LOAD_JSON_APPEND) ? ",append" : "", - LF_ISSET(LOAD_JSON_NO_OVERWRITE) ? ",overwrite=false" : ""); + LF_ISSET(LOAD_JSON_NO_OVERWRITE) ? ",overwrite=false" : "")) != 0) { + ret = util_err(session, ret, NULL); + goto err; + } if ((ret = session->open_cursor( session, uri, NULL, config, &cursor)) != 0) { - ret = util_err(session, ret, "%s: session.open", uri); + ret = util_err(session, ret, "%s: session.open_cursor", uri); goto err; } keyformat = cursor->key_format; @@ -256,7 +264,7 @@ json_data(WT_SESSION *session, nfield = 0; JSON_EXPECT(session, ins, '{'); if (ins->kvraw == NULL) { - if ((ins->kvraw = (char *)malloc(1)) == NULL) { + if ((ins->kvraw = malloc(1)) == NULL) { ret = util_err(session, errno, NULL); goto err; } @@ -358,8 +366,11 @@ json_top_level(WT_SESSION *session, JSON_INPUT_STATE *ins, uint32_t flags) while (json_peek(session, ins) == 's') { JSON_EXPECT(session, ins, 's'); tableuri = realloc(tableuri, ins->toklen); - snprintf(tableuri, ins->toklen, "%.*s", - (int)(ins->toklen - 2), ins->tokstart + 1); + if ((ret = __wt_snprintf(tableuri, ins->toklen, + "%.*s", (int)(ins->toklen - 2), ins->tokstart + 1)) != 0) { + ret = util_err(session, ret, NULL); + goto err; + } JSON_EXPECT(session, ins, ':'); if (!hasversion) { if (strcmp(tableuri, DUMP_JSON_VERSION_MARKER) != 0) { diff --git a/src/third_party/wiredtiger/src/utilities/util_loadtext.c b/src/third_party/wiredtiger/src/utilities/util_loadtext.c index f9c5b6e9a1f..7602d43f8c9 100644 --- a/src/third_party/wiredtiger/src/utilities/util_loadtext.c +++ b/src/third_party/wiredtiger/src/utilities/util_loadtext.c @@ -15,9 +15,11 @@ static int usage(void); int util_loadtext(WT_SESSION *session, int argc, char *argv[]) { + WT_DECL_RET; int ch; - const char *uri; + char *uri; + uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "f:")) != EOF) switch (ch) { case 'f': /* input file */ @@ -35,10 +37,13 @@ util_loadtext(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the uri. */ if (argc != 1) return (usage()); - if ((uri = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); - return (text(session, uri)); + ret = text(session, uri); + + free(uri); + return (ret); } /* @@ -61,7 +66,7 @@ text(WT_SESSION *session, const char *uri) */ if ((ret = session->open_cursor( session, uri, NULL, "append,overwrite", &cursor)) != 0) - return (util_err(session, ret, "%s: session.open", uri)); + return (util_err(session, ret, "%s: session.open_cursor", uri)); /* * We're about to load strings, make sure the formats match. diff --git a/src/third_party/wiredtiger/src/utilities/util_main.c b/src/third_party/wiredtiger/src/utilities/util_main.c index 2054b94e3ce..c6f225bb667 100644 --- a/src/third_party/wiredtiger/src/utilities/util_main.c +++ b/src/third_party/wiredtiger/src/utilities/util_main.c @@ -20,7 +20,43 @@ static const char *command; /* Command name */ #define REC_LOGOFF "log=(enabled=false)" #define REC_RECOVER "log=(recover=on)" -static int usage(void); +static void +usage(void) +{ + fprintf(stderr, + "WiredTiger Data Engine (version %d.%d)\n", + WIREDTIGER_VERSION_MAJOR, WIREDTIGER_VERSION_MINOR); + fprintf(stderr, + "global options:\n" + "\t" "-C\t" "wiredtiger_open configuration\n" + "\t" "-h\t" "database directory\n" + "\t" "-L\t" "turn logging off for debug-mode\n" + "\t" "-R\t" "run recovery if configured\n" + "\t" "-V\t" "display library version and exit\n" + "\t" "-v\t" "verbose\n"); + fprintf(stderr, + "commands:\n" + "\t" "alter\t alter an object\n" + "\t" "backup\t database backup\n" + "\t" "compact\t compact an object\n" + "\t" "copyright copyright information\n" + "\t" "create\t create an object\n" + "\t" "drop\t drop an object\n" + "\t" "dump\t dump an object\n" + "\t" "list\t list database objects\n" + "\t" "load\t load an object\n" + "\t" "loadtext load an object from a text file\n" + "\t" "printlog display the database log\n" + "\t" "read\t read values from an object\n" + "\t" "rebalance rebalance an object\n" + "\t" "rename\t rename an object\n" + "\t" "salvage\t salvage a file\n" + "\t" "stat\t display statistics for an object\n" + "\t" "truncate truncate an object, removing all content\n" + "\t" "upgrade\t upgrade an object\n" + "\t" "verify\t verify an object\n" + "\t" "write\t write values to an object\n"); +} int main(int argc, char *argv[]) @@ -73,8 +109,9 @@ main(int argc, char *argv[]) cmd_config = __wt_optarg; break; case 'E': /* secret key */ + free(secretkey); /* lint: set more than once */ if ((secretkey = strdup(__wt_optarg)) == NULL) { - ret = util_err(NULL, errno, NULL); + (void)util_err(NULL, errno, NULL); goto err; } memset(__wt_optarg, 0, strlen(__wt_optarg)); @@ -92,24 +129,27 @@ main(int argc, char *argv[]) break; case 'V': /* version */ printf("%s\n", wiredtiger_version(NULL, NULL, NULL)); - return (EXIT_SUCCESS); + goto done; case 'v': /* verbose */ verbose = true; break; case '?': default: - return (usage()); + usage(); + goto err; } if (logoff && recover) { fprintf(stderr, "Only one of -L and -R is allowed.\n"); - return (EXIT_FAILURE); + goto err; } argc -= __wt_optind; argv += __wt_optind; /* The next argument is the command name. */ - if (argc < 1) - return (usage()); + if (argc < 1) { + usage(); + goto err; + } command = argv[0]; /* Reset getopt. */ @@ -117,6 +157,10 @@ main(int argc, char *argv[]) func = NULL; switch (command[0]) { + case 'a': + if (strcmp(command, "alter") == 0) + func = util_alter; + break; case 'b': if (strcmp(command, "backup") == 0) func = util_backup; @@ -126,7 +170,7 @@ main(int argc, char *argv[]) func = util_compact; else if (strcmp(command, "copyright") == 0) { util_copyright(); - return (EXIT_SUCCESS); + goto done; } else if (strcmp(command, "create") == 0) { func = util_create; config = "create"; @@ -171,6 +215,10 @@ main(int argc, char *argv[]) config = "statistics=(all)"; } break; + case 't' : + if (strcmp(command, "truncate") == 0) + func = util_truncate; + break; case 'u': if (strcmp(command, "upgrade") == 0) func = util_upgrade; @@ -186,8 +234,10 @@ main(int argc, char *argv[]) default: break; } - if (func == NULL) - return (usage()); + if (func == NULL) { + usage(); + goto err; + } /* Build the configuration string. */ len = 10; /* some slop */ @@ -204,30 +254,39 @@ main(int argc, char *argv[]) } len += strlen(rec_config); if ((p = malloc(len)) == NULL) { - ret = util_err(NULL, errno, NULL); + (void)util_err(NULL, errno, NULL); goto err; } - (void)snprintf(p, len, "%s,%s,%s%s%s%s", + if ((ret = __wt_snprintf(p, len, "%s,%s,%s%s%s%s", config == NULL ? "" : config, - cmd_config == NULL ? "" : cmd_config, rec_config, p1, p2, p3); + cmd_config == NULL ? "" : cmd_config, + rec_config, p1, p2, p3)) != 0) { + (void)util_err(NULL, ret, NULL); + goto err; + } config = p; /* Open the database and a session. */ if ((ret = wiredtiger_open(home, verbose ? verbose_handler : NULL, config, &conn)) != 0) { - ret = util_err(NULL, ret, NULL); + (void)util_err(NULL, ret, NULL); goto err; } if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) { - ret = util_err(NULL, ret, NULL); + (void)util_err(NULL, ret, NULL); goto err; } /* Call the function. */ ret = func(session, argc, argv); + if (0) { +err: ret = 1; + } +done: + /* Close the database. */ -err: if (conn != NULL && (tret = conn->close(conn, NULL)) != 0 && ret == 0) + if (conn != NULL && (tret = conn->close(conn, NULL)) != 0 && ret == 0) ret = tret; free(p); @@ -236,51 +295,14 @@ err: if (conn != NULL && (tret = conn->close(conn, NULL)) != 0 && ret == 0) return (ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE); } -static int -usage(void) -{ - fprintf(stderr, - "WiredTiger Data Engine (version %d.%d)\n", - WIREDTIGER_VERSION_MAJOR, WIREDTIGER_VERSION_MINOR); - fprintf(stderr, - "global options:\n" - "\t" "-C\t" "wiredtiger_open configuration\n" - "\t" "-h\t" "database directory\n" - "\t" "-L\t" "turn logging off for debug-mode\n" - "\t" "-R\t" "run recovery if configured\n" - "\t" "-V\t" "display library version and exit\n" - "\t" "-v\t" "verbose\n"); - fprintf(stderr, - "commands:\n" - "\t" "backup\t database backup\n" - "\t" "compact\t compact an object\n" - "\t" "copyright copyright information\n" - "\t" "create\t create an object\n" - "\t" "drop\t drop an object\n" - "\t" "dump\t dump an object\n" - "\t" "list\t list database objects\n" - "\t" "load\t load an object\n" - "\t" "loadtext load an object from a text file\n" - "\t" "printlog display the database log\n" - "\t" "read\t read values from an object\n" - "\t" "rebalance rebalance an object\n" - "\t" "rename\t rename an object\n" - "\t" "salvage\t salvage a file\n" - "\t" "stat\t display statistics for an object\n" - "\t" "upgrade\t upgrade an object\n" - "\t" "verify\t verify an object\n" - "\t" "write\t write values to an object\n"); - - return (EXIT_FAILURE); -} - /* - * util_name -- + * util_uri -- * Build a name. */ char * -util_name(WT_SESSION *session, const char *s, const char *type) +util_uri(WT_SESSION *session, const char *s, const char *type) { + WT_DECL_RET; size_t len; char *name; @@ -304,8 +326,12 @@ util_name(WT_SESSION *session, const char *s, const char *type) * the default type for the operation. */ if (strchr(s, ':') != NULL) - strcpy(name, s); + WT_ERR(__wt_snprintf(name, len, "%s", s)); else - snprintf(name, len, "%s:%s", type, s); + WT_ERR(__wt_snprintf(name, len, "%s:%s", type, s)); return (name); + +err: free(name); + (void)util_err(session, ret, NULL); + return (NULL); } diff --git a/src/third_party/wiredtiger/src/utilities/util_misc.c b/src/third_party/wiredtiger/src/utilities/util_misc.c index 0905bfa97be..e26185a0096 100644 --- a/src/third_party/wiredtiger/src/utilities/util_misc.c +++ b/src/third_party/wiredtiger/src/utilities/util_misc.c @@ -140,7 +140,10 @@ util_flush(WT_SESSION *session, const char *uri) if ((buf = malloc(len)) == NULL) return (util_err(session, errno, NULL)); - (void)snprintf(buf, len, "target=(\"%s\")", uri); + if ((ret = __wt_snprintf(buf, len, "target=(\"%s\")", uri)) != 0) { + free(buf); + return (util_err(session, ret, NULL)); + } ret = session->checkpoint(session, buf); free(buf); diff --git a/src/third_party/wiredtiger/src/utilities/util_printlog.c b/src/third_party/wiredtiger/src/utilities/util_printlog.c index e7fa2134934..5f3ed43905b 100644 --- a/src/third_party/wiredtiger/src/utilities/util_printlog.c +++ b/src/third_party/wiredtiger/src/utilities/util_printlog.c @@ -14,8 +14,8 @@ int util_printlog(WT_SESSION *session, int argc, char *argv[]) { WT_DECL_RET; - int ch; uint32_t flags; + int ch; flags = 0; while ((ch = __wt_getopt(progname, argc, argv, "f:x")) != EOF) @@ -41,17 +41,9 @@ util_printlog(WT_SESSION *session, int argc, char *argv[]) if (argc != 0) return (usage()); - ret = __wt_txn_printlog(session, flags); - - if (ret != 0) { - fprintf(stderr, "%s: printlog failed: %s\n", - progname, session->strerror(session, ret)); - goto err; - } + if ((ret = __wt_txn_printlog(session, flags)) != 0) + (void)util_err(session, ret, "printlog"); - if (0) { -err: ret = 1; - } return (ret); } diff --git a/src/third_party/wiredtiger/src/utilities/util_read.c b/src/third_party/wiredtiger/src/utilities/util_read.c index 2e766377aa9..393949b6a1c 100644 --- a/src/third_party/wiredtiger/src/utilities/util_read.c +++ b/src/third_party/wiredtiger/src/utilities/util_read.c @@ -18,8 +18,9 @@ util_read(WT_SESSION *session, int argc, char *argv[]) uint64_t recno; int ch; bool rkey, rval; - const char *uri, *value; + char *uri, *value; + uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "")) != EOF) switch (ch) { case '?': @@ -32,13 +33,19 @@ util_read(WT_SESSION *session, int argc, char *argv[]) /* The remaining arguments are a uri followed by a list of keys. */ if (argc < 2) return (usage()); - if ((uri = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); - /* Open the object. */ - if ((ret = session->open_cursor( - session, uri, NULL, NULL, &cursor)) != 0) - return (util_err(session, ret, "%s: session.open", uri)); + /* + * Open the object; free allocated memory immediately to simplify + * future error handling. + */ + if ((ret = + session->open_cursor(session, uri, NULL, NULL, &cursor)) != 0) + (void)util_err(session, ret, "%s: session.open_cursor", uri); + free(uri); + if (ret != 0) + return (ret); /* * A simple search only makes sense if the key format is a string or a diff --git a/src/third_party/wiredtiger/src/utilities/util_rebalance.c b/src/third_party/wiredtiger/src/utilities/util_rebalance.c index 45f161487e5..c188ea17d22 100644 --- a/src/third_party/wiredtiger/src/utilities/util_rebalance.c +++ b/src/third_party/wiredtiger/src/utilities/util_rebalance.c @@ -15,9 +15,9 @@ util_rebalance(WT_SESSION *session, int argc, char *argv[]) { WT_DECL_RET; int ch; - char *name; + char *uri; - name = NULL; + uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "")) != EOF) switch (ch) { case '?': @@ -30,25 +30,21 @@ util_rebalance(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the table name. */ if (argc != 1) return (usage()); - if ((name = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); - if ((ret = session->rebalance(session, name, NULL)) != 0) { - fprintf(stderr, "%s: rebalance(%s): %s\n", - progname, name, session->strerror(session, ret)); - goto err; + if ((ret = session->rebalance(session, uri, NULL)) != 0) + (void)util_err(session, ret, "session.rebalance: %s", uri); + else { + /* + * Verbose configures a progress counter, move to the next + * line. + */ + if (verbose) + printf("\n"); } - /* Verbose configures a progress counter, move to the next line. */ - if (verbose) - printf("\n"); - - if (0) { -err: ret = 1; - } - - free(name); - + free(uri); return (ret); } diff --git a/src/third_party/wiredtiger/src/utilities/util_rename.c b/src/third_party/wiredtiger/src/utilities/util_rename.c index aee299c6e63..bb2d40cd103 100644 --- a/src/third_party/wiredtiger/src/utilities/util_rename.c +++ b/src/third_party/wiredtiger/src/utilities/util_rename.c @@ -30,22 +30,15 @@ util_rename(WT_SESSION *session, int argc, char *argv[]) /* The remaining arguments are the object uri and new name. */ if (argc != 2) return (usage()); - if ((uri = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); newuri = argv[1]; - if ((ret = session->rename(session, uri, newuri, NULL)) != 0) { - fprintf(stderr, "%s: rename %s to %s: %s\n", - progname, uri, newuri, session->strerror(session, ret)); - goto err; - } - - if (0) { -err: ret = 1; - } + if ((ret = session->rename(session, uri, newuri, NULL)) != 0) + (void)util_err( + session, ret, "session.rename: %s, %s", uri, newuri); free(uri); - return (ret); } diff --git a/src/third_party/wiredtiger/src/utilities/util_salvage.c b/src/third_party/wiredtiger/src/utilities/util_salvage.c index 679d1074457..6cc2278b846 100644 --- a/src/third_party/wiredtiger/src/utilities/util_salvage.c +++ b/src/third_party/wiredtiger/src/utilities/util_salvage.c @@ -16,10 +16,10 @@ util_salvage(WT_SESSION *session, int argc, char *argv[]) WT_DECL_RET; int ch; const char *force; - char *name; + char *uri; force = NULL; - name = NULL; + uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "F")) != EOF) switch (ch) { case 'F': @@ -35,25 +35,21 @@ util_salvage(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the file name. */ if (argc != 1) return (usage()); - if ((name = util_name(session, *argv, "file")) == NULL) + if ((uri = util_uri(session, *argv, "file")) == NULL) return (1); - if ((ret = session->salvage(session, name, force)) != 0) { - fprintf(stderr, "%s: salvage(%s): %s\n", - progname, name, session->strerror(session, ret)); - goto err; + if ((ret = session->salvage(session, uri, force)) != 0) + (void)util_err(session, ret, "session.salvage: %s", uri); + else { + /* + * Verbose configures a progress counter, move to the next + * line. + */ + if (verbose) + printf("\n"); } - /* Verbose configures a progress counter, move to the next line. */ - if (verbose) - printf("\n"); - - if (0) { -err: ret = 1; - } - - free(name); - + free(uri); return (ret); } diff --git a/src/third_party/wiredtiger/src/utilities/util_stat.c b/src/third_party/wiredtiger/src/utilities/util_stat.c index 4376f559ceb..0692afe2819 100644 --- a/src/third_party/wiredtiger/src/utilities/util_stat.c +++ b/src/third_party/wiredtiger/src/utilities/util_stat.c @@ -55,7 +55,7 @@ util_stat(WT_SESSION *session, int argc, char *argv[]) objname = (char *)""; break; case 1: - if ((objname = util_name(session, *argv, "table")) == NULL) + if ((objname = util_uri(session, *argv, "table")) == NULL) return (1); objname_free = true; break; @@ -68,7 +68,10 @@ util_stat(WT_SESSION *session, int argc, char *argv[]) fprintf(stderr, "%s: %s\n", progname, strerror(errno)); goto err; } - snprintf(uri, urilen, "statistics:%s", objname); + if ((ret = __wt_snprintf(uri, urilen, "statistics:%s", objname)) != 0) { + fprintf(stderr, "%s: %s\n", progname, strerror(ret)); + goto err; + } if ((ret = session->open_cursor(session, uri, NULL, config, &cursor)) != 0) { @@ -82,8 +85,8 @@ util_stat(WT_SESSION *session, int argc, char *argv[]) (ret = cursor->next(cursor)) == 0 && (ret = cursor->get_value(cursor, &desc, &pval, NULL)) == 0) if (printf("%s=%s\n", desc, pval) < 0) { - ret = errno; - break; + (void)util_err(session, errno, "printf"); + goto err; } if (ret == WT_NOTFOUND) ret = 0; diff --git a/src/third_party/wiredtiger/src/utilities/util_truncate.c b/src/third_party/wiredtiger/src/utilities/util_truncate.c new file mode 100644 index 00000000000..35de02345c8 --- /dev/null +++ b/src/third_party/wiredtiger/src/utilities/util_truncate.c @@ -0,0 +1,52 @@ +/*- + * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "util.h" + +static int usage(void); + +int +util_truncate(WT_SESSION *session, int argc, char *argv[]) +{ + WT_DECL_RET; + int ch; + char *uri; + + uri = NULL; + while ((ch = __wt_getopt(progname, argc, argv, "")) != EOF) + switch (ch) { + case '?': + default: + return (usage()); + } + + argc -= __wt_optind; + argv += __wt_optind; + + /* The remaining argument is the uri. */ + if (argc != 1) + return (usage()); + if ((uri = util_uri(session, *argv, "table")) == NULL) + return (1); + + if ((ret = session->truncate(session, uri, NULL, NULL, NULL)) != 0) + (void)util_err(session, ret, "session.truncate: %s", uri); + + free(uri); + return (ret); +} + +static int +usage(void) +{ + (void)fprintf(stderr, + "usage: %s %s " + "truncate uri\n", + progname, usage_prefix); + return (1); +} diff --git a/src/third_party/wiredtiger/src/utilities/util_upgrade.c b/src/third_party/wiredtiger/src/utilities/util_upgrade.c index 63b23f28c16..f89bd46e133 100644 --- a/src/third_party/wiredtiger/src/utilities/util_upgrade.c +++ b/src/third_party/wiredtiger/src/utilities/util_upgrade.c @@ -15,9 +15,9 @@ util_upgrade(WT_SESSION *session, int argc, char *argv[]) { WT_DECL_RET; int ch; - char *name; + char *uri; - name = NULL; + uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "")) != EOF) switch (ch) { case '?': @@ -30,25 +30,21 @@ util_upgrade(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the table name. */ if (argc != 1) return (usage()); - if ((name = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); - if ((ret = session->upgrade(session, name, NULL)) != 0) { - fprintf(stderr, "%s: upgrade(%s): %s\n", - progname, name, session->strerror(session, ret)); - goto err; + if ((ret = session->upgrade(session, uri, NULL)) != 0) + (void)util_err(session, ret, "session.upgrade: %s", uri); + else { + /* + * Verbose configures a progress counter, move to the next + * line. + */ + if (verbose) + printf("\n"); } - /* Verbose configures a progress counter, move to the next line. */ - if (verbose) - printf("\n"); - - if (0) { -err: ret = 1; - } - - free(name); - + free(uri); return (ret); } diff --git a/src/third_party/wiredtiger/src/utilities/util_verify.c b/src/third_party/wiredtiger/src/utilities/util_verify.c index 82bdd780cd3..ace1be7a5de 100644 --- a/src/third_party/wiredtiger/src/utilities/util_verify.c +++ b/src/third_party/wiredtiger/src/utilities/util_verify.c @@ -17,10 +17,10 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) size_t size; int ch; bool dump_address, dump_blocks, dump_layout, dump_pages; - char *config, *dump_offsets, *name; + char *config, *dump_offsets, *uri; dump_address = dump_blocks = dump_layout = dump_pages = false; - config = dump_offsets = name = NULL; + config = dump_offsets = uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "d:")) != EOF) switch (ch) { case 'd': @@ -55,7 +55,7 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) /* The remaining argument is the table name. */ if (argc != 1) return (usage()); - if ((name = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); /* Build the configuration string as necessary. */ @@ -69,10 +69,10 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) strlen("dump_offsets[],") + (dump_offsets == NULL ? 0 : strlen(dump_offsets)) + 20; if ((config = malloc(size)) == NULL) { - (void)util_err(session, errno, NULL); + ret = util_err(session, errno, NULL); goto err; } - snprintf(config, size, + if ((ret = __wt_snprintf(config, size, "%s%s%s%s%s%s%s", dump_address ? "dump_address," : "", dump_blocks ? "dump_blocks," : "", @@ -80,25 +80,24 @@ util_verify(WT_SESSION *session, int argc, char *argv[]) dump_offsets != NULL ? "dump_offsets=[" : "", dump_offsets != NULL ? dump_offsets : "", dump_offsets != NULL ? "]," : "", - dump_pages ? "dump_pages," : ""); - } - if ((ret = session->verify(session, name, config)) != 0) { - fprintf(stderr, "%s: verify(%s): %s\n", - progname, name, session->strerror(session, ret)); - goto err; + dump_pages ? "dump_pages," : "")) != 0) { + (void)util_err(session, ret, NULL); + goto err; + } } - - /* Verbose configures a progress counter, move to the next line. */ - if (verbose) - printf("\n"); - - if (0) { -err: ret = 1; + if ((ret = session->verify(session, uri, config)) != 0) + (void)util_err(session, ret, "session.verify: %s", uri); + else { + /* + * Verbose configures a progress counter, move to the next + * line. + */ + if (verbose) + printf("\n"); } - free(config); - free(name); - +err: free(config); + free(uri); return (ret); } diff --git a/src/third_party/wiredtiger/src/utilities/util_write.c b/src/third_party/wiredtiger/src/utilities/util_write.c index 7d9bce02b36..1d3e6937f8d 100644 --- a/src/third_party/wiredtiger/src/utilities/util_write.c +++ b/src/third_party/wiredtiger/src/utilities/util_write.c @@ -18,10 +18,10 @@ util_write(WT_SESSION *session, int argc, char *argv[]) uint64_t recno; int ch; bool append, overwrite, rkey; - const char *uri; - char config[100]; + char *uri, config[100]; append = overwrite = false; + uri = NULL; while ((ch = __wt_getopt(progname, argc, argv, "ao")) != EOF) switch (ch) { case 'a': @@ -47,15 +47,25 @@ util_write(WT_SESSION *session, int argc, char *argv[]) } else if (argc < 3 || ((argc - 1) % 2 != 0)) return (usage()); - if ((uri = util_name(session, *argv, "table")) == NULL) + if ((uri = util_uri(session, *argv, "table")) == NULL) return (1); - /* Open the object. */ - (void)snprintf(config, sizeof(config), "%s,%s", - append ? "append=true" : "", overwrite ? "overwrite=true" : ""); - if ((ret = session->open_cursor( - session, uri, NULL, config, &cursor)) != 0) - return (util_err(session, ret, "%s: session.open", uri)); + /* + * Open the object; free allocated memory immediately to simplify + * future error handling. + */ + if ((ret = __wt_snprintf(config, sizeof(config), "%s,%s", + append ? "append=true" : "", + overwrite ? "overwrite=true" : "")) != 0) { + free(uri); + return (util_err(session, ret, NULL)); + } + if ((ret = + session->open_cursor(session, uri, NULL, config, &cursor)) != 0) + (void)util_err(session, ret, "%s: session.open_cursor", uri); + free(uri); + if (ret != 0) + return (ret); /* * A simple search only makes sense if the key format is a string or a diff --git a/src/third_party/wiredtiger/test/bloom/test_bloom.c b/src/third_party/wiredtiger/test/bloom/test_bloom.c index 67249ff887e..b6299bbbadc 100644 --- a/src/third_party/wiredtiger/test/bloom/test_bloom.c +++ b/src/third_party/wiredtiger/test/bloom/test_bloom.c @@ -29,8 +29,6 @@ #include "test_util.h" static struct { - char *progname; /* Program name */ - WT_CONNECTION *wt_conn; /* WT_CONNECTION handle */ WT_SESSION *wt_session; /* WT_SESSION handle */ @@ -61,10 +59,7 @@ main(int argc, char *argv[]) { int ch; - if ((g.progname = strrchr(argv[0], DIR_DELIM)) == NULL) - g.progname = argv[0]; - else - ++g.progname; + (void)testutil_set_progname(argv); /* Set default configuration values. */ g.c_cache = 10; @@ -75,7 +70,7 @@ main(int argc, char *argv[]) g.c_srand = 3233456; /* Set values from the command line. */ - while ((ch = __wt_getopt(g.progname, argc, argv, "c:f:k:o:s:")) != EOF) + while ((ch = __wt_getopt(progname, argc, argv, "c:f:k:o:s:")) != EOF) switch (ch) { case 'c': /* Cache size */ g.c_cache = (u_int)atoi(__wt_optarg); @@ -126,9 +121,9 @@ setup(void) * Open configuration -- put command line configuration options at the * end so they can override "standard" configuration. */ - snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "create,error_prefix=\"%s\",cache_size=%" PRIu32 "MB,%s", - g.progname, g.c_cache, g.config_open == NULL ? "" : g.config_open); + progname, g.c_cache, g.config_open == NULL ? "" : g.config_open)); testutil_check(wiredtiger_open(NULL, NULL, config, &conn)); @@ -246,7 +241,7 @@ populate_entries(void) void usage(void) { - fprintf(stderr, "usage: %s [-cfkos]\n", g.progname); + fprintf(stderr, "usage: %s [-cfkos]\n", progname); fprintf(stderr, "%s", "\t-c cache size\n" "\t-f number of bits per item\n" diff --git a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c index ef49a9492ce..84d2765843a 100644 --- a/src/third_party/wiredtiger/test/checkpoint/checkpointer.c +++ b/src/third_party/wiredtiger/test/checkpoint/checkpointer.c @@ -74,7 +74,7 @@ checkpointer(void *arg) WT_UNUSED(arg); - __wt_thread_id(tid, sizeof(tid)); + testutil_check(__wt_thread_id(tid, sizeof(tid))); printf("checkpointer thread starting: tid: %s\n", tid); (void)real_checkpointer(); @@ -107,8 +107,9 @@ real_checkpointer(void) "WiredTigerCheckpoint", strlen("WiredTigerCheckpoint")) == 0) checkpoint_config = NULL; else { + testutil_check(__wt_snprintf( + _buf, sizeof(_buf), "name=%s", g.checkpoint_name)); checkpoint_config = _buf; - snprintf(checkpoint_config, 128, "name=%s", g.checkpoint_name); } while (g.running) { /* Execute a checkpoint */ @@ -147,7 +148,8 @@ verify_checkpoint(WT_SESSION *session) ret = t_ret = 0; key_count = 0; - snprintf(ckpt, 128, "checkpoint=%s", g.checkpoint_name); + testutil_check(__wt_snprintf( + ckpt, sizeof(ckpt), "checkpoint=%s", g.checkpoint_name)); cursors = calloc((size_t)g.ntables, sizeof(*cursors)); if (cursors == NULL) return (log_print_err("verify_checkpoint", ENOMEM, 1)); @@ -159,7 +161,8 @@ verify_checkpoint(WT_SESSION *session) */ if (g.cookies[i].type == LSM) continue; - snprintf(next_uri, 128, "table:__wt%04d", i); + testutil_check(__wt_snprintf( + next_uri, sizeof(next_uri), "table:__wt%04d", i)); if ((ret = session->open_cursor( session, next_uri, NULL, ckpt, &cursors[i])) != 0) { (void)log_print_err( @@ -296,7 +299,8 @@ diagnose_key_error( session = cursor1->session; key1_orig = key2_orig = 0; - snprintf(ckpt, 128, "checkpoint=%s", g.checkpoint_name); + testutil_check(__wt_snprintf( + ckpt, sizeof(ckpt), "checkpoint=%s", g.checkpoint_name)); /* Save the failed keys. */ if (cursor1->get_key(cursor1, &key1_orig) != 0 || @@ -338,7 +342,8 @@ diagnose_key_error( * Now try opening new cursors on the checkpoints and see if we * get the same missing key via searching. */ - snprintf(next_uri, 128, "table:__wt%04d", index1); + testutil_check(__wt_snprintf( + next_uri, sizeof(next_uri), "table:__wt%04d", index1)); if (session->open_cursor(session, next_uri, NULL, ckpt, &c) != 0) return (1); c->set_key(c, key1_orig); @@ -350,7 +355,8 @@ diagnose_key_error( if (c->close(c) != 0) return (1); - snprintf(next_uri, 128, "table:__wt%04d", index2); + testutil_check(__wt_snprintf( + next_uri, sizeof(next_uri), "table:__wt%04d", index2)); if (session->open_cursor(session, next_uri, NULL, ckpt, &c) != 0) return (1); c->set_key(c, key1_orig); @@ -367,7 +373,8 @@ live_check: * Now try opening cursors on the live checkpoint to see if we get the * same missing key via searching. */ - snprintf(next_uri, 128, "table:__wt%04d", index1); + testutil_check(__wt_snprintf( + next_uri, sizeof(next_uri), "table:__wt%04d", index1)); if (session->open_cursor(session, next_uri, NULL, NULL, &c) != 0) return (1); c->set_key(c, key1_orig); @@ -376,7 +383,8 @@ live_check: if (c->close(c) != 0) return (1); - snprintf(next_uri, 128, "table:__wt%04d", index2); + testutil_check(__wt_snprintf( + next_uri, sizeof(next_uri), "table:__wt%04d", index2)); if (session->open_cursor(session, next_uri, NULL, NULL, &c) != 0) return (1); c->set_key(c, key2_orig); diff --git a/src/third_party/wiredtiger/test/checkpoint/smoke.sh b/src/third_party/wiredtiger/test/checkpoint/smoke.sh index 123d4e00df5..39b1f428c2c 100755 --- a/src/third_party/wiredtiger/test/checkpoint/smoke.sh +++ b/src/third_party/wiredtiger/test/checkpoint/smoke.sh @@ -6,8 +6,8 @@ set -e echo "checkpoint: 3 mixed tables" $TEST_WRAPPER ./t -T 3 -t m -# We are done if short tests are requested -test -z "$TESTUTIL_DISABLE_LONG_TESTS" || exit 0 +# We are done unless long tests are enabled. +test "$TESTUTIL_ENABLE_LONG_TESTS" = "1" || exit 0 echo "checkpoint: 6 column-store tables" $TEST_WRAPPER ./t -T 6 -t c diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c index 4998019ad8e..e7e1a0b81a5 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c @@ -50,10 +50,7 @@ main(int argc, char *argv[]) char *working_dir; const char *config_open; - if ((g.progname = strrchr(argv[0], DIR_DELIM)) == NULL) - g.progname = argv[0]; - else - ++g.progname; + (void)testutil_set_progname(argv); config_open = NULL; ret = 0; @@ -68,7 +65,7 @@ main(int argc, char *argv[]) runs = 1; while ((ch = __wt_getopt( - g.progname, argc, argv, "c:C:h:k:l:n:r:t:T:W:")) != EOF) + progname, argc, argv, "c:C:h:k:l:n:r:t:T:W:")) != EOF) switch (ch) { case 'c': g.checkpoint_name = __wt_optarg; @@ -132,7 +129,7 @@ main(int argc, char *argv[]) testutil_work_dir_from_path(g.home, 512, working_dir); - printf("%s: process %" PRIu64 "\n", g.progname, (uint64_t)getpid()); + printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid()); for (cnt = 1; (runs == 0 || cnt <= runs) && g.status == 0; ++cnt) { printf(" %d: %d workers, %d tables\n", cnt, g.nworkers, g.ntables); @@ -202,11 +199,11 @@ wt_connect(const char *config_open) testutil_make_work_dir(g.home); - snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "create,statistics=(fast),error_prefix=\"%s\",cache_size=1GB%s%s", - g.progname, + progname, config_open == NULL ? "" : ",", - config_open == NULL ? "" : config_open); + config_open == NULL ? "" : config_open)); if ((ret = wiredtiger_open( g.home, &event_handler, config, &g.conn)) != 0) @@ -297,10 +294,10 @@ log_print_err(const char *m, int e, int fatal) g.running = 0; g.status = e; } - fprintf(stderr, "%s: %s: %s\n", g.progname, m, wiredtiger_strerror(e)); + fprintf(stderr, "%s: %s: %s\n", progname, m, wiredtiger_strerror(e)); if (g.logfp != NULL) fprintf(g.logfp, "%s: %s: %s\n", - g.progname, m, wiredtiger_strerror(e)); + progname, m, wiredtiger_strerror(e)); return (e); } @@ -333,7 +330,7 @@ usage(void) "usage: %s " "[-S] [-C wiredtiger-config] [-k keys] [-l log]\n\t" "[-n ops] [-c checkpoint] [-r runs] [-t f|r|v] [-W workers]\n", - g.progname); + progname); fprintf(stderr, "%s", "\t-C specify wiredtiger_open configuration arguments\n" "\t-c checkpoint name to used named checkpoints\n" diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h index 0d0d02447d5..347bd2c6e89 100644 --- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h +++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.h @@ -58,7 +58,6 @@ typedef struct { u_int nkeys; /* Keys to load */ u_int nops; /* Operations per thread */ FILE *logfp; /* Message log file. */ - char *progname; /* Program name */ int nworkers; /* Number workers configured */ int ntables; /* Number tables configured */ int ntables_created; /* Number tables opened */ diff --git a/src/third_party/wiredtiger/test/checkpoint/workers.c b/src/third_party/wiredtiger/test/checkpoint/workers.c index e4fe7bd1b29..82d1b8685c4 100644 --- a/src/third_party/wiredtiger/test/checkpoint/workers.c +++ b/src/third_party/wiredtiger/test/checkpoint/workers.c @@ -39,14 +39,12 @@ static int create_table(WT_SESSION *session, COOKIE *cookie) { int ret; - char *p, *end, config[128]; + char config[128]; - p = config; - end = config + sizeof(config); - p += snprintf(p, (size_t)(end - p), - "key_format=%s,value_format=S", cookie->type == COL ? "r" : "q"); - if (cookie->type == LSM) - (void)snprintf(p, (size_t)(end - p), ",type=lsm"); + testutil_check(__wt_snprintf(config, sizeof(config), + "key_format=%s,value_format=S,%s", + cookie->type == COL ? "r" : "q", + cookie->type == LSM ? ",type=lsm" : "")); if ((ret = session->create(session, cookie->uri, config)) != 0) if (ret != EEXIST) @@ -88,8 +86,9 @@ start_workers(table_type type) (table_type)((i % MAX_TABLE_TYPE) + 1); else g.cookies[i].type = type; - (void)snprintf(g.cookies[i].uri, 128, - "%s%04d", URI_BASE, g.cookies[i].id); + testutil_check(__wt_snprintf( + g.cookies[i].uri, sizeof(g.cookies[i].uri), + "%s%04d", URI_BASE, g.cookies[i].id)); /* Should probably be atomic to avoid races. */ if ((ret = create_table(session, &g.cookies[i])) != 0) @@ -132,7 +131,8 @@ worker_op(WT_CURSOR *cursor, uint64_t keyno, u_int new_val) char valuebuf[64]; cursor->set_key(cursor, keyno); - (void)snprintf(valuebuf, sizeof(valuebuf), "%037u", new_val); + testutil_check(__wt_snprintf( + valuebuf, sizeof(valuebuf), "%037u", new_val)); cursor->set_value(cursor, valuebuf); if ((ret = cursor->insert(cursor)) != 0) { if (ret == WT_ROLLBACK) @@ -153,7 +153,7 @@ worker(void *arg) WT_UNUSED(arg); - __wt_thread_id(tid, sizeof(tid)); + testutil_check(__wt_thread_id(tid, sizeof(tid))); printf("worker thread starting: tid: %s\n", tid); (void)real_worker(); diff --git a/src/third_party/wiredtiger/test/csuite/Makefile.am b/src/third_party/wiredtiger/test/csuite/Makefile.am index a96492c1e71..f2b4fcacdc8 100644 --- a/src/third_party/wiredtiger/test/csuite/Makefile.am +++ b/src/third_party/wiredtiger/test/csuite/Makefile.am @@ -4,8 +4,13 @@ LDADD = $(top_builddir)/test/utility/libtest_util.la \ $(top_builddir)/libwiredtiger.la AM_LDFLAGS = -static +noinst_PROGRAMS= + +test_scope_SOURCES = scope/main.c +noinst_PROGRAMS += test_scope + test_wt1965_col_efficiency_SOURCES = wt1965_col_efficiency/main.c -noinst_PROGRAMS = test_wt1965_col_efficiency +noinst_PROGRAMS += test_wt1965_col_efficiency test_wt2403_lsm_workload_SOURCES = wt2403_lsm_workload/main.c noinst_PROGRAMS += test_wt2403_lsm_workload @@ -37,9 +42,24 @@ noinst_PROGRAMS += test_wt2834_join_bloom_fix test_wt2853_perf_SOURCES = wt2853_perf/main.c noinst_PROGRAMS += test_wt2853_perf +test_wt2909_checkpoint_integrity_SOURCES = wt2909_checkpoint_integrity/main.c +noinst_PROGRAMS += test_wt2909_checkpoint_integrity + test_wt2999_join_extractor_SOURCES = wt2999_join_extractor/main.c noinst_PROGRAMS += test_wt2999_join_extractor +test_wt3120_filesys_SOURCES = wt3120_filesys/main.c +noinst_PROGRAMS += test_wt3120_filesys + +test_wt3135_search_near_collator_SOURCES = wt3135_search_near_collator/main.c +noinst_PROGRAMS += test_wt3135_search_near_collator + +test_wt3184_dup_index_collator_SOURCES = wt3184_dup_index_collator/main.c +noinst_PROGRAMS += test_wt3184_dup_index_collator + +test_rwlock_SOURCES = rwlock/main.c +noinst_PROGRAMS += test_rwlock + # Run this during a "make check" smoke test. TESTS = $(noinst_PROGRAMS) LOG_COMPILER = $(TEST_WRAPPER) diff --git a/src/third_party/wiredtiger/test/csuite/rwlock/main.c b/src/third_party/wiredtiger/test/csuite/rwlock/main.c new file mode 100644 index 00000000000..04813182478 --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/rwlock/main.c @@ -0,0 +1,184 @@ +/*- + * Public Domain 2014-2017 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * JIRA ticket reference: HELP-4355 + * Test rwlock collapse under load. + */ +#define MAX_THREADS 1000 +#define READS_PER_WRITE 10000 +//#define READS_PER_WRITE 1000000 +//#define READS_PER_WRITE 100 + +#define CHECK_CORRECTNESS 1 +//#define USE_POSIX 1 + +static WT_RWLOCK rwlock; +static pthread_rwlock_t p_rwlock; +static bool running; +static uint64_t shared_counter; + +void *thread_rwlock(void *); +void *thread_dump(void *); + +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + struct timespec te, ts; + pthread_t dump_id, id[MAX_THREADS]; + int i; + + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + opts->nthreads = 100; + opts->nops = 1000000; /* per thread */ + testutil_check(testutil_parse_opts(argc, argv, opts)); + running = true; + + testutil_make_work_dir(opts->home); + testutil_check(wiredtiger_open(opts->home, NULL, + "create,session_max=1000,statistics=(fast)", &opts->conn)); + + testutil_check(__wt_rwlock_init(NULL, &rwlock)); + testutil_check(pthread_rwlock_init(&p_rwlock, NULL)); + + testutil_check(pthread_create( + &dump_id, NULL, thread_dump, (void *)opts)); + + __wt_epoch(NULL, &ts); + for (i = 0; i < (int)opts->nthreads; ++i) + testutil_check(pthread_create( + &id[i], NULL, thread_rwlock, (void *)opts)); + + while (--i >= 0) + testutil_check(pthread_join(id[i], NULL)); + __wt_epoch(NULL, &te); + printf("%.2lf\n", WT_TIMEDIFF_MS(te, ts) / 1000.0); + + running = false; + testutil_check(pthread_join(dump_id, NULL)); + + testutil_check(pthread_rwlock_destroy(&p_rwlock)); + testutil_cleanup(opts); + return (EXIT_SUCCESS); +} + +/* + * Acquire a rwlock, every Nth operation, acquire exclusive. + */ +void * +thread_rwlock(void *arg) +{ + TEST_OPTS *opts; + WT_SESSION *wt_session; + WT_SESSION_IMPL *session; + uint64_t i, counter; + bool writelock; + + opts = (TEST_OPTS *)arg; + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &wt_session)); + session = (WT_SESSION_IMPL *)wt_session; + + printf("Running rwlock thread\n"); + for (i = 1; i <= opts->nops; ++i) { + writelock = (i % READS_PER_WRITE == 0); + +#ifdef USE_POSIX + if (writelock) + testutil_check(pthread_rwlock_wrlock(&p_rwlock)); + else + testutil_check(pthread_rwlock_rdlock(&p_rwlock)); +#else + if (writelock) + __wt_writelock(session, &rwlock); + else + __wt_readlock(session, &rwlock); +#endif + + /* + * Do a tiny amount of work inside the lock so the compiler + * can't optimize everything away. + */ + (void)__wt_atomic_add64(&counter, 1); + +#ifdef CHECK_CORRECTNESS + if (writelock) + counter = ++shared_counter; + else + counter = shared_counter; + + __wt_yield(); + + testutil_assert(counter == shared_counter); +#endif + +#ifdef USE_POSIX + testutil_check(pthread_rwlock_unlock(&p_rwlock)); +#else + if (writelock) + __wt_writeunlock(session, &rwlock); + else + __wt_readunlock(session, &rwlock); +#endif + + if (i % 10000 == 0) { + printf("%s", session->id == 20 ? ".\n" : "."); + fflush(stdout); + } + } + + opts->running = false; + + return (NULL); +} + +void * +thread_dump(void *arg) { + WT_UNUSED(arg); + + while (running) { + sleep(1); + printf("\n" + "rwlock { current %" PRIu8 ", next %" PRIu8 + ", reader %" PRIu8 ", readers_active %" PRIu16 + ", readers_queued %" PRIu16 " }\n", + rwlock.u.s.current, + rwlock.u.s.next, + rwlock.u.s.reader, + rwlock.u.s.readers_active, + rwlock.u.s.readers_queued); + } + + return (NULL); +} diff --git a/src/third_party/wiredtiger/test/csuite/scope/main.c b/src/third_party/wiredtiger/test/csuite/scope/main.c new file mode 100644 index 00000000000..15dabd97c40 --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/scope/main.c @@ -0,0 +1,288 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +#define KEY "key" +#define VALUE "value" + +static int ignore_errors; + +static int +handle_error(WT_EVENT_HANDLER *handler, + WT_SESSION *session, int error, const char *message) +{ + (void)(handler); + + /* Skip the error messages we're expecting to see. */ + if (ignore_errors > 0 && + (strstr(message, "requires key be set") != NULL || + strstr(message, "requires value be set") != NULL)) { + --ignore_errors; + return (0); + } + + (void)fprintf(stderr, "%s: %s\n", + message, session->strerror(session, error)); + return (0); +} + +static WT_EVENT_HANDLER event_handler = { + handle_error, + NULL, + NULL, + NULL +}; + +static void +cursor_scope_ops(WT_SESSION *session, const char *uri) +{ + struct { + const char *op; + enum { INSERT, SEARCH, SEARCH_NEAR, + REMOVE, REMOVE_POS, RESERVE, UPDATE } func; + const char *config; + } *op, ops[] = { + /* + * The ops order is fixed and shouldn't change, that is, insert + * has to happen first so search, update and remove operations + * are possible, and remove has to be last. + */ + { "insert", INSERT, NULL, }, + { "search", SEARCH, NULL, }, + { "search", SEARCH_NEAR, NULL, }, +#if 0 + { "reserve", RESERVE, NULL, }, +#endif + { "update", UPDATE, NULL, }, + { "remove", REMOVE, NULL, }, + { "remove", REMOVE_POS, NULL, }, + { NULL, INSERT, NULL } + }; + WT_CURSOR *cursor; + uint64_t keyr; + const char *key, *value; + char keybuf[100], valuebuf[100]; + int exact; + bool recno; + + /* Reserve requires a running transaction. */ + testutil_check(session->begin_transaction(session, NULL)); + + cursor = NULL; + for (op = ops; op->op != NULL; op++) { + key = value = NULL; + + /* Open a cursor. */ + if (cursor != NULL) + testutil_check(cursor->close(cursor)); + testutil_check(session->open_cursor( + session, uri, NULL, op->config, &cursor)); + recno = strcmp(cursor->key_format, "r") == 0; + + /* + * Set up application buffers so we can detect overwrites + * or failure to copy application information into library + * memory. + */ + if (recno) + cursor->set_key(cursor, (uint64_t)1); + else { + strcpy(keybuf, KEY); + cursor->set_key(cursor, keybuf); + } + strcpy(valuebuf, VALUE); + cursor->set_value(cursor, valuebuf); + + /* + * The application must keep key and value memory valid until + * the next operation that positions the cursor, modifies the + * data, or resets or closes the cursor. + * + * Modifying either the key or value buffers is not permitted. + */ + switch (op->func) { + case INSERT: + testutil_check(cursor->insert(cursor)); + break; + case SEARCH: + testutil_check(cursor->search(cursor)); + break; + case SEARCH_NEAR: + testutil_check(cursor->search_near(cursor, &exact)); + break; + case REMOVE_POS: + /* + * Remove has two modes, one where the remove is based + * on a cursor position, the other where it's based on + * a set key. The results are different, so test them + * separately. + */ + testutil_check(cursor->search(cursor)); + /* FALLTHROUGH */ + case REMOVE: + testutil_check(cursor->remove(cursor)); + break; + case RESERVE: +#if 0 + testutil_check(cursor->reserve(cursor)); +#endif + break; + case UPDATE: + testutil_check(cursor->update(cursor)); + break; + } + + /* + * The cursor should no longer reference application memory, + * and application buffers can be safely overwritten. + */ + memset(keybuf, 'K', sizeof(keybuf)); + memset(valuebuf, 'V', sizeof(valuebuf)); + + /* + * Check that get_key/get_value behave as expected after the + * operation. + */ + switch (op->func) { + case INSERT: + case REMOVE: + /* + * Insert and remove configured with a search key do + * not position the cursor and have no key or value. + * + * There should be two error messages, ignore them. + */ + ignore_errors = 2; + if (recno) + testutil_assert( + cursor->get_key(cursor, &keyr) != 0); + else + testutil_assert( + cursor->get_key(cursor, &key) != 0); + testutil_assert(cursor->get_value(cursor, &value) != 0); + testutil_assert(ignore_errors == 0); + break; + case REMOVE_POS: + /* + * Remove configured with a cursor position has a key, + * but no value. + * + * There should be one error message, ignore it. + */ + if (recno) { + testutil_assert( + cursor->get_key(cursor, &keyr) == 0); + testutil_assert(keyr == 1); + } else { + testutil_assert( + cursor->get_key(cursor, &key) == 0); + testutil_assert(key != keybuf); + testutil_assert(strcmp(key, KEY) == 0); + } + ignore_errors = 1; + testutil_assert(cursor->get_value(cursor, &value) != 0); + testutil_assert(ignore_errors == 0); + break; + case RESERVE: + case SEARCH: + case SEARCH_NEAR: + case UPDATE: + /* + * Reserve, search, search-near and update position the + * cursor and have both a key and value. + * + * Any key/value should not reference application + * memory. + */ + if (recno) { + testutil_assert( + cursor->get_key(cursor, &keyr) == 0); + testutil_assert(keyr == 1); + } else { + testutil_assert( + cursor->get_key(cursor, &key) == 0); + testutil_assert(key != keybuf); + testutil_assert(strcmp(key, KEY) == 0); + } + testutil_assert(cursor->get_value(cursor, &value) == 0); + testutil_assert(value != valuebuf); + testutil_assert(strcmp(value, VALUE) == 0); + break; + } + + /* + * We have more than one remove operation, add the key back + * in. + */ + if (op->func == REMOVE || op->func == REMOVE_POS) { + if (recno) + cursor->set_key(cursor, (uint64_t)1); + else { + cursor->set_key(cursor, KEY); + } + cursor->set_value(cursor, VALUE); + testutil_check(cursor->insert(cursor)); + } + } +} + +static void +run(WT_CONNECTION *conn, const char *uri, const char *config) +{ + WT_SESSION *session; + + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + testutil_check(session->create(session, uri, config)); + cursor_scope_ops(session, uri); + testutil_check(session->close(session, NULL)); +} + +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check( + wiredtiger_open(opts->home, &event_handler, "create", &opts->conn)); + + run(opts->conn, "file:file.SS", "key_format=S,value_format=S"); + run(opts->conn, "file:file.rS", "key_format=r,value_format=S"); + run(opts->conn, "lsm:lsm.SS", "key_format=S,value_format=S"); + run(opts->conn, "lsm:lsm.rS", "key_format=r,value_format=S"); + run(opts->conn, "table:table.SS", "key_format=S,value_format=S"); + run(opts->conn, "table:table.rS", "key_format=r,value_format=S"); + + testutil_cleanup(opts); + + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c b/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c index a7235d81b31..e5b73d5e642 100644 --- a/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt1965_col_efficiency/main.c @@ -132,7 +132,8 @@ main(int argc, char *argv[]) testutil_check(opts->conn->open_session( opts->conn, NULL, NULL, &session)); - sprintf(table_format, "key_format=r,value_format="); + testutil_check(__wt_snprintf( + table_format, sizeof(table_format), "key_format=r,value_format=")); for (i = 0; i < NR_FIELDS; i++) strcat(table_format, "Q"); diff --git a/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c b/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c index 4b352b26051..9876582fffa 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2246_col_append/main.c @@ -68,8 +68,8 @@ page_init(uint64_t n) else { if (recno % 3 == 0) ++vrecno; - snprintf(buf, - sizeof(buf), "%" PRIu64 " VALUE ------", vrecno); + testutil_check(__wt_snprintf(buf, + sizeof(buf), "%" PRIu64 " VALUE ------", vrecno)); cursor->set_value(cursor, buf); } testutil_check(cursor->insert(cursor)); @@ -101,9 +101,10 @@ main(int argc, char *argv[]) uint64_t i, id; char buf[100]; + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + opts = &_opts; - if (testutil_disable_long_tests()) - return (0); memset(opts, 0, sizeof(*opts)); opts->table_type = TABLE_ROW; opts->n_append_threads = N_APPEND_THREADS; @@ -111,19 +112,19 @@ main(int argc, char *argv[]) testutil_check(testutil_parse_opts(argc, argv, opts)); testutil_make_work_dir(opts->home); - snprintf(buf, sizeof(buf), + testutil_check(__wt_snprintf(buf, sizeof(buf), "create," "cache_size=%s," "eviction=(threads_max=5)," "statistics=(fast)", - opts->table_type == TABLE_FIX ? "500MB" : "2GB"); + opts->table_type == TABLE_FIX ? "500MB" : "2GB")); testutil_check(wiredtiger_open(opts->home, NULL, buf, &opts->conn)); testutil_check( opts->conn->open_session(opts->conn, NULL, NULL, &session)); - snprintf(buf, sizeof(buf), + testutil_check(__wt_snprintf(buf, sizeof(buf), "key_format=r,value_format=%s," "allocation_size=4K,leaf_page_max=64K", - opts->table_type == TABLE_FIX ? "8t" : "S"); + opts->table_type == TABLE_FIX ? "8t" : "S")); testutil_check(session->create(session, opts->uri, buf)); testutil_check(session->close(session, NULL)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c index 239a3f300d0..617490fec4d 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2323_join_visibility/main.c @@ -92,10 +92,11 @@ main(int argc, char *argv[]) TEST_OPTS *opts, _opts; const char *tablename; + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + opts = &_opts; sharedopts = &_sharedopts; - if (testutil_disable_long_tests()) - return (0); memset(opts, 0, sizeof(*opts)); memset(sharedopts, 0, sizeof(*sharedopts)); @@ -105,14 +106,18 @@ main(int argc, char *argv[]) tablename = strchr(opts->uri, ':'); testutil_assert(tablename != NULL); tablename++; - snprintf(sharedopts->posturi, sizeof(sharedopts->posturi), - "index:%s:post", tablename); - snprintf(sharedopts->baluri, sizeof(sharedopts->baluri), - "index:%s:bal", tablename); - snprintf(sharedopts->flaguri, sizeof(sharedopts->flaguri), - "index:%s:flag", tablename); - snprintf(sharedopts->joinuri, sizeof(sharedopts->joinuri), - "join:%s", opts->uri); + testutil_check(__wt_snprintf( + sharedopts->posturi, sizeof(sharedopts->posturi), + "index:%s:post", tablename)); + testutil_check(__wt_snprintf( + sharedopts->baluri, sizeof(sharedopts->baluri), + "index:%s:bal", tablename)); + testutil_check(__wt_snprintf( + sharedopts->flaguri, sizeof(sharedopts->flaguri), + "index:%s:flag", tablename)); + testutil_check(__wt_snprintf( + sharedopts->joinuri, sizeof(sharedopts->joinuri), + "join:%s", opts->uri)); testutil_check(wiredtiger_open(opts->home, NULL, "create,cache_size=1G", &opts->conn)); @@ -349,19 +354,21 @@ static void *thread_join(void *arg) balcur->set_key(balcur, 0); testutil_check(balcur->search(balcur)); if (sharedopts->bloom) - sprintf(cfg, "compare=lt,strategy=bloom,count=%d", - N_RECORDS); + testutil_check(__wt_snprintf(cfg, sizeof(cfg), + "compare=lt,strategy=bloom,count=%d", N_RECORDS)); else - sprintf(cfg, "compare=lt"); + testutil_check(__wt_snprintf( + cfg, sizeof(cfg), "compare=lt")); testutil_check(session->join(session, joincur, balcur, cfg)); flagcur->set_key(flagcur, 0); testutil_check(flagcur->search(flagcur)); if (sharedopts->bloom) - sprintf(cfg, "compare=eq,strategy=bloom,count=%d", - N_RECORDS); + testutil_check(__wt_snprintf(cfg, sizeof(cfg), + "compare=eq,strategy=bloom,count=%d", N_RECORDS)); else - sprintf(cfg, "compare=eq"); + testutil_check(__wt_snprintf( + cfg, sizeof(cfg), "compare=eq")); testutil_check(session->join(session, joincur, flagcur, cfg)); /* Expect no values returned */ diff --git a/src/third_party/wiredtiger/test/csuite/wt2447_join_main_table/main.c b/src/third_party/wiredtiger/test/csuite/wt2447_join_main_table/main.c index 1368e7c8c09..656cea04145 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2447_join_main_table/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2447_join_main_table/main.c @@ -102,9 +102,12 @@ main(int argc, char *argv[]) tablename = strchr(opts->uri, ':'); testutil_assert(tablename != NULL); tablename++; - snprintf(index1uri, sizeof(index1uri), "index:%s:index1", tablename); - snprintf(index2uri, sizeof(index2uri), "index:%s:index2", tablename); - snprintf(joinuri, sizeof(joinuri), "join:%s", opts->uri); + testutil_check(__wt_snprintf( + index1uri, sizeof(index1uri), "index:%s:index1", tablename)); + testutil_check(__wt_snprintf( + index2uri, sizeof(index2uri), "index:%s:index2", tablename)); + testutil_check(__wt_snprintf( + joinuri, sizeof(joinuri), "join:%s", opts->uri)); testutil_check(wiredtiger_open(opts->home, NULL, "statistics=(all),create", &opts->conn)); @@ -150,7 +153,8 @@ main(int argc, char *argv[]) cursor2->set_key(cursor2, half + 1); testutil_check(cursor2->search(cursor2)); - sprintf(bloom_cfg, "compare=lt,strategy=bloom,count=%d", half); + testutil_check(__wt_snprintf(bloom_cfg, sizeof(bloom_cfg), + "compare=lt,strategy=bloom,count=%d", half)); testutil_check(session->open_cursor(session, joinuri, NULL, NULL, &jcursor)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c b/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c index ae18760a829..ba17d485e07 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2535_insert_race/main.c @@ -49,9 +49,10 @@ main(int argc, char *argv[]) uint64_t current_value; int i; + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + opts = &_opts; - if (testutil_disable_long_tests()) - return (0); memset(opts, 0, sizeof(*opts)); opts->nthreads = 10; opts->nrecords = 1000; diff --git a/src/third_party/wiredtiger/test/csuite/wt2592_join_schema/main.c b/src/third_party/wiredtiger/test/csuite/wt2592_join_schema/main.c index 0ec1c765d99..be3eff6136c 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2592_join_schema/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2592_join_schema/main.c @@ -82,9 +82,12 @@ main(int argc, char *argv[]) tablename = strchr(opts->uri, ':'); testutil_assert(tablename != NULL); tablename++; - snprintf(countryuri, sizeof(countryuri), "index:%s:country", tablename); - snprintf(yearuri, sizeof(yearuri), "index:%s:year", tablename); - snprintf(joinuri, sizeof(joinuri), "join:%s", opts->uri); + testutil_check(__wt_snprintf( + countryuri, sizeof(countryuri), "index:%s:country", tablename)); + testutil_check(__wt_snprintf( + yearuri, sizeof(yearuri), "index:%s:year", tablename)); + testutil_check(__wt_snprintf( + joinuri, sizeof(joinuri), "join:%s", opts->uri)); testutil_check(wiredtiger_open(opts->home, NULL, "create,cache_size=200M", &opts->conn)); diff --git a/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c b/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c index 7c80496f1b6..e128df29f41 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2834_join_bloom_fix/main.c @@ -59,11 +59,11 @@ main(int argc, char *argv[]) char flaguri[256]; char joinuri[256]; + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + opts = &_opts; - if (testutil_disable_long_tests()) - return (0); memset(opts, 0, sizeof(*opts)); - testutil_check(testutil_parse_opts(argc, argv, opts)); testutil_make_work_dir(opts->home); @@ -83,10 +83,14 @@ main(int argc, char *argv[]) tablename = strchr(opts->uri, ':'); testutil_assert(tablename != NULL); tablename++; - snprintf(posturi, sizeof(posturi), "index:%s:post", tablename); - snprintf(balanceuri, sizeof(balanceuri), "index:%s:balance", tablename); - snprintf(flaguri, sizeof(flaguri), "index:%s:flag", tablename); - snprintf(joinuri, sizeof(joinuri), "join:%s", opts->uri); + testutil_check(__wt_snprintf( + posturi, sizeof(posturi), "index:%s:post", tablename)); + testutil_check(__wt_snprintf( + balanceuri, sizeof(balanceuri), "index:%s:balance", tablename)); + testutil_check(__wt_snprintf( + flaguri, sizeof(flaguri), "index:%s:flag", tablename)); + testutil_check(__wt_snprintf( + joinuri, sizeof(joinuri), "join:%s", opts->uri)); testutil_check(session->create(session, posturi, "columns=(post)")); testutil_check(session->create(session, balanceuri, @@ -126,14 +130,14 @@ main(int argc, char *argv[]) balancecur->set_key(balancecur, 0); testutil_check(balancecur->search(balancecur)); - sprintf(cfg, "compare=lt,strategy=bloom,count=%d", - N_RECORDS / 100); + testutil_check(__wt_snprintf(cfg, sizeof(cfg), + "compare=lt,strategy=bloom,count=%d", N_RECORDS / 100)); testutil_check(session->join(session, joincur, balancecur, cfg)); flagcur->set_key(flagcur, 0); testutil_check(flagcur->search(flagcur)); - sprintf(cfg, "compare=eq,strategy=bloom,count=%d", - N_RECORDS / 100); + testutil_check(__wt_snprintf(cfg, sizeof(cfg), + "compare=eq,strategy=bloom,count=%d", N_RECORDS / 100)); testutil_check(session->join(session, joincur, flagcur, cfg)); /* Expect no values returned */ diff --git a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c index 6cec9634cd1..46ba71372e5 100644 --- a/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c +++ b/src/third_party/wiredtiger/test/csuite/wt2853_perf/main.c @@ -82,11 +82,11 @@ main(int argc, char *argv[]) int i, nfail; const char *tablename; + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + opts = &_opts; sharedopts = &_sharedopts; - - if (testutil_disable_long_tests()) - return (0); memset(opts, 0, sizeof(*opts)); memset(sharedopts, 0, sizeof(*sharedopts)); memset(insert_args, 0, sizeof(insert_args)); @@ -114,12 +114,15 @@ main(int argc, char *argv[]) tablename = strchr(opts->uri, ':'); testutil_assert(tablename != NULL); tablename++; - snprintf(sharedopts->posturi, sizeof(sharedopts->posturi), - "index:%s:post", tablename); - snprintf(sharedopts->baluri, sizeof(sharedopts->baluri), - "index:%s:bal", tablename); - snprintf(sharedopts->flaguri, sizeof(sharedopts->flaguri), - "index:%s:flag", tablename); + testutil_check(__wt_snprintf( + sharedopts->posturi, sizeof(sharedopts->posturi), + "index:%s:post", tablename)); + testutil_check(__wt_snprintf( + sharedopts->baluri, sizeof(sharedopts->baluri), + "index:%s:bal", tablename)); + testutil_check(__wt_snprintf( + sharedopts->flaguri, sizeof(sharedopts->flaguri), + "index:%s:flag", tablename)); testutil_check(session->create(session, sharedopts->posturi, "columns=(post)")); diff --git a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c new file mode 100644 index 00000000000..ce7bd72fa3f --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c @@ -0,0 +1,669 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +#include +#include +#include + +/* + * JIRA ticket reference: WT-2909 + * Test case description: + * + * This test attempts to check the integrity of checkpoints by injecting + * failures (by means of a custom file system) and then trying to recover. To + * insulate the top level program from various crashes that may occur when + * injecting failures, the "populate" code runs in another process, and is + * expected to sometimes fail. Then the top level program runs recovery (with + * the normal file system) and checks the results. Any failure at the top level + * indicates a checkpoint integrity problem. + * + * Each subtest uses the same kind of schema and data, the only variance is + * when the faults are injected. At the moment, this test only injects during + * checkpoints, and only injects write failures. It varies in the number of + * successful writes that occur before an injected failure (during a checkpoint + * operation), this can be indicated with "-o N". When N is not specified, the + * test attempts to find the optimal range of N for testing. Clearly when N is + * large, then the checkpoint may be successfully written, and the data + * represented by the checkpoint will be fully present. When N is small, + * nothing of interest is written and no data is present. To find the sweet + * spot where interesting failures occur, the test does a binary search to find + * the approximate N that divides the "small" and "large" cases. This is not + * strictly deterministic, a given N may give different results on different + * runs. But approximate optimal N can be determined, allowing a series of + * additional tests clustered around this N. + * + * The data is stored in two tables, one having indices. Both tables have + * the same keys and are updated with the same key in a single transaction. + * + * Failure mode: + * If one table is out of step with the other, that is detected as a failure at + * the top level. If an index is missing values (or has extra values), that is + * likewise a failure at the top level. If the tables or the home directory + * cannot be opened, that is a top level error. The tables must be present + * as an initial checkpoint is done without any injected fault. + */ + +/* + * This program does not run on Windows. The non-portable aspects at minimum + * are fork/exec the use of environment variables (used by fail_fs), and file + * name and build locations of dynamically loaded libraries. + */ +#define BIG_SIZE (1024 * 10) +#define BIG_CONTENTS "" +#define MAX_ARGS 20 +#define MAX_OP_RANGE 1000 +#define STDERR_FILE "stderr.txt" +#define STDOUT_FILE "stdout.txt" +#define TESTS_PER_OP_VALUE 3 +#define VERBOSE_PRINT 10000 + +static int check_results(TEST_OPTS *, uint64_t *); +static void check_values(WT_CURSOR *, int, int, int, char *); +static int create_big_string(char **); +static void cursor_count_items(WT_CURSOR *, uint64_t *); +static void disable_failures(void); +static void enable_failures(uint64_t, uint64_t); +static void generate_key(uint64_t, int *); +static void generate_value(uint32_t, uint64_t, char *, int *, int *, int *, + char **); +static void run_check_subtest(TEST_OPTS *, const char *, uint64_t, bool, + uint64_t *); +static void run_check_subtest_range(TEST_OPTS *, const char *, bool); +static int run_process(TEST_OPTS *, const char *, char *[], int *); +static void subtest_main(int, char *[], bool); +static void subtest_populate(TEST_OPTS *, bool); + +extern int __wt_optind; + +#define WT_FAIL_FS_LIB "../../ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so" + +/* + * check_results -- + * Check all the tables and verify the results. + */ +static int +check_results(TEST_OPTS *opts, uint64_t *foundp) +{ + WT_CURSOR *maincur, *maincur2, *v0cur, *v1cur, *v2cur; + WT_SESSION *session; + uint64_t count, idxcount, nrecords; + uint32_t rndint; + int key, key_got, ret, v0, v1, v2; + char *bigref, *big; + + testutil_check(create_big_string(&bigref)); + nrecords = opts->nrecords; + testutil_check(wiredtiger_open(opts->home, NULL, + "create,log=(enabled)", &opts->conn)); + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check(session->open_cursor(session, "table:subtest", NULL, + NULL, &maincur)); + testutil_check(session->open_cursor(session, "table:subtest2", NULL, + NULL, &maincur2)); + testutil_check(session->open_cursor(session, "index:subtest:v0", NULL, + NULL, &v0cur)); + testutil_check(session->open_cursor(session, "index:subtest:v1", NULL, + NULL, &v1cur)); + testutil_check(session->open_cursor(session, "index:subtest:v2", NULL, + NULL, &v2cur)); + + count = 0; + while ((ret = maincur->next(maincur)) == 0) { + testutil_check(maincur2->next(maincur2)); + testutil_check(maincur2->get_key(maincur2, &key_got)); + testutil_check(maincur2->get_value(maincur2, &rndint)); + + generate_key(count, &key); + generate_value(rndint, count, bigref, &v0, &v1, &v2, &big); + testutil_assert(key == key_got); + + /* Check the key/values in main table. */ + testutil_check(maincur->get_key(maincur, &key_got)); + testutil_assert(key == key_got); + check_values(maincur, v0, v1, v2, big); + + /* Check the values in the indices. */ + v0cur->set_key(v0cur, v0); + testutil_check(v0cur->search(v0cur)); + check_values(v0cur, v0, v1, v2, big); + v1cur->set_key(v1cur, v1); + testutil_check(v1cur->search(v1cur)); + check_values(v1cur, v0, v1, v2, big); + v2cur->set_key(v2cur, v2); + testutil_check(v2cur->search(v2cur)); + check_values(v2cur, v0, v1, v2, big); + + count++; + if (count % VERBOSE_PRINT == 0 && opts->verbose) + printf("checked %" PRIu64 "/%" PRIu64 "\n", count, + nrecords); + } + if (count % VERBOSE_PRINT != 0 && opts->verbose) + printf("checked %" PRIu64 "/%" PRIu64 "\n", count, nrecords); + + /* + * Always expect at least one entry, as populate does a + * checkpoint after the first insert. + */ + testutil_assert(count > 0); + testutil_assert(ret == WT_NOTFOUND); + testutil_assert(maincur2->next(maincur2) == WT_NOTFOUND); + cursor_count_items(v0cur, &idxcount); + testutil_assert(count == idxcount); + cursor_count_items(v1cur, &idxcount); + testutil_assert(count == idxcount); + cursor_count_items(v2cur, &idxcount); + testutil_assert(count == idxcount); + + testutil_check(opts->conn->close(opts->conn, NULL)); + opts->conn = NULL; + + free(bigref); + *foundp = count; + return (0); +} + +/* + * check_values -- + * Check that the values in the cursor match the given values. + */ +static void +check_values(WT_CURSOR *cursor, int v0, int v1, int v2, char *big) +{ + int v0_got, v1_got, v2_got; + char *big_got; + + testutil_check(cursor->get_value(cursor, &v0_got, &v1_got, &v2_got, + &big_got)); + testutil_assert(v0 == v0_got); + testutil_assert(v1 == v1_got); + testutil_assert(v2 == v2_got); + testutil_assert(strcmp(big, big_got) == 0); +} + +/* + * create_big_string -- + * Create and fill the "reference" big array. + */ +static int create_big_string(char **bigp) +{ + size_t i, mod; + char *big; + + if ((big = malloc(BIG_SIZE + 1)) == NULL) + return (ENOMEM); + mod = strlen(BIG_CONTENTS); + for (i = 0; i < BIG_SIZE; i++) { + big[i] = BIG_CONTENTS[i % mod]; + } + big[BIG_SIZE] = '\0'; + *bigp = big; + return (0); +} + +/* + * cursor_count_items -- + * Count the number of items in the table by traversing + * through the cursor. + */ +static void +cursor_count_items(WT_CURSOR *cursor, uint64_t *countp) +{ + int ret; + + *countp = 0; + + testutil_check(cursor->reset(cursor)); + while ((ret = cursor->next(cursor)) == 0) + (*countp)++; + testutil_assert(ret == WT_NOTFOUND); +} + +/* + * disable_failures -- + * Disable failures in the fail file system. + */ +static void +disable_failures(void) +{ + testutil_check(setenv("WT_FAIL_FS_ENABLE", "0", 1)); +} + +/* + * enable_failures -- + * Enable failures in the fail file system. + */ +static void +enable_failures(uint64_t allow_writes, uint64_t allow_reads) +{ + char value[100]; + + testutil_check(setenv("WT_FAIL_FS_ENABLE", "1", 1)); + testutil_check(__wt_snprintf( + value, sizeof(value), "%" PRIu64, allow_writes)); + testutil_check(setenv("WT_FAIL_FS_WRITE_ALLOW", value, 1)); + testutil_check(__wt_snprintf( + value, sizeof(value), "%" PRIu64, allow_reads)); + testutil_check(setenv("WT_FAIL_FS_READ_ALLOW", value, 1)); +} + +/* + * generate_key -- + * Generate a key used by the "subtest" and "subtest2" tables. + */ +static void +generate_key(uint64_t i, int *keyp) +{ + *keyp = (int)i; +} + +/* + * generate_value -- + * Generate values for the "subtest" table. + */ +static void +generate_value(uint32_t rndint, uint64_t i, char *bigref, + int *v0p, int *v1p, int *v2p, char **bigp) +{ + *v0p = (int)(i * 7); + *v1p = (int)(i * 10007); + *v2p = (int)(i * 100000007); + *bigp = &bigref[rndint % BIG_SIZE]; +} + +/* + * run_check_subtest -- + * Run the subtest with the given parameters and check the results. + */ +static void +run_check_subtest(TEST_OPTS *opts, const char *debugger, uint64_t nops, + bool close_test, uint64_t *nresultsp) +{ + int estatus, narg; + char rarg[20], sarg[20], *subtest_args[MAX_ARGS]; + + narg = 0; + if (debugger != NULL) { + subtest_args[narg++] = (char *)debugger; + subtest_args[narg++] = (char *)"--"; + } + + subtest_args[narg++] = (char *)opts->progname; + /* "subtest" must appear before arguments */ + if (close_test) + subtest_args[narg++] = (char *)"subtest_close"; + else + subtest_args[narg++] = (char *)"subtest"; + subtest_args[narg++] = (char *)"-h"; + subtest_args[narg++] = opts->home; + subtest_args[narg++] = (char *)"-v"; /* subtest is always verbose */ + subtest_args[narg++] = (char *)"-p"; + subtest_args[narg++] = (char *)"-o"; + testutil_check(__wt_snprintf(sarg, sizeof(sarg), "%" PRIu64, nops)); + subtest_args[narg++] = sarg; /* number of operations */ + subtest_args[narg++] = (char *)"-n"; + testutil_check(__wt_snprintf( + rarg, sizeof(rarg), "%" PRIu64, opts->nrecords)); + subtest_args[narg++] = rarg; /* number of records */ + subtest_args[narg++] = NULL; + testutil_assert(narg <= MAX_ARGS); + if (opts->verbose) + printf("running a separate process with %" PRIu64 + " operations until fail...\n", nops); + testutil_clean_work_dir(opts->home); + testutil_check(run_process( + opts, debugger != NULL ? debugger : opts->progname, + subtest_args, &estatus)); + if (opts->verbose) + printf("process exited %d\n", estatus); + + /* + * Verify results in parent process. + */ + testutil_check(check_results(opts, nresultsp)); +} + +/* + * run_check_subtest_range -- + * + * Run successive tests via binary search that determines the approximate + * crossover point between when data is recoverable or not. Once that is + * determined, run the subtest in a range near that crossover point. + * + * The theory is that running at the crossover point will tend to trigger + * "interesting" failures at the borderline when the checkpoint is about to, + * or has, succeeded. If any of those failures creates a WT home directory + * that cannot be recovered, the top level test will fail. + */ +static void +run_check_subtest_range(TEST_OPTS *opts, const char *debugger, bool close_test) +{ + uint64_t cutoff, high, low, mid, nops, nresults; + int i; + bool got_failure, got_success; + + if (opts->verbose) + printf("Determining best range of operations until failure, " + "with close_test %s.\n", + (close_test ? "enabled" : "disabled")); + + run_check_subtest(opts, debugger, 1, close_test, &cutoff); + low = 0; + high = MAX_OP_RANGE; + mid = (low + high) / 2; + while (mid != low) { + run_check_subtest(opts, debugger, mid, close_test, + &nresults); + if (nresults > cutoff) + high = mid; + else + low = mid; + mid = (low + high) / 2; + } + /* + * mid is the number of ops that is the crossover point. + * Run some tests near that point to try to trigger weird + * failures. If mid is too low or too high, it indicates + * there is a fundamental problem with the test. + */ + testutil_assert(mid > 1 && mid < MAX_OP_RANGE - 1); + if (opts->verbose) + printf("Retesting around %" PRIu64 " operations.\n", + mid); + + got_failure = false; + got_success = false; + for (nops = mid - 10; nops < mid + 10; nops++) { + for (i = 0; i < TESTS_PER_OP_VALUE; i++) { + run_check_subtest(opts, debugger, nops, + close_test, &nresults); + if (nresults > cutoff) + got_failure = true; + else + got_success = true; + } + } + /* + * Check that it really ran with a crossover point. + */ + testutil_assert(got_failure); + testutil_assert(got_success); +} + +/* + * run_process -- + * Run a program with arguments, wait until it completes. + */ +static int +run_process(TEST_OPTS *opts, const char *prog, char *argv[], int *status) +{ + int pid; + char **arg; + + if (opts->verbose) { + printf("running: "); + for (arg = argv; *arg != NULL; arg++) + printf("%s ", *arg); + printf("\n"); + } + if ((pid = fork()) == 0) { + (void)execv(prog, argv); + testutil_die(errno, "%s", prog); + } else if (pid < 0) + return (errno); + + (void)waitpid(pid, status, 0); + return (0); +} + +/* + * subtest_main -- + * The main program for the subtest + */ +static void +subtest_main(int argc, char *argv[], bool close_test) +{ + TEST_OPTS *opts, _opts; + WT_SESSION *session; + char config[1024], filename[1024]; + struct rlimit rlim; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + memset(&rlim, 0, sizeof(rlim)); + + /* No core files during fault injection tests. */ + testutil_check(setrlimit(RLIMIT_CORE, &rlim)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + /* Redirect stderr, stdout. */ + testutil_check(__wt_snprintf( + filename, sizeof(filename), "%s/%s", opts->home, STDERR_FILE)); + testutil_assert(freopen(filename, "a", stderr) != NULL); + testutil_check(__wt_snprintf( + filename, sizeof(filename), "%s/%s", opts->home, STDOUT_FILE)); + testutil_assert(freopen(filename, "a", stdout) != NULL); + testutil_check(__wt_snprintf(config, sizeof(config), + "create,cache_size=250M,log=(enabled)," + "transaction_sync=(enabled,method=none),extensions=(" + WT_FAIL_FS_LIB + "=(early_load,config={environment=true,verbose=true})]")); + + testutil_check(wiredtiger_open(opts->home, NULL, config, &opts->conn)); + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check(session->create(session, "table:subtest", + "key_format=i,value_format=iiiS," + "columns=(id,v0,v1,v2,big)")); + + testutil_check(session->create(session, "table:subtest2", + "key_format=i,value_format=i")); + + testutil_check(session->create(session, "index:subtest:v0", + "columns=(v0)")); + testutil_check(session->create(session, "index:subtest:v1", + "columns=(v1)")); + testutil_check(session->create(session, "index:subtest:v2", + "columns=(v2)")); + + testutil_check(session->close(session, NULL)); + + subtest_populate(opts, close_test); + + testutil_cleanup(opts); +} + +/* + * This macro is used as a substitute for testutil_check, except that it is + * aware of when a failure may be expected due to the effects of the fail_fs. + * This macro is used only in subtest_populate(), it uses local variables. + */ +#define CHECK(expr) { \ + int _ret; \ + _ret = expr; \ + if (_ret != 0) { \ + if (!failmode || \ + (_ret != WT_RUN_RECOVERY && _ret != EIO)) { \ + fprintf(stderr, " BAD RETURN %d for \"%s\"\n", \ + _ret, #expr); \ + testutil_check(_ret); \ + } else \ + failed = true; \ + } \ +} + +/* + * subtest_populate -- + * Populate the tables. + */ +static void +subtest_populate(TEST_OPTS *opts, bool close_test) +{ + WT_CURSOR *maincur, *maincur2; + WT_RAND_STATE rnd; + WT_SESSION *session; + uint64_t i, nrecords; + uint32_t rndint; + int key, v0, v1, v2; + char *big, *bigref; + bool failed, failmode; + + failmode = failed = false; + __wt_random_init_seed(NULL, &rnd); + CHECK(create_big_string(&bigref)); + nrecords = opts->nrecords; + + CHECK(opts->conn->open_session( + opts->conn, NULL, NULL, &session)); + + CHECK(session->open_cursor(session, "table:subtest", NULL, + NULL, &maincur)); + + CHECK(session->open_cursor(session, "table:subtest2", NULL, + NULL, &maincur2)); + + for (i = 0; i < nrecords && !failed; i++) { + rndint = __wt_random(&rnd); + generate_key(i, &key); + generate_value(rndint, i, bigref, &v0, &v1, &v2, &big); + CHECK(session->begin_transaction(session, NULL)); + maincur->set_key(maincur, key); + maincur->set_value(maincur, v0, v1, v2, big); + CHECK(maincur->insert(maincur)); + + maincur2->set_key(maincur2, key); + maincur2->set_value(maincur2, rndint); + CHECK(maincur2->insert(maincur2)); + CHECK(session->commit_transaction(session, NULL)); + + if (i == 0) + /* + * Force an initial checkpoint, that helps to + * distinguish a clear failure from just not running + * long enough. + */ + CHECK(session->checkpoint(session, NULL)); + + if ((i + 1) % VERBOSE_PRINT == 0 && opts->verbose) + printf(" %" PRIu64 "/%" PRIu64 "\n", + (i + 1), nrecords); + /* Attempt to isolate the failures to checkpointing. */ + if (i == (nrecords/100)) { + enable_failures(opts->nops, 1000000); + failmode = true; /* CHECK should expect failures. */ + CHECK(session->checkpoint(session, NULL)); + failmode = false; + disable_failures(); + if (failed && opts->verbose) + printf("checkpoint failed (expected).\n"); + } + } + + /* + * Closing handles after an extreme fail is likely to cause + * cascading failures (or crashes), so recommended practice is + * to immediately exit. We're interested in testing both with + * and without the recommended practice. + */ + if (failed) { + if (!close_test) { + fprintf(stderr, "exit early.\n"); + exit(0); + } else + fprintf(stderr, "closing after failure.\n"); + } + + free(bigref); + CHECK(maincur->close(maincur)); + CHECK(maincur2->close(maincur2)); + CHECK(session->close(session, NULL)); +} + +/* + * main -- + * The main program for the test. When invoked with "subtest" + * argument, run the subtest. Otherwise, run a separate process + * for each needed subtest, and check the results. + */ +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + uint64_t nresults; + const char *debugger; + + if (!testutil_enable_long_tests()) /* Ignore unless requested */ + return (EXIT_SUCCESS); + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + debugger = NULL; + + testutil_check(testutil_parse_opts(argc, argv, opts)); + argc -= __wt_optind; + argv += __wt_optind; + if (opts->nrecords == 0) + opts->nrecords = 50000; + + while (argc > 0) { + if (strcmp(argv[0], "subtest") == 0) { + subtest_main(argc, argv, false); + return (0); + } else if (strcmp(argv[0], "subtest_close") == 0) { + subtest_main(argc, argv, true); + return (0); + } else if (strcmp(argv[0], "gdb") == 0) + debugger = "/usr/bin/gdb"; + else + testutil_assert(false); + argc--; + argv++; + } + if (opts->verbose) { + printf("Number of operations until failure: %" PRIu64 + " (change with -o N)\n", opts->nops); + printf("Number of records: %" PRIu64 + " (change with -n N)\n", opts->nrecords); + } + if (opts->nops == 0) { + run_check_subtest_range(opts, debugger, false); + run_check_subtest_range(opts, debugger, true); + } else + run_check_subtest(opts, debugger, opts->nops, + opts->nrecords, &nresults); + + testutil_clean_work_dir(opts->home); + testutil_cleanup(opts); + + return (0); +} diff --git a/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c b/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c new file mode 100644 index 00000000000..2fae85017d4 --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/wt3120_filesys/main.c @@ -0,0 +1,99 @@ +/*- + * Public Domain 2014-2017 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * JIRA ticket reference: WT-3120 + * Test case description: A simple file system extension built into + * a shared library. + * Failure mode: Loading the file system and closing the connection + * is enough to evoke the failure. This test does slightly more + * than that. + */ + +#define WT_FAIL_FS_LIB "../../ext/test/fail_fs/.libs/libwiredtiger_fail_fs.so" + +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + WT_CURSOR *cursor; + WT_SESSION *session; + char *kstr, *vstr; + char buf[1024]; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(__wt_snprintf(buf, sizeof(buf), + "create,extensions=(" WT_FAIL_FS_LIB "=(early_load=true))")); + testutil_check(wiredtiger_open(opts->home, NULL, buf, &opts->conn)); + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(session->create(session, opts->uri, + "key_format=S,value_format=S")); + + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, + &cursor)); + cursor->set_key(cursor, "a"); + cursor->set_value(cursor, "0"); + testutil_check(cursor->insert(cursor)); + cursor->set_key(cursor, "b"); + cursor->set_value(cursor, "1"); + testutil_check(cursor->insert(cursor)); + testutil_check(cursor->close(cursor)); + testutil_check(session->close(session, NULL)); + + /* Force to disk and re-open. */ + testutil_check(opts->conn->close(opts->conn, NULL)); + testutil_check(wiredtiger_open(opts->home, NULL, NULL, &opts->conn)); + + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + testutil_check(session->open_cursor(session, opts->uri, NULL, NULL, + &cursor)); + testutil_check(cursor->next(cursor)); + testutil_check(cursor->get_key(cursor, &kstr)); + testutil_check(cursor->get_value(cursor, &vstr)); + testutil_assert(strcmp(kstr, "a") == 0); + testutil_assert(strcmp(vstr, "0") == 0); + testutil_check(cursor->next(cursor)); + testutil_check(cursor->get_key(cursor, &kstr)); + testutil_check(cursor->get_value(cursor, &vstr)); + testutil_assert(strcmp(kstr, "b") == 0); + testutil_assert(strcmp(vstr, "1") == 0); + testutil_assert(cursor->next(cursor) == WT_NOTFOUND); + testutil_check(cursor->close(cursor)); + testutil_check(session->close(session, NULL)); + printf("Success\n"); + + testutil_cleanup(opts); + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/test/csuite/wt3135_search_near_collator/main.c b/src/third_party/wiredtiger/test/csuite/wt3135_search_near_collator/main.c new file mode 100644 index 00000000000..8783034a7d8 --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/wt3135_search_near_collator/main.c @@ -0,0 +1,360 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * JIRA ticket reference: WT-3135 + * Test case description: Each set of data is ordered and contains + * five elements (0-4). We insert elements 1 and 3, and then do + * search_near and search for each element. For each set of data, we perform + * these tests first using a custom collator, and second using a custom collator + * and extractor. In each case there are index keys having variable length. + * Failure mode: In the reported test case, the custom compare routine is + * given a truncated key to compare, and the unpack functions return errors + * because the truncation appeared in the middle of a key. + */ + +#define TEST_ENTRY_COUNT 5 +typedef const char *TEST_SET[TEST_ENTRY_COUNT]; +static TEST_SET test_sets[] = { + { "0", "01", "012", "0123", "01234" }, + { "A", "B", "C", "D", "E" }, + { "5", "54", "543", "5432", "54321" }, + { "54321", "5433", "544", "55", "6" } +}; +#define TEST_SET_COUNT (sizeof(test_sets) / sizeof(test_sets[0])) + +static bool +item_str_equal(WT_ITEM *item, const char *str) +{ + return (item->size == strlen(str) + 1 && strncmp((char *)item->data, + str, item->size) == 0); +} + +static int +compare_int(int64_t a, int64_t b) +{ + return (a < b ? -1 : (a > b ? 1 : 0)); +} + +static int +index_compare_primary(WT_PACK_STREAM *s1, WT_PACK_STREAM *s2, int *cmp) +{ + int64_t pkey1, pkey2; + int rc1, rc2; + + rc1 = wiredtiger_unpack_int(s1, &pkey1); + rc2 = wiredtiger_unpack_int(s2, &pkey2); + + if (rc1 == 0 && rc2 == 0) + *cmp = compare_int(pkey1, pkey2); + else if (rc1 != 0 && rc2 != 0) + *cmp = 0; + else if (rc1 != 0) + *cmp = -1; + else + *cmp = 1; + return (0); +} + +static int +index_compare_S(WT_COLLATOR *collator, WT_SESSION *session, + const WT_ITEM *key1, const WT_ITEM *key2, int *cmp) +{ + WT_PACK_STREAM *s1, *s2; + const char *skey1, *skey2; + + (void)collator; + + testutil_check(wiredtiger_unpack_start(session, "Si", key1->data, + key1->size, &s1)); + testutil_check(wiredtiger_unpack_start(session, "Si", key2->data, + key2->size, &s2)); + + testutil_check(wiredtiger_unpack_str(s1, &skey1)); + testutil_check(wiredtiger_unpack_str(s2, &skey2)); + + if ((*cmp = strcmp(skey1, skey2)) == 0) + testutil_check(index_compare_primary(s1, s2, cmp)); + + testutil_check(wiredtiger_pack_close(s1, NULL)); + testutil_check(wiredtiger_pack_close(s2, NULL)); + + return (0); +} + +static int +index_compare_u(WT_COLLATOR *collator, WT_SESSION *session, + const WT_ITEM *key1, const WT_ITEM *key2, int *cmp) +{ + WT_ITEM skey1, skey2; + WT_PACK_STREAM *s1, *s2; + + (void)collator; + + testutil_check(wiredtiger_unpack_start(session, "ui", key1->data, + key1->size, &s1)); + testutil_check(wiredtiger_unpack_start(session, "ui", key2->data, + key2->size, &s2)); + + testutil_check(wiredtiger_unpack_item(s1, &skey1)); + testutil_check(wiredtiger_unpack_item(s2, &skey2)); + + if ((*cmp = strcmp(skey1.data, skey2.data)) == 0) + testutil_check(index_compare_primary(s1, s2, cmp)); + + testutil_check(wiredtiger_pack_close(s1, NULL)); + testutil_check(wiredtiger_pack_close(s2, NULL)); + + return (0); +} + +static int +index_extractor_u(WT_EXTRACTOR *extractor, WT_SESSION *session, + const WT_ITEM *key, const WT_ITEM *value, WT_CURSOR *result_cursor) +{ + (void)extractor; + (void)session; + (void)key; + + result_cursor->set_key(result_cursor, value); + return result_cursor->insert(result_cursor); +} + +static WT_COLLATOR collator_S = { index_compare_S, NULL, NULL }; +static WT_COLLATOR collator_u = { index_compare_u, NULL, NULL }; +static WT_EXTRACTOR extractor_u = { index_extractor_u, NULL, NULL }; + +/* + * Check search() and search_near() using the test string indicated + * by test_index. + */ +static void +search_using_str(WT_CURSOR *cursor, TEST_SET test_set, int test_index) +{ + int exact, ret; + const char *result; + const char *str_01, *str_0123, *test_str; + + testutil_assert(test_index >= 0 && test_index <= 4); + str_01 = test_set[1]; + str_0123 = test_set[3]; + test_str = test_set[test_index]; + + cursor->set_key(cursor, test_str); + testutil_check(cursor->search_near(cursor, &exact)); + testutil_check(cursor->get_key(cursor, &result)); + + if (test_index == 0) + testutil_assert(strcmp(result, str_01) == 0 && exact > 0); + else if (test_index == 1) + testutil_assert(strcmp(result, str_01) == 0 && exact == 0); + else if (test_index == 2) + testutil_assert((strcmp(result, str_0123) == 0 && exact > 0) || + (strcmp(result, str_01) == 0 && exact < 0)); + else if (test_index == 3) + testutil_assert(strcmp(result, str_0123) == 0 && exact == 0); + else if (test_index == 4) + testutil_assert(strcmp(result, str_0123) == 0 && exact < 0); + + cursor->set_key(cursor, test_str); + ret = cursor->search(cursor); + + if (test_index == 0 || test_index == 2 || test_index == 4) + testutil_assert(ret == WT_NOTFOUND); + else if (test_index == 1 || test_index == 3) + testutil_assert(ret == 0); +} + +/* + * Check search() and search_near() using the test string indicated + * by test_index against a table containing a variable sized item. + */ +static void +search_using_item(WT_CURSOR *cursor, TEST_SET test_set, int test_index) +{ + WT_ITEM item; + size_t testlen; + int exact, ret; + const char *str_01, *str_0123, *test_str; + + testutil_assert(test_index >= 0 && test_index <= 4); + str_01 = test_set[1]; + str_0123 = test_set[3]; + test_str = test_set[test_index]; + + testlen = strlen(test_str) + 1; + item.data = test_str; + item.size = testlen; + cursor->set_key(cursor, &item); + testutil_check(cursor->search_near(cursor, &exact)); + testutil_check(cursor->get_key(cursor, &item)); + + if (test_index == 0) + testutil_assert(item_str_equal(&item, str_01) && exact > 0); + else if (test_index == 1) + testutil_assert(item_str_equal(&item, str_01) && exact == 0); + else if (test_index == 2) + testutil_assert((item_str_equal(&item, str_0123) && exact > 0) + || (item_str_equal(&item, str_01) && exact < 0)); + else if (test_index == 3) + testutil_assert(item_str_equal(&item, str_0123) && exact == 0); + else if (test_index == 4) + testutil_assert(item_str_equal(&item, str_0123) && exact < 0); + + item.data = test_str; + item.size = testlen; + cursor->set_key(cursor, &item); + ret = cursor->search(cursor); + + if (test_index == 0 || test_index == 2 || test_index == 4) + testutil_assert(ret == WT_NOTFOUND); + else if (test_index == 1 || test_index == 3) + testutil_assert(ret == 0); +} + +/* + * For each set of data, perform tests. + */ +static void +test_one_set(WT_SESSION *session, TEST_SET set) +{ + WT_CURSOR *cursor; + WT_ITEM item; + int32_t i; + + /* + * Part 1: Using a custom collator, insert some elements + * and verify results from search_near. + */ + + testutil_check(session->create(session, + "table:main", "key_format=i,value_format=S,columns=(k,v)")); + testutil_check(session->create(session, + "index:main:def_collator", "columns=(v)")); + testutil_check(session->create(session, + "index:main:custom_collator", + "columns=(v),collator=collator_S")); + + /* Insert only elements #1 and #3. */ + testutil_check(session->open_cursor(session, + "table:main", NULL, NULL, &cursor)); + cursor->set_key(cursor, 0); + cursor->set_value(cursor, set[1]); + testutil_check(cursor->insert(cursor)); + cursor->set_key(cursor, 1); + cursor->set_value(cursor, set[3]); + testutil_check(cursor->insert(cursor)); + testutil_check(cursor->close(cursor)); + + /* Check all elements in def_collator index. */ + testutil_check(session->open_cursor(session, + "index:main:def_collator", NULL, NULL, &cursor)); + for (i = 0; i < (int32_t)TEST_ENTRY_COUNT; i++) + search_using_str(cursor, set, i); + testutil_check(cursor->close(cursor)); + + /* Check all elements in custom_collator index */ + testutil_check(session->open_cursor(session, + "index:main:custom_collator", NULL, NULL, &cursor)); + for (i = 0; i < (int32_t)TEST_ENTRY_COUNT; i++) + search_using_str(cursor, set, i); + testutil_check(cursor->close(cursor)); + + /* + * Part 2: perform the same checks using a custom collator and + * extractor. + */ + testutil_check(session->create(session, + "table:main2", "key_format=i,value_format=u,columns=(k,v)")); + + testutil_check(session->create(session, "index:main2:idx_w_coll", + "key_format=u,collator=collator_u,extractor=extractor_u")); + + testutil_check(session->open_cursor(session, + "table:main2", NULL, NULL, &cursor)); + + memset(&item, 0, sizeof(item)); + item.size = strlen(set[1]) + 1; + item.data = set[1]; + cursor->set_key(cursor, 1); + cursor->set_value(cursor, &item); + testutil_check(cursor->insert(cursor)); + + item.size = strlen(set[3]) + 1; + item.data = set[3]; + cursor->set_key(cursor, 3); + cursor->set_value(cursor, &item); + testutil_check(cursor->insert(cursor)); + + testutil_check(cursor->close(cursor)); + + testutil_check(session->open_cursor(session, + "index:main2:idx_w_coll", NULL, NULL, &cursor)); + for (i = 0; i < (int32_t)TEST_ENTRY_COUNT; i++) + search_using_item(cursor, set, i); + testutil_check(cursor->close(cursor)); + + testutil_check(session->drop(session, "table:main", NULL)); + testutil_check(session->drop(session, "table:main2", NULL)); +} + +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + WT_SESSION *session; + size_t i; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(wiredtiger_open(opts->home, NULL, "create", + &opts->conn)); + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + /* Add any collators and extractors used by tests */ + testutil_check(opts->conn->add_collator(opts->conn, "collator_S", + &collator_S, NULL)); + testutil_check(opts->conn->add_collator(opts->conn, "collator_u", + &collator_u, NULL)); + testutil_check(opts->conn->add_extractor(opts->conn, "extractor_u", + &extractor_u, NULL)); + + for (i = 0; i < TEST_SET_COUNT; i++) { + printf("test set %" WT_SIZET_FMT "\n", i); + test_one_set(session, test_sets[i]); + } + + testutil_check(session->close(session, NULL)); + testutil_cleanup(opts); + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/test/csuite/wt3184_dup_index_collator/main.c b/src/third_party/wiredtiger/test/csuite/wt3184_dup_index_collator/main.c new file mode 100644 index 00000000000..c969e7a1d7e --- /dev/null +++ b/src/third_party/wiredtiger/test/csuite/wt3184_dup_index_collator/main.c @@ -0,0 +1,168 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "test_util.h" + +/* + * JIRA ticket reference: WT-3184 + * Test case description: Each set of data is ordered and contains + * five elements (0-4). We insert elements 1 and 3, and then do + * search_near and search for each element. For each set of data, we perform + * these tests first using a custom collator, and second using a custom collator + * and extractor. In each case there are index keys having variable length. + * Failure mode: In the reported test case, the custom compare routine is + * given a truncated key to compare, and the unpack functions return errors + * because the truncation appeared in the middle of a key. + */ + +static int +compare_int(int32_t a, int32_t b) +{ + return (a < b ? -1 : (a > b ? 1 : 0)); +} + +static int32_t +item_to_int(WT_ITEM *item) +{ + testutil_assert(item->size == sizeof(int32_t)); + return (*(int32_t *)item->data); +} + +static int +compare_int_items(WT_ITEM *itema, WT_ITEM *itemb) +{ + testutil_assert(itema->size == sizeof(int32_t)); + testutil_assert(itemb->size == sizeof(int32_t)); + return (compare_int(item_to_int(itema), item_to_int(itemb))); +} + +static void +print_int_item(const char *str, const WT_ITEM *item) +{ + if (item->size > 0) { + testutil_assert(item->size == sizeof(int32_t)); + printf("%s%" PRId32, str, *(int32_t *)item->data); + } else + printf("%s", str); +} + +static int +index_compare(WT_COLLATOR *collator, WT_SESSION *session, + const WT_ITEM *key1, const WT_ITEM *key2, int *cmp) +{ + WT_ITEM ikey1, pkey1, ikey2, pkey2; + + (void)collator; + testutil_check(wiredtiger_struct_unpack(session, + key1->data, key1->size, "uu", &ikey1, &pkey1)); + testutil_check(wiredtiger_struct_unpack(session, + key2->data, key2->size, "uu", &ikey2, &pkey2)); + + print_int_item("index_compare: index key1 = ", &ikey1); + print_int_item(", primary key1 = ", &pkey1); + print_int_item(", index key2 = ", &ikey2); + print_int_item(", primary key2 = ", &pkey2); + printf("\n"); + + if ((*cmp = compare_int_items(&ikey1, &ikey2)) != 0) + return (0); + + if (pkey1.size != 0 && pkey2.size != 0) + *cmp = compare_int_items(&pkey1, &pkey2); + else if (pkey1.size != 0) + *cmp = 1; + else if (pkey2.size != 0) + *cmp = -1; + else + *cmp = 0; + + return (0); +} + +static WT_COLLATOR index_coll = { index_compare, NULL, NULL }; + +int +main(int argc, char *argv[]) +{ + TEST_OPTS *opts, _opts; + WT_CURSOR *cursor, *cursor1; + WT_ITEM got, k, v; + WT_SESSION *session; + int32_t ki, vi; + + opts = &_opts; + memset(opts, 0, sizeof(*opts)); + testutil_check(testutil_parse_opts(argc, argv, opts)); + testutil_make_work_dir(opts->home); + + testutil_check(wiredtiger_open(opts->home, NULL, "create", + &opts->conn)); + testutil_check( + opts->conn->open_session(opts->conn, NULL, NULL, &session)); + + testutil_check(opts->conn->add_collator(opts->conn, "index_coll", + &index_coll, NULL)); + + testutil_check(session->create(session, + "table:main", "key_format=u,value_format=u,columns=(k,v)")); + testutil_check(session->create(session, + "index:main:index", "columns=(v),collator=index_coll")); + + printf("adding new record\n"); + testutil_check(session->open_cursor(session, "table:main", NULL, NULL, + &cursor)); + + ki = 13; + vi = 17; + + k.data = &ki; k.size = sizeof(ki); + v.data = &vi; v.size = sizeof(vi); + + cursor->set_key(cursor, &k); + cursor->set_value(cursor, &v); + testutil_check(cursor->insert(cursor)); + testutil_check(cursor->close(cursor)); + + printf("positioning index cursor\n"); + + testutil_check(session->open_cursor(session, "index:main:index", NULL, + NULL, &cursor)); + cursor->set_key(cursor, &v); + testutil_check(cursor->search(cursor)); + + printf("duplicating cursor\n"); + testutil_check(session->open_cursor(session, NULL, cursor, NULL, + &cursor1)); + testutil_check(cursor->get_value(cursor, &got)); + testutil_assert(item_to_int(&got) == 17); + testutil_check(cursor1->get_value(cursor1, &got)); + testutil_assert(item_to_int(&got) == 17); + + testutil_check(session->close(session, NULL)); + testutil_cleanup(opts); + return (EXIT_SUCCESS); +} diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order.c index 85b8c68e545..d3c64b54ab5 100644 --- a/src/third_party/wiredtiger/test/cursor_order/cursor_order.c +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order.c @@ -29,7 +29,6 @@ #include "cursor_order.h" static char home[512]; /* Program working dir */ -static char *progname; /* Program name */ static FILE *logfp; /* Log file */ static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); @@ -51,10 +50,7 @@ main(int argc, char *argv[]) int ch, cnt, runs; char *config_open, *working_dir; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); cfg = &_cfg; config_open = NULL; @@ -185,19 +181,15 @@ wt_connect(SHARED_CONFIG *cfg, char *config_open) }; int ret; char config[512]; - size_t print_count; testutil_clean_work_dir(home); testutil_make_work_dir(home); - print_count = (size_t)snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "create,statistics=(all),error_prefix=\"%s\",%s%s", progname, config_open == NULL ? "" : ",", - config_open == NULL ? "" : config_open); - - if (print_count >= sizeof(config)) - testutil_die(EINVAL, "Config string too long"); + config_open == NULL ? "" : config_open)); if ((ret = wiredtiger_open( home, &event_handler, config, &cfg->conn)) != 0) diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order_file.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order_file.c index 5dc7194b5fb..42d7af54de4 100644 --- a/src/third_party/wiredtiger/test/cursor_order/cursor_order_file.c +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order_file.c @@ -34,23 +34,21 @@ file_create(SHARED_CONFIG *cfg, const char *name) WT_CONNECTION *conn; WT_SESSION *session; int ret; - char *p, *end, config[128]; + char config[128]; conn = cfg->conn; if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) testutil_die(ret, "conn.session"); - p = config; - end = config + sizeof(config); - p += snprintf(p, (size_t)(end - p), + testutil_check(__wt_snprintf(config, sizeof(config), "key_format=%s," "internal_page_max=%d," "split_deepen_min_child=200," - "leaf_page_max=%d,", - cfg->ftype == ROW ? "S" : "r", 16 * 1024, 128 * 1024); - if (cfg->ftype == FIX) - (void)snprintf(p, (size_t)(end - p), ",value_format=3t"); + "leaf_page_max=%d," + "%s", + cfg->ftype == ROW ? "S" : "r", 16 * 1024, 128 * 1024, + cfg->ftype == FIX ? ",value_format=3t" : "")); if ((ret = session->create(session, name, config)) != 0) if (ret != EEXIST) @@ -67,9 +65,10 @@ load(SHARED_CONFIG *cfg, const char *name) WT_CURSOR *cursor; WT_ITEM *value, _value; WT_SESSION *session; - char keybuf[64], valuebuf[64]; - int64_t keyno; + size_t len; + uint64_t keyno; int ret; + char keybuf[64], valuebuf[64]; conn = cfg->conn; @@ -83,9 +82,10 @@ load(SHARED_CONFIG *cfg, const char *name) testutil_die(ret, "cursor.open"); value = &_value; - for (keyno = 1; keyno <= (int64_t)cfg->nkeys; ++keyno) { + for (keyno = 1; keyno <= cfg->nkeys; ++keyno) { if (cfg->ftype == ROW) { - snprintf(keybuf, sizeof(keybuf), "%016u", (u_int)keyno); + testutil_check(__wt_snprintf( + keybuf, sizeof(keybuf), "%016" PRIu64, keyno)); cursor->set_key(cursor, keybuf); } else cursor->set_key(cursor, (uint32_t)keyno); @@ -93,8 +93,10 @@ load(SHARED_CONFIG *cfg, const char *name) if (cfg->ftype == FIX) cursor->set_value(cursor, 0x01); else { - value->size = (uint32_t)snprintf( - valuebuf, sizeof(valuebuf), "%37u", (u_int)keyno); + testutil_check(__wt_snprintf_len_set( + valuebuf, sizeof(valuebuf), + &len, "%37" PRIu64, keyno)); + value->size = (uint32_t)len; cursor->set_value(cursor, value); } if ((ret = cursor->insert(cursor)) != 0) diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c index 58da49b2991..299f22684c9 100644 --- a/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c +++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order_ops.c @@ -69,7 +69,8 @@ ops_start(SHARED_CONFIG *cfg) run_info[i].cfg = cfg; if (i == 0 || cfg->multiple_files) { run_info[i].name = dmalloc(64); - snprintf(run_info[i].name, 64, FNAME, (int)i); + testutil_check(__wt_snprintf( + run_info[i].name, 64, FNAME, (int)i)); /* Vary by orders of magnitude */ if (cfg->vary_nops) @@ -93,8 +94,8 @@ ops_start(SHARED_CONFIG *cfg) run_info[offset].name = dmalloc(64); /* Have reverse scans read from tables with writes. */ name_index = i % cfg->append_inserters; - snprintf( - run_info[offset].name, 64, FNAME, (int)name_index); + testutil_check(__wt_snprintf( + run_info[offset].name, 64, FNAME, (int)name_index)); /* Vary by orders of magnitude */ if (cfg->vary_nops) @@ -231,7 +232,7 @@ reverse_scan(void *arg) id = (uintmax_t)arg; s = &run_info[id]; cfg = s->cfg; - __wt_thread_id(tid, sizeof(tid)); + testutil_check(__wt_thread_id(tid, sizeof(tid))); __wt_random_init(&s->rnd); printf(" reverse scan thread %2" PRIuMAX @@ -272,6 +273,7 @@ append_insert_op( { WT_ITEM *value, _value; uint64_t keyno; + size_t len; int ret; char keybuf[64], valuebuf[64]; @@ -281,7 +283,8 @@ append_insert_op( keyno = __wt_atomic_add64(&cfg->key_range, 1); if (cfg->ftype == ROW) { - snprintf(keybuf, sizeof(keybuf), "%016u", (u_int)keyno); + testutil_check(__wt_snprintf( + keybuf, sizeof(keybuf), "%016" PRIu64, keyno)); cursor->set_key(cursor, keybuf); } else cursor->set_key(cursor, (uint32_t)keyno); @@ -291,8 +294,9 @@ append_insert_op( if (cfg->ftype == FIX) cursor->set_value(cursor, 0x10); else { - value->size = (uint32_t)snprintf( - valuebuf, sizeof(valuebuf), "XXX %37u", (u_int)keyno); + testutil_check(__wt_snprintf_len_set( + valuebuf, sizeof(valuebuf), &len, "XXX %37" PRIu64, keyno)); + value->size = (uint32_t)len; cursor->set_value(cursor, value); } if ((ret = cursor->insert(cursor)) != 0) @@ -318,7 +322,7 @@ append_insert(void *arg) id = (uintmax_t)arg; s = &run_info[id]; cfg = s->cfg; - __wt_thread_id(tid, sizeof(tid)); + testutil_check(__wt_thread_id(tid, sizeof(tid))); __wt_random_init(&s->rnd); printf("write thread %2" PRIuMAX " starting: tid: %s, file: %s\n", diff --git a/src/third_party/wiredtiger/test/fops/file.c b/src/third_party/wiredtiger/test/fops/file.c index ea15f1ee80d..d1cd22ab391 100644 --- a/src/third_party/wiredtiger/test/fops/file.c +++ b/src/third_party/wiredtiger/test/fops/file.c @@ -51,7 +51,7 @@ obj_bulk(void) if ((ret = c->close(c)) != 0) testutil_die(ret, "cursor.close"); } else if (ret != ENOENT && ret != EBUSY && ret != EINVAL) - testutil_die(ret, "session.open_cursor"); + testutil_die(ret, "session.open_cursor bulk"); } if ((ret = session->close(session, NULL)) != 0) testutil_die(ret, "session.close"); @@ -71,7 +71,8 @@ obj_bulk_unique(int force) /* Generate a unique object name. */ if ((ret = pthread_rwlock_wrlock(&single)) != 0) testutil_die(ret, "pthread_rwlock_wrlock single"); - (void)snprintf(new_uri, sizeof(new_uri), "%s.%u", uri, ++uid); + testutil_check(__wt_snprintf( + new_uri, sizeof(new_uri), "%s.%u", uri, ++uid)); if ((ret = pthread_rwlock_unlock(&single)) != 0) testutil_die(ret, "pthread_rwlock_unlock single"); @@ -79,12 +80,17 @@ obj_bulk_unique(int force) testutil_die(ret, "session.create: %s", new_uri); __wt_yield(); - if ((ret = - session->open_cursor(session, new_uri, NULL, "bulk", &c)) != 0) - testutil_die(ret, "session.open_cursor: %s", new_uri); - - if ((ret = c->close(c)) != 0) - testutil_die(ret, "cursor.close"); + /* + * Opening a bulk cursor may have raced with a forced checkpoint + * which created a checkpoint of the empty file, and triggers an EINVAL + */ + if ((ret = session->open_cursor( + session, new_uri, NULL, "bulk", &c)) == 0) { + if ((ret = c->close(c)) != 0) + testutil_die(ret, "cursor.close"); + } else if (ret != EINVAL) + testutil_die(ret, + "session.open_cursor bulk unique: %s, new_uri"); while ((ret = session->drop( session, new_uri, force ? "force" : NULL)) != 0) @@ -147,7 +153,8 @@ obj_create_unique(int force) /* Generate a unique object name. */ if ((ret = pthread_rwlock_wrlock(&single)) != 0) testutil_die(ret, "pthread_rwlock_wrlock single"); - (void)snprintf(new_uri, sizeof(new_uri), "%s.%u", uri, ++uid); + testutil_check(__wt_snprintf( + new_uri, sizeof(new_uri), "%s.%u", uri, ++uid)); if ((ret = pthread_rwlock_unlock(&single)) != 0) testutil_die(ret, "pthread_rwlock_unlock single"); @@ -190,9 +197,13 @@ obj_checkpoint(void) if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) testutil_die(ret, "conn.session"); - /* Force the checkpoint so it has to be taken. */ + /* + * Force the checkpoint so it has to be taken. Forced checkpoints can + * race with other metadata operations and return EBUSY - we'd expect + * applications using forced checkpoints to retry on EBUSY. + */ if ((ret = session->checkpoint(session, "force")) != 0) - if (ret != ENOENT) + if (ret != EBUSY && ret != ENOENT) testutil_die(ret, "session.checkpoint"); if ((ret = session->close(session, NULL)) != 0) diff --git a/src/third_party/wiredtiger/test/fops/t.c b/src/third_party/wiredtiger/test/fops/t.c index 7b4a7cf8fca..07ac07349e3 100644 --- a/src/third_party/wiredtiger/test/fops/t.c +++ b/src/third_party/wiredtiger/test/fops/t.c @@ -34,7 +34,6 @@ u_int nops; /* Operations */ const char *uri; /* Object */ const char *config; /* Object config */ -static char *progname; /* Program name */ static FILE *logfp; /* Log file */ static char home[512]; @@ -71,22 +70,15 @@ main(int argc, char *argv[]) int ch, cnt, ret, runs; char *config_open, *working_dir; - working_dir = NULL; - - /* Remove directories */ - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); if ((ret = pthread_rwlock_init(&single, NULL)) != 0) testutil_die(ret, "pthread_rwlock_init: single"); - config_open = NULL; nops = 1000; nthreads = 10; runs = 1; - + config_open = working_dir = NULL; while ((ch = __wt_getopt(progname, argc, argv, "C:h:l:n:r:t:")) != EOF) switch (ch) { case 'C': /* wiredtiger_open config */ @@ -165,11 +157,11 @@ wt_startup(char *config_open) testutil_make_work_dir(home); - snprintf(config_buf, sizeof(config_buf), + testutil_check(__wt_snprintf(config_buf, sizeof(config_buf), "create,error_prefix=\"%s\",cache_size=5MB%s%s", progname, config_open == NULL ? "" : ",", - config_open == NULL ? "" : config_open); + config_open == NULL ? "" : config_open)); if ((ret = wiredtiger_open( home, &event_handler, config_buf, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); @@ -225,6 +217,11 @@ handle_message(WT_EVENT_HANDLER *handler, (void)(handler); (void)(session); + /* Ignore messages about failing to create forced checkpoints. */ + if (strstr( + message, "forced or named checkpoint") != NULL) + return (0); + if (logfp != NULL) return (fprintf(logfp, "%s\n", message) < 0 ? -1 : 0); diff --git a/src/third_party/wiredtiger/test/format/backup.c b/src/third_party/wiredtiger/test/format/backup.c index 69fdf771de9..8aa614fa970 100644 --- a/src/third_party/wiredtiger/test/format/backup.c +++ b/src/third_party/wiredtiger/test/format/backup.c @@ -63,7 +63,7 @@ copy_file(WT_SESSION *session, const char *name) len = strlen("BACKUP") + strlen(name) + 10; first = dmalloc(len); - (void)snprintf(first, len, "BACKUP/%s", name); + testutil_check(__wt_snprintf(first, len, "BACKUP/%s", name)); testutil_check(__wt_copy_and_sync(session, name, first)); /* @@ -72,7 +72,7 @@ copy_file(WT_SESSION *session, const char *name) */ len = strlen("BACKUP_COPY") + strlen(name) + 10; second = dmalloc(len); - (void)snprintf(second, len, "BACKUP_COPY/%s", name); + testutil_check(__wt_snprintf(second, len, "BACKUP_COPY/%s", name)); testutil_check(__wt_copy_and_sync(session, first, second)); free(first); diff --git a/src/third_party/wiredtiger/test/format/config.c b/src/third_party/wiredtiger/test/format/config.c index 839ff5058de..22b40f7164d 100644 --- a/src/third_party/wiredtiger/test/format/config.c +++ b/src/third_party/wiredtiger/test/format/config.c @@ -44,6 +44,7 @@ static void config_map_compression(const char *, u_int *); static void config_map_encryption(const char *, u_int *); static void config_map_file_type(const char *, u_int *); static void config_map_isolation(const char *, u_int *); +static void config_pct(void); static void config_reset(void); /* @@ -62,39 +63,42 @@ config_setup(void) config_in_memory(); /* - * Choose a data source type and a file type: they're interrelated (LSM - * trees are only compatible with row-store) and other items depend on - * them. + * Choose a file format and a data source: they're interrelated (LSM is + * only compatible with row-store) and other items depend on them. */ + if (!config_is_perm("file_type")) { + if (config_is_perm("data_source") && DATASOURCE("lsm")) + config_single("file_type=row", 0); + else + switch (mmrand(NULL, 1, 10)) { + case 1: /* 10% */ + config_single("file_type=fix", 0); + break; + case 2: case 3: case 4: /* 30% */ + config_single("file_type=var", 0); + break; /* 60% */ + case 5: case 6: case 7: case 8: case 9: case 10: + config_single("file_type=row", 0); + break; + } + } + config_map_file_type(g.c_file_type, &g.type); + if (!config_is_perm("data_source")) switch (mmrand(NULL, 1, 3)) { case 1: config_single("data_source=file", 0); break; case 2: - if (!g.c_in_memory) { - config_single("data_source=lsm", 0); - break; - } - /* FALLTHROUGH */ - case 3: config_single("data_source=table", 0); break; - } - - if (!config_is_perm("file_type")) - switch (DATASOURCE("lsm") ? 5 : mmrand(NULL, 1, 10)) { - case 1: - config_single("file_type=fix", 0); - break; - case 2: case 3: case 4: - config_single("file_type=var", 0); - break; - case 5: case 6: case 7: case 8: case 9: case 10: - config_single("file_type=row", 0); + case 3: + if (g.c_in_memory || g.type != ROW) + config_single("data_source=table", 0); + else + config_single("data_source=lsm", 0); break; } - config_map_file_type(g.c_file_type, &g.type); /* * If data_source and file_type were both "permanent", we may still @@ -103,7 +107,7 @@ config_setup(void) if (DATASOURCE("lsm") && g.type != ROW) { fprintf(stderr, "%s: lsm data_source is only compatible with row file_type\n", - g.progname); + progname); exit(EXIT_FAILURE); } @@ -159,31 +163,19 @@ config_setup(void) config_encryption(); config_isolation(); config_lrt(); + config_pct(); /* - * Periodically, set the delete percentage to 0 so salvage gets run, - * as long as the delete percentage isn't nailed down. - * Don't do it on the first run, all our smoke tests would hit it. - */ - if (!g.replay && g.run_cnt % 10 == 9 && !config_is_perm("delete_pct")) - config_single("delete_pct=0", 0); - - /* - * If this is an LSM run, set the cache size and crank up the insert - * percentage. + * If this is an LSM run, ensure cache size sanity. + * Ensure there is at least 1MB of cache per thread. */ - if (DATASOURCE("lsm")) { - if (!config_is_perm("cache")) + if (!config_is_perm("cache")) { + if (DATASOURCE("lsm")) g.c_cache = 30 * g.c_chunk_size; - - if (!config_is_perm("insert_pct")) - g.c_insert_pct = mmrand(NULL, 50, 85); + if (g.c_cache < g.c_threads) + g.c_cache = g.c_threads; } - /* Ensure there is at least 1MB of cache per thread. */ - if (!config_is_perm("cache") && g.c_cache < g.c_threads) - g.c_cache = g.c_threads; - /* Give in-memory configuration a final review. */ config_in_memory_check(); @@ -265,8 +257,8 @@ config_compression(const char *conf_name) */ cstr = "none"; if (strcmp(conf_name, "logging_compression") == 0 && g.c_logging == 0) { - (void)snprintf( - confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr); + testutil_check(__wt_snprintf( + confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr)); config_single(confbuf, 0); return; } @@ -301,7 +293,7 @@ config_compression(const char *conf_name) break; #endif #ifdef HAVE_BUILTIN_EXTENSION_ZSTD - case 15: case 16 case 17: /* 15% zstd */ + case 15: case 16: case 17: /* 15% zstd */ cstr = "zstd"; break; #endif @@ -310,7 +302,8 @@ config_compression(const char *conf_name) break; } - (void)snprintf(confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr); + testutil_check(__wt_snprintf( + confbuf, sizeof(confbuf), "%s=%s", conf_name, cstr)); config_single(confbuf, 0); } @@ -473,7 +466,7 @@ config_lrt(void) * stores. */ if (g.type == FIX) { - if (config_is_perm("long_running_txn")) + if (config_is_perm("long_running_txn") && g.c_long_running_txn) testutil_die(EINVAL, "long_running_txn not supported with fixed-length " "column store"); @@ -481,6 +474,83 @@ config_lrt(void) } } +/* + * config_pct -- + * Configure operation percentages. + */ +static void +config_pct(void) +{ + static struct { + const char *name; /* Operation */ + uint32_t *vp; /* Value store */ + u_int order; /* Order of assignment */ + } list[] = { +#define CONFIG_DELETE_ENTRY 0 + { "delete_pct", &g.c_delete_pct, 0 }, + { "insert_pct", &g.c_insert_pct, 0 }, + { "read_pct", &g.c_read_pct, 0 }, + { "write_pct", &g.c_write_pct, 0 }, + }; + u_int i, max_order, max_slot, n, pct; + + /* + * Walk the list of operations, checking for an illegal configuration + * and creating a random order in the list. + */ + pct = 0; + for (i = 0; i < WT_ELEMENTS(list); ++i) + if (config_is_perm(list[i].name)) + pct += *list[i].vp; + else + list[i].order = mmrand(NULL, 1, 1000); + if (pct > 100) + testutil_die(EINVAL, + "operation percentages total to more than 100%%"); + + /* + * If the delete percentage isn't nailed down, periodically set it to + * 0 so salvage gets run. Don't do it on the first run, all our smoke + * tests would hit it. + */ + if (!config_is_perm("delete_pct") && !g.replay && g.run_cnt % 10 == 9) { + list[CONFIG_DELETE_ENTRY].order = 0; + *list[CONFIG_DELETE_ENTRY].vp = 0; + } + + /* + * Walk the list, allocating random numbers of operations in a random + * order. + * + * If the "order" field is non-zero, we need to create a value for this + * operation. Find the largest order field in the array; if one non-zero + * order field is found, it's the last entry and gets the remainder of + * the operations. + */ + for (pct = 100 - pct;;) { + for (i = n = + max_order = max_slot = 0; i < WT_ELEMENTS(list); ++i) { + if (list[i].order != 0) + ++n; + if (list[i].order > max_order) { + max_order = list[i].order; + max_slot = i; + } + } + if (n == 0) + break; + if (n == 1) { + *list[max_slot].vp = pct; + break; + } + *list[max_slot].vp = mmrand(NULL, 0, pct); + list[max_slot].order = 0; + pct -= *list[max_slot].vp; + } + testutil_assert(g.c_delete_pct + + g.c_insert_pct + g.c_read_pct + g.c_write_pct == 100); +} + /* * config_error -- * Display configuration information on error. @@ -609,13 +679,14 @@ void config_single(const char *s, int perm) { CONFIG *cp; - long v; + long vlong; + uint32_t v; char *p; const char *ep; if ((ep = strchr(s, '=')) == NULL) { fprintf(stderr, - "%s: %s: illegal configuration value\n", g.progname, s); + "%s: %s: illegal configuration value\n", progname, s); exit(EXIT_FAILURE); } @@ -674,34 +745,35 @@ config_single(const char *s, int perm) return; } - v = -1; + vlong = -1; if (F_ISSET(cp, C_BOOL)) { if (strncmp(ep, "off", strlen("off")) == 0) - v = 0; + vlong = 0; else if (strncmp(ep, "on", strlen("on")) == 0) - v = 1; + vlong = 1; } - if (v == -1) { - v = strtol(ep, &p, 10); + if (vlong == -1) { + vlong = strtol(ep, &p, 10); if (*p != '\0') { fprintf(stderr, "%s: %s: illegal numeric value\n", - g.progname, s); + progname, s); exit(EXIT_FAILURE); } } + v = (uint32_t)vlong; if (F_ISSET(cp, C_BOOL)) { if (v != 0 && v != 1) { fprintf(stderr, "%s: %s: value of boolean not 0 or 1\n", - g.progname, s); + progname, s); exit(EXIT_FAILURE); } } else if (v < cp->min || v > cp->maxset) { fprintf(stderr, "%s: %s: value outside min/max values of %" PRIu32 "-%" PRIu32 "\n", - g.progname, s, cp->min, cp->maxset); + progname, s, cp->min, cp->maxset); exit(EXIT_FAILURE); } - *cp->v = (uint32_t)v; + *cp->v = v; } /* @@ -817,7 +889,7 @@ config_find(const char *s, size_t len) return (cp); fprintf(stderr, - "%s: %s: unknown configuration keyword\n", g.progname, s); + "%s: %s: unknown configuration keyword\n", progname, s); config_error(); exit(EXIT_FAILURE); } diff --git a/src/third_party/wiredtiger/test/format/config.h b/src/third_party/wiredtiger/test/format/config.h index 9bfba3cd0df..b5feb7a5321 100644 --- a/src/third_party/wiredtiger/test/format/config.h +++ b/src/third_party/wiredtiger/test/format/config.h @@ -65,6 +65,10 @@ static CONFIG c[] = { "if timed run should drop core", /* 0% */ C_BOOL, 0, 0, 0, &g.c_abort, NULL }, + { "alter", + "if altering the table is enabled", /* 10% */ + C_BOOL, 10, 0, 0, &g.c_alter, NULL }, + { "auto_throttle", "if LSM inserts are throttled", /* 90% */ C_BOOL, 90, 0, 0, &g.c_auto_throttle, NULL }, @@ -127,7 +131,7 @@ static CONFIG c[] = { { "delete_pct", "percent operations that are deletes", - 0x0, 0, 45, 90, &g.c_delete_pct, NULL }, + C_IGNORE, 0, 0, 100, &g.c_delete_pct, NULL }, { "dictionary", "if values are dictionary compressed", /* 20% */ @@ -167,7 +171,7 @@ static CONFIG c[] = { { "insert_pct", "percent operations that are inserts", - 0x0, 0, 45, 90, &g.c_insert_pct, NULL }, + C_IGNORE, 0, 0, 100, &g.c_insert_pct, NULL }, { "internal_key_truncation", "if internal keys are truncated", /* 95% */ @@ -250,6 +254,14 @@ static CONFIG c[] = { "quiet run (same as -q)", C_IGNORE|C_BOOL, 0, 0, 0, &g.c_quiet, NULL }, + { "read_pct", + "percent operations that are reads", + C_IGNORE, 0, 0, 100, &g.c_read_pct, NULL }, + + { "rebalance", + "rebalance testing", /* 100% */ + C_BOOL, 100, 1, 0, &g.c_rebalance, NULL }, + { "repeat_data_pct", "percent duplicate values in row- or var-length column-stores", 0x0, 0, 90, 90, &g.c_repeat_data_pct, NULL }, @@ -266,17 +278,13 @@ static CONFIG c[] = { "the number of runs", C_IGNORE, 0, UINT_MAX, UINT_MAX, &g.c_runs, NULL }, - { "rebalance", - "rebalance testing", /* 100% */ - C_BOOL, 100, 1, 0, &g.c_rebalance, NULL }, - { "salvage", "salvage testing", /* 100% */ C_BOOL, 100, 1, 0, &g.c_salvage, NULL }, { "split_pct", "page split size as a percentage of the maximum page size", - 0x0, 40, 85, 85, &g.c_split_pct, NULL }, + 0x0, 50, 100, 100, &g.c_split_pct, NULL }, { "statistics", "maintain statistics", /* 20% */ @@ -316,7 +324,7 @@ static CONFIG c[] = { { "write_pct", "percent operations that are writes", - 0x0, 0, 90, 90, &g.c_write_pct, NULL }, + C_IGNORE, 0, 0, 100, &g.c_write_pct, NULL }, { NULL, NULL, 0x0, 0, 0, 0, NULL, NULL } }; diff --git a/src/third_party/wiredtiger/test/format/format.h b/src/third_party/wiredtiger/test/format/format.h index 820bc020c9b..41cc48c4278 100644 --- a/src/third_party/wiredtiger/test/format/format.h +++ b/src/third_party/wiredtiger/test/format/format.h @@ -79,8 +79,6 @@ #define FORMAT_OPERATION_REPS 3 /* 3 thread operations sets */ typedef struct { - char *progname; /* Program name */ - char *home; /* Home directory */ char *home_backup; /* Hot-backup directory */ char *home_backup_init; /* Initialize backup command */ @@ -140,6 +138,7 @@ typedef struct { char *config_open; /* Command-line configuration */ uint32_t c_abort; /* Config values */ + uint32_t c_alter; uint32_t c_auto_throttle; uint32_t c_backups; uint32_t c_bitcnt; @@ -191,6 +190,7 @@ typedef struct { uint32_t c_reverse; uint32_t c_rows; uint32_t c_runs; + uint32_t c_read_pct; uint32_t c_rebalance; uint32_t c_salvage; uint32_t c_split_pct; @@ -242,7 +242,7 @@ typedef struct { } GLOBAL; extern GLOBAL g; -typedef struct WT_COMPILER_TYPE_ALIGN(WT_CACHE_LINE_ALIGNMENT) { +typedef struct { WT_RAND_STATE rnd; /* thread RNG state */ uint64_t search; /* operations */ @@ -276,6 +276,7 @@ void bdb_remove(uint64_t, int *); void bdb_update(const void *, size_t, const void *, size_t); #endif +void *alter(void *); void *backup(void *); void *compact(void *); void config_clear(void); diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c index 283e2912daa..72e885bd0d6 100644 --- a/src/third_party/wiredtiger/test/format/ops.c +++ b/src/third_party/wiredtiger/test/format/ops.c @@ -28,14 +28,17 @@ #include "format.h" -static int col_insert(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t *); -static int col_remove(WT_CURSOR *, WT_ITEM *, uint64_t); -static int col_update(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t); +static int col_insert(TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t *); +static int col_remove(WT_CURSOR *, WT_ITEM *, uint64_t, bool); +static int col_update( + TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t, bool); static int nextprev(WT_CURSOR *, int); static void *ops(void *); -static int row_insert(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t); -static int row_remove(WT_CURSOR *, WT_ITEM *, uint64_t); -static int row_update(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t); +static int row_insert( + TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t, bool); +static int row_remove(WT_CURSOR *, WT_ITEM *, uint64_t, bool); +static int row_update( + TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t, bool); static void table_append_init(void); #ifdef HAVE_BERKELEY_DB @@ -50,10 +53,10 @@ static void print_item(const char *, WT_ITEM *); void wts_ops(int lastrun) { - TINFO *tinfo, total; + TINFO **tinfo_list, *tinfo, total; WT_CONNECTION *conn; WT_SESSION *session; - pthread_t backup_tid, compact_tid, lrt_tid; + pthread_t alter_tid, backup_tid, compact_tid, lrt_tid; int64_t fourths, thread_ops; uint32_t i; int running; @@ -61,6 +64,7 @@ wts_ops(int lastrun) conn = g.wts_conn; session = NULL; /* -Wconditional-uninitialized */ + memset(&alter_tid, 0, sizeof(alter_tid)); memset(&backup_tid, 0, sizeof(backup_tid)); memset(&compact_tid, 0, sizeof(compact_tid)); memset(&lrt_tid, 0, sizeof(lrt_tid)); @@ -102,19 +106,24 @@ wts_ops(int lastrun) "=============== thread ops start ==============="); } - /* Create thread structure; start the worker threads. */ - tinfo = dcalloc((size_t)g.c_threads, sizeof(*tinfo)); + /* + * Create the per-thread structures and start the worker threads. + * Allocate the thread structures separately to minimize false sharing. + */ + tinfo_list = dcalloc((size_t)g.c_threads, sizeof(TINFO *)); for (i = 0; i < g.c_threads; ++i) { - tinfo[i].id = (int)i + 1; - tinfo[i].state = TINFO_RUNNING; - testutil_check( - pthread_create(&tinfo[i].tid, NULL, ops, &tinfo[i])); + tinfo_list[i] = tinfo = dcalloc(1, sizeof(TINFO)); + tinfo->id = (int)i + 1; + tinfo->state = TINFO_RUNNING; + testutil_check(pthread_create(&tinfo->tid, NULL, ops, tinfo)); } /* * If a multi-threaded run, start optional backup, compaction and * long-running reader threads. */ + if (g.c_alter) + testutil_check(pthread_create(&alter_tid, NULL, alter, NULL)); if (g.c_backups) testutil_check(pthread_create(&backup_tid, NULL, backup, NULL)); if (g.c_compact) @@ -128,21 +137,22 @@ wts_ops(int lastrun) /* Clear out the totals each pass. */ memset(&total, 0, sizeof(total)); for (i = 0, running = 0; i < g.c_threads; ++i) { - total.commit += tinfo[i].commit; - total.deadlock += tinfo[i].deadlock; - total.insert += tinfo[i].insert; - total.remove += tinfo[i].remove; - total.rollback += tinfo[i].rollback; - total.search += tinfo[i].search; - total.update += tinfo[i].update; - - switch (tinfo[i].state) { + tinfo = tinfo_list[i]; + total.commit += tinfo->commit; + total.deadlock += tinfo->deadlock; + total.insert += tinfo->insert; + total.remove += tinfo->remove; + total.rollback += tinfo->rollback; + total.search += tinfo->search; + total.update += tinfo->update; + + switch (tinfo->state) { case TINFO_RUNNING: running = 1; break; case TINFO_COMPLETE: - tinfo[i].state = TINFO_JOINED; - (void)pthread_join(tinfo[i].tid, NULL); + tinfo->state = TINFO_JOINED; + (void)pthread_join(tinfo->tid, NULL); break; case TINFO_JOINED: break; @@ -154,7 +164,7 @@ wts_ops(int lastrun) */ if (fourths == 0 || (thread_ops != -1 && - tinfo[i].ops >= (uint64_t)thread_ops)) { + tinfo->ops >= (uint64_t)thread_ops)) { /* * On the last execution, optionally drop core * for recovery testing. @@ -163,7 +173,7 @@ wts_ops(int lastrun) static char *core = NULL; *core = 0; } - tinfo[i].quit = 1; + tinfo->quit = 1; } } track("ops", 0ULL, &total); @@ -173,10 +183,14 @@ wts_ops(int lastrun) if (fourths != -1) --fourths; } - free(tinfo); + for (i = 0; i < g.c_threads; ++i) + free(tinfo_list[i]); + free(tinfo_list); /* Wait for the backup, compaction, long-running reader threads. */ g.workers_finished = 1; + if (g.c_alter) + (void)pthread_join(alter_tid, NULL); if (g.c_backups) (void)pthread_join(backup_tid, NULL); if (g.c_compact) @@ -232,6 +246,9 @@ typedef struct { bool insert; /* Insert operation */ } SNAP_OPS; +#define SNAP_TRACK \ + (snap != NULL && (size_t)(snap - snap_list) < WT_ELEMENTS(snap_list)) + /* * snap_track -- * Add a single snapshot isolation returned value to the list. @@ -384,15 +401,16 @@ snap_check(WT_CURSOR *cursor, static void * ops(void *arg) { + enum { INSERT, READ, REMOVE, UPDATE } op; SNAP_OPS *snap, snap_list[64]; TINFO *tinfo; WT_CONNECTION *conn; - WT_CURSOR *cursor, *cursor_insert; + WT_CURSOR *cursor; WT_DECL_RET; WT_ITEM *key, _key, *value, _value; WT_SESSION *session; uint64_t keyno, ckpt_op, reset_op, session_op; - uint32_t op, rnd; + uint32_t rnd; u_int i; int dir; char *ckpt_config, ckpt_name[64]; @@ -418,9 +436,9 @@ ops(void *arg) val_gen_setup(&tinfo->rnd, value); /* Set the first operation where we'll create sessions and cursors. */ - session_op = 0; + cursor = NULL; session = NULL; - cursor = cursor_insert = NULL; + session_op = 0; /* Set the first operation where we'll perform checkpoint operations. */ ckpt_op = g.c_checkpoints ? mmrand(&tinfo->rnd, 100, 10000) : 0; @@ -455,7 +473,7 @@ ops(void *arg) * 10% of the time, perform some read-only operations * from a checkpoint. * - * Skip that if we single-threaded and doing checks + * Skip that if we are single-threaded and doing checks * against a Berkeley DB database, because that won't * work because the Berkeley DB database records won't * match the checkpoint. Also skip if we are using @@ -474,24 +492,11 @@ ops(void *arg) readonly = true; } else { /* - * Open two cursors: one for overwriting and one - * for append (if it's a column-store). - * - * The reason is when testing with existing - * records, we don't track if a record was - * deleted or not, which means we must use - * cursor->insert with overwriting configured. - * But, in column-store files where we're - * testing with new, appended records, we don't - * want to have to specify the record number, - * which requires an append configuration. + * Configure "append", in the case of column + * stores, we append when inserting new rows. */ - testutil_check(session->open_cursor(session, - g.uri, NULL, "overwrite", &cursor)); - if (g.type == FIX || g.type == VAR) - testutil_check(session->open_cursor( - session, g.uri, - NULL, "append", &cursor_insert)); + testutil_check(session->open_cursor( + session, g.uri, NULL, "append", &cursor)); /* Pick the next session/cursor close/open. */ session_op += mmrand(&tinfo->rnd, 100, 5000); @@ -525,8 +530,9 @@ ops(void *arg) pthread_rwlock_trywrlock(&g.backup_lock) == EBUSY) ckpt_config = NULL; else { - (void)snprintf(ckpt_name, sizeof(ckpt_name), - "name=thread-%d", tinfo->id); + testutil_check(__wt_snprintf( + ckpt_name, sizeof(ckpt_name), + "name=thread-%d", tinfo->id)); ckpt_config = ckpt_name; } @@ -552,8 +558,9 @@ ops(void *arg) strcpy(ckpt_name, "checkpoint=WiredTigerCheckpoint"); else - (void)snprintf(ckpt_name, sizeof(ckpt_name), - "checkpoint=thread-%d", tinfo->id); + testutil_check(__wt_snprintf( + ckpt_name, sizeof(ckpt_name), + "checkpoint=thread-%d", tinfo->id)); ckpt_available = true; skip_checkpoint: /* Pick the next checkpoint operation. */ @@ -589,111 +596,174 @@ skip_checkpoint: /* Pick the next checkpoint operation. */ intxn = true; } + /* Select a row. */ keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows); positioned = false; + /* Select an operation. */ + op = READ; + if (!readonly) { + i = mmrand(&tinfo->rnd, 1, 100); + if (i < g.c_delete_pct) + op = REMOVE; + else if (i < g.c_delete_pct + g.c_insert_pct) + op = INSERT; + else if (i < + g.c_delete_pct + g.c_insert_pct + g.c_write_pct) + op = UPDATE; + else + op = READ; + } + /* - * Perform some number of operations: the percentage of deletes, - * inserts and writes are specified, reads are the rest. The - * percentages don't have to add up to 100, a high percentage - * of deletes will mean fewer inserts and writes. Modifications - * are always followed by a read to confirm it worked. + * Inserts, removes and updates can be done following a cursor + * set-key, or based on a cursor position taken from a previous + * search. If not already doing a read, position the cursor at + * an existing point in the tree 20% of the time. */ - op = readonly ? UINT32_MAX : mmrand(&tinfo->rnd, 1, 100); - if (op < g.c_delete_pct) { - ++tinfo->remove; + positioned = false; + if (op != READ && mmrand(&tinfo->rnd, 1, 5) == 1) { + ++tinfo->search; + ret = read_row(cursor, key, value, keyno); + if (ret == 0) { + positioned = true; + if (SNAP_TRACK) + snap_track(snap++, keyno, NULL, value); + } else { + positioned = false; + if (ret == WT_ROLLBACK && intxn) + goto deadlock; + testutil_assert(ret == WT_NOTFOUND); + } + } +#if 0 + /* Optionally reserve a row. */ + if (!readonly && intxn && mmrand(&tinfo->rnd, 0, 20) == 1) { switch (g.type) { case ROW: - ret = row_remove(cursor, key, keyno); + ret = + row_reserve(cursor, key, keyno, positioned); break; case FIX: case VAR: - ret = col_remove(cursor, key, keyno); + ret = col_reserve(cursor, keyno, positioned); break; } if (ret == 0) { positioned = true; - if (snap != NULL && (size_t) - (snap - snap_list) < WT_ELEMENTS(snap_list)) - snap_track(snap++, keyno, NULL, NULL); + __wt_yield(); } else { positioned = false; if (ret == WT_ROLLBACK && intxn) goto deadlock; + testutil_assert(ret == WT_NOTFOUND); } - } else if (op < g.c_delete_pct + g.c_insert_pct) { - ++tinfo->insert; + } +#endif + /* Perform the operation. */ + switch (op) { + case INSERT: switch (g.type) { case ROW: - key_gen_insert(&tinfo->rnd, key, keyno); - val_gen(&tinfo->rnd, value, keyno); - ret = row_insert(cursor, key, value, keyno); + ret = row_insert(tinfo, + cursor, key, value, keyno, positioned); break; case FIX: case VAR: /* - * We can only append so many new records, if - * we've reached that limit, update a record - * instead of doing an insert. + * We can only append so many new records, once + * we reach that limit, update a record instead + * of inserting. */ if (g.append_cnt >= g.append_max) - goto skip_insert; + goto update_instead_of_insert; - /* Insert, then reset the insert cursor. */ - val_gen(&tinfo->rnd, value, g.rows + 1); ret = col_insert( - cursor_insert, key, value, &keyno); - testutil_check( - cursor_insert->reset(cursor_insert)); + tinfo, cursor, key, value, &keyno); break; } + + /* Insert never leaves the cursor positioned. */ positioned = false; if (ret == 0) { - if (snap != NULL && (size_t) - (snap - snap_list) < WT_ELEMENTS(snap_list)) + ++tinfo->insert; + if (SNAP_TRACK) snap_track(snap++, keyno, g.type == ROW ? key : NULL, value); - } else + } else { if (ret == WT_ROLLBACK && intxn) goto deadlock; - } else if ( - op < g.c_delete_pct + g.c_insert_pct + g.c_write_pct) { - ++tinfo->update; + testutil_assert(ret == 0 || ret == WT_ROLLBACK); + } + break; + case READ: + ++tinfo->search; + ret = read_row(cursor, key, value, keyno); + if (ret == 0) { + positioned = true; + if (SNAP_TRACK) + snap_track(snap++, keyno, NULL, value); + } else { + positioned = false; + if (ret == WT_ROLLBACK && intxn) + goto deadlock; + testutil_assert(ret == WT_NOTFOUND); + } + break; + case REMOVE: switch (g.type) { case ROW: - key_gen(key, keyno); - val_gen(&tinfo->rnd, value, keyno); - ret = row_update(cursor, key, value, keyno); + ret = + row_remove(cursor, key, keyno, positioned); break; case FIX: case VAR: -skip_insert: val_gen(&tinfo->rnd, value, keyno); - ret = col_update(cursor, key, value, keyno); + ret = + col_remove(cursor, key, keyno, positioned); break; } if (ret == 0) { - positioned = true; - if (snap != NULL && (size_t) - (snap - snap_list) < WT_ELEMENTS(snap_list)) - snap_track(snap++, keyno, NULL, value); + ++tinfo->remove; + /* + * Don't set positioned: it's unchanged from the + * previous state, but not necessarily set. + */ + if (SNAP_TRACK) + snap_track(snap++, keyno, NULL, NULL); } else { positioned = false; if (ret == WT_ROLLBACK && intxn) goto deadlock; + testutil_assert(ret == WT_NOTFOUND); + } + break; + case UPDATE: +update_instead_of_insert: + ++tinfo->update; + + /* Update the row. */ + switch (g.type) { + case ROW: + ret = row_update(tinfo, + cursor, key, value, keyno, positioned); + break; + case FIX: + case VAR: + ret = col_update(tinfo, + cursor, key, value, keyno, positioned); + break; } - } else { - ++tinfo->search; - ret = read_row(cursor, key, value, keyno); if (ret == 0) { positioned = true; - if (snap != NULL && (size_t) - (snap - snap_list) < WT_ELEMENTS(snap_list)) + if (SNAP_TRACK) snap_track(snap++, keyno, NULL, value); } else { positioned = false; if (ret == WT_ROLLBACK && intxn) goto deadlock; + testutil_assert(ret == 0 || ret == WT_ROLLBACK); } + break; } /* @@ -716,8 +786,8 @@ skip_insert: val_gen(&tinfo->rnd, value, keyno); testutil_check(cursor->reset(cursor)); /* - * If we're in a transaction, commit 40% of the time and - * rollback 10% of the time. + * Continue if not in a transaction, else add more operations + * to the transaction half the time. */ if (!intxn || (rnd = mmrand(&tinfo->rnd, 1, 10)) > 5) continue; @@ -730,6 +800,10 @@ skip_insert: val_gen(&tinfo->rnd, value, keyno); cursor, snap_list, snap, key, value)) == WT_ROLLBACK) goto deadlock; + /* + * If we're in a transaction, commit 40% of the time and + * rollback 10% of the time. + */ switch (rnd) { case 1: case 2: case 3: case 4: /* 40% */ testutil_check( @@ -1029,27 +1103,94 @@ nextprev(WT_CURSOR *cursor, int next) return (ret); } +#if 0 +/* + * row_reserve -- + * Reserve a row in a row-store file. + */ +static int +row_reserve(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, bool positioned) +{ + WT_DECL_RET; + + if (!positioned) { + key_gen(key, keyno); + cursor->set_key(cursor, key); + } + + if (g.logging == LOG_OPS) + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, + "%-10s{%.*s}", "reserve", (int)key->size, key->data); + + switch (ret = cursor->reserve(cursor)) { + case 0: + break; + case WT_CACHE_FULL: + case WT_ROLLBACK: + return (WT_ROLLBACK); + case WT_NOTFOUND: + return (WT_NOTFOUND); + default: + testutil_die(ret, + "row_reserve: reserve row %" PRIu64 " by key", keyno); + } + return (0); +} + +/* + * col_reserve -- + * Reserve a row in a column-store file. + */ +static int +col_reserve(WT_CURSOR *cursor, uint64_t keyno, bool positioned) +{ + WT_DECL_RET; + + if (!positioned) + cursor->set_key(cursor, keyno); + + if (g.logging == LOG_OPS) + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, + "%-10s%" PRIu64, "reserve", keyno); + + switch (ret = cursor->reserve(cursor)) { + case 0: + break; + case WT_CACHE_FULL: + case WT_ROLLBACK: + return (WT_ROLLBACK); + case WT_NOTFOUND: + return (WT_NOTFOUND); + default: + testutil_die(ret, "col_reserve: %" PRIu64, keyno); + } + return (0); +} +#endif + /* * row_update -- * Update a row in a row-store file. */ static int -row_update(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) +row_update(TINFO *tinfo, WT_CURSOR *cursor, + WT_ITEM *key, WT_ITEM *value, uint64_t keyno, bool positioned) { WT_DECL_RET; - WT_SESSION *session; - session = cursor->session; + if (!positioned) { + key_gen(key, keyno); + cursor->set_key(cursor, key); + } + val_gen(&tinfo->rnd, value, keyno); + cursor->set_value(cursor, value); - /* Log the operation */ if (g.logging == LOG_OPS) - (void)g.wt_api->msg_printf(g.wt_api, session, + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, "%-10s{%.*s}, {%.*s}", "put", (int)key->size, key->data, (int)value->size, value->data); - cursor->set_key(cursor, key); - cursor->set_value(cursor, value); switch (ret = cursor->update(cursor)) { case 0: break; @@ -1075,32 +1216,32 @@ row_update(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) * Update a row in a column-store file. */ static int -col_update(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) +col_update(TINFO *tinfo, WT_CURSOR *cursor, + WT_ITEM *key, WT_ITEM *value, uint64_t keyno, bool positioned) { WT_DECL_RET; - WT_SESSION *session; - session = cursor->session; + if (!positioned) + cursor->set_key(cursor, keyno); + val_gen(&tinfo->rnd, value, keyno); + if (g.type == FIX) + cursor->set_value(cursor, *(uint8_t *)value->data); + else + cursor->set_value(cursor, value); - /* Log the operation */ if (g.logging == LOG_OPS) { if (g.type == FIX) - (void)g.wt_api->msg_printf(g.wt_api, session, + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, "%-10s%" PRIu64 " {0x%02" PRIx8 "}", "update", keyno, ((uint8_t *)value->data)[0]); else - (void)g.wt_api->msg_printf(g.wt_api, session, + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, "%-10s%" PRIu64 " {%.*s}", "update", keyno, (int)value->size, (char *)value->data); } - cursor->set_key(cursor, keyno); - if (g.type == FIX) - cursor->set_value(cursor, *(uint8_t *)value->data); - else - cursor->set_value(cursor, value); switch (ret = cursor->update(cursor)) { case 0: break; @@ -1227,22 +1368,29 @@ table_append(uint64_t keyno) * Insert a row in a row-store file. */ static int -row_insert(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) +row_insert(TINFO *tinfo, WT_CURSOR *cursor, + WT_ITEM *key, WT_ITEM *value, uint64_t keyno, bool positioned) { WT_DECL_RET; - WT_SESSION *session; - session = cursor->session; + /* + * If we positioned the cursor already, it's a test of an update using + * the insert method. Otherwise, generate a unique key and insert. + */ + if (!positioned) { + key_gen_insert(&tinfo->rnd, key, keyno); + cursor->set_key(cursor, key); + } + val_gen(&tinfo->rnd, value, keyno); + cursor->set_value(cursor, value); /* Log the operation */ if (g.logging == LOG_OPS) - (void)g.wt_api->msg_printf(g.wt_api, session, + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, "%-10s{%.*s}, {%.*s}", "insert", (int)key->size, key->data, (int)value->size, value->data); - cursor->set_key(cursor, key); - cursor->set_value(cursor, value); switch (ret = cursor->insert(cursor)) { case 0: break; @@ -1268,14 +1416,13 @@ row_insert(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno) * Insert an element in a column-store file. */ static int -col_insert(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t *keynop) +col_insert(TINFO *tinfo, + WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t *keynop) { WT_DECL_RET; - WT_SESSION *session; uint64_t keyno; - session = cursor->session; - + val_gen(&tinfo->rnd, value, g.rows + 1); if (g.type == FIX) cursor->set_value(cursor, *(uint8_t *)value->data); else @@ -1296,12 +1443,12 @@ col_insert(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t *keynop) if (g.logging == LOG_OPS) { if (g.type == FIX) - (void)g.wt_api->msg_printf(g.wt_api, session, + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, "%-10s%" PRIu64 " {0x%02" PRIx8 "}", "insert", keyno, ((uint8_t *)value->data)[0]); else - (void)g.wt_api->msg_printf(g.wt_api, session, + (void)g.wt_api->msg_printf(g.wt_api, cursor->session, "%-10s%" PRIu64 " {%.*s}", "insert", keyno, (int)value->size, (char *)value->data); @@ -1324,21 +1471,19 @@ col_insert(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t *keynop) * Remove an row from a row-store file. */ static int -row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno) +row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, bool positioned) { WT_DECL_RET; - WT_SESSION *session; - session = cursor->session; - - key_gen(key, keyno); + if (!positioned) { + key_gen(key, keyno); + cursor->set_key(cursor, key); + } - /* Log the operation */ if (g.logging == LOG_OPS) - (void)g.wt_api->msg_printf( - g.wt_api, session, "%-10s%" PRIu64, "remove", keyno); + (void)g.wt_api->msg_printf(g.wt_api, + cursor->session, "%-10s%" PRIu64, "remove", keyno); - cursor->set_key(cursor, key); /* We use the cursor in overwrite mode, check for existence. */ if ((ret = cursor->search(cursor)) == 0) ret = cursor->remove(cursor); @@ -1374,19 +1519,17 @@ row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno) * Remove a row from a column-store file. */ static int -col_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno) +col_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, bool positioned) { WT_DECL_RET; - WT_SESSION *session; - session = cursor->session; + if (!positioned) + cursor->set_key(cursor, keyno); - /* Log the operation */ if (g.logging == LOG_OPS) - (void)g.wt_api->msg_printf( - g.wt_api, session, "%-10s%" PRIu64, "remove", keyno); + (void)g.wt_api->msg_printf(g.wt_api, + cursor->session, "%-10s%" PRIu64, "remove", keyno); - cursor->set_key(cursor, keyno); /* We use the cursor in overwrite mode, check for existence. */ if ((ret = cursor->search(cursor)) == 0) ret = cursor->remove(cursor); @@ -1437,7 +1580,7 @@ notfound_chk(const char *f, int wt_ret, int bdb_notfound, uint64_t keyno) return (1); if (bdb_notfound) { - fprintf(stderr, "%s: %s:", g.progname, f); + fprintf(stderr, "%s: %s:", progname, f); if (keyno != 0) fprintf(stderr, " row %" PRIu64 ":", keyno); fprintf(stderr, @@ -1445,7 +1588,7 @@ notfound_chk(const char *f, int wt_ret, int bdb_notfound, uint64_t keyno) testutil_die(0, NULL); } if (wt_ret == WT_NOTFOUND) { - fprintf(stderr, "%s: %s:", g.progname, f); + fprintf(stderr, "%s: %s:", progname, f); if (keyno != 0) fprintf(stderr, " row %" PRIu64 ":", keyno); fprintf(stderr, diff --git a/src/third_party/wiredtiger/test/format/rebalance.c b/src/third_party/wiredtiger/test/format/rebalance.c index 9849b7df82b..e35c62e7255 100644 --- a/src/third_party/wiredtiger/test/format/rebalance.c +++ b/src/third_party/wiredtiger/test/format/rebalance.c @@ -41,10 +41,10 @@ wts_rebalance(void) track("rebalance", 0ULL, NULL); /* Dump the current object. */ - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), ".." DIR_DELIM_STR ".." DIR_DELIM_STR "wt" " -h %s dump -f %s/rebalance.orig %s", - g.home, g.home, g.uri); + g.home, g.home, g.uri)); testutil_checkfmt(system(cmd), "command failed: %s", cmd); /* Rebalance, then verify the object. */ @@ -66,21 +66,21 @@ wts_rebalance(void) wts_verify("post-rebalance verify"); wts_close(); - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), ".." DIR_DELIM_STR ".." DIR_DELIM_STR "wt" " -h %s dump -f %s/rebalance.new %s", - g.home, g.home, g.uri); + g.home, g.home, g.uri)); testutil_checkfmt(system(cmd), "command failed: %s", cmd); /* Compare the old/new versions of the object. */ #ifdef _WIN32 - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "fc /b %s\\rebalance.orig %s\\rebalance.new > NUL", - g.home, g.home); + g.home, g.home)); #else - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "cmp %s/rebalance.orig %s/rebalance.new > /dev/null", - g.home, g.home); + g.home, g.home)); #endif testutil_checkfmt(system(cmd), "command failed: %s", cmd); } diff --git a/src/third_party/wiredtiger/test/format/salvage.c b/src/third_party/wiredtiger/test/format/salvage.c index 69805fb1018..f82dc34dd5f 100644 --- a/src/third_party/wiredtiger/test/format/salvage.c +++ b/src/third_party/wiredtiger/test/format/salvage.c @@ -70,29 +70,31 @@ corrupt(void) * It's a little tricky: if the data source is a file, we're looking * for "wt", if the data source is a table, we're looking for "wt.wt". */ - (void)snprintf(buf, sizeof(buf), "%s/%s", g.home, WT_NAME); + testutil_check(__wt_snprintf( + buf, sizeof(buf), "%s/%s", g.home, WT_NAME)); if ((fd = open(buf, O_RDWR)) != -1) { #ifdef _WIN32 - (void)snprintf(copycmd, sizeof(copycmd), + testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), "copy %s\\%s %s\\slvg.copy\\%s.corrupted", - g.home, WT_NAME, g.home, WT_NAME); + g.home, WT_NAME, g.home, WT_NAME)); #else - (void)snprintf(copycmd, sizeof(copycmd), + testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), "cp %s/%s %s/slvg.copy/%s.corrupted", - g.home, WT_NAME, g.home, WT_NAME); + g.home, WT_NAME, g.home, WT_NAME)); #endif goto found; } - (void)snprintf(buf, sizeof(buf), "%s/%s.wt", g.home, WT_NAME); + testutil_check(__wt_snprintf( + buf, sizeof(buf), "%s/%s.wt", g.home, WT_NAME)); if ((fd = open(buf, O_RDWR)) != -1) { #ifdef _WIN32 - (void)snprintf(copycmd, sizeof(copycmd), + testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), "copy %s\\%s.wt %s\\slvg.copy\\%s.wt.corrupted", - g.home, WT_NAME, g.home, WT_NAME); + g.home, WT_NAME, g.home, WT_NAME)); #else - (void)snprintf(copycmd, sizeof(copycmd), + testutil_check(__wt_snprintf(copycmd, sizeof(copycmd), "cp %s/%s.wt %s/slvg.copy/%s.wt.corrupted", - g.home, WT_NAME, g.home, WT_NAME); + g.home, WT_NAME, g.home, WT_NAME)); #endif goto found; } @@ -103,7 +105,8 @@ found: if (fstat(fd, &sb) == -1) offset = mmrand(NULL, 0, (u_int)sb.st_size); len = (size_t)(20 + (sb.st_size / 100) * 2); - (void)snprintf(buf, sizeof(buf), "%s/slvg.corrupt", g.home); + testutil_check(__wt_snprintf( + buf, sizeof(buf), "%s/slvg.corrupt", g.home)); if ((fp = fopen(buf, "w")) == NULL) testutil_die(errno, "salvage-corrupt: open: %s", buf); (void)fprintf(fp, diff --git a/src/third_party/wiredtiger/test/format/t.c b/src/third_party/wiredtiger/test/format/t.c index 7701595776c..c6686ae8b91 100644 --- a/src/third_party/wiredtiger/test/format/t.c +++ b/src/third_party/wiredtiger/test/format/t.c @@ -49,14 +49,7 @@ main(int argc, char *argv[]) config = NULL; -#ifdef _WIN32 - g.progname = "t_format.exe"; -#else - if ((g.progname = strrchr(argv[0], DIR_DELIM)) == NULL) - g.progname = argv[0]; - else - ++g.progname; -#endif + (void)testutil_set_progname(argv); #if 0 /* Configure the GNU malloc for debugging. */ @@ -74,7 +67,7 @@ main(int argc, char *argv[]) home = NULL; onerun = 0; while ((ch = __wt_getopt( - g.progname, argc, argv, "1C:c:H:h:Llqrt:")) != EOF) + progname, argc, argv, "1C:c:H:h:Llqrt:")) != EOF) switch (ch) { case '1': /* One run */ onerun = 1; @@ -179,7 +172,7 @@ main(int argc, char *argv[]) testutil_check(pthread_rwlock_init(&g.checkpoint_lock, NULL)); testutil_check(pthread_rwlock_init(&g.death_lock, NULL)); - printf("%s: process %" PRIdMAX "\n", g.progname, (intmax_t)getpid()); + printf("%s: process %" PRIdMAX "\n", progname, (intmax_t)getpid()); while (++g.run_cnt <= g.c_runs || g.c_runs == 0 ) { startup(); /* Start a run */ @@ -344,7 +337,7 @@ usage(void) "usage: %s [-1Llqr] [-C wiredtiger-config]\n " "[-c config-file] [-H mount] [-h home] " "[name=value ...]\n", - g.progname); + progname); fprintf(stderr, "%s", "\t-1 run once\n" "\t-C specify wiredtiger_open configuration arguments\n" diff --git a/src/third_party/wiredtiger/test/format/util.c b/src/third_party/wiredtiger/test/format/util.c index a709aa93a2e..983d03e2525 100644 --- a/src/third_party/wiredtiger/test/format/util.c +++ b/src/third_party/wiredtiger/test/format/util.c @@ -241,20 +241,23 @@ val_gen(WT_RAND_STATE *rnd, WT_ITEM *value, uint64_t keyno) void track(const char *tag, uint64_t cnt, TINFO *tinfo) { - static int lastlen = 0; - int len; + static size_t lastlen = 0; + size_t len; char msg[128]; if (g.c_quiet || tag == NULL) return; if (tinfo == NULL && cnt == 0) - len = snprintf(msg, sizeof(msg), "%4d: %s", g.run_cnt, tag); + testutil_check(__wt_snprintf_len_set( + msg, sizeof(msg), &len, "%4d: %s", g.run_cnt, tag)); else if (tinfo == NULL) - len = snprintf( - msg, sizeof(msg), "%4d: %s: %" PRIu64, g.run_cnt, tag, cnt); + testutil_check(__wt_snprintf_len_set( + msg, sizeof(msg), &len, + "%4d: %s: %" PRIu64, g.run_cnt, tag, cnt)); else - len = snprintf(msg, sizeof(msg), + testutil_check(__wt_snprintf_len_set( + msg, sizeof(msg), &len, "%4d: %s: " "search %" PRIu64 "%s, " "insert %" PRIu64 "%s, " @@ -268,7 +271,7 @@ track(const char *tag, uint64_t cnt, TINFO *tinfo) tinfo->update > M(9) ? tinfo->update / M(1) : tinfo->update, tinfo->update > M(9) ? "M" : "", tinfo->remove > M(9) ? tinfo->remove / M(1) : tinfo->remove, - tinfo->remove > M(9) ? "M" : ""); + tinfo->remove > M(9) ? "M" : "")); if (lastlen > len) { memset(msg + len, ' ', (size_t)(lastlen - len)); @@ -297,27 +300,30 @@ path_setup(const char *home) /* Log file. */ len = strlen(g.home) + strlen("log") + 2; g.home_log = dmalloc(len); - snprintf(g.home_log, len, "%s/%s", g.home, "log"); + testutil_check(__wt_snprintf(g.home_log, len, "%s/%s", g.home, "log")); /* RNG log file. */ len = strlen(g.home) + strlen("rand") + 2; g.home_rand = dmalloc(len); - snprintf(g.home_rand, len, "%s/%s", g.home, "rand"); + testutil_check(__wt_snprintf( + g.home_rand, len, "%s/%s", g.home, "rand")); /* Run file. */ len = strlen(g.home) + strlen("CONFIG") + 2; g.home_config = dmalloc(len); - snprintf(g.home_config, len, "%s/%s", g.home, "CONFIG"); + testutil_check(__wt_snprintf( + g.home_config, len, "%s/%s", g.home, "CONFIG")); /* Statistics file. */ len = strlen(g.home) + strlen("stats") + 2; g.home_stats = dmalloc(len); - snprintf(g.home_stats, len, "%s/%s", g.home, "stats"); + testutil_check(__wt_snprintf( + g.home_stats, len, "%s/%s", g.home, "stats")); /* BDB directory. */ len = strlen(g.home) + strlen("bdb") + 2; g.home_bdb = dmalloc(len); - snprintf(g.home_bdb, len, "%s/%s", g.home, "bdb"); + testutil_check(__wt_snprintf(g.home_bdb, len, "%s/%s", g.home, "bdb")); /* * Home directory initialize command: create the directory if it doesn't @@ -336,21 +342,23 @@ path_setup(const char *home) "cd %s & mkdir KVS" len = strlen(g.home) * 7 + strlen(CMD) + 1; g.home_init = dmalloc(len); - snprintf(g.home_init, len, CMD, - g.home, g.home, g.home, g.home, g.home, g.home, g.home); + testutil_check(__wt_snprintf(g.home_init, len, CMD, + g.home, g.home, g.home, g.home, g.home, g.home, g.home)); #else #define CMD "test -e %s || mkdir %s; " \ "cd %s > /dev/null && rm -rf `ls | sed /rand/d`; " \ "mkdir KVS" len = strlen(g.home) * 3 + strlen(CMD) + 1; g.home_init = dmalloc(len); - snprintf(g.home_init, len, CMD, g.home, g.home, g.home); + testutil_check(__wt_snprintf( + g.home_init, len, CMD, g.home, g.home, g.home)); #endif /* Primary backup directory. */ len = strlen(g.home) + strlen("BACKUP") + 2; g.home_backup = dmalloc(len); - snprintf(g.home_backup, len, "%s/%s", g.home, "BACKUP"); + testutil_check(__wt_snprintf( + g.home_backup, len, "%s/%s", g.home, "BACKUP")); /* * Backup directory initialize command, remove and re-create the primary @@ -365,9 +373,9 @@ path_setup(const char *home) len = strlen(g.home) * 4 + strlen("BACKUP") * 2 + strlen("BACKUP_COPY") * 2 + strlen(CMD) + 1; g.home_backup_init = dmalloc(len); - snprintf(g.home_backup_init, len, CMD, + testutil_check(__wt_snprintf(g.home_backup_init, len, CMD, g.home, "BACKUP", g.home, "BACKUP_COPY", - g.home, "BACKUP", g.home, "BACKUP_COPY"); + g.home, "BACKUP", g.home, "BACKUP_COPY")); /* * Salvage command, save the interesting files so we can replay the @@ -390,7 +398,7 @@ path_setup(const char *home) #endif len = strlen(g.home) + strlen(CMD) + 1; g.home_salvage_copy = dmalloc(len); - snprintf(g.home_salvage_copy, len, CMD, g.home); + testutil_check(__wt_snprintf(g.home_salvage_copy, len, CMD, g.home)); } /* @@ -459,3 +467,48 @@ fclose_and_clear(FILE **fpp) testutil_die(errno, "fclose"); return; } + +/* + * alter -- + * Periodically alter a table's metadata. + */ +void * +alter(void *arg) +{ + WT_CONNECTION *conn; + WT_SESSION *session; + u_int period; + bool access_value; + char buf[32]; + + (void)(arg); + conn = g.wts_conn; + + /* + * Only alter the access pattern hint. If we alter the cache resident + * setting we may end up with a setting that fills cache and doesn't + * allow it to be evicted. + */ + access_value = false; + + /* Open a session */ + testutil_check(conn->open_session(conn, NULL, NULL, &session)); + + while (!g.workers_finished) { + period = mmrand(NULL, 1, 10); + + testutil_check(__wt_snprintf(buf, sizeof(buf), + "access_pattern_hint=%s", + access_value ? "random" : "none")); + access_value = !access_value; + if (session->alter(session, g.uri, buf) != 0) + break; + while (period > 0 && !g.workers_finished) { + --period; + sleep(1); + } + } + + testutil_check(session->close(session, NULL)); + return (NULL); +} diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c index da234ce53c7..6aa4784d1c1 100644 --- a/src/third_party/wiredtiger/test/format/wts.c +++ b/src/third_party/wiredtiger/test/format/wts.c @@ -120,8 +120,15 @@ static WT_EVENT_HANDLER event_handler = { NULL /* Close handler. */ }; -#undef REMAIN -#define REMAIN(p, end) (size_t)((p) >= (end) ? 0 : (end) - (p)) +#define CONFIG_APPEND(p, ...) do { \ + size_t __len; \ + testutil_check( \ + __wt_snprintf_len_set(p, max, &__len, __VA_ARGS__)); \ + if (__len > max) \ + __len = max; \ + p += __len; \ + max -= __len; \ +} while (0) /* * wts_open -- @@ -132,42 +139,42 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) { WT_CONNECTION *conn; WT_DECL_RET; - char *config, *end, *p, helium_config[1024]; + size_t max; + char *config, *p, helium_config[1024]; *connp = NULL; config = p = g.wiredtiger_open_config; - end = config + sizeof(g.wiredtiger_open_config); + max = sizeof(g.wiredtiger_open_config); - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, "create=true," "cache_size=%" PRIu32 "MB," "checkpoint_sync=false," "error_prefix=\"%s\"", - g.c_cache, g.progname); + g.c_cache, progname); /* In-memory configuration. */ if (g.c_in_memory != 0) - p += snprintf(p, REMAIN(p, end), ",in_memory=1"); + CONFIG_APPEND(p, ",in_memory=1"); /* LSM configuration. */ if (DATASOURCE("lsm")) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",lsm_manager=(worker_thread_max=%" PRIu32 "),", g.c_lsm_worker_threads); - if (DATASOURCE("lsm") || g.c_cache < 20) { - p += snprintf(p, REMAIN(p, end), ",eviction_dirty_trigger=95"); - } + if (DATASOURCE("lsm") || g.c_cache < 20) + CONFIG_APPEND(p, ",eviction_dirty_trigger=95"); /* Eviction worker configuration. */ if (g.c_evict_max != 0) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",eviction=(threads_max=%" PRIu32 ")", g.c_evict_max); /* Logging configuration. */ if (g.c_logging) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",log=(enabled=true,archive=%d,prealloc=%d" ",compressor=\"%s\")", g.c_logging_archive ? 1 : 0, @@ -175,21 +182,21 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) compressor(g.c_logging_compression_flag)); if (g.c_encryption) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",encryption=(name=%s)", encryptor(g.c_encryption_flag)); /* Miscellaneous. */ #ifdef HAVE_POSIX_MEMALIGN - p += snprintf(p, REMAIN(p, end), ",buffer_alignment=512"); + CONFIG_APPEND(p, ",buffer_alignment=512"); #endif - p += snprintf(p, REMAIN(p, end), ",mmap=%d", g.c_mmap ? 1 : 0); + CONFIG_APPEND(p, ",mmap=%d", g.c_mmap ? 1 : 0); if (g.c_direct_io) - p += snprintf(p, REMAIN(p, end), ",direct_io=(data)"); + CONFIG_APPEND(p, ",direct_io=(data)"); if (g.c_data_extend) - p += snprintf(p, REMAIN(p, end), ",file_extend=(data=8MB)"); + CONFIG_APPEND(p, ",file_extend=(data=8MB)"); /* * Run the statistics server and/or maintain statistics in the engine. @@ -198,18 +205,18 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) if (g.c_statistics_server) { if (mmrand(NULL, 0, 5) == 1 && memcmp(g.uri, "file:", strlen("file:")) == 0) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",statistics=(fast)" ",statistics_log=(wait=5,sources=(\"file:\"))"); else - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",statistics=(fast),statistics_log=(wait=5)"); } else - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",statistics=(%s)", g.c_statistics ? "fast" : "none"); /* Extensions. */ - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",extensions=[" "\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\"],", g.c_reverse ? REVERSE_PATH : "", @@ -227,11 +234,11 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) * override the standard configuration. */ if (g.c_config_open != NULL) - p += snprintf(p, REMAIN(p, end), ",%s", g.c_config_open); + CONFIG_APPEND(p, ",%s", g.c_config_open); if (g.config_open != NULL) - p += snprintf(p, REMAIN(p, end), ",%s", g.config_open); + CONFIG_APPEND(p, ",%s", g.config_open); - if (REMAIN(p, end) == 0) + if (max == 0) testutil_die(ENOMEM, "wiredtiger_open configuration buffer too small"); @@ -259,12 +266,13 @@ wts_open(const char *home, bool set_api, WT_CONNECTION **connp) if (DATASOURCE("helium")) { if (g.helium_mount == NULL) testutil_die(EINVAL, "no Helium mount point specified"); - (void)snprintf(helium_config, sizeof(helium_config), + testutil_check( + __wt_snprintf(helium_config, sizeof(helium_config), "entry=wiredtiger_extension_init,config=[" "helium_verbose=0," "dev1=[helium_devices=\"he://./%s\"," "helium_o_volume_truncate=1]]", - g.helium_mount); + g.helium_mount)); if ((ret = conn->load_extension( conn, HELIUM_PATH, helium_config)) != 0) testutil_die(ret, @@ -299,13 +307,13 @@ wts_init(void) { WT_CONNECTION *conn; WT_SESSION *session; + size_t max; uint32_t maxintlpage, maxintlkey, maxleafpage, maxleafkey, maxleafvalue; - char config[4096], *end, *p; + char config[4096], *p; conn = g.wts_conn; - p = config; - end = config + sizeof(config); + max = sizeof(config); /* * Ensure that we can service at least one operation per-thread @@ -326,7 +334,7 @@ wts_init(void) if (maxleafpage > 512) maxleafpage >>= 1; } - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, "key_format=%s," "allocation_size=512,%s" "internal_page_max=%" PRIu32 ",leaf_page_max=%" PRIu32, @@ -340,43 +348,35 @@ wts_init(void) */ maxintlkey = mmrand(NULL, maxintlpage / 50, maxintlpage / 40); if (maxintlkey > 20) - p += snprintf(p, REMAIN(p, end), - ",internal_key_max=%" PRIu32, maxintlkey); + CONFIG_APPEND(p, ",internal_key_max=%" PRIu32, maxintlkey); maxleafkey = mmrand(NULL, maxleafpage / 50, maxleafpage / 40); if (maxleafkey > 20) - p += snprintf(p, REMAIN(p, end), - ",leaf_key_max=%" PRIu32, maxleafkey); + CONFIG_APPEND(p, ",leaf_key_max=%" PRIu32, maxleafkey); maxleafvalue = mmrand(NULL, maxleafpage * 10, maxleafpage / 40); if (maxleafvalue > 40 && maxleafvalue < 100 * 1024) - p += snprintf(p, REMAIN(p, end), - ",leaf_value_max=%" PRIu32, maxleafvalue); + CONFIG_APPEND(p, ",leaf_value_max=%" PRIu32, maxleafvalue); switch (g.type) { case FIX: - p += snprintf(p, REMAIN(p, end), - ",value_format=%" PRIu32 "t", g.c_bitcnt); + CONFIG_APPEND(p, ",value_format=%" PRIu32 "t", g.c_bitcnt); break; case ROW: if (g.c_huffman_key) - p += snprintf(p, REMAIN(p, end), - ",huffman_key=english"); + CONFIG_APPEND(p, ",huffman_key=english"); if (g.c_prefix_compression) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",prefix_compression_min=%" PRIu32, g.c_prefix_compression_min); else - p += snprintf(p, REMAIN(p, end), - ",prefix_compression=false"); + CONFIG_APPEND(p, ",prefix_compression=false"); if (g.c_reverse) - p += snprintf(p, REMAIN(p, end), - ",collator=reverse"); + CONFIG_APPEND(p, ",collator=reverse"); /* FALLTHROUGH */ case VAR: if (g.c_huffman_value) - p += snprintf(p, REMAIN(p, end), - ",huffman_value=english"); + CONFIG_APPEND(p, ",huffman_value=english"); if (g.c_dictionary) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",dictionary=%" PRIu32, mmrand(NULL, 123, 517)); break; } @@ -384,66 +384,63 @@ wts_init(void) /* Configure checksums. */ switch (g.c_checksum_flag) { case CHECKSUM_OFF: - p += snprintf(p, REMAIN(p, end), ",checksum=\"off\""); + CONFIG_APPEND(p, ",checksum=\"off\""); break; case CHECKSUM_ON: - p += snprintf(p, REMAIN(p, end), ",checksum=\"on\""); + CONFIG_APPEND(p, ",checksum=\"on\""); break; case CHECKSUM_UNCOMPRESSED: - p += snprintf(p, REMAIN(p, end), ",checksum=\"uncompressed\""); + CONFIG_APPEND(p, ",checksum=\"uncompressed\""); break; } /* Configure compression. */ if (g.c_compression_flag != COMPRESS_NONE) - p += snprintf(p, REMAIN(p, end), ",block_compressor=\"%s\"", + CONFIG_APPEND(p, ",block_compressor=\"%s\"", compressor(g.c_compression_flag)); /* Configure Btree internal key truncation. */ - p += snprintf(p, REMAIN(p, end), ",internal_key_truncate=%s", + CONFIG_APPEND(p, ",internal_key_truncate=%s", g.c_internal_key_truncation ? "true" : "false"); /* Configure Btree page key gap. */ - p += snprintf(p, REMAIN(p, end), ",key_gap=%" PRIu32, g.c_key_gap); + CONFIG_APPEND(p, ",key_gap=%" PRIu32, g.c_key_gap); /* Configure Btree split page percentage. */ - p += snprintf(p, REMAIN(p, end), ",split_pct=%" PRIu32, g.c_split_pct); + CONFIG_APPEND(p, ",split_pct=%" PRIu32, g.c_split_pct); /* Configure LSM and data-sources. */ if (DATASOURCE("helium")) - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",type=helium,helium_o_compress=%d,helium_o_truncate=1", g.c_compression_flag == COMPRESS_NONE ? 0 : 1); if (DATASOURCE("kvsbdb")) - p += snprintf(p, REMAIN(p, end), ",type=kvsbdb"); + CONFIG_APPEND(p, ",type=kvsbdb"); if (DATASOURCE("lsm")) { - p += snprintf(p, REMAIN(p, end), ",type=lsm,lsm=("); - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, ",type=lsm,lsm=("); + CONFIG_APPEND(p, "auto_throttle=%s,", g.c_auto_throttle ? "true" : "false"); - p += snprintf(p, REMAIN(p, end), - "chunk_size=%" PRIu32 "MB,", g.c_chunk_size); + CONFIG_APPEND(p, "chunk_size=%" PRIu32 "MB,", g.c_chunk_size); /* * We can't set bloom_oldest without bloom, and we want to test * with Bloom filters on most of the time anyway. */ if (g.c_bloom_oldest) g.c_bloom = 1; - p += snprintf(p, REMAIN(p, end), - "bloom=%s,", g.c_bloom ? "true" : "false"); - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, "bloom=%s,", g.c_bloom ? "true" : "false"); + CONFIG_APPEND(p, "bloom_bit_count=%" PRIu32 ",", g.c_bloom_bit_count); - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, "bloom_hash_count=%" PRIu32 ",", g.c_bloom_hash_count); - p += snprintf(p, REMAIN(p, end), + CONFIG_APPEND(p, "bloom_oldest=%s,", g.c_bloom_oldest ? "true" : "false"); - p += snprintf(p, REMAIN(p, end), - "merge_max=%" PRIu32 ",", g.c_merge_max); - p += snprintf(p, REMAIN(p, end), ",)"); + CONFIG_APPEND(p, "merge_max=%" PRIu32 ",", g.c_merge_max); + CONFIG_APPEND(p, ",)"); } - if (REMAIN(p, end) == 0) + if (max == 0) testutil_die(ENOMEM, "WT_SESSION.create configuration buffer too small"); @@ -490,14 +487,14 @@ wts_dump(const char *tag, int dump_bdb) len = strlen(g.home) + strlen(BERKELEY_DB_PATH) + strlen(g.uri) + 100; cmd = dmalloc(len); - (void)snprintf(cmd, len, + testutil_check(__wt_snprintf(cmd, len, "sh s_dumpcmp -h %s %s %s %s %s %s", g.home, dump_bdb ? "-b " : "", dump_bdb ? BERKELEY_DB_PATH : "", g.type == FIX || g.type == VAR ? "-c" : "", g.uri == NULL ? "" : "-n", - g.uri == NULL ? "" : g.uri); + g.uri == NULL ? "" : g.uri)); testutil_checkfmt(system(cmd), "%s: dump comparison failed", tag); free(cmd); @@ -587,7 +584,7 @@ wts_stats(void) fprintf(fp, "\n\n====== Data source statistics:\n"); len = strlen("statistics:") + strlen(g.uri) + 1; stat_name = dmalloc(len); - snprintf(stat_name, len, "statistics:%s", g.uri); + testutil_check(__wt_snprintf(stat_name, len, "statistics:%s", g.uri)); testutil_check(session->open_cursor( session, stat_name, NULL, NULL, &cursor)); free(stat_name); diff --git a/src/third_party/wiredtiger/test/huge/huge.c b/src/third_party/wiredtiger/test/huge/huge.c index 17e2db353d5..2b0d5f498e3 100644 --- a/src/third_party/wiredtiger/test/huge/huge.c +++ b/src/third_party/wiredtiger/test/huge/huge.c @@ -29,7 +29,6 @@ #include "test_util.h" static char home[512]; /* Program working dir */ -static const char *progname; /* Program name */ static uint8_t *big; /* Big key/value buffer */ #define GIGABYTE (1073741824) @@ -167,14 +166,10 @@ main(int argc, char *argv[]) int ch, small; char *working_dir; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); small = 0; working_dir = NULL; - while ((ch = __wt_getopt(progname, argc, argv, "h:s")) != EOF) switch (ch) { case 'h': diff --git a/src/third_party/wiredtiger/test/java/com/wiredtiger/test/CursorTest03.java b/src/third_party/wiredtiger/test/java/com/wiredtiger/test/CursorTest03.java new file mode 100644 index 00000000000..64f33f4d7b6 --- /dev/null +++ b/src/third_party/wiredtiger/test/java/com/wiredtiger/test/CursorTest03.java @@ -0,0 +1,175 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +package com.wiredtiger.test; + +import com.wiredtiger.db.Connection; +import com.wiredtiger.db.Cursor; +import com.wiredtiger.db.SearchStatus; +import com.wiredtiger.db.Session; +import com.wiredtiger.db.WiredTigerPackingException; +import com.wiredtiger.db.WiredTigerException; +import com.wiredtiger.db.wiredtiger; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; +import org.junit.Assert; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/* + * Test cases for WT-3238. + * + * Most WiredTiger methods return int, and our SWIG typemaps for Java add + * checking that throws exceptions for non-zero returns. Certain methods + * (Cursor.compare, Cursor.equals) are declared as returning int in Java, + * but should not throw exceptions for normal returns (which may be + * non-zero). + */ +public class CursorTest03 { + Connection conn; + Session s; + static String values[] = { "key0", "key1" }; + + @Test + public void cursor_int_methods() + throws WiredTigerPackingException { + setup(); + + Cursor c1 = s.open_cursor("table:t", null, null); + Cursor c2 = s.open_cursor("table:t", null, null); + for (String s : values) { + c1.putKeyString(s); + c1.putValueString(s); + c1.insert(); + } + c1.reset(); + + // "key1" compared to "key1" + c1.putKeyString(values[1]); + Assert.assertEquals(c1.search_near(), SearchStatus.FOUND); + c2.putKeyString(values[1]); + Assert.assertEquals(c2.search_near(), SearchStatus.FOUND); + Assert.assertEquals(c1.compare(c2), 0); + Assert.assertEquals(c2.compare(c1), 0); + Assert.assertEquals(c1.compare(c1), 0); + Assert.assertEquals(c1.equals(c2), 1); + Assert.assertEquals(c2.equals(c1), 1); + Assert.assertEquals(c1.equals(c1), 1); + + // "key0" compared to "key1" + c1.putKeyString(values[0]); + Assert.assertEquals(c1.search_near(), SearchStatus.FOUND); + Assert.assertEquals(c1.compare(c2), -1); + Assert.assertEquals(c2.compare(c1), 1); + Assert.assertEquals(c1.equals(c2), 0); + Assert.assertEquals(c2.equals(c1), 0); + + c1.close(); + c2.close(); + teardown(); + } + + public void expectException(Cursor c1, Cursor c2) + { + boolean caught = false; + try { + c1.compare(c2); + } + catch (WiredTigerException wte) { + caught = true; + } + Assert.assertTrue(caught); + + caught = false; + try { + c1.equals(c2); + } + catch (WiredTigerException wte) { + caught = true; + } + Assert.assertTrue(caught); + } + + @Test + public void cursor_int_methods_errors() + throws WiredTigerPackingException { + setup(); + + Cursor c1 = s.open_cursor("table:t", null, null); + Cursor c2 = s.open_cursor("table:t", null, null); + Cursor cx = s.open_cursor("table:t2", null, null); + for (String s : values) { + c1.putKeyString(s); + c1.putValueString(s); + c1.insert(); + cx.putKeyString(s); + cx.putValueString(s); + cx.insert(); + } + c1.reset(); + cx.reset(); + + // With both cursors not set, should be an exception. + expectException(c1, c2); + expectException(c1, c2); + + // With any one cursor not set, should be an exception. + c1.putKeyString(values[1]); + Assert.assertEquals(c1.search_near(), SearchStatus.FOUND); + expectException(c1, c2); + expectException(c1, c2); + + // With two cursors from different tables, should be an exception. + cx.putKeyString(values[1]); + Assert.assertEquals(cx.search_near(), SearchStatus.FOUND); + expectException(c1, cx); + expectException(c1, cx); + + c1.close(); + c2.close(); + cx.close(); + teardown(); + } + + private void setup() { + conn = wiredtiger.open("WT_HOME", "create"); + s = conn.open_session(null); + s.create("table:t", "key_format=S,value_format=S"); + s.create("table:t2", "key_format=S,value_format=S"); + } + + private void teardown() { + s.drop("table:t", ""); + s.drop("table:t2", ""); + s.close(""); + conn.close(""); + } + +} + diff --git a/src/third_party/wiredtiger/test/java/com/wiredtiger/test/WiredTigerSuite.java b/src/third_party/wiredtiger/test/java/com/wiredtiger/test/WiredTigerSuite.java index 5bd98d53fac..9322d30671a 100644 --- a/src/third_party/wiredtiger/test/java/com/wiredtiger/test/WiredTigerSuite.java +++ b/src/third_party/wiredtiger/test/java/com/wiredtiger/test/WiredTigerSuite.java @@ -38,6 +38,7 @@ import org.junit.runners.Suite; ConfigTest.class, CursorTest.class, CursorTest02.class, + CursorTest03.class, ExceptionTest.class, PackTest.class, PackTest02.class, diff --git a/src/third_party/wiredtiger/test/manydbs/manydbs.c b/src/third_party/wiredtiger/test/manydbs/manydbs.c index 7e986d47af3..42020d6ce9a 100644 --- a/src/third_party/wiredtiger/test/manydbs/manydbs.c +++ b/src/third_party/wiredtiger/test/manydbs/manydbs.c @@ -32,7 +32,6 @@ #define HOME_BASE "WT_TEST" static char home[HOME_SIZE]; /* Base home directory */ static char hometmp[HOME_SIZE]; /* Each conn home directory */ -static const char *progname; /* Program name */ static const char * const uri = "table:main"; #define WTOPEN_CFG_COMMON \ @@ -129,10 +128,8 @@ main(int argc, char *argv[]) const char *working_dir, *wt_cfg; char cmd[128]; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); + dbs = MAX_DBS; working_dir = HOME_BASE; idle = false; @@ -171,7 +168,8 @@ main(int argc, char *argv[]) testutil_make_work_dir(home); __wt_random_init(&rnd); for (i = 0; i < dbs; ++i) { - snprintf(hometmp, HOME_SIZE, "%s/%s.%d", home, HOME_BASE, i); + testutil_check(__wt_snprintf( + hometmp, HOME_SIZE, "%s/%s.%d", home, HOME_BASE, i)); testutil_make_work_dir(hometmp); /* * Open each database. Rotate different configurations diff --git a/src/third_party/wiredtiger/test/mciproject.yml b/src/third_party/wiredtiger/test/mciproject.yml index eb74914eb46..6456475aa00 100644 --- a/src/third_party/wiredtiger/test/mciproject.yml +++ b/src/third_party/wiredtiger/test/mciproject.yml @@ -65,7 +65,7 @@ tasks: ./build_posix/reconf ${configure_env_vars|} ./configure --enable-diagnostic --enable-python --enable-zlib --enable-strict --enable-verbose ${make_command|make} ${smp_command|} 2>&1 - ${make_command|make} VERBOSE=1 check 2>&1 + TESTUTIL_ENABLE_LONG_TESTS=1 ${make_command|make} VERBOSE=1 check 2>&1 fi - command: archive.targz_pack params: diff --git a/src/third_party/wiredtiger/test/readonly/readonly.c b/src/third_party/wiredtiger/test/readonly/readonly.c index a4b79f5859f..66c7a0ca692 100644 --- a/src/third_party/wiredtiger/test/readonly/readonly.c +++ b/src/third_party/wiredtiger/test/readonly/readonly.c @@ -39,7 +39,6 @@ static char home_rd[HOME_SIZE + sizeof(HOME_RD_SUFFIX)]; #define HOME_RD2_SUFFIX ".RDNOLOCK" /* Read-only dir no lock file */ static char home_rd2[HOME_SIZE + sizeof(HOME_RD2_SUFFIX)]; -static const char *progname; /* Program name */ static const char *saved_argv0; /* Program command */ static const char * const uri = "table:main"; @@ -172,10 +171,8 @@ main(int argc, char *argv[]) char cmd[512]; uint8_t buf[MAX_VAL]; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); + /* * Needed unaltered for system command later. */ @@ -209,10 +206,12 @@ main(int argc, char *argv[]) * Set up all the directory names. */ testutil_work_dir_from_path(home, sizeof(home), working_dir); - (void)snprintf(home_wr, sizeof(home_wr), "%s%s", home, HOME_WR_SUFFIX); - (void)snprintf(home_rd, sizeof(home_rd), "%s%s", home, HOME_RD_SUFFIX); - (void)snprintf( - home_rd2, sizeof(home_rd2), "%s%s", home, HOME_RD2_SUFFIX); + testutil_check(__wt_snprintf( + home_wr, sizeof(home_wr), "%s%s", home, HOME_WR_SUFFIX)); + testutil_check(__wt_snprintf( + home_rd, sizeof(home_rd), "%s%s", home, HOME_RD_SUFFIX)); + testutil_check(__wt_snprintf( + home_rd2, sizeof(home_rd2), "%s%s", home, HOME_RD2_SUFFIX)); if (!child) { testutil_make_work_dir(home); testutil_make_work_dir(home_wr); @@ -271,22 +270,22 @@ main(int argc, char *argv[]) * Copy the database. Remove any lock file from one copy * and chmod the copies to be read-only permissions. */ - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "cp -rp %s/* %s; rm -f %s/WiredTiger.lock", - home, home_wr, home_wr); + home, home_wr, home_wr)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "cp -rp %s/* %s; chmod 0555 %s; chmod -R 0444 %s/*", - home, home_rd, home_rd, home_rd); + home, home_rd, home_rd, home_rd)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); - (void)snprintf(cmd, sizeof(cmd), + testutil_check(__wt_snprintf(cmd, sizeof(cmd), "cp -rp %s/* %s; rm -f %s/WiredTiger.lock; " "chmod 0555 %s; chmod -R 0444 %s/*", - home, home_rd2, home_rd2, home_rd2, home_rd2); + home, home_rd2, home_rd2, home_rd2, home_rd2)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); @@ -330,8 +329,8 @@ main(int argc, char *argv[]) * * The child will exit with success if its test passes. */ - (void)snprintf( - cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); if (WEXITSTATUS(status) != 0) @@ -340,8 +339,8 @@ main(int argc, char *argv[]) /* * Scenario 2. Run child with writable config. */ - (void)snprintf( - cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); if (WEXITSTATUS(status) != 0) @@ -361,8 +360,8 @@ main(int argc, char *argv[]) /* * Scenario 3. Child read-only. */ - (void)snprintf( - cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "%s -h %s -R", saved_argv0, working_dir)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); if (WEXITSTATUS(status) != 0) @@ -371,8 +370,8 @@ main(int argc, char *argv[]) /* * Scenario 4. Run child with writable config. */ - (void)snprintf( - cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "%s -h %s -W", saved_argv0, working_dir)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); if (WEXITSTATUS(status) != 0) @@ -393,11 +392,12 @@ main(int argc, char *argv[]) * We need to chmod the read-only databases back so that they can * be removed by scripts. */ - (void)snprintf(cmd, sizeof(cmd), "chmod 0777 %s %s", home_rd, home_rd2); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "chmod 0777 %s %s", home_rd, home_rd2)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); - (void)snprintf(cmd, sizeof(cmd), "chmod -R 0666 %s/* %s/*", - home_rd, home_rd2); + testutil_check(__wt_snprintf( + cmd, sizeof(cmd), "chmod -R 0666 %s/* %s/*", home_rd, home_rd2)); if ((status = system(cmd)) < 0) testutil_die(status, "system: %s", cmd); printf(" *** Readonly test successful ***\n"); diff --git a/src/third_party/wiredtiger/test/recovery/random-abort.c b/src/third_party/wiredtiger/test/recovery/random-abort.c index 21460646017..febe6530534 100644 --- a/src/third_party/wiredtiger/test/recovery/random-abort.c +++ b/src/third_party/wiredtiger/test/recovery/random-abort.c @@ -31,10 +31,14 @@ #include #include -static char home[512]; /* Program working dir */ -static const char *progname; /* Program name */ +static char home[1024]; /* Program working dir */ + +/* + * These two names for the URI and file system must be maintained in tandem. + */ static const char * const uri = "table:main"; -bool inmem; +static const char * const fs_main = "main.wt"; +static bool inmem; #define MAX_TH 12 #define MIN_TH 5 @@ -90,14 +94,16 @@ thread_run(void *arg) /* * The value is the name of the record file with our id appended. */ - snprintf(buf, sizeof(buf), RECORDS_FILE, td->id); + testutil_check(__wt_snprintf(buf, sizeof(buf), RECORDS_FILE, td->id)); /* * Set up a large value putting our id in it. Write it in there a * bunch of times, but the rest of the buffer can just be zero. */ - snprintf(lgbuf, sizeof(lgbuf), "th-%" PRIu32, td->id); + testutil_check(__wt_snprintf( + lgbuf, sizeof(lgbuf), "th-%" PRIu32, td->id)); for (i = 0; i < 128; i += strlen(lgbuf)) - snprintf(&large[i], lsize - i, "%s", lgbuf); + testutil_check(__wt_snprintf( + &large[i], lsize - i, "%s", lgbuf)); /* * Keep a separate file with the records we wrote for checking. */ @@ -120,7 +126,8 @@ thread_run(void *arg) * Write our portion of the key space until we're killed. */ for (i = td->start; ; ++i) { - snprintf(kname, sizeof(kname), "%" PRIu64, i); + testutil_check(__wt_snprintf( + kname, sizeof(kname), "%" PRIu64, i)); cursor->set_key(cursor, kname); /* * Every 30th record write a very large record that exceeds the @@ -211,6 +218,7 @@ extern char *__wt_optarg; int main(int argc, char *argv[]) { + struct stat sb; FILE *fp; WT_CONNECTION *conn; WT_CURSOR *cursor; @@ -222,12 +230,9 @@ main(int argc, char *argv[]) pid_t pid; bool fatal, rand_th, rand_time, verify_only; const char *working_dir; - char fname[64], kname[64]; + char fname[64], kname[64], statname[1024]; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); inmem = false; nth = MIN_TH; @@ -263,7 +268,7 @@ main(int argc, char *argv[]) if (argc != 0) usage(); - testutil_work_dir_from_path(home, 512, working_dir); + testutil_work_dir_from_path(home, sizeof(home), working_dir); /* * If the user wants to verify they need to tell us how many threads * there were so we can find the old record files. @@ -305,8 +310,16 @@ main(int argc, char *argv[]) /* parent */ /* * Sleep for the configured amount of time before killing - * the child. + * the child. Start the timeout from the time we notice that + * the table has been created. That allows the test to run + * correctly on really slow machines. Verify the process ID + * still exists in case the child aborts for some reason we + * don't stay in this loop forever. */ + testutil_check(__wt_snprintf( + statname, sizeof(statname), "%s/%s", home, fs_main)); + while (stat(statname, &sb) != 0 && kill(pid, 0) == 0) + sleep(1); sleep(timeout); /* @@ -339,12 +352,10 @@ main(int argc, char *argv[]) fatal = false; for (i = 0; i < nth; ++i) { middle = 0; - snprintf(fname, sizeof(fname), RECORDS_FILE, i); - if ((fp = fopen(fname, "r")) == NULL) { - fprintf(stderr, - "Failed to open %s. i %" PRIu32 "\n", fname, i); - testutil_die(errno, "fopen"); - } + testutil_check(__wt_snprintf( + fname, sizeof(fname), RECORDS_FILE, i)); + if ((fp = fopen(fname, "r")) == NULL) + testutil_die(errno, "fopen: %s", fname); /* * For every key in the saved file, verify that the key exists @@ -370,7 +381,8 @@ main(int argc, char *argv[]) fname, key, last_key); break; } - snprintf(kname, sizeof(kname), "%" PRIu64, key); + testutil_check(__wt_snprintf( + kname, sizeof(kname), "%" PRIu64, key)); cursor->set_key(cursor, kname); if ((ret = cursor->search(cursor)) != 0) { if (ret != WT_NOTFOUND) diff --git a/src/third_party/wiredtiger/test/recovery/truncated-log.c b/src/third_party/wiredtiger/test/recovery/truncated-log.c index c265263d44c..a127d8c1c63 100644 --- a/src/third_party/wiredtiger/test/recovery/truncated-log.c +++ b/src/third_party/wiredtiger/test/recovery/truncated-log.c @@ -30,13 +30,7 @@ #include -#ifdef _WIN32 -/* snprintf is not supported on <= VS2013 */ -#define snprintf _snprintf -#endif - -static char home[512]; /* Program working dir */ -static const char *progname; /* Program name */ +static char home[1024]; /* Program working dir */ static const char * const uri = "table:main"; #define RECORDS_FILE "records" @@ -138,7 +132,8 @@ usage(void) * Child process creates the database and table, and then writes data into * the table until it is killed by the parent. */ -static void fill_db(void)WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); +static void fill_db(void) + WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); static void fill_db(void) { @@ -194,9 +189,9 @@ fill_db(void) max_key = min_key * 2; first = true; for (i = 0; i < max_key; ++i) { - snprintf(k, sizeof(k), "key%03d", (int)i); - snprintf(v, sizeof(v), "value%0*d", - (int)(V_SIZE - strlen("value")), (int)i); + testutil_check(__wt_snprintf(k, sizeof(k), "key%03d", (int)i)); + testutil_check(__wt_snprintf(v, sizeof(v), "value%0*d", + (int)(V_SIZE - (strlen("value") + 1)), (int)i)); cursor->set_key(cursor, k); cursor->set_value(cursor, v); if ((ret = cursor->insert(cursor)) != 0) @@ -271,10 +266,7 @@ main(int argc, char *argv[]) pid_t pid; const char *working_dir; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); working_dir = "WT_TEST.truncated-log"; while ((ch = __wt_getopt(progname, argc, argv, "h:")) != EOF) @@ -290,7 +282,7 @@ main(int argc, char *argv[]) if (argc != 0) usage(); - testutil_work_dir_from_path(home, 512, working_dir); + testutil_work_dir_from_path(home, sizeof(home), working_dir); testutil_make_work_dir(home); /* diff --git a/src/third_party/wiredtiger/test/salvage/salvage.c b/src/third_party/wiredtiger/test/salvage/salvage.c index b8553bbd72d..83f9c6349bc 100644 --- a/src/third_party/wiredtiger/test/salvage/salvage.c +++ b/src/third_party/wiredtiger/test/salvage/salvage.c @@ -54,8 +54,6 @@ void run(int); void t(int, u_int, int); int usage(void); -static const char *progname; /* Program name */ - static FILE *res_fp; /* Results file */ static u_int page_type; /* File types */ static int value_unique; /* Values are unique */ @@ -70,10 +68,7 @@ main(int argc, char *argv[]) u_int ptype; int ch, r; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); r = 0; ptype = 0; @@ -445,7 +440,8 @@ run(int r) process(); - snprintf(buf, sizeof(buf), "cmp %s %s > /dev/null", DUMP, RSLT); + testutil_check(__wt_snprintf( + buf, sizeof(buf), "cmp %s %s > /dev/null", DUMP, RSLT)); if (system(buf)) { fprintf(stderr, "check failed, salvage results were incorrect\n"); @@ -490,28 +486,28 @@ build(int ikey, int ivalue, int cnt) switch (page_type) { case WT_PAGE_COL_FIX: - (void)snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "key_format=r,value_format=7t," "allocation_size=%d," "internal_page_max=%d,internal_item_max=%d," "leaf_page_max=%d,leaf_item_max=%d", - PSIZE, PSIZE, OSIZE, PSIZE, OSIZE); + PSIZE, PSIZE, OSIZE, PSIZE, OSIZE)); break; case WT_PAGE_COL_VAR: - (void)snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "key_format=r," "allocation_size=%d," "internal_page_max=%d,internal_item_max=%d," "leaf_page_max=%d,leaf_item_max=%d", - PSIZE, PSIZE, OSIZE, PSIZE, OSIZE); + PSIZE, PSIZE, OSIZE, PSIZE, OSIZE)); break; case WT_PAGE_ROW_LEAF: - (void)snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "key_format=u," "allocation_size=%d," "internal_page_max=%d,internal_item_max=%d," "leaf_page_max=%d,leaf_item_max=%d", - PSIZE, PSIZE, OSIZE, PSIZE, OSIZE); + PSIZE, PSIZE, OSIZE, PSIZE, OSIZE)); break; default: assert(0); @@ -525,7 +521,8 @@ build(int ikey, int ivalue, int cnt) case WT_PAGE_COL_VAR: break; case WT_PAGE_ROW_LEAF: - snprintf(kbuf, sizeof(kbuf), "%010d KEY------", ikey); + testutil_check(__wt_snprintf( + kbuf, sizeof(kbuf), "%010d KEY------", ikey)); key.data = kbuf; key.size = 20; cursor->set_key(cursor, &key); @@ -538,8 +535,8 @@ build(int ikey, int ivalue, int cnt) break; case WT_PAGE_COL_VAR: case WT_PAGE_ROW_LEAF: - snprintf(vbuf, sizeof(vbuf), - "%010d VALUE----", value_unique ? ivalue : 37); + testutil_check(__wt_snprintf(vbuf, sizeof(vbuf), + "%010d VALUE----", value_unique ? ivalue : 37)); value.data = vbuf; value.size = 20; cursor->set_value(cursor, &value); @@ -626,9 +623,9 @@ process(void) /* Salvage. */ config[0] = '\0'; if (verbose) - snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "error_prefix=\"%s\",verbose=[salvage,verify],", - progname); + progname)); strcat(config, "log=(enabled=false),"); CHECK(wiredtiger_open(NULL, NULL, config, &conn) == 0); diff --git a/src/third_party/wiredtiger/test/suite/run.py b/src/third_party/wiredtiger/test/suite/run.py index ba6d9f78503..97c58bfdccf 100644 --- a/src/third_party/wiredtiger/test/suite/run.py +++ b/src/third_party/wiredtiger/test/suite/run.py @@ -324,7 +324,8 @@ if __name__ == '__main__': # All global variables should be set before any test classes are loaded. # That way, verbose printing can be done at the class definition level. wttest.WiredTigerTestCase.globalSetup(preserve, timestamp, gdbSub, - verbose, dirarg, longtest) + verbose, wt_builddir, dirarg, + longtest) # Without any tests listed as arguments, do discovery if len(testargs) == 0: diff --git a/src/third_party/wiredtiger/test/suite/test_alter01.py b/src/third_party/wiredtiger/test/suite/test_alter01.py new file mode 100644 index 00000000000..dfdf6b7a17e --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_alter01.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_alter01.py +# Smoke-test the session alter operations. +class test_alter01(wttest.WiredTigerTestCase): + name = "alter01" + entries = 100 + # Settings for access_pattern_hint + types = [ + ('file', dict(uri='file:', use_cg=False, use_index=False)), + ('lsm', dict(uri='lsm:', use_cg=False, use_index=False)), + ('table-cg', dict(uri='table:', use_cg=True, use_index=False)), + ('table-index', dict(uri='table:', use_cg=False, use_index=True)), + ('table-simple', dict(uri='table:', use_cg=False, use_index=False)), + ] + hints = [ + ('default', dict(acreate='')), + ('none', dict(acreate='none')), + ('random', dict(acreate='random')), + ('sequential', dict(acreate='sequential')), + ] + access_alter=('', 'none', 'random', 'sequential') + # Settings for cache_resident + resid = [ + ('default', dict(ccreate='')), + ('false', dict(ccreate='false')), + ('true', dict(ccreate='true')), + ] + reopen = [ + ('no-reopen', dict(reopen=False)), + ('reopen', dict(reopen=True)), + ] + cache_alter=('', 'false', 'true') + scenarios = make_scenarios(types, hints, resid, reopen) + + def verify_metadata(self, metastr): + if metastr == '': + return + cursor = self.session.open_cursor('metadata:', None, None) + # + # Walk through all the metadata looking for the entries that are + # the file URIs for components of the table. + # + found = False + while True: + ret = cursor.next() + if ret != 0: + break + key = cursor.get_key() + check_meta = ((key.find("lsm:") != -1 or key.find("file:") != -1) \ + and key.find(self.name) != -1) + if check_meta: + value = cursor[key] + found = True + self.assertTrue(value.find(metastr) != -1) + cursor.close() + self.assertTrue(found == True) + + # Alter: Change the access pattern hint after creation + def test_alter01_access(self): + uri = self.uri + self.name + create_params = 'key_format=i,value_format=i,' + complex_params = '' + # + # If we're not explicitly setting the parameter, then don't + # modify create_params to test using the default. + # + if self.acreate != '': + access_param = 'access_pattern_hint=%s' % self.acreate + create_params += '%s,' % access_param + complex_params += '%s,' % access_param + else: + # NOTE: This is hard-coding the default value. If the default + # changes then this will fail and need to be fixed. + access_param = 'access_pattern_hint=none' + if self.ccreate != '': + cache_param = 'cache_resident=%s' % self.ccreate + create_params += '%s,' % cache_param + complex_params += '%s,' % cache_param + else: + # NOTE: This is hard-coding the default value. If the default + # changes then this will fail and need to be fixed. + cache_param = 'cache_resident=false' + + cgparam = '' + if self.use_cg or self.use_index: + cgparam = 'columns=(k,v),' + if self.use_cg: + cgparam += 'colgroups=(g0),' + + self.session.create(uri, create_params + cgparam) + # Add in column group or index settings. + if self.use_cg: + cgparam = 'columns=(v),' + suburi = 'colgroup:' + self.name + ':g0' + self.session.create(suburi, complex_params + cgparam) + if self.use_index: + suburi = 'index:' + self.name + ':i0' + self.session.create(suburi, complex_params + cgparam) + + # Put some data in table. + c = self.session.open_cursor(uri, None) + for k in range(self.entries): + c[k+1] = 1 + c.close() + + # Verify the string in the metadata + self.verify_metadata(access_param) + self.verify_metadata(cache_param) + + # Run through all combinations of the alter commands + # for all allowed settings. This tests having only one or + # the other set as well as having both set. It will also + # cover trying to change the setting to its current value. + for a in self.access_alter: + alter_param = '' + access_str = '' + if a != '': + access_str = 'access_pattern_hint=%s' % a + for c in self.cache_alter: + alter_param = '%s' % access_str + cache_str = '' + if c != '': + cache_str = 'cache_resident=%s' % c + alter_param += ',%s' % cache_str + if alter_param != '': + self.session.alter(uri, alter_param) + if self.reopen: + self.reopen_conn() + special = self.use_cg or self.use_index + if not special: + self.verify_metadata(access_str) + self.verify_metadata(cache_str) + else: + self.session.alter(suburi, alter_param) + self.verify_metadata(access_str) + self.verify_metadata(cache_str) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_async01.py b/src/third_party/wiredtiger/test/suite/test_async01.py index cbb3dad8de6..158c16a9381 100644 --- a/src/third_party/wiredtiger/test/suite/test_async01.py +++ b/src/third_party/wiredtiger/test/suite/test_async01.py @@ -132,7 +132,7 @@ class test_async01(wttest.WiredTigerTestCase, suite_subprocess): ]) # Enable async for this test. - def conn_config(self, dir): + def conn_config(self): return 'async=(enabled=true,ops_max=%s,' % self.async_ops + \ 'threads=%s)' % self.async_threads diff --git a/src/third_party/wiredtiger/test/suite/test_async02.py b/src/third_party/wiredtiger/test/suite/test_async02.py index 50652da6dfd..28435fe85b2 100644 --- a/src/third_party/wiredtiger/test/suite/test_async02.py +++ b/src/third_party/wiredtiger/test/suite/test_async02.py @@ -129,7 +129,7 @@ class test_async02(wttest.WiredTigerTestCase, suite_subprocess): ]) # Enable async for this test. - def conn_config(self, dir): + def conn_config(self): return 'async=(enabled=true,ops_max=%s,' % self.async_ops + \ 'threads=%s)' % self.async_threads diff --git a/src/third_party/wiredtiger/test/suite/test_backup03.py b/src/third_party/wiredtiger/test/suite/test_backup03.py index 73d05f0b0a1..c1ed3cc9e1a 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup03.py +++ b/src/third_party/wiredtiger/test/suite/test_backup03.py @@ -74,7 +74,7 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): ('backup_9', dict(big=3,list=[])), # Backup everything ] - scenarios = make_scenarios(list) + scenarios = make_scenarios(list, prune=3, prunelong=1000) # Create a large cache, otherwise this test runs quite slowly. conn_config = 'cache_size=1G' diff --git a/src/third_party/wiredtiger/test/suite/test_backup04.py b/src/third_party/wiredtiger/test/suite/test_backup04.py index 919649fed57..be52a5e1e97 100644 --- a/src/third_party/wiredtiger/test/suite/test_backup04.py +++ b/src/third_party/wiredtiger/test/suite/test_backup04.py @@ -60,7 +60,7 @@ class test_backup_target(wttest.WiredTigerTestCase, suite_subprocess): ]) # Create a large cache, otherwise this test runs quite slowly. - def conn_config(self, dir): + def conn_config(self): return 'cache_size=1G,log=(archive=false,enabled,file_max=%s)' % \ self.logmax diff --git a/src/third_party/wiredtiger/test/suite/test_bug011.py b/src/third_party/wiredtiger/test/suite/test_bug011.py index 29bb08ec2e5..5e0721b93f1 100644 --- a/src/third_party/wiredtiger/test/suite/test_bug011.py +++ b/src/third_party/wiredtiger/test/suite/test_bug011.py @@ -30,25 +30,28 @@ import random, wiredtiger, wttest from wtdataset import SimpleDataSet # test_bug011.py -# Eviction working on more files than there are hazard pointers. +# Eviction working on more trees than the eviction server can walk +# simultaneously. There is a builtin limit of 1000 trees, we open double +# that, which makes this a long-running test. class test_bug011(wttest.WiredTigerTestCase): """ Test having eviction working on more files than the number of allocated hazard pointers. """ table_name = 'test_bug011' - ntables = 50 + ntables = 2000 nrows = 10000 nops = 10000 # Add connection configuration for this test. - def conn_config(self, dir): - return 'cache_size=10MB,eviction_dirty_target=99,eviction_dirty_trigger=99,hazard_max=' + str(self.ntables / 2) + def conn_config(self): + return 'cache_size=1GB' + @wttest.longtest("Eviction copes with lots of files") def test_eviction(self): cursors = [] datasets = [] for i in range(0, self.ntables): - this_uri = 'table:%s-%03d' % (self.table_name, i) + this_uri = 'table:%s-%05d' % (self.table_name, i) ds = SimpleDataSet(self, this_uri, self.nrows, config='allocation_size=1KB,leaf_page_max=1KB') ds.populate() @@ -57,9 +60,9 @@ class test_bug011(wttest.WiredTigerTestCase): # Switch over to on-disk trees with multiple leaf pages self.reopen_conn() - # Make sure we have a cursor for the table so it stays in cache. + # Make sure we have a cursor for every table so it stays in cache. for i in range(0, self.ntables): - this_uri = 'table:%s-%03d' % (self.table_name, i) + this_uri = 'table:%s-%05d' % (self.table_name, i) cursors.append(self.session.open_cursor(this_uri, None)) # Make use of the cache. diff --git a/src/third_party/wiredtiger/test/suite/test_collator.py b/src/third_party/wiredtiger/test/suite/test_collator.py index 3fae4ff47cb..7ce135c8976 100644 --- a/src/third_party/wiredtiger/test/suite/test_collator.py +++ b/src/third_party/wiredtiger/test/suite/test_collator.py @@ -48,34 +48,10 @@ class test_collator(wttest.WiredTigerTestCase): nentries = 100 nindices = 4 - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name, libname) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + libname + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' - - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor'), - ('collators', 'revint', 'revint_collator')]) - connarg = 'create,error_prefix="{0}: ",{1}'.format( - self.shortid(), extarg) - conn = self.wiredtiger_open(dir, connarg) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('extractors', 'csv') + extlist.extension('collators', 'revint') def create_indices(self): # Create self.nindices index files, each with a column from the CSV diff --git a/src/third_party/wiredtiger/test/suite/test_compact02.py b/src/third_party/wiredtiger/test/suite/test_compact02.py index fe31aa1cbe1..803600eea14 100644 --- a/src/third_party/wiredtiger/test/suite/test_compact02.py +++ b/src/third_party/wiredtiger/test/suite/test_compact02.py @@ -38,7 +38,7 @@ from wtscenario import make_scenarios class test_compact02(wttest.WiredTigerTestCase): types = [ - ('file', dict(uri='file:test_compact02')), + ('table', dict(uri='table:test_compact02')), ] cacheSize = [ ('default', dict(cacheSize='')), @@ -69,8 +69,9 @@ class test_compact02(wttest.WiredTigerTestCase): # 1. Create a table with the data, alternating record size. # 2. Checkpoint and get stats on the table to confirm the size. # 3. Delete the half of the records with the larger record size. - # 4. Call compact. - # 5. Get stats on compacted table. + # 4. Checkpoint so compact finds something to work with. + # 5. Call compact. + # 6. Get stats on compacted table. # nrecords = 22000 bigvalue = "abcdefghi" * 1074 # 9*1074 == 9666 @@ -80,8 +81,11 @@ class test_compact02(wttest.WiredTigerTestCase): # Return the size of the file def getSize(self): + # To allow this to work on systems without ftruncate, + # get the portion of the file allocated, via 'statistics=(all)', + # not the physical file size, via 'statistics=(size)'. cstat = self.session.open_cursor( - 'statistics:' + self.uri, None, 'statistics=(size)') + 'statistics:' + self.uri, None, 'statistics=(all)') sz = cstat[stat.dsrc.block_size][2] cstat.close() return sz @@ -95,8 +99,9 @@ class test_compact02(wttest.WiredTigerTestCase): def ConnectionOpen(self, cacheSize): self.home = '.' conn_params = 'create,' + \ - cacheSize + ',error_prefix="%s: ",' % self.shortid() + \ - 'statistics=(fast),eviction_dirty_target=99,eviction_dirty_trigger=99' + cacheSize + ',error_prefix="%s",' % self.shortid() + \ + 'statistics=(all),' + \ + 'eviction_dirty_target=99,eviction_dirty_trigger=99' try: self.conn = wiredtiger.wiredtiger_open(self.home, conn_params) except wiredtiger.WiredTigerError as e: @@ -137,10 +142,13 @@ class test_compact02(wttest.WiredTigerTestCase): c.close() self.pr('Removed total ' + str((count * 9666) / mb) + 'MB') - # 4. Call compact. + # 4. Checkpoint + self.session.checkpoint() + + # 5. Call compact. self.session.compact(self.uri, None) - # 5. Get stats on compacted table. + # 6. Get stats on compacted table. sz = self.getSize() self.pr('After compact ' + str(sz / mb) + 'MB') diff --git a/src/third_party/wiredtiger/test/suite/test_compress01.py b/src/third_party/wiredtiger/test/suite/test_compress01.py index 606f7b63235..ef1064d294e 100644 --- a/src/third_party/wiredtiger/test/suite/test_compress01.py +++ b/src/third_party/wiredtiger/test/suite/test_compress01.py @@ -51,22 +51,10 @@ class test_compress01(wttest.WiredTigerTestCase): nrecords = 10000 bigvalue = "abcdefghij" * 1000 - # Load the compression extension, compression is enabled elsewhere. - def conn_config(self, dir): - return self.extensionArg(self.compress) - - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, name): - if name == None: - return '' - - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext/compressors') - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('compression extension "' + extfile + '" not built') - return ',extensions=["' + extfile + '"]' + # Load the compression extension, skip the test if missing + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('compressors', self.compress) # Create a table, add keys with both big and small values, then verify them. def test_compress(self): diff --git a/src/third_party/wiredtiger/test/suite/test_config03.py b/src/third_party/wiredtiger/test/suite/test_config03.py index 88ca6ae3f39..89038d71319 100644 --- a/src/third_party/wiredtiger/test/suite/test_config03.py +++ b/src/third_party/wiredtiger/test/suite/test_config03.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!usr/bin/env python # # Public Domain 2014-2016 MongoDB, Inc. # Public Domain 2008-2014 WiredTiger, Inc. @@ -48,8 +48,6 @@ class test_config03(test_base03.test_base03): eviction_trigger_scenarios = wtscenario.quick_scenarios( 's_eviction_trigger', [50, 90, 95, 99], None) - hazard_max_scenarios = wtscenario.quick_scenarios('s_hazard_max', - [15, 50, 500], [0.4, 0.8, 0.8]) multiprocess_scenarios = wtscenario.quick_scenarios('s_multiprocess', [True,False], [1.0,1.0]) session_max_scenarios = wtscenario.quick_scenarios('s_session_max', @@ -66,14 +64,14 @@ class test_config03(test_base03.test_base03): verbose_scenarios = wtscenario.quick_scenarios('s_verbose', [None], None) config_vars = [ 'cache_size', 'create', 'error_prefix', 'eviction_target', - 'eviction_trigger', 'hazard_max', 'multiprocess', - 'session_max', 'verbose' ] + 'eviction_trigger', 'multiprocess', 'session_max', + 'verbose' ] scenarios = wtscenario.make_scenarios( cache_size_scenarios, create_scenarios, error_prefix_scenarios, eviction_target_scenarios, eviction_trigger_scenarios, - hazard_max_scenarios, multiprocess_scenarios, session_max_scenarios, - transactional_scenarios, verbose_scenarios, prune=1000) + multiprocess_scenarios, session_max_scenarios, + transactional_scenarios, verbose_scenarios, prune=100, prunelong=1000) #wttest.WiredTigerTestCase.printVerbose(2, 'test_config03: running ' + \ # str(len(scenarios)) + ' of ' + \ diff --git a/src/third_party/wiredtiger/test/suite/test_config04.py b/src/third_party/wiredtiger/test/suite/test_config04.py index 204aa7e27d5..db8a5f4a16a 100644 --- a/src/third_party/wiredtiger/test/suite/test_config04.py +++ b/src/third_party/wiredtiger/test/suite/test_config04.py @@ -154,10 +154,6 @@ class test_config04(wttest.WiredTigerTestCase): 'eviction_trigger=86'), "/eviction target must be lower than the eviction trigger/") - def test_hazard_max(self): - # Note: There isn't any direct way to know that this was set. - self.common_test('hazard_max=50') - def test_invalid_config(self): msg = '/Unbalanced brackets/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, diff --git a/src/third_party/wiredtiger/test/suite/test_cursor01.py b/src/third_party/wiredtiger/test/suite/test_cursor01.py index 8c66042eec0..2a33b15dccc 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor01.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor01.py @@ -99,6 +99,7 @@ class test_cursor01(wttest.WiredTigerTestCase): self.pr('creating cursor') cursor = self.session.open_cursor(tablearg, None, None) self.assertCursorHasNoKeyValue(cursor) + self.assertEqual(cursor.uri, tablearg) for i in range(0, self.nentries): cursor[self.genkey(i)] = self.genvalue(i) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor07.py b/src/third_party/wiredtiger/test/suite/test_cursor07.py index d6078183fc1..19db718fd11 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor07.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor07.py @@ -49,7 +49,7 @@ class test_cursor07(wttest.WiredTigerTestCase, suite_subprocess): ('reopen', dict(reopen=True)) ]) # Enable logging for this test. - def conn_config(self, dir): + def conn_config(self): return 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ 'transaction_sync="(method=dsync,enabled)"' diff --git a/src/third_party/wiredtiger/test/suite/test_cursor08.py b/src/third_party/wiredtiger/test/suite/test_cursor08.py index 3f8f50defa7..cc76f528aa9 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor08.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor08.py @@ -54,24 +54,14 @@ class test_cursor08(wttest.WiredTigerTestCase, suite_subprocess): ] scenarios = make_scenarios(reopens, compress) # Load the compression extension, and enable it for logging. - def conn_config(self, dir): + def conn_config(self): return 'log=(archive=false,enabled,file_max=%s,' % self.logmax + \ 'compressor=%s),' % self.compress + \ - 'transaction_sync="(method=dsync,enabled)",' + \ - self.extensionArg(self.compress) + 'transaction_sync="(method=dsync,enabled)"' - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, name): - if name == None or name == 'none': - return '' - - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext/compressors') - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('compression extension "' + extfile + '" not built') - return ',extensions=["' + extfile + '"]' + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('compressors', self.compress) def test_log_cursor(self): # print "Creating %s with config '%s'" % (self.uri, self.create_params) diff --git a/src/third_party/wiredtiger/test/suite/test_cursor10.py b/src/third_party/wiredtiger/test/suite/test_cursor10.py new file mode 100644 index 00000000000..6cabfde9f1f --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_cursor10.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtscenario import make_scenarios + +# test_cursor10.py +# Cursors with projections. +class test_cursor10(wttest.WiredTigerTestCase): + """ + Test cursor search and search_near + """ + table_name1 = 'test_cursor10' + nentries = 20 + + scenarios = make_scenarios([ + ('row', dict(key_format='S', uri='table')), + ('col', dict(key_format='r', uri='table')) + ]) + + def genkey(self, i): + if self.key_format == 'S': + return 'key' + str(i).zfill(5) # return key00001, key00002, etc. + else: + return long(i+1) + + def genvalue(self, i): + return [ 'v0:' + str(i), i+1, 'v2' + str(i+2), i+3 ] + + def extractkey(self, k): + if self.key_format == 'S': + return int(k[3:]) + else: + return long(k-1) + + def test_projection(self): + """ + Create entries, and read back in a regular and projected cursor + """ + tablearg = self.uri + ":" + self.table_name1 + create_args = 'columns=(k,v0,v1,v2,v3),value_format=SiSi,key_format=' \ + + self.key_format + self.session.create(tablearg, create_args) + + cursor = self.session.open_cursor(tablearg, None, None) + for i in range(0, self.nentries): + cursor.set_key(self.genkey(i)) + values = self.genvalue(i) + cursor.set_value(*values) + cursor.insert() + cursor.close() + cursor = self.session.open_cursor(tablearg, None, None) + count = 0 + for k,v0,v1,v2,v3 in cursor: + i = self.extractkey(k) + self.assertEqual(self.genkey(i), k) + self.assertEqual(self.genvalue(i), [v0,v1,v2,v3]) + count += 1 + self.assertEqual(count, self.nentries) + cursor.close() + cursor = self.session.open_cursor(tablearg + '(v3,v2,v1,v0,k)',\ + None, None) + count = 0 + for k1,v3,v2,v1,v0,k2 in cursor: + self.assertEqual(k1, k2) + i = self.extractkey(k1) + self.assertEqual(self.genkey(i), k1) + self.assertEqual(self.genvalue(i), [v0,v1,v2,v3]) + count += 1 + self.assertEqual(count, self.nentries) + cursor.close() + + def test_index_projection(self): + """ + Create entries, and read back in an index cursor with a projection + """ + tablearg = self.uri + ":" + self.table_name1 + indexarg = 'index:' + self.table_name1 + ':index1' + create_args = 'columns=(k,v0,v1,v2,v3),value_format=SiSi,key_format=' \ + + self.key_format + self.session.create(tablearg, create_args) + self.session.create(indexarg, 'columns=(v0,v2,v1,v3)') + cursor = self.session.open_cursor(tablearg, None, None) + for i in range(0, self.nentries): + cursor.set_key(self.genkey(i)) + values = self.genvalue(i) + cursor.set_value(*values) + cursor.insert() + cursor.close() + cursor = self.session.open_cursor(tablearg + '(v3,v2,v1,v0,k)',\ + None, None) + count = 0 + for k1,v3,v2,v1,v0,k2 in cursor: + self.assertEqual(k1, k2) + i = self.extractkey(k1) + self.assertEqual(self.genkey(i), k1) + self.assertEqual(self.genvalue(i), [v0,v1,v2,v3]) + count += 1 + self.assertEqual(count, self.nentries) + cursor.close() +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor11.py b/src/third_party/wiredtiger/test/suite/test_cursor11.py new file mode 100644 index 00000000000..e159ec499e6 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_cursor11.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet, SimpleIndexDataSet +from wtdataset import SimpleLSMDataSet, ComplexDataSet, ComplexLSMDataSet +from wtscenario import make_scenarios + +# test_cursor11.py +# WT_CURSOR position tests: remove (if not already positioned), and insert +# leave the cursor without position or information. +class test_cursor11(wttest.WiredTigerTestCase): + + keyfmt = [ + ('integer', dict(keyfmt='i')), + ('recno', dict(keyfmt='r')), + ('string', dict(keyfmt='S')), + ] + types = [ + ('file', dict(uri='file', ds=SimpleDataSet)), + ('lsm', dict(uri='lsm', ds=SimpleDataSet)), + ('table-complex', dict(uri='table', ds=ComplexDataSet)), + ('table-complex-lsm', dict(uri='table', ds=ComplexLSMDataSet)), + ('table-index', dict(uri='table', ds=SimpleIndexDataSet)), + ('table-simple', dict(uri='table', ds=SimpleDataSet)), + ('table-simple-lsm', dict(uri='table', ds=SimpleLSMDataSet)), + ] + scenarios = make_scenarios(types, keyfmt) + + def skip(self): + return self.keyfmt == 'r' and \ + (self.ds.is_lsm() or self.uri == 'lsm') + + # Do a remove using the cursor after setting a position, and confirm + # the key and position remain set but no value. + def test_cursor_remove_with_position(self): + if self.skip(): + return + + # Build an object. + uri = self.uri + ':test_cursor11' + ds = self.ds(self, uri, 50, key_format=self.keyfmt) + ds.populate() + s = self.conn.open_session() + c = s.open_cursor(uri, None) + + c.set_key(ds.key(25)) + self.assertEquals(c.search(), 0) + self.assertEquals(c.next(), 0) + self.assertEquals(c.get_key(), ds.key(26)) + c.remove() + self.assertEquals(c.get_key(), ds.key(26)) + msg = '/requires value be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_value, msg) + self.assertEquals(c.next(), 0) + self.assertEquals(c.get_key(), ds.key(27)) + + # Do a remove using the cursor without setting a position, and confirm + # no key, value or position remains. + def test_cursor_remove_without_position(self): + if self.skip(): + return + + # Build an object. + uri = self.uri + ':test_cursor11' + ds = self.ds(self, uri, 50, key_format=self.keyfmt) + ds.populate() + s = self.conn.open_session() + c = s.open_cursor(uri, None) + + c.set_key(ds.key(25)) + c.remove() + msg = '/requires key be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_key, msg) + msg = '/requires value be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_value, msg) + self.assertEquals(c.next(), 0) + self.assertEquals(c.get_key(), ds.key(1)) + + # Do a remove using the key after also setting a position, and confirm + # no key, value or position remains. + def test_cursor_remove_with_key_and_position(self): + if self.skip(): + return + + # Build an object. + uri = self.uri + ':test_cursor11' + ds = self.ds(self, uri, 50, key_format=self.keyfmt) + ds.populate() + s = self.conn.open_session() + c = s.open_cursor(uri, None) + + c.set_key(ds.key(25)) + self.assertEquals(c.search(), 0) + c.set_key(ds.key(25)) + c.remove() + msg = '/requires key be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_key, msg) + msg = '/requires value be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_value, msg) + self.assertEquals(c.next(), 0) + self.assertEquals(c.get_key(), ds.key(1)) + + # Do an insert and confirm no key, value or position remains. + def test_cursor_insert(self): + if self.skip(): + return + + # Build an object. + uri = self.uri + ':test_cursor11' + ds = self.ds(self, uri, 50, key_format=self.keyfmt) + ds.populate() + s = self.conn.open_session() + c = s.open_cursor(uri, None) + + c.set_key(ds.key(25)) + c.set_value(ds.value(300)) + c.insert() + msg = '/requires key be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_key, msg) + msg = '/requires value be set/' + self.assertRaisesWithMessage( + wiredtiger.WiredTigerError, c.get_value, msg) + self.assertEquals(c.next(), 0) + self.assertEquals(c.get_key(), ds.key(1)) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_cursor_random.py b/src/third_party/wiredtiger/test/suite/test_cursor_random.py index 3bda6dc9946..ee0f85a29ee 100644 --- a/src/third_party/wiredtiger/test/suite/test_cursor_random.py +++ b/src/third_party/wiredtiger/test/suite/test_cursor_random.py @@ -71,6 +71,15 @@ class test_cursor_random(wttest.WiredTigerTestCase): self.assertEquals(cursor.reset(), 0) cursor.close() + # Check that next_random fails with an empty tree, repeatedly. + def test_cursor_random_empty(self): + uri = self.type + self.session.create(uri, 'key_format=S,value_format=S') + cursor = self.session.open_cursor(uri, None, self.config) + for i in range(1,5): + self.assertTrue(cursor.next(), wiredtiger.WT_NOTFOUND) + cursor.close + # Check that next_random works with a single value, repeatedly. def test_cursor_random_single_record(self): uri = self.type @@ -127,6 +136,46 @@ class test_cursor_random(wttest.WiredTigerTestCase): def test_cursor_random_multiple_page_records(self): self.cursor_random_multiple_page_records(0) + # Check that next_random fails in the presence of a set of values, some of + # which are deleted. + def test_cursor_random_deleted_partial(self): + uri = self.type + ds = self.dataset(self, uri, 10000, + config='allocation_size=512,leaf_page_max=512') + ds.populate() + + # Close the connection so everything is forced to disk. + self.reopen_conn() + + start = self.session.open_cursor(uri, None) + start.set_key(ds.key(10)) + end = self.session.open_cursor(uri, None) + end.set_key(ds.key(10000-10)) + self.session.truncate(None, start, end, None) + self.assertEqual(start.close(), 0) + self.assertEqual(end.close(), 0) + + cursor = self.session.open_cursor(uri, None, self.config) + for i in range(1,10): + self.assertEqual(cursor.next(), 0) + + # Check that next_random fails in the presence of a set of values, all of + # which are deleted. + def test_cursor_random_deleted_all(self): + uri = self.type + ds = self.dataset(self, uri, 10000, + config='allocation_size=512,leaf_page_max=512') + ds.populate() + + # Close the connection so everything is forced to disk. + self.reopen_conn() + + self.session.truncate(uri, None, None, None) + + cursor = self.session.open_cursor(uri, None, self.config) + for i in range(1,10): + self.assertTrue(cursor.next(), wiredtiger.WT_NOTFOUND) + # Check that opening a random cursor on column-store returns not-supported. class test_cursor_random_column(wttest.WiredTigerTestCase): scenarios = make_scenarios([ diff --git a/src/third_party/wiredtiger/test/suite/test_dump.py b/src/third_party/wiredtiger/test/suite/test_dump.py index f6a83c32489..3127c7aef00 100644 --- a/src/third_party/wiredtiger/test/suite/test_dump.py +++ b/src/third_party/wiredtiger/test/suite/test_dump.py @@ -32,7 +32,7 @@ import wiredtiger, wttest from suite_subprocess import suite_subprocess from wtscenario import make_scenarios from wtdataset import SimpleDataSet, SimpleIndexDataSet, SimpleLSMDataSet, \ - ComplexDataSet, ComplexLSMDataSet + ComplexDataSet, ComplexLSMDataSet, ProjectionDataSet, ProjectionIndexDataSet # test_dump.py # Utilities: wt dump @@ -62,6 +62,10 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): ('table-simple-lsm', dict(uri='table:', dataset=SimpleLSMDataSet)), ('table-complex', dict(uri='table:', dataset=ComplexDataSet)), ('table-complex-lsm', dict(uri='table:', dataset=ComplexLSMDataSet)), + ('table-simple-proj', dict(uri='table:', + dataset=ProjectionDataSet, projection=True)), + ('table-index-proj', dict(uri='table:', + dataset=ProjectionIndexDataSet, projection=True)), ] scenarios = make_scenarios(types, keyfmt, dumpfmt) @@ -158,5 +162,53 @@ class test_dump(wttest.WiredTigerTestCase, suite_subprocess): pop = self.dataset(self, uri2, self.nentries, key_format=self.keyfmt) pop.check() +# test_dump_projection +# Utilities: wt dump +# Test the dump utility with projections +class test_dump_projection(wttest.WiredTigerTestCase, suite_subprocess): + dir = 'dump.dir' # Backup directory name + + name = 'test_dump' + nentries = 2500 + uri = 'table:' + + # Dump, re-load and do a content comparison. + def test_dump(self): + + # Create the object. + uri = self.uri + self.name + pop = ProjectionDataSet(self, uri, self.nentries, key_format='S') + pop.populate() + + # Check some cases with invalid projections. + self.runWt(['dump', '-x', uri + '('], \ + outfilename='bad1.out', errfilename='err1.out', failure=True) + self.check_non_empty_file('err1.out') + self.runWt(['dump', '-x', uri + '(xx)'], \ + outfilename='bad2.out', errfilename='err2.out', failure=True) + self.check_non_empty_file('err2.out') + self.runWt(['dump', '-x', uri + pop.projection[:-1]], \ + outfilename='bad3.out', errfilename='err3.out', failure=True) + self.check_non_empty_file('err3.out') + + # Dump the object with a valid projection. + self.runWt(['dump', '-x', uri + pop.projection], outfilename='dump.out') + + # Re-load the object in a new home. + os.mkdir(self.dir) + self.runWt(['-h', self.dir, 'load', '-f', 'dump.out']) + + # Check the database contents. + self.runWt(['list'], outfilename='list.out') + self.runWt(['-h', self.dir, 'list'], outfilename='list.out.new') + s1 = set(open('list.out').read().split()) + s2 = set(open('list.out.new').read().split()) + self.assertEqual(not s1.symmetric_difference(s2), True) + + # Check the object's contents. + self.reopen_conn(self.dir) + pop_reload = ProjectionDataSet(self, uri, self.nentries, key_format='S') + pop_reload.check() + if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt01.py b/src/third_party/wiredtiger/test/suite/test_encrypt01.py index 746c9d13e96..317bed93246 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt01.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt01.py @@ -66,41 +66,20 @@ class test_encrypt01(wttest.WiredTigerTestCase): nrecords = 5000 bigvalue = "abcdefghij" * 1001 # len(bigvalue) = 10010 - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('encryptors', self.sys_encrypt) + extlist.extension('encryptors', self.file_encrypt) + extlist.extension('compressors', self.block_compress) + extlist.extension('compressors', self.log_compress) + + def conn_config(self): encarg = 'encryption=(name={0}{1}),'.format( self.sys_encrypt, self.sys_encrypt_args) comparg = '' if self.log_compress != None: comparg='log=(compressor={0}),'.format(self.log_compress) - extarg = self.extensionArg([('encryptors', self.sys_encrypt), - ('encryptors', self.file_encrypt), - ('compressors', self.block_compress), - ('compressors', self.log_compress)]) - conn = self.wiredtiger_open(dir, - 'create,error_prefix="{0}: ",{1}{2}{3}'.format( - self.shortid(), encarg, comparg, extarg)) - self.pr(`conn`) - return conn - - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' + return encarg + comparg # Create a table, add keys with both big and small values, then verify them. def test_encrypt(self): diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt02.py b/src/third_party/wiredtiger/test/suite/test_encrypt02.py index 648686274c4..d950be067e2 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt02.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt02.py @@ -39,51 +39,27 @@ from wtscenario import make_scenarios class test_encrypt02(wttest.WiredTigerTestCase, suite_subprocess): uri = 'file:test_encrypt02' encrypt_type = [ - ('noarg', dict( encrypt='rotn', encrypt_args='name=rotn', - secret_arg=None)), - ('keyid', dict( encrypt='rotn', encrypt_args='name=rotn,keyid=11', - secret_arg=None)), - ('pass', dict( encrypt='rotn', encrypt_args='name=rotn', - secret_arg='ABC')), - ('keyid-pass', dict( encrypt='rotn', encrypt_args='name=rotn,keyid=11', - secret_arg='ABC')), + ('noarg', dict( encrypt_args='name=rotn', secret_arg=None)), + ('keyid', dict( encrypt_args='name=rotn,keyid=11', secret_arg=None)), + ('pass', dict( encrypt_args='name=rotn', secret_arg='ABC')), + ('keyid-pass', dict( + encrypt_args='name=rotn,keyid=11', secret_arg='ABC')), ] scenarios = make_scenarios(encrypt_type) + def conn_extensions(self, extlist): + # Load the compression extension, skip the test if missing + extlist.skip_if_missing = True + extlist.extension('encryptors', 'rotn') + nrecords = 5000 bigvalue = "abcdefghij" * 1001 # len(bigvalue) = 10010 - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' - - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): + def conn_config(self): secretarg = '' if self.secret_arg != None: secretarg = ',secretkey=' + self.secret_arg - encarg = 'encryption=({0}{1})'.format(self.encrypt_args, secretarg) - extarg = self.extensionArg([('encryptors', self.encrypt)]) - connarg = 'create,error_prefix="{0}: ",{1},{2}'.format( - self.shortid(), encarg, extarg) - conn = self.wiredtiger_open(dir, connarg) - self.pr(`conn`) - return conn + return 'encryption=({0}{1})'.format(self.encrypt_args, secretarg) # Create a table, add keys with both big and small values, then verify them. def test_pass(self): diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt03.py b/src/third_party/wiredtiger/test/suite/test_encrypt03.py index cf459190637..302572bd044 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt03.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt03.py @@ -50,37 +50,14 @@ class test_encrypt03(wttest.WiredTigerTestCase): ] scenarios = make_scenarios(types, encrypt) - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - encarg = 'encryption=(name={0}{1}),'.format( - self.sys_encrypt, self.sys_encrypt_args) - extarg = self.extensionArg([('encryptors', self.sys_encrypt), - ('encryptors', self.file_encrypt)]) - self.pr('encarg = ' + encarg + ' extarg = ' + extarg) - conn = self.wiredtiger_open(dir, - 'create,error_prefix="{0}: ",{1}{2}'.format( - self.shortid(), encarg, extarg)) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('encryptors', self.sys_encrypt) + extlist.extension('encryptors', self.file_encrypt) - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' + def conn_config(self): + return 'encryption=(name={0}{1}),'.format( + self.sys_encrypt, self.sys_encrypt_args) # Create a table with encryption values that are in error. def test_encrypt(self): diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt04.py b/src/third_party/wiredtiger/test/suite/test_encrypt04.py index a244cf97961..19c0b85d427 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt04.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt04.py @@ -77,9 +77,16 @@ class test_encrypt04(wttest.WiredTigerTestCase, suite_subprocess): wttest.WiredTigerTestCase.__init__(self, *args, **kwargs) self.part = 1 + def conn_extensions(self, extlist): + extarg = None + if self.expect_forceerror: + extarg='(config=\"rotn_force_error=true\")' + extlist.skip_if_missing = True + extlist.extension('encryptors', self.name, extarg) + # Override WiredTigerTestCase, we have extensions. def setUpConnectionOpen(self, dir): - forceerror = None + self.expect_forceerror = False if self.part == 1: self.name = self.name1 self.keyid = self.keyid1 @@ -93,21 +100,20 @@ class test_encrypt04(wttest.WiredTigerTestCase, suite_subprocess): self.fileinclear = self.fileinclear2 if \ hasattr(self, 'fileinclear2') else False if hasattr(self, 'forceerror1') and hasattr(self, 'forceerror2'): - forceerror = "rotn_force_error=true" - self.expect_forceerror = forceerror != None + self.expect_forceerror = True self.got_forceerror = False encarg = 'encryption=(name={0},keyid={1},secretkey={2}),'.format( self.name, self.keyid, self.secretkey) - # If forceerror is set for this test, add a config arg to - # the extension string. That signals rotn to return a (-1000) - # error code, which we'll detect here. - extarg = self.extensionArg([('encryptors', self.name, forceerror)]) + # If forceerror is set for this test, conn_extensions adds a + # config arg to the extension string. That signals rotn to + # return a (-1000) error code, which we'll detect here. + extarg = self.extensionsConfig() self.pr('encarg = ' + encarg + ' extarg = ' + extarg) completed = False try: conn = self.wiredtiger_open(dir, - 'create,error_prefix="{0}: ",{1}{2}'.format( + 'create,error_prefix="{0}",{1}{2}'.format( self.shortid(), encarg, extarg)) except (BaseException) as err: # Capture the recognizable error created by rotn @@ -135,29 +141,6 @@ class test_encrypt04(wttest.WiredTigerTestCase, suite_subprocess): self.assertEqual(cursor.search(), 0) self.assertEquals(cursor.get_value(), val) - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name, extarg) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - extfile = '"' + extfile + '"' - if not extfile in extfiles: - s = extfile - if extarg != None: - s += "=(config=\"" + extarg + "\")" - extfiles.append(s) - if len(extfiles) == 0: - return '' - else: - return ',extensions=[' + ','.join(extfiles) + ']' - # Evaluate expression, which either must succeed (if expect_okay) # or must fail (if !expect_okay). def check_okay(self, expect_okay, expr): diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt05.py b/src/third_party/wiredtiger/test/suite/test_encrypt05.py index 19a3522b3d5..d8862321821 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt05.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt05.py @@ -49,41 +49,20 @@ class test_encrypt05(wttest.WiredTigerTestCase): nrecords = 500 bigvalue = 'a' * 500 # we use values that will definitely give compression - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('encryptors', self.sys_encrypt) + extlist.extension('encryptors', self.file_encrypt) + extlist.extension('compressors', self.block_compress) + extlist.extension('compressors', self.log_compress) + + def conn_config(self): encarg = 'encryption=(name={0}{1}),'.format( self.sys_encrypt, self.sys_encrypt_args) comparg = '' if self.log_compress != None: comparg='log=(compressor={0}),'.format(self.log_compress) - extarg = self.extensionArg([('encryptors', self.sys_encrypt), - ('encryptors', self.file_encrypt), - ('compressors', self.block_compress), - ('compressors', self.log_compress)]) - conn = self.wiredtiger_open(dir, - 'create,error_prefix="{0}: ",{1}{2}{3}'.format( - self.shortid(), encarg, comparg, extarg)) - self.pr(`conn`) - return conn - - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' + return encarg + comparg def getvalue(self, r, n): if n < len(self.bigvalue): diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt06.py b/src/third_party/wiredtiger/test/suite/test_encrypt06.py index 893c4ba3095..72718e53b2b 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt06.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt06.py @@ -89,38 +89,15 @@ class test_encrypt06(wttest.WiredTigerTestCase): scenarios = make_scenarios(encrypt, storagetype) nrecords = 1000 - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - encarg = 'encryption=(name={0}{1}),'.format( + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('encryptors', self.sys_encrypt) + extlist.extension('encryptors', self.file0_encrypt) + extlist.extension('encryptors', self.file1_encrypt) + + def conn_config(self): + return 'encryption=(name={0}{1}),'.format( self.sys_encrypt, self.sys_encrypt_args) - comparg = '' - extarg = self.extensionArg([('encryptors', self.sys_encrypt), - ('encryptors', self.file0_encrypt), - ('encryptors', self.file1_encrypt)]) - self.open_params = 'create,error_prefix="{0}: ",{1}{2}{3}'.format( - self.shortid(), encarg, comparg, extarg) - conn = self.wiredtiger_open(dir, self.open_params) - self.pr(`conn`) - return conn - - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' def encrypt_file_params(self, name, args): if name == None: diff --git a/src/third_party/wiredtiger/test/suite/test_encrypt07.py b/src/third_party/wiredtiger/test/suite/test_encrypt07.py index 97ab1987d4f..81c9f1a49ea 100644 --- a/src/third_party/wiredtiger/test/suite/test_encrypt07.py +++ b/src/third_party/wiredtiger/test/suite/test_encrypt07.py @@ -44,35 +44,14 @@ class test_encrypt07(test_salvage.test_salvage): nrecords = 5000 bigvalue = "abcdefghij" * 1007 # len(bigvalue) = 10070 - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - encarg = 'encryption=(name={0}{1}),'.format( - self.sys_encrypt, self.sys_encrypt_args) - extarg = self.extensionArg([('encryptors', self.sys_encrypt)]) - conn = self.wiredtiger_open(dir, - 'create,error_prefix="{0}: ",{1}{2}'.format( - self.shortid(), encarg, extarg)) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + # Load the compression extension, skip the test if missing + extlist.skip_if_missing = True + extlist.extension('encryptors', self.sys_encrypt) - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' + def conn_config(self): + return 'encryption=(name={0}{1}),'.format( + self.sys_encrypt, self.sys_encrypt_args) def rot13(self, s): return codecs.encode(s, 'rot_13') diff --git a/src/third_party/wiredtiger/test/suite/test_hazard.py b/src/third_party/wiredtiger/test/suite/test_hazard.py new file mode 100644 index 00000000000..f2891fce526 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_hazard.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_hazard.py +# Hazard pointer tests. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet + +# Regression tests. +class test_hazard(wttest.WiredTigerTestCase): + + # Allocate a large number of hazard pointers in a session, forcing the + # hazard pointer array to repeatedly grow. + def test_hazard(self): + uri = "table:hazard" + ds = SimpleDataSet(self, uri, 1000) + ds.populate() + + # Open 10,000 cursors and pin a page to set a hazard pointer. + cursors = [] + for i in range(0, 10000): + c = self.session.open_cursor(uri, None) + c.set_key(ds.key(10)) + c.search() + cursors.append(c) + + # Close the cursors, clearing the hazard pointer. + for c in cursors: + c.close() + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_index02.py b/src/third_party/wiredtiger/test/suite/test_index02.py index 9f39df003b1..4f424e5d3d2 100644 --- a/src/third_party/wiredtiger/test/suite/test_index02.py +++ b/src/third_party/wiredtiger/test/suite/test_index02.py @@ -27,20 +27,26 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest +from wtscenario import make_scenarios # test_index02.py # test search_near in indices class test_index02(wttest.WiredTigerTestCase): '''Test search_near in indices''' + scenarios = make_scenarios([ + ('index', dict(indexconfig='columns=(v)', ncol=1)), + ('index-with-key', dict(indexconfig='columns=(v,k)', ncol=2)), + ]) + basename = 'test_index02' tablename = 'table:' + basename indexname = 'index:' + basename + ":inverse" - def test_search_near(self): - '''Create a table, look for a nonexistent key''' + def test_search_near_exists(self): + '''Create a table, look for an existing key''' self.session.create(self.tablename, 'key_format=r,value_format=Q,columns=(k,v)') - self.session.create(self.indexname, 'columns=(v)') + self.session.create(self.indexname, self.indexconfig) cur = self.session.open_cursor(self.tablename, None, "append") cur.set_value(1) cur.insert() @@ -52,17 +58,60 @@ class test_index02(wttest.WiredTigerTestCase): cur.insert() cur.set_value(10) cur.insert() + cur.close() + + # Retry after reopening + for runs in xrange(2): + # search near should find a match + cur = self.session.open_cursor(self.indexname, None, None) + if self.ncol == 1: + cur.set_key(5) + else: + cur.set_key(5, 3) + self.assertEqual(cur.search_near(), 0) + + # Retry after reopening + self.reopen_conn() + + def test_search_near_between(self): + '''Create a table, look for a non-existing key''' + self.session.create(self.tablename, 'key_format=i,value_format=i,columns=(k,v)') + self.session.create(self.indexname, self.indexconfig) + cur = self.session.open_cursor(self.tablename) + for k in xrange(3): + cur[k] = 5 * k + 10 + cur.close() + + search_keys = [ 1, 11, 15, 19, 21 ] # search near should find a match - cur2 = self.session.open_cursor(self.indexname, None, None) - cur2.set_key(5) - self.assertEqual(cur2.search_near(), 0) + for runs in xrange(2): + cur = self.session.open_cursor(self.indexname, None, None) + for k in search_keys: + if self.ncol == 1: + cur.set_key(k) + else: + cur.set_key(k, 1) # [15,1] will completely match + exact = cur.search_near() + if self.ncol == 1: + found_key = cur.get_key() + else: + [ found_key, index ] = cur.get_key() + self.pr("search_near for " + str(k) + " found " + str(found_key) + " with exact " + str(exact)) + self.assertEqual(exact, cmp(found_key, k), "for key " + str(k)) + self.reopen_conn() - # Retry after reopening - self.reopen_conn() - cur3 = self.session.open_cursor(self.indexname, None, None) - cur3.set_key(5) - self.assertEqual(cur3.search_near(), 0) + def test_search_near_empty(self): + '''Create an empty table, look for a key''' + self.session.create(self.tablename, 'key_format=i,value_format=i,columns=(k,v)') + self.session.create(self.indexname, self.indexconfig) + + cur = self.session.open_cursor(self.indexname, None, None) + if self.ncol == 1: + cur.set_key(3) + else: + cur.set_key(3, 1) + self.assertEqual(cur.search_near(), wiredtiger.WT_NOTFOUND) if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_inmem01.py b/src/third_party/wiredtiger/test/suite/test_inmem01.py index 1af43bbd9d9..388485db29b 100644 --- a/src/third_party/wiredtiger/test/suite/test_inmem01.py +++ b/src/third_party/wiredtiger/test/suite/test_inmem01.py @@ -59,7 +59,7 @@ class test_inmem01(wttest.WiredTigerTestCase): ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt, value_format=self.valuefmt, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, - lambda:ds.populate(), msg) + ds.populate, msg) # Figure out the last key we successfully inserted, and check all # previous inserts are still there. @@ -77,7 +77,7 @@ class test_inmem01(wttest.WiredTigerTestCase): ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt, value_format=self.valuefmt, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, - lambda:ds.populate(), msg) + ds.populate, msg) # Now that the database contains as much data as will fit into # the configured cache, verify removes succeed. @@ -93,7 +93,7 @@ class test_inmem01(wttest.WiredTigerTestCase): ds = SimpleDataSet(self, self.uri, 10000000, key_format=self.keyfmt, value_format=self.valuefmt, config=self.table_config) self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, - lambda:ds.populate(), msg) + ds.populate, msg) cursor = self.session.open_cursor(self.uri, None) cursor.prev() @@ -120,5 +120,50 @@ class test_inmem01(wttest.WiredTigerTestCase): break self.assertTrue(inserted) + # Custom "keep filling" helper + def fill(self, cursor, ds, start, end): + for i in xrange(start + 1, end + 1): + cursor[ds.key(i)] = ds.value(i) + + # Keep adding data to the cache until it becomes really full, make sure + # that reads aren't blocked. + @wttest.longtest("Try to wedge an in-memory cache") + def test_wedge(self): + # Try to really wedge the cache full + ds = SimpleDataSet(self, self.uri, 0, key_format=self.keyfmt, + value_format=self.valuefmt, config=self.table_config) + ds.populate() + cursor = self.session.open_cursor(self.uri, None) + + run = 0 + start, last_key = -1000, 0 + while last_key - start > 100: + msg = '/WT_CACHE_FULL.*/' + start = last_key + self.assertRaisesHavingMessage(wiredtiger.WiredTigerError, + lambda: self.fill(cursor, ds, start, 10000000), msg) + cursor.reset() + sleep(1) + + # Figure out the last key we successfully inserted, and check all + # previous inserts are still there. + cursor.prev() + last_key = int(cursor.get_key()) + run += 1 + self.pr('Finished iteration ' + str(run) + ', last_key = ' + str(last_key)) + + self.pr('Checking ' + str(last_key) + ' keys') + ds = SimpleDataSet(self, self.uri, last_key, key_format=self.keyfmt, + value_format=self.valuefmt, config=self.table_config) + + # This test is *much* slower for fixed-length column stores: we fit + # many more records into the cache, so don't do as many passes through + # the data. + checks = 10 if self.valuefmt.endswith('t') else 100 + for run in xrange(checks): + ds.check() + self.pr('Finished check ' + str(run)) + sleep(1) + if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_intpack.py b/src/third_party/wiredtiger/test/suite/test_intpack.py index b0cece09494..ae391e68fca 100644 --- a/src/third_party/wiredtiger/test/suite/test_intpack.py +++ b/src/third_party/wiredtiger/test/suite/test_intpack.py @@ -126,8 +126,8 @@ class PackTester: class test_intpack(wttest.WiredTigerTestCase): name = 'test_intpack' - # We have to be a bit verbose here with naming, as there can be problems with - # case insensitive test names:w + # We have to be a bit verbose here with naming, scenario names are + # case insensitive and must be unique. scenarios = make_scenarios([ ('int8_t_b', dict(formatcode='b', low=-128, high=127, nbits=8)), diff --git a/src/third_party/wiredtiger/test/suite/test_join01.py b/src/third_party/wiredtiger/test/suite/test_join01.py index f3b13026896..bdd86a06d4f 100644 --- a/src/third_party/wiredtiger/test/suite/test_join01.py +++ b/src/third_party/wiredtiger/test/suite/test_join01.py @@ -69,7 +69,7 @@ class test_join01(wttest.WiredTigerTestCase): ] scenarios = make_scenarios(type_scen, bloom0_scen, bloom1_scen, projection_scen, nested_scen, stats_scen, - order_scen) + order_scen, prune=50, prunelong=1000) # We need statistics for these tests. conn_config = 'statistics=(all)' @@ -110,8 +110,9 @@ class test_join01(wttest.WiredTigerTestCase): while jc.next() == 0: [k] = jc.get_keys() i = k - 1 - if do_proj: # our projection test simply reverses the values - [v2,v1,v0] = jc.get_values() + if do_proj: # our projection reverses the values and adds the key + [v2,v1,v0,kproj] = jc.get_values() + self.assertEquals(k, kproj) else: [v0,v1,v2] = jc.get_values() self.assertEquals(self.gen_values(i), [v0,v1,v2]) @@ -136,7 +137,7 @@ class test_join01(wttest.WiredTigerTestCase): if self.ref == 'index': expectstats.append('join: index:join01:index0: ' + statdesc) elif self.do_proj: - expectstats.append('join: table:join01(v2,v1,v0): ' + statdesc) + expectstats.append('join: table:join01(v2,v1,v0,k): ' + statdesc) else: expectstats.append('join: table:join01: ' + statdesc) self.check_stats(statcur, expectstats) @@ -228,7 +229,7 @@ class test_join01(wttest.WiredTigerTestCase): c.close() if do_proj: - proj_suffix = '(v2,v1,v0)' # Reversed values + proj_suffix = '(v2,v1,v0,k)' # Reversed values plus key else: proj_suffix = '' # Default projection (v0,v1,v2) diff --git a/src/third_party/wiredtiger/test/suite/test_join03.py b/src/third_party/wiredtiger/test/suite/test_join03.py index edab7146a6b..dd8111f6ead 100644 --- a/src/third_party/wiredtiger/test/suite/test_join03.py +++ b/src/third_party/wiredtiger/test/suite/test_join03.py @@ -36,33 +36,9 @@ class test_join03(wttest.WiredTigerTestCase): table_name1 = 'test_join03' nentries = 100 - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name, libname) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + libname + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' - - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor')]) - connarg = 'create,error_prefix="{0}: ",{1}'.format( - self.shortid(), extarg) - conn = self.wiredtiger_open(dir, connarg) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('extractors', 'csv') def gen_key(self, i): return [ i + 1 ] diff --git a/src/third_party/wiredtiger/test/suite/test_join04.py b/src/third_party/wiredtiger/test/suite/test_join04.py index a71418d9f05..e65b8b53333 100644 --- a/src/third_party/wiredtiger/test/suite/test_join04.py +++ b/src/third_party/wiredtiger/test/suite/test_join04.py @@ -36,33 +36,9 @@ class test_join04(wttest.WiredTigerTestCase): table_name1 = 'test_join04' nentries = 100 - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name, libname) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + libname + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' - - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor')]) - connarg = 'create,error_prefix="{0}: ",{1}'.format( - self.shortid(), extarg) - conn = self.wiredtiger_open(dir, connarg) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('extractors', 'csv') # JIRA WT-2308: # Test extractors with equality joins diff --git a/src/third_party/wiredtiger/test/suite/test_join07.py b/src/third_party/wiredtiger/test/suite/test_join07.py index 2a32e678d72..8fae3539246 100644 --- a/src/third_party/wiredtiger/test/suite/test_join07.py +++ b/src/third_party/wiredtiger/test/suite/test_join07.py @@ -200,33 +200,9 @@ class test_join07(wttest.WiredTigerTestCase): scenarios = make_scenarios(extractscen) - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name, libname) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + libname + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' - - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor')]) - connarg = 'create,error_prefix="{0}: ",{1}'.format( - self.shortid(), extarg) - conn = self.wiredtiger_open(dir, connarg) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('extractors', 'csv') def expect(self, token, expected): if token == None or token.kind not in expected: diff --git a/src/third_party/wiredtiger/test/suite/test_jsondump02.py b/src/third_party/wiredtiger/test/suite/test_jsondump02.py index 8482851fb94..60863c4aa97 100644 --- a/src/third_party/wiredtiger/test/suite/test_jsondump02.py +++ b/src/third_party/wiredtiger/test/suite/test_jsondump02.py @@ -234,6 +234,24 @@ class test_jsondump02(wttest.WiredTigerTestCase, suite_subprocess): ('"ikey" : 4,\n"Skey" : "key4"', '"S1" : "val16",\n"i2" : 16,\n"S3" : "val64",\n"i4" : 64')) self.check_json(self.table_uri4, table4_json) + # This projection has 3 value fields reversed with a key at the end. + table4_json_projection = ( + ('"ikey" : 1,\n"Skey" : "key1"', + '"i4" : 1,\n"S3" : "val1",\n"i2" : 1,\n"ikey" : 1'), + ('"ikey" : 2,\n"Skey" : "key2"', + '"i4" : 8,\n"S3" : "val8",\n"i2" : 4,\n"ikey" : 2'), + ('"ikey" : 3,\n"Skey" : "key3"', + '"i4" : 27,\n"S3" : "val27",\n"i2" : 9,\n"ikey" : 3'), + ('"ikey" : 4,\n"Skey" : "key4"', + '"i4" : 64,\n"S3" : "val64",\n"i2" : 16,\n"ikey" : 4')) + # bad projection URI + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.check_json(self.table_uri4 + '(i4,S3,i2,ikey', + table4_json_projection), + '/Unbalanced brackets/') + # This projection should work. + self.check_json(self.table_uri4 + '(i4,S3,i2,ikey)', + table4_json_projection) # The dump config currently is not supported for the index type. self.check_json(uri4index1, ( ('"Skey" : "key1"', diff --git a/src/third_party/wiredtiger/test/suite/test_lsm01.py b/src/third_party/wiredtiger/test/suite/test_lsm01.py index b44df4bae14..f705b09b0a4 100644 --- a/src/third_party/wiredtiger/test/suite/test_lsm01.py +++ b/src/third_party/wiredtiger/test/suite/test_lsm01.py @@ -57,7 +57,7 @@ class test_lsm01(wttest.WiredTigerTestCase): scenarios = wtscenario.make_scenarios( chunk_size_scenarios, merge_max_scenarios, bloom_scenarios, bloom_bit_scenarios, bloom_hash_scenarios, record_count_scenarios, - prune=500) + prune=100, prunelong=500) # Test drop of an object. def test_lsm(self): diff --git a/src/third_party/wiredtiger/test/suite/test_nsnap04.py b/src/third_party/wiredtiger/test/suite/test_nsnap04.py index 60901dd2ee3..8d491540d74 100644 --- a/src/third_party/wiredtiger/test/suite/test_nsnap04.py +++ b/src/third_party/wiredtiger/test/suite/test_nsnap04.py @@ -38,14 +38,18 @@ class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename nrows_per_itr = 10 - def check_named_snapshot(self, snapshot, expected): + def check_named_snapshot(self, snapshot, expected, skip_snapshot=False): new_session = self.conn.open_session() c = new_session.open_cursor(self.uri) - new_session.begin_transaction("snapshot=" + str(snapshot)) + if skip_snapshot: + new_session.begin_transaction() + else: + new_session.begin_transaction("snapshot=" + str(snapshot)) count = 0 for row in c: count += 1 new_session.commit_transaction() + new_session.close() # print "Checking snapshot %d, expect %d, found %d" % (snapshot, expected, count) self.assertEqual(count, expected) @@ -80,5 +84,34 @@ class test_nsnap04(wttest.WiredTigerTestCase, suite_subprocess): self.session.snapshot("name=0") self.check_named_snapshot(0, 2 * self.nrows_per_itr) + def test_include_updates(self): + # Populate a table + end = start = 0 + SimpleDataSet(self, self.uri, 0, key_format='i').populate() + + snapshots = [] + c = self.session.open_cursor(self.uri) + for i in xrange(self.nrows_per_itr): + c[i] = "some value" + + self.session.begin_transaction("isolation=snapshot") + count = 0 + for row in c: + count += 1 + self.session.snapshot("name=0,include_updates=true") + + self.check_named_snapshot(0, self.nrows_per_itr) + + # Insert some more content using the active session. + for i in xrange(self.nrows_per_itr): + c[self.nrows_per_itr + i] = "some value" + + self.check_named_snapshot(0, 2 * self.nrows_per_itr) + # Ensure transactions not tracking the snapshot don't see the updates + self.check_named_snapshot(0, self.nrows_per_itr, skip_snapshot=True) + self.session.commit_transaction() + # Ensure content is visible to non-snapshot transactions after commit + self.check_named_snapshot(0, 2 * self.nrows_per_itr, skip_snapshot=True) + if __name__ == '__main__': wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_overwrite.py b/src/third_party/wiredtiger/test/suite/test_overwrite.py index 4739abaa578..c894de99bd0 100644 --- a/src/third_party/wiredtiger/test/suite/test_overwrite.py +++ b/src/third_party/wiredtiger/test/suite/test_overwrite.py @@ -27,32 +27,47 @@ # OTHER DEALINGS IN THE SOFTWARE. import wiredtiger, wttest -from wtdataset import SimpleDataSet +from wtdataset import SimpleDataSet, SimpleIndexDataSet +from wtdataset import SimpleLSMDataSet, ComplexDataSet, ComplexLSMDataSet from wtscenario import make_scenarios # test_overwrite.py # cursor overwrite configuration method class test_overwrite(wttest.WiredTigerTestCase): name = 'overwrite' - scenarios = make_scenarios([ - ('file-r', dict(type='file:', keyfmt='r', dataset=SimpleDataSet)), - ('file-S', dict(type='file:', keyfmt='S', dataset=SimpleDataSet)), - ('lsm-S', dict(type='lsm:', keyfmt='S', dataset=SimpleDataSet)), - ('table-r', dict(type='table:', keyfmt='r', dataset=SimpleDataSet)), - ('table-S', dict(type='table:', keyfmt='S', dataset=SimpleDataSet)), - ]) + keyfmt = [ + ('integer', dict(keyfmt='i')), + ('recno', dict(keyfmt='r')), + ('string', dict(keyfmt='S')), + ] + types = [ + ('file', dict(uri='file:', ds=SimpleDataSet)), + ('lsm', dict(uri='lsm:', ds=SimpleDataSet)), + ('table-complex', dict(uri='table:', ds=ComplexDataSet)), + ('table-complex-lsm', dict(uri='table:', ds=ComplexLSMDataSet)), + ('table-index', dict(uri='table:', ds=SimpleIndexDataSet)), + ('table-simple', dict(uri='table:', ds=SimpleDataSet)), + ('table-simple-lsm', dict(uri='table:', ds=SimpleLSMDataSet)), + ] + scenarios = make_scenarios(types, keyfmt) + def skip(self): + return self.keyfmt == 'r' and \ + (self.ds.is_lsm() or self.uri == 'lsm') # Confirm a cursor configured with/without overwrite correctly handles # non-existent records during insert, remove and update operations. def test_overwrite_insert(self): - uri = self.type + self.name - ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + if self.skip(): + return + + uri = self.uri + self.name + ds = self.ds(self, uri, 100, key_format=self.keyfmt) ds.populate() # Insert of an existing record with overwrite off fails. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(5)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1000)) self.assertRaises(wiredtiger.WiredTigerError, lambda: cursor.insert()) # One additional test for the insert method: duplicate the cursor with @@ -63,30 +78,33 @@ class test_overwrite(wttest.WiredTigerTestCase): cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(5)) dupc = self.session.open_cursor(None, cursor, "overwrite=true") - dupc.set_value('XXXXXXXXXX') + dupc.set_value(ds.value(1001)) self.assertEquals(dupc.insert(), 0) # Insert of an existing record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(ds.key(6)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1002)) self.assertEquals(cursor.insert(), 0) # Insert of a non-existent record with overwrite off succeeds. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(200)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1003)) self.assertEquals(cursor.insert(), 0) # Insert of a non-existent record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(ds.key(201)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1004)) self.assertEquals(cursor.insert(), 0) def test_overwrite_remove(self): - uri = self.type + self.name - ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + if self.skip(): + return + + uri = self.uri + self.name + ds = self.ds(self, uri, 100, key_format=self.keyfmt) ds.populate() # Remove of an existing record with overwrite off succeeds. @@ -110,32 +128,35 @@ class test_overwrite(wttest.WiredTigerTestCase): self.assertEquals(cursor.remove(), 0) def test_overwrite_update(self): - uri = self.type + self.name - ds = self.dataset(self, uri, 100, key_format=self.keyfmt) + if self.skip(): + return + + uri = self.uri + self.name + ds = self.ds(self, uri, 100, key_format=self.keyfmt) ds.populate() # Update of an existing record with overwrite off succeeds. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(5)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1005)) self.assertEquals(cursor.update(), 0) # Update of an existing record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(ds.key(6)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1006)) self.assertEquals(cursor.update(), 0) # Update of a non-existent record with overwrite off fails. cursor = self.session.open_cursor(uri, None, "overwrite=false") cursor.set_key(ds.key(200)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1007)) self.assertEquals(cursor.update(), wiredtiger.WT_NOTFOUND) # Update of a non-existent record with overwrite on succeeds. cursor = self.session.open_cursor(uri, None) cursor.set_key(ds.key(201)) - cursor.set_value('XXXXXXXXXX') + cursor.set_value(ds.value(1008)) self.assertEquals(cursor.update(), 0) if __name__ == '__main__': diff --git a/src/third_party/wiredtiger/test/suite/test_perf001.py b/src/third_party/wiredtiger/test/suite/test_perf001.py index b22ed2baeb0..6331a3f64d6 100644 --- a/src/third_party/wiredtiger/test/suite/test_perf001.py +++ b/src/third_party/wiredtiger/test/suite/test_perf001.py @@ -40,7 +40,8 @@ class test_perf001(wttest.WiredTigerTestCase): scenarios = make_scenarios([ #('file-file', dict(tabletype='file',indextype='file')), - ('file-lsm', dict(tabletype='file',indextype='lsm')), + ('file-lsm', dict(tabletype='file',indextype='lsm', cfg='', + conn_config="statistics=(fast),statistics_log=(wait=1)")), #('lsm-file', dict(tabletype='lsm',indextype='file')), #('lsm-lsm', dict(tabletype='lsm',indextype='lsm')), ]) diff --git a/src/third_party/wiredtiger/test/suite/test_readonly01.py b/src/third_party/wiredtiger/test/suite/test_readonly01.py index e4b431ca1da..f41280a3283 100644 --- a/src/third_party/wiredtiger/test/suite/test_readonly01.py +++ b/src/third_party/wiredtiger/test/suite/test_readonly01.py @@ -75,8 +75,7 @@ class test_readonly01(wttest.WiredTigerTestCase, suite_subprocess): scenarios = make_scenarios(basecfg_list, dir_list, log_list, types) - def conn_config(self, dir): - self.home = dir + def conn_config(self): params = \ 'error_prefix="%s",' % self.shortid() + \ '%s' % self.logcfg + \ diff --git a/src/third_party/wiredtiger/test/suite/test_reconfig01.py b/src/third_party/wiredtiger/test/suite/test_reconfig01.py index e76becac76a..cbc8bca5740 100644 --- a/src/third_party/wiredtiger/test/suite/test_reconfig01.py +++ b/src/third_party/wiredtiger/test/suite/test_reconfig01.py @@ -64,6 +64,18 @@ class test_reconfig01(wttest.WiredTigerTestCase): # same ops_max of 512 and thread of 8. self.conn.reconfigure("async=(enabled=true)") + def test_reconfig_eviction(self): + # Increase the max number of running threads (default 8). + self.conn.reconfigure("eviction=(threads_max=10)") + # Increase the min number of running threads (default 1). + self.conn.reconfigure("eviction=(threads_min=5)") + # Decrease the max number of running threads. + self.conn.reconfigure("eviction=(threads_max=7)") + # Decrease the min number of running threads. + self.conn.reconfigure("eviction=(threads_min=2)") + # Set min and max the same. + self.conn.reconfigure("eviction=(threads_min=6,threads_max=6)") + def test_reconfig_lsm_manager(self): # We create and populate a tiny LSM so that we can start off with # the LSM threads running and change the numbers of threads. diff --git a/src/third_party/wiredtiger/test/suite/test_reconfig02.py b/src/third_party/wiredtiger/test/suite/test_reconfig02.py index 36a78a1805f..042d3bbe71f 100644 --- a/src/third_party/wiredtiger/test/suite/test_reconfig02.py +++ b/src/third_party/wiredtiger/test/suite/test_reconfig02.py @@ -62,7 +62,7 @@ class test_reconfig02(wttest.WiredTigerTestCase): self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.conn.reconfigure("log=(path=foo)"), msg) self.assertRaisesWithMessage(wiredtiger.WiredTigerError, - lambda: self.conn.reconfigure("log=(recovery=true)"), msg) + lambda: self.conn.reconfigure("log=(recover=true)"), msg) # Logging starts on, but prealloc is off. Verify it is off. # Reconfigure it on and run again, making sure that log files @@ -109,6 +109,7 @@ class test_reconfig02(wttest.WiredTigerTestCase): # Now turn on archive, sleep a bit to allow the archive thread # to run and then confirm that all original logs are gone. self.conn.reconfigure("log=(archive=true)") + self.session.checkpoint("force") time.sleep(2) cur_logs = fnmatch.filter(os.listdir('.'), "*Log*") for o in orig_logs: diff --git a/src/third_party/wiredtiger/test/suite/test_reconfig04.py b/src/third_party/wiredtiger/test/suite/test_reconfig04.py new file mode 100644 index 00000000000..51d9b91c1f4 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_reconfig04.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest + +# test_reconfig04.py +# Test WT_SESSION::reconfigure +class test_reconfig04(wttest.WiredTigerTestCase): + def test_session_reconfigure(self): + self.session.reconfigure('ignore_cache_size=false') + + self.session.reconfigure('isolation=snapshot') + self.session.reconfigure('isolation=read-committed') + self.session.reconfigure('isolation=read-uncommitted') + + self.session.reconfigure('ignore_cache_size=true') + self.session.reconfigure('isolation=snapshot') + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_schema05.py b/src/third_party/wiredtiger/test/suite/test_schema05.py index 28ad51b3c92..d536a629373 100644 --- a/src/third_party/wiredtiger/test/suite/test_schema05.py +++ b/src/third_party/wiredtiger/test/suite/test_schema05.py @@ -57,33 +57,9 @@ class test_schema05(wttest.WiredTigerTestCase): ('index-after', { 'create_index' : 2 }), ]) - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, exts): - extfiles = [] - for ext in exts: - (dirname, name, libname) = ext - if name != None and name != 'none': - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext', dirname) - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + libname + '.so') - if not os.path.exists(extfile): - self.skipTest('extension "' + extfile + '" not built') - if not extfile in extfiles: - extfiles.append(extfile) - if len(extfiles) == 0: - return '' - else: - return ',extensions=["' + '","'.join(extfiles) + '"]' - - # Override WiredTigerTestCase, we have extensions. - def setUpConnectionOpen(self, dir): - extarg = self.extensionArg([('extractors', 'csv', 'csv_extractor')]) - connarg = 'create,error_prefix="{0}: ",{1}'.format( - self.shortid(), extarg) - conn = self.wiredtiger_open(dir, connarg) - self.pr(`conn`) - return conn + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('extractors', 'csv') def create_indices(self): # Create self.nindices index files, each with a column from the CSV diff --git a/src/third_party/wiredtiger/test/suite/test_schema07.py b/src/third_party/wiredtiger/test/suite/test_schema07.py index ac397c6e1a1..3e4b1d28a4d 100644 --- a/src/third_party/wiredtiger/test/suite/test_schema07.py +++ b/src/third_party/wiredtiger/test/suite/test_schema07.py @@ -33,8 +33,7 @@ import wiredtiger, wttest class test_schema07(wttest.WiredTigerTestCase): tablename = 'table:test_schema07' - def conn_config(self, dir): - return 'cache_size=10MB' + conn_config = 'cache_size=10MB' @wttest.longtest("Creating many tables shouldn't fill the cache") def test_many_tables(self): diff --git a/src/third_party/wiredtiger/test/suite/test_shared_cache01.py b/src/third_party/wiredtiger/test/suite/test_shared_cache01.py index 70560a625ee..c3bd946cc4b 100644 --- a/src/third_party/wiredtiger/test/suite/test_shared_cache01.py +++ b/src/third_party/wiredtiger/test/suite/test_shared_cache01.py @@ -73,7 +73,7 @@ class test_shared_cache01(wttest.WiredTigerTestCase): os.mkdir(name) next_conn = self.wiredtiger_open( name, - 'create,error_prefix="' + self.shortid() + ': "' + + 'create,error_prefix="%s",' % self.shortid() + pool_opts + extra_opts) self.conns.append(next_conn) self.sessions.append(next_conn.open_session(None)) diff --git a/src/third_party/wiredtiger/test/suite/test_shared_cache02.py b/src/third_party/wiredtiger/test/suite/test_shared_cache02.py index 7cde6c86695..67f9bf7c6b7 100644 --- a/src/third_party/wiredtiger/test/suite/test_shared_cache02.py +++ b/src/third_party/wiredtiger/test/suite/test_shared_cache02.py @@ -73,7 +73,7 @@ class test_shared_cache02(wttest.WiredTigerTestCase): os.mkdir(name) next_conn = self.wiredtiger_open( name, - 'create,error_prefix="' + self.shortid() + ': "' + + 'create,error_prefix="%s",' % self.shortid() + pool_opts + extra_opts) self.conns.append(next_conn) self.sessions.append(next_conn.open_session(None)) diff --git a/src/third_party/wiredtiger/test/suite/test_stat02.py b/src/third_party/wiredtiger/test/suite/test_stat02.py index cecda7f1ddc..45af283ed02 100644 --- a/src/third_party/wiredtiger/test/suite/test_stat02.py +++ b/src/third_party/wiredtiger/test/suite/test_stat02.py @@ -59,7 +59,7 @@ class test_stat_cursor_config(wttest.WiredTigerTestCase): scenarios = make_scenarios(uri, data_config, cursor_config) # Turn on statistics for this test. - def conn_config(self, dir): + def conn_config(self): return 'statistics=(%s)' % self.data_config # For each database/cursor configuration, confirm the right combinations diff --git a/src/third_party/wiredtiger/test/suite/test_sweep01.py b/src/third_party/wiredtiger/test/suite/test_sweep01.py index 71f8fcb180e..5559190caca 100644 --- a/src/third_party/wiredtiger/test/suite/test_sweep01.py +++ b/src/third_party/wiredtiger/test/suite/test_sweep01.py @@ -116,10 +116,15 @@ class test_sweep01(wttest.WiredTigerTestCase, suite_subprocess): # Give slow machines time to process files. stat_cursor = self.session.open_cursor('statistics:', None, None) this_nfile = stat_cursor[stat.conn.file_open][2] + removed = stat_cursor[stat.conn.dh_sweep_remove][2] stat_cursor.close() self.pr("==== loop " + str(sleep)) self.pr("this_nfile " + str(this_nfile)) - if this_nfile == final_nfile: + self.pr("removed " + str(removed)) + # On slow machines there can be a lag where files get closed but + # the sweep server cannot yet remove the handles. So wait for the + # removed statistic to indicate forward progress too. + if this_nfile == final_nfile and removed != remove1: break c.close() self.pr("Sweep loop took " + str(sleep)) diff --git a/src/third_party/wiredtiger/test/suite/test_truncate01.py b/src/third_party/wiredtiger/test/suite/test_truncate01.py index 2319eeddbef..98b741ba6a4 100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate01.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate01.py @@ -128,6 +128,7 @@ class test_truncate_cursor_order(wttest.WiredTigerTestCase): msg = '/the start cursor position is after the stop cursor position/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.truncate(None, c1, c2, None), msg) + c1.set_key(ds.key(10)) c2.set_key(ds.key(20)) self.session.truncate(None, c1, c2, None) @@ -183,11 +184,11 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): # those tests to file objects. types = [ ('file', dict(type='file:', valuefmt='S', - config='allocation_size=512,leaf_page_max=512')), + config='allocation_size=512,leaf_page_max=512', P=0.25)), ('file8t', dict(type='file:', valuefmt='8t', - config='allocation_size=512,leaf_page_max=512')), + config='allocation_size=512,leaf_page_max=512', P=0.25)), ('table', dict(type='table:', valuefmt='S', - config='allocation_size=512,leaf_page_max=512')), + config='allocation_size=512,leaf_page_max=512', P=0.5)), ] keyfmt = [ ('integer', dict(keyfmt='i')), @@ -203,7 +204,8 @@ class test_truncate_cursor(wttest.WiredTigerTestCase): ('big', dict(nentries=1000,skip=37)), ] - scenarios = make_scenarios(types, keyfmt, size, reopen) + scenarios = make_scenarios(types, keyfmt, size, reopen, + prune=10, prunelong=1000) # Set a cursor key. def cursorKey(self, ds, uri, key): diff --git a/src/third_party/wiredtiger/test/suite/test_truncate02.py b/src/third_party/wiredtiger/test/suite/test_truncate02.py index 73fed362354..729825b26d4 100644 --- a/src/third_party/wiredtiger/test/suite/test_truncate02.py +++ b/src/third_party/wiredtiger/test/suite/test_truncate02.py @@ -85,7 +85,8 @@ class test_truncate_fast_delete(wttest.WiredTigerTestCase): ('txn2', dict(commit=False)), ] - scenarios = make_scenarios(types, keyfmt, overflow, reads, writes, txn) + scenarios = make_scenarios(types, keyfmt, overflow, reads, writes, txn, + prune=20, prunelong=1000) # Return the number of records visible to the cursor; test both forward # and backward iteration, they are different code paths in this case. diff --git a/src/third_party/wiredtiger/test/suite/test_txn02.py b/src/third_party/wiredtiger/test/suite/test_txn02.py index a0c2c12a47c..01626057b9e 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn02.py +++ b/src/third_party/wiredtiger/test/suite/test_txn02.py @@ -93,11 +93,10 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): checklog_calls = 100 if wttest.islongtest() else 2 checklog_mod = (len(scenarios) / checklog_calls + 1) - def setUpConnectionOpen(self, dir): - self.home = dir + def conn_config(self): # Cycle through the different transaction_sync values in a # deterministic manner. - self.txn_sync = self.sync_list[ + txn_sync = self.sync_list[ self.scenario_number % len(self.sync_list)] # # We don't want to run zero fill with only the same settings, such @@ -107,17 +106,9 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): zerofill = 'false' if self.scenario_number % freq == 0: zerofill = 'true' - self.backup_dir = os.path.join(self.home, "WT_BACKUP") - conn_params = \ - 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ - 'log=(zero_fill=%s),' % zerofill + \ - 'create,error_prefix="%s: ",' % self.shortid() + \ - 'transaction_sync="%s",' % self.txn_sync - # print "Creating conn at '%s' with config '%s'" % (dir, conn_params) - conn = self.wiredtiger_open(dir, conn_params) - self.pr(`conn`) - self.session2 = conn.open_session() - return conn + return 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ + 'log=(zero_fill=%s),' % zerofill + \ + 'transaction_sync="%s",' % txn_sync # Check that a cursor (optionally started in a new transaction), sees the # expected values. @@ -176,8 +167,10 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): backup_conn = self.wiredtiger_open(self.backup_dir, backup_conn_params) try: - self.check(backup_conn.open_session(), None, committed) + session = backup_conn.open_session() finally: + session.checkpoint("force") + self.check(backup_conn.open_session(), None, committed) # Sleep long enough so that the archive thread is guaranteed # to run before we close the connection. time.sleep(1.0) @@ -204,6 +197,8 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): self.assertEqual(cur_logs, pr_logs) def test_ops(self): + self.backup_dir = os.path.join(self.home, "WT_BACKUP") + self.session2 = self.conn.open_session() # print "Creating %s with config '%s'" % (self.uri, self.create_params) self.session.create(self.uri, self.create_params) # Set up the table with entries for 1, 2, 10 and 11. @@ -226,6 +221,7 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): # Close and reopen the connection and cursor. if reopen == 'reopen': self.reopen_conn() + self.session2 = self.conn.open_session() c = self.session.open_cursor(self.uri, None, 'overwrite') self.session.begin_transaction( diff --git a/src/third_party/wiredtiger/test/suite/test_txn04.py b/src/third_party/wiredtiger/test/suite/test_txn04.py index ade39272f84..d8f6774ded1 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn04.py +++ b/src/third_party/wiredtiger/test/suite/test_txn04.py @@ -63,24 +63,15 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): txn1s = [('t1c', dict(txn1='commit')), ('t1r', dict(txn1='rollback'))] scenarios = make_scenarios(types, op1s, txn1s) - # Overrides WiredTigerTestCase - def setUpConnectionOpen(self, dir): - self.home = dir + + def conn_config(self): # Cycle through the different transaction_sync values in a # deterministic manner. - self.txn_sync = self.sync_list[ + txn_sync = self.sync_list[ self.scenario_number % len(self.sync_list)] - self.backup_dir = os.path.join(self.home, "WT_BACKUP") # Set archive false on the home directory. - conn_params = \ - 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ - 'create,error_prefix="%s: ",' % self.shortid() + \ - 'transaction_sync="%s",' % self.txn_sync - # print "Creating conn at '%s' with config '%s'" % (dir, conn_params) - conn = self.wiredtiger_open(dir, conn_params) - self.pr(`conn`) - self.session2 = conn.open_session() - return conn + return 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ + 'transaction_sync="%s",' % txn_sync # Check that a cursor (optionally started in a new transaction), sees the # expected values. @@ -146,6 +137,7 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): # The runWt command closes our connection and sessions so # we need to reopen them here. self.hot_backup(None, committed) + self.session2 = self.conn.open_session() c = self.session.open_cursor(self.uri, None, 'overwrite') c.set_value(1) # Then do the given modification. @@ -193,6 +185,8 @@ class test_txn04(wttest.WiredTigerTestCase, suite_subprocess): self.hot_backup(self.uri, committed) def test_ops(self): + self.backup_dir = os.path.join(self.home, "WT_BACKUP") + self.session2 = self.conn.open_session() with self.expectedStdoutPattern('recreating metadata'): self.ops() diff --git a/src/third_party/wiredtiger/test/suite/test_txn05.py b/src/third_party/wiredtiger/test/suite/test_txn05.py index 9e84fe7d3fe..7aaff221ba4 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn05.py +++ b/src/third_party/wiredtiger/test/suite/test_txn05.py @@ -64,23 +64,15 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): txn1s = [('t1c', dict(txn1='commit')), ('t1r', dict(txn1='rollback'))] scenarios = make_scenarios(types, op1s, txn1s) - # Overrides WiredTigerTestCase - def setUpConnectionOpen(self, dir): - self.home = dir + + def conn_config(self): # Cycle through the different transaction_sync values in a # deterministic manner. - self.txn_sync = self.sync_list[ + txn_sync = self.sync_list[ self.scenario_number % len(self.sync_list)] - self.backup_dir = os.path.join(self.home, "WT_BACKUP") - conn_params = \ - 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ - 'create,error_prefix="%s: ",' % self.shortid() + \ - 'transaction_sync="%s",' % self.txn_sync - # print "Creating conn at '%s' with config '%s'" % (dir, conn_params) - conn = self.wiredtiger_open(dir, conn_params) - self.pr(`conn`) - self.session2 = conn.open_session() - return conn + # Set archive false on the home directory. + return 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ + 'transaction_sync="%s",' % txn_sync # Check that a cursor (optionally started in a new transaction), sees the # expected values. @@ -139,8 +131,12 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): backup_conn = self.wiredtiger_open(self.backup_dir, backup_conn_params) try: - self.check(backup_conn.open_session(), None, committed) + session = backup_conn.open_session() finally: + self.check(session, None, committed) + # Force a checkpoint because we don't record the recovery + # checkpoint as available for archiving. + session.checkpoint("force") # Sleep long enough so that the archive thread is guaranteed # to run before we close the connection. time.sleep(1.0) @@ -163,6 +159,8 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): self.runWt(['-h', self.backup_dir, 'printlog'], outfilename='printlog.out') def test_ops(self): + self.backup_dir = os.path.join(self.home, "WT_BACKUP") + self.session2 = self.conn.open_session() # print "Creating %s with config '%s'" % (self.uri, self.create_params) self.session.create(self.uri, self.create_params) # Set up the table with entries for 1-5. diff --git a/src/third_party/wiredtiger/test/suite/test_txn06.py b/src/third_party/wiredtiger/test/suite/test_txn06.py index 2bff97f6aac..c91dc6a623b 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn06.py +++ b/src/third_party/wiredtiger/test/suite/test_txn06.py @@ -40,10 +40,10 @@ class test_txn06(wttest.WiredTigerTestCase, suite_subprocess): source_uri = 'table:' + tablename + "_src" nrows = 100000 - def setUpConnectionOpen(self, *args): + def conn_config(self): if not wiredtiger.verbose_build(): self.skipTest('requires a verbose build') - return super(test_txn06, self).setUpConnectionOpen(*args) + return '' def test_long_running(self): # Populate a table diff --git a/src/third_party/wiredtiger/test/suite/test_txn07.py b/src/third_party/wiredtiger/test/suite/test_txn07.py index f9577bad7f2..e26cf5aaaea 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn07.py +++ b/src/third_party/wiredtiger/test/suite/test_txn07.py @@ -70,43 +70,20 @@ class test_txn07(wttest.WiredTigerTestCase, suite_subprocess): ('none', dict(compress='')), ] - scenarios = make_scenarios(types, op1s, txn1s, compress) - # Overrides WiredTigerTestCase - def setUpConnectionOpen(self, dir): - self.home = dir - # Cycle through the different transaction_sync values in a - # deterministic manner. - self.txn_sync = self.sync_list[ - self.scenario_number % len(self.sync_list)] - self.backup_dir = os.path.join(self.home, "WT_BACKUP") - conn_params = \ - 'log=(archive=false,enabled,file_max=%s,' % self.logmax + \ - 'compressor=%s)' % self.compress + \ - self.extensionArg(self.compress) + \ - ',create,error_prefix="%s: ",' % self.shortid() + \ - "statistics=(fast)," + \ - 'transaction_sync="%s",' % self.txn_sync - # print "Creating conn at '%s' with config '%s'" % (dir, conn_params) - try: - conn = self.wiredtiger_open(dir, conn_params) - except wiredtiger.WiredTigerError as e: - print "Failed conn at '%s' with config '%s'" % (dir, conn_params) - self.pr(`conn`) - self.session2 = conn.open_session() - return conn - - # Return the wiredtiger_open extension argument for a shared library. - def extensionArg(self, name): - if name == None or name == '': - return '' - - testdir = os.path.dirname(__file__) - extdir = os.path.join(run.wt_builddir, 'ext/compressors') - extfile = os.path.join( - extdir, name, '.libs', 'libwiredtiger_' + name + '.so') - if not os.path.exists(extfile): - self.skipTest('compression extension "' + extfile + '" not built') - return ',extensions=["' + extfile + '"]' + scenarios = make_scenarios(types, op1s, txn1s, compress, + prune=30, prunelong=1000) + + def conn_config(self): + return 'log=(archive=false,enabled,file_max=%s,' % self.logmax + \ + 'compressor=%s)' % self.compress + \ + ',create,error_prefix="%s",' % self.shortid() + \ + "statistics=(fast)," + \ + 'transaction_sync="%s",' % \ + self.sync_list[self.scenario_number % len(self.sync_list)] + + def conn_extensions(self, extlist): + extlist.skip_if_missing = True + extlist.extension('compressors', self.compress) # Check that a cursor (optionally started in a new transaction), sees the # expected values. @@ -139,7 +116,7 @@ class test_txn07(wttest.WiredTigerTestCase, suite_subprocess): self.backup(self.backup_dir) backup_conn_params = 'log=(enabled,file_max=%s,' % self.logmax + \ 'compressor=%s)' % self.compress + \ - self.extensionArg(self.compress) + self.extensionsConfig() backup_conn = self.wiredtiger_open(self.backup_dir, backup_conn_params) try: self.check(backup_conn.open_session(), None, committed) @@ -147,6 +124,9 @@ class test_txn07(wttest.WiredTigerTestCase, suite_subprocess): backup_conn.close() def test_ops(self): + self.backup_dir = os.path.join(self.home, "WT_BACKUP") + self.session2 = self.conn.open_session() + # print "Creating %s with config '%s'" % (self.uri, self.create_params) self.session.create(self.uri, self.create_params) # Set up the table with entries for 1-5. diff --git a/src/third_party/wiredtiger/test/suite/test_txn08.py b/src/third_party/wiredtiger/test/suite/test_txn08.py index f0cdf08df07..04faed9d45a 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn08.py +++ b/src/third_party/wiredtiger/test/suite/test_txn08.py @@ -41,7 +41,7 @@ class test_txn08(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename # Turn on logging for this test. - def conn_config(self, dir): + def conn_config(self): return 'log=(archive=false,enabled,file_max=%s),' % self.logmax + \ 'transaction_sync="(method=dsync,enabled)"' diff --git a/src/third_party/wiredtiger/test/suite/test_txn09.py b/src/third_party/wiredtiger/test/suite/test_txn09.py index cfad8270ab1..768d714e248 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn09.py +++ b/src/third_party/wiredtiger/test/suite/test_txn09.py @@ -80,19 +80,9 @@ class test_txn09(wttest.WiredTigerTestCase, suite_subprocess): op1s, txn1s, op2s, txn2s, op3s, txn3s, op4s, txn4s, prune=20, prunelong=5000) - # Overrides WiredTigerTestCase - def setUpConnectionOpen(self, dir): - self.home = dir - conn_params = \ - 'create,error_prefix="%s: ",' % self.shortid() + \ - 'log=(archive=false,enabled=%s),' % int(self.log_enabled) + \ - 'transaction_sync=(enabled=false),' - - # print "Opening conn at '%s' with config '%s'" % (dir, conn_params) - conn = self.wiredtiger_open(dir, conn_params) - self.pr(`conn`) - self.session2 = conn.open_session() - return conn + def conn_config(self): + return 'log=(archive=false,enabled=%s),' % int(self.log_enabled) + \ + 'transaction_sync=(enabled=false)' # Check that a cursor (optionally started in a new transaction), sees the # expected values. @@ -141,6 +131,7 @@ class test_txn09(wttest.WiredTigerTestCase, suite_subprocess): # Close and reopen the connection and cursor, toggling the log self.log_enabled = not self.log_enabled self.reopen_conn() + self.session2 = self.conn.open_session() c = self.session.open_cursor(self.uri, None, 'overwrite') self.session.begin_transaction( diff --git a/src/third_party/wiredtiger/test/suite/test_txn11.py b/src/third_party/wiredtiger/test/suite/test_txn11.py index 147bf3a76c0..3c02b1e86e3 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn11.py +++ b/src/third_party/wiredtiger/test/suite/test_txn11.py @@ -44,7 +44,7 @@ class test_txn11(wttest.WiredTigerTestCase, suite_subprocess): uri = 'table:' + tablename # Turn on logging for this test. - def conn_config(self, dir): + def conn_config(self): return 'log=(archive=%s,' % self.archive + \ 'enabled,file_max=%s,prealloc=false),' % self.logmax + \ 'transaction_sync=(enabled=false),' diff --git a/src/third_party/wiredtiger/test/suite/test_txn13.py b/src/third_party/wiredtiger/test/suite/test_txn13.py index ae0250c06e8..2bf49486b3a 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn13.py +++ b/src/third_party/wiredtiger/test/suite/test_txn13.py @@ -50,7 +50,7 @@ class test_txn13(wttest.WiredTigerTestCase, suite_subprocess): ]) # Turn on logging for this test. - def conn_config(self, dir): + def conn_config(self): return 'log=(archive=false,enabled,file_max=%s)' % self.logmax + \ ',cache_size=8G' diff --git a/src/third_party/wiredtiger/test/suite/test_txn15.py b/src/third_party/wiredtiger/test/suite/test_txn15.py index c061c093b02..a2bfb626338 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn15.py +++ b/src/third_party/wiredtiger/test/suite/test_txn15.py @@ -41,7 +41,7 @@ class test_txn15(wttest.WiredTigerTestCase, suite_subprocess): create_params = 'key_format=i,value_format=i' entries = 100 # Turn on logging for this test. - def conn_config(self, dir): + def conn_config(self): return 'statistics=(fast),' + \ 'log=(archive=false,enabled,file_max=100K),' + \ 'use_environment=false,' + \ diff --git a/src/third_party/wiredtiger/test/suite/test_util14.py b/src/third_party/wiredtiger/test/suite/test_util14.py new file mode 100644 index 00000000000..e2a9f41f0d4 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_util14.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os +from suite_subprocess import suite_subprocess +import wiredtiger, wttest + +# test_util14.py +# Utilities: wt truncate +class test_util14(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util14.a' + nentries = 1000 + + def test_truncate_process(self): + """ + Test truncate in a 'wt' process + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.assertTrue(os.path.exists(self.tablename + ".wt")) + cursor = self.session.open_cursor('table:' + self.tablename, None, None) + for i in range(0, self.nentries): + cursor[str(i)] = str(i) + cursor.close() + + self.runWt(["truncate", "table:" + self.tablename]) + + """ + Test to confirm table exists and is empty + """ + outfile="outfile.txt" + errfile="errfile.txt" + self.assertTrue(os.path.exists(self.tablename + ".wt")) + self.runWt(["read", 'table:' + self.tablename, 'NoMatch'], + outfilename=outfile, errfilename=errfile, failure=True) + self.check_empty_file(outfile) + self.check_file_contains(errfile, 'NoMatch: not found\n') + + """ + Tests for error cases + 1. Missing URI + 2. Invalid URI + 3. Valid but incorrect URI + 4. Double URI + """ + self.runWt(["truncate"], + outfilename=outfile, errfilename=errfile, failure=True) + self.check_empty_file(outfile) + self.check_file_contains(errfile, 'usage:') + + self.runWt(["truncate", "foobar"], + outfilename=outfile, errfilename=errfile, failure=True) + self.check_empty_file(outfile) + self.check_file_contains(errfile, 'No such file or directory') + + self.runWt(["truncate", 'table:xx' + self.tablename], + outfilename=outfile, errfilename=errfile, failure=True) + self.check_empty_file(outfile) + self.check_file_contains(errfile, 'No such file or directory') + + self.runWt(["truncate", 'table:' + self.tablename, 'table:' + self.tablename], + outfilename=outfile, errfilename=errfile, failure=True) + self.check_empty_file(outfile) + self.check_file_contains(errfile, 'usage:') + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_util15.py b/src/third_party/wiredtiger/test/suite/test_util15.py new file mode 100644 index 00000000000..33096e71bee --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_util15.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os +from suite_subprocess import suite_subprocess +import wiredtiger, wttest + +# test_util15.py +# Utilities: wt alter +class test_util15(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util15.a' + + def test_alter_process(self): + """ + Test alter in a 'wt' process + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.assertTrue(os.path.exists(self.tablename + ".wt")) + + """ + Alter access pattern and confirm + """ + acc_pat_seq="access_pattern_hint=sequential" + self.runWt(["alter", "table:" + self.tablename, acc_pat_seq]) + cursor = self.session.open_cursor("metadata:create", None, None) + cursor.set_key("table:" + self.tablename) + self.assertEqual(cursor.search(),0) + string = cursor.get_value() + cursor.close() + self.assertTrue(acc_pat_seq in string) + + """ + Alter access pattern again and confirm + """ + acc_pat_rand="access_pattern_hint=random" + self.runWt(["alter", "table:" + self.tablename, acc_pat_rand]) + cursor = self.session.open_cursor("metadata:create", None, None) + cursor.set_key("table:" + self.tablename) + self.assertEqual(cursor.search(),0) + string = cursor.get_value() + cursor.close() + self.assertTrue(acc_pat_rand in string) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_util16.py b/src/third_party/wiredtiger/test/suite/test_util16.py new file mode 100644 index 00000000000..00e68c1017a --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_util16.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os +from suite_subprocess import suite_subprocess +import wiredtiger, wttest + +# test_util16.py +# Utilities: wt rename +class test_util16(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util16.a' + tablename2 = 'test_util16.b' + nentries = 1000 + + def test_rename_process(self): + """ + Test alter in a 'wt' process + """ + params = 'key_format=S,value_format=S' + self.session.create('table:' + self.tablename, params) + self.assertTrue(os.path.exists(self.tablename + ".wt")) + cursor = self.session.open_cursor('table:' + self.tablename, None, None) + for i in range(0, self.nentries): + cursor[str(i)] = str(i) + cursor.close() + + self.runWt(["rename", "table:" + self.tablename, "table:" + self.tablename2]) + self.assertTrue(os.path.exists(self.tablename2 + ".wt")) + cursor = self.session.open_cursor('table:' + self.tablename2, None, None) + count = 0 + while cursor.next() == 0: + count +=1 + cursor.close() + self.assertEquals(self.nentries, count) + + self.runWt(["rename", "table:" + self.tablename2, "table:" + self.tablename]) + self.assertTrue(os.path.exists(self.tablename + ".wt")) + cursor = self.session.open_cursor('table:' + self.tablename, None, None) + count = 0 + while cursor.next() == 0: + count +=1 + cursor.close() + self.assertEquals(self.nentries, count) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_util17.py b/src/third_party/wiredtiger/test/suite/test_util17.py new file mode 100644 index 00000000000..decc1fabf1d --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_util17.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import os +from suite_subprocess import suite_subprocess +import wiredtiger, wttest + +# test_util17.py +# Utilities: wt stat +class test_util17(wttest.WiredTigerTestCase, suite_subprocess): + tablename = 'test_util17.a' + + def test_stat_process(self): + """ + Test stat in a 'wt' process + This test is just here to confirm that stat produces a correct looking + output, it isn't here to do statistics validation. + """ + params = 'key_format=S,value_format=S' + outfile = "wt-stat.out" + expected_string = "cursor: cursor create calls=" + self.session.create('table:' + self.tablename, params) + self.assertTrue(os.path.exists(self.tablename + ".wt")) + self.runWt(["stat"], outfilename=outfile) + self.check_file_contains(outfile, expected_string) + + expected_string = "cache_walk: Entries in the root page=1" + self.runWt(["stat", "table:" + self.tablename ], outfilename=outfile) + self.check_file_contains(outfile, expected_string) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/wtdataset.py b/src/third_party/wiredtiger/test/suite/wtdataset.py index cb77735510c..946b97d995f 100644 --- a/src/third_party/wiredtiger/test/suite/wtdataset.py +++ b/src/third_party/wiredtiger/test/suite/wtdataset.py @@ -41,6 +41,7 @@ class BaseDataSet(object): self.key_format = kwargs.get('key_format', 'S') self.value_format = kwargs.get('value_format', 'S') self.config = kwargs.get('config', '') + self.projection = kwargs.get('projection', '') def create(self): self.testcase.session.create(self.uri, 'key_format=' + self.key_format @@ -49,7 +50,7 @@ class BaseDataSet(object): def fill(self): c = self.testcase.session.open_cursor(self.uri, None) - for i in range(1, self.rows + 1): + for i in xrange(1, self.rows + 1): c[self.key(i)] = self.value(i) c.close() @@ -103,7 +104,8 @@ class BaseDataSet(object): def check(self): self.testcase.pr('check: ' + self.uri) - cursor = self.testcase.session.open_cursor(self.uri, None) + cursor = self.testcase.session.open_cursor( + self.uri + self.projection, None, None) self.check_cursor(cursor) cursor.close() @@ -166,7 +168,7 @@ class SimpleIndexDataSet(SimpleDataSet): # Check values in the index. idxcursor = self.testcase.session.open_cursor(self.indexname) - for i in range(1, self.rows + 1): + for i in xrange(1, self.rows + 1): k = self.key(i) v = self.value(i) ik = (v, k) # The index key is columns=(v,k). @@ -289,6 +291,94 @@ class ComplexLSMDataSet(ComplexDataSet): def is_lsm(cls): return True +class ProjectionDataSet(SimpleDataSet): + """ + ProjectionDataSet creates a table with predefined data identical to + SimpleDataSet (single key and value), but when checking it, uses + a cursor with a projection. + """ + def __init__(self, testcase, uri, rows, **kwargs): + kwargs['config'] = kwargs.get('config', '') + ',columns=(k,v0)' + kwargs['projection'] = '(v0,v0,v0)' + super(ProjectionDataSet, self).__init__(testcase, uri, rows, **kwargs) + + # A value suitable for checking the value returned by a cursor. + def comparable_value(self, i): + v0 = self.value(i) + return [v0, v0, v0] + + def check_cursor(self, cursor): + i = 0 + for key, got0, got1, got2 in cursor: + i += 1 + self.testcase.assertEqual(key, self.key(i)) + if cursor.value_format == '8t' and got0 == 0: # deleted + continue + self.testcase.assertEqual([got0, got1, got2], + self.comparable_value(i)) + self.testcase.assertEqual(i, self.rows) + +class ProjectionIndexDataSet(BaseDataSet): + """ + ProjectionIndexDataSet creates a table with three values and + an index. Checks are made against a projection of the main table + and a projection of the index. + """ + def __init__(self, testcase, uri, rows, **kwargs): + self.origconfig = kwargs.get('config', '') + self.indexname = 'index:' + uri.split(":")[1] + ':index0' + kwargs['config'] = self.origconfig + ',columns=(k,v0,v1,v2)' + kwargs['value_format'] = kwargs.get('value_format', 'SiS') + kwargs['projection'] = '(v1,v2,v0)' + super(ProjectionIndexDataSet, self).__init__( + testcase, uri, rows, **kwargs) + + def value(self, i): + return ('v0:' + str(i), i*i, 'v2:' + str(i)) + + # Suitable for checking the value returned by a cursor using a projection. + def comparable_value(self, i): + return [i*i, 'v2:' + str(i), 'v0:' + str(i)] + + def create(self): + super(ProjectionIndexDataSet, self).create() + self.testcase.session.create(self.indexname, 'columns=(v2,v1),' + + self.origconfig) + + def check_cursor(self, cursor): + i = 0 + for key, got0, got1, got2 in cursor: + i += 1 + self.testcase.assertEqual(key, self.key(i)) + if cursor.value_format == '8t' and got0 == 0: # deleted + continue + self.testcase.assertEqual([got0, got1, got2], + self.comparable_value(i)) + self.testcase.assertEqual(i, self.rows) + + def check_index_cursor(self, cursor): + for i in xrange(1, self.rows + 1): + k = self.key(i) + v = self.value(i) + ik = (v[2], v[1]) # The index key is (v2,v2) + expect = [v[1],k,v[2],v[0]] + self.testcase.assertEqual(expect, cursor[ik]) + + def check(self): + BaseDataSet.check(self) + + # Check values in the index. + idxcursor = self.testcase.session.open_cursor( + self.indexname + '(v1,k,v2,v0)') + self.check_index_cursor(idxcursor) + idxcursor.close() + + def index_count(self): + return 1 + + def index_name(self, i): + return self.indexname + # create a key based on a cursor as a shortcut to creating a SimpleDataSet def simple_key(cursor, i): return BaseDataSet.key_by_format(i, cursor.key_format) diff --git a/src/third_party/wiredtiger/test/suite/wttest.py b/src/third_party/wiredtiger/test/suite/wttest.py index 4d6df0bc8bd..e91838544b9 100644 --- a/src/third_party/wiredtiger/test/suite/wttest.py +++ b/src/third_party/wiredtiger/test/suite/wttest.py @@ -37,9 +37,8 @@ except ImportError: import unittest from contextlib import contextmanager -import os, re, shutil, sys, time, traceback -import wtscenario -import wiredtiger +import glob, os, re, shutil, sys, time, traceback +import wiredtiger, wtscenario def shortenWithEllipsis(s, maxlen): if len(s) > maxlen: @@ -152,6 +151,14 @@ class TestSuiteConnection(object): else: return getattr(self._conn, attr) +# Just like a list of strings, but with a convenience function +class ExtensionList(list): + skipIfMissing = False + def extension(self, dirname, name, extarg=None): + if name != None and name != 'none': + ext = '' if extarg == None else '=' + extarg + self.append(dirname + '/' + name + ext) + class WiredTigerTestCase(unittest.TestCase): _globalSetup = False _printOnceSeen = {} @@ -160,9 +167,16 @@ class WiredTigerTestCase(unittest.TestCase): # Can be a string or a callable function or lambda expression. conn_config = '' + # conn_extensions can be overridden to add a list of extensions to load. + # Each entry is a string (directory and extension name) and optional config. + # Example: + # conn_extensions = ('extractors/csv_extractor', + # 'test/fail_fs={allow_writes=100}') + conn_extensions = () + @staticmethod def globalSetup(preserveFiles = False, useTimestamp = False, - gdbSub = False, verbose = 1, dirarg = None, + gdbSub = False, verbose = 1, builddir = None, dirarg = None, longtest = False): WiredTigerTestCase._preserveFiles = preserveFiles d = 'WT_TEST' if dirarg == None else dirarg @@ -172,6 +186,7 @@ class WiredTigerTestCase(unittest.TestCase): os.makedirs(d) wtscenario.set_long_run(longtest) WiredTigerTestCase._parentTestdir = d + WiredTigerTestCase._builddir = builddir WiredTigerTestCase._origcwd = os.getcwd() WiredTigerTestCase._resultfile = open(os.path.join(d, 'results.txt'), "w", 0) # unbuffered WiredTigerTestCase._gdbSubprocess = gdbSub @@ -224,16 +239,70 @@ class WiredTigerTestCase(unittest.TestCase): return "%s.%s.%s" % (self.__module__, self.className(), self._testMethodName) - # Can be overridden, but first consider setting self.conn_config . + # Return the wiredtiger_open extension argument for + # any needed shared library. + def extensionsConfig(self): + exts = self.conn_extensions + if hasattr(exts, '__call__'): + exts = ExtensionList() + self.conn_extensions(exts) + result = '' + extfiles = {} + skipIfMissing = False + if hasattr(exts, 'skip_if_missing'): + skipIfMissing = exts.skip_if_missing + for ext in exts: + extconf = '' + if '=' in ext: + splits = ext.split('=', 1) + ext = splits[0] + extconf = '=' + splits[1] + splits = ext.split('/') + if len(splits) != 2: + raise Exception(self.shortid() + + ": " + ext + + ": extension is not named /") + libname = splits[1] + dirname = splits[0] + pat = os.path.join(WiredTigerTestCase._builddir, 'ext', + dirname, libname, '.libs', 'libwiredtiger_*.so') + filenames = glob.glob(pat) + if len(filenames) == 0: + if skipIfMissing: + self.skipTest('extension "' + ext + '" not built') + continue + else: + raise Exception(self.shortid() + + ": " + ext + + ": no extensions library found matching: " + pat) + elif len(filenames) > 1: + raise Exception(self.shortid() + + ": " + ext + + ": multiple extensions libraries found matching: " + pat) + complete = '"' + filenames[0] + '"' + extconf + if ext in extfiles: + if extfiles[ext] != complete: + raise Exception(self.shortid() + + ": non-matching extension arguments in " + + str(exts)) + else: + extfiles[ext] = complete + if len(extfiles) != 0: + result = ',extensions=[' + ','.join(extfiles.values()) + ']' + return result + + # Can be overridden, but first consider setting self.conn_config + # or self.conn_extensions def setUpConnectionOpen(self, home): self.home = home config = self.conn_config if hasattr(config, '__call__'): - config = config(home) + config = self.conn_config() + config += self.extensionsConfig() # In case the open starts additional threads, flush first to # avoid confusion. sys.stdout.flush() - conn_param = 'create,error_prefix="%s: ",%s' % (self.shortid(), config) + conn_param = 'create,error_prefix="%s",%s' % (self.shortid(), config) try: conn = self.wiredtiger_open(home, conn_param) except wiredtiger.WiredTigerError as e: @@ -287,6 +356,7 @@ class WiredTigerTestCase(unittest.TestCase): self.testsubdir = self.className() + '.' + str(self.__class__.wt_ntests) self.testdir = os.path.join(WiredTigerTestCase._parentTestdir, self.testsubdir) self.__class__.wt_ntests += 1 + self.starttime = time.time() if WiredTigerTestCase._verbose > 2: self.prhead('started in ' + self.testdir, True) # tearDown needs connections list, set it here in case the open fails. @@ -355,6 +425,9 @@ class WiredTigerTestCase(unittest.TestCase): else: self.pr('preserving directory ' + self.testdir) + elapsed = time.time() - self.starttime + if elapsed > 0.001 and WiredTigerTestCase._verbose >= 2: + print "%s: %.2f seconds" % (str(self), elapsed) if not passed and not skipped: print "ERROR in " + str(self) self.pr('FAIL') diff --git a/src/third_party/wiredtiger/test/syscall/Makefile.am b/src/third_party/wiredtiger/test/syscall/Makefile.am new file mode 100644 index 00000000000..1d55b06c388 --- /dev/null +++ b/src/third_party/wiredtiger/test/syscall/Makefile.am @@ -0,0 +1,7 @@ +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include +LDADD = $(top_builddir)/libwiredtiger.la +AM_LDFLAGS = -static +noinst_PROGRAMS = + +test_wt2336_base_SOURCES = wt2336_base/main.c +noinst_PROGRAMS += test_wt2336_base diff --git a/src/third_party/wiredtiger/test/syscall/syscall.py b/src/third_party/wiredtiger/test/syscall/syscall.py new file mode 100644 index 00000000000..59c2f347146 --- /dev/null +++ b/src/third_party/wiredtiger/test/syscall/syscall.py @@ -0,0 +1,857 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2016 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# syscall.py +# Command line syscall test runner +# +# For each .run file, run the corresponding program and collect strace +# output, comparing it to the contents of the .run file. +# +# run files are first preprocessed, which means the use of #ifdefs, #defines +# and #includes are allowed, as well as /**/ and // comments. +# Expressions are evaluated using the Python parser, so that: +# hex and octal numbers are accepted, constant values can be or-ed. +# +from __future__ import print_function +import argparse, distutils.spawn, fnmatch, os, platform, re, shutil, \ + subprocess, sys + +# A class that represents a context in which predefined constants can be +# set, and new variables can be assigned. +class VariableContext(object): + def __getitem__(self, key): + if key not in dir(self) or key.startswith('__'): + raise KeyError(key) + return getattr(self, key) + + def __setitem__(self, key, value): + setattr(self, key, value) + +################################################################ +# Changable parameters +# We expect these values to evolve as tests are added or modified. + +# Generally, system calls must be wrapped in an ASSERT_XX() "macro". +# Exceptions are calls in this list that return 0 on success, or +# those that are hardcoded in Runner.call_compare() +calls_returning_zero = [ 'close', 'ftruncate', 'fdatasync', 'rename' ] + +# Encapsulate all the defines we can use in our scripts. +# When this program is run, we'll find out their actual values on +# the host system. +defines_used = [ + 'HAVE_FTRUNCATE', 'O_ACCMODE', 'O_APPEND', 'O_ASYNC', + 'O_CLOEXEC', 'O_CREAT', 'O_EXCL', 'O_EXLOCK', 'O_NOATIME', + 'O_NOFOLLOW', 'O_NONBLOCK', 'O_RDONLY', 'O_RDWR', 'O_SHLOCK', + 'O_TRUNC', 'O_WRONLY' ] + +################################################################ + +# Patterns that are used to match the .run file and/or the output. +ident = r'([a-zA-Z_][a-zA-Z0-9_]*)' +outputpat = re.compile(r'OUTPUT\("([^"]*)"\)') +argpat = re.compile(r'''((?:[^,"']|"[^"]*"|'[^']*')+)''') +discardpat = re.compile(r';') + +# e.g. fd = open("blah", 0, 0); +assignpat = re.compile(ident + r'\s*=\s*' + ident + r'(\([^;]*\));') + +# e.g. ASSERT_EQ(close(fd), 0); +assertpat = re.compile(r'ASSERT_([ENLG][QET])\s*\(\s*' + ident + r'\s*(\(.*\))\s*,\s*([a-zA-Z0-9_]+)\);') + +# e.g. close(fd); must return 0 +callpat = re.compile(ident + r'(\(.*\));') + +# e.g. open("blah", 0x0, 0x0) = 6 0 +# We capture the errno (e.g. "0" or "Err#60"), but don't do anything with it. +# We don't currently test anything that is errno dependent. +dtruss_pat = re.compile(ident + r'(\(.*\))\s*=\s*(-*[0-9xA-F]+)\s+([-A-Za-z#0-9]*)') +# At the top of the dtruss output is a fixed string. +dtruss_init_pat = re.compile(r'\s*SYSCALL\(args\)\s*=\s*return\s*') + +strace_pat = re.compile(ident + r'(\(.*\))\s*=\s(-*[0-9]+)()') + +tracepat = re.compile(r'TRACE\("([^"]*)"\)') +runpat = re.compile(r'RUN\(([^\)]*)\)') +systempat = re.compile(r'SYSTEM\("([^"]*)"\)') +# If tracepat matches, set map['trace_syscalls'] to the 0'th group, etc. +headpatterns = [ [ tracepat, 'trace_syscalls', 0], + [ systempat, 'required_system', 0], + [ runpat, 'run_args', 0] ] + +pwrite_in = r'pwrite64' +pwrite_out = r'pwrite' + +# To create breakpoints while debugging this script +def bp(): + import pdb + pdb.set_trace() + +def msg(s): + print("syscall.py: " + s, file=sys.stderr) + +def die(s): + msg(s) + sys.exit(1) + +# If wttop appears as a prefix of pathname, strip it off. +def simplify_path(wttop, pathname): + wttop = os.path.join(wttop, "") + if pathname.startswith(wttop): + pathname = pathname[len(wttop):] + return pathname + +def printfile(pathname, abbrev): + print("================================================================") + print(abbrev + " (" + pathname + "):") + with open(pathname, 'r') as f: + shutil.copyfileobj(f, sys.stdout) + print("================================================================") + +# A line from a file: a modified string with the file name and line number +# associated with it. +class FileLine(str): + filename = None + linenum = 0 + def __new__(cls, value, *args, **kwargs): + return super(FileLine, cls).__new__(cls, value) + + def prefix(self): + return self.filename + ':' + str(self.linenum) + ': ' + + def range_prefix(self, otherline): + if self == otherline: + othernum = '' + elif otherline == None: + othernum = '-EOF' + else: + othernum = '-' + str(otherline.linenum) + return self.filename + ':' + str(self.linenum) + othernum + ': ' + +# Manage reading from a file, tracking line numbers. +class Reader(object): + # 'raw' means we don't ignore any lines + # 'is_cpp' means input lines beginning with '#' indicate file/linenumber + def __init__(self, wttop, filename, f, raw = True, is_cpp = False): + self.wttop = wttop + self.orig_filename = filename + self.filename = filename + self.f = f + self.linenum = 1 + self.raw = raw + self.is_cpp = is_cpp + if not self.f: + die(self.filename + ': cannot open') + + def __enter__(self): + if not self.f: + return False + return self + + def __exit__(self, typ, value, traceback): + if self.f: + self.f.close() + self.f = None + + # Return True if the line is to be ignored. + def ignore(self, line): + if self.raw: + return False + return line == '' + + # strip a line of comments + def strip_line(self, line): + if not line: + return None + line = line.strip() + if self.is_cpp and line.startswith('#'): + parts = line.split() + if len(parts) < 3 or not parts[1].isdigit(): + msg('bad cpp input: ' + line) + line = '' + self.linenum = int(parts[1]) - 1 + self.filename = parts[2].strip('"') + if self.filename == '': + self.filename = self.orig_filename + if '//' in line: + if line.startswith('//'): + line = '' + else: + # This isn't exactly right, it would see "; //" + # within a string or comment. + m = re.match(r'^(.*;|.*\.\.\.)\s*//', line) + if m: + line = m.groups()[0].strip() + return line + + def readline(self): + line = self.strip_line(self.f.readline()) + while line != None and self.ignore(line): + self.linenum += 1 + line = self.strip_line(self.f.readline()) + if line: + line = FileLine(line) + line.filename = self.filename + line.linenum = self.linenum + self.linenum += 1 + else: + line = '' # make this somewhat compatible with file.readline + return line + + def close(self): + self.f.close() + +# Read from a regular file. +class FileReader(Reader): + def __init__(self, wttop, filename, raw = True): + return super(FileReader, self).__init__(wttop, filename, + open(filename), raw, False) + +# Read from the C preprocessor run on a file. +class PreprocessedReader(Reader): + def __init__(self, wttop, filename, predefines, raw = True): + sourcedir = os.path.dirname(filename) + cmd = ['cc', '-E', '-I' + sourcedir] + for name in dir(predefines): + if not name.startswith('__'): + cmd.append('-D' + name + '=' + str(predefines[name])) + cmd.append('-') + proc = subprocess.Popen(cmd, stdin=open(filename), + stdout=subprocess.PIPE) + super(PreprocessedReader, self).__init__(wttop, filename, + proc.stdout, raw, True) + +# Track options discovered in the 'head' section of the .run file. +class HeadOpts: + def __init__(self): + self.run_args = None + self.required_system = None + self.trace_syscalls = None + +# Manage a run of the target program characterized by a .run file, +# comparing output from the run and reporting differences. +class Runner: + def __init__(self, wttopdir, runfilename, exedir, testexe, + strace, args, variables, defines): + self.variables = variables + self.defines = defines + self.wttopdir = wttopdir + self.runfilename = runfilename + self.testexe = testexe + self.exedir = exedir + self.strace = strace + self.args = args + self.headopts = HeadOpts() + self.dircreated = False + self.strip_syscalls = None + outfilename = args.outfilename + errfilename = args.errfilename + if outfilename == None: + self.outfilename = os.path.join(exedir, 'stdout.txt') + else: + self.outfilename = outfilename + if errfilename == None: + self.errfilename = os.path.join(exedir, 'stderr.txt') + else: + self.errfilename = errfilename + + self.runfile = PreprocessedReader(self.wttopdir, runfilename, + self.defines, False) + + def init(self, systemtype): + # Read up until 'RUN()', setting attributes of self.headopts + runline = '?' + m = None + while runline: + runline = self.runfile.readline() + m = None + for pat,attr,group in headpatterns: + m = re.match(pat, runline) + if m: + setattr(self.headopts, attr, m.groups()[group]) + break + if not m: + self.fail(runline, "unknown header option: " + runline) + return [ False, False ] + if self.headopts.run_args != None: # found RUN()? + break + if not self.headopts.trace_syscalls: + msg("'" + self.runfile.filename + "': needs TRACE(...)") + return [ False, False ] + runargs = self.headopts.run_args.strip() + if len(runargs) > 0: + if len(runargs) < 2 or runargs[0] != '"' or runargs[-1] != '"': + msg("'" + self.runfile.filename + + "': Missing double quotes in RUN arguments") + return [ False, False ] + runargs = runargs[1:-1] + self.runargs = runargs.split() + #print("SYSCALLS: " + self.headopts.trace_syscalls + if self.headopts.required_system != None and \ + self.headopts.required_system != systemtype: + msg("skipping '" + self.runfile.filename + "': for '" + + self.headopts.required_system + "', this system is '" + + systemtype + "'") + return [ False, True ] + return [ True, False ] + + def close(self, forcePreserve): + self.runfile.close() + if self.exedir and self.dircreated and \ + not self.args.preserve and not forcePreserve: + os.chdir('..') + shutil.rmtree(self.exedir) + + def fail(self, line, s): + # make it work if line is None or is a plain string. + try: + prefix = simplify_path(self.wttopdir, line.prefix()) + except: + prefix = 'syscall.py: ' + print(prefix + s, file=sys.stderr) + + def failrange(self, line, lineto, s): + # make it work if line is None or is a plain string. + try: + prefix = simplify_path(self.wttopdir, line.range_prefix(lineto)) + except: + prefix = 'syscall.py: ' + print(prefix + s, file=sys.stderr) + + def str_match(self, s1, s2): + fuzzyRight = False + if len(s1) < 2 or len(s2) < 2: + return False + if s1[-3:] == '...': + fuzzyRight = True + s1 = s1[:-3] + if s2[-3:] == '...': + s2 = s2[:-3] + if s1[0] != '"' or s1[-1] != '"' or s2[0] != '"' or s2[-1] != '"': + return False + s1 = s1[1:-1] + s2 = s2[1:-1] + # We allow a trailing \0 + if s1[-2:] == '\\0': + s1 = s1[:-2] + if s2[-2:] == '\\0': + s2 = s2[:-2] + if fuzzyRight: + return s2.startswith(s2) + else: + return s1 == s2 + + def expr_eval(self, s): + return eval(s, {}, self.variables) + + def arg_match(self, a1, a2): + a1 = a1.strip() + a2 = a2.strip() + if a1 == a2: + return True + if len(a1) == 0 or len(a2) == 0: + return False + if a1[0] == '"': + return self.str_match(a1, a2) + #print(' arg_match: <' + a1 + '> <' + a2 + '>') + try: + a1value = self.expr_eval(a1) + except Exception: + self.fail(a1, 'unknown expression: ' + a1) + return False + try: + a2value = self.expr_eval(a2) + except Exception: + self.fail(a2, 'unknown expression: ' + a2) + return False + return a1value == a2value or int(a1value) == int(a2value) + + def split_args(self, s): + if s[0] == '(': + s = s[1:] + if s[-1] == ')': + s = s[:-1] + return argpat.split(s)[1::2] + + def args_match(self, args1, args2): + #print('args_match: ' + str(s1) + ', ' + str(s2)) + pos = 0 + for a1 in args1: + a1 = a1.strip() + if a1 == '...': # match anything? + return True + if pos >= len(args2): + return False + if not self.arg_match(a1, args2[pos]): + return False + pos += 1 + if pos < len(args2): + return False + return True + + # func(args); is shorthand for for ASSERT_EQ(func(args), xxx); + # where xxx may be 0 or may be derived from one of the args. + def call_compare(self, callname, result, eargs, errline): + if callname in calls_returning_zero: + return self.compare("EQ", result, "0", errline) + elif callname == 'pwrite' or callname == 'pwrite64': + return self.compare("EQ", + re.sub(pwrite_in, pwrite_out, result), + re.sub(pwrite_in, pwrite_out, eargs[2]), + errline) + else: + self.fail(errline, 'call ' + callname + + ': not known, use ASSERT_EQ()') + + def compare(self, compareop, left, right, errline): + l = self.expr_eval(left) + r = self.expr_eval(right) + if (compareop == "EQ" and l == r) or \ + (compareop == "NE" and l != r) or \ + (compareop == "LT" and l < r) or \ + (compareop == "LE" and l <= r) or \ + (compareop == "GT" and l > r) or \ + (compareop == "GE" and l >= r): + return True + else: + self.fail(errline, + 'call returned value: ' + left + ', comparison: (' + + left + ' ' + compareop + ' ' + right + + ') at line: ' + errline) + return False + + def match_report(self, runline, errline, verbose, skiplines, result, desc): + if result: + if verbose: + print('MATCH:') + print(' ' + runline.prefix() + runline) + print(' ' + errline.prefix() + errline) + else: + if verbose: + if not skiplines: + msg('Expecting ' + desc) + print(' ' + runline.prefix() + runline + + ' does not match:') + print(' ' + errline.prefix() + errline) + else: + print(' (... match) ' + errline.prefix() + errline) + return result + + def match(self, runline, errline, verbose, skiplines): + m = re.match(outputpat, runline) + if m: + outwant = m.groups()[0] + return self.match_report(runline, errline, verbose, skiplines, + errline == outwant, 'output line') + if self.args.systype == 'Linux': + em = re.match(strace_pat, errline) + elif self.args.systype == 'Darwin': + em = re.match(dtruss_pat, errline) + if not em: + self.fail(errline, 'Unknown strace/dtruss output: ' + errline) + return False + gotcall = re.sub(pwrite_in, pwrite_out, em.groups()[0]) + # filtering syscalls here if needed. If it's not a match, + # mark the errline so it is retried. + if self.strip_syscalls != None and gotcall not in self.strip_syscalls: + errline.skip = True + return False + m = re.match(assignpat, runline) + if m: + if m.groups()[1] != gotcall: + return self.match_report(runline, errline, verbose, skiplines, + False, 'syscall to match assignment') + + rargs = self.split_args(m.groups()[2]) + eargs = self.split_args(em.groups()[1]) + result = self.args_match(rargs, eargs) + if result: + self.variables[m.groups()[0]] = em.groups()[2] + return self.match_report(runline, errline, verbose, skiplines, + result, 'syscall to match assignment') + + # pattern groups using example ASSERT_EQ(close(fd), 0); + # 0 : comparison op ("EQ") + # 1 : function call name "close" + # 2 : function call args "(fd)" + # 3 : comparitor "0" + m = re.match(assertpat, runline) + if m: + if m.groups()[1] != gotcall: + return self.match_report(runline, errline, verbose, skiplines, + False, 'syscall to match ASSERT') + + rargs = self.split_args(m.groups()[2]) + eargs = self.split_args(em.groups()[1]) + result = self.args_match(rargs, eargs) + if not result: + return self.match_report(runline, errline, verbose, skiplines, + result, 'syscall to match ASSERT') + result = self.compare(m.groups()[0], em.groups()[2], + m.groups()[3], errline) + return self.match_report(runline, errline, verbose, skiplines, + result, 'ASSERT') + + # A call without an enclosing ASSERT is reduced to an ASSERT, + # depending on the particular system call. + m = re.match(callpat, runline) + if m: + if m.groups()[0] != gotcall: + return self.match_report(runline, errline, verbose, skiplines, + False, 'syscall') + + rargs = self.split_args(m.groups()[1]) + eargs = self.split_args(em.groups()[1]) + result = self.args_match(rargs, eargs) + if not result: + return self.match_report(runline, errline, verbose, skiplines, + result, 'syscall') + result = self.call_compare(m.groups()[0], em.groups()[2], + eargs, errline) + return self.match_report(runline, errline, verbose, skiplines, + result, 'syscall') + + self.fail(runline, 'unrecognized pattern in runfile:' + runline) + return False + + def match_lines(self): + outfile = FileReader(self.wttopdir, self.outfilename, True) + errfile = FileReader(self.wttopdir, self.errfilename, True) + + if outfile.readline(): + self.fail(None, 'output file has content, expected to be empty') + return False + + with outfile, errfile: + runlines = self.order_runfile(self.runfile) + errline = errfile.readline() + errline = re.sub(pwrite_in, pwrite_out, errline) + if re.match(dtruss_init_pat, errline): + errline = errfile.readline() + skiplines = False + for runline in runlines: + runline = re.sub(pwrite_in, pwrite_out, runline) + if runline == '...': + skiplines = True + if self.args.verbose: + print('Fuzzy matching:') + print(' ' + runline.prefix() + runline) + continue + first_errline = errline + while errline and not self.match(runline, errline, + self.args.verbose, skiplines): + if skiplines or hasattr(errline, 'skip'): + errline = errfile.readline() + else: + self.fail(runline, "expecting " + runline) + self.failrange(first_errline, errline, "does not match") + return False + if not errline: + self.fail(runline, "failed to match line: " + runline) + self.failrange(first_errline, errline, "does not match") + return False + errline = errfile.readline() + if re.match(dtruss_init_pat, errline): + errline = errfile.readline() + skiplines = False + if errline and not skiplines: + self.fail(errline, "extra lines seen starting at " + errline) + return False + return True + + def order_runfile(self, f): + # In OS X, dtruss is implemented using dtrace's apparently buffered + # printf writes to stdout, but that is all redirected to stderr. + # Because of that, the test program's writes to stderr do not + # interleave with dtruss output as it does with Linux's strace + # (which writes directly to stderr). On OS X, we get the program's + # output first, we compensate for this by moving all the + # OUTPUT statements in the runfile to match first. This simple + # approach will break if there is more data generated by OUTPUT + # statements than a stdio buffer's size. + matchout = (self.args.systype == 'Darwin') + out = [] + nonout = [] + s = f.readline() + while s: + if matchout and re.match(outputpat, s): + out.append(s) + elif not re.match(discardpat, s): + nonout.append(s) + s = f.readline() + out.extend(nonout) + return out + + def run(self): + if not self.exedir: + self.fail(None, "Execution directory not set") + return False + if not os.path.isfile(self.testexe): + msg("'" + self.testexe + "': no such file") + return False + + shutil.rmtree(self.exedir, ignore_errors=True) + os.mkdir(self.exedir) + self.dircreated = True + os.chdir(self.exedir) + + callargs = list(self.strace) + trace_syscalls = self.headopts.trace_syscalls + if self.args.systype == 'Linux': + callargs.extend(['-e', 'trace=' + trace_syscalls ]) + elif self.args.systype == 'Darwin': + # dtrace has no option to limit the syscalls to be traced, + # so we'll filter the output. + self.strip_syscalls = re.sub(pwrite_in, pwrite_out, + self.headopts.trace_syscalls).split(',') + callargs.append(self.testexe) + callargs.extend(self.runargs) + + outfile = open(self.outfilename, 'w') + errfile = open(self.errfilename, 'w') + if self.args.verbose: + print('RUNNING: ' + str(callargs)) + subret = subprocess.call(callargs, stdout=outfile, stderr=errfile) + outfile.close() + errfile.close() + if subret != 0: + msg("'" + self.testexe + "': exit value " + str(subret)) + printfile(self.outfilename, "output") + printfile(self.errfilename, "error") + return False + return True + +# Run the syscall program. +class SyscallCommand: + def __init__(self, disttop, builddir): + self.disttop = disttop + self.builddir = builddir + + def parse_args(self, argv): + srcdir = os.path.join(self.disttop, 'test', 'syscall') + self.exetopdir = os.path.join(self.builddir, 'test', 'syscall') + self.incdir1 = os.path.join(self.disttop, 'src', 'include') + self.incdir2 = self.builddir + + ap = argparse.ArgumentParser('Syscall test runner') + ap.add_argument('--systype', + help='override system type (Linux/Windows/Darwin)') + ap.add_argument('--errfile', dest='errfilename', + help='do not run the program, use this file as stderr') + ap.add_argument('--outfile', dest='outfilename', + help='do not run the program, use this file as stdout') + ap.add_argument('--preserve', action="store_true", + help='keep the WT_TEST.* directories') + ap.add_argument('--verbose', action="store_true", + help='add some verbose information') + ap.add_argument('tests', nargs='*', + help='the tests to run (defaults to all)') + args = ap.parse_args() + + if not args.systype: + args.systype = platform.system() # Linux, Windows, Darwin + + self.dorun = True + if args.errfilename or args.outfilename: + if len(args.tests) != 1: + msg("one test is required when --errfile or --outfile" + + " is specified") + return False + if not args.outfilename: + args.outfilename = os.devnull + if not args.errfilename: + args.errfilename = os.devnull + self.dorun = False + + # for now, we permit Linux and Darwin + straceexe = None + if args.systype == 'Linux': + strace = [ 'strace' ] + straceexe = 'strace' + elif args.systype == 'Darwin': + strace = [ 'sudo', 'dtruss' ] + straceexe = 'dtruss' + else: + msg("systype '" + args.systype + "' unsupported") + return False + if not distutils.spawn.find_executable(straceexe): + msg("strace: does not exist") + return False + self.args = args + self.strace = strace + return True + + def runone(self, runfilename, exedir, testexe, args): + result = True + runner = Runner(self.disttop, runfilename, exedir, testexe, + self.strace, args, self.variables, self.defines) + okay, skip = runner.init(args.systype) + if not okay: + if not skip: + result = False + else: + if testexe: + print('running ' + testexe) + if not runner.run(): + result = False + if result: + print('comparing:') + print(' ' + simplify_path(self.disttop, runfilename)) + print(' ' + simplify_path(self.disttop, runner.errfilename)) + result = runner.match_lines() + if not result and args.verbose: + printfile(runfilename, "runfile") + printfile(runner.errfilename, "trace output") + runner.close(not result) + if not result: + print('************************ FAILED ************************') + print(' see results in ' + exedir) + print('') + return result + + # Create a C program to get values for all defines we need. + # The output of the program is Python code that we'll execute + # directly to set the values. + def build_system_defines(self): + # variables is a symbol table that is used to + # evaluate expressions both in the .run file and + # in the output file. This is needed for strace, + # which shows system call flags in symbolic form. + self.variables = VariableContext() + # defines is a symbol table that is used to + # create preprocessor defines, effectively evaluating + # all flag defines in the .run file. + self.defines = VariableContext() + program = \ + '#include \n' + \ + '#include \n' + \ + '#include \n' + \ + 'int main() {\n' + for define in defines_used: + program += '#ifdef ' + define + '\n' + # output is Python that sets attributes of 'o'. + program += ' printf("o.' + define + '=%d\\n", ' + \ + define + ');\n' + program += '#endif\n' + program += \ + ' return(0);\n' + \ + '}\n' + probe_c = os.path.join(self.exetopdir, "syscall_probe.c") + probe_exe = os.path.join(self.exetopdir, "syscall_probe") + with open(probe_c, "w") as f: + f.write(program) + ccargs = ['cc', '-o', probe_exe] + ccargs.append('-I' + self.incdir1) + ccargs.append('-I' + self.incdir2) + if self.args.systype == 'Linux': + ccargs.append('-D_GNU_SOURCE') + ccargs.append(probe_c) + subret = subprocess.call(ccargs) + if subret != 0: + msg("probe compilation returned " + str(subret)) + return False + proc = subprocess.Popen([probe_exe], stdout=subprocess.PIPE) + out, err = proc.communicate() + subret = proc.returncode + if subret != 0 or err: + msg("probe run returned " + str(subret) + ", error=" + str(err)) + return False + if self.args.verbose: + print('probe output:\n' + out) + o = self.defines # The 'o' object will be modified. + exec(out) # Run the produced Python. + o = self.variables # Set these in variables too, so strace + exec(out) # symbolic output is evaluated. + if not self.args.preserve: + os.remove(probe_c) + os.remove(probe_exe) + return True + + def execute(self): + args = self.args + result = True + if not self.build_system_defines(): + die('cannot build system defines') + if not self.dorun: + for testname in args.tests: + result = self.runone(testname, None, None, args) and result + else: + if len(args.tests) > 0: + tests = [] + for arg in args.tests: + abspath = os.path.abspath(arg) + tests.append([os.path.dirname(abspath), [], + [os.path.basename(abspath)]]) + else: + tests = os.walk(syscalldir) + os.chdir(self.exetopdir) + for path, subdirs, files in tests: + testnum = -1 if len(files) <= 1 else 0 + for name in files: + if fnmatch.fnmatch(name, '*.run'): + testname = os.path.basename(os.path.normpath(path)) + runfilename = os.path.join(path, name) + testexe = os.path.join(self.exetopdir, + 'test_' + testname) + exedir = os.path.join(self.exetopdir, + 'WT_TEST.' + testname) + # If there are multiple tests in this directory, + # give each one its own execution dir. + if testnum >= 0: + exedir += '.' + str(testnum) + testnum += 1 + result = self.runone(runfilename, exedir, + testexe, args) and result + return result + +# Set paths, determining the top of the build. +syscalldir = sys.path[0] +wt_disttop = os.path.dirname(os.path.dirname(syscalldir)) + +# Note: this code is borrowed from test/suite/run.py +# Check for a local build that contains the wt utility. First check in +# current working directory, then in build_posix and finally in the disttop +# directory. This isn't ideal - if a user has multiple builds in a tree we +# could pick the wrong one. +if os.path.isfile(os.path.join(os.getcwd(), 'wt')): + wt_builddir = os.getcwd() +elif os.path.isfile(os.path.join(wt_disttop, 'wt')): + wt_builddir = wt_disttop +elif os.path.isfile(os.path.join(wt_disttop, 'build_posix', 'wt')): + wt_builddir = os.path.join(wt_disttop, 'build_posix') +elif os.path.isfile(os.path.join(wt_disttop, 'wt.exe')): + wt_builddir = wt_disttop +else: + die('unable to find useable WiredTiger build') + +cmd = SyscallCommand(wt_disttop, wt_builddir) +if not cmd.parse_args(sys.argv): + die('bad usage') +if not cmd.execute(): + sys.exit(1) +sys.exit(0) diff --git a/src/third_party/wiredtiger/test/syscall/wt2336_base/base.run b/src/third_party/wiredtiger/test/syscall/wt2336_base/base.run new file mode 100644 index 00000000000..7d2c42ce64e --- /dev/null +++ b/src/third_party/wiredtiger/test/syscall/wt2336_base/base.run @@ -0,0 +1,180 @@ +// Public Domain 2014-2016 MongoDB, Inc. +// Public Domain 2008-2014 WiredTiger, Inc. +// +// This is free and unencumbered software released into the public domain. +// +// Anyone is free to copy, modify, publish, use, compile, sell, or +// distribute this software, either in source code form or as a compiled +// binary, for any purpose, commercial or non-commercial, and by any +// means. +// +// In jurisdictions that recognize copyright laws, the author or authors +// of this software dedicate any and all copyright interest in the +// software to the public domain. We make this dedication for the benefit +// of the public at large and to the detriment of our heirs and +// successors. We intend this dedication to be an overt act of +// relinquishment in perpetuity of all present and future rights to this +// software under copyright law. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +// IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// +// base.run +// Command line syscall test runner +// +#ifdef __linux__ +SYSTEM("Linux"); +#define OPEN_EXISTING(name, flags) open(name, flags) +#else /* __linux__ */ +SYSTEM("Darwin"); +#define O_NOATIME 0 +#define OPEN_EXISTING(name, flags) open(name, flags, 0) +#endif /* __linux__ */ + +#ifdef HAVE_FTRUNCATE +#ifdef __linux__ +#define FTRUNCATE(fd, len) ftruncate(fd, len) +#else +#define FTRUNCATE(fd, len) ftruncate(fd, len, len) +#endif +#else +#define FTRUNCATE(fd, len) /* do nothing */ +#endif + +TRACE("close,fdatasync,fsync,ftruncate,open,pwrite64,rename"); +RUN(""); +... +OUTPUT("--------------wiredtiger_open"); +// lock == 3 +lock = open("./WiredTiger.lock", O_RDWR|O_CREAT|O_CLOEXEC, 0666); +pwrite64(lock, "WiredTiger lock file\n", 0x15, 0x0); +fd = open("./WiredTiger", O_RDWR|O_CREAT|O_CLOEXEC, 0666); +pwrite64(fd, "WiredTiger\nWiredTiger"..., ...); +#ifdef __linux__ +fdatasync(fd); +#endif /* __linux__ */ +close(fd); + +... // On Linux, there are calls to open and read "/proc/meminfo" here. + +fd = open("./WiredTiger.basecfg.set", O_RDWR|O_CREAT|O_EXCL|O_CLOEXEC, 0666); +pwrite64(fd, "# Do not modify this file."..., ...); +#ifdef __linux__ +fdatasync(fd); +#endif /* __linux__ */ +close(fd); +rename("./WiredTiger.basecfg.set", "./WiredTiger.basecfg"); + +#ifdef __linux__ +dir = open("./", O_RDONLY); +fdatasync(dir); +close(dir); +#endif + +fd = open("./WiredTiger.wt", O_RDWR|O_CREAT|O_EXCL|O_NOATIME|O_CLOEXEC, 0666); + +#ifdef __linux__ +dir = open("./", O_RDONLY); +fdatasync(dir); +close(dir); +#endif /* __linux__ */ + +pwrite64(fd, ""..., 0x1000, 0x0); +#ifdef __linux__ +fdatasync(fd); +#endif /* __linux__ */ +close(fd); + +wt = OPEN_EXISTING("./WiredTiger.wt\0", O_RDWR|O_NOATIME|O_CLOEXEC); +FTRUNCATE(wt, 0x1000); + +fd = open("./WiredTiger.turtle.set\0", O_RDWR|O_CREAT|O_EXCL|O_CLOEXEC, 0666); +pwrite64(fd, "WiredTiger version string\nWiredTiger"..., ...); +#ifdef __linux__ +fdatasync(fd); +#endif /* __linux__ */ +close(fd); +rename("./WiredTiger.turtle.set", "./WiredTiger.turtle"); + +... // There is a second open of turtle here, is it important? + +fd = open("./WiredTigerLAS.wt", O_RDWR|O_CREAT|O_EXCL|O_NOATIME|O_CLOEXEC, 0666); + +#ifdef __linux__ +dir = open("./", O_RDONLY); +fdatasync(dir); +close(dir); +#endif /* __linux__ */ + +pwrite64(fd, ""..., 0x1000, 0x0); + +#ifdef __linux__ +fdatasync(fd); +#endif /* __linux__ */ + +close(fd); +fd = OPEN_EXISTING("./WiredTigerLAS.wt", O_RDWR|O_NOATIME|O_CLOEXEC); +FTRUNCATE(fd, 0x1000); +fd = OPEN_EXISTING("./WiredTiger.turtle", O_RDWR|O_CLOEXEC); +close(fd); +pwrite64(wt, ""..., 0x1000, 0x1000); +pwrite64(wt, ""..., 0x1000, 0x2000); +pwrite64(wt, ""..., 0x1000, 0x3000); +#ifdef __linux__ +fdatasync(wt); +#endif /* __linux__ */ +fd = OPEN_EXISTING("./WiredTiger.turtle", O_RDWR|O_CLOEXEC); + +close(fd); +fd = open("./WiredTiger.turtle.set", O_RDWR|O_CREAT|O_EXCL|O_CLOEXEC, 0666); +pwrite64(fd, "WiredTiger version string\nWiredTiger"..., ...); +#ifdef __linux__ +fdatasync(fd); +#endif /* __linux__ */ +close(fd); +rename("./WiredTiger.turtle.set", "./WiredTiger.turtle"); +#ifdef __linux__ +dir = open("./", O_RDONLY); +fdatasync(dir); +close(dir); +fdatasync(wt); +#endif /* __linux__ */ + +OUTPUT("--------------open_session"); +OUTPUT("--------------create"); + +hello = open("./hello.wt", O_RDWR|O_CREAT|O_EXCL|O_NOATIME|O_CLOEXEC, 0666); +#ifdef __linux__ +dir = open("./", O_RDONLY); +fdatasync(dir); +close(dir); +#endif /* __linux__ */ +pwrite64(hello, "A\330\001"..., 0x1000, 0x0); +#ifdef __linux__ +fdatasync(hello); +#endif /* __linux__ */ +close(hello); +hello = OPEN_EXISTING("./hello.wt", O_RDWR|O_NOATIME|O_CLOEXEC); +FTRUNCATE(hello, 0x1000); +turtle = OPEN_EXISTING("./WiredTiger.turtle", O_RDWR|O_CLOEXEC); +close(turtle); +... // more changes to turtle +OUTPUT("--------------rename"); +#ifdef __linux__ +FTRUNCATE(hello, 0x1000); +#endif /* __linux__ */ +close(hello); +rename("./hello.wt", "./world.wt"); +// Changes to turtle, more writes to wt. +turtle = OPEN_EXISTING("./WiredTiger.turtle", O_RDWR|O_CLOEXEC); +close(turtle); +... +OUTPUT("--------------drop"); +... +OUTPUT("--------------WT_CONNECTION::close"); +... diff --git a/src/third_party/wiredtiger/test/syscall/wt2336_base/main.c b/src/third_party/wiredtiger/test/syscall/wt2336_base/main.c new file mode 100644 index 00000000000..22420371dd0 --- /dev/null +++ b/src/third_party/wiredtiger/test/syscall/wt2336_base/main.c @@ -0,0 +1,69 @@ +#include + +#include // TODO +#include // TODO +#include + +static void fail(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn)); + +static void +fail(int ret) { + fprintf(stderr, + "%s: %d (%s)\n", + "wt2336_fileop_basic", ret, wiredtiger_strerror(ret)); + exit(ret); +} + +#define SEPARATOR "--------------" + +int +main(int argc, char *argv[]) +{ + int ret; + WT_CONNECTION *conn; + WT_SESSION *session; + + (void)argc; + (void)argv; + fprintf(stderr, SEPARATOR "wiredtiger_open\n"); + if ((ret = wiredtiger_open(".", NULL, "create", &conn)) != 0) + fail(ret); + + usleep(100); + fflush(stderr); + fprintf(stderr, SEPARATOR "open_session\n"); + fflush(stderr); + + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) + fail(ret); + + usleep(100); + fflush(stderr); + fprintf(stderr, SEPARATOR "create\n"); + fflush(stderr); + + if ((ret = session->create( + session, "table:hello", "key_format=S,value_format=S")) != 0) + fail(ret); + + usleep(100); + fprintf(stderr, SEPARATOR "rename\n"); + + if ((ret = session->rename( + session, "table:hello", "table:world", NULL)) != 0) + fail(ret); + + fflush(stdout); + fprintf(stderr, SEPARATOR "drop\n"); + fflush(stdout); + + if ((ret = session->drop(session, "table:world", NULL)) != 0) + fail(ret); + + fprintf(stderr, SEPARATOR "WT_CONNECTION::close\n"); + + if ((ret = conn->close(conn, NULL)) != 0) + fail(ret); + + return (0); +} diff --git a/src/third_party/wiredtiger/test/thread/file.c b/src/third_party/wiredtiger/test/thread/file.c index 81ec6ad44f8..7a7d16c4cd6 100644 --- a/src/third_party/wiredtiger/test/thread/file.c +++ b/src/third_party/wiredtiger/test/thread/file.c @@ -33,20 +33,18 @@ file_create(const char *name) { WT_SESSION *session; int ret; - char *p, *end, config[128]; + char config[128]; if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) testutil_die(ret, "conn.session"); - p = config; - end = config + sizeof(config); - p += snprintf(p, (size_t)(end - p), + testutil_check(__wt_snprintf(config, sizeof(config), "key_format=%s," "internal_page_max=%d," - "leaf_page_max=%d,", - ftype == ROW ? "u" : "r", 16 * 1024, 128 * 1024); - if (ftype == FIX) - (void)snprintf(p, (size_t)(end - p), ",value_format=3t"); + "leaf_page_max=%d," + "%s", + ftype == ROW ? "u" : "r", 16 * 1024, 128 * 1024, + ftype == FIX ? ",value_format=3t" : "")); if ((ret = session->create(session, name, config)) != 0) if (ret != EEXIST) @@ -62,9 +60,10 @@ load(const char *name) WT_CURSOR *cursor; WT_ITEM *key, _key, *value, _value; WT_SESSION *session; - char keybuf[64], valuebuf[64]; - u_int keyno; + uint64_t keyno; + size_t len; int ret; + char keybuf[64], valuebuf[64]; file_create(name); @@ -79,18 +78,22 @@ load(const char *name) value = &_value; for (keyno = 1; keyno <= nkeys; ++keyno) { if (ftype == ROW) { + testutil_check(__wt_snprintf_len_set( + keybuf, sizeof(keybuf), + &len, "%017" PRIu64, keyno)); key->data = keybuf; - key->size = (uint32_t) - snprintf(keybuf, sizeof(keybuf), "%017u", keyno); + key->size = (uint32_t)len; cursor->set_key(cursor, key); } else - cursor->set_key(cursor, (uint32_t)keyno); - value->data = valuebuf; + cursor->set_key(cursor, keyno); if (ftype == FIX) cursor->set_value(cursor, 0x01); else { - value->size = (uint32_t) - snprintf(valuebuf, sizeof(valuebuf), "%37u", keyno); + testutil_check(__wt_snprintf_len_set( + valuebuf, sizeof(valuebuf), + &len, "%37" PRIu64, keyno)); + value->data = valuebuf; + value->size = (uint32_t)len; cursor->set_value(cursor, value); } if ((ret = cursor->insert(cursor)) != 0) diff --git a/src/third_party/wiredtiger/test/thread/rw.c b/src/third_party/wiredtiger/test/thread/rw.c index c6107a06c49..e8a2650ca51 100644 --- a/src/third_party/wiredtiger/test/thread/rw.c +++ b/src/third_party/wiredtiger/test/thread/rw.c @@ -66,7 +66,8 @@ rw_start(u_int readers, u_int writers) for (i = 0; i < writers; ++i) { if (i == 0 || multiple_files) { run_info[i].name = dmalloc(64); - snprintf(run_info[i].name, 64, FNAME, i); + testutil_check(__wt_snprintf( + run_info[i].name, 64, FNAME, i)); /* Vary by orders of magnitude */ if (vary_nops) @@ -88,8 +89,8 @@ rw_start(u_int readers, u_int writers) run_info[offset].name = dmalloc(64); /* Have readers read from tables with writes. */ name_index = i % writers; - snprintf( - run_info[offset].name, 64, FNAME, name_index); + testutil_check(__wt_snprintf( + run_info[offset].name, 64, FNAME, name_index)); /* Vary by orders of magnitude */ if (vary_nops) @@ -158,7 +159,8 @@ static inline void reader_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) { WT_ITEM *key, _key; - u_int keyno; + size_t len; + uint64_t keyno; int ret; char keybuf[64]; @@ -166,17 +168,18 @@ reader_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) keyno = __wt_random(&s->rnd) % nkeys + 1; if (ftype == ROW) { + testutil_check(__wt_snprintf_len_set( + keybuf, sizeof(keybuf), &len, "%017" PRIu64, keyno)); key->data = keybuf; - key->size = (uint32_t) - snprintf(keybuf, sizeof(keybuf), "%017u", keyno); + key->size = (uint32_t)len; cursor->set_key(cursor, key); } else - cursor->set_key(cursor, (uint32_t)keyno); + cursor->set_key(cursor, keyno); if ((ret = cursor->search(cursor)) != 0 && ret != WT_NOTFOUND) testutil_die(ret, "cursor.search"); if (log_print) testutil_check(session->log_printf(session, - "Reader Thread %p key %017u", pthread_self(), keyno)); + "Reader Thread %p key %017" PRIu64, pthread_self(), keyno)); } /* @@ -195,7 +198,7 @@ reader(void *arg) id = (int)(uintptr_t)arg; s = &run_info[id]; - __wt_thread_id(tid, sizeof(tid)); + testutil_check(__wt_thread_id(tid, sizeof(tid))); __wt_random_init(&s->rnd); printf(" read thread %2d starting: tid: %s, file: %s\n", @@ -242,7 +245,8 @@ static inline void writer_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) { WT_ITEM *key, _key, *value, _value; - u_int keyno; + uint64_t keyno; + size_t len; int ret; char keybuf[64], valuebuf[64]; @@ -251,12 +255,13 @@ writer_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) keyno = __wt_random(&s->rnd) % nkeys + 1; if (ftype == ROW) { + testutil_check(__wt_snprintf_len_set( + keybuf, sizeof(keybuf), &len, "%017" PRIu64, keyno)); key->data = keybuf; - key->size = (uint32_t) - snprintf(keybuf, sizeof(keybuf), "%017u", keyno); + key->size = (uint32_t)len; cursor->set_key(cursor, key); } else - cursor->set_key(cursor, (uint32_t)keyno); + cursor->set_key(cursor, keyno); if (keyno % 5 == 0) { ++s->remove; if ((ret = @@ -268,8 +273,10 @@ writer_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) if (ftype == FIX) cursor->set_value(cursor, 0x10); else { - value->size = (uint32_t)snprintf( - valuebuf, sizeof(valuebuf), "XXX %37u", keyno); + testutil_check(__wt_snprintf_len_set( + valuebuf, sizeof(valuebuf), + &len, "XXX %37" PRIu64, keyno)); + value->size = (uint32_t)len; cursor->set_value(cursor, value); } if ((ret = cursor->update(cursor)) != 0) @@ -277,7 +284,7 @@ writer_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s) } if (log_print) testutil_check(session->log_printf(session, - "Writer Thread %p key %017u", pthread_self(), keyno)); + "Writer Thread %p key %017" PRIu64, pthread_self(), keyno)); } /* @@ -296,7 +303,7 @@ writer(void *arg) id = (int)(uintptr_t)arg; s = &run_info[id]; - __wt_thread_id(tid, sizeof(tid)); + testutil_check(__wt_thread_id(tid, sizeof(tid))); __wt_random_init(&s->rnd); printf("write thread %2d starting: tid: %s, file: %s\n", diff --git a/src/third_party/wiredtiger/test/thread/stats.c b/src/third_party/wiredtiger/test/thread/stats.c index 67a2c02719b..839d65e8a4d 100644 --- a/src/third_party/wiredtiger/test/thread/stats.c +++ b/src/third_party/wiredtiger/test/thread/stats.c @@ -65,7 +65,8 @@ stats(void) /* File statistics. */ if (!multiple_files) { - (void)snprintf(name, sizeof(name), "statistics:" FNAME, 0); + testutil_check(__wt_snprintf( + name, sizeof(name), "statistics:" FNAME, 0)); if ((ret = session->open_cursor( session, name, NULL, NULL, &cursor)) != 0) testutil_die(ret, "session.open_cursor"); diff --git a/src/third_party/wiredtiger/test/thread/t.c b/src/third_party/wiredtiger/test/thread/t.c index baadbf2adb9..d2ed4c74bb7 100644 --- a/src/third_party/wiredtiger/test/thread/t.c +++ b/src/third_party/wiredtiger/test/thread/t.c @@ -37,7 +37,6 @@ int multiple_files; /* File per thread */ int session_per_op; /* New session per operation */ static char home[512]; /* Program working dir */ -static char *progname; /* Program name */ static FILE *logfp; /* Log file */ static int handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *); @@ -59,10 +58,7 @@ main(int argc, char *argv[]) int ch, cnt, runs; char *config_open, *working_dir; - if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) - progname = argv[0]; - else - ++progname; + (void)testutil_set_progname(argv); config_open = NULL; working_dir = NULL; @@ -189,19 +185,15 @@ wt_connect(char *config_open) }; int ret; char config[512]; - size_t print_count; testutil_clean_work_dir(home); testutil_make_work_dir(home); - print_count = (size_t)snprintf(config, sizeof(config), + testutil_check(__wt_snprintf(config, sizeof(config), "create,statistics=(all),error_prefix=\"%s\",%s%s", progname, config_open == NULL ? "" : ",", - config_open == NULL ? "" : config_open); - - if (print_count >= sizeof(config)) - testutil_die(EINVAL, "Config string too long"); + config_open == NULL ? "" : config_open)); if ((ret = wiredtiger_open(home, &event_handler, config, &conn)) != 0) testutil_die(ret, "wiredtiger_open"); diff --git a/src/third_party/wiredtiger/test/utility/misc.c b/src/third_party/wiredtiger/test/utility/misc.c index 1491c9a6938..934dac86a7b 100644 --- a/src/third_party/wiredtiger/test/utility/misc.c +++ b/src/third_party/wiredtiger/test/utility/misc.c @@ -28,6 +28,7 @@ #include "test_util.h" void (*custom_die)(void) = NULL; +const char *progname = "program name not set"; /* * die -- @@ -42,7 +43,9 @@ testutil_die(int e, const char *fmt, ...) if (custom_die != NULL) (*custom_die)(); + fprintf(stderr, "%s: FAILED", progname); if (fmt != NULL) { + fprintf(stderr, ": "); va_start(ap, fmt); vfprintf(stderr, fmt, ap); va_end(ap); @@ -54,6 +57,20 @@ testutil_die(int e, const char *fmt, ...) exit(EXIT_FAILURE); } +/* + * testutil_set_progname -- + * Set the global program name for error handling. + */ +const char * +testutil_set_progname(char * const *argv) +{ + if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL) + progname = argv[0]; + else + ++progname; + return (progname); +} + /* * testutil_work_dir_from_path -- * Takes a buffer, its size and the intended work directory. @@ -78,7 +95,7 @@ testutil_work_dir_from_path(char *buffer, size_t len, const char *dir) * Remove the work directory. */ void -testutil_clean_work_dir(char *dir) +testutil_clean_work_dir(const char *dir) { size_t len; int ret; @@ -91,14 +108,14 @@ testutil_clean_work_dir(char *dir) if ((buf = malloc(len)) == NULL) testutil_die(ENOMEM, "Failed to allocate memory"); - snprintf(buf, len, "%s %s %s %s", DIR_EXISTS_COMMAND, dir, - RM_COMMAND, dir); + testutil_check(__wt_snprintf( + buf, len, "%s %s %s %s", DIR_EXISTS_COMMAND, dir, RM_COMMAND, dir)); #else len = strlen(dir) + strlen(RM_COMMAND) + 1; if ((buf = malloc(len)) == NULL) testutil_die(ENOMEM, "Failed to allocate memory"); - snprintf(buf, len, "%s%s", RM_COMMAND, dir); + testutil_check(__wt_snprintf(buf, len, "%s%s", RM_COMMAND, dir)); #endif if ((ret = system(buf)) != 0 && ret != ENOENT) @@ -125,7 +142,7 @@ testutil_make_work_dir(char *dir) testutil_die(ENOMEM, "Failed to allocate memory"); /* mkdir shares syntax between Windows and Linux */ - snprintf(buf, len, "%s%s", MKDIR_COMMAND, dir); + testutil_check(__wt_snprintf(buf, len, "%s%s", MKDIR_COMMAND, dir)); if ((ret = system(buf)) != 0) testutil_die(ret, "%s", buf); free(buf); @@ -149,20 +166,25 @@ testutil_cleanup(TEST_OPTS *opts) } /* - * testutil_disable_long_tests -- - * Return if TESTUTIL_DISABLE_LONG_TESTS is set. + * testutil_enable_long_tests -- + * Return if TESTUTIL_ENABLE_LONG_TESTS is set. */ bool -testutil_disable_long_tests(void) +testutil_enable_long_tests(void) { const char *res; + bool enable_long_tests; if (__wt_getenv(NULL, - "TESTUTIL_DISABLE_LONG_TESTS", &res) == WT_NOTFOUND) + "TESTUTIL_ENABLE_LONG_TESTS", &res) == WT_NOTFOUND) return (false); + /* Accept anything other than "TESTUTIL_ENABLE_LONG_TESTS=0". */ + enable_long_tests = res[0] != '0'; + free((void *)res); - return (true); + + return (enable_long_tests); } /* diff --git a/src/third_party/wiredtiger/test/utility/parse_opts.c b/src/third_party/wiredtiger/test/utility/parse_opts.c index 74a1c021d5d..c3eff3360de 100644 --- a/src/third_party/wiredtiger/test/utility/parse_opts.c +++ b/src/third_party/wiredtiger/test/utility/parse_opts.c @@ -43,10 +43,7 @@ testutil_parse_opts(int argc, char * const *argv, TEST_OPTS *opts) opts->running = true; opts->verbose = false; - if ((opts->progname = strrchr(argv[0], DIR_DELIM)) == NULL) - opts->progname = argv[0]; - else - ++opts->progname; + opts->progname = testutil_set_progname(argv); while ((ch = __wt_getopt(opts->progname, argc, argv, "A:h:n:o:pR:T:t:vW:")) != EOF) @@ -118,13 +115,15 @@ testutil_parse_opts(int argc, char * const *argv, TEST_OPTS *opts) if (opts->home == NULL) { len = strlen("WT_TEST.") + strlen(opts->progname) + 10; opts->home = dmalloc(len); - snprintf(opts->home, len, "WT_TEST.%s", opts->progname); + testutil_check(__wt_snprintf( + opts->home, len, "WT_TEST.%s", opts->progname)); } /* Setup the default URI string */ len = strlen("table:") + strlen(opts->progname) + 10; opts->uri = dmalloc(len); - snprintf(opts->uri, len, "table:%s", opts->progname); + testutil_check(__wt_snprintf( + opts->uri, len, "table:%s", opts->progname)); return (0); } diff --git a/src/third_party/wiredtiger/test/utility/test_util.h b/src/third_party/wiredtiger/test/utility/test_util.h index f6a9cd68e02..406ed2c4961 100644 --- a/src/third_party/wiredtiger/test/utility/test_util.h +++ b/src/third_party/wiredtiger/test/utility/test_util.h @@ -48,7 +48,7 @@ /* Generic option parsing structure shared by all test cases. */ typedef struct { char *home; - char *progname; + const char *progname; enum { TABLE_COL=1, /* Fixed-length column store */ TABLE_FIX=2, /* Variable-length column store */ TABLE_ROW=3 /* Row-store */ @@ -183,12 +183,15 @@ void *dmalloc(size_t); void *drealloc(void *, size_t); void *dstrdup(const void *); void *dstrndup(const char *, size_t); -void testutil_clean_work_dir(char *); +void testutil_clean_work_dir(const char *); void testutil_cleanup(TEST_OPTS *); -bool testutil_disable_long_tests(void); +bool testutil_enable_long_tests(void); void testutil_make_work_dir(char *); int testutil_parse_opts(int, char * const *, TEST_OPTS *); void testutil_work_dir_from_path(char *, size_t, const char *); void *thread_append(void *); void *thread_insert_append(void *); void *thread_prev(void *); + +extern const char *progname; +const char *testutil_set_progname(char * const *); diff --git a/src/third_party/wiredtiger/test/utility/thread.c b/src/third_party/wiredtiger/test/utility/thread.c index 38465b2f02b..122ad554442 100644 --- a/src/third_party/wiredtiger/test/utility/thread.c +++ b/src/third_party/wiredtiger/test/utility/thread.c @@ -57,8 +57,8 @@ thread_append(void *arg) if (opts->table_type == TABLE_FIX) cursor->set_value(cursor, buf[0]); else { - snprintf(buf, sizeof(buf), - "%" PRIu64 " VALUE ------", recno); + testutil_check(__wt_snprintf(buf, sizeof(buf), + "%" PRIu64 " VALUE ------", recno)); cursor->set_value(cursor, buf); } testutil_check(cursor->insert(cursor)); @@ -94,7 +94,8 @@ thread_insert_append(void *arg) session, opts->uri, NULL, NULL, &cursor)); for (i = 0; i < opts->nrecords; ++i) { - snprintf(kbuf, sizeof(kbuf), "%010d KEY------", (int)i); + testutil_check(__wt_snprintf( + kbuf, sizeof(kbuf), "%010d KEY------", (int)i)); cursor->set_key(cursor, kbuf); cursor->set_value(cursor, "========== VALUE ======="); testutil_check(cursor->insert(cursor)); diff --git a/src/third_party/wiredtiger/test/windows/windows_shim.h b/src/third_party/wiredtiger/test/windows/windows_shim.h index a6050e92bca..8985904fb19 100644 --- a/src/third_party/wiredtiger/test/windows/windows_shim.h +++ b/src/third_party/wiredtiger/test/windows/windows_shim.h @@ -36,6 +36,8 @@ #include #include +#include "wt_internal.h" + #define inline __inline /* Define some POSIX types */ @@ -52,12 +54,7 @@ typedef int u_int; /* snprintf does not exist on <= VS 2013 */ #if _MSC_VER < 1900 -#define snprintf _wt_snprintf - -_Check_return_opt_ int __cdecl _wt_snprintf( - _Out_writes_(_MaxCount) char * _DstBuf, - _In_ size_t _MaxCount, - _In_z_ _Printf_format_string_ const char * _Format, ...); +#define snprintf __wt_snprintf #endif /* @@ -87,7 +84,7 @@ int usleep(useconds_t useconds); /* - * Emulate the support we need for the tests + * Emulate the support we need for tests and example code. */ typedef CRITICAL_SECTION pthread_mutex_t; typedef CONDITION_VARIABLE pthread_cond_t; diff --git a/src/third_party/wiredtiger/test/wtperf/test_conf_dump.py b/src/third_party/wiredtiger/test/wtperf/test_conf_dump.py new file mode 100644 index 00000000000..ef7f276a1d0 --- /dev/null +++ b/src/third_party/wiredtiger/test/wtperf/test_conf_dump.py @@ -0,0 +1,296 @@ +# Usage: python test_conf_dump.py +# +# This script tests if the config file dumped in the test directory corresponds +# correctly to the wtperf config file used. Command line options to wtperf are +# also taken into account. +# +# Following expectations are checked for: +# 1. If provided through multiple sources, "conn_config" and "table_config" +# configuration options are appended to each other. All other options get +# replaced by a higher precedent source. +# 2. The precedence order for the options in an increasing order is as follows: +# default option, +# provided through config file, +# provided through option -o +# provided through option -C (for conn_config) or -T (for table_config) +# +# Test fails if any config option is missing or has a wrong value. Test also +# fails if the value for the option is not replaced/appended in the correct +# order of precedence as stated above. + +import os, re, subprocess, sys + +OP_FILE = "WT_TEST/CONFIG.wtperf" +TMP_CONF = "__tmp.wtperf" +WTPERF_BIN = "./wtperf" +WTPERF_DIR = "../../build_posix/bench/wtperf/" + +CONF_NOT_PROVIDED = -2 + +# Generate a wtperf conf file to use +def generate_conf_file(file_name): + f = open(file_name, 'w') + f.write( +'''conn_config="cache_size=16GB,eviction=(threads_max=4),log=(enabled=false),session_max=33" +table_config="leaf_page_max=32k,internal_page_max=16k,allocation_size=4k,split_pct=90,type=file" +close_conn=false +icount=1500 +create=true +compression="snappy" +checkpoint_interval=5 +checkpoint_threads=1 +populate_threads=1 +report_interval=5 +session_count_idle=50 +session_count_idle=60 +session_count_idle=70 +session_count_idle=80 +run_time=5 +sample_interval=5 +sample_rate=1 +table_count=2 +threads=((count=6,updates=1)) +value_sz=1000 +warmup=2 +''') + f.close() + +# Build a command from the given options and execute wtperf +def execute_wtperf(conf_file, option_C = "", option_T = "", option_o = ""): + # Generate the command to run, execute wtperf + cmd = WTPERF_BIN + " -O " + conf_file + if option_C: + cmd += " -C " + option_C + if option_T: + cmd += " -T " + option_T + if option_o: + # Any quotes in option_o need to be escaped before providing it as part + # of the command + option_o_cmd_str = option_o.replace('"', '\\"') + cmd += " -o " + option_o_cmd_str + + print "Running: ", cmd + subprocess.check_call(cmd, shell=True) + print "=========================\n" + +# Build a dictionary of config key and it's value from the given config file. +# Optionally take -C, -T and -o and overwrite/append values as per correct +# precedence +def build_dict_from_conf( + conf_file, option_C = "", option_T = "", option_o = ""): + # Open given conf file and make a dictionary of passed arguments and values + with open(conf_file) as f: + lines = f.read().splitlines() + + # Maintain precedence order of config file, -o, -C/-T + # Build a dict of config options, appending values for table_config and + # conn_config, if specified multiple times. Replace with the latest in + # case of all other configuration keys. + key_val_dict = {} + for line in lines: + if re.match('^\s*#', line) is None: + key_val_pair = line.split('=', 1) + if ((key_val_pair[0] == 'table_config' or + key_val_pair[0] == 'conn_config') and + key_val_pair[0] in key_val_dict): + tmp_val = key_val_dict[key_val_pair[0]][:-1] + tmp_val += "," + tmp_val += key_val_pair[1][1:] + key_val_dict[key_val_pair[0]] = tmp_val + else: + key_val_dict[key_val_pair[0]] = key_val_pair[1] + + # If provided, put option o in the dict + if option_o: + opt_o_key_val_list = option_o.split(',') + for op_o_key_val in opt_o_key_val_list: + key_val_pair = op_o_key_val.split('=', 1) + if ((key_val_pair[0] == 'table_config' or + key_val_pair[0] == 'conn_config') and + key_val_pair[0] in key_val_dict): + tmp_val = key_val_dict[key_val_pair[0]][:-1] + tmp_val += "," + tmp_val += key_val_pair[1][1:] + key_val_dict[key_val_pair[0]] = tmp_val + else: + key_val_dict[key_val_pair[0]] = key_val_pair[1] + + # If provided, put option C in the dict + if option_C: + tmp_val = key_val_dict["conn_config"][:-1] + tmp_val += "," + tmp_val += option_C[1:] + key_val_dict["conn_config"] = tmp_val + + # If provided, put option T in the dict + if option_T: + tmp_val = key_val_dict["table_config"][:-1] + tmp_val += "," + tmp_val += option_T[1:] + key_val_dict["table_config"] = tmp_val + + return key_val_dict + +# Extract configuration value for the given key from the given config file +def extract_config_from_file(conf_file, key): + ret_val = "" + with open(conf_file) as f: + lines = f.read().splitlines() + for line in lines: + if re.match('^\s*#', line) is None: + key_val_pair = line.split('=', 1) + if key_val_pair[0] == key: + ret_val = key_val_pair[1] + return ret_val + +# Extract configuration value for the given key from the given "-o" string +def extract_config_from_opt_o(option_o, key): + ret_val = "" + opt_o_key_val_list = option_o.split(',') + for op_o_key_val in opt_o_key_val_list: + key_val_pair = op_o_key_val.split('=', 1) + if key_val_pair[0] == key: + ret_val = key_val_pair[1] + return ret_val + +# Execute test: +# Run wtperf with given config and check if the dumped config file matches the +# given inputs +def run_test(conf_file, option_C = "", option_T = "", option_o = ""): + # Run wtperf + execute_wtperf(conf_file, option_C, option_T, option_o) + + key_val_dict_ip = build_dict_from_conf( + conf_file, option_C, option_T, option_o) + key_val_dict_op = build_dict_from_conf(OP_FILE) + + conn_config_from_file = extract_config_from_file(conf_file, "conn_config") + table_config_from_file = extract_config_from_file(conf_file, "table_config") + conn_config_from_opt_o = "" + table_config_from_opt_o = "" + if option_o: + conn_config_from_opt_o = extract_config_from_opt_o( + option_o, "conn_config") + table_config_from_opt_o = extract_config_from_opt_o( + option_o, "table_config") + + # Check if dumped output conf matches with input file and options + match = True + for key in key_val_dict_ip: + match_itr = True + + # Check if we see this config key in the dumped file + if not key in key_val_dict_op: + print "Key '", key, "' not found in dumped file ", OP_FILE + match = match_itr = False + continue + + # Check if values from all sources of conn_config are presented in the + # conn_config in dumped file. Also check of their relative ordering as + # per precedence rules defined. + if (key == 'conn_config' and + (conn_config_from_file or conn_config_from_opt_o or option_C)): + # Should find these config in order: file < option o < option C + file_loc = CONF_NOT_PROVIDED + option_o_loc = CONF_NOT_PROVIDED + option_C_loc = CONF_NOT_PROVIDED + op_conn_config = key_val_dict_op['conn_config'] + + if conn_config_from_file: + file_loc = op_conn_config.find(conn_config_from_file[1:-1]) + if conn_config_from_opt_o: + option_o_loc = op_conn_config.find(conn_config_from_opt_o[1:-1]) + if option_C: + option_C_loc = op_conn_config.find(option_C[1:-1]) + + # Check if value from any of the sources is missing + if ((conn_config_from_file and file_loc == -1) or + (conn_config_from_opt_o and option_o_loc == -1) or + (option_C and option_C_loc == -1)): + print "Part of conn_config missing in dumped file ", OP_FILE + match_itr = False + + # Check if the values got appended in the correct order + if match_itr: + if ((option_o_loc != CONF_NOT_PROVIDED and + option_o_loc < file_loc) or + (option_C_loc != CONF_NOT_PROVIDED and + (option_C_loc < file_loc or option_C_loc < option_o_loc))): + print "Detected incorrect config append order:" + match_itr = False + + # Check if values from all sources of table_config are presented in the + # table_config in dumped file. Also check of their relative ordering as + # per precedence rules defined. + if (key == 'table_config' and + (table_config_from_file or table_config_from_opt_o or option_T)): + # Should find these config in order: file < option o < option T + file_loc = CONF_NOT_PROVIDED + option_o_loc = CONF_NOT_PROVIDED + option_T_loc = CONF_NOT_PROVIDED + op_table_config = key_val_dict_op['table_config'] + + if table_config_from_file: + file_loc = op_table_config.find(table_config_from_file[1:-1]) + if table_config_from_opt_o: + option_o_loc = op_table_config.find( + table_config_from_opt_o[1:-1]) + if option_T: + option_T_loc = op_table_config.find(option_T[1:-1]) + + # Check if value from any of the sources is missing + if ((table_config_from_file and file_loc == -1) or + (table_config_from_opt_o and option_o_loc == -1) or + (option_T and option_T_loc == -1)): + print "Part of table_config missing in dumped file ", OP_FILE + match_itr = False + + # Check if the values got appended in the correct order + if match_itr: + if ((option_o_loc != CONF_NOT_PROVIDED and + option_o_loc < file_loc) or + (option_T_loc != CONF_NOT_PROVIDED and + (option_T_loc < file_loc or option_T_loc < option_o_loc))): + print "Detected incorrect config append order:" + match_itr = False + + if (key != 'table_config' and key != 'conn_config' and + key_val_dict_ip[key] != key_val_dict_op[key]): + print "Config mismatch between:" + match_itr = False + + if match_itr is False: + print "Input Config:", key, '=', key_val_dict_ip[key] + print "Dumped Config:", key, '=', key_val_dict_op[key] + print "\n" + + match = match and match_itr + + return match + +# ----------------- Execute Test -------------- +# If a wtperf conf file is provided use it, else generate a temp conf file +os.chdir(WTPERF_DIR) +if len(sys.argv) == 2: + conf_file = sys.argv[1] +else: + conf_file = TMP_CONF + generate_conf_file(conf_file) + +# Run a test with no options +if not run_test(conf_file): + exit(-1) + +# Run a test with -C, -T, -o provided +option_o = "verbose=2,conn_config=\"session_max=135\",table_config=\"type=lsm\",sample_interval=2,run_time=0,sample_rate=2,readonly=false" +option_C = "\"cache_size=10GB,session_max=115\"" +option_T = "\"allocation_size=8k,split_pct=92\"" +if not run_test(conf_file, option_C, option_T, option_o): + exit(-1) + +# Cleanup generated temp files +subprocess.check_call("rm -rf WT_TEST/", shell=True) +if len(sys.argv) == 1 and conf_file == TMP_CONF: + subprocess.check_call("rm " + TMP_CONF, shell=True) + +print "All tests succeeded" diff --git a/src/third_party/wiredtiger/tools/wtstats/stat_data.py b/src/third_party/wiredtiger/tools/wtstats/stat_data.py index 635e710c469..a94ce524ae3 100644 --- a/src/third_party/wiredtiger/tools/wtstats/stat_data.py +++ b/src/third_party/wiredtiger/tools/wtstats/stat_data.py @@ -34,6 +34,9 @@ no_scale_per_second_list = [ 'reconciliation: split objects currently awaiting free', 'session: open cursor count', 'session: open session count', + 'session: table alter failed calls', + 'session: table alter successful calls', + 'session: table alter unchanged and skipped', 'session: table compact failed calls', 'session: table compact successful calls', 'session: table create failed calls', @@ -91,6 +94,7 @@ no_scale_per_second_list = [ 'btree: row-store leaf pages', 'cache: bytes currently in the cache', 'cache: overflow values cached in memory', + 'cache: tracked dirty bytes in the cache', 'cache_walk: Average difference between current eviction generation when the page was last considered', 'cache_walk: Average on-disk page image size seen', 'cache_walk: Clean pages currently in cache', @@ -124,6 +128,8 @@ no_clear_list = [ 'cache: eviction currently operating in aggressive mode', 'cache: eviction empty score', 'cache: eviction state', + 'cache: eviction worker thread active', + 'cache: eviction worker thread stable number', 'cache: files with active eviction walks', 'cache: maximum bytes configured', 'cache: maximum page size at eviction', @@ -147,6 +153,9 @@ no_clear_list = [ 'reconciliation: split objects currently awaiting free', 'session: open cursor count', 'session: open session count', + 'session: table alter failed calls', + 'session: table alter successful calls', + 'session: table alter unchanged and skipped', 'session: table compact failed calls', 'session: table compact successful calls', 'session: table create failed calls', @@ -180,6 +189,7 @@ no_clear_list = [ 'transaction: transaction range of IDs currently pinned by named snapshots', 'btree: btree checkpoint generation', 'cache: bytes currently in the cache', + 'cache: tracked dirty bytes in the cache', 'cache_walk: Average difference between current eviction generation when the page was last considered', 'cache_walk: Average on-disk page image size seen', 'cache_walk: Clean pages currently in cache', diff --git a/src/third_party/wiredtiger/tools/wtstats/wtstats.html.template b/src/third_party/wiredtiger/tools/wtstats/wtstats.html.template index ef0f60bb4f2..8e26e519456 100644 --- a/src/third_party/wiredtiger/tools/wtstats/wtstats.html.template +++ b/src/third_party/wiredtiger/tools/wtstats/wtstats.html.template @@ -10011,19 +10011,18 @@ svg.multiline .line { }
\ No newline at end of file diff --git a/version.json b/version.json index a0b32133df4..ad5f47b99da 100644 --- a/version.json +++ b/version.json @@ -1,4 +1,4 @@ { - "githash": "009580ad490190ba33d1c6253ebd8d91808923e4", - "version": "3.2.11" + "githash": "056bf45128114e44c5358c7a8776fb582363e094", + "version": "3.2.16" } \ No newline at end of file -- cgit v1.2.3 From 930687d86a670280fe7e5f98534183a571eaecd1 Mon Sep 17 00:00:00 2001 From: Apollon Oikonomopoulos Date: Sat, 7 Oct 2017 23:39:33 +0300 Subject: New upstream version 3.2.17 --- etc/evergreen.yml | 86 +---- etc/longevity.yml | 11 +- etc/system_perf.yml | 175 +++------- jstests/aggregation/bugs/server6118.js | 17 +- jstests/aggregation/bugs/server6179.js | 19 +- jstests/aggregation/bugs/server7781.js | 17 +- jstests/aggregation/bugs/server9444.js | 154 +++++---- jstests/auth/mongos_cache_invalidation.js | 11 +- jstests/core/group9.js | 20 ++ jstests/core/mr4.js | 6 +- jstests/gle/gle_sharded_write.js | 376 ++++++++++----------- jstests/multiVersion/libs/multi_rs.js | 7 +- jstests/noPassthrough/backup_restore.js | 11 +- jstests/noPassthrough/initial_sync_cloner_dups.js | 3 +- .../no_balance_collection.js | 8 +- .../apply_ops_insert_write_conflict_nonatomic.js | 8 + jstests/replsets/election_not_blocked.js | 2 +- jstests/replsets/initial_sync1.js | 6 +- jstests/replsets/initial_sync2.js | 20 +- .../libs/apply_ops_insert_write_conflict.js | 88 +++++ jstests/replsets/maintenance.js | 2 +- jstests/replsets/oplog_truncated_on_recovery.js | 2 +- .../priority_takeover_one_node_higher_priority.js | 14 +- .../read_committed_with_catalog_changes.js | 4 +- jstests/replsets/replsetadd_profile.js | 2 +- jstests/replsets/replsetprio1.js | 8 +- jstests/replsets/request_primary_stepdown.js | 2 +- jstests/replsets/resync_with_write_load.js | 2 +- jstests/replsets/rollback.js | 2 +- jstests/replsets/rollback2.js | 2 +- jstests/replsets/rollback3.js | 2 +- jstests/replsets/rollback5.js | 2 +- jstests/replsets/rollback_auth.js | 2 +- jstests/replsets/rollback_cmd_unrollbackable.js | 2 +- jstests/replsets/rollback_collMod_fatal.js | 2 +- jstests/replsets/rollback_different_h.js | 2 +- jstests/replsets/rollback_dropdb.js | 2 +- jstests/replsets/rollback_fake_cmd.js | 2 +- jstests/replsets/rollback_index.js | 2 +- jstests/replsets/rslib.js | 2 +- jstests/replsets/stepdown.js | 2 +- jstests/replsets/stepdown_kill_other_ops.js | 2 +- jstests/replsets/stepdown_killop.js | 2 +- jstests/replsets/stepdown_long_wait_time.js | 2 +- jstests/replsets/sync_passive.js | 2 +- jstests/replsets/two_nodes_priority_take_over.js | 2 +- jstests/sharding/balance_repl.js | 21 +- jstests/sharding/balance_tags2.js | 15 +- jstests/sharding/explain_cmd.js | 357 +++++++++---------- .../sharding/explain_find_and_modify_sharded.js | 2 +- jstests/sharding/hash_shard_unique_compound.js | 60 ++-- jstests/sharding/mapReduce_inSharded_outSharded.js | 128 +++---- jstests/sharding/migrateBig.js | 46 ++- jstests/sharding/migrateBig_balancer.js | 19 +- jstests/sharding/printShardingStatus.js | 2 +- jstests/sharding/shard3.js | 10 +- jstests/sharding/split_with_force_small.js | 99 +++--- jstests/sharding/stale_version_write.js | 46 +-- jstests/slow2/mr_during_migrate.js | 159 +++++---- src/mongo/base/validate_locale.cpp | 13 +- src/mongo/db/commands/mr.cpp | 6 +- src/mongo/db/dbhelpers.cpp | 5 +- src/mongo/db/exec/group.cpp | 3 +- src/mongo/db/ops/update_driver.cpp | 12 +- src/mongo/db/repl/oplog.cpp | 6 +- src/mongo/dbtests/jstests.cpp | 44 +++ src/mongo/s/SConscript | 2 +- src/mongo/s/catalog/catalog_manager.h | 6 +- src/mongo/s/chunk_manager.cpp | 3 + src/mongo/scripting/deadline_monitor.cpp | 2 +- src/mongo/scripting/deadline_monitor.h | 12 +- src/mongo/scripting/engine.cpp | 3 + src/mongo/scripting/engine.h | 2 + src/mongo/scripting/mozjs/bson.cpp | 10 +- src/mongo/scripting/mozjs/implscope.cpp | 10 + src/mongo/scripting/mozjs/implscope.h | 5 + src/mongo/scripting/mozjs/objectwrapper.cpp | 7 +- src/mongo/scripting/mozjs/proxyscope.cpp | 4 + src/mongo/scripting/mozjs/proxyscope.h | 2 + src/mongo/shell/replsettest.js | 2 +- src/third_party/wiredtiger/dist/s_string.ok | 1 + src/third_party/wiredtiger/dist/stat_data.py | 1 + src/third_party/wiredtiger/import.data | 2 +- src/third_party/wiredtiger/src/btree/row_key.c | 2 + .../wiredtiger/src/docs/programming.dox | 15 +- src/third_party/wiredtiger/src/docs/spell.ok | 1 + .../wiredtiger/src/docs/tune-build-options.dox | 9 + src/third_party/wiredtiger/src/evict/evict_lru.c | 7 + src/third_party/wiredtiger/src/include/extern.h | 1 + src/third_party/wiredtiger/src/include/gcc.h | 2 +- src/third_party/wiredtiger/src/include/lint.h | 1 + src/third_party/wiredtiger/src/include/misc.i | 25 ++ src/third_party/wiredtiger/src/include/msvc.h | 4 +- src/third_party/wiredtiger/src/include/session.h | 7 + src/third_party/wiredtiger/src/include/stat.h | 1 + src/third_party/wiredtiger/src/include/txn.h | 2 + .../wiredtiger/src/include/wiredtiger.in | 346 ++++++++++--------- .../wiredtiger/src/os_common/os_alloc.c | 2 + .../wiredtiger/src/os_common/os_getopt.c | 8 +- src/third_party/wiredtiger/src/os_posix/os_dir.c | 6 + src/third_party/wiredtiger/src/os_posix/os_time.c | 22 +- src/third_party/wiredtiger/src/os_win/os_time.c | 9 +- .../wiredtiger/src/schema/schema_list.c | 31 ++ .../wiredtiger/src/session/session_api.c | 17 +- src/third_party/wiredtiger/src/support/stat.c | 3 + src/third_party/wiredtiger/src/txn/txn.c | 22 +- src/third_party/wiredtiger/src/txn/txn_log.c | 11 + version.json | 4 +- 108 files changed, 1528 insertions(+), 1274 deletions(-) create mode 100644 jstests/core/group9.js create mode 100644 jstests/replsets/apply_ops_insert_write_conflict_nonatomic.js create mode 100644 jstests/replsets/libs/apply_ops_insert_write_conflict.js create mode 100644 src/third_party/wiredtiger/src/docs/tune-build-options.dox diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 9bef88f9a0e..97b0f49b755 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -90,7 +90,6 @@ variables: - ubuntu1404-rocksdb - debian71 - debian81 - - solaris-64-bit - osx-107-ssl - osx-107 @@ -201,7 +200,7 @@ functions: rm -rf rocksdb git clone https://github.com/facebook/rocksdb.git cd rocksdb - make static_lib + make USE_RTTI=1 static_lib fi "build new tools" : @@ -5394,89 +5393,6 @@ buildvariants: - ubuntu1604-packer - name: push -########################################### -# Solaris buildvariants # -########################################### - -- name: solaris-64-bit - display_name: "* Solaris" - modules: - - mongo-tools - run_on: - - solaris - expansions: - push_path: sunos5 - push_bucket: downloads.mongodb.org - push_name: sunos5 - push_arch: x86_64 - gorootvars: PATH=/opt/mongodbtoolchain/v2/bin:$PATH - tooltags: -gccgoflags "-lsocket -lnsl" - compile_flags: CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ -j$(kstat cpu | sort -u | grep -c "^module") --release CCFLAGS="-m64" LINKFLAGS="-m64 -static-libstdc++ -static-libgcc" OBJCOPY=/opt/mongodbtoolchain/bin/objcopy - num_jobs_available: $(kstat cpu | sort -u | grep -c "^module") - tasks: - - name: compile - - name: aggregation - - name: aggregation_WT - - name: auth - - name: auth_WT - - name: bulk_gle_passthrough - - name: bulk_gle_passthrough_WT - - name: concurrency - - name: concurrency_WT - - name: concurrency_replication - - name: concurrency_replication_WT - - name: concurrency_sharded - - name: concurrency_sharded_WT - - name: concurrency_sharded_sccc - - name: concurrency_sharded_sccc_WT - - name: dbtest - - name: dbtest_WT - - name: disk - - name: durability - - name: failpoints - - name: httpinterface - - name: jsCore - - name: jsCore_compatibility - - name: jsCore_compatibility_WT - - name: jsCore_small_oplog - - name: jsCore_small_oplog_rs - - name: jsCore_small_oplog_rs_WT - - name: jsCore_small_oplog_WT - - name: jsCore_WT - - name: mmap - - name: mongosTest - - name: noPassthrough - - name: noPassthroughWithMongod - - name: noPassthroughWithMongod_WT - - name: noPassthrough_WT - - name: parallel - - name: parallel_compatibility - - name: parallel_compatibility_WT - - name: parallel_WT - - name: replicasets - - name: replicasets_WT - - name: replication - - name: replication_WT - - name: sharding - - name: sharding_csrs_upgrade - - name: sharded_collections_jscore_passthrough - - name: sharded_collections_jscore_passthrough_WT - - name: sharding_jscore_passthrough - - name: sharding_jscore_passthrough_WT - - name: sharding_jscore_passthrough_wire_ops_WT - - name: sharding_WT - - name: sharding_csrs_upgrade_WT - - name: slow1 - - name: slow1_WT - - name: slow2 - - name: slow2_WT - - name: tool - - name: tool_WT - - name: unittests - - name: push - distros: - - rhel70 - ########################################### # Debian buildvariants # ########################################### diff --git a/etc/longevity.yml b/etc/longevity.yml index b5cb674a354..af830baf318 100644 --- a/etc/longevity.yml +++ b/etc/longevity.yml @@ -141,15 +141,15 @@ functions: "infrastructure provisioning": - command: shell.exec - # call infrastructure-provisioning.sh. This will either create a cluster, or update tags on existing instances. + # call infrastructure-provisioning.py. This will either create a cluster, or update tags on existing instances. params: working_dir: work script: | set -e set -v source ./dsienv.sh - export PRODUCTION=true - $DSI_PATH/bin/infrastructure_provisioning.sh ${cluster} + source ./venv/bin/activate + $DSI_PATH/bin/infrastructure_provisioning.py "configure mongodb cluster": - command: shell.exec @@ -190,10 +190,11 @@ functions: set -o verbose source ./dsienv.sh # Longevity runs so rarely, we simply teardown the cluster when done. - # Note that nowadays infrastructure_teardown.sh is actually copying the terraform.tfstate into /data/infrastructure_provisioning + # Note that nowadays infrastructure_teardown.py is actually copying the terraform.tfstate into /data/infrastructure_provisioning # but as of this writing the rhel70-perf-longevity distro didn't actually use the teardown hook. source ./dsienv.sh - $DSI_PATH/bin/infrastructure_teardown.sh + source ./venv/bin/activate + $DSI_PATH/bin/infrastructure_teardown.py echo "Cluster DESTROYED." echo echo "All perf results" diff --git a/etc/system_perf.yml b/etc/system_perf.yml index 641b34bac49..e654824801c 100644 --- a/etc/system_perf.yml +++ b/etc/system_perf.yml @@ -25,8 +25,8 @@ post: params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: work/reports/graphs/timeseries-p1.html - remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/timeseries-p1-${task_name}-${build_id}.html + local_file: work/reports/graphs/timeseries-mongod.0.html + remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/timeseries-mongod.0-${task_name}-${build_id}.html bucket: mciuploads permissions: public-read content_type: text/html @@ -103,7 +103,6 @@ functions: ext: ${ext} script_flags : ${script_flags} dsi_rev: ${dsi_rev} - compare_task: ${compare_task} workloads_rev: ${workloads_rev} # compositions of expansions @@ -151,15 +150,15 @@ functions: "infrastructure provisioning": - command: shell.exec - # call infrastructure-provisioning.sh. This will either create a cluster, or update tags on existing instances. + # call infrastructure-provisioning.py. This will either create a cluster, or update tags on existing instances. params: working_dir: work script: | set -e set -v source ./dsienv.sh - export PRODUCTION=true - $DSI_PATH/bin/infrastructure_provisioning.sh ${cluster} + source ./venv/bin/activate + $DSI_PATH/bin/infrastructure_provisioning.py "configure mongodb cluster": - command: shell.exec @@ -237,45 +236,6 @@ functions: OVERRIDEFILE="../src/dsi/dsi/analysis/v3.2/system_perf_override.json" python -u ../src/dsi/dsi/analysis/post_run_check.py ${script_flags} --reports-analysis reports --perf-file reports/perf.json --rev ${revision} -f history.json -t tags.json --refTag $TAG --overrideFile $OVERRIDEFILE --project_id sys-perf --variant ${build_variant} --task ${task_name} - "compare": - - command: shell.exec - params: - script: | - set -o verbose - rm -rf ./src ./work - mkdir src - mkdir work - - command: manifest.load - - command: git.get_project - params: - directory: src - revisions: # for each module include revision as : ${_rev} - dsi: ${dsi_rev} - - command: json.get - params: - task: ${compare_task} - variant : ${variant1} - file: "work/standalone.json" - name: "perf" - - command: json.get - params: - task: ${compare_task} - variant : ${variant2} - file: "work/oplog.json" - name: "perf" - - command: shell.exec - type : test - params: - working_dir: work - script: | - set -o errexit - set -o verbose - python -u ../src/dsi/dsi/analysis/compare.py -b standalone.json -c oplog.json - - command: "json.send" - params: - name: "perf" - file: "work/perf.json" - ####################################### # Tasks # ####################################### @@ -357,6 +317,50 @@ tasks: vars: script_flags: --ycsb-throughput-analysis reports +- name: industry_benchmarks_wmajority_WT + depends_on: + - name: compile + variant: linux-standalone + commands: + - func: "prepare environment" + vars: + storageEngine: "wiredTiger" + test: "ycsb-wmajority" + - func: "infrastructure provisioning" + - func: "configure mongodb cluster" + vars: + storageEngine: "wiredTiger" + - func: "run test" + vars: + storageEngine: "wiredTiger" + test: "ycsb-wmajority" + - func: "make test log artifact" + - func: "analyze" + vars: + script_flags: --ycsb-throughput-analysis reports + +- name: industry_benchmarks_wmajority_MMAPv1 + depends_on: + - name: compile + variant: linux-standalone + commands: + - func: "prepare environment" + vars: + storageEngine: "mmapv1" + test: "ycsb-wmajority" + - func: "infrastructure provisioning" + - func: "configure mongodb cluster" + vars: + storageEngine: "mmapv1" + - func: "run test" + vars: + storageEngine: "mmapv1" + test: "ycsb-wmajority" + - func: "make test log artifact" + - func: "analyze" + vars: + script_flags: --ycsb-throughput-analysis reports + - name: core_workloads_WT depends_on: - name: compile @@ -518,70 +522,6 @@ tasks: - func: "make test log artifact" - func: "analyze" -- name: industry_benchmarks_WT_oplog_comp - depends_on: - - name: industry_benchmarks_WT - variant: linux-standalone - status : "*" - - name: industry_benchmarks_WT - variant: linux-1-node-replSet - status: "*" - commands: - - func: "compare" - vars: - compare_task: "industry_benchmarks_WT" - variant1: "linux-standalone" - variant2: "linux-1-node-replSet" - - func: "analyze" - -- name: industry_benchmarks_MMAPv1_oplog_comp - depends_on: - - name: industry_benchmarks_MMAPv1 - variant: linux-standalone - status: "*" - - name: industry_benchmarks_MMAPv1 - variant: linux-1-node-replSet - status: "*" - commands: - - func: "compare" - vars: - compare_task: "industry_benchmarks_MMAPv1" - variant1: "linux-standalone" - variant2: "linux-1-node-replSet" - - func: "analyze" - -- name: core_workloads_WT_oplog_comp - depends_on: - - name: core_workloads_WT - variant: linux-standalone - status: "*" - - name: core_workloads_WT - variant: linux-1-node-replSet - status: "*" - commands: - - func: "compare" - vars: - compare_task: "core_workloads_WT" - variant1: "linux-standalone" - variant2: "linux-1-node-replSet" - - func: "analyze" - -- name: core_workloads_MMAPv1_oplog_comp - depends_on: - - name: core_workloads_MMAPv1 - variant: linux-standalone - status: "*" - - name: core_workloads_MMAPv1 - variant: linux-1-node-replSet - status: "*" - commands: - - func: "compare" - vars: - compare_task: "core_workloads_MMAPv1" - variant1: "linux-standalone" - variant2: "linux-1-node-replSet" - - func: "analyze" - - name: initialsync_WT depends_on: - name: compile @@ -706,6 +646,8 @@ buildvariants: - name: industry_benchmarks_WT - name: core_workloads_WT - name: industry_benchmarks_MMAPv1 + - name: industry_benchmarks_wmajority_WT + - name: industry_benchmarks_wmajority_MMAPv1 - name: core_workloads_MMAPv1 - name: mongos_workloads_WT - name: mongos_workloads_MMAPv1 @@ -728,6 +670,8 @@ buildvariants: - name: industry_benchmarks_WT - name: core_workloads_WT - name: industry_benchmarks_MMAPv1 + - name: industry_benchmarks_wmajority_WT + - name: industry_benchmarks_wmajority_MMAPv1 - name: core_workloads_MMAPv1 - name: non_sharded_workloads_WT - name: non_sharded_workloads_MMAPv1 @@ -748,16 +692,3 @@ buildvariants: - name: initialsync_WT - name: initialsync_MMAPv1 -- name: linux-oplog-compare - display_name: Linux Oplog Compare - batchtime: 10080 # 7 days - modules: *modules - expansions: - project: *project - run_on: - - "rhel70-perf-single" - tasks: - - name: industry_benchmarks_WT_oplog_comp - - name: core_workloads_WT_oplog_comp - - name: industry_benchmarks_MMAPv1_oplog_comp - - name: core_workloads_MMAPv1_oplog_comp diff --git a/jstests/aggregation/bugs/server6118.js b/jstests/aggregation/bugs/server6118.js index f891135de72..3c55ae5ce33 100644 --- a/jstests/aggregation/bugs/server6118.js +++ b/jstests/aggregation/bugs/server6118.js @@ -1,12 +1,12 @@ // SERVER-6118: support for sharded sorts (function() { + 'use strict'; - var s = new ShardingTest({name: "aggregation_sort1", shards: 2, mongos: 1}); - s.stopBalancer(); + var s = new ShardingTest({shards: 2}); - s.adminCommand({enablesharding: "test"}); + assert.commandWorked(s.s0.adminCommand({enablesharding: "test"})); s.ensurePrimaryShard('test', 'shard0001'); - s.adminCommand({shardcollection: "test.data", key: {_id: 1}}); + assert.commandWorked(s.s0.adminCommand({shardcollection: "test.data", key: {_id: 1}})); var d = s.getDB("test"); @@ -20,12 +20,12 @@ bulkOp.execute(); // Split the data into 3 chunks - s.adminCommand({split: "test.data", middle: {_id: 33}}); - s.adminCommand({split: "test.data", middle: {_id: 66}}); + assert.commandWorked(s.s0.adminCommand({split: "test.data", middle: {_id: 33}})); + assert.commandWorked(s.s0.adminCommand({split: "test.data", middle: {_id: 66}})); // Migrate the middle chunk to another shard - s.adminCommand( - {movechunk: "test.data", find: {_id: 50}, to: s.getOther(s.getServer("test")).name}); + assert.commandWorked(s.s0.adminCommand( + {movechunk: "test.data", find: {_id: 50}, to: s.getOther(s.getServer("test")).name})); // Check that the results are in order. var result = d.data.aggregate({$sort: {_id: 1}}).toArray(); @@ -36,5 +36,4 @@ } s.stop(); - })(); diff --git a/jstests/aggregation/bugs/server6179.js b/jstests/aggregation/bugs/server6179.js index 1109ddaa67e..503e91a70d1 100644 --- a/jstests/aggregation/bugs/server6179.js +++ b/jstests/aggregation/bugs/server6179.js @@ -1,12 +1,12 @@ // SERVER-6179: support for two $groups in sharded agg (function() { + 'use strict'; - var s = new ShardingTest({name: "aggregation_multiple_group", shards: 2, mongos: 1}); - s.stopBalancer(); + var s = new ShardingTest({shards: 2}); - s.adminCommand({enablesharding: "test"}); + assert.commandWorked(s.s0.adminCommand({enablesharding: "test"})); s.ensurePrimaryShard('test', 'shard0001'); - s.adminCommand({shardcollection: "test.data", key: {_id: 1}}); + assert.commandWorked(s.s0.adminCommand({shardcollection: "test.data", key: {_id: 1}})); var d = s.getDB("test"); @@ -20,18 +20,18 @@ bulkOp.execute(); // Split the data into 3 chunks - s.adminCommand({split: "test.data", middle: {_id: 33}}); - s.adminCommand({split: "test.data", middle: {_id: 66}}); + assert.commandWorked(s.s0.adminCommand({split: "test.data", middle: {_id: 33}})); + assert.commandWorked(s.s0.adminCommand({split: "test.data", middle: {_id: 66}})); // Migrate the middle chunk to another shard - s.adminCommand( - {movechunk: "test.data", find: {_id: 50}, to: s.getOther(s.getServer("test")).name}); + assert.commandWorked(s.s0.adminCommand( + {movechunk: "test.data", find: {_id: 50}, to: s.getOther(s.getServer("test")).name})); // Check that we get results rather than an error var result = d.data.aggregate({$group: {_id: '$_id', i: {$first: '$i'}}}, {$group: {_id: '$i', avg_id: {$avg: '$_id'}}}, {$sort: {_id: 1}}).toArray(); - expected = [ + var expected = [ {"_id": 0, "avg_id": 45}, {"_id": 1, "avg_id": 46}, {"_id": 2, "avg_id": 47}, @@ -47,5 +47,4 @@ assert.eq(result, expected); s.stop(); - })(); diff --git a/jstests/aggregation/bugs/server7781.js b/jstests/aggregation/bugs/server7781.js index 230a8a64c9f..c3918aeb8d2 100644 --- a/jstests/aggregation/bugs/server7781.js +++ b/jstests/aggregation/bugs/server7781.js @@ -1,5 +1,6 @@ // SERVER-7781 $geoNear pipeline stage (function() { + 'use strict'; load('jstests/libs/geo_near_random.js'); load('jstests/aggregation/extras/utils.js'); @@ -59,10 +60,12 @@ shards.push(shard._id); }); - db.adminCommand({shardCollection: db[coll].getFullName(), key: {rand: 1}}); + assert.commandWorked( + db.adminCommand({shardCollection: db[coll].getFullName(), key: {rand: 1}})); for (var i = 1; i < 10; i++) { // split at 0.1, 0.2, ... 0.9 - db.adminCommand({split: db[coll].getFullName(), middle: {rand: i / 10}}); + assert.commandWorked( + db.adminCommand({split: db[coll].getFullName(), middle: {rand: i / 10}})); db.adminCommand({ moveChunk: db[coll].getFullName(), find: {rand: i / 10}, @@ -87,13 +90,13 @@ // test with defaults var queryPoint = pointMaker.mkPt(0.25); // stick to center of map - geoCmd = { + var geoCmd = { geoNear: coll, near: queryPoint, includeLocs: true, spherical: true }; - aggCmd = { + var aggCmd = { $geoNear: { near: queryPoint, includeLocs: 'stats.loc', @@ -134,7 +137,7 @@ geoCmd.num = 40; geoCmd.near = queryPoint; aggCmd.$geoNear.near = queryPoint; - aggArr = [aggCmd, {$limit: 50}, {$limit: 60}, {$limit: 40}]; + var aggArr = [aggCmd, {$limit: 50}, {$limit: 60}, {$limit: 40}]; checkOutput(db.runCommand(geoCmd), db[coll].aggregate(aggArr), 40); // Test $geoNear with an initial batchSize of 0. Regression test for SERVER-20935. @@ -157,13 +160,11 @@ test(db, false, '2dsphere'); var sharded = new ShardingTest({shards: 3, mongos: 1}); - sharded.stopBalancer(); - sharded.adminCommand({enablesharding: "test"}); + assert.commandWorked(sharded.s0.adminCommand({enablesharding: "test"})); sharded.ensurePrimaryShard('test', 'shard0001'); test(sharded.getDB('test'), true, '2d'); test(sharded.getDB('test'), true, '2dsphere'); sharded.stop(); - })(); diff --git a/jstests/aggregation/bugs/server9444.js b/jstests/aggregation/bugs/server9444.js index ad5f4b03ca6..f3dc2748b0a 100644 --- a/jstests/aggregation/bugs/server9444.js +++ b/jstests/aggregation/bugs/server9444.js @@ -1,76 +1,80 @@ // server-9444 support disk storage of intermediate results in aggregation - -var t = db.server9444; -t.drop(); - -var sharded = (typeof(RUNNING_IN_SHARDED_AGG_TEST) != 'undefined'); // see end of testshard1.js -if (sharded) { - db.adminCommand({shardcollection: t.getFullName(), key: {"_id": 'hashed'}}); -} - -var memoryLimitMB = sharded ? 200 : 100; - -function loadData() { - var bigStr = Array(1024 * 1024 + 1).toString(); // 1MB of ',' - for (var i = 0; i < memoryLimitMB + 1; i++) - t.insert({_id: i, bigStr: i + bigStr, random: Math.random()}); - - assert.gt(t.stats().size, memoryLimitMB * 1024 * 1024); -} -loadData(); - -function test(pipeline, outOfMemoryCode) { - // ensure by default we error out if exceeding memory limit - var res = t.runCommand('aggregate', {pipeline: pipeline}); - assert.commandFailed(res); - assert.eq(res.code, outOfMemoryCode); - - // ensure allowDiskUse: false does what it says - var res = t.runCommand('aggregate', {pipeline: pipeline, allowDiskUse: false}); - assert.commandFailed(res); - assert.eq(res.code, outOfMemoryCode); - - // allowDiskUse only supports bool. In particular, numbers aren't allowed. - var res = t.runCommand('aggregate', {pipeline: pipeline, allowDiskUse: 1}); - assert.commandFailed(res); - assert.eq(res.code, 16949); - - // ensure we work when allowDiskUse === true - var res = t.aggregate(pipeline, {allowDiskUse: true}); - assert.eq(res.itcount(), t.count()); // all tests output one doc per input doc -} - -var groupCode = 16945; -var sortCode = 16819; -var sortLimitCode = 16820; - -test([{$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}], groupCode); - -// sorting with _id would use index which doesn't require extsort -test([{$sort: {random: 1}}], sortCode); -test([{$sort: {bigStr: 1}}], sortCode); // big key and value - -// make sure sort + large limit won't crash the server (SERVER-10136) -test([{$sort: {bigStr: 1}}, {$limit: 1000 * 1000 * 1000}], sortLimitCode); - -// test combining two extSorts in both same and different orders -test([{$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}, {$sort: {_id: 1}}], groupCode); -test([{$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}, {$sort: {_id: -1}}], groupCode); -test([{$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}, {$sort: {random: 1}}], groupCode); -test([{$sort: {random: 1}}, {$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}], sortCode); - -var origDB = db; -if (sharded) { - // Stop balancer first before dropping so there will be no contention on the ns lock. - // It's alright to modify the global db variable since sharding tests never run in parallel. - db = db.getSiblingDB('config'); - sh.stopBalancer(); -} - -// don't leave large collection laying around -t.drop(); - -if (sharded) { - sh.startBalancer(); - db = origDB; -} +(function() { + 'use strict'; + + var t = db.server9444; + t.drop(); + + var sharded = (typeof(RUNNING_IN_SHARDED_AGG_TEST) != 'undefined'); // see end of testshard1.js + if (sharded) { + assert.commandWorked( + db.adminCommand({shardcollection: t.getFullName(), key: {"_id": 'hashed'}})); + } + + var memoryLimitMB = sharded ? 200 : 100; + + function loadData() { + var bigStr = Array(1024 * 1024 + 1).toString(); // 1MB of ',' + for (var i = 0; i < memoryLimitMB + 1; i++) + t.insert({_id: i, bigStr: i + bigStr, random: Math.random()}); + + assert.gt(t.stats().size, memoryLimitMB * 1024 * 1024); + } + loadData(); + + function test(pipeline, outOfMemoryCode) { + // ensure by default we error out if exceeding memory limit + var res = t.runCommand('aggregate', {pipeline: pipeline}); + assert.commandFailed(res); + assert.eq(res.code, outOfMemoryCode); + + // ensure allowDiskUse: false does what it says + var res = t.runCommand('aggregate', {pipeline: pipeline, allowDiskUse: false}); + assert.commandFailed(res); + assert.eq(res.code, outOfMemoryCode); + + // allowDiskUse only supports bool. In particular, numbers aren't allowed. + var res = t.runCommand('aggregate', {pipeline: pipeline, allowDiskUse: 1}); + assert.commandFailed(res); + assert.eq(res.code, 16949); + + // ensure we work when allowDiskUse === true + var res = t.aggregate(pipeline, {allowDiskUse: true}); + assert.eq(res.itcount(), t.count()); // all tests output one doc per input doc + } + + var groupCode = 16945; + var sortCode = 16819; + var sortLimitCode = 16820; + + test([{$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}], groupCode); + + // sorting with _id would use index which doesn't require extsort + test([{$sort: {random: 1}}], sortCode); + test([{$sort: {bigStr: 1}}], sortCode); // big key and value + + // make sure sort + large limit won't crash the server (SERVER-10136) + test([{$sort: {bigStr: 1}}, {$limit: 1000 * 1000 * 1000}], sortLimitCode); + + // test combining two extSorts in both same and different orders + test([{$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}, {$sort: {_id: 1}}], groupCode); + test([{$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}, {$sort: {_id: -1}}], groupCode); + test([{$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}, {$sort: {random: 1}}], groupCode); + test([{$sort: {random: 1}}, {$group: {_id: '$_id', bigStr: {$first: '$bigStr'}}}], sortCode); + + var origDB = db; + if (sharded) { + // Stop balancer first before dropping so there will be no contention on the ns lock. + // It's alright to modify the global db variable since sharding tests never run in parallel. + db = db.getSiblingDB('config'); + sh.stopBalancer(); + } + + // don't leave large collection laying around + t.drop(); + + if (sharded) { + sh.startBalancer(); + db = origDB; + } +})(); diff --git a/jstests/auth/mongos_cache_invalidation.js b/jstests/auth/mongos_cache_invalidation.js index 60700956e39..4594a18f8c4 100644 --- a/jstests/auth/mongos_cache_invalidation.js +++ b/jstests/auth/mongos_cache_invalidation.js @@ -209,13 +209,10 @@ db3.auth('spencer', 'pwd'); // s0/db1 should update its cache instantly assert.commandFailedWithCode(db1.foo.runCommand("collStats"), authzErrorCode); - // s1/db2 should update its cache in 5 seconds. - assert.soon( - function() { - return db2.foo.runCommand("collStats").code == authzErrorCode; - }, - "Mongos did not update its user cache after 5 seconds", - 6 * 1000); // Give an extra 1 second to avoid races + // s1/db2 should update its cache in 10 seconds. + assert.soon(function() { + return db2.foo.runCommand("collStats").code == authzErrorCode; + }, "Mongos did not update its user cache after 10 seconds", 10 * 1000); // We manually invalidate the cache on s2/db3. db3.adminCommand("invalidateUserCache"); diff --git a/jstests/core/group9.js b/jstests/core/group9.js new file mode 100644 index 00000000000..a7d9a0d128e --- /dev/null +++ b/jstests/core/group9.js @@ -0,0 +1,20 @@ +(function() { + 'use strict'; + var t = db.group_owned; + t.drop(); + + assert.writeOK(t.insert({_id: 1, subdoc: {id: 1}})); + assert.writeOK(t.insert({_id: 2, subdoc: {id: 2}})); + + var result = t.group({ + key: {'subdoc.id': 1}, + reduce: function(doc, value) { + value.subdoc = doc.subdoc; + return value; + }, + initial: {}, + finalize: function(res) {} + }); + + assert(result.length == 2); +}()); diff --git a/jstests/core/mr4.js b/jstests/core/mr4.js index ae5e11528af..2b8e93c3b35 100644 --- a/jstests/core/mr4.js +++ b/jstests/core/mr4.js @@ -9,7 +9,7 @@ t.save({x: 4, tags: ["b", "c"]}); m = function() { this.tags.forEach(function(z) { - emit(z, {count: xx}); + emit(z, {count: xx.val}); }); }; @@ -23,7 +23,7 @@ r = function(key, values) { }; }; -res = t.mapReduce(m, r, {out: "mr4_out", scope: {xx: 1}}); +res = t.mapReduce(m, r, {out: "mr4_out", scope: {xx: {val: 1}}}); z = res.convertToSingleObject(); assert.eq(3, Object.keySet(z).length, "A1"); @@ -33,7 +33,7 @@ assert.eq(3, z.c.count, "A4"); res.drop(); -res = t.mapReduce(m, r, {scope: {xx: 2}, out: "mr4_out"}); +res = t.mapReduce(m, r, {scope: {xx: {val: 2}}, out: "mr4_out"}); z = res.convertToSingleObject(); assert.eq(3, Object.keySet(z).length, "A1"); diff --git a/jstests/gle/gle_sharded_write.js b/jstests/gle/gle_sharded_write.js index f1feffed5b2..8d2a21cd758 100644 --- a/jstests/gle/gle_sharded_write.js +++ b/jstests/gle/gle_sharded_write.js @@ -2,192 +2,192 @@ // Ensures GLE correctly reports basic write stats and failures // Note that test should work correctly with and without write commands. // - -var st = new ShardingTest({shards: 2, mongos: 1}); -st.stopBalancer(); - -var mongos = st.s0; -var admin = mongos.getDB("admin"); -var config = mongos.getDB("config"); -var coll = mongos.getCollection(jsTestName() + ".coll"); -var shards = config.shards.find().toArray(); - -assert.commandWorked(admin.runCommand({enableSharding: coll.getDB().toString()})); -printjson(admin.runCommand({movePrimary: coll.getDB().toString(), to: shards[0]._id})); -assert.commandWorked(admin.runCommand({shardCollection: coll.toString(), key: {_id: 1}})); -assert.commandWorked(admin.runCommand({split: coll.toString(), middle: {_id: 0}})); -assert.commandWorked( - admin.runCommand({moveChunk: coll.toString(), find: {_id: 0}, to: shards[1]._id})); - -st.printShardingStatus(); - -var gle = null; - -// -// Successful insert -coll.remove({}); -coll.insert({_id: -1}); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert('err' in gle); -assert(!gle.err); -assert.eq(coll.count(), 1); - -// -// Successful update -coll.remove({}); -coll.insert({_id: 1}); -coll.update({_id: 1}, {$set: {foo: "bar"}}); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert('err' in gle); -assert(!gle.err); -assert(gle.updatedExisting); -assert.eq(gle.n, 1); -assert.eq(coll.count(), 1); - -// -// Successful multi-update -coll.remove({}); -coll.insert({_id: 1}); -coll.update({}, {$set: {foo: "bar"}}, false, true); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert('err' in gle); -assert(!gle.err); -assert(gle.updatedExisting); -assert.eq(gle.n, 1); -assert.eq(coll.count(), 1); - -// -// Successful upsert -coll.remove({}); -coll.update({_id: 1}, {_id: 1}, true); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert('err' in gle); -assert(!gle.err); -assert(!gle.updatedExisting); -assert.eq(gle.n, 1); -assert.eq(gle.upserted, 1); -assert.eq(coll.count(), 1); - -// -// Successful upserts -coll.remove({}); -coll.update({_id: -1}, {_id: -1}, true); -coll.update({_id: 1}, {_id: 1}, true); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert('err' in gle); -assert(!gle.err); -assert(!gle.updatedExisting); -assert.eq(gle.n, 1); -assert.eq(gle.upserted, 1); -assert.eq(coll.count(), 2); - -// -// Successful remove -coll.remove({}); -coll.insert({_id: 1}); -coll.remove({_id: 1}); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert('err' in gle); -assert(!gle.err); -assert.eq(gle.n, 1); -assert.eq(coll.count(), 0); - -// -// Error on one host during update -coll.remove({}); -coll.update({_id: 1}, {$invalid: "xxx"}, true); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert(gle.err); -assert(gle.code); -assert(!gle.errmsg); -assert(gle.singleShard); -assert.eq(coll.count(), 0); - -// -// Error on two hosts during remove -coll.remove({}); -coll.remove({$invalid: 'remove'}); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert(gle.err); -assert(gle.code); -assert(!gle.errmsg); -assert(gle.shards); -assert.eq(coll.count(), 0); - -// -// Repeated calls to GLE should work -coll.remove({}); -coll.update({_id: 1}, {$invalid: "xxx"}, true); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert(gle.err); -assert(gle.code); -assert(!gle.errmsg); -assert(gle.singleShard); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert(gle.err); -assert(gle.code); -assert(!gle.errmsg); -assert(gle.singleShard); -assert.eq(coll.count(), 0); - -// -// Geo $near is not supported on mongos -coll.ensureIndex({loc: "2dsphere"}); -coll.remove({}); -var query = { - loc: { - $near: { - $geometry: {type: "Point", coordinates: [0, 0]}, - $maxDistance: 1000, +(function() { + 'use strict'; + + var st = new ShardingTest({shards: 2, mongos: 1}); + + var mongos = st.s0; + var admin = mongos.getDB("admin"); + var config = mongos.getDB("config"); + var coll = mongos.getCollection(jsTestName() + ".coll"); + var shards = config.shards.find().toArray(); + + assert.commandWorked(admin.runCommand({enableSharding: coll.getDB().toString()})); + printjson(admin.runCommand({movePrimary: coll.getDB().toString(), to: shards[0]._id})); + assert.commandWorked(admin.runCommand({shardCollection: coll.toString(), key: {_id: 1}})); + assert.commandWorked(admin.runCommand({split: coll.toString(), middle: {_id: 0}})); + assert.commandWorked( + admin.runCommand({moveChunk: coll.toString(), find: {_id: 0}, to: shards[1]._id})); + + st.printShardingStatus(); + + var gle = null; + + // + // Successful insert + coll.remove({}); + coll.insert({_id: -1}); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert('err' in gle); + assert(!gle.err); + assert.eq(coll.count(), 1); + + // + // Successful update + coll.remove({}); + coll.insert({_id: 1}); + coll.update({_id: 1}, {$set: {foo: "bar"}}); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert('err' in gle); + assert(!gle.err); + assert(gle.updatedExisting); + assert.eq(gle.n, 1); + assert.eq(coll.count(), 1); + + // + // Successful multi-update + coll.remove({}); + coll.insert({_id: 1}); + coll.update({}, {$set: {foo: "bar"}}, false, true); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert('err' in gle); + assert(!gle.err); + assert(gle.updatedExisting); + assert.eq(gle.n, 1); + assert.eq(coll.count(), 1); + + // + // Successful upsert + coll.remove({}); + coll.update({_id: 1}, {_id: 1}, true); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert('err' in gle); + assert(!gle.err); + assert(!gle.updatedExisting); + assert.eq(gle.n, 1); + assert.eq(gle.upserted, 1); + assert.eq(coll.count(), 1); + + // + // Successful upserts + coll.remove({}); + coll.update({_id: -1}, {_id: -1}, true); + coll.update({_id: 1}, {_id: 1}, true); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert('err' in gle); + assert(!gle.err); + assert(!gle.updatedExisting); + assert.eq(gle.n, 1); + assert.eq(gle.upserted, 1); + assert.eq(coll.count(), 2); + + // + // Successful remove + coll.remove({}); + coll.insert({_id: 1}); + coll.remove({_id: 1}); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert('err' in gle); + assert(!gle.err); + assert.eq(gle.n, 1); + assert.eq(coll.count(), 0); + + // + // Error on one host during update + coll.remove({}); + coll.update({_id: 1}, {$invalid: "xxx"}, true); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert(gle.err); + assert(gle.code); + assert(!gle.errmsg); + assert(gle.singleShard); + assert.eq(coll.count(), 0); + + // + // Error on two hosts during remove + coll.remove({}); + coll.remove({$invalid: 'remove'}); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert(gle.err); + assert(gle.code); + assert(!gle.errmsg); + assert(gle.shards); + assert.eq(coll.count(), 0); + + // + // Repeated calls to GLE should work + coll.remove({}); + coll.update({_id: 1}, {$invalid: "xxx"}, true); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert(gle.err); + assert(gle.code); + assert(!gle.errmsg); + assert(gle.singleShard); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert(gle.err); + assert(gle.code); + assert(!gle.errmsg); + assert(gle.singleShard); + assert.eq(coll.count(), 0); + + // + // Geo $near is not supported on mongos + coll.ensureIndex({loc: "2dsphere"}); + coll.remove({}); + var query = { + loc: { + $near: { + $geometry: {type: "Point", coordinates: [0, 0]}, + $maxDistance: 1000, + } } - } -}; -printjson(coll.remove(query)); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert(gle.err); -assert(gle.code); -assert(!gle.errmsg); -assert(gle.shards); -assert.eq(coll.count(), 0); - -// -// First shard down -// - -// -// Successful bulk insert on two hosts, host dies before gle (error contacting host) -coll.remove({}); -coll.insert([{_id: 1}, {_id: -1}]); -// Wait for write to be written to shards before shutting it down. -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -MongoRunner.stopMongod(st.shard0); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -// Should get an error about contacting dead host. -assert(!gle.ok); -assert(gle.errmsg); - -// -// Failed insert on two hosts, first host dead -// NOTE: This is DIFFERENT from 2.4, since we don't need to contact a host we didn't get -// successful writes from. -coll.remove({_id: 1}); -coll.insert([{_id: 1}, {_id: -1}]); -printjson(gle = coll.getDB().runCommand({getLastError: 1})); -assert(gle.ok); -assert(gle.err); -assert.eq(coll.count({_id: 1}), 1); - -jsTest.log("DONE!"); - -st.stop(); + }; + printjson(coll.remove(query)); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert(gle.err); + assert(gle.code); + assert(!gle.errmsg); + assert(gle.shards); + assert.eq(coll.count(), 0); + + // + // First shard down + // + + // + // Successful bulk insert on two hosts, host dies before gle (error contacting host) + coll.remove({}); + coll.insert([{_id: 1}, {_id: -1}]); + // Wait for write to be written to shards before shutting it down. + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + MongoRunner.stopMongod(st.shard0); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + // Should get an error about contacting dead host. + assert(!gle.ok); + assert(gle.errmsg); + + // + // Failed insert on two hosts, first host dead + // NOTE: This is DIFFERENT from 2.4, since we don't need to contact a host we didn't get + // successful writes from. + coll.remove({_id: 1}); + coll.insert([{_id: 1}, {_id: -1}]); + printjson(gle = coll.getDB().runCommand({getLastError: 1})); + assert(gle.ok); + assert(gle.err); + assert.eq(coll.count({_id: 1}), 1); + + st.stop(); +})(); diff --git a/jstests/multiVersion/libs/multi_rs.js b/jstests/multiVersion/libs/multi_rs.js index 673be42d3df..da976c7f4dc 100644 --- a/jstests/multiVersion/libs/multi_rs.js +++ b/jstests/multiVersion/libs/multi_rs.js @@ -50,7 +50,12 @@ ReplSetTest.prototype.upgradeNode = function(node, opts, user, pwd) { var isMaster = node.getDB('admin').runCommand({isMaster: 1}); if (!isMaster.arbiterOnly) { - assert.commandWorked(node.adminCommand("replSetMaintenance")); + // Must retry this command, as it might return "currently running for election" and fail. + // Node might still be running for an election that will fail because it lost the election + // race with another node, at test initialization. See SERVER-23133. + assert.soon(function() { + return (node.adminCommand("replSetMaintenance").ok); + }); this.waitForState(node, ReplSetTest.State.RECOVERING); } diff --git a/jstests/noPassthrough/backup_restore.js b/jstests/noPassthrough/backup_restore.js index 95d111e2cd3..0074c8c1f8d 100644 --- a/jstests/noPassthrough/backup_restore.js +++ b/jstests/noPassthrough/backup_restore.js @@ -226,8 +226,8 @@ rst.start(secondary.nodeId, {}, true); } - // Wait up to 60 seconds until restarted node is in state secondary - rst.waitForState(rst.getSecondaries(), ReplSetTest.State.SECONDARY, 60 * 1000); + // Wait up to 5 minutes until restarted node is in state secondary. + rst.waitForState(rst.getSecondaries(), ReplSetTest.State.SECONDARY); // Add new hidden node to replSetTest var hiddenCfg = { @@ -263,8 +263,7 @@ // Wait up to 60 seconds until the new hidden node is in state RECOVERING. rst.waitForState(rst.nodes[numNodes], - [ReplSetTest.State.RECOVERING, ReplSetTest.State.SECONDARY], - 60 * 1000); + [ReplSetTest.State.RECOVERING, ReplSetTest.State.SECONDARY]); // Stop CRUD client and FSM client. assert(checkProgram(crudPid), testName + ' CRUD client was not running at end of test'); @@ -272,8 +271,8 @@ stopMongoProgramByPid(crudPid); stopMongoProgramByPid(fsmPid); - // Wait up to 60 seconds until the new hidden node is in state SECONDARY. - rst.waitForState(rst.nodes[numNodes], ReplSetTest.State.SECONDARY, 60 * 1000); + // Wait up to 5 minutes until the new hidden node is in state SECONDARY. + rst.waitForState(rst.nodes[numNodes], ReplSetTest.State.SECONDARY); // Wait for secondaries to finish catching up before shutting down. assert.writeOK(primary.getDB("test").foo.insert( diff --git a/jstests/noPassthrough/initial_sync_cloner_dups.js b/jstests/noPassthrough/initial_sync_cloner_dups.js index 1208dc8a16e..c0a3e71c0c1 100644 --- a/jstests/noPassthrough/initial_sync_cloner_dups.js +++ b/jstests/noPassthrough/initial_sync_cloner_dups.js @@ -79,8 +79,7 @@ // Wait for the secondary to get ReplSetInitiate command. replTest.waitForState( secondary, - [ReplSetTest.State.STARTUP_2, ReplSetTest.State.RECOVERING, ReplSetTest.State.SECONDARY], - 60 * 1000); + [ReplSetTest.State.STARTUP_2, ReplSetTest.State.RECOVERING, ReplSetTest.State.SECONDARY]); // This fail point will cause the first intial sync to fail, and leave an op in the buffer to // verify the fix from SERVER-17807 diff --git a/jstests/noPassthroughWithMongod/no_balance_collection.js b/jstests/noPassthroughWithMongod/no_balance_collection.js index cfec6199ca2..1c2f1aae009 100644 --- a/jstests/noPassthroughWithMongod/no_balance_collection.js +++ b/jstests/noPassthroughWithMongod/no_balance_collection.js @@ -1,14 +1,11 @@ // Tests whether the noBalance flag disables balancing for collections -var st = new ShardingTest({shards: 2, mongos: 1, verbose: 1}); +var st = new ShardingTest({shards: 2, mongos: 1}); // First, test that shell helpers require an argument assert.throws(sh.disableBalancing, [], "sh.disableBalancing requires a collection"); assert.throws(sh.enableBalancing, [], "sh.enableBalancing requires a collection"); -// Initially stop balancing -st.stopBalancer(); - var shardAName = st._shardNames[0]; var shardBName = st._shardNames[1]; @@ -70,10 +67,11 @@ jsTest.log("Chunks for " + collB + " are balanced."); // Re-disable balancing for collB sh.disableBalancing(collB); + // Wait for the balancer to fully finish the last migration and write the changelog // MUST set db var here, ugly but necessary db = st.s0.getDB("config"); -sh.waitForBalancer(true); +st.waitForBalancerRound(); // Make sure auto-migrates on insert don't move chunks var lastMigration = sh._lastMigration(collB); diff --git a/jstests/replsets/apply_ops_insert_write_conflict_nonatomic.js b/jstests/replsets/apply_ops_insert_write_conflict_nonatomic.js new file mode 100644 index 00000000000..4ef394f5682 --- /dev/null +++ b/jstests/replsets/apply_ops_insert_write_conflict_nonatomic.js @@ -0,0 +1,8 @@ +(function() { + 'use strict'; + + load("jstests/replsets/libs/apply_ops_insert_write_conflict.js"); + + new ApplyOpsInsertWriteConflictTest( + {testName: 'apply_ops_insert_write_conflict_nonatomic', atomic: false}).run(); +}()); diff --git a/jstests/replsets/election_not_blocked.js b/jstests/replsets/election_not_blocked.js index 95b53be1ebc..88d66715929 100644 --- a/jstests/replsets/election_not_blocked.js +++ b/jstests/replsets/election_not_blocked.js @@ -25,7 +25,7 @@ // so it cannot vote while fsync locked in PV1. Use PV0 explicitly here. protocolVersion: 0 }); - replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary(); // do a write diff --git a/jstests/replsets/initial_sync1.js b/jstests/replsets/initial_sync1.js index 55a454b765c..51d355d80c1 100644 --- a/jstests/replsets/initial_sync1.js +++ b/jstests/replsets/initial_sync1.js @@ -77,9 +77,7 @@ wait(function() { return config2.version == config.version && (config3 && config3.version == config.version); }); -replTest.waitForState(slave2, - [ReplSetTest.State.SECONDARY, ReplSetTest.State.RECOVERING], - 60 * 1000); +replTest.waitForState(slave2, [ReplSetTest.State.SECONDARY, ReplSetTest.State.RECOVERING]); print("7. Kill the secondary in the middle of syncing"); replTest.stop(slave1); @@ -91,7 +89,7 @@ replTest.waitForState(slave2, ReplSetTest.State.SECONDARY, 60 * 1000); print("9. Bring the secondary back up"); replTest.start(slave1, {}, true); reconnect(slave1); -replTest.waitForState(slave1, [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY], 60 * 1000); +replTest.waitForState(slave1, [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY]); print("10. Insert some stuff"); master = replTest.getPrimary(); diff --git a/jstests/replsets/initial_sync2.js b/jstests/replsets/initial_sync2.js index bab1063c072..69c91d30f04 100644 --- a/jstests/replsets/initial_sync2.js +++ b/jstests/replsets/initial_sync2.js @@ -25,7 +25,7 @@ var doTest = function() { var conns = replTest.startSet(); replTest.initiate(); - replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 5 * 60 * 1000); + replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary(); var foo = master.getDB("foo"); @@ -73,27 +73,27 @@ var doTest = function() { }); admin_s2.runCommand({replSetFreeze: 999999}); - replTest.waitForState( - replTest.nodes[2], [ReplSetTest.State.SECONDARY, ReplSetTest.State.RECOVERING], 60 * 1000); + replTest.waitForState(replTest.nodes[2], + [ReplSetTest.State.SECONDARY, ReplSetTest.State.RECOVERING]); jsTest.log("7. Kill #1 in the middle of syncing"); replTest.stop(0); jsTest.log("8. Check that #3 makes it into secondary state"); - replTest.waitForState( - replTest.nodes[2], [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY], 60 * 1000); + replTest.waitForState(replTest.nodes[2], + [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY]); jsTest.log("9. Bring #1 back up"); replTest.start(0, {}, true); - replTest.waitForState( - replTest.nodes[0], [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY], 60 * 1000); + replTest.waitForState(replTest.nodes[0], + [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY]); jsTest.log("10. Initial sync should succeed"); - replTest.waitForState( - replTest.nodes[2], [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY], 60 * 1000); + replTest.waitForState(replTest.nodes[2], + [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY]); jsTest.log("11. Ensure #1 becomes primary"); - replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); jsTest.log("12. Everyone happy eventually"); replTest.awaitReplication(2 * 60 * 1000); diff --git a/jstests/replsets/libs/apply_ops_insert_write_conflict.js b/jstests/replsets/libs/apply_ops_insert_write_conflict.js new file mode 100644 index 00000000000..9bbdea7e08e --- /dev/null +++ b/jstests/replsets/libs/apply_ops_insert_write_conflict.js @@ -0,0 +1,88 @@ +/** + * Sets up a test for WriteConflictException handling in applyOps with an insert workload. + */ +var ApplyOpsInsertWriteConflictTest = function(options) { + 'use strict'; + + if (!(this instanceof ApplyOpsInsertWriteConflictTest)) { + return new ApplyOpsInsertWriteConflictTest(options); + } + + // Capture the 'this' reference + var self = this; + + self.options = options; + + /** + * Runs the test. + */ + this.run = function() { + var options = this.options; + + var replTest = new ReplSetTest({nodes: 1}); + replTest.startSet(); + replTest.initiate(); + + var primary = replTest.getPrimary(); + var primaryDB = primary.getDB('test'); + + var t = primaryDB.getCollection(options.testName); + t.drop(); + + assert.commandWorked(primaryDB.createCollection(t.getName())); + + var numOps = 1000; + var ops = Array(numOps).fill('ignored').map((unused, i) => { + return { + op: 'i', + ns: t.getFullName(), + o: {_id: i} + }; + }); + + if (!options.atomic) { + // Adding a command to the list of operations to prevent the applyOps command from + // applying + // all the operations atomically. + ops.push({ns: "test.$cmd", op: "c", o: {applyOps: []}}); + numOps++; + } + + // Probabilities for WCE are chosen based on empirical testing. + // The probability for WCE during an atomic applyOps should be much smaller than that for + // the non-atomic case because we have to attempt to re-apply the entire batch of 'numOps' + // operations on WCE in the atomic case. + var probability = (options.atomic ? 0.1 : 5.0) / numOps; + + // Set up failpoint to trigger WriteConflictException during write operations. + assert.commandWorked( + primaryDB.adminCommand({setParameter: 1, traceWriteConflictExceptions: true})); + assert.commandWorked(primaryDB.adminCommand({ + configureFailPoint: 'WTWriteConflictException', + mode: {activationProbability: probability} + })); + + // This logs each operation being applied. + var previousLogLevel = + assert.commandWorked(primaryDB.setLogLevel(3, 'replication')).was.replication.verbosity; + + var applyOpsResult = primaryDB.adminCommand({applyOps: ops}); + + // Reset log level. + primaryDB.setLogLevel(previousLogLevel, 'replication'); + + assert.eq( + numOps, + applyOpsResult.applied, + 'number of operations applied did not match list of generated insert operations. ' + + 'applyOps result: ' + tojson(applyOpsResult)); + applyOpsResult.results.forEach((operationSucceeded, i) => { + assert(operationSucceeded, + 'applyOps failed: operation with index ' + i + ' failed: operation: ' + + tojson(ops[i], '', true) + '. applyOps result: ' + tojson(applyOpsResult)); + }); + assert.commandWorked(applyOpsResult); + + replTest.stopSet(); + }; +}; diff --git a/jstests/replsets/maintenance.js b/jstests/replsets/maintenance.js index b1fe94efc0e..8b4765212b8 100644 --- a/jstests/replsets/maintenance.js +++ b/jstests/replsets/maintenance.js @@ -5,7 +5,7 @@ var conns = replTest.startSet({verbose: 1}); var config = replTest.getReplSetConfig(); config.members[0].priority = 2; replTest.initiate(config); -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); // Make sure we have a master var master = replTest.getPrimary(); diff --git a/jstests/replsets/oplog_truncated_on_recovery.js b/jstests/replsets/oplog_truncated_on_recovery.js index 4d469178691..477da7b4d92 100644 --- a/jstests/replsets/oplog_truncated_on_recovery.js +++ b/jstests/replsets/oplog_truncated_on_recovery.js @@ -74,7 +74,7 @@ log(assert.commandWorked(localDB.adminCommand("replSetGetStatus"))); log("restart primary"); replTest.restart(master); - replTest.waitForState(master, ReplSetTest.State.RECOVERING, 90000); + replTest.waitForState(master, ReplSetTest.State.RECOVERING); assert.soon(function() { var mv; diff --git a/jstests/replsets/priority_takeover_one_node_higher_priority.js b/jstests/replsets/priority_takeover_one_node_higher_priority.js index 81f7717a0ee..de20f71c854 100644 --- a/jstests/replsets/priority_takeover_one_node_higher_priority.js +++ b/jstests/replsets/priority_takeover_one_node_higher_priority.js @@ -13,7 +13,7 @@ replSet.startSet(); replSet.initiate(); - replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY); var primary = replSet.getPrimary(); replSet.awaitSecondaryNodes(); @@ -21,19 +21,19 @@ // Primary should step down long enough for election to occur on secondary. var config = assert.commandWorked(primary.adminCommand({replSetGetConfig: 1})).config; - var electionTimeoutMillis = config.settings.electionTimeoutMillis; - var stepDownGuardMillis = electionTimeoutMillis * 2; var stepDownException = assert.throws(function() { - primary.adminCommand({replSetStepDown: stepDownGuardMillis / 1000}); + primary.adminCommand({replSetStepDown: replSet.kDefaultTimeoutMS / 1000}); }); assert.neq(-1, tojson(stepDownException).indexOf('error doing query'), 'replSetStepDown did not disconnect client'); // Step down primary and wait for node 1 to be promoted to primary. - replSet.waitForState(replSet.nodes[1], ReplSetTest.State.PRIMARY, 60 * 1000); + replSet.waitForState(replSet.nodes[1], ReplSetTest.State.PRIMARY); + + // Unfreeze node 0 so it can seek election. + assert.commandWorked(primary.adminCommand({replSetFreeze: 0})); // Eventually node 0 will stand for election again because it has a higher priorty. - replSet.waitForState( - replSet.nodes[0], ReplSetTest.State.PRIMARY, stepDownGuardMillis + 60 * 1000); + replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY); })(); diff --git a/jstests/replsets/read_committed_with_catalog_changes.js b/jstests/replsets/read_committed_with_catalog_changes.js index dae14da31de..4290391bd73 100644 --- a/jstests/replsets/read_committed_with_catalog_changes.js +++ b/jstests/replsets/read_committed_with_catalog_changes.js @@ -208,13 +208,13 @@ load("jstests/replsets/rslib.js"); // For startSetIfSupportsReadMajority. // they may be passed in to a ScopedThread. function assertReadsBlock(coll) { var res = - coll.runCommand('find', {"readConcern": {"level": "majority"}, "maxTimeMS": 1000}); + coll.runCommand('find', {"readConcern": {"level": "majority"}, "maxTimeMS": 5000}); assert.commandFailedWithCode(res, ErrorCodes.ExceededTimeLimit, "Expected read of " + coll.getFullName() + " to block"); } - function assertReadsSucceed(coll, timeoutMs = 1000) { + function assertReadsSucceed(coll, timeoutMs = 20000) { var res = coll.runCommand('find', {"readConcern": {"level": "majority"}, "maxTimeMS": timeoutMs}); assert.commandWorked(res, 'reading from ' + coll.getFullName()); diff --git a/jstests/replsets/replsetadd_profile.js b/jstests/replsets/replsetadd_profile.js index 641e7ca7cfd..2e396a61eb7 100644 --- a/jstests/replsets/replsetadd_profile.js +++ b/jstests/replsets/replsetadd_profile.js @@ -19,7 +19,7 @@ masterCollection.save({a: 1}); var newNode = replTest.add(); replTest.reInitiate(); -replTest.waitForState(replTest.nodes[1], ReplSetTest.State.SECONDARY, 60 * 1000); +replTest.waitForState(replTest.nodes[1], ReplSetTest.State.SECONDARY); // Allow documents to propagate to new replica set member. replTest.awaitReplication(); diff --git a/jstests/replsets/replsetprio1.js b/jstests/replsets/replsetprio1.js index 16beb851b81..5aee0a33a92 100644 --- a/jstests/replsets/replsetprio1.js +++ b/jstests/replsets/replsetprio1.js @@ -16,10 +16,10 @@ }); // 2 should be master (give this a while to happen, as other nodes might first be elected) - replTest.waitForState(nodes[2], ReplSetTest.State.PRIMARY, 120000); + replTest.waitForState(nodes[2], ReplSetTest.State.PRIMARY); // wait for 1 to not appear to be master (we are about to make it master and need a clean slate // here) - replTest.waitForState(nodes[1], ReplSetTest.State.SECONDARY, 60000); + replTest.waitForState(nodes[1], ReplSetTest.State.SECONDARY); // Wait for election oplog entry to be replicated, to ensure 0 will vote for 1 after stopping 2. replTest.awaitReplication(); @@ -28,7 +28,7 @@ replTest.stop(2); // 1 should eventually be master - replTest.waitForState(nodes[1], ReplSetTest.State.PRIMARY, 60000); + replTest.waitForState(nodes[1], ReplSetTest.State.PRIMARY); // do some writes on 1 var master = replTest.getPrimary(); @@ -42,7 +42,7 @@ // bring 2 back up, 2 should wait until caught up and then become master replTest.restart(2); - replTest.waitForState(nodes[2], ReplSetTest.State.PRIMARY, 60000); + replTest.waitForState(nodes[2], ReplSetTest.State.PRIMARY); // make sure nothing was rolled back master = replTest.getPrimary(); diff --git a/jstests/replsets/request_primary_stepdown.js b/jstests/replsets/request_primary_stepdown.js index 02050bd55f4..3e56946397b 100644 --- a/jstests/replsets/request_primary_stepdown.js +++ b/jstests/replsets/request_primary_stepdown.js @@ -15,7 +15,7 @@ conf.protocolVersion = 0; replSet.initiate(conf); - replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY); replSet.awaitSecondaryNodes(); replSet.awaitReplication(); var primary = replSet.getPrimary(); diff --git a/jstests/replsets/resync_with_write_load.js b/jstests/replsets/resync_with_write_load.js index 1a782ffacbe..13b6042ab58 100644 --- a/jstests/replsets/resync_with_write_load.js +++ b/jstests/replsets/resync_with_write_load.js @@ -19,7 +19,7 @@ var config = { ] }; var r = replTest.initiate(config); -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); // Make sure we have a master var master = replTest.getPrimary(); var a_conn = conns[0]; diff --git a/jstests/replsets/rollback.js b/jstests/replsets/rollback.js index 56aaed37d6f..6baeb88666a 100644 --- a/jstests/replsets/rollback.js +++ b/jstests/replsets/rollback.js @@ -46,7 +46,7 @@ load("jstests/replsets/rslib.js"); }); // Make sure we have a master - replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary(); var a_conn = conns[0]; var A = a_conn.getDB("admin"); diff --git a/jstests/replsets/rollback2.js b/jstests/replsets/rollback2.js index cd4f5a049b7..1cbf8194880 100644 --- a/jstests/replsets/rollback2.js +++ b/jstests/replsets/rollback2.js @@ -42,7 +42,7 @@ load("jstests/replsets/rslib.js"); }); // Make sure we have a master and that that master is node A - replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary(); var a_conn = conns[0]; a_conn.setSlaveOk(); diff --git a/jstests/replsets/rollback3.js b/jstests/replsets/rollback3.js index 59b685c7bad..5eb9ba574d1 100644 --- a/jstests/replsets/rollback3.js +++ b/jstests/replsets/rollback3.js @@ -47,7 +47,7 @@ load("jstests/replsets/rslib.js"); }); // Make sure we have a master and that that master is node A - replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary(); var a_conn = conns[0]; a_conn.setSlaveOk(); diff --git a/jstests/replsets/rollback5.js b/jstests/replsets/rollback5.js index cc0007822dd..81614447607 100644 --- a/jstests/replsets/rollback5.js +++ b/jstests/replsets/rollback5.js @@ -23,7 +23,7 @@ var r = replTest.initiate({ }); // Make sure we have a master -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary(); var a_conn = conns[0]; var b_conn = conns[1]; diff --git a/jstests/replsets/rollback_auth.js b/jstests/replsets/rollback_auth.js index 0c0b35b91ed..4b31a0e527e 100644 --- a/jstests/replsets/rollback_auth.js +++ b/jstests/replsets/rollback_auth.js @@ -39,7 +39,7 @@ }); // Make sure we have a master - replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary(); var a_conn = conns[0]; var b_conn = conns[1]; diff --git a/jstests/replsets/rollback_cmd_unrollbackable.js b/jstests/replsets/rollback_cmd_unrollbackable.js index 41b8f77f74f..01c0bc5bf38 100644 --- a/jstests/replsets/rollback_cmd_unrollbackable.js +++ b/jstests/replsets/rollback_cmd_unrollbackable.js @@ -26,7 +26,7 @@ var AID = replTest.getNodeId(a_conn); var BID = replTest.getNodeId(b_conn); // get master and do an initial write -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary(); assert(master === conns[0], "conns[0] assumed to be master"); assert(a_conn.host === master.host, "a_conn assumed to be master"); diff --git a/jstests/replsets/rollback_collMod_fatal.js b/jstests/replsets/rollback_collMod_fatal.js index 61af67b15d0..76a43c8cd4c 100644 --- a/jstests/replsets/rollback_collMod_fatal.js +++ b/jstests/replsets/rollback_collMod_fatal.js @@ -25,7 +25,7 @@ var b_conn = conns[1]; var AID = replTest.getNodeId(a_conn); var BID = replTest.getNodeId(b_conn); -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); // get master and do an initial write var master = replTest.getPrimary(); diff --git a/jstests/replsets/rollback_different_h.js b/jstests/replsets/rollback_different_h.js index 4b9aede1bbc..37f6f0a71cd 100644 --- a/jstests/replsets/rollback_different_h.js +++ b/jstests/replsets/rollback_different_h.js @@ -36,7 +36,7 @@ var b_conn = conns[1]; var AID = replTest.getNodeId(a_conn); var BID = replTest.getNodeId(b_conn); -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); // get master and do an initial write var master = replTest.getPrimary(); diff --git a/jstests/replsets/rollback_dropdb.js b/jstests/replsets/rollback_dropdb.js index b8f7d8d09ee..5853f7c47ce 100644 --- a/jstests/replsets/rollback_dropdb.js +++ b/jstests/replsets/rollback_dropdb.js @@ -25,7 +25,7 @@ var b_conn = conns[1]; var AID = replTest.getNodeId(a_conn); var BID = replTest.getNodeId(b_conn); -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); // get master and do an initial write var master = replTest.getPrimary(); diff --git a/jstests/replsets/rollback_fake_cmd.js b/jstests/replsets/rollback_fake_cmd.js index 6eba30d2c16..fcc5fbaf39b 100644 --- a/jstests/replsets/rollback_fake_cmd.js +++ b/jstests/replsets/rollback_fake_cmd.js @@ -36,7 +36,7 @@ var b_conn = conns[1]; var AID = replTest.getNodeId(a_conn); var BID = replTest.getNodeId(b_conn); -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); // get master and do an initial write var master = replTest.getPrimary(); diff --git a/jstests/replsets/rollback_index.js b/jstests/replsets/rollback_index.js index d3bf747680c..66b2be66f63 100644 --- a/jstests/replsets/rollback_index.js +++ b/jstests/replsets/rollback_index.js @@ -38,7 +38,7 @@ var b_conn = conns[1]; var AID = replTest.getNodeId(a_conn); var BID = replTest.getNodeId(b_conn); -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); // get master and do an initial write var master = replTest.getPrimary(); diff --git a/jstests/replsets/rslib.js b/jstests/replsets/rslib.js index b97f79c00d1..3b41531d760 100644 --- a/jstests/replsets/rslib.js +++ b/jstests/replsets/rslib.js @@ -138,7 +138,7 @@ var startSetIfSupportsReadMajority; } printjson(state); return true; - }, "not all members ready", timeout || 60000); + }, "not all members ready", timeout || 10 * 60 * 1000); print("All members are now in state PRIMARY, SECONDARY, or ARBITER"); }; diff --git a/jstests/replsets/stepdown.js b/jstests/replsets/stepdown.js index 5a8388da4d4..bed42cff3e5 100644 --- a/jstests/replsets/stepdown.js +++ b/jstests/replsets/stepdown.js @@ -20,7 +20,7 @@ var replTest = new ReplSetTest({ }); var nodes = replTest.startSet(); replTest.initiate(); -replTest.waitForState(nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary(); // do a write diff --git a/jstests/replsets/stepdown_kill_other_ops.js b/jstests/replsets/stepdown_kill_other_ops.js index 930206046c1..d360624bc7a 100644 --- a/jstests/replsets/stepdown_kill_other_ops.js +++ b/jstests/replsets/stepdown_kill_other_ops.js @@ -15,7 +15,7 @@ ] }); - replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY); var primary = replSet.getPrimary(); assert.eq(primary.host, nodes[0], "primary assumed to be node 0"); diff --git a/jstests/replsets/stepdown_killop.js b/jstests/replsets/stepdown_killop.js index 5c0e0ffae91..1b86d5fb8d4 100644 --- a/jstests/replsets/stepdown_killop.js +++ b/jstests/replsets/stepdown_killop.js @@ -23,7 +23,7 @@ ] }); - replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY); var secondary = replSet.getSecondary(); jsTestLog('Disable replication on the SECONDARY ' + secondary.host); diff --git a/jstests/replsets/stepdown_long_wait_time.js b/jstests/replsets/stepdown_long_wait_time.js index 60e0fdb4247..eb1bf7007d9 100644 --- a/jstests/replsets/stepdown_long_wait_time.js +++ b/jstests/replsets/stepdown_long_wait_time.js @@ -22,7 +22,7 @@ ] }); - replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + replSet.waitForState(replSet.nodes[0], ReplSetTest.State.PRIMARY); var primary = replSet.getPrimary(); var secondary = replSet.getSecondary(); diff --git a/jstests/replsets/sync_passive.js b/jstests/replsets/sync_passive.js index 4899385563f..c0be375b98b 100644 --- a/jstests/replsets/sync_passive.js +++ b/jstests/replsets/sync_passive.js @@ -29,7 +29,7 @@ config.members[0].priority = 2; config.members[2].priority = 0; replTest.initiate(config); -replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); +replTest.waitForState(replTest.nodes[0], ReplSetTest.State.PRIMARY); var master = replTest.getPrimary().getDB("test"); var server0 = master; diff --git a/jstests/replsets/two_nodes_priority_take_over.js b/jstests/replsets/two_nodes_priority_take_over.js index f6e62fe681d..ae0d53e92b6 100644 --- a/jstests/replsets/two_nodes_priority_take_over.js +++ b/jstests/replsets/two_nodes_priority_take_over.js @@ -29,7 +29,7 @@ if (false) { }); // The first node will be the primary at the beginning. - rst.waitForState(rst.nodes[0], ReplSetTest.State.PRIMARY, 60 * 1000); + rst.waitForState(rst.nodes[0], ReplSetTest.State.PRIMARY); // Get the term when replset is stable. var res = rst.getPrimary().adminCommand("replSetGetStatus"); diff --git a/jstests/sharding/balance_repl.js b/jstests/sharding/balance_repl.js index 46404646995..59cc694fc42 100644 --- a/jstests/sharding/balance_repl.js +++ b/jstests/sharding/balance_repl.js @@ -3,7 +3,7 @@ // (function() { - "use strict"; + 'use strict'; // The mongod secondaries are set to priority 0 and votes 0 to prevent the primaries // from stepping down during migrations on slow evergreen builders. @@ -27,14 +27,16 @@ } assert.writeOK(bulk.execute()); - s.adminCommand({enablesharding: "test"}); + assert.commandWorked(s.s0.adminCommand({enablesharding: "test"})); s.ensurePrimaryShard('test', 'test-rs0'); - s.adminCommand({shardcollection: "test.foo", key: {_id: 1}}); + assert.commandWorked(s.s0.adminCommand({shardcollection: "test.foo", key: {_id: 1}})); - for (i = 0; i < 20; i++) - s.adminCommand({split: "test.foo", middle: {_id: i * 100}}); + for (i = 0; i < 20; i++) { + assert.commandWorked(s.s0.adminCommand({split: "test.foo", middle: {_id: i * 100}})); + } assert.eq(2100, db.foo.find().itcount()); + var coll = db.foo; coll.setSlaveOk(); @@ -42,10 +44,9 @@ var other = s.config.shards.findOne({_id: {$ne: serverName}}); for (i = 0; i < 20; i++) { - // Needs to waitForDelete because we'll be performing a slaveOk query, - // and secondaries don't have a chunk manager so it doesn't know how to - // filter out docs it doesn't own. - assert(s.adminCommand({ + // Needs to waitForDelete because we'll be performing a slaveOk query, and secondaries don't + // have a chunk manager so it doesn't know how to filter out docs it doesn't own. + assert.commandWorked(s.s0.adminCommand({ moveChunk: "test.foo", find: {_id: i * 100}, to: other._id, @@ -53,9 +54,9 @@ writeConcern: {w: 2}, _waitForDelete: true })); + assert.eq(2100, coll.find().itcount()); } s.stop(); - }()); diff --git a/jstests/sharding/balance_tags2.js b/jstests/sharding/balance_tags2.js index e4bf370d1cd..a7f1161d6dc 100644 --- a/jstests/sharding/balance_tags2.js +++ b/jstests/sharding/balance_tags2.js @@ -1,27 +1,26 @@ // Test balancing all chunks to one shard by tagging the full shard-key range on that collection -var s = new ShardingTest( - {name: "balance_tags2", shards: 3, mongos: 1, other: {chunkSize: 1, enableBalancer: true}}); +var s = new ShardingTest({shards: 3, mongos: 1, other: {chunkSize: 1, enableBalancer: true}}); -s.adminCommand({enablesharding: "test"}); +assert.commandWorked(s.s0.adminCommand({enablesharding: "test"})); s.ensurePrimaryShard('test', 'shard0001'); var db = s.getDB("test"); var bulk = db.foo.initializeUnorderedBulkOp(); -for (i = 0; i < 21; i++) { +for (var i = 0; i < 21; i++) { bulk.insert({_id: i, x: i}); } assert.writeOK(bulk.execute()); -sh.shardCollection("test.foo", {_id: 1}); +assert.commandWorked(s.s0.adminCommand({shardCollection: "test.foo", key: {_id: 1}})); -sh.stopBalancer(); +s.stopBalancer(); -for (i = 0; i < 20; i++) { +for (var i = 0; i < 20; i++) { sh.splitAt("test.foo", {_id: i}); } -sh.startBalancer(); +s.startBalancer(); sh.status(true); diff --git a/jstests/sharding/explain_cmd.js b/jstests/sharding/explain_cmd.js index 3b8a8ef1240..b4ec0db35e9 100644 --- a/jstests/sharding/explain_cmd.js +++ b/jstests/sharding/explain_cmd.js @@ -1,174 +1,185 @@ // Tests for the mongos explain command. - -// Create a cluster with 3 shards. -var st = new ShardingTest({shards: 2}); -st.stopBalancer(); - -var db = st.s.getDB("test"); -var explain; - -// Setup a collection that will be sharded. The shard key will be 'a'. There's also an index on 'b'. -var collSharded = db.getCollection("mongos_explain_cmd"); -collSharded.drop(); -collSharded.ensureIndex({a: 1}); -collSharded.ensureIndex({b: 1}); - -// Enable sharding. -assert.commandWorked(db.adminCommand({enableSharding: db.getName()})); -st.ensurePrimaryShard(db.getName(), 'shard0001'); -db.adminCommand({shardCollection: collSharded.getFullName(), key: {a: 1}}); - -// Pre-split the collection to ensure that both shards have chunks. Explicitly -// move chunks since the balancer is disabled. -for (var i = 1; i <= 2; i++) { - assert.commandWorked(db.adminCommand({split: collSharded.getFullName(), middle: {a: i}})); - - var shardName = "shard000" + (i - 1); - printjson(db.adminCommand({moveChunk: collSharded.getFullName(), find: {a: i}, to: shardName})); -} - -// Put data on each shard. -for (var i = 0; i < 3; i++) { - collSharded.insert({_id: i, a: i, b: 1}); -} - -printjson(sh.status()); - -// Test a scatter-gather count command. -assert.eq(3, collSharded.count({b: 1})); - -// Explain the scatter-gather count. -explain = db.runCommand( - {explain: {count: collSharded.getName(), query: {b: 1}}, verbosity: "allPlansExecution"}); - -// Validate some basic properties of the result. -printjson(explain); -assert.commandWorked(explain); -assert("queryPlanner" in explain); -assert("executionStats" in explain); -assert.eq(2, explain.queryPlanner.winningPlan.shards.length); -assert.eq(2, explain.executionStats.executionStages.shards.length); - -// An explain of a command that doesn't exist should fail gracefully. -explain = db.runCommand({ - explain: {nonexistent: collSharded.getName(), query: {b: 1}}, - verbosity: "allPlansExecution" -}); -printjson(explain); -assert.commandFailed(explain); - -// ------- - -// Setup a collection that is not sharded. -var collUnsharded = db.getCollection("mongos_explain_cmd_unsharded"); -collUnsharded.drop(); -collUnsharded.ensureIndex({a: 1}); -collUnsharded.ensureIndex({b: 1}); - -for (var i = 0; i < 3; i++) { - collUnsharded.insert({_id: i, a: i, b: 1}); -} -assert.eq(3, collUnsharded.count({b: 1})); - -explain = db.runCommand({ - explain: { - group: { - ns: collUnsharded.getName(), - key: "a", - cond: "b", - $reduce: function(curr, result) {}, - initial: {} - } - }, - verbosity: "allPlansExecution" -}); - -// Basic validation: a group command can only be passed through to an unsharded collection, -// so we should confirm that the mongos stage is always SINGLE_SHARD. -printjson(explain); -assert.commandWorked(explain); -assert("queryPlanner" in explain); -assert("executionStats" in explain); -assert.eq("SINGLE_SHARD", explain.queryPlanner.winningPlan.stage); - -// The same group should fail over the sharded collection, because group is only supported -// if it is passed through to an unsharded collection. -explain = db.runCommand({ - explain: { - group: { - ns: collSharded.getName(), - key: "a", - cond: "b", - $reduce: function(curr, result) {}, - initial: {} - } - }, - verbosity: "allPlansExecution" -}); -printjson(explain); -assert.commandFailed(explain); - -// ------- - -// Explain a delete operation and verify that it hits all shards without the shard key -explain = db.runCommand({ - explain: {delete: collSharded.getName(), deletes: [{q: {b: 1}, limit: 0}]}, - verbosity: "allPlansExecution" -}); -assert.commandWorked(explain, tojson(explain)); -assert.eq(explain.queryPlanner.winningPlan.stage, "SHARD_WRITE"); -assert.eq(explain.queryPlanner.winningPlan.shards.length, 2); -assert.eq(explain.queryPlanner.winningPlan.shards[0].winningPlan.stage, "DELETE"); -assert.eq(explain.queryPlanner.winningPlan.shards[1].winningPlan.stage, "DELETE"); -// Check that the deletes didn't actually happen. -assert.eq(3, collSharded.count({b: 1})); - -// Explain a delete operation and verify that it hits only one shard with the shard key -explain = db.runCommand({ - explain: {delete: collSharded.getName(), deletes: [{q: {a: 1}, limit: 0}]}, - verbosity: "allPlansExecution" -}); -assert.commandWorked(explain, tojson(explain)); -assert.eq(explain.queryPlanner.winningPlan.shards.length, 1); -// Check that the deletes didn't actually happen. -assert.eq(3, collSharded.count({b: 1})); - -// Check that we fail gracefully if we try to do an explain of a write batch that has more -// than one operation in it. -explain = db.runCommand({ - explain: - {delete: collSharded.getName(), deletes: [{q: {a: 1}, limit: 1}, {q: {a: 2}, limit: 1}]}, - verbosity: "allPlansExecution" -}); -assert.commandFailed(explain, tojson(explain)); - -// Explain a multi upsert operation and verify that it hits all shards -explain = db.runCommand({ - explain: {update: collSharded.getName(), updates: [{q: {}, u: {$set: {b: 10}}, multi: true}]}, - verbosity: "allPlansExecution" -}); -assert.commandWorked(explain, tojson(explain)); -assert.eq(explain.queryPlanner.winningPlan.shards.length, 2); -assert.eq(explain.queryPlanner.winningPlan.stage, "SHARD_WRITE"); -assert.eq(explain.queryPlanner.winningPlan.shards.length, 2); -assert.eq(explain.queryPlanner.winningPlan.shards[0].winningPlan.stage, "UPDATE"); -assert.eq(explain.queryPlanner.winningPlan.shards[1].winningPlan.stage, "UPDATE"); -// Check that the update didn't actually happen. -assert.eq(0, collSharded.count({b: 10})); - -// Explain an upsert operation and verify that it hits only a single shard -explain = db.runCommand({ - explain: {update: collSharded.getName(), updates: [{q: {a: 10}, u: {a: 10}, upsert: true}]}, - verbosity: "allPlansExecution" -}); -assert.commandWorked(explain, tojson(explain)); -assert.eq(explain.queryPlanner.winningPlan.shards.length, 1); -// Check that the upsert didn't actually happen. -assert.eq(0, collSharded.count({a: 10})); - -// Explain an upsert operation which cannot be targeted, ensure an error is thrown -explain = db.runCommand({ - explain: {update: collSharded.getName(), updates: [{q: {b: 10}, u: {b: 10}, upsert: true}]}, - verbosity: "allPlansExecution" -}); -assert.commandFailed(explain, tojson(explain)); +(function() { + 'use strict'; + + // Create a cluster with 3 shards. + var st = new ShardingTest({shards: 2}); + + var db = st.s.getDB("test"); + var explain; + + // Setup a collection that will be sharded. The shard key will be 'a'. There's also an index on + // 'b'. + var collSharded = db.getCollection("mongos_explain_cmd"); + collSharded.drop(); + collSharded.ensureIndex({a: 1}); + collSharded.ensureIndex({b: 1}); + + // Enable sharding. + assert.commandWorked(db.adminCommand({enableSharding: db.getName()})); + st.ensurePrimaryShard(db.getName(), 'shard0001'); + db.adminCommand({shardCollection: collSharded.getFullName(), key: {a: 1}}); + + // Pre-split the collection to ensure that both shards have chunks. Explicitly + // move chunks since the balancer is disabled. + for (var i = 1; i <= 2; i++) { + assert.commandWorked(db.adminCommand({split: collSharded.getFullName(), middle: {a: i}})); + + var shardName = "shard000" + (i - 1); + printjson( + db.adminCommand({moveChunk: collSharded.getFullName(), find: {a: i}, to: shardName})); + } + + // Put data on each shard. + for (var i = 0; i < 3; i++) { + collSharded.insert({_id: i, a: i, b: 1}); + } + + st.printShardingStatus(); + + // Test a scatter-gather count command. + assert.eq(3, collSharded.count({b: 1})); + + // Explain the scatter-gather count. + explain = db.runCommand( + {explain: {count: collSharded.getName(), query: {b: 1}}, verbosity: "allPlansExecution"}); + + // Validate some basic properties of the result. + printjson(explain); + assert.commandWorked(explain); + assert("queryPlanner" in explain); + assert("executionStats" in explain); + assert.eq(2, explain.queryPlanner.winningPlan.shards.length); + assert.eq(2, explain.executionStats.executionStages.shards.length); + + // An explain of a command that doesn't exist should fail gracefully. + explain = db.runCommand({ + explain: {nonexistent: collSharded.getName(), query: {b: 1}}, + verbosity: "allPlansExecution" + }); + printjson(explain); + assert.commandFailed(explain); + + // ------- + + // Setup a collection that is not sharded. + var collUnsharded = db.getCollection("mongos_explain_cmd_unsharded"); + collUnsharded.drop(); + collUnsharded.ensureIndex({a: 1}); + collUnsharded.ensureIndex({b: 1}); + + for (var i = 0; i < 3; i++) { + collUnsharded.insert({_id: i, a: i, b: 1}); + } + assert.eq(3, collUnsharded.count({b: 1})); + + explain = db.runCommand({ + explain: { + group: { + ns: collUnsharded.getName(), + key: "a", + cond: "b", + $reduce: function(curr, result) {}, + initial: {} + } + }, + verbosity: "allPlansExecution" + }); + + // Basic validation: a group command can only be passed through to an unsharded collection, + // so we should confirm that the mongos stage is always SINGLE_SHARD. + printjson(explain); + assert.commandWorked(explain); + assert("queryPlanner" in explain); + assert("executionStats" in explain); + assert.eq("SINGLE_SHARD", explain.queryPlanner.winningPlan.stage); + + // The same group should fail over the sharded collection, because group is only supported + // if it is passed through to an unsharded collection. + explain = db.runCommand({ + explain: { + group: { + ns: collSharded.getName(), + key: "a", + cond: "b", + $reduce: function(curr, result) {}, + initial: {} + } + }, + verbosity: "allPlansExecution" + }); + printjson(explain); + assert.commandFailed(explain); + + // ------- + + // Explain a delete operation and verify that it hits all shards without the shard key + explain = db.runCommand({ + explain: {delete: collSharded.getName(), deletes: [{q: {b: 1}, limit: 0}]}, + verbosity: "allPlansExecution" + }); + assert.commandWorked(explain, tojson(explain)); + assert.eq(explain.queryPlanner.winningPlan.stage, "SHARD_WRITE"); + assert.eq(explain.queryPlanner.winningPlan.shards.length, 2); + assert.eq(explain.queryPlanner.winningPlan.shards[0].winningPlan.stage, "DELETE"); + assert.eq(explain.queryPlanner.winningPlan.shards[1].winningPlan.stage, "DELETE"); + // Check that the deletes didn't actually happen. + assert.eq(3, collSharded.count({b: 1})); + + // Explain a delete operation and verify that it hits only one shard with the shard key + explain = db.runCommand({ + explain: {delete: collSharded.getName(), deletes: [{q: {a: 1}, limit: 0}]}, + verbosity: "allPlansExecution" + }); + assert.commandWorked(explain, tojson(explain)); + assert.eq(explain.queryPlanner.winningPlan.shards.length, 1); + // Check that the deletes didn't actually happen. + assert.eq(3, collSharded.count({b: 1})); + + // Check that we fail gracefully if we try to do an explain of a write batch that has more + // than one operation in it. + explain = db.runCommand({ + explain: { + delete: collSharded.getName(), + deletes: [{q: {a: 1}, limit: 1}, {q: {a: 2}, limit: 1}] + }, + verbosity: "allPlansExecution" + }); + assert.commandFailed(explain, tojson(explain)); + + // Explain a multi upsert operation and verify that it hits all shards + explain = db.runCommand({ + explain: + {update: collSharded.getName(), updates: [{q: {}, u: {$set: {b: 10}}, multi: true}]}, + verbosity: "allPlansExecution" + }); + assert.commandWorked(explain, tojson(explain)); + assert.eq(explain.queryPlanner.winningPlan.shards.length, 2); + assert.eq(explain.queryPlanner.winningPlan.stage, "SHARD_WRITE"); + assert.eq(explain.queryPlanner.winningPlan.shards.length, 2); + assert.eq(explain.queryPlanner.winningPlan.shards[0].winningPlan.stage, "UPDATE"); + assert.eq(explain.queryPlanner.winningPlan.shards[1].winningPlan.stage, "UPDATE"); + // Check that the update didn't actually happen. + assert.eq(0, collSharded.count({b: 10})); + + // Explain an upsert operation and verify that it hits only a single shard + explain = db.runCommand({ + explain: + {update: collSharded.getName(), updates: [{q: {a: 10}, u: {a: 10}, upsert: true}]}, + verbosity: "allPlansExecution" + }); + assert.commandWorked(explain, tojson(explain)); + assert.eq(explain.queryPlanner.winningPlan.shards.length, 1); + // Check that the upsert didn't actually happen. + assert.eq(0, collSharded.count({a: 10})); + + // Explain an upsert operation which cannot be targeted, ensure an error is thrown + explain = db.runCommand({ + explain: + {update: collSharded.getName(), updates: [{q: {b: 10}, u: {b: 10}, upsert: true}]}, + verbosity: "allPlansExecution" + }); + assert.commandFailed(explain, tojson(explain)); + + st.stop(); +})(); diff --git a/jstests/sharding/explain_find_and_modify_sharded.js b/jstests/sharding/explain_find_and_modify_sharded.js index 40af14f6265..e8c69adc222 100644 --- a/jstests/sharding/explain_find_and_modify_sharded.js +++ b/jstests/sharding/explain_find_and_modify_sharded.js @@ -9,7 +9,6 @@ // Create a cluster with 2 shards. var st = new ShardingTest({shards: 2}); - st.stopBalancer(); var testDB = st.s.getDB('test'); var shardKey = { @@ -85,4 +84,5 @@ assert.commandWorked(res); assertExplainResult(res, 'executionStats', 'executionStages', 'shard0001', 'DELETE'); + st.stop(); })(); diff --git a/jstests/sharding/hash_shard_unique_compound.js b/jstests/sharding/hash_shard_unique_compound.js index 5d6d466c1f9..abaf45260b9 100644 --- a/jstests/sharding/hash_shard_unique_compound.js +++ b/jstests/sharding/hash_shard_unique_compound.js @@ -2,44 +2,42 @@ // Does 2 things and checks for consistent error: // 1.) shard collection on hashed "a", ensure unique index {a:1, b:1} // 2.) reverse order +(function() { + 'use strict'; -var s = new ShardingTest({name: jsTestName(), shards: 1, mongos: 1, verbose: 1}); -var dbName = "test"; -var collName = "foo"; -var ns = dbName + "." + collName; -var db = s.getDB(dbName); -var coll = db.getCollection(collName); + var s = new ShardingTest({shards: 1, mongos: 1}); + var dbName = "test"; + var collName = "foo"; + var ns = dbName + "." + collName; + var db = s.getDB(dbName); + var coll = db.getCollection(collName); -// Enable sharding on DB -var res = db.adminCommand({enablesharding: dbName}); + // Enable sharding on DB + assert.commandWorked(db.adminCommand({enablesharding: dbName})); -// for simplicity start by turning off balancer -var res = s.stopBalancer(); + // Shard a fresh collection using a hashed shard key + assert.commandWorked(db.adminCommand({shardcollection: ns, key: {a: "hashed"}})); -// shard a fresh collection using a hashed shard key -coll.drop(); -assert.commandWorked(db.adminCommand({shardcollection: ns, key: {a: "hashed"}})); -db.printShardingStatus(); + // Create unique index + assert.commandWorked(coll.ensureIndex({a: 1, b: 1}, {unique: true})); -// Create unique index -assert.commandWorked(coll.ensureIndex({a: 1, b: 1}, {unique: true})); + jsTest.log("------ indexes -------"); + jsTest.log(tojson(coll.getIndexes())); -jsTest.log("------ indexes -------"); -jsTest.log(tojson(coll.getIndexes())); + // Second Part + jsTest.log("------ dropping sharded collection to start part 2 -------"); + coll.drop(); -// Second Part -jsTest.log("------ dropping sharded collection to start part 2 -------"); -coll.drop(); + // Create unique index + assert.commandWorked(coll.ensureIndex({a: 1, b: 1}, {unique: true})); -// Create unique index -assert.commandWorked(coll.ensureIndex({a: 1, b: 1}, {unique: true})); + // shard a fresh collection using a hashed shard key + assert.commandWorked(db.adminCommand({shardcollection: ns, key: {a: "hashed"}}), + "shardcollection didn't worked 2"); -// shard a fresh collection using a hashed shard key -assert.commandWorked(db.adminCommand({shardcollection: ns, key: {a: "hashed"}}), - "shardcollection didn't worked 2"); + s.printShardingStatus(); + jsTest.log("------ indexes 2-------"); + jsTest.log(tojson(coll.getIndexes())); -db.printShardingStatus(); -jsTest.log("------ indexes 2-------"); -jsTest.log(tojson(coll.getIndexes())); - -s.stop(); + s.stop(); +})(); diff --git a/jstests/sharding/mapReduce_inSharded_outSharded.js b/jstests/sharding/mapReduce_inSharded_outSharded.js index d1aba2599f0..5190a1fe4ba 100644 --- a/jstests/sharding/mapReduce_inSharded_outSharded.js +++ b/jstests/sharding/mapReduce_inSharded_outSharded.js @@ -1,60 +1,70 @@ -var verifyOutput = function(out) { - printjson(out); - assert.eq(out.counts.input, 51200, "input count is wrong"); - assert.eq(out.counts.emit, 51200, "emit count is wrong"); - assert.gt(out.counts.reduce, 99, "reduce count is wrong"); - assert.eq(out.counts.output, 512, "output count is wrong"); -}; - -var st = new ShardingTest( - {shards: 2, verbose: 1, mongos: 1, other: {chunkSize: 1, enableBalancer: true}}); - -st.adminCommand({enablesharding: "mrShard"}); -st.ensurePrimaryShard('mrShard', 'shard0001'); -st.adminCommand({shardcollection: "mrShard.srcSharded", key: {"_id": 1}}); - -var db = st.getDB("mrShard"); - -var bulk = db.srcSharded.initializeUnorderedBulkOp(); -for (j = 0; j < 100; j++) { - for (i = 0; i < 512; i++) { - bulk.insert({j: j, i: i}); +(function() { + "use strict"; + + var verifyOutput = function(out) { + printjson(out); + assert.eq(out.counts.input, 51200, "input count is wrong"); + assert.eq(out.counts.emit, 51200, "emit count is wrong"); + assert.gt(out.counts.reduce, 99, "reduce count is wrong"); + assert.eq(out.counts.output, 512, "output count is wrong"); + }; + + var st = new ShardingTest( + {shards: 2, verbose: 1, mongos: 1, other: {chunkSize: 1, enableBalancer: true}}); + + var admin = st.s0.getDB('admin'); + + assert.commandWorked(admin.runCommand({enablesharding: "mrShard"})); + st.ensurePrimaryShard('mrShard', 'shard0001'); + assert.commandWorked( + admin.runCommand({shardcollection: "mrShard.srcSharded", key: {"_id": 1}})); + + var db = st.s0.getDB("mrShard"); + + var bulk = db.srcSharded.initializeUnorderedBulkOp(); + for (var j = 0; j < 100; j++) { + for (var i = 0; i < 512; i++) { + bulk.insert({j: j, i: i}); + } + } + assert.writeOK(bulk.execute()); + + function map() { + emit(this.i, 1); + } + function reduce(key, values) { + return Array.sum(values); } -} -assert.writeOK(bulk.execute()); - -function map() { - emit(this.i, 1); -} -function reduce(key, values) { - return Array.sum(values); -} - -// sharded src sharded dst -var suffix = "InShardedOutSharded"; - -var out = - db.srcSharded.mapReduce(map, reduce, {out: {replace: "mrReplace" + suffix, sharded: true}}); -verifyOutput(out); - -out = db.srcSharded.mapReduce(map, reduce, {out: {merge: "mrMerge" + suffix, sharded: true}}); -verifyOutput(out); - -out = db.srcSharded.mapReduce(map, reduce, {out: {reduce: "mrReduce" + suffix, sharded: true}}); -verifyOutput(out); - -out = db.srcSharded.mapReduce(map, reduce, {out: {inline: 1}}); -verifyOutput(out); -assert(out.results != 'undefined', "no results for inline"); - -out = db.srcSharded.mapReduce( - map, reduce, {out: {replace: "mrReplace" + suffix, db: "mrShardOtherDB", sharded: true}}); -verifyOutput(out); - -out = db.runCommand({ - mapReduce: "srcSharded", // use new name mapReduce rather than mapreduce - map: map, - reduce: reduce, - out: "mrBasic" + "srcSharded", -}); -verifyOutput(out); + + // sharded src sharded dst + var suffix = "InShardedOutSharded"; + + var out = db.srcSharded.mapReduce( + map, reduce, {out: {replace: "mrReplace" + suffix, sharded: true}}); + verifyOutput(out); + + out = db.srcSharded.mapReduce(map, reduce, {out: {merge: "mrMerge" + suffix, sharded: true}}); + verifyOutput(out); + + out = db.srcSharded.mapReduce(map, reduce, {out: {reduce: "mrReduce" + suffix, sharded: true}}); + verifyOutput(out); + + out = db.srcSharded.mapReduce(map, reduce, {out: {inline: 1}}); + verifyOutput(out); + assert(out.results != 'undefined', "no results for inline"); + + out = db.srcSharded.mapReduce( + map, reduce, {out: {replace: "mrReplace" + suffix, db: "mrShardOtherDB", sharded: true}}); + verifyOutput(out); + + out = db.runCommand({ + mapReduce: "srcSharded", // use new name mapReduce rather than mapreduce + map: map, + reduce: reduce, + out: "mrBasic" + "srcSharded", + }); + verifyOutput(out); + + st.stop(); + +})(); diff --git a/jstests/sharding/migrateBig.js b/jstests/sharding/migrateBig.js index e11782baed2..6e6be382795 100644 --- a/jstests/sharding/migrateBig.js +++ b/jstests/sharding/migrateBig.js @@ -1,64 +1,63 @@ (function() { + 'use strict'; var s = new ShardingTest({name: "migrateBig", shards: 2, other: {chunkSize: 1}}); - s.config.settings.update({_id: "balancer"}, {$set: {_waitForDelete: true}}, true); - s.adminCommand({enablesharding: "test"}); + assert.writeOK( + s.config.settings.update({_id: "balancer"}, {$set: {_waitForDelete: true}}, true)); + assert.commandWorked(s.s0.adminCommand({enablesharding: "test"})); s.ensurePrimaryShard('test', 'shard0001'); - s.adminCommand({shardcollection: "test.foo", key: {x: 1}}); + assert.commandWorked(s.s0.adminCommand({shardcollection: "test.foo", key: {x: 1}})); - db = s.getDB("test"); - coll = db.foo; + var db = s.getDB("test"); + var coll = db.foo; - big = ""; + var big = ""; while (big.length < 10000) big += "eliot"; var bulk = coll.initializeUnorderedBulkOp(); - for (x = 0; x < 100; x++) { + for (var x = 0; x < 100; x++) { bulk.insert({x: x, big: big}); } assert.writeOK(bulk.execute()); - db.printShardingStatus(); - - s.adminCommand({split: "test.foo", middle: {x: 30}}); - s.adminCommand({split: "test.foo", middle: {x: 66}}); - s.adminCommand( - {movechunk: "test.foo", find: {x: 90}, to: s.getOther(s.getServer("test")).name}); + assert.commandWorked(s.s0.adminCommand({split: "test.foo", middle: {x: 30}})); + assert.commandWorked(s.s0.adminCommand({split: "test.foo", middle: {x: 66}})); + assert.commandWorked(s.s0.adminCommand( + {movechunk: "test.foo", find: {x: 90}, to: s.getOther(s.getServer("test")).name})); db.printShardingStatus(); print("YO : " + s.getServer("test").host); - direct = new Mongo(s.getServer("test").host); + var direct = new Mongo(s.getServer("test").host); print("direct : " + direct); - directDB = direct.getDB("test"); + var directDB = direct.getDB("test"); - for (done = 0; done < 2 * 1024 * 1024; done += big.length) { + for (var done = 0; done < 2 * 1024 * 1024; done += big.length) { assert.writeOK(directDB.foo.insert({x: 50 + Math.random(), big: big})); } db.printShardingStatus(); assert.throws(function() { - s.adminCommand( - {movechunk: "test.foo", find: {x: 50}, to: s.getOther(s.getServer("test")).name}); + assert.commandWorked(s.s0.adminCommand( + {movechunk: "test.foo", find: {x: 50}, to: s.getOther(s.getServer("test")).name})); }, [], "move should fail"); - for (i = 0; i < 20; i += 2) { + for (var i = 0; i < 20; i += 2) { try { - s.adminCommand({split: "test.foo", middle: {x: i}}); + assert.commandWorked(s.s0.adminCommand({split: "test.foo", middle: {x: i}})); } catch (e) { - // we may have auto split on some of these - // which is ok + // We may have auto split on some of these, which is ok print(e); } } db.printShardingStatus(); - s.config.settings.update({_id: "balancer"}, {$set: {stopped: false}}, true); + s.startBalancer(); assert.soon(function() { var x = s.chunkDiff("foo", "test"); @@ -73,5 +72,4 @@ assert.eq(coll.count(), coll.find().itcount()); s.stop(); - })(); diff --git a/jstests/sharding/migrateBig_balancer.js b/jstests/sharding/migrateBig_balancer.js index cd44a225a62..906ed341c7c 100644 --- a/jstests/sharding/migrateBig_balancer.js +++ b/jstests/sharding/migrateBig_balancer.js @@ -1,11 +1,12 @@ (function() { + 'use strict'; var st = new ShardingTest({name: 'migrateBig_balancer', shards: 2, other: {enableBalancer: true}}); var mongos = st.s; var admin = mongos.getDB("admin"); - db = mongos.getDB("test"); + var db = mongos.getDB("test"); var coll = db.getCollection("stuff"); assert.commandWorked(admin.runCommand({enablesharding: coll.getDB().getName()})); @@ -18,7 +19,7 @@ for (var i = 0; i < nsq; i++) data += data; - dataObj = {}; + var dataObj = {}; for (var i = 0; i < n; i++) dataObj["data-" + i] = data; @@ -30,19 +31,16 @@ assert.eq(40, coll.count(), "prep1"); - printjson(coll.stats()); - - admin.printShardingStatus(); - - admin.runCommand({shardcollection: "" + coll, key: {_id: 1}}); + assert.commandWorked(admin.runCommand({shardcollection: "" + coll, key: {_id: 1}})); + st.printShardingStatus(); assert.lt( 5, mongos.getDB("config").chunks.find({ns: "test.stuff"}).count(), "not enough chunks"); assert.soon(function() { // On *extremely* slow or variable systems, we've seen migrations fail in the critical - // section and - // kill the server. Do an explicit check for this. SERVER-8781 + // section and kill the server. Do an explicit check for this. SERVER-8781 + // // TODO: Remove once we can better specify what systems to run what tests on. try { assert.commandWorked(st.shard0.getDB("admin").runCommand({ping: 1})); @@ -53,7 +51,7 @@ throw e; } - res = mongos.getDB("config").chunks.group({ + var res = mongos.getDB("config").chunks.group({ cond: {ns: "test.stuff"}, key: {shard: 1}, reduce: function(doc, out) { @@ -68,5 +66,4 @@ }, "never migrated", 10 * 60 * 1000, 1000); st.stop(); - })(); diff --git a/jstests/sharding/printShardingStatus.js b/jstests/sharding/printShardingStatus.js index 05e6eca0d4f..5bfa70c2d8f 100644 --- a/jstests/sharding/printShardingStatus.js +++ b/jstests/sharding/printShardingStatus.js @@ -3,6 +3,7 @@ // headings and the names of sharded collections and their shard keys. (function() { + 'use strict'; var st = new ShardingTest({shards: 1, mongos: 2, config: 1, other: {smallfiles: true}}); @@ -233,5 +234,4 @@ assert(mongos.getDB("test").dropDatabase()); st.stop(); - })(); diff --git a/jstests/sharding/shard3.js b/jstests/sharding/shard3.js index 6800e3f4370..290a9f79719 100644 --- a/jstests/sharding/shard3.js +++ b/jstests/sharding/shard3.js @@ -1,5 +1,4 @@ (function() { - // Include helpers for analyzing explain output. load("jstests/libs/analyze_plan.js"); @@ -17,11 +16,14 @@ } assert(sh.getBalancerState(), "A1"); - sh.setBalancerState(false); + + sh.stopBalancer(); assert(!sh.getBalancerState(), "A2"); - sh.setBalancerState(true); + + sh.startBalancer(); assert(sh.getBalancerState(), "A3"); - sh.setBalancerState(false); + + sh.stopBalancer(); assert(!sh.getBalancerState(), "A4"); s.config.databases.find().forEach(printjson); diff --git a/jstests/sharding/split_with_force_small.js b/jstests/sharding/split_with_force_small.js index 0148c924993..be21049650e 100644 --- a/jstests/sharding/split_with_force_small.js +++ b/jstests/sharding/split_with_force_small.js @@ -1,70 +1,69 @@ // // Tests autosplit locations with force : true, for small collections // +(function() { + 'use strict'; -var options = { - chunkSize: 1 // MB -}; + var st = new ShardingTest( + {shards: 1, mongos: 1, other: {chunkSize: 1, mongosOptions: {noAutoSplit: ""}}}); -var st = new ShardingTest({shards: 1, mongos: 1, other: options}); -st.stopBalancer(); + var mongos = st.s0; + var admin = mongos.getDB("admin"); + var config = mongos.getDB("config"); + var shardAdmin = st.shard0.getDB("admin"); + var coll = mongos.getCollection("foo.bar"); -var mongos = st.s0; -var admin = mongos.getDB("admin"); -var config = mongos.getDB("config"); -var shardAdmin = st.shard0.getDB("admin"); -var coll = mongos.getCollection("foo.bar"); + assert.commandWorked(admin.runCommand({enableSharding: coll.getDB() + ""})); + assert.commandWorked(admin.runCommand({shardCollection: coll + "", key: {_id: 1}})); + assert.commandWorked(admin.runCommand({split: coll + "", middle: {_id: 0}})); -assert(admin.runCommand({enableSharding: coll.getDB() + ""}).ok); -assert(admin.runCommand({shardCollection: coll + "", key: {_id: 1}}).ok); -assert(admin.runCommand({split: coll + "", middle: {_id: 0}}).ok); + jsTest.log("Insert a bunch of data into the low chunk of a collection," + + " to prevent relying on stats."); -jsTest.log("Insert a bunch of data into the low chunk of a collection," + - " to prevent relying on stats."); + var data128k = "x"; + for (var i = 0; i < 7; i++) + data128k += data128k; -var data128k = "x"; -for (var i = 0; i < 7; i++) - data128k += data128k; + var bulk = coll.initializeUnorderedBulkOp(); + for (var i = 0; i < 1024; i++) { + bulk.insert({_id: -(i + 1)}); + } + assert.writeOK(bulk.execute()); -var bulk = coll.initializeUnorderedBulkOp(); -for (var i = 0; i < 1024; i++) { - bulk.insert({_id: -(i + 1)}); -} -assert.writeOK(bulk.execute()); + jsTest.log("Insert 32 docs into the high chunk of a collection"); -jsTest.log("Insert 32 docs into the high chunk of a collection"); + bulk = coll.initializeUnorderedBulkOp(); + for (var i = 0; i < 32; i++) { + bulk.insert({_id: i}); + } + assert.writeOK(bulk.execute()); -bulk = coll.initializeUnorderedBulkOp(); -for (var i = 0; i < 32; i++) { - bulk.insert({_id: i}); -} -assert.writeOK(bulk.execute()); + jsTest.log("Split off MaxKey chunk..."); -jsTest.log("Split off MaxKey chunk..."); + assert.commandWorked(admin.runCommand({split: coll + "", middle: {_id: 32}})); -assert(admin.runCommand({split: coll + "", middle: {_id: 32}}).ok); + jsTest.log("Keep splitting chunk multiple times..."); -jsTest.log("Keep splitting chunk multiple times..."); - -st.printShardingStatus(); - -for (var i = 0; i < 5; i++) { - assert(admin.runCommand({split: coll + "", find: {_id: 0}}).ok); st.printShardingStatus(); -} -// Make sure we can't split further than 5 (2^5) times -assert(!admin.runCommand({split: coll + "", find: {_id: 0}}).ok); + for (var i = 0; i < 5; i++) { + assert.commandWorked(admin.runCommand({split: coll + "", find: {_id: 0}})); + st.printShardingStatus(); + } + + // Make sure we can't split further than 5 (2^5) times + assert.commandFailed(admin.runCommand({split: coll + "", find: {_id: 0}})); -var chunks = config.chunks.find({'min._id': {$gte: 0, $lt: 32}}).sort({min: 1}).toArray(); -printjson(chunks); + var chunks = config.chunks.find({'min._id': {$gte: 0, $lt: 32}}).sort({min: 1}).toArray(); + printjson(chunks); -// Make sure the chunks grow by 2x (except the first) -var nextSize = 1; -for (var i = 0; i < chunks.size; i++) { - assert.eq(coll.count({_id: {$gte: chunks[i].min._id, $lt: chunks[i].max._id}}), nextSize); - if (i != 0) - nextSize += nextSize; -} + // Make sure the chunks grow by 2x (except the first) + var nextSize = 1; + for (var i = 0; i < chunks.size; i++) { + assert.eq(coll.count({_id: {$gte: chunks[i].min._id, $lt: chunks[i].max._id}}), nextSize); + if (i != 0) + nextSize += nextSize; + } -st.stop(); + st.stop(); +})(); diff --git a/jstests/sharding/stale_version_write.js b/jstests/sharding/stale_version_write.js index e5885dcfa41..bd603124548 100644 --- a/jstests/sharding/stale_version_write.js +++ b/jstests/sharding/stale_version_write.js @@ -1,37 +1,37 @@ // Tests whether a reset sharding version triggers errors +(function() { + 'use strict'; -jsTest.log("Starting sharded cluster..."); + var st = new ShardingTest({shards: 1, mongos: 2}); -var st = new ShardingTest({shards: 1, mongos: 2, verbose: 2}); + var mongosA = st.s0; + var mongosB = st.s1; -st.stopBalancer(); + jsTest.log("Adding new collections..."); -var mongosA = st.s0; -var mongosB = st.s1; + var collA = mongosA.getCollection(jsTestName() + ".coll"); + assert.writeOK(collA.insert({hello: "world"})); -jsTest.log("Adding new collections..."); + var collB = mongosB.getCollection("" + collA); + assert.writeOK(collB.insert({hello: "world"})); -var collA = mongosA.getCollection(jsTestName() + ".coll"); -assert.writeOK(collA.insert({hello: "world"})); + jsTest.log("Enabling sharding..."); -var collB = mongosB.getCollection("" + collA); -assert.writeOK(collB.insert({hello: "world"})); + assert.commandWorked(mongosA.getDB("admin").adminCommand({enableSharding: "" + collA.getDB()})); + assert.commandWorked( + mongosA.getDB("admin").adminCommand({shardCollection: "" + collA, key: {_id: 1}})); -jsTest.log("Enabling sharding..."); + // MongoD doesn't know about the config shard version *until* MongoS tells it + collA.findOne(); -printjson(mongosA.getDB("admin").runCommand({enableSharding: "" + collA.getDB()})); -printjson(mongosA.getDB("admin").runCommand({shardCollection: "" + collA, key: {_id: 1}})); + jsTest.log("Trigger shard version mismatch..."); -// MongoD doesn't know about the config shard version *until* MongoS tells it -collA.findOne(); + assert.writeOK(collB.insert({goodbye: "world"})); -jsTest.log("Trigger shard version mismatch..."); + print("Inserted..."); -assert.writeOK(collB.insert({goodbye: "world"})); + assert.eq(3, collA.find().itcount()); + assert.eq(3, collB.find().itcount()); -print("Inserted..."); - -assert.eq(3, collA.find().itcount()); -assert.eq(3, collB.find().itcount()); - -st.stop(); + st.stop(); +})(); diff --git a/jstests/slow2/mr_during_migrate.js b/jstests/slow2/mr_during_migrate.js index cb439aeb241..1b3f55721f4 100644 --- a/jstests/slow2/mr_during_migrate.js +++ b/jstests/slow2/mr_during_migrate.js @@ -1,113 +1,112 @@ // Do parallel ops with migrates occurring +(function() { + 'use strict'; -var st = new ShardingTest({shards: 10, mongos: 2, verbose: 2}); + var st = new ShardingTest({shards: 10, mongos: 2, verbose: 2}); -jsTest.log("Doing parallel operations..."); + var mongos = st.s0; + var admin = mongos.getDB("admin"); + var coll = st.s.getCollection(jsTest.name() + ".coll"); -// Stop balancer, since it'll just get in the way of these -st.stopBalancer(); + var numDocs = 1024 * 1024; + var dataSize = 1024; // bytes, must be power of 2 -var mongos = st.s0; -var admin = mongos.getDB("admin"); -var coll = st.s.getCollection(jsTest.name() + ".coll"); + var data = "x"; + while (data.length < dataSize) + data += data; -var numDocs = 1024 * 1024; -var dataSize = 1024; // bytes, must be power of 2 + var bulk = coll.initializeUnorderedBulkOp(); + for (var i = 0; i < numDocs; i++) { + bulk.insert({_id: i, data: data}); + } + assert.writeOK(bulk.execute()); -var data = "x"; -while (data.length < dataSize) - data += data; + // Make sure everything got inserted + assert.eq(numDocs, coll.find().itcount()); -var bulk = coll.initializeUnorderedBulkOp(); -for (var i = 0; i < numDocs; i++) { - bulk.insert({_id: i, data: data}); -} -assert.writeOK(bulk.execute()); + jsTest.log("Inserted " + sh._dataFormat(dataSize * numDocs) + " of data."); -// Make sure everything got inserted -assert.eq(numDocs, coll.find().itcount()); + // Shard collection + st.shardColl(coll, {_id: 1}, false); -jsTest.log("Inserted " + sh._dataFormat(dataSize * numDocs) + " of data."); + st.printShardingStatus(); -// Shard collection -st.shardColl(coll, {_id: 1}, false); + jsTest.log("Sharded collection now initialized, starting migrations..."); -st.printShardingStatus(); - -jsTest.log("Sharded collection now initialized, starting migrations..."); + var checkMigrate = function() { + print("Result of migrate : "); + printjson(this); + }; -var checkMigrate = function() { - print("Result of migrate : "); - printjson(this); -}; + // Creates a number of migrations of random chunks to diff shard servers + var ops = []; + for (var i = 0; i < st._connections.length; i++) { + ops.push({ + op: "command", + ns: "admin", + command: { + moveChunk: "" + coll, + find: {_id: {"#RAND_INT": [0, numDocs]}}, + to: st._connections[i].shardName, + _waitForDelete: true + }, + showResult: true + }); + } -// Creates a number of migrations of random chunks to diff shard servers -var ops = []; -for (var i = 0; i < st._connections.length; i++) { - ops.push({ - op: "command", - ns: "admin", - command: { - moveChunk: "" + coll, - find: {_id: {"#RAND_INT": [0, numDocs]}}, - to: st._connections[i].shardName, - _waitForDelete: true - }, - showResult: true - }); -} + // TODO: Also migrate output collection -// TODO: Also migrate output collection + jsTest.log("Starting migrations now..."); -jsTest.log("Starting migrations now..."); + var bid = benchStart({ops: ops, host: st.s.host, parallel: 1, handleErrors: false}); -var bid = benchStart({ops: ops, host: st.s.host, parallel: 1, handleErrors: false}); + //####################### + // Tests during migration -//####################### -// Tests during migration + var numTests = 5; -var numTests = 5; + for (var t = 0; t < numTests; t++) { + jsTest.log("Test #" + t); -for (var t = 0; t < numTests; t++) { - jsTest.log("Test #" + t); + var mongos = st.s1; // use other mongos so we get stale shard versions + var coll = mongos.getCollection(coll + ""); + var outputColl = mongos.getCollection(coll + "_output"); - var mongos = st.s1; // use other mongos so we get stale shard versions - var coll = mongos.getCollection(coll + ""); - var outputColl = mongos.getCollection(coll + "_output"); + var numTypes = 32; + var map = function() { + emit(this._id % 32 /* must be hardcoded */, {c: 1}); + }; - var numTypes = 32; - var map = function() { - emit(this._id % 32 /* must be hardcoded */, {c: 1}); - }; - var reduce = function(k, vals) { - var total = 0; - for (var i = 0; i < vals.length; i++) - total += vals[i].c; - return { - c: total + var reduce = function(k, vals) { + var total = 0; + for (var i = 0; i < vals.length; i++) + total += vals[i].c; + return { + c: total + }; }; - }; - printjson(coll.find({_id: 0}).itcount()); + printjson(coll.find({_id: 0}).itcount()); - jsTest.log("Starting new mapReduce run #" + t); + jsTest.log("Starting new mapReduce run #" + t); - // assert.eq( coll.find().itcount(), numDocs ) + // assert.eq( coll.find().itcount(), numDocs ) - coll.getMongo().getDB("admin").runCommand({setParameter: 1, traceExceptions: true}); + coll.getMongo().getDB("admin").runCommand({setParameter: 1, traceExceptions: true}); - printjson(coll.mapReduce( - map, reduce, {out: {replace: outputColl.getName(), db: outputColl.getDB() + ""}})); + printjson(coll.mapReduce( + map, reduce, {out: {replace: outputColl.getName(), db: outputColl.getDB() + ""}})); - jsTest.log("MapReduce run #" + t + " finished."); + jsTest.log("MapReduce run #" + t + " finished."); - assert.eq(outputColl.find().itcount(), numTypes); + assert.eq(outputColl.find().itcount(), numTypes); - outputColl.find().forEach(function(x) { - assert.eq(x.value.c, numDocs / numTypes); - }); -} + outputColl.find().forEach(function(x) { + assert.eq(x.value.c, numDocs / numTypes); + }); + } -printjson(benchFinish(bid)); + printjson(benchFinish(bid)); -st.stop(); + st.stop(); +})(); diff --git a/src/mongo/base/validate_locale.cpp b/src/mongo/base/validate_locale.cpp index 5a4320c34c5..81207d4d89d 100644 --- a/src/mongo/base/validate_locale.cpp +++ b/src/mongo/base/validate_locale.cpp @@ -30,6 +30,7 @@ #include #include "mongo/base/init.h" +#include "mongo/util/mongoutils/str.h" namespace mongo { @@ -38,13 +39,15 @@ MONGO_INITIALIZER_GENERAL(ValidateLocale, MONGO_NO_PREREQUISITES, MONGO_DEFAULT_ try { // Validate that boost can correctly load the user's locale boost::filesystem::path("/").has_root_directory(); - } catch (const std::runtime_error&) { - return Status(ErrorCodes::BadValue, - "Invalid or no user locale set." + } catch (const std::runtime_error& e) { + return Status( + ErrorCodes::BadValue, + str::stream() + << "Invalid or no user locale set. " #ifndef _WIN32 - " Please ensure LANG and/or LC_* environment variables are set correctly." + << " Please ensure LANG and/or LC_* environment variables are set correctly. " #endif - ); + << e.what()); } return Status::OK(); } diff --git a/src/mongo/db/commands/mr.cpp b/src/mongo/db/commands/mr.cpp index f93dee2b554..ced9f2bbdb7 100644 --- a/src/mongo/db/commands/mr.cpp +++ b/src/mongo/db/commands/mr.cpp @@ -308,7 +308,7 @@ Config::Config(const string& _dbname, const BSONObj& cmdObj) { // scope and code if (cmdObj["scope"].type() == Object) - scopeSetup = cmdObj["scope"].embeddedObjectUserCheck(); + scopeSetup = cmdObj["scope"].embeddedObjectUserCheck().getOwned(); mapper.reset(new JSMapper(cmdObj["map"])); reducer.reset(new JSReducer(cmdObj["reduce"])); @@ -316,7 +316,7 @@ Config::Config(const string& _dbname, const BSONObj& cmdObj) { finalizer.reset(new JSFinalizer(cmdObj["finalize"])); if (cmdObj["mapparams"].type() == Array) { - mapParams = cmdObj["mapparams"].embeddedObjectUserCheck(); + mapParams = cmdObj["mapparams"].embeddedObjectUserCheck().getOwned(); } } @@ -787,6 +787,7 @@ void State::init() { AuthorizationSession::get(ClientBasic::getCurrent())->getAuthenticatedUserNamesToken(); _scope.reset(globalScriptEngine->newScopeForCurrentThread()); _scope->registerOperation(_txn); + _scope->requireOwnedObjects(); _scope->setLocalDB(_config.dbname); _scope->loadStored(_txn, true); @@ -1435,6 +1436,7 @@ public: BSONObj o; PlanExecutor::ExecState execState; while (PlanExecutor::ADVANCED == (execState = exec->getNext(&o, NULL))) { + o = o.getOwned(); // we will be accessing outside of the lock // check to see if this is a new object we don't own yet // because of a chunk migration if (collMetadata) { diff --git a/src/mongo/db/dbhelpers.cpp b/src/mongo/db/dbhelpers.cpp index d3c62ea0ef8..ced194addbf 100644 --- a/src/mongo/db/dbhelpers.cpp +++ b/src/mongo/db/dbhelpers.cpp @@ -448,11 +448,12 @@ long long Helpers::removeRange(OperationContext* txn, txn, repl::ReplClientInfo::forClient(txn->getClient()).getLastOp(), writeConcern); - if (replStatus.status.code() == ErrorCodes::ExceededTimeLimit) { + if (replStatus.status.code() == ErrorCodes::ExceededTimeLimit || + replStatus.status.code() == ErrorCodes::WriteConcernFailed) { warning(LogComponent::kSharding) << "replication to secondaries for removeRange at " "least 60 seconds behind"; } else { - massertStatusOK(replStatus.status); + uassertStatusOK(replStatus.status); } millisWaitingForReplication += replStatus.duration; } diff --git a/src/mongo/db/exec/group.cpp b/src/mongo/db/exec/group.cpp index 4ce1d4031db..31a72df4772 100644 --- a/src/mongo/db/exec/group.cpp +++ b/src/mongo/db/exec/group.cpp @@ -148,7 +148,8 @@ Status GroupStage::processObject(const BSONObj& obj) { } } - _scope->setObject("obj", obj, true); + BSONObj objCopy = obj.getOwned(); + _scope->setObject("obj", objCopy, true); _scope->setNumber("n", n - 1); try { diff --git a/src/mongo/db/ops/update_driver.cpp b/src/mongo/db/ops/update_driver.cpp index c0c11082a2e..a99376174a8 100644 --- a/src/mongo/db/ops/update_driver.cpp +++ b/src/mongo/db/ops/update_driver.cpp @@ -183,6 +183,13 @@ Status UpdateDriver::populateDocumentWithQueryFields(const BSONObj& query, return populateDocumentWithQueryFields(*cq, immutablePaths, doc); } +namespace { + +const FieldRef idPath("_id"); +const vector emptyImmutablePaths; + +} // namespace + Status UpdateDriver::populateDocumentWithQueryFields(const CanonicalQuery& query, const vector* immutablePathsPtr, mutablebson::Document& doc) const { @@ -192,13 +199,12 @@ Status UpdateDriver::populateDocumentWithQueryFields(const CanonicalQuery& query if (isDocReplacement()) { FieldRefSet pathsToExtract; - // TODO: Refactor update logic, make _id just another immutable field - static const FieldRef idPath("_id"); - static const vector emptyImmutablePaths; const vector& immutablePaths = immutablePathsPtr ? *immutablePathsPtr : emptyImmutablePaths; pathsToExtract.fillFrom(immutablePaths); + + // TODO: Refactor update logic, make _id just another immutable field pathsToExtract.insert(&idPath); // Extract only immutable fields from replacement-style diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp index 63858050014..2c764baf9e3 100644 --- a/src/mongo/db/repl/oplog.cpp +++ b/src/mongo/db/repl/oplog.cpp @@ -825,11 +825,7 @@ Status applyOperation_inlock(OperationContext* txn, Status status{ErrorCodes::NotYetInitialized, ""}; { WriteUnitOfWork wuow(txn); - try { - status = collection->insertDocument(txn, o, true); - } catch (DBException dbe) { - status = dbe.toStatus(); - } + status = collection->insertDocument(txn, o, true); if (status.isOK()) { wuow.commit(); } diff --git a/src/mongo/dbtests/jstests.cpp b/src/mongo/dbtests/jstests.cpp index 53cead63a3a..04a9180dbbd 100644 --- a/src/mongo/dbtests/jstests.cpp +++ b/src/mongo/dbtests/jstests.cpp @@ -2361,6 +2361,49 @@ public: } }; +class RequiresOwnedObjects { +public: + void run() { + char buf[] = {5, 0, 0, 0, 0}; + BSONObj unowned(buf); + BSONObj owned = unowned.getOwned(); + + ASSERT(!unowned.isOwned()); + ASSERT(owned.isOwned()); + + // Ensure that by default we can bind owned and unowned + { + unique_ptr s(globalScriptEngine->newScope()); + s->setObject("unowned", unowned, true); + s->setObject("owned", owned, true); + } + + // After we set the flag, we should only be able to set owned + { + unique_ptr s(globalScriptEngine->newScope()); + s->requireOwnedObjects(); + s->setObject("owned", owned, true); + + bool threwException = false; + try { + s->setObject("unowned", unowned, true); + } catch (...) { + threwException = true; + + auto status = exceptionToStatus(); + + ASSERT_EQUALS(status.code(), ErrorCodes::BadValue); + } + + ASSERT(threwException); + + // after resetting, we can set unowned's again + s->reset(); + s->setObject("unowned", unowned, true); + } + } +}; + class All : public Suite { public: All() : Suite("js") { @@ -2422,6 +2465,7 @@ public: add(); add(); + add(); add(); add(); diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript index 34efb20cf88..cf8d04827a6 100644 --- a/src/mongo/s/SConscript +++ b/src/mongo/s/SConscript @@ -75,9 +75,9 @@ env.Library( '$BUILD_DIR/mongo/s/catalog/forwarding_catalog_manager', '$BUILD_DIR/mongo/s/catalog/replset/catalog_manager_replica_set', '$BUILD_DIR/mongo/s/coreshard', - '$BUILD_DIR/mongo/s/mongoscore', '$BUILD_DIR/mongo/util/clock_source_mock', '$BUILD_DIR/mongo/util/net/message_port_mock', + 'mongoscore', ], LIBDEPS_TAGS=[ # Depends on coreshard, but that would be circular diff --git a/src/mongo/s/catalog/catalog_manager.h b/src/mongo/s/catalog/catalog_manager.h index 5257f53f7db..2f75ba2da69 100644 --- a/src/mongo/s/catalog/catalog_manager.h +++ b/src/mongo/s/catalog/catalog_manager.h @@ -451,9 +451,6 @@ public: */ virtual bool isMetadataConsistentFromLastCheck(OperationContext* txn) = 0; -protected: - CatalogManager() = default; - /** * Obtains a reference to the distributed lock manager instance to use for synchronizing * system-wide changes. @@ -462,6 +459,9 @@ protected: * be cached. */ virtual DistLockManager* getDistLockManager() = 0; + +protected: + CatalogManager() = default; }; } // namespace mongo diff --git a/src/mongo/s/chunk_manager.cpp b/src/mongo/s/chunk_manager.cpp index f785e748c54..29e596948d4 100644 --- a/src/mongo/s/chunk_manager.cpp +++ b/src/mongo/s/chunk_manager.cpp @@ -98,6 +98,9 @@ public: string shardFor(OperationContext* txn, const string& hostName) const final { const auto shard = grid.shardRegistry()->getShard(txn, hostName); + uassert(ErrorCodes::ShardNotFound, + str::stream() << "Shard " << hostName << " not found.", + shard); return shard->getId(); } diff --git a/src/mongo/scripting/deadline_monitor.cpp b/src/mongo/scripting/deadline_monitor.cpp index 5eb0f52e5de..f75c34a0cc2 100644 --- a/src/mongo/scripting/deadline_monitor.cpp +++ b/src/mongo/scripting/deadline_monitor.cpp @@ -34,7 +34,7 @@ namespace mongo { -MONGO_EXPORT_SERVER_PARAMETER(scriptingEngineInterruptIntervalMS, int, 1000); +MONGO_EXPORT_SERVER_PARAMETER(scriptingEngineInterruptIntervalMS, int, 0); int getScriptingEngineInterruptInterval() { return scriptingEngineInterruptIntervalMS.load(); diff --git a/src/mongo/scripting/deadline_monitor.h b/src/mongo/scripting/deadline_monitor.h index b1c2855dd46..05acd5c349c 100644 --- a/src/mongo/scripting/deadline_monitor.h +++ b/src/mongo/scripting/deadline_monitor.h @@ -138,7 +138,7 @@ private: const Date_t now = Date_t::now(); const auto interruptInterval = Milliseconds{getScriptingEngineInterruptInterval()}; - if (now - lastInterruptCycle > interruptInterval) { + if ((interruptInterval.count() > 0) && (now - lastInterruptCycle > interruptInterval)) { for (const auto& task : _tasks) { if (task.second > now) task.first->interrupt(); @@ -148,9 +148,13 @@ private: // wait for a task to be added or a deadline to expire if (_nearestDeadlineWallclock > now) { - if (_nearestDeadlineWallclock == Date_t::max() || - _nearestDeadlineWallclock - now > interruptInterval) { - _newDeadlineAvailable.wait_for(lk, interruptInterval); + if (_nearestDeadlineWallclock == Date_t::max()) { + if ((interruptInterval.count() > 0) && + (_nearestDeadlineWallclock - now > interruptInterval)) { + _newDeadlineAvailable.wait_for(lk, interruptInterval); + } else { + _newDeadlineAvailable.wait(lk); + } } else { _newDeadlineAvailable.wait_until(lk, _nearestDeadlineWallclock.toSystemTimePoint()); diff --git a/src/mongo/scripting/engine.cpp b/src/mongo/scripting/engine.cpp index 7e6628344ab..71c42f2fe74 100644 --- a/src/mongo/scripting/engine.cpp +++ b/src/mongo/scripting/engine.cpp @@ -422,6 +422,9 @@ public: void advanceGeneration() { _real->advanceGeneration(); } + void requireOwnedObjects() override { + _real->requireOwnedObjects(); + } bool isKillPending() const { return _real->isKillPending(); } diff --git a/src/mongo/scripting/engine.h b/src/mongo/scripting/engine.h index d31311f0114..2fbae5c347e 100644 --- a/src/mongo/scripting/engine.h +++ b/src/mongo/scripting/engine.h @@ -105,6 +105,8 @@ public: virtual void advanceGeneration() = 0; + virtual void requireOwnedObjects() = 0; + virtual ScriptingFunction createFunction(const char* code); /** diff --git a/src/mongo/scripting/mozjs/bson.cpp b/src/mongo/scripting/mozjs/bson.cpp index 1e7e4b7c190..3783bc0755a 100644 --- a/src/mongo/scripting/mozjs/bson.cpp +++ b/src/mongo/scripting/mozjs/bson.cpp @@ -59,13 +59,17 @@ namespace { * the appearance of mutable state on the read/write versions. */ struct BSONHolder { - BSONHolder(const BSONObj& obj, const BSONObj* parent, std::size_t generation, bool ro) + BSONHolder(const BSONObj& obj, const BSONObj* parent, const MozJSImplScope* scope, bool ro) : _obj(obj), - _generation(generation), + _generation(scope->getGeneration()), _isOwned(obj.isOwned() || (parent && parent->isOwned())), _resolved(false), _readOnly(ro), _altered(false) { + uassert( + ErrorCodes::BadValue, + "Attempt to bind an unowned BSON Object to a JS scope marked as requiring ownership", + _isOwned || (!scope->requiresOwnedObjects())); if (parent) { _parent.emplace(*parent); } @@ -107,7 +111,7 @@ void BSONInfo::make( auto scope = getScope(cx); scope->getProto().newObject(obj); - JS_SetPrivate(obj, new BSONHolder(bson, parent, scope->getGeneration(), ro)); + JS_SetPrivate(obj, new BSONHolder(bson, parent, scope, ro)); } void BSONInfo::finalize(JSFreeOp* fop, JSObject* obj) { diff --git a/src/mongo/scripting/mozjs/implscope.cpp b/src/mongo/scripting/mozjs/implscope.cpp index 3b260c1558c..e85efd97505 100644 --- a/src/mongo/scripting/mozjs/implscope.cpp +++ b/src/mongo/scripting/mozjs/implscope.cpp @@ -322,6 +322,7 @@ MozJSImplScope::MozJSImplScope(MozJSScriptEngine* engine) _status(Status::OK()), _quickExit(false), _generation(0), + _requireOwnedObjects(false), _hasOutOfMemoryException(false), _binDataProto(_context), _bsonProto(_context), @@ -749,6 +750,7 @@ void MozJSImplScope::reset() { unregisterOperation(); _pendingKill.store(false); _pendingGC.store(false); + _requireOwnedObjects = false; advanceGeneration(); } @@ -869,6 +871,14 @@ void MozJSImplScope::advanceGeneration() { _generation++; } +void MozJSImplScope::requireOwnedObjects() { + _requireOwnedObjects = true; +} + +bool MozJSImplScope::requiresOwnedObjects() const { + return _requireOwnedObjects; +} + const std::string& MozJSImplScope::getParentStack() const { return _parentStack; } diff --git a/src/mongo/scripting/mozjs/implscope.h b/src/mongo/scripting/mozjs/implscope.h index bfff70a93b0..edfc236e90c 100644 --- a/src/mongo/scripting/mozjs/implscope.h +++ b/src/mongo/scripting/mozjs/implscope.h @@ -305,6 +305,10 @@ public: void advanceGeneration() override; + void requireOwnedObjects() override; + + bool requiresOwnedObjects() const; + JS::HandleId getInternedStringId(InternedString name) { return _internedStrings.getInternedString(name); } @@ -373,6 +377,7 @@ private: bool _quickExit; std::string _parentStack; std::size_t _generation; + bool _requireOwnedObjects; bool _hasOutOfMemoryException; WrapType _binDataProto; diff --git a/src/mongo/scripting/mozjs/objectwrapper.cpp b/src/mongo/scripting/mozjs/objectwrapper.cpp index 77ea9a1eca8..8781f792484 100644 --- a/src/mongo/scripting/mozjs/objectwrapper.cpp +++ b/src/mongo/scripting/mozjs/objectwrapper.cpp @@ -564,7 +564,12 @@ ObjectWrapper::WriteFieldRecursionFrame::WriteFieldRecursionFrame(JSContext* cx, ids.infallibleAppend(rid); } } else { - JS::AutoIdArray rids(cx, JS_Enumerate(cx, thisv)); + auto ridArrayPtr = JS_Enumerate(cx, thisv); + if (!ridArrayPtr) { + throwCurrentJSException( + cx, ErrorCodes::JSInterpreterFailure, "Failure to enumerate object"); + } + JS::AutoIdArray rids(cx, ridArrayPtr); if (!ids.reserve(rids.length())) { throwCurrentJSException( diff --git a/src/mongo/scripting/mozjs/proxyscope.cpp b/src/mongo/scripting/mozjs/proxyscope.cpp index bb4cd5c06ff..9bb11ce400b 100644 --- a/src/mongo/scripting/mozjs/proxyscope.cpp +++ b/src/mongo/scripting/mozjs/proxyscope.cpp @@ -120,6 +120,10 @@ void MozJSProxyScope::advanceGeneration() { run([&] { _implScope->advanceGeneration(); }); } +void MozJSProxyScope::requireOwnedObjects() { + run([&] { _implScope->requireOwnedObjects(); }); +} + double MozJSProxyScope::getNumber(const char* field) { double out; run([&] { out = _implScope->getNumber(field); }); diff --git a/src/mongo/scripting/mozjs/proxyscope.h b/src/mongo/scripting/mozjs/proxyscope.h index 451981330a1..4dd69a3ebe9 100644 --- a/src/mongo/scripting/mozjs/proxyscope.h +++ b/src/mongo/scripting/mozjs/proxyscope.h @@ -129,6 +129,8 @@ public: void advanceGeneration() override; + void requireOwnedObjects() override; + double getNumber(const char* field) override; int getNumberInt(const char* field) override; long long getNumberLongLong(const char* field) override; diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js index 4a64f5e50a0..89ab91da580 100644 --- a/src/mongo/shell/replsettest.js +++ b/src/mongo/shell/replsettest.js @@ -89,7 +89,7 @@ var ReplSetTest = function(opts) { var _unbridgedPorts; var _unbridgedNodes; - this.kDefaultTimeoutMS = 5 * 60 * 1000; + this.kDefaultTimeoutMS = 10 * 60 * 1000; // Publicly exposed variables diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index 1f7f7d9fd3a..f3852d00ac8 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -731,6 +731,7 @@ fsyncLock fsyncs ftruncate func +fvisibility gcc gdb ge diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index ac79ffd029a..512892eb44d 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -150,6 +150,7 @@ connection_stats = [ ConnStat('read_io', 'total read I/Os'), ConnStat('rwlock_read', 'pthread mutex shared lock read-lock calls'), ConnStat('rwlock_write', 'pthread mutex shared lock write-lock calls'), + ConnStat('time_travel', 'detected system time went backwards'), ConnStat('write_io', 'total write I/Os'), ########################################## diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index abf3fe5cb9c..8bd00db3aa2 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "b8f590dea0400666ef26e21adf11c5997bb5ef1b", + "commit": "827b48a34227243c809d41fac3dc909ed46b0c5e", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-3.2" diff --git a/src/third_party/wiredtiger/src/btree/row_key.c b/src/third_party/wiredtiger/src/btree/row_key.c index 032fdf7d897..5bb09832eed 100644 --- a/src/third_party/wiredtiger/src/btree/row_key.c +++ b/src/third_party/wiredtiger/src/btree/row_key.c @@ -471,6 +471,8 @@ __wt_row_ikey_alloc(WT_SESSION_IMPL *session, { WT_IKEY *ikey; + WT_ASSERT(session, key != NULL); /* quiet clang scan-build */ + /* * Allocate memory for the WT_IKEY structure and the key, then copy * the key into place. diff --git a/src/third_party/wiredtiger/src/docs/programming.dox b/src/third_party/wiredtiger/src/docs/programming.dox index aa76bef4614..205e7544c6c 100644 --- a/src/third_party/wiredtiger/src/docs/programming.dox +++ b/src/third_party/wiredtiger/src/docs/programming.dox @@ -65,19 +65,20 @@ each of which is ordered by one or more columns. - @subpage_single wtperf - @subpage_single wtstats

-- @subpage_single tune_memory_allocator -- @subpage_single tune_page_size_and_comp -- @subpage_single tune_cache +- @subpage_single tune_build_options - @subpage_single tune_bulk_load +- @subpage_single tune_cache +- @subpage_single tune_checksum +- @subpage_single tune_close - @subpage_single tune_cursor_persist -- @subpage_single tune_read_only - @subpage_single tune_durability -- @subpage_single tune_checksum - @subpage_single tune_file_alloc +- @subpage_single tune_memory_allocator +- @subpage_single tune_mutex +- @subpage_single tune_page_size_and_comp +- @subpage_single tune_read_only - @subpage_single tune_system_buffer_cache - @subpage_single tune_transparent_huge_pages -- @subpage_single tune_close -- @subpage_single tune_mutex - @subpage_single tune_zone_reclaim */ diff --git a/src/third_party/wiredtiger/src/docs/spell.ok b/src/third_party/wiredtiger/src/docs/spell.ok index bc2e16b1122..5d629f4c49f 100644 --- a/src/third_party/wiredtiger/src/docs/spell.ok +++ b/src/third_party/wiredtiger/src/docs/spell.ok @@ -237,6 +237,7 @@ fput freelist fsync ftruncate +fvisibility gcc gdbm ge diff --git a/src/third_party/wiredtiger/src/docs/tune-build-options.dox b/src/third_party/wiredtiger/src/docs/tune-build-options.dox new file mode 100644 index 00000000000..79cd60b1105 --- /dev/null +++ b/src/third_party/wiredtiger/src/docs/tune-build-options.dox @@ -0,0 +1,9 @@ +/*! @page tune_build_options gcc/clang build options + +WiredTiger can be built using the gcc/clang \c -fvisibility=hidden flag, +which may significantly reduce the size and load time of the WiredTiger +library when built as a dynamic shared object, and allow the optimizer +to produce better code (for example, by eliminating most lookups in the +procedure linkage table). + + */ diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 26bbf9f679b..cc3c5a5c824 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -941,6 +941,13 @@ __evict_tune_workers(WT_SESSION_IMPL *session) conn = S2C(session); cache = conn->cache; + /* + * If we have a fixed number of eviction threads, there is no value in + * calculating if we should do any tuning. + */ + if (conn->evict_threads_max == conn->evict_threads_min) + return (0); + WT_ASSERT(session, conn->evict_threads.threads[0]->session == session); pgs_evicted_cur = pgs_evicted_persec_cur = 0; diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index bf3279d0f94..dfd2d03707f 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -570,6 +570,7 @@ extern int __wt_schema_destroy_index(WT_SESSION_IMPL *session, WT_INDEX **idxp) extern int __wt_schema_destroy_table(WT_SESSION_IMPL *session, WT_TABLE **tablep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_remove_table(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_close_tables(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_sweep_tables(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_colgroup_name(WT_SESSION_IMPL *session, WT_TABLE *table, const char *cgname, size_t len, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_open_colgroups(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_open_index(WT_SESSION_IMPL *session, WT_TABLE *table, const char *idxname, size_t len, WT_INDEX **indexp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/gcc.h b/src/third_party/wiredtiger/src/include/gcc.h index 22d78fc165a..6c712813d29 100644 --- a/src/third_party/wiredtiger/src/include/gcc.h +++ b/src/third_party/wiredtiger/src/include/gcc.h @@ -9,7 +9,7 @@ #define WT_PTRDIFFT_FMT "td" /* ptrdiff_t format string */ #define WT_SIZET_FMT "zu" /* size_t format string */ -/* Add GCC-specific attributes to types and function declarations. */ +/* GCC-specific attributes. */ #define WT_PACKED_STRUCT_BEGIN(name) \ struct __attribute__ ((__packed__)) name { #define WT_PACKED_STRUCT_END \ diff --git a/src/third_party/wiredtiger/src/include/lint.h b/src/third_party/wiredtiger/src/include/lint.h index 2d0f47988b7..813b9182683 100644 --- a/src/third_party/wiredtiger/src/include/lint.h +++ b/src/third_party/wiredtiger/src/include/lint.h @@ -9,6 +9,7 @@ #define WT_PTRDIFFT_FMT "td" /* ptrdiff_t format string */ #define WT_SIZET_FMT "zu" /* size_t format string */ +/* Lint-specific attributes. */ #define WT_PACKED_STRUCT_BEGIN(name) \ struct name { #define WT_PACKED_STRUCT_END \ diff --git a/src/third_party/wiredtiger/src/include/misc.i b/src/third_party/wiredtiger/src/include/misc.i index 7040886cf82..fad10f01103 100644 --- a/src/third_party/wiredtiger/src/include/misc.i +++ b/src/third_party/wiredtiger/src/include/misc.i @@ -54,6 +54,31 @@ __wt_seconds(WT_SESSION_IMPL *session, time_t *timep) *timep = t.tv_sec; } +/* + * __wt_time_check_monotonic -- + * Check and prevent time running backward. If we detect that it has, we + * set the time structure to the previous values, making time stand still + * until we see a time in the future of the highest value seen so far. + */ +static inline void +__wt_time_check_monotonic(WT_SESSION_IMPL *session, struct timespec *tsp) +{ + /* + * Detect time going backward. If so, use the last + * saved timestamp. + */ + if (session == NULL) + return; + + if (tsp->tv_sec < session->last_epoch.tv_sec || + (tsp->tv_sec == session->last_epoch.tv_sec && + tsp->tv_nsec < session->last_epoch.tv_nsec)) { + WT_STAT_CONN_INCR(session, time_travel); + *tsp = session->last_epoch; + } else + session->last_epoch = *tsp; +} + /* * __wt_verbose -- * Verbose message. diff --git a/src/third_party/wiredtiger/src/include/msvc.h b/src/third_party/wiredtiger/src/include/msvc.h index 6c5c8b67647..c9399be3185 100644 --- a/src/third_party/wiredtiger/src/include/msvc.h +++ b/src/third_party/wiredtiger/src/include/msvc.h @@ -16,9 +16,7 @@ #define WT_PTRDIFFT_FMT "Id" /* ptrdiff_t format string */ #define WT_SIZET_FMT "Iu" /* size_t format string */ -/* - * Add MSVC-specific attributes and pragmas to types and function declarations. - */ +/* MSVC-specific attributes. */ #define WT_PACKED_STRUCT_BEGIN(name) \ __pragma(pack(push,1)) \ struct name { diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index 1b2dfd1ed2b..d05dee68641 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -66,6 +66,7 @@ struct __wt_session_impl { /* Session handle reference list */ TAILQ_HEAD(__dhandles, __wt_data_handle_cache) dhandles; time_t last_sweep; /* Last sweep for dead handles */ + struct timespec last_epoch; /* Last epoch time returned */ /* Cursors closed with the session */ TAILQ_HEAD(__cursors, __wt_cursor) cursors; @@ -97,6 +98,12 @@ struct __wt_session_impl { */ TAILQ_HEAD(__tables, __wt_table) tables; + /* + * Updated when the table cache is swept of all tables older than the + * current schema generation. + */ + uint64_t table_sweep_gen; + /* Current rwlock for callback. */ WT_RWLOCK *current_rwlock; uint8_t current_rwticket; diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index 6c274484bcb..db48a841571 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -361,6 +361,7 @@ struct __wt_connection_stats { int64_t cache_eviction_clean; int64_t cond_auto_wait_reset; int64_t cond_auto_wait; + int64_t time_travel; int64_t file_open; int64_t memory_allocation; int64_t memory_free; diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index 7e802c188ab..fdf9c714afa 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -93,6 +93,8 @@ struct __wt_txn_global { * the global transaction state. */ WT_RWLOCK scan_rwlock; + /* Protects logging, checkpoints and transaction visibility. */ + WT_RWLOCK visibility_rwlock; /* * Track information about the running checkpoint. The transaction diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index ddecb2ac765..821efdf5fa1 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -39,6 +39,16 @@ extern "C" { #define __F(func) (*(func)) #endif +/* + * We support configuring WiredTiger with the gcc/clang -fvisibility=hidden + * flags, but that requires public APIs be specifically marked. + */ +#if defined(DOXYGEN) || defined(SWIG) || !defined(__GNUC__) +#define WT_ATTRIBUTE_LIBRARY_VISIBLE +#else +#define WT_ATTRIBUTE_LIBRARY_VISIBLE __attribute__((visibility("default"))) +#endif + #ifdef SWIG %{ #include @@ -2553,7 +2563,7 @@ struct __wt_connection { */ int wiredtiger_open(const char *home, WT_EVENT_HANDLER *errhandler, const char *config, - WT_CONNECTION **connectionp); + WT_CONNECTION **connectionp) WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Return information about a WiredTiger error as a string (see @@ -2564,7 +2574,7 @@ int wiredtiger_open(const char *home, * @param error a return value from a WiredTiger, ISO C, or POSIX standard API * @returns a string representation of the error */ -const char *wiredtiger_strerror(int error); +const char *wiredtiger_strerror(int error) WT_ATTRIBUTE_LIBRARY_VISIBLE; #if !defined(SWIG) /*! @@ -2701,7 +2711,8 @@ struct __wt_event_handler { * @errors */ int wiredtiger_struct_pack(WT_SESSION *session, - void *buffer, size_t size, const char *format, ...); + void *buffer, size_t size, const char *format, ...) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Calculate the size required to pack a structure. @@ -2719,7 +2730,7 @@ int wiredtiger_struct_pack(WT_SESSION *session, * @errors */ int wiredtiger_struct_size(WT_SESSION *session, - size_t *sizep, const char *format, ...); + size_t *sizep, const char *format, ...) WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Unpack a structure from a buffer. @@ -2736,7 +2747,8 @@ int wiredtiger_struct_size(WT_SESSION *session, * @errors */ int wiredtiger_struct_unpack(WT_SESSION *session, - const void *buffer, size_t size, const char *format, ...); + const void *buffer, size_t size, const char *format, ...) + WT_ATTRIBUTE_LIBRARY_VISIBLE; #if !defined(SWIG) @@ -2763,7 +2775,8 @@ typedef struct __wt_pack_stream WT_PACK_STREAM; * @errors */ int wiredtiger_pack_start(WT_SESSION *session, - const char *format, void *buffer, size_t size, WT_PACK_STREAM **psp); + const char *format, void *buffer, size_t size, WT_PACK_STREAM **psp) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Start an unpacking operation from a buffer with the given format string. @@ -2779,7 +2792,8 @@ int wiredtiger_pack_start(WT_SESSION *session, * @errors */ int wiredtiger_unpack_start(WT_SESSION *session, - const char *format, const void *buffer, size_t size, WT_PACK_STREAM **psp); + const char *format, const void *buffer, size_t size, WT_PACK_STREAM **psp) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Close a packing stream. @@ -2788,7 +2802,8 @@ int wiredtiger_unpack_start(WT_SESSION *session, * @param[out] usedp the number of bytes in the buffer used by the stream * @errors */ -int wiredtiger_pack_close(WT_PACK_STREAM *ps, size_t *usedp); +int wiredtiger_pack_close(WT_PACK_STREAM *ps, size_t *usedp) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Pack an item into a packing stream. @@ -2797,7 +2812,8 @@ int wiredtiger_pack_close(WT_PACK_STREAM *ps, size_t *usedp); * @param item an item to pack * @errors */ -int wiredtiger_pack_item(WT_PACK_STREAM *ps, WT_ITEM *item); +int wiredtiger_pack_item(WT_PACK_STREAM *ps, WT_ITEM *item) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Pack a signed integer into a packing stream. @@ -2806,7 +2822,8 @@ int wiredtiger_pack_item(WT_PACK_STREAM *ps, WT_ITEM *item); * @param i a signed integer to pack * @errors */ -int wiredtiger_pack_int(WT_PACK_STREAM *ps, int64_t i); +int wiredtiger_pack_int(WT_PACK_STREAM *ps, int64_t i) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Pack a string into a packing stream. @@ -2815,7 +2832,8 @@ int wiredtiger_pack_int(WT_PACK_STREAM *ps, int64_t i); * @param s a string to pack * @errors */ -int wiredtiger_pack_str(WT_PACK_STREAM *ps, const char *s); +int wiredtiger_pack_str(WT_PACK_STREAM *ps, const char *s) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Pack an unsigned integer into a packing stream. @@ -2824,7 +2842,8 @@ int wiredtiger_pack_str(WT_PACK_STREAM *ps, const char *s); * @param u an unsigned integer to pack * @errors */ -int wiredtiger_pack_uint(WT_PACK_STREAM *ps, uint64_t u); +int wiredtiger_pack_uint(WT_PACK_STREAM *ps, uint64_t u) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Unpack an item from a packing stream. @@ -2833,7 +2852,8 @@ int wiredtiger_pack_uint(WT_PACK_STREAM *ps, uint64_t u); * @param item an item to unpack * @errors */ -int wiredtiger_unpack_item(WT_PACK_STREAM *ps, WT_ITEM *item); +int wiredtiger_unpack_item(WT_PACK_STREAM *ps, WT_ITEM *item) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Unpack a signed integer from a packing stream. @@ -2842,7 +2862,8 @@ int wiredtiger_unpack_item(WT_PACK_STREAM *ps, WT_ITEM *item); * @param[out] ip the unpacked signed integer * @errors */ -int wiredtiger_unpack_int(WT_PACK_STREAM *ps, int64_t *ip); +int wiredtiger_unpack_int(WT_PACK_STREAM *ps, int64_t *ip) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Unpack a string from a packing stream. @@ -2851,7 +2872,8 @@ int wiredtiger_unpack_int(WT_PACK_STREAM *ps, int64_t *ip); * @param[out] sp the unpacked string * @errors */ -int wiredtiger_unpack_str(WT_PACK_STREAM *ps, const char **sp); +int wiredtiger_unpack_str(WT_PACK_STREAM *ps, const char **sp) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * Unpack an unsigned integer from a packing stream. @@ -2860,7 +2882,8 @@ int wiredtiger_unpack_str(WT_PACK_STREAM *ps, const char **sp); * @param[out] up the unpacked unsigned integer * @errors */ -int wiredtiger_unpack_uint(WT_PACK_STREAM *ps, uint64_t *up); +int wiredtiger_unpack_uint(WT_PACK_STREAM *ps, uint64_t *up) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! @} */ /*! @@ -2938,7 +2961,8 @@ struct __wt_config_item { * @snippet ex_all.c Validate a configuration string */ int wiredtiger_config_validate(WT_SESSION *session, - WT_EVENT_HANDLER *errhandler, const char *name, const char *config); + WT_EVENT_HANDLER *errhandler, const char *name, const char *config) + WT_ATTRIBUTE_LIBRARY_VISIBLE; #endif /*! @@ -2958,7 +2982,8 @@ int wiredtiger_config_validate(WT_SESSION *session, * @snippet ex_config_parse.c Create a configuration parser */ int wiredtiger_config_parser_open(WT_SESSION *session, - const char *config, size_t len, WT_CONFIG_PARSER **config_parserp); + const char *config, size_t len, WT_CONFIG_PARSER **config_parserp) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /*! * A handle that can be used to search and traverse configuration strings @@ -3047,7 +3072,8 @@ struct __wt_config_parser { * @param patchp a location where the patch version number is returned * @returns a string representation of the version */ -const char *wiredtiger_version(int *majorp, int *minorp, int *patchp); +const char *wiredtiger_version(int *majorp, int *minorp, int *patchp) + WT_ATTRIBUTE_LIBRARY_VISIBLE; /******************************************* * Error returns @@ -4546,304 +4572,306 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_COND_AUTO_WAIT_RESET 1102 /*! connection: auto adjusting condition wait calls */ #define WT_STAT_CONN_COND_AUTO_WAIT 1103 +/*! connection: detected system time went backwards */ +#define WT_STAT_CONN_TIME_TRAVEL 1104 /*! connection: files currently open */ -#define WT_STAT_CONN_FILE_OPEN 1104 +#define WT_STAT_CONN_FILE_OPEN 1105 /*! connection: memory allocations */ -#define WT_STAT_CONN_MEMORY_ALLOCATION 1105 +#define WT_STAT_CONN_MEMORY_ALLOCATION 1106 /*! connection: memory frees */ -#define WT_STAT_CONN_MEMORY_FREE 1106 +#define WT_STAT_CONN_MEMORY_FREE 1107 /*! connection: memory re-allocations */ -#define WT_STAT_CONN_MEMORY_GROW 1107 +#define WT_STAT_CONN_MEMORY_GROW 1108 /*! connection: pthread mutex condition wait calls */ -#define WT_STAT_CONN_COND_WAIT 1108 +#define WT_STAT_CONN_COND_WAIT 1109 /*! connection: pthread mutex shared lock read-lock calls */ -#define WT_STAT_CONN_RWLOCK_READ 1109 +#define WT_STAT_CONN_RWLOCK_READ 1110 /*! connection: pthread mutex shared lock write-lock calls */ -#define WT_STAT_CONN_RWLOCK_WRITE 1110 +#define WT_STAT_CONN_RWLOCK_WRITE 1111 /*! connection: total fsync I/Os */ -#define WT_STAT_CONN_FSYNC_IO 1111 +#define WT_STAT_CONN_FSYNC_IO 1112 /*! connection: total read I/Os */ -#define WT_STAT_CONN_READ_IO 1112 +#define WT_STAT_CONN_READ_IO 1113 /*! connection: total write I/Os */ -#define WT_STAT_CONN_WRITE_IO 1113 +#define WT_STAT_CONN_WRITE_IO 1114 /*! cursor: cursor create calls */ -#define WT_STAT_CONN_CURSOR_CREATE 1114 +#define WT_STAT_CONN_CURSOR_CREATE 1115 /*! cursor: cursor insert calls */ -#define WT_STAT_CONN_CURSOR_INSERT 1115 +#define WT_STAT_CONN_CURSOR_INSERT 1116 /*! cursor: cursor next calls */ -#define WT_STAT_CONN_CURSOR_NEXT 1116 +#define WT_STAT_CONN_CURSOR_NEXT 1117 /*! cursor: cursor prev calls */ -#define WT_STAT_CONN_CURSOR_PREV 1117 +#define WT_STAT_CONN_CURSOR_PREV 1118 /*! cursor: cursor remove calls */ -#define WT_STAT_CONN_CURSOR_REMOVE 1118 +#define WT_STAT_CONN_CURSOR_REMOVE 1119 /*! cursor: cursor reset calls */ -#define WT_STAT_CONN_CURSOR_RESET 1119 +#define WT_STAT_CONN_CURSOR_RESET 1120 /*! cursor: cursor restarted searches */ -#define WT_STAT_CONN_CURSOR_RESTART 1120 +#define WT_STAT_CONN_CURSOR_RESTART 1121 /*! cursor: cursor search calls */ -#define WT_STAT_CONN_CURSOR_SEARCH 1121 +#define WT_STAT_CONN_CURSOR_SEARCH 1122 /*! cursor: cursor search near calls */ -#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1122 +#define WT_STAT_CONN_CURSOR_SEARCH_NEAR 1123 /*! cursor: cursor update calls */ -#define WT_STAT_CONN_CURSOR_UPDATE 1123 +#define WT_STAT_CONN_CURSOR_UPDATE 1124 /*! cursor: truncate calls */ -#define WT_STAT_CONN_CURSOR_TRUNCATE 1124 +#define WT_STAT_CONN_CURSOR_TRUNCATE 1125 /*! data-handle: connection data handles currently active */ -#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1125 +#define WT_STAT_CONN_DH_CONN_HANDLE_COUNT 1126 /*! data-handle: connection sweep candidate became referenced */ -#define WT_STAT_CONN_DH_SWEEP_REF 1126 +#define WT_STAT_CONN_DH_SWEEP_REF 1127 /*! data-handle: connection sweep dhandles closed */ -#define WT_STAT_CONN_DH_SWEEP_CLOSE 1127 +#define WT_STAT_CONN_DH_SWEEP_CLOSE 1128 /*! data-handle: connection sweep dhandles removed from hash list */ -#define WT_STAT_CONN_DH_SWEEP_REMOVE 1128 +#define WT_STAT_CONN_DH_SWEEP_REMOVE 1129 /*! data-handle: connection sweep time-of-death sets */ -#define WT_STAT_CONN_DH_SWEEP_TOD 1129 +#define WT_STAT_CONN_DH_SWEEP_TOD 1130 /*! data-handle: connection sweeps */ -#define WT_STAT_CONN_DH_SWEEPS 1130 +#define WT_STAT_CONN_DH_SWEEPS 1131 /*! data-handle: session dhandles swept */ -#define WT_STAT_CONN_DH_SESSION_HANDLES 1131 +#define WT_STAT_CONN_DH_SESSION_HANDLES 1132 /*! data-handle: session sweep attempts */ -#define WT_STAT_CONN_DH_SESSION_SWEEPS 1132 +#define WT_STAT_CONN_DH_SESSION_SWEEPS 1133 /*! lock: checkpoint lock acquisitions */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1133 +#define WT_STAT_CONN_LOCK_CHECKPOINT_COUNT 1134 /*! lock: checkpoint lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1134 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_APPLICATION 1135 /*! lock: checkpoint lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1135 +#define WT_STAT_CONN_LOCK_CHECKPOINT_WAIT_INTERNAL 1136 /*! lock: handle-list lock eviction thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_HANDLE_LIST_WAIT_EVICTION 1136 +#define WT_STAT_CONN_LOCK_HANDLE_LIST_WAIT_EVICTION 1137 /*! lock: metadata lock acquisitions */ -#define WT_STAT_CONN_LOCK_METADATA_COUNT 1137 +#define WT_STAT_CONN_LOCK_METADATA_COUNT 1138 /*! lock: metadata lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1138 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_APPLICATION 1139 /*! lock: metadata lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1139 +#define WT_STAT_CONN_LOCK_METADATA_WAIT_INTERNAL 1140 /*! lock: schema lock acquisitions */ -#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1140 +#define WT_STAT_CONN_LOCK_SCHEMA_COUNT 1141 /*! lock: schema lock application thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1141 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_APPLICATION 1142 /*! lock: schema lock internal thread wait time (usecs) */ -#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1142 +#define WT_STAT_CONN_LOCK_SCHEMA_WAIT_INTERNAL 1143 /*! lock: table lock acquisitions */ -#define WT_STAT_CONN_LOCK_TABLE_COUNT 1143 +#define WT_STAT_CONN_LOCK_TABLE_COUNT 1144 /*! * lock: table lock application thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1144 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_APPLICATION 1145 /*! * lock: table lock internal thread time waiting for the table lock * (usecs) */ -#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1145 +#define WT_STAT_CONN_LOCK_TABLE_WAIT_INTERNAL 1146 /*! log: busy returns attempting to switch slots */ -#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1146 +#define WT_STAT_CONN_LOG_SLOT_SWITCH_BUSY 1147 /*! log: consolidated slot closures */ -#define WT_STAT_CONN_LOG_SLOT_CLOSES 1147 +#define WT_STAT_CONN_LOG_SLOT_CLOSES 1148 /*! log: consolidated slot join active slot closed */ -#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1148 +#define WT_STAT_CONN_LOG_SLOT_ACTIVE_CLOSED 1149 /*! log: consolidated slot join races */ -#define WT_STAT_CONN_LOG_SLOT_RACES 1149 +#define WT_STAT_CONN_LOG_SLOT_RACES 1150 /*! log: consolidated slot join transitions */ -#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1150 +#define WT_STAT_CONN_LOG_SLOT_TRANSITIONS 1151 /*! log: consolidated slot joins */ -#define WT_STAT_CONN_LOG_SLOT_JOINS 1151 +#define WT_STAT_CONN_LOG_SLOT_JOINS 1152 /*! log: consolidated slot transitions unable to find free slot */ -#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1152 +#define WT_STAT_CONN_LOG_SLOT_NO_FREE_SLOTS 1153 /*! log: consolidated slot unbuffered writes */ -#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1153 +#define WT_STAT_CONN_LOG_SLOT_UNBUFFERED 1154 /*! log: log bytes of payload data */ -#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1154 +#define WT_STAT_CONN_LOG_BYTES_PAYLOAD 1155 /*! log: log bytes written */ -#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1155 +#define WT_STAT_CONN_LOG_BYTES_WRITTEN 1156 /*! log: log files manually zero-filled */ -#define WT_STAT_CONN_LOG_ZERO_FILLS 1156 +#define WT_STAT_CONN_LOG_ZERO_FILLS 1157 /*! log: log flush operations */ -#define WT_STAT_CONN_LOG_FLUSH 1157 +#define WT_STAT_CONN_LOG_FLUSH 1158 /*! log: log force write operations */ -#define WT_STAT_CONN_LOG_FORCE_WRITE 1158 +#define WT_STAT_CONN_LOG_FORCE_WRITE 1159 /*! log: log force write operations skipped */ -#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1159 +#define WT_STAT_CONN_LOG_FORCE_WRITE_SKIP 1160 /*! log: log records compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1160 +#define WT_STAT_CONN_LOG_COMPRESS_WRITES 1161 /*! log: log records not compressed */ -#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1161 +#define WT_STAT_CONN_LOG_COMPRESS_WRITE_FAILS 1162 /*! log: log records too small to compress */ -#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1162 +#define WT_STAT_CONN_LOG_COMPRESS_SMALL 1163 /*! log: log release advances write LSN */ -#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1163 +#define WT_STAT_CONN_LOG_RELEASE_WRITE_LSN 1164 /*! log: log scan operations */ -#define WT_STAT_CONN_LOG_SCANS 1164 +#define WT_STAT_CONN_LOG_SCANS 1165 /*! log: log scan records requiring two reads */ -#define WT_STAT_CONN_LOG_SCAN_REREADS 1165 +#define WT_STAT_CONN_LOG_SCAN_REREADS 1166 /*! log: log server thread advances write LSN */ -#define WT_STAT_CONN_LOG_WRITE_LSN 1166 +#define WT_STAT_CONN_LOG_WRITE_LSN 1167 /*! log: log server thread write LSN walk skipped */ -#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1167 +#define WT_STAT_CONN_LOG_WRITE_LSN_SKIP 1168 /*! log: log sync operations */ -#define WT_STAT_CONN_LOG_SYNC 1168 +#define WT_STAT_CONN_LOG_SYNC 1169 /*! log: log sync time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DURATION 1169 +#define WT_STAT_CONN_LOG_SYNC_DURATION 1170 /*! log: log sync_dir operations */ -#define WT_STAT_CONN_LOG_SYNC_DIR 1170 +#define WT_STAT_CONN_LOG_SYNC_DIR 1171 /*! log: log sync_dir time duration (usecs) */ -#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1171 +#define WT_STAT_CONN_LOG_SYNC_DIR_DURATION 1172 /*! log: log write operations */ -#define WT_STAT_CONN_LOG_WRITES 1172 +#define WT_STAT_CONN_LOG_WRITES 1173 /*! log: logging bytes consolidated */ -#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1173 +#define WT_STAT_CONN_LOG_SLOT_CONSOLIDATED 1174 /*! log: maximum log file size */ -#define WT_STAT_CONN_LOG_MAX_FILESIZE 1174 +#define WT_STAT_CONN_LOG_MAX_FILESIZE 1175 /*! log: number of pre-allocated log files to create */ -#define WT_STAT_CONN_LOG_PREALLOC_MAX 1175 +#define WT_STAT_CONN_LOG_PREALLOC_MAX 1176 /*! log: pre-allocated log files not ready and missed */ -#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1176 +#define WT_STAT_CONN_LOG_PREALLOC_MISSED 1177 /*! log: pre-allocated log files prepared */ -#define WT_STAT_CONN_LOG_PREALLOC_FILES 1177 +#define WT_STAT_CONN_LOG_PREALLOC_FILES 1178 /*! log: pre-allocated log files used */ -#define WT_STAT_CONN_LOG_PREALLOC_USED 1178 +#define WT_STAT_CONN_LOG_PREALLOC_USED 1179 /*! log: records processed by log scan */ -#define WT_STAT_CONN_LOG_SCAN_RECORDS 1179 +#define WT_STAT_CONN_LOG_SCAN_RECORDS 1180 /*! log: total in-memory size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_MEM 1180 +#define WT_STAT_CONN_LOG_COMPRESS_MEM 1181 /*! log: total log buffer size */ -#define WT_STAT_CONN_LOG_BUFFER_SIZE 1181 +#define WT_STAT_CONN_LOG_BUFFER_SIZE 1182 /*! log: total size of compressed records */ -#define WT_STAT_CONN_LOG_COMPRESS_LEN 1182 +#define WT_STAT_CONN_LOG_COMPRESS_LEN 1183 /*! log: written slots coalesced */ -#define WT_STAT_CONN_LOG_SLOT_COALESCED 1183 +#define WT_STAT_CONN_LOG_SLOT_COALESCED 1184 /*! log: yields waiting for previous log file close */ -#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1184 +#define WT_STAT_CONN_LOG_CLOSE_YIELDS 1185 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1185 +#define WT_STAT_CONN_REC_PAGE_DELETE_FAST 1186 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_CONN_REC_PAGES 1186 +#define WT_STAT_CONN_REC_PAGES 1187 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_CONN_REC_PAGES_EVICTION 1187 +#define WT_STAT_CONN_REC_PAGES_EVICTION 1188 /*! reconciliation: pages deleted */ -#define WT_STAT_CONN_REC_PAGE_DELETE 1188 +#define WT_STAT_CONN_REC_PAGE_DELETE 1189 /*! reconciliation: split bytes currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1189 +#define WT_STAT_CONN_REC_SPLIT_STASHED_BYTES 1190 /*! reconciliation: split objects currently awaiting free */ -#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1190 +#define WT_STAT_CONN_REC_SPLIT_STASHED_OBJECTS 1191 /*! session: open cursor count */ -#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1191 +#define WT_STAT_CONN_SESSION_CURSOR_OPEN 1192 /*! session: open session count */ -#define WT_STAT_CONN_SESSION_OPEN 1192 +#define WT_STAT_CONN_SESSION_OPEN 1193 /*! session: table alter failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1193 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_FAIL 1194 /*! session: table alter successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1194 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SUCCESS 1195 /*! session: table alter unchanged and skipped */ -#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1195 +#define WT_STAT_CONN_SESSION_TABLE_ALTER_SKIP 1196 /*! session: table compact failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1196 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_FAIL 1197 /*! session: table compact successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1197 +#define WT_STAT_CONN_SESSION_TABLE_COMPACT_SUCCESS 1198 /*! session: table create failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1198 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_FAIL 1199 /*! session: table create successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1199 +#define WT_STAT_CONN_SESSION_TABLE_CREATE_SUCCESS 1200 /*! session: table drop failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1200 +#define WT_STAT_CONN_SESSION_TABLE_DROP_FAIL 1201 /*! session: table drop successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1201 +#define WT_STAT_CONN_SESSION_TABLE_DROP_SUCCESS 1202 /*! session: table rebalance failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1202 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_FAIL 1203 /*! session: table rebalance successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1203 +#define WT_STAT_CONN_SESSION_TABLE_REBALANCE_SUCCESS 1204 /*! session: table rename failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1204 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_FAIL 1205 /*! session: table rename successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1205 +#define WT_STAT_CONN_SESSION_TABLE_RENAME_SUCCESS 1206 /*! session: table salvage failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1206 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_FAIL 1207 /*! session: table salvage successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1207 +#define WT_STAT_CONN_SESSION_TABLE_SALVAGE_SUCCESS 1208 /*! session: table truncate failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1208 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_FAIL 1209 /*! session: table truncate successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1209 +#define WT_STAT_CONN_SESSION_TABLE_TRUNCATE_SUCCESS 1210 /*! session: table verify failed calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1210 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_FAIL 1211 /*! session: table verify successful calls */ -#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1211 +#define WT_STAT_CONN_SESSION_TABLE_VERIFY_SUCCESS 1212 /*! thread-state: active filesystem fsync calls */ -#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1212 +#define WT_STAT_CONN_THREAD_FSYNC_ACTIVE 1213 /*! thread-state: active filesystem read calls */ -#define WT_STAT_CONN_THREAD_READ_ACTIVE 1213 +#define WT_STAT_CONN_THREAD_READ_ACTIVE 1214 /*! thread-state: active filesystem write calls */ -#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1214 +#define WT_STAT_CONN_THREAD_WRITE_ACTIVE 1215 /*! thread-yield: application thread time evicting (usecs) */ -#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1215 +#define WT_STAT_CONN_APPLICATION_EVICT_TIME 1216 /*! thread-yield: application thread time waiting for cache (usecs) */ -#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1216 +#define WT_STAT_CONN_APPLICATION_CACHE_TIME 1217 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1217 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1218 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1218 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1219 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1219 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1220 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1220 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1221 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1221 +#define WT_STAT_CONN_PAGE_SLEEP 1222 /*! transaction: number of named snapshots created */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1222 +#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1223 /*! transaction: number of named snapshots dropped */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1223 +#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1224 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1224 +#define WT_STAT_CONN_TXN_BEGIN 1225 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1225 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1226 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1226 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1227 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1227 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1228 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1228 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1229 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1229 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1230 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1230 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1231 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1231 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1232 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1232 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1233 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1233 +#define WT_STAT_CONN_TXN_CHECKPOINT 1234 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1234 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1235 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1235 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1236 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1236 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1237 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1237 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1238 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1238 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1239 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1239 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1240 /*! * transaction: transaction range of IDs currently pinned by named * snapshots */ -#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1240 +#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1241 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1241 +#define WT_STAT_CONN_TXN_SYNC 1242 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1242 +#define WT_STAT_CONN_TXN_COMMIT 1243 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1243 +#define WT_STAT_CONN_TXN_ROLLBACK 1244 /*! * @} diff --git a/src/third_party/wiredtiger/src/os_common/os_alloc.c b/src/third_party/wiredtiger/src/os_common/os_alloc.c index ef96ed09ea7..c54bcc718f2 100644 --- a/src/third_party/wiredtiger/src/os_common/os_alloc.c +++ b/src/third_party/wiredtiger/src/os_common/os_alloc.c @@ -266,6 +266,8 @@ __wt_strndup(WT_SESSION_IMPL *session, const void *str, size_t len, void *retp) WT_RET(__wt_malloc(session, len + 1, &p)); + WT_ASSERT(session, p != NULL); /* quiet clang scan-build */ + /* * Don't change this to strncpy, we rely on this function to duplicate * "strings" that contain nul bytes. diff --git a/src/third_party/wiredtiger/src/os_common/os_getopt.c b/src/third_party/wiredtiger/src/os_common/os_getopt.c index 960776c3999..fa21123ba0e 100644 --- a/src/third_party/wiredtiger/src/os_common/os_getopt.c +++ b/src/third_party/wiredtiger/src/os_common/os_getopt.c @@ -59,13 +59,17 @@ #include "wt_internal.h" -extern int __wt_opterr, __wt_optind, __wt_optopt, __wt_optreset; +extern int __wt_opterr WT_ATTRIBUTE_LIBRARY_VISIBLE; +extern int __wt_optind WT_ATTRIBUTE_LIBRARY_VISIBLE; +extern int __wt_optopt WT_ATTRIBUTE_LIBRARY_VISIBLE; +extern int __wt_optreset WT_ATTRIBUTE_LIBRARY_VISIBLE; + int __wt_opterr = 1, /* if error message should be printed */ __wt_optind = 1, /* index into parent argv vector */ __wt_optopt, /* character checked for validity */ __wt_optreset; /* reset getopt */ -extern char *__wt_optarg; +extern char *__wt_optarg WT_ATTRIBUTE_LIBRARY_VISIBLE; char *__wt_optarg; /* argument associated with option */ #define BADCH (int)'?' diff --git a/src/third_party/wiredtiger/src/os_posix/os_dir.c b/src/third_party/wiredtiger/src/os_posix/os_dir.c index 627278540d1..b1b6571e4ba 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_dir.c +++ b/src/third_party/wiredtiger/src/os_posix/os_dir.c @@ -37,7 +37,13 @@ __wt_posix_directory_list(WT_FILE_SYSTEM *file_system, dirallocsz = 0; entries = NULL; + /* + * If opendir fails, we should have a NULL pointer with an error value, + * but various static analysis programs remain unconvinced, check both. + */ WT_SYSCALL_RETRY(((dirp = opendir(directory)) == NULL ? -1 : 0), ret); + if (dirp == NULL && ret == 0) + ret = EINVAL; if (ret != 0) WT_RET_MSG(session, ret, "%s: directory-list: opendir", directory); diff --git a/src/third_party/wiredtiger/src/os_posix/os_time.c b/src/third_party/wiredtiger/src/os_posix/os_time.c index 6f150ee8ffe..fe337fea7cf 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_time.c +++ b/src/third_party/wiredtiger/src/os_posix/os_time.c @@ -16,6 +16,7 @@ void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { + struct timespec tmp; WT_DECL_RET; /* @@ -27,21 +28,34 @@ __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) tsp->tv_sec = 0; tsp->tv_nsec = 0; + /* + * Read into a local variable so that we're comparing the correct + * value when we check for monotonic increasing time. There are + * many places we read into an unlocked global variable. + */ #if defined(HAVE_CLOCK_GETTIME) - WT_SYSCALL_RETRY(clock_gettime(CLOCK_REALTIME, tsp), ret); - if (ret == 0) + WT_SYSCALL_RETRY(clock_gettime(CLOCK_REALTIME, &tmp), ret); + if (ret == 0) { + __wt_time_check_monotonic(session, &tmp); + tsp->tv_sec = tmp.tv_sec; + tsp->tv_nsec = tmp.tv_nsec; return; + } WT_PANIC_MSG(session, ret, "clock_gettime"); #elif defined(HAVE_GETTIMEOFDAY) + { struct timeval v; WT_SYSCALL_RETRY(gettimeofday(&v, NULL), ret); if (ret == 0) { - tsp->tv_sec = v.tv_sec; - tsp->tv_nsec = v.tv_usec * WT_THOUSAND; + tmp.tv_sec = v.tv_sec; + tmp.tv_nsec = v.tv_usec * WT_THOUSAND; + __wt_time_check_monotonic(session, &tmp); + *tsp = tmp; return; } WT_PANIC_MSG(session, ret, "gettimeofday"); + } #else NO TIME-OF-DAY IMPLEMENTATION: see src/os_posix/os_time.c #endif diff --git a/src/third_party/wiredtiger/src/os_win/os_time.c b/src/third_party/wiredtiger/src/os_win/os_time.c index 6aa5b3719f6..ba71341ab22 100644 --- a/src/third_party/wiredtiger/src/os_win/os_time.c +++ b/src/third_party/wiredtiger/src/os_win/os_time.c @@ -15,17 +15,18 @@ void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) { + struct timespec tmp; FILETIME time; uint64_t ns100; - WT_UNUSED(session); - GetSystemTimeAsFileTime(&time); ns100 = (((int64_t)time.dwHighDateTime << 32) + time.dwLowDateTime) - 116444736000000000LL; - tsp->tv_sec = ns100 / 10000000; - tsp->tv_nsec = (long)((ns100 % 10000000) * 100); + tmp.tv_sec = ns100 / 10000000; + tmp.tv_nsec = (long)((ns100 % 10000000) * 100); + __wt_time_check_monotonic(session, &tmp); + *tsp = tmp; } /* diff --git a/src/third_party/wiredtiger/src/schema/schema_list.c b/src/third_party/wiredtiger/src/schema/schema_list.c index 74ef5135a4a..3dc51b6cb43 100644 --- a/src/third_party/wiredtiger/src/schema/schema_list.c +++ b/src/third_party/wiredtiger/src/schema/schema_list.c @@ -249,3 +249,34 @@ __wt_schema_close_tables(WT_SESSION_IMPL *session) WT_TRET(__wt_schema_remove_table(session, table)); return (ret); } + +/* + * __wt_schema_sweep_tables -- + * Close all idle, obsolete tables in a session. + */ +int +__wt_schema_sweep_tables(WT_SESSION_IMPL *session) +{ + WT_TABLE *table, *next; + uint64_t schema_gen; + bool old_table_busy; + + WT_ORDERED_READ(schema_gen, S2C(session)->schema_gen); + if (schema_gen == session->table_sweep_gen) + return (0); + + old_table_busy = false; + TAILQ_FOREACH_SAFE(table, &session->tables, q, next) + if (table->schema_gen != schema_gen) { + if (table->refcnt == 0) + WT_RET(__wt_schema_remove_table( + session, table)); + else + old_table_busy = true; + } + + if (!old_table_busy) + session->table_sweep_gen = schema_gen; + + return (0); +} diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index b7daf0e2e02..5ce6135cfca 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -818,6 +818,8 @@ __session_reset(WT_SESSION *wt_session) WT_TRET(__wt_session_reset_cursors(session, true)); + WT_TRET(__wt_schema_sweep_tables(session)); + /* Release common session resources. */ WT_TRET(__wt_session_release_resources(session)); @@ -1105,7 +1107,6 @@ int __wt_session_range_truncate(WT_SESSION_IMPL *session, const char *uri, WT_CURSOR *start, WT_CURSOR *stop) { - WT_CURSOR *cursor; WT_DECL_RET; int cmp; bool local_start; @@ -1134,12 +1135,13 @@ __wt_session_range_truncate(WT_SESSION_IMPL *session, } /* - * Cursor truncate is only supported for some objects, check for the - * supporting methods we need, range_truncate and compare. + * Cursor truncate is only supported for some objects, check for a + * supporting compare method. */ - cursor = start == NULL ? stop : start; - if (cursor->compare == NULL) - WT_ERR(__wt_bad_object_type(session, cursor->uri)); + if (start != NULL && start->compare == NULL) + WT_ERR(__wt_bad_object_type(session, start->uri)); + if (stop != NULL && stop->compare == NULL) + WT_ERR(__wt_bad_object_type(session, stop->uri)); /* * If both cursors set, check they're correctly ordered with respect to @@ -1150,6 +1152,9 @@ __wt_session_range_truncate(WT_SESSION_IMPL *session, * reference the same object and the keys are set. */ if (start != NULL && stop != NULL) { + /* quiet clang scan-build */ + WT_ASSERT(session, start->compare != NULL); + WT_ERR(start->compare(start, stop, &cmp)); if (cmp > 0) WT_ERR_MSG(session, EINVAL, diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 2c2217f8c20..8b72e653658 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -728,6 +728,7 @@ static const char * const __stats_connection_desc[] = { "cache: unmodified pages evicted", "connection: auto adjusting condition resets", "connection: auto adjusting condition wait calls", + "connection: detected system time went backwards", "connection: files currently open", "connection: memory allocations", "connection: memory frees", @@ -1014,6 +1015,7 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) stats->cache_eviction_clean = 0; stats->cond_auto_wait_reset = 0; stats->cond_auto_wait = 0; + stats->time_travel = 0; /* not clearing file_open */ stats->memory_allocation = 0; stats->memory_free = 0; @@ -1320,6 +1322,7 @@ __wt_stat_connection_aggregate( to->cache_eviction_clean += WT_STAT_READ(from, cache_eviction_clean); to->cond_auto_wait_reset += WT_STAT_READ(from, cond_auto_wait_reset); to->cond_auto_wait += WT_STAT_READ(from, cond_auto_wait); + to->time_travel += WT_STAT_READ(from, time_travel); to->file_open += WT_STAT_READ(from, file_open); to->memory_allocation += WT_STAT_READ(from, memory_allocation); to->memory_free += WT_STAT_READ(from, memory_free); diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index ea7faa2e966..76fdf71e715 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -503,13 +503,17 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_TXN *txn; + WT_TXN_GLOBAL *txn_global; WT_TXN_OP *op; u_int i; - bool did_update; + bool did_update, locked; txn = &session->txn; conn = S2C(session); + txn_global = &conn->txn_global; did_update = txn->mod_count != 0; + locked = false; + WT_ASSERT(session, !F_ISSET(txn, WT_TXN_ERROR) || !did_update); if (!F_ISSET(txn, WT_TXN_RUNNING)) @@ -580,6 +584,14 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) * This is particularly important for checkpoints. */ __wt_txn_release_snapshot(session); + /* + * We hold the visibility lock for reading from the time + * we write our log record until the time we release our + * transaction so that the LSN any checkpoint gets will + * always reflect visible data. + */ + __wt_readlock(session, &txn_global->visibility_rwlock); + locked = true; ret = __wt_txn_log_commit(session, cfg); } @@ -590,8 +602,12 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) * Nothing can fail after this point. */ if (ret != 0) { + if (locked) + __wt_readunlock(session, + &txn_global->visibility_rwlock); WT_TRET(__wt_txn_rollback(session, cfg)); return (ret); + } /* Free memory associated with updates. */ @@ -600,6 +616,8 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) txn->mod_count = 0; __wt_txn_release(session); + if (locked) + __wt_readunlock(session, &txn_global->visibility_rwlock); return (0); } @@ -770,6 +788,7 @@ __wt_txn_global_init(WT_SESSION_IMPL *session, const char *cfg[]) &txn_global->id_lock, "transaction id lock")); WT_RET(__wt_rwlock_init(session, &txn_global->scan_rwlock)); WT_RET(__wt_rwlock_init(session, &txn_global->nsnap_rwlock)); + WT_RET(__wt_rwlock_init(session, &txn_global->visibility_rwlock)); txn_global->nsnap_oldest_id = WT_TXN_NONE; TAILQ_INIT(&txn_global->nsnaph); @@ -801,6 +820,7 @@ __wt_txn_global_destroy(WT_SESSION_IMPL *session) __wt_spin_destroy(session, &txn_global->id_lock); __wt_rwlock_destroy(session, &txn_global->scan_rwlock); __wt_rwlock_destroy(session, &txn_global->nsnap_rwlock); + __wt_rwlock_destroy(session, &txn_global->visibility_rwlock); __wt_free(session, txn_global->states); } diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c index 2931dc1ce82..cb3b3436786 100644 --- a/src/third_party/wiredtiger/src/txn/txn_log.c +++ b/src/third_party/wiredtiger/src/txn/txn_log.c @@ -294,11 +294,13 @@ __wt_txn_checkpoint_log( WT_ITEM *ckpt_snapshot, empty; WT_LSN *ckpt_lsn; WT_TXN *txn; + WT_TXN_GLOBAL *txn_global; uint8_t *end, *p; size_t recsize; uint32_t i, rectype = WT_LOGREC_CHECKPOINT; const char *fmt = WT_UNCHECKED_STRING(IIIIu); + txn_global = &S2C(session)->txn_global; txn = &session->txn; ckpt_lsn = &txn->ckpt_lsn; @@ -319,6 +321,15 @@ __wt_txn_checkpoint_log( case WT_TXN_LOG_CKPT_PREPARE: txn->full_ckpt = true; WT_ERR(__wt_log_flush_lsn(session, ckpt_lsn, true)); + /* + * We take and immediately release the visibility lock. + * Acquiring the write lock guarantees that any transaction + * that has written to the log has also made its transaction + * visible at this time. + */ + __wt_writelock(session, &txn_global->visibility_rwlock); + __wt_writeunlock(session, &txn_global->visibility_rwlock); + /* * We need to make sure that the log records in the checkpoint * LSN are on disk. In particular to make sure that the diff --git a/version.json b/version.json index ad5f47b99da..84ff3f9e40e 100644 --- a/version.json +++ b/version.json @@ -1,4 +1,4 @@ { - "githash": "056bf45128114e44c5358c7a8776fb582363e094", - "version": "3.2.16" + "githash": "186656d79574f7dfe0831a7e7821292ab380f667", + "version": "3.2.17" } \ No newline at end of file -- cgit v1.2.3 From c49e99631589113663b1a3ac691870421965a315 Mon Sep 17 00:00:00 2001 From: Apollon Oikonomopoulos Date: Thu, 22 Mar 2018 12:04:55 +0200 Subject: New upstream version 3.4.14 --- README | 56 +- buildscripts/burn_in_tests.py | 5 +- buildscripts/hang_analyzer.py | 54 +- buildscripts/package_test/.kitchen.yml | 4 +- .../package_test/recipes/install_mongodb.rb | 10 + ...gregation_read_concern_majority_passthrough.yml | 39 +- .../suites/read_concern_majority_passthrough.yml | 99 +- .../replica_sets_initsync_jscore_passthrough.yml | 1 + .../replica_sets_resync_jscore_passthrough.yml | 1 + .../suites/sharding_continuous_config_stepdown.yml | 1 + ...harding_last_stable_mongos_and_mixed_shards.yml | 2 + .../suites/write_concern_majority_passthrough.yml | 78 + buildscripts/resmokelib/core/programs.py | 17 +- buildscripts/resmokelib/logging/buildlogger.py | 152 +- buildscripts/resmokelib/logging/config.py | 8 +- .../resmokelib/testing/fixtures/interface.py | 13 +- .../resmokelib/testing/fixtures/masterslave.py | 9 +- .../resmokelib/testing/fixtures/replicaset.py | 32 +- .../resmokelib/testing/fixtures/shardedcluster.py | 33 +- .../resmokelib/testing/fixtures/standalone.py | 10 +- buildscripts/resmokelib/testing/testcases.py | 27 +- buildscripts/scons.py | 28 +- .../tests/resmokelib/logging/test_buildlogger.py | 67 + distsrc/README | 88 +- docs/building.md | 2 +- etc/drivers_nightly.yml | 14 +- etc/evergreen.yml | 400 +- etc/longevity.yml | 115 +- etc/perf.yml | 17 +- etc/system_perf.yml | 403 +- jstests/aggregation/extras/utils.js | 2 +- jstests/auth/mongos_cache_invalidation.js | 11 +- jstests/auth/scram-credentials-invalid.js | 45 + jstests/auth/system_authorization_indexes.js | 66 + jstests/concurrency/fsm_libs/cluster.js | 19 + jstests/concurrency/fsm_workloads/compact.js | 17 +- .../fsm_workloads/remove_multiple_documents.js | 11 +- jstests/concurrency/fsm_workloads/remove_where.js | 9 + jstests/core/apply_ops1.js | 49 +- jstests/core/apply_ops_atomicity.js | 10 +- jstests/core/autocomplete.js | 42 + jstests/core/batch_write_command_delete.js | 2 + jstests/core/batch_write_command_insert.js | 2 + jstests/core/batch_write_command_update.js | 2 + jstests/core/bypass_doc_validation.js | 2 + jstests/core/capped6.js | 7 + jstests/core/collation_plan_cache.js | 9 +- jstests/core/collection_info_cache_race.js | 4 +- jstests/core/collmod.js | 2 + jstests/core/collmod_bad_spec.js | 2 + .../commands_that_do_not_write_do_not_accept_wc.js | 2 + jstests/core/constructors.js | 2 + jstests/core/count10.js | 7 + jstests/core/count_plan_summary.js | 10 +- jstests/core/crud_api.js | 2 + jstests/core/diagdata.js | 26 +- jstests/core/doc_validation.js | 2 + jstests/core/doc_validation_invalid_validators.js | 2 + jstests/core/doc_validation_options.js | 1 + jstests/core/dropdb_race.js | 2 + jstests/core/elemMatchProjection.js | 50 +- jstests/core/error2.js | 1 + jstests/core/eval0.js | 2 + jstests/core/eval1.js | 1 + jstests/core/eval3.js | 1 + jstests/core/eval4.js | 1 + jstests/core/eval5.js | 1 + jstests/core/eval6.js | 1 + jstests/core/eval7.js | 2 + jstests/core/eval9.js | 2 + jstests/core/eval_mr.js | 2 + jstests/core/eval_nolock.js | 1 + jstests/core/evala.js | 1 + jstests/core/evalb.js | 2 + jstests/core/evald.js | 2 + jstests/core/evale.js | 2 + jstests/core/evalg.js | 2 + jstests/core/evalh.js | 2 + jstests/core/evalj.js | 2 + jstests/core/existsa.js | 212 +- jstests/core/fsync.js | 2 + jstests/core/fts_dotted_prefix_fields.js | 15 + jstests/core/fts_trailing_fields.js | 22 + jstests/core/function_string_representations.js | 38 + jstests/core/geo_2d_trailing_fields.js | 45 + jstests/core/geo_s2cursorlimitskip.js | 7 + jstests/core/geo_update_btree.js | 2 + jstests/core/getlog2.js | 7 + jstests/core/index_elemmatch2.js | 63 + jstests/core/index_filter_commands.js | 20 +- jstests/core/index_stats.js | 7 + jstests/core/js3.js | 1 + jstests/core/js7.js | 2 + jstests/core/js9.js | 2 + jstests/core/js_jit.js | 40 + jstests/core/list_collections1.js | 8 +- jstests/core/list_collections_filter.js | 4 +- jstests/core/list_indexes.js | 8 +- jstests/core/list_indexes_invalidation.js | 2 +- jstests/core/list_namespaces_invalidation.js | 2 + jstests/core/max_time_ms.js | 8 + jstests/core/mr4.js | 6 +- jstests/core/mr_killop.js | 3 +- jstests/core/mr_optim.js | 15 +- jstests/core/no_db_created.js | 4 +- jstests/core/notablescan.js | 7 + jstests/core/operation_latency_histogram.js | 6 + jstests/core/plan_cache_clear.js | 7 + jstests/core/plan_cache_list_plans.js | 7 + jstests/core/plan_cache_list_shapes.js | 7 + jstests/core/plan_cache_shell_helpers.js | 7 + jstests/core/profile2.js | 2 +- jstests/core/profile_getmore.js | 13 +- jstests/core/profile_insert.js | 2 + jstests/core/recursion.js | 6 +- jstests/core/regex_not_id.js | 6 +- jstests/core/remove8.js | 1 + jstests/core/rename4.js | 2 + jstests/core/rename_stayTemp.js | 2 +- jstests/core/shell_connection_strings.js | 33 + jstests/core/shell_writeconcern.js | 3 + jstests/core/stages_delete.js | 6 + jstests/core/startup_log.js | 209 +- jstests/core/storefunc.js | 2 + jstests/core/top.js | 6 + jstests/core/update_affects_indexes.js | 100 + jstests/core/update_multi5.js | 4 +- jstests/core/views/invalid_system_views.js | 2 + jstests/core/views/views_all_commands.js | 2 + jstests/core/views/views_basic.js | 2 +- jstests/core/views/views_change.js | 3 +- jstests/core/views/views_collation.js | 117 + jstests/core/views/views_find.js | 2 +- jstests/core/views/views_rename.js | 17 + jstests/core/views/views_stats.js | 6 + jstests/core/views/views_validation.js | 1 + jstests/core/write_result.js | 2 + jstests/hooks/validate_collections.js | 15 + jstests/libs/analyze_plan.js | 71 + jstests/libs/ftdc.js | 102 + jstests/libs/override_methods/override_helpers.js | 117 + .../set_majority_read_and_write_concerns.js | 178 - .../set_read_and_write_concerns.js | 216 + .../set_read_preference_secondary.js | 162 + .../initial_sync_last_stable_from_latest.js | 15 + .../initial_sync_latest_from_last_stable.js | 14 + jstests/multiVersion/initialsync.js | 58 - jstests/multiVersion/libs/initial_sync.js | 50 + jstests/multiVersion/libs/multi_rs.js | 1 + .../set_feature_compatibility_version.js | 34 + jstests/noPassthrough/ftdc_setdirectory.js | 118 + jstests/noPassthrough/geo_near_fcv32.js | 85 + jstests/noPassthrough/libs/backup_restore.js | 20 +- .../noPassthrough/non_atomic_apply_ops_logging.js | 62 + jstests/noPassthrough/partial_unique_indexes.js | 47 + .../skip_sharding_configuration_checks.js | 53 + .../apply_ops_index_collation.js | 80 + jstests/noPassthroughWithMongod/ftdc_params.js | 54 +- .../host_connection_string_validation.js | 4 +- .../initial_sync_oplog_rollover.js | 2 +- .../replset_host_connection_validation.js | 64 +- jstests/replsets/apply_batches_totalMillis.js | 63 + ...apply_ops_concurrent_non_atomic_different_db.js | 11 + ...ly_ops_concurrent_non_atomic_same_collection.js | 11 + .../apply_ops_concurrent_non_atomic_same_db.js | 11 + jstests/replsets/clean_shutdown_oplog_state.js | 2 +- jstests/replsets/index_delete.js | 2 +- .../initial_sync_rename_collection_unsafe.js | 61 + jstests/replsets/last_vote.js | 6 - .../libs/apply_ops_concurrent_non_atomic.js | 232 + .../libs/apply_ops_insert_write_conflict.js | 10 +- .../no_flapping_during_network_partition.js | 2 +- .../replsets/noop_writes_wait_for_write_concern.js | 235 + .../read_committed_with_catalog_changes.js | 2 +- jstests/replsets/startParallelShell.js | 33 + jstests/replsets/too_stale_secondary.js | 2 +- jstests/sharding/auth.js | 3 +- jstests/sharding/auto_rebalance_parallel.js | 68 +- jstests/sharding/autosplit.js | 7 +- jstests/sharding/mapReduce_inSharded_outSharded.js | 128 +- jstests/sharding/migrateBig_balancer.js | 8 +- .../sharding/movechunk_commit_changelog_stats.js | 41 + jstests/sharding/printShardingStatus.js | 11 +- .../sharding/shard_existing_coll_chunk_count.js | 165 + jstests/sharding/shard_identity_rollback.js | 5 +- jstests/sharding/write_cmd_auto_split.js | 6 +- rpm/init.d-mongod | 1 + rpm/init.d-mongod.suse | 1 + rpm/mongod.service | 3 + site_scons/site_tools/jstoh.py | 8 +- src/mongo/SConscript | 18 +- src/mongo/base/error_codes.err | 6 + src/mongo/base/parse_number_test.cpp | 6 +- src/mongo/base/validate_locale.cpp | 13 +- src/mongo/bson/bson_comparator_interface_base.h | 11 + src/mongo/bson/bsonelement_comparator_interface.h | 13 + src/mongo/bson/bsonobj.cpp | 1 + src/mongo/bson/bsonobj.h | 4 - src/mongo/bson/util/builder.h | 11 +- src/mongo/client/connection_string.h | 17 + src/mongo/client/fetcher.cpp | 18 +- src/mongo/client/fetcher.h | 11 +- src/mongo/client/fetcher_test.cpp | 4 +- src/mongo/client/mongo_uri.cpp | 340 +- src/mongo/client/mongo_uri.h | 41 + src/mongo/client/mongo_uri_test.cpp | 292 +- .../mongo-uri-host-identifiers.json | 203 + .../client/mongo_uri_tests/mongo-uri-invalid.json | 247 + .../client/mongo_uri_tests/mongo-uri-options.json | 25 + .../mongo-uri-unix-sockets-absolute.json | 373 + .../mongo-uri-unix-sockets-relative.json | 466 + .../mongo_uri_tests/mongo-uri-valid-auth.json | 311 + .../client/mongo_uri_tests/mongo-uri-warnings.json | 56 + src/mongo/crypto/mechanism_scram.cpp | 4 + src/mongo/crypto/mechanism_scram.h | 15 +- src/mongo/db/SConscript | 7 +- src/mongo/db/auth/SConscript | 1 + src/mongo/db/auth/auth_index_d.cpp | 147 +- .../db/auth/sasl_scramsha1_server_conversation.cpp | 7 + src/mongo/db/auth/user.h | 11 + src/mongo/db/auth/user_document_parser_test.cpp | 2 + src/mongo/db/bson/dotted_path_support.cpp | 2 +- src/mongo/db/bson/dotted_path_support.h | 3 +- src/mongo/db/catalog/apply_ops.cpp | 439 +- src/mongo/db/catalog/apply_ops.h | 2 +- src/mongo/db/catalog/database.cpp | 2 +- src/mongo/db/catalog/rename_collection.cpp | 10 +- src/mongo/db/commands/apply_ops_cmd.cpp | 23 +- src/mongo/db/commands/create_indexes.cpp | 8 - src/mongo/db/commands/dbcommands.cpp | 25 +- .../db/commands/feature_compatibility_version.cpp | 40 +- .../db/commands/feature_compatibility_version.h | 13 +- src/mongo/db/commands/find_and_modify.cpp | 19 - src/mongo/db/commands/mr.cpp | 6 +- src/mongo/db/commands/pipeline_command.cpp | 5 - src/mongo/db/concurrency/SConscript | 13 + src/mongo/db/concurrency/d_concurrency.cpp | 8 + src/mongo/db/concurrency/d_concurrency.h | 6 +- src/mongo/db/concurrency/d_concurrency_test.cpp | 321 +- .../global_lock_acquisition_tracker.cpp | 45 + .../concurrency/global_lock_acquisition_tracker.h | 62 + src/mongo/db/concurrency/lock_manager.cpp | 2 + src/mongo/db/concurrency/lock_state.cpp | 9 +- src/mongo/db/concurrency/locker.h | 14 + src/mongo/db/db.cpp | 14 +- src/mongo/db/dbhelpers.cpp | 34 +- src/mongo/db/dbhelpers.h | 5 +- src/mongo/db/dbwebserver.cpp | 1 + src/mongo/db/ftdc/SConscript | 31 +- src/mongo/db/ftdc/collector.cpp | 1 + src/mongo/db/ftdc/compressor_test.cpp | 2 +- src/mongo/db/ftdc/constants.h | 6 + src/mongo/db/ftdc/controller.cpp | 31 +- src/mongo/db/ftdc/controller.h | 15 +- src/mongo/db/ftdc/controller_test.cpp | 2 +- src/mongo/db/ftdc/ftdc_mongod.cpp | 323 +- src/mongo/db/ftdc/ftdc_mongod.h | 56 +- src/mongo/db/ftdc/ftdc_mongos.cpp | 153 + src/mongo/db/ftdc/ftdc_mongos.h | 43 + src/mongo/db/ftdc/ftdc_server.cpp | 350 + src/mongo/db/ftdc/ftdc_server.h | 103 + src/mongo/db/ftdc/ftdc_system_stats.h | 1 + src/mongo/db/ftdc/util.cpp | 13 + src/mongo/db/ftdc/util.h | 5 + src/mongo/db/ftdc/util_test.cpp | 21 + src/mongo/db/fts/unicode/string.cpp | 9 +- src/mongo/db/index/expression_keys_private.cpp | 5 +- src/mongo/db/instance.cpp | 9 +- src/mongo/db/log_process_details.cpp | 17 + src/mongo/db/matcher/expression_leaf.cpp | 40 +- src/mongo/db/matcher/expression_leaf.h | 22 +- src/mongo/db/matcher/expression_leaf_test.cpp | 66 +- src/mongo/db/matcher/expression_parser.cpp | 7 +- src/mongo/db/mongod_options.cpp | 23 +- src/mongo/db/mongod_options.h | 2 + src/mongo/db/namespace_string-inl.h | 2 +- src/mongo/db/namespace_string_test.cpp | 12 + src/mongo/db/op_observer_impl.cpp | 36 +- src/mongo/db/operation_context.cpp | 2 +- src/mongo/db/operation_context.h | 34 + src/mongo/db/operation_context_test.cpp | 2 +- src/mongo/db/ops/delete.cpp | 8 - src/mongo/db/ops/modifier_add_to_set.cpp | 8 + src/mongo/db/ops/modifier_add_to_set_test.cpp | 37 + src/mongo/db/ops/modifier_bit.cpp | 8 + src/mongo/db/ops/modifier_bit_test.cpp | 37 + src/mongo/db/ops/modifier_compare.cpp | 7 + src/mongo/db/ops/modifier_compare_test.cpp | 37 + src/mongo/db/ops/modifier_current_date.cpp | 10 + src/mongo/db/ops/modifier_current_date_test.cpp | 37 + src/mongo/db/ops/modifier_inc.cpp | 7 + src/mongo/db/ops/modifier_inc_test.cpp | 37 + src/mongo/db/ops/modifier_interface.h | 6 + src/mongo/db/ops/modifier_push.cpp | 9 + src/mongo/db/ops/modifier_push_test.cpp | 83 + src/mongo/db/ops/modifier_set.cpp | 6 + src/mongo/db/ops/modifier_set_test.cpp | 37 + src/mongo/db/ops/update.cpp | 17 - src/mongo/db/ops/update_driver.cpp | 20 +- src/mongo/db/pipeline/SConscript | 2 +- src/mongo/db/pipeline/document_source_geo_near.cpp | 10 +- .../db/pipeline/parsed_aggregation_projection.cpp | 46 +- .../db/pipeline/parsed_aggregation_projection.h | 37 +- .../parsed_aggregation_projection_test.cpp | 69 + src/mongo/db/query/SConscript | 44 +- src/mongo/db/query/canonical_query_test.cpp | 3 +- src/mongo/db/query/index_bounds_builder.cpp | 8 + src/mongo/db/query/index_bounds_builder.h | 11 + src/mongo/db/query/index_bounds_builder_test.cpp | 50 + src/mongo/db/query/plan_enumerator.cpp | 52 +- src/mongo/db/query/plan_enumerator.h | 11 + src/mongo/db/query/planner_access.cpp | 95 +- src/mongo/db/query/planner_ixselect.cpp | 53 +- src/mongo/db/query/planner_ixselect.h | 3 +- src/mongo/db/query/planner_ixselect_test.cpp | 79 + src/mongo/db/query/query_planner_geo_test.cpp | 62 + src/mongo/db/query/query_planner_text_test.cpp | 73 + src/mongo/db/query/query_solution.h | 7 + src/mongo/db/range_deleter.cpp | 20 +- src/mongo/db/range_deleter.h | 2 + src/mongo/db/range_deleter_db_env.cpp | 2 + src/mongo/db/range_deleter_db_env.h | 5 +- src/mongo/db/range_deleter_mock_env.cpp | 1 + src/mongo/db/range_deleter_mock_env.h | 1 + src/mongo/db/repl/SConscript | 2 +- src/mongo/db/repl/bgsync.cpp | 46 +- src/mongo/db/repl/bgsync.h | 2 +- src/mongo/db/repl/collection_cloner.cpp | 6 +- src/mongo/db/repl/database_cloner.cpp | 3 +- src/mongo/db/repl/databases_cloner.cpp | 71 +- src/mongo/db/repl/databases_cloner.h | 30 + src/mongo/db/repl/databases_cloner_test.cpp | 90 + src/mongo/db/repl/initial_syncer.cpp | 3 +- src/mongo/db/repl/master_slave.cpp | 41 +- src/mongo/db/repl/oplog.cpp | 131 +- src/mongo/db/repl/oplog.h | 9 + src/mongo/db/repl/oplog_fetcher.cpp | 33 +- src/mongo/db/repl/oplog_fetcher.h | 10 + src/mongo/db/repl/repl_client_info.cpp | 18 +- src/mongo/db/repl/repl_set_config.cpp | 11 +- ...replication_coordinator_external_state_impl.cpp | 7 +- src/mongo/db/repl/replication_coordinator_impl.cpp | 20 + .../repl/replication_coordinator_impl_elect_v1.cpp | 39 +- .../replication_coordinator_impl_elect_v1_test.cpp | 11 +- .../replication_coordinator_impl_heartbeat.cpp | 28 +- src/mongo/db/repl/replication_executor_test.cpp | 15 + src/mongo/db/repl/sync_source_feedback.cpp | 39 +- src/mongo/db/repl/sync_source_feedback.h | 9 +- src/mongo/db/repl/sync_source_resolver.cpp | 78 +- src/mongo/db/repl/sync_source_resolver.h | 15 +- src/mongo/db/repl/sync_tail.cpp | 58 +- src/mongo/db/repl/topology_coordinator_impl.cpp | 62 +- src/mongo/db/repl/topology_coordinator_impl.h | 6 +- .../db/repl/topology_coordinator_impl_test.cpp | 35 +- .../db/repl/topology_coordinator_impl_v1_test.cpp | 35 +- src/mongo/db/s/SConscript | 1 - src/mongo/db/s/balancer/balancer.cpp | 5 +- src/mongo/db/s/balancer/balancer.h | 3 +- .../balancer_chunk_selection_policy_impl.cpp | 13 +- .../balancer_chunk_selection_policy_impl.h | 3 +- src/mongo/db/s/balancer/balancer_policy.cpp | 78 +- src/mongo/db/s/balancer/balancer_policy.h | 64 +- src/mongo/db/s/balancer/balancer_policy_test.cpp | 146 +- src/mongo/db/s/collection_range_deleter.cpp | 222 - src/mongo/db/s/collection_range_deleter.h | 77 - src/mongo/db/s/collection_sharding_state.h | 2 - src/mongo/db/s/metadata_manager.cpp | 124 +- src/mongo/db/s/metadata_manager.h | 52 - src/mongo/db/s/metadata_manager_test.cpp | 192 +- src/mongo/db/s/migration_chunk_cloner_source.h | 6 +- .../db/s/migration_chunk_cloner_source_legacy.cpp | 23 +- .../db/s/migration_chunk_cloner_source_legacy.h | 2 +- src/mongo/db/s/migration_destination_manager.cpp | 40 +- src/mongo/db/s/migration_destination_manager.h | 5 +- ...gration_destination_manager_legacy_commands.cpp | 11 +- src/mongo/db/s/migration_source_manager.cpp | 22 +- src/mongo/db/s/migration_source_manager.h | 3 + src/mongo/db/s/sharding_state.cpp | 109 +- src/mongo/db/s/sharding_state.h | 29 - src/mongo/db/sorter/SConscript | 2 +- src/mongo/db/sorter/sorter.cpp | 33 +- src/mongo/db/stats/range_deleter_server_status.cpp | 5 + src/mongo/db/storage/SConscript | 15 +- src/mongo/db/storage/encryption_hooks.cpp | 96 + src/mongo/db/storage/encryption_hooks.h | 99 + .../ephemeral_for_test_record_store.cpp | 11 +- src/mongo/db/storage/key_string.cpp | 6 +- src/mongo/db/storage/key_string_test.cpp | 102 +- src/mongo/db/storage/mmap_v1/mmap_v1_engine.cpp | 4 + src/mongo/db/storage/wiredtiger/SConscript | 1 + .../wiredtiger/wiredtiger_customization_hooks.cpp | 34 +- .../wiredtiger/wiredtiger_customization_hooks.h | 75 +- .../db/storage/wiredtiger/wiredtiger_index.cpp | 1 + .../db/storage/wiredtiger/wiredtiger_kv_engine.cpp | 8 + .../storage/wiredtiger/wiredtiger_record_store.cpp | 29 +- .../wiredtiger/wiredtiger_recovery_unit.cpp | 15 +- .../storage/wiredtiger/wiredtiger_recovery_unit.h | 6 + .../wiredtiger/wiredtiger_server_status.cpp | 3 + .../wiredtiger/wiredtiger_session_cache.cpp | 56 +- .../storage/wiredtiger/wiredtiger_session_cache.h | 19 +- .../storage/wiredtiger/wiredtiger_size_storer.cpp | 2 +- .../wiredtiger/wiredtiger_snapshot_manager.cpp | 1 + .../db/storage/wiredtiger/wiredtiger_util.cpp | 8 +- src/mongo/db/views/resolved_view.cpp | 29 +- src/mongo/db/views/resolved_view.h | 22 +- src/mongo/db/views/resolved_view_test.cpp | 91 +- src/mongo/db/views/view_catalog.cpp | 9 +- src/mongo/db/views/view_catalog_test.cpp | 38 + src/mongo/db/views/view_sharding_check.cpp | 6 +- src/mongo/dbtests/dbhelper_tests.cpp | 8 +- src/mongo/dbtests/jstests.cpp | 44 + src/mongo/dbtests/rollbacktests.cpp | 12 +- src/mongo/executor/connection_pool.cpp | 247 +- src/mongo/executor/network_interface_asio_auth.cpp | 1 + src/mongo/executor/task_executor.h | 2 + src/mongo/executor/task_executor_test_common.cpp | 8 + src/mongo/rpc/get_status_from_command_result.cpp | 43 + src/mongo/rpc/get_status_from_command_result.h | 13 + src/mongo/s/catalog/type_tags.cpp | 27 +- src/mongo/s/catalog/type_tags_test.cpp | 5 +- src/mongo/s/client/shard_remote.cpp | 3 +- src/mongo/s/commands/SConscript | 1 + src/mongo/s/commands/cluster_ftdc_commands.cpp | 27 +- .../s/commands/cluster_shard_collection_cmd.cpp | 18 +- src/mongo/s/commands/cluster_write.cpp | 25 +- src/mongo/s/move_chunk_request.cpp | 3 +- src/mongo/s/server.cpp | 6 + src/mongo/s/sharding_initialization.cpp | 24 + src/mongo/scripting/deadline_monitor.cpp | 2 +- src/mongo/scripting/deadline_monitor.h | 6 +- src/mongo/scripting/deadline_monitor_test.cpp | 13 + src/mongo/scripting/engine.cpp | 16 + src/mongo/scripting/engine.h | 4 +- src/mongo/scripting/mozjs/PosixNSPR.cpp | 13 +- src/mongo/scripting/mozjs/bson.cpp | 10 +- src/mongo/scripting/mozjs/db.cpp | 53 +- src/mongo/scripting/mozjs/db.h | 5 +- src/mongo/scripting/mozjs/dbcollection.cpp | 10 +- src/mongo/scripting/mozjs/dbcollection.h | 5 +- src/mongo/scripting/mozjs/dbquery.cpp | 25 +- src/mongo/scripting/mozjs/dbquery.h | 6 +- src/mongo/scripting/mozjs/implscope.cpp | 195 +- src/mongo/scripting/mozjs/implscope.h | 8 + src/mongo/scripting/mozjs/mongohelpers.js | 8 + src/mongo/scripting/mozjs/proxyscope.cpp | 4 + src/mongo/scripting/mozjs/proxyscope.h | 2 + src/mongo/shell/bench.cpp | 14 +- src/mongo/shell/dbshell.cpp | 169 +- src/mongo/shell/mongo.js | 5 +- src/mongo/shell/replsettest.js | 16 +- src/mongo/shell/servers_misc.js | 20 +- src/mongo/shell/shardingtest.js | 6 +- src/mongo/shell/utils.js | 15 +- src/mongo/shell/utils_sh.js | 130 +- src/mongo/transport/SConscript | 4 +- src/mongo/transport/message_compressor_manager.cpp | 30 +- .../transport/message_compressor_manager_test.cpp | 89 +- src/mongo/transport/message_compressor_snappy.cpp | 17 +- src/mongo/util/SConscript | 11 + src/mongo/util/assert_util.cpp | 5 +- src/mongo/util/cmdline_utils/censor_cmdline.cpp | 2 + src/mongo/util/concurrency/with_lock.h | 111 + src/mongo/util/net/hostandport.cpp | 4 +- src/mongo/util/net/miniwebserver.cpp | 1 + src/mongo/util/net/ssl_manager.cpp | 64 +- src/mongo/util/options_parser/options_parser.cpp | 5 + .../util/options_parser/options_parser_test.cpp | 76 + src/mongo/util/password.cpp | 4 +- src/mongo/util/producer_consumer_queue.h | 559 + src/mongo/util/producer_consumer_queue_test.cpp | 531 + src/mongo/util/tcmalloc_server_status_section.cpp | 26 + src/third_party/SConscript | 2 +- .../gperftools-2.5/src/base/spinlock.cc | 77 + src/third_party/gperftools-2.5/src/base/spinlock.h | 6 + src/third_party/gperftools-2.5/src/page_heap.cc | 15 + src/third_party/gperftools-2.5/src/page_heap.h | 14 +- src/third_party/gperftools-2.5/src/tcmalloc.cc | 53 + src/third_party/pcre-8.39/132html | 313 - src/third_party/pcre-8.39/AUTHORS | 45 - src/third_party/pcre-8.39/COPYING | 5 - src/third_party/pcre-8.39/ChangeLog | 6010 ---------- src/third_party/pcre-8.39/CheckMan | 67 - src/third_party/pcre-8.39/CleanTxt | 113 - src/third_party/pcre-8.39/Detrail | 35 - src/third_party/pcre-8.39/HACKING | 528 - src/third_party/pcre-8.39/INSTALL | 370 - src/third_party/pcre-8.39/LICENCE | 93 - src/third_party/pcre-8.39/NEWS | 725 -- src/third_party/pcre-8.39/NON-AUTOTOOLS-BUILD | 772 -- src/third_party/pcre-8.39/NON-UNIX-USE | 7 - src/third_party/pcre-8.39/PrepareRelease | 258 - src/third_party/pcre-8.39/README | 1002 -- src/third_party/pcre-8.39/RunGrepTest | 593 - src/third_party/pcre-8.39/RunTest | 1014 -- src/third_party/pcre-8.39/RunTest.bat | 616 - src/third_party/pcre-8.39/SConscript | 68 - src/third_party/pcre-8.39/ar-lib | 270 - src/third_party/pcre-8.39/build_posix/config.h | 348 - src/third_party/pcre-8.39/build_solaris/config.h | 348 - src/third_party/pcre-8.39/build_windows/config.h | 57 - src/third_party/pcre-8.39/dftables.c | 212 - src/third_party/pcre-8.39/libpcre.pc.in | 13 - src/third_party/pcre-8.39/libpcre16.pc.in | 13 - src/third_party/pcre-8.39/libpcre32.pc.in | 13 - src/third_party/pcre-8.39/libpcrecpp.pc.in | 12 - src/third_party/pcre-8.39/libpcreposix.pc.in | 13 - src/third_party/pcre-8.39/makevp.bat | 66 - src/third_party/pcre-8.39/makevp_c.txt | 21 - src/third_party/pcre-8.39/makevp_l.txt | 21 - src/third_party/pcre-8.39/pcre-config.in | 133 - src/third_party/pcre-8.39/pcre.h | 677 -- src/third_party/pcre-8.39/pcre.h.generic | 677 -- src/third_party/pcre-8.39/pcre.h.in | 677 -- src/third_party/pcre-8.39/pcre16_byte_order.c | 45 - src/third_party/pcre-8.39/pcre16_chartables.c | 45 - src/third_party/pcre-8.39/pcre16_compile.c | 45 - src/third_party/pcre-8.39/pcre16_config.c | 45 - src/third_party/pcre-8.39/pcre16_dfa_exec.c | 45 - src/third_party/pcre-8.39/pcre16_exec.c | 45 - src/third_party/pcre-8.39/pcre16_fullinfo.c | 45 - src/third_party/pcre-8.39/pcre16_get.c | 45 - src/third_party/pcre-8.39/pcre16_globals.c | 45 - src/third_party/pcre-8.39/pcre16_jit_compile.c | 45 - src/third_party/pcre-8.39/pcre16_maketables.c | 45 - src/third_party/pcre-8.39/pcre16_newline.c | 45 - src/third_party/pcre-8.39/pcre16_ord2utf16.c | 90 - src/third_party/pcre-8.39/pcre16_printint.c | 45 - src/third_party/pcre-8.39/pcre16_refcount.c | 45 - src/third_party/pcre-8.39/pcre16_string_utils.c | 45 - src/third_party/pcre-8.39/pcre16_study.c | 45 - src/third_party/pcre-8.39/pcre16_tables.c | 45 - src/third_party/pcre-8.39/pcre16_ucd.c | 45 - src/third_party/pcre-8.39/pcre16_utf16_utils.c | 130 - src/third_party/pcre-8.39/pcre16_valid_utf16.c | 137 - src/third_party/pcre-8.39/pcre16_version.c | 45 - src/third_party/pcre-8.39/pcre16_xclass.c | 45 - src/third_party/pcre-8.39/pcre32_byte_order.c | 45 - src/third_party/pcre-8.39/pcre32_chartables.c | 45 - src/third_party/pcre-8.39/pcre32_compile.c | 45 - src/third_party/pcre-8.39/pcre32_config.c | 45 - src/third_party/pcre-8.39/pcre32_dfa_exec.c | 45 - src/third_party/pcre-8.39/pcre32_exec.c | 45 - src/third_party/pcre-8.39/pcre32_fullinfo.c | 45 - src/third_party/pcre-8.39/pcre32_get.c | 45 - src/third_party/pcre-8.39/pcre32_globals.c | 45 - src/third_party/pcre-8.39/pcre32_jit_compile.c | 45 - src/third_party/pcre-8.39/pcre32_maketables.c | 45 - src/third_party/pcre-8.39/pcre32_newline.c | 45 - src/third_party/pcre-8.39/pcre32_ord2utf32.c | 82 - src/third_party/pcre-8.39/pcre32_printint.c | 45 - src/third_party/pcre-8.39/pcre32_refcount.c | 45 - src/third_party/pcre-8.39/pcre32_string_utils.c | 45 - src/third_party/pcre-8.39/pcre32_study.c | 45 - src/third_party/pcre-8.39/pcre32_tables.c | 45 - src/third_party/pcre-8.39/pcre32_ucd.c | 45 - src/third_party/pcre-8.39/pcre32_utf32_utils.c | 141 - src/third_party/pcre-8.39/pcre32_valid_utf32.c | 124 - src/third_party/pcre-8.39/pcre32_version.c | 45 - src/third_party/pcre-8.39/pcre32_xclass.c | 45 - src/third_party/pcre-8.39/pcre_byte_order.c | 319 - src/third_party/pcre-8.39/pcre_chartables.c | 198 - src/third_party/pcre-8.39/pcre_chartables.c.dist | 198 - src/third_party/pcre-8.39/pcre_compile.c | 9749 --------------- src/third_party/pcre-8.39/pcre_config.c | 190 - src/third_party/pcre-8.39/pcre_dfa_exec.c | 3674 ------ src/third_party/pcre-8.39/pcre_exec.c | 7173 ----------- src/third_party/pcre-8.39/pcre_fullinfo.c | 245 - src/third_party/pcre-8.39/pcre_get.c | 669 -- src/third_party/pcre-8.39/pcre_globals.c | 86 - src/third_party/pcre-8.39/pcre_internal.h | 2798 ----- src/third_party/pcre-8.39/pcre_jit_compile.c | 11726 ------------------ src/third_party/pcre-8.39/pcre_jit_test.c | 1764 --- src/third_party/pcre-8.39/pcre_maketables.c | 156 - src/third_party/pcre-8.39/pcre_newline.c | 210 - src/third_party/pcre-8.39/pcre_ord2utf8.c | 94 - src/third_party/pcre-8.39/pcre_printint.c | 834 -- src/third_party/pcre-8.39/pcre_refcount.c | 92 - src/third_party/pcre-8.39/pcre_scanner.cc | 199 - src/third_party/pcre-8.39/pcre_scanner.h | 172 - src/third_party/pcre-8.39/pcre_scanner_unittest.cc | 161 - src/third_party/pcre-8.39/pcre_string_utils.c | 211 - src/third_party/pcre-8.39/pcre_stringpiece.cc | 43 - src/third_party/pcre-8.39/pcre_stringpiece.h | 180 - src/third_party/pcre-8.39/pcre_stringpiece.h.in | 180 - .../pcre-8.39/pcre_stringpiece_unittest.cc | 152 - src/third_party/pcre-8.39/pcre_study.c | 1686 --- src/third_party/pcre-8.39/pcre_tables.c | 727 -- src/third_party/pcre-8.39/pcre_ucd.c | 3630 ------ src/third_party/pcre-8.39/pcre_valid_utf8.c | 301 - src/third_party/pcre-8.39/pcre_version.c | 98 - src/third_party/pcre-8.39/pcre_xclass.c | 268 - src/third_party/pcre-8.39/pcrecpp.cc | 924 -- src/third_party/pcre-8.39/pcrecpp.h | 710 -- src/third_party/pcre-8.39/pcrecpp_internal.h | 71 - src/third_party/pcre-8.39/pcrecpp_unittest.cc | 1291 -- src/third_party/pcre-8.39/pcrecpparg.h | 174 - src/third_party/pcre-8.39/pcrecpparg.h.in | 174 - src/third_party/pcre-8.39/pcredemo.c | 406 - src/third_party/pcre-8.39/pcregexp.pas | 845 -- src/third_party/pcre-8.39/pcregrep.c | 3325 ------ src/third_party/pcre-8.39/pcreposix.c | 432 - src/third_party/pcre-8.39/pcreposix.h | 146 - src/third_party/pcre-8.39/pcretest.c | 5754 --------- src/third_party/pcre-8.39/perltest.pl | 242 - src/third_party/pcre-8.39/ucp.h | 224 - src/third_party/pcre-8.41/132html | 313 + src/third_party/pcre-8.41/AUTHORS | 45 + src/third_party/pcre-8.41/COPYING | 5 + src/third_party/pcre-8.41/ChangeLog | 6104 ++++++++++ src/third_party/pcre-8.41/CheckMan | 67 + src/third_party/pcre-8.41/CleanTxt | 113 + src/third_party/pcre-8.41/Detrail | 35 + src/third_party/pcre-8.41/HACKING | 528 + src/third_party/pcre-8.41/INSTALL | 370 + src/third_party/pcre-8.41/LICENCE | 93 + src/third_party/pcre-8.41/NEWS | 737 ++ src/third_party/pcre-8.41/NON-AUTOTOOLS-BUILD | 772 ++ src/third_party/pcre-8.41/NON-UNIX-USE | 7 + src/third_party/pcre-8.41/PrepareRelease | 258 + src/third_party/pcre-8.41/README | 1002 ++ src/third_party/pcre-8.41/RunGrepTest | 593 + src/third_party/pcre-8.41/RunTest | 1014 ++ src/third_party/pcre-8.41/RunTest.bat | 616 + src/third_party/pcre-8.41/SConscript | 68 + src/third_party/pcre-8.41/ar-lib | 270 + src/third_party/pcre-8.41/build_posix/config.h | 348 + src/third_party/pcre-8.41/build_solaris/config.h | 348 + src/third_party/pcre-8.41/build_windows/config.h | 57 + src/third_party/pcre-8.41/dftables.c | 212 + src/third_party/pcre-8.41/libpcre.pc.in | 13 + src/third_party/pcre-8.41/libpcre16.pc.in | 13 + src/third_party/pcre-8.41/libpcre32.pc.in | 13 + src/third_party/pcre-8.41/libpcrecpp.pc.in | 12 + src/third_party/pcre-8.41/libpcreposix.pc.in | 13 + src/third_party/pcre-8.41/makevp.bat | 66 + src/third_party/pcre-8.41/makevp_c.txt | 21 + src/third_party/pcre-8.41/makevp_l.txt | 21 + src/third_party/pcre-8.41/pcre-config.in | 133 + src/third_party/pcre-8.41/pcre.h | 677 ++ src/third_party/pcre-8.41/pcre.h.generic | 677 ++ src/third_party/pcre-8.41/pcre.h.in | 677 ++ src/third_party/pcre-8.41/pcre16_byte_order.c | 45 + src/third_party/pcre-8.41/pcre16_chartables.c | 45 + src/third_party/pcre-8.41/pcre16_compile.c | 45 + src/third_party/pcre-8.41/pcre16_config.c | 45 + src/third_party/pcre-8.41/pcre16_dfa_exec.c | 45 + src/third_party/pcre-8.41/pcre16_exec.c | 45 + src/third_party/pcre-8.41/pcre16_fullinfo.c | 45 + src/third_party/pcre-8.41/pcre16_get.c | 45 + src/third_party/pcre-8.41/pcre16_globals.c | 45 + src/third_party/pcre-8.41/pcre16_jit_compile.c | 45 + src/third_party/pcre-8.41/pcre16_maketables.c | 45 + src/third_party/pcre-8.41/pcre16_newline.c | 45 + src/third_party/pcre-8.41/pcre16_ord2utf16.c | 90 + src/third_party/pcre-8.41/pcre16_printint.c | 45 + src/third_party/pcre-8.41/pcre16_refcount.c | 45 + src/third_party/pcre-8.41/pcre16_string_utils.c | 45 + src/third_party/pcre-8.41/pcre16_study.c | 45 + src/third_party/pcre-8.41/pcre16_tables.c | 45 + src/third_party/pcre-8.41/pcre16_ucd.c | 45 + src/third_party/pcre-8.41/pcre16_utf16_utils.c | 130 + src/third_party/pcre-8.41/pcre16_valid_utf16.c | 137 + src/third_party/pcre-8.41/pcre16_version.c | 45 + src/third_party/pcre-8.41/pcre16_xclass.c | 45 + src/third_party/pcre-8.41/pcre32_byte_order.c | 45 + src/third_party/pcre-8.41/pcre32_chartables.c | 45 + src/third_party/pcre-8.41/pcre32_compile.c | 45 + src/third_party/pcre-8.41/pcre32_config.c | 45 + src/third_party/pcre-8.41/pcre32_dfa_exec.c | 45 + src/third_party/pcre-8.41/pcre32_exec.c | 45 + src/third_party/pcre-8.41/pcre32_fullinfo.c | 45 + src/third_party/pcre-8.41/pcre32_get.c | 45 + src/third_party/pcre-8.41/pcre32_globals.c | 45 + src/third_party/pcre-8.41/pcre32_jit_compile.c | 45 + src/third_party/pcre-8.41/pcre32_maketables.c | 45 + src/third_party/pcre-8.41/pcre32_newline.c | 45 + src/third_party/pcre-8.41/pcre32_ord2utf32.c | 82 + src/third_party/pcre-8.41/pcre32_printint.c | 45 + src/third_party/pcre-8.41/pcre32_refcount.c | 45 + src/third_party/pcre-8.41/pcre32_string_utils.c | 45 + src/third_party/pcre-8.41/pcre32_study.c | 45 + src/third_party/pcre-8.41/pcre32_tables.c | 45 + src/third_party/pcre-8.41/pcre32_ucd.c | 45 + src/third_party/pcre-8.41/pcre32_utf32_utils.c | 141 + src/third_party/pcre-8.41/pcre32_valid_utf32.c | 124 + src/third_party/pcre-8.41/pcre32_version.c | 45 + src/third_party/pcre-8.41/pcre32_xclass.c | 45 + src/third_party/pcre-8.41/pcre_byte_order.c | 319 + src/third_party/pcre-8.41/pcre_chartables.c | 198 + src/third_party/pcre-8.41/pcre_chartables.c.dist | 198 + src/third_party/pcre-8.41/pcre_compile.c | 9795 ++++++++++++++++ src/third_party/pcre-8.41/pcre_config.c | 190 + src/third_party/pcre-8.41/pcre_dfa_exec.c | 3674 ++++++ src/third_party/pcre-8.41/pcre_exec.c | 7173 +++++++++++ src/third_party/pcre-8.41/pcre_fullinfo.c | 245 + src/third_party/pcre-8.41/pcre_get.c | 669 ++ src/third_party/pcre-8.41/pcre_globals.c | 86 + src/third_party/pcre-8.41/pcre_internal.h | 2807 +++++ src/third_party/pcre-8.41/pcre_jit_compile.c | 11744 +++++++++++++++++++ src/third_party/pcre-8.41/pcre_jit_test.c | 1765 +++ src/third_party/pcre-8.41/pcre_maketables.c | 156 + src/third_party/pcre-8.41/pcre_newline.c | 210 + src/third_party/pcre-8.41/pcre_ord2utf8.c | 94 + src/third_party/pcre-8.41/pcre_printint.c | 834 ++ src/third_party/pcre-8.41/pcre_refcount.c | 92 + src/third_party/pcre-8.41/pcre_scanner.cc | 199 + src/third_party/pcre-8.41/pcre_scanner.h | 172 + src/third_party/pcre-8.41/pcre_scanner_unittest.cc | 162 + src/third_party/pcre-8.41/pcre_string_utils.c | 211 + src/third_party/pcre-8.41/pcre_stringpiece.cc | 43 + src/third_party/pcre-8.41/pcre_stringpiece.h | 180 + src/third_party/pcre-8.41/pcre_stringpiece.h.in | 180 + .../pcre-8.41/pcre_stringpiece_unittest.cc | 153 + src/third_party/pcre-8.41/pcre_study.c | 1686 +++ src/third_party/pcre-8.41/pcre_tables.c | 727 ++ src/third_party/pcre-8.41/pcre_ucd.c | 3644 ++++++ src/third_party/pcre-8.41/pcre_valid_utf8.c | 301 + src/third_party/pcre-8.41/pcre_version.c | 98 + src/third_party/pcre-8.41/pcre_xclass.c | 268 + src/third_party/pcre-8.41/pcrecpp.cc | 924 ++ src/third_party/pcre-8.41/pcrecpp.h | 710 ++ src/third_party/pcre-8.41/pcrecpp_internal.h | 71 + src/third_party/pcre-8.41/pcrecpp_unittest.cc | 1292 ++ src/third_party/pcre-8.41/pcrecpparg.h | 174 + src/third_party/pcre-8.41/pcrecpparg.h.in | 174 + src/third_party/pcre-8.41/pcredemo.c | 406 + src/third_party/pcre-8.41/pcregexp.pas | 845 ++ src/third_party/pcre-8.41/pcregrep.c | 3344 ++++++ src/third_party/pcre-8.41/pcreposix.c | 432 + src/third_party/pcre-8.41/pcreposix.h | 146 + src/third_party/pcre-8.41/pcretest.c | 5773 +++++++++ src/third_party/pcre-8.41/perltest.pl | 242 + src/third_party/pcre-8.41/ucp.h | 224 + src/third_party/scripts/pcre_get_sources.sh | 2 +- .../wiredtiger/build_linux/wiredtiger_config.h | 3 + .../wiredtiger/build_posix/configure.ac.in | 38 + .../wiredtiger/build_win/wiredtiger_config.h | 3 + src/third_party/wiredtiger/dist/api_data.py | 3 + src/third_party/wiredtiger/dist/filelist | 1 + src/third_party/wiredtiger/dist/flags.py | 2 +- src/third_party/wiredtiger/dist/s_string.ok | 2 + src/third_party/wiredtiger/dist/stat_data.py | 8 + src/third_party/wiredtiger/import.data | 2 +- .../wiredtiger/src/async/async_worker.c | 5 +- src/third_party/wiredtiger/src/btree/bt_cursor.c | 42 +- src/third_party/wiredtiger/src/btree/bt_delete.c | 9 +- src/third_party/wiredtiger/src/btree/bt_handle.c | 41 +- src/third_party/wiredtiger/src/btree/bt_random.c | 3 +- src/third_party/wiredtiger/src/btree/bt_walk.c | 121 +- src/third_party/wiredtiger/src/config/config_def.c | 46 +- src/third_party/wiredtiger/src/conn/conn_api.c | 47 +- src/third_party/wiredtiger/src/conn/conn_dhandle.c | 19 +- src/third_party/wiredtiger/src/conn/conn_log.c | 35 +- src/third_party/wiredtiger/src/conn/conn_open.c | 3 +- src/third_party/wiredtiger/src/cursor/cur_index.c | 4 +- src/third_party/wiredtiger/src/cursor/cur_join.c | 2 +- src/third_party/wiredtiger/src/cursor/cur_table.c | 4 +- src/third_party/wiredtiger/src/evict/evict_lru.c | 218 +- src/third_party/wiredtiger/src/evict/evict_page.c | 40 +- src/third_party/wiredtiger/src/include/btmem.h | 4 +- src/third_party/wiredtiger/src/include/btree.h | 38 +- src/third_party/wiredtiger/src/include/cache.h | 3 +- .../wiredtiger/src/include/connection.h | 7 +- src/third_party/wiredtiger/src/include/extern.h | 11 +- .../wiredtiger/src/include/extern_posix.h | 2 +- .../wiredtiger/src/include/extern_win.h | 2 +- src/third_party/wiredtiger/src/include/flags.h | 8 +- src/third_party/wiredtiger/src/include/misc.i | 39 - src/third_party/wiredtiger/src/include/os.h | 4 +- src/third_party/wiredtiger/src/include/session.h | 6 + src/third_party/wiredtiger/src/include/stat.h | 8 + src/third_party/wiredtiger/src/include/txn.h | 2 + src/third_party/wiredtiger/src/include/txn.i | 10 - .../wiredtiger/src/include/wiredtiger.in | 78 +- src/third_party/wiredtiger/src/log/log.c | 66 +- src/third_party/wiredtiger/src/log/log_slot.c | 20 +- src/third_party/wiredtiger/src/lsm/lsm_manager.c | 4 +- src/third_party/wiredtiger/src/meta/meta_table.c | 17 +- src/third_party/wiredtiger/src/meta/meta_turtle.c | 6 + .../wiredtiger/src/os_posix/os_mtx_cond.c | 33 +- src/third_party/wiredtiger/src/os_posix/os_time.c | 27 +- src/third_party/wiredtiger/src/os_win/os_time.c | 15 +- .../wiredtiger/src/reconcile/rec_write.c | 218 +- .../wiredtiger/src/schema/schema_alter.c | 13 +- .../wiredtiger/src/schema/schema_create.c | 6 +- .../wiredtiger/src/schema/schema_drop.c | 4 +- .../wiredtiger/src/schema/schema_list.c | 55 +- .../wiredtiger/src/schema/schema_open.c | 68 +- .../wiredtiger/src/schema/schema_rename.c | 2 +- .../wiredtiger/src/schema/schema_stat.c | 11 +- .../wiredtiger/src/schema/schema_truncate.c | 2 +- .../wiredtiger/src/schema/schema_worker.c | 9 +- .../wiredtiger/src/session/session_api.c | 15 +- .../wiredtiger/src/session/session_dhandle.c | 5 +- src/third_party/wiredtiger/src/support/err.c | 13 +- src/third_party/wiredtiger/src/support/rand.c | 12 - src/third_party/wiredtiger/src/support/stat.c | 29 + src/third_party/wiredtiger/src/support/time.c | 89 + src/third_party/wiredtiger/src/txn/txn.c | 21 +- src/third_party/wiredtiger/src/txn/txn_log.c | 31 +- src/third_party/wiredtiger/src/txn/txn_recover.c | 38 +- src/third_party/wiredtiger/test/mciproject.yml | 14 - .../wiredtiger/test/recovery/random-abort.c | 4 +- .../wiredtiger/test/suite/test_bug018.py | 98 + .../wiredtiger/test/suite/test_inmem01.py | 3 + src/third_party/wiredtiger/test/suite/test_las.py | 60 + .../wiredtiger/test/suite/test_txn02.py | 1 - .../wiredtiger/test/suite/test_txn05.py | 11 +- .../wiredtiger/test/suite/test_txn09.py | 4 +- .../wiredtiger/test/suite/test_txn16.py | 140 + version.json | 4 +- 811 files changed, 97371 insertions(+), 86723 deletions(-) create mode 100644 buildscripts/resmokeconfig/suites/write_concern_majority_passthrough.yml create mode 100644 buildscripts/tests/resmokelib/logging/test_buildlogger.py create mode 100644 jstests/auth/scram-credentials-invalid.js create mode 100644 jstests/auth/system_authorization_indexes.js create mode 100644 jstests/core/autocomplete.js create mode 100644 jstests/core/fts_dotted_prefix_fields.js create mode 100644 jstests/core/fts_trailing_fields.js create mode 100644 jstests/core/function_string_representations.js create mode 100644 jstests/core/geo_2d_trailing_fields.js create mode 100644 jstests/core/index_elemmatch2.js create mode 100644 jstests/core/js_jit.js create mode 100644 jstests/core/shell_connection_strings.js create mode 100644 jstests/core/update_affects_indexes.js create mode 100644 jstests/core/views/views_rename.js create mode 100644 jstests/libs/ftdc.js create mode 100644 jstests/libs/override_methods/override_helpers.js delete mode 100644 jstests/libs/override_methods/set_majority_read_and_write_concerns.js create mode 100644 jstests/libs/override_methods/set_read_and_write_concerns.js create mode 100644 jstests/libs/override_methods/set_read_preference_secondary.js create mode 100644 jstests/multiVersion/initial_sync_last_stable_from_latest.js create mode 100644 jstests/multiVersion/initial_sync_latest_from_last_stable.js delete mode 100644 jstests/multiVersion/initialsync.js create mode 100644 jstests/multiVersion/libs/initial_sync.js create mode 100644 jstests/noPassthrough/ftdc_setdirectory.js create mode 100644 jstests/noPassthrough/geo_near_fcv32.js create mode 100644 jstests/noPassthrough/non_atomic_apply_ops_logging.js create mode 100644 jstests/noPassthrough/partial_unique_indexes.js create mode 100644 jstests/noPassthrough/skip_sharding_configuration_checks.js create mode 100644 jstests/noPassthroughWithMongod/apply_ops_index_collation.js create mode 100644 jstests/replsets/apply_batches_totalMillis.js create mode 100644 jstests/replsets/apply_ops_concurrent_non_atomic_different_db.js create mode 100644 jstests/replsets/apply_ops_concurrent_non_atomic_same_collection.js create mode 100644 jstests/replsets/apply_ops_concurrent_non_atomic_same_db.js create mode 100644 jstests/replsets/initial_sync_rename_collection_unsafe.js create mode 100644 jstests/replsets/libs/apply_ops_concurrent_non_atomic.js create mode 100644 jstests/replsets/noop_writes_wait_for_write_concern.js create mode 100644 jstests/replsets/startParallelShell.js create mode 100644 jstests/sharding/movechunk_commit_changelog_stats.js create mode 100644 jstests/sharding/shard_existing_coll_chunk_count.js create mode 100644 src/mongo/client/mongo_uri_tests/mongo-uri-host-identifiers.json create mode 100644 src/mongo/client/mongo_uri_tests/mongo-uri-invalid.json create mode 100644 src/mongo/client/mongo_uri_tests/mongo-uri-options.json create mode 100644 src/mongo/client/mongo_uri_tests/mongo-uri-unix-sockets-absolute.json create mode 100644 src/mongo/client/mongo_uri_tests/mongo-uri-unix-sockets-relative.json create mode 100644 src/mongo/client/mongo_uri_tests/mongo-uri-valid-auth.json create mode 100644 src/mongo/client/mongo_uri_tests/mongo-uri-warnings.json create mode 100644 src/mongo/db/concurrency/global_lock_acquisition_tracker.cpp create mode 100644 src/mongo/db/concurrency/global_lock_acquisition_tracker.h create mode 100644 src/mongo/db/ftdc/ftdc_mongos.cpp create mode 100644 src/mongo/db/ftdc/ftdc_mongos.h create mode 100644 src/mongo/db/ftdc/ftdc_server.cpp create mode 100644 src/mongo/db/ftdc/ftdc_server.h delete mode 100644 src/mongo/db/s/collection_range_deleter.cpp delete mode 100644 src/mongo/db/s/collection_range_deleter.h create mode 100644 src/mongo/db/storage/encryption_hooks.cpp create mode 100644 src/mongo/db/storage/encryption_hooks.h create mode 100644 src/mongo/util/concurrency/with_lock.h create mode 100644 src/mongo/util/producer_consumer_queue.h create mode 100644 src/mongo/util/producer_consumer_queue_test.cpp delete mode 100755 src/third_party/pcre-8.39/132html delete mode 100644 src/third_party/pcre-8.39/AUTHORS delete mode 100644 src/third_party/pcre-8.39/COPYING delete mode 100644 src/third_party/pcre-8.39/ChangeLog delete mode 100755 src/third_party/pcre-8.39/CheckMan delete mode 100755 src/third_party/pcre-8.39/CleanTxt delete mode 100755 src/third_party/pcre-8.39/Detrail delete mode 100644 src/third_party/pcre-8.39/HACKING delete mode 100644 src/third_party/pcre-8.39/INSTALL delete mode 100644 src/third_party/pcre-8.39/LICENCE delete mode 100644 src/third_party/pcre-8.39/NEWS delete mode 100644 src/third_party/pcre-8.39/NON-AUTOTOOLS-BUILD delete mode 100644 src/third_party/pcre-8.39/NON-UNIX-USE delete mode 100755 src/third_party/pcre-8.39/PrepareRelease delete mode 100644 src/third_party/pcre-8.39/README delete mode 100755 src/third_party/pcre-8.39/RunGrepTest delete mode 100755 src/third_party/pcre-8.39/RunTest delete mode 100644 src/third_party/pcre-8.39/RunTest.bat delete mode 100644 src/third_party/pcre-8.39/SConscript delete mode 100755 src/third_party/pcre-8.39/ar-lib delete mode 100644 src/third_party/pcre-8.39/build_posix/config.h delete mode 100644 src/third_party/pcre-8.39/build_solaris/config.h delete mode 100755 src/third_party/pcre-8.39/build_windows/config.h delete mode 100644 src/third_party/pcre-8.39/dftables.c delete mode 100644 src/third_party/pcre-8.39/libpcre.pc.in delete mode 100644 src/third_party/pcre-8.39/libpcre16.pc.in delete mode 100644 src/third_party/pcre-8.39/libpcre32.pc.in delete mode 100644 src/third_party/pcre-8.39/libpcrecpp.pc.in delete mode 100644 src/third_party/pcre-8.39/libpcreposix.pc.in delete mode 100644 src/third_party/pcre-8.39/makevp.bat delete mode 100644 src/third_party/pcre-8.39/makevp_c.txt delete mode 100644 src/third_party/pcre-8.39/makevp_l.txt delete mode 100644 src/third_party/pcre-8.39/pcre-config.in delete mode 100644 src/third_party/pcre-8.39/pcre.h delete mode 100644 src/third_party/pcre-8.39/pcre.h.generic delete mode 100644 src/third_party/pcre-8.39/pcre.h.in delete mode 100644 src/third_party/pcre-8.39/pcre16_byte_order.c delete mode 100644 src/third_party/pcre-8.39/pcre16_chartables.c delete mode 100644 src/third_party/pcre-8.39/pcre16_compile.c delete mode 100644 src/third_party/pcre-8.39/pcre16_config.c delete mode 100644 src/third_party/pcre-8.39/pcre16_dfa_exec.c delete mode 100644 src/third_party/pcre-8.39/pcre16_exec.c delete mode 100644 src/third_party/pcre-8.39/pcre16_fullinfo.c delete mode 100644 src/third_party/pcre-8.39/pcre16_get.c delete mode 100644 src/third_party/pcre-8.39/pcre16_globals.c delete mode 100644 src/third_party/pcre-8.39/pcre16_jit_compile.c delete mode 100644 src/third_party/pcre-8.39/pcre16_maketables.c delete mode 100644 src/third_party/pcre-8.39/pcre16_newline.c delete mode 100644 src/third_party/pcre-8.39/pcre16_ord2utf16.c delete mode 100644 src/third_party/pcre-8.39/pcre16_printint.c delete mode 100644 src/third_party/pcre-8.39/pcre16_refcount.c delete mode 100644 src/third_party/pcre-8.39/pcre16_string_utils.c delete mode 100644 src/third_party/pcre-8.39/pcre16_study.c delete mode 100644 src/third_party/pcre-8.39/pcre16_tables.c delete mode 100644 src/third_party/pcre-8.39/pcre16_ucd.c delete mode 100644 src/third_party/pcre-8.39/pcre16_utf16_utils.c delete mode 100644 src/third_party/pcre-8.39/pcre16_valid_utf16.c delete mode 100644 src/third_party/pcre-8.39/pcre16_version.c delete mode 100644 src/third_party/pcre-8.39/pcre16_xclass.c delete mode 100644 src/third_party/pcre-8.39/pcre32_byte_order.c delete mode 100644 src/third_party/pcre-8.39/pcre32_chartables.c delete mode 100644 src/third_party/pcre-8.39/pcre32_compile.c delete mode 100644 src/third_party/pcre-8.39/pcre32_config.c delete mode 100644 src/third_party/pcre-8.39/pcre32_dfa_exec.c delete mode 100644 src/third_party/pcre-8.39/pcre32_exec.c delete mode 100644 src/third_party/pcre-8.39/pcre32_fullinfo.c delete mode 100644 src/third_party/pcre-8.39/pcre32_get.c delete mode 100644 src/third_party/pcre-8.39/pcre32_globals.c delete mode 100644 src/third_party/pcre-8.39/pcre32_jit_compile.c delete mode 100644 src/third_party/pcre-8.39/pcre32_maketables.c delete mode 100644 src/third_party/pcre-8.39/pcre32_newline.c delete mode 100644 src/third_party/pcre-8.39/pcre32_ord2utf32.c delete mode 100644 src/third_party/pcre-8.39/pcre32_printint.c delete mode 100644 src/third_party/pcre-8.39/pcre32_refcount.c delete mode 100644 src/third_party/pcre-8.39/pcre32_string_utils.c delete mode 100644 src/third_party/pcre-8.39/pcre32_study.c delete mode 100644 src/third_party/pcre-8.39/pcre32_tables.c delete mode 100644 src/third_party/pcre-8.39/pcre32_ucd.c delete mode 100644 src/third_party/pcre-8.39/pcre32_utf32_utils.c delete mode 100644 src/third_party/pcre-8.39/pcre32_valid_utf32.c delete mode 100644 src/third_party/pcre-8.39/pcre32_version.c delete mode 100644 src/third_party/pcre-8.39/pcre32_xclass.c delete mode 100644 src/third_party/pcre-8.39/pcre_byte_order.c delete mode 100644 src/third_party/pcre-8.39/pcre_chartables.c delete mode 100644 src/third_party/pcre-8.39/pcre_chartables.c.dist delete mode 100644 src/third_party/pcre-8.39/pcre_compile.c delete mode 100644 src/third_party/pcre-8.39/pcre_config.c delete mode 100644 src/third_party/pcre-8.39/pcre_dfa_exec.c delete mode 100644 src/third_party/pcre-8.39/pcre_exec.c delete mode 100644 src/third_party/pcre-8.39/pcre_fullinfo.c delete mode 100644 src/third_party/pcre-8.39/pcre_get.c delete mode 100644 src/third_party/pcre-8.39/pcre_globals.c delete mode 100644 src/third_party/pcre-8.39/pcre_internal.h delete mode 100644 src/third_party/pcre-8.39/pcre_jit_compile.c delete mode 100644 src/third_party/pcre-8.39/pcre_jit_test.c delete mode 100644 src/third_party/pcre-8.39/pcre_maketables.c delete mode 100644 src/third_party/pcre-8.39/pcre_newline.c delete mode 100644 src/third_party/pcre-8.39/pcre_ord2utf8.c delete mode 100644 src/third_party/pcre-8.39/pcre_printint.c delete mode 100644 src/third_party/pcre-8.39/pcre_refcount.c delete mode 100644 src/third_party/pcre-8.39/pcre_scanner.cc delete mode 100644 src/third_party/pcre-8.39/pcre_scanner.h delete mode 100644 src/third_party/pcre-8.39/pcre_scanner_unittest.cc delete mode 100644 src/third_party/pcre-8.39/pcre_string_utils.c delete mode 100644 src/third_party/pcre-8.39/pcre_stringpiece.cc delete mode 100644 src/third_party/pcre-8.39/pcre_stringpiece.h delete mode 100644 src/third_party/pcre-8.39/pcre_stringpiece.h.in delete mode 100644 src/third_party/pcre-8.39/pcre_stringpiece_unittest.cc delete mode 100644 src/third_party/pcre-8.39/pcre_study.c delete mode 100644 src/third_party/pcre-8.39/pcre_tables.c delete mode 100644 src/third_party/pcre-8.39/pcre_ucd.c delete mode 100644 src/third_party/pcre-8.39/pcre_valid_utf8.c delete mode 100644 src/third_party/pcre-8.39/pcre_version.c delete mode 100644 src/third_party/pcre-8.39/pcre_xclass.c delete mode 100644 src/third_party/pcre-8.39/pcrecpp.cc delete mode 100644 src/third_party/pcre-8.39/pcrecpp.h delete mode 100644 src/third_party/pcre-8.39/pcrecpp_internal.h delete mode 100644 src/third_party/pcre-8.39/pcrecpp_unittest.cc delete mode 100644 src/third_party/pcre-8.39/pcrecpparg.h delete mode 100644 src/third_party/pcre-8.39/pcrecpparg.h.in delete mode 100644 src/third_party/pcre-8.39/pcredemo.c delete mode 100644 src/third_party/pcre-8.39/pcregexp.pas delete mode 100644 src/third_party/pcre-8.39/pcregrep.c delete mode 100644 src/third_party/pcre-8.39/pcreposix.c delete mode 100644 src/third_party/pcre-8.39/pcreposix.h delete mode 100644 src/third_party/pcre-8.39/pcretest.c delete mode 100755 src/third_party/pcre-8.39/perltest.pl delete mode 100644 src/third_party/pcre-8.39/ucp.h create mode 100755 src/third_party/pcre-8.41/132html create mode 100644 src/third_party/pcre-8.41/AUTHORS create mode 100644 src/third_party/pcre-8.41/COPYING create mode 100644 src/third_party/pcre-8.41/ChangeLog create mode 100755 src/third_party/pcre-8.41/CheckMan create mode 100755 src/third_party/pcre-8.41/CleanTxt create mode 100755 src/third_party/pcre-8.41/Detrail create mode 100644 src/third_party/pcre-8.41/HACKING create mode 100644 src/third_party/pcre-8.41/INSTALL create mode 100644 src/third_party/pcre-8.41/LICENCE create mode 100644 src/third_party/pcre-8.41/NEWS create mode 100644 src/third_party/pcre-8.41/NON-AUTOTOOLS-BUILD create mode 100644 src/third_party/pcre-8.41/NON-UNIX-USE create mode 100755 src/third_party/pcre-8.41/PrepareRelease create mode 100644 src/third_party/pcre-8.41/README create mode 100755 src/third_party/pcre-8.41/RunGrepTest create mode 100755 src/third_party/pcre-8.41/RunTest create mode 100644 src/third_party/pcre-8.41/RunTest.bat create mode 100644 src/third_party/pcre-8.41/SConscript create mode 100755 src/third_party/pcre-8.41/ar-lib create mode 100644 src/third_party/pcre-8.41/build_posix/config.h create mode 100644 src/third_party/pcre-8.41/build_solaris/config.h create mode 100755 src/third_party/pcre-8.41/build_windows/config.h create mode 100644 src/third_party/pcre-8.41/dftables.c create mode 100644 src/third_party/pcre-8.41/libpcre.pc.in create mode 100644 src/third_party/pcre-8.41/libpcre16.pc.in create mode 100644 src/third_party/pcre-8.41/libpcre32.pc.in create mode 100644 src/third_party/pcre-8.41/libpcrecpp.pc.in create mode 100644 src/third_party/pcre-8.41/libpcreposix.pc.in create mode 100644 src/third_party/pcre-8.41/makevp.bat create mode 100644 src/third_party/pcre-8.41/makevp_c.txt create mode 100644 src/third_party/pcre-8.41/makevp_l.txt create mode 100644 src/third_party/pcre-8.41/pcre-config.in create mode 100644 src/third_party/pcre-8.41/pcre.h create mode 100644 src/third_party/pcre-8.41/pcre.h.generic create mode 100644 src/third_party/pcre-8.41/pcre.h.in create mode 100644 src/third_party/pcre-8.41/pcre16_byte_order.c create mode 100644 src/third_party/pcre-8.41/pcre16_chartables.c create mode 100644 src/third_party/pcre-8.41/pcre16_compile.c create mode 100644 src/third_party/pcre-8.41/pcre16_config.c create mode 100644 src/third_party/pcre-8.41/pcre16_dfa_exec.c create mode 100644 src/third_party/pcre-8.41/pcre16_exec.c create mode 100644 src/third_party/pcre-8.41/pcre16_fullinfo.c create mode 100644 src/third_party/pcre-8.41/pcre16_get.c create mode 100644 src/third_party/pcre-8.41/pcre16_globals.c create mode 100644 src/third_party/pcre-8.41/pcre16_jit_compile.c create mode 100644 src/third_party/pcre-8.41/pcre16_maketables.c create mode 100644 src/third_party/pcre-8.41/pcre16_newline.c create mode 100644 src/third_party/pcre-8.41/pcre16_ord2utf16.c create mode 100644 src/third_party/pcre-8.41/pcre16_printint.c create mode 100644 src/third_party/pcre-8.41/pcre16_refcount.c create mode 100644 src/third_party/pcre-8.41/pcre16_string_utils.c create mode 100644 src/third_party/pcre-8.41/pcre16_study.c create mode 100644 src/third_party/pcre-8.41/pcre16_tables.c create mode 100644 src/third_party/pcre-8.41/pcre16_ucd.c create mode 100644 src/third_party/pcre-8.41/pcre16_utf16_utils.c create mode 100644 src/third_party/pcre-8.41/pcre16_valid_utf16.c create mode 100644 src/third_party/pcre-8.41/pcre16_version.c create mode 100644 src/third_party/pcre-8.41/pcre16_xclass.c create mode 100644 src/third_party/pcre-8.41/pcre32_byte_order.c create mode 100644 src/third_party/pcre-8.41/pcre32_chartables.c create mode 100644 src/third_party/pcre-8.41/pcre32_compile.c create mode 100644 src/third_party/pcre-8.41/pcre32_config.c create mode 100644 src/third_party/pcre-8.41/pcre32_dfa_exec.c create mode 100644 src/third_party/pcre-8.41/pcre32_exec.c create mode 100644 src/third_party/pcre-8.41/pcre32_fullinfo.c create mode 100644 src/third_party/pcre-8.41/pcre32_get.c create mode 100644 src/third_party/pcre-8.41/pcre32_globals.c create mode 100644 src/third_party/pcre-8.41/pcre32_jit_compile.c create mode 100644 src/third_party/pcre-8.41/pcre32_maketables.c create mode 100644 src/third_party/pcre-8.41/pcre32_newline.c create mode 100644 src/third_party/pcre-8.41/pcre32_ord2utf32.c create mode 100644 src/third_party/pcre-8.41/pcre32_printint.c create mode 100644 src/third_party/pcre-8.41/pcre32_refcount.c create mode 100644 src/third_party/pcre-8.41/pcre32_string_utils.c create mode 100644 src/third_party/pcre-8.41/pcre32_study.c create mode 100644 src/third_party/pcre-8.41/pcre32_tables.c create mode 100644 src/third_party/pcre-8.41/pcre32_ucd.c create mode 100644 src/third_party/pcre-8.41/pcre32_utf32_utils.c create mode 100644 src/third_party/pcre-8.41/pcre32_valid_utf32.c create mode 100644 src/third_party/pcre-8.41/pcre32_version.c create mode 100644 src/third_party/pcre-8.41/pcre32_xclass.c create mode 100644 src/third_party/pcre-8.41/pcre_byte_order.c create mode 100644 src/third_party/pcre-8.41/pcre_chartables.c create mode 100644 src/third_party/pcre-8.41/pcre_chartables.c.dist create mode 100644 src/third_party/pcre-8.41/pcre_compile.c create mode 100644 src/third_party/pcre-8.41/pcre_config.c create mode 100644 src/third_party/pcre-8.41/pcre_dfa_exec.c create mode 100644 src/third_party/pcre-8.41/pcre_exec.c create mode 100644 src/third_party/pcre-8.41/pcre_fullinfo.c create mode 100644 src/third_party/pcre-8.41/pcre_get.c create mode 100644 src/third_party/pcre-8.41/pcre_globals.c create mode 100644 src/third_party/pcre-8.41/pcre_internal.h create mode 100644 src/third_party/pcre-8.41/pcre_jit_compile.c create mode 100644 src/third_party/pcre-8.41/pcre_jit_test.c create mode 100644 src/third_party/pcre-8.41/pcre_maketables.c create mode 100644 src/third_party/pcre-8.41/pcre_newline.c create mode 100644 src/third_party/pcre-8.41/pcre_ord2utf8.c create mode 100644 src/third_party/pcre-8.41/pcre_printint.c create mode 100644 src/third_party/pcre-8.41/pcre_refcount.c create mode 100644 src/third_party/pcre-8.41/pcre_scanner.cc create mode 100644 src/third_party/pcre-8.41/pcre_scanner.h create mode 100644 src/third_party/pcre-8.41/pcre_scanner_unittest.cc create mode 100644 src/third_party/pcre-8.41/pcre_string_utils.c create mode 100644 src/third_party/pcre-8.41/pcre_stringpiece.cc create mode 100644 src/third_party/pcre-8.41/pcre_stringpiece.h create mode 100644 src/third_party/pcre-8.41/pcre_stringpiece.h.in create mode 100644 src/third_party/pcre-8.41/pcre_stringpiece_unittest.cc create mode 100644 src/third_party/pcre-8.41/pcre_study.c create mode 100644 src/third_party/pcre-8.41/pcre_tables.c create mode 100644 src/third_party/pcre-8.41/pcre_ucd.c create mode 100644 src/third_party/pcre-8.41/pcre_valid_utf8.c create mode 100644 src/third_party/pcre-8.41/pcre_version.c create mode 100644 src/third_party/pcre-8.41/pcre_xclass.c create mode 100644 src/third_party/pcre-8.41/pcrecpp.cc create mode 100644 src/third_party/pcre-8.41/pcrecpp.h create mode 100644 src/third_party/pcre-8.41/pcrecpp_internal.h create mode 100644 src/third_party/pcre-8.41/pcrecpp_unittest.cc create mode 100644 src/third_party/pcre-8.41/pcrecpparg.h create mode 100644 src/third_party/pcre-8.41/pcrecpparg.h.in create mode 100644 src/third_party/pcre-8.41/pcredemo.c create mode 100644 src/third_party/pcre-8.41/pcregexp.pas create mode 100644 src/third_party/pcre-8.41/pcregrep.c create mode 100644 src/third_party/pcre-8.41/pcreposix.c create mode 100644 src/third_party/pcre-8.41/pcreposix.h create mode 100644 src/third_party/pcre-8.41/pcretest.c create mode 100755 src/third_party/pcre-8.41/perltest.pl create mode 100644 src/third_party/pcre-8.41/ucp.h create mode 100644 src/third_party/wiredtiger/src/support/time.c create mode 100644 src/third_party/wiredtiger/test/suite/test_bug018.py create mode 100644 src/third_party/wiredtiger/test/suite/test_las.py create mode 100644 src/third_party/wiredtiger/test/suite/test_txn16.py diff --git a/README b/README index 858f8a1d2c8..5a772b66e7d 100644 --- a/README +++ b/README @@ -4,23 +4,26 @@ Welcome to MongoDB 3.4! COMPONENTS - mongod - The database process. - mongos - Sharding controller. + mongod - The database server. + mongos - Sharding router. mongo - The database shell (uses interactive javascript). UTILITIES - mongodump - MongoDB dump tool - for backups, snapshots, etc. - mongorestore - MongoDB restore a dump - mongoexport - Export a single collection to JSON or CSV - mongoimport - Import from JSON or CSV - mongofiles - Utility for putting and getting files from MongoDB GridFS - mongostat - Show performance statistics - mongoreplay - Workload capture and analysis tool + mongodump - Create a binary dump of the contents of a database. + mongorestore - Restore data from the output created by mongodump. + mongoexport - Export the contents of a collection to JSON or CSV. + mongoimport - Import data from JSON, CSV or TSV. + mongofiles - Put, get and delete files from GridFS. + mongostat - Show the status of a running mongod/mongos. + bsondump - Convert BSON files into human-readable formats. + mongooplog - Poll the oplog and apply to a local server. + mongoreplay - Traffic capture and replay tool. + mongotop - Track time spent reading and writing data. BUILDING - See docs/building.md, also www.mongodb.org search for "Building". + See docs/building.md. RUNNING @@ -30,7 +33,7 @@ RUNNING To run a single server database: - $ mkdir /data/db + $ sudo mkdir -p /data/db $ ./mongod $ $ # The mongo javascript shell connects to localhost and test database by default: @@ -40,7 +43,12 @@ RUNNING DRIVERS Client drivers for most programming languages are available at - mongodb.org. Use the shell ("mongo") for administrative tasks. + https://docs.mongodb.com/manual/applications/drivers/. Use the shell + ("mongo") for administrative tasks. + +BUG REPORTS + + See https://github.com/mongodb/mongo/wiki/Submit-Bug-Reports. PACKAGING @@ -49,26 +57,28 @@ PACKAGING DOCUMENTATION - http://www.mongodb.org/ + https://docs.mongodb.com/manual/ + +CLOUD HOSTED MONGODB -CLOUD MANAGED MONGODB + https://www.mongodb.com/cloud/atlas - http://cloud.mongodb.com/ +MAIL LISTS -MAIL LISTS AND IRC + https://groups.google.com/forum/#!forum/mongodb-user - http://dochub.mongodb.org/core/community + A forum for technical questions about using MongoDB. + + https://groups.google.com/forum/#!forum/mongodb-dev + + A forum for technical questions about building and developing MongoDB. LEARN MONGODB - http://university.mongodb.com/ + https://university.mongodb.com/ LICENSE Most MongoDB source files (src/mongo folder and below) are made available - under the terms of the GNU Affero General Public License (AGPL). See + under the terms of the GNU Affero General Public License (GNU AGPLv3). See individual files for details. - - As an exception, the files in the client/, debian/, rpm/, - utils/mongoutils, and all subdirectories thereof are made available under - the terms of the Apache License, version 2.0. diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py index d171a5a6bc1..6e39685ef8b 100644 --- a/buildscripts/burn_in_tests.py +++ b/buildscripts/burn_in_tests.py @@ -17,7 +17,7 @@ import sys import urlparse import yaml -API_SERVER_DEFAULT = "http://evergreen-api.mongodb.com:8080" +API_SERVER_DEFAULT = "https://evergreen.mongodb.com" # Get relative imports to work when the package is not installed on the PYTHONPATH. if __name__ == "__main__" and __package__ is None: @@ -418,6 +418,8 @@ def main(): tests_by_task = _load_tests_file(values.test_list_file) # If there are no tests to run, carry on. if tests_by_task is None: + test_results = {"failures": 0, "results": []} + _write_report_file(test_results, values.report_file) sys.exit(0) # Run the executor finder. @@ -438,6 +440,7 @@ def main(): # If there are no changed tests, exit cleanly. if not changed_tests: print "No new or modified tests found." + _write_report_file({}, values.test_list_outfile) sys.exit(0) suites = resmokelib.parser.get_suites(values, changed_tests) tests_by_executor = create_executor_list(suites, exclude_suites) diff --git a/buildscripts/hang_analyzer.py b/buildscripts/hang_analyzer.py index 73e1cbe6d41..09245904a7f 100755 --- a/buildscripts/hang_analyzer.py +++ b/buildscripts/hang_analyzer.py @@ -24,6 +24,7 @@ import signal import subprocess import sys import tempfile +import traceback import time from distutils import spawn from optparse import OptionParser @@ -53,7 +54,7 @@ def call(a, logger): if ret != 0: logger.error("Bad exit code %d" % (ret)) - raise Exception() + raise Exception("Bad exit code %d from %s" % (ret, " ".join(a))) # Copied from python 2.7 version of subprocess.py @@ -375,14 +376,14 @@ class GDBDumper(object): base, ext = os.path.splitext(logger.mongo_process_filename) raw_stacks_filename = base + '_raw_stacks' + ext raw_stacks_commands = [ - 'echo \\nWriting raw stacks to %s.\\n' % raw_stacks_filename, - # This sends output to log file rather than stdout until we turn logging off. - 'set logging redirect on', - 'set logging file ' + raw_stacks_filename, - 'set logging on', - 'thread apply all bt', - 'set logging off', - ] + 'echo \\nWriting raw stacks to %s.\\n' % raw_stacks_filename, + # This sends output to log file rather than stdout until we turn logging off. + 'set logging redirect on', + 'set logging file ' + raw_stacks_filename, + 'set logging on', + 'thread apply all bt', + 'set logging off', + ] cmds = [ "set interactive-mode off", @@ -732,28 +733,38 @@ def main(): # a signal handler to wait for the signal since it supports POSIX signals. if _is_windows: root_logger.info("Calling SetEvent to signal python process %s with PID %d" % - (process_name, pid)) + (process_name, pid)) signal_event_object(root_logger, pid) else: root_logger.info("Sending signal SIGUSR1 to python process %s with PID %d" % - (process_name, pid)) + (process_name, pid)) signal_process(root_logger, pid, signal.SIGUSR1) + trapped_exceptions = [] + # Dump all processes, except python & java. for (pid, process_name) in [(p, pn) for (p, pn) in processes if not re.match("^(java|python)", pn)]: process_logger = get_process_logger(options.debugger_output, pid, process_name) - dbg.dump_info( - root_logger, - process_logger, - pid, - process_name, - options.dump_core and check_dump_quota(max_dump_size_bytes, dbg.get_dump_ext())) + try: + dbg.dump_info( + root_logger, + process_logger, + pid, + process_name, + options.dump_core and check_dump_quota(max_dump_size_bytes, dbg.get_dump_ext())) + except Exception as err: + root_logger.info("Error encountered when invoking debugger %s" % err) + trapped_exceptions.append(traceback.format_exc()) # Dump java processes using jstack. for (pid, process_name) in [(p, pn) for (p, pn) in processes if pn.startswith("java")]: process_logger = get_process_logger(options.debugger_output, pid, process_name) - jstack.dump_info(root_logger, process_logger, pid, process_name) + try: + jstack.dump_info(root_logger, process_logger, pid, process_name) + except Exception as err: + root_logger.info("Error encountered when invoking debugger %s" % err) + trapped_exceptions.append(traceback.format_exc()) # Signal go processes to ensure they print out stack traces, and die on POSIX OSes. # On Windows, this will simply kill the process since python emulates SIGABRT as @@ -761,10 +772,15 @@ def main(): # Note: The stacktrace output may be captured elsewhere (i.e. resmoke). for (pid, process_name) in [(p, pn) for (p, pn) in processes if pn in go_processes]: root_logger.info("Sending signal SIGABRT to go process %s with PID %d" % - (process_name, pid)) + (process_name, pid)) signal_process(root_logger, pid, signal.SIGABRT) root_logger.info("Done analyzing all processes for hangs") + for exception in trapped_exceptions: + root_logger.info(exception) + if trapped_exceptions: + sys.exit(1) + if __name__ == "__main__": main() diff --git a/buildscripts/package_test/.kitchen.yml b/buildscripts/package_test/.kitchen.yml index e9b6989d221..3b4a476551c 100644 --- a/buildscripts/package_test/.kitchen.yml +++ b/buildscripts/package_test/.kitchen.yml @@ -2,11 +2,13 @@ driver: name: ec2 region: us-east-1 + vpc_mode: true + vpc_id: <%= ENV['KITCHEN_VPC'] %> subnet_id: <%= ENV['KITCHEN_SUBNET'] %> security_group_ids: - <%= ENV['KITCHEN_SECURITY_GROUP'] %> aws_ssh_key_id: <%= ENV['KITCHEN_SSH_KEY_ID'] %> - interface: dns + interface: private associate_public_ip: true verifier: diff --git a/buildscripts/package_test/recipes/install_mongodb.rb b/buildscripts/package_test/recipes/install_mongodb.rb index dcf7499289e..9b4ea703b2b 100644 --- a/buildscripts/package_test/recipes/install_mongodb.rb +++ b/buildscripts/package_test/recipes/install_mongodb.rb @@ -84,6 +84,16 @@ if platform_family? 'suse' EOD end + %w( + SLES12-Pool + SLES12-Updates + ).each do |repo| + execute "add #{repo}" do + command "zypper addrepo --check --refresh --name \"#{repo}\" http://smt-ec2.susecloud.net/repo/SUSE/Products/SLE-SERVER/12/x86_64/product?credentials=SMT-http_smt-ec2_susecloud_net 'SMT-http_smt-ec2_susecloud_net:#{repo}'" + not_if "zypper lr | grep #{repo}" + end + end + execute 'install mongod' do command 'zypper -n install `find . -name "*server*.rpm"`' cwd homedir diff --git a/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml b/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml index bc227b09279..72b838b3c07 100644 --- a/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml @@ -6,11 +6,28 @@ selector: - jstests/aggregation/expressions/*.js - jstests/aggregation/sources/*/*.js exclude_files: - - jstests/aggregation/bugs/server18198.js # Uses a mocked mongo client to test read preference. - - jstests/aggregation/mongos_slaveok.js # Majority read on secondary requires afterOpTime. - - jstests/aggregation/sources/facet/use_cases.js # Cannot specify write concern when - # secondaryThrottle is not set. - - jstests/aggregation/testSlave.js # Majority read on secondary requires afterOpTime. + # These tests fail due to the inability to specify a writeConcern when secondaryThrottle is not + # set as part of the moveChunk command. + - jstests/aggregation/sources/facet/use_cases.js + # These test fail because afterOpTime is required to guarantee a secondary has advanced its + # majority-committed snapshot. + - jstests/aggregation/mongos_slaveok.js + - jstests/aggregation/testSlave.js + exclude_with_any_tags: + ## + # The next three tags correspond to the special errors thrown by the + # set_read_and_write_concerns.js override when it refuses to replace the readConcern or + # writeConcern of a particular command. Above each tag are the message(s) that cause the tag to + # be warranted. + ## + # "Cowardly refusing to override read concern of command: ..." + - assumes_read_concern_unchanged + # "Cowardly refusing to override write concern of command: ..." + - assumes_write_concern_unchanged + # "Cowardly refusing to run test with overridden write concern when it uses a command that can + # only perform w=1 writes: ..." + - requires_collmod_command + - requires_eval_command executor: js_test: @@ -18,13 +35,19 @@ executor: shell_options: global_vars: TestData: + defaultReadConcernLevel: majority enableMajorityReadConcern: '' - eval: "var testingReplication = true; load('jstests/libs/override_methods/set_majority_read_and_write_concerns.js');" + eval: >- + var testingReplication = true; + load('jstests/libs/override_methods/set_read_and_write_concerns.js'); readMode: commands hooks: - - class: ValidateCollections + # The CheckReplDBHash hook waits until all operations have replicated to and have been applied + # on the secondaries, so we run the ValidateCollections hook after it to ensure we're + # validating the entire contents of the collection. - class: CheckReplOplogs - class: CheckReplDBHash + - class: ValidateCollections fixture: class: ReplicaSetFixture mongod_options: @@ -33,5 +56,3 @@ executor: enableTestCommands: 1 numInitialSyncAttempts: 1 num_nodes: 2 - # Needs to be set for any ephemeral or no-journaling storage engine - write_concern_majority_journal_default: false diff --git a/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml b/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml index f47e424e1cf..22e84bc9c63 100644 --- a/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/read_concern_majority_passthrough.yml @@ -3,79 +3,47 @@ selector: roots: - jstests/core/**/*.js exclude_files: - # Tests that won't work with an injected 'majority' readConcern - # and/or an injected 'majority' writeConcern. Where a function is - # listed the reason is we don't have a reliable solution to override - # the write concern for that function. - - jstests/core/batch_write_command*.js # these tests use various write concerns - - jstests/core/bench_test*.js # benchRun() used for writes - - jstests/core/capped_update.js # uses godinsert and can't run under replication. - - jstests/core/crud_api.js # has specific w:0 tests - - jstests/core/error2.js # db.eval() used - - jstests/core/eval0.js # db.eval() used - - jstests/core/eval1.js # db.eval() used - - jstests/core/eval3.js # db.eval() used - - jstests/core/eval4.js # db.eval() used - - jstests/core/eval5.js # db.eval() used - - jstests/core/eval6.js # db.eval() used - - jstests/core/eval7.js # db.eval() used - - jstests/core/eval9.js # db.eval() used - - jstests/core/evala.js # db.eval() used - - jstests/core/evalb.js # db.eval() used - - jstests/core/evald.js # db.eval() used - - jstests/core/evale.js # db.eval() used - - jstests/core/evalg.js # db.eval() used - - jstests/core/eval_mr.js # db.eval() used - - jstests/core/eval_nolock.js # db.eval() used - - jstests/core/geo_s2cursorlimitskip.js # drops system.profile collection and counts ops. - - jstests/core/js3.js # db.dbEval() used - - jstests/core/js7.js # db.eval() used - - jstests/core/js9.js # db.eval() used - - jstests/core/mr_merge.js # mr temp tables aren't replicated - - jstests/core/mr_merge2.js # mr temp tables aren't replicated - - jstests/core/mr_outreduce.js # mr temp tables aren't replicated - - jstests/core/mr_outreduce2.js # mr temp tables aren't replicated - - jstests/core/opcounters_active.js # off by n problem with opcounters - - jstests/core/opcounters_write_cmd.js # off by n problem with opcounters - - jstests/core/profile_agg.js # system.profile not replicated - - jstests/core/profile_count.js # system.profile not replicated - - jstests/core/profile_delete.js # system.profile not replicated - - jstests/core/profile_distinct.js # system.profile not replicated - - jstests/core/profile_find.js # system.profile not replicated - - jstests/core/profile_findandmodify.js # system.profile not replicated - - jstests/core/profile_geonear.js # system.profile not replicated - - jstests/core/profile_getmore.js # system.profile not replicated - - jstests/core/profile_group.js # system.profile not replicated - - jstests/core/profile_insert.js # system.profile not replicated - - jstests/core/profile_mapreduce.js # system.profile not replicated - - jstests/core/profile_update.js # system.profile not replicated - - jstests/core/profile1.js # system.profile not replicated - - jstests/core/profile2.js # system.profile not replicated - - jstests/core/profile3.js # system.profile not replicated - - jstests/core/read_after_optime.js # verifies read after optime fails on standalone - - jstests/core/remove8.js # db.eval() used - - jstests/core/rename4.js # db.eval() used - - jstests/core/shell1.js # tests setSlaveOk() variations on standalone mongod - - jstests/core/shellkillop.js # db.eval() used - - jstests/core/shell_writeconcern.js # checks write concern shell helpers - - jstests/core/storefunc.js # db.eval() used - - jstests/core/write_result.js # Tests invalid writeConcern, we shouldn't override. - # Tests that need triaging & remediation | blacklist decision - # Comments list possible problem point under review. - - jstests/core/capped6.js # Uses captrunc test command. - - jstests/core/convert_to_capped_nonexistant.js # Uses convertToCapped and captrunc command. - - jstests/core/stages_delete.js # Uses stageDebug command for deletes. + # These tests are not expected to pass with replica-sets: + - jstests/core/dbadmin.js + - jstests/core/opcounters_write_cmd.js + - jstests/core/read_after_optime.js + - jstests/core/capped_update.js + # These tests use benchRun(), which isn't configured to use the overridden writeConcern. + - jstests/core/bench_test*.js + exclude_with_any_tags: + ## + # The next three tags correspond to the special errors thrown by the + # set_read_and_write_concerns.js override when it refuses to replace the readConcern or + # writeConcern of a particular command. Above each tag are the message(s) that cause the tag to + # be warranted. + ## + # "Cowardly refusing to override read concern of command: ..." + - assumes_read_concern_unchanged + # "Cowardly refusing to override write concern of command: ..." + - assumes_write_concern_unchanged + # "Cowardly refusing to run test with overridden write concern when it uses a command that can + # only perform w=1 writes: ..." + - requires_collmod_command + - requires_eval_command executor: js_test: config: shell_options: - eval: "var testingReplication = true; load('jstests/libs/override_methods/set_majority_read_and_write_concerns.js');" + global_vars: + TestData: + defaultReadConcernLevel: majority + eval: >- + var testingReplication = true; + load('jstests/libs/override_methods/set_read_and_write_concerns.js'); readMode: commands hooks: - - class: ValidateCollections + # The CheckReplDBHash hook waits until all operations have replicated to and have been applied + # on the secondaries, so we run the ValidateCollections hook after it to ensure we're + # validating the entire contents of the collection. - class: CheckReplOplogs - class: CheckReplDBHash + - class: ValidateCollections - class: CleanEveryN n: 20 fixture: @@ -84,7 +52,6 @@ executor: set_parameters: enableTestCommands: 1 numInitialSyncAttempts: 1 + writePeriodicNoops: 1 enableMajorityReadConcern: '' num_nodes: 2 - # Needs to be set for any ephemeral or no-journaling storage engine - write_concern_majority_journal_default: false diff --git a/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml index 832e0ab9bfc..114ebcab372 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_initsync_jscore_passthrough.yml @@ -14,6 +14,7 @@ selector: - jstests/core/commands_that_do_not_write_do_not_accept_wc.js - jstests/core/constructors.js - jstests/core/eval_mr.js + - jstests/core/function_string_representations.js - jstests/core/geo_big_polygon3.js - jstests/core/geo_mapreduce.js - jstests/core/geo_mapreduce2.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml index 3b48456c92e..1e51713a475 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_resync_jscore_passthrough.yml @@ -14,6 +14,7 @@ selector: - jstests/core/commands_that_do_not_write_do_not_accept_wc.js - jstests/core/constructors.js - jstests/core/eval_mr.js + - jstests/core/function_string_representations.js - jstests/core/geo_big_polygon3.js - jstests/core/geo_mapreduce.js - jstests/core/geo_mapreduce2.js diff --git a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml index 0bafa32a1e5..acf8ae53e96 100644 --- a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml +++ b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml @@ -50,6 +50,7 @@ selector: - jstests/sharding/shard2.js - jstests/sharding/shard3.js - jstests/sharding/shard_collection_basic.js + - jstests/sharding/shard_existing_coll_chunk_count.js - jstests/sharding/sharding_balance1.js - jstests/sharding/sharding_balance2.js - jstests/sharding/sharding_balance3.js diff --git a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml index f4c741badd4..c7ff1b24395 100644 --- a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml +++ b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml @@ -95,6 +95,8 @@ selector: # TODO (SERVER-27379): Cannot create views when featureCompatibilityVersion is 3.2. # Unblacklist when the featureCompatibilityVersion is 3.4 or higher. - jstests/sharding/views.js + # SERVER-20392: Reenable when backported to 3.4 and released as last-stable. + - jstests/sharding/shard_existing_coll_chunk_count.js executor: js_test: diff --git a/buildscripts/resmokeconfig/suites/write_concern_majority_passthrough.yml b/buildscripts/resmokeconfig/suites/write_concern_majority_passthrough.yml new file mode 100644 index 00000000000..388046d2782 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/write_concern_majority_passthrough.yml @@ -0,0 +1,78 @@ +test_kind: js_test + +selector: + js_test: + roots: + - jstests/core/**/*.js + exclude_files: + # These tests are not expected to pass with replica-sets: + - jstests/core/dbadmin.js + - jstests/core/opcounters_write_cmd.js + - jstests/core/read_after_optime.js + - jstests/core/capped_update.js + # The connection_string_validation.js test does not expect the mongo shell to be using a replica + # set connection string. + - jstests/core/connection_string_validation.js + # These tests attempt to read from the "system.profile" collection, which may be missing entries + # if a write was performed on the primary of the replica set instead. + - jstests/core/*profile*.js + # The shellkillop.js test spawns a parallel shell without using startParallelShell() and + # therefore doesn't inherit the w="majority" write concern when performing its writes. + - jstests/core/shellkillop.js + exclude_with_any_tags: + ## + # The next three tags correspond to the special errors thrown by the + # set_read_and_write_concerns.js override when it refuses to replace the readConcern or + # writeConcern of a particular command. Above each tag are the message(s) that cause the tag to + # be warranted. + ## + # "Cowardly refusing to override read concern of command: ..." + - assumes_read_concern_unchanged + # "Cowardly refusing to override write concern of command: ..." + - assumes_write_concern_unchanged + # "Cowardly refusing to run test with overridden write concern when it uses a command that can + # only perform w=1 writes: ..." + - requires_collmod_command + - requires_eval_command + ## + # The next tag corresponds to the special error thrown by the set_read_preference_secondary.js + # override when it refuses to replace the readPreference of a particular command. Above each tag + # are the message(s) that cause the tag to be warranted. + ## + # "Cowardly refusing to override read preference of command: ..." + # "Cowardly refusing to run test with overridden read preference when it reads from a + # non-replicated collection: ..." + - assumes_read_preference_unchanged + +executor: + js_test: + config: + shell_options: + global_vars: + TestData: + defaultReadConcernLevel: local + eval: >- + testingReplication = true; + load('jstests/libs/override_methods/set_read_and_write_concerns.js'); + load('jstests/libs/override_methods/set_read_preference_secondary.js'); + readMode: commands + hooks: + # The CheckReplDBHash hook waits until all operations have replicated to and have been applied + # on the secondaries, so we run the ValidateCollections hook after it to ensure we're + # validating the entire contents of the collection. + - class: CheckReplOplogs + - class: CheckReplDBHash + - class: ValidateCollections + - class: CleanEveryN + n: 20 + fixture: + class: ReplicaSetFixture + mongod_options: + set_parameters: + enableTestCommands: 1 + numInitialSyncAttempts: 1 + # This suite requires w="majority" writes to be applied on all secondaries. By using a 2-node + # replica set and having secondaries vote, the majority of the replica set is all nodes. + num_nodes: 2 + voting_secondaries: true + use_replica_set_connection_string: true diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index e4cadc8e006..d94cd438ee0 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -115,8 +115,8 @@ def mongos_program(logger, executable=None, process_kwargs=None, **kwargs): return _process.Process(logger, args, **process_kwargs) -def mongo_shell_program(logger, executable=None, filename=None, process_kwargs=None, - isMainTest=True, **kwargs): +def mongo_shell_program(logger, executable=None, connection_string=None, filename=None, + process_kwargs=None, isMainTest=True, **kwargs): """ Returns a Process instance that starts a mongo shell with arguments constructed from 'kwargs'. @@ -184,9 +184,22 @@ def mongo_shell_program(logger, executable=None, filename=None, process_kwargs=N if config.SHELL_WRITE_MODE is not None: kwargs["writeMode"] = config.SHELL_WRITE_MODE + if connection_string is not None: + # The --host and --port options are ignored by the mongo shell when an explicit connection + # string is specified. We remove these options to avoid any ambiguity with what server the + # logged mongo shell invocation will connect to. + if "port" in kwargs: + kwargs.pop("port") + + if "host" in kwargs: + kwargs.pop("host") + # Apply the rest of the command line arguments. _apply_kwargs(args, kwargs) + if connection_string is not None: + args.append(connection_string) + # Have the mongos shell run the specified file. args.append(filename) diff --git a/buildscripts/resmokelib/logging/buildlogger.py b/buildscripts/resmokelib/logging/buildlogger.py index f058372df8c..e8e37d2c004 100644 --- a/buildscripts/resmokelib/logging/buildlogger.py +++ b/buildscripts/resmokelib/logging/buildlogger.py @@ -5,7 +5,9 @@ Defines handlers for communicating with a buildlogger server. from __future__ import absolute_import import functools +import httplib import urllib2 +import json from . import handlers from . import loggers @@ -131,6 +133,47 @@ def new_test_id(build_id, build_config, test_filename, test_command): return response["id"] +class _LogsSplitter(object): + """Class with static methods used to split list of log lines into smaller batches.""" + + @staticmethod + def split_logs(log_lines, max_size): + """ + Splits the log lines into batches of size less than or equal to max_size. + + Args: + log_lines: A list of log lines. + max_size: The maximum size in bytes a batch of log lines can have in JSON. + Returns: + A list of list of log lines. Each item is a list is a list of log lines + satisfying the size requirement. + """ + if not max_size: + return [log_lines] + + def line_size(line): + """ + Computes the encoded JSON size of a log line as part of an array. + 2 is added to each string size to account for the array representation of the logs, + as each line is preceded by a '[' or a space and followed by a ',' or a ']'. + """ + return len(json.dumps(line, encoding="utf-8")) + 2 + + curr_logs = [] + curr_logs_size = 0 + split_logs = [] + for line in log_lines: + size = line_size(line) + if curr_logs_size + size > max_size: + split_logs.append(curr_logs) + curr_logs = [] + curr_logs_size = 0 + curr_logs.append(line) + curr_logs_size += size + split_logs.append(curr_logs) + return split_logs + + class _BaseBuildloggerHandler(handlers.BufferedHandler): """ Base class of the buildlogger handler for the global logs and the @@ -138,8 +181,8 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler): """ def __init__(self, - build_id, build_config, + endpoint, capacity=_SEND_AFTER_LINES, interval_secs=_SEND_AFTER_SECS): """ @@ -157,8 +200,9 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler): username, password) - self.build_id = build_id + self.endpoint = endpoint self.retry_buffer = [] + self.max_size = None def process_record(self, record): """ @@ -177,12 +221,55 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler): """ Convenience method for subclasses to use when making POST requests. """ - return self.http_handler.post(*args, **kwargs) def _append_logs(self, log_lines): - raise NotImplementedError("_append_logs must be implemented by _BaseBuildloggerHandler" - " subclasses") + """ + Sends a POST request to the handlers endpoint with the logs that have been captured. + + Returns: + The number of log lines that have been successfully sent. + """ + lines_sent = 0 + for chunk in _LogsSplitter.split_logs(log_lines, self.max_size): + chunk_lines_sent = self.__append_logs_chunk(chunk) + lines_sent += chunk_lines_sent + if chunk_lines_sent < len(chunk): + # Not all lines have been sent. We stop here. + break + return lines_sent + + def __append_logs_chunk(self, log_lines_chunk): + """ + Sends a log lines chunk, handles 413 Request Entity Too Large errors and retries + if necessary. + + Returns: + The number of log lines that have been successfully sent. + """ + try: + self.post(self.endpoint, data=log_lines_chunk) + return len(log_lines_chunk) + except urllib2.HTTPError as err: + # Handle the "Request Entity Too Large" error, set the max size and retry. + if err.code == httplib.REQUEST_ENTITY_TOO_LARGE: + response_data = json.load(err, "utf-8") + if isinstance(response_data, dict) and "max_size" in response_data: + new_max_size = response_data["max_size"] + if self.max_size and new_max_size >= self.max_size: + loggers._BUILDLOGGER_FALLBACK.exception( + "Received an HTTP 413 code, but already had max_size set") + return 0 + loggers._BUILDLOGGER_FALLBACK.warning( + "Received an HTTP 413 code, updating the request max_size to %s", + new_max_size) + self.max_size = new_max_size + return self._append_logs(log_lines_chunk) + loggers._BUILDLOGGER_FALLBACK.exception("Encountered an error.") + return 0 + except: + loggers._BUILDLOGGER_FALLBACK.exception("Encountered an error.") + return 0 def _flush_buffer_with_lock(self, buf, close_called): """ @@ -196,9 +283,10 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler): self.retry_buffer.extend(buf) - if self._append_logs(self.retry_buffer): - self.retry_buffer = [] - elif close_called: + nb_sent = self._append_logs(self.retry_buffer) + if nb_sent: + self.retry_buffer = self.retry_buffer[nb_sent:] + if close_called and self.retry_buffer: # Request to the buildlogger server returned an error, so use the fallback logger to # avoid losing the log messages entirely. for (_, message) in self.retry_buffer: @@ -215,29 +303,14 @@ class BuildloggerTestHandler(_BaseBuildloggerHandler): Buildlogger handler for the test logs. """ - def __init__(self, build_id, build_config, test_id, **kwargs): - """ - Initializes the buildlogger handler with the build id, test id, - and credentials. - """ - - _BaseBuildloggerHandler.__init__(self, build_id, build_config, **kwargs) - - self.test_id = test_id - - @_log_on_error - def _append_logs(self, log_lines): - """ - Sends a POST request to the APPEND_TEST_LOGS_ENDPOINT with the - logs that have been captured. - """ + def __init__(self, build_config, build_id, test_id, + capacity=_SEND_AFTER_LINES, interval_secs=_SEND_AFTER_SECS): + """Initializes the buildlogger handler with the credentials, build id, and test id.""" endpoint = APPEND_TEST_LOGS_ENDPOINT % { - "build_id": self.build_id, - "test_id": self.test_id, + "build_id": build_id, + "test_id": test_id, } - - response = self.post(endpoint, data=log_lines) - return response is not None + _BaseBuildloggerHandler.__init__(self, build_config, endpoint, capacity, interval_secs) @_log_on_error def _finish_test(self, failed=False): @@ -245,12 +318,7 @@ class BuildloggerTestHandler(_BaseBuildloggerHandler): Sends a POST request to the APPEND_TEST_LOGS_ENDPOINT with the test status. """ - endpoint = APPEND_TEST_LOGS_ENDPOINT % { - "build_id": self.build_id, - "test_id": self.test_id, - } - - self.post(endpoint, headers={ + self.post(self.endpoint, headers={ "X-Sendlogs-Test-Done": "true", "X-Sendlogs-Test-Failed": "true" if failed else "false", }) @@ -271,12 +339,8 @@ class BuildloggerGlobalHandler(_BaseBuildloggerHandler): Buildlogger handler for the global logs. """ - @_log_on_error - def _append_logs(self, log_lines): - """ - Sends a POST request to the APPEND_GLOBAL_LOGS_ENDPOINT with - the logs that have been captured. - """ - endpoint = APPEND_GLOBAL_LOGS_ENDPOINT % {"build_id": self.build_id} - response = self.post(endpoint, data=log_lines) - return response is not None + def __init__(self, build_config, build_id, + capacity=_SEND_AFTER_LINES, interval_secs=_SEND_AFTER_SECS): + """Initializes the buildlogger handler with the credentials and build id.""" + endpoint = APPEND_GLOBAL_LOGS_ENDPOINT % {"build_id": build_id} + _BaseBuildloggerHandler.__init__(self, build_config, endpoint, capacity, interval_secs) diff --git a/buildscripts/resmokelib/logging/config.py b/buildscripts/resmokelib/logging/config.py index c3960bbafd3..c3216b2d879 100644 --- a/buildscripts/resmokelib/logging/config.py +++ b/buildscripts/resmokelib/logging/config.py @@ -67,8 +67,8 @@ def apply_buildlogger_global_handler(logger, logging_config, build_id=None, buil log_format = logger_info.get("format", _DEFAULT_FORMAT) formatter = formatters.ISO8601Formatter(fmt=log_format) - handler = buildlogger.BuildloggerGlobalHandler(build_id, - build_config, + handler = buildlogger.BuildloggerGlobalHandler(build_config, + build_id, **handler_info) handler.setFormatter(formatter) else: @@ -98,8 +98,8 @@ def apply_buildlogger_test_handler(logger, log_format = logger_info.get("format", _DEFAULT_FORMAT) formatter = formatters.ISO8601Formatter(fmt=log_format) - handler = buildlogger.BuildloggerTestHandler(build_id, - build_config, + handler = buildlogger.BuildloggerTestHandler(build_config, + build_id, test_id, **handler_info) handler.setFormatter(formatter) diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py index b4b0066a5aa..83418502ead 100644 --- a/buildscripts/resmokelib/testing/fixtures/interface.py +++ b/buildscripts/resmokelib/testing/fixtures/interface.py @@ -79,13 +79,22 @@ class Fixture(object): """ return True - def get_connection_string(self): + def get_internal_connection_string(self): """ Returns the connection string for this fixture. This is NOT a driver connection string, but a connection string of the format expected by the mongo::ConnectionString class. """ - raise NotImplementedError("get_connection_string must be implemented by Fixture subclasses") + raise NotImplementedError( + "get_internal_connection_string must be implemented by Fixture subclasses") + + def get_driver_connection_url(self): + """ + Return the mongodb connection string as defined here: + https://docs.mongodb.com/manual/reference/connection-string/ + """ + raise NotImplementedError( + "get_driver_connection_url must be implemented by Fixture subclasses") def __str__(self): return "%s (Job #%d)" % (self.__class__.__name__, self.job_num) diff --git a/buildscripts/resmokelib/testing/fixtures/masterslave.py b/buildscripts/resmokelib/testing/fixtures/masterslave.py index 469c7ac0816..fb444cfe097 100644 --- a/buildscripts/resmokelib/testing/fixtures/masterslave.py +++ b/buildscripts/resmokelib/testing/fixtures/masterslave.py @@ -5,7 +5,6 @@ Master/slave fixture for executing JSTests against. from __future__ import absolute_import import os.path -import socket import pymongo @@ -160,6 +159,12 @@ class MasterSlaveFixture(interface.ReplFixture): mongod_options = self.mongod_options.copy() mongod_options.update(self.slave_options) mongod_options["slave"] = "" - mongod_options["source"] = "%s:%d" % (socket.gethostname(), self.port) + mongod_options["source"] = self.master.get_internal_connection_string() mongod_options["dbpath"] = os.path.join(self._dbpath_prefix, "slave") return self._new_mongod(mongod_logger, mongod_options) + + def get_internal_connection_string(self): + return self.master.get_internal_connection_string() + + def get_driver_connection_url(self): + return self.master.get_driver_connection_url() diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py index 141bbd60ea1..0ec3de30280 100644 --- a/buildscripts/resmokelib/testing/fixtures/replicaset.py +++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py @@ -37,7 +37,8 @@ class ReplicaSetFixture(interface.ReplFixture): write_concern_majority_journal_default=None, auth_options=None, replset_config_options=None, - voting_secondaries=True): + voting_secondaries=False, + use_replica_set_connection_string=False): interface.ReplFixture.__init__(self, logger, job_num) @@ -50,6 +51,7 @@ class ReplicaSetFixture(interface.ReplFixture): self.auth_options = auth_options self.replset_config_options = utils.default_if_none(replset_config_options, {}) self.voting_secondaries = voting_secondaries + self.use_replica_set_connection_string = use_replica_set_connection_string # The dbpath in mongod_options is used as the dbpath prefix for replica set members and # takes precedence over other settings. The ShardedClusterFixture uses this parameter to @@ -97,7 +99,7 @@ class ReplicaSetFixture(interface.ReplFixture): # Initiate the replica set. members = [] for (i, node) in enumerate(self.nodes): - member_info = {"_id": i, "host": node.get_connection_string()} + member_info = {"_id": i, "host": node.get_internal_connection_string()} if i > 0: member_info["priority"] = 0 if i >= 7 or not self.voting_secondaries: @@ -107,7 +109,7 @@ class ReplicaSetFixture(interface.ReplFixture): members.append(member_info) if self.initial_sync_node: members.append({"_id": self.initial_sync_node_idx, - "host": self.initial_sync_node.get_connection_string(), + "host": self.initial_sync_node.get_internal_connection_string(), "priority": 0, "hidden": 1, "votes": 0}) @@ -291,11 +293,27 @@ class ReplicaSetFixture(interface.ReplFixture): return logging.loggers.new_logger(logger_name, parent=self.logger) - def get_connection_string(self): + def get_internal_connection_string(self): if self.replset_name is None: - raise ValueError("Must call setup() before calling get_connection_string()") + raise ValueError("Must call setup() before calling get_internal_connection_string()") - conn_strs = [node.get_connection_string() for node in self.nodes] + conn_strs = [node.get_internal_connection_string() for node in self.nodes] if self.initial_sync_node: - conn_strs.append(self.initial_sync_node.get_connection_string()) + conn_strs.append(self.initial_sync_node.get_internal_connection_string()) return self.replset_name + "/" + ",".join(conn_strs) + + def get_driver_connection_url(self): + if self.replset_name is None: + raise ValueError("Must call setup() before calling get_driver_connection_url()") + + if self.use_replica_set_connection_string: + # We use a replica set connection string when all nodes are electable because we + # anticipate the client will want to gracefully handle any failovers. + conn_strs = [node.get_internal_connection_string() for node in self.nodes] + if self.initial_sync_node: + conn_strs.append(self.initial_sync_node.get_internal_connection_string()) + return "mongodb://" + ",".join(conn_strs) + "/?replicaSet=" + self.replset_name + else: + # We return a direct connection to the expected pimary when only the first node is + # electable because we want the client to error out if a stepdown occurs. + return self.nodes[0].get_driver_connection_url() diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py index ac7e597f24b..2e2db535d6d 100644 --- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py +++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py @@ -170,11 +170,14 @@ class ShardedClusterFixture(interface.Fixture): all(shard.is_running() for shard in self.shards) and self.mongos is not None and self.mongos.is_running()) - def get_connection_string(self): + def get_internal_connection_string(self): if self.mongos is None: - raise ValueError("Must call setup() before calling get_connection_string()") + raise ValueError("Must call setup() before calling get_internal_connection_string()") - return "%s:%d" % (socket.gethostname(), self.mongos.port) + return self.mongos.get_internal_connection_string() + + def get_driver_connection_url(self): + return "mongodb://" + self.get_internal_connection_string() def _new_configsvr(self): """ @@ -229,16 +232,11 @@ class ShardedClusterFixture(interface.Fixture): mongos_logger = logging.loggers.new_logger(logger_name, parent=self.logger) mongos_options = copy.deepcopy(self.mongos_options) - configdb_hostname = socket.gethostname() if self.separate_configsvr: - configdb_replset = ShardedClusterFixture._CONFIGSVR_REPLSET_NAME - configdb_port = self.configsvr.port - mongos_options["configdb"] = "%s/%s:%d" % (configdb_replset, - configdb_hostname, - configdb_port) + mongos_options["configdb"] = self.configsvr.get_internal_connection_string() else: - mongos_options["configdb"] = "%s:%d" % (configdb_hostname, self.shards[0].port) + mongos_options["configdb"] = "localhost:%d" % (self.shards[0].port) return _MongoSFixture(mongos_logger, self.job_num, @@ -254,9 +252,9 @@ class ShardedClusterFixture(interface.Fixture): for more details. """ - hostname = socket.gethostname() - self.logger.info("Adding %s:%d as a shard..." % (hostname, shard.port)) - client.admin.command({"addShard": "%s:%d" % (hostname, shard.port)}) + connection_string = shard.get_internal_connection_string() + self.logger.info("Adding %s as a shard...", connection_string) + client.admin.command({"addShard": connection_string}) class _MongoSFixture(interface.Fixture): @@ -356,3 +354,12 @@ class _MongoSFixture(interface.Fixture): def is_running(self): return self.mongos is not None and self.mongos.poll() is None + + def get_internal_connection_string(self): + if self.mongos is None: + raise ValueError("Must call setup() before calling get_internal_connection_string()") + + return "localhost:%d" % self.port + + def get_driver_connection_url(self): + return "mongodb://" + self.get_internal_connection_string() diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index ba62b3d2b8c..bc69775c285 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -7,7 +7,6 @@ from __future__ import absolute_import import os import os.path import shutil -import socket import time import pymongo @@ -146,8 +145,11 @@ class MongoDFixture(interface.Fixture): def is_running(self): return self.mongod is not None and self.mongod.poll() is None - def get_connection_string(self): + def get_internal_connection_string(self): if self.mongod is None: - raise ValueError("Must call setup() before calling get_connection_string()") + raise ValueError("Must call setup() before calling get_internal_connection_string()") - return "%s:%d" % (socket.gethostname(), self.port) + return "localhost:%d" % self.port + + def get_driver_connection_url(self): + return "mongodb://" + self.get_internal_connection_string() diff --git a/buildscripts/resmokelib/testing/testcases.py b/buildscripts/resmokelib/testing/testcases.py index b4029fc6ea8..21d35215a29 100644 --- a/buildscripts/resmokelib/testing/testcases.py +++ b/buildscripts/resmokelib/testing/testcases.py @@ -187,7 +187,7 @@ class CPPIntegrationTestCase(TestCase): def configure(self, fixture, *args, **kwargs): TestCase.configure(self, fixture, *args, **kwargs) - self.program_options["connectionString"] = self.fixture.get_connection_string() + self.program_options["connectionString"] = self.fixture.get_internal_connection_string() def run_test(self): try: @@ -362,6 +362,19 @@ class JSTestCase(TestCase): # Directory already exists. pass + process_kwargs = self.shell_options.get("process_kwargs", {}).copy() + + if "KRB5_CONFIG" in process_kwargs and "KRB5CCNAME" not in process_kwargs: + # Use a job-specific credential cache for JavaScript tests involving Kerberos. + krb5_dir = os.path.join(data_dir, "krb5") + try: + os.makedirs(krb5_dir) + except os.error: + pass + process_kwargs["KRB5CCNAME"] = "DIR:" + os.path.join(krb5_dir, ".") + + self.shell_options["process_kwargs"] = process_kwargs + def _get_data_dir(self, global_vars): """ Returns the value that the mongo shell should set for the @@ -408,11 +421,13 @@ class JSTestCase(TestCase): is_main_test = True if thread_id > 0: is_main_test = False - return core.programs.mongo_shell_program(logger, - executable=self.shell_executable, - filename=self.js_filename, - isMainTest=is_main_test, - **self.shell_options) + return core.programs.mongo_shell_program( + logger, + executable=self.shell_executable, + filename=self.js_filename, + connection_string=self.fixture.get_driver_connection_url(), + isMainTest=is_main_test, + **self.shell_options) def _run_test_in_thread(self, thread_id): # Make a logger for each thread. diff --git a/buildscripts/scons.py b/buildscripts/scons.py index e7860162eb6..b0b9cfa834e 100755 --- a/buildscripts/scons.py +++ b/buildscripts/scons.py @@ -1,16 +1,26 @@ -#!/usr/bin/python +#!/usr/bin/env python2 + +from __future__ import print_function import os -import subprocess import sys SCONS_VERSION = os.environ.get('SCONS_VERSION', "2.5.0") -mongodb_root = os.path.dirname(os.path.dirname(__file__)) -scons_dir = os.path.join(mongodb_root, 'src', 'third_party', 'scons-' + SCONS_VERSION) +mongodb_root = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) +scons_dir = os.path.join(mongodb_root, 'src', 'third_party','scons-' + SCONS_VERSION, + 'scons-local-' + SCONS_VERSION) + +if not os.path.exists(scons_dir): + print("Could not find SCons in '%s'" % (scons_dir)) + sys.exit(1) + +sys.path = [scons_dir] + sys.path + +try: + import SCons.Script +except ImportError: + print("Could not find SCons in '%s'" % (scons_dir)) + sys.exit(1) -if sys.platform == 'win32': - args = [sys.executable, os.path.join(scons_dir, 'scons.py')] + sys.argv[1:] - sys.exit(subprocess.call(args)) -else: - os.execv(os.path.join(scons_dir, 'scons.py'), sys.argv) +SCons.Script.main() diff --git a/buildscripts/tests/resmokelib/logging/test_buildlogger.py b/buildscripts/tests/resmokelib/logging/test_buildlogger.py new file mode 100644 index 00000000000..7bc705948db --- /dev/null +++ b/buildscripts/tests/resmokelib/logging/test_buildlogger.py @@ -0,0 +1,67 @@ +"""Unit tests for the buildscripts.resmokelib.logging.buildlogger module.""" + +from __future__ import absolute_import + +import json +import unittest + +from buildscripts.resmokelib.logging import buildlogger + + +class TestLogsSplitter(unittest.TestCase): + """Unit tests for the _LogsSplitter class.""" + + def test_split_no_logs(self): + logs = [] + max_size = 10 + self.assertEqual([[]], buildlogger._LogsSplitter.split_logs(logs, max_size)) + + def test_split_no_max_size(self): + logs = self.__generate_logs(size=30) + max_size = None + self.assertEqual([logs], buildlogger._LogsSplitter.split_logs(logs, max_size)) + + def test_split_max_size_smaller(self): + logs = self.__generate_logs(size=20) + max_size = 30 + self.assertEqual([logs], buildlogger._LogsSplitter.split_logs(logs, max_size)) + + def test_split_max_size_equal(self): + logs = self.__generate_logs(size=30) + max_size = 30 + self.assertEqual([logs], buildlogger._LogsSplitter.split_logs(logs, max_size)) + + def test_split_max_size_larger(self): + logs = self.__generate_logs(size=31) + max_size = 30 + self.assertEqual( + [logs[0:-1], logs[-1:]], + buildlogger._LogsSplitter.split_logs(logs, max_size)) + + logs = self.__generate_logs(size=149) + max_size = 19 + self.assertEqual( + [logs[0:3], logs[3:6], logs[6:9], logs[9:12], logs[12:15], + logs[15:18], logs[18:21], logs[21:24], logs[24:27], logs[27:]], + buildlogger._LogsSplitter.split_logs(logs, max_size)) + + def check_split_sizes(self, splits, max_size): + for split in splits: + self.assertTrue(TestLogsSplitter.size(split) <= max_size) + + def __generate_logs(self, size): + # The size of [ "x" ] is 5. This is the minimum size we generate. + self.assertTrue(size >= 5) + # Each new "x" adds 5 to the size. + nb_lines = size / 5 + # Each additional "x" on a line adds 1 to the size. + last_line_extra = size % 5 + logs = ["x"] * nb_lines + logs[-1] += "x" * last_line_extra + self.assertEqual(size, TestLogsSplitter.size(logs)) + return logs + + @staticmethod + def size(logs): + """Returns the size of the log lines when represented in JSON.""" + return len(json.dumps(logs, encoding="utf-8")) diff --git a/distsrc/README b/distsrc/README index 8cc83742981..5a772b66e7d 100644 --- a/distsrc/README +++ b/distsrc/README @@ -1,22 +1,30 @@ MongoDB README -Welcome to MongoDB! - +Welcome to MongoDB 3.4! + COMPONENTS - bin/mongod - The database process. - bin/mongos - Sharding controller. - bin/mongo - The database shell (uses interactive javascript). - -UTILITIES - - bin/mongodump - MongoDB dump tool - for backups, snapshots, etc.. - bin/mongorestore - MongoDB restore a dump - bin/mongoexport - Export a single collection to test (JSON, CSV) - bin/mongoimport - Import from JSON or CSV - bin/mongofiles - Utility for putting and getting files from MongoDB GridFS - bin/mongostat - Show performance statistics - + mongod - The database server. + mongos - Sharding router. + mongo - The database shell (uses interactive javascript). + +UTILITIES + + mongodump - Create a binary dump of the contents of a database. + mongorestore - Restore data from the output created by mongodump. + mongoexport - Export the contents of a collection to JSON or CSV. + mongoimport - Import data from JSON, CSV or TSV. + mongofiles - Put, get and delete files from GridFS. + mongostat - Show the status of a running mongod/mongos. + bsondump - Convert BSON files into human-readable formats. + mongooplog - Poll the oplog and apply to a local server. + mongoreplay - Traffic capture and replay tool. + mongotop - Track time spent reading and writing data. + +BUILDING + + See docs/building.md. + RUNNING For command line options invoke: @@ -25,28 +33,52 @@ RUNNING To run a single server database: - $ mkdir /data/db + $ sudo mkdir -p /data/db $ ./mongod $ $ # The mongo javascript shell connects to localhost and test database by default: - $ ./mongo + $ ./mongo > help DRIVERS - Client drivers for most programming languages are available at mongodb.org. Use the - shell ("mongo") for administrative tasks. + Client drivers for most programming languages are available at + https://docs.mongodb.com/manual/applications/drivers/. Use the shell + ("mongo") for administrative tasks. + +BUG REPORTS + + See https://github.com/mongodb/mongo/wiki/Submit-Bug-Reports. + +PACKAGING + + Packages are created dynamically by the package.py script located in the + buildscripts directory. This will generate RPM and Debian packages. DOCUMENTATION - http://www.mongodb.org/ - -MAIL LISTS AND IRC - - http://dochub.mongodb.org/core/community + https://docs.mongodb.com/manual/ + +CLOUD HOSTED MONGODB + + https://www.mongodb.com/cloud/atlas + +MAIL LISTS + + https://groups.google.com/forum/#!forum/mongodb-user + + A forum for technical questions about using MongoDB. + + https://groups.google.com/forum/#!forum/mongodb-dev + + A forum for technical questions about building and developing MongoDB. + +LEARN MONGODB + + https://university.mongodb.com/ -32 BIT BUILD NOTES +LICENSE - MongoDB uses memory mapped files. If built as a 32 bit executable, you will - not be able to work with large (multi-gigabyte) databases. However, 32 bit - builds work fine with small development databases. + Most MongoDB source files (src/mongo folder and below) are made available + under the terms of the GNU Affero General Public License (GNU AGPLv3). See + individual files for details. diff --git a/docs/building.md b/docs/building.md index 6008a9b606f..48f1a188396 100644 --- a/docs/building.md +++ b/docs/building.md @@ -4,7 +4,7 @@ Building MongoDB To build MongoDB, you will need: * A modern C++ compiler. One of the following is required. - * GCC 4.8.2 or newer + * GCC 5.3.0 or newer * Clang 3.4 (or Apple XCode 5.1.1 Clang) or newer * Visual Studio 2013 Update 2 or newer * Python 2.7 diff --git a/etc/drivers_nightly.yml b/etc/drivers_nightly.yml index f039112ea85..464f844ac57 100644 --- a/etc/drivers_nightly.yml +++ b/etc/drivers_nightly.yml @@ -3,7 +3,7 @@ ##################################################### ### Instructions for Adding a Driver ### -# The Major sections are: +## The Major sections are: # VARIABLES # FUNCTIONS (MongoDB and Drivers) @@ -218,8 +218,9 @@ functions: - command: shell.exec type: test params: - working_dir: "src/${DRIVER_WORKING_DIRECTORY}" + working_dir: src script: | + cd ${DRIVER_WORKING_DIRECTORY} ${PREPARE_SHELL} echo $DRIVERS_TOOLS echo $MONGO_ORCHESTRATION_HOME @@ -318,8 +319,9 @@ functions: "setup drivers environment": - command: shell.exec params: - working_dir: "src/${DRIVER_WORKING_DIRECTORY}" + working_dir: src script: | + cd "${DRIVER_WORKING_DIRECTORY}" # Get the current unique version of this checkout if [ "${is_patch}" = "true" ]; then CURRENT_VERSION=$(git describe)-patch-${version_id} @@ -433,10 +435,7 @@ tasks: if [ `which strip` ]; then echo "found strip" find . -maxdepth 1 -type f -iname "mongo*" -exec strip {} \; - strip mongo/mongo fi - # On RHEL, the mongo shell binary is in a mongo directory - mv mongo/mongo ./mongodb-binaries/bin/ || true find . -maxdepth 1 -type f -iname "mongo*" -exec mv {} ./mongodb-binaries/bin/ \; ${compress} mongodb-binaries.${ext|tgz} mongodb-binaries/ @@ -584,7 +583,6 @@ axes: display_name: "Python 2.6" variables: ### There is no python toolchain on RHEL 6.2, using system python - ### PYTHON_BINARY: "/opt/python/2.6/bin/python" PYTHON_BINARY: "/usr/bin/python" DRIVER_WORKING_DIRECTORY: "mongo-python-driver" DRIVER_TEST_UPLOAD_DIRECTORY: "src/mongo-python-driver/xunit-results/TEST-*.xml" @@ -629,7 +627,7 @@ modules: - name: mongo-python-driver repo: git@github.com:mongodb/mongo-python-driver.git - branch: evergreen + branch: master ####################################### # BUILDVARIANTS # diff --git a/etc/evergreen.yml b/etc/evergreen.yml index fae0563fbde..520a49b77f5 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -113,7 +113,7 @@ variables: - enterprise-debian71-64 - enterprise-debian81-64 - enterprise-linux-64-amazon-ami - - enterprise-osx-107 + - enterprise-osx-108 - enterprise-rhel-62-64-bit - enterprise-rhel-62-64-bit-coverage - enterprise-rhel-62-64-bit-inmem @@ -145,17 +145,15 @@ variables: - linux-64-ephemeralForTest - linux-64-lsm - linux-64-repeated-execution - - osx-107 - - osx-107-debug - - osx-107-ssl + - osx-108 + - osx-108-debug + - osx-108-ssl - rhel62 - rhel70 - - solaris-64-bit - suse11 - suse12 - ubuntu1204 - ubuntu1404 - - ubuntu1404-rocksdb - ubuntu1604 - ubuntu1604-arm64 - ubuntu1604-asan @@ -238,9 +236,6 @@ functions: if [ "Windows_NT" = "$OS" ]; then typeperf -qx PhysicalDisk | grep Disk | grep -v _Total > disk_counters.txt typeperf -cf disk_counters.txt -si 5 -o mongo-diskstats - # Solaris: iostat -T d option for timestamp. - elif iostat -T d -Mx > /dev/null 2>&1; then - iostat -T d -Mx 5 > mongo-diskstats # Linux: iostat -t option for timestamp. elif iostat -tdmx > /dev/null 2>&1; then iostat -tdmx 5 > mongo-diskstats @@ -285,11 +280,8 @@ functions: get_pids() { proc_pids=$(pgrep $1); } get_process_info() { proc_name=$(ps -p $1 -o comm=); - # prstat is available on Solaris - if [ ! -z $(which prstat 2> /dev/null) ]; then - proc_threads=$(prstat -p $1 1 1 | grep $1 | cut -f2 -d "/"); # /proc is available on Linux platforms - elif [ -f /proc/$1/status ]; then + if [ -f /proc/$1/status ]; then ${set_sudo} proc_threads=$($sudo grep Threads /proc/$1/status | sed "s/\s//g" | cut -f2 -d ":"); else @@ -365,19 +357,6 @@ functions: mv mongodb*/mongo.{debug,dSYM,pdb} mongodb*/mongod.{debug,dSYM,pdb} mongodb*/mongos.{debug,dSYM,pdb} . 2>/dev/null || true rm -r mongodb*/*.{debug,dSYM,pdb} mongo-debugsymbols.tgz 2>/dev/null || true - "build rocksdb" : - command: shell.exec - params: - script: | - set -o errexit - set -o verbose - if [ "${build_rocksdb|}" = "true" ]; then - rm -rf rocksdb - git clone https://github.com/facebook/rocksdb.git - cd rocksdb - make CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ static_lib - fi - "build new tools" : command: shell.exec params: @@ -441,6 +420,13 @@ functions: set -o errexit set -o verbose + # Set the TMPDIR environment variable to be a directory in the task's working + # directory so that temporary files created by processes spawned by resmoke.py get + # cleaned up after the task completes. This also ensures the spawned processes + # aren't impacted by limited space in the mount point for the /tmp directory. + export TMPDIR="${workdir}/tmp" + mkdir -p $TMPDIR + # check if virtualenv is set up if [ -d "venv" ]; then if [ "Windows_NT" = "$OS" ]; then @@ -524,7 +510,7 @@ functions: path_value="$path_value:${task_path_suffix}" fi - ${resmoke_wrapper} PATH="$path_value" ${san_symbolizer} ${san_options} ${rlp_environment} ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/resmoke.py ${resmoke_args} $extra_args ${test_flags} --log=buildlogger --reportFile=report.json + ${resmoke_wrapper} PATH="$path_value" ${san_symbolizer} ${lang_environment} ${san_options} ${rlp_environment} ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/resmoke.py ${resmoke_args} $extra_args ${test_flags} --log=buildlogger --reportFile=report.json "do jepsen setup" : @@ -852,10 +838,18 @@ functions: script: | set -o errexit + # Override the aws credentials with the kitchen specific credentials + cat < ~/.aws/credentials + [default] + aws_access_key_id = ${kitchen_aws_key} + aws_secret_access_key = ${kitchen_aws_secret} + EOF + export KITCHEN_ARTIFACTS_URL="https://s3.amazonaws.com/mciuploads/${project}/${build_variant}/${revision}/artifacts/${build_id}.tgz" export KITCHEN_SECURITY_GROUP="${kitchen_security_group}" export KITCHEN_SSH_KEY_ID="${kitchen_ssh_key_id}" export KITCHEN_SUBNET="${kitchen_subnet}" + export KITCHEN_VPC="${kitchen_vpc}" for i in {1..3} do @@ -1125,6 +1119,12 @@ post: if [ ! -f $new_binary_file ]; then mv $binary_file_location $new_binary_file fi + # On Windows if a .pdb symbol file exists, include it in the archive. + pdb_file=$(echo $binary_file_location | sed "s/\.exe/.pdb/") + if [ -f $pdb_file ]; then + new_pdb_file=unittest_binaries/$(echo $pdb_file | sed "s/.*\///") + mv $pdb_file $new_pdb_file + fi done done - command: archive.targz_pack @@ -1239,14 +1239,12 @@ tasks: directory: src revisions: # for each module include revision as : ${_rev} enterprise: ${enterprise_rev} - rocksdb: ${rocksdb_rev} - command: git.apply_patch params: directory: src - func: "get buildnumber" - func: "setup credentials" - func: "build new tools" # noop if ${newtools} is not "true" - - func: "build rocksdb" # noop if ${build_rocksdb} is not "true" - command: shell.exec params: working_dir: src @@ -1295,14 +1293,6 @@ tasks: ${strip_command|/usr/bin/strip} mongobridge fi - # If this is a scheduled build, we check for changes against the last scheduled commit. - if [ "${is_patch}" != "true" ]; then - burn_in_args="--checkEvergreen" - fi - - # Capture a list of new and modified tests. - ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/burn_in_tests.py --branch=${branch_name} --buildVariant=${build_variant} --testListOutfile=jstests/new_tests.json --noExec $burn_in_args - # On windows we need to make sure the paths in unittests.txt are compatible with cygwin tar sed 's|\\|/|g' build/unittests.txt > build/unittests-tarlist.txt ${tar|tar} -czvf mongodb-unittests.tgz -T build/unittests-tarlist.txt build/unittests.txt @@ -1421,7 +1411,6 @@ tasks: directory: src revisions: # for each module include revision as : ${_rev} enterprise: ${enterprise_rev} - rocksdb: ${rocksdb_rev} - command: git.apply_patch params: directory: src @@ -1440,7 +1429,38 @@ tasks: depends_on: - name: compile commands: + - command: git.get_project + # The repository is cloned in a directory distinct from src for the modified test detection + # because the extraction of the artifacts performed in the 'do setup' causes + # 'git diff --name-only' to see all tests as modified on Windows (git 1.9.5). See SERVER-30634. + params: + directory: burn_in_tests_clonedir + revisions: # for each module include revision as : ${_rev} + enterprise: ${enterprise_rev} + - command: shell.exec - func: "do setup" + - func: "set up virtualenv" + - command: shell.exec + params: + working_dir: burn_in_tests_clonedir + script: | + set -o errexit + # Create a symbolic link to the venv in the src directory so activate_virtualenv can use it + # if it exists. + ln -s ../src/venv venv + ${activate_virtualenv} + set -o verbose + # If this is a scheduled build, we check for changes against the last scheduled commit. + if [ "${is_patch}" != "true" ]; then + burn_in_args="--checkEvergreen" + fi + # Copy the dbtest executable from the src dir because burn_in_tests.py calls it to get the + # list of dbtest suites. + cp ../src/dbtest${exe} . + # Capture a list of new and modified tests. + ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/burn_in_tests.py --branch=${branch_name} --buildVariant=${build_variant} --testListOutfile=jstests/new_tests.json --noExec $burn_in_args + # Copy the results to the src dir. + cp jstests/new_tests.json ../src/jstests/new_tests.json - func: "do multiversion setup" - func: "run tests" vars: @@ -2716,6 +2736,17 @@ tasks: resmoke_args: --suites=read_concern_majority_passthrough --storageEngine=wiredTiger run_multiple_jobs: true +- <<: *task_template + name: write_concern_majority_passthrough_WT + depends_on: + - name: jsCore_WT + commands: + - func: "do setup" + - func: "run tests" + vars: + resmoke_args: --suites=write_concern_majority_passthrough --storageEngine=wiredTiger + run_multiple_jobs: true + - <<: *task_template name: replica_sets commands: @@ -3097,7 +3128,7 @@ tasks: /usr/local/bin/notary-client.py --key-name "server-3.4" --auth-token-file ${workdir}/src/signing_auth_token --comment "Evergreen Automatic Signing ${revision} - ${build_variant} - ${branch_name}" --notary-url http://notary-service.build.10gen.cc:5000 --skip-missing mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz} mongodb-shell-${push_name}-${push_arch}-${suffix}.${ext|tgz} mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz} mongodb-win32-${push_arch}-${suffix}.msi mongodb-src-${src_suffix}.${ext|tar.gz} if [ "${has_packages|}" = "true" ]; then - CURATOR_RELEASE="88f34a9f1c79db7ea9597b6e85eb5995d03714ba" + CURATOR_RELEASE="ea8d75dcc1a587111e7418e2428fb67e267af9fe" curl -L -O http://boxes.10gen.com/build/curator/curator-dist-rhel70-$CURATOR_RELEASE.tar.gz tar -zxvf curator-dist-rhel70-$CURATOR_RELEASE.tar.gz ./curator repo --config ./etc/repo_config.yaml --distro ${packager_distro} --edition ${repo_edition} --version ${version} --arch ${packager_arch} --packages repo @@ -3513,11 +3544,6 @@ modules: prefix: src/mongo/db/modules branch: v3.4 -- name: rocksdb - repo: git@github.com:mongodb-partners/mongo-rocks.git - prefix: src/mongo/db/modules - branch: v3.4 - ####################################### # Buildvariants # ####################################### @@ -3650,6 +3676,7 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: write_concern_majority_passthrough_WT - name: push - name: linux-64-repeated-execution @@ -3767,6 +3794,7 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: write_concern_majority_passthrough_WT - name: linux-64-duroff display_name: Linux (No Journal) @@ -4159,6 +4187,7 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: linux push_arch: x86_64-ubuntu1604 + lang_environment: LANG=C compile_flags: --ssl MONGO_DISTMOD=ubuntu1604 -j$(grep -c ^processor /proc/cpuinfo) --release CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy multiversion_platform_arch: "ubuntu1604" multiversion_edition: "targeted" @@ -4283,6 +4312,7 @@ buildvariants: push_arch: arm64-enterprise-ubuntu1604 compile_flags: --ssl MONGO_DISTMOD=ubuntu1604 -j$(grep -c ^processor /proc/cpuinfo) CCFLAGS="-march=armv8-a+crc -mtune=generic" --release CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy num_jobs_available: $(grep -c ^processor /proc/cpuinfo) + max_jobs: 8 # Avoid starting too many mongod's on ARM test servers variant_excluded_flags: requires_mmapv1 has_packages: true packager_script: packager-enterprise.py @@ -4370,6 +4400,7 @@ buildvariants: push_arch: arm64-ubuntu1604 compile_flags: --ssl MONGO_DISTMOD=ubuntu1604 -j$(grep -c ^processor /proc/cpuinfo) --release CCFLAGS="-march=armv8-a+crc -mtune=generic" CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy num_jobs_available: $(grep -c ^processor /proc/cpuinfo) + max_jobs: 8 # Avoid starting too many mongod's on ARM test servers variant_excluded_flags: requires_mmapv1 has_packages: true packager_script: packager.py @@ -4381,62 +4412,8 @@ buildvariants: - name: compile distros: - ubuntu1604-arm64-large - - name: aggregation_WT - - name: aggregation_auth - - name: auth_WT - name: dbtest_WT - - name: failpoints - - name: failpoints_auth - - name: gle_auth_WT - - name: gle_auth_write_cmd_WT - - name: gle_auth_basics_passthrough_WT - - name: gle_auth_basics_passthrough_write_cmd_WT - - name: sharding_gle_auth_basics_passthrough_WT - - name: sharding_gle_auth_basics_passthrough_write_cmd_WT - name: jsCore_WT - - name: jsCore_compatibility_WT - - name: jsCore_decimal_WT - - name: jstestfuzz_WT - - name: jstestfuzz_concurrent_WT - - name: jstestfuzz_concurrent_replication_WT - - name: jstestfuzz_concurrent_sharded_WT - - name: jstestfuzz_replication_WT - - name: jstestfuzz_sharded_WT - - name: mongosTest - - name: noPassthrough_WT - - name: noPassthroughWithMongod_WT - - name: bulk_gle_passthrough_WT - - name: parallel_WT - - name: parallel_compatibility_WT - - name: concurrency_WT - distros: - # There is a known performance issue with the ubuntu1604-arm64-1.linaro.build.10gen.cc host - # that's part of the ubuntu1604-arm64-large pool. We run the concurrency tests only on the - # hosts from the ubuntu1604-arm64-small pool to work around this issue. - - ubuntu1604-arm64-small - - name: concurrency_replication_WT - distros: - - ubuntu1604-arm64-small - - name: concurrency_sharded_WT - distros: - - ubuntu1604-arm64-small - - name: concurrency_simultaneous_WT - distros: - - ubuntu1604-arm64-small - - name: replica_sets_WT - - name: replica_sets_auth - - name: replica_sets_jscore_passthrough_WT - - name: master_slave_WT - - name: master_slave_auth - - name: master_slave_jscore_passthrough_WT - - name: sharding_WT - - name: sharding_auth - - name: slow1_WT - - name: serial_run_WT - - name: sharding_jscore_passthrough_WT - - name: ssl - - name: sslSpecial - - name: tool_WT - name: unittests - name: push distros: @@ -4529,6 +4506,7 @@ buildvariants: - ubuntu1604-zseries-large - ubuntu1604-zseries-small batchtime: 1440 # 1 day + stepback: false expansions: gorootvars: PATH=/opt/mongodbtoolchain/v2/bin:$PATH tooltags: -gccgoflags "$(pkg-config --libs --cflags libssl libcrypto libsasl2)" -tags 'sasl ssl' @@ -4536,7 +4514,7 @@ buildvariants: push_bucket: downloads.10gen.com push_name: linux push_arch: s390x-enterprise-ubuntu1604 - compile_flags: --ssl MONGO_DISTMOD=ubuntu1604 --release -j$(grep -c ^processor /proc/cpuinfo) CCFLAGS="-march=z196 -mtune=zEC12" CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy + compile_flags: --ssl MONGO_DISTMOD=ubuntu1604 --release -j3 CCFLAGS="-march=z196 -mtune=zEC12" CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy num_jobs_available: 2 variant_excluded_flags: requires_mmapv1 has_packages: true @@ -4621,6 +4599,7 @@ buildvariants: - name: sslSpecial - name: tool_WT - name: unittests + - name: write_concern_majority_passthrough_WT - name: push distros: - ubuntu1604-test @@ -5055,6 +5034,7 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: write_concern_majority_passthrough_WT - name: push distros: - rhel70-small @@ -5194,6 +5174,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: write_concern_majority_passthrough_WT + distros: + - windows-64-vs2015-large - name: enterprise-windows-64 display_name: "* Enterprise Windows" @@ -5447,8 +5430,8 @@ buildvariants: # OSX buildvariants # ########################################### -- name: osx-107 - display_name: OS X 10.7 +- name: osx-108 + display_name: OS X 10.8 run_on: - macos-1012 batchtime: 1440 # 1 day @@ -5457,8 +5440,8 @@ buildvariants: push_bucket: downloads.mongodb.org push_name: osx push_arch: x86_64 - gorootvars: CGO_CFLAGS=-mmacosx-version-min=10.7 CGO_LDFLAGS=-mmacosx-version-min=10.7 - compile_flags: --allocator=system -j$(sysctl -n hw.logicalcpu) --release --osx-version-min=10.7 --libc++ + gorootvars: CGO_CFLAGS=-mmacosx-version-min=10.8 CGO_LDFLAGS=-mmacosx-version-min=10.8 + compile_flags: --allocator=system -j$(sysctl -n hw.logicalcpu) --release --osx-version-min=10.8 --libc++ python: python2.7 num_jobs_available: 1 build_mongoreplay: true @@ -5547,8 +5530,8 @@ buildvariants: distros: - rhel70-small -- name: osx-107-ssl - display_name: SSL OS X 10.7 +- name: osx-108-ssl + display_name: SSL OS X 10.8 run_on: - macos-1012 batchtime: 1440 # 1 day @@ -5558,8 +5541,8 @@ buildvariants: push_name: osx-ssl push_arch: x86_64 tooltags: "-tags ssl" - gorootvars: CGO_CPPFLAGS=-I/opt/mongodbtoolchain/v2/include CGO_CFLAGS=-mmacosx-version-min=10.7 CGO_LDFLAGS=-mmacosx-version-min=10.7 - compile_flags: --ssl --allocator=system -j$(sysctl -n hw.logicalcpu) --release --osx-version-min=10.7 --libc++ CPPPATH=/opt/mongodbtoolchain/v2/include + gorootvars: CGO_CPPFLAGS=-I/opt/mongodbtoolchain/v2/include CGO_CFLAGS=-mmacosx-version-min=10.8 CGO_LDFLAGS=-mmacosx-version-min=10.8 + compile_flags: --ssl --allocator=system -j$(sysctl -n hw.logicalcpu) --release --osx-version-min=10.8 --libc++ CPPPATH=/opt/mongodbtoolchain/v2/include python: python2.7 num_jobs_available: 1 build_mongoreplay: true @@ -5644,12 +5627,13 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: write_concern_majority_passthrough_WT - name: push distros: - rhel70-small -- name: osx-107-debug - display_name: "* OS X 10.7 DEBUG" +- name: osx-108-debug + display_name: "* OS X 10.8 DEBUG" run_on: - macos-1012 expansions: @@ -5658,8 +5642,8 @@ buildvariants: push_name: osx-debug push_arch: x86_64 num_jobs_available: 1 - gorootvars: CGO_CFLAGS=-mmacosx-version-min=10.7 CGO_LDFLAGS=-mmacosx-version-min=10.7 - compile_flags: --dbg=on --opt=on --allocator=system -j$(sysctl -n hw.logicalcpu) --osx-version-min=10.7 --libc++ + gorootvars: CGO_CFLAGS=-mmacosx-version-min=10.8 CGO_LDFLAGS=-mmacosx-version-min=10.8 + compile_flags: --dbg=on --opt=on --allocator=system -j$(sysctl -n hw.logicalcpu) --osx-version-min=10.8 --libc++ python: python2.7 build_mongoreplay: true tasks: @@ -5689,8 +5673,8 @@ buildvariants: - name: tool_WT - name: unittests -- name: enterprise-osx-107 - display_name: Enterprise OS X 10.7 +- name: enterprise-osx-108 + display_name: Enterprise OS X 10.8 modules: - enterprise run_on: @@ -5702,8 +5686,8 @@ buildvariants: push_name: osx push_arch: x86_64-enterprise tooltags: "-tags 'ssl sasl'" - gorootvars: CGO_CPPFLAGS=-I/opt/mongodbtoolchain/v2/include CGO_CFLAGS=-mmacosx-version-min=10.7 CGO_LDFLAGS=-mmacosx-version-min=10.7 - compile_flags: --ssl --allocator=system -j$(sysctl -n hw.logicalcpu) --release --osx-version-min=10.7 --libc++ CPPPATH=/opt/mongodbtoolchain/v2/include + gorootvars: CGO_CPPFLAGS=-I/opt/mongodbtoolchain/v2/include CGO_CFLAGS=-mmacosx-version-min=10.8 CGO_LDFLAGS=-mmacosx-version-min=10.8 + compile_flags: --ssl --allocator=system -j$(sysctl -n hw.logicalcpu) --release --osx-version-min=10.8 --libc++ CPPPATH=/opt/mongodbtoolchain/v2/include python: python2.7 num_jobs_available: 1 build_mongoreplay: true @@ -5961,6 +5945,9 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: write_concern_majority_passthrough_WT + distros: + - rhel62-large - name: package distros: - ubuntu1604-packer @@ -6113,6 +6100,7 @@ buildvariants: - name: ssl - name: sslSpecial - name: unittests + - name: write_concern_majority_passthrough_WT - name: enterprise-rhel-70-64-bit display_name: Enterprise RHEL 7.0 @@ -6526,6 +6514,7 @@ buildvariants: - name: sslSpecial - name: tool_WT - name: unittests + - name: write_concern_majority_passthrough_WT - name: push distros: - rhel70-small @@ -6628,6 +6617,7 @@ buildvariants: - name: sslSpecial - name: tool_WT - name: unittests + - name: write_concern_majority_passthrough_WT - name: push distros: - rhel70-small @@ -6763,6 +6753,7 @@ buildvariants: # - name: tool - name: tool_WT - name: unittests + - name: write_concern_majority_passthrough_WT - name: push distros: - rhel62-small @@ -6921,6 +6912,7 @@ buildvariants: push_path: linux push_bucket: downloads.10gen.com push_name: linux + lang_environment: LANG=C push_arch: x86_64-enterprise-ubuntu1604 compile_flags: --ssl MONGO_DISTMOD=ubuntu1604 --release -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy num_jobs_available: $(grep -c ^processor /proc/cpuinfo) @@ -7053,6 +7045,7 @@ buildvariants: - suse12-zseries-build - suse12-zseries-test batchtime: 1440 # 1 day + stepback: false expansions: gorootvars: PATH=/opt/mongodbtoolchain/v2/bin:$PATH tooltags: -gccgoflags "$(pkg-config --libs --cflags libssl libsasl2)" -tags 'sasl ssl' @@ -7060,7 +7053,7 @@ buildvariants: push_bucket: downloads.10gen.com push_name: linux push_arch: s390x-enterprise-suse12 - compile_flags: --ssl MONGO_DISTMOD=suse12 --release -j$(( $(grep -c ^processor /proc/cpuinfo) / 2 )) CCFLAGS="-march=z196 -mtune=zEC12" CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy + compile_flags: --ssl MONGO_DISTMOD=suse12 --release -j3 CCFLAGS="-march=z196 -mtune=zEC12" CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy num_jobs_available: $(grep -c ^processor /proc/cpuinfo) variant_excluded_flags: requires_mmapv1 has_packages: true @@ -7142,6 +7135,7 @@ buildvariants: - name: sslSpecial - name: tool_WT - name: unittests + - name: write_concern_majority_passthrough_WT - name: push distros: - suse12-test @@ -7327,6 +7321,7 @@ buildvariants: run_on: - suse11-zseries-build batchtime: 1440 # 1 day + stepback: false expansions: gorootvars: PATH=/opt/mongodbtoolchain/v2/bin:$PATH tooltags: -gccgoflags "$(pkg-config --libs --cflags libssl libsasl2)" -tags 'sasl ssl' @@ -7334,7 +7329,7 @@ buildvariants: push_bucket: downloads.10gen.com push_name: linux push_arch: s390x-enterprise-suse11 - compile_flags: --ssl MONGO_DISTMOD=suse11 --release -j$(grep -c ^processor /proc/cpuinfo) CCFLAGS="-march=z9-109 -mtune=z10" CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy --use-s390x-crc32=off + compile_flags: --ssl MONGO_DISTMOD=suse11 --release -j3 CCFLAGS="-march=z9-109 -mtune=z10" CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy --use-s390x-crc32=off num_jobs_available: $(grep -c ^processor /proc/cpuinfo) variant_excluded_flags: requires_mmapv1 has_packages: true @@ -7413,6 +7408,7 @@ buildvariants: - name: sslSpecial - name: tool_WT - name: unittests + - name: write_concern_majority_passthrough_WT - name: push distros: - suse11-test @@ -7534,94 +7530,6 @@ buildvariants: - name: push -########################################### -# Solaris buildvariants # -########################################### - -- name: solaris-64-bit - display_name: "* Solaris" - run_on: - - solaris - expansions: - push_path: sunos5 - push_bucket: downloads.mongodb.org - push_name: sunos5 - push_arch: x86_64 - gorootvars: PATH=/opt/mongodbtoolchain/v2/bin:$PATH - tooltags: -gccgoflags "-lsocket -lnsl" - compile_flags: CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy -j$(kstat cpu | sort -u | grep -c "^module") --release CCFLAGS="-m64" LINKFLAGS="-m64 -static-libstdc++ -static-libgcc" OBJCOPY=/opt/mongodbtoolchain/bin/objcopy - num_jobs_available: $(( $(kstat cpu | sort -u | grep -c "^module") / 2 )) - use_scons_cache: true - tasks: - - name: compile - - name: aggregation - - name: aggregation_WT - - name: auth - - name: auth_WT - - name: bulk_gle_passthrough - - name: bulk_gle_passthrough_WT - - name: concurrency - - name: concurrency_WT - - name: concurrency_replication - - name: concurrency_replication_WT - - name: concurrency_sharded - - name: concurrency_sharded_WT - - name: concurrency_simultaneous - - name: concurrency_simultaneous_WT - - name: dbtest - - name: dbtest_WT - - name: disk - - name: durability - - name: failpoints - - name: httpinterface - - name: jsCore - - name: jsCore_compatibility - - name: jsCore_compatibility_WT - - name: jsCore_decimal - - name: jsCore_decimal_WT - - name: jsCore_WT - - name: jstestfuzz - - name: jstestfuzz_WT - - name: jstestfuzz_replication - - name: jstestfuzz_replication_WT - - name: jstestfuzz_sharded - - name: jstestfuzz_sharded_WT - - name: mmap - - name: mongosTest - - name: noPassthrough - - name: noPassthroughWithMongod - - name: noPassthroughWithMongod_WT - - name: noPassthrough_WT - - name: parallel - - name: parallel_compatibility - - name: parallel_compatibility_WT - - name: parallel_WT - - name: replica_sets - - name: replica_sets_WT - - name: replica_sets_jscore_passthrough - - name: replica_sets_jscore_passthrough_WT - - name: master_slave - - name: master_slave_WT - - name: master_slave_jscore_passthrough - - name: master_slave_jscore_passthrough_WT - - name: sharding - - name: sharded_collections_jscore_passthrough - - name: sharded_collections_jscore_passthrough_WT - - name: sharding_jscore_passthrough - - name: sharding_jscore_passthrough_WT - - name: sharding_jscore_passthrough_wire_ops_WT - - name: sharding_WT - - name: slow1 - - name: slow1_WT - - name: serial_run - - name: serial_run_WT - - name: tool - - name: tool_WT - - name: unittests - - name: push - distros: - - rhel70-small - ########################################### # Debian buildvariants # ########################################### @@ -8075,6 +7983,7 @@ buildvariants: - name: sslSpecial - name: tool - name: unittests + - name: write_concern_majority_passthrough_WT - name: linux-64-ephemeralForTest display_name: Linux (ephemeralForTest) @@ -8232,6 +8141,7 @@ buildvariants: - name: sslSpecial - name: tool - name: unittests + - name: write_concern_majority_passthrough_WT - name: enterprise-rhel-72-s390x-inmem display_name: Enterprise RHEL 7.2 s390x (inMemory) DEBUG @@ -8319,75 +8229,7 @@ buildvariants: - name: sslSpecial - name: tool - name: unittests - -- name: ubuntu1404-rocksdb - display_name: Ubuntu 14.04 (RocksDB) - modules: - - rocksdb - run_on: - - ubuntu1404-test - batchtime: 1440 # 1 day - expansions: - build_rocksdb: true - compile_flags: -j$(grep -c ^processor /proc/cpuinfo) --dbg=off --opt=on CPPPATH=$(readlink -f ../rocksdb/include/) LIBPATH=$(readlink -f ../rocksdb/) LIBS=rocksdb CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy VARIANT_DIR=release --runtime-hardening=off - num_jobs_available: $(grep -c ^processor /proc/cpuinfo) - test_flags: --storageEngine=rocksdb - use_scons_cache: true - build_mongoreplay: true - tasks: - - name: compile - distros: - - ubuntu1404-build - - name: aggregation - - name: aggregation_WT - - name: aggregation_auth - - name: auth - - name: dbtest - # - name: disk - # - name: durability - # - name: dur_jscore_passthrough - - name: failpoints - - name: failpoints_auth - - name: gle_auth - - name: gle_auth_write_cmd - - name: gle_auth_basics_passthrough - - name: gle_auth_basics_passthrough_write_cmd - - name: sharding_gle_auth_basics_passthrough - - name: sharding_gle_auth_basics_passthrough_write_cmd - - name: jsCore - - name: jsCore_auth - - name: jsCore_compatibility - - name: jstestfuzz - - name: jstestfuzz_concurrent_WT - - name: jstestfuzz_concurrent_replication_WT - - name: jstestfuzz_concurrent_sharded_WT - - name: jstestfuzz_replication - - name: jstestfuzz_sharded - - name: noPassthrough - - name: noPassthroughWithMongod - - name: bulk_gle_passthrough - - name: parallel - - name: parallel_compatibility - - name: concurrency - - name: concurrency_replication - - name: concurrency_sharded - # We use machines with more RAM to avoid OOM issues when validating collections. - distros: - - ubuntu1404-build - - name: concurrency_simultaneous - - name: replica_sets - - name: replica_sets_auth - - name: replica_sets_jscore_passthrough - - name: master_slave - - name: master_slave_WT - - name: master_slave_auth - - name: master_slave_jscore_passthrough - - name: sharding - - name: sharding_auth - - name: slow1 - - name: serial_run - - name: sharding_jscore_passthrough - - name: unittests + - name: write_concern_majority_passthrough_WT ########################################### # Experimental buildvariants # @@ -8405,6 +8247,7 @@ buildvariants: tooltags: "-tags 'ssl'" # We need llvm-symbolizer in the PATH for ASAN for clang-3.7 or later. variant_path_suffix: /usr/lib/llvm-3.8/bin + lang_environment: LANG=C san_options: LSAN_OPTIONS="suppressions=etc/lsan.suppressions" ASAN_OPTIONS=detect_leaks=1 compile_flags: CC=/usr/bin/clang-3.8 CXX=/usr/bin/clang++-3.8 CPPDEFINES="_GLIBCXX_USE_CXX11_ABI=0" --dbg=on --opt=on --allocator=system --sanitize=address --ssl -j$(grep -c ^processor /proc/cpuinfo) --nostrip VARIANT_DIR=build multiversion_platform_arch: "ubuntu1604" @@ -8538,6 +8381,7 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: write_concern_majority_passthrough_WT - name: ubuntu1604-asan display_name: ~ ASAN SSL Ubuntu 16.04 @@ -8549,6 +8393,7 @@ buildvariants: tooltags: "-tags 'ssl'" # We need llvm-symbolizer in the PATH for ASAN for clang-3.7 or later. variant_path_suffix: /usr/lib/llvm-3.8/bin + lang_environment: LANG=C san_options: LSAN_OPTIONS="suppressions=etc/lsan.suppressions" ASAN_OPTIONS=detect_leaks=1 compile_flags: CC=/usr/bin/clang-3.8 CXX=/usr/bin/clang++-3.8 CPPDEFINES="_GLIBCXX_USE_CXX11_ABI=0" --opt=on --allocator=system --sanitize=address --ssl -j$(grep -c ^processor /proc/cpuinfo) --nostrip VARIANT_DIR=build num_jobs_available: $(($(grep -c ^processor /proc/cpuinfo) / 3)) # Avoid starting too many mongod's under ASAN build. @@ -8580,6 +8425,7 @@ buildvariants: variant_path_suffix: /usr/lib/llvm-3.8/bin gorootvars: GOROOT=/opt/go PATH="/opt/go/bin:$PATH" tooltags: "-tags 'ssl sasl'" + lang_environment: LANG=C san_options: UBSAN_OPTIONS="print_stacktrace=1" compile_flags: CC=/usr/bin/clang-3.8 CXX=/usr/bin/clang++-3.8 --dbg=on --opt=on --allocator=system --sanitize=undefined --ssl -j$(grep -c ^processor /proc/cpuinfo) --nostrip CXXFLAGS="-nostdlib -nostdinc++" LIBPATH=/opt/mongodbtoolchain/v2/lib/gcc/x86_64-mongodb-linux/5.4.0 CPPPATH="/opt/mongodbtoolchain/v2/include/c++/5.4.0/ /opt/mongodbtoolchain/v2/include/c++/5.4.0/x86_64-mongodb-linux" CPPDEFINES="_GLIBCXX_USE_CXX11_ABI=0" VARIANT_DIR=build multiversion_platform_arch: "ubuntu1604" @@ -8712,6 +8558,7 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: write_concern_majority_passthrough_WT - name: enterprise-ubuntu-dynamic-1604-64-bit display_name: "* Shared Library Enterprise Ubuntu 16.04" @@ -8720,6 +8567,7 @@ buildvariants: expansions: gorootvars: GOROOT=/opt/go PATH="/opt/go/bin:$PATH" tooltags: "-tags 'ssl sasl'" + lang_environment: LANG=C compile_flags: --ssl MONGO_DISTMOD=ubuntu1604 -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy --link-model=dynamic num_jobs_available: $(grep -c ^processor /proc/cpuinfo) use_scons_cache: true @@ -8861,6 +8709,7 @@ buildvariants: - name: tool - name: tool_WT - name: unittests + - name: write_concern_majority_passthrough_WT - name: initsync-3dot2-rhel-62 display_name: "~ Initial Sync 3.2 Enterprise RHEL 6.2" @@ -8962,3 +8811,6 @@ buildvariants: - rhel62-large - name: slow1 - name: slow1_WT + - name: write_concern_majority_passthrough_WT + distros: + - rhel62-large diff --git a/etc/longevity.yml b/etc/longevity.yml index f6da942e0c1..9cdf9502fa7 100644 --- a/etc/longevity.yml +++ b/etc/longevity.yml @@ -9,45 +9,22 @@ post: params: working_dir: work script: | - set -v source ./dsienv.sh - $DSI_PATH/bin/make_artifact.sh + make_artifact.sh - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: work/reports.tgz - remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/${task_name}-${build_id}.${ext|tgz} + local_file: work/dsi-artifacts.tgz + remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/dsi-artifacts-${task_name}-${build_id}-${execution}.${ext|tgz} bucket: mciuploads permissions: public-read content_type: ${content_type|application/x-gzip} - display_name: test-log - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: work/reports/graphs/timeseries-p1.html - remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/timeseries-p1-${task_name}-${build_id}.html - bucket: mciuploads - permissions: public-read - content_type: text/html - display_name: timeseries-p1.html + display_name: Dsi Artifacts - Execution ${execution} - command: attach.results params: file_location: work/report.json - - command: shell.exec - # destroy the cluster - params: - working_dir: work - script: | - set -e - set -o verbose - # call terraform destroy twice to avoid AWS timeout - yes yes | ./terraform destroy --var-file=cluster.json - yes yes | ./terraform destroy --var-file=cluster.json - # clean all file to be safe - rm -rf * - echo "Cluster DESTROYED." + - func: "destroy cluster" - command: shell.exec params: working_dir: src @@ -79,12 +56,17 @@ functions: working_dir: work script: | cat > bootstrap.yml < runtime.yml < bootstrap.yml < runtime.yml < : ${_rev} - dsi: ${dsi_rev} - - command: json.get - params: - task: ${compare_task} - variant : ${variant1} - file: "work/standalone.json" - name: "perf" - - command: json.get - params: - task: ${compare_task} - variant : ${variant2} - file: "work/oplog.json" - name: "perf" - - command: shell.exec - type : test - params: - working_dir: work - script: | - set -o errexit - set -o verbose - python -u ../src/dsi/dsi/analysis/compare.py -b standalone.json -c oplog.json - - command: "json.send" - params: - name: "perf" - file: "work/perf.json" + TAG="3.2.17-Baseline" + OVERRIDEFILE="../src/dsi/dsi/analysis/${branch_name}/system_perf_override.json" + python -u ../src/dsi/dsi/analysis/post_run_check.py ${script_flags} --reports-analysis reports --perf-file perf.json --rev ${revision} -f history.json -t tags.json --refTag $TAG --overrideFile $OVERRIDEFILE --project_id sys-perf --variant ${build_variant} --task ${task_name} ####################################### # Tasks # @@ -323,8 +266,17 @@ tasks: then echo "Fetching JS test DB correctness checks from directory jstests/hooks" cp -a jstests/hooks/* mongodb/jstests/hooks + + echo "Now adding our own special run_validate_collections.js wrapper" + mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js + + cat << EOF > mongodb/jstests/hooks/run_validate_collections.js + print("NOTE: run_validate_collections.js will skip the oplog!"); + TestData = { skipValidationNamespaces: ['local.oplog.rs'] }; + load('jstests/hooks/run_validate_collections.actual.js'); + EOF fi - tar cvf mongodb.tar mongodb + tar cf mongodb.tar mongodb gzip mongodb.tar - command: s3.put params: @@ -341,25 +293,20 @@ tasks: depends_on: - name: compile variant: linux-standalone + priority: 5 commands: - func: "prepare environment" vars: storageEngine: "wiredTiger" test: "ycsb" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "wiredTiger" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "wiredTiger" - test: "ycsb" - - func: "make test log artifact" - func: "analyze" vars: script_flags: --ycsb-throughput-analysis reports - name: industry_benchmarks_MMAPv1 + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -368,20 +315,46 @@ tasks: vars: storageEngine: "mmapv1" test: "ycsb" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" + - func: "deploy cluster" + - func: "run test" + - func: "analyze" vars: - storageEngine: "mmapv1" + script_flags: --ycsb-throughput-analysis reports + +- name: industry_benchmarks_wmajority_WT + priority: 5 + depends_on: + - name: compile + variant: linux-standalone + commands: + - func: "prepare environment" + vars: + storageEngine: "wiredTiger" + test: "ycsb-wmajority" + - func: "deploy cluster" - func: "run test" + - func: "analyze" + vars: + script_flags: --ycsb-throughput-analysis reports + +- name: industry_benchmarks_wmajority_MMAPv1 + priority: 5 + depends_on: + - name: compile + variant: linux-standalone + commands: + - func: "prepare environment" vars: storageEngine: "mmapv1" - test: "ycsb" - - func: "make test log artifact" + test: "ycsb-wmajority" + - func: "deploy cluster" + - func: "run test" - func: "analyze" vars: script_flags: --ycsb-throughput-analysis reports - name: core_workloads_WT + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -390,18 +363,12 @@ tasks: vars: storageEngine: "wiredTiger" test: "core" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "wiredTiger" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "wiredTiger" - test: "core" - - func: "make test log artifact" - func: "analyze" - name: core_workloads_MMAPv1 + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -410,18 +377,12 @@ tasks: vars: storageEngine: "mmapv1" test: "core" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "mmapv1" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "mmapv1" - test: "core" - - func: "make test log artifact" - func: "analyze" - name: non_sharded_workloads_WT + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -430,18 +391,12 @@ tasks: vars: storageEngine: "wiredTiger" test: "non_sharded" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "wiredTiger" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "wiredTiger" - test: "non_sharded" - - func: "make test log artifact" - func: "analyze" - name: non_sharded_workloads_MMAPv1 + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -450,18 +405,12 @@ tasks: vars: storageEngine: "mmapv1" test: "non_sharded" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "mmapv1" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "mmapv1" - test: "non_sharded" - - func: "make test log artifact" - func: "analyze" - name: mongos_workloads_WT + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -470,18 +419,12 @@ tasks: vars: storageEngine: "wiredTiger" test: "mongos" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "wiredTiger" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "wiredTiger" - test: "mongos" - - func: "make test log artifact" - func: "analyze" - name: mongos_workloads_MMAPv1 + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -491,18 +434,12 @@ tasks: vars: storageEngine: "mmapv1" test: "mongos" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "mmapv1" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "mmapv1" - test: "mongos" - - func: "make test log artifact" - func: "analyze" - name: move_chunk_workloads_WT + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -511,18 +448,12 @@ tasks: vars: storageEngine: "wiredTiger" test: "move_chunk" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "wiredTiger" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "wiredTiger" - test: "move_chunk" - - func: "make test log artifact" - func: "analyze" - name: move_chunk_workloads_MMAPv1 + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -531,18 +462,12 @@ tasks: vars: storageEngine: "mmapv1" test: "move_chunk" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "mmapv1" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "mmapv1" - test: "move_chunk" - - func: "make test log artifact" - func: "analyze" - name: secondary_performance_WT + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -550,21 +475,15 @@ tasks: - func: "prepare environment" vars: storageEngine: "wiredTiger" + # Unfortunately the dash/underscore style is different for mongodb_setup and test_control test: "secondary_performance" - setup: "replica-2node" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "wiredTiger" - setup: "replica-2node" + setup: "secondary-performance" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "wiredTiger" - test: "secondary_performance" - - func: "make test log artifact" - func: "analyze" - name: secondary_performance_MMAPv1 + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -572,85 +491,15 @@ tasks: - func: "prepare environment" vars: storageEngine: "mmapv1" + # Unfortunately the dash/underscore style is different for mongodb_setup and test_control test: "secondary_performance" - setup: "replica-2node" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "mmapv1" - setup: "replica-2node" + setup: "secondary-performance" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "mmapv1" - test: "secondary_performance" - - func: "make test log artifact" - - func: "analyze" - -- name: industry_benchmarks_WT_oplog_comp - depends_on: - - name: industry_benchmarks_WT - variant: linux-standalone - status : "*" - - name: industry_benchmarks_WT - variant: linux-1-node-replSet - status: "*" - commands: - - func: "compare" - vars: - compare_task: "industry_benchmarks_WT" - variant1: "linux-standalone" - variant2: "linux-1-node-replSet" - - func: "analyze" - -- name: industry_benchmarks_MMAPv1_oplog_comp - depends_on: - - name: industry_benchmarks_MMAPv1 - variant: linux-standalone - status: "*" - - name: industry_benchmarks_MMAPv1 - variant: linux-1-node-replSet - status: "*" - commands: - - func: "compare" - vars: - compare_task: "industry_benchmarks_MMAPv1" - variant1: "linux-standalone" - variant2: "linux-1-node-replSet" - - func: "analyze" - -- name: core_workloads_WT_oplog_comp - depends_on: - - name: core_workloads_WT - variant: linux-standalone - status: "*" - - name: core_workloads_WT - variant: linux-1-node-replSet - status: "*" - commands: - - func: "compare" - vars: - compare_task: "core_workloads_WT" - variant1: "linux-standalone" - variant2: "linux-1-node-replSet" - - func: "analyze" - -- name: core_workloads_MMAPv1_oplog_comp - depends_on: - - name: core_workloads_MMAPv1 - variant: linux-standalone - status: "*" - - name: core_workloads_MMAPv1 - variant: linux-1-node-replSet - status: "*" - commands: - - func: "compare" - vars: - compare_task: "core_workloads_MMAPv1" - variant1: "linux-standalone" - variant2: "linux-1-node-replSet" - func: "analyze" - name: initialsync_WT + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -658,19 +507,13 @@ tasks: - func: "prepare environment" vars: storageEngine: "wiredTiger" - test: "initialSync" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "wiredTiger" + test: "initialsync" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "wiredTiger" - test: "initialSync" - - func: "make test log artifact" - func: "analyze" - name: initialsync_MMAPv1 + priority: 5 depends_on: - name: compile variant : linux-standalone @@ -678,19 +521,13 @@ tasks: - func: "prepare environment" vars: storageEngine: "mmapv1" - test: "initialSync" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "mmapv1" + test: "initialsync" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "mmapv1" - test: "initialSync" - - func: "make test log artifact" - func: "analyze" - name: initialsync-logkeeper_WT + priority: 5 depends_on: - name: compile variant: linux-standalone @@ -700,18 +537,10 @@ tasks: vars: storageEngine: "wiredTiger" test: "initialsync-logkeeper" - - func: "infrastructure provisioning" - - func: "configure mongodb cluster" - vars: - storageEngine: "wiredTiger" + - func: "deploy cluster" - func: "run test" - vars: - storageEngine: "wiredTiger" - test: "initialsync-logkeeper" - - func: "make test log artifact" - func: "analyze" - ####################################### # Modules # ####################################### @@ -798,6 +627,8 @@ buildvariants: - name: industry_benchmarks_WT - name: core_workloads_WT - name: industry_benchmarks_MMAPv1 + - name: industry_benchmarks_wmajority_WT + - name: industry_benchmarks_wmajority_MMAPv1 - name: core_workloads_MMAPv1 - name: mongos_workloads_WT - name: mongos_workloads_MMAPv1 @@ -820,6 +651,8 @@ buildvariants: - name: industry_benchmarks_WT - name: core_workloads_WT - name: industry_benchmarks_MMAPv1 + - name: industry_benchmarks_wmajority_WT + - name: industry_benchmarks_wmajority_MMAPv1 - name: core_workloads_MMAPv1 - name: secondary_performance_WT - name: secondary_performance_MMAPv1 @@ -858,19 +691,3 @@ buildvariants: - "rhel70-perf-initialsync-logkeeper" tasks: - name: initialsync-logkeeper_WT - -- name: linux-oplog-compare - display_name: Linux Oplog Compare - batchtime: 10080 # 7 days - modules: *modules - expansions: - compile_flags: -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/v2/bin/gcc CXX=/opt/mongodbtoolchain/v2/bin/g++ OBJCOPY=/opt/mongodbtoolchain/v2/bin/objcopy - use_scons_cache: true - project: *project - run_on: - - "rhel70-perf-single" - tasks: - - name: industry_benchmarks_WT_oplog_comp - - name: core_workloads_WT_oplog_comp - - name: industry_benchmarks_MMAPv1_oplog_comp - - name: core_workloads_MMAPv1_oplog_comp diff --git a/jstests/aggregation/extras/utils.js b/jstests/aggregation/extras/utils.js index 2dd1038388c..68b2597c533 100644 --- a/jstests/aggregation/extras/utils.js +++ b/jstests/aggregation/extras/utils.js @@ -276,7 +276,7 @@ function assertErrorCode(coll, pipe, code, errmsg) { var cursorRes = coll.runCommand("aggregate", cmd); if (cursorRes.ok) { var followupBatchSize = 0; // default - var cursor = new DBCommandCursor(coll.getMongo(), cursorRes, followupBatchSize); + var cursor = new DBCommandCursor(cursorRes._mongo, cursorRes, followupBatchSize); var error = assert.throws(function() { cursor.itcount(); diff --git a/jstests/auth/mongos_cache_invalidation.js b/jstests/auth/mongos_cache_invalidation.js index e36445cac5e..b53656de3a1 100644 --- a/jstests/auth/mongos_cache_invalidation.js +++ b/jstests/auth/mongos_cache_invalidation.js @@ -61,7 +61,7 @@ db3.auth('spencer', 'pwd'); * At this point we have 3 handles to the "test" database, each of which are on connections to * different mongoses. "db1", "db2", and "db3" are all auth'd as spencer@test and will be used * to verify that user and role data changes get propaged to their mongoses. - * "db2" is connected to a mongos with a 10 second user cache invalidation interval, + * "db2" is connected to a mongos with a 5 second user cache invalidation interval, * while "db3" is connected to a mongos with a 10 minute cache invalidation interval. */ @@ -202,12 +202,9 @@ db3.auth('spencer', 'pwd'); assert.commandFailedWithCode(db1.foo.runCommand("collStats"), authzErrorCode); // s1/db2 should update its cache in 10 seconds. - assert.soon( - function() { - return db2.foo.runCommand("collStats").code == authzErrorCode; - }, - "Mongos did not update its user cache after 10 seconds", - 6 * 1000); // Give an extra 1 second to avoid races + assert.soon(function() { + return db2.foo.runCommand("collStats").code == authzErrorCode; + }, "Mongos did not update its user cache after 10 seconds", 10 * 1000); // We manually invalidate the cache on s2/db3. db3.adminCommand("invalidateUserCache"); diff --git a/jstests/auth/scram-credentials-invalid.js b/jstests/auth/scram-credentials-invalid.js new file mode 100644 index 00000000000..16c0c204d12 --- /dev/null +++ b/jstests/auth/scram-credentials-invalid.js @@ -0,0 +1,45 @@ +// Ensure that attempting to use SCRAM-SHA-1 auth on a +// user with invalid SCRAM-SHA-1 credentials fails gracefully. + +(function() { + 'use strict'; + + function runTest(mongod) { + assert(mongod); + const admin = mongod.getDB('admin'); + const test = mongod.getDB('test'); + + admin.createUser({user: 'admin', pwd: 'pass', roles: jsTest.adminUserRoles}); + assert(admin.auth('admin', 'pass')); + + test.createUser({user: 'user', pwd: 'pass', roles: jsTest.basicUserRoles}); + + // Give the test user an invalid set of SCRAM-SHA-1 credentials. + assert.eq(admin.system.users + .update({_id: "test.user"}, { + $set: { + "credentials.SCRAM-SHA-1": { + salt: "AAAA", + storedKey: "AAAA", + serverKey: "AAAA", + iterationCount: 10000 + } + } + }) + .nModified, + 1, + "Should have updated one document for user@test"); + admin.logout(); + + assert(!test.auth({user: 'user', pwd: 'pass'})); + + assert.soon(function() { + const log = cat(mongod.fullOptions.logFile); + return /Unable to perform SCRAM-SHA-1 auth.* invalid SCRAM credentials/.test(log); + }, "No warning issued for invalid SCRAM-SHA-1 credendials doc", 30 * 1000, 5 * 1000); + } + + const mongod = MongoRunner.runMongod({auth: "", useLogFiles: true}); + runTest(mongod); + MongoRunner.stopMongod(mongod); +})(); diff --git a/jstests/auth/system_authorization_indexes.js b/jstests/auth/system_authorization_indexes.js new file mode 100644 index 00000000000..496f78578a4 --- /dev/null +++ b/jstests/auth/system_authorization_indexes.js @@ -0,0 +1,66 @@ +/** Ensure that authorization system collections' indexes are correctly generated. + * + * This test requires users to persist across a restart. + * @tags: [requires_persistence] + */ + +(function() { + let conn = MongoRunner.runMongod({smallfiles: ""}); + let db = conn.getDB("admin"); + + // TEST: User and role collections start off with no indexes + assert.eq(0, db.system.users.getIndexes().length); + assert.eq(0, db.system.roles.getIndexes().length); + + // TEST: User and role creation generates indexes + db.createUser({user: "user", pwd: "pwd", roles: []}); + assert.eq(2, db.system.users.getIndexes().length); + + db.createRole({role: "role", privileges: [], roles: []}); + assert.eq(2, db.system.roles.getIndexes().length); + + // TEST: Destroying admin.system.users index and restarting will recreate it + assert.commandWorked(db.system.users.dropIndexes()); + MongoRunner.stopMongod(conn); + conn = MongoRunner.runMongod({restart: conn, cleanData: false}); + db = conn.getDB("admin"); + assert.eq(2, db.system.users.getIndexes().length); + assert.eq(2, db.system.roles.getIndexes().length); + + // TEST: Destroying admin.system.roles index and restarting will recreate it + assert.commandWorked(db.system.roles.dropIndexes()); + MongoRunner.stopMongod(conn); + conn = MongoRunner.runMongod({restart: conn, cleanData: false}); + db = conn.getDB("admin"); + assert.eq(2, db.system.users.getIndexes().length); + assert.eq(2, db.system.roles.getIndexes().length); + + // TEST: Destroying both authorization indexes and restarting will recreate them + assert.commandWorked(db.system.users.dropIndexes()); + assert.commandWorked(db.system.roles.dropIndexes()); + MongoRunner.stopMongod(conn); + conn = MongoRunner.runMongod({restart: conn, cleanData: false}); + db = conn.getDB("admin"); + assert.eq(2, db.system.users.getIndexes().length); + assert.eq(2, db.system.roles.getIndexes().length); + + // TEST: Destroying the admin.system.users index and restarting will recreate it, even if + // admin.system.roles does not exist + db.dropDatabase(); + db.createUser({user: "user", pwd: "pwd", roles: []}); + assert.commandWorked(db.system.users.dropIndexes()); + MongoRunner.stopMongod(conn); + conn = MongoRunner.runMongod({restart: conn, cleanData: false}); + db = conn.getDB("admin"); + assert.eq(2, db.system.users.getIndexes().length); + + // TEST: Destroying the admin.system.roles index and restarting will recreate it, even if + // admin.system.users does not exist + db.dropDatabase(); + db.createRole({role: "role", privileges: [], roles: []}); + assert.commandWorked(db.system.roles.dropIndexes()); + MongoRunner.stopMongod(conn); + conn = MongoRunner.runMongod({restart: conn, cleanData: false}); + db = conn.getDB("admin"); + assert.eq(2, db.system.roles.getIndexes().length); +})(); diff --git a/jstests/concurrency/fsm_libs/cluster.js b/jstests/concurrency/fsm_libs/cluster.js index a5705a409af..f3e2429f898 100644 --- a/jstests/concurrency/fsm_libs/cluster.js +++ b/jstests/concurrency/fsm_libs/cluster.js @@ -573,6 +573,25 @@ var Cluster = function(options) { return data; }; + + this.isRunningWiredTigerLSM = function isRunningWiredTigerLSM() { + var adminDB = this.getDB('admin'); + + if (this.isSharded()) { + // Get the storage engine the sharded cluster is configured to use from one of the + // shards since mongos won't report it. + adminDB = st.shard0.getDB('admin'); + } + + var res = adminDB.runCommand({getCmdLineOpts: 1}); + assert.commandWorked(res, 'failed to get command line options'); + + var wiredTigerOptions = res.parsed.storage.wiredTiger || {}; + var wiredTigerCollectionConfig = wiredTigerOptions.collectionConfig || {}; + var wiredTigerConfigString = wiredTigerCollectionConfig.configString || ''; + + return wiredTigerConfigString === 'type=lsm'; + }; }; /** diff --git a/jstests/concurrency/fsm_workloads/compact.js b/jstests/concurrency/fsm_workloads/compact.js index 8f91f52bf5e..e86e6ef555f 100644 --- a/jstests/concurrency/fsm_workloads/compact.js +++ b/jstests/concurrency/fsm_workloads/compact.js @@ -87,12 +87,27 @@ var $config = (function() { dropCollections(db, pattern); }; + var skip = function skip(cluster) { + if (cluster.isRunningWiredTigerLSM()) { + // There is a known hang during concurrent FSM workloads with the compact command used + // with wiredTiger LSM variants. Bypass this command for the wiredTiger LSM variant + // until a fix is available for WT-2523. + return { + skip: true, + msg: 'WT-2523: compact command can cause hang using WT LSM index during ' + + 'concurrent workloads' + }; + } + return {skip: false}; + }; + return { threadCount: 15, iterations: 10, states: states, transitions: transitions, teardown: teardown, - data: data + data: data, + skip: skip }; })(); diff --git a/jstests/concurrency/fsm_workloads/remove_multiple_documents.js b/jstests/concurrency/fsm_workloads/remove_multiple_documents.js index bfd64cd6790..1461ca0cc03 100644 --- a/jstests/concurrency/fsm_workloads/remove_multiple_documents.js +++ b/jstests/concurrency/fsm_workloads/remove_multiple_documents.js @@ -34,8 +34,17 @@ var $config = (function() { } }; + var skip = function skip(cluster) { + // When the balancer is enabled, the nRemoved result may be inaccurate as + // a chunk migration may be active, causing the count function to assert. + if (cluster.isBalancerEnabled()) { + return {skip: true, msg: 'does not run when balancer is enabled.'}; + } + return {skip: false}; + }; + var transitions = {init: {count: 1}, count: {remove: 1}, remove: {remove: 0.825, count: 0.125}}; - return {threadCount: 10, iterations: 20, states: states, transitions: transitions}; + return {threadCount: 10, iterations: 20, states: states, transitions: transitions, skip: skip}; })(); diff --git a/jstests/concurrency/fsm_workloads/remove_where.js b/jstests/concurrency/fsm_workloads/remove_where.js index f9c0e6a2c03..3ef214c769b 100644 --- a/jstests/concurrency/fsm_workloads/remove_where.js +++ b/jstests/concurrency/fsm_workloads/remove_where.js @@ -38,5 +38,14 @@ var $config = extendWorkload($config, function($config, $super) { /* no-op to prevent index from being created */ }; + $config.skip = function skip(cluster) { + // When the balancer is enabled, the nRemoved result may be inaccurate as + // a chunk migration may be active, causing the count function to assert. + if (cluster.isBalancerEnabled()) { + return {skip: true, msg: 'does not run when balancer is enabled.'}; + } + return {skip: false}; + }; + return $config; }); diff --git a/jstests/core/apply_ops1.js b/jstests/core/apply_ops1.js index 8ce15b1df22..7088145c718 100644 --- a/jstests/core/apply_ops1.js +++ b/jstests/core/apply_ops1.js @@ -286,9 +286,54 @@ 'applyOps should fail on unknown operation type "x" with valid "ns" value'); assert.eq(0, t.find().count(), "Non-zero amount of documents in collection to start"); + + /** + * Test function for running CRUD operations on non-existent namespaces using various + * combinations of invalid namespaces (collection/database), allowAtomic and alwaysUpsert. + * + * Leave 'expectedErrorCode' undefined if this command is expected to run successfully. + */ + function testCrudOperationOnNonExistentNamespace(optype, o, o2, expectedErrorCode) { + expectedErrorCode = expectedErrorCode || ErrorCodes.OK; + const t2 = db.getSiblingDB('apply_ops1_no_such_db').getCollection('t'); + [t, t2].forEach(coll => { + const op = {op: optype, ns: coll.getFullName(), o: o, o2: o2}; + [false, true].forEach(allowAtomic => { + [false, true].forEach(alwaysUpsert => { + const cmd = { + applyOps: [op], + allowAtomic: allowAtomic, + alwaysUpsert: alwaysUpsert + }; + jsTestLog('Testing applyOps on non-existent namespace: ' + tojson(cmd)); + if (expectedErrorCode === ErrorCodes.OK) { + assert.commandWorked(db.adminCommand(cmd)); + } else { + assert.commandFailedWithCode(db.adminCommand(cmd), expectedErrorCode); + } + }); + }); + }); + } + + // Insert and update operations on non-existent collections/databases should return + // NamespaceNotFound. + testCrudOperationOnNonExistentNamespace('i', {_id: 0}, {}, ErrorCodes.NamespaceNotFound); + testCrudOperationOnNonExistentNamespace('u', {x: 0}, {_id: 0}, ErrorCodes.NamespaceNotFound); + + // Delete operations on non-existent collections/databases should return OK for idempotency + // reasons. + testCrudOperationOnNonExistentNamespace('d', {_id: 0}, {}); + assert.commandFailed( - db.adminCommand({applyOps: [{"op": "i", "ns": t.getFullName(), "o": {_id: 5, x: 17}}]}), - "Applying an insert operation on a non-existent collection should fail"); + db.adminCommand({ + applyOps: [{ + "op": "c", + "ns": "admin.$cmd", + "o": {applyOps: [{"op": "i", "ns": t.getFullName(), "o": {_id: 5, x: 17}}]} + }] + }), + "Applying a nested insert operation on a non-existent collection should fail"); assert.commandWorked(db.createCollection(t.getName())); var a = assert.commandWorked( diff --git a/jstests/core/apply_ops_atomicity.js b/jstests/core/apply_ops_atomicity.js index 911ce32a311..e704815a889 100644 --- a/jstests/core/apply_ops_atomicity.js +++ b/jstests/core/apply_ops_atomicity.js @@ -29,10 +29,12 @@ var newDBName = "apply_ops_atomicity"; var newDB = db.getSiblingDB(newDBName); assert.commandWorked(newDB.dropDatabase()); - // Do an update on a non-existent database, since only 'u' ops can implicitly create - // collections. - assert.commandWorked(newDB.runCommand( - {applyOps: [{op: "u", ns: newDBName + ".foo", o: {_id: 5, x: 17}, o2: {_id: 5, x: 16}}]})); + // Updates on a non-existent database no longer implicitly create collections and will fail with + // a NamespaceNotFound error. + assert.commandFailedWithCode(newDB.runCommand({ + applyOps: [{op: "u", ns: newDBName + ".foo", o: {_id: 5, x: 17}, o2: {_id: 5, x: 16}}] + }), + ErrorCodes.NamespaceNotFound); var sawTooManyLocksError = false; diff --git a/jstests/core/autocomplete.js b/jstests/core/autocomplete.js new file mode 100644 index 00000000000..6eb6e21a7a3 --- /dev/null +++ b/jstests/core/autocomplete.js @@ -0,0 +1,42 @@ +/** + * Validate auto complete works for various javascript types implemented by C++. + */ +(function() { + 'use strict'; + + function testAutoComplete(prefix) { + // This method updates a global object with an array of strings on success. + shellAutocomplete(prefix); + return __autocomplete__; + } + + // Create a collection + db.auto_complete_coll.insert({}); + + // Validate DB auto completion + const db_stuff = testAutoComplete('db.'); + + // Verify we enumerate built-in methods + assert.contains('db.prototype', db_stuff); + assert.contains('db.hasOwnProperty', db_stuff); + assert.contains('db.toString(', db_stuff); + + // Verify we have some methods we added + assert.contains('db.adminCommand(', db_stuff); + assert.contains('db.runCommand(', db_stuff); + + // Verify we enumerate collections + assert.contains('db.auto_complete_coll', db_stuff); + + // Validate Collection autocompletion + const coll_stuff = testAutoComplete('db.auto_complete_coll.'); + + // Verify we enumerate built-in methods + assert.contains('db.auto_complete_coll.prototype', coll_stuff); + assert.contains('db.auto_complete_coll.hasOwnProperty', coll_stuff); + assert.contains('db.auto_complete_coll.toString(', coll_stuff); + + // Verify we have some methods we added + assert.contains('db.auto_complete_coll.aggregate(', coll_stuff); + assert.contains('db.auto_complete_coll.runCommand(', coll_stuff); +})(); \ No newline at end of file diff --git a/jstests/core/batch_write_command_delete.js b/jstests/core/batch_write_command_delete.js index 2aefcea6a7f..99b5f8e3a61 100644 --- a/jstests/core/batch_write_command_delete.js +++ b/jstests/core/batch_write_command_delete.js @@ -1,3 +1,5 @@ +// @tags: [assumes_write_concern_unchanged] + // // Ensures that mongod respects the batch write protocols for delete // diff --git a/jstests/core/batch_write_command_insert.js b/jstests/core/batch_write_command_insert.js index 274f35513e7..3fa6cf98756 100644 --- a/jstests/core/batch_write_command_insert.js +++ b/jstests/core/batch_write_command_insert.js @@ -1,3 +1,5 @@ +// @tags: [assumes_write_concern_unchanged] + // // Ensures that mongod respects the batch write protocol for inserts // diff --git a/jstests/core/batch_write_command_update.js b/jstests/core/batch_write_command_update.js index 2d9d2d699b2..987525e2515 100644 --- a/jstests/core/batch_write_command_update.js +++ b/jstests/core/batch_write_command_update.js @@ -1,3 +1,5 @@ +// @tags: [assumes_write_concern_unchanged] + // // Ensures that mongod respects the batch write protocols for updates // diff --git a/jstests/core/bypass_doc_validation.js b/jstests/core/bypass_doc_validation.js index d9bca81ab6d..c2cdfff0bc9 100644 --- a/jstests/core/bypass_doc_validation.js +++ b/jstests/core/bypass_doc_validation.js @@ -1,5 +1,7 @@ // Test the bypassDocumentValidation flag with some database commands. The test uses relevant shell // helpers when they're available for the respective server commands. +// +// @tags: [requires_collmod_command] (function() { 'use strict'; diff --git a/jstests/core/capped6.js b/jstests/core/capped6.js index e94e7ea44e8..ca216fbe96a 100644 --- a/jstests/core/capped6.js +++ b/jstests/core/capped6.js @@ -1,4 +1,11 @@ // Test NamespaceDetails::cappedTruncateAfter via "captrunc" command +// +// @tags: [ +// # This test attempts to perform read operations on a capped collection after truncating +// # documents using the captrunc command. The writes from the captrunc command aren't guaranteed +// # to become visible until a later w="majority" write occurs. +// assumes_write_concern_unchanged, +// ] (function() { var coll = db.capped6; diff --git a/jstests/core/collation_plan_cache.js b/jstests/core/collation_plan_cache.js index 0eec77388e4..790bbbadaa6 100644 --- a/jstests/core/collation_plan_cache.js +++ b/jstests/core/collation_plan_cache.js @@ -1,4 +1,11 @@ // Integration testing for the plan cache and index filter commands with collation. +// +// @tags: [ +// # This test attempts to perform queries and introspect the server's plan cache entries. The +// # former operation may be routed to a secondary in the replica set, whereas the latter must be +// # routed to the primary. +// assumes_read_preference_unchanged, +// ] (function() { 'use strict'; @@ -237,4 +244,4 @@ assert.eq(0, coll.runCommand('planCacheListFilters').filters.length, 'unexpected number of plan cache filters'); -})(); \ No newline at end of file +})(); diff --git a/jstests/core/collection_info_cache_race.js b/jstests/core/collection_info_cache_race.js index d57fc3340db..8fcde050e99 100644 --- a/jstests/core/collection_info_cache_race.js +++ b/jstests/core/collection_info_cache_race.js @@ -5,9 +5,9 @@ var coll = db.collection_info_cache_race; coll.drop(); assert.commandWorked(db.createCollection(coll.getName(), {autoIndexId: false})); // Fails when SERVER-16502 was not fixed, due to invariant -assert.writeOK(coll.save({_id: false}, {writeConcern: {w: 1}})); +assert.writeOK(coll.save({_id: false})); coll.drop(); assert.commandWorked(db.createCollection(coll.getName(), {autoIndexId: false})); assert.eq(null, coll.findOne()); -assert.writeOK(coll.save({_id: false}, {writeConcern: {w: 1}})); +assert.writeOK(coll.save({_id: false})); diff --git a/jstests/core/collmod.js b/jstests/core/collmod.js index 16f9694560c..e366041bd99 100644 --- a/jstests/core/collmod.js +++ b/jstests/core/collmod.js @@ -1,5 +1,7 @@ // Basic js tests for the collMod command. // Test setting the usePowerOf2Sizes flag, and modifying TTL indexes. +// +// @tags: [requires_collmod_command] function debug(x) { // printjson( x ); diff --git a/jstests/core/collmod_bad_spec.js b/jstests/core/collmod_bad_spec.js index ccce81fd4b1..c3d5e7a148e 100644 --- a/jstests/core/collmod_bad_spec.js +++ b/jstests/core/collmod_bad_spec.js @@ -2,6 +2,8 @@ // // Tests that a collMod with a bad specification does not cause any changes, and does not crash the // server. +// +// @tags: [requires_collmod_command] (function() { "use strict"; diff --git a/jstests/core/commands_that_do_not_write_do_not_accept_wc.js b/jstests/core/commands_that_do_not_write_do_not_accept_wc.js index ef5f8762a42..a9c72e108a1 100644 --- a/jstests/core/commands_that_do_not_write_do_not_accept_wc.js +++ b/jstests/core/commands_that_do_not_write_do_not_accept_wc.js @@ -2,6 +2,8 @@ * This file tests commands that do not support write concern. It passes both valid and invalid * writeConcern fields to commands and expects the commands to fail with a writeConcernNotSupported * error. + * + * @tags: [assumes_write_concern_unchanged] */ (function() { diff --git a/jstests/core/constructors.js b/jstests/core/constructors.js index 9e2cd26bbe8..93842780b42 100644 --- a/jstests/core/constructors.js +++ b/jstests/core/constructors.js @@ -1,4 +1,6 @@ // Tests to see what validity checks are done for 10gen specific object construction +// +// @tags: [requires_eval_command] // Takes a list of constructors and returns a new list with an extra entry for each constructor with // "new" prepended diff --git a/jstests/core/count10.js b/jstests/core/count10.js index 2a1853c399a..453775c97f5 100644 --- a/jstests/core/count10.js +++ b/jstests/core/count10.js @@ -1,4 +1,11 @@ // Test that interrupting a count returns an error code. +// +// @tags: [ +// # This test attempts to perform a count command and find it using the currentOp command. The +// # former operation may be routed to a secondary in the replica set, whereas the latter must be +// # routed to the primary. +// assumes_read_preference_unchanged, +// ] t = db.count10; t.drop(); diff --git a/jstests/core/count_plan_summary.js b/jstests/core/count_plan_summary.js index 48891d21e8e..365f289c457 100644 --- a/jstests/core/count_plan_summary.js +++ b/jstests/core/count_plan_summary.js @@ -1,5 +1,11 @@ -// Test that the plan summary string appears in db.currentOp() for -// count operations. SERVER-14064. +// Test that the plan summary string appears in db.currentOp() for count operations. SERVER-14064. +// +// @tags: [ +// # This test attempts to perform a find command and find it using the currentOp command. The +// # former operation may be routed to a secondary in the replica set, whereas the latter must be +// # routed to the primary. +// assumes_read_preference_unchanged, +// ] var t = db.jstests_count_plan_summary; t.drop(); diff --git a/jstests/core/crud_api.js b/jstests/core/crud_api.js index c9dbfb40c85..d572d4b90a5 100644 --- a/jstests/core/crud_api.js +++ b/jstests/core/crud_api.js @@ -1,3 +1,5 @@ +// @tags: [assumes_write_concern_unchanged] + (function() { "use strict"; diff --git a/jstests/core/diagdata.js b/jstests/core/diagdata.js index 490e4a3eb2b..6938f8c5102 100644 --- a/jstests/core/diagdata.js +++ b/jstests/core/diagdata.js @@ -1,4 +1,5 @@ // Test that verifies getDiagnosticData returns FTDC data +load('jstests/libs/ftdc.js'); (function() { "use strict"; @@ -6,28 +7,5 @@ // Verify we require admin database assert.commandFailed(db.diagdata.runCommand("getDiagnosticData")); - // We need to retry a few times if run this test immediately after mongod is started as FTDC may - // not have run yet. - var foundGoodDocument = false; - - for (var i = 0; i < 60; ++i) { - var result = db.adminCommand("getDiagnosticData"); - assert.commandWorked(result); - - var data = result.data; - - if (!data.hasOwnProperty("start")) { - // Wait a little longer for FTDC to start - sleep(500); - } else { - // Check for a few common properties to ensure we got data - assert(data.hasOwnProperty("serverStatus"), - "does not have 'serverStatus' in '" + tojson(data) + "'"); - assert(data.hasOwnProperty("end"), "does not have 'end' in '" + tojson(data) + "'"); - foundGoodDocument = true; - } - } - assert(foundGoodDocument, - "getDiagnosticData failed to return a non-empty command, is FTDC running?"); - + verifyGetDiagnosticData(db.getSiblingDB('admin')); })(); diff --git a/jstests/core/doc_validation.js b/jstests/core/doc_validation.js index a30763869e7..95dbae1b219 100644 --- a/jstests/core/doc_validation.js +++ b/jstests/core/doc_validation.js @@ -1,4 +1,6 @@ // Test basic inserts and updates with document validation. +// +// @tags: [requires_collmod_command] (function() { "use strict"; diff --git a/jstests/core/doc_validation_invalid_validators.js b/jstests/core/doc_validation_invalid_validators.js index b78b31c0977..70262f5de43 100644 --- a/jstests/core/doc_validation_invalid_validators.js +++ b/jstests/core/doc_validation_invalid_validators.js @@ -1,5 +1,7 @@ // Verify invalid validator statements won't work and that we // can't create validated collections on restricted databases. +// +// @tags: [requires_collmod_command] (function() { "use strict"; diff --git a/jstests/core/doc_validation_options.js b/jstests/core/doc_validation_options.js index 8a96685e48f..d1434af6d45 100644 --- a/jstests/core/doc_validation_options.js +++ b/jstests/core/doc_validation_options.js @@ -1,3 +1,4 @@ +// @tags: [requires_collmod_command] (function() { "use strict"; diff --git a/jstests/core/dropdb_race.js b/jstests/core/dropdb_race.js index bd5e7e5ddba..d8b49d08174 100644 --- a/jstests/core/dropdb_race.js +++ b/jstests/core/dropdb_race.js @@ -1,4 +1,6 @@ // test dropping a db with simultaneous commits +// +// @tags: [assumes_write_concern_unchanged] m = db.getMongo(); baseName = "jstests_dur_droprace"; diff --git a/jstests/core/elemMatchProjection.js b/jstests/core/elemMatchProjection.js index 4e80e8a296e..5060d8a6346 100644 --- a/jstests/core/elemMatchProjection.js +++ b/jstests/core/elemMatchProjection.js @@ -4,35 +4,59 @@ t.drop(); date1 = new Date(); +// Generate monotonically increasing _id values. ObjectIds generated by the shell are not guaranteed +// to be monotically increasing, and we will depend on the _id sort order later in the test. +var currentId = 0; +function nextId() { + return ++currentId; +} + // Insert various styles of arrays for (i = 0; i < 100; i++) { - t.insert({group: 1, x: [1, 2, 3, 4, 5]}); - t.insert({group: 2, x: [{a: 1, b: 2}, {a: 2, c: 3}, {a: 1, d: 5}]}); + t.insert({_id: nextId(), group: 1, x: [1, 2, 3, 4, 5]}); + t.insert({_id: nextId(), group: 2, x: [{a: 1, b: 2}, {a: 2, c: 3}, {a: 1, d: 5}]}); t.insert({ + _id: nextId(), group: 3, x: [{a: 1, b: 2}, {a: 2, c: 3}, {a: 1, d: 5}], y: [{aa: 1, bb: 2}, {aa: 2, cc: 3}, {aa: 1, dd: 5}] }); - t.insert({group: 3, x: [{a: 1, b: 3}, {a: -6, c: 3}]}); - t.insert({group: 4, x: [{a: 1, b: 4}, {a: -6, c: 3}]}); - t.insert({group: 5, x: [new Date(), 5, 10, 'string', new ObjectId(), 123.456]}); + t.insert({_id: nextId(), group: 3, x: [{a: 1, b: 3}, {a: -6, c: 3}]}); + t.insert({_id: nextId(), group: 4, x: [{a: 1, b: 4}, {a: -6, c: 3}]}); + t.insert({_id: nextId(), group: 5, x: [new Date(), 5, 10, 'string', new ObjectId(), 123.456]}); t.insert({ + _id: nextId(), group: 6, x: [{a: 'string', b: date1}, {a: new ObjectId(), b: 1.2345}, {a: 'string2', b: date1}] }); - t.insert({group: 7, x: [{y: [1, 2, 3, 4]}]}); - t.insert({group: 8, x: [{y: [{a: 1, b: 2}, {a: 3, b: 4}]}]}); - t.insert({group: 9, x: [{y: [{a: 1, b: 2}, {a: 3, b: 4}]}, {z: [{a: 1, b: 2}, {a: 3, b: 4}]}]}); - t.insert({group: 10, x: [{a: 1, b: 2}, {a: 3, b: 4}], y: [{c: 1, d: 2}, {c: 3, d: 4}]}); - t.insert({group: 10, x: [{a: 1, b: 2}, {a: 3, b: 4}], y: [{c: 1, d: 2}, {c: 3, d: 4}]}); + t.insert({_id: nextId(), group: 7, x: [{y: [1, 2, 3, 4]}]}); + t.insert({_id: nextId(), group: 8, x: [{y: [{a: 1, b: 2}, {a: 3, b: 4}]}]}); + t.insert({ + _id: nextId(), + group: 9, + x: [{y: [{a: 1, b: 2}, {a: 3, b: 4}]}, {z: [{a: 1, b: 2}, {a: 3, b: 4}]}] + }); + t.insert({ + _id: nextId(), + group: 10, + x: [{a: 1, b: 2}, {a: 3, b: 4}], + y: [{c: 1, d: 2}, {c: 3, d: 4}] + }); + t.insert({ + _id: nextId(), + group: 10, + x: [{a: 1, b: 2}, {a: 3, b: 4}], + y: [{c: 1, d: 2}, {c: 3, d: 4}] + }); t.insert({ + _id: nextId(), group: 11, x: [{a: 1, b: 2}, {a: 2, c: 3}, {a: 1, d: 5}], covered: [{aa: 1, bb: 2}, {aa: 2, cc: 3}, {aa: 1, dd: 5}] }); - t.insert({group: 12, x: {y: [{a: 1, b: 1}, {a: 1, b: 2}]}}); - t.insert({group: 13, x: [{a: 1, b: 1}, {a: 1, b: 2}]}); - t.insert({group: 13, x: [{a: 1, b: 2}, {a: 1, b: 1}]}); + t.insert({_id: nextId(), group: 12, x: {y: [{a: 1, b: 1}, {a: 1, b: 2}]}}); + t.insert({_id: nextId(), group: 13, x: [{a: 1, b: 1}, {a: 1, b: 2}]}); + t.insert({_id: nextId(), group: 13, x: [{a: 1, b: 2}, {a: 1, b: 1}]}); } t.ensureIndex({ group: 1, diff --git a/jstests/core/error2.js b/jstests/core/error2.js index 6f0b95bc17e..fb6a8e6e3b2 100644 --- a/jstests/core/error2.js +++ b/jstests/core/error2.js @@ -1,4 +1,5 @@ // Test that client gets stack trace on failed invoke +// @tags: [requires_eval_command] f = db.jstests_error2; diff --git a/jstests/core/eval0.js b/jstests/core/eval0.js index 5802f2597cb..c21c6be66c6 100644 --- a/jstests/core/eval0.js +++ b/jstests/core/eval0.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + assert.writeOK(db.evalprep.insert({}), "db must exist for eval to succeed"); db.evalprep.drop(); assert.eq(17, diff --git a/jstests/core/eval1.js b/jstests/core/eval1.js index 8b139cae02a..b5bffac892e 100644 --- a/jstests/core/eval1.js +++ b/jstests/core/eval1.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.eval1; t.drop(); diff --git a/jstests/core/eval3.js b/jstests/core/eval3.js index c4f8be21056..b95837a6817 100644 --- a/jstests/core/eval3.js +++ b/jstests/core/eval3.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.eval3; t.drop(); diff --git a/jstests/core/eval4.js b/jstests/core/eval4.js index 0d120b393de..9b0c2a49d82 100644 --- a/jstests/core/eval4.js +++ b/jstests/core/eval4.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.eval4; t.drop(); diff --git a/jstests/core/eval5.js b/jstests/core/eval5.js index 46bd679dd77..815365ac8b9 100644 --- a/jstests/core/eval5.js +++ b/jstests/core/eval5.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.eval5; t.drop(); diff --git a/jstests/core/eval6.js b/jstests/core/eval6.js index 31258f6917b..96b43b3516c 100644 --- a/jstests/core/eval6.js +++ b/jstests/core/eval6.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.eval6; t.drop(); diff --git a/jstests/core/eval7.js b/jstests/core/eval7.js index 80197fcdde6..3bace093db1 100644 --- a/jstests/core/eval7.js +++ b/jstests/core/eval7.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + assert.writeOK(db.evalprep.insert({}), "db must exist for eval to succeed"); db.evalprep.drop(); diff --git a/jstests/core/eval9.js b/jstests/core/eval9.js index 1480ff6519c..82c230a8a95 100644 --- a/jstests/core/eval9.js +++ b/jstests/core/eval9.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + assert.writeOK(db.evalprep.insert({}), "db must exist for eval to succeed"); db.evalprep.drop(); diff --git a/jstests/core/eval_mr.js b/jstests/core/eval_mr.js index 4a3dc8dad6c..33ca96eea01 100644 --- a/jstests/core/eval_mr.js +++ b/jstests/core/eval_mr.js @@ -1,4 +1,6 @@ // Test that the eval command can't be used to invoke the mapReduce command. SERVER-17889. +// +// @tags: [requires_eval_command] (function() { "use strict"; db.eval_mr.drop(); diff --git a/jstests/core/eval_nolock.js b/jstests/core/eval_nolock.js index 9511784becb..0fde2666f5d 100644 --- a/jstests/core/eval_nolock.js +++ b/jstests/core/eval_nolock.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.eval_nolock; t.drop(); diff --git a/jstests/core/evala.js b/jstests/core/evala.js index 7ccf33ac754..09241eeedee 100644 --- a/jstests/core/evala.js +++ b/jstests/core/evala.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.evala; t.drop(); diff --git a/jstests/core/evalb.js b/jstests/core/evalb.js index 3391c4cc4f2..2de16f4ea83 100644 --- a/jstests/core/evalb.js +++ b/jstests/core/evalb.js @@ -1,5 +1,7 @@ // Check the return value of a db.eval function running a database query, and ensure the function's // contents are logged in the profile log. +// +// @tags: [requires_eval_command] // Use a reserved database name to avoid a conflict in the parallel test suite. var stddb = db; diff --git a/jstests/core/evald.js b/jstests/core/evald.js index 8049d2ba8ae..43e74fc4600 100644 --- a/jstests/core/evald.js +++ b/jstests/core/evald.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + t = db.jstests_evald; t.drop(); diff --git a/jstests/core/evale.js b/jstests/core/evale.js index 1ddc8519fc6..20384e0d741 100644 --- a/jstests/core/evale.js +++ b/jstests/core/evale.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + t = db.jstests_evale; t.drop(); diff --git a/jstests/core/evalg.js b/jstests/core/evalg.js index 18503659217..56cb1dedd5f 100644 --- a/jstests/core/evalg.js +++ b/jstests/core/evalg.js @@ -1,4 +1,6 @@ // SERVER-17499: Test behavior of getMore on aggregation cursor under eval command. +// +// @tags: [requires_eval_command] db.evalg.drop(); for (var i = 0; i < 102; ++i) { db.evalg.insert({}); diff --git a/jstests/core/evalh.js b/jstests/core/evalh.js index 11e672f6bf4..b9c0d486d59 100644 --- a/jstests/core/evalh.js +++ b/jstests/core/evalh.js @@ -1,5 +1,7 @@ /** * Test that db.eval does not support auth. + * + * @tags: [requires_eval_command] */ (function() { 'use strict'; diff --git a/jstests/core/evalj.js b/jstests/core/evalj.js index f2326fff365..d6ef46430de 100644 --- a/jstests/core/evalj.js +++ b/jstests/core/evalj.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + (function() { "use strict"; diff --git a/jstests/core/existsa.js b/jstests/core/existsa.js index e9430b489a3..d98fd3f2d68 100644 --- a/jstests/core/existsa.js +++ b/jstests/core/existsa.js @@ -1,101 +1,111 @@ -// Sparse indexes are disallowed for $exists:false queries. SERVER-3918 - -t = db.jstests_existsa; -t.drop(); - -t.save({}); -t.save({a: 1}); -t.save({a: {x: 1}, b: 1}); - -/** Configure testing of an index { :1 }. */ -function setIndex(_indexKeyField) { - indexKeyField = _indexKeyField; - indexKeySpec = {}; - indexKeySpec[indexKeyField] = 1; - t.ensureIndex(indexKeySpec, {sparse: true}); -} -setIndex('a'); - -/** @return count when hinting the index to use. */ -function hintedCount(query) { - return t.find(query).hint(indexKeySpec).itcount(); -} - -/** The query field does not exist and the sparse index is not used without a hint. */ -function assertMissing(query, expectedMissing, expectedIndexedMissing) { - expectedMissing = expectedMissing || 1; - expectedIndexedMissing = expectedIndexedMissing || 0; - assert.eq(expectedMissing, t.count(query)); - // We also shouldn't get a different count depending on whether - // an index is used or not. - assert.eq(expectedIndexedMissing, hintedCount(query)); -} - -/** The query field exists and the sparse index is used without a hint. */ -function assertExists(query, expectedExists) { - expectedExists = expectedExists || 2; - assert.eq(expectedExists, t.count(query)); - // An $exists:true predicate generates no index filters. Add another predicate on the index key - // to trigger use of the index. - andClause = {}; - andClause[indexKeyField] = {$ne: null}; - Object.extend(query, {$and: [andClause]}); - assert.eq(expectedExists, t.count(query)); - assert.eq(expectedExists, hintedCount(query)); -} - -/** The query field exists and the sparse index is not used without a hint. */ -function assertExistsUnindexed(query, expectedExists) { - expectedExists = expectedExists || 2; - assert.eq(expectedExists, t.count(query)); - // Even with another predicate on the index key, the sparse index is disallowed. - andClause = {}; - andClause[indexKeyField] = {$ne: null}; - Object.extend(query, {$and: [andClause]}); - assert.eq(expectedExists, t.count(query)); - assert.eq(expectedExists, hintedCount(query)); -} - -// $exists:false queries match the proper number of documents and disallow the sparse index. -assertMissing({a: {$exists: false}}); -assertMissing({a: {$not: {$exists: true}}}); -assertMissing({$and: [{a: {$exists: false}}]}); -assertMissing({$or: [{a: {$exists: false}}]}); -assertMissing({$nor: [{a: {$exists: true}}]}); -assertMissing({'a.x': {$exists: false}}, 2, 1); - -// Currently a sparse index is disallowed even if the $exists:false query is on a different field. -assertMissing({b: {$exists: false}}, 2, 1); -assertMissing({b: {$exists: false}, a: {$ne: 6}}, 2, 1); -assertMissing({b: {$not: {$exists: true}}}, 2, 1); - -// Top level $exists:true queries match the proper number of documents -// and use the sparse index on { a : 1 }. -assertExists({a: {$exists: true}}); - -// Nested $exists queries match the proper number of documents and disallow the sparse index. -assertExistsUnindexed({$nor: [{a: {$exists: false}}]}); -assertExistsUnindexed({$nor: [{'a.x': {$exists: false}}]}, 1); -assertExistsUnindexed({a: {$not: {$exists: false}}}); - -// Nested $exists queries disallow the sparse index in some cases where it is not strictly -// necessary to do so. (Descriptive tests.) -assertExistsUnindexed({$nor: [{b: {$exists: false}}]}, 1); // Unindexed field. -assertExists({$or: [{a: {$exists: true}}]}); // $exists:true not $exists:false. - -// Behavior is similar with $elemMatch. -t.drop(); -t.save({a: [{}]}); -t.save({a: [{b: 1}]}); -t.save({a: [{b: 1}]}); -setIndex('a.b'); - -assertMissing({a: {$elemMatch: {b: {$exists: false}}}}); -// A $elemMatch predicate is treated as nested, and the index should be used for $exists:true. -assertExists({a: {$elemMatch: {b: {$exists: true}}}}); - -// A non sparse index will not be disallowed. -t.drop(); -t.save({}); -t.ensureIndex({a: 1}); -assert.eq(1, t.find({a: {$exists: false}}).itcount()); +/** + * Tests that sparse indexes are disallowed for $exists:false queries. + */ +(function() { + "use strict"; + + const coll = db.jstests_existsa; + coll.drop(); + + assert.writeOK(coll.insert({})); + assert.writeOK(coll.insert({a: 1})); + assert.writeOK(coll.insert({a: {x: 1}, b: 1})); + + let indexKeySpec = {}; + let indexKeyField = ''; + + /** Configure testing of an index { :1 }. */ + function setIndex(_indexKeyField) { + indexKeyField = _indexKeyField; + indexKeySpec = {}; + indexKeySpec[indexKeyField] = 1; + coll.ensureIndex(indexKeySpec, {sparse: true}); + } + setIndex('a'); + + /** @return count when hinting the index to use. */ + function hintedCount(query) { + return coll.find(query).hint(indexKeySpec).itcount(); + } + + /** The query field does not exist and the sparse index is not used without a hint. */ + function assertMissing(query, expectedMissing = 1, expectedIndexedMissing = 0) { + assert.eq(expectedMissing, coll.count(query)); + // We also shouldn't get a different count depending on whether + // an index is used or not. + assert.eq(expectedIndexedMissing, hintedCount(query)); + } + + /** The query field exists and the sparse index is used without a hint. */ + function assertExists(query, expectedExists = 2) { + assert.eq(expectedExists, coll.count(query)); + // An $exists:true predicate generates no index filters. Add another predicate on the index + // key to trigger use of the index. + let andClause = {}; + andClause[indexKeyField] = {$ne: null}; + Object.extend(query, {$and: [andClause]}); + assert.eq(expectedExists, coll.count(query)); + assert.eq(expectedExists, hintedCount(query)); + } + + /** The query field exists and the sparse index is not used without a hint. */ + function assertExistsUnindexed(query, expectedExists = 2) { + assert.eq(expectedExists, coll.count(query)); + // Even with another predicate on the index key, the sparse index is disallowed. + let andClause = {}; + andClause[indexKeyField] = {$ne: null}; + Object.extend(query, {$and: [andClause]}); + assert.eq(expectedExists, coll.count(query)); + assert.eq(expectedExists, hintedCount(query)); + } + + // $exists:false queries match the proper number of documents and disallow the sparse index. + assertMissing({a: {$exists: false}}); + assertMissing({a: {$not: {$exists: true}}}); + assertMissing({$and: [{a: {$exists: false}}]}); + assertMissing({$or: [{a: {$exists: false}}]}); + assertMissing({$nor: [{a: {$exists: true}}]}); + assertMissing({'a.x': {$exists: false}}, 2, 1); + + // Currently a sparse index is disallowed even if the $exists:false query is on a different + // field. + assertMissing({b: {$exists: false}}, 2, 1); + assertMissing({b: {$exists: false}, a: {$ne: 6}}, 2, 1); + assertMissing({b: {$not: {$exists: true}}}, 2, 1); + + // Top level $exists:true queries match the proper number of documents + // and use the sparse index on { a : 1 }. + assertExists({a: {$exists: true}}); + + // Nested $exists queries match the proper number of documents and disallow the sparse index. + assertExistsUnindexed({$nor: [{a: {$exists: false}}]}); + assertExistsUnindexed({$nor: [{'a.x': {$exists: false}}]}, 1); + assertExistsUnindexed({a: {$not: {$exists: false}}}); + + // Nested $exists queries disallow the sparse index in some cases where it is not strictly + // necessary to do so. (Descriptive tests.) + assertExistsUnindexed({$nor: [{b: {$exists: false}}]}, 1); // Unindexed field. + assertExists({$or: [{a: {$exists: true}}]}); // $exists:true not $exists:false. + + // Behavior is similar with $elemMatch. + coll.drop(); + assert.writeOK(coll.insert({a: [{}]})); + assert.writeOK(coll.insert({a: [{b: 1}]})); + assert.writeOK(coll.insert({a: [{b: [1]}]})); + setIndex('a.b'); + + assertMissing({a: {$elemMatch: {b: {$exists: false}}}}); + + // A $elemMatch predicate is treated as nested, and the index should be used for $exists:true. + assertExists({a: {$elemMatch: {b: {$exists: true}}}}); + + // A $not within $elemMatch should not attempt to use a sparse index for $exists:false. + assertExistsUnindexed({'a.b': {$elemMatch: {$not: {$exists: false}}}}, 1); + assertExistsUnindexed({'a.b': {$elemMatch: {$gt: 0, $not: {$exists: false}}}}, 1); + + // A non sparse index will not be disallowed. + coll.drop(); + assert.writeOK(coll.insert({})); + coll.ensureIndex({a: 1}); + assert.eq(1, coll.find({a: {$exists: false}}).itcount()); +})(); diff --git a/jstests/core/fsync.js b/jstests/core/fsync.js index 13ff20e4177..2ad625d88ba 100644 --- a/jstests/core/fsync.js +++ b/jstests/core/fsync.js @@ -6,6 +6,8 @@ * - Confirm that writes can progress after fsyncUnlock * - Confirm that the command can be run repeatedly without breaking things * - Confirm that the pseudo commands and eval can perform fsyncLock/Unlock + * + * @tags: [requires_eval_command] */ (function() { "use strict"; diff --git a/jstests/core/fts_dotted_prefix_fields.js b/jstests/core/fts_dotted_prefix_fields.js new file mode 100644 index 00000000000..f811c4a7203 --- /dev/null +++ b/jstests/core/fts_dotted_prefix_fields.js @@ -0,0 +1,15 @@ +// Test that text search works correct when the text index has dotted paths as the non-text +// prefixes. +(function() { + "use strict"; + + let coll = db.fts_dotted_prefix_fields; + coll.drop(); + assert.commandWorked(coll.createIndex({"a.x": 1, "a.y": 1, "b.x": 1, "b.y": 1, words: "text"})); + assert.writeOK(coll.insert({a: {x: 1, y: 2}, b: {x: 3, y: 4}, words: "lorem ipsum dolor sit"})); + assert.writeOK(coll.insert({a: {x: 1, y: 2}, b: {x: 5, y: 4}, words: "lorem ipsum dolor sit"})); + + assert.eq(1, + coll.find({$text: {$search: "lorem ipsum"}, "a.x": 1, "a.y": 2, "b.x": 3, "b.y": 4}) + .itcount()); +}()); diff --git a/jstests/core/fts_trailing_fields.js b/jstests/core/fts_trailing_fields.js new file mode 100644 index 00000000000..2c7f79b423d --- /dev/null +++ b/jstests/core/fts_trailing_fields.js @@ -0,0 +1,22 @@ +// Tests for predicates which can use the trailing field of a text index. +(function() { + "use strict"; + + const coll = db.fts_trailing_fields; + + coll.drop(); + assert.commandWorked(coll.createIndex({a: 1, b: "text", c: 1})); + assert.writeOK(coll.insert({a: 2, b: "lorem ipsum"})); + + assert.eq(0, coll.find({a: 2, $text: {$search: "lorem"}, c: {$exists: true}}).itcount()); + assert.eq(1, coll.find({a: 2, $text: {$search: "lorem"}, c: null}).itcount()); + assert.eq(1, coll.find({a: 2, $text: {$search: "lorem"}, c: {$exists: false}}).itcount()); + + // An equality predicate on the leading field isn't useful, but it shouldn't cause any problems. + // Same with an $elemMatch predicate on one of the trailing fields. + coll.drop(); + assert.commandWorked(coll.createIndex({a: 1, b: "text", "c.d": 1})); + assert.writeOK(coll.insert({a: 2, b: "lorem ipsum", c: {d: 3}})); + assert.eq(0, coll.find({a: [1, 2], $text: {$search: "lorem"}}).itcount()); + assert.eq(0, coll.find({a: 2, $text: {$search: "lorem"}, c: {$elemMatch: {d: 3}}}).itcount()); +}()); diff --git a/jstests/core/function_string_representations.js b/jstests/core/function_string_representations.js new file mode 100644 index 00000000000..af66e9160a9 --- /dev/null +++ b/jstests/core/function_string_representations.js @@ -0,0 +1,38 @@ +/** Demonstrate that mapReduce can accept functions represented by strings. + * Some drivers do not have a type which represents a Javascript function. These languages represent + * the arguments to mapReduce as strings. + */ + +(function() { + "use strict"; + + var col = db.function_string_representations; + col.drop(); + assert.writeOK(col.insert({ + _id: "abc123", + ord_date: new Date("Oct 04, 2012"), + status: 'A', + price: 25, + items: [{sku: "mmm", qty: 5, price: 2.5}, {sku: "nnn", qty: 5, price: 2.5}] + })); + + var mapFunction = "function() {emit(this._id, this.price);}"; + var reduceFunction = "function(keyCustId, valuesPrices) {return Array.sum(valuesPrices);}"; + assert.commandWorked(col.mapReduce(mapFunction, reduceFunction, {out: "map_reduce_example"})); + + // Provided strings may end with semicolons and/or whitespace + mapFunction += " ; "; + reduceFunction += " ; "; + assert.commandWorked(col.mapReduce(mapFunction, reduceFunction, {out: "map_reduce_example"})); + + // $where exhibits the same behavior + var whereFunction = "function() {return this.price === 25;}"; + assert.eq(1, col.find({$where: whereFunction}).itcount()); + + whereFunction += ";"; + assert.eq(1, col.find({$where: whereFunction}).itcount()); + + // db.eval does not need to be tested, as it accepts code fragments, not functions. + // system.js does not need to be tested, as its contents types' are preserved, and + // strings are not promoted into functions. +})(); diff --git a/jstests/core/geo_2d_trailing_fields.js b/jstests/core/geo_2d_trailing_fields.js new file mode 100644 index 00000000000..8f9c881ae4c --- /dev/null +++ b/jstests/core/geo_2d_trailing_fields.js @@ -0,0 +1,45 @@ +// Tests for predicates which can use the trailing field of a 2d index. +(function() { + "use strict"; + + const coll = db.geo_2d_trailing_fields; + + const isMaster = assert.commandWorked(db.adminCommand({isMaster: 1})); + const isMongos = (isMaster.msg === "isdbgrid"); + + coll.drop(); + assert.commandWorked(coll.createIndex({a: "2d", b: 1})); + assert.writeOK(coll.insert({a: [0, 0]})); + + // Verify that $near queries handle existence predicates over the trailing fields correctly. + if (!isMongos) { + assert.eq(0, coll.find({a: {$near: [0, 0]}, b: {$exists: true}}).itcount()); + assert.eq(1, coll.find({a: {$near: [0, 0]}, b: null}).itcount()); + assert.eq(1, coll.find({a: {$near: [0, 0]}, b: {$exists: false}}).itcount()); + } + + // Verify that non-near 2d queries handle existence predicates over the trailing fields + // correctly. + assert.eq(0, + coll.find({a: {$geoWithin: {$center: [[0, 0], 1]}}, b: {$exists: true}}).itcount()); + assert.eq(1, coll.find({a: {$geoWithin: {$center: [[0, 0], 1]}}, b: null}).itcount()); + assert.eq(1, + coll.find({a: {$geoWithin: {$center: [[0, 0], 1]}}, b: {$exists: false}}).itcount()); + + coll.drop(); + assert.commandWorked(coll.createIndex({a: "2d", "b.c": 1})); + assert.writeOK(coll.insert({a: [0, 0], b: [{c: 2}, {c: 3}]})); + + // Verify that $near queries correctly handle predicates which cannot be covered due to array + // semantics. + if (!isMongos) { + assert.eq(0, coll.find({a: {$near: [0, 0]}, "b.c": [2, 3]}).itcount()); + assert.eq(0, coll.find({a: {$near: [0, 0]}, "b.c": {$type: "array"}}).itcount()); + } + + // Verify that non-near 2d queries correctly handle predicates which cannot be covered due to + // array semantics. + assert.eq(0, coll.find({a: {$geoWithin: {$center: [[0, 0], 1]}}, "b.c": [2, 3]}).itcount()); + assert.eq( + 0, coll.find({a: {$geoWithin: {$center: [[0, 0], 1]}}, "b.c": {$type: "array"}}).itcount()); +}()); diff --git a/jstests/core/geo_s2cursorlimitskip.js b/jstests/core/geo_s2cursorlimitskip.js index 427fbf8fe29..dc645dc68af 100644 --- a/jstests/core/geo_s2cursorlimitskip.js +++ b/jstests/core/geo_s2cursorlimitskip.js @@ -1,4 +1,11 @@ // Test various cursor behaviors +// +// @tags: [ +// # This test attempts to enable profiling on a server and then get profiling data by reading +// # from the "system.profile" collection. The former operation must be routed to the primary in +// # a replica set, whereas the latter may be routed to a secondary. +// assumes_read_preference_unchanged, +// ] var testDB = db.getSiblingDB("geo_s2cursorlimitskip"); var t = testDB.geo_s2getmmm; diff --git a/jstests/core/geo_update_btree.js b/jstests/core/geo_update_btree.js index a85d4274415..b4fa24df57e 100644 --- a/jstests/core/geo_update_btree.js +++ b/jstests/core/geo_update_btree.js @@ -1,4 +1,6 @@ // Tests whether the geospatial search is stable under btree updates +// +// @tags: [assumes_write_concern_unchanged] var coll = db.getCollection("jstests_geo_update_btree"); coll.drop(); diff --git a/jstests/core/getlog2.js b/jstests/core/getlog2.js index 597a85e20ee..e5287ea8c1b 100644 --- a/jstests/core/getlog2.js +++ b/jstests/core/getlog2.js @@ -1,4 +1,11 @@ // tests getlog as well as slow querying logging +// +// @tags: [ +// # This test attempts to perform a find command and see that it ran using the getLog command. +// # The former operation may be routed to a secondary in the replica set, whereas the latter must +// # be routed to the primary. +// assumes_read_preference_unchanged, +// ] glcol = db.getLogTest2; glcol.drop(); diff --git a/jstests/core/index_elemmatch2.js b/jstests/core/index_elemmatch2.js new file mode 100644 index 00000000000..ecd24035284 --- /dev/null +++ b/jstests/core/index_elemmatch2.js @@ -0,0 +1,63 @@ +/** + * Test that queries containing $elemMatch correctly use an index if each child expression is + * compatible with the index. + */ +(function() { + "use strict"; + + load("jstests/libs/analyze_plan.js"); + + const coll = db.elemMatch_index; + coll.drop(); + + assert.writeOK(coll.insert({a: 1})); + assert.writeOK(coll.insert({a: [{}]})); + assert.writeOK(coll.insert({a: [1, null]})); + assert.writeOK(coll.insert({a: [{type: "Point", coordinates: [0, 0]}]})); + + assert.commandWorked(coll.createIndex({a: 1}, {sparse: true})); + + function assertIndexResults(coll, query, useIndex, nReturned) { + const explainPlan = coll.find(query).explain("executionStats"); + assert.eq(isIxscan(explainPlan.queryPlanner.winningPlan), useIndex); + assert.eq(explainPlan.executionStats.nReturned, nReturned); + } + + assertIndexResults(coll, {a: {$elemMatch: {$exists: false}}}, false, 0); + + // An $elemMatch predicate is treated as nested, and the index should be used for $exists:true. + assertIndexResults(coll, {a: {$elemMatch: {$exists: true}}}, true, 3); + + // $not within $elemMatch should not attempt to use a sparse index for $exists:false. + assertIndexResults(coll, {a: {$elemMatch: {$not: {$exists: false}}}}, false, 3); + assertIndexResults(coll, {a: {$elemMatch: {$gt: 0, $not: {$exists: false}}}}, false, 1); + + // $geo within $elemMatch should not attempt to use a non-geo index. + assertIndexResults( + coll, + { + a: { + $elemMatch: { + $geoWithin: { + $geometry: + {type: "Polygon", coordinates: [[[0, 0], [0, 1], [1, 0], [0, 0]]]} + } + } + } + }, + false, + 1); + + // $in with a null value within $elemMatch should use a sparse index. + assertIndexResults(coll, {a: {$elemMatch: {$in: [null]}}}, true, 1); + + // $eq with a null value within $elemMatch should use a sparse index. + assertIndexResults(coll, {a: {$elemMatch: {$eq: null}}}, true, 1); + + // A negated regex within $elemMatch should not use an index, sparse or not. + assertIndexResults(coll, {a: {$elemMatch: {$not: {$in: [/^a/]}}}}, false, 3); + + coll.dropIndexes(); + assert.commandWorked(coll.createIndex({a: 1})); + assertIndexResults(coll, {a: {$elemMatch: {$not: {$in: [/^a/]}}}}, false, 3); +})(); diff --git a/jstests/core/index_filter_commands.js b/jstests/core/index_filter_commands.js index 8684be3b2b9..58f78d0514e 100644 --- a/jstests/core/index_filter_commands.js +++ b/jstests/core/index_filter_commands.js @@ -6,20 +6,24 @@ * Displays index filters for all query shapes in a collection. * * - planCacheClearFilters - * Clears index filter for a single query shape or, - * if the query shape is omitted, all filters for the collection. + * Clears index filter for a single query shape or, if the query shape is omitted, all filters for + * the collection. * * - planCacheSetFilter * Sets index filter for a query shape. Overrides existing filter. * - * Not a lot of data access in this test suite. Hint commands - * manage a non-persistent mapping in the server of - * query shape to list of index specs. + * Not a lot of data access in this test suite. Hint commands manage a non-persistent mapping in the + * server of query shape to list of index specs. * - * Only time we might need to execute a query is to check the plan - * cache state. We would do this with the planCacheListPlans command - * on the same query shape with the index filters. + * Only time we might need to execute a query is to check the plan cache state. We would do this + * with the planCacheListPlans command on the same query shape with the index filters. * + * @tags: [ + * # This test attempts to perform queries with plan cache filters set up. The former operation + * # may be routed to a secondary in the replica set, whereas the latter must be routed to the + * # primary. + * assumes_read_preference_unchanged, + * ] */ load("jstests/libs/analyze_plan.js"); diff --git a/jstests/core/index_stats.js b/jstests/core/index_stats.js index 60b37fd571e..ee4d13d4d0a 100644 --- a/jstests/core/index_stats.js +++ b/jstests/core/index_stats.js @@ -1,3 +1,10 @@ +// @tags: [ +// # This test attempts to perform write operations and get index usage statistics using the +// # $indexStats stage. The former operation must be routed to the primary in a replica set, +// # whereas the latter may be routed to a secondary. +// assumes_read_preference_unchanged, +// ] + (function() { "use strict"; diff --git a/jstests/core/js3.js b/jstests/core/js3.js index 4d46c25bbf7..c808e7ec75a 100644 --- a/jstests/core/js3.js +++ b/jstests/core/js3.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.jstests_js3; diff --git a/jstests/core/js7.js b/jstests/core/js7.js index 810f4692d4f..99cddd114f8 100644 --- a/jstests/core/js7.js +++ b/jstests/core/js7.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + t = db.jstests_js7; t.drop(); diff --git a/jstests/core/js9.js b/jstests/core/js9.js index 515fa883aea..e0703ac39ea 100644 --- a/jstests/core/js9.js +++ b/jstests/core/js9.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + c = db.jstests_js9; c.drop(); diff --git a/jstests/core/js_jit.js b/jstests/core/js_jit.js new file mode 100644 index 00000000000..4ccdd2917ae --- /dev/null +++ b/jstests/core/js_jit.js @@ -0,0 +1,40 @@ +/** + * Validate various native types continue to work when run in JITed code. + * + * In SERVER-30362, the JIT would not compile natives types which had custom getProperty + * implementations correctly. We force the JIT to kick in by using large loops. + */ +(function() { + 'use strict'; + + function testDBCollection() { + const c = new DBCollection(null, null, "foo", "test.foo"); + for (let i = 0; i < 100000; i++) { + if (c.toString() != "test.foo") { + throw i; + } + } + } + + function testDB() { + const c = new DB(null, "test"); + for (let i = 0; i < 100000; i++) { + if (c.toString() != "test") { + throw i; + } + } + } + + function testDBQuery() { + const c = DBQuery('a', 'b', 'c', 'd'); + for (let i = 0; i < 100000; i++) { + if (c.toString() != "DBQuery: d -> null") { + throw i; + } + } + } + + testDBCollection(); + testDB(); + testDBQuery(); +})(); \ No newline at end of file diff --git a/jstests/core/list_collections1.js b/jstests/core/list_collections1.js index c8c3f92fbc9..ff1aac304ff 100644 --- a/jstests/core/list_collections1.js +++ b/jstests/core/list_collections1.js @@ -72,8 +72,8 @@ // var getListCollectionsCursor = function(options, subsequentBatchSize) { - return new DBCommandCursor( - mydb.getMongo(), mydb.runCommand("listCollections", options), subsequentBatchSize); + var res = mydb.runCommand("listCollections", options); + return new DBCommandCursor(res._mongo, res, subsequentBatchSize); }; var cursorCountMatching = function(cursor, pred) { @@ -282,9 +282,9 @@ assert.commandWorked(mydb.createCollection("quux")); res = mydb.runCommand("listCollections", {cursor: {batchSize: 0}}); - cursor = new DBCommandCursor(mydb.getMongo(), res, 2); + cursor = new DBCommandCursor(res._mongo, res, 2); cursor.close(); - cursor = new DBCommandCursor(mydb.getMongo(), res, 2); + cursor = new DBCommandCursor(res._mongo, res, 2); assert.throws(function() { cursor.hasNext(); }); diff --git a/jstests/core/list_collections_filter.js b/jstests/core/list_collections_filter.js index e0d18f055d0..0516d57a417 100644 --- a/jstests/core/list_collections_filter.js +++ b/jstests/core/list_collections_filter.js @@ -19,8 +19,8 @@ filter = {}; } - var cursor = new DBCommandCursor(mydb.getMongo(), - mydb.runCommand("listCollections", {filter: filter})); + var res = mydb.runCommand("listCollections", {filter: filter}); + var cursor = new DBCommandCursor(res._mongo, res); function stripToName(result) { return result.name; } diff --git a/jstests/core/list_indexes.js b/jstests/core/list_indexes.js index de0f4473980..6ec9a7a13e1 100644 --- a/jstests/core/list_indexes.js +++ b/jstests/core/list_indexes.js @@ -27,8 +27,8 @@ // var getListIndexesCursor = function(coll, options, subsequentBatchSize) { - return new DBCommandCursor( - coll.getDB().getMongo(), coll.runCommand("listIndexes", options), subsequentBatchSize); + var res = coll.runCommand("listIndexes", options); + return new DBCommandCursor(res._mongo, res, subsequentBatchSize); }; var cursorGetIndexSpecs = function(cursor) { @@ -163,9 +163,9 @@ assert.commandWorked(coll.ensureIndex({c: 1}, {unique: true})); res = coll.runCommand("listIndexes", {cursor: {batchSize: 0}}); - cursor = new DBCommandCursor(coll.getDB().getMongo(), res, 2); + cursor = new DBCommandCursor(res._mongo, res, 2); cursor.close(); - cursor = new DBCommandCursor(coll.getDB().getMongo(), res, 2); + cursor = new DBCommandCursor(res._mongo, res, 2); assert.throws(function() { cursor.hasNext(); }); diff --git a/jstests/core/list_indexes_invalidation.js b/jstests/core/list_indexes_invalidation.js index b8cbe5eb134..9fe94de5efc 100644 --- a/jstests/core/list_indexes_invalidation.js +++ b/jstests/core/list_indexes_invalidation.js @@ -19,7 +19,7 @@ printjson(res); // Ensure the cursor has data, rename or drop the collection, and exhaust the cursor. - let cursor = new DBCommandCursor(db.getMongo(), res); + let cursor = new DBCommandCursor(res._mongo, res); let errMsg = 'expected more data from command ' + tojson(cmd) + ', with result ' + tojson(res); assert(cursor.hasNext(), errMsg); diff --git a/jstests/core/list_namespaces_invalidation.js b/jstests/core/list_namespaces_invalidation.js index 6f8033b5fe4..85eb26510bb 100644 --- a/jstests/core/list_namespaces_invalidation.js +++ b/jstests/core/list_namespaces_invalidation.js @@ -1,4 +1,6 @@ // SERVER-27996/SERVER-28022 Missing invalidation for system.namespaces writes +// +// @tags: [requires_collmod_command] (function() { 'use strict'; let dbInvalidName = 'system_namespaces_invalidations'; diff --git a/jstests/core/max_time_ms.js b/jstests/core/max_time_ms.js index 0442ffcba68..efe3dabcc6d 100644 --- a/jstests/core/max_time_ms.js +++ b/jstests/core/max_time_ms.js @@ -1,4 +1,12 @@ // Tests query/command option $maxTimeMS. +// +// @tags: [ +// # This test attempts to perform read operations after having enabled the maxTimeAlwaysTimeOut +// # failpoint. The former operations may be routed to a secondary in the replica set, whereas the +// # latter must be routed to the primary. +// assumes_read_preference_unchanged, +// requires_collmod_command, +// ] var t = db.max_time_ms; var exceededTimeLimit = 50; // ErrorCodes::ExceededTimeLimit diff --git a/jstests/core/mr4.js b/jstests/core/mr4.js index 58ea303f7e8..683583acecd 100644 --- a/jstests/core/mr4.js +++ b/jstests/core/mr4.js @@ -9,7 +9,7 @@ t.save({x: 4, tags: ["b", "c"]}); m = function() { this.tags.forEach(function(z) { - emit(z, {count: xx}); + emit(z, {count: xx.val}); }); }; @@ -21,7 +21,7 @@ r = function(key, values) { return {count: total}; }; -res = t.mapReduce(m, r, {out: "mr4_out", scope: {xx: 1}}); +res = t.mapReduce(m, r, {out: "mr4_out", scope: {xx: {val: 1}}}); z = res.convertToSingleObject(); assert.eq(3, Object.keySet(z).length, "A1"); @@ -31,7 +31,7 @@ assert.eq(3, z.c.count, "A4"); res.drop(); -res = t.mapReduce(m, r, {scope: {xx: 2}, out: "mr4_out"}); +res = t.mapReduce(m, r, {scope: {xx: {val: 2}}, out: "mr4_out"}); z = res.convertToSingleObject(); assert.eq(3, Object.keySet(z).length, "A1"); diff --git a/jstests/core/mr_killop.js b/jstests/core/mr_killop.js index 78e98f0bcaa..e85986e2fc0 100644 --- a/jstests/core/mr_killop.js +++ b/jstests/core/mr_killop.js @@ -4,7 +4,7 @@ t = db.jstests_mr_killop; t.drop(); t2 = db.jstests_mr_killop_out; t2.drop(); - +db.adminCommand({"configureFailPoint": 'mr_killop_test_fp', "mode": 'alwaysOn'}); function debug(x) { // printjson( x ); } @@ -171,3 +171,4 @@ var loop = function() { }; runMRTests(loop, false); runFinalizeTests(loop, false); +db.adminCommand({"configureFailPoint": 'mr_killop_test_fp', "mode": 'off'}); diff --git a/jstests/core/mr_optim.js b/jstests/core/mr_optim.js index 7437753ca67..1c525ae3de3 100644 --- a/jstests/core/mr_optim.js +++ b/jstests/core/mr_optim.js @@ -3,8 +3,17 @@ t = db.mr_optim; t.drop(); +// We drop the output collection to ensure the test can be run multiple times successfully. We +// explicitly avoid using the DBCollection#drop() shell helper to avoid implicitly sharding the +// collection during the sharded_collections_jscore_passthrough.yml test suite when reading the +// results from the output collection in the reformat() function. +var res = db.runCommand({drop: "mr_optim_out"}); +if (res.ok !== 1) { + assert.commandFailedWithCode(res, ErrorCodes.NamespaceNotFound); +} + for (var i = 0; i < 1000; ++i) { - t.save({a: Math.random(1000), b: Math.random(10000)}); + assert.writeOK(t.save({a: Math.random(1000), b: Math.random(10000)})); } function m() { @@ -21,7 +30,7 @@ function reformat(r) { if (r.results) cursor = r.results; else - cursor = r.find(); + cursor = r.find().sort({_id: 1}); cursor.forEach(function(z) { x[z._id] = z.value; }); @@ -43,4 +52,4 @@ res.drop(); assert.eq(x, x2, "object from inline and collection are not equal"); -t.drop(); \ No newline at end of file +t.drop(); diff --git a/jstests/core/no_db_created.js b/jstests/core/no_db_created.js index 3491914d470..b67b193494c 100644 --- a/jstests/core/no_db_created.js +++ b/jstests/core/no_db_created.js @@ -1,4 +1,6 @@ // checks that operations do not create a database +// +// @tags: [requires_collmod_command] (function() { "use strict"; @@ -32,4 +34,4 @@ noDB(mydb); assert.writeOK(coll.insert({})); mydb.dropDatabase(); -}()); \ No newline at end of file +}()); diff --git a/jstests/core/notablescan.js b/jstests/core/notablescan.js index 80306c08cf2..bb4c170a603 100644 --- a/jstests/core/notablescan.js +++ b/jstests/core/notablescan.js @@ -1,4 +1,11 @@ // check notablescan mode +// +// @tags: [ +// # This test attempts to perform read operations after having enabled the notablescan server +// # parameter. The former operations may be routed to a secondary in the replica set, whereas the +// # latter must be routed to the primary. +// assumes_read_preference_unchanged, +// ] t = db.test_notablescan; t.drop(); diff --git a/jstests/core/operation_latency_histogram.js b/jstests/core/operation_latency_histogram.js index 1e3f1a59b95..947a8be6520 100644 --- a/jstests/core/operation_latency_histogram.js +++ b/jstests/core/operation_latency_histogram.js @@ -1,4 +1,10 @@ // Checks that histogram counters for collections are updated as we expect. +// +// This test attempts to perform write operations and get latency statistics using the $collStats +// stage. The former operation must be routed to the primary in a replica set, whereas the latter +// may be routed to a secondary. +// +// @tags: [assumes_read_preference_unchanged] (function() { "use strict"; diff --git a/jstests/core/plan_cache_clear.js b/jstests/core/plan_cache_clear.js index 8f9cf0ea302..778239616b5 100644 --- a/jstests/core/plan_cache_clear.js +++ b/jstests/core/plan_cache_clear.js @@ -1,5 +1,12 @@ // Test clearing of the plan cache, either manually through the planCacheClear command, // or due to system events such as an index build. +// +// @tags: [ +// # This test attempts to perform queries and introspect/manipulate the server's plan cache +// # entries. The former operation may be routed to a secondary in the replica set, whereas the +// # latter must be routed to the primary. +// assumes_read_preference_unchanged, +// ] var t = db.jstests_plan_cache_clear; t.drop(); diff --git a/jstests/core/plan_cache_list_plans.js b/jstests/core/plan_cache_list_plans.js index 7ca599483ff..3359980ab07 100644 --- a/jstests/core/plan_cache_list_plans.js +++ b/jstests/core/plan_cache_list_plans.js @@ -1,4 +1,11 @@ // Test the planCacheListPlans command. +// +// @tags: [ +// # This test attempts to perform queries and introspect the server's plan cache entries. The +// # former operation may be routed to a secondary in the replica set, whereas the latter must be +// # routed to the primary. +// assumes_read_preference_unchanged, +// ] var t = db.jstests_plan_cache_list_plans; t.drop(); diff --git a/jstests/core/plan_cache_list_shapes.js b/jstests/core/plan_cache_list_shapes.js index 1c9ecdf9e1b..61c3111cd8a 100644 --- a/jstests/core/plan_cache_list_shapes.js +++ b/jstests/core/plan_cache_list_shapes.js @@ -1,5 +1,12 @@ // Test the planCacheListQueryShapes command, which returns a list of query shapes // for the queries currently cached in the collection. +// +// @tags: [ +// # This test attempts to perform queries with plan cache filters set up. The former operation +// # may be routed to a secondary in the replica set, whereas the latter must be routed to the +// # primary. +// assumes_read_preference_unchanged, +// ] var t = db.jstests_plan_cache_list_shapes; t.drop(); diff --git a/jstests/core/plan_cache_shell_helpers.js b/jstests/core/plan_cache_shell_helpers.js index dc990b19dcc..6c4ff185014 100644 --- a/jstests/core/plan_cache_shell_helpers.js +++ b/jstests/core/plan_cache_shell_helpers.js @@ -1,4 +1,11 @@ // Test the shell helpers which wrap the plan cache commands. +// +// @tags: [ +// # This test attempts to perform queries and introspect the server's plan cache entries. The +// # former operation may be routed to a secondary in the replica set, whereas the latter must be +// # routed to the primary. +// assumes_read_preference_unchanged, +// ] var t = db.jstests_plan_cache_shell_helpers; t.drop(); diff --git a/jstests/core/profile2.js b/jstests/core/profile2.js index bb1605abd1e..9da6410d2ac 100644 --- a/jstests/core/profile2.js +++ b/jstests/core/profile2.js @@ -24,7 +24,7 @@ assert(result.hasOwnProperty('millis')); assert(result.hasOwnProperty('query')); assert.eq('string', typeof(result.query)); // String value is truncated. -assert(result.query.match(/filter: { a: "a+\.\.\." } }$/)); +assert(result.query.match(/filter: { a: "a+\.\.\." }/)); assert.commandWorked(coll.getDB().runCommand({profile: 0})); coll.getDB().system.profile.drop(); diff --git a/jstests/core/profile_getmore.js b/jstests/core/profile_getmore.js index a9272567b1a..3f6f492597e 100644 --- a/jstests/core/profile_getmore.js +++ b/jstests/core/profile_getmore.js @@ -24,12 +24,13 @@ var cursor = coll.find({a: {$gt: 0}}).sort({a: 1}).batchSize(2); cursor.next(); // Perform initial query and consume first of 2 docs returned. - var cursorId = getLatestProfilerEntry(testDB).cursorid; // Save cursorid from find. + var cursorId = + getLatestProfilerEntry(testDB, {op: "query"}).cursorid; // Save cursorid from find. cursor.next(); // Consume second of 2 docs from initial query. cursor.next(); // getMore performed, leaving open cursor. - var profileObj = getLatestProfilerEntry(testDB); + var profileObj = getLatestProfilerEntry(testDB, {op: "getmore"}); assert.eq(profileObj.ns, coll.getFullName(), tojson(profileObj)); assert.eq(profileObj.op, "getmore", tojson(profileObj)); @@ -67,7 +68,7 @@ cursor.next(); // Consume second of 2 docs from initial query. cursor.next(); // getMore performed, leaving open cursor. - profileObj = getLatestProfilerEntry(testDB); + profileObj = getLatestProfilerEntry(testDB, {op: "getmore"}); assert.eq(profileObj.hasSortStage, true, tojson(profileObj)); @@ -83,7 +84,7 @@ cursor.next(); // Perform initial query and consume first of 3 docs returned. cursor.itcount(); // Exhaust the cursor. - profileObj = getLatestProfilerEntry(testDB); + profileObj = getLatestProfilerEntry(testDB, {op: "getmore"}); assert(profileObj.hasOwnProperty("cursorid"), tojson(profileObj)); // cursorid should always be present on getMore. @@ -101,12 +102,12 @@ assert.commandWorked(coll.createIndex({a: 1})); var cursor = coll.aggregate([{$match: {a: {$gte: 0}}}], {cursor: {batchSize: 0}}); - var cursorId = getLatestProfilerEntry(testDB).cursorid; + var cursorId = getLatestProfilerEntry(testDB, {"command.aggregate": coll.getName()}).cursorid; assert.neq(0, cursorId); cursor.next(); // Consume the result set. - profileObj = getLatestProfilerEntry(testDB); + profileObj = getLatestProfilerEntry(testDB, {op: "getmore"}); assert.eq(profileObj.ns, coll.getFullName(), tojson(profileObj)); assert.eq(profileObj.op, "getmore", tojson(profileObj)); diff --git a/jstests/core/profile_insert.js b/jstests/core/profile_insert.js index 3994c896bda..836ec2ecff2 100644 --- a/jstests/core/profile_insert.js +++ b/jstests/core/profile_insert.js @@ -1,4 +1,6 @@ // Confirms that profiled insert execution contains all expected metrics with proper values. +// +// @tags: [assumes_write_concern_unchanged] (function() { "use strict"; diff --git a/jstests/core/recursion.js b/jstests/core/recursion.js index 926250be20d..1db491c8ae9 100644 --- a/jstests/core/recursion.js +++ b/jstests/core/recursion.js @@ -1,5 +1,7 @@ -// Basic tests for a form of stack recursion that's been shown to cause C++ -// side stack overflows in the past. See SERVER-19614. +// Basic tests for a form of stack recursion that's been shown to cause C++ side stack overflows in +// the past. See SERVER-19614. +// +// @tags: [requires_eval_command] (function() { "use strict"; diff --git a/jstests/core/regex_not_id.js b/jstests/core/regex_not_id.js index 1f15250f240..35b2c858867 100644 --- a/jstests/core/regex_not_id.js +++ b/jstests/core/regex_not_id.js @@ -3,10 +3,10 @@ var testColl = db.regex_not_id; testColl.drop(); -assert.writeOK(testColl.insert({_id: "ABCDEF1"}, {writeConcern: {w: 1}})); +assert.writeOK(testColl.insert({_id: "ABCDEF1"})); // Should be an error. -assert.writeError(testColl.insert({_id: /^A/}, {writeConcern: {w: 1}})); +assert.writeError(testColl.insert({_id: /^A/})); // _id doesn't have to be first; still disallowed -assert.writeError(testColl.insert({xxx: "ABCDEF", _id: /ABCDEF/}, {writeConcern: {w: 1}})); \ No newline at end of file +assert.writeError(testColl.insert({xxx: "ABCDEF", _id: /ABCDEF/})); diff --git a/jstests/core/remove8.js b/jstests/core/remove8.js index 563e4708cf9..3c9fd6a11a1 100644 --- a/jstests/core/remove8.js +++ b/jstests/core/remove8.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.remove8; t.drop(); diff --git a/jstests/core/rename4.js b/jstests/core/rename4.js index 185193deaa9..756918db5f6 100644 --- a/jstests/core/rename4.js +++ b/jstests/core/rename4.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + t = db.jstests_rename4; t.drop(); diff --git a/jstests/core/rename_stayTemp.js b/jstests/core/rename_stayTemp.js index d8451af2d2d..8dee8f89fea 100644 --- a/jstests/core/rename_stayTemp.js +++ b/jstests/core/rename_stayTemp.js @@ -11,7 +11,7 @@ function ns(coll) { function istemp(name) { var result = db.runCommand("listCollections", {filter: {name: name}}); assert(result.ok); - var collections = new DBCommandCursor(db.getMongo(), result).toArray(); + var collections = new DBCommandCursor(result._mongo, result).toArray(); assert.eq(1, collections.length); return collections[0].options.temp ? true : false; } diff --git a/jstests/core/shell_connection_strings.js b/jstests/core/shell_connection_strings.js new file mode 100644 index 00000000000..22861e6ce25 --- /dev/null +++ b/jstests/core/shell_connection_strings.js @@ -0,0 +1,33 @@ +// Test mongo shell connect strings. +(function() { + 'use strict'; + + const mongod = new MongoURI(db.getMongo().host).servers[0]; + const host = mongod.host; + const port = mongod.port; + + function testConnect(ok, ...args) { + const exitCode = runMongoProgram('mongo', '--eval', ';', ...args); + if (ok) { + assert.eq(exitCode, 0, "failed to connect with `" + args.join(' ') + "`"); + } else { + assert.neq( + exitCode, 0, "unexpectedly succeeded connecting with `" + args.join(' ') + "`"); + } + } + + testConnect(true, `${host}:${port}`); + testConnect(true, `${host}:${port}/test`); + testConnect(true, `${host}:${port}/admin`); + testConnect(true, host, '--port', port); + testConnect(true, '--host', host, '--port', port, 'test'); + testConnect(true, '--host', host, '--port', port, 'admin'); + testConnect(true, `mongodb://${host}:${port}/test`); + testConnect(true, `mongodb://${host}:${port}/test?connectTimeoutMS=10000`); + + // if a full URI is provided, you cannot also specify host or port + testConnect(false, `${host}/test`, '--port', port); + testConnect(false, `mongodb://${host}:${port}/test`, '--port', port); + testConnect(false, `mongodb://${host}:${port}/test`, '--host', host); + testConnect(false, `mongodb://${host}:${port}/test`, '--host', host, '--port', port); +})(); diff --git a/jstests/core/shell_writeconcern.js b/jstests/core/shell_writeconcern.js index f3f190061cf..e3e7a23a9aa 100644 --- a/jstests/core/shell_writeconcern.js +++ b/jstests/core/shell_writeconcern.js @@ -1,7 +1,10 @@ "use strict"; + // check that shell writeconcern work correctly // 1.) tests that it can be set on each level and is inherited // 2.) tests that each operation (update/insert/remove/save) take and ensure a write concern +// +// @tags: [assumes_write_concern_unchanged] var collA = db.shell_wc_a; var collB = db.shell_wc_b; diff --git a/jstests/core/stages_delete.js b/jstests/core/stages_delete.js index f8e7380c75a..98a270cedf3 100644 --- a/jstests/core/stages_delete.js +++ b/jstests/core/stages_delete.js @@ -1,3 +1,9 @@ +// @tags: [ +// # This test attempts to remove documents using the stageDebug command, which doesn't support +// # specifying a writeConcern. +// assumes_write_concern_unchanged, +// ] + // Test basic delete stage functionality. var coll = db.stages_delete; var collScanStage = {cscan: {args: {direction: 1}, filter: {deleteMe: true}}}; diff --git a/jstests/core/startup_log.js b/jstests/core/startup_log.js index 3b0cbe3464d..c73013d1744 100644 --- a/jstests/core/startup_log.js +++ b/jstests/core/startup_log.js @@ -1,101 +1,108 @@ -load('jstests/aggregation/extras/utils.js'); - -(function() { - 'use strict'; - - // Check that smallArray is entirely contained by largeArray - // returns false if a member of smallArray is not in largeArray - function arrayIsSubset(smallArray, largeArray) { - for (var i = 0; i < smallArray.length; i++) { - if (!Array.contains(largeArray, smallArray[i])) { - print("Could not find " + smallArray[i] + " in largeArray"); - return false; - } - } - - return true; - } - - // Test startup_log - var stats = db.getSisterDB("local").startup_log.stats(); - assert(stats.capped); - - var latestStartUpLog = - db.getSisterDB("local").startup_log.find().sort({$natural: -1}).limit(1).next(); - var serverStatus = db._adminCommand("serverStatus"); - var cmdLine = db._adminCommand("getCmdLineOpts").parsed; - - // Test that the startup log has the expected keys - var verbose = false; - var expectedKeys = - ["_id", "hostname", "startTime", "startTimeLocal", "cmdLine", "pid", "buildinfo"]; - var keys = Object.keySet(latestStartUpLog); - assert(arrayEq(expectedKeys, keys, verbose), 'startup_log keys failed'); - - // Tests _id implicitly - should be comprised of host-timestamp - // Setup expected startTime and startTimeLocal from the supplied timestamp - var _id = latestStartUpLog._id.split('-'); // _id should consist of host-timestamp - var _idUptime = _id.pop(); - var _idHost = _id.join('-'); - var uptimeSinceEpochRounded = Math.floor(_idUptime / 1000) * 1000; - var startTime = new Date(uptimeSinceEpochRounded); // Expected startTime - - assert.eq(_idHost, latestStartUpLog.hostname, "Hostname doesn't match one from _id"); - assert.eq(serverStatus.host.split(':')[0], - latestStartUpLog.hostname, - "Hostname doesn't match one in server status"); - assert.closeWithinMS(startTime, - latestStartUpLog.startTime, - "StartTime doesn't match one from _id", - 2000); // Expect less than 2 sec delta - assert.eq(cmdLine, latestStartUpLog.cmdLine, "cmdLine doesn't match that from getCmdLineOpts"); - assert.eq(serverStatus.pid, latestStartUpLog.pid, "pid doesn't match that from serverStatus"); - - // Test buildinfo - var buildinfo = db.runCommand("buildinfo"); - delete buildinfo.ok; // Delete extra meta info not in startup_log - var isMaster = db._adminCommand("ismaster"); - - // Test buildinfo has the expected keys - var expectedKeys = [ - "version", - "gitVersion", - "allocator", - "versionArray", - "javascriptEngine", - "openssl", - "buildEnvironment", - "debug", - "maxBsonObjectSize", - "bits", - "modules" - ]; - - var keys = Object.keySet(latestStartUpLog.buildinfo); - // Disabled to check - assert(arrayIsSubset(expectedKeys, keys), - "buildinfo keys failed! \n expected:\t" + expectedKeys + "\n actual:\t" + keys); - assert.eq(buildinfo, - latestStartUpLog.buildinfo, - "buildinfo doesn't match that from buildinfo command"); - - // Test version and version Array - var version = latestStartUpLog.buildinfo.version.split('-')[0]; - var versionArray = latestStartUpLog.buildinfo.versionArray; - var versionArrayCleaned = versionArray.slice(0, 3); - if (versionArray[3] == -100) { - versionArrayCleaned[2] -= 1; - } - - assert.eq(serverStatus.version, - latestStartUpLog.buildinfo.version, - "Mongo version doesn't match that from ServerStatus"); - assert.eq( - version, versionArrayCleaned.join('.'), "version doesn't match that from the versionArray"); - var jsEngine = latestStartUpLog.buildinfo.javascriptEngine; - assert((jsEngine == "none") || jsEngine.startsWith("mozjs")); - assert.eq(isMaster.maxBsonObjectSize, - latestStartUpLog.buildinfo.maxBsonObjectSize, - "maxBsonObjectSize doesn't match one from ismaster"); - -})(); +/** + * This test attempts to read from the "local.startup_log" collection and assert that it has an + * entry matching the server's response from the "getCmdLineOpts" command. The former operation may + * be routed to a secondary in the replica set, whereas the latter must be routed to the primary. + * + * @tags: [assumes_read_preference_unchanged] + */ +load('jstests/aggregation/extras/utils.js'); + +(function() { + 'use strict'; + + // Check that smallArray is entirely contained by largeArray + // returns false if a member of smallArray is not in largeArray + function arrayIsSubset(smallArray, largeArray) { + for (var i = 0; i < smallArray.length; i++) { + if (!Array.contains(largeArray, smallArray[i])) { + print("Could not find " + smallArray[i] + " in largeArray"); + return false; + } + } + + return true; + } + + // Test startup_log + var stats = db.getSisterDB("local").startup_log.stats(); + assert(stats.capped); + + var latestStartUpLog = + db.getSisterDB("local").startup_log.find().sort({$natural: -1}).limit(1).next(); + var serverStatus = db._adminCommand("serverStatus"); + var cmdLine = db._adminCommand("getCmdLineOpts").parsed; + + // Test that the startup log has the expected keys + var verbose = false; + var expectedKeys = + ["_id", "hostname", "startTime", "startTimeLocal", "cmdLine", "pid", "buildinfo"]; + var keys = Object.keySet(latestStartUpLog); + assert(arrayEq(expectedKeys, keys, verbose), 'startup_log keys failed'); + + // Tests _id implicitly - should be comprised of host-timestamp + // Setup expected startTime and startTimeLocal from the supplied timestamp + var _id = latestStartUpLog._id.split('-'); // _id should consist of host-timestamp + var _idUptime = _id.pop(); + var _idHost = _id.join('-'); + var uptimeSinceEpochRounded = Math.floor(_idUptime / 1000) * 1000; + var startTime = new Date(uptimeSinceEpochRounded); // Expected startTime + + assert.eq(_idHost, latestStartUpLog.hostname, "Hostname doesn't match one from _id"); + assert.eq(serverStatus.host.split(':')[0], + latestStartUpLog.hostname, + "Hostname doesn't match one in server status"); + assert.closeWithinMS(startTime, + latestStartUpLog.startTime, + "StartTime doesn't match one from _id", + 2000); // Expect less than 2 sec delta + assert.eq(cmdLine, latestStartUpLog.cmdLine, "cmdLine doesn't match that from getCmdLineOpts"); + assert.eq(serverStatus.pid, latestStartUpLog.pid, "pid doesn't match that from serverStatus"); + + // Test buildinfo + var buildinfo = db.runCommand("buildinfo"); + delete buildinfo.ok; // Delete extra meta info not in startup_log + var isMaster = db._adminCommand("ismaster"); + + // Test buildinfo has the expected keys + var expectedKeys = [ + "version", + "gitVersion", + "allocator", + "versionArray", + "javascriptEngine", + "openssl", + "buildEnvironment", + "debug", + "maxBsonObjectSize", + "bits", + "modules" + ]; + + var keys = Object.keySet(latestStartUpLog.buildinfo); + // Disabled to check + assert(arrayIsSubset(expectedKeys, keys), + "buildinfo keys failed! \n expected:\t" + expectedKeys + "\n actual:\t" + keys); + assert.eq(buildinfo, + latestStartUpLog.buildinfo, + "buildinfo doesn't match that from buildinfo command"); + + // Test version and version Array + var version = latestStartUpLog.buildinfo.version.split('-')[0]; + var versionArray = latestStartUpLog.buildinfo.versionArray; + var versionArrayCleaned = versionArray.slice(0, 3); + if (versionArray[3] == -100) { + versionArrayCleaned[2] -= 1; + } + + assert.eq(serverStatus.version, + latestStartUpLog.buildinfo.version, + "Mongo version doesn't match that from ServerStatus"); + assert.eq( + version, versionArrayCleaned.join('.'), "version doesn't match that from the versionArray"); + var jsEngine = latestStartUpLog.buildinfo.javascriptEngine; + assert((jsEngine == "none") || jsEngine.startsWith("mozjs")); + assert.eq(isMaster.maxBsonObjectSize, + latestStartUpLog.buildinfo.maxBsonObjectSize, + "maxBsonObjectSize doesn't match one from ismaster"); + +})(); diff --git a/jstests/core/storefunc.js b/jstests/core/storefunc.js index 8598e9cc62b..15abc56421e 100644 --- a/jstests/core/storefunc.js +++ b/jstests/core/storefunc.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + // Use a private sister database to avoid conflicts with other tests that use system.js var testdb = db.getSisterDB("storefunc"); var res; diff --git a/jstests/core/top.js b/jstests/core/top.js index 819b41b0981..3d98f5a7b2d 100644 --- a/jstests/core/top.js +++ b/jstests/core/top.js @@ -1,5 +1,11 @@ /** * 1. check top numbers are correct + * + * This test attempts to perform read operations and get statistics using the top command. The + * former operation may be routed to a secondary in the replica set, whereas the latter must be + * routed to the primary. + * + * @tags: [assumes_read_preference_unchanged] */ (function() { load("jstests/libs/stats.js"); diff --git a/jstests/core/update_affects_indexes.js b/jstests/core/update_affects_indexes.js new file mode 100644 index 00000000000..91db3ebe565 --- /dev/null +++ b/jstests/core/update_affects_indexes.js @@ -0,0 +1,100 @@ +// This is a regression test for SERVER-32048. It checks that index keys are correctly updated when +// an update modifier implicitly creates a new array element. +(function() { + "use strict"; + + let coll = db.update_affects_indexes; + coll.drop(); + let indexKeyPattern = {"a.b": 1}; + assert.commandWorked(coll.createIndex(indexKeyPattern)); + + // Tests that the document 'docId' has all the index keys in 'expectedKeys' and none of the + // index keys in 'unexpectedKeys'. + function assertExpectedIndexKeys(docId, expectedKeys, unexpectedKeys) { + for (let key of expectedKeys) { + let res = coll.find(docId).hint(indexKeyPattern).min(key).returnKey().toArray(); + assert.eq(1, res.length, tojson(res)); + assert.eq(key, res[0]); + } + + for (let key of unexpectedKeys) { + let res = coll.find(docId).hint(indexKeyPattern).min(key).returnKey().toArray(); + if (res.length > 0) { + assert.eq(1, res.length, tojson(res)); + assert.neq(key, res[0]); + } + } + } + + // $set implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 0, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 0}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 0}, {$set: {"a.1.c": 0}})); + assertExpectedIndexKeys({_id: 0}, [{"a.b": 0}, {"a.b": null}], []); + + // $set implicitly creates array element beyond end of array. + assert.writeOK(coll.insert({_id: 1, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 1}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 1}, {$set: {"a.3.c": 0}})); + assertExpectedIndexKeys({_id: 1}, [{"a.b": 0}, {"a.b": null}], []); + + // $set implicitly creates array element in empty array (no index key changes needed). + assert.writeOK(coll.insert({_id: 2, a: []})); + assertExpectedIndexKeys({_id: 2}, [{"a.b": null}], []); + assert.writeOK(coll.update({_id: 2}, {$set: {"a.0.c": 0}})); + assertExpectedIndexKeys({_id: 2}, [{"a.b": null}], []); + + // $inc implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 3, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 3}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 3}, {$inc: {"a.1.c": 0}})); + assertExpectedIndexKeys({_id: 3}, [{"a.b": 0}, {"a.b": null}], []); + + // $mul implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 4, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 4}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 4}, {$mul: {"a.1.c": 0}})); + assertExpectedIndexKeys({_id: 4}, [{"a.b": 0}, {"a.b": null}], []); + + // $addToSet implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 5, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 5}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 5}, {$addToSet: {"a.1.c": 0}})); + assertExpectedIndexKeys({_id: 5}, [{"a.b": 0}, {"a.b": null}], []); + + // $bit implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 6, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 6}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 6}, {$bit: {"a.1.c": {and: NumberInt(1)}}})); + assertExpectedIndexKeys({_id: 6}, [{"a.b": 0}, {"a.b": null}], []); + + // $min implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 7, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 7}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 7}, {$min: {"a.1.c": 0}})); + assertExpectedIndexKeys({_id: 7}, [{"a.b": 0}, {"a.b": null}], []); + + // $max implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 8, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 8}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 8}, {$max: {"a.1.c": 0}})); + assertExpectedIndexKeys({_id: 8}, [{"a.b": 0}, {"a.b": null}], []); + + // $currentDate implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 9, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 9}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 9}, {$currentDate: {"a.1.c": true}})); + assertExpectedIndexKeys({_id: 9}, [{"a.b": 0}, {"a.b": null}], []); + + // $push implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 10, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 10}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 10}, {$push: {"a.1.c": 0}})); + assertExpectedIndexKeys({_id: 10}, [{"a.b": 0}, {"a.b": null}], []); + + // $pushAll implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 11, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 11}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 11}, {$pushAll: {"a.1.c": [0]}})); + assertExpectedIndexKeys({_id: 11}, [{"a.b": 0}, {"a.b": null}], []); +}()); diff --git a/jstests/core/update_multi5.js b/jstests/core/update_multi5.js index e610462a620..9e9550bc554 100644 --- a/jstests/core/update_multi5.js +++ b/jstests/core/update_multi5.js @@ -7,8 +7,8 @@ assert.writeOK(t.insert({path: 'r1', subscribers: [1, 2]})); assert.writeOK(t.insert({path: 'r2', subscribers: [3, 4]})); - var res = assert.writeOK(t.update( - {}, {$addToSet: {subscribers: 5}}, {upsert: false, multi: true, writeConcern: {w: 1}})); + var res = + assert.writeOK(t.update({}, {$addToSet: {subscribers: 5}}, {upsert: false, multi: true})); assert.eq(res.nMatched, 2, tojson(res)); diff --git a/jstests/core/views/invalid_system_views.js b/jstests/core/views/invalid_system_views.js index 3ba282d2ca1..49ac600ba8b 100644 --- a/jstests/core/views/invalid_system_views.js +++ b/jstests/core/views/invalid_system_views.js @@ -1,6 +1,8 @@ /** * Tests that invalid view definitions in system.views do not impact valid commands on existing * collections. + * + * @tags: [requires_collmod_command] */ (function() { "use strict"; diff --git a/jstests/core/views/views_all_commands.js b/jstests/core/views/views_all_commands.js index 0115672dcda..e1798803a1b 100644 --- a/jstests/core/views/views_all_commands.js +++ b/jstests/core/views/views_all_commands.js @@ -48,6 +48,8 @@ * * skipStandalone * If true, do not run this command on a standalone mongod. + * + * @tags: [requires_collmod_command] */ (function() { diff --git a/jstests/core/views/views_basic.js b/jstests/core/views/views_basic.js index d6032b105df..b4a2f177982 100644 --- a/jstests/core/views/views_basic.js +++ b/jstests/core/views/views_basic.js @@ -13,7 +13,7 @@ let res = viewsDB.runCommand(cmd); assert.commandWorked(res); - let cursor = new DBCommandCursor(db.getMongo(), res, 5); + let cursor = new DBCommandCursor(res._mongo, res, 5); let actual = cursor.toArray(); assert(arrayEq(actual, expected), "actual: " + tojson(cursor.toArray()) + ", expected:" + tojson(expected)); diff --git a/jstests/core/views/views_change.js b/jstests/core/views/views_change.js index 002284095c5..a62e7e7f04f 100644 --- a/jstests/core/views/views_change.js +++ b/jstests/core/views/views_change.js @@ -1,6 +1,7 @@ /** * Tests the behavior of views when the backing view or collection is changed. - * @tags: [requires_find_command] + * + * @tags: [requires_collmod_command, requires_find_command] */ (function() { "use strict"; diff --git a/jstests/core/views/views_collation.js b/jstests/core/views/views_collation.js index 9e4ed7feb30..38557358ce9 100644 --- a/jstests/core/views/views_collation.js +++ b/jstests/core/views/views_collation.js @@ -1,9 +1,13 @@ /** * Tests the behavior of operations when interacting with a view's default collation. + * + * @tags: [requires_collmod_command] */ (function() { "use strict"; + load("jstests/libs/analyze_plan.js"); + let viewsDB = db.getSiblingDB("views_collation"); assert.commandWorked(viewsDB.dropDatabase()); assert.commandWorked(viewsDB.runCommand({create: "simpleCollection"})); @@ -58,6 +62,15 @@ assert.commandWorked(viewsDB.runCommand({count: "filView"})); assert.commandWorked(viewsDB.runCommand({distinct: "filView", key: "x"})); + // Explain of operations that do not specify a collation succeed. + assert.commandWorked(viewsDB.runCommand({aggregate: "filView", pipeline: [], explain: true})); + assert.commandWorked( + viewsDB.runCommand({explain: {find: "filView"}, verbosity: "allPlansExecution"})); + assert.commandWorked( + viewsDB.runCommand({explain: {count: "filView"}, verbosity: "allPlansExecution"})); + assert.commandWorked(viewsDB.runCommand( + {explain: {distinct: "filView", key: "x"}, verbosity: "allPlansExecution"})); + // Operations with a matching collation succeed. assert.commandWorked( viewsDB.runCommand({aggregate: "filView", pipeline: [], collation: {locale: "fil"}})); @@ -66,6 +79,18 @@ assert.commandWorked( viewsDB.runCommand({distinct: "filView", key: "x", collation: {locale: "fil"}})); + // Explain of operations with a matching collation succeed. + assert.commandWorked(viewsDB.runCommand( + {aggregate: "filView", pipeline: [], explain: true, collation: {locale: "fil"}})); + assert.commandWorked(viewsDB.runCommand( + {explain: {find: "filView", collation: {locale: "fil"}}, verbosity: "allPlansExecution"})); + assert.commandWorked(viewsDB.runCommand( + {explain: {count: "filView", collation: {locale: "fil"}}, verbosity: "allPlansExecution"})); + assert.commandWorked(viewsDB.runCommand({ + explain: {distinct: "filView", key: "x", collation: {locale: "fil"}}, + verbosity: "allPlansExecution" + })); + // Attempting to override the non-simple default collation of a view fails. assert.commandFailedWithCode( viewsDB.runCommand({aggregate: "filView", pipeline: [], collation: {locale: "en"}}), @@ -90,6 +115,46 @@ viewsDB.runCommand({distinct: "filView", key: "x", collation: {locale: "simple"}}), ErrorCodes.OptionNotSupportedOnView); + // Attempting to override the default collation of a view with explain fails. + assert.commandFailedWithCode( + viewsDB.runCommand( + {aggregate: "filView", pipeline: [], explain: true, collation: {locale: "en"}}), + ErrorCodes.OptionNotSupportedOnView); + assert.commandFailedWithCode( + viewsDB.runCommand( + {aggregate: "filView", pipeline: [], explain: true, collation: {locale: "simple"}}), + ErrorCodes.OptionNotSupportedOnView); + assert.commandFailedWithCode(viewsDB.runCommand({ + explain: {find: "filView", collation: {locale: "fr"}}, + verbosity: "allPlansExecution" + }), + ErrorCodes.OptionNotSupportedOnView); + assert.commandFailedWithCode(viewsDB.runCommand({ + explain: {find: "filView", collation: {locale: "simple"}}, + verbosity: "allPlansExecution" + }), + ErrorCodes.OptionNotSupportedOnView); + assert.commandFailedWithCode(viewsDB.runCommand({ + explain: {count: "filView", collation: {locale: "zh"}}, + verbosity: "allPlansExecution" + }), + ErrorCodes.OptionNotSupportedOnView); + assert.commandFailedWithCode(viewsDB.runCommand({ + explain: {count: "filView", collation: {locale: "simple"}}, + verbosity: "allPlansExecution" + }), + ErrorCodes.OptionNotSupportedOnView); + assert.commandFailedWithCode(viewsDB.runCommand({ + explain: {distinct: "filView", key: "x", collation: {locale: "es"}}, + verbosity: "allPlansExecution" + }), + ErrorCodes.OptionNotSupportedOnView); + assert.commandFailedWithCode(viewsDB.runCommand({ + explain: {distinct: "filView", key: "x", collation: {locale: "simple"}}, + verbosity: "allPlansExecution" + }), + ErrorCodes.OptionNotSupportedOnView); + const lookupSimpleView = { $lookup: {from: "simpleView", localField: "x", foreignField: "x", as: "result"} }; @@ -253,4 +318,56 @@ viewsDB.runCommand( {collMod: "esView", viewOn: "simpleCollection", pipeline: [graphLookupFilView]}), ErrorCodes.OptionNotSupportedOnView); + + // Make sure that when an operation does not specify the collation, it correctly uses the + // default collation associated with the view. For this, we set up a new backing collection with + // a case-insensitive view. + assert.commandWorked(viewsDB.runCommand({create: "case_sensitive_coll"})); + assert.commandWorked(viewsDB.runCommand({ + create: "case_insensitive_view", + viewOn: "case_sensitive_coll", + collation: {locale: "en", strength: 1} + })); + + assert.writeOK(viewsDB.case_sensitive_coll.insert({f: "case"})); + assert.writeOK(viewsDB.case_sensitive_coll.insert({f: "Case"})); + assert.writeOK(viewsDB.case_sensitive_coll.insert({f: "CASE"})); + + let explain, cursorStage; + + // Test that aggregate against a view with a default collation correctly uses the collation. + assert.eq(1, viewsDB.case_sensitive_coll.aggregate([{$match: {f: "case"}}]).itcount()); + assert.eq(3, viewsDB.case_insensitive_view.aggregate([{$match: {f: "case"}}]).itcount()); + explain = viewsDB.case_insensitive_view.explain().aggregate([{$match: {f: "case"}}]); + cursorStage = getAggPlanStage(explain, "$cursor"); + assert.neq(null, cursorStage, tojson(explain)); + assert.eq(1, cursorStage.$cursor.queryPlanner.collation.strength, tojson(cursorStage)); + + // Test that count against a view with a default collation correctly uses the collation. + assert.eq(1, viewsDB.case_sensitive_coll.count({f: "case"})); + assert.eq(3, viewsDB.case_insensitive_view.count({f: "case"})); + explain = viewsDB.case_insensitive_view.explain().count({f: "case"}); + cursorStage = getAggPlanStage(explain, "$cursor"); + assert.neq(null, cursorStage, tojson(explain)); + assert.eq(1, cursorStage.$cursor.queryPlanner.collation.strength, tojson(cursorStage)); + + // Test that distinct against a view with a default collation correctly uses the collation. + assert.eq(3, viewsDB.case_sensitive_coll.distinct("f").length); + assert.eq(1, viewsDB.case_insensitive_view.distinct("f").length); + explain = viewsDB.case_insensitive_view.explain().distinct("f"); + cursorStage = getAggPlanStage(explain, "$cursor"); + assert.neq(null, cursorStage, tojson(explain)); + assert.eq(1, cursorStage.$cursor.queryPlanner.collation.strength, tojson(cursorStage)); + + // Test that find against a view with a default collation correctly uses the collation. + let findRes = viewsDB.runCommand({find: "case_sensitive_coll", filter: {f: "case"}}); + assert.commandWorked(findRes); + assert.eq(1, findRes.cursor.firstBatch.length); + findRes = viewsDB.runCommand({find: "case_insensitive_view", filter: {f: "case"}}); + assert.commandWorked(findRes); + assert.eq(3, findRes.cursor.firstBatch.length); + explain = viewsDB.runCommand({explain: {find: "case_insensitive_view", filter: {f: "case"}}}); + cursorStage = getAggPlanStage(explain, "$cursor"); + assert.neq(null, cursorStage, tojson(explain)); + assert.eq(1, cursorStage.$cursor.queryPlanner.collation.strength, tojson(cursorStage)); }()); diff --git a/jstests/core/views/views_find.js b/jstests/core/views/views_find.js index c196e980ce5..53c9d4e8b98 100644 --- a/jstests/core/views/views_find.js +++ b/jstests/core/views/views_find.js @@ -15,7 +15,7 @@ let assertFindResultEq = function(cmd, expected, ordered) { let res = viewsDB.runCommand(cmd); assert.commandWorked(res); - let arr = new DBCommandCursor(db.getMongo(), res, 5).toArray(); + let arr = new DBCommandCursor(res._mongo, res, 5).toArray(); let errmsg = tojson({expected: expected, got: arr}); if (typeof(ordered) === "undefined" || !ordered) diff --git a/jstests/core/views/views_rename.js b/jstests/core/views/views_rename.js new file mode 100644 index 00000000000..ad656671ea1 --- /dev/null +++ b/jstests/core/views/views_rename.js @@ -0,0 +1,17 @@ +(function() { + // SERVER-30406 Test that renaming system.views correctly invalidates the view catalog + 'use strict'; + db.view.drop(); + db.coll.drop(); + assert.commandWorked(db.createView("view", "coll", [])); + assert.writeOK(db.coll.insert({_id: 1})); + assert.eq(db.view.find().count(), 1, "couldn't find document in view"); + assert.commandWorked(db.system.views.renameCollection("views", /*dropTarget*/ true)); + assert.eq(db.view.find().count(), + 0, + "find on view should have returned no results after renaming away system.views"); + assert.commandWorked(db.views.renameCollection("system.views")); + assert.eq(db.view.find().count(), + 1, + "find on view should have worked again after renaming system.views back in place"); +})(); diff --git a/jstests/core/views/views_stats.js b/jstests/core/views/views_stats.js index 75feb857c9a..22261e9fa81 100644 --- a/jstests/core/views/views_stats.js +++ b/jstests/core/views/views_stats.js @@ -1,4 +1,10 @@ // Test that top and latency histogram statistics are recorded for views. +// +// This test attempts to perform write operations and get latency statistics using the $collStats +// stage. The former operation must be routed to the primary in a replica set, whereas the latter +// may be routed to a secondary. +// +// @tags: [assumes_read_preference_unchanged] (function() { "use strict"; diff --git a/jstests/core/views/views_validation.js b/jstests/core/views/views_validation.js index 84c7f1d3510..b32750a082f 100644 --- a/jstests/core/views/views_validation.js +++ b/jstests/core/views/views_validation.js @@ -1,3 +1,4 @@ +// @tags: [requires_collmod_command] (function() { "use strict"; let viewsDb = db.getSiblingDB("views_validation"); diff --git a/jstests/core/write_result.js b/jstests/core/write_result.js index 86486089c68..453be4ca1c9 100644 --- a/jstests/core/write_result.js +++ b/jstests/core/write_result.js @@ -1,6 +1,8 @@ // // Tests the behavior of single writes using write commands // +// @tags: [assumes_write_concern_unchanged] +// var coll = db.write_result; coll.drop(); diff --git a/jstests/hooks/validate_collections.js b/jstests/hooks/validate_collections.js index aeb38a98bf5..e64bb39d305 100644 --- a/jstests/hooks/validate_collections.js +++ b/jstests/hooks/validate_collections.js @@ -71,6 +71,21 @@ function validateCollections(db, obj) { filter = {$or: [filter, {type: {$exists: false}}]}; } + // Optionally skip collections. + if (Array.isArray(jsTest.options().skipValidationNamespaces) && + jsTest.options().skipValidationNamespaces.length > 0) { + let skippedCollections = []; + for (let ns of jsTest.options().skipValidationNamespaces) { + // Strip off the database name from 'ns' to extract the collName. + const collName = ns.replace(new RegExp('^' + db.getName() + '\.'), ''); + // Skip the collection 'collName' if the db name was removed from 'ns'. + if (collName !== ns) { + skippedCollections.push({name: {$ne: collName}}); + } + } + filter = {$and: [filter, ...skippedCollections]}; + } + let collInfo = db.getCollectionInfos(filter); for (var collDocument of collInfo) { var coll = db.getCollection(collDocument["name"]); diff --git a/jstests/libs/analyze_plan.js b/jstests/libs/analyze_plan.js index 62511ae1fac..5e52ad6aba3 100644 --- a/jstests/libs/analyze_plan.js +++ b/jstests/libs/analyze_plan.js @@ -115,3 +115,74 @@ function getChunkSkips(root) { return 0; } + +/** + * Given the root stage of agg explain's JSON representation of a query plan ('root'), returns all + * subdocuments whose stage is 'stage'. This can either be an agg stage name like "$cursor" or + * "$sort", or a query stage name like "IXSCAN" or "SORT". + * + * Returns an empty array if the plan does not have the requested stage. Asserts that agg explain + * structure matches expected format. + */ +function getAggPlanStages(root, stage) { + let results = []; + + function getDocumentSources(docSourceArray) { + let results = []; + for (let i = 0; i < docSourceArray.length; i++) { + let properties = Object.getOwnPropertyNames(docSourceArray[i]); + assert.eq(1, properties.length); + if (properties[0] === stage) { + results.push(docSourceArray[i]); + } + } + return results; + } + + if (root.hasOwnProperty("stages")) { + assert(root.stages.constructor === Array); + + results = results.concat(getDocumentSources(root.stages)); + + assert(root.stages[0].hasOwnProperty("$cursor")); + assert(root.stages[0].$cursor.hasOwnProperty("queryPlanner")); + assert(root.stages[0].$cursor.queryPlanner.hasOwnProperty("winningPlan")); + results = + results.concat(getPlanStages(root.stages[0].$cursor.queryPlanner.winningPlan, stage)); + } + + if (root.hasOwnProperty("shards")) { + for (let elem in root.shards) { + assert(root.shards[elem].stages.constructor === Array); + + results = results.concat(getDocumentSources(root.shards[elem].stages)); + + assert(root.shards[elem].stages[0].hasOwnProperty("$cursor")); + assert(root.shards[elem].stages[0].$cursor.hasOwnProperty("queryPlanner")); + assert(root.shards[elem].stages[0].$cursor.queryPlanner.hasOwnProperty("winningPlan")); + results = results.concat( + getPlanStages(root.shards[elem].stages[0].$cursor.queryPlanner.winningPlan, stage)); + } + } + + return results; +} + +/** + * Given the root stage of agg explain's JSON representation of a query plan ('root'), returns the + * subdocument with its stage as 'stage'. Returns null if the plan does not have such a stage. + * Asserts that no more than one stage is a match. + */ +function getAggPlanStage(root, stage) { + let planStageList = getAggPlanStages(root, stage); + + if (planStageList.length === 0) { + return null; + } else { + assert.eq(1, + planStageList.length, + "getAggPlanStage expects to find 0 or 1 matching stages. planStageList: " + + tojson(planStageList)); + return planStageList[0]; + } +} diff --git a/jstests/libs/ftdc.js b/jstests/libs/ftdc.js new file mode 100644 index 00000000000..7d327d39852 --- /dev/null +++ b/jstests/libs/ftdc.js @@ -0,0 +1,102 @@ +/** + * Utility test functions for FTDC + */ +'use strict'; + +/** + * Verify that getDiagnosticData is working correctly. + */ +function verifyGetDiagnosticData(adminDb) { + // We need to retry a few times if run this test immediately after mongod is started as FTDC may + // not have run yet. + var foundGoodDocument = false; + + for (var i = 0; i < 60 && foundGoodDocument == false; ++i) { + var result = adminDb.runCommand("getDiagnosticData"); + assert.commandWorked(result); + + var data = result.data; + + if (!data.hasOwnProperty("start")) { + // Wait a little longer for FTDC to start + jsTestLog("Running getDiagnosticData: " + tojson(result)); + + sleep(500); + } else { + // Check for a few common properties to ensure we got data + assert(data.hasOwnProperty("serverStatus"), + "does not have 'serverStatus' in '" + tojson(data) + "'"); + assert(data.hasOwnProperty("end"), "does not have 'end' in '" + tojson(data) + "'"); + foundGoodDocument = true; + + jsTestLog("Got good getDiagnosticData: " + tojson(result)); + } + } + + assert(foundGoodDocument, + "getDiagnosticData failed to return a non-empty command, is FTDC running?"); +} + +/** + * Validate all the common FTDC parameters are set correctly and can be manipulated. + */ +function verifyCommonFTDCParameters(adminDb, isEnabled) { + // Are we running against MongoS? + var isMongos = ("isdbgrid" == adminDb.runCommand("ismaster").msg); + + // Check the defaults are correct + // + function getparam(field) { + var q = {getParameter: 1}; + q[field] = 1; + + var ret = adminDb.runCommand(q); + return ret[field]; + } + + // Verify the defaults are as we documented them + assert.eq(getparam("diagnosticDataCollectionEnabled"), isEnabled); + assert.eq(getparam("diagnosticDataCollectionPeriodMillis"), 1000); + assert.eq(getparam("diagnosticDataCollectionDirectorySizeMB"), 200); + assert.eq(getparam("diagnosticDataCollectionFileSizeMB"), 10); + assert.eq(getparam("diagnosticDataCollectionSamplesPerChunk"), 300); + assert.eq(getparam("diagnosticDataCollectionSamplesPerInterimUpdate"), 10); + + function setparam(obj) { + var ret = adminDb.runCommand(Object.extend({setParameter: 1}, obj)); + return ret; + } + + if (!isMongos) { + // The MongoS specific behavior for diagnosticDataCollectionEnabled is tested in + // ftdc_setdirectory.js. + assert.commandWorked(setparam({"diagnosticDataCollectionEnabled": 1})); + } + assert.commandWorked(setparam({"diagnosticDataCollectionPeriodMillis": 100})); + assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 10})); + assert.commandWorked(setparam({"diagnosticDataCollectionFileSizeMB": 1})); + assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerChunk": 2})); + assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerInterimUpdate": 2})); + + // Negative tests - set values below minimums + assert.commandFailed(setparam({"diagnosticDataCollectionPeriodMillis": 1})); + assert.commandFailed(setparam({"diagnosticDataCollectionDirectorySizeMB": 1})); + assert.commandFailed(setparam({"diagnosticDataCollectionSamplesPerChunk": 1})); + assert.commandFailed(setparam({"diagnosticDataCollectionSamplesPerInterimUpdate": 1})); + + // Negative test - set file size bigger then directory size + assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 10})); + assert.commandFailed(setparam({"diagnosticDataCollectionFileSizeMB": 100})); + + // Negative test - set directory size less then file size + assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 100})); + assert.commandWorked(setparam({"diagnosticDataCollectionFileSizeMB": 50})); + assert.commandFailed(setparam({"diagnosticDataCollectionDirectorySizeMB": 10})); + + // Reset + assert.commandWorked(setparam({"diagnosticDataCollectionFileSizeMB": 10})); + assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 200})); + assert.commandWorked(setparam({"diagnosticDataCollectionPeriodMillis": 1000})); + assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerChunk": 300})); + assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerInterimUpdate": 10})); +} \ No newline at end of file diff --git a/jstests/libs/override_methods/override_helpers.js b/jstests/libs/override_methods/override_helpers.js new file mode 100644 index 00000000000..12bd4b54739 --- /dev/null +++ b/jstests/libs/override_methods/override_helpers.js @@ -0,0 +1,117 @@ +/** + * The OverrideHelpers object defines convenience methods for overriding commands and functions in + * the mongo shell. + */ +var OverrideHelpers = (function() { + "use strict"; + + function isAggregationWithOutStage(commandName, commandObj) { + if (commandName !== "aggregate" || typeof commandObj !== "object" || commandObj === null) { + return false; + } + + if (!Array.isArray(commandObj.pipeline) || commandObj.pipeline.length === 0) { + return false; + } + + const lastStage = commandObj.pipeline[commandObj.pipeline.length - 1]; + if (typeof lastStage !== "object" || lastStage === null) { + return false; + } + + return Object.keys(lastStage)[0] === "$out"; + } + + function isMapReduceWithInlineOutput(commandName, commandObj) { + if ((commandName !== "mapReduce" && commandName !== "mapreduce") || + typeof commandObj !== "object" || commandObj === null) { + return false; + } + + if (typeof commandObj.out !== "object") { + return false; + } + + return commandObj.out.hasOwnProperty("inline"); + } + + function prependOverrideInParallelShell(overrideFile) { + const startParallelShellOriginal = startParallelShell; + + startParallelShell = function(jsCode, port, noConnect) { + let newCode; + if (typeof jsCode === "function") { + // Load the override file and immediately invoke the supplied function. + newCode = `load("${overrideFile}"); (${jsCode})();`; + } else { + newCode = `load("${overrideFile}"); ${jsCode};`; + } + + return startParallelShellOriginal(newCode, port, noConnect); + }; + } + + function overrideRunCommand(overrideFunc) { + const DBQueryOriginal = DBQuery; + const mongoRunCommandOriginal = Mongo.prototype.runCommand; + const mongoRunCommandWithMetadataOriginal = Mongo.prototype.runCommandWithMetadata; + + DBQuery = function( + mongo, db, collection, ns, query, fields, limit, skip, batchSize, options) { + // If the query isn't being run against the "$cmd" or "$cmd.sys" namespaces, then it + // represents an OP_QUERY find on that collection. We skip calling overrideFunc() in + // this case because the operation doesn't represent a command. + if (!(collection instanceof DBCollection && + (collection.getName() === "$cmd" || collection.getName().startsWith("$cmd.")))) { + return DBQueryOriginal.apply(this, arguments); + } + + // Due to the function signatures of Mongo.prototype.runCommand() and + // Mongo.prototype.runCommandWithMetadata(), the overrideFunc() function expects that + // the Mongo connection object is passed as the first argument and also represents the + // 'this' parameter. As a workaround, we bind the appropriate 'this' value to the + // DBQueryOriginal constructor ahead of time. + const commandName = Object.keys(query)[0]; + return overrideFunc( + mongo, + db.getName(), + commandName, + query, + DBQueryOriginal.bind(this), + (query) => + [mongo, db, collection, ns, query, fields, limit, skip, batchSize, options]); + }; + + // Copy any properties (e.g. DBQuery.Option) that are set on DBQueryOriginal. + Object.keys(DBQueryOriginal).forEach(function(key) { + DBQuery[key] = DBQueryOriginal[key]; + }); + + Mongo.prototype.runCommand = function(dbName, commandObj, options) { + const commandName = Object.keys(commandObj)[0]; + return overrideFunc(this, + dbName, + commandName, + commandObj, + mongoRunCommandOriginal, + (commandObj) => [dbName, commandObj, options]); + }; + + Mongo.prototype.runCommandWithMetadata = function(dbName, metadata, commandArgs) { + const commandName = Object.keys(commandArgs)[0]; + return overrideFunc(this, + dbName, + commandName, + commandArgs, + mongoRunCommandWithMetadataOriginal, + (commandArgs) => [dbName, metadata, commandArgs]); + }; + } + + return { + isAggregationWithOutStage: isAggregationWithOutStage, + isMapReduceWithInlineOutput: isMapReduceWithInlineOutput, + prependOverrideInParallelShell: prependOverrideInParallelShell, + overrideRunCommand: overrideRunCommand, + }; +})(); diff --git a/jstests/libs/override_methods/set_majority_read_and_write_concerns.js b/jstests/libs/override_methods/set_majority_read_and_write_concerns.js deleted file mode 100644 index d3bb4449ed4..00000000000 --- a/jstests/libs/override_methods/set_majority_read_and_write_concerns.js +++ /dev/null @@ -1,178 +0,0 @@ -/** - * Use prototype overrides to set a read concern of "majority" and a write concern of "majority" - * while running core tests. - */ -(function() { - "use strict"; - var defaultWriteConcern = { - w: "majority", - // Use a "signature" value that won't typically match a value assigned in normal use. - wtimeout: 60321 - }; - var defaultReadConcern = {level: "majority"}; - - var originalDBQuery = DBQuery; - - DBQuery = function(mongo, db, collection, ns, query, fields, limit, skip, batchSize, options) { - if (ns.endsWith("$cmd")) { - if (query.hasOwnProperty("writeConcern") && - bsonWoCompare(query.writeConcern, defaultWriteConcern) !== 0) { - jsTestLog("Warning: DBQuery overriding existing writeConcern of: " + - tojson(query.writeConcern)); - query.writeConcern = defaultWriteConcern; - } - } - - return originalDBQuery.apply(this, arguments); - }; - - DBQuery.Option = originalDBQuery.Option; - - var originalStartParallelShell = startParallelShell; - startParallelShell = function(jsCode, port, noConnect) { - var newCode; - var overridesFile = "jstests/libs/override_methods/set_majority_read_and_write_concerns.js"; - - if (typeof(jsCode) === "function") { - // Load the override file and immediately invoke the supplied function. - newCode = `load("${overridesFile}"); (${jsCode})();`; - } else { - newCode = `load("${overridesFile}"); ${jsCode};`; - } - - return originalStartParallelShell(newCode, port, noConnect); - }; - - DB.prototype._runCommandImpl = function(dbName, obj, options) { - var cmdName = ""; - for (var fieldName in obj) { - cmdName = fieldName; - break; - } - - // These commands directly support a writeConcern argument. - var commandsToForceWriteConcern = [ - "_mergeAuthzCollections", - "appendOplogNote", - "applyOps", - "authSchemaUpgrade", - "captrunc", - "cleanupOrphaned", - "clone", - "cloneCollection", - "cloneCollectionAsCapped", - // "collMod", SERVER-25196 - not supported - "convertToCapped", - "copydb", - "create", - "createIndexes", - "createRole", - "createUser", - "delete", - "drop", - "dropDatabase", - "dropAllRolesFromDatabase", - "dropAllUsersFromDatabase", - "dropDatabase", - "dropIndexes", - "dropRole", - "dropUser", - "emptycapped", - "findAndModify", - "findandmodify", - "godinsert", - "grantPrivilegesToRole", - "grantRolesToRole", - "grantRolesToUser", - "insert", - "mapReduceFinish", - "mergeAuthzCollections", - "moveChunk", - "movePrimary", - "reIndex", - "remove", - "renameCollection", - "resvChunkStart", - "revokePriviligesFromRole", - "revokeRolesFromRole", - "revokeRolesFromUser", - "update", - "updateRole", - "updateUser", - ]; - - // These are reading commands that support majority readConcern. - var commandsToForceReadConcern = [ - "count", - "distinct", - "find", - "geoNear", - "geoSearch", - "group", - ]; - - var forceWriteConcern = Array.contains(commandsToForceWriteConcern, cmdName); - var forceReadConcern = Array.contains(commandsToForceReadConcern, cmdName); - - if (cmdName === "aggregate") { - // Aggregate can be either a read or a write depending on whether it has a $out stage. - // $out is required to be the last stage of the pipeline. - var stages = obj.pipeline; - const lastStage = stages && Array.isArray(stages) && (stages.length !== 0) - ? stages[stages.length - 1] - : undefined; - const hasOut = - lastStage && (typeof lastStage === 'object') && lastStage.hasOwnProperty('$out'); - if (hasOut) { - forceWriteConcern = true; - } else { - forceReadConcern = true; - } - } - - else if (cmdName === "mapReduce") { - var stages = obj.pipeline; - const lastStage = stages && Array.isArray(stages) && (stages.length !== 0) - ? stages[stages.length - 1] - : undefined; - const hasOut = - lastStage && (typeof lastStage === 'object') && lastStage.hasOwnProperty('$out'); - if (hasOut) { - forceWriteConcern = true; - } - } - - if (forceWriteConcern) { - if (obj.hasOwnProperty("writeConcern")) { - if (bsonWoCompare(obj.writeConcern, defaultWriteConcern) !== 0) { - jsTestLog("Warning: _runCommandImpl overriding existing writeConcern of: " + - tojson(obj.writeConcern)); - obj.writeConcern = defaultWriteConcern; - } - } else { - obj.writeConcern = defaultWriteConcern; - } - - } else if (forceReadConcern) { - if (obj.hasOwnProperty("readConcern")) { - if (bsonWoCompare(obj.readConcern, defaultReadConcern) !== 0) { - jsTestLog("Warning: _runCommandImpl overriding existing readConcern of: " + - tojson(obj.readConcern)); - obj.readConcern = defaultReadConcern; - } - } else { - obj.readConcern = defaultReadConcern; - } - } - - var res = this.getMongo().runCommand(dbName, obj, options); - - return res; - }; - - // Use a majority write concern if the operation does not specify one. - DBCollection.prototype.getWriteConcern = function() { - return new WriteConcern(defaultWriteConcern); - }; - -})(); diff --git a/jstests/libs/override_methods/set_read_and_write_concerns.js b/jstests/libs/override_methods/set_read_and_write_concerns.js new file mode 100644 index 00000000000..f371644c8f7 --- /dev/null +++ b/jstests/libs/override_methods/set_read_and_write_concerns.js @@ -0,0 +1,216 @@ +/** + * Use prototype overrides to set read concern and write concern while running tests. + */ +(function() { + "use strict"; + + load("jstests/libs/override_methods/override_helpers.js"); + + if (typeof TestData === "undefined" || !TestData.hasOwnProperty("defaultReadConcernLevel")) { + throw new Error( + "The readConcern level to use must be set as the 'defaultReadConcernLevel'" + + " property on the global TestData object"); + } + + const kDefaultReadConcern = {level: TestData.defaultReadConcernLevel}; + const kDefaultWriteConcern = + (TestData.hasOwnProperty("defaultWriteConcern")) ? TestData.defaultWriteConcern : { + w: "majority", + // Use a "signature" value that won't typically match a value assigned in normal use. + // This way the wtimeout set by this override is distinguishable in the server logs. + wtimeout: 5 * 60 * 1000 + 321, // 300321ms + }; + + const kCommandsSupportingReadConcern = new Set([ + "aggregate", + "count", + "distinct", + "find", + "geoNear", + "geoSearch", + "group", + "parallelCollectionScan", + ]); + + const kCommandsSupportingWriteConcern = new Set([ + "_configsvrAddShard", + "_configsvrAddShardToZone", + "_configsvrCommitChunkMerge", + "_configsvrCommitChunkMigration", + "_configsvrCommitChunkSplit", + "_configsvrCreateDatabase", + "_configsvrEnableSharding", + "_configsvrMoveChunk", + "_configsvrMovePrimary", + "_configsvrRemoveShard", + "_configsvrRemoveShardFromZone", + "_configsvrShardCollection", + "_configsvrUpdateZoneKeyRange", + "_mergeAuthzCollections", + "_recvChunkStart", + "appendOplogNote", + "applyOps", + "authSchemaUpgrade", + "aggregate", + "captrunc", + "cleanupOrphaned", + "clone", + "cloneCollection", + "cloneCollectionAsCapped", + // "collMod", SERVER-25196 - not supported + "convertToCapped", + "copydb", + "create", + "createIndexes", + "createRole", + "createUser", + "delete", + "deleteIndexes", + "drop", + "dropAllRolesFromDatabase", + "dropAllUsersFromDatabase", + "dropDatabase", + "dropIndexes", + "dropRole", + "dropUser", + "emptycapped", + "findAndModify", + "findandmodify", + "godinsert", + "grantPrivilegesToRole", + "grantRolesToRole", + "grantRolesToUser", + "insert", + "mapReduce", + "mapreduce", + "mapreduce.shardedfinish", + "moveChunk", + "renameCollection", + "revokePrivilegesFromRole", + "revokeRolesFromRole", + "revokeRolesFromUser", + "setFeatureCompatibilityVersion", + "update", + "updateRole", + "updateUser", + ]); + + function runCommandWithReadAndWriteConcerns( + conn, dbName, commandName, commandObj, func, makeFuncArgs) { + if (typeof commandObj !== "object" || commandObj === null) { + return func.apply(conn, makeFuncArgs(commandObj)); + } + + // If the command is in a wrapped form, then we look for the actual command object inside + // the query/$query object. + let commandObjUnwrapped = commandObj; + if (commandName === "query" || commandName === "$query") { + commandObjUnwrapped = commandObj[commandName]; + commandName = Object.keys(commandObjUnwrapped)[0]; + } + + if (commandName === "collMod" || commandName === "eval" || commandName === "$eval") { + throw new Error("Cowardly refusing to run test with overridden write concern when it" + + " uses a command that can only perform w=1 writes: " + + tojson(commandObj)); + } + + let shouldForceReadConcern = kCommandsSupportingReadConcern.has(commandName); + let shouldForceWriteConcern = kCommandsSupportingWriteConcern.has(commandName); + + if (commandName === "aggregate") { + if (OverrideHelpers.isAggregationWithOutStage(commandName, commandObjUnwrapped)) { + // The $out stage can only be used with readConcern={level: "local"}. + shouldForceReadConcern = false; + } else { + // A writeConcern can only be used with a $out stage. + shouldForceWriteConcern = false; + } + + if (commandObjUnwrapped.explain) { + // Attempting to specify a readConcern while explaining an aggregation would always + // return an error prior to SERVER-30582 and it otherwise only compatible with + // readConcern={level: "local"}. + shouldForceReadConcern = false; + } + } else if (OverrideHelpers.isMapReduceWithInlineOutput(commandName, commandObjUnwrapped)) { + // A writeConcern can only be used with non-inline output. + shouldForceWriteConcern = false; + } else if (commandObj[commandName] === "system.profile") { + // Writes to the "system.profile" collection aren't guaranteed to be visible in the same + // majority-committed snapshot as the command they originated from. We don't override + // the readConcern for operations on the "system.profile" collection so that tests which + // assert on its contents continue to succeed. + shouldForceReadConcern = false; + } + + const inWrappedForm = commandObj !== commandObjUnwrapped; + + if (shouldForceReadConcern) { + // We create a copy of 'commandObj' to avoid mutating the parameter the caller + // specified. + commandObj = Object.assign({}, commandObj); + if (inWrappedForm) { + commandObjUnwrapped = Object.assign({}, commandObjUnwrapped); + commandObj[Object.keys(commandObj)[0]] = commandObjUnwrapped; + } else { + commandObjUnwrapped = commandObj; + } + + if (commandObjUnwrapped.hasOwnProperty("readConcern")) { + let readConcern = commandObjUnwrapped.readConcern; + + if (typeof readConcern !== "object" || readConcern === null || + (readConcern.hasOwnProperty("level") && + bsonWoCompare({_: readConcern.level}, {_: kDefaultReadConcern.level}) !== 0)) { + throw new Error("Cowardly refusing to override read concern of command: " + + tojson(commandObj)); + } + + // We create a copy of the readConcern object to avoid mutating the parameter the + // caller specified. + readConcern = Object.assign({}, readConcern, kDefaultReadConcern); + commandObjUnwrapped.readConcern = readConcern; + } else { + commandObjUnwrapped.readConcern = kDefaultReadConcern; + } + } + + if (shouldForceWriteConcern) { + // We create a copy of 'commandObj' to avoid mutating the parameter the caller + // specified. + commandObj = Object.assign({}, commandObj); + if (inWrappedForm) { + commandObjUnwrapped = Object.assign({}, commandObjUnwrapped); + commandObj[Object.keys(commandObj)[0]] = commandObjUnwrapped; + } else { + commandObjUnwrapped = commandObj; + } + + if (commandObjUnwrapped.hasOwnProperty("writeConcern")) { + let writeConcern = commandObjUnwrapped.writeConcern; + + if (typeof writeConcern !== "object" || writeConcern === null || + (writeConcern.hasOwnProperty("w") && + bsonWoCompare({_: writeConcern.w}, {_: kDefaultWriteConcern.w}) !== 0)) { + throw new Error("Cowardly refusing to override write concern of command: " + + tojson(commandObj)); + } + + // We create a copy of the writeConcern object to avoid mutating the parameter the + // caller specified. + writeConcern = Object.assign({}, writeConcern, kDefaultWriteConcern); + commandObjUnwrapped.writeConcern = writeConcern; + } else { + commandObjUnwrapped.writeConcern = kDefaultWriteConcern; + } + } + + return func.apply(conn, makeFuncArgs(commandObj)); + } + + OverrideHelpers.prependOverrideInParallelShell( + "jstests/libs/override_methods/set_read_and_write_concerns.js"); + + OverrideHelpers.overrideRunCommand(runCommandWithReadAndWriteConcerns); +})(); diff --git a/jstests/libs/override_methods/set_read_preference_secondary.js b/jstests/libs/override_methods/set_read_preference_secondary.js new file mode 100644 index 00000000000..d1d26433c5c --- /dev/null +++ b/jstests/libs/override_methods/set_read_preference_secondary.js @@ -0,0 +1,162 @@ +/** + * Use prototype overrides to set read preference to "secondary" when running tests. + */ +(function() { + "use strict"; + + load("jstests/libs/override_methods/override_helpers.js"); + + const kReadPreferenceSecondary = {mode: "secondary"}; + const kCommandsSupportingReadPreference = new Set([ + "aggregate", + "collStats", + "count", + "dbStats", + "distinct", + "find", + "geoNear", + "geoSearch", + "group", + "mapReduce", + "mapreduce", + "parallelCollectionScan", + ]); + + // This list of cursor-generating commands is incomplete. For example, "listCollections", + // "listIndexes", "parallelCollectionScan", and "repairCursor" are all missing from this list. + // If we ever add tests that attempt to run getMore or killCursors on cursors generated from + // those commands, then we should update the contents of this list and also handle any + // differences in the server's response format. + const kCursorGeneratingCommands = new Set(["aggregate", "find"]); + + const CursorTracker = (function() { + const kNoCursor = new NumberLong(0); + + const connectionsByCursorId = {}; + + return { + getConnectionUsedForCursor: function getConnectionUsedForCursor(cursorId) { + return (cursorId instanceof NumberLong) ? connectionsByCursorId[cursorId] + : undefined; + }, + + setConnectionUsedForCursor: function setConnectionUsedForCursor(cursorId, cursorConn) { + if (cursorId instanceof NumberLong && + !bsonBinaryEqual({_: cursorId}, {_: kNoCursor})) { + connectionsByCursorId[cursorId] = cursorConn; + } + }, + }; + })(); + + function runCommandWithReadPreferenceSecondary( + conn, dbName, commandName, commandObj, func, makeFuncArgs) { + if (typeof commandObj !== "object" || commandObj === null) { + return func.apply(conn, makeFuncArgs(commandObj)); + } + + // If the command is in a wrapped form, then we look for the actual command object inside + // the query/$query object. + let commandObjUnwrapped = commandObj; + if (commandName === "query" || commandName === "$query") { + commandObjUnwrapped = commandObj[commandName]; + commandName = Object.keys(commandObjUnwrapped)[0]; + } + + if (commandObj[commandName] === "system.profile") { + throw new Error("Cowardly refusing to run test with overridden read preference" + + " when it reads from a non-replicated collection: " + + tojson(commandObj)); + } + + if (conn.isReplicaSetConnection()) { + // When a "getMore" or "killCursors" command is issued on a replica set connection, we + // attempt to automatically route the command to the server the cursor(s) were + // originally established on. This makes it possible to use the + // set_read_preference_secondary.js override without needing to update calls of + // DB#runCommand() to explicitly track the connection that was used. If the connection + // is actually a direct connection to a mongod or mongos process, or if the cursor id + // cannot be found in the CursorTracker, then we'll fall back to using DBClientRS's + // server selection and send the operation to the current primary. It is possible that + // the test is trying to exercise the behavior around when an unknown cursor id is sent + // to the server. + if (commandName === "getMore") { + const cursorId = commandObjUnwrapped[commandName]; + const cursorConn = CursorTracker.getConnectionUsedForCursor(cursorId); + if (cursorConn !== undefined) { + return func.apply(cursorConn, makeFuncArgs(commandObj)); + } + } else if (commandName === "killCursors") { + const cursorIds = commandObjUnwrapped.cursors; + if (Array.isArray(cursorIds)) { + let cursorConn; + + for (let cursorId of cursorIds) { + const otherCursorConn = CursorTracker.getConnectionUsedForCursor(cursorId); + if (cursorConn === undefined) { + cursorConn = otherCursorConn; + } else if (otherCursorConn !== undefined) { + // We set 'cursorConn' back to undefined and break out of the loop so + // that we don't attempt to automatically route the "killCursors" + // command when there are cursors from different servers. + cursorConn = undefined; + break; + } + } + + if (cursorConn !== undefined) { + return func.apply(cursorConn, makeFuncArgs(commandObj)); + } + } + } + } + + let shouldForceReadPreference = kCommandsSupportingReadPreference.has(commandName); + if (OverrideHelpers.isAggregationWithOutStage(commandName, commandObjUnwrapped)) { + // An aggregation with a $out stage must be sent to the primary. + shouldForceReadPreference = false; + } else if ((commandName === "mapReduce" || commandName === "mapreduce") && + !OverrideHelpers.isMapReduceWithInlineOutput(commandName, commandObjUnwrapped)) { + // A map-reduce operation with non-inline output must be sent to the primary. + shouldForceReadPreference = false; + } + + if (shouldForceReadPreference) { + if (commandObj === commandObjUnwrapped) { + // We wrap the command object using a "query" field rather than a "$query" field to + // match the implementation of DB.prototype._attachReadPreferenceToCommand(). + commandObj = {query: commandObj}; + } else { + // We create a copy of 'commandObj' to avoid mutating the parameter the caller + // specified. + commandObj = Object.assign({}, commandObj); + } + + if (commandObj.hasOwnProperty("$readPreference") && + !bsonBinaryEqual({_: commandObj.$readPreference}, {_: kReadPreferenceSecondary})) { + throw new Error("Cowardly refusing to override read preference of command: " + + tojson(commandObj)); + } + + commandObj.$readPreference = kReadPreferenceSecondary; + } + + const serverResponse = func.apply(conn, makeFuncArgs(commandObj)); + + if (conn.isReplicaSetConnection() && kCursorGeneratingCommands.has(commandName) && + serverResponse.ok === 1 && serverResponse.hasOwnProperty("cursor")) { + // We associate the cursor id returned by the server with the connection that was used + // to establish it so that we can attempt to automatically route subsequent "getMore" + // and "killCursors" commands. + CursorTracker.setConnectionUsedForCursor(serverResponse.cursor.id, + serverResponse._mongo); + } + + return serverResponse; + } + + OverrideHelpers.prependOverrideInParallelShell( + "jstests/libs/override_methods/set_read_preference_secondary.js"); + + OverrideHelpers.overrideRunCommand(runCommandWithReadPreferenceSecondary); +})(); diff --git a/jstests/multiVersion/initial_sync_last_stable_from_latest.js b/jstests/multiVersion/initial_sync_last_stable_from_latest.js new file mode 100644 index 00000000000..3f036c17fe9 --- /dev/null +++ b/jstests/multiVersion/initial_sync_last_stable_from_latest.js @@ -0,0 +1,15 @@ +/** + * Multiversion initial sync test. Tests that initial sync succeeds when a 'last-stable' version + * secondary syncs from a 'latest' version replica set. + */ + +'use strict'; + +load("./jstests/multiVersion/libs/initial_sync.js"); + +var testName = "multiversion_initial_sync_last_stable_from_latest"; +let replSetVersion = "latest"; +let newSecondaryVersion = "last-stable"; +let fcv = "3.2"; + +multversionInitialSyncTest(testName, replSetVersion, newSecondaryVersion, {}, fcv); diff --git a/jstests/multiVersion/initial_sync_latest_from_last_stable.js b/jstests/multiVersion/initial_sync_latest_from_last_stable.js new file mode 100644 index 00000000000..fd92b64e87a --- /dev/null +++ b/jstests/multiVersion/initial_sync_latest_from_last_stable.js @@ -0,0 +1,14 @@ +/** + * Multiversion initial sync test. Tests that initial sync succeeds when a 'latest' version + * secondary syncs from a 'last-stable' version replica set. + */ + +'use strict'; + +load("./jstests/multiVersion/libs/initial_sync.js"); + +var testName = "multiversion_initial_sync_latest_from_last_stable"; +let replSetVersion = "last-stable"; +let newSecondaryVersion = "latest"; + +multversionInitialSyncTest(testName, replSetVersion, newSecondaryVersion, {}); diff --git a/jstests/multiVersion/initialsync.js b/jstests/multiVersion/initialsync.js deleted file mode 100644 index e9a424fd05c..00000000000 --- a/jstests/multiVersion/initialsync.js +++ /dev/null @@ -1,58 +0,0 @@ -// Multiversion initial sync test. -load("./jstests/multiVersion/libs/multi_rs.js"); -load("./jstests/replsets/rslib.js"); - -var oldVersion = "last-stable"; -var newVersion = "latest"; - -var name = "multiversioninitsync"; - -var multitest = function(replSetVersion, newNodeVersion) { - var nodes = {n1: {binVersion: replSetVersion}, n2: {binVersion: replSetVersion}}; - - print("Start up a two-node " + replSetVersion + " replica set."); - var rst = new ReplSetTest({name: name, nodes: nodes}); - rst.startSet(); - var config = rst.getReplSetConfig(); - // Set protocol version to 0 for 3.2 replset. - if (replSetVersion == newVersion) { - config.protocolVersion = 0; - } - rst.initiate(config); - - // Wait for a primary node. - var primary = rst.getPrimary(); - - // Insert some data and wait for replication. - for (var i = 0; i < 25; i++) { - primary.getDB("foo").foo.insert({_id: i}); - } - rst.awaitReplication(); - - print("Bring up a new node with version " + newNodeVersion + " and add to set."); - rst.add({binVersion: newNodeVersion}); - rst.reInitiate(); - - // Wait for a primary node. - var primary = rst.getPrimary(); - var secondaries = rst.getSecondaries(); - - print("Wait for new node to be synced."); - rst.awaitReplication(); - - rst.stopSet(); -}; - -// ***************************************** -// Test A: -// "Latest" version secondary is synced from -// an old ReplSet. -// ***************************************** -multitest(oldVersion, newVersion); - -// ***************************************** -// Test B: -// Old Secondary is synced from a "latest" -// version ReplSet. -// ***************************************** -multitest(newVersion, oldVersion); diff --git a/jstests/multiVersion/libs/initial_sync.js b/jstests/multiVersion/libs/initial_sync.js new file mode 100644 index 00000000000..866f825c8b9 --- /dev/null +++ b/jstests/multiVersion/libs/initial_sync.js @@ -0,0 +1,50 @@ + +'use strict'; + +load("./jstests/multiVersion/libs/multi_rs.js"); +load("./jstests/replsets/rslib.js"); + +/** + * Test that starts up a replica set with 2 nodes of version 'replSetVersion', inserts some data, + * then adds a new node to the replica set with version 'newNodeVersion' and waits for initial sync + * to complete. If the 'fcv' argument is given, sets the feature compatibility version of the + * replica set to 'fcv' before adding the third node. + */ +var multversionInitialSyncTest = function( + name, replSetVersion, newNodeVersion, configSettings, fcv) { + + var nodes = {n1: {binVersion: replSetVersion}, n2: {binVersion: replSetVersion}}; + + jsTestLog("Starting up a two-node '" + replSetVersion + "' version replica set."); + var rst = new ReplSetTest({name: name, nodes: nodes}); + rst.startSet(); + + var conf = rst.getReplSetConfig(); + conf.settings = configSettings; + rst.initiate(conf); + + // Wait for a primary node. + var primary = rst.getPrimary(); + + // Set 'featureCompatibilityVersion' if given. + if (fcv) { + jsTestLog("Setting FCV to '" + fcv + "' on the primary."); + assert.commandWorked(primary.adminCommand({setFeatureCompatibilityVersion: fcv})); + } + + // Insert some data and wait for replication. + for (var i = 0; i < 25; i++) { + primary.getDB("foo").foo.insert({_id: i}); + } + rst.awaitReplication(); + + jsTestLog("Bringing up a new node with version '" + newNodeVersion + "' and adding to set."); + rst.add({binVersion: newNodeVersion}); + rst.reInitiate(); + + jsTestLog("Waiting for new node to be synced."); + rst.awaitReplication(); + rst.awaitSecondaryNodes(); + + rst.stopSet(); +}; \ No newline at end of file diff --git a/jstests/multiVersion/libs/multi_rs.js b/jstests/multiVersion/libs/multi_rs.js index da976c7f4dc..87d5995ef48 100644 --- a/jstests/multiVersion/libs/multi_rs.js +++ b/jstests/multiVersion/libs/multi_rs.js @@ -25,6 +25,7 @@ ReplSetTest.prototype.upgradeSet = function(options, user, pwd) { var node = nodesToUpgrade[i]; if (node == primary) { node = this.stepdown(node); + this.waitForState(node, ReplSetTest.State.SECONDARY); primary = this.getPrimary(); } diff --git a/jstests/multiVersion/set_feature_compatibility_version.js b/jstests/multiVersion/set_feature_compatibility_version.js index d5c73f26efb..73a5785aec9 100644 --- a/jstests/multiVersion/set_feature_compatibility_version.js +++ b/jstests/multiVersion/set_feature_compatibility_version.js @@ -54,6 +54,23 @@ // featureCompatibilityVersion cannot be set via setParameter. assert.commandFailed(adminDB.runCommand({setParameter: 1, featureCompatibilityVersion: "3.2"})); + // setFeatureCompatibilityVersion fails to downgrade to FCV=3.2 if the write fails. + assert.commandWorked(adminDB.runCommand({ + configureFailPoint: "failCollectionUpdates", + data: {collectionNS: "admin.system.version"}, + mode: "alwaysOn" + })); + assert.commandFailed(adminDB.runCommand({setFeatureCompatibilityVersion: "3.2"})); + res = adminDB.runCommand({getParameter: 1, featureCompatibilityVersion: 1}); + assert.commandWorked(res); + assert.eq(res.featureCompatibilityVersion, "3.4"); + assert.eq(adminDB.system.version.findOne({_id: "featureCompatibilityVersion"}).version, "3.4"); + assert.commandWorked(adminDB.runCommand({ + configureFailPoint: "failCollectionUpdates", + data: {collectionNS: "admin.system.version"}, + mode: "off" + })); + // featureCompatibilityVersion can be set to 3.2. assert.commandWorked(adminDB.runCommand({setFeatureCompatibilityVersion: "3.2"})); res = adminDB.runCommand({getParameter: 1, featureCompatibilityVersion: 1}); @@ -70,6 +87,23 @@ "Expected index with name 'incompatible_with_version_32' to have been removed: " + tojson(allIndexes)); + // setFeatureCompatibilityVersion fails to upgrade to FCV=3.4 if the write fails. + assert.commandWorked(adminDB.runCommand({ + configureFailPoint: "failCollectionUpdates", + data: {collectionNS: "admin.system.version"}, + mode: "alwaysOn" + })); + assert.commandFailed(adminDB.runCommand({setFeatureCompatibilityVersion: "3.4"})); + res = adminDB.runCommand({getParameter: 1, featureCompatibilityVersion: 1}); + assert.commandWorked(res); + assert.eq(res.featureCompatibilityVersion, "3.2"); + assert.eq(adminDB.system.version.findOne({_id: "featureCompatibilityVersion"}).version, "3.2"); + assert.commandWorked(adminDB.runCommand({ + configureFailPoint: "failCollectionUpdates", + data: {collectionNS: "admin.system.version"}, + mode: "off" + })); + // featureCompatibilityVersion can be set to 3.4. assert.commandWorked(adminDB.runCommand({setFeatureCompatibilityVersion: "3.4"})); res = adminDB.runCommand({getParameter: 1, featureCompatibilityVersion: 1}); diff --git a/jstests/noPassthrough/ftdc_setdirectory.js b/jstests/noPassthrough/ftdc_setdirectory.js new file mode 100644 index 00000000000..c1173055e2a --- /dev/null +++ b/jstests/noPassthrough/ftdc_setdirectory.js @@ -0,0 +1,118 @@ +/** + * Test that verifies FTDC works in mongos. + */ +load('jstests/libs/ftdc.js'); + +(function() { + 'use strict'; + let testPath1 = MongoRunner.toRealPath('ftdc_setdir1'); + let testPath2 = MongoRunner.toRealPath('ftdc_setdir2'); + let testPath3 = MongoRunner.toRealPath('ftdc_setdir3'); + let testLog3 = testPath3 + "mongos_ftdc.log"; + + // Make the log file directory for mongos. + mkdir(testPath3); + + // Startup 3 mongos: + // 1. Normal MongoS with no log file to verify FTDC can be startup at runtime with a path. + // 2. MongoS with explict diagnosticDataCollectionDirectoryPath setParameter at startup. + // 3. MongoS with log file to verify automatic FTDC path computation works. + let st = new ShardingTest({ + shards: 1, + mongos: { + s0: {verbose: 0}, + s1: {setParameter: {diagnosticDataCollectionDirectoryPath: testPath2}}, + s2: {logpath: testLog3} + } + }); + + let admin1 = st.s0.getDB('admin'); + let admin2 = st.s1.getDB('admin'); + let admin3 = st.s2.getDB('admin'); + + function setParam(admin, obj) { + var ret = admin.runCommand(Object.extend({setParameter: 1}, obj)); + return ret; + } + + function getParam(admin, field) { + var q = {getParameter: 1}; + q[field] = 1; + + var ret = admin.runCommand(q); + assert.commandWorked(ret); + return ret[field]; + } + + // Verify FTDC can be started at runtime. + function verifyFTDCDisabledOnStartup() { + jsTestLog("Running verifyFTDCDisabledOnStartup"); + verifyCommonFTDCParameters(admin1, false); + + // 1. Try to enable and fail + assert.commandFailed(setParam(admin1, {"diagnosticDataCollectionEnabled": 1})); + + // 2. Set path and succeed + assert.commandWorked( + setParam(admin1, {"diagnosticDataCollectionDirectoryPath": testPath2})); + + // 3. Set path again and fail + assert.commandFailed( + setParam(admin1, {"diagnosticDataCollectionDirectoryPath": testPath2})); + + // 4. Enable successfully + assert.commandWorked(setParam(admin1, {"diagnosticDataCollectionEnabled": 1})); + + // 5. Validate getDiagnosticData returns FTDC data now + jsTestLog("Verifying FTDC getDiagnosticData"); + verifyGetDiagnosticData(admin1); + } + + // Verify FTDC is already running if there was a path set at startup. + function verifyFTDCStartsWithPath() { + jsTestLog("Running verifyFTDCStartsWithPath"); + verifyCommonFTDCParameters(admin2, true); + + // 1. Set path fail + assert.commandFailed( + setParam(admin2, {"diagnosticDataCollectionDirectoryPath": testPath2})); + + // 2. Enable successfully + assert.commandWorked(setParam(admin2, {"diagnosticDataCollectionEnabled": 1})); + + // 3. Validate getDiagnosticData returns FTDC data now + jsTestLog("Verifying FTDC getDiagnosticData"); + verifyGetDiagnosticData(admin2); + } + + function normpath(path) { + return path.replace(/\\/g, "/"); + } + + // Verify FTDC is already running if there was a path set at startup. + function verifyFTDCStartsWithLogFile() { + jsTestLog("Running verifyFTDCStartsWithLogFile"); + verifyCommonFTDCParameters(admin3, true); + + // 1. Verify that path is computed correctly. + let computedPath = getParam(admin3, "diagnosticDataCollectionDirectoryPath"); + assert.eq(normpath(computedPath), normpath(testPath3 + "mongos_ftdc.diagnostic.data")); + + // 2. Set path fail + assert.commandFailed( + setParam(admin3, {"diagnosticDataCollectionDirectoryPath": testPath2})); + + // 3. Enable successfully + assert.commandWorked(setParam(admin3, {"diagnosticDataCollectionEnabled": 1})); + + // 4. Validate getDiagnosticData returns FTDC data now + jsTestLog("Verifying FTDC getDiagnosticData"); + verifyGetDiagnosticData(admin3); + } + + verifyFTDCDisabledOnStartup(); + verifyFTDCStartsWithPath(); + verifyFTDCStartsWithLogFile(); + + st.stop(); +})(); diff --git a/jstests/noPassthrough/geo_near_fcv32.js b/jstests/noPassthrough/geo_near_fcv32.js new file mode 100644 index 00000000000..10b331a411d --- /dev/null +++ b/jstests/noPassthrough/geo_near_fcv32.js @@ -0,0 +1,85 @@ +/** + * Confirms that $geoNear aggregation and geoNear command succeed when FCV is 3.2. + */ +(function() { + "use strict"; + + const conn = MongoRunner.runMongod({}); + assert.neq(null, conn, "mongod was unable to start up"); + + const testDB = conn.getDB("geo_near_fcv32"); + testDB.test32.drop(); + assert.commandWorked(testDB.adminCommand({setFeatureCompatibilityVersion: "3.2"})); + + // Create 2dsphere index. + assert.commandWorked(testDB.test32.createIndex({loc: "2dsphere"})); + + // Assert that $geoNear aggregate command does not fail due to collation errors when FCV is 3.2. + assert.eq(0, + testDB.test32 + .aggregate([{ + $geoNear: { + near: {type: "Point", coordinates: [1.23, 1.23]}, + distanceField: "distance", + spherical: true + } + }]) + .itcount()); + + // Assert that specifying the simple collation in $geoNear aggregate fails with FCV 3.2. + assert.throws(function() { + testDB.test32.aggregate([{ + $geoNear: { + near: {type: "Point", coordinates: [1.23, 1.23]}, + distanceField: "distance", + spherical: true, + collation: {locale: "simple"}, + } + }]); + }); + + // Assert that specifying the simple collation in geoNear command fails with FCV 3.2. + assert.commandFailed(testDB.runCommand({ + geoNear: "test32", + near: {type: "Point", coordinates: [1.23, 1.23]}, + spherical: true, + collation: {locale: "simple"} + })); + + assert.commandWorked(testDB.adminCommand({setFeatureCompatibilityVersion: "3.4"})); + + // Create collection with case sensitive collation. + assert.commandWorked( + testDB.createCollection("test34", {collation: {locale: "en_US", strength: 2}})); + assert.commandWorked(testDB.test34.createIndex({loc: "2dsphere"})); + assert.writeOK(testDB.test34.insert({loc: [1.23, 1.23], str: "A"})); + + // Assert that after upgrading FCV to 3.4 $geoNear aggregate inherits the collection's default + // collation. + assert.eq(1, + testDB.test34 + .aggregate([{ + $geoNear: { + near: {type: "Point", coordinates: [1.23, 1.23]}, + distanceField: "distance", + spherical: true, + query: {str: "a"}, + } + }]) + .itcount()); + + // Assert that the $geoNear aggregate accepts a specific collation and overrides the default + // collation in FCV 3.4. + assert.eq(1, + testDB.test34 + .aggregate([{ + $geoNear: { + near: {type: "Point", coordinates: [1.23, 1.23]}, + distanceField: "distance", + spherical: true, + query: {str: "à"}, + } + }], + {collation: {locale: "en_US", strength: 1}}) + .itcount()); +})(); diff --git a/jstests/noPassthrough/libs/backup_restore.js b/jstests/noPassthrough/libs/backup_restore.js index b4e02415081..e66f0f8af97 100644 --- a/jstests/noPassthrough/libs/backup_restore.js +++ b/jstests/noPassthrough/libs/backup_restore.js @@ -352,15 +352,31 @@ var BackupRestoreTest = function(options) { stopMongoProgramByPid(fsmPid); // Wait up to 5 minutes until the new hidden node is in state SECONDARY. + jsTestLog('CRUD and FSM clients stopped. Waiting for hidden node ' + hiddenHost + + ' to become SECONDARY'); rst.waitForState(hiddenNode, ReplSetTest.State.SECONDARY); // Wait for secondaries to finish catching up before shutting down. + jsTestLog( + 'Hidden node ' + hiddenHost + + ' is now SECONDARY. Waiting for CRUD and FSM operations to be applied on all nodes.'); + rst.awaitReplication(); + + jsTestLog('CRUD and FSM operations successfully applied on all nodes. ' + + 'Waiting for all nodes to agree on the primary.'); rst.awaitNodesAgreeOnPrimary(); + + jsTestLog('All nodes agree on the primary. Getting current primary.'); primary = rst.getPrimary(); - assert.writeOK(primary.getDB("test").foo.insert( - {}, {writeConcern: {w: rst.nodes.length, wtimeout: 10 * 60 * 1000}})); + + jsTestLog('Inserting single document into primary ' + primary.host + + ' with writeConcern w:' + rst.nodes.length); + var writeResult = assert.writeOK(primary.getDB("test").foo.insert( + {}, {writeConcern: {w: rst.nodes.length, wtimeout: ReplSetTest.kDefaultTimeoutMS}})); // Stop set. + jsTestLog('Insert operation successful: ' + tojson(writeResult) + + '. Stopping replica set.'); rst.stopSet(); // Cleanup the files from the test diff --git a/jstests/noPassthrough/non_atomic_apply_ops_logging.js b/jstests/noPassthrough/non_atomic_apply_ops_logging.js new file mode 100644 index 00000000000..f93e9d38189 --- /dev/null +++ b/jstests/noPassthrough/non_atomic_apply_ops_logging.js @@ -0,0 +1,62 @@ +// SERVER-28594 Ensure non-atomic ops are individually logged in applyOps +// and atomic ops are collectively logged in applyOps. +(function() { + "use strict"; + + let rst = new ReplSetTest({nodes: 1}); + rst.startSet(); + rst.initiate(); + + let primary = rst.getPrimary(); + let testDB = primary.getDB("test"); + let oplogColl = primary.getDB("local").oplog.rs; + let testCollName = "testColl"; + let rerenamedCollName = "rerenamedColl"; + + testDB.runCommand({drop: testCollName}); + testDB.runCommand({drop: rerenamedCollName}); + assert.commandWorked(testDB.runCommand({create: testCollName})); + let testColl = testDB[testCollName]; + + // Ensure atomic apply ops logging only produces one oplog entry + // per call to apply ops and does not log individual operations + // separately. + assert.commandWorked(testDB.runCommand({ + applyOps: [ + {op: "i", ns: testColl.getFullName(), o: {_id: 1, a: "foo"}}, + {op: "i", ns: testColl.getFullName(), o: {_id: 2, a: "bar"}} + ] + })); + assert.eq(oplogColl.find({"o.applyOps": {"$exists": true}}).count(), 1); + assert.eq(oplogColl.find({"op": "i"}).count(), 0); + // Ensure non-atomic apply ops logging produces an oplog entry for + // each operation in the apply ops call and no record of applyOps + // appears for these operations. + assert.commandWorked(testDB.runCommand({ + applyOps: [ + { + op: "c", + ns: "test.$cmd", + o: { + renameCollection: "test.testColl", + to: "test.renamedColl", + stayTemp: false, + dropTarget: false + } + }, + { + op: "c", + ns: "test.$cmd", + o: { + renameCollection: "test.renamedColl", + to: "test." + rerenamedCollName, + stayTemp: false, + dropTarget: false + } + } + ] + })); + assert.eq(oplogColl.find({"o.renameCollection": {"$exists": true}}).count(), 2); + assert.eq(oplogColl.find({"o.applyOps": {"$exists": true}}).count(), 1); + rst.stopSet(); +})(); diff --git a/jstests/noPassthrough/partial_unique_indexes.js b/jstests/noPassthrough/partial_unique_indexes.js new file mode 100644 index 00000000000..7c376872ade --- /dev/null +++ b/jstests/noPassthrough/partial_unique_indexes.js @@ -0,0 +1,47 @@ +/** + * SERVER-32001: Test that indexing paths for non-unique, partial, unique, partial&unique + * crud operations correctly handle WriteConflictExceptions. + */ +(function() { + "strict"; + + let conn = MongoRunner.runMongod(); + let testDB = conn.getDB("test"); + + let t = testDB.jstests_parallel_allops; + t.drop(); + + t.createIndex({x: 1, _id: 1}, {partialFilterExpression: {_id: {$lt: 500}}, unique: true}); + t.createIndex({y: -1, _id: 1}, {unique: true}); + t.createIndex({x: -1}, {partialFilterExpression: {_id: {$gte: 500}}, unique: false}); + t.createIndex({y: 1}, {unique: false}); + + let _id = {"#RAND_INT": [0, 1000]}; + let ops = [ + {op: "remove", ns: t.getFullName(), query: {_id}}, + {op: "update", ns: t.getFullName(), query: {_id}, update: {$inc: {x: 1}}, upsert: true}, + {op: "update", ns: t.getFullName(), query: {_id}, update: {$inc: {y: 1}}, upsert: true}, + ]; + + let seconds = 5; + let parallel = 5; + let host = testDB.getMongo().host; + + let benchArgs = {ops, seconds, parallel, host}; + + assert.commandWorked(testDB.adminCommand({ + configureFailPoint: 'WTWriteConflictExceptionForReads', + mode: {activationProbability: 0.01} + })); + assert.commandWorked(testDB.adminCommand( + {configureFailPoint: 'WTWriteConflictException', mode: {activationProbability: 0.01}})); + res = benchRun(benchArgs); + printjson({res}); + + assert.commandWorked( + testDB.adminCommand({configureFailPoint: 'WTWriteConflictException', mode: "off"})); + assert.commandWorked( + testDB.adminCommand({configureFailPoint: 'WTWriteConflictExceptionForReads', mode: "off"})); + res = t.validate(); + assert(res.valid, tojson(res)); +})(); diff --git a/jstests/noPassthrough/skip_sharding_configuration_checks.js b/jstests/noPassthrough/skip_sharding_configuration_checks.js new file mode 100644 index 00000000000..ff3e95440eb --- /dev/null +++ b/jstests/noPassthrough/skip_sharding_configuration_checks.js @@ -0,0 +1,53 @@ +/** + * Starts standalone RS with skipShardingConfigurationChecks. + * @tags: [requires_persistence] + */ +(function() { + 'use strict'; + + function expectState(rst, state) { + assert.soon(function() { + var status = rst.status(); + if (status.myState != state) { + print("Waiting for state " + state + " in replSetGetStatus output: " + + tojson(status)); + } + return status.myState == state; + }); + } + + let configSvr = MongoRunner.runMongod( + {configsvr: "", setParameter: 'skipShardingConfigurationChecks=true'}); + assert.eq(configSvr, null); + + let shardSvr = + MongoRunner.runMongod({shardsvr: "", setParameter: 'skipShardingConfigurationChecks=true'}); + assert.eq(shardSvr, null); + + var st = new ShardingTest({name: "skipConfig", shards: {rs0: {nodes: 1}}}); + var configRS = st.configRS; + var shardRS = st.rs0; + + st.stopAllMongos(); + shardRS.stopSet(15, true); + configRS.stopSet(undefined, true); + + jsTestLog("Restarting configRS as a standalone ReplicaSet"); + + for (let i = 0; i < configRS.nodes.length; i++) { + delete configRS.nodes[i].fullOptions.configsvr; + configRS.nodes[i].fullOptions.setParameter = 'skipShardingConfigurationChecks=true'; + } + configRS.startSet({}, true); + expectState(configRS, ReplSetTest.State.PRIMARY); + configRS.stopSet(); + + jsTestLog("Restarting shardRS as a standalone ReplicaSet"); + for (let i = 0; i < shardRS.nodes.length; i++) { + delete shardRS.nodes[i].fullOptions.shardsvr; + shardRS.nodes[i].fullOptions.setParameter = 'skipShardingConfigurationChecks=true'; + } + shardRS.startSet({}, true); + expectState(shardRS, ReplSetTest.State.PRIMARY); + shardRS.stopSet(); +})(); diff --git a/jstests/noPassthroughWithMongod/apply_ops_index_collation.js b/jstests/noPassthroughWithMongod/apply_ops_index_collation.js new file mode 100644 index 00000000000..6248bc63b0f --- /dev/null +++ b/jstests/noPassthroughWithMongod/apply_ops_index_collation.js @@ -0,0 +1,80 @@ +// Cannot implicitly shard accessed collections because of collection existing when none +// expected. +// @tags: [assumes_no_implicit_collection_creation_after_drop] + +// Tests creation of indexes using applyOps for collections with a non-simple default collation. +// Indexes created through applyOps should be built exactly according to their index spec, without +// inheriting the collection default collation, since this is how the oplog entries are replicated. +// TODO SERVER-31435: Move this test into core once applyOps with createIndexes replicates +// correctly. +(function() { + "use strict"; + + load("jstests/libs/get_index_helpers.js"); + + const coll = db.apply_ops_index_collation; + coll.drop(); + assert.commandWorked(db.createCollection(coll.getName(), {collation: {locale: "fr_CA"}})); + + // An index created using an insert-style oplog entry with a non-simple collation does not + // inherit the collection default collation. + let res = assert.commandWorked(db.adminCommand({ + applyOps: [{ + op: "i", + ns: db.system.indexes.getFullName(), + o: { + v: 2, + key: {c: 1}, + name: "c_1_en", + ns: coll.getFullName(), + collation: { + locale: "en_US", + caseLevel: false, + caseFirst: "off", + strength: 3, + numericOrdering: false, + alternate: "non-ignorable", + maxVariable: "punct", + normalization: false, + backwards: false, + version: "57.1" + } + } + }] + })); + let allIndexes = coll.getIndexes(); + let spec = GetIndexHelpers.findByName(allIndexes, "c_1_en"); + assert.neq(null, spec, "Index 'c_1_en' not found: " + tojson(allIndexes)); + assert.eq(2, spec.v, tojson(spec)); + assert.eq("en_US", spec.collation.locale, tojson(spec)); + + // An index created using an insert-style oplog entry with a simple collation does not inherit + // the collection default collation. + res = assert.commandWorked(db.adminCommand({ + applyOps: [{ + op: "i", + ns: db.system.indexes.getFullName(), + o: {v: 2, key: {c: 1}, name: "c_1", ns: coll.getFullName()} + }] + })); + allIndexes = coll.getIndexes(); + spec = GetIndexHelpers.findByName(allIndexes, "c_1"); + assert.neq(null, spec, "Index 'c_1' not found: " + tojson(allIndexes)); + assert.eq(2, spec.v, tojson(spec)); + assert(!spec.hasOwnProperty("collation"), tojson(spec)); + + // A v=1 index created using an insert-style oplog entry does not inherit the collection default + // collation. + res = assert.commandWorked(db.adminCommand({ + applyOps: [{ + op: "i", + ns: db.system.indexes.getFullName(), + o: {v: 1, key: {d: 1}, name: "d_1", ns: coll.getFullName()} + }] + })); + allIndexes = coll.getIndexes(); + spec = GetIndexHelpers.findByName(allIndexes, "d_1"); + assert.neq(null, spec, "Index 'd_1' not found: " + tojson(allIndexes)); + assert.eq(1, spec.v, tojson(spec)); + assert(!spec.hasOwnProperty("collation"), tojson(spec)); +})(); diff --git a/jstests/noPassthroughWithMongod/ftdc_params.js b/jstests/noPassthroughWithMongod/ftdc_params.js index ced6c1b5675..08714040fcb 100644 --- a/jstests/noPassthroughWithMongod/ftdc_params.js +++ b/jstests/noPassthroughWithMongod/ftdc_params.js @@ -1,58 +1,10 @@ // FTDC test cases // +load('jstests/libs/ftdc.js'); + (function() { 'use strict'; var admin = db.getSiblingDB("admin"); - // Check the defaults are correct - // - function getparam(field) { - var q = {getParameter: 1}; - q[field] = 1; - - var ret = admin.runCommand(q); - return ret[field]; - } - - // Verify the defaults are as we documented them - assert.eq(getparam("diagnosticDataCollectionEnabled"), true); - assert.eq(getparam("diagnosticDataCollectionPeriodMillis"), 1000); - assert.eq(getparam("diagnosticDataCollectionDirectorySizeMB"), 200); - assert.eq(getparam("diagnosticDataCollectionFileSizeMB"), 10); - assert.eq(getparam("diagnosticDataCollectionSamplesPerChunk"), 300); - assert.eq(getparam("diagnosticDataCollectionSamplesPerInterimUpdate"), 10); - - function setparam(obj) { - var ret = admin.runCommand(Object.extend({setParameter: 1}, obj)); - return ret; - } - - assert.commandWorked(setparam({"diagnosticDataCollectionEnabled": 1})); - assert.commandWorked(setparam({"diagnosticDataCollectionPeriodMillis": 100})); - assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 10})); - assert.commandWorked(setparam({"diagnosticDataCollectionFileSizeMB": 1})); - assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerChunk": 2})); - assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerInterimUpdate": 2})); - - // Negative tests - set values below minimums - assert.commandFailed(setparam({"diagnosticDataCollectionPeriodMillis": 1})); - assert.commandFailed(setparam({"diagnosticDataCollectionDirectorySizeMB": 1})); - assert.commandFailed(setparam({"diagnosticDataCollectionSamplesPerChunk": 1})); - assert.commandFailed(setparam({"diagnosticDataCollectionSamplesPerInterimUpdate": 1})); - - // Negative test - set file size bigger then directory size - assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 10})); - assert.commandFailed(setparam({"diagnosticDataCollectionFileSizeMB": 100})); - - // Negative test - set directory size less then file size - assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 100})); - assert.commandWorked(setparam({"diagnosticDataCollectionFileSizeMB": 50})); - assert.commandFailed(setparam({"diagnosticDataCollectionDirectorySizeMB": 10})); - - // Reset - assert.commandWorked(setparam({"diagnosticDataCollectionFileSizeMB": 10})); - assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 200})); - assert.commandWorked(setparam({"diagnosticDataCollectionPeriodMillis": 1000})); - assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerChunk": 300})); - assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerInterimUpdate": 10})); + verifyCommonFTDCParameters(admin, true); })(); diff --git a/jstests/noPassthroughWithMongod/host_connection_string_validation.js b/jstests/noPassthroughWithMongod/host_connection_string_validation.js index a252ef39230..f7bac51db88 100644 --- a/jstests/noPassthroughWithMongod/host_connection_string_validation.js +++ b/jstests/noPassthroughWithMongod/host_connection_string_validation.js @@ -89,8 +89,8 @@ print("Testing " + (isGood ? "good" : "bad") + " connection string " + i + "..."); print(" * testing " + connectionString); testHost(connectionString, isGood); - print(" * testing mongodb://" + connectionString); - testHost("mongodb://" + connectionString, isGood); + print(" * testing mongodb://" + encodeURIComponent(connectionString)); + testHost("mongodb://" + encodeURIComponent(connectionString), isGood); } var i; diff --git a/jstests/noPassthroughWithMongod/initial_sync_oplog_rollover.js b/jstests/noPassthroughWithMongod/initial_sync_oplog_rollover.js index 3a47e14447c..d186a8f59ec 100644 --- a/jstests/noPassthroughWithMongod/initial_sync_oplog_rollover.js +++ b/jstests/noPassthroughWithMongod/initial_sync_oplog_rollover.js @@ -34,7 +34,7 @@ assert.writeOK(coll.insert({a: 1})); function getFirstOplogEntry(conn) { - return conn.getDB('local').oplog.rs.find().sort({ts: 1}).limit(1)[0]; + return conn.getDB('local').oplog.rs.find().sort({$natural: 1}).limit(1)[0]; } var firstOplogEntry = getFirstOplogEntry(primary); diff --git a/jstests/noPassthroughWithMongod/replset_host_connection_validation.js b/jstests/noPassthroughWithMongod/replset_host_connection_validation.js index 7e48de363e2..e65387dd705 100644 --- a/jstests/noPassthroughWithMongod/replset_host_connection_validation.js +++ b/jstests/noPassthroughWithMongod/replset_host_connection_validation.js @@ -1,5 +1,6 @@ // Test --host with a replica set. (function() { + 'use strict'; const replSetName = 'hostTestReplSetName'; @@ -26,35 +27,56 @@ // Pass the inner test's exit code back as the outer test's exit code if (exitCode != 0) { - doassert("inner test failed with exit code " + exitcode); + doassert("inner test failed with exit code " + exitCode); } return; } - const testHost = function(host) { - const exitCode = runMongoProgram('mongo', '--eval', ';', '--host', host); - if (exitCode !== 0) { - doassert("failed to connect with `--host " + host + - "`, but expected success. Exit code: " + exitCode); + function testHost(host, uri, ok) { + const exitCode = runMongoProgram('mongo', '--eval', ';', '--host', host, uri); + if (ok) { + assert.eq(exitCode, 0, "failed to connect with `--host " + host + "`"); + } else { + assert.neq(exitCode, 0, "unexpectedly succeeded to connect with `--host " + host + "`"); } - }; - - const connStrings = [ - `localhost:${port}`, - `${replSetName}/localhost:${port}`, - `mongodb://localhost:${port}/admin?replicaSet=${replSetName}`, - `mongodb://localhost:${port}`, - ]; - - function runConnectionStringTestFor(i, connectionString) { - print("Testing connection string " + i + "..."); - print(" * testing " + connectionString); - testHost(connectionString); } - for (let i = 0; i < connStrings.length; ++i) { - runConnectionStringTestFor(i, connStrings[i]); + function runConnectionStringTestFor(connectionString, uri, ok) { + print("* Testing: --host " + connectionString + " " + uri); + if (!ok) { + print(" This should fail"); + } + testHost(connectionString, uri, ok); + } + + function expSuccess(str) { + runConnectionStringTestFor(str, '', true); + if (!str.startsWith('mongodb://')) { + runConnectionStringTestFor(str, 'dbname', true); + } } + function expFailure(str) { + runConnectionStringTestFor(str, '', false); + } + + expSuccess(`localhost:${port}`); + expSuccess(`${replSetName}/localhost:${port}`); + expSuccess(`${replSetName}/localhost:${port},[::1]:${port}`); + expSuccess(`${replSetName}/localhost:${port},`); + expSuccess(`${replSetName}/localhost:${port},,`); + expSuccess(`mongodb://localhost:${port}/admin?replicaSet=${replSetName}`); + expSuccess(`mongodb://localhost:${port}`); + + expFailure(','); + expFailure(',,'); + expFailure(`${replSetName}/`); + expFailure(`${replSetName}/,`); + expFailure(`${replSetName}/,,`); + expFailure(`${replSetName}//not/a/socket`); + expFailure(`mongodb://localhost:${port}/admin?replicaSet=`); + expFailure('mongodb://localhost:'); + expFailure(`mongodb://:${port}`); + jsTest.log("SUCCESSFUL test completion"); })(); diff --git a/jstests/replsets/apply_batches_totalMillis.js b/jstests/replsets/apply_batches_totalMillis.js new file mode 100644 index 00000000000..9e093211cb6 --- /dev/null +++ b/jstests/replsets/apply_batches_totalMillis.js @@ -0,0 +1,63 @@ +/** + * serverStatus.metrics.repl.apply.batches.totalMillis is a cumulative measure of how much time a + * node spends applying batches. This test checks that it includes the time spent waiting for + * batches to finish, by comparing the time recorded after replicating a small and a large load. + */ + +(function() { + "use strict"; + + // Gets the value of metrics.repl.apply.batches.totalMillis. + function getTotalMillis(node) { + return assert.commandWorked(node.adminCommand({serverStatus: 1})) + .metrics.repl.apply.batches.totalMillis; + } + + // Do a bulk insert of documents as: {{key: 0}, {key: 1}, {key: 2}, ... , {key: num-1}} + function performBulkInsert(coll, key, num) { + let bulk = coll.initializeUnorderedBulkOp(); + for (let i = 0; i < num; i++) { + let doc = {}; + doc[key] = i; + bulk.insert(doc); + } + assert.writeOK(bulk.execute()); + rst.awaitReplication(); + } + + let name = "apply_batches_totalMillis"; + let rst = new ReplSetTest({name: name, nodes: 2}); + rst.startSet(); + rst.initiate(); + + let primary = rst.getPrimary(); + let secondary = rst.getSecondary(); + let coll = primary.getDB(name)["foo"]; + + // Perform an initial write on the system and ensure steady state. + assert.writeOK(coll.insert({init: 0})); + rst.awaitReplication(); + let baseTime = getTotalMillis(secondary); + + // Introduce a small load and wait for it to be replicated. + performBulkInsert(coll, "small", 1000); + + // Record the time spent applying the small load. + let timeAfterSmall = getTotalMillis(secondary); + let deltaSmall = timeAfterSmall - baseTime; + + // Insert a significantly larger load. + performBulkInsert(coll, "large", 20000); + + // Record the time spent applying the large load. + let timeAfterLarge = getTotalMillis(secondary); + let deltaLarge = timeAfterLarge - timeAfterSmall; + + jsTestLog(`Recorded deltas: {small: ${deltaSmall}ms, large: ${deltaLarge}ms}.`); + + // We should have recorded at least as much time on the second load as we did on the first. + // This is a crude comparison that is only taken to check that the timer is used correctly. + assert(deltaLarge >= deltaSmall, "Expected a higher net totalMillis for the larger load."); + rst.stopSet(); + +})(); \ No newline at end of file diff --git a/jstests/replsets/apply_ops_concurrent_non_atomic_different_db.js b/jstests/replsets/apply_ops_concurrent_non_atomic_different_db.js new file mode 100644 index 00000000000..05cb6f9e996 --- /dev/null +++ b/jstests/replsets/apply_ops_concurrent_non_atomic_different_db.js @@ -0,0 +1,11 @@ +(function() { + 'use strict'; + + load('jstests/replsets/libs/apply_ops_concurrent_non_atomic.js'); + + new ApplyOpsConcurrentNonAtomicTest({ + ns1: 'test1.coll1', + ns2: 'test2.coll2', + requiresDocumentLevelConcurrency: false, + }).run(); +}()); diff --git a/jstests/replsets/apply_ops_concurrent_non_atomic_same_collection.js b/jstests/replsets/apply_ops_concurrent_non_atomic_same_collection.js new file mode 100644 index 00000000000..004eeaaa52f --- /dev/null +++ b/jstests/replsets/apply_ops_concurrent_non_atomic_same_collection.js @@ -0,0 +1,11 @@ +(function() { + 'use strict'; + + load('jstests/replsets/libs/apply_ops_concurrent_non_atomic.js'); + + new ApplyOpsConcurrentNonAtomicTest({ + ns1: 'test.coll', + ns2: 'test.coll', + requiresDocumentLevelConcurrency: true, + }).run(); +}()); diff --git a/jstests/replsets/apply_ops_concurrent_non_atomic_same_db.js b/jstests/replsets/apply_ops_concurrent_non_atomic_same_db.js new file mode 100644 index 00000000000..10f874382a5 --- /dev/null +++ b/jstests/replsets/apply_ops_concurrent_non_atomic_same_db.js @@ -0,0 +1,11 @@ +(function() { + 'use strict'; + + load('jstests/replsets/libs/apply_ops_concurrent_non_atomic.js'); + + new ApplyOpsConcurrentNonAtomicTest({ + ns1: 'test.coll1', + ns2: 'test.coll2', + requiresDocumentLevelConcurrency: false, + }).run(); +}()); diff --git a/jstests/replsets/clean_shutdown_oplog_state.js b/jstests/replsets/clean_shutdown_oplog_state.js index 5ac2f72d556..91b31fc66d7 100644 --- a/jstests/replsets/clean_shutdown_oplog_state.js +++ b/jstests/replsets/clean_shutdown_oplog_state.js @@ -10,7 +10,7 @@ var rst = new ReplSetTest({ name: "name", nodes: 2, - oplogSize: 100, + oplogSize: 500, }); rst.startSet(); diff --git a/jstests/replsets/index_delete.js b/jstests/replsets/index_delete.js index f7c45e2f2e0..1c352a18581 100644 --- a/jstests/replsets/index_delete.js +++ b/jstests/replsets/index_delete.js @@ -60,7 +60,7 @@ try { } else { return false; } - }, "index not started on secondary", 30000, 50); + }, "index not started on secondary"); } finally { // Turn off failpoint and let the index build resumes. assert.commandWorked( diff --git a/jstests/replsets/initial_sync_rename_collection_unsafe.js b/jstests/replsets/initial_sync_rename_collection_unsafe.js new file mode 100644 index 00000000000..a105e1c9287 --- /dev/null +++ b/jstests/replsets/initial_sync_rename_collection_unsafe.js @@ -0,0 +1,61 @@ +/** + * Tests that renameCollection commands do not abort initial sync when users specify + * 'allowUnsafeRenamesDuringInitialSync'. + */ + +(function() { + 'use strict'; + + load("jstests/libs/check_log.js"); + + var parameters = TestData.setParameters; + if (parameters && parameters.indexOf("use3dot2InitialSync=true") != -1) { + jsTest.log("Skipping this test because use3dot2InitialSync was provided."); + return; + } + + const basename = 'initial_sync_rename_collection_unsafe'; + + const rst = new ReplSetTest({name: basename, nodes: 1}); + rst.startSet(); + rst.initiate(); + + const dbName = 'd'; + const primary = rst.getPrimary(); + const primaryDB = primary.getDB(dbName); + + assert.writeOK(primaryDB['foo'].save({})); + + jsTestLog('Bring up a new node'); + const secondary = rst.add({setParameter: {allowUnsafeRenamesDuringInitialSync: true}}); + assert.commandWorked(secondary.adminCommand( + {configureFailPoint: 'initialSyncHangBeforeCopyingDatabases', mode: 'alwaysOn'})); + rst.reInitiate(); + assert.eq(primary, rst.getPrimary(), 'Primary changed after reconfig'); + + // Wait for fail point message to be logged. + checkLog.contains(secondary, + 'initial sync - initialSyncHangBeforeCopyingDatabases fail point enabled'); + + jsTestLog('Rename collection on the primary'); + assert.commandWorked(primaryDB['foo'].renameCollection('renamed')); + + assert.commandWorked(secondary.adminCommand( + {configureFailPoint: 'initialSyncHangBeforeCopyingDatabases', mode: 'off'})); + + checkLog.contains(secondary, 'allowUnsafeRenamesDuringInitialSync set to true'); + + jsTestLog('Wait for both nodes to be up-to-date'); + rst.awaitSecondaryNodes(); + rst.awaitReplication(); + + jsTestLog('Check that all collections were renamed correctly on the secondary'); + const secondaryDB = secondary.getDB(dbName); + assert.eq(secondaryDB['renamed'].find().itcount(), 1, 'renamed collection does not exist'); + assert.eq(secondaryDB['foo'].find().itcount(), 0, 'collection `foo` exists after rename'); + + let res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1, initialSync: 1})); + assert.eq(res.initialSyncStatus.failedInitialSyncAttempts, 0); + + rst.stopSet(); +})(); diff --git a/jstests/replsets/last_vote.js b/jstests/replsets/last_vote.js index b9e0474217c..44d349f3237 100644 --- a/jstests/replsets/last_vote.js +++ b/jstests/replsets/last_vote.js @@ -134,9 +134,6 @@ "replSetRequestVotes response had the wrong term: " + tojson(response)); assert(!response.voteGranted, "node granted vote in term before last vote doc: " + tojson(response)); - assert.eq(response.reason, - "candidate's term is lower than mine", - "replSetRequestVotes response had the wrong reason: " + tojson(response)); assertNodeHasLastVote(node0, term, rst.nodes[0]); assertCurrentTerm(node0, term); @@ -178,9 +175,6 @@ "replSetRequestVotes response had the wrong term: " + tojson(response)); assert(!response.voteGranted, "node granted vote in term of last vote doc: " + tojson(response)); - assert.eq(response.reason, - "already voted for another candidate this term", - "replSetRequestVotes response had the wrong reason: " + tojson(response)); assertNodeHasLastVote(node0, term, rst.nodes[0]); assertCurrentTerm(node0, term); diff --git a/jstests/replsets/libs/apply_ops_concurrent_non_atomic.js b/jstests/replsets/libs/apply_ops_concurrent_non_atomic.js new file mode 100644 index 00000000000..5e170378de2 --- /dev/null +++ b/jstests/replsets/libs/apply_ops_concurrent_non_atomic.js @@ -0,0 +1,232 @@ +/** + * This test ensures that multiple non-atomic applyOps commands can run concurrently. + * Prior to SERVER-29802, applyOps would acquire the global lock regardless of the + * atomicity of the operations (as a whole) being applied. + * + * Every instance of ApplyOpsConcurrentNonAtomicTest is configured with an "options" document + * with the following format: + * { + * ns1: , + * ns1: , + * requiresDocumentLevelConcurrency: , + * } + * + * ns1: + * Fully qualified namespace of first set of CRUD operations. For simplicity, only insert + * operations will be used. The set of documents generated for the inserts into ns1 will have + * _id values distinct from those generated for ns2. + * + * ns2: + * Fully qualified namespace of second set of CRUD operations. This may be the same namespace as + * ns1. As with ns1, only insert operations will be used. + * + * requiresDocumentLevelConcurrency: + * Set to true if this test case can only be run with a storage engine that supports document + * level concurrency. + */ +var ApplyOpsConcurrentNonAtomicTest = function(options) { + 'use strict'; + + load('jstests/concurrency/fsm_workload_helpers/server_types.js'); + + if (!(this instanceof ApplyOpsConcurrentNonAtomicTest)) { + return new ApplyOpsConcurrentNonAtomicTest(options); + } + + // Capture the 'this' reference + var self = this; + + self.options = options; + + /** + * Logs message using test name as prefix. + */ + function testLog(message) { + jsTestLog('ApplyOpsConcurrentNonAtomicTest: ' + message); + } + + /** + * Creates an array of insert operations for applyOps into collection 'coll'. + */ + function generateInsertOps(coll, numOps, id) { + // Explicit 'use strict' to prevent mozjs from injecting its own "use strict" directive + // (with incorrect indentation) when we convert this function into a string for + // startParallelShell(). + 'use strict'; + const ops = Array(numOps).fill('ignored').map((unused, i) => { + return {op: 'i', ns: coll.getFullName(), o: {_id: (id * numOps + i), id: id}}; + }); + return ops; + } + + /** + * Runs applyOps in non-atomic mode to insert 'numOps' documents into collection 'coll'. + */ + function applyOpsInsertNonAtomic(coll, numOps, id) { + 'use strict'; + const ops = generateInsertOps(coll, numOps, id); + const mydb = coll.getDB(); + assert.commandWorked(mydb.runCommand({applyOps: ops, allowAtomic: false}), + 'failed to insert documents into ' + coll.getFullName()); + } + + /** + * Parses 'numOps' and collection namespace from 'options' and runs applyOps to inserted + * generated documents. + * + * options format: + * { + * ns: , + * numOps: , + * id: , + * } + * + * ns: + * Fully qualified namespace of collection to insert documents into. + * + * numOps: + * Number of insert operations to generate for applyOps command. + * + * id: + * Index of collection for applyOps. Used with 'numOps' to generate _id values that will not + * collide with collections with different indexes. + */ + function insertFunction(options) { + 'use strict'; + + const coll = db.getMongo().getCollection(options.ns); + const numOps = options.numOps; + const id = options.id; + + testLog('Starting to apply ' + numOps + ' operations in collection ' + coll.getFullName()); + applyOpsInsertNonAtomic(coll, numOps, id); + testLog('Successfully applied ' + numOps + ' operations in collection ' + + coll.getFullName()); + } + + /** + * Creates a function for startParallelShell() to run that will insert documents into + * collection 'coll' using applyOps. + */ + function createInsertFunction(coll, numOps, id) { + const options = { + ns: coll.getFullName(), + numOps: numOps, + id: id, + }; + const functionName = 'insertFunction_' + coll.getFullName().replace(/\./g, '_'); + const s = // + '\n\n' + // + 'const testLog = ' + testLog + ';\n\n' + // + 'const generateInsertOps = ' + generateInsertOps + ';\n\n' + // + 'const applyOpsInsertNonAtomic = ' + applyOpsInsertNonAtomic + ';\n\n' + // + 'const ' + functionName + ' = ' + insertFunction + ';\n\n' + // + functionName + '(' + tojson(options) + ');'; // + return s; + } + + /** + * Returns number of insert operations reported by serverStatus. + * In 3.4 'opcountersRepl', not 'opcounters' was previously the correct field. Now non-atomic + * ops are now replicated as they are applied and are counted toward the global op counter. + */ + function getInsertOpCount(serverStatus) { + return serverStatus.opcounters.insert; + } + + /** + * Runs the test. + */ + this.run = function() { + const options = this.options; + + assert(options.ns1, 'collection 1 namespace not provided'); + assert(options.ns2, 'collection 2 namespace not provided'); + + const replTest = new ReplSetTest({nodes: 1}); + replTest.startSet(); + replTest.initiate(); + + const primary = replTest.getPrimary(); + const adminDb = primary.getDB('admin'); + + if (options.requiresDocumentLevelConcurrency && + !supportsDocumentLevelConcurrency(adminDb)) { + testLog('Skipping test because storage engine does not support document level ' + + 'concurrency.'); + return; + } + + const coll1 = primary.getCollection(options.ns1); + const db1 = coll1.getDB(); + const coll2 = primary.getCollection(options.ns2); + const db2 = coll2.getDB(); + + assert.commandWorked(db1.createCollection(coll1.getName())); + if (coll1.getFullName() !== coll2.getFullName()) { + assert.commandWorked(db2.createCollection(coll2.getName())); + } + + // Enable fail point to pause applyOps between operations. + assert.commandWorked(primary.adminCommand( + {configureFailPoint: 'applyOpsPauseBetweenOperations', mode: 'alwaysOn'})); + + // This logs each operation being applied. + const previousLogLevel = + assert.commandWorked(primary.setLogLevel(3, 'replication')).was.replication.verbosity; + + testLog('Applying operations in collections ' + coll1.getFullName() + ' and ' + + coll2.getFullName()); + + const numOps = 100; + const insertProcess1 = + startParallelShell(createInsertFunction(coll1, numOps, 0), replTest.getPort(0)); + const insertProcess2 = + startParallelShell(createInsertFunction(coll2, numOps, 1), replTest.getPort(0)); + + // The fail point will prevent applyOps from advancing past the first operation in each + // batch of operations. If applyOps is applying both sets of operations concurrently without + // holding the global lock, the insert opcounter will eventually be incremented to 2. + try { + let insertOpCount = 0; + const expectedFinalOpCount = 2; + assert.soon( + function() { + const serverStatus = adminDb.serverStatus(); + insertOpCount = getInsertOpCount(serverStatus); + // This assertion may fail if the fail point is not implemented correctly within + // applyOps. This allows us to fail fast instead of waiting for the + // assert.soon() function to time out. + assert.lte(insertOpCount, + expectedFinalOpCount, + 'Expected at most ' + expectedFinalOpCount + + ' documents inserted with fail point enabled. ' + + 'Most recent insert operation count = ' + insertOpCount); + return insertOpCount === expectedFinalOpCount; + }, + 'Insert operation count did not reach ' + expectedFinalOpCount + + ' as expected with fail point enabled. Most recent insert operation count = ' + + insertOpCount); + } finally { + assert.commandWorked(primary.adminCommand( + {configureFailPoint: 'applyOpsPauseBetweenOperations', mode: 'off'})); + } + + insertProcess1(); + insertProcess2(); + + testLog('Successfully applied operations in collections ' + coll1.getFullName() + ' and ' + + coll2.getFullName()); + + // Reset log level. + primary.setLogLevel(previousLogLevel, 'replication'); + + const serverStatus = adminDb.serverStatus(); + assert.eq(200, + getInsertOpCount(serverStatus), + 'incorrect number of insert operations in server status after applyOps: ' + + tojson(serverStatus)); + + replTest.stopSet(); + }; +}; diff --git a/jstests/replsets/libs/apply_ops_insert_write_conflict.js b/jstests/replsets/libs/apply_ops_insert_write_conflict.js index 88b299c08b6..d0a47f5b6d8 100644 --- a/jstests/replsets/libs/apply_ops_insert_write_conflict.js +++ b/jstests/replsets/libs/apply_ops_insert_write_conflict.js @@ -36,14 +36,6 @@ var ApplyOpsInsertWriteConflictTest = function(options) { return {op: 'i', ns: t.getFullName(), o: {_id: i}}; }); - if (!options.atomic) { - // Adding a command to the list of operations to prevent the applyOps command from - // applying - // all the operations atomically. - ops.push({ns: "test.$cmd", op: "c", o: {applyOps: []}}); - numOps++; - } - // Probabilities for WCE are chosen based on empirical testing. // The probability for WCE during an atomic applyOps should be much smaller than that for // the non-atomic case because we have to attempt to re-apply the entire batch of 'numOps' @@ -62,7 +54,7 @@ var ApplyOpsInsertWriteConflictTest = function(options) { var previousLogLevel = assert.commandWorked(primaryDB.setLogLevel(3, 'replication')).was.replication.verbosity; - var applyOpsResult = primaryDB.adminCommand({applyOps: ops}); + var applyOpsResult = primaryDB.adminCommand({applyOps: ops, allowAtomic: options.atomic}); // Reset log level. primaryDB.setLogLevel(previousLogLevel, 'replication'); diff --git a/jstests/replsets/no_flapping_during_network_partition.js b/jstests/replsets/no_flapping_during_network_partition.js index 5a289b2afdd..daab9be2bca 100644 --- a/jstests/replsets/no_flapping_during_network_partition.js +++ b/jstests/replsets/no_flapping_during_network_partition.js @@ -39,7 +39,7 @@ primary.disconnect(secondary); jsTestLog("Wait long enough for the secondary to call for an election."); - checkLog.contains(secondary, "can see a healthy primary of equal or greater priority"); + checkLog.contains(secondary, "can see a healthy primary"); jsTestLog("Verify the primary and secondary do not change during the partition."); assert.eq(primary, replTest.getPrimary()); diff --git a/jstests/replsets/noop_writes_wait_for_write_concern.js b/jstests/replsets/noop_writes_wait_for_write_concern.js new file mode 100644 index 00000000000..61ffc518df4 --- /dev/null +++ b/jstests/replsets/noop_writes_wait_for_write_concern.js @@ -0,0 +1,235 @@ +/** + * This file tests that if a user initiates a write that becomes a noop due to being a duplicate + * operation, that we still wait for write concern. This is because we must wait for write concern + * on the write that made this a noop so that we can be sure it doesn't get rolled back if we + * acknowledge it. + */ + +(function() { + "use strict"; + load('jstests/libs/write_concern_util.js'); + + var name = 'noop_writes_wait_for_write_concern'; + var replTest = new ReplSetTest({ + name: name, + nodes: [{}, {rsConfig: {priority: 0}}, {rsConfig: {priority: 0}}], + }); + replTest.startSet(); + replTest.initiate(); + // Stops node 1 so that all w:3 write concerns time out. We have 3 data bearing nodes so that + // 'dropDatabase' can satisfy its implicit writeConcern: majority but still time out from the + // explicit w:3 write concern. + replTest.stop(1); + + var primary = replTest.getPrimary(); + assert.eq(primary, replTest.nodes[0]); + var dbName = 'testDB'; + var db = primary.getDB(dbName); + var collName = 'testColl'; + var coll = db[collName]; + + function dropTestCollection() { + coll.drop(); + assert.eq(0, coll.find().itcount(), "test collection not empty"); + } + + // Each entry in this array contains a command whose noop write concern behavior needs to be + // tested. Entries have the following structure: + // { + // req: , // Command request object that will result in a noop + // // write after the setup function is called. + // + // setupFunc: , // Function to run to ensure that the request is a + // // noop. + // + // confirmFunc: , // Function to run after the command is run to ensure + // // that it executed properly. Accepts the result of + // // the noop request to validate it. + // } + var commands = []; + + commands.push({ + req: {applyOps: [{op: "i", ns: coll.getFullName(), o: {_id: 1}}]}, + setupFunc: function() { + assert.writeOK(coll.insert({_id: 1})); + }, + confirmFunc: function(res) { + assert.commandWorked(res); + assert.eq(res.applied, 1); + assert.eq(res.results[0], true); + assert.eq(coll.find().itcount(), 1); + assert.eq(coll.count({_id: 1}), 1); + } + }); + + // 'update' where the document to update does not exist. + commands.push({ + req: {update: collName, updates: [{q: {a: 1}, u: {b: 2}}]}, + setupFunc: function() { + assert.writeOK(coll.insert({a: 1})); + assert.writeOK(coll.update({a: 1}, {b: 2})); + }, + confirmFunc: function(res) { + assert.commandWorked(res); + assert.eq(res.n, 0); + assert.eq(res.nModified, 0); + assert.eq(coll.find().itcount(), 1); + assert.eq(coll.count({b: 2}), 1); + } + }); + + // 'update' where the update has already been done. + commands.push({ + req: {update: collName, updates: [{q: {a: 1}, u: {$set: {b: 2}}}]}, + setupFunc: function() { + assert.writeOK(coll.insert({a: 1})); + assert.writeOK(coll.update({a: 1}, {$set: {b: 2}})); + }, + confirmFunc: function(res) { + assert.commandWorked(res); + assert.eq(res.n, 1); + assert.eq(res.nModified, 0); + assert.eq(coll.find().itcount(), 1); + assert.eq(coll.count({a: 1, b: 2}), 1); + } + }); + + commands.push({ + req: {delete: collName, deletes: [{q: {a: 1}, limit: 1}]}, + setupFunc: function() { + assert.writeOK(coll.insert({a: 1})); + assert.writeOK(coll.remove({a: 1})); + }, + confirmFunc: function(res) { + assert.commandWorked(res); + assert.eq(res.n, 0); + assert.eq(coll.count({a: 1}), 0); + } + }); + + commands.push({ + req: {createIndexes: collName, indexes: [{key: {a: 1}, name: "a_1"}]}, + setupFunc: function() { + assert.writeOK(coll.insert({a: 1})); + assert.commandWorked( + db.runCommand({createIndexes: collName, indexes: [{key: {a: 1}, name: "a_1"}]})); + }, + confirmFunc: function(res) { + assert.commandWorked(res); + assert.eq(res.numIndexesBefore, res.numIndexesAfter); + } + }); + + // 'findAndModify' where the document to update does not exist. + commands.push({ + req: {findAndModify: collName, query: {a: 1}, update: {b: 2}}, + setupFunc: function() { + assert.writeOK(coll.insert({a: 1})); + assert.commandWorked( + db.runCommand({findAndModify: collName, query: {a: 1}, update: {b: 2}})); + }, + confirmFunc: function(res) { + assert.commandWorked(res); + assert.eq(res.lastErrorObject.updatedExisting, false); + assert.eq(coll.find().itcount(), 1); + assert.eq(coll.count({b: 2}), 1); + } + }); + + // 'findAndModify' where the update has already been done. + commands.push({ + req: {findAndModify: collName, query: {a: 1}, update: {$set: {b: 2}}}, + setupFunc: function() { + assert.writeOK(coll.insert({a: 1})); + assert.commandWorked( + db.runCommand({findAndModify: collName, query: {a: 1}, update: {$set: {b: 2}}})); + }, + confirmFunc: function(res) { + assert.commandWorked(res); + assert.eq(res.lastErrorObject.updatedExisting, true); + assert.eq(coll.find().itcount(), 1); + assert.eq(coll.count({a: 1, b: 2}), 1); + } + }); + + commands.push({ + req: {dropDatabase: 1}, + setupFunc: function() { + assert.writeOK(coll.insert({a: 1})); + assert.commandWorked(db.runCommand({dropDatabase: 1})); + }, + confirmFunc: function(res) { + assert.commandWorked(res); + } + }); + + commands.push({ + req: {drop: collName}, + setupFunc: function() { + assert.writeOK(coll.insert({a: 1})); + assert.commandWorked(db.runCommand({drop: collName})); + }, + confirmFunc: function(res) { + assert.commandFailedWithCode(res, ErrorCodes.NamespaceNotFound); + } + }); + + commands.push({ + req: {create: collName}, + setupFunc: function() { + assert.commandWorked(db.runCommand({create: collName})); + }, + confirmFunc: function(res) { + assert.commandFailedWithCode(res, ErrorCodes.NamespaceExists); + } + }); + + commands.push({ + req: {insert: collName, documents: [{_id: 1}]}, + setupFunc: function() { + assert.writeOK(coll.insert({_id: 1})); + }, + confirmFunc: function(res) { + assert.commandWorked(res); + assert.eq(res.n, 0); + assert.eq(res.writeErrors[0].code, ErrorCodes.DuplicateKey); + assert.eq(coll.count({_id: 1}), 1); + } + }); + + function testCommandWithWriteConcern(cmd) { + // Provide a small wtimeout that we expect to time out. + cmd.req.writeConcern = {w: 3, wtimeout: 1000}; + jsTest.log("Testing " + tojson(cmd.req)); + + dropTestCollection(); + + cmd.setupFunc(); + + // We run the command on a different connection. If the the command were run on the + // same connection, then the client last op for the noop write would be set by the setup + // operation. By using a fresh connection the client last op begins as null. + // This test explicitly tests that write concern for noop writes works when the + // client last op has not already been set by a duplicate operation. + var shell2 = new Mongo(primary.host); + + // We check the error code of 'res' in the 'confirmFunc'. + var res = shell2.getDB(dbName).runCommand(cmd.req); + + try { + // Tests that the command receives a write concern error. If we don't wait for write + // concern on noop writes then we won't get a write concern error. + assertWriteConcernError(res); + cmd.confirmFunc(res); + } catch (e) { + // Make sure that we print out the response. + printjson(res); + throw e; + } + } + + commands.forEach(function(cmd) { + testCommandWithWriteConcern(cmd); + }); + +})(); \ No newline at end of file diff --git a/jstests/replsets/read_committed_with_catalog_changes.js b/jstests/replsets/read_committed_with_catalog_changes.js index 02d26759e26..7928b16d3ae 100644 --- a/jstests/replsets/read_committed_with_catalog_changes.js +++ b/jstests/replsets/read_committed_with_catalog_changes.js @@ -214,7 +214,7 @@ load("jstests/replsets/rslib.js"); // For startSetIfSupportsReadMajority. "Expected read of " + coll.getFullName() + " to block"); } - function assertReadsSucceed(coll, timeoutMs = 10000) { + function assertReadsSucceed(coll, timeoutMs = 20000) { var res = coll.runCommand('find', {"readConcern": {"level": "majority"}, "maxTimeMS": timeoutMs}); assert.commandWorked(res, 'reading from ' + coll.getFullName()); diff --git a/jstests/replsets/startParallelShell.js b/jstests/replsets/startParallelShell.js new file mode 100644 index 00000000000..beca88d19a9 --- /dev/null +++ b/jstests/replsets/startParallelShell.js @@ -0,0 +1,33 @@ +// Test startParallelShell() in a replica set. + +var db; + +(function() { + 'use strict'; + + const setName = 'rs0'; + const replSet = new ReplSetTest({name: setName, nodes: 3}); + const nodes = replSet.nodeList(); + replSet.startSet(); + replSet.initiate(); + + const url = replSet.getURL(); + print("* Connecting to " + url); + const mongo = new Mongo(url); + db = mongo.getDB('admin'); + assert.eq(url, mongo.host, "replSet.getURL() should match active connection string"); + + print("* Starting parallel shell on --host " + db.getMongo().host); + startParallelShell('db.coll0.insert({test: "connString only"});'); + assert.soon(function() { + return db.coll0.find({test: "connString only"}).count() === 1; + }); + + const uri = new MongoURI(url); + const port0 = uri.servers[0].port; + print("* Starting parallel shell w/ --port " + port0); + startParallelShell('db.coll0.insert({test: "explicit port"});', port0); + assert.soon(function() { + return db.coll0.find({test: "explicit port"}).count() === 1; + }); +})(); diff --git a/jstests/replsets/too_stale_secondary.js b/jstests/replsets/too_stale_secondary.js index 369662e5f16..1a0c28b3454 100644 --- a/jstests/replsets/too_stale_secondary.js +++ b/jstests/replsets/too_stale_secondary.js @@ -34,7 +34,7 @@ "use strict"; function getFirstOplogEntry(conn) { - return conn.getDB('local').oplog.rs.find().sort({ts: 1}).limit(1)[0]; + return conn.getDB('local').oplog.rs.find().sort({$natural: 1}).limit(1)[0]; } /** diff --git a/jstests/sharding/auth.js b/jstests/sharding/auth.js index 61b25f10dde..037532acaec 100644 --- a/jstests/sharding/auth.js +++ b/jstests/sharding/auth.js @@ -38,7 +38,7 @@ name: "auth", mongos: 1, shards: 0, - other: {keyFile: "jstests/libs/key1", chunkSize: 1, enableAutoSplit: true}, + other: {keyFile: "jstests/libs/key1", chunkSize: 1, enableAutoSplit: false}, }); if (s.getDB('admin').runCommand('buildInfo').bits < 64) { @@ -167,6 +167,7 @@ s.getDB("test").foo.remove({}); var num = 10000; + assert.commandWorked(s.s.adminCommand({split: "test.foo", middle: {x: num / 2}})); var bulk = s.getDB("test").foo.initializeUnorderedBulkOp(); for (i = 0; i < num; i++) { bulk.insert( diff --git a/jstests/sharding/auto_rebalance_parallel.js b/jstests/sharding/auto_rebalance_parallel.js index 955319b8c5d..c7078a6898a 100644 --- a/jstests/sharding/auto_rebalance_parallel.js +++ b/jstests/sharding/auto_rebalance_parallel.js @@ -1,43 +1,71 @@ /** * Tests that the cluster is balanced in parallel in one balancer round (standalone). */ + (function() { 'use strict'; var st = new ShardingTest({shards: 4}); + var config = st.s0.getDB('config'); assert.commandWorked(st.s0.adminCommand({enableSharding: 'TestDB'})); st.ensurePrimaryShard('TestDB', st.shard0.shardName); - assert.commandWorked(st.s0.adminCommand({shardCollection: 'TestDB.TestColl', key: {Key: 1}})); - var coll = st.s0.getDB('TestDB').TestColl; + function prepareCollectionForBalance(collName) { + assert.commandWorked(st.s0.adminCommand({shardCollection: collName, key: {Key: 1}})); + + var coll = st.s0.getCollection(collName); + + // Create 4 chunks initially and ensure they get balanced within 1 balancer round + assert.writeOK(coll.insert({Key: 1, Value: 'Test value 1'})); + assert.writeOK(coll.insert({Key: 10, Value: 'Test value 10'})); + assert.writeOK(coll.insert({Key: 20, Value: 'Test value 20'})); + assert.writeOK(coll.insert({Key: 30, Value: 'Test value 30'})); + + assert.commandWorked(st.splitAt(collName, {Key: 10})); + assert.commandWorked(st.splitAt(collName, {Key: 20})); + assert.commandWorked(st.splitAt(collName, {Key: 30})); + + // Move two of the chunks to shard0001 so we have option to do parallel balancing + assert.commandWorked(st.moveChunk(collName, {Key: 20}, st.shard1.shardName)); + assert.commandWorked(st.moveChunk(collName, {Key: 30}, st.shard1.shardName)); - // Create 4 chunks initially and ensure they get balanced within 1 balancer round - assert.writeOK(coll.insert({Key: 1, Value: 'Test value 1'})); - assert.writeOK(coll.insert({Key: 10, Value: 'Test value 10'})); - assert.writeOK(coll.insert({Key: 20, Value: 'Test value 20'})); - assert.writeOK(coll.insert({Key: 30, Value: 'Test value 30'})); + assert.eq(2, config.chunks.find({ns: collName, shard: st.shard0.shardName}).itcount()); + assert.eq(2, config.chunks.find({ns: collName, shard: st.shard1.shardName}).itcount()); + } - assert.commandWorked(st.splitAt('TestDB.TestColl', {Key: 10})); - assert.commandWorked(st.splitAt('TestDB.TestColl', {Key: 20})); - assert.commandWorked(st.splitAt('TestDB.TestColl', {Key: 30})); + function checkCollectionBalanced(collName) { + assert.eq(1, config.chunks.find({ns: collName, shard: st.shard0.shardName}).itcount()); + assert.eq(1, config.chunks.find({ns: collName, shard: st.shard1.shardName}).itcount()); + assert.eq(1, config.chunks.find({ns: collName, shard: st.shard2.shardName}).itcount()); + assert.eq(1, config.chunks.find({ns: collName, shard: st.shard3.shardName}).itcount()); + } - // Move two of the chunks to shard0001 so we have option to do parallel balancing - assert.commandWorked(st.moveChunk('TestDB.TestColl', {Key: 20}, st.shard1.shardName)); - assert.commandWorked(st.moveChunk('TestDB.TestColl', {Key: 30}, st.shard1.shardName)); + function countMoves(collName) { + return config.changelog.find({what: 'moveChunk.start', ns: collName}).itcount(); + } - assert.eq(2, st.s0.getDB('config').chunks.find({shard: st.shard0.shardName}).itcount()); - assert.eq(2, st.s0.getDB('config').chunks.find({shard: st.shard1.shardName}).itcount()); + prepareCollectionForBalance('TestDB.TestColl1'); + prepareCollectionForBalance('TestDB.TestColl2'); + + // Count the moveChunk start attempts accurately and ensure that only the correct number of + // migrations are scheduled + const testColl1InitialMoves = countMoves('TestDB.TestColl1'); + const testColl2InitialMoves = countMoves('TestDB.TestColl2'); - // Do enable the balancer and wait for a single balancer round st.startBalancer(); st.awaitBalancerRound(); + st.awaitBalancerRound(); st.stopBalancer(); - assert.eq(1, st.s0.getDB('config').chunks.find({shard: st.shard0.shardName}).itcount()); - assert.eq(1, st.s0.getDB('config').chunks.find({shard: st.shard1.shardName}).itcount()); - assert.eq(1, st.s0.getDB('config').chunks.find({shard: st.shard2.shardName}).itcount()); - assert.eq(1, st.s0.getDB('config').chunks.find({shard: st.shard3.shardName}).itcount()); + checkCollectionBalanced('TestDB.TestColl1'); + checkCollectionBalanced('TestDB.TestColl2'); + + assert.eq(2, countMoves('TestDB.TestColl1') - testColl1InitialMoves); + assert.eq(2, countMoves('TestDB.TestColl2') - testColl2InitialMoves); + + // Ensure there are no migration errors reported + assert.eq(0, config.changelog.find({what: 'moveChunk.error'}).itcount()); st.stop(); })(); diff --git a/jstests/sharding/autosplit.js b/jstests/sharding/autosplit.js index bb34021487f..0eba386b6a3 100644 --- a/jstests/sharding/autosplit.js +++ b/jstests/sharding/autosplit.js @@ -4,7 +4,12 @@ (function() { 'use strict'; - var s = new ShardingTest({name: "auto1", shards: 2, mongos: 1, other: {enableAutoSplit: true}}); + var s = new ShardingTest({ + name: "auto1", + shards: 2, + mongos: 1, + other: {enableAutoSplit: true, chunkSize: 10}, + }); assert.commandWorked(s.s0.adminCommand({enablesharding: "test"})); s.ensurePrimaryShard('test', 'shard0001'); diff --git a/jstests/sharding/mapReduce_inSharded_outSharded.js b/jstests/sharding/mapReduce_inSharded_outSharded.js index d1aba2599f0..d73eb517e98 100644 --- a/jstests/sharding/mapReduce_inSharded_outSharded.js +++ b/jstests/sharding/mapReduce_inSharded_outSharded.js @@ -1,60 +1,70 @@ -var verifyOutput = function(out) { - printjson(out); - assert.eq(out.counts.input, 51200, "input count is wrong"); - assert.eq(out.counts.emit, 51200, "emit count is wrong"); - assert.gt(out.counts.reduce, 99, "reduce count is wrong"); - assert.eq(out.counts.output, 512, "output count is wrong"); -}; - -var st = new ShardingTest( - {shards: 2, verbose: 1, mongos: 1, other: {chunkSize: 1, enableBalancer: true}}); - -st.adminCommand({enablesharding: "mrShard"}); -st.ensurePrimaryShard('mrShard', 'shard0001'); -st.adminCommand({shardcollection: "mrShard.srcSharded", key: {"_id": 1}}); - -var db = st.getDB("mrShard"); - -var bulk = db.srcSharded.initializeUnorderedBulkOp(); -for (j = 0; j < 100; j++) { - for (i = 0; i < 512; i++) { - bulk.insert({j: j, i: i}); +(function() { + "use strict"; + + var verifyOutput = function(out) { + printjson(out); + assert.eq(out.counts.input, 51200, "input count is wrong"); + assert.eq(out.counts.emit, 51200, "emit count is wrong"); + assert.gt(out.counts.reduce, 99, "reduce count is wrong"); + assert.eq(out.counts.output, 512, "output count is wrong"); + }; + + var st = new ShardingTest( + {shards: 2, verbose: 1, mongos: 1, other: {chunkSize: 1, enableBalancer: true}}); + + var admin = st.s0.getDB('admin'); + + assert.commandWorked(admin.runCommand({enablesharding: "mrShard"})); + st.ensurePrimaryShard('mrShard', 'shard0001'); + assert.commandWorked( + admin.runCommand({shardcollection: "mrShard.srcSharded", key: {"_id": 1}})); + + var db = st.s0.getDB("mrShard"); + + var bulk = db.srcSharded.initializeUnorderedBulkOp(); + for (var j = 0; j < 100; j++) { + for (var i = 0; i < 512; i++) { + bulk.insert({j: j, i: i}); + } + } + assert.writeOK(bulk.execute()); + + function map() { + emit(this.i, 1); + } + function reduce(key, values) { + return Array.sum(values); } -} -assert.writeOK(bulk.execute()); - -function map() { - emit(this.i, 1); -} -function reduce(key, values) { - return Array.sum(values); -} - -// sharded src sharded dst -var suffix = "InShardedOutSharded"; - -var out = - db.srcSharded.mapReduce(map, reduce, {out: {replace: "mrReplace" + suffix, sharded: true}}); -verifyOutput(out); - -out = db.srcSharded.mapReduce(map, reduce, {out: {merge: "mrMerge" + suffix, sharded: true}}); -verifyOutput(out); - -out = db.srcSharded.mapReduce(map, reduce, {out: {reduce: "mrReduce" + suffix, sharded: true}}); -verifyOutput(out); - -out = db.srcSharded.mapReduce(map, reduce, {out: {inline: 1}}); -verifyOutput(out); -assert(out.results != 'undefined', "no results for inline"); - -out = db.srcSharded.mapReduce( - map, reduce, {out: {replace: "mrReplace" + suffix, db: "mrShardOtherDB", sharded: true}}); -verifyOutput(out); - -out = db.runCommand({ - mapReduce: "srcSharded", // use new name mapReduce rather than mapreduce - map: map, - reduce: reduce, - out: "mrBasic" + "srcSharded", -}); -verifyOutput(out); + + // sharded src sharded dst + var suffix = "InShardedOutSharded"; + + var out = + db.srcSharded.mapReduce(map, reduce, {out: {replace: "mrReplace" + suffix, sharded: true}}); + verifyOutput(out); + + out = db.srcSharded.mapReduce(map, reduce, {out: {merge: "mrMerge" + suffix, sharded: true}}); + verifyOutput(out); + + out = db.srcSharded.mapReduce(map, reduce, {out: {reduce: "mrReduce" + suffix, sharded: true}}); + verifyOutput(out); + + out = db.srcSharded.mapReduce(map, reduce, {out: {inline: 1}}); + verifyOutput(out); + assert(out.results != 'undefined', "no results for inline"); + + out = db.srcSharded.mapReduce( + map, reduce, {out: {replace: "mrReplace" + suffix, db: "mrShardOtherDB", sharded: true}}); + verifyOutput(out); + + out = db.runCommand({ + mapReduce: "srcSharded", // use new name mapReduce rather than mapreduce + map: map, + reduce: reduce, + out: "mrBasic" + "srcSharded", + }); + verifyOutput(out); + + st.stop(); + +})(); diff --git a/jstests/sharding/migrateBig_balancer.js b/jstests/sharding/migrateBig_balancer.js index 9eb50c2168e..03e98fa5493 100644 --- a/jstests/sharding/migrateBig_balancer.js +++ b/jstests/sharding/migrateBig_balancer.js @@ -31,11 +31,8 @@ assert.eq(40, coll.count(), "prep1"); - printjson(coll.stats()); - - admin.printShardingStatus(); - - admin.runCommand({shardcollection: "" + coll, key: {_id: 1}}); + assert.commandWorked(admin.runCommand({shardcollection: "" + coll, key: {_id: 1}})); + st.printShardingStatus(); assert.lt( 5, mongos.getDB("config").chunks.find({ns: "test.stuff"}).count(), "not enough chunks"); @@ -56,5 +53,4 @@ }, "never migrated", 10 * 60 * 1000, 1000); st.stop(); - })(); diff --git a/jstests/sharding/movechunk_commit_changelog_stats.js b/jstests/sharding/movechunk_commit_changelog_stats.js new file mode 100644 index 00000000000..d257bb6ed94 --- /dev/null +++ b/jstests/sharding/movechunk_commit_changelog_stats.js @@ -0,0 +1,41 @@ +// +// Tests that the changelog entry for moveChunk.commit contains stats on the migration. +// + +(function() { + 'use strict'; + + var st = new ShardingTest({mongos: 1, shards: 2}); + var kDbName = 'db'; + + var mongos = st.s0; + var shard0 = st.shard0.shardName; + var shard1 = st.shard1.shardName; + + assert.commandWorked(mongos.adminCommand({enableSharding: kDbName})); + st.ensurePrimaryShard(kDbName, shard0); + + function assertCountsInChangelog() { + let changeLog = st.s.getDB('config').changelog.find({what: 'moveChunk.commit'}).toArray(); + assert.gt(changeLog.length, 0); + for (let i = 0; i < changeLog.length; i++) { + assert(changeLog[i].details.hasOwnProperty('counts') || + changeLog[i].details.hasOwnProperty('clonedBytes')); + } + } + + var ns = kDbName + '.fooHashed'; + assert.commandWorked(mongos.adminCommand({shardCollection: ns, key: {_id: 'hashed'}})); + + var aChunk = mongos.getDB('config').chunks.findOne({_id: RegExp(ns), shard: shard0}); + assert(aChunk); + + // Assert counts field exists in the changelog entry for moveChunk.commit + assert.commandWorked( + mongos.adminCommand({moveChunk: ns, bounds: [aChunk.min, aChunk.max], to: shard1})); + assertCountsInChangelog(); + + mongos.getDB(kDbName).fooHashed.drop(); + + st.stop(); +})(); \ No newline at end of file diff --git a/jstests/sharding/printShardingStatus.js b/jstests/sharding/printShardingStatus.js index 85330311fd4..cdc36999e51 100644 --- a/jstests/sharding/printShardingStatus.js +++ b/jstests/sharding/printShardingStatus.js @@ -7,6 +7,8 @@ var st = new ShardingTest({shards: 1, mongos: 2, config: 1, other: {smallfiles: true}}); + var standalone = MongoRunner.runMongod(); + var mongos = st.s0; var admin = mongos.getDB("admin"); @@ -85,9 +87,10 @@ testBasicVerboseOnly(outputVerbose); // Take a copy of the config db, in order to test the harder-to-setup cases below. + // Copy into a standalone to also test running printShardingStatus() against a config dump. // TODO: Replace this manual copy with copydb once SERVER-13080 is fixed. var config = mongos.getDB("config"); - var configCopy = mongos.getDB("configCopy"); + var configCopy = standalone.getDB("configCopy"); config.getCollectionInfos().forEach(function(c) { // Create collection with options. assert.commandWorked(configCopy.createCollection(c.name, c.options)); @@ -140,11 +143,11 @@ configCopy.mongos.remove({}); var output = grabStatusOutput(configCopy, false); - assertPresentInOutput(output, "most recently active mongoses:\n\tnone", "no mongoses"); + assertPresentInOutput(output, "most recently active mongoses:\n none", "no mongoses"); var output = grabStatusOutput(configCopy, true); assertPresentInOutput( - output, "most recently active mongoses:\n\tnone", "no mongoses (verbose)"); + output, "most recently active mongoses:\n none", "no mongoses (verbose)"); assert(mongos.getDB(dbName).dropDatabase()); @@ -237,5 +240,7 @@ assert(mongos.getDB("test").dropDatabase()); + MongoRunner.stopMongod(standalone); + st.stop(); })(); diff --git a/jstests/sharding/shard_existing_coll_chunk_count.js b/jstests/sharding/shard_existing_coll_chunk_count.js new file mode 100644 index 00000000000..60145fef712 --- /dev/null +++ b/jstests/sharding/shard_existing_coll_chunk_count.js @@ -0,0 +1,165 @@ +/** + * This test confirms that after sharding a collection with some pre-existing data, + * the resulting chunks aren't auto-split too aggressively. + */ +(function() { + 'use strict'; + + var s = new ShardingTest({ + name: "shard_existing_coll_chunk_count", + shards: 1, + mongos: 1, + other: {enableAutoSplit: true}, + }); + + assert.commandWorked(s.s.adminCommand({enablesharding: "test"})); + + var collNum = 0; + var overhead = Object.bsonsize({_id: ObjectId(), i: 1, pad: ""}); + + var getNumberChunks = function(ns) { + return s.configRS.getPrimary().getDB("config").getCollection("chunks").count({ns}); + }; + + var runCase = function(opts) { + // Expected options. + assert.gte(opts.docSize, 0); + assert.gte(opts.stages.length, 2); + + // Compute padding. + if (opts.docSize < overhead) { + var pad = ""; + } else { + var pad = (new Array(opts.docSize - overhead + 1)).join(' '); + } + + collNum++; + var db = s.getDB("test"); + var collName = "coll" + collNum; + var coll = db.getCollection(collName); + var i = 0; + var limit = 0; + var stageNum = 0; + var stage = opts.stages[stageNum]; + + // Insert initial docs. + var bulk = coll.initializeUnorderedBulkOp(); + limit += stage.numDocsToInsert; + for (; i < limit; i++) { + bulk.insert({i, pad}); + } + assert.writeOK(bulk.execute()); + + // Create shard key index. + assert.commandWorked(coll.createIndex({i: 1})); + + // Shard collection. + assert.commandWorked(s.s.adminCommand({shardcollection: coll.getFullName(), key: {i: 1}})); + + // Confirm initial number of chunks. + var numChunks = getNumberChunks(coll.getFullName()); + assert.eq(numChunks, + stage.expectedNumChunks, + 'in ' + coll.getFullName() + ' expected ' + stage.expectedNumChunks + + ' initial chunks, but found ' + numChunks + '\nopts: ' + tojson(opts) + + '\nchunks:\n' + s.getChunksString(coll.getFullName())); + + // Do the rest of the stages. + for (stageNum = 1; stageNum < opts.stages.length; stageNum++) { + stage = opts.stages[stageNum]; + + // Insert the later docs (one at a time, to maximise the autosplit effects). + limit += stage.numDocsToInsert; + for (; i < limit; i++) { + coll.insert({i, pad}); + } + + // Confirm number of chunks for this stage. + var numChunks = getNumberChunks(coll.getFullName()); + assert.eq(numChunks, + stage.expectedNumChunks, + 'in ' + coll.getFullName() + ' expected ' + stage.expectedNumChunks + + ' chunks for stage ' + stageNum + ', but found ' + numChunks + + '\nopts: ' + tojson(opts) + '\nchunks:\n' + + s.getChunksString(coll.getFullName())); + } + }; + + // Original problematic case. + runCase({ + docSize: 0, + stages: [ + {numDocsToInsert: 20000, expectedNumChunks: 1}, + {numDocsToInsert: 7, expectedNumChunks: 1}, + {numDocsToInsert: 1000, expectedNumChunks: 1}, + ], + }); + + // Original problematic case (worse). + runCase({ + docSize: 0, + stages: [ + {numDocsToInsert: 90000, expectedNumChunks: 1}, + {numDocsToInsert: 7, expectedNumChunks: 1}, + {numDocsToInsert: 1000, expectedNumChunks: 1}, + ], + }); + + // Pathological case #1. + runCase({ + docSize: 522, + stages: [ + {numDocsToInsert: 8191, expectedNumChunks: 1}, + {numDocsToInsert: 2, expectedNumChunks: 1}, + {numDocsToInsert: 1000, expectedNumChunks: 1}, + ], + }); + + // Pathological case #2. + runCase({ + docSize: 522, + stages: [ + {numDocsToInsert: 8192, expectedNumChunks: 1}, + {numDocsToInsert: 8192, expectedNumChunks: 1}, + ], + }); + + // Lower chunksize to 1MB, and restart the mongos for it to take. + assert.writeOK( + s.getDB("config").getCollection("settings").update({_id: "chunksize"}, {$set: {value: 1}}, { + upsert: true + })); + s.restartMongos(0); + + // Original problematic case, scaled down to smaller chunksize. + runCase({ + docSize: 0, + stages: [ + {numDocsToInsert: 10000, expectedNumChunks: 1}, + {numDocsToInsert: 10, expectedNumChunks: 1}, + {numDocsToInsert: 20, expectedNumChunks: 1}, + {numDocsToInsert: 40, expectedNumChunks: 1}, + {numDocsToInsert: 1000, expectedNumChunks: 1}, + ], + }); + + // Docs just smaller than half chunk size. + runCase({ + docSize: 510 * 1024, + stages: [ + {numDocsToInsert: 10, expectedNumChunks: 6}, + {numDocsToInsert: 10, expectedNumChunks: 12}, + ], + }); + + // Docs just larger than half chunk size. + runCase({ + docSize: 514 * 1024, + stages: [ + {numDocsToInsert: 10, expectedNumChunks: 10}, + {numDocsToInsert: 10, expectedNumChunks: 20}, + ], + }); + + s.stop(); +})(); diff --git a/jstests/sharding/shard_identity_rollback.js b/jstests/sharding/shard_identity_rollback.js index c7b6fedaacc..37cc8b10726 100644 --- a/jstests/sharding/shard_identity_rollback.js +++ b/jstests/sharding/shard_identity_rollback.js @@ -89,8 +89,9 @@ } }, function() { - var oldPriOplog = priConn.getDB('local').oplog.rs.find().sort({ts: -1}).toArray(); - var newPriOplog = newPriConn.getDB('local').oplog.rs.find().sort({ts: -1}).toArray(); + var oldPriOplog = priConn.getDB('local').oplog.rs.find().sort({$natural: -1}).toArray(); + var newPriOplog = + newPriConn.getDB('local').oplog.rs.find().sort({$natural: -1}).toArray(); return "timed out waiting for original primary to shut down after rollback. " + "Old primary oplog: " + tojson(oldPriOplog) + "; new primary oplog: " + tojson(newPriOplog); diff --git a/jstests/sharding/write_cmd_auto_split.js b/jstests/sharding/write_cmd_auto_split.js index 1cf9b5ab39a..95151b1e7e9 100644 --- a/jstests/sharding/write_cmd_auto_split.js +++ b/jstests/sharding/write_cmd_auto_split.js @@ -40,7 +40,7 @@ assert.eq(1, configDB.chunks.find().itcount()); - for (var x = 0; x < 1100; x++) { + for (var x = 0; x < 3100; x++) { assert.writeOK(testDB.runCommand({ update: 'update', updates: [{q: {x: x}, u: {x: x, v: doc1k}, upsert: true}], @@ -80,7 +80,7 @@ // Note: Estimated 'chunk size' tracked by mongos is initialized with a random value so // we are going to be conservative. - for (var x = 0; x < 1100; x += 400) { + for (var x = 0; x < 3100; x += 400) { var docs = []; for (var y = 0; y < 400; y++) { @@ -101,7 +101,7 @@ assert.eq(1, configDB.chunks.find().itcount()); - for (var x = 0; x < 1100; x += 400) { + for (var x = 0; x < 3100; x += 400) { var docs = []; for (var y = 0; y < 400; y++) { diff --git a/rpm/init.d-mongod b/rpm/init.d-mongod index 56539ef4d42..4e172b9f15c 100755 --- a/rpm/init.d-mongod +++ b/rpm/init.d-mongod @@ -64,6 +64,7 @@ start() ulimit -n 64000 ulimit -m unlimited ulimit -u 64000 + ulimit -l unlimited echo -n $"Starting mongod: " daemon --user "$MONGO_USER" --check $mongod "$NUMACTL $mongod $OPTIONS >/dev/null 2>&1" diff --git a/rpm/init.d-mongod.suse b/rpm/init.d-mongod.suse index fae1fb8b3f3..2216507a009 100644 --- a/rpm/init.d-mongod.suse +++ b/rpm/init.d-mongod.suse @@ -65,6 +65,7 @@ start() ulimit -n 64000 ulimit -m unlimited ulimit -u 64000 + ulimit -l unlimited echo -n "Starting mongod: " $NUMACTL /sbin/start_daemon -u "$MONGO_USER" -p "$PIDFILEPATH" $mongod $OPTIONS >/dev/null 2>&1 diff --git a/rpm/mongod.service b/rpm/mongod.service index 708bf663614..e4d48d08c1e 100644 --- a/rpm/mongod.service +++ b/rpm/mongod.service @@ -13,6 +13,7 @@ ExecStartPre=/usr/bin/chown mongod:mongod /var/run/mongodb ExecStartPre=/usr/bin/chmod 0755 /var/run/mongodb PermissionsStartOnly=true PIDFile=/var/run/mongodb/mongod.pid +Type=forking # file size LimitFSIZE=infinity # cpu time @@ -23,6 +24,8 @@ LimitAS=infinity LimitNOFILE=64000 # processes/threads LimitNPROC=64000 +# locked memory +LimitMEMLOCK=infinity # total threads (user+kernel) TasksMax=infinity TasksAccounting=false diff --git a/site_scons/site_tools/jstoh.py b/site_scons/site_tools/jstoh.py index 62424a6cd4f..dc90b324b28 100644 --- a/site_scons/site_tools/jstoh.py +++ b/site_scons/site_tools/jstoh.py @@ -8,8 +8,8 @@ def jsToHeader(target, source): h = [ '#include "mongo/base/string_data.h"', + '#include "mongo/scripting/engine.h"', 'namespace mongo {', - 'struct JSFile{ const char* name; const StringData& source; };', 'namespace JSFiles{', ] @@ -21,7 +21,7 @@ def jsToHeader(target, source): objname = os.path.split(filename)[1].split('.')[0] stringname = '_jscode_raw_' + objname - h.append('const char ' + stringname + "[] = {") + h.append('constexpr char ' + stringname + "[] = {") with open(filename, 'r') as f: for line in f: @@ -30,8 +30,8 @@ def jsToHeader(target, source): h.append("0};") # symbols aren't exported w/o this h.append('extern const JSFile %s;' % objname) - h.append('const JSFile %s = { "%s", StringData(%s) };' % - (objname, filename.replace('\\', '/'), stringname)) + h.append('const JSFile %s = { "%s", StringData(%s, sizeof(%s) - 1) };' % + (objname, filename.replace('\\', '/'), stringname, stringname)) h.append("} // namespace JSFiles") h.append("} // namespace mongo") diff --git a/src/mongo/SConscript b/src/mongo/SConscript index 2cfb2060e7f..1208446d70f 100644 --- a/src/mongo/SConscript +++ b/src/mongo/SConscript @@ -341,6 +341,7 @@ env.Install( LIBDEPS=[ 'db/conn_pool_options', 'db/commands/core', + 'db/ftdc/ftdc_mongos', 'db/mongodandmongos', 's/client/sharding_connection_hook', 's/commands/cluster_commands', @@ -409,12 +410,17 @@ if not has_option('noshell') and usemozjs: mongo_shell = shellEnv.Program( "mongo", ["shell/dbshell.cpp"] + env.WindowsResourceFile("shell/shell.rc"), - LIBDEPS=["$BUILD_DIR/third_party/shim_pcrecpp", - "shell_core", - "db/server_options_core", - "client/clientdriver", - "$BUILD_DIR/mongo/util/password", - ]) + LIBDEPS=[ + "$BUILD_DIR/third_party/shim_pcrecpp", + "shell_core", + "db/server_options_core", + "client/clientdriver", + "$BUILD_DIR/mongo/util/password", + ], + LIBDEPS_PRIVATE=[ + "$BUILD_DIR/mongo/client/connection_string", + ] + ) shellEnv.Install( '#/', mongo_shell ) else: diff --git a/src/mongo/base/error_codes.err b/src/mongo/base/error_codes.err index c8def6bd695..98d41461fce 100644 --- a/src/mongo/base/error_codes.err +++ b/src/mongo/base/error_codes.err @@ -202,6 +202,12 @@ error_code("CannotBuildIndexKeys", 201) error_code("NetworkInterfaceExceededTimeLimit", 202) error_code("TooManyLocks", 208) error_code("UpdateOperationFailed", 218) +error_code("FTDCPathNotSet", 219) +error_code("FTDCPathAlreadySet", 220) +error_code("ProducerConsumerQueueBatchTooLarge", 247) +error_code("ProducerConsumerQueueEndClosed", 248) + +# Error codes 4000-8999 are reserved. # Non-sequential error codes (for compatibility only) error_code("SocketException", 9001) diff --git a/src/mongo/base/parse_number_test.cpp b/src/mongo/base/parse_number_test.cpp index 0ed768a7dd3..76b1018af21 100644 --- a/src/mongo/base/parse_number_test.cpp +++ b/src/mongo/base/parse_number_test.cpp @@ -290,9 +290,9 @@ TEST(Double, TestParsingNormal) { // not parseable by the Windows SDK libc or the Solaris libc in the mode we build. // See SERVER-14131. - ASSERT_PARSES(double, "0xff", 0xff); - ASSERT_PARSES(double, "-0xff", -0xff); - ASSERT_PARSES(double, "0xabcab.defdefP-10", 0xabcab.defdefP-10); + ASSERT_PARSES(double, "0xff", 255); + ASSERT_PARSES(double, "-0xff", -255); + ASSERT_PARSES(double, "0xabcab.defdefP-10", 687.16784283419838); #endif } diff --git a/src/mongo/base/validate_locale.cpp b/src/mongo/base/validate_locale.cpp index ffb7190c804..b51eac53edb 100644 --- a/src/mongo/base/validate_locale.cpp +++ b/src/mongo/base/validate_locale.cpp @@ -32,6 +32,7 @@ #include #include "mongo/base/init.h" +#include "mongo/util/mongoutils/str.h" namespace mongo { @@ -40,13 +41,15 @@ MONGO_INITIALIZER_GENERAL(ValidateLocale, MONGO_NO_PREREQUISITES, MONGO_DEFAULT_ try { // Validate that boost can correctly load the user's locale boost::filesystem::path("/").has_root_directory(); - } catch (const std::runtime_error&) { - return Status(ErrorCodes::BadValue, - "Invalid or no user locale set." + } catch (const std::runtime_error& e) { + return Status( + ErrorCodes::BadValue, + str::stream() + << "Invalid or no user locale set. " #ifndef _WIN32 - " Please ensure LANG and/or LC_* environment variables are set correctly." + << " Please ensure LANG and/or LC_* environment variables are set correctly. " #endif - ); + << e.what()); } #ifdef _WIN32 diff --git a/src/mongo/bson/bson_comparator_interface_base.h b/src/mongo/bson/bson_comparator_interface_base.h index f70bec7b872..29d80916f72 100644 --- a/src/mongo/bson/bson_comparator_interface_base.h +++ b/src/mongo/bson/bson_comparator_interface_base.h @@ -28,9 +28,11 @@ #pragma once +#include #include #include #include +#include #include "mongo/base/disallow_copying.h" #include "mongo/base/string_data_comparator_interface.h" @@ -51,6 +53,9 @@ class BSONComparatorInterfaceBase { MONGO_DISALLOW_COPYING(BSONComparatorInterfaceBase); public: + BSONComparatorInterfaceBase(BSONComparatorInterfaceBase&& other) = default; + BSONComparatorInterfaceBase& operator=(BSONComparatorInterfaceBase&& other) = default; + /** * A deferred comparison between two objects of type T, which can be converted into a boolean * via the evaluate() method. @@ -122,6 +127,8 @@ public: using Set = std::set; + using FlatSet = boost::container::flat_set; + using UnorderedSet = stdx::unordered_set; template @@ -207,6 +214,10 @@ protected: return Set(init, LessThan(this)); } + FlatSet makeFlatSet(const std::vector& elements) const { + return FlatSet(elements.begin(), elements.end(), LessThan(this)); + } + UnorderedSet makeUnorderedSet(std::initializer_list init = {}) const { return UnorderedSet(init, 0, Hasher(this), EqualTo(this)); } diff --git a/src/mongo/bson/bsonelement_comparator_interface.h b/src/mongo/bson/bsonelement_comparator_interface.h index c6223a2d352..d69397a68db 100644 --- a/src/mongo/bson/bsonelement_comparator_interface.h +++ b/src/mongo/bson/bsonelement_comparator_interface.h @@ -33,6 +33,9 @@ namespace mongo { +typedef std::set BSONElementSet; +typedef std::multiset BSONElementMultiSet; + /** * A BSONElement::ComparatorInterface is an abstract class for comparing BSONElement objects. Usage * for comparing two BSON elements, 'lhs' and 'rhs', where 'comparator' is an instance of a class @@ -60,6 +63,14 @@ public: return makeSet(init); } + /** + * Constructs a BSONEltFlatSet whose equivalence classes are given by this comparator. This + * comparator must outlive the returned set. + */ + FlatSet makeBSONEltFlatSet(const std::vector& elements) const { + return makeFlatSet(elements); + } + /** * Constructs a BSONEltUnorderedSet whose equivalence classes are given by this * comparator. This comparator must outlive the returned set. @@ -91,6 +102,8 @@ public: using BSONEltSet = BSONComparatorInterfaceBase::Set; +using BSONEltFlatSet = BSONComparatorInterfaceBase::FlatSet; + using BSONEltUnorderedSet = BSONComparatorInterfaceBase::UnorderedSet; template diff --git a/src/mongo/bson/bsonobj.cpp b/src/mongo/bson/bsonobj.cpp index d9af2c706a9..bc0045f5681 100644 --- a/src/mongo/bson/bsonobj.cpp +++ b/src/mongo/bson/bsonobj.cpp @@ -33,6 +33,7 @@ #include "mongo/base/data_range.h" #include "mongo/bson/bson_validate.h" +#include "mongo/bson/bsonelement_comparator_interface.h" #include "mongo/db/json.h" #include "mongo/util/allocator.h" #include "mongo/util/hex.h" diff --git a/src/mongo/bson/bsonobj.h b/src/mongo/bson/bsonobj.h index 22ab6ff182f..0036fea6518 100644 --- a/src/mongo/bson/bsonobj.h +++ b/src/mongo/bson/bsonobj.h @@ -42,7 +42,6 @@ #include "mongo/base/string_data_comparator_interface.h" #include "mongo/bson/bson_comparator_interface_base.h" #include "mongo/bson/bsonelement.h" -#include "mongo/bson/bsonelement_comparator_interface.h" #include "mongo/bson/bsontypes.h" #include "mongo/bson/oid.h" #include "mongo/bson/timestamp.h" @@ -53,9 +52,6 @@ namespace mongo { -typedef std::set BSONElementSet; -typedef std::multiset BSONElementMSet; - /** C++ representation of a "BSON" object -- that is, an extended JSON-style object in a binary representation. diff --git a/src/mongo/bson/util/builder.h b/src/mongo/bson/util/builder.h index b959c59236e..ba6945e3fbc 100644 --- a/src/mongo/bson/util/builder.h +++ b/src/mongo/bson/util/builder.h @@ -320,15 +320,16 @@ private: } /* "slow" portion of 'grow()' */ void NOINLINE_DECL grow_reallocate(int minSize) { + if (minSize > BufferMaxSize) { + std::stringstream ss; + ss << "BufBuilder attempted to grow() to " << minSize << " bytes, past the 64MB limit."; + msgasserted(13548, ss.str().c_str()); + } + int a = 64; while (a < minSize) a = a * 2; - if (a > BufferMaxSize) { - std::stringstream ss; - ss << "BufBuilder attempted to grow() to " << a << " bytes, past the 64MB limit."; - msgasserted(13548, ss.str().c_str()); - } _buf.realloc(a); size = a; } diff --git a/src/mongo/client/connection_string.h b/src/mongo/client/connection_string.h index e8fc8eb2637..2e888b56dad 100644 --- a/src/mongo/client/connection_string.h +++ b/src/mongo/client/connection_string.h @@ -28,11 +28,13 @@ #pragma once +#include #include #include #include "mongo/base/status_with.h" #include "mongo/base/string_data.h" +#include "mongo/bson/util/builder.h" #include "mongo/stdx/mutex.h" #include "mongo/util/assert_util.h" #include "mongo/util/net/hostandport.h" @@ -157,6 +159,10 @@ public: return _string < other._string; } + + friend std::ostream& operator<<(std::ostream&, const ConnectionString&); + friend StringBuilder& operator<<(StringBuilder&, const ConnectionString&); + private: /** * Creates a SET connection string with the specified set name and servers. @@ -179,4 +185,15 @@ private: static stdx::mutex _connectHookMutex; static ConnectionHook* _connectHook; }; + +inline std::ostream& operator<<(std::ostream& ss, const ConnectionString& cs) { + ss << cs._string; + return ss; +} + +inline StringBuilder& operator<<(StringBuilder& sb, const ConnectionString& cs) { + sb << cs._string; + return sb; +} + } // namespace mongo diff --git a/src/mongo/client/fetcher.cpp b/src/mongo/client/fetcher.cpp index 741efe9afab..4918bbf76c2 100644 --- a/src/mongo/client/fetcher.cpp +++ b/src/mongo/client/fetcher.cpp @@ -171,7 +171,8 @@ Fetcher::Fetcher(executor::TaskExecutor* executor, const BSONObj& findCmdObj, const CallbackFn& work, const BSONObj& metadata, - Milliseconds timeout, + Milliseconds findNetworkTimeout, + Milliseconds getMoreNetworkTimeout, std::unique_ptr firstCommandRetryPolicy) : _executor(executor), _source(source), @@ -179,10 +180,11 @@ Fetcher::Fetcher(executor::TaskExecutor* executor, _cmdObj(findCmdObj.getOwned()), _metadata(metadata.getOwned()), _work(work), - _timeout(timeout), + _findNetworkTimeout(findNetworkTimeout), + _getMoreNetworkTimeout(getMoreNetworkTimeout), _firstRemoteCommandScheduler( _executor, - RemoteCommandRequest(_source, _dbname, _cmdObj, _metadata, nullptr, _timeout), + RemoteCommandRequest(_source, _dbname, _cmdObj, _metadata, nullptr, _findNetworkTimeout), stdx::bind(&Fetcher::_callback, this, stdx::placeholders::_1, kFirstBatchFieldName), std::move(firstCommandRetryPolicy)) { uassert(ErrorCodes::BadValue, "callback function cannot be null", work); @@ -204,10 +206,6 @@ BSONObj Fetcher::getMetadataObject() const { return _metadata; } -Milliseconds Fetcher::getTimeout() const { - return _timeout; -} - std::string Fetcher::toString() const { return getDiagnosticString(); } @@ -221,7 +219,8 @@ std::string Fetcher::getDiagnosticString() const { output << " query: " << _cmdObj; output << " query metadata: " << _metadata; output << " active: " << _isActive_inlock(); - output << " timeout: " << _timeout; + output << " findNetworkTimeout: " << _findNetworkTimeout; + output << " getMoreNetworkTimeout: " << _getMoreNetworkTimeout; output << " shutting down?: " << _isShuttingDown_inlock(); output << " first: " << _first; output << " firstCommandScheduler: " << _firstRemoteCommandScheduler.toString(); @@ -316,7 +315,8 @@ Status Fetcher::_scheduleGetMore(const BSONObj& cmdObj) { } StatusWith scheduleResult = _executor->scheduleRemoteCommand( - RemoteCommandRequest(_source, _dbname, cmdObj, _metadata, nullptr, _timeout), + RemoteCommandRequest( + _source, _dbname, cmdObj, _metadata, nullptr, _getMoreNetworkTimeout), stdx::bind(&Fetcher::_callback, this, stdx::placeholders::_1, kNextBatchFieldName)); if (!scheduleResult.isOK()) { diff --git a/src/mongo/client/fetcher.h b/src/mongo/client/fetcher.h index 050ac50f249..3af4c6f329d 100644 --- a/src/mongo/client/fetcher.h +++ b/src/mongo/client/fetcher.h @@ -130,7 +130,8 @@ public: const BSONObj& cmdObj, const CallbackFn& work, const BSONObj& metadata = rpc::ServerSelectionMetadata(true, boost::none).toBSON(), - Milliseconds timeout = RemoteCommandRequest::kNoTimeout, + Milliseconds findNetworkTimeout = RemoteCommandRequest::kNoTimeout, + Milliseconds getMoreNetworkTimeout = RemoteCommandRequest::kNoTimeout, std::unique_ptr firstCommandRetryPolicy = RemoteCommandRetryScheduler::makeNoRetryPolicy()); @@ -151,11 +152,6 @@ public: */ BSONObj getMetadataObject() const; - /** - * Returns timeout for remote commands to complete. - */ - Milliseconds getTimeout() const; - /** * Returns diagnostic information. */ @@ -260,7 +256,8 @@ private: executor::TaskExecutor::CallbackHandle _getMoreCallbackHandle; // Socket timeout - Milliseconds _timeout; + Milliseconds _findNetworkTimeout; + Milliseconds _getMoreNetworkTimeout; // First remote command scheduler. RemoteCommandRetryScheduler _firstRemoteCommandScheduler; diff --git a/src/mongo/client/fetcher_test.cpp b/src/mongo/client/fetcher_test.cpp index 00dae071d3a..66fb026c6b6 100644 --- a/src/mongo/client/fetcher_test.cpp +++ b/src/mongo/client/fetcher_test.cpp @@ -242,6 +242,7 @@ TEST_F(FetcherTest, InvalidConstruction) { unreachableCallback, rpc::makeEmptyMetadata(), RemoteCommandRequest::kNoTimeout, + RemoteCommandRequest::kNoTimeout, std::unique_ptr()), UserException, ErrorCodes::BadValue, @@ -273,7 +274,6 @@ TEST_F(FetcherTest, RemoteCommandRequestShouldContainCommandParametersPassedToCo ASSERT_EQUALS(source, fetcher->getSource()); ASSERT_BSONOBJ_EQ(findCmdObj, fetcher->getCommandObject()); ASSERT_BSONOBJ_EQ(metadataObj, fetcher->getMetadataObject()); - ASSERT_EQUALS(timeout, fetcher->getTimeout()); ASSERT_OK(fetcher->schedule()); @@ -284,6 +284,7 @@ TEST_F(FetcherTest, RemoteCommandRequestShouldContainCommandParametersPassedToCo ASSERT_TRUE(net->hasReadyRequests()); auto noi = net->getNextReadyRequest(); request = noi->getRequest(); + ASSERT_EQUALS(timeout, request.timeout); } ASSERT_EQUALS(source, request.target); @@ -1048,6 +1049,7 @@ TEST_F(FetcherTest, FetcherAppliesRetryPolicyToFirstCommandButNotToGetMoreReques makeCallback(), rpc::makeEmptyMetadata(), executor::RemoteCommandRequest::kNoTimeout, + executor::RemoteCommandRequest::kNoTimeout, std::move(policy)); callbackHook = appendGetMoreRequest; diff --git a/src/mongo/client/mongo_uri.cpp b/src/mongo/client/mongo_uri.cpp index e737479d05b..e2356881290 100644 --- a/src/mongo/client/mongo_uri.cpp +++ b/src/mongo/client/mongo_uri.cpp @@ -32,131 +32,323 @@ #include "mongo/client/mongo_uri.h" +#include + #include "mongo/base/status_with.h" #include "mongo/bson/bsonobjbuilder.h" #include "mongo/client/dbclientinterface.h" #include "mongo/client/sasl_client_authenticate.h" +#include "mongo/db/namespace_string.h" +#include "mongo/util/hex.h" #include "mongo/util/mongoutils/str.h" -#include "mongo/util/password_digest.h" #include #include +#include #include -#include -#include + +namespace { +constexpr std::array hexits{ + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}; +const mongo::StringData kURIPrefix{"mongodb://"}; +} + +/** + * RFC 3986 Section 2.1 - Percent Encoding + * + * Encode data elements in a way which will allow them to be embedded + * into a mongodb:// URI safely. + */ +void mongo::uriEncode(std::ostream& ss, StringData toEncode, StringData passthrough) { + for (const auto& c : toEncode) { + if ((c == '-') || (c == '_') || (c == '.') || (c == '~') || isalnum(c) || + (passthrough.find(c) != std::string::npos)) { + ss << c; + } else { + // Encoding anything not included in section 2.3 "Unreserved characters" + ss << '%' << hexits[(c >> 4) & 0xF] << hexits[c & 0xF]; + } + } +} + +mongo::StatusWith mongo::uriDecode(StringData toDecode) { + StringBuilder out; + for (size_t i = 0; i < toDecode.size(); ++i) { + const auto c = toDecode[i]; + if (c == '%') { + if (i + 2 > toDecode.size()) { + return Status(ErrorCodes::FailedToParse, + "Encountered partial escape sequence at end of string"); + } + out << fromHex(toDecode.substr(i + 1, 2)); + i += 2; + } else { + out << c; + } + } + return out.str(); +} namespace mongo { namespace { -const char kMongoDBURL[] = - // scheme: non-capturing - "mongodb://" - // credentials: two inner captures for user and password - "(?:([^:]+)(?::([^@]+))?@)?" +/** + * Helper Method for MongoURI::parse() to split a string into exactly 2 pieces by a char delimeter + */ +std::pair partitionForward(StringData str, const char c) { + const auto delim = str.find(c); + if (delim == std::string::npos) { + return {str, StringData()}; + } + return {str.substr(0, delim), str.substr(delim + 1)}; +} - // servers: grabs all host:port or UNIX socket names - "((?:[^\\/]+|/.+\\.sock)(?:,(?:[^\\/]+|/.+\\.sock))*)" +/** + * Helper method for MongoURI::parse() to split a string into exactly 2 pieces by a char delimiter + * searching backward from the end of the string. + */ +std::pair partitionBackward(StringData str, const char c) { + const auto delim = str.rfind(c); + if (delim == std::string::npos) { + return {StringData(), str}; + } + return {str.substr(0, delim), str.substr(delim + 1)}; +} - // database and options are grouped together - "(?:/" +/** + * Breakout method for parsing application/x-www-form-urlencoded option pairs + * + * foo=bar&baz=qux&... + */ +StatusWith parseOptions(StringData options, StringData url) { + MongoURI::OptionsMap ret; + if (options.empty()) { + return ret; + } - // database: matches anything but the chars that cannot be part of a MongoDB database name which - // are (in order) - forward slash, back slash, dot, space, double-quote, dollar sign, asterisk, - // less than, greater than, colon, pipe, question mark. - "([^/\\\\\\.\\ \"\\$*<>:\\|\\?]*)?" + if (options.find('?') != std::string::npos) { + return Status(ErrorCodes::FailedToParse, + str::stream() + << "URI Cannot Contain multiple questions marks for mongodb:// URL: " + << url); + } - // options - "(?:\\?([^&=?]+=[^&=?]+(?:&[^&=?]+=[^&=?]+)*))?" + const auto optionsStr = options.toString(); + for (auto i = + boost::make_split_iterator(optionsStr, boost::first_finder("&", boost::is_iequal())); + i != std::remove_reference::type{}; + ++i) { + const auto opt = boost::copy_range(*i); + if (opt.empty()) { + return Status(ErrorCodes::FailedToParse, + str::stream() + << "Missing a key/value pair in the options for mongodb:// URL: " + << url); + } - // close db/options group - ")?"; + const auto kvPair = partitionForward(opt, '='); + const auto keyRaw = kvPair.first; + if (keyRaw.empty()) { + return Status( + ErrorCodes::FailedToParse, + str::stream() + << "Missing a key for key/value pair in the options for mongodb:// URL: " + << url); + } + const auto key = uriDecode(keyRaw); + if (!key.isOK()) { + return Status( + ErrorCodes::FailedToParse, + str::stream() << "Key '" << keyRaw + << "' in options cannot properly be URL decoded for mongodb:// URL: " + << url); + } + const auto valRaw = kvPair.second; + if (valRaw.empty()) { + return Status(ErrorCodes::FailedToParse, + str::stream() << "Missing value for key '" << keyRaw + << "' in the options for mongodb:// URL: " + << url); + } + const auto val = uriDecode(valRaw); + if (!val.isOK()) { + return Status( + ErrorCodes::FailedToParse, + str::stream() << "Value '" << valRaw << "' for key '" << keyRaw + << "' in options cannot properly be URL decoded for mongodb:// URL: " + << url); + } -} // namespace + ret[key.getValue()] = val.getValue(); + } + + return ret; +} +} // namespace StatusWith MongoURI::parse(const std::string& url) { - if (!boost::algorithm::starts_with(url, "mongodb://")) { - auto cs_status = ConnectionString::parse(url); + const StringData urlSD(url); + + // 1. Validate and remove the scheme prefix mongodb:// + if (!urlSD.startsWith(kURIPrefix)) { + const auto cs_status = ConnectionString::parse(url); if (!cs_status.isOK()) { return cs_status.getStatus(); } - return MongoURI(cs_status.getValue()); } + const auto uriWithoutPrefix = urlSD.substr(kURIPrefix.size()); + + // 2. Split the string by the first, unescaped / (if any), yielding: + // split[0]: User information and host identifers + // split[1]: Auth database and connection options + const auto userAndDb = partitionForward(uriWithoutPrefix, '/'); + const auto userAndHostInfo = userAndDb.first; + const auto databaseAndOptions = userAndDb.second; + + // 2.b Make sure that there are no question marks in the left side of the / + // as any options after the ? must still have the / delimeter + if (databaseAndOptions.empty() && userAndHostInfo.find('?') != std::string::npos) { + return Status( + ErrorCodes::FailedToParse, + str::stream() + << "URI must contain slash delimeter between hosts and options for mongodb:// URL: " + << url); + } - const boost::regex mongoUrlRe(kMongoDBURL); - - boost::smatch matches; - if (!boost::regex_match(url, matches, mongoUrlRe)) { + // 3. Split the user information and host identifiers string by the last, unescaped @, yielding: + // split[0]: User information + // split[1]: Host identifiers; + const auto userAndHost = partitionBackward(userAndHostInfo, '@'); + const auto userInfo = userAndHost.first; + const auto hostIdentifiers = userAndHost.second; + + // 4. Validate, split (if applicable), and URL decode the user information, yielding: + // split[0] = username + // split[1] = password + const auto userAndPass = partitionForward(userInfo, ':'); + const auto usernameSD = userAndPass.first; + const auto passwordSD = userAndPass.second; + + const auto containsColonOrAt = [](StringData str) { + return (str.find(':') != std::string::npos) || (str.find('@') != std::string::npos); + }; + + if (containsColonOrAt(usernameSD)) { return Status(ErrorCodes::FailedToParse, - str::stream() << "Failed to parse mongodb:// URL: " << url); + str::stream() << "Username must be URL Encoded for mongodb:// URL: " << url); } - - // We have the whole input plus 5 top level captures (user, password, host, db, options). - invariant(matches.size() == 6); - - if (!matches[3].matched) { - return Status(ErrorCodes::FailedToParse, "No server(s) specified"); + if (containsColonOrAt(passwordSD)) { + return Status(ErrorCodes::FailedToParse, + str::stream() << "Password must be URL Encoded for mongodb:// URL: " << url); } - std::map options; + // Get the username and make sure it did not fail to decode + const auto usernameWithStatus = uriDecode(usernameSD); + if (!usernameWithStatus.isOK()) { + return Status( + ErrorCodes::FailedToParse, + str::stream() << "Username cannot properly be URL decoded for mongodb:// URL: " << url); + } + const auto username = usernameWithStatus.getValue(); + + // Get the password and make sure it did not fail to decode + const auto passwordWithStatus = uriDecode(passwordSD); + if (!passwordWithStatus.isOK()) + return Status( + ErrorCodes::FailedToParse, + str::stream() << "Password cannot properly be URL decoded for mongodb:// URL: " << url); + const auto password = passwordWithStatus.getValue(); + + // 5. Validate, split, and URL decode the host identifiers. + const auto hostIdentifiersStr = hostIdentifiers.toString(); + std::vector servers; + for (auto i = boost::make_split_iterator(hostIdentifiersStr, + boost::first_finder(",", boost::is_iequal())); + i != std::remove_reference::type{}; + ++i) { + const auto hostWithStatus = uriDecode(boost::copy_range(*i)); + if (!hostWithStatus.isOK()) { + return Status( + ErrorCodes::FailedToParse, + str::stream() << "Host cannot properly be URL decoded for mongodb:// URL: " << url); + } - if (matches[5].matched) { - const std::string optionsMatch = matches[5].str(); + const auto host = hostWithStatus.getValue(); + if (host.empty()) { + continue; + } - std::vector> optionsTokens; - boost::algorithm::split(optionsTokens, optionsMatch, boost::algorithm::is_any_of("=&")); + if ((host.find('/') != std::string::npos) && !StringData(host).endsWith(".sock")) { + return Status( + ErrorCodes::FailedToParse, + str::stream() << "'" << host << "' in '" << url + << "' appears to be a unix socket, but does not end in '.sock'"); + } - if (optionsTokens.size() % 2 != 0) { - return Status(ErrorCodes::FailedToParse, - str::stream() - << "Missing a key or value in the options for mongodb:// URL: " - << url); - ; + const auto statusHostAndPort = HostAndPort::parse(host); + if (!statusHostAndPort.isOK()) { + return statusHostAndPort.getStatus(); } + servers.push_back(statusHostAndPort.getValue()); + } + if (servers.empty()) { + return Status(ErrorCodes::FailedToParse, "No server(s) specified"); + } - for (size_t i = 0; i != optionsTokens.size(); i = i + 2) - options[std::string(optionsTokens[i].begin(), optionsTokens[i].end())] = - std::string(optionsTokens[i + 1].begin(), optionsTokens[i + 1].end()); + // 6. Split the auth database and connection options string by the first, unescaped ?, yielding: + // split[0] = auth database + // split[1] = connection options + const auto dbAndOpts = partitionForward(databaseAndOptions, '?'); + const auto databaseSD = dbAndOpts.first; + const auto connectionOptions = dbAndOpts.second; + const auto databaseWithStatus = uriDecode(databaseSD); + if (!databaseWithStatus.isOK()) { + return Status(ErrorCodes::FailedToParse, + str::stream() + << "Database name cannot properly be URL decoded for mongodb:// URL: " + << url); + } + const auto database = databaseWithStatus.getValue(); + + // 7. Validate the database contains no prohibited characters + // Prohibited characters: + // slash ("/"), backslash ("\"), space (" "), double-quote ("""), or dollar sign ("$") + // period (".") is also prohibited, but drivers MAY allow periods + if (!database.empty() && + !NamespaceString::validDBName(database, + NamespaceString::DollarInDbNameBehavior::Disallow)) { + return Status(ErrorCodes::FailedToParse, + str::stream() + << "Database name cannot have reserved characters for mongodb:// URL: " + << url); } - OptionsMap::const_iterator optIter; + // 8. Validate, split, and URL decode the connection options + const auto optsWith = parseOptions(connectionOptions, url); + if (!optsWith.isOK()) { + return optsWith.getStatus(); + } + const auto options = optsWith.getValue(); // If a replica set option was specified, store it in the 'setName' field. - bool haveSetName; + const auto optIter = options.find("replicaSet"); std::string setName; - if ((haveSetName = ((optIter = options.find("replicaSet")) != options.end()))) { + if (optIter != options.end()) { setName = optIter->second; + invariant(!setName.empty()); } - std::vector servers; - - { - std::vector servers_split; - const std::string serversStr = matches[3].str(); - boost::algorithm::split(servers_split, serversStr, boost::is_any_of(",")); - for (auto&& s : servers_split) { - auto statusHostAndPort = HostAndPort::parse(s); - if (!statusHostAndPort.isOK()) { - return statusHostAndPort.getStatus(); - } - - servers.push_back(statusHostAndPort.getValue()); - } - } - - const bool direct = !haveSetName && (servers.size() == 1); - - if (!direct && setName.empty()) { + if ((servers.size() > 1) && setName.empty()) { return Status(ErrorCodes::FailedToParse, "Cannot list multiple servers in URL without 'replicaSet' option"); } ConnectionString cs( - direct ? ConnectionString::MASTER : ConnectionString::SET, servers, setName); - return MongoURI( - std::move(cs), matches[1].str(), matches[2].str(), matches[4].str(), std::move(options)); + setName.empty() ? ConnectionString::MASTER : ConnectionString::SET, servers, setName); + return MongoURI(std::move(cs), username, password, database, std::move(options)); } } // namespace mongo diff --git a/src/mongo/client/mongo_uri.h b/src/mongo/client/mongo_uri.h index 7b6364761d5..f7a7c4aeb81 100644 --- a/src/mongo/client/mongo_uri.h +++ b/src/mongo/client/mongo_uri.h @@ -29,12 +29,14 @@ #pragma once #include +#include #include #include #include "mongo/base/status_with.h" #include "mongo/base/string_data.h" #include "mongo/bson/bsonobj.h" +#include "mongo/bson/util/builder.h" #include "mongo/client/connection_string.h" #include "mongo/stdx/mutex.h" #include "mongo/util/assert_util.h" @@ -42,6 +44,25 @@ namespace mongo { +/** + * Encode a string for embedding in a URI. + * Replaces reserved bytes with %xx sequences. + * + * Optionally allows passthrough characters to remain unescaped. + */ +void uriEncode(std::ostream& ss, StringData str, StringData passthrough = ""_sd); +inline std::string uriEncode(StringData str, StringData passthrough = ""_sd) { + std::ostringstream ss; + uriEncode(ss, str, passthrough); + return ss.str(); +} + +/** + * Decode a URI encoded string. + * Replaces + and %xx sequences with their original byte. + */ +StatusWith uriDecode(StringData str); + /** * MongoURI handles parsing of URIs for mongodb, and falls back to old-style * ConnectionString parsing. It's used primarily by the shell. @@ -49,6 +70,13 @@ namespace mongo { * * mongodb://[usr:pwd@]host1[:port1]...[,hostN[:portN]]][/[db][?options]] * + * While this format is generally RFC 3986 compliant, some exceptions do exist: + * 1. The 'host' field, as defined by section 3.2.2 is expanded in the following ways: + * a. Multiple hosts may be specified as a comma separated list. + * b. Hosts may take the form of absolute paths for unix domain sockets. + * i. Sockets must end in the suffix '.sock' + * 2. The 'fragment' field, as defined by section 3.5 is not permitted. + * * For a complete list of URI string options, see * https://wiki.mongodb.com/display/DH/Connection+String+Format * @@ -125,6 +153,9 @@ public: MongoURI() = default; + friend std::ostream& operator<<(std::ostream&, const MongoURI&); + friend StringBuilder& operator<<(StringBuilder&, const MongoURI&); + private: MongoURI(ConnectionString connectString, const std::string& user, @@ -146,4 +177,14 @@ private: OptionsMap _options; }; +inline std::ostream& operator<<(std::ostream& ss, const MongoURI& uri) { + ss << uri._connectString; + return ss; +} + +inline StringBuilder& operator<<(StringBuilder& sb, const MongoURI& uri) { + sb << uri._connectString; + return sb; +} + } // namespace mongo diff --git a/src/mongo/client/mongo_uri_test.cpp b/src/mongo/client/mongo_uri_test.cpp index 6a5f6c9882a..c32b1c0b5c5 100644 --- a/src/mongo/client/mongo_uri_test.cpp +++ b/src/mongo/client/mongo_uri_test.cpp @@ -28,11 +28,17 @@ #include "mongo/platform/basic.h" -#include "mongo/client/mongo_uri.h" +#include #include "mongo/base/string_data.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsontypes.h" +#include "mongo/bson/json.h" +#include "mongo/client/mongo_uri.h" #include "mongo/unittest/unittest.h" +#include + namespace { using mongo::MongoURI; @@ -60,16 +66,77 @@ const URITestCase validCases[] = { {"mongodb://user@127.0.0.1", "user", "", kMaster, "", 1, 0, ""}, - {"mongodb://127.0.0.1/dbName?foo=a&c=b", "", "", kMaster, "", 1, 2, "dbName"}, - {"mongodb://localhost/?foo=bar", "", "", kMaster, "", 1, 1, ""}, + {"mongodb://localhost,/?foo=bar", "", "", kMaster, "", 1, 1, ""}, + {"mongodb://user:pwd@127.0.0.1:1234", "user", "pwd", kMaster, "", 1, 0, ""}, {"mongodb://user@127.0.0.1:1234", "user", "", kMaster, "", 1, 0, ""}, {"mongodb://127.0.0.1:1234/dbName?foo=a&c=b", "", "", kMaster, "", 1, 2, "dbName"}, + {"mongodb://127.0.0.1/dbName?foo=a&c=b", "", "", kMaster, "", 1, 2, "dbName"}, + + {"mongodb://user:pwd@127.0.0.1,/dbName?foo=a&c=b", "user", "pwd", kMaster, "", 1, 2, "dbName"}, + + {"mongodb://user:pwd@127.0.0.1,127.0.0.2/dbname?a=b&replicaSet=replName", + "user", + "pwd", + kSet, + "replName", + 2, + 2, + "dbname"}, + + {"mongodb://needs%20encoding%25%23!%3C%3E:pwd@127.0.0.1,127.0.0.2/" + "dbname?a=b&replicaSet=replName", + "needs encoding%#!<>", + "pwd", + kSet, + "replName", + 2, + 2, + "dbname"}, + + {"mongodb://needs%20encoding%25%23!%3C%3E:pwd@127.0.0.1,127.0.0.2/" + "db@name?a=b&replicaSet=replName", + "needs encoding%#!<>", + "pwd", + kSet, + "replName", + 2, + 2, + "db@name"}, + + {"mongodb://user:needs%20encoding%25%23!%3C%3E@127.0.0.1,127.0.0.2/" + "dbname?a=b&replicaSet=replName", + "user", + "needs encoding%#!<>", + kSet, + "replName", + 2, + 2, + "dbname"}, + + {"mongodb://user:pwd@127.0.0.1,127.0.0.2/dbname?a=b&replicaSet=needs%20encoding%25%23!%3C%3E", + "user", + "pwd", + kSet, + "needs encoding%#!<>", + 2, + 2, + "dbname"}, + + {"mongodb://user:pwd@127.0.0.1,127.0.0.2/needsencoding%40hello?a=b&replicaSet=replName", + "user", + "pwd", + kSet, + "replName", + 2, + 2, + "needsencoding@hello"}, + {"mongodb://user:pwd@127.0.0.1,127.0.0.2/?replicaSet=replName", "user", "pwd", @@ -259,21 +326,27 @@ const URITestCase validCases[] = { 1, 2, ""}, - {"mongodb:///tmp/mongodb-27017.sock", "", "", kMaster, "", 1, 0, ""}, - {"mongodb:///tmp/mongodb-27017.sock,/tmp/mongodb-27018.sock/?replicaSet=replName", + {"mongodb://%2Ftmp%2Fmongodb-27017.sock", "", "", kMaster, "", 1, 0, ""}, + + {"mongodb://%2Ftmp%2Fmongodb-27017.sock,%2Ftmp%2Fmongodb-27018.sock/?replicaSet=replName", "", "", kSet, "replName", 2, 1, - ""}}; + ""}, +}; const InvalidURITestCase invalidCases[] = { // No host. {"mongodb://"}, + {"mongodb://usr:pwd@/dbname?a=b"}, + + // Username and password must be encoded (cannot have ':' or '@') + {"mongodb://usr:pwd:@127.0.0.1/dbName?foo=a&c=b"}, // Needs a "/" after the hosts and before the options. {"mongodb://localhost:27017,localhost:27018?replicaSet=missingSlash"}, @@ -282,32 +355,109 @@ const InvalidURITestCase invalidCases[] = { {"mongodb://localhost:27017localhost:27018"}, // Domain sockets have to end in ".sock". - {"mongodb:///notareal/domainsock"}, + {"mongodb://%2Fnotareal%2Fdomainsock"}, + + // Database name cannot contain slash ("/"), backslash ("\"), space (" "), double-quote ("""), + // or dollar sign ("$") + {"mongodb://usr:pwd@localhost:27017/db$name?a=b"}, + {"mongodb://usr:pwd@localhost:27017/db/name?a=b"}, + {"mongodb://usr:pwd@localhost:27017/db\\name?a=b"}, + {"mongodb://usr:pwd@localhost:27017/db name?a=b"}, + {"mongodb://usr:pwd@localhost:27017/db\"name?a=b"}, + + // Options must have a key + {"mongodb://usr:pwd@localhost:27017/dbname?=b"}, + + // Cannot skip a key value pair + {"mongodb://usr:pwd@localhost:27017/dbname?a=b&&b=c"}, + + // Multiple Unix domain sockets and auth DB resembling a socket (relative path) + {"mongodb://rel%2Fmongodb-27017.sock,rel%2Fmongodb-27018.sock/admin.sock?replicaSet=replName"}, + + // Multiple Unix domain sockets with auth DB resembling a path (relative path) + {"mongodb://rel%2Fmongodb-27017.sock,rel%2Fmongodb-27018.sock/admin.shoe?replicaSet=replName"}, + + // Multiple Unix domain sockets and auth DB resembling a socket (absolute path) + {"mongodb://%2Ftmp%2Fmongodb-27017.sock,%2Ftmp%2Fmongodb-27018.sock/" + "admin.sock?replicaSet=replName"}, + + // Multiple Unix domain sockets with auth DB resembling a path (absolute path) + {"mongodb://%2Ftmp%2Fmongodb-27017.sock,%2Ftmp%2Fmongodb-27018.sock/" + "admin.shoe?replicaSet=replName"}, + + // Missing value in key value pair for options + {"mongodb://127.0.0.1:1234/dbName?foo=a&c=b&d"}, + {"mongodb://127.0.0.1:1234/dbName?foo=a&c=b&d="}, + {"mongodb://127.0.0.1:1234/dbName?foo=a&h=&c=b&d=6"}, + {"mongodb://127.0.0.1:1234/dbName?foo=a&h&c=b&d=6"}, + + // Missing a hostname, or unparsable hostname(s) + {"mongodb://,/dbName"}, + {"mongodb://user:pwd@,/dbName"}, + {"mongodb://localhost:1234:5678/dbName"}, // Options can't have multiple question marks. Only one. {"mongodb://localhost:27017/?foo=a?c=b&d=e?asdf=foo"}, + + // Missing a key in key value pair for options + {"mongodb://127.0.0.1:1234/dbName?foo=a&=d&c=b"}, + + // Missing an entire key-value pair + {"mongodb://127.0.0.1:1234/dbName?foo=a&&c=b"}, }; +// Helper Method to take a filename for a json file and return the array of tests inside of it +mongo::BSONObj getBsonFromJsonFile(std::string fileName) { + boost::filesystem::path directoryPath = boost::filesystem::current_path(); + boost::filesystem::path filePath(directoryPath / "src" / "mongo" / "client" / + "mongo_uri_tests" / fileName); + std::string filename(filePath.string()); + std::ifstream infile(filename.c_str()); + std::string data((std::istreambuf_iterator(infile)), std::istreambuf_iterator()); + if (data.empty()) { + // v3.4 test infra is unable to see these extra files. + return mongo::BSONObj(); + } + mongo::BSONObj obj = mongo::fromjson(data); + ASSERT_TRUE(obj.valid(mongo::BSONVersion::kLatest)); + ASSERT_TRUE(obj.hasField("tests")); + mongo::BSONObj arr = obj.getField("tests").embeddedObject().getOwned(); + ASSERT_TRUE(arr.couldBeArray()); + return arr; +} + +// Helper method to take a BSONElement and either extract its string or return an empty string +std::string returnStringFromElementOrNull(mongo::BSONElement element) { + ASSERT_TRUE(!element.eoo()); + if (element.type() == mongo::jstNULL) { + return std::string(); + } + ASSERT_EQ(element.type(), mongo::String); + return element.String(); +} + +// Helper method to take a valid test case, parse() it, and assure the output is correct +void testValidURIFormat(URITestCase testCase) { + mongo::unittest::log() << "Testing URI: " << testCase.URI << '\n'; + std::string errMsg; + const auto cs_status = MongoURI::parse(testCase.URI); + ASSERT_OK(cs_status); + auto result = cs_status.getValue(); + ASSERT_EQ(testCase.uname, result.getUser()); + ASSERT_EQ(testCase.password, result.getPassword()); + ASSERT_EQ(testCase.type, result.type()); + ASSERT_EQ(testCase.setname, result.getSetName()); + ASSERT_EQ(testCase.numservers, result.getServers().size()); + ASSERT_EQ(testCase.numOptions, result.getOptions().size()); + ASSERT_EQ(testCase.database, result.getDatabase()); +} + TEST(MongoURI, GoodTrickyURIs) { const size_t numCases = sizeof(validCases) / sizeof(validCases[0]); for (size_t i = 0; i != numCases; ++i) { const URITestCase testCase = validCases[i]; - mongo::unittest::log() << "Testing URI: " << testCase.URI << '\n'; - std::string errMsg; - auto cs_status = MongoURI::parse(testCase.URI); - if (!cs_status.getStatus().toString().empty()) { - mongo::unittest::log() << "error with uri: " << cs_status.getStatus().toString(); - } - ASSERT_TRUE(cs_status.isOK()); - auto result = cs_status.getValue(); - ASSERT_EQ(testCase.uname, result.getUser()); - ASSERT_EQ(testCase.password, result.getPassword()); - ASSERT_EQ(testCase.type, result.type()); - ASSERT_EQ(testCase.setname, result.getSetName()); - ASSERT_EQ(testCase.numservers, result.getServers().size()); - ASSERT_EQ(testCase.numOptions, result.getOptions().size()); - ASSERT_EQ(testCase.database, result.getDatabase()); + testValidURIFormat(testCase); } } @@ -318,7 +468,7 @@ TEST(MongoURI, InvalidURIs) { const InvalidURITestCase testCase = invalidCases[i]; mongo::unittest::log() << "Testing URI: " << testCase.URI << '\n'; auto cs_status = MongoURI::parse(testCase.URI); - ASSERT_FALSE(cs_status.isOK()); + ASSERT_NOT_OK(cs_status); } } @@ -357,4 +507,100 @@ TEST(MongoURI, CloneURIForServer) { ASSERT_EQ(clonedURIOptions.at("ssl"), "true"); } +/** + * These tests come from the Mongo Uri Specifications for the drivers found at: + * https://github.com/mongodb/specifications/tree/master/source/connection-string/tests + * They have been slighly altered as the Drivers specification is slighly different + * from the server specification. + */ +TEST(MongoURI, specTests) { + const std::string files[] = { + "mongo-uri-valid-auth.json", + "mongo-uri-options.json", + "mongo-uri-unix-sockets-absolute.json", + "mongo-uri-unix-sockets-relative.json", + "mongo-uri-warnings.json", + "mongo-uri-host-identifiers.json", + "mongo-uri-invalid.json", + }; + + for (const auto& file : files) { + const auto testBson = getBsonFromJsonFile(file); + + for (const auto& testElement : testBson) { + ASSERT_EQ(testElement.type(), mongo::Object); + const auto test = testElement.Obj(); + + // First extract the valid field and the uri field + const auto validDoc = test.getField("valid"); + ASSERT_FALSE(validDoc.eoo()); + ASSERT_TRUE(validDoc.isBoolean()); + const auto valid = validDoc.Bool(); + + const auto uriDoc = test.getField("uri"); + ASSERT_FALSE(uriDoc.eoo()); + ASSERT_EQ(uriDoc.type(), mongo::String); + const auto uri = uriDoc.String(); + + if (!valid) { + // This uri string is invalid --> parse the uri and ensure it fails + const InvalidURITestCase testCase = {uri}; + mongo::unittest::log() << "Testing URI: " << testCase.URI << '\n'; + auto cs_status = MongoURI::parse(testCase.URI); + ASSERT_NOT_OK(cs_status); + } else { + // This uri is valid -- > parse the remaining necessary fields + + // parse the auth options + std::string database, username, password; + + const auto auth = test.getField("auth"); + ASSERT_FALSE(auth.eoo()); + if (auth.type() != mongo::jstNULL) { + ASSERT_EQ(auth.type(), mongo::Object); + const auto authObj = auth.embeddedObject(); + database = returnStringFromElementOrNull(authObj.getField("db")); + username = returnStringFromElementOrNull(authObj.getField("username")); + password = returnStringFromElementOrNull(authObj.getField("password")); + } + + // parse the hosts + const auto hosts = test.getField("hosts"); + ASSERT_FALSE(hosts.eoo()); + ASSERT_EQ(hosts.type(), mongo::Array); + const auto numHosts = static_cast(hosts.Obj().nFields()); + + // parse the options + mongo::ConnectionString::ConnectionType connectionType = kMaster; + size_t numOptions = 0; + std::string setName; + const auto optionsElement = test.getField("options"); + ASSERT_FALSE(optionsElement.eoo()); + if (optionsElement.type() != mongo::jstNULL) { + ASSERT_EQ(optionsElement.type(), mongo::Object); + const auto optionsObj = optionsElement.Obj(); + numOptions = optionsObj.nFields(); + const auto replsetElement = optionsObj.getField("replicaSet"); + if (!replsetElement.eoo()) { + ASSERT_EQ(replsetElement.type(), mongo::String); + setName = replsetElement.String(); + connectionType = kSet; + } + } + + // Create the URITestCase abnd + const URITestCase testCase = {uri, + username, + password, + connectionType, + setName, + numHosts, + numOptions, + database}; + testValidURIFormat(testCase); + } + } + } +} + } // namespace diff --git a/src/mongo/client/mongo_uri_tests/mongo-uri-host-identifiers.json b/src/mongo/client/mongo_uri_tests/mongo-uri-host-identifiers.json new file mode 100644 index 00000000000..8a1fda580a8 --- /dev/null +++ b/src/mongo/client/mongo_uri_tests/mongo-uri-host-identifiers.json @@ -0,0 +1,203 @@ +{ + "tests": [ + { + "description": "Single IPv4 host without port", + "uri": "mongodb://127.0.0.1", + "valid": true, + "warning": false, + "hosts": [ + { + "type": "ipv4", + "host": "127.0.0.1", + "port": null + } + ], + "auth": null, + "options": null + }, + { + "description": "Single IPv4 host with port", + "uri": "mongodb://127.0.0.1:27018", + "valid": true, + "warning": false, + "hosts": [ + { + "type": "ipv4", + "host": "127.0.0.1", + "port": 27018 + } + ], + "auth": null, + "options": null + }, + { + "description": "Single IP literal host without port", + "uri": "mongodb://[::1]", + "valid": true, + "warning": false, + "hosts": [ + { + "type": "ip_literal", + "host": "::1", + "port": null + } + ], + "auth": null, + "options": null + }, + { + "description": "Single IP literal host with port", + "uri": "mongodb://[::1]:27019", + "valid": true, + "warning": false, + "hosts": [ + { + "type": "ip_literal", + "host": "::1", + "port": 27019 + } + ], + "auth": null, + "options": null + }, + { + "description": "Single hostname without port", + "uri": "mongodb://example.com", + "valid": true, + "warning": false, + "hosts": [ + { + "type": "hostname", + "host": "example.com", + "port": null + } + ], + "auth": null, + "options": null + }, + { + "description": "Single hostname with port", + "uri": "mongodb://example.com:27020", + "valid": true, + "warning": false, + "hosts": [ + { + "type": "hostname", + "host": "example.com", + "port": 27020 + } + ], + "auth": null, + "options": null + }, + { + "description": "Single hostname (resembling IPv4) without port", + "uri": "mongodb://256.0.0.1", + "valid": true, + "warning": false, + "hosts": [ + { + "type": "hostname", + "host": "256.0.0.1", + "port": null + } + ], + "auth": null, + "options": null + }, + { + "description": "Multiple hosts (mixed formats)", + "uri": "mongodb://127.0.0.1,[::1]:27018,example.com:27019", + "valid": false, + "warning": false, + "hosts": [ + { + "type": "ipv4", + "host": "127.0.0.1", + "port": null + }, + { + "type": "ip_literal", + "host": "::1", + "port": 27018 + }, + { + "type": "hostname", + "host": "example.com", + "port": 27019 + } + ], + "auth": null, + "options": null + }, + { + "description": "Multiple hosts (mixed formats)", + "uri": "mongodb://127.0.0.1,[::1]:27018,example.com:27019/?replicaSet=replset", + "valid": true, + "warning": false, + "hosts": [ + { + "type": "ipv4", + "host": "127.0.0.1", + "port": null + }, + { + "type": "ip_literal", + "host": "::1", + "port": 27018 + }, + { + "type": "hostname", + "host": "example.com", + "port": 27019 + } + ], + "auth": null, + "options": { + "replicaSet": "replset" + } + }, + { + "description": "UTF-8 hosts", + "uri": "mongodb://bücher.example.com,umläut.example.com/", + "valid": false, + "warning": false, + "hosts": [ + { + "type": "hostname", + "host": "bücher.example.com", + "port": null + }, + { + "type": "hostname", + "host": "umläut.example.com", + "port": null + } + ], + "auth": null, + "options": null + }, + { + "description": "UTF-8 hosts", + "uri": "mongodb://bücher.example.com,umläut.example.com/?replicaSet=replset", + "valid": true, + "warning": false, + "hosts": [ + { + "type": "hostname", + "host": "bücher.example.com", + "port": null + }, + { + "type": "hostname", + "host": "umläut.example.com", + "port": null + } + ], + "auth": null, + "options": { + "replicaSet": "replset" + } + } + ] +} \ No newline at end of file diff --git a/src/mongo/client/mongo_uri_tests/mongo-uri-invalid.json b/src/mongo/client/mongo_uri_tests/mongo-uri-invalid.json new file mode 100644 index 00000000000..3ee82bd535e --- /dev/null +++ b/src/mongo/client/mongo_uri_tests/mongo-uri-invalid.json @@ -0,0 +1,247 @@ +{ + "tests": [ + { + "auth": null, + "description": "Empty string", + "hosts": null, + "options": null, + "uri": "", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Missing host", + "hosts": null, + "options": null, + "uri": "mongodb://", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Double colon in host identifier", + "hosts": null, + "options": null, + "uri": "mongodb://localhost::27017", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Double colon in host identifier and trailing slash", + "hosts": null, + "options": null, + "uri": "mongodb://localhost::27017/", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Double colon in host identifier with missing host and port", + "hosts": null, + "options": null, + "uri": "mongodb://::", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Double colon in host identifier with missing port", + "hosts": null, + "options": null, + "uri": "mongodb://localhost,localhost::", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Double colon in host identifier and second host", + "hosts": null, + "options": null, + "uri": "mongodb://localhost::27017,abc", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Invalid port (negative number) with hostname", + "hosts": null, + "options": null, + "uri": "mongodb://localhost:-1", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Invalid port (zero) with hostname", + "hosts": null, + "options": null, + "uri": "mongodb://localhost:0/", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Invalid port (positive number) with hostname", + "hosts": null, + "options": null, + "uri": "mongodb://localhost:65536", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Invalid port (positive number) with hostname and trailing slash", + "hosts": null, + "options": null, + "uri": "mongodb://localhost:65536/", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Invalid port (non-numeric string) with hostname", + "hosts": null, + "options": null, + "uri": "mongodb://localhost:foo", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Invalid port (negative number) with IP literal", + "hosts": null, + "options": null, + "uri": "mongodb://[::1]:-1", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Invalid port (zero) with IP literal", + "hosts": null, + "options": null, + "uri": "mongodb://[::1]:0/", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Invalid port (positive number) with IP literal", + "hosts": null, + "options": null, + "uri": "mongodb://[::1]:65536", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Invalid port (positive number) with IP literal and trailing slash", + "hosts": null, + "options": null, + "uri": "mongodb://[::1]:65536/", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Invalid port (non-numeric string) with IP literal", + "hosts": null, + "options": null, + "uri": "mongodb://[::1]:foo", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Missing delimiting slash between hosts and options", + "hosts": null, + "options": null, + "uri": "mongodb://example.com?w=1", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Incomplete key value pair for option", + "hosts": null, + "options": null, + "uri": "mongodb://example.com/?w", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Username with password containing an unescaped colon", + "hosts": null, + "options": null, + "uri": "mongodb://alice:foo:bar@127.0.0.1", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Username with password containing an unescaped colon", + "hosts": null, + "options": null, + "uri": "mongodb://alice:foo:bar@127.0.0.1", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Username containing an unescaped at-sign", + "hosts": null, + "options": null, + "uri": "mongodb://alice@@127.0.0.1", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Username with password containing an unescaped at-sign", + "hosts": null, + "options": null, + "uri": "mongodb://alice@foo:bar@127.0.0.1", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Username containing an unescaped slash", + "hosts": null, + "options": null, + "uri": "mongodb://alice/@localhost/db", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Username containing unescaped slash with password", + "hosts": null, + "options": null, + "uri": "mongodb://alice/bob:foo@localhost/db", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Username with password containing an unescaped slash", + "hosts": null, + "options": null, + "uri": "mongodb://alice:foo/bar@localhost/db", + "valid": false, + "warning": null + }, + { + "auth": null, + "description": "Host with unescaped slash", + "hosts": null, + "options": null, + "uri": "mongodb:///tmp/mongodb-27017.sock/", + "valid": false, + "warning": null + } + ] +} diff --git a/src/mongo/client/mongo_uri_tests/mongo-uri-options.json b/src/mongo/client/mongo_uri_tests/mongo-uri-options.json new file mode 100644 index 00000000000..4c2bded9e72 --- /dev/null +++ b/src/mongo/client/mongo_uri_tests/mongo-uri-options.json @@ -0,0 +1,25 @@ +{ + "tests": [ + { + "description": "Option names are normalized to lowercase", + "uri": "mongodb://alice:secret@example.com/admin?AUTHMechanism=MONGODB-CR", + "valid": true, + "warning": false, + "hosts": [ + { + "type": "hostname", + "host": "example.com", + "port": null + } + ], + "auth": { + "username": "alice", + "password": "secret", + "db": "admin" + }, + "options": { + "authmechanism": "MONGODB-CR" + } + } + ] +} diff --git a/src/mongo/client/mongo_uri_tests/mongo-uri-unix-sockets-absolute.json b/src/mongo/client/mongo_uri_tests/mongo-uri-unix-sockets-absolute.json new file mode 100644 index 00000000000..d167c315bbc --- /dev/null +++ b/src/mongo/client/mongo_uri_tests/mongo-uri-unix-sockets-absolute.json @@ -0,0 +1,373 @@ +{ + "tests": [ + { + "auth": null, + "description": "Unix domain socket (absolute path with trailing slash)", + "hosts": [ + { + "host": "/tmp/mongodb-27017.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://%2Ftmp%2Fmongodb-27017.sock/", + "valid": true, + "warning": false + }, + { + "auth": null, + "description": "Unix domain socket (absolute path without trailing slash)", + "hosts": [ + { + "host": "/tmp/mongodb-27017.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://%2Ftmp%2Fmongodb-27017.sock", + "valid": true, + "warning": false + }, + { + "auth": null, + "description": "Unix domain socket (absolute path with spaces in path)", + "hosts": [ + { + "host": "/tmp/ /mongodb-27017.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://%2Ftmp%2F %2Fmongodb-27017.sock", + "valid": true, + "warning": false + }, + { + "auth": null, + "description": "Multiple Unix domain sockets (absolute paths)", + "hosts": [ + { + "host": "/tmp/mongodb-27017.sock", + "port": null, + "type": "unix" + }, + { + "host": "/tmp/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://%2Ftmp%2Fmongodb-27017.sock,%2Ftmp%2Fmongodb-27018.sock", + "valid": false, + "warning": false + }, + { + "auth": null, + "description": "Multiple hosts (absolute path and ipv4)", + "hosts": [ + { + "host": "127.0.0.1", + "port": 27017, + "type": "ipv4" + }, + { + "host": "/tmp/mongodb-27017.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://127.0.0.1:27017,%2Ftmp%2Fmongodb-27017.sock", + "valid": false, + "warning": false + }, + { + "auth": null, + "description": "Multiple hosts (absolute path and hostname resembling relative path)", + "hosts": [ + { + "host": "mongodb-27017.sock", + "port": null, + "type": "hostname" + }, + { + "host": "/tmp/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://mongodb-27017.sock,%2Ftmp%2Fmongodb-27018.sock", + "valid": false, + "warning": false + }, + { + "auth": null, + "description": "Multiple Unix domain sockets (absolute paths)", + "hosts": [ + { + "host": "/tmp/mongodb-27017.sock", + "port": null, + "type": "unix" + }, + { + "host": "/tmp/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": { + "replicaSet": "replset" + }, + "uri": "mongodb://%2Ftmp%2Fmongodb-27017.sock,%2Ftmp%2Fmongodb-27018.sock/?replicaSet=replset", + "valid": true, + "warning": false + }, + { + "auth": null, + "description": "Multiple hosts (absolute path and ipv4)", + "hosts": [ + { + "host": "127.0.0.1", + "port": 27017, + "type": "ipv4" + }, + { + "host": "/tmp/mongodb-27017.sock", + "port": null, + "type": "unix" + } + ], + "options": { + "replicaSet": "replset" + }, + "uri": "mongodb://127.0.0.1:27017,%2Ftmp%2Fmongodb-27017.sock/?replicaSet=replset", + "valid": true, + "warning": false + }, + { + "auth": null, + "description": "Multiple hosts (absolute path and hostname resembling relative path)", + "hosts": [ + { + "host": "mongodb-27017.sock", + "port": null, + "type": "hostname" + }, + { + "host": "/tmp/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": { + "replicaSet": "replset" + }, + "uri": "mongodb://mongodb-27017.sock,%2Ftmp%2Fmongodb-27018.sock/?replicaSet=replset", + "valid": true, + "warning": false + }, + { + "auth": { + "db": "admin", + "password": "foo", + "username": "alice" + }, + "description": "Unix domain socket with auth database (absolute path)", + "hosts": [ + { + "host": "/tmp/mongodb-27017.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://alice:foo@%2Ftmp%2Fmongodb-27017.sock/admin", + "valid": true, + "warning": false + }, + { + "auth": null, + "description": "Unix domain socket with path resembling socket file (absolute path with trailing slash)", + "hosts": [ + { + "host": "/tmp/path.to.sock/mongodb-27017.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://%2Ftmp%2Fpath.to.sock%2Fmongodb-27017.sock/", + "valid": true, + "warning": false + }, + { + "auth": null, + "description": "Unix domain socket with path resembling socket file (absolute path without trailing slash)", + "hosts": [ + { + "host": "/tmp/path.to.sock/mongodb-27017.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://%2Ftmp%2Fpath.to.sock%2Fmongodb-27017.sock", + "valid": true, + "warning": false + }, + { + "auth": { + "db": "admin", + "password": "bar", + "username": "bob" + }, + "description": "Unix domain socket with path resembling socket file and auth (absolute path)", + "hosts": [ + { + "host": "/tmp/path.to.sock/mongodb-27017.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://bob:bar@%2Ftmp%2Fpath.to.sock%2Fmongodb-27017.sock/admin", + "valid": true, + "warning": false + }, + { + "auth": { + "db": "admin", + "password": null, + "username": null + }, + "description": "Multiple Unix domain sockets and auth DB (absolute path)", + "hosts": [ + { + "host": "/tmp/mongodb-27017.sock", + "port": null, + "type": "unix" + }, + { + "host": "/tmp/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://%2Ftmp%2Fmongodb-27017.sock,%2Ftmp%2Fmongodb-27018.sock/admin", + "valid": false, + "warning": false + }, + { + "auth": { + "db": "admin", + "password": null, + "username": null + }, + "description": "Multiple Unix domain sockets and auth DB (absolute path)", + "hosts": [ + { + "host": "/tmp/mongodb-27017.sock", + "port": null, + "type": "unix" + }, + { + "host": "/tmp/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": { + "replicaSet": "replset" + }, + "uri": "mongodb://%2Ftmp%2Fmongodb-27017.sock,%2Ftmp%2Fmongodb-27018.sock/admin?replicaSet=replset", + "valid": true, + "warning": false + }, + { + "auth": { + "db": "admin", + "password": "bar", + "username": "bob" + }, + "description": "Multiple Unix domain sockets with auth and query string (absolute path)", + "hosts": [ + { + "host": "/tmp/mongodb-27017.sock", + "port": null, + "type": "unix" + }, + { + "host": "/tmp/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": { + "w": 1 + }, + "uri": "mongodb://bob:bar@%2Ftmp%2Fmongodb-27017.sock,%2Ftmp%2Fmongodb-27018.sock/admin?w=1", + "valid": false, + "warning": false + }, + { + "auth": { + "db": "admin", + "password": "bar", + "username": "bob" + }, + "description": "Multiple Unix domain sockets with auth and query string (absolute path)", + "hosts": [ + { + "host": "/tmp/mongodb-27017.sock", + "port": null, + "type": "unix" + }, + { + "host": "/tmp/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": { + "w": 1, + "replicaSet":"replset" + }, + "uri": "mongodb://bob:bar@%2Ftmp%2Fmongodb-27017.sock,%2Ftmp%2Fmongodb-27018.sock/admin?w=1&replicaSet=replset", + "valid": true, + "warning": false + }, + { + "auth": { + "db": "admin", + "password": "bar", + "username": "bob" + }, + "description": "Multiple Unix domain sockets with auth and query string (absolute path)", + "hosts": [ + { + "host": "/tmp/mongodb-27017.sock", + "port": null, + "type": "unix" + }, + { + "host": "/tmp/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": { + "w": 1, + "replicaSet":"replset" + }, + "uri": "mongodb://bob:bar@%2Ftmp%2Fmongodb-27017.sock,%2Ftmp%2Fmongodb-27018.sock/admin?replicaSet=replset&w=1", + "valid": true, + "warning": false + } + ] +} diff --git a/src/mongo/client/mongo_uri_tests/mongo-uri-unix-sockets-relative.json b/src/mongo/client/mongo_uri_tests/mongo-uri-unix-sockets-relative.json new file mode 100644 index 00000000000..8da45c4e1b9 --- /dev/null +++ b/src/mongo/client/mongo_uri_tests/mongo-uri-unix-sockets-relative.json @@ -0,0 +1,466 @@ +{ + "tests": [ + { + "auth": null, + "description": "Unix domain socket (relative path with trailing slash)", + "hosts": [ + { + "host": "rel/mongodb-27017.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://rel%2Fmongodb-27017.sock/", + "valid": true, + "warning": false + }, + { + "auth": null, + "description": "Unix domain socket (relative path without trailing slash)", + "hosts": [ + { + "host": "rel/mongodb-27017.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://rel%2Fmongodb-27017.sock", + "valid": true, + "warning": false + }, + { + "auth": null, + "description": "Unix domain socket (relative path with spaces)", + "hosts": [ + { + "host": "rel/ /mongodb-27017.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://rel%2F %2Fmongodb-27017.sock", + "valid": true, + "warning": false + }, + { + "auth": null, + "description": "Multiple Unix domain sockets (relative paths)", + "hosts": [ + { + "host": "rel/mongodb-27017.sock", + "port": null, + "type": "unix" + }, + { + "host": "rel/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://rel%2Fmongodb-27017.sock,rel%2Fmongodb-27018.sock", + "valid": false, + "warning": false + }, + { + "auth": null, + "description": "Multiple Unix domain sockets (relative paths)", + "hosts": [ + { + "host": "rel/mongodb-27017.sock", + "port": null, + "type": "unix" + }, + { + "host": "rel/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": { + "replicaSet": "replset" + }, + "uri": "mongodb://rel%2Fmongodb-27017.sock,rel%2Fmongodb-27018.sock/?replicaSet=replset", + "valid": true, + "warning": false + }, + { + "auth": null, + "description": "Multiple Unix domain sockets (relative and absolute paths)", + "hosts": [ + { + "host": "rel/mongodb-27017.sock", + "port": null, + "type": "unix" + }, + { + "host": "/tmp/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://rel%2Fmongodb-27017.sock,%2Ftmp%2Fmongodb-27018.sock", + "valid": false, + "warning": false + }, + { + "auth": null, + "description": "Multiple Unix domain sockets (relative and absolute paths)", + "hosts": [ + { + "host": "rel/mongodb-27017.sock", + "port": null, + "type": "unix" + }, + { + "host": "/tmp/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": { + "replicaSet": "replset" + }, + "uri": "mongodb://rel%2Fmongodb-27017.sock,%2Ftmp%2Fmongodb-27018.sock/?replicaSet=replset", + "valid": true, + "warning": false + }, + { + "auth": null, + "description": "Multiple hosts (relative path and ipv4)", + "hosts": [ + { + "host": "127.0.0.1", + "port": 27017, + "type": "ipv4" + }, + { + "host": "rel/mongodb-27017.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://127.0.0.1:27017,rel%2Fmongodb-27017.sock", + "valid": false, + "warning": false + }, + { + "auth": null, + "description": "Multiple hosts (relative path and ipv4)", + "hosts": [ + { + "host": "127.0.0.1", + "port": 27017, + "type": "ipv4" + }, + { + "host": "rel/mongodb-27017.sock", + "port": null, + "type": "unix" + } + ], + "options": { + "replicaSet": "replset" + }, + "uri": "mongodb://127.0.0.1:27017,rel%2Fmongodb-27017.sock/?replicaSet=replset", + "valid": true, + "warning": false + }, + { + "auth": null, + "description": "Multiple hosts (relative path and hostname resembling relative path)", + "hosts": [ + { + "host": "mongodb-27017.sock", + "port": null, + "type": "hostname" + }, + { + "host": "rel/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://mongodb-27017.sock,rel%2Fmongodb-27018.sock", + "valid": false, + "warning": false + }, + { + "auth": null, + "description": "Multiple hosts (relative path and hostname resembling relative path)", + "hosts": [ + { + "host": "mongodb-27017.sock", + "port": null, + "type": "hostname" + }, + { + "host": "rel/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": { + "replicaSet": "replset" + }, + "uri": "mongodb://mongodb-27017.sock,rel%2Fmongodb-27018.sock/?replicaSet=replset", + "valid": true, + "warning": false + }, + { + "auth": { + "db": "admin", + "password": "foo", + "username": "alice" + }, + "description": "Unix domain socket with auth database (relative path)", + "hosts": [ + { + "host": "rel/mongodb-27017.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://alice:foo@rel%2Fmongodb-27017.sock/admin", + "valid": true, + "warning": false + }, + { + "auth": null, + "description": "Unix domain socket with path resembling socket file (relative path with trailing slash)", + "hosts": [ + { + "host": "rel/path.to.sock/mongodb-27017.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://rel%2Fpath.to.sock%2Fmongodb-27017.sock/", + "valid": true, + "warning": false + }, + { + "auth": null, + "description": "Unix domain socket with path resembling socket file (relative path without trailing slash)", + "hosts": [ + { + "host": "rel/path.to.sock/mongodb-27017.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://rel%2Fpath.to.sock%2Fmongodb-27017.sock", + "valid": true, + "warning": false + }, + { + "auth": { + "db": "admin", + "password": "bar", + "username": "bob" + }, + "description": "Unix domain socket with path resembling socket file and auth (relative path)", + "hosts": [ + { + "host": "rel/path.to.sock/mongodb-27017.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://bob:bar@rel%2Fpath.to.sock%2Fmongodb-27017.sock/admin", + "valid": true, + "warning": false + }, + { + "auth": { + "db": "admin", + "password": null, + "username": null + }, + "description": "Multiple Unix domain sockets and auth DB resembling a socket (relative path)", + "hosts": [ + { + "host": "rel/mongodb-27017.sock", + "port": null, + "type": "unix" + }, + { + "host": "rel/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://rel%2Fmongodb-27017.sock,rel%2Fmongodb-27018.sock/admin", + "valid": false, + "warning": false + }, + { + "auth": { + "db": "admin", + "password": null, + "username": null + }, + "description": "Multiple Unix domain sockets and auth DB resembling a socket (relative path)", + "hosts": [ + { + "host": "rel/mongodb-27017.sock", + "port": null, + "type": "unix" + }, + { + "host": "rel/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": { + "replicaSet": "replset" + }, + "uri": "mongodb://rel%2Fmongodb-27017.sock,rel%2Fmongodb-27018.sock/admin?replicaSet=replset", + "valid": true, + "warning": false + }, + { + "auth": { + "db": "admin", + "password": null, + "username": null + }, + "description": "Multiple Unix domain sockets with auth DB resembling a path (relative path)", + "hosts": [ + { + "host": "rel/mongodb-27017.sock", + "port": null, + "type": "unix" + }, + { + "host": "rel/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": null, + "uri": "mongodb://rel%2Fmongodb-27017.sock,rel%2Fmongodb-27018.sock/admin", + "valid": false, + "warning": false + }, + { + "auth": { + "db": "admin", + "password": null, + "username": null + }, + "description": "Multiple Unix domain sockets with auth DB resembling a path (relative path)", + "hosts": [ + { + "host": "rel/mongodb-27017.sock", + "port": null, + "type": "unix" + }, + { + "host": "rel/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": { + "replicaSet": "replset" + }, + "uri": "mongodb://rel%2Fmongodb-27017.sock,rel%2Fmongodb-27018.sock/admin?replicaSet=replset", + "valid": true, + "warning": false + }, + { + "auth": { + "db": "admin", + "password": "bar", + "username": "bob" + }, + "description": "Multiple Unix domain sockets with auth and query string (relative path)", + "hosts": [ + { + "host": "rel/mongodb-27017.sock", + "port": null, + "type": "unix" + }, + { + "host": "rel/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": { + "w": 1 + }, + "uri": "mongodb://bob:bar@rel%2Fmongodb-27017.sock,rel%2Fmongodb-27018.sock/admin?w=1", + "valid": false, + "warning": false + }, + { + "auth": { + "db": "admin", + "password": "bar", + "username": "bob" + }, + "description": "Multiple Unix domain sockets with auth and query string (relative path)", + "hosts": [ + { + "host": "rel/mongodb-27017.sock", + "port": null, + "type": "unix" + }, + { + "host": "rel/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": { + "w": 1, + "replicaSet": "replset" + }, + "uri": "mongodb://bob:bar@rel%2Fmongodb-27017.sock,rel%2Fmongodb-27018.sock/admin?w=1&replicaSet=replset", + "valid": true, + "warning": false + }, + { + "auth": { + "db": "admin", + "password": "bar", + "username": "bob" + }, + "description": "Multiple Unix domain sockets with auth and query string (relative path)", + "hosts": [ + { + "host": "rel/mongodb-27017.sock", + "port": null, + "type": "unix" + }, + { + "host": "rel/mongodb-27018.sock", + "port": null, + "type": "unix" + } + ], + "options": { + "w": 1, + "replicaSet": "replset", + "b": 4 + }, + "uri": "mongodb://bob:bar@rel%2Fmongodb-27017.sock,rel%2Fmongodb-27018.sock/admin?w=1&replicaSet=replset&b=4", + "valid": true, + "warning": false + } + ] +} diff --git a/src/mongo/client/mongo_uri_tests/mongo-uri-valid-auth.json b/src/mongo/client/mongo_uri_tests/mongo-uri-valid-auth.json new file mode 100644 index 00000000000..a44236e7b62 --- /dev/null +++ b/src/mongo/client/mongo_uri_tests/mongo-uri-valid-auth.json @@ -0,0 +1,311 @@ +{ + "tests": [ + { + "auth": { + "db": null, + "password": "foo", + "username": "alice" + }, + "description": "User info for single IPv4 host without database", + "hosts": [ + { + "host": "127.0.0.1", + "port": null, + "type": "ipv4" + } + ], + "options": null, + "uri": "mongodb://alice:foo@127.0.0.1", + "valid": true, + "warning": false + }, + { + "auth": { + "db": "test", + "password": "foo", + "username": "alice" + }, + "description": "User info for single IPv4 host with database", + "hosts": [ + { + "host": "127.0.0.1", + "port": null, + "type": "ipv4" + } + ], + "options": null, + "uri": "mongodb://alice:foo@127.0.0.1/test", + "valid": true, + "warning": false + }, + { + "auth": { + "db": null, + "password": "bar", + "username": "bob" + }, + "description": "User info for single IP literal host without database", + "hosts": [ + { + "host": "::1", + "port": 27018, + "type": "ip_literal" + } + ], + "options": null, + "uri": "mongodb://bob:bar@[::1]:27018", + "valid": true, + "warning": false + }, + { + "auth": { + "db": "admin", + "password": "bar", + "username": "bob" + }, + "description": "User info for single IP literal host with database", + "hosts": [ + { + "host": "::1", + "port": 27018, + "type": "ip_literal" + } + ], + "options": null, + "uri": "mongodb://bob:bar@[::1]:27018/admin", + "valid": true, + "warning": false + }, + { + "auth": { + "db": null, + "password": "baz", + "username": "eve" + }, + "description": "User info for single hostname without database", + "hosts": [ + { + "host": "example.com", + "port": null, + "type": "hostname" + } + ], + "options": null, + "uri": "mongodb://eve:baz@example.com", + "valid": true, + "warning": false + }, + { + "auth": { + "db": "db2", + "password": "baz", + "username": "eve" + }, + "description": "User info for single hostname with database", + "hosts": [ + { + "host": "example.com", + "port": null, + "type": "hostname" + } + ], + "options": null, + "uri": "mongodb://eve:baz@example.com/db2", + "valid": true, + "warning": false + }, + { + "auth": { + "db": null, + "password": "secret", + "username": "alice" + }, + "description": "User info for multiple hosts without database", + "hosts": [ + { + "host": "127.0.0.1", + "port": null, + "type": "ipv4" + }, + { + "host": "example.com", + "port": 27018, + "type": "hostname" + } + ], + "options": null, + "uri": "mongodb://alice:secret@127.0.0.1,example.com:27018", + "valid": false, + "warning": false + }, + { + "auth": { + "db": "admin", + "password": "secret", + "username": "alice" + }, + "description": "User info for multiple hosts with database", + "hosts": [ + { + "host": "example.com", + "port": null, + "type": "hostname" + }, + { + "host": "::1", + "port": 27019, + "type": "ip_literal" + } + ], + "options": null, + "uri": "mongodb://alice:secret@example.com,[::1]:27019/admin", + "valid": false, + "warning": false + }, + { + "auth": { + "db": null, + "password": null, + "username": "alice" + }, + "description": "Username without password", + "hosts": [ + { + "host": "127.0.0.1", + "port": null, + "type": "ipv4" + } + ], + "options": null, + "uri": "mongodb://alice@127.0.0.1", + "valid": true, + "warning": false + }, + { + "auth": { + "db": null, + "password": "", + "username": "alice" + }, + "description": "Username with empty password", + "hosts": [ + { + "host": "127.0.0.1", + "port": null, + "type": "ipv4" + } + ], + "options": null, + "uri": "mongodb://alice:@127.0.0.1", + "valid": true, + "warning": false + }, + { + "auth": { + "db": "my=db", + "password": null, + "username": "@l:ce/=" + }, + "description": "Escaped username and database without password", + "hosts": [ + { + "host": "example.com", + "port": null, + "type": "hostname" + } + ], + "options": null, + "uri": "mongodb://%40l%3Ace%2F%3D@example.com/my%3Ddb", + "valid": true, + "warning": false + }, + { + "auth": { + "db": "admin=", + "password": "fizzb@zz=", + "username": "$am" + }, + "description": "Escaped user info and database (MONGODB-CR)", + "hosts": [ + { + "host": "127.0.0.1", + "port": null, + "type": "ipv4" + } + ], + "options": { + "authmechanism": "MONGODB-CR" + }, + "uri": "mongodb://%24am:fizzb%40zz%3D@127.0.0.1/admin%3D?authMechanism=MONGODB-CR", + "valid": true, + "warning": false + }, + { + "auth": { + "db": null, + "password": null, + "username": "CN=myName,OU=myOrgUnit,O=myOrg,L=myLocality,ST=myState,C=myCountry" + }, + "description": "Escaped username (MONGODB-X509)", + "hosts": [ + { + "host": "localhost", + "port": null, + "type": "hostname" + } + ], + "options": { + "authmechanism": "MONGODB-X509" + }, + "uri": "mongodb://CN%3DmyName%2COU%3DmyOrgUnit%2CO%3DmyOrg%2CL%3DmyLocality%2CST%3DmyState%2CC%3DmyCountry@localhost/?authMechanism=MONGODB-X509", + "valid": true, + "warning": false + }, + { + "auth": { + "db": null, + "password": "secret", + "username": "user@EXAMPLE.COM" + }, + "description": "Escaped username (GSSAPI)", + "hosts": [ + { + "host": "localhost", + "port": null, + "type": "hostname" + } + ], + "options": { + "authmechanism": "GSSAPI", + "authmechanismproperties": { + "CANONICALIZE_HOST_NAME": true, + "SERVICE_NAME": "other" + } + }, + "uri": "mongodb://user%40EXAMPLE.COM:secret@localhost/?authMechanismProperties=SERVICE_NAME:other,CANONICALIZE_HOST_NAME:true&authMechanism=GSSAPI", + "valid": true, + "warning": false + }, + { + "auth": { + "db": "admin", + "password": "secret", + "username": "alice" + }, + "description": "At-signs in options aren't part of the userinfo", + "hosts": [ + { + "host": "example.com", + "port": null, + "type": "hostname" + } + ], + "options": { + "replicaSet": "my@replicaset" + }, + "uri": "mongodb://alice:secret@example.com/admin?replicaSet=my@replicaset", + "valid": true, + "warning": false + } + ] +} diff --git a/src/mongo/client/mongo_uri_tests/mongo-uri-warnings.json b/src/mongo/client/mongo_uri_tests/mongo-uri-warnings.json new file mode 100644 index 00000000000..a04246847ab --- /dev/null +++ b/src/mongo/client/mongo_uri_tests/mongo-uri-warnings.json @@ -0,0 +1,56 @@ +{ + "tests": [ + { + "description": "Unrecognized option keys are ignored", + "uri": "mongodb://example.com/?foo=bar", + "valid": true, + "warning": true, + "hosts": [ + { + "type": "hostname", + "host": "example.com", + "port": null + } + ], + "auth": null, + "options": { + "foo": "bar" + } + }, + { + "description": "Unsupported option values are ignored", + "uri": "mongodb://example.com/?fsync=ifPossible", + "valid": true, + "warning": true, + "hosts": [ + { + "type": "hostname", + "host": "example.com", + "port": null + } + ], + "auth": null, + "options": { + "fsync": "ifPossible" + } + }, + { + "description": "Deprecated (or unknown) options are ignored if replacement exists", + "uri": "mongodb://example.com/?wtimeout=5&wtimeoutMS=10", + "valid": true, + "warning": true, + "hosts": [ + { + "type": "hostname", + "host": "example.com", + "port": null + } + ], + "auth": null, + "options": { + "wtimeoutMS": 10, + "wtimeout": 5 + } + } + ] +} \ No newline at end of file diff --git a/src/mongo/crypto/mechanism_scram.cpp b/src/mongo/crypto/mechanism_scram.cpp index 8cd59142d67..6a2757d383d 100644 --- a/src/mongo/crypto/mechanism_scram.cpp +++ b/src/mongo/crypto/mechanism_scram.cpp @@ -226,6 +226,10 @@ bool verifyClientProof(StringData clientProof, StringData storedKey, StringData SHA1Block computedStoredKey = SHA1Block::computeHash(clientSignature.data(), clientSignature.size()); + if (storedKey.size() != computedStoredKey.size()) { + return false; + } + return consttimeMemEqual(reinterpret_cast(storedKey.rawData()), computedStoredKey.data(), computedStoredKey.size()); diff --git a/src/mongo/crypto/mechanism_scram.h b/src/mongo/crypto/mechanism_scram.h index b12cebbf2e1..6045f037078 100644 --- a/src/mongo/crypto/mechanism_scram.h +++ b/src/mongo/crypto/mechanism_scram.h @@ -53,15 +53,22 @@ const std::string serverKeyFieldName = "serverKey"; */ struct SCRAMPresecrets { SCRAMPresecrets(std::string hashedPassword, - std::vector salt, - size_t iterationCount) + std::vector salt_, + size_t iterationCount_) : hashedPassword(std::move(hashedPassword)), - salt(std::move(salt)), - iterationCount(iterationCount) {} + salt(std::move(salt_)), + iterationCount(iterationCount_) { + uassert(ErrorCodes::BadValue, + "Invalid salt for SCRAM mechanism", + salt.size() >= kSaltLengthMin); + } std::string hashedPassword; std::vector salt; size_t iterationCount; + +private: + static const size_t kSaltLengthMin = 16; }; inline bool operator==(const SCRAMPresecrets& lhs, const SCRAMPresecrets& rhs) { diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index e2be152f126..ded6480875c 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -445,6 +445,7 @@ env.Library( LIBDEPS=[ '$BUILD_DIR/mongo/db/serveronly', '$BUILD_DIR/mongo/util/options_parser/options_parser_init', + 'repl/replica_set_messages', 'server_options', ], ) @@ -766,9 +767,12 @@ env.Library( 'log_process_details.cpp', ], LIBDEPS=[ + 'repl/repl_coordinator_global', + 'repl/repl_coordinator_interface', + 'repl/replica_set_messages', + 'server_options', '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/util/net/network', - 'server_options', ], ) @@ -818,3 +822,4 @@ asioEnv.CppIntegrationTest( '$BUILD_DIR/mongo/executor/network_interface_asio_fixture', ], ) + diff --git a/src/mongo/db/auth/SConscript b/src/mongo/db/auth/SConscript index 792314035b5..fc85149de7f 100644 --- a/src/mongo/db/auth/SConscript +++ b/src/mongo/db/auth/SConscript @@ -138,6 +138,7 @@ env.Library('authmongod', ], LIBDEPS=[ 'authservercommon', + '$BUILD_DIR/mongo/db/catalog/index_key_validate', '$BUILD_DIR/mongo/db/db_raii', '$BUILD_DIR/mongo/db/dbdirectclient', '$BUILD_DIR/mongo/db/dbhelpers', diff --git a/src/mongo/db/auth/auth_index_d.cpp b/src/mongo/db/auth/auth_index_d.cpp index 427900c8377..47987ac40c4 100644 --- a/src/mongo/db/auth/auth_index_d.cpp +++ b/src/mongo/db/auth/auth_index_d.cpp @@ -34,15 +34,20 @@ #include "mongo/base/init.h" #include "mongo/base/status.h" +#include "mongo/client/index_spec.h" #include "mongo/db/auth/authorization_manager.h" #include "mongo/db/auth/authorization_manager_global.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/index_catalog.h" +#include "mongo/db/catalog/index_create.h" +#include "mongo/db/catalog/index_key_validate.h" #include "mongo/db/client.h" -#include "mongo/db/client.h" +#include "mongo/db/concurrency/write_conflict_exception.h" +#include "mongo/db/curop.h" #include "mongo/db/db_raii.h" #include "mongo/db/index/index_descriptor.h" #include "mongo/db/jsobj.h" +#include "mongo/db/storage/storage_options.h" #include "mongo/util/assert_util.h" #include "mongo/util/log.h" @@ -58,6 +63,8 @@ BSONObj v3SystemUsersKeyPattern; BSONObj v3SystemRolesKeyPattern; std::string v3SystemUsersIndexName; std::string v3SystemRolesIndexName; +IndexSpec v3SystemUsersIndexSpec; +IndexSpec v3SystemRolesIndexSpec; MONGO_INITIALIZER(AuthIndexKeyPatterns)(InitializerContext*) { v1SystemUsersKeyPattern = BSON("user" << 1 << "userSource" << 1); @@ -76,13 +83,66 @@ MONGO_INITIALIZER(AuthIndexKeyPatterns)(InitializerContext*) { << AuthorizationManager::ROLE_DB_FIELD_NAME << "_1"); + v3SystemUsersIndexSpec.addKeys(v3SystemUsersKeyPattern); + v3SystemUsersIndexSpec.unique(); + v3SystemUsersIndexSpec.name(v3SystemUsersIndexName); + + v3SystemRolesIndexSpec.addKeys(v3SystemRolesKeyPattern); + v3SystemRolesIndexSpec.unique(); + v3SystemRolesIndexSpec.name(v3SystemRolesIndexName); + return Status::OK(); } +void generateSystemIndexForExistingCollection(OperationContext* opCtx, + Collection* collection, + const NamespaceString& ns, + const IndexSpec& spec) { + // Do not try and generate any system indexes in read only mode. + if (storageGlobalParams.readOnly) { + warning() << "Running in queryable backup mode. Unable to create authorization index on " + << ns; + return; + } + + try { + auto indexSpecStatus = index_key_validate::validateIndexSpec( + spec.toBSON(), ns, serverGlobalParams.featureCompatibility); + BSONObj indexSpec = fassertStatusOK(40452, indexSpecStatus); + + log() << "No authorization index detected on " << ns + << " collection. Attempting to recover by creating an index with spec: " << indexSpec; + + MultiIndexBlock indexer(opCtx, collection); + + MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { + fassertStatusOK(40453, indexer.init(indexSpec)); + } + MONGO_WRITE_CONFLICT_RETRY_LOOP_END(opCtx, "authorization index regeneration", ns.ns()); + + fassertStatusOK(40454, indexer.insertAllDocumentsInCollection()); + + MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { + WriteUnitOfWork wunit(opCtx); + + indexer.commit(); + + wunit.commit(); + } + MONGO_WRITE_CONFLICT_RETRY_LOOP_END(opCtx, "authorization index regeneration", ns.ns()); + + log() << "Authorization index construction on " << ns << " is complete"; + } catch (const DBException& e) { + severe() << "Failed to regenerate index for " << ns << ". Exception: " << e.what(); + throw; + } +} + } // namespace Status verifySystemIndexes(OperationContext* txn) { - const NamespaceString systemUsers = AuthorizationManager::usersCollectionNamespace; + const NamespaceString& systemUsers = AuthorizationManager::usersCollectionNamespace; + const NamespaceString& systemRoles = AuthorizationManager::rolesCollectionNamespace; // Make sure the old unique index from v2.4 on system.users doesn't exist. ScopedTransaction scopedXact(txn, MODE_IX); @@ -91,23 +151,54 @@ Status verifySystemIndexes(OperationContext* txn) { return Status::OK(); } - Collection* collection = autoDb.getDb()->getCollection(NamespaceString(systemUsers)); - if (!collection) { - return Status::OK(); + Collection* collection = autoDb.getDb()->getCollection(systemUsers); + if (collection) { + IndexCatalog* indexCatalog = collection->getIndexCatalog(); + invariant(indexCatalog); + + // Make sure the old unique index from v2.4 on system.users doesn't exist. + std::vector indexes; + indexCatalog->findIndexesByKeyPattern(txn, v1SystemUsersKeyPattern, false, &indexes); + + if (!indexes.empty()) { + fassert(ErrorCodes::AmbiguousIndexKeyPattern, indexes.size() == 1); + return Status(ErrorCodes::AuthSchemaIncompatible, + "Old 2.4 style user index identified. " + "The authentication schema needs to be updated by " + "running authSchemaUpgrade on a 2.6 server."); + } + + // Ensure that system indexes exist for the user collection + indexCatalog->findIndexesByKeyPattern(txn, v3SystemUsersKeyPattern, false, &indexes); + if (indexes.empty()) { + try { + generateSystemIndexForExistingCollection( + txn, collection, systemUsers, v3SystemUsersIndexSpec); + } catch (...) { + return exceptionToStatus(); + } + } } - IndexCatalog* indexCatalog = collection->getIndexCatalog(); - std::vector indexes; - indexCatalog->findIndexesByKeyPattern(txn, v1SystemUsersKeyPattern, false, &indexes); - - if (indexCatalog && !indexes.empty()) { - fassert(ErrorCodes::AmbiguousIndexKeyPattern, indexes.size() == 1); - return Status(ErrorCodes::AuthSchemaIncompatible, - "Old 2.4 style user index identified. " - "The authentication schema needs to be updated by " - "running authSchemaUpgrade on a 2.6 server."); + // Ensure that system indexes exist for the roles collection, if it exists. + collection = autoDb.getDb()->getCollection(systemRoles); + if (collection) { + IndexCatalog* indexCatalog = collection->getIndexCatalog(); + invariant(indexCatalog); + + std::vector indexes; + indexCatalog->findIndexesByKeyPattern(txn, v3SystemRolesKeyPattern, false, &indexes); + if (indexes.empty()) { + try { + generateSystemIndexForExistingCollection( + txn, collection, systemRoles, v3SystemRolesIndexSpec); + } catch (...) { + return exceptionToStatus(); + } + } } + return Status::OK(); } @@ -115,19 +206,21 @@ void createSystemIndexes(OperationContext* txn, Collection* collection) { invariant(collection); const NamespaceString& ns = collection->ns(); if (ns == AuthorizationManager::usersCollectionNamespace) { - collection->getIndexCatalog()->createIndexOnEmptyCollection( - txn, - BSON("name" << v3SystemUsersIndexName << "ns" << collection->ns().ns() << "key" - << v3SystemUsersKeyPattern - << "unique" - << true)); + auto indexSpec = fassertStatusOK( + 40455, + index_key_validate::validateIndexSpec( + v3SystemUsersIndexSpec.toBSON(), ns, serverGlobalParams.featureCompatibility)); + + fassertStatusOK( + 40456, collection->getIndexCatalog()->createIndexOnEmptyCollection(txn, indexSpec)); } else if (ns == AuthorizationManager::rolesCollectionNamespace) { - collection->getIndexCatalog()->createIndexOnEmptyCollection( - txn, - BSON("name" << v3SystemRolesIndexName << "ns" << collection->ns().ns() << "key" - << v3SystemRolesKeyPattern - << "unique" - << true)); + auto indexSpec = fassertStatusOK( + 40457, + index_key_validate::validateIndexSpec( + v3SystemRolesIndexSpec.toBSON(), ns, serverGlobalParams.featureCompatibility)); + + fassertStatusOK( + 40458, collection->getIndexCatalog()->createIndexOnEmptyCollection(txn, indexSpec)); } } diff --git a/src/mongo/db/auth/sasl_scramsha1_server_conversation.cpp b/src/mongo/db/auth/sasl_scramsha1_server_conversation.cpp index f48ab9b8953..900d0b818fc 100644 --- a/src/mongo/db/auth/sasl_scramsha1_server_conversation.cpp +++ b/src/mongo/db/auth/sasl_scramsha1_server_conversation.cpp @@ -198,6 +198,12 @@ StatusWith SaslSCRAMSHA1ServerConversation::_firstStep(std::vector _creds.scram.serverKey = scramCreds[scram::serverKeyFieldName].String(); } + if (!_creds.scram.isValid()) { + return Status(ErrorCodes::AuthenticationFailed, + "Unable to perform SCRAM-SHA-1 authentication for a user with missing " + "or invalid SCRAM credentials"); + } + // Generate server-first-message // Create text-based nonce as base64 encoding of a binary blob of length multiple of 3 const int nonceLenQWords = 3; @@ -280,6 +286,7 @@ StatusWith SaslSCRAMSHA1ServerConversation::_secondStep(const std::vector< // ClientSignature := HMAC(StoredKey, AuthMessage) // ClientKey := ClientSignature XOR ClientProof // ServerSignature := HMAC(ServerKey, AuthMessage) + invariant(_creds.scram.isValid()); if (!scram::verifyClientProof( base64::decode(clientProof), base64::decode(_creds.scram.storedKey), _authMessage)) { diff --git a/src/mongo/db/auth/user.h b/src/mongo/db/auth/user.h index d4aea7e442b..cdef6e52ddd 100644 --- a/src/mongo/db/auth/user.h +++ b/src/mongo/db/auth/user.h @@ -66,6 +66,17 @@ public: std::string salt; std::string serverKey; std::string storedKey; + + bool isValid() const { + // 160bit -> 20octets -> * 4/3 -> 26.667 -> padded to 28 + const size_t kEncodedSHA1Length = 28; + // 128bit -> 16octets -> * 4/3 -> 21.333 -> padded to 24 + const size_t kEncodedSaltLength = 24; + + return (salt.size() == kEncodedSaltLength) && + (serverKey.size() == kEncodedSHA1Length) && + (storedKey.size() == kEncodedSHA1Length); + } }; struct CredentialData { CredentialData() : password(""), scram(), isExternal(false) {} diff --git a/src/mongo/db/auth/user_document_parser_test.cpp b/src/mongo/db/auth/user_document_parser_test.cpp index 273eaff86f5..9ab688750d8 100644 --- a/src/mongo/db/auth/user_document_parser_test.cpp +++ b/src/mongo/db/auth/user_document_parser_test.cpp @@ -434,6 +434,7 @@ TEST_F(V2UserDocumentParsing, V2CredentialExtraction) { << BSON("MONGODB-CR" << "a")))); ASSERT(user->getCredentials().password == "a"); + ASSERT(!user->getCredentials().scram.isValid()); ASSERT(!user->getCredentials().isExternal); // Credentials are {external:true if users's db is $external @@ -446,6 +447,7 @@ TEST_F(V2UserDocumentParsing, V2CredentialExtraction) { << "credentials" << BSON("external" << true)))); ASSERT(user->getCredentials().password.empty()); + ASSERT(!user->getCredentials().scram.isValid()); ASSERT(user->getCredentials().isExternal); } diff --git a/src/mongo/db/bson/dotted_path_support.cpp b/src/mongo/db/bson/dotted_path_support.cpp index 1a9fda88501..31a92a5c54c 100644 --- a/src/mongo/db/bson/dotted_path_support.cpp +++ b/src/mongo/db/bson/dotted_path_support.cpp @@ -176,7 +176,7 @@ void extractAllElementsAlongPath(const BSONObj& obj, void extractAllElementsAlongPath(const BSONObj& obj, StringData path, - BSONElementMSet& elements, + BSONElementMultiSet& elements, bool expandArrayOnTrailingField, std::set* arrayComponents) { const size_t initialDepth = 0; diff --git a/src/mongo/db/bson/dotted_path_support.h b/src/mongo/db/bson/dotted_path_support.h index c9c6676c75a..0a3d072c56a 100644 --- a/src/mongo/db/bson/dotted_path_support.h +++ b/src/mongo/db/bson/dotted_path_support.h @@ -31,6 +31,7 @@ #include #include +#include "mongo/bson/bsonelement_comparator_interface.h" #include "mongo/bson/bsonobj.h" namespace mongo { @@ -104,7 +105,7 @@ void extractAllElementsAlongPath(const BSONObj& obj, void extractAllElementsAlongPath(const BSONObj& obj, StringData path, - BSONElementMSet& elements, + BSONElementMultiSet& elements, bool expandArrayOnTrailingField = true, std::set* arrayComponents = nullptr); diff --git a/src/mongo/db/catalog/apply_ops.cpp b/src/mongo/db/catalog/apply_ops.cpp index 672a37a10c7..ec5c7681f0e 100644 --- a/src/mongo/db/catalog/apply_ops.cpp +++ b/src/mongo/db/catalog/apply_ops.cpp @@ -32,6 +32,7 @@ #include "mongo/db/catalog/apply_ops.h" +#include "mongo/bson/util/bson_extract.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/database.h" #include "mongo/db/catalog/database_holder.h" @@ -42,21 +43,31 @@ #include "mongo/db/curop.h" #include "mongo/db/db_raii.h" #include "mongo/db/dbdirectclient.h" +#include "mongo/db/index/index_descriptor.h" #include "mongo/db/matcher/extensions_callback_disallow_extensions.h" #include "mongo/db/matcher/matcher.h" #include "mongo/db/op_observer.h" #include "mongo/db/operation_context.h" +#include "mongo/db/query/collation/collation_spec.h" #include "mongo/db/repl/oplog.h" #include "mongo/db/repl/replication_coordinator_global.h" #include "mongo/db/service_context.h" +#include "mongo/rpc/get_status_from_command_result.h" +#include "mongo/util/fail_point_service.h" #include "mongo/util/log.h" namespace mongo { namespace { + +const auto kPreconditionFieldName = "preCondition"_sd; + +// If enabled, causes loop in _applyOps() to hang after applying current operation. +MONGO_FP_DECLARE(applyOpsPauseBetweenOperations); + /** * Return true iff the applyOpsCmd can be executed in a single WriteUnitOfWork. */ -bool canBeAtomic(const BSONObj& applyOpCmd) { +bool _areOpsCrudOnly(const BSONObj& applyOpCmd) { for (const auto& elem : applyOpCmd.firstElement().Obj()) { const char* names[] = {"ns", "op"}; BSONElement fields[2]; @@ -89,14 +100,11 @@ bool canBeAtomic(const BSONObj& applyOpCmd) { return true; } -Status _applyOps(OperationContext* txn, +Status _applyOps(OperationContext* opCtx, const std::string& dbName, const BSONObj& applyOpCmd, BSONObjBuilder* result, int* numApplied) { - dassert(txn->lockState()->isLockHeldForMode( - ResourceId(RESOURCE_GLOBAL, ResourceId::SINGLETON_GLOBAL), MODE_X)); - BSONObj ops = applyOpCmd.firstElement().Obj(); // apply @@ -107,148 +115,169 @@ Status _applyOps(OperationContext* txn, BSONArrayBuilder ab; const bool alwaysUpsert = applyOpCmd.hasField("alwaysUpsert") ? applyOpCmd["alwaysUpsert"].trueValue() : true; - const bool haveWrappingWUOW = txn->lockState()->inAWriteUnitOfWork(); - - { - repl::UnreplicatedWritesBlock uwb(txn); + const bool haveWrappingWUOW = opCtx->lockState()->inAWriteUnitOfWork(); - while (i.more()) { - BSONElement e = i.next(); - const BSONObj& opObj = e.Obj(); + while (i.more()) { + BSONElement e = i.next(); + const BSONObj& opObj = e.Obj(); - // Ignore 'n' operations. - const char* opType = opObj["op"].valuestrsafe(); - if (*opType == 'n') - continue; + // Ignore 'n' operations. + const char* opType = opObj["op"].valuestrsafe(); + if (*opType == 'n') + continue; - const std::string ns = opObj["ns"].String(); + const NamespaceString nss(opObj["ns"].String()); - // Need to check this here, or OldClientContext may fail an invariant. - if (*opType != 'c' && !NamespaceString(ns).isValid()) - return {ErrorCodes::InvalidNamespace, "invalid ns: " + ns}; + // Need to check this here, or OldClientContext may fail an invariant. + if (*opType != 'c' && !nss.isValid()) + return {ErrorCodes::InvalidNamespace, "invalid ns: " + nss.ns()}; - Status status(ErrorCodes::InternalError, ""); + Status status(ErrorCodes::InternalError, ""); - if (haveWrappingWUOW) { - invariant(*opType != 'c'); + if (haveWrappingWUOW) { + invariant(opCtx->lockState()->isW()); + invariant(*opType != 'c'); + auto db = dbHolder().get(opCtx, nss.ns()); + if (!db) { + throw DBException( + "cannot create a database in atomic applyOps mode; will retry without " + "atomicity", + ErrorCodes::NamespaceNotFound); + } - if (!dbHolder().get(txn, ns)) { - throw DBException( - "cannot create a database in atomic applyOps mode; will retry without " - "atomicity", - ErrorCodes::NamespaceNotFound); - } + // When processing an update on a non-existent collection, applyOperation_inlock() + // returns UpdateOperationFailed on updates and allows the collection to be + // implicitly created on upserts. We detect both cases here and fail early with + // NamespaceNotFound. + auto collection = db->getCollection(nss); + if (!collection && !nss.isSystemDotIndexes() && (*opType == 'i' || *opType == 'u')) { + throw DBException(str::stream() << "cannot apply insert or update operation on " + "a non-existent namespace " + << nss.ns() + << ": " + << redact(opObj), + ErrorCodes::NamespaceNotFound); + } - OldClientContext ctx(txn, ns); - status = repl::applyOperation_inlock(txn, ctx.db(), opObj, alwaysUpsert); - if (!status.isOK()) - return status; - logOpForDbHash(txn, ns.c_str()); - } else { - try { - // Run operations under a nested lock as a hack to prevent yielding. - // - // The list of operations is supposed to be applied atomically; yielding - // would break atomicity by allowing an interruption or a shutdown to occur - // after only some operations are applied. We are already locked globally - // at this point, so taking a DBLock on the namespace creates a nested lock, - // and yields are disallowed for operations that hold a nested lock. - // - // We do not have a wrapping WriteUnitOfWork so it is possible for a journal - // commit to happen with a subset of ops applied. - Lock::GlobalWrite globalWriteLockDisallowTempRelease(txn->lockState()); - - // Ensures that yielding will not happen (see the comment above). - DEV { - Locker::LockSnapshot lockSnapshot; - invariant(!txn->lockState()->saveLockStateAndUnlock(&lockSnapshot)); - }; - - MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { - if (*opType == 'c') { - status = repl::applyCommand_inlock(txn, opObj, true); + OldClientContext ctx(opCtx, nss.ns()); + status = repl::applyOperation_inlock(opCtx, ctx.db(), opObj, alwaysUpsert); + if (!status.isOK()) + return status; + logOpForDbHash(opCtx, nss.ns().c_str()); + } else { + try { + MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { + if (*opType == 'c') { + invariant(opCtx->lockState()->isW()); + uassertStatusOK(status = repl::applyCommand_inlock(opCtx, opObj, true)); + } else { + const char* names[] = {"o", "ns"}; + BSONElement fields[2]; + opObj.getFields(2, names, fields); + BSONElement& fieldO = fields[0]; + BSONElement& fieldNs = fields[1]; + const StringData ns = fieldNs.valueStringData(); + NamespaceString requestNss{ns}; + + if (nss.isSystemDotIndexes()) { + BSONObj indexSpec; + NamespaceString indexNss; + std::tie(indexSpec, indexNss) = + repl::prepForApplyOpsIndexInsert(fieldO, opObj, requestNss); + if (!indexSpec["collation"]) { + // If the index spec does not include a collation, explicitly + // specify the simple collation, so the index does not inherit the + // collection default collation. + auto indexVersion = indexSpec["v"]; + // The index version is populated by prepForApplyOpsIndexInsert(). + invariant(indexVersion); + if (indexVersion.isNumber() && + (indexVersion.numberInt() >= + static_cast(IndexDescriptor::IndexVersion::kV2))) { + BSONObjBuilder bob; + bob.append("collation", CollationSpec::kSimpleSpec); + bob.appendElements(indexSpec); + indexSpec = bob.obj(); + } + } + BSONObjBuilder command; + command.append("createIndexes", indexNss.coll()); + { + BSONArrayBuilder indexes(command.subarrayStart("indexes")); + indexes.append(indexSpec); + indexes.doneFast(); + } + const BSONObj commandObj = command.done(); + + DBDirectClient client(opCtx); + BSONObj infoObj; + client.runCommand(nsToDatabase(ns), commandObj, infoObj); + status = getStatusFromCommandResult(infoObj); } else { - OldClientContext ctx(txn, ns); - - status = - repl::applyOperation_inlock(txn, ctx.db(), opObj, alwaysUpsert); + AutoGetCollection autoColl(opCtx, nss, MODE_IX); + if (!autoColl.getCollection() && !nss.isSystemDotIndexes()) { + // For idempotency reasons, return success on delete operations. + if (*opType == 'd') { + status = Status::OK(); + } else { + throw DBException( + str::stream() + << "cannot apply insert or update operation on" + " a non-existent namespace " + << nss.ns() + << ": " + << mongo::redact(opObj), + ErrorCodes::NamespaceNotFound); + } + } else { + OldClientContext ctx(opCtx, nss.ns()); + status = repl::applyOperation_inlock( + opCtx, ctx.db(), opObj, alwaysUpsert); + } } } - MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "applyOps", ns); - } catch (const DBException& ex) { - ab.append(false); - result->append("applied", ++(*numApplied)); - result->append("code", ex.getCode()); - result->append("codeName", - ErrorCodes::errorString(ErrorCodes::fromInt(ex.getCode()))); - result->append("errmsg", ex.what()); - result->append("results", ab.arr()); - return Status(ErrorCodes::UnknownError, ex.what()); } - WriteUnitOfWork wuow(txn); - logOpForDbHash(txn, ns.c_str()); - wuow.commit(); + MONGO_WRITE_CONFLICT_RETRY_LOOP_END(opCtx, "applyOps", nss.ns()); + } catch (const DBException& ex) { + ab.append(false); + result->append("applied", ++(*numApplied)); + result->append("code", ex.getCode()); + result->append("codeName", + ErrorCodes::errorString(ErrorCodes::fromInt(ex.getCode()))); + result->append("errmsg", ex.what()); + result->append("results", ab.arr()); + return ex.toStatus(); } - - ab.append(status.isOK()); - if (!status.isOK()) { - log() << "applyOps error applying: " << status; - errors++; - } - - (*numApplied)++; + WriteUnitOfWork wuow(opCtx); + logOpForDbHash(opCtx, nss.ns().c_str()); + wuow.commit(); } - result->append("applied", *numApplied); - result->append("results", ab.arr()); - } // set replicatedWrites back to original value - - if (txn->writesAreReplicated()) { - // We want this applied atomically on slaves - // so we re-wrap without the pre-condition for speed - - std::string tempNS = str::stream() << dbName << ".$cmd"; - - // TODO: possibly use mutable BSON to remove preCondition field - // once it is available - BSONObjBuilder cmdBuilder; - - for (auto elem : applyOpCmd) { - auto name = elem.fieldNameStringData(); - if (name == "preCondition") - continue; - if (name == "bypassDocumentValidation") - continue; - cmdBuilder.append(elem); + ab.append(status.isOK()); + if (!status.isOK()) { + log() << "applyOps error applying: " << status; + errors++; } - const BSONObj cmdRewritten = cmdBuilder.done(); - - auto opObserver = getGlobalServiceContext()->getOpObserver(); - invariant(opObserver); - if (haveWrappingWUOW) { - opObserver->onApplyOps(txn, tempNS, cmdRewritten); - } else { - // When executing applyOps outside of a wrapping WriteUnitOfWOrk, always logOp the - // command regardless of whether the individial ops succeeded and rely on any - // failures to also on secondaries. This isn't perfect, but it's what the command - // has always done and is part of its "correct" behavior. - while (true) { - try { - WriteUnitOfWork wunit(txn); - opObserver->onApplyOps(txn, tempNS, cmdRewritten); - - wunit.commit(); - break; - } catch (const WriteConflictException& wce) { - LOG(2) << "WriteConflictException while logging applyOps command, retrying."; - txn->recoveryUnit()->abandonSnapshot(); - continue; - } + (*numApplied)++; + + if (MONGO_FAIL_POINT(applyOpsPauseBetweenOperations)) { + // While holding a database lock under MMAPv1, we would be implicitly holding the + // flush lock here. This would prevent other threads from acquiring the global + // lock or any database locks. We release all locks temporarily while the fail + // point is enabled to allow other threads to make progress. + boost::optional release; + auto storageEngine = opCtx->getServiceContext()->getGlobalStorageEngine(); + if (storageEngine->isMmapV1() && !opCtx->lockState()->isW()) { + release.emplace(opCtx->lockState()); } + MONGO_FAIL_POINT_PAUSE_WHILE_SET(applyOpsPauseBetweenOperations); } } + result->append("applied", *numApplied); + result->append("results", ab.arr()); + if (errors != 0) { return Status(ErrorCodes::UnknownError, "applyOps had one or more errors applying ops"); } @@ -256,94 +285,144 @@ Status _applyOps(OperationContext* txn, return Status::OK(); } -Status preconditionOK(OperationContext* txn, const BSONObj& applyOpCmd, BSONObjBuilder* result) { - dassert(txn->lockState()->isLockHeldForMode( - ResourceId(RESOURCE_GLOBAL, ResourceId::SINGLETON_GLOBAL), MODE_X)); - - if (applyOpCmd["preCondition"].type() == Array) { - BSONObjIterator i(applyOpCmd["preCondition"].Obj()); - while (i.more()) { - BSONObj preCondition = i.next().Obj(); - if (preCondition["ns"].type() != BSONType::String) { - return {ErrorCodes::InvalidNamespace, - str::stream() << "ns in preCondition must be a string, but found type: " - << typeName(preCondition["ns"].type())}; - } - const NamespaceString nss(preCondition["ns"].valueStringData()); - if (!nss.isValid()) { - return {ErrorCodes::InvalidNamespace, "invalid ns: " + nss.ns()}; - } +bool _hasPrecondition(const BSONObj& applyOpCmd) { + return applyOpCmd[kPreconditionFieldName].type() == Array; +} - DBDirectClient db(txn); - BSONObj realres = db.findOne(nss.ns(), preCondition["q"].Obj()); +Status _checkPrecondition(OperationContext* opCtx, + const BSONObj& applyOpCmd, + BSONObjBuilder* result) { + invariant(opCtx->lockState()->isW()); + invariant(_hasPrecondition(applyOpCmd)); + + for (auto elem : applyOpCmd[kPreconditionFieldName].Obj()) { + auto preCondition = elem.Obj(); + if (preCondition["ns"].type() != BSONType::String) { + return {ErrorCodes::InvalidNamespace, + str::stream() << "ns in preCondition must be a string, but found type: " + << typeName(preCondition["ns"].type())}; + } + const NamespaceString nss(preCondition["ns"].valueStringData()); + if (!nss.isValid()) { + return {ErrorCodes::InvalidNamespace, "invalid ns: " + nss.ns()}; + } - // Get collection default collation. - Database* database = dbHolder().get(txn, nss.db()); - if (!database) { - return {ErrorCodes::NamespaceNotFound, - "database in ns does not exist: " + nss.ns()}; - } - Collection* collection = database->getCollection(nss.ns()); - if (!collection) { - return {ErrorCodes::NamespaceNotFound, - "collection in ns does not exist: " + nss.ns()}; - } - const CollatorInterface* collator = collection->getDefaultCollator(); - - // Apply-ops would never have a $where/$text matcher. Using the "DisallowExtensions" - // callback ensures that parsing will throw an error if $where or $text are found. - Matcher matcher( - preCondition["res"].Obj(), ExtensionsCallbackDisallowExtensions(), collator); - if (!matcher.matches(realres)) { - result->append("got", realres); - result->append("whatFailed", preCondition); - return {ErrorCodes::BadValue, "preCondition failed"}; - } + DBDirectClient db(opCtx); + BSONObj realres = db.findOne(nss.ns(), preCondition["q"].Obj()); + + // Get collection default collation. + Database* database = dbHolder().get(opCtx, nss.db()); + if (!database) { + return {ErrorCodes::NamespaceNotFound, "database in ns does not exist: " + nss.ns()}; + } + Collection* collection = database->getCollection(nss.ns()); + if (!collection) { + return {ErrorCodes::NamespaceNotFound, "collection in ns does not exist: " + nss.ns()}; + } + const CollatorInterface* collator = collection->getDefaultCollator(); + + // Apply-ops would never have a $where/$text matcher. Using the "DisallowExtensions" + // callback ensures that parsing will throw an error if $where or $text are found. + Matcher matcher( + preCondition["res"].Obj(), ExtensionsCallbackDisallowExtensions(), collator); + if (!matcher.matches(realres)) { + result->append("got", realres); + result->append("whatFailed", preCondition); + return {ErrorCodes::BadValue, "preCondition failed"}; } } + return Status::OK(); } } // namespace -Status applyOps(OperationContext* txn, +Status applyOps(OperationContext* opCtx, const std::string& dbName, const BSONObj& applyOpCmd, BSONObjBuilder* result) { - ScopedTransaction scopedXact(txn, MODE_X); - Lock::GlobalWrite globalWriteLock(txn->lockState()); + bool allowAtomic = false; + uassertStatusOK( + bsonExtractBooleanFieldWithDefault(applyOpCmd, "allowAtomic", true, &allowAtomic)); + auto areOpsCrudOnly = _areOpsCrudOnly(applyOpCmd); + auto isAtomic = allowAtomic && areOpsCrudOnly; + auto hasPrecondition = _hasPrecondition(applyOpCmd); + + ScopedTransaction scopedXact(opCtx, MODE_X); + boost::optional globalWriteLock; + boost::optional dbWriteLock; + + // There's only one case where we are allowed to take the database lock instead of the global + // lock - no preconditions; only CRUD ops; and non-atomic mode. + if (!hasPrecondition && areOpsCrudOnly && !allowAtomic) { + dbWriteLock.emplace(opCtx->lockState(), dbName, MODE_IX); + } else { + globalWriteLock.emplace(opCtx->lockState()); + } - bool userInitiatedWritesAndNotPrimary = txn->writesAreReplicated() && + bool userInitiatedWritesAndNotPrimary = opCtx->writesAreReplicated() && !repl::getGlobalReplicationCoordinator()->canAcceptWritesForDatabase(dbName); if (userInitiatedWritesAndNotPrimary) return Status(ErrorCodes::NotMaster, str::stream() << "Not primary while applying ops to database " << dbName); - Status preconditionStatus = preconditionOK(txn, applyOpCmd, result); - if (!preconditionStatus.isOK()) { - return preconditionStatus; + if (hasPrecondition) { + auto status = _checkPrecondition(opCtx, applyOpCmd, result); + if (!status.isOK()) { + return status; + } } int numApplied = 0; - if (!canBeAtomic(applyOpCmd)) - return _applyOps(txn, dbName, applyOpCmd, result, &numApplied); + if (!isAtomic) + return _applyOps(opCtx, dbName, applyOpCmd, result, &numApplied); // Perform write ops atomically + invariant(globalWriteLock); try { MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { BSONObjBuilder intermediateResult; - WriteUnitOfWork wunit(txn); + WriteUnitOfWork wunit(opCtx); numApplied = 0; - uassertStatusOK(_applyOps(txn, dbName, applyOpCmd, &intermediateResult, &numApplied)); + { + // Suppress replication for atomic operations until end of applyOps. + repl::UnreplicatedWritesBlock uwb(opCtx); + uassertStatusOK( + _applyOps(opCtx, dbName, applyOpCmd, &intermediateResult, &numApplied)); + } + // Generate oplog entry for all atomic ops collectively. + if (opCtx->writesAreReplicated()) { + // We want this applied atomically on slaves so we rewrite the oplog entry without + // the pre-condition for speed. + + std::string tempNS = str::stream() << dbName << ".$cmd"; + + BSONObjBuilder cmdBuilder; + + for (auto elem : applyOpCmd) { + auto name = elem.fieldNameStringData(); + if (name == kPreconditionFieldName) + continue; + if (name == "bypassDocumentValidation") + continue; + cmdBuilder.append(elem); + } + + const BSONObj cmdRewritten = cmdBuilder.done(); + + auto opObserver = getGlobalServiceContext()->getOpObserver(); + invariant(opObserver); + opObserver->onApplyOps(opCtx, tempNS, cmdRewritten); + } wunit.commit(); result->appendElements(intermediateResult.obj()); } - MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "applyOps", dbName); + MONGO_WRITE_CONFLICT_RETRY_LOOP_END(opCtx, "applyOps", dbName); } catch (const DBException& ex) { if (ex.getCode() == ErrorCodes::NamespaceNotFound) { // Retry in non-atomic mode, since MMAP cannot implicitly create a new database // within an active WriteUnitOfWork. - return _applyOps(txn, dbName, applyOpCmd, result, &numApplied); + return _applyOps(opCtx, dbName, applyOpCmd, result, &numApplied); } BSONArrayBuilder ab; ++numApplied; diff --git a/src/mongo/db/catalog/apply_ops.h b/src/mongo/db/catalog/apply_ops.h index 588d3bb370b..3a742891573 100644 --- a/src/mongo/db/catalog/apply_ops.h +++ b/src/mongo/db/catalog/apply_ops.h @@ -37,7 +37,7 @@ class OperationContext; * Applies ops contained in "applyOpCmd" and populates fields in "result" to be returned to the * user. */ -Status applyOps(OperationContext* txn, +Status applyOps(OperationContext* opCtx, const std::string& dbName, const BSONObj& applyOpCmd, BSONObjBuilder* result); diff --git a/src/mongo/db/catalog/database.cpp b/src/mongo/db/catalog/database.cpp index 9e24afd5ffb..23fdfb84e95 100644 --- a/src/mongo/db/catalog/database.cpp +++ b/src/mongo/db/catalog/database.cpp @@ -475,8 +475,8 @@ Status Database::renameCollection(OperationContext* txn, Top::get(txn->getClient()->getServiceContext()).collectionDropped(fromNS.toString()); } - txn->recoveryUnit()->registerChange(new AddCollectionChange(txn, this, toNS)); Status s = _dbEntry->renameCollection(txn, fromNS, toNS, stayTemp); + txn->recoveryUnit()->registerChange(new AddCollectionChange(txn, this, toNS)); _collections[toNS] = _getOrCreateCollectionInstance(txn, toNS); return s; } diff --git a/src/mongo/db/catalog/rename_collection.cpp b/src/mongo/db/catalog/rename_collection.cpp index d9d25a951dc..f5f62ce9636 100644 --- a/src/mongo/db/catalog/rename_collection.cpp +++ b/src/mongo/db/catalog/rename_collection.cpp @@ -71,7 +71,15 @@ Status renameCollection(OperationContext* txn, DisableDocumentValidation validationDisabler(txn); ScopedTransaction transaction(txn, MODE_X); - Lock::GlobalWrite globalWriteLock(txn->lockState()); + boost::optional globalWriteLock; + boost::optional dbWriteLock; + + // If the rename is known not to be a cross-database rename, just a database lock suffices. + if (source.db() == target.db()) + dbWriteLock.emplace(txn->lockState(), source.db(), MODE_X); + else + globalWriteLock.emplace(txn->lockState()); + // We stay in source context the whole time. This is mostly to set the CurOp namespace. OldClientContext ctx(txn, source.ns()); diff --git a/src/mongo/db/commands/apply_ops_cmd.cpp b/src/mongo/db/commands/apply_ops_cmd.cpp index c252b8fd073..262e8ef93d3 100644 --- a/src/mongo/db/commands/apply_ops_cmd.cpp +++ b/src/mongo/db/commands/apply_ops_cmd.cpp @@ -116,22 +116,17 @@ public: } } - auto client = txn->getClient(); - auto lastOpAtOperationStart = repl::ReplClientInfo::forClient(client).getLastOp(); - ScopeGuard lastOpSetterGuard = - MakeObjGuard(repl::ReplClientInfo::forClient(client), - &repl::ReplClientInfo::setLastOpToSystemLastOpTime, - txn); - + // TODO (SERVER-30217): When a write concern is provided to the applyOps command, we + // normally wait on the OpTime of whichever operation successfully completed last. This is + // erroneous, however, if the last operation in the array happens to be a write no-op and + // thus isn’t assigned an OpTime. Let the second to last operation in the applyOps be write + // A, the last operation in applyOps be write B. Let B do a no-op write and let the + // operation that caused B to be a no-op be C. If C has an OpTime after A but before B, + // then we won’t wait for C to be replicated and it could be rolled back, even though B + // was acknowledged. To fix this, we should wait for replication of the node’s last applied + // OpTime if the last write operation was a no-op write. auto applyOpsStatus = appendCommandStatus(result, applyOps(txn, dbname, cmdObj, &result)); - if (repl::ReplClientInfo::forClient(client).getLastOp() != lastOpAtOperationStart) { - // If this operation has already generated a new lastOp, don't bother setting it - // here. No-op applyOps will not generate a new lastOp, so we still need the guard to - // fire in that case. - lastOpSetterGuard.Dismiss(); - } - return applyOpsStatus; } diff --git a/src/mongo/db/commands/create_indexes.cpp b/src/mongo/db/commands/create_indexes.cpp index 489e080f7a6..ee0133da46b 100644 --- a/src/mongo/db/commands/create_indexes.cpp +++ b/src/mongo/db/commands/create_indexes.cpp @@ -290,12 +290,6 @@ public: const int numIndexesBefore = collection->getIndexCatalog()->numIndexesTotal(txn); result.append("numIndexesBefore", numIndexesBefore); - auto client = txn->getClient(); - ScopeGuard lastOpSetterGuard = - MakeObjGuard(repl::ReplClientInfo::forClient(client), - &repl::ReplClientInfo::setLastOpToSystemLastOpTime, - txn); - MultiIndexBlock indexer(txn, collection); indexer.allowBackgroundBuilding(); indexer.allowInterruption(); @@ -403,8 +397,6 @@ public: result.append("numIndexesAfter", collection->getIndexCatalog()->numIndexesTotal(txn)); - lastOpSetterGuard.Dismiss(); - return true; } diff --git a/src/mongo/db/commands/dbcommands.cpp b/src/mongo/db/commands/dbcommands.cpp index 37bcf783d47..38eae01ce9d 100644 --- a/src/mongo/db/commands/dbcommands.cpp +++ b/src/mongo/db/commands/dbcommands.cpp @@ -61,6 +61,7 @@ #include "mongo/db/commands.h" #include "mongo/db/commands/server_status.h" #include "mongo/db/commands/shutdown.h" +#include "mongo/db/concurrency/global_lock_acquisition_tracker.h" #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/db_raii.h" #include "mongo/db/dbdirectclient.h" @@ -1256,13 +1257,21 @@ namespace { void _waitForWriteConcernAndAddToCommandResponse(OperationContext* opCtx, const std::string& commandName, + const repl::OpTime& lastOpBeforeRun, BSONObjBuilder* commandResponseBuilder) { + auto lastOpAfterRun = repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp(); + + // Ensures that if we tried to do a write, we wait for write concern, even if that write was + // a noop. + if ((lastOpAfterRun == lastOpBeforeRun) && + GlobalLockAcquisitionTracker::get(opCtx).getGlobalExclusiveLockTaken()) { + repl::ReplClientInfo::forClient(opCtx->getClient()).setLastOpToSystemLastOpTime(opCtx); + lastOpAfterRun = repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp(); + } + WriteConcernResult res; auto waitForWCStatus = - waitForWriteConcern(opCtx, - repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp(), - opCtx->getWriteConcern(), - &res); + waitForWriteConcern(opCtx, lastOpAfterRun, opCtx->getWriteConcern(), &res); Command::appendCommandWCStatus(*commandResponseBuilder, waitForWCStatus, res); // SERVER-22421: This code is to ensure error response backwards compatibility with the @@ -1558,13 +1567,17 @@ bool Command::run(OperationContext* txn, return result; } + auto lastOpBeforeRun = repl::ReplClientInfo::forClient(txn->getClient()).getLastOp(); + // Change the write concern while running the command. const auto oldWC = txn->getWriteConcern(); ON_BLOCK_EXIT([&] { txn->setWriteConcern(oldWC); }); txn->setWriteConcern(wcResult.getValue()); - ON_BLOCK_EXIT( - [&] { _waitForWriteConcernAndAddToCommandResponse(txn, getName(), &inPlaceReplyBob); }); + ON_BLOCK_EXIT([&] { + _waitForWriteConcernAndAddToCommandResponse( + txn, getName(), lastOpBeforeRun, &inPlaceReplyBob); + }); result = run(txn, db, cmd, 0, errmsg, inPlaceReplyBob); diff --git a/src/mongo/db/commands/feature_compatibility_version.cpp b/src/mongo/db/commands/feature_compatibility_version.cpp index f354efa88d9..00c6de52a5c 100644 --- a/src/mongo/db/commands/feature_compatibility_version.cpp +++ b/src/mongo/db/commands/feature_compatibility_version.cpp @@ -248,12 +248,8 @@ void FeatureCompatibilityVersion::set(OperationContext* txn, StringData version) client.runCommand(nss.db().toString(), makeUpdateCommand(version, WriteConcernOptions::Majority), updateResult); - uassertStatusOK(getStatusFromCommandResult(updateResult)); - uassertStatusOK(getWriteConcernStatusFromCommandResult(updateResult)); + uassertStatusOK(getStatusFromWriteCommandReply(updateResult)); - // We then update the value of the featureCompatibilityVersion server parameter. - serverGlobalParams.featureCompatibility.version.store( - ServerGlobalParams::FeatureCompatibility::Version::k34); } else if (version == FeatureCompatibilityVersionCommandParser::kVersion32) { // We update the featureCompatibilityVersion document stored in the "admin.system.version" // collection. We do this before dropping the v=2 index in order to maintain the invariant @@ -262,7 +258,7 @@ void FeatureCompatibilityVersion::set(OperationContext* txn, StringData version) // concern to this update because we're going to do so anyway for the "dropIndexes" command. BSONObj updateResult; client.runCommand(nss.db().toString(), makeUpdateCommand(version, BSONObj()), updateResult); - uassertStatusOK(getStatusFromCommandResult(updateResult)); + uassertStatusOK(getStatusFromWriteCommandReply(updateResult)); // We then drop the v=2 index on the "admin.system.version" collection to enable 3.2 // secondaries to sync from this mongod. @@ -278,10 +274,6 @@ void FeatureCompatibilityVersion::set(OperationContext* txn, StringData version) uassertStatusOK(status); } uassertStatusOK(getWriteConcernStatusFromCommandResult(dropIndexesResult)); - - // We then update the value of the featureCompatibilityVersion server parameter. - serverGlobalParams.featureCompatibility.version.store( - ServerGlobalParams::FeatureCompatibility::Version::k32); } } @@ -323,7 +315,8 @@ void FeatureCompatibilityVersion::setIfCleanStartup(OperationContext* txn, } // We then insert the featureCompatibilityVersion document into the "admin.system.version" - // collection. We do this after creating the v=2 index in order to maintain the invariant + // collection. The server parameter will be updated on commit by the op observer. + // We do this after creating the v=2 index in order to maintain the invariant // that if the featureCompatibilityVersion is 3.4, then 'k32IncompatibleIndexSpec' index // exists on the "admin.system.version" collection. If we happened to fail to insert the // document when starting up, then on a subsequent start-up we'd no longer consider the data @@ -334,14 +327,10 @@ void FeatureCompatibilityVersion::setIfCleanStartup(OperationContext* txn, BSON("_id" << FeatureCompatibilityVersion::kParameterName << FeatureCompatibilityVersion::kVersionField << FeatureCompatibilityVersionCommandParser::kVersion34))); - - // We then update the value of the featureCompatibilityVersion server parameter. - serverGlobalParams.featureCompatibility.version.store( - ServerGlobalParams::FeatureCompatibility::Version::k34); } } -void FeatureCompatibilityVersion::onInsertOrUpdate(const BSONObj& doc) { +void FeatureCompatibilityVersion::onInsertOrUpdate(OperationContext* opCtx, const BSONObj& doc) { auto idElement = doc["_id"]; if (idElement.type() != BSONType::String || idElement.String() != FeatureCompatibilityVersion::kParameterName) { @@ -350,10 +339,11 @@ void FeatureCompatibilityVersion::onInsertOrUpdate(const BSONObj& doc) { auto newVersion = uassertStatusOK(FeatureCompatibilityVersion::parse(doc)); log() << "setting featureCompatibilityVersion to " << getFeatureCompatibilityVersionString(newVersion); - serverGlobalParams.featureCompatibility.version.store(newVersion); + opCtx->recoveryUnit()->onCommit( + [newVersion]() { serverGlobalParams.featureCompatibility.version.store(newVersion); }); } -void FeatureCompatibilityVersion::onDelete(const BSONObj& doc) { +void FeatureCompatibilityVersion::onDelete(OperationContext* opCtx, const BSONObj& doc) { auto idElement = doc["_id"]; if (idElement.type() != BSONType::String || idElement.String() != FeatureCompatibilityVersion::kParameterName) { @@ -361,15 +351,19 @@ void FeatureCompatibilityVersion::onDelete(const BSONObj& doc) { } log() << "setting featureCompatibilityVersion to " << FeatureCompatibilityVersionCommandParser::kVersion32; - serverGlobalParams.featureCompatibility.version.store( - ServerGlobalParams::FeatureCompatibility::Version::k32); + opCtx->recoveryUnit()->onCommit([]() { + serverGlobalParams.featureCompatibility.version.store( + ServerGlobalParams::FeatureCompatibility::Version::k32); + }); } -void FeatureCompatibilityVersion::onDropCollection() { +void FeatureCompatibilityVersion::onDropCollection(OperationContext* opCtx) { log() << "setting featureCompatibilityVersion to " << FeatureCompatibilityVersionCommandParser::kVersion32; - serverGlobalParams.featureCompatibility.version.store( - ServerGlobalParams::FeatureCompatibility::Version::k32); + opCtx->recoveryUnit()->onCommit([]() { + serverGlobalParams.featureCompatibility.version.store( + ServerGlobalParams::FeatureCompatibility::Version::k32); + }); } /** diff --git a/src/mongo/db/commands/feature_compatibility_version.h b/src/mongo/db/commands/feature_compatibility_version.h index 4bcb4b56e55..8e7969c2e70 100644 --- a/src/mongo/db/commands/feature_compatibility_version.h +++ b/src/mongo/db/commands/feature_compatibility_version.h @@ -76,21 +76,22 @@ public: /** * Examines a document inserted or updated in admin.system.version. If it is the - * featureCompatibilityVersion document, validates the document and updates the server - * parameter. + * featureCompatibilityVersion document, validates the document and on commit, updates + * the server parameter. */ - static void onInsertOrUpdate(const BSONObj& doc); + static void onInsertOrUpdate(OperationContext* opCtx, const BSONObj& doc); /** * Examines the _id of a document removed from admin.system.version. If it is the * featureCompatibilityVersion document, resets the server parameter to its default value (3.2). + * on commit. */ - static void onDelete(const BSONObj& doc); + static void onDelete(OperationContext* opCtx, const BSONObj& doc); /** - * Resets the server parameter to its default value (3.2). + * Resets the server parameter to its default value (3.2) on commit. */ - static void onDropCollection(); + static void onDropCollection(OperationContext* opCtx); }; } // namespace mongo diff --git a/src/mongo/db/commands/find_and_modify.cpp b/src/mongo/db/commands/find_and_modify.cpp index 79ee34203ce..61dbc4f6509 100644 --- a/src/mongo/db/commands/find_and_modify.cpp +++ b/src/mongo/db/commands/find_and_modify.cpp @@ -364,18 +364,6 @@ public: if (shouldBypassDocumentValidationForCommand(cmdObj)) maybeDisableValidation.emplace(txn); - auto client = txn->getClient(); - auto lastOpAtOperationStart = repl::ReplClientInfo::forClient(client).getLastOp(); - ScopeGuard lastOpSetterGuard = - MakeObjGuard(repl::ReplClientInfo::forClient(client), - &repl::ReplClientInfo::setLastOpToSystemLastOpTime, - txn); - - // If this is the local database, don't set last op. - if (dbName == "local") { - lastOpSetterGuard.Dismiss(); - } - auto curOp = CurOp::get(txn); OpDebug* opDebug = &curOp->debug(); @@ -567,13 +555,6 @@ public: } MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "findAndModify", nsString.ns()); - if (repl::ReplClientInfo::forClient(client).getLastOp() != lastOpAtOperationStart) { - // If this operation has already generated a new lastOp, don't bother setting it here. - // No-op updates will not generate a new lastOp, so we still need the guard to fire in - // that case. - lastOpSetterGuard.Dismiss(); - } - return true; } diff --git a/src/mongo/db/commands/mr.cpp b/src/mongo/db/commands/mr.cpp index 4aeed6d5978..136656b8dd2 100644 --- a/src/mongo/db/commands/mr.cpp +++ b/src/mongo/db/commands/mr.cpp @@ -314,7 +314,7 @@ Config::Config(const string& _dbname, const BSONObj& cmdObj) { // scope and code if (cmdObj["scope"].type() == Object) - scopeSetup = cmdObj["scope"].embeddedObjectUserCheck(); + scopeSetup = cmdObj["scope"].embeddedObjectUserCheck().getOwned(); mapper.reset(new JSMapper(cmdObj["map"])); reducer.reset(new JSReducer(cmdObj["reduce"])); @@ -322,7 +322,7 @@ Config::Config(const string& _dbname, const BSONObj& cmdObj) { finalizer.reset(new JSFinalizer(cmdObj["finalize"])); if (cmdObj["mapparams"].type() == Array) { - mapParams = cmdObj["mapparams"].embeddedObjectUserCheck(); + mapParams = cmdObj["mapparams"].embeddedObjectUserCheck().getOwned(); } } @@ -836,6 +836,7 @@ void State::init() { AuthorizationSession::get(Client::getCurrent())->getAuthenticatedUserNamesToken(); _scope.reset(getGlobalScriptEngine()->newScopeForCurrentThread()); _scope->registerOperation(_txn); + _scope->requireOwnedObjects(); _scope->setLocalDB(_config.dbname); _scope->loadStored(_txn, true); @@ -1522,6 +1523,7 @@ public: BSONObj o; PlanExecutor::ExecState execState; while (PlanExecutor::ADVANCED == (execState = exec->getNext(&o, NULL))) { + o = o.getOwned(); // we will be accessing outside of the lock // check to see if this is a new object we don't own yet // because of a chunk migration if (collMetadata) { diff --git a/src/mongo/db/commands/pipeline_command.cpp b/src/mongo/db/commands/pipeline_command.cpp index 6fe6de34381..03f8ec87e30 100644 --- a/src/mongo/db/commands/pipeline_command.cpp +++ b/src/mongo/db/commands/pipeline_command.cpp @@ -407,10 +407,6 @@ public: return appendCommandStatus(result, resolvedView.getStatus()); } - auto collationSpec = ctx.getView()->defaultCollator() - ? ctx.getView()->defaultCollator()->getSpec().toBSON().getOwned() - : CollationSpec::kSimpleSpec; - // With the view & collation resolved, we can relinquish locks. ctx.releaseLocksForView(); @@ -424,7 +420,6 @@ public: if (!newRequest.isOK()) { return appendCommandStatus(result, newRequest.getStatus()); } - newRequest.getValue().setCollation(collationSpec); bool status = runParsed( txn, origNss, newRequest.getValue(), newCmd.getValue(), errmsg, result); diff --git a/src/mongo/db/concurrency/SConscript b/src/mongo/db/concurrency/SConscript index 942b7a4f7f1..023d3b6b09e 100644 --- a/src/mongo/db/concurrency/SConscript +++ b/src/mongo/db/concurrency/SConscript @@ -12,6 +12,17 @@ env.Library( ] ) +env.Library( + target='global_lock_acquisition_tracker', + source=[ + 'global_lock_acquisition_tracker.cpp' + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/util/decorable', + ] +) + + env.Library( target='lock_manager', source=[ @@ -21,6 +32,7 @@ env.Library( 'lock_stats.cpp', ], LIBDEPS=[ + 'global_lock_acquisition_tracker', '$BUILD_DIR/mongo/util/background_job', '$BUILD_DIR/mongo/base', # Temporary crutch since the ssl cleanup is hard coded in background.cpp @@ -44,6 +56,7 @@ env.CppUnitTest( LIBDEPS=[ '$BUILD_DIR/mongo/db/service_context_noop_init', '$BUILD_DIR/mongo/util/progress_meter', + 'global_lock_acquisition_tracker', 'lock_manager', ] ) diff --git a/src/mongo/db/concurrency/d_concurrency.cpp b/src/mongo/db/concurrency/d_concurrency.cpp index f7dabcc2248..6e7bd1ee7d6 100644 --- a/src/mongo/db/concurrency/d_concurrency.cpp +++ b/src/mongo/db/concurrency/d_concurrency.cpp @@ -34,6 +34,7 @@ #include +#include "mongo/db/concurrency/global_lock_acquisition_tracker.h" #include "mongo/db/namespace_string.h" #include "mongo/db/server_parameters.h" #include "mongo/db/service_context.h" @@ -91,6 +92,13 @@ void Lock::GlobalLock::waitForLock(unsigned timeoutMs) { if (_result != LOCK_OK && _locker->shouldConflictWithSecondaryBatchApplication()) { _pbwm.unlock(); } + + if (_locker->isWriteLocked() && haveClient()) { + auto opCtx = cc().getOperationContext(); + if (opCtx) { + GlobalLockAcquisitionTracker::get(opCtx).setGlobalExclusiveLockTaken(); + } + } } void Lock::GlobalLock::_unlock() { diff --git a/src/mongo/db/concurrency/d_concurrency.h b/src/mongo/db/concurrency/d_concurrency.h index 6274814ca1c..b8df869089e 100644 --- a/src/mongo/db/concurrency/d_concurrency.h +++ b/src/mongo/db/concurrency/d_concurrency.h @@ -163,6 +163,9 @@ public: /** * Enqueues lock but does not block on lock acquisition. * Call waitForLock() to complete locking process. + * + * Does not set that the global lock was taken on the GlobalLockAcquisitionTracker. Call + * waitForLock to do so. */ GlobalLock(Locker* locker, LockMode lockMode, unsigned timeoutMs, EnqueueOnly enqueueOnly); @@ -171,7 +174,8 @@ public: } /** - * Waits for lock to be granted. + * Waits for lock to be granted. Sets that the global lock was taken on the + * GlobalLockAcquisitionTracker. */ void waitForLock(unsigned timeoutMs); diff --git a/src/mongo/db/concurrency/d_concurrency_test.cpp b/src/mongo/db/concurrency/d_concurrency_test.cpp index 22aae759650..799aae8a7e3 100644 --- a/src/mongo/db/concurrency/d_concurrency_test.cpp +++ b/src/mongo/db/concurrency/d_concurrency_test.cpp @@ -34,6 +34,7 @@ #include #include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/concurrency/global_lock_acquisition_tracker.h" #include "mongo/db/concurrency/lock_manager_test_help.h" #include "mongo/db/operation_context.h" #include "mongo/stdx/functional.h" @@ -80,12 +81,13 @@ private: class UseGlobalThrottling { public: explicit UseGlobalThrottling(OperationContext* opCtx, int numTickets) - : _opCtx(opCtx), _holder(1) { + : _opCtx(opCtx), _holder(numTickets) { _opCtx->lockState()->setGlobalThrottling(&_holder, &_holder); } - ~UseGlobalThrottling() { + ~UseGlobalThrottling() noexcept(false) { // Reset the global setting as we're about to destroy the ticket holder. _opCtx->lockState()->setGlobalThrottling(nullptr, nullptr); + ASSERT_EQ(_holder.used(), 0); } private: @@ -114,6 +116,16 @@ makeKClientsWithLockers(int k) { return clients; } +/** + * Returns an operation context that has an MMAPV1 locker attached to it. + */ +ServiceContext::UniqueOperationContext makeMMAPOperationContext() { + auto opCtx = cc().makeOperationContext(); + opCtx->releaseLockState(); + opCtx->setLockState(stdx::make_unique()); + return opCtx; +} + /** * Calls fn the given number of iterations, spread out over up to maxThreads threads. * The threadNr passed is an integer between 0 and maxThreads exclusive. Logs timing @@ -292,6 +304,86 @@ TEST(DConcurrency, GlobalLockX_Timeout) { } } +TEST(DConcurrency, GlobalLockXSetsGlobalLockTakenOnOperationContext) { + Client::initThreadIfNotAlready(); + auto opCtx = makeMMAPOperationContext(); + ASSERT_FALSE(GlobalLockAcquisitionTracker::get(opCtx.get()).getGlobalExclusiveLockTaken()); + + { + Lock::GlobalLock globalWrite(opCtx->lockState(), MODE_X, 0); + ASSERT(globalWrite.isLocked()); + } + ASSERT_TRUE(GlobalLockAcquisitionTracker::get(opCtx.get()).getGlobalExclusiveLockTaken()); +} + +TEST(DConcurrency, GlobalLockIXSetsGlobalLockTakenOnOperationContext) { + Client::initThreadIfNotAlready(); + auto opCtx = makeMMAPOperationContext(); + ASSERT_FALSE(GlobalLockAcquisitionTracker::get(opCtx.get()).getGlobalExclusiveLockTaken()); + { + Lock::GlobalLock globalWrite(opCtx->lockState(), MODE_IX, 0); + ASSERT(globalWrite.isLocked()); + } + ASSERT_TRUE(GlobalLockAcquisitionTracker::get(opCtx.get()).getGlobalExclusiveLockTaken()); +} + +TEST(DConcurrency, GlobalLockSDoesNotSetGlobalLockTakenOnOperationContext) { + Client::initThreadIfNotAlready(); + auto opCtx = makeMMAPOperationContext(); + ASSERT_FALSE(GlobalLockAcquisitionTracker::get(opCtx.get()).getGlobalExclusiveLockTaken()); + { + Lock::GlobalLock globalRead(opCtx->lockState(), MODE_S, 0); + ASSERT(globalRead.isLocked()); + } + ASSERT_FALSE(GlobalLockAcquisitionTracker::get(opCtx.get()).getGlobalExclusiveLockTaken()); +} + +TEST(DConcurrency, GlobalLockISDoesNotSetGlobalLockTakenOnOperationContext) { + Client::initThreadIfNotAlready(); + auto opCtx = makeMMAPOperationContext(); + ASSERT_FALSE(GlobalLockAcquisitionTracker::get(opCtx.get()).getGlobalExclusiveLockTaken()); + { + Lock::GlobalLock globalRead(opCtx->lockState(), MODE_IS, 0); + ASSERT(globalRead.isLocked()); + } + ASSERT_FALSE(GlobalLockAcquisitionTracker::get(opCtx.get()).getGlobalExclusiveLockTaken()); +} + +TEST(DConcurrency, DBLockXSetsGlobalLockTakenOnOperationContext) { + Client::initThreadIfNotAlready(); + auto opCtx = makeMMAPOperationContext(); + ASSERT_FALSE(GlobalLockAcquisitionTracker::get(opCtx.get()).getGlobalExclusiveLockTaken()); + + { Lock::DBLock dbWrite(opCtx->lockState(), "db", MODE_X); } + ASSERT_TRUE(GlobalLockAcquisitionTracker::get(opCtx.get()).getGlobalExclusiveLockTaken()); +} + +TEST(DConcurrency, DBLockSDoesNotSetGlobalLockTakenOnOperationContext) { + Client::initThreadIfNotAlready(); + auto opCtx = makeMMAPOperationContext(); + ASSERT_FALSE(GlobalLockAcquisitionTracker::get(opCtx.get()).getGlobalExclusiveLockTaken()); + + { Lock::DBLock dbRead(opCtx->lockState(), "db", MODE_S); } + ASSERT_FALSE(GlobalLockAcquisitionTracker::get(opCtx.get()).getGlobalExclusiveLockTaken()); +} + +TEST(DConcurrency, GlobalLockXDoesNotSetGlobalLockTakenWhenLockAcquisitionTimesOut) { + Client::initThreadIfNotAlready(); + auto clients = makeKClientsWithLockers(1); + + // Take a global lock so that the next one times out. + Lock::GlobalLock globalWrite0(clients[0].second.get()->lockState(), MODE_X, 0); + ASSERT(globalWrite0.isLocked()); + + auto opCtx = makeMMAPOperationContext(); + ASSERT_FALSE(GlobalLockAcquisitionTracker::get(opCtx.get()).getGlobalExclusiveLockTaken()); + { + Lock::GlobalLock globalWrite1(opCtx->lockState(), MODE_X, 1); + ASSERT_FALSE(globalWrite1.isLocked()); + } + ASSERT_FALSE(GlobalLockAcquisitionTracker::get(opCtx.get()).getGlobalExclusiveLockTaken()); +} + TEST(DConcurrency, GlobalLockS_NoTimeoutDueToGlobalLockS) { MMAPV1LockerImpl ls; Lock::GlobalRead globalRead(&ls); @@ -720,6 +812,231 @@ TEST(DConcurrency, Throttling) { ASSERT(!overlongWait); } +TEST(DConcurrency, NoThrottlingWhenNotAcquiringTickets) { + auto clientOpctxPairs = makeKClientsWithLockers(2); + auto opctx1 = clientOpctxPairs[0].second.get(); + auto opctx2 = clientOpctxPairs[1].second.get(); + // Limit the locker to 1 ticket at a time. + UseGlobalThrottling throttle(opctx1, 1); + + // Prevent the enforcement of ticket throttling. + opctx1->lockState()->setShouldAcquireTicket(false); + + // Both locks should be acquired immediately because there is no throttling. + Lock::GlobalRead R1(opctx1->lockState(), 0); + ASSERT(R1.isLocked()); + + Lock::GlobalRead R2(opctx2->lockState(), 0); + ASSERT(R2.isLocked()); +} + +TEST(DConcurrency, CompatibleFirstWithSXIS) { + auto clientOpctxPairs = makeKClientsWithLockers(3); + auto opctx1 = clientOpctxPairs[0].second.get(); + auto opctx2 = clientOpctxPairs[1].second.get(); + auto opctx3 = clientOpctxPairs[2].second.get(); + + // Build a queue of MODE_S <- MODE_X <- MODE_IS, with MODE_S granted. + Lock::GlobalRead lockS(opctx1->lockState()); + ASSERT(lockS.isLocked()); + Lock::GlobalLock lockX(opctx2->lockState(), MODE_X, UINT_MAX, Lock::GlobalLock::EnqueueOnly()); + ASSERT(!lockX.isLocked()); + + // A MODE_IS should be granted due to compatibleFirst policy. + Lock::GlobalLock lockIS(opctx3->lockState(), MODE_IS, 0); + ASSERT(lockIS.isLocked()); + + lockX.waitForLock(0); + ASSERT(!lockX.isLocked()); +} + + +TEST(DConcurrency, CompatibleFirstWithXSIXIS) { + auto clientOpctxPairs = makeKClientsWithLockers(4); + auto opctx1 = clientOpctxPairs[0].second.get(); + auto opctx2 = clientOpctxPairs[1].second.get(); + auto opctx3 = clientOpctxPairs[2].second.get(); + auto opctx4 = clientOpctxPairs[3].second.get(); + + // Build a queue of MODE_X <- MODE_S <- MODE_IX <- MODE_IS, with MODE_X granted. + boost::optional lockX; + lockX.emplace(opctx1->lockState()); + ASSERT(lockX->isLocked()); + boost::optional lockS; + lockS.emplace(opctx2->lockState(), MODE_S, UINT_MAX, Lock::GlobalLock::EnqueueOnly()); + ASSERT(!lockS->isLocked()); + Lock::GlobalLock lockIX( + opctx3->lockState(), MODE_IX, UINT_MAX, Lock::GlobalLock::EnqueueOnly()); + ASSERT(!lockIX.isLocked()); + Lock::GlobalLock lockIS( + opctx4->lockState(), MODE_IS, UINT_MAX, Lock::GlobalLock::EnqueueOnly()); + ASSERT(!lockIS.isLocked()); + + + // Now release the MODE_X and ensure that MODE_S will switch policy to compatibleFirst + lockX.reset(); + lockS->waitForLock(0); + ASSERT(lockS->isLocked()); + ASSERT(!lockIX.isLocked()); + lockIS.waitForLock(0); + ASSERT(lockIS.isLocked()); + + // Now release the MODE_S and ensure that MODE_IX gets locked. + lockS.reset(); + lockIX.waitForLock(0); + ASSERT(lockIX.isLocked()); +} + +TEST(DConcurrency, CompatibleFirstWithXSXIXIS) { + auto clientOpctxPairs = makeKClientsWithLockers(5); + auto opctx1 = clientOpctxPairs[0].second.get(); + auto opctx2 = clientOpctxPairs[1].second.get(); + auto opctx3 = clientOpctxPairs[2].second.get(); + auto opctx4 = clientOpctxPairs[3].second.get(); + auto opctx5 = clientOpctxPairs[4].second.get(); + + // Build a queue of MODE_X <- MODE_S <- MODE_X <- MODE_IX <- MODE_IS, with the first MODE_X + // granted and check that releasing it will result in the MODE_IS being granted. + boost::optional lockXgranted; + lockXgranted.emplace(opctx1->lockState()); + ASSERT(lockXgranted->isLocked()); + + boost::optional lockX; + lockX.emplace(opctx3->lockState(), MODE_X, UINT_MAX, Lock::GlobalLock::EnqueueOnly()); + ASSERT(!lockX->isLocked()); + + // Now request MODE_S: it will be first in the pending list due to EnqueueAtFront policy. + boost::optional lockS; + lockS.emplace(opctx2->lockState(), MODE_S, UINT_MAX, Lock::GlobalLock::EnqueueOnly()); + ASSERT(!lockS->isLocked()); + + Lock::GlobalLock lockIX( + opctx4->lockState(), MODE_IX, UINT_MAX, Lock::GlobalLock::EnqueueOnly()); + ASSERT(!lockIX.isLocked()); + Lock::GlobalLock lockIS( + opctx5->lockState(), MODE_IS, UINT_MAX, Lock::GlobalLock::EnqueueOnly()); + ASSERT(!lockIS.isLocked()); + + + // Now release the granted MODE_X and ensure that MODE_S will switch policy to compatibleFirst, + // not locking the MODE_X or MODE_IX, but instead granting the final MODE_IS. + lockXgranted.reset(); + lockS->waitForLock(0); + ASSERT(lockS->isLocked()); + + lockX->waitForLock(0); + ASSERT(!lockX->isLocked()); + lockIX.waitForLock(0); + ASSERT(!lockIX.isLocked()); + + lockIS.waitForLock(0); + ASSERT(lockIS.isLocked()); +} + +TEST(DConcurrency, CompatibleFirstStress) { + int numThreads = 8; + int testMicros = 500000; + AtomicUInt64 readOnlyInterval{0}; + AtomicBool done{false}; + std::vector acquisitionCount(numThreads); + std::vector timeoutCount(numThreads); + std::vector busyWaitCount(numThreads); + auto clientOpctxPairs = makeKClientsWithLockers(numThreads); + + // Do some busy waiting to trigger different timings. The atomic load prevents compilers + // from optimizing the loop away. + auto busyWait = [&done, &busyWaitCount](int threadId, long long iters) { + while (iters-- > 0) { + for (int i = 0; i < 100 && !done.load(); i++) { + busyWaitCount[threadId]++; + } + } + }; + + std::vector threads; + + // Thread putting state in/out of read-only CompatibleFirst mode. + threads.emplace_back([&]() { + Timer t; + auto endTime = t.micros() + testMicros; + uint64_t readOnlyIntervalCount = 0; + OperationContext* opCtx = clientOpctxPairs[0].second.get(); + for (int iters = 0; (t.micros() < endTime); iters++) { + busyWait(0, iters % 20); + Lock::GlobalRead readLock(opCtx->lockState(), iters % 2); + if (!readLock.isLocked()) { + timeoutCount[0]++; + continue; + } + acquisitionCount[0]++; + readOnlyInterval.store(++readOnlyIntervalCount); + busyWait(0, iters % 200); + readOnlyInterval.store(0); + }; + done.store(true); + }); + + for (int threadId = 1; threadId < numThreads; threadId++) { + threads.emplace_back([&, threadId]() { + Timer t; + for (int iters = 0; !done.load(); iters++) { + OperationContext* opCtx = clientOpctxPairs[threadId].second.get(); + boost::optional lock; + switch (threadId) { + case 1: + case 2: + case 3: + case 4: { + // Here, actually try to acquire a lock without waiting, and check whether + // we should have gotten the lock or not. Use MODE_IS in 95% of the cases, + // and MODE_S in only 5, as that stressing the partitioning scheme and + // policy changes more as thread 0 acquires/releases its MODE_S lock. + busyWait(threadId, iters % 100); + auto interval = readOnlyInterval.load(); + lock.emplace(opCtx->lockState(), + iters % 20 ? MODE_IS : MODE_S, + 0, + Lock::GlobalLock::EnqueueOnly()); + // If thread 0 is holding the MODE_S lock while we tried to acquire a + // MODE_IS or MODE_S lock, the CompatibleFirst policy guarantees success. + auto newInterval = readOnlyInterval.load(); + invariant(!interval || interval != newInterval || lock->isLocked()); + lock->waitForLock(0); + break; + } + case 5: + busyWait(threadId, iters % 150); + lock.emplace(opCtx->lockState(), MODE_X, iters % 2); + busyWait(threadId, iters % 10); + break; + case 6: + lock.emplace(opCtx->lockState(), iters % 25 ? MODE_IX : MODE_S, iters % 2); + busyWait(threadId, iters % 100); + break; + case 7: + busyWait(threadId, iters % 100); + lock.emplace(opCtx->lockState(), iters % 20 ? MODE_IS : MODE_X, 0); + break; + default: + MONGO_UNREACHABLE; + } + if (lock->isLocked()) + acquisitionCount[threadId]++; + else + timeoutCount[threadId]++; + }; + }); + } + + for (auto& thread : threads) + thread.join(); + for (int threadId = 0; threadId < numThreads; threadId++) { + log() << "thread " << threadId << " stats: " << acquisitionCount[threadId] + << " acquisitions, " << timeoutCount[threadId] << " timeouts, " + << busyWaitCount[threadId] / 1000000 << "M busy waits"; + } +} + // These tests exercise single- and multi-threaded performance of uncontended lock acquisition. It // is neither practical nor useful to run them on debug builds. diff --git a/src/mongo/db/concurrency/global_lock_acquisition_tracker.cpp b/src/mongo/db/concurrency/global_lock_acquisition_tracker.cpp new file mode 100644 index 00000000000..21ac9272c93 --- /dev/null +++ b/src/mongo/db/concurrency/global_lock_acquisition_tracker.cpp @@ -0,0 +1,45 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/concurrency/global_lock_acquisition_tracker.h" + +namespace mongo { + +const OperationContext::Decoration GlobalLockAcquisitionTracker::get = + OperationContext::declareDecoration(); + +bool GlobalLockAcquisitionTracker::getGlobalExclusiveLockTaken() const { + return _globalExclusiveLockTaken; +} + +void GlobalLockAcquisitionTracker::setGlobalExclusiveLockTaken() { + _globalExclusiveLockTaken = true; +} +} // namespace mongo diff --git a/src/mongo/db/concurrency/global_lock_acquisition_tracker.h b/src/mongo/db/concurrency/global_lock_acquisition_tracker.h new file mode 100644 index 00000000000..5bd68453f57 --- /dev/null +++ b/src/mongo/db/concurrency/global_lock_acquisition_tracker.h @@ -0,0 +1,62 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/operation_context.h" + +namespace mongo { + +/** + * The GlobalLockAcquisitionTracker keeps track of if the global lock has ever been taken in X or + * IX mode. This class is used to track if we ever did a transaction with the intent to do a + * write, so that we can enforce write concern on noop writes. + */ +class GlobalLockAcquisitionTracker { +public: + static const OperationContext::Decoration get; + + // Decoration requires a default constructor. + GlobalLockAcquisitionTracker() = default; + + /** + * Returns whether we have ever taken a global lock in X or IX mode in this operation. + */ + bool getGlobalExclusiveLockTaken() const; + + /** + * Sets that we have ever taken a global lock in X or IX mode in this operation. + */ + void setGlobalExclusiveLockTaken(); + +private: + // Set to true when the global lock is first taken in X or IX mode. Never set back to false. + bool _globalExclusiveLockTaken = false; +}; + +} // namespace mongo diff --git a/src/mongo/db/concurrency/lock_manager.cpp b/src/mongo/db/concurrency/lock_manager.cpp index f90141e79be..8ae7919012e 100644 --- a/src/mongo/db/concurrency/lock_manager.cpp +++ b/src/mongo/db/concurrency/lock_manager.cpp @@ -592,7 +592,9 @@ bool LockManager::unlock(LockRequest* request) { lock->decGrantedModeCount(request->mode); if (request->compatibleFirst) { + invariant(lock->compatibleFirstCount > 0); lock->compatibleFirstCount--; + invariant(lock->compatibleFirstCount == 0 || !lock->grantedList.empty()); } _onLockModeChanged(lock, lock->grantedCounts[request->mode] == 0); diff --git a/src/mongo/db/concurrency/lock_state.cpp b/src/mongo/db/concurrency/lock_state.cpp index 45ea279c137..3625cef24f7 100644 --- a/src/mongo/db/concurrency/lock_state.cpp +++ b/src/mongo/db/concurrency/lock_state.cpp @@ -311,7 +311,7 @@ LockResult LockerImpl::_lockGlobalBegin(LockMode mode, Milliseconds dassert(isLocked() == (_modeForTicket != MODE_NONE)); if (_modeForTicket == MODE_NONE) { const bool reader = isSharedLockMode(mode); - auto holder = ticketHolders[mode]; + auto holder = shouldAcquireTicket() ? ticketHolders[mode] : nullptr; if (holder) { _clientState.store(reader ? kQueuedReader : kQueuedWriter); if (timeout == Milliseconds::max()) { @@ -787,7 +787,10 @@ LockResult LockerImpl::lockComplete(ResourceId resId, } } - // Cleanup the state, since this is an unused lock now + // Cleanup the state, since this is an unused lock now. + // Note: in case of the _notify object returning LOCK_TIMEOUT, it is possible to find that the + // lock was still granted after all, but we don't try to take advantage of that and will return + // a timeout. if (result != LOCK_OK) { LockRequestsMap::Iterator it = _requests.find(resId); _unlockImpl(&it); @@ -807,7 +810,7 @@ bool LockerImpl::_unlockImpl(LockRequestsMap::Iterator* it) { if (globalLockManager.unlock(it->objAddr())) { if (it->key() == resourceIdGlobal) { invariant(_modeForTicket != MODE_NONE); - auto holder = ticketHolders[_modeForTicket]; + auto holder = shouldAcquireTicket() ? ticketHolders[_modeForTicket] : nullptr; _modeForTicket = MODE_NONE; if (holder) { holder->release(); diff --git a/src/mongo/db/concurrency/locker.h b/src/mongo/db/concurrency/locker.h index 37af569124f..c1443865326 100644 --- a/src/mongo/db/concurrency/locker.h +++ b/src/mongo/db/concurrency/locker.h @@ -334,11 +334,25 @@ public: return _shouldConflictWithSecondaryBatchApplication; } + /** + * If set to false, this opts out of the ticket mechanism. This should be used sparingly + * for special purpose threads, such as FTDC. + */ + void setShouldAcquireTicket(bool newValue) { + invariant(!isLocked()); + _shouldAcquireTicket = newValue; + } + bool shouldAcquireTicket() const { + return _shouldAcquireTicket; + } + + protected: Locker() {} private: bool _shouldConflictWithSecondaryBatchApplication = true; + bool _shouldAcquireTicket = true; }; } // namespace mongo diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index 2dbb2b1065e..c6272425bf5 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -103,10 +103,10 @@ #include "mongo/db/startup_warnings_mongod.h" #include "mongo/db/stats/counters.h" #include "mongo/db/stats/snapshots.h" +#include "mongo/db/storage/encryption_hooks.h" #include "mongo/db/storage/mmap_v1/mmap_v1_options.h" #include "mongo/db/storage/storage_engine.h" #include "mongo/db/storage/storage_options.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h" #include "mongo/db/ttl.h" #include "mongo/db/wire_version.h" #include "mongo/executor/network_interface_factory.h" @@ -596,7 +596,7 @@ ExitCode _initAndListen(int listenPort) { getGlobalServiceContext()->initializeGlobalStorageEngine(); #ifdef MONGO_CONFIG_WIREDTIGER_ENABLED - if (WiredTigerCustomizationHooks::get(getGlobalServiceContext())->restartRequired()) { + if (EncryptionHooks::get(getGlobalServiceContext())->restartRequired()) { exitCleanly(EXIT_CLEAN); } #endif @@ -704,7 +704,11 @@ ExitCode _initAndListen(int listenPort) { Status status = authindex::verifySystemIndexes(startupOpCtx.get()); if (!status.isOK()) { log() << redact(status); - exitCleanly(EXIT_NEED_UPGRADE); + if (status.code() == ErrorCodes::AuthSchemaIncompatible) { + exitCleanly(EXIT_NEED_UPGRADE); + } else { + quickExit(EXIT_FAILURE); + } } // SERVER-14090: Verify that auth schema version is schemaVersion26Final. @@ -752,7 +756,7 @@ ExitCode _initAndListen(int listenPort) { if (!storageGlobalParams.readOnly) { logStartup(startupOpCtx.get()); - startFTDC(); + startMongoDFTDC(); getDeleter()->startWorkers(); @@ -1108,7 +1112,7 @@ static void shutdownTask() { #endif // Shutdown Full-Time Data Capture - stopFTDC(); + stopMongoDFTDC(); if (txn) { ShardingState::get(txn)->shutDown(txn); diff --git a/src/mongo/db/dbhelpers.cpp b/src/mongo/db/dbhelpers.cpp index ebed6807c7e..96eb011018d 100644 --- a/src/mongo/db/dbhelpers.cpp +++ b/src/mongo/db/dbhelpers.cpp @@ -62,8 +62,8 @@ #include "mongo/db/s/sharding_state.h" #include "mongo/db/service_context.h" #include "mongo/db/storage/data_protector.h" +#include "mongo/db/storage/encryption_hooks.h" #include "mongo/db/storage/storage_options.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h" #include "mongo/db/write_concern.h" #include "mongo/db/write_concern_options.h" #include "mongo/s/shard_key_pattern.h" @@ -270,6 +270,7 @@ long long Helpers::removeRange(OperationContext* txn, const KeyRange& range, BoundInclusion boundInclusion, const WriteConcernOptions& writeConcern, + Milliseconds& replWaitDuration, RemoveSaver* callback, bool fromMigrate, bool onlyRemoveOrphanedDocs) { @@ -283,6 +284,7 @@ long long Helpers::removeRange(OperationContext* txn, BSONObj max; { + ScopedTransaction scopedXact(txn, MODE_IS); AutoGetCollectionForRead ctx(txn, ns); Collection* collection = ctx.getCollection(); if (!collection) { @@ -326,11 +328,12 @@ long long Helpers::removeRange(OperationContext* txn, long long numDeleted = 0; - Milliseconds millisWaitingForReplication{0}; + replWaitDuration = Milliseconds::zero(); while (1) { // Scoping for write lock. { + ScopedTransaction scopedXact(txn, MODE_IX); AutoGetCollection ctx(txn, NamespaceString(ns), MODE_IX, MODE_IX); Collection* collection = ctx.getCollection(); if (!collection) @@ -438,20 +441,21 @@ long long Helpers::removeRange(OperationContext* txn, txn, repl::ReplClientInfo::forClient(txn->getClient()).getLastOp(), writeConcern); - if (replStatus.status.code() == ErrorCodes::ExceededTimeLimit) { + if (replStatus.status.code() == ErrorCodes::ExceededTimeLimit || + replStatus.status.code() == ErrorCodes::WriteConcernFailed) { warning(LogComponent::kSharding) << "replication to secondaries for removeRange at " "least 60 seconds behind"; } else { uassertStatusOK(replStatus.status); } - millisWaitingForReplication += replStatus.duration; + replWaitDuration += replStatus.duration; } } if (writeConcern.shouldWaitForOtherNodes()) log(LogComponent::kSharding) << "Helpers::removeRangeUnlocked time spent waiting for replication: " - << durationCount(millisWaitingForReplication) << "ms" << endl; + << durationCount(replWaitDuration) << "ms" << endl; MONGO_LOG_COMPONENT(1, LogComponent::kSharding) << "end removal of " << min << " to " << max << " in " << ns << " (took " @@ -485,19 +489,19 @@ Helpers::RemoveSaver::RemoveSaver(const string& a, const string& b, const string ss << why << "." << terseCurrentTime(false) << "." << NUM++ << ".bson"; _file /= ss.str(); - auto hooks = WiredTigerCustomizationHooks::get(getGlobalServiceContext()); - if (hooks->enabled()) { - _protector = hooks->getDataProtector(); - _file += hooks->getProtectedPathSuffix(); + auto encryptionHooks = EncryptionHooks::get(getGlobalServiceContext()); + if (encryptionHooks->enabled()) { + _protector = encryptionHooks->getDataProtector(); + _file += encryptionHooks->getProtectedPathSuffix(); } } Helpers::RemoveSaver::~RemoveSaver() { if (_protector && _out) { - auto hooks = WiredTigerCustomizationHooks::get(getGlobalServiceContext()); - invariant(hooks->enabled()); + auto encryptionHooks = EncryptionHooks::get(getGlobalServiceContext()); + invariant(encryptionHooks->enabled()); - size_t protectedSizeMax = hooks->additionalBytesForProtectedBuffer(); + size_t protectedSizeMax = encryptionHooks->additionalBytesForProtectedBuffer(); std::unique_ptr protectedBuffer(new uint8_t[protectedSizeMax]); size_t resultLen; @@ -560,10 +564,10 @@ Status Helpers::RemoveSaver::goingToDelete(const BSONObj& o) { std::unique_ptr protectedBuffer; if (_protector) { - auto hooks = WiredTigerCustomizationHooks::get(getGlobalServiceContext()); - invariant(hooks->enabled()); + auto encryptionHooks = EncryptionHooks::get(getGlobalServiceContext()); + invariant(encryptionHooks->enabled()); - size_t protectedSizeMax = dataSize + hooks->additionalBytesForProtectedBuffer(); + size_t protectedSizeMax = dataSize + encryptionHooks->additionalBytesForProtectedBuffer(); protectedBuffer.reset(new uint8_t[protectedSizeMax]); size_t resultLen; diff --git a/src/mongo/db/dbhelpers.h b/src/mongo/db/dbhelpers.h index 650f265265d..59a4a317db7 100644 --- a/src/mongo/db/dbhelpers.h +++ b/src/mongo/db/dbhelpers.h @@ -141,11 +141,13 @@ struct Helpers { /** * Takes a namespace range, specified by a min and max and qualified by an index pattern, * and removes all the documents in that range found by iterating - * over the given index. Caller is responsible for insuring that min/max are + * over the given index. Caller is responsible for ensuring that min/max are * compatible with the given keyPattern (e.g min={a:100} is compatible with * keyPattern={a:1,b:1} since it can be extended to {a:100,b:minKey}, but * min={b:100} is not compatible). * + * Returns time spent waiting for majority replication in replWaitDuration. + * * Caller must hold a write lock on 'ns' * * Returns -1 when no usable index exists @@ -157,6 +159,7 @@ struct Helpers { const KeyRange& range, BoundInclusion boundInclusion, const WriteConcernOptions& secondaryThrottle, + Milliseconds& replWaitDuration, RemoveSaver* callback = NULL, bool fromMigrate = false, bool onlyRemoveOrphanedDocs = false); diff --git a/src/mongo/db/dbwebserver.cpp b/src/mongo/db/dbwebserver.cpp index b202e0500b9..50fa68a8176 100644 --- a/src/mongo/db/dbwebserver.cpp +++ b/src/mongo/db/dbwebserver.cpp @@ -65,6 +65,7 @@ namespace mongo { using std::map; +using std::string; using std::stringstream; using std::vector; diff --git a/src/mongo/db/ftdc/SConscript b/src/mongo/db/ftdc/SConscript index 0d16aec93f2..81df9ad0f4d 100644 --- a/src/mongo/db/ftdc/SConscript +++ b/src/mongo/db/ftdc/SConscript @@ -21,7 +21,6 @@ ftdcEnv.Library( LIBDEPS=[ '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/bson/util/bson_extract', - '$BUILD_DIR/mongo/db/server_options_core', '$BUILD_DIR/mongo/db/service_context', '$BUILD_DIR/third_party/s2/s2', # For VarInt '$BUILD_DIR/third_party/shim_zlib', @@ -40,24 +39,44 @@ elif env.TargetOSIs('windows'): ] env.Library( - target='ftdc_mongod', + target='ftdc_server', source=[ - 'ftdc_commands.cpp', - 'ftdc_mongod.cpp', + 'ftdc_server.cpp', 'ftdc_system_stats.cpp', 'ftdc_system_stats_${TARGET_OS}.cpp', ], LIBDEPS=[ '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/commands', - '$BUILD_DIR/mongo/db/repl/repl_coordinator_global', '$BUILD_DIR/mongo/db/server_parameters', - '$BUILD_DIR/mongo/db/storage/storage_options', '$BUILD_DIR/mongo/util/processinfo', 'ftdc' ] + platform_libs, ) +env.Library( + target='ftdc_mongod', + source=[ + 'ftdc_commands.cpp', + 'ftdc_mongod.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/db/repl/repl_coordinator_global', + '$BUILD_DIR/mongo/db/storage/storage_options', + 'ftdc_server' + ], +) + +env.Library( + target='ftdc_mongos', + source=[ + 'ftdc_mongos.cpp', + ], + LIBDEPS=[ + 'ftdc_server' + ], +) + env.CppUnitTest( target='ftdc_test', source=[ diff --git a/src/mongo/db/ftdc/collector.cpp b/src/mongo/db/ftdc/collector.cpp index 611f12dff5a..7c35fb12bb6 100644 --- a/src/mongo/db/ftdc/collector.cpp +++ b/src/mongo/db/ftdc/collector.cpp @@ -67,6 +67,7 @@ std::tuple FTDCCollectorCollection::collect(Client* client) { // batches that are taking a long time. auto txn = client->makeOperationContext(); txn->lockState()->setShouldConflictWithSecondaryBatchApplication(false); + txn->lockState()->setShouldAcquireTicket(false); for (auto& collector : _collectors) { BSONObjBuilder subObjBuilder(builder.subobjStart(collector->name())); diff --git a/src/mongo/db/ftdc/compressor_test.cpp b/src/mongo/db/ftdc/compressor_test.cpp index 688197a392a..0c01bd58040 100644 --- a/src/mongo/db/ftdc/compressor_test.cpp +++ b/src/mongo/db/ftdc/compressor_test.cpp @@ -368,7 +368,7 @@ TEST(FTDCCompressor, TestNumbersCompat) { } // Test various date time types -TEST(AFTDCCompressor, TestDateTimeTypes) { +TEST(FTDCCompressor, TestDateTimeTypes) { TestTie c; for (int i = 0; i < 10; i++) { BSONObjBuilder builder1; diff --git a/src/mongo/db/ftdc/constants.h b/src/mongo/db/ftdc/constants.h index a989fd35b64..3fe5191ff44 100644 --- a/src/mongo/db/ftdc/constants.h +++ b/src/mongo/db/ftdc/constants.h @@ -26,6 +26,10 @@ * then also delete it in the license file. */ +#pragma once + +#include "mongo/base/string_data.h" + namespace mongo { extern const char kFTDCInterimFile[]; @@ -42,4 +46,6 @@ extern const char kFTDCDocsField[]; extern const char kFTDCCollectStartField[]; extern const char kFTDCCollectEndField[]; +constexpr StringData kFTDCDefaultDirectory = "diagnostic.data"_sd; + } // namespace mongo diff --git a/src/mongo/db/ftdc/controller.cpp b/src/mongo/db/ftdc/controller.cpp index 9e2987db0f9..bc3d4df444c 100644 --- a/src/mongo/db/ftdc/controller.cpp +++ b/src/mongo/db/ftdc/controller.cpp @@ -47,9 +47,19 @@ namespace mongo { -void FTDCController::setEnabled(bool enabled) { +Status FTDCController::setEnabled(bool enabled) { stdx::lock_guard lock(_mutex); + + if (_path.empty()) { + return Status(ErrorCodes::FTDCPathNotSet, + str::stream() << "FTDC cannot be enabled without setting the set parameter " + "'diagnosticDataCollectionDirectoryPath' first."); + } + _configTemp.enabled = enabled; + _condvar.notify_one(); + + return Status::OK(); } void FTDCController::setPeriod(Milliseconds millis) { @@ -82,6 +92,23 @@ void FTDCController::setMaxSamplesPerInterimMetricChunk(size_t size) { _condvar.notify_one(); } +Status FTDCController::setDirectory(const boost::filesystem::path& path) { + stdx::lock_guard lock(_mutex); + + if (!_path.empty()) { + return Status(ErrorCodes::FTDCPathAlreadySet, + str::stream() << "FTDC path has already been set to '" << _path.string() + << "'. It cannot be changed."); + } + + _path = path; + + // Do not notify for the change since it has to be enabled via setEnabled. + + return Status::OK(); +} + + void FTDCController::addPeriodicCollector(std::unique_ptr collector) { { stdx::lock_guard lock(_mutex); @@ -203,7 +230,7 @@ void FTDCController::doLoop() { } // TODO: consider only running this thread if we are enabled - // for now, we just keep an idle thread as it is simplier + // for now, we just keep an idle thread as it is simpler if (_config.enabled) { // Delay initialization of FTDCFileManager until we are sure the user has enabled // FTDC diff --git a/src/mongo/db/ftdc/controller.h b/src/mongo/db/ftdc/controller.h index 44b5e838506..e73d570598d 100644 --- a/src/mongo/db/ftdc/controller.h +++ b/src/mongo/db/ftdc/controller.h @@ -62,8 +62,12 @@ public: /* * Set whether the controller is enabled, and collects data. + * + * Returns ErrorCodes::FTDCPathNotSet if no log path has been specified for FTDC. This occurs + * in MongoS in some situations since MongoS is not required to have a storage directory like + * MongoD does. */ - void setEnabled(bool enabled); + Status setEnabled(bool enabled); /** * Set the period for data collection. @@ -94,6 +98,13 @@ public: */ void setMaxSamplesPerInterimMetricChunk(size_t size); + /* + * Set the path to store FTDC files if not already set. + * + * Returns ErrorCodes::FTDCPathAlreadySet if the path has already been set. + */ + Status setDirectory(const boost::filesystem::path& path); + /** * Add a metric collector to collect periodically. i.e., serverStatus */ @@ -172,7 +183,7 @@ private: State _state{State::kNotStarted}; // Directory to store files - const boost::filesystem::path _path; + boost::filesystem::path _path; // Mutex to protect the condvar, configuration changes, and most recent periodic document. stdx::mutex _mutex; diff --git a/src/mongo/db/ftdc/controller_test.cpp b/src/mongo/db/ftdc/controller_test.cpp index 365f06580cd..c75a3fe44d6 100644 --- a/src/mongo/db/ftdc/controller_test.cpp +++ b/src/mongo/db/ftdc/controller_test.cpp @@ -254,7 +254,7 @@ TEST(FTDCControllerTest, TestStartAsDisabled) { ASSERT_EQUALS(files0.size(), 0UL); - c.setEnabled(true); + ASSERT_OK(c.setEnabled(true)); c1Ptr->setSignalOnCount(50); diff --git a/src/mongo/db/ftdc/ftdc_mongod.cpp b/src/mongo/db/ftdc/ftdc_mongod.cpp index 094a2b05d06..1f53ec345ea 100644 --- a/src/mongo/db/ftdc/ftdc_mongod.cpp +++ b/src/mongo/db/ftdc/ftdc_mongod.cpp @@ -31,295 +31,18 @@ #include "mongo/db/ftdc/ftdc_mongod.h" #include -#include -#include -#include "mongo/base/init.h" -#include "mongo/base/status.h" -#include "mongo/bson/bsonobjbuilder.h" -#include "mongo/db/commands.h" -#include "mongo/db/ftdc/collector.h" -#include "mongo/db/ftdc/config.h" +#include "mongo/db/ftdc/constants.h" #include "mongo/db/ftdc/controller.h" -#include "mongo/db/ftdc/ftdc_system_stats.h" -#include "mongo/db/jsobj.h" +#include "mongo/db/ftdc/ftdc_server.h" #include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/repl/replication_coordinator_global.h" -#include "mongo/db/server_parameters.h" -#include "mongo/db/service_context.h" #include "mongo/db/storage/storage_options.h" namespace mongo { namespace { - -const auto getFTDCController = ServiceContext::declareDecoration>(); - -FTDCController* getGlobalFTDCController() { - if (!hasGlobalServiceContext()) { - return nullptr; - } - - return getFTDCController(getGlobalServiceContext()).get(); -} - -std::atomic localEnabledFlag(FTDCConfig::kEnabledDefault); // NOLINT - -class ExportedFTDCEnabledParameter - : public ExportedServerParameter { -public: - ExportedFTDCEnabledParameter() - : ExportedServerParameter( - ServerParameterSet::getGlobal(), - "diagnosticDataCollectionEnabled", - &localEnabledFlag) {} - - virtual Status validate(const bool& potentialNewValue) { - auto controller = getGlobalFTDCController(); - if (controller) { - controller->setEnabled(potentialNewValue); - } - - return Status::OK(); - } - -} exportedFTDCEnabledParameter; - -std::atomic localPeriodMillis(FTDCConfig::kPeriodMillisDefault); // NOLINT - -class ExportedFTDCPeriodParameter - : public ExportedServerParameter { -public: - ExportedFTDCPeriodParameter() - : ExportedServerParameter( - ServerParameterSet::getGlobal(), - "diagnosticDataCollectionPeriodMillis", - &localPeriodMillis) {} - - virtual Status validate(const std::int32_t& potentialNewValue) { - if (potentialNewValue < 100) { - return Status( - ErrorCodes::BadValue, - "diagnosticDataCollectionPeriodMillis must be greater than or equal to 100ms"); - } - - auto controller = getGlobalFTDCController(); - if (controller) { - controller->setPeriod(Milliseconds(potentialNewValue)); - } - - return Status::OK(); - } - -} exportedFTDCPeriodParameter; - -// Scale the values down since are defaults are in bytes, but the user interface is MB -std::atomic localMaxDirectorySizeMB( // NOLINT - FTDCConfig::kMaxDirectorySizeBytesDefault / (1024 * 1024)); - -std::atomic localMaxFileSizeMB(FTDCConfig::kMaxFileSizeBytesDefault / // NOLINT - (1024 * 1024)); - -class ExportedFTDCDirectorySizeParameter - : public ExportedServerParameter { -public: - ExportedFTDCDirectorySizeParameter() - : ExportedServerParameter( - ServerParameterSet::getGlobal(), - "diagnosticDataCollectionDirectorySizeMB", - &localMaxDirectorySizeMB) {} - - virtual Status validate(const std::int32_t& potentialNewValue) { - if (potentialNewValue < 10) { - return Status( - ErrorCodes::BadValue, - "diagnosticDataCollectionDirectorySizeMB must be greater than or equal to 10"); - } - - if (potentialNewValue < localMaxFileSizeMB) { - return Status( - ErrorCodes::BadValue, - str::stream() - << "diagnosticDataCollectionDirectorySizeMB must be greater than or equal to '" - << localMaxFileSizeMB - << "' which is the current value of diagnosticDataCollectionFileSizeMB."); - } - - auto controller = getGlobalFTDCController(); - if (controller) { - controller->setMaxDirectorySizeBytes(potentialNewValue * 1024 * 1024); - } - - return Status::OK(); - } - -} exportedFTDCDirectorySizeParameter; - -class ExportedFTDCFileSizeParameter - : public ExportedServerParameter { -public: - ExportedFTDCFileSizeParameter() - : ExportedServerParameter( - ServerParameterSet::getGlobal(), - "diagnosticDataCollectionFileSizeMB", - &localMaxFileSizeMB) {} - - virtual Status validate(const std::int32_t& potentialNewValue) { - if (potentialNewValue < 1) { - return Status(ErrorCodes::BadValue, - "diagnosticDataCollectionFileSizeMB must be greater than or equal to 1"); - } - - if (potentialNewValue > localMaxDirectorySizeMB) { - return Status( - ErrorCodes::BadValue, - str::stream() - << "diagnosticDataCollectionFileSizeMB must be less than or equal to '" - << localMaxDirectorySizeMB - << "' which is the current value of diagnosticDataCollectionDirectorySizeMB."); - } - - auto controller = getGlobalFTDCController(); - if (controller) { - controller->setMaxFileSizeBytes(potentialNewValue * 1024 * 1024); - } - - return Status::OK(); - } - -} exportedFTDCFileSizeParameter; - -std::atomic localMaxSamplesPerArchiveMetricChunk( // NOLINT - FTDCConfig::kMaxSamplesPerArchiveMetricChunkDefault); - -class ExportedFTDCArchiveChunkSizeParameter - : public ExportedServerParameter { -public: - ExportedFTDCArchiveChunkSizeParameter() - : ExportedServerParameter( - ServerParameterSet::getGlobal(), - "diagnosticDataCollectionSamplesPerChunk", - &localMaxSamplesPerArchiveMetricChunk) {} - - virtual Status validate(const std::int32_t& potentialNewValue) { - if (potentialNewValue < 2) { - return Status( - ErrorCodes::BadValue, - "diagnosticDataCollectionSamplesPerChunk must be greater than or equal to 2"); - } - - auto controller = getGlobalFTDCController(); - if (controller) { - controller->setMaxSamplesPerArchiveMetricChunk(potentialNewValue); - } - - return Status::OK(); - } - -} exportedFTDCArchiveChunkSizeParameter; - -std::atomic localMaxSamplesPerInterimMetricChunk( // NOLINT - FTDCConfig::kMaxSamplesPerInterimMetricChunkDefault); - -class ExportedFTDCInterimChunkSizeParameter - : public ExportedServerParameter { -public: - ExportedFTDCInterimChunkSizeParameter() - : ExportedServerParameter( - ServerParameterSet::getGlobal(), - "diagnosticDataCollectionSamplesPerInterimUpdate", - &localMaxSamplesPerInterimMetricChunk) {} - - virtual Status validate(const std::int32_t& potentialNewValue) { - if (potentialNewValue < 2) { - return Status(ErrorCodes::BadValue, - "diagnosticDataCollectionSamplesPerInterimUpdate must be greater than or " - "equal to 2"); - } - - auto controller = getGlobalFTDCController(); - if (controller) { - controller->setMaxSamplesPerInterimMetricChunk(potentialNewValue); - } - - return Status::OK(); - } - -} exportedFTDCInterimChunkSizeParameter; - -class FTDCSimpleInternalCommandCollector final : public FTDCCollectorInterface { -public: - FTDCSimpleInternalCommandCollector(StringData command, - StringData name, - StringData ns, - BSONObj cmdObj) - : _name(name.toString()), _ns(ns.toString()), _cmdObj(std::move(cmdObj)) { - _command = Command::findCommand(command); - invariant(_command); - } - - void collect(OperationContext* txn, BSONObjBuilder& builder) override { - std::string errmsg; - - bool ret = _command->run(txn, _ns, _cmdObj, 0, errmsg, builder); - - // Some commands return errmsgs when they return false (collstats) - // Some commands return bson objs when they return false (replGetStatus) - // We append the status as needed to ensure readers of the collected data can check the - // status of any individual command. - _command->appendCommandStatus(builder, ret, errmsg); - } - - std::string name() const override { - return _name; - } - -private: - std::string _name; - std::string _ns; - BSONObj _cmdObj; - - // Not owned - Command* _command; -}; - -} // namespace - -// Register the FTDC system -// Note: This must be run before the server parameters are parsed during startup -// so that the FTDCController is initialized. -// -void startFTDC() { - boost::filesystem::path dir(storageGlobalParams.dbpath); - dir /= "diagnostic.data"; - - - FTDCConfig config; - config.period = Milliseconds(localPeriodMillis.load()); - config.enabled = localEnabledFlag; - config.maxFileSizeBytes = localMaxFileSizeMB * 1024 * 1024; - config.maxDirectorySizeBytes = localMaxDirectorySizeMB * 1024 * 1024; - config.maxSamplesPerArchiveMetricChunk = localMaxSamplesPerArchiveMetricChunk; - config.maxSamplesPerInterimMetricChunk = localMaxSamplesPerInterimMetricChunk; - - auto controller = stdx::make_unique(dir, config); - - // Install periodic collectors - // These are collected on the period interval in FTDCConfig. - // NOTE: For each command here, there must be an equivalent privilege check in - // GetDiagnosticDataCommand - - // CmdServerStatus - // The "sharding" section is filtered out because at this time it only consists of strings in - // migration status. This section triggers too many schema changes in the serverStatus which - // hurt ftdc compression efficiency, because its output varies depending on the list of active - // migrations. - controller->addPeriodicCollector(stdx::make_unique( - "serverStatus", - "serverStatus", - "", - BSON("serverStatus" << 1 << "tcMalloc" << true << "sharding" << false))); - +void registerMongoDCollectors(FTDCController* controller) { // These metrics are only collected if replication is enabled if (repl::getGlobalReplicationCoordinator()->getReplicationMode() != repl::ReplicationCoordinator::modeNone) { @@ -335,43 +58,19 @@ void startFTDC() { BSON("collStats" << "oplog.rs"))); } - - // Install System Metric Collector as a periodic collector - installSystemMetricsCollector(controller.get()); - - // Install file rotation collectors - // These are collected on each file rotation. - - // CmdBuildInfo - controller->addOnRotateCollector(stdx::make_unique( - "buildInfo", "buildInfo", "", BSON("buildInfo" << 1))); - - // CmdGetCmdLineOpts - controller->addOnRotateCollector(stdx::make_unique( - "getCmdLineOpts", "getCmdLineOpts", "", BSON("getCmdLineOpts" << 1))); - - // HostInfoCmd - controller->addOnRotateCollector(stdx::make_unique( - "hostInfo", "hostInfo", "", BSON("hostInfo" << 1))); - - // Install the new controller - auto& staticFTDC = getFTDCController(getGlobalServiceContext()); - - staticFTDC = std::move(controller); - - staticFTDC->start(); } -void stopFTDC() { - auto controller = getGlobalFTDCController(); +} // namespace - if (controller) { - controller->stop(); - } +void startMongoDFTDC() { + boost::filesystem::path dir(storageGlobalParams.dbpath); + dir /= kFTDCDefaultDirectory.toString(); + + startFTDC(dir, FTDCStartMode::kStart, registerMongoDCollectors); } -FTDCController* FTDCController::get(ServiceContext* serviceContext) { - return getFTDCController(serviceContext).get(); +void stopMongoDFTDC() { + stopFTDC(); } } // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_mongod.h b/src/mongo/db/ftdc/ftdc_mongod.h index 1e4f20b8b17..b4409dde902 100644 --- a/src/mongo/db/ftdc/ftdc_mongod.h +++ b/src/mongo/db/ftdc/ftdc_mongod.h @@ -1,30 +1,30 @@ /** -* Copyright (C) 2015 MongoDB Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -* -* As a special exception, the copyright holders give permission to link the -* code of portions of this program with the OpenSSL library under certain -* conditions as described in each individual source file and distribute -* linked combinations including the program with the OpenSSL library. You -* must comply with the GNU Affero General Public License in all respects -* for all of the code used other than as permitted herein. If you modify -* file(s) with this exception, you may extend this exception to your -* version of the file(s), but you are not obligated to do so. If you do not -* wish to do so, delete this exception statement from your version. If you -* delete this exception statement from all source files in the program, -* then also delete it in the license file. -*/ + * Copyright (C) 2015 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ #pragma once @@ -34,11 +34,11 @@ namespace mongo { * Start Full Time Data Capture * Starts 1 thread. */ -void startFTDC(); +void startMongoDFTDC(); /** * Stop Full Time Data Capture */ -void stopFTDC(); +void stopMongoDFTDC(); } // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_mongos.cpp b/src/mongo/db/ftdc/ftdc_mongos.cpp new file mode 100644 index 00000000000..bdacaa7b6e1 --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_mongos.cpp @@ -0,0 +1,153 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kFTDC + +#include "mongo/platform/basic.h" + +#include "mongo/db/ftdc/ftdc_mongos.h" + +#include + +#include "mongo/db/ftdc/controller.h" +#include "mongo/db/ftdc/ftdc_server.h" +#include "mongo/db/server_parameters.h" +#include "mongo/stdx/thread.h" +#include "mongo/util/log.h" + +namespace mongo { + +namespace { + +/** + * Expose diagnosticDataCollectionDirectoryPath set parameter to specify the MongoS FTDC path. + */ +class ExportedFTDCDirectoryPathParameter : public ServerParameter { +public: + ExportedFTDCDirectoryPathParameter() + : ServerParameter(ServerParameterSet::getGlobal(), + "diagnosticDataCollectionDirectoryPath", + true, + true) {} + + + void append(OperationContext* opCtx, BSONObjBuilder& b, const std::string& name) final { + stdx::lock_guard guard(_lock); + b.append(name, _path.generic_string()); + } + + Status set(const BSONElement& newValueElement) { + if (newValueElement.type() != String) { + return Status(ErrorCodes::BadValue, + "diagnosticDataCollectionDirectoryPath only supports type string"); + } + + std::string str = newValueElement.str(); + return setFromString(str); + } + + Status setFromString(const std::string& str) final { + stdx::lock_guard guard(_lock); + + FTDCController* controller = nullptr; + + if (hasGlobalServiceContext()) { + controller = FTDCController::get(getGlobalServiceContext()); + } + + if (controller) { + Status s = controller->setDirectory(str); + if (!s.isOK()) { + return s; + } + } + + _path = str; + + return Status::OK(); + } + + boost::filesystem::path getDirectory() { + stdx::lock_guard guard(_lock); + return _path; + } + + void setDirectory(boost::filesystem::path& path) { + stdx::lock_guard guard(_lock); + _path = path; + } + +private: + // Lock to guard _path + stdx::mutex _lock; + + // Directory location of ftdc files, guarded by _lock + boost::filesystem::path _path; +} exportedFTDCDirectoryPathParameter; + +void registerMongoSCollectors(FTDCController* controller) { + // PoolStats + controller->addPeriodicCollector(stdx::make_unique( + "connPoolStats", "connPoolStats", "", BSON("connPoolStats" << 1))); +} + +} // namespace + +void startMongoSFTDC() { + // Get the path to use for FTDC: + // 1. Check if the user set one. + // 2. If not, check if the user has a logpath and derive one. + // 3. Otherwise, tell the user FTDC cannot run. + + // Only attempt to enable FTDC if we have a path to log files to. + FTDCStartMode startMode = FTDCStartMode::kStart; + auto directory = exportedFTDCDirectoryPathParameter.getDirectory(); + + if (directory.empty()) { + if (serverGlobalParams.logpath.empty()) { + warning() << "FTDC is disabled because neither '--logpath' nor set parameter " + "'diagnosticDataCollectionDirectoryPath' are specified."; + startMode = FTDCStartMode::kSkipStart; + } else { + directory = FTDCUtil::getMongoSPath(serverGlobalParams.logpath); + + // Update the server parameter with the computed path. + // Note: If the computed FTDC directory conflicts with an existing file, then FTDC will + // warn about the conflict, and not startup. It will not terminate MongoS in this + // situation. + exportedFTDCDirectoryPathParameter.setDirectory(directory); + } + } + + startFTDC(directory, startMode, registerMongoSCollectors); +} + +void stopMongoSFTDC() { + stopFTDC(); +} + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_mongos.h b/src/mongo/db/ftdc/ftdc_mongos.h new file mode 100644 index 00000000000..4d8ff8fc08b --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_mongos.h @@ -0,0 +1,43 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +namespace mongo { + +/** + * Start Full Time Data Capture + */ +void startMongoSFTDC(); + +/** + * Stop Full Time Data Capture + */ +void stopMongoSFTDC(); + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_server.cpp b/src/mongo/db/ftdc/ftdc_server.cpp new file mode 100644 index 00000000000..167587a8722 --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_server.cpp @@ -0,0 +1,350 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/ftdc/ftdc_server.h" + +#include +#include +#include + +#include "mongo/base/status.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/commands.h" +#include "mongo/db/ftdc/collector.h" +#include "mongo/db/ftdc/config.h" +#include "mongo/db/ftdc/controller.h" +#include "mongo/db/ftdc/ftdc_system_stats.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/server_parameters.h" +#include "mongo/db/service_context.h" +#include "mongo/stdx/memory.h" + +namespace mongo { + +namespace { + +const auto getFTDCController = ServiceContext::declareDecoration>(); + +FTDCController* getGlobalFTDCController() { + if (!hasGlobalServiceContext()) { + return nullptr; + } + + return getFTDCController(getGlobalServiceContext()).get(); +} + +std::atomic localEnabledFlag(FTDCConfig::kEnabledDefault); // NOLINT + +class ExportedFTDCEnabledParameter + : public ExportedServerParameter { +public: + ExportedFTDCEnabledParameter() + : ExportedServerParameter( + ServerParameterSet::getGlobal(), + "diagnosticDataCollectionEnabled", + &localEnabledFlag) {} + + virtual Status validate(const bool& potentialNewValue) { + auto controller = getGlobalFTDCController(); + if (controller) { + return controller->setEnabled(potentialNewValue); + } + + return Status::OK(); + } + +} exportedFTDCEnabledParameter; + +std::atomic localPeriodMillis(FTDCConfig::kPeriodMillisDefault); // NOLINT + +class ExportedFTDCPeriodParameter + : public ExportedServerParameter { +public: + ExportedFTDCPeriodParameter() + : ExportedServerParameter( + ServerParameterSet::getGlobal(), + "diagnosticDataCollectionPeriodMillis", + &localPeriodMillis) {} + + virtual Status validate(const std::int32_t& potentialNewValue) { + if (potentialNewValue < 100) { + return Status( + ErrorCodes::BadValue, + "diagnosticDataCollectionPeriodMillis must be greater than or equal to 100ms"); + } + + auto controller = getGlobalFTDCController(); + if (controller) { + controller->setPeriod(Milliseconds(potentialNewValue)); + } + + return Status::OK(); + } + +} exportedFTDCPeriodParameter; + +// Scale the values down since are defaults are in bytes, but the user interface is MB +std::atomic localMaxDirectorySizeMB(FTDCConfig::kMaxDirectorySizeBytesDefault / + (1024 * 1024)); // NOLINT + +std::atomic localMaxFileSizeMB(FTDCConfig::kMaxFileSizeBytesDefault / + (1024 * 1024)); // NOLINT + +class ExportedFTDCDirectorySizeParameter + : public ExportedServerParameter { +public: + ExportedFTDCDirectorySizeParameter() + : ExportedServerParameter( + ServerParameterSet::getGlobal(), + "diagnosticDataCollectionDirectorySizeMB", + &localMaxDirectorySizeMB) {} + + virtual Status validate(const std::int32_t& potentialNewValue) { + if (potentialNewValue < 10) { + return Status( + ErrorCodes::BadValue, + "diagnosticDataCollectionDirectorySizeMB must be greater than or equal to 10"); + } + + if (potentialNewValue < localMaxFileSizeMB.load()) { + return Status( + ErrorCodes::BadValue, + str::stream() + << "diagnosticDataCollectionDirectorySizeMB must be greater than or equal to '" + << localMaxFileSizeMB.load() + << "' which is the current value of diagnosticDataCollectionFileSizeMB."); + } + + auto controller = getGlobalFTDCController(); + if (controller) { + controller->setMaxDirectorySizeBytes(potentialNewValue * 1024 * 1024); + } + + return Status::OK(); + } + +} exportedFTDCDirectorySizeParameter; + +class ExportedFTDCFileSizeParameter + : public ExportedServerParameter { +public: + ExportedFTDCFileSizeParameter() + : ExportedServerParameter( + ServerParameterSet::getGlobal(), + "diagnosticDataCollectionFileSizeMB", + &localMaxFileSizeMB) {} + + virtual Status validate(const std::int32_t& potentialNewValue) { + if (potentialNewValue < 1) { + return Status(ErrorCodes::BadValue, + "diagnosticDataCollectionFileSizeMB must be greater than or equal to 1"); + } + + if (potentialNewValue > localMaxDirectorySizeMB.load()) { + return Status( + ErrorCodes::BadValue, + str::stream() + << "diagnosticDataCollectionFileSizeMB must be less than or equal to '" + << localMaxDirectorySizeMB.load() + << "' which is the current value of diagnosticDataCollectionDirectorySizeMB."); + } + + auto controller = getGlobalFTDCController(); + if (controller) { + controller->setMaxFileSizeBytes(potentialNewValue * 1024 * 1024); + } + + return Status::OK(); + } + +} exportedFTDCFileSizeParameter; + +std::atomic localMaxSamplesPerArchiveMetricChunk( // NOLINT + FTDCConfig::kMaxSamplesPerArchiveMetricChunkDefault); + +class ExportedFTDCArchiveChunkSizeParameter + : public ExportedServerParameter { +public: + ExportedFTDCArchiveChunkSizeParameter() + : ExportedServerParameter( + ServerParameterSet::getGlobal(), + "diagnosticDataCollectionSamplesPerChunk", + &localMaxSamplesPerArchiveMetricChunk) {} + + virtual Status validate(const std::int32_t& potentialNewValue) { + if (potentialNewValue < 2) { + return Status( + ErrorCodes::BadValue, + "diagnosticDataCollectionSamplesPerChunk must be greater than or equal to 2"); + } + + auto controller = getGlobalFTDCController(); + if (controller) { + controller->setMaxSamplesPerArchiveMetricChunk(potentialNewValue); + } + + return Status::OK(); + } + +} exportedFTDCArchiveChunkSizeParameter; + +std::atomic localMaxSamplesPerInterimMetricChunk( // NOLINT + FTDCConfig::kMaxSamplesPerInterimMetricChunkDefault); + +class ExportedFTDCInterimChunkSizeParameter + : public ExportedServerParameter { +public: + ExportedFTDCInterimChunkSizeParameter() + : ExportedServerParameter( + ServerParameterSet::getGlobal(), + "diagnosticDataCollectionSamplesPerInterimUpdate", + &localMaxSamplesPerInterimMetricChunk) {} + + virtual Status validate(const std::int32_t& potentialNewValue) { + if (potentialNewValue < 2) { + return Status(ErrorCodes::BadValue, + "diagnosticDataCollectionSamplesPerInterimUpdate must be greater than or " + "equal to 2"); + } + + auto controller = getGlobalFTDCController(); + if (controller) { + controller->setMaxSamplesPerInterimMetricChunk(potentialNewValue); + } + + return Status::OK(); + } + +} exportedFTDCInterimChunkSizeParameter; +} // namespace + +FTDCSimpleInternalCommandCollector::FTDCSimpleInternalCommandCollector(StringData command, + StringData name, + StringData ns, + BSONObj cmdObj) + : _name(name.toString()), _ns(ns.toString()), _cmdObj(std::move(cmdObj)) { + _command = Command::findCommand(command); + invariant(_command); +} + +void FTDCSimpleInternalCommandCollector::collect(OperationContext* opCtx, BSONObjBuilder& builder) { + std::string errmsg; + + bool ret = _command->run(opCtx, _ns, _cmdObj, 0, errmsg, builder); + + // Some commands return errmsgs when they return false (collstats) + // Some commands return bson objs when they return false (replGetStatus) + // We append the status as needed to ensure readers of the collected data can check the + // status of any individual command. + _command->appendCommandStatus(builder, ret, errmsg); +} + +std::string FTDCSimpleInternalCommandCollector::name() const { + return _name; +} + +// Register the FTDC system +// Note: This must be run before the server parameters are parsed during startup +// so that the FTDCController is initialized. +// +void startFTDC(boost::filesystem::path& path, + FTDCStartMode startupMode, + RegisterCollectorsFunction registerCollectors) { + FTDCConfig config; + config.period = Milliseconds(localPeriodMillis.load()); + // Only enable FTDC if our caller says to enable FTDC, MongoS may not have a valid path to write + // files to so update the diagnosticDataCollectionEnabled set parameter to reflect that. + localEnabledFlag.store(startupMode == FTDCStartMode::kStart && localEnabledFlag.load()); + config.enabled = localEnabledFlag.load(); + config.maxFileSizeBytes = localMaxFileSizeMB.load() * 1024 * 1024; + config.maxDirectorySizeBytes = localMaxDirectorySizeMB.load() * 1024 * 1024; + config.maxSamplesPerArchiveMetricChunk = localMaxSamplesPerArchiveMetricChunk.load(); + config.maxSamplesPerInterimMetricChunk = localMaxSamplesPerInterimMetricChunk.load(); + + auto controller = stdx::make_unique(path, config); + + // Install periodic collectors + // These are collected on the period interval in FTDCConfig. + // NOTE: For each command here, there must be an equivalent privilege check in + // GetDiagnosticDataCommand + + // CmdServerStatus + // The "sharding" section is filtered out because at this time it only consists of strings in + // migration status. This section triggers too many schema changes in the serverStatus which + // hurt ftdc compression efficiency, because its output varies depending on the list of active + // migrations. + // TODO: do we need to enable "sharding" on MongoS? + controller->addPeriodicCollector(stdx::make_unique( + "serverStatus", + "serverStatus", + "", + BSON("serverStatus" << 1 << "tcMalloc" << true << "sharding" << false))); + + registerCollectors(controller.get()); + + // Install System Metric Collector as a periodic collector + installSystemMetricsCollector(controller.get()); + + // Install file rotation collectors + // These are collected on each file rotation. + + // CmdBuildInfo + controller->addOnRotateCollector(stdx::make_unique( + "buildInfo", "buildInfo", "", BSON("buildInfo" << 1))); + + // CmdGetCmdLineOpts + controller->addOnRotateCollector(stdx::make_unique( + "getCmdLineOpts", "getCmdLineOpts", "", BSON("getCmdLineOpts" << 1))); + + // HostInfoCmd + controller->addOnRotateCollector(stdx::make_unique( + "hostInfo", "hostInfo", "", BSON("hostInfo" << 1))); + + // Install the new controller + auto& staticFTDC = getFTDCController(getGlobalServiceContext()); + + staticFTDC = std::move(controller); + + staticFTDC->start(); +} + +void stopFTDC() { + auto controller = getGlobalFTDCController(); + + if (controller) { + controller->stop(); + } +} + +FTDCController* FTDCController::get(ServiceContext* serviceContext) { + return getFTDCController(serviceContext).get(); +} + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_server.h b/src/mongo/db/ftdc/ftdc_server.h new file mode 100644 index 00000000000..4b4583ae153 --- /dev/null +++ b/src/mongo/db/ftdc/ftdc_server.h @@ -0,0 +1,103 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include + +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/commands.h" +#include "mongo/db/ftdc/collector.h" +#include "mongo/db/ftdc/controller.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/operation_context.h" +#include "mongo/stdx/functional.h" + +namespace mongo { + +/** + * Function that allows FTDC server components to register their own collectors as needed. + */ +using RegisterCollectorsFunction = stdx::function; + +/** + * An enum that decides whether FTDC will startup as part of startup or if its deferred to later. + */ +enum class FTDCStartMode { + + /** + * Skip starting FTDC since it missing a file storage location. + */ + kSkipStart, + + /** + * Start FTDC because it has a path to store files. + */ + kStart, +}; + +/** + * Start Full Time Data Capture + * Starts 1 thread. + * + * See MongoD and MongoS specific functions. + */ +void startFTDC(boost::filesystem::path& path, + FTDCStartMode startupMode, + RegisterCollectorsFunction registerCollectors); + +/** + * Stop Full Time Data Capture + * + * See MongoD and MongoS specific functions. + */ +void stopFTDC(); + +/** + * A simple FTDC Collector that runs Commands. + */ +class FTDCSimpleInternalCommandCollector final : public FTDCCollectorInterface { +public: + FTDCSimpleInternalCommandCollector(StringData command, + StringData name, + StringData ns, + BSONObj cmdObj); + + void collect(OperationContext* opCtx, BSONObjBuilder& builder) override; + std::string name() const override; + +private: + std::string _name; + std::string _ns; + BSONObj _cmdObj; + + // Not owned + Command* _command; +}; + +} // namespace mongo diff --git a/src/mongo/db/ftdc/ftdc_system_stats.h b/src/mongo/db/ftdc/ftdc_system_stats.h index 20d21ef4f39..97bc0a67cf5 100644 --- a/src/mongo/db/ftdc/ftdc_system_stats.h +++ b/src/mongo/db/ftdc/ftdc_system_stats.h @@ -25,6 +25,7 @@ * delete this exception statement from all source files in the program, * then also delete it in the license file. */ +#pragma once #include diff --git a/src/mongo/db/ftdc/util.cpp b/src/mongo/db/ftdc/util.cpp index d56eb8ca380..243ba90b666 100644 --- a/src/mongo/db/ftdc/util.cpp +++ b/src/mongo/db/ftdc/util.cpp @@ -103,6 +103,19 @@ Date_t roundTime(Date_t now, Milliseconds period) { return Date_t::fromMillisSinceEpoch(next_time); } +boost::filesystem::path getMongoSPath(const boost::filesystem::path& logFile) { + auto base = logFile; + + // Keep stripping file extensions until we are only left with the file name + while (base.has_extension()) { + auto full_path = base.generic_string(); + base = full_path.substr(0, full_path.size() - base.extension().size()); + } + + base += "." + kFTDCDefaultDirectory.toString(); + return base; +} + } // namespace FTDCUtil diff --git a/src/mongo/db/ftdc/util.h b/src/mongo/db/ftdc/util.h index 0cb4d19ce7f..4816c534f96 100644 --- a/src/mongo/db/ftdc/util.h +++ b/src/mongo/db/ftdc/util.h @@ -190,6 +190,11 @@ boost::filesystem::path getInterimTempFile(const boost::filesystem::path& file); */ Date_t roundTime(Date_t now, Milliseconds period); +/** + * Get the storage path for MongoS from the log file path. + */ +boost::filesystem::path getMongoSPath(const boost::filesystem::path& logFile); + } // namespace FTDCUtil } // namespace mongo diff --git a/src/mongo/db/ftdc/util_test.cpp b/src/mongo/db/ftdc/util_test.cpp index aa99006c73a..0adf0f73c15 100644 --- a/src/mongo/db/ftdc/util_test.cpp +++ b/src/mongo/db/ftdc/util_test.cpp @@ -46,4 +46,25 @@ TEST(FTDCUtilTest, TestRoundTime) { checkTime(14, 13, 7); } +// Validate the MongoS FTDC path is computed correctly from a log file path. +TEST(FTDCUtilTest, TestMongoSPath) { + + std::vector> testCases = { + {"/var/log/mongos.log", "/var/log/mongos.diagnostic.data"}, + {"/var/log/mongos.foo.log", "/var/log/mongos.diagnostic.data"}, + {"/var/log/log_file", "/var/log/log_file.diagnostic.data"}, + {"./mongos.log", "./mongos.diagnostic.data"}, + {"../mongos.log", "../mongos.diagnostic.data"}, + {"c:\\var\\log\\mongos.log", "c:\\var\\log\\mongos.diagnostic.data"}, + {"c:\\var\\log\\mongos.foo.log", "c:\\var\\log\\mongos.diagnostic.data"}, + {"c:\\var\\log\\log_file", "c:\\var\\log\\log_file.diagnostic.data"}, + {"/var/some.log/mongos.log", "/var/some.log/mongos.diagnostic.data"}, + {"/var/some.log/log_file", "/var/some.log/log_file.diagnostic.data"}, + }; + + for (const auto& p : testCases) { + ASSERT_EQUALS(FTDCUtil::getMongoSPath(p.first), p.second); + } +} + } // namespace mongo diff --git a/src/mongo/db/fts/unicode/string.cpp b/src/mongo/db/fts/unicode/string.cpp index 10737acc3ed..8c1890f3669 100644 --- a/src/mongo/db/fts/unicode/string.cpp +++ b/src/mongo/db/fts/unicode/string.cpp @@ -30,6 +30,7 @@ #include #include +#include #include "mongo/db/fts/unicode/byte_vector.h" #include "mongo/platform/bits.h" @@ -272,9 +273,15 @@ bool String::substrMatch(const std::string& str, auto haystack = caseFoldAndStripDiacritics(&haystackBuf, str, options, cfMode); auto needle = caseFoldAndStripDiacritics(&needleBuf, find, options, cfMode); - // Case sensitive and diacritic sensitive. +// Case sensitive and diacritic sensitive. +#if BOOST_VERSION < 106200 return boost::algorithm::boyer_moore_search( haystack.begin(), haystack.end(), needle.begin(), needle.end()) != haystack.end(); +#else + return boost::algorithm::boyer_moore_search( + haystack.begin(), haystack.end(), needle.begin(), needle.end()) != + std::make_pair(haystack.end(), haystack.end()); +#endif } } // namespace unicode diff --git a/src/mongo/db/index/expression_keys_private.cpp b/src/mongo/db/index/expression_keys_private.cpp index d58c5ce1dc1..9fe3365f393 100644 --- a/src/mongo/db/index/expression_keys_private.cpp +++ b/src/mongo/db/index/expression_keys_private.cpp @@ -32,6 +32,7 @@ #include +#include "mongo/bson/bsonelement_comparator_interface.h" #include "mongo/bson/simple_bsonobj_comparator.h" #include "mongo/db/bson/dotted_path_support.h" #include "mongo/db/field_ref.h" @@ -247,7 +248,7 @@ void ExpressionKeysPrivate::get2DKeys(const BSONObj& obj, const TwoDIndexingParams& params, BSONObjSet* keys, std::vector* locs) { - BSONElementMSet bSet; + BSONElementMultiSet bSet; // Get all the nested location fields, but don't return individual elements from // the last array, if it exists. @@ -256,7 +257,7 @@ void ExpressionKeysPrivate::get2DKeys(const BSONObj& obj, if (bSet.empty()) return; - for (BSONElementMSet::iterator setI = bSet.begin(); setI != bSet.end(); ++setI) { + for (BSONElementMultiSet::iterator setI = bSet.begin(); setI != bSet.end(); ++setI) { BSONElement geo = *setI; if (geo.eoo() || !geo.isABSONObj()) diff --git a/src/mongo/db/instance.cpp b/src/mongo/db/instance.cpp index c10d0a3174b..5bbf3339a37 100644 --- a/src/mongo/db/instance.cpp +++ b/src/mongo/db/instance.cpp @@ -206,14 +206,15 @@ void receivedCommand(OperationContext* txn, const int32_t responseToMsgId = message.header().getId(); - DbMessage dbMessage(message); - QueryMessage queryMessage(dbMessage); - CurOp* op = CurOp::get(txn); - rpc::LegacyReplyBuilder builder{}; try { + DbMessage dbMessage(message); + + // Can throw, so make sure it's under the try statement. + QueryMessage queryMessage(dbMessage); + // This will throw if the request is on an invalid namespace. rpc::LegacyRequest request{&message}; // Auth checking for Commands happens later. diff --git a/src/mongo/db/log_process_details.cpp b/src/mongo/db/log_process_details.cpp index 125106fd331..ea6e9d945a8 100644 --- a/src/mongo/db/log_process_details.cpp +++ b/src/mongo/db/log_process_details.cpp @@ -34,6 +34,9 @@ #include "mongo/db/log_process_details.h" +#include "mongo/db/repl/repl_set_config.h" +#include "mongo/db/repl/replication_coordinator.h" +#include "mongo/db/repl/replication_coordinator_global.h" #include "mongo/db/server_options.h" #include "mongo/db/server_options_helpers.h" #include "mongo/util/log.h" @@ -51,6 +54,7 @@ void logProcessDetails() { auto&& vii = VersionInfoInterface::instance(); log() << mongodVersion(vii); vii.logBuildInfo(); + printCommandLineOpts(); } @@ -59,6 +63,19 @@ void logProcessDetailsForLogRotate() { << (is32bit() ? " 32" : " 64") << "-bit " << "host=" << getHostNameCached(); + auto replCoord = repl::getGlobalReplicationCoordinator(); + if (replCoord != nullptr && + replCoord->getReplicationMode() == repl::ReplicationCoordinator::modeReplSet) { + auto rsConfig = replCoord->getConfig(); + + if (rsConfig.isInitialized()) { + log() << "Replica Set Config: " << rsConfig.toBSON(); + log() << "Replica Set Member State: " << (replCoord->getMemberState()).toString(); + } else { + log() << "Node currently has no Replica Set Config."; + } + } + logProcessDetails(); } diff --git a/src/mongo/db/matcher/expression_leaf.cpp b/src/mongo/db/matcher/expression_leaf.cpp index d562bff141e..0bf8d8685bc 100644 --- a/src/mongo/db/matcher/expression_leaf.cpp +++ b/src/mongo/db/matcher/expression_leaf.cpp @@ -202,7 +202,7 @@ void ComparisonMatchExpression::debugString(StringBuilder& debug, int level) con } void ComparisonMatchExpression::serialize(BSONObjBuilder* out) const { - string opString = ""; + std::string opString = ""; switch (matchType()) { case LT: opString = "$lt"; @@ -656,29 +656,31 @@ bool InMatchExpression::equivalent(const MatchExpression* other) const { void InMatchExpression::_doSetCollator(const CollatorInterface* collator) { _collator = collator; + _eltCmp = BSONElementComparator(BSONElementComparator::FieldNamesMode::kIgnore, _collator); // We need to re-compute '_equalitySet', since our set comparator has changed. - BSONElementSet equalitiesWithNewComparator( - _originalEqualityVector.begin(), _originalEqualityVector.end(), collator); - _equalitySet = std::move(equalitiesWithNewComparator); + _equalitySet = _eltCmp.makeBSONEltFlatSet(_originalEqualityVector); } -Status InMatchExpression::addEquality(const BSONElement& elt) { - if (elt.type() == BSONType::RegEx) { - return Status(ErrorCodes::BadValue, "InMatchExpression equality cannot be a regex"); - } - if (elt.type() == BSONType::Undefined) { - return Status(ErrorCodes::BadValue, "InMatchExpression equality cannot be undefined"); - } +Status InMatchExpression::setEqualities(std::vector equalities) { + for (auto&& equality : equalities) { + if (equality.type() == BSONType::RegEx) { + return Status(ErrorCodes::BadValue, "InMatchExpression equality cannot be a regex"); + } + if (equality.type() == BSONType::Undefined) { + return Status(ErrorCodes::BadValue, "InMatchExpression equality cannot be undefined"); + } - if (elt.type() == BSONType::jstNULL) { - _hasNull = true; - } - if (elt.type() == BSONType::Array && elt.Obj().isEmpty()) { - _hasEmptyArray = true; + if (equality.type() == BSONType::jstNULL) { + _hasNull = true; + } else if (equality.type() == BSONType::Array && equality.Obj().isEmpty()) { + _hasEmptyArray = true; + } } - _equalitySet.insert(elt); - _originalEqualityVector.push_back(elt); + _originalEqualityVector = std::move(equalities); + + _equalitySet = _eltCmp.makeBSONEltFlatSet(_originalEqualityVector); + return Status::OK(); } @@ -884,7 +886,7 @@ void BitTestMatchExpression::debugString(StringBuilder& debug, int level) const } void BitTestMatchExpression::serialize(BSONObjBuilder* out) const { - string opString = ""; + std::string opString = ""; switch (matchType()) { case BITS_ALL_SET: diff --git a/src/mongo/db/matcher/expression_leaf.h b/src/mongo/db/matcher/expression_leaf.h index 128d3070737..5761f2ca2a2 100644 --- a/src/mongo/db/matcher/expression_leaf.h +++ b/src/mongo/db/matcher/expression_leaf.h @@ -30,9 +30,11 @@ #pragma once +#include "mongo/bson/bsonelement_comparator.h" #include "mongo/bson/bsonmisc.h" #include "mongo/bson/bsonobj.h" #include "mongo/db/matcher/expression.h" +#include "mongo/db/query/collation/collator_interface.h" #include "mongo/stdx/memory.h" #include "mongo/stdx/unordered_map.h" @@ -330,7 +332,10 @@ public: */ class InMatchExpression : public LeafMatchExpression { public: - InMatchExpression() : LeafMatchExpression(MATCH_IN) {} + InMatchExpression() + : LeafMatchExpression(MATCH_IN), + _eltCmp(BSONElementComparator::FieldNamesMode::kIgnore, _collator), + _equalitySet(_eltCmp.makeBSONEltFlatSet(_originalEqualityVector)) {} Status init(StringData path); @@ -349,11 +354,11 @@ public: */ virtual void _doSetCollator(const CollatorInterface* collator); - Status addEquality(const BSONElement& elt); + Status setEqualities(std::vector equalities); Status addRegex(std::unique_ptr expr); - const BSONElementSet& getEqualities() const { + const BSONEltFlatSet& getEqualities() const { return _equalitySet; } @@ -380,17 +385,20 @@ private: // Whether or not '_equalities' has an empty array element in it. bool _hasEmptyArray = false; - // Collator used to compare elements. By default, simple binary comparison will be used. + // Collator used to construct '_eltCmp'; const CollatorInterface* _collator = nullptr; - // Set of equality elements associated with this expression. '_collator' is used as a comparator - // for this set. - BSONElementSet _equalitySet; + // Comparator used to compare elements. By default, simple binary comparison will be used. + BSONElementComparator _eltCmp; // Original container of equality elements, including duplicates. Needed for re-computing // '_equalitySet' in case '_collator' changes after elements have been added. std::vector _originalEqualityVector; + // Set of equality elements associated with this expression. '_eltCmp' is used as a comparator + // for this set. + BSONEltFlatSet _equalitySet; + // Container of regex elements this object owns. std::vector> _regexes; }; diff --git a/src/mongo/db/matcher/expression_leaf_test.cpp b/src/mongo/db/matcher/expression_leaf_test.cpp index 293ed59c85c..1c3f7795a95 100644 --- a/src/mongo/db/matcher/expression_leaf_test.cpp +++ b/src/mongo/db/matcher/expression_leaf_test.cpp @@ -1501,7 +1501,8 @@ TEST(InMatchExpression, MatchesElementSingle) { BSONObj match = BSON("a" << 1); BSONObj notMatch = BSON("a" << 2); InMatchExpression in; - in.addEquality(operand.firstElement()); + std::vector equalities{operand.firstElement()}; + ASSERT_OK(in.setEqualities(std::move(equalities))); ASSERT(in.matchesSingleElement(match["a"])); ASSERT(!in.matchesSingleElement(notMatch["a"])); } @@ -1519,10 +1520,8 @@ TEST(InMatchExpression, MatchesEmpty) { TEST(InMatchExpression, MatchesElementMultiple) { BSONObj operand = BSON_ARRAY(1 << "r" << true << 1); InMatchExpression in; - in.addEquality(operand[0]); - in.addEquality(operand[1]); - in.addEquality(operand[2]); - in.addEquality(operand[3]); + std::vector equalities{operand[0], operand[1], operand[2], operand[3]}; + ASSERT_OK(in.setEqualities(std::move(equalities))); BSONObj matchFirst = BSON("a" << 1); BSONObj matchSecond = BSON("a" @@ -1540,7 +1539,8 @@ TEST(InMatchExpression, MatchesScalar) { BSONObj operand = BSON_ARRAY(5); InMatchExpression in; in.init("a"); - in.addEquality(operand.firstElement()); + std::vector equalities{operand.firstElement()}; + ASSERT_OK(in.setEqualities(std::move(equalities))); ASSERT(in.matchesBSON(BSON("a" << 5.0), NULL)); ASSERT(!in.matchesBSON(BSON("a" << 4), NULL)); @@ -1550,7 +1550,8 @@ TEST(InMatchExpression, MatchesArrayValue) { BSONObj operand = BSON_ARRAY(5); InMatchExpression in; in.init("a"); - in.addEquality(operand.firstElement()); + std::vector equalities{operand.firstElement()}; + ASSERT_OK(in.setEqualities(std::move(equalities))); ASSERT(in.matchesBSON(BSON("a" << BSON_ARRAY(5.0 << 6)), NULL)); ASSERT(!in.matchesBSON(BSON("a" << BSON_ARRAY(6 << 7)), NULL)); @@ -1562,7 +1563,8 @@ TEST(InMatchExpression, MatchesNull) { InMatchExpression in; in.init("a"); - in.addEquality(operand.firstElement()); + std::vector equalities{operand.firstElement()}; + ASSERT_OK(in.setEqualities(std::move(equalities))); ASSERT(in.matchesBSON(BSONObj(), NULL)); ASSERT(in.matchesBSON(BSON("a" << BSONNULL), NULL)); @@ -1576,15 +1578,16 @@ TEST(InMatchExpression, MatchesUndefined) { InMatchExpression in; in.init("a"); - Status s = in.addEquality(operand.firstElement()); - ASSERT_NOT_OK(s); + std::vector equalities{operand.firstElement()}; + ASSERT_NOT_OK(in.setEqualities(std::move(equalities))); } TEST(InMatchExpression, MatchesMinKey) { BSONObj operand = BSON_ARRAY(MinKey); InMatchExpression in; in.init("a"); - in.addEquality(operand.firstElement()); + std::vector equalities{operand.firstElement()}; + ASSERT_OK(in.setEqualities(std::move(equalities))); ASSERT(in.matchesBSON(BSON("a" << MinKey), NULL)); ASSERT(!in.matchesBSON(BSON("a" << MaxKey), NULL)); @@ -1595,7 +1598,8 @@ TEST(InMatchExpression, MatchesMaxKey) { BSONObj operand = BSON_ARRAY(MaxKey); InMatchExpression in; in.init("a"); - in.addEquality(operand.firstElement()); + std::vector equalities{operand.firstElement()}; + ASSERT_OK(in.setEqualities(std::move(equalities))); ASSERT(in.matchesBSON(BSON("a" << MaxKey), NULL)); ASSERT(!in.matchesBSON(BSON("a" << MinKey), NULL)); @@ -1606,9 +1610,8 @@ TEST(InMatchExpression, MatchesFullArray) { BSONObj operand = BSON_ARRAY(BSON_ARRAY(1 << 2) << 4 << 5); InMatchExpression in; in.init("a"); - in.addEquality(operand[0]); - in.addEquality(operand[1]); - in.addEquality(operand[2]); + std::vector equalities{operand[0], operand[1], operand[2]}; + ASSERT_OK(in.setEqualities(std::move(equalities))); ASSERT(in.matchesBSON(BSON("a" << BSON_ARRAY(1 << 2)), NULL)); ASSERT(!in.matchesBSON(BSON("a" << BSON_ARRAY(1 << 2 << 3)), NULL)); @@ -1620,8 +1623,8 @@ TEST(InMatchExpression, ElemMatchKey) { BSONObj operand = BSON_ARRAY(5 << 2); InMatchExpression in; in.init("a"); - in.addEquality(operand[0]); - in.addEquality(operand[1]); + std::vector equalities{operand[0], operand[1]}; + ASSERT_OK(in.setEqualities(std::move(equalities))); MatchDetails details; details.requestElemMatchKey(); @@ -1639,7 +1642,8 @@ TEST(InMatchExpression, InMatchExpressionsWithDifferentNumbersOfElementsAreUnequ << "string"); InMatchExpression eq1; InMatchExpression eq2; - eq1.addEquality(obj.firstElement()); + std::vector equalities{obj.firstElement()}; + ASSERT_OK(eq1.setEqualities(std::move(equalities))); ASSERT(!eq1.equivalent(&eq2)); } @@ -1675,8 +1679,12 @@ TEST(InMatchExpression, InMatchExpressionsWithCollationEquivalentElementsAreEqua InMatchExpression eq2; eq2.setCollator(&collator2); - eq1.addEquality(obj1.firstElement()); - eq2.addEquality(obj2.firstElement()); + std::vector equalities1{obj1.firstElement()}; + ASSERT_OK(eq1.setEqualities(std::move(equalities1))); + + std::vector equalities2{obj2.firstElement()}; + ASSERT_OK(eq2.setEqualities(std::move(equalities2))); + ASSERT(eq1.equivalent(&eq2)); } @@ -1692,8 +1700,12 @@ TEST(InMatchExpression, InMatchExpressionsWithCollationNonEquivalentElementsAreU InMatchExpression eq2; eq2.setCollator(&collator2); - eq1.addEquality(obj1.firstElement()); - eq2.addEquality(obj2.firstElement()); + std::vector equalities1{obj1.firstElement()}; + ASSERT_OK(eq1.setEqualities(std::move(equalities1))); + + std::vector equalities2{obj2.firstElement()}; + ASSERT_OK(eq2.setEqualities(std::move(equalities2))); + ASSERT(!eq1.equivalent(&eq2)); } @@ -1702,7 +1714,8 @@ TEST(InMatchExpression, StringMatchingWithNullCollatorUsesBinaryComparison) { BSONObj notMatch = BSON("a" << "string2"); InMatchExpression in; - in.addEquality(operand.firstElement()); + std::vector equalities{operand.firstElement()}; + ASSERT_OK(in.setEqualities(std::move(equalities))); ASSERT(!in.matchesSingleElement(notMatch["a"])); } @@ -1713,7 +1726,8 @@ TEST(InMatchExpression, StringMatchingRespectsCollation) { CollatorInterfaceMock collator(CollatorInterfaceMock::MockType::kAlwaysEqual); InMatchExpression in; in.setCollator(&collator); - in.addEquality(operand.firstElement()); + std::vector equalities{operand.firstElement()}; + ASSERT_OK(in.setEqualities(std::move(equalities))); ASSERT(in.matchesSingleElement(match["a"])); } @@ -1726,8 +1740,8 @@ TEST(InMatchExpression, ChangingCollationAfterAddingEqualitiesPreservesEqualitie CollatorInterfaceMock collatorReverseString(CollatorInterfaceMock::MockType::kReverseString); InMatchExpression in; in.setCollator(&collatorAlwaysEqual); - in.addEquality(obj1.firstElement()); - in.addEquality(obj2.firstElement()); + std::vector equalities{obj1.firstElement(), obj2.firstElement()}; + ASSERT_OK(in.setEqualities(std::move(equalities))); ASSERT(in.getEqualities().size() == 1); in.setCollator(&collatorReverseString); ASSERT(in.getEqualities().size() == 2); diff --git a/src/mongo/db/matcher/expression_parser.cpp b/src/mongo/db/matcher/expression_parser.cpp index 2d1399edcf9..9bfc2d75f3e 100644 --- a/src/mongo/db/matcher/expression_parser.cpp +++ b/src/mongo/db/matcher/expression_parser.cpp @@ -588,6 +588,7 @@ Status MatchExpressionParser::_parseInExpression(InMatchExpression* inExpression const BSONObj& theArray, const CollatorInterface* collator) { inExpression->setCollator(collator); + std::vector equalities; BSONObjIterator i(theArray); while (i.more()) { BSONElement e = i.next(); @@ -606,12 +607,10 @@ Status MatchExpressionParser::_parseInExpression(InMatchExpression* inExpression if (!s.isOK()) return s; } else { - Status s = inExpression->addEquality(e); - if (!s.isOK()) - return s; + equalities.push_back(e); } } - return Status::OK(); + return inExpression->setEqualities(std::move(equalities)); } StatusWithMatchExpression MatchExpressionParser::_parseType(const char* name, diff --git a/src/mongo/db/mongod_options.cpp b/src/mongo/db/mongod_options.cpp index 307d6ace41a..c4b2283ef40 100644 --- a/src/mongo/db/mongod_options.cpp +++ b/src/mongo/db/mongod_options.cpp @@ -58,6 +58,7 @@ using std::cout; using std::endl; using std::string; + MongodGlobalParams mongodGlobalParams; extern DiagLog _diaglog; @@ -685,14 +686,13 @@ Status validateMongodOptions(const moe::Environment& params) { } } - bool isClusterRoleShard = false; + bool isClusterRoleShard = params.count("shardsvr"); if (params.count("sharding.clusterRole")) { auto clusterRole = params["sharding.clusterRole"].as(); - isClusterRoleShard = (clusterRole == "shardsvr"); + isClusterRoleShard = isClusterRoleShard || (clusterRole == "shardsvr"); } - if ((isClusterRoleShard || params.count("shardsvr")) && - !params.count("sharding._overrideShardIdentity")) { + if (isClusterRoleShard && !params.count("sharding._overrideShardIdentity")) { return Status( ErrorCodes::BadValue, "shardsvr cluster role with queryableBackupMode requires _overrideShardIdentity"); @@ -1328,6 +1328,21 @@ Status storeMongodOptions(const moe::Environment& params) { log() << endl; } + bool isClusterRoleShard = params.count("shardsvr"); + bool isClusterRoleConfig = params.count("configsvr"); + if (params.count("sharding.clusterRole")) { + auto clusterRole = params["sharding.clusterRole"].as(); + isClusterRoleShard = isClusterRoleShard || (clusterRole == "shardsvr"); + isClusterRoleConfig = isClusterRoleConfig || (clusterRole == "configsvr"); + } + + if ((isClusterRoleShard || isClusterRoleConfig) && skipShardingConfigurationChecks) { + auto clusterRoleStr = isClusterRoleConfig ? "--configsvr" : "--shardsvr"; + return Status(ErrorCodes::BadValue, + str::stream() << "Can not specify " << clusterRoleStr + << " and set skipShardingConfigurationChecks=true"); + } + setGlobalReplSettings(replSettings); return Status::OK(); } diff --git a/src/mongo/db/mongod_options.h b/src/mongo/db/mongod_options.h index 27fb778cfdf..481edacc96f 100644 --- a/src/mongo/db/mongod_options.h +++ b/src/mongo/db/mongod_options.h @@ -50,6 +50,8 @@ struct MongodGlobalParams { MongodGlobalParams() : scriptingEnabled(true) {} }; +extern bool skipShardingConfigurationChecks; + extern MongodGlobalParams mongodGlobalParams; Status addMongodOptions(moe::OptionSection* options); diff --git a/src/mongo/db/namespace_string-inl.h b/src/mongo/db/namespace_string-inl.h index 0ff45fca39e..7ad830f2481 100644 --- a/src/mongo/db/namespace_string-inl.h +++ b/src/mongo/db/namespace_string-inl.h @@ -118,7 +118,7 @@ inline bool NamespaceString::validCollectionName(StringData coll) { return true; } -inline NamespaceString::NamespaceString() : _ns(), _dotIndex(0) {} +inline NamespaceString::NamespaceString() : _ns(), _dotIndex(std::string::npos) {} inline NamespaceString::NamespaceString(StringData nsIn) { _ns = nsIn.toString(); // copy to our buffer _dotIndex = _ns.find('.'); diff --git a/src/mongo/db/namespace_string_test.cpp b/src/mongo/db/namespace_string_test.cpp index 69884bd5ca9..69742c797bf 100644 --- a/src/mongo/db/namespace_string_test.cpp +++ b/src/mongo/db/namespace_string_test.cpp @@ -272,4 +272,16 @@ TEST(NamespaceStringTest, makeListIndexesNSIsCorrect) { ASSERT(ns.isListIndexesCursorNS()); ASSERT_EQUALS(NamespaceString("DB.COLL"), ns.getTargetNSForListIndexes()); } + +TEST(NamespaceStringTest, EmptyNSStringReturnsEmptyColl) { + NamespaceString nss{}; + ASSERT_TRUE(nss.toString().empty()); + ASSERT_EQ(nss.coll(), StringData{}); +} + +TEST(NamespaceStringTest, EmptyNSStringReturnsEmptyDb) { + NamespaceString nss{}; + ASSERT_TRUE(nss.toString().empty()); + ASSERT_EQ(nss.db(), StringData{}); +} } diff --git a/src/mongo/db/op_observer_impl.cpp b/src/mongo/db/op_observer_impl.cpp index 00560851a7a..91bc3317d48 100644 --- a/src/mongo/db/op_observer_impl.cpp +++ b/src/mongo/db/op_observer_impl.cpp @@ -42,9 +42,12 @@ #include "mongo/db/server_options.h" #include "mongo/db/views/durable_view_catalog.h" #include "mongo/scripting/engine.h" +#include "mongo/util/fail_point_service.h" namespace mongo { +MONGO_FP_DECLARE(failCollectionUpdates); + void OpObserverImpl::onCreateIndex(OperationContext* txn, const std::string& ns, BSONObj indexDoc, @@ -80,7 +83,7 @@ void OpObserverImpl::onInserts(OperationContext* txn, if (nss.ns() == FeatureCompatibilityVersion::kCollection) { for (auto it = begin; it != end; it++) { - FeatureCompatibilityVersion::onInsertOrUpdate(*it); + FeatureCompatibilityVersion::onInsertOrUpdate(txn, *it); } } @@ -94,6 +97,20 @@ void OpObserverImpl::onInserts(OperationContext* txn, } void OpObserverImpl::onUpdate(OperationContext* txn, const OplogUpdateEntryArgs& args) { + MONGO_FAIL_POINT_BLOCK(failCollectionUpdates, extraData) { + auto collElem = extraData.getData()["collectionNS"]; + // If the failpoint specifies no collection or matches the existing one, fail. + if (!collElem || args.ns == collElem.String()) { + uasserted(40654, + str::stream() << "failCollectionUpdates failpoint enabled, namespace: " + << args.ns + << ", update: " + << args.update + << " on document with " + << args.criteria); + } + } + // Do not log a no-op operation; see SERVER-21738 if (args.update.isEmpty()) { return; @@ -119,7 +136,7 @@ void OpObserverImpl::onUpdate(OperationContext* txn, const OplogUpdateEntryArgs& } if (args.ns == FeatureCompatibilityVersion::kCollection) { - FeatureCompatibilityVersion::onInsertOrUpdate(args.updatedDoc); + FeatureCompatibilityVersion::onInsertOrUpdate(txn, args.updatedDoc); } } @@ -162,7 +179,7 @@ void OpObserverImpl::onDelete(OperationContext* txn, DurableViewCatalog::onExternalChange(txn, ns); } if (ns.ns() == FeatureCompatibilityVersion::kCollection) { - FeatureCompatibilityVersion::onDelete(deleteState.idDoc); + FeatureCompatibilityVersion::onDelete(txn, deleteState.idDoc); } } @@ -221,7 +238,7 @@ void OpObserverImpl::onDropDatabase(OperationContext* txn, const std::string& db repl::logOp(txn, "c", dbName.c_str(), cmdObj, nullptr, false); if (NamespaceString(dbName).db() == FeatureCompatibilityVersion::kDatabase) { - FeatureCompatibilityVersion::onDropCollection(); + FeatureCompatibilityVersion::onDropCollection(txn); } getGlobalAuthorizationManager()->logOp(txn, "c", dbName.c_str(), cmdObj, nullptr); @@ -243,7 +260,7 @@ void OpObserverImpl::onDropCollection(OperationContext* txn, } if (collectionName.ns() == FeatureCompatibilityVersion::kCollection) { - FeatureCompatibilityVersion::onDropCollection(); + FeatureCompatibilityVersion::onDropCollection(txn); } getGlobalAuthorizationManager()->logOp(txn, "c", dbName.c_str(), cmdObj, nullptr); @@ -276,11 +293,10 @@ void OpObserverImpl::onRenameCollection(OperationContext* txn, << dropTarget); repl::logOp(txn, "c", dbName.c_str(), cmdObj, nullptr, false); - if (fromCollection.coll() == DurableViewCatalog::viewsCollectionName() || - toCollection.coll() == DurableViewCatalog::viewsCollectionName()) { - DurableViewCatalog::onExternalChange( - txn, NamespaceString(DurableViewCatalog::viewsCollectionName())); - } + if (fromCollection.isSystemDotViews()) + DurableViewCatalog::onExternalChange(txn, fromCollection); + if (toCollection.isSystemDotViews()) + DurableViewCatalog::onExternalChange(txn, toCollection); getGlobalAuthorizationManager()->logOp(txn, "c", dbName.c_str(), cmdObj, nullptr); logOpForDbHash(txn, dbName.c_str()); diff --git a/src/mongo/db/operation_context.cpp b/src/mongo/db/operation_context.cpp index 8151fe97dfa..41b54eb6611 100644 --- a/src/mongo/db/operation_context.cpp +++ b/src/mongo/db/operation_context.cpp @@ -325,7 +325,7 @@ StatusWith OperationContext::waitForConditionOrInterruptNoAsser deadline = std::min(deadline, getDeadline()); } - const auto waitStatus = [&] { + const auto waitStatus = [&]() -> stdx::cv_status { if (Date_t::max() == deadline) { cv.wait(m); return stdx::cv_status::no_timeout; diff --git a/src/mongo/db/operation_context.h b/src/mongo/db/operation_context.h index 2c8be360a77..79455e47309 100644 --- a/src/mongo/db/operation_context.h +++ b/src/mongo/db/operation_context.h @@ -201,6 +201,40 @@ public: return stdx::cv_status::no_timeout; } + /** + * Waits on condition "cv" for "pred" until "pred" returns true, or the given "deadline" + * expires, or this operation is interrupted, or this operation's own deadline expires. + * + * + * If the operation deadline expires or the operation is interrupted, throws a DBException. If + * the given "deadline" expires, returns pred. Otherwise, returns true. + */ + template + bool waitForConditionOrInterruptUntilPred(stdx::condition_variable& cv, + stdx::unique_lock& m, + Date_t deadline, + Pred pred) { + while (!pred()) { + if (stdx::cv_status::timeout == waitForConditionOrInterruptUntil(cv, m, deadline)) { + return pred(); + } + } + return true; + } + + /** + * Same as the predicate form of waitForConditionOrInterruptUntilPred, but takes a relative + * amount of time to wait instead of an absolute time point. + */ + template + bool waitForConditionOrInterruptFor(stdx::condition_variable& cv, + stdx::unique_lock& m, + Milliseconds duration, + Pred pred) { + return waitForConditionOrInterruptUntilPred( + cv, m, getServiceContext()->getPreciseClockSource()->now() + duration, pred); + } + /** * Same as waitForConditionOrInterruptUntil, except returns StatusWith and * non-ok status indicates the error instead of a DBException. diff --git a/src/mongo/db/operation_context_test.cpp b/src/mongo/db/operation_context_test.cpp index 06c6575208a..ca77b8c316b 100644 --- a/src/mongo/db/operation_context_test.cpp +++ b/src/mongo/db/operation_context_test.cpp @@ -231,7 +231,7 @@ public: Date_t maxTime) { auto barrier = std::make_shared(2); - auto task = stdx::packaged_task([=] { + auto task = stdx::packaged_task([=]() -> stdx::cv_status { if (maxTime < Date_t::max()) { txn->setDeadlineByDate(maxTime); } diff --git a/src/mongo/db/ops/delete.cpp b/src/mongo/db/ops/delete.cpp index 7f509308ad4..6a7da15ee74 100644 --- a/src/mongo/db/ops/delete.cpp +++ b/src/mongo/db/ops/delete.cpp @@ -63,19 +63,11 @@ long long deleteObjects(OperationContext* txn, ParsedDelete parsedDelete(txn, &request); uassertStatusOK(parsedDelete.parseRequest()); - auto client = txn->getClient(); - auto lastOpAtOperationStart = repl::ReplClientInfo::forClient(client).getLastOp(); - std::unique_ptr exec = uassertStatusOK( getExecutorDelete(txn, &CurOp::get(txn)->debug(), collection, &parsedDelete)); uassertStatusOK(exec->executePlan()); - // No-ops need to reset lastOp in the client, for write concern. - if (repl::ReplClientInfo::forClient(client).getLastOp() == lastOpAtOperationStart) { - repl::ReplClientInfo::forClient(client).setLastOpToSystemLastOpTime(txn); - } - return DeleteStage::getNumDeleted(*exec); } diff --git a/src/mongo/db/ops/modifier_add_to_set.cpp b/src/mongo/db/ops/modifier_add_to_set.cpp index a565800d749..416d487e0f4 100644 --- a/src/mongo/db/ops/modifier_add_to_set.cpp +++ b/src/mongo/db/ops/modifier_add_to_set.cpp @@ -225,6 +225,8 @@ Status ModifierAddToSet::prepare(mb::Element root, StringData matchedField, Exec // Locate the field name in 'root'. Status status = pathsupport::findLongestPrefix( _fieldRef, root, &_preparedState->idxFound, &_preparedState->elemFound); + const auto elemFoundIsArray = + _preparedState->elemFound.ok() && _preparedState->elemFound.getType() == BSONType::Array; // FindLongestPrefix may say the path does not exist at all, which is fine here, or // that the path was not viable or otherwise wrong, in which case, the mod cannot @@ -248,6 +250,12 @@ Status ModifierAddToSet::prepare(mb::Element root, StringData matchedField, Exec if (!_preparedState->elemFound.ok() || _preparedState->idxFound < (_fieldRef.numParts() - 1)) { // If no target element exists, we will simply be creating a new array. _preparedState->addAll = true; + + if (elemFoundIsArray) { + // Report that an existing array will gain a new element as a result of this mod. + execInfo->indexOfArrayWithNewElement[0] = _preparedState->idxFound; + } + return Status::OK(); } diff --git a/src/mongo/db/ops/modifier_add_to_set_test.cpp b/src/mongo/db/ops/modifier_add_to_set_test.cpp index 235a65239e6..63d5a7136d4 100644 --- a/src/mongo/db/ops/modifier_add_to_set_test.cpp +++ b/src/mongo/db/ops/modifier_add_to_set_test.cpp @@ -459,4 +459,41 @@ TEST(Collation, AddToSetWithEachRespectsCollation) { ASSERT_EQUALS(doc, fromjson("{ a : ['abc', 'bdc'] }")); } + +TEST(IndexedMod, PrepareReportCreatedArrayElement) { + Document doc(fromjson("{a: [{b: 0}]}")); + Mod mod(fromjson("{$addToSet: {'a.1.c': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.c"); + ASSERT_TRUE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_EQUALS(*execInfo.indexOfArrayWithNewElement[0], 0u); + ASSERT_FALSE(execInfo.noOp); +} + +TEST(IndexedMod, PrepareDoNotReportModifiedArrayElement) { + Document doc(fromjson("{a: [{b: 0}]}")); + Mod mod(fromjson("{$addToSet: {'a.0.c': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0.c"); + ASSERT_FALSE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_FALSE(execInfo.noOp); +} + +TEST(IndexedMod, PrepareDoNotReportCreatedNumericObjectField) { + Document doc(fromjson("{a: {'0': {b: 0}}}")); + Mod mod(fromjson("{$addToSet: {'a.1.c': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.c"); + ASSERT_FALSE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_FALSE(execInfo.noOp); +} } // namespace diff --git a/src/mongo/db/ops/modifier_bit.cpp b/src/mongo/db/ops/modifier_bit.cpp index fb7ae3f45eb..99296911e95 100644 --- a/src/mongo/db/ops/modifier_bit.cpp +++ b/src/mongo/db/ops/modifier_bit.cpp @@ -164,6 +164,8 @@ Status ModifierBit::prepare(mutablebson::Element root, // Locate the field name in 'root'. Status status = pathsupport::findLongestPrefix( _fieldRef, root, &_preparedState->idxFound, &_preparedState->elemFound); + const auto elemFoundIsArray = + _preparedState->elemFound.ok() && _preparedState->elemFound.getType() == BSONType::Array; // FindLongestPrefix may say the path does not exist at all, which is fine here, or @@ -189,6 +191,12 @@ Status ModifierBit::prepare(mutablebson::Element root, // If no target element exists, the value we will write is the result of applying // the operation to a zero-initialized integer element. _preparedState->newValue = apply(SafeNum(static_cast(0))); + + if (elemFoundIsArray) { + // Report that an existing array will gain a new element as a result of this mod. + execInfo->indexOfArrayWithNewElement[0] = _preparedState->idxFound; + } + return Status::OK(); } diff --git a/src/mongo/db/ops/modifier_bit_test.cpp b/src/mongo/db/ops/modifier_bit_test.cpp index dbc4d7ba0b2..5b8b0131ea7 100644 --- a/src/mongo/db/ops/modifier_bit_test.cpp +++ b/src/mongo/db/ops/modifier_bit_test.cpp @@ -755,4 +755,41 @@ TEST(DbUpdateTests, Bit1_4_Combined) { ASSERT_EQUALS(BSON("$set" << BSON("x" << ((3 | 2) & 8))), logDoc); } +TEST(IndexedMod, PrepareReportCreatedArrayElement) { + Document doc(fromjson("{a: [{b: 0}]}")); + Mod mod(fromjson("{$bit: {'a.1.c': {and: NumberInt(1)}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.c"); + ASSERT_TRUE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_EQUALS(*execInfo.indexOfArrayWithNewElement[0], 0u); + ASSERT_FALSE(execInfo.noOp); +} + +TEST(IndexedMod, PrepareDoNotReportModifiedArrayElement) { + Document doc(fromjson("{a: [{b: NumberInt(0)}]}")); + Mod mod(fromjson("{$bit: {'a.0.c': {or: NumberInt(1)}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0.c"); + ASSERT_FALSE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_FALSE(execInfo.noOp); +} + +TEST(IndexedMod, PrepareDoNotReportCreatedNumericObjectField) { + Document doc(fromjson("{a: {'0': {b: 0}}}")); + Mod mod(fromjson("{$bit: {'a.1.c': {and: NumberInt(1)}}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.c"); + ASSERT_FALSE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_FALSE(execInfo.noOp); +} + } // namespace diff --git a/src/mongo/db/ops/modifier_compare.cpp b/src/mongo/db/ops/modifier_compare.cpp index 4e366f90ee2..7db7abd09ca 100644 --- a/src/mongo/db/ops/modifier_compare.cpp +++ b/src/mongo/db/ops/modifier_compare.cpp @@ -110,6 +110,8 @@ Status ModifierCompare::prepare(mutablebson::Element root, // be created during the apply. Status status = pathsupport::findLongestPrefix( _updatePath, root, &_preparedState->idxFound, &_preparedState->elemFound); + const auto elemFoundIsArray = + _preparedState->elemFound.ok() && _preparedState->elemFound.getType() == BSONType::Array; // FindLongestPrefix may say the path does not exist at all, which is fine here, or // that the path was not viable or otherwise wrong, in which case, the mod cannot @@ -128,6 +130,11 @@ Status ModifierCompare::prepare(mutablebson::Element root, _preparedState->idxFound == (_updatePath.numParts() - 1)); if (!destExists) { execInfo->noOp = false; + + if (elemFoundIsArray) { + // Report that an existing array will gain a new element as a result of this mod. + execInfo->indexOfArrayWithNewElement[0] = _preparedState->idxFound; + } } else { const int compareVal = _preparedState->elemFound.compareWithBSONElement(_val, _collator, false); diff --git a/src/mongo/db/ops/modifier_compare_test.cpp b/src/mongo/db/ops/modifier_compare_test.cpp index 9bc036b776b..edaba6a2a68 100644 --- a/src/mongo/db/ops/modifier_compare_test.cpp +++ b/src/mongo/db/ops/modifier_compare_test.cpp @@ -341,4 +341,41 @@ TEST(Collation, MaxRespectsCollationFromSetCollator) { ASSERT_OK(mod.apply()); ASSERT_EQUALS(fromjson("{a : 'abd'}"), doc); } + +TEST(IndexedMod, PrepareReportCreatedArrayElement) { + Document doc(fromjson("{a: [{b: 0}]}")); + Mod mod(fromjson("{$min: {'a.1.c': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.c"); + ASSERT_TRUE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_EQUALS(*execInfo.indexOfArrayWithNewElement[0], 0u); + ASSERT_FALSE(execInfo.noOp); +} + +TEST(IndexedMod, PrepareDoNotReportModifiedArrayElement) { + Document doc(fromjson("{a: [{b: 0}]}")); + Mod mod(fromjson("{$min: {'a.0.c': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0.c"); + ASSERT_FALSE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_FALSE(execInfo.noOp); +} + +TEST(IndexedMod, PrepareDoNotReportCreatedNumericObjectField) { + Document doc(fromjson("{a: {'0': {b: 0}}}")); + Mod mod(fromjson("{$min: {'a.1.c': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.c"); + ASSERT_FALSE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_FALSE(execInfo.noOp); +} } // namespace diff --git a/src/mongo/db/ops/modifier_current_date.cpp b/src/mongo/db/ops/modifier_current_date.cpp index cd328f5fe94..4496e0cd2cd 100644 --- a/src/mongo/db/ops/modifier_current_date.cpp +++ b/src/mongo/db/ops/modifier_current_date.cpp @@ -161,6 +161,8 @@ Status ModifierCurrentDate::prepare(mutablebson::Element root, // be created during the apply. Status status = pathsupport::findLongestPrefix( _updatePath, root, &_preparedState->idxFound, &_preparedState->elemFound); + const auto elemFoundIsArray = + _preparedState->elemFound.ok() && _preparedState->elemFound.getType() == BSONType::Array; // FindLongestPrefix may say the path does not exist at all, which is fine here, or // that the path was not viable or otherwise wrong, in which case, the mod cannot @@ -175,6 +177,14 @@ Status ModifierCurrentDate::prepare(mutablebson::Element root, // there is any conflict among mods. execInfo->fieldRef[0] = &_updatePath; + if (!_preparedState->elemFound.ok() || + _preparedState->idxFound < (_updatePath.numParts() - 1)) { + if (elemFoundIsArray) { + // Report that an existing array will gain a new element as a result of this mod. + execInfo->indexOfArrayWithNewElement[0] = _preparedState->idxFound; + } + } + return Status::OK(); } diff --git a/src/mongo/db/ops/modifier_current_date_test.cpp b/src/mongo/db/ops/modifier_current_date_test.cpp index 81bcd4d04e0..a3746e445da 100644 --- a/src/mongo/db/ops/modifier_current_date_test.cpp +++ b/src/mongo/db/ops/modifier_current_date_test.cpp @@ -359,4 +359,41 @@ TEST(DottedTimestampInput, EmptyStartDoc) { validateOplogEntry(oplogFormat, logDoc); } +TEST(BoolInput, PrepareReportCreatedArrayElement) { + Document doc(fromjson("{a: [{b: 0}]}")); + Mod mod(fromjson("{$currentDate: {'a.1.c': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.c"); + ASSERT_TRUE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_EQUALS(*execInfo.indexOfArrayWithNewElement[0], 0u); + ASSERT_FALSE(execInfo.noOp); +} + +TEST(BoolInput, PrepareDoNotReportModifiedArrayElement) { + Document doc(fromjson("{a: [{b: 0}]}")); + Mod mod(fromjson("{$currentDate: {'a.0.c': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0.c"); + ASSERT_FALSE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_FALSE(execInfo.noOp); +} + +TEST(BoolInput, PrepareDoNotReportCreatedNumericObjectField) { + Document doc(fromjson("{a: {'0': {b: 0}}}")); + Mod mod(fromjson("{$currentDate: {'a.1.c': true}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.c"); + ASSERT_FALSE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_FALSE(execInfo.noOp); +} + } // namespace diff --git a/src/mongo/db/ops/modifier_inc.cpp b/src/mongo/db/ops/modifier_inc.cpp index 314ac6a5024..f2d51b390da 100644 --- a/src/mongo/db/ops/modifier_inc.cpp +++ b/src/mongo/db/ops/modifier_inc.cpp @@ -135,6 +135,8 @@ Status ModifierInc::prepare(mutablebson::Element root, // be created during the apply. Status status = pathsupport::findLongestPrefix( _fieldRef, root, &_preparedState->idxFound, &_preparedState->elemFound); + const auto elemFoundIsArray = + _preparedState->elemFound.ok() && _preparedState->elemFound.getType() == BSONType::Array; // FindLongestPrefix may say the path does not exist at all, which is fine here, or // that the path was not viable or otherwise wrong, in which case, the mod cannot @@ -166,6 +168,11 @@ Status ModifierInc::prepare(mutablebson::Element root, if (_mode == MODE_MUL) _preparedState->newValue *= SafeNum(static_cast(0)); + if (elemFoundIsArray) { + // Report that an existing array will gain a new element as a result of this mod. + execInfo->indexOfArrayWithNewElement[0] = _preparedState->idxFound; + } + return Status::OK(); } diff --git a/src/mongo/db/ops/modifier_inc_test.cpp b/src/mongo/db/ops/modifier_inc_test.cpp index 02fbbd4474a..3784abbcdff 100644 --- a/src/mongo/db/ops/modifier_inc_test.cpp +++ b/src/mongo/db/ops/modifier_inc_test.cpp @@ -630,4 +630,41 @@ TEST(Multiplication, ApplyMissingElementDouble) { ASSERT_EQUALS(mongo::NumberDouble, doc.root().rightChild().getType()); } +TEST(IndexedMod, PrepareReportCreatedArrayElement) { + Document doc(fromjson("{a: [{b: 0}]}")); + Mod mod(fromjson("{$inc: {'a.1.c': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.c"); + ASSERT_TRUE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_EQUALS(*execInfo.indexOfArrayWithNewElement[0], 0u); + ASSERT_FALSE(execInfo.noOp); +} + +TEST(IndexedMod, PrepareDoNotReportModifiedArrayElement) { + Document doc(fromjson("{a: [{b: 0}]}")); + Mod mod(fromjson("{$inc: {'a.0.c': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0.c"); + ASSERT_FALSE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_FALSE(execInfo.noOp); +} + +TEST(IndexedMod, PrepareDoNotReportCreatedNumericObjectField) { + Document doc(fromjson("{a: {'0': {b: 0}}}")); + Mod mod(fromjson("{$inc: {'a.1.c': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.c"); + ASSERT_FALSE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_FALSE(execInfo.noOp); +} + } // namespace diff --git a/src/mongo/db/ops/modifier_interface.h b/src/mongo/db/ops/modifier_interface.h index d833dcf07a1..2294802e18d 100644 --- a/src/mongo/db/ops/modifier_interface.h +++ b/src/mongo/db/ops/modifier_interface.h @@ -205,11 +205,17 @@ struct ModifierInterface::ExecInfo { ExecInfo() : noOp(false), context(ANY_CONTEXT) { for (int i = 0; i < MAX_NUM_FIELDS; i++) { fieldRef[i] = NULL; + indexOfArrayWithNewElement[i] = boost::none; } } // The fields of concern to the driver: no other op may modify the fields listed here. FieldRef* fieldRef[MAX_NUM_FIELDS]; // not owned here + + // For each modified field ref, the index of the path component representing an existing array + // that gained a new element. + boost::optional indexOfArrayWithNewElement[MAX_NUM_FIELDS]; + bool noOp; UpdateContext context; }; diff --git a/src/mongo/db/ops/modifier_push.cpp b/src/mongo/db/ops/modifier_push.cpp index 8bc671ecf46..ef19e45c8a2 100644 --- a/src/mongo/db/ops/modifier_push.cpp +++ b/src/mongo/db/ops/modifier_push.cpp @@ -434,6 +434,8 @@ Status ModifierPush::prepare(mutablebson::Element root, // be created during the apply. Status status = pathsupport::findLongestPrefix( _fieldRef, root, &_preparedState->idxFound, &_preparedState->elemFound); + const auto elemFoundIsArray = + _preparedState->elemFound.ok() && _preparedState->elemFound.getType() == BSONType::Array; // FindLongestPrefix may say the path does not exist at all, which is fine here, or // that the path was not viable or otherwise wrong, in which case, the mod cannot @@ -463,6 +465,13 @@ Status ModifierPush::prepare(mutablebson::Element root, // there is any conflict among mods. execInfo->fieldRef[0] = &_fieldRef; + if (!_preparedState->elemFound.ok() || _preparedState->idxFound < (_fieldRef.numParts() - 1)) { + if (elemFoundIsArray) { + // Report that an existing array will gain a new element as a result of this mod. + execInfo->indexOfArrayWithNewElement[0] = _preparedState->idxFound; + } + } + return Status::OK(); } diff --git a/src/mongo/db/ops/modifier_push_test.cpp b/src/mongo/db/ops/modifier_push_test.cpp index 49b9d01b002..a7de5d9f0f0 100644 --- a/src/mongo/db/ops/modifier_push_test.cpp +++ b/src/mongo/db/ops/modifier_push_test.cpp @@ -1473,4 +1473,87 @@ TEST(ToPosition, Back) { ASSERT_EQUALS(fromjson("{$set: {'a.1':1}}"), logDoc); } +TEST(IndexedMod, PrepareReportCreatedArrayElement) { + Document doc(fromjson("{a: [{b: 0}]}")); + Mod mod(fromjson("{$push: {'a.1.c': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.c"); + ASSERT_TRUE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_EQUALS(*execInfo.indexOfArrayWithNewElement[0], 0u); + ASSERT_FALSE(execInfo.noOp); +} + +TEST(IndexedMod, PrepareReportCreatedArrayElementPushAll) { + Document doc(fromjson("{a: [{b: 0}]}")); + auto modObj = fromjson("{$pushAll: {'a.1.c': [2]}}"); + ModifierPush mod(ModifierPush::PUSH_ALL); + ASSERT_OK(mod.init(modObj["$pushAll"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.c"); + ASSERT_TRUE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_EQUALS(*execInfo.indexOfArrayWithNewElement[0], 0u); + ASSERT_FALSE(execInfo.noOp); +} + +TEST(IndexedMod, PrepareDoNotReportModifiedArrayElement) { + Document doc(fromjson("{a: [{b: 0}]}")); + Mod mod(fromjson("{$push: {'a.0.c': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0.c"); + ASSERT_FALSE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_FALSE(execInfo.noOp); +} + +TEST(IndexedMod, PrepareDoNotReportModifiedArrayElementPushAll) { + Document doc(fromjson("{a: [{b: 0}]}")); + auto modObj = fromjson("{$pushAll: {'a.0.c': [2]}}"); + ModifierPush mod(ModifierPush::PUSH_ALL); + ASSERT_OK(mod.init(modObj["$pushAll"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0.c"); + ASSERT_FALSE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_FALSE(execInfo.noOp); +} + +TEST(IndexedMod, PrepareDoNotReportCreatedNumericObjectField) { + Document doc(fromjson("{a: {'0': {b: 0}}}")); + Mod mod(fromjson("{$push: {'a.1.c': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.c"); + ASSERT_FALSE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_FALSE(execInfo.noOp); +} + +TEST(IndexedMod, PrepareDoNotReportCreatedNumericObjectFieldPushAll) { + Document doc(fromjson("{a: {'0': {b: 0}}}")); + auto modObj = fromjson("{$pushAll: {'a.1.c': [2]}}"); + ModifierPush mod(ModifierPush::PUSH_ALL); + ASSERT_OK(mod.init(modObj["$pushAll"].embeddedObject().firstElement(), + ModifierInterface::Options::normal())); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(mod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.c"); + ASSERT_FALSE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_FALSE(execInfo.noOp); +} + } // unnamed namespace diff --git a/src/mongo/db/ops/modifier_set.cpp b/src/mongo/db/ops/modifier_set.cpp index 59f59c555ce..03b703b860c 100644 --- a/src/mongo/db/ops/modifier_set.cpp +++ b/src/mongo/db/ops/modifier_set.cpp @@ -127,6 +127,8 @@ Status ModifierSet::prepare(mutablebson::Element root, // be created during the apply. Status status = pathsupport::findLongestPrefix( _fieldRef, root, &_preparedState->idxFound, &_preparedState->elemFound); + const auto elemFoundIsArray = + _preparedState->elemFound.ok() && _preparedState->elemFound.getType() == BSONType::Array; // FindLongestPrefix may say the path does not exist at all, which is fine here, or // that the path was not viable or otherwise wrong, in which case, the mod cannot @@ -155,6 +157,10 @@ Status ModifierSet::prepare(mutablebson::Element root, // If the field path is not fully present, then this mod cannot be in place, nor is it a noOp. if (!_preparedState->elemFound.ok() || _preparedState->idxFound < (_fieldRef.numParts() - 1)) { + if (elemFoundIsArray) { + // Report that an existing array will gain a new element as a result of this mod. + execInfo->indexOfArrayWithNewElement[0] = _preparedState->idxFound; + } return Status::OK(); } diff --git a/src/mongo/db/ops/modifier_set_test.cpp b/src/mongo/db/ops/modifier_set_test.cpp index 1579b096722..b83270da957 100644 --- a/src/mongo/db/ops/modifier_set_test.cpp +++ b/src/mongo/db/ops/modifier_set_test.cpp @@ -429,6 +429,43 @@ TEST(IndexedMod, PrepareNonViablePath) { ASSERT_NOT_OK(setMod.prepare(doc.root(), "", &execInfo)); } +TEST(IndexedMod, PrepareReportCreatedArrayElement) { + Document doc(fromjson("{a: [{b: 0}]}")); + Mod setMod(fromjson("{$set: {'a.1.c': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.c"); + ASSERT_TRUE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_EQUALS(*execInfo.indexOfArrayWithNewElement[0], 0u); + ASSERT_FALSE(execInfo.noOp); +} + +TEST(IndexedMod, PrepareDoNotReportModifiedArrayElement) { + Document doc(fromjson("{a: [{b: 0}]}")); + Mod setMod(fromjson("{$set: {'a.0.c': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.0.c"); + ASSERT_FALSE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_FALSE(execInfo.noOp); +} + +TEST(IndexedMod, PrepareDoNotReportCreatedNumericObjectField) { + Document doc(fromjson("{a: {'0': {b: 0}}}")); + Mod setMod(fromjson("{$set: {'a.1.c': 2}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a.1.c"); + ASSERT_FALSE(execInfo.indexOfArrayWithNewElement[0]); + ASSERT_FALSE(execInfo.noOp); +} + TEST(IndexedMod, PrepareApplyInPlace) { Document doc(fromjson("{a: [{b: 0},{b: 1},{b: 1}]}")); Mod setMod(fromjson("{$set: {'a.2.b': 2}}")); diff --git a/src/mongo/db/ops/update.cpp b/src/mongo/db/ops/update.cpp index 5e0763f9eac..bc3c595dc6c 100644 --- a/src/mongo/db/ops/update.cpp +++ b/src/mongo/db/ops/update.cpp @@ -63,20 +63,9 @@ UpdateResult update(OperationContext* txn, Database* db, const UpdateRequest& re // Explain should never use this helper. invariant(!request.isExplain()); - auto client = txn->getClient(); - auto lastOpAtOperationStart = repl::ReplClientInfo::forClient(client).getLastOp(); - ScopeGuard lastOpSetterGuard = MakeObjGuard(repl::ReplClientInfo::forClient(client), - &repl::ReplClientInfo::setLastOpToSystemLastOpTime, - txn); - const NamespaceString& nsString = request.getNamespaceString(); Collection* collection = db->getCollection(nsString.ns()); - // If this is the local database, don't set last op. - if (db->name() == "local") { - lastOpSetterGuard.Dismiss(); - } - // The update stage does not create its own collection. As such, if the update is // an upsert, create the collection that the update stage inserts into beforehand. if (!collection && request.isUpsert()) { @@ -116,12 +105,6 @@ UpdateResult update(OperationContext* txn, Database* db, const UpdateRequest& re uassertStatusOK(getExecutorUpdate(txn, nullOpDebug, collection, &parsedUpdate)); uassertStatusOK(exec->executePlan()); - if (repl::ReplClientInfo::forClient(client).getLastOp() != lastOpAtOperationStart) { - // If this operation has already generated a new lastOp, don't bother setting it here. - // No-op updates will not generate a new lastOp, so we still need the guard to fire in that - // case. - lastOpSetterGuard.Dismiss(); - } const UpdateStats* updateStats = UpdateStage::getUpdateStats(exec.get()); diff --git a/src/mongo/db/ops/update_driver.cpp b/src/mongo/db/ops/update_driver.cpp index e5a63d64ccb..dc47ca4175f 100644 --- a/src/mongo/db/ops/update_driver.cpp +++ b/src/mongo/db/ops/update_driver.cpp @@ -291,11 +291,21 @@ Status UpdateDriver::update(StringData matchedField, // is not a no-op and it is in a valid context -- then we switch back to a // non-in-place mode. // - // TODO: make mightBeIndexed and fieldRef like each other. - if (!_affectIndices && !execInfo.noOp && _indexedFields && - _indexedFields->mightBeIndexed(execInfo.fieldRef[i]->dottedField())) { - _affectIndices = true; - doc->disableInPlaceUpdates(); + // To determine if indexes are affected: If we did not create a new element in an array, + // check whether the full path affects indexes. If we did create a new element in an + // array, check whether the array itself might affect any indexes. This is necessary + // because if there is an index {"a.b": 1}, and we set "a.1.c" and implicitly create an + // array element in "a", then we may need to add a null key to the index {"a.b": 1}, + // even though "a.1.c" does not appear to affect the index. + if (!_affectIndices && !execInfo.noOp && _indexedFields) { + auto pathLengthForIndexCheck = execInfo.indexOfArrayWithNewElement[i] + ? *execInfo.indexOfArrayWithNewElement[i] + 1 + : execInfo.fieldRef[i]->numParts(); + if (_indexedFields->mightBeIndexed( + execInfo.fieldRef[i]->dottedSubstring(0, pathLengthForIndexCheck))) { + _affectIndices = true; + doc->disableInPlaceUpdates(); + } } } diff --git a/src/mongo/db/pipeline/SConscript b/src/mongo/db/pipeline/SConscript index 98a66a02f48..ca8617f3d3d 100644 --- a/src/mongo/db/pipeline/SConscript +++ b/src/mongo/db/pipeline/SConscript @@ -252,8 +252,8 @@ docSourceEnv.Library( '$BUILD_DIR/mongo/db/matcher/expression_algo', '$BUILD_DIR/mongo/db/service_context', '$BUILD_DIR/mongo/db/stats/top', + '$BUILD_DIR/mongo/db/storage/encryption_hooks', '$BUILD_DIR/mongo/db/storage/storage_options', - '$BUILD_DIR/mongo/db/storage/wiredtiger/storage_wiredtiger_customization_hooks', '$BUILD_DIR/third_party/shim_snappy', ], LIBDEPS_TAGS=[ diff --git a/src/mongo/db/pipeline/document_source_geo_near.cpp b/src/mongo/db/pipeline/document_source_geo_near.cpp index 17e9997bbf5..a385eeeb07e 100644 --- a/src/mongo/db/pipeline/document_source_geo_near.cpp +++ b/src/mongo/db/pipeline/document_source_geo_near.cpp @@ -150,10 +150,12 @@ BSONObj DocumentSourceGeoNear::buildGeoNearCmd() const { geoNear.append("minDistance", minDistance); geoNear.append("query", query); - if (pExpCtx->getCollator()) { - geoNear.append("collation", pExpCtx->getCollator()->getSpec().toBSON()); - } else { - geoNear.append("collation", CollationSpec::kSimpleSpec); + if (!pExpCtx->collation.isEmpty()) { + if (pExpCtx->getCollator()) { + geoNear.append("collation", pExpCtx->getCollator()->getSpec().toBSON()); + } else { + geoNear.append("collation", CollationSpec::kSimpleSpec); + } } geoNear.append("spherical", spherical); diff --git a/src/mongo/db/pipeline/parsed_aggregation_projection.cpp b/src/mongo/db/pipeline/parsed_aggregation_projection.cpp index 69b196faab3..c8d7bf06d7f 100644 --- a/src/mongo/db/pipeline/parsed_aggregation_projection.cpp +++ b/src/mongo/db/pipeline/parsed_aggregation_projection.cpp @@ -46,6 +46,8 @@ namespace mongo { namespace parsed_aggregation_projection { +using expression::isPathPrefixOf; + // // ProjectionSpecValidator // @@ -54,22 +56,36 @@ Status ProjectionSpecValidator::validate(const BSONObj& spec) { return ProjectionSpecValidator(spec).validate(); } -Status ProjectionSpecValidator::ensurePathDoesNotConflictOrThrow(StringData path) { - for (auto&& seenPath : _seenPaths) { - if ((path == seenPath) || (expression::isPathPrefixOf(path, seenPath)) || - (expression::isPathPrefixOf(seenPath, path))) { - return Status(ErrorCodes::FailedToParse, - str::stream() << "specification contains two conflicting paths. " - "Cannot specify both '" - << path - << "' and '" - << seenPath - << "': " - << _rawObj.toString(), - 40176); - } +Status ProjectionSpecValidator::ensurePathDoesNotConflictOrThrow(const std::string& path) { + auto result = _seenPaths.emplace(path); + auto pos = result.first; + + // Check whether the path was a duplicate of an existing path. + auto conflictingPath = boost::make_optional(!result.second, *pos); + + // Check whether the preceding path prefixes this path. + if (!conflictingPath && pos != _seenPaths.begin()) { + conflictingPath = + boost::make_optional(isPathPrefixOf(*std::prev(pos), path), *std::prev(pos)); + } + + // Check whether this path prefixes the subsequent path. + if (!conflictingPath && std::next(pos) != _seenPaths.end()) { + conflictingPath = + boost::make_optional(isPathPrefixOf(path, *std::next(pos)), *std::next(pos)); + } + + if (conflictingPath) { + return Status(ErrorCodes::FailedToParse, + str::stream() << "specification contains two conflicting paths. " + "Cannot specify both '" + << path + << "' and '" + << *conflictingPath + << "': " + << _rawObj.toString(), + 40176); } - _seenPaths.emplace_back(path.toString()); return Status::OK(); } diff --git a/src/mongo/db/pipeline/parsed_aggregation_projection.h b/src/mongo/db/pipeline/parsed_aggregation_projection.h index 8d92e4ab768..4fd9731774e 100644 --- a/src/mongo/db/pipeline/parsed_aggregation_projection.h +++ b/src/mongo/db/pipeline/parsed_aggregation_projection.h @@ -69,7 +69,7 @@ private: * For example, a user is not allowed to specify {'a': 1, 'a.b': 1}, or some similar conflicting * paths. */ - Status ensurePathDoesNotConflictOrThrow(StringData path); + Status ensurePathDoesNotConflictOrThrow(const std::string& path); /** * Returns the relevant error if an invalid projection specification is detected. @@ -99,9 +99,40 @@ private: // The original object. Used to generate more helpful error messages. const BSONObj& _rawObj; + // Custom comparator that orders fieldpath strings by path prefix first, then by field. + struct PathPrefixComparator { + static constexpr char dot = '.'; + + // Returns true if the lhs value should sort before the rhs, false otherwise. + bool operator()(const std::string& lhs, const std::string& rhs) const { + for (size_t pos = 0, len = std::min(lhs.size(), rhs.size()); pos < len; ++pos) { + auto &lchar = lhs[pos], &rchar = rhs[pos]; + if (lchar == rchar) { + continue; + } + + // Consider the path delimiter '.' as being less than all other characters, so that + // paths sort directly before any paths they prefix and directly after any paths + // which prefix them. + if (lchar == dot) { + return true; + } else if (rchar == dot) { + return false; + } + + // Otherwise, default to normal character comparison. + return lchar < rchar; + } + + // If we get here, then we have reached the end of lhs and/or rhs and all of their path + // segments up to this point match. If lhs is shorter than rhs, then lhs prefixes rhs + // and should sort before it. + return lhs.size() < rhs.size(); + } + }; + // Tracks which paths we've seen to ensure no two paths conflict with each other. - // Can be a vector since we iterate through it. - std::vector _seenPaths; + std::set _seenPaths; }; /** diff --git a/src/mongo/db/pipeline/parsed_aggregation_projection_test.cpp b/src/mongo/db/pipeline/parsed_aggregation_projection_test.cpp index 71dd1c2378f..cb5d50b2771 100644 --- a/src/mongo/db/pipeline/parsed_aggregation_projection_test.cpp +++ b/src/mongo/db/pipeline/parsed_aggregation_projection_test.cpp @@ -126,6 +126,75 @@ TEST(ParsedAggregationProjectionErrors, ShouldRejectFieldsWithSharedPrefix) { UserException); } +TEST(ParsedAggregationProjectionErrors, ShouldRejectPathConflictsWithNonAlphaNumericCharacters) { + const boost::intrusive_ptr expCtx(new ExpressionContextForTest()); + // Include/exclude non-alphanumeric fields with a shared prefix. First assert that the non- + // alphanumeric fields are accepted when no prefixes are present. + ASSERT(ParsedAggregationProjection::create( + expCtx, BSON("a.b-c" << true << "a.b" << true << "a.b?c" << true << "a.b c" << true))); + ASSERT(ParsedAggregationProjection::create( + expCtx, BSON("a.b c" << false << "a.b?c" << false << "a.b" << false << "a.b-c" << false))); + + // Then assert that we throw when we introduce a prefixed field. + ASSERT_THROWS( + ParsedAggregationProjection::create( + expCtx, + BSON("a.b-c" << true << "a.b" << true << "a.b?c" << true << "a.b c" << true << "a.b.d" + << true)), + AssertionException); + ASSERT_THROWS( + ParsedAggregationProjection::create( + expCtx, + BSON("a.b.d" << false << "a.b c" << false << "a.b?c" << false << "a.b" << false + << "a.b-c" + << false)), + AssertionException); + + // Adding the same field twice. + ASSERT_THROWS(ParsedAggregationProjection::create( + expCtx, BSON("a.b?c" << wrapInLiteral(1) << "a.b?c" << wrapInLiteral(0))), + AssertionException); + ASSERT_THROWS(ParsedAggregationProjection::create( + expCtx, BSON("a.b c" << wrapInLiteral(0) << "a.b c" << wrapInLiteral(1))), + AssertionException); + + // Mix of include/exclude and adding a shared prefix. + ASSERT_THROWS( + ParsedAggregationProjection::create( + expCtx, + BSON("a.b-c" << true << "a.b" << wrapInLiteral(1) << "a.b?c" << true << "a.b c" << true + << "a.b.d" + << true)), + AssertionException); + ASSERT_THROWS(ParsedAggregationProjection::create( + expCtx, + BSON("a.b.d" << false << "a.b c" << false << "a.b?c" << false << "a.b" + << wrapInLiteral(0) + << "a.b-c" + << false)), + AssertionException); + + // Adding a shared prefix twice. + ASSERT_THROWS(ParsedAggregationProjection::create( + expCtx, + BSON("a.b-c" << wrapInLiteral(1) << "a.b" << wrapInLiteral(1) << "a.b?c" + << wrapInLiteral(1) + << "a.b c" + << wrapInLiteral(1) + << "a.b.d" + << wrapInLiteral(0))), + AssertionException); + ASSERT_THROWS(ParsedAggregationProjection::create( + expCtx, + BSON("a.b.d" << wrapInLiteral(1) << "a.b c" << wrapInLiteral(1) << "a.b?c" + << wrapInLiteral(1) + << "a.b" + << wrapInLiteral(0) + << "a.b-c" + << wrapInLiteral(1))), + AssertionException); +} + TEST(ParsedAggregationProjectionErrors, ShouldRejectMixOfIdAndSubFieldsOfId) { const boost::intrusive_ptr expCtx(new ExpressionContextForTest()); // Include/exclude _id twice. diff --git a/src/mongo/db/query/SConscript b/src/mongo/db/query/SConscript index a6c14a5f7dc..521f44cecb8 100644 --- a/src/mongo/db/query/SConscript +++ b/src/mongo/db/query/SConscript @@ -364,47 +364,11 @@ env.CppUnitTest( env.CppUnitTest( target="query_planner_test", source=[ - "query_planner_test.cpp" - ], - LIBDEPS=[ - "query_planner_test_fixture", - ], -) - -env.CppUnitTest( - target="query_planner_array_test", - source=[ - "query_planner_array_test.cpp" - ], - LIBDEPS=[ - "query_planner_test_fixture", - ], -) - -env.CppUnitTest( - target="query_planner_geo_test", - source=[ - "query_planner_geo_test.cpp" - ], - LIBDEPS=[ - "query_planner_test_fixture", - ], -) - -env.CppUnitTest( - target="query_planner_partialidx_test", - source=[ - "query_planner_partialidx_test.cpp" - ], - LIBDEPS=[ - "query_planner_test_fixture", - ], -) - -env.CppUnitTest( - target="query_planner_collation_test", - source=[ + "query_planner_array_test.cpp", "query_planner_collation_test.cpp", + "query_planner_geo_test.cpp", + "query_planner_partialidx_test.cpp", + "query_planner_test.cpp", ], LIBDEPS=[ "collation/collator_interface_mock", diff --git a/src/mongo/db/query/canonical_query_test.cpp b/src/mongo/db/query/canonical_query_test.cpp index c7a1f2a05a2..31ba2737ee9 100644 --- a/src/mongo/db/query/canonical_query_test.cpp +++ b/src/mongo/db/query/canonical_query_test.cpp @@ -548,7 +548,8 @@ TEST(CanonicalQueryTest, NormalizeWithInPreservesCollator) { BSONObj obj = fromjson("{'': 'string'}"); auto inMatchExpression = stdx::make_unique(); inMatchExpression->setCollator(&collator); - inMatchExpression->addEquality(obj.firstElement()); + std::vector equalities{obj.firstElement()}; + ASSERT_OK(inMatchExpression->setEqualities(std::move(equalities))); unique_ptr matchExpression( CanonicalQuery::normalizeTree(inMatchExpression.release())); ASSERT(matchExpression->matchType() == MatchExpression::MatchType::EQ); diff --git a/src/mongo/db/query/index_bounds_builder.cpp b/src/mongo/db/query/index_bounds_builder.cpp index b16ad43dc5e..eeb1c2f1114 100644 --- a/src/mongo/db/query/index_bounds_builder.cpp +++ b/src/mongo/db/query/index_bounds_builder.cpp @@ -249,6 +249,14 @@ bool typeMatch(const BSONObj& obj) { return first.canonicalType() == second.canonicalType(); } +bool IndexBoundsBuilder::canUseCoveredMatching(const MatchExpression* expr, + const IndexEntry& index) { + IndexBoundsBuilder::BoundsTightness tightness; + OrderedIntervalList oil; + translate(expr, BSONElement{}, index, &oil, &tightness); + return tightness >= IndexBoundsBuilder::INEXACT_COVERED; +} + // static void IndexBoundsBuilder::translate(const MatchExpression* expr, const BSONElement& elt, diff --git a/src/mongo/db/query/index_bounds_builder.h b/src/mongo/db/query/index_bounds_builder.h index b397f81b295..311b3ee61f6 100644 --- a/src/mongo/db/query/index_bounds_builder.h +++ b/src/mongo/db/query/index_bounds_builder.h @@ -69,6 +69,17 @@ public: */ static void allValuesForField(const BSONElement& elt, OrderedIntervalList* out); + /** + * Returns true if 'expr' can correctly be assigned as an INEXACT_COVERED predicate to an index + * scan over 'index'. + * + * The result of this function is not meaningful when the predicate applies to special fields + * such as "hashed", "2d", or "2dsphere". That is, the caller is responsible for ensuring that + * 'expr' is a candidate for covered matching over a regular ascending/descending field of the + * index. + */ + static bool canUseCoveredMatching(const MatchExpression* expr, const IndexEntry& index); + /** * Turn the MatchExpression in 'expr' into a set of index bounds. The field that 'expr' is * concerned with is indexed according to the keypattern element 'elt' from index 'index'. diff --git a/src/mongo/db/query/index_bounds_builder_test.cpp b/src/mongo/db/query/index_bounds_builder_test.cpp index 45f11a2cc1d..a8ae3651d32 100644 --- a/src/mongo/db/query/index_bounds_builder_test.cpp +++ b/src/mongo/db/query/index_bounds_builder_test.cpp @@ -2137,4 +2137,54 @@ TEST(IndexBoundsBuilderTest, InWithStringAgainstHashedIndexWithCollatorUsesHashO ASSERT_EQUALS(tightness, IndexBoundsBuilder::INEXACT_FETCH); } +TEST(IndexBoundsBuilderTest, CanUseCoveredMatchingForEqualityPredicate) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$eq: 3}}"); + unique_ptr expr(parseMatchExpression(obj)); + ASSERT_TRUE(IndexBoundsBuilder::canUseCoveredMatching(expr.get(), testIndex)); +} + +TEST(IndexBoundsBuilderTest, CannotUseCoveredMatchingForEqualityToArrayPredicate) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$eq: [1, 2, 3]}}"); + unique_ptr expr(parseMatchExpression(obj)); + ASSERT_FALSE(IndexBoundsBuilder::canUseCoveredMatching(expr.get(), testIndex)); +} + +TEST(IndexBoundsBuilderTest, CannotUseCoveredMatchingForEqualityToNullPredicate) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: null}"); + unique_ptr expr(parseMatchExpression(obj)); + ASSERT_FALSE(IndexBoundsBuilder::canUseCoveredMatching(expr.get(), testIndex)); +} + +TEST(IndexBoundsBuilderTest, CannotUseCoveredMatchingForTypeArrayPredicate) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$type: 'array'}}"); + unique_ptr expr(parseMatchExpression(obj)); + ASSERT_FALSE(IndexBoundsBuilder::canUseCoveredMatching(expr.get(), testIndex)); +} + +TEST(IndexBoundsBuilderTest, CannotUseCoveredMatchingForExistsTruePredicate) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$exists: true}}"); + unique_ptr expr(parseMatchExpression(obj)); + ASSERT_FALSE(IndexBoundsBuilder::canUseCoveredMatching(expr.get(), testIndex)); +} + +TEST(IndexBoundsBuilderTest, CannotUseCoveredMatchingForExistsFalsePredicate) { + IndexEntry testIndex = IndexEntry(BSONObj()); + BSONObj obj = fromjson("{a: {$exists: false}}"); + unique_ptr expr(parseMatchExpression(obj)); + ASSERT_FALSE(IndexBoundsBuilder::canUseCoveredMatching(expr.get(), testIndex)); +} + +TEST(IndexBoundsBuilderTest, CanUseCoveredMatchingForExistsTrueWithSparseIndex) { + IndexEntry testIndex = IndexEntry(BSONObj()); + testIndex.sparse = true; + BSONObj obj = fromjson("{a: {$exists: true}}"); + unique_ptr expr(parseMatchExpression(obj)); + ASSERT_TRUE(IndexBoundsBuilder::canUseCoveredMatching(expr.get(), testIndex)); +} + } // namespace diff --git a/src/mongo/db/query/plan_enumerator.cpp b/src/mongo/db/query/plan_enumerator.cpp index 789cd405607..de80606df8d 100644 --- a/src/mongo/db/query/plan_enumerator.cpp +++ b/src/mongo/db/query/plan_enumerator.cpp @@ -579,6 +579,32 @@ bool PlanEnumerator::prepMemo(MatchExpression* node, PrepMemoContext context) { return false; } +void PlanEnumerator::assignToNonMultikeyMandatoryIndex( + const IndexEntry& index, + const std::vector& predsOverLeadingField, + const IndexToPredMap& idxToNotFirst, + OneIndexAssignment* indexAssign) { + // Text indexes are typically multikey because there is an index key for each token in the + // source text. However, the leading and trailing non-text fields of the index cannot be + // multikey. As a result, we should use non-multikey predicate assignment rules for such + // indexes. + invariant(!index.multikey || index.type == IndexType::INDEX_TEXT); + + // Since the index is not multikey, all predicates over the leading field can be assigned. + indexAssign->preds = predsOverLeadingField; + + // Since everything in assign.preds prefixes the index, they all go at position '0' in the + // index, the first position. + indexAssign->positions.resize(indexAssign->preds.size(), 0); + + // And now we begin compound analysis. Find everything that could use assign.index but isn't a + // pred over the first field of that index. + auto compIt = idxToNotFirst.find(indexAssign->index); + if (compIt != idxToNotFirst.end()) { + compound(compIt->second, index, indexAssign); + } +} + bool PlanEnumerator::enumerateMandatoryIndex(const IndexToPredMap& idxToFirst, const IndexToPredMap& idxToNotFirst, MatchExpression* mandatoryPred, @@ -613,7 +639,12 @@ bool PlanEnumerator::enumerateMandatoryIndex(const IndexToPredMap& idxToFirst, const vector& predsOverLeadingField = it->second; - if (thisIndex.multikey && !thisIndex.multikeyPaths.empty()) { + // Text indexes should be treated like non-multikey indexes, since the non-text fields are + // prohibited from containing arrays. + if (thisIndex.type == IndexType::INDEX_TEXT) { + assignToNonMultikeyMandatoryIndex( + thisIndex, predsOverLeadingField, idxToNotFirst, &indexAssign); + } else if (thisIndex.multikey && !thisIndex.multikeyPaths.empty()) { // 2dsphere indexes are the only special index type that should ever have path-level // multikey information. invariant(INDEX_2DSPHERE == thisIndex.type); @@ -719,22 +750,9 @@ bool PlanEnumerator::enumerateMandatoryIndex(const IndexToPredMap& idxToFirst, } } } else { - // For non-multikey, we don't have to do anything too special. - // Just assign all "first" predicates and try to compound like usual. - indexAssign.preds = it->second; - - // Since everything in assign.preds prefixes the index, they all go - // at position '0' in the index, the first position. - indexAssign.positions.resize(indexAssign.preds.size(), 0); - - // And now we begin compound analysis. - - // Find everything that could use assign.index but isn't a pred over - // the first field of that index. - IndexToPredMap::const_iterator compIt = idxToNotFirst.find(indexAssign.index); - if (compIt != idxToNotFirst.end()) { - compound(compIt->second, thisIndex, &indexAssign); - } + // The index is not multikey. + assignToNonMultikeyMandatoryIndex( + thisIndex, predsOverLeadingField, idxToNotFirst, &indexAssign); } // The mandatory predicate must be assigned. diff --git a/src/mongo/db/query/plan_enumerator.h b/src/mongo/db/query/plan_enumerator.h index 5543bc68ca9..a29c1dd4e76 100644 --- a/src/mongo/db/query/plan_enumerator.h +++ b/src/mongo/db/query/plan_enumerator.h @@ -433,6 +433,17 @@ private: const std::set& mandatoryIndices, AndAssignment* andAssignment); + /** + * Assigns predicates in 'predsOverLeadingField' and 'idxToNotFirst' to 'indexAssign'. Assumes + * that the index is not multikey. Also assumes that that the index is of a type used to answer + * "mandatory predicates" such as text or geoNear. + */ + void assignToNonMultikeyMandatoryIndex( + const IndexEntry& index, + const std::vector& predsOverLeadingField, + const IndexToPredMap& idxToNotFirst, + OneIndexAssignment* indexAssign); + /** * Try to assign predicates in 'tryCompound' to 'thisIndex' as compound assignments. * Output the assignments in 'assign'. diff --git a/src/mongo/db/query/planner_access.cpp b/src/mongo/db/query/planner_access.cpp index 9a6f52e18d0..90c4873ffbb 100644 --- a/src/mongo/db/query/planner_access.cpp +++ b/src/mongo/db/query/planner_access.cpp @@ -224,6 +224,17 @@ QuerySolutionNode* QueryPlannerAccess::makeLeafNode( TextMatchExpressionBase* textExpr = static_cast(expr); TextNode* ret = new TextNode(index); ret->ftsQuery = textExpr->getFTSQuery().clone(); + + // Count the number of prefix fields before the "text" field. + for (auto&& keyPatternElt : ret->index.keyPattern) { + // We know that the only key pattern with a type of String is the _fts field + // which is immediately after all prefix fields. + if (BSONType::String == keyPatternElt.type()) { + break; + } + ++(ret->numPrefixFields); + } + return ret; } else { // Note that indexKeyPattern.firstElement().fieldName() may not equal expr->path() @@ -335,10 +346,24 @@ void QueryPlannerAccess::mergeWithLeafNode(MatchExpression* expr, ScanBuildingSt const StageType type = node->getType(); - // Text data is covered, but not exactly. Text covering is unlike any other covering - // so we deal with it in addFilterToSolutionNode. if (STAGE_TEXT == type) { - scanState->tightness = IndexBoundsBuilder::INEXACT_COVERED; + auto textNode = static_cast(node); + + if (pos < textNode->numPrefixFields) { + // This predicate is assigned to one of the prefix fields of the text index. Such + // predicates must always be equalities and must always be attached to the TEXT node. In + // order to ensure this happens, we assign INEXACT_COVERED tightness. + scanState->tightness = IndexBoundsBuilder::INEXACT_COVERED; + } else { + // The predicate is assigned to one of the trailing fields of the text index. We + // currently don't generate bounds for predicates assigned to trailing fields of a text + // index, but rather attempt to attach a covered filter. However, certain predicates can + // never be correctly covered (e.g. $exists), so we assign the tightness accordingly. + scanState->tightness = IndexBoundsBuilder::canUseCoveredMatching(expr, index) + ? IndexBoundsBuilder::INEXACT_COVERED + : IndexBoundsBuilder::INEXACT_FETCH; + } + return; } @@ -347,20 +372,23 @@ void QueryPlannerAccess::mergeWithLeafNode(MatchExpression* expr, ScanBuildingSt if (STAGE_GEO_NEAR_2D == type) { invariant(INDEX_2D == index.type); - // 2D indexes are weird - the "2d" field stores a normally-indexed BinData field, but - // additional array fields are *not* exploded into multi-keys - they are stored directly - // as arrays in the index. Also, no matter what the index expression, the "2d" field is - // always first. - // This means that we can only generically accumulate bounds for 2D indexes over the - // first "2d" field (pos == 0) - MatchExpressions over other fields in the 2D index may - // be covered (can be evaluated using only the 2D index key). The additional fields - // must not affect the index scan bounds, since they are not stored in an - // IndexScan-compatible format. + // 2D indexes have a special format - the "2d" field stores a normally-indexed BinData + // field, but additional array fields are *not* exploded into multi-keys - they are stored + // directly as arrays in the index. Also, no matter what the index expression, the "2d" + // field is always first. + // + // This means that we can only generically accumulate bounds for 2D indexes over the first + // "2d" field (pos == 0) - MatchExpressions over other fields in the 2D index may be covered + // (can be evaluated using only the 2D index key). The additional fields must not affect + // the index scan bounds, since they are not stored in an IndexScan-compatible format. if (pos > 0) { - // Marking this field as covered allows the planner to accumulate a MatchExpression - // over the returned 2D index keys instead of adding to the index bounds. - scanState->tightness = IndexBoundsBuilder::INEXACT_COVERED; + // The predicate is over a trailing field of the "2d" index. If possible, we assign it + // as a covered filter (the INEXACT_COVERED case). Otherwise, the filter must be + // evaluated after fetching the full documents. + scanState->tightness = IndexBoundsBuilder::canUseCoveredMatching(expr, index) + ? IndexBoundsBuilder::INEXACT_COVERED + : IndexBoundsBuilder::INEXACT_FETCH; return; } @@ -374,10 +402,15 @@ void QueryPlannerAccess::mergeWithLeafNode(MatchExpression* expr, ScanBuildingSt verify(type == STAGE_IXSCAN); IndexScanNode* scan = static_cast(node); - // See STAGE_GEO_NEAR_2D above - 2D indexes can only accumulate scan bounds over the - // first "2d" field (pos == 0) + // See STAGE_GEO_NEAR_2D above - 2D indexes can only accumulate scan bounds over the first + // "2d" field (pos == 0). if (INDEX_2D == index.type && pos > 0) { - scanState->tightness = IndexBoundsBuilder::INEXACT_COVERED; + // The predicate is over a trailing field of the "2d" index. If possible, we assign it + // as a covered filter (the INEXACT_COVERED case). Otherwise, the filter must be + // evaluated after fetching the full documents. + scanState->tightness = IndexBoundsBuilder::canUseCoveredMatching(expr, index) + ? IndexBoundsBuilder::INEXACT_COVERED + : IndexBoundsBuilder::INEXACT_FETCH; return; } @@ -416,25 +449,9 @@ void QueryPlannerAccess::mergeWithLeafNode(MatchExpression* expr, ScanBuildingSt void QueryPlannerAccess::finishTextNode(QuerySolutionNode* node, const IndexEntry& index) { TextNode* tn = static_cast(node); - // Figure out what positions are prefix positions. We build an index key prefix from - // the predicates over the text index prefix keys. - // For example, say keyPattern = { a: 1, _fts: "text", _ftsx: 1, b: 1 } - // prefixEnd should be 1. - size_t prefixEnd = 0; - BSONObjIterator it(tn->index.keyPattern); - // Count how many prefix terms we have. - while (it.more()) { - // We know that the only key pattern with a type of String is the _fts field - // which is immediately after all prefix fields. - if (String == it.next().type()) { - break; - } - ++prefixEnd; - } - // If there's no prefix, the filter is already on the node and the index prefix is null. // We can just return. - if (!prefixEnd) { + if (!tn->numPrefixFields) { return; } @@ -448,7 +465,7 @@ void QueryPlannerAccess::finishTextNode(QuerySolutionNode* node, const IndexEntr if (MatchExpression::AND != textFilterMe->matchType()) { // Only one prefix term. - invariant(1 == prefixEnd); + invariant(1u == tn->numPrefixFields); // Sanity check: must be an EQ. invariant(MatchExpression::EQ == textFilterMe->matchType()); @@ -460,10 +477,10 @@ void QueryPlannerAccess::finishTextNode(QuerySolutionNode* node, const IndexEntr // Indexed by the keyPattern position index assignment. We want to add // prefixes in order but we must order them first. - vector prefixExprs(prefixEnd, NULL); + vector prefixExprs(tn->numPrefixFields, nullptr); AndMatchExpression* amExpr = static_cast(textFilterMe); - invariant(amExpr->numChildren() >= prefixEnd); + invariant(amExpr->numChildren() >= tn->numPrefixFields); // Look through the AND children. The prefix children we want to // stash in prefixExprs. @@ -474,7 +491,7 @@ void QueryPlannerAccess::finishTextNode(QuerySolutionNode* node, const IndexEntr invariant(NULL != ixtag); // Skip this child if it's not part of a prefix, or if we've already assigned a // predicate to this prefix position. - if (ixtag->pos >= prefixEnd || prefixExprs[ixtag->pos] != NULL) { + if (ixtag->pos >= tn->numPrefixFields || prefixExprs[ixtag->pos] != NULL) { ++curChild; continue; } diff --git a/src/mongo/db/query/planner_ixselect.cpp b/src/mongo/db/query/planner_ixselect.cpp index 321e856d1b7..2bf2eedc25e 100644 --- a/src/mongo/db/query/planner_ixselect.cpp +++ b/src/mongo/db/query/planner_ixselect.cpp @@ -48,6 +48,28 @@ namespace mongo { +namespace { + +/** + * Checks whether the given index is compatible with each child of the given $elemMatch expression. + * Assumes that the match expression is of type ELEM_MATCH_VALUE. + */ +bool elemMatchValueCompatible(const BSONElement& elt, + const IndexEntry& index, + MatchExpression* elemMatch, + const CollatorInterface* collator) { + invariant(elemMatch->matchType() == MatchExpression::ELEM_MATCH_VALUE); + for (size_t child = 0; child < elemMatch->numChildren(); ++child) { + if (!QueryPlannerIXSelect::compatible( + elt, index, elemMatch->getChild(child), collator, true)) { + return false; + } + } + return true; +} + +} // namespace + static double fieldWithDefault(const BSONObj& infoObj, const string& name, double def) { BSONElement e = infoObj[name]; if (e.isNumber()) { @@ -168,7 +190,8 @@ void QueryPlannerIXSelect::findRelevantIndices(const unordered_set& fiel bool QueryPlannerIXSelect::compatible(const BSONElement& elt, const IndexEntry& index, MatchExpression* node, - const CollatorInterface* collator) { + const CollatorInterface* collator, + bool elemMatchChild) { if ((boundsGeneratingNodeContainsComparisonToType(node, BSONType::String) || boundsGeneratingNodeContainsComparisonToType(node, BSONType::Array) || boundsGeneratingNodeContainsComparisonToType(node, BSONType::Object)) && @@ -195,16 +218,18 @@ bool QueryPlannerIXSelect::compatible(const BSONElement& elt, MatchExpression::MatchType exprtype = node->matchType(); if (indexedFieldType.empty()) { - // Can't check for null w/a sparse index. - if (exprtype == MatchExpression::EQ && index.sparse) { + // Can't use a sparse index for $eq with a null element, unless the equality is within a + // $elemMatch expression since the latter implies a match on the literal element 'null'. + if (exprtype == MatchExpression::EQ && index.sparse && !elemMatchChild) { const EqualityMatchExpression* expr = static_cast(node); if (expr->getData().isNull()) { return false; } } - // Can't check for $in w/ null element w/a sparse index. - if (exprtype == MatchExpression::MATCH_IN && index.sparse) { + // Can't use a sparse index for $in with a null element, unless the $eq is within a + // $elemMatch expression since the latter implies a match on the literal element 'null'. + if (exprtype == MatchExpression::MATCH_IN && index.sparse && !elemMatchChild) { const InMatchExpression* expr = static_cast(node); if (expr->hasNull()) { return false; @@ -220,7 +245,10 @@ bool QueryPlannerIXSelect::compatible(const BSONElement& elt, // the expression is a NOT. if (exprtype == MatchExpression::NOT) { // Don't allow indexed NOT on special index types such as geo or text indices. - if (INDEX_BTREE != index.type) { + // TODO: SERVER-30994 should remove this check entirely and allow $not on the + // 'non-special' fields of non-btree indices. + // (e.g. {a: 1, geo: "2dsphere"}) + if (INDEX_BTREE != index.type && !elemMatchChild) { return false; } @@ -386,12 +414,19 @@ void QueryPlannerIXSelect::rateIndices(MatchExpression* node, BSONObjIterator it(indices[i].keyPattern); BSONElement elt = it.next(); if (elt.fieldName() == fullPath && compatible(elt, indices[i], node, collator)) { - rt->first.push_back(i); + if (node->matchType() != MatchExpression::ELEM_MATCH_VALUE || + elemMatchValueCompatible(elt, indices[i], node, collator)) { + rt->first.push_back(i); + } } while (it.more()) { elt = it.next(); - if (elt.fieldName() == fullPath && compatible(elt, indices[i], node, collator)) { - rt->notFirst.push_back(i); + if (elt.fieldName() == fullPath && + compatible(elt, indices[i], node, collator, false)) { + if (node->matchType() != MatchExpression::ELEM_MATCH_VALUE || + elemMatchValueCompatible(elt, indices[i], node, collator)) { + rt->notFirst.push_back(i); + } } } } diff --git a/src/mongo/db/query/planner_ixselect.h b/src/mongo/db/query/planner_ixselect.h index 914e9265d9b..7cb34973810 100644 --- a/src/mongo/db/query/planner_ixselect.h +++ b/src/mongo/db/query/planner_ixselect.h @@ -72,7 +72,8 @@ public: static bool compatible(const BSONElement& elt, const IndexEntry& index, MatchExpression* node, - const CollatorInterface* collator); + const CollatorInterface* collator, + bool elemMatchChild = false); /** * Determine how useful all of our relevant 'indices' are to all predicates in the subtree diff --git a/src/mongo/db/query/planner_ixselect_test.cpp b/src/mongo/db/query/planner_ixselect_test.cpp index 1c273db8716..0db164a68be 100644 --- a/src/mongo/db/query/planner_ixselect_test.cpp +++ b/src/mongo/db/query/planner_ixselect_test.cpp @@ -45,6 +45,8 @@ using namespace mongo; namespace { +constexpr CollatorInterface* kSimpleCollator = nullptr; + using std::unique_ptr; using std::string; using std::vector; @@ -303,6 +305,83 @@ TEST(QueryPlannerIXSelectTest, RateIndicesTaggedNodePathArrayNegation) { testRateIndicesTaggedNodePaths("{a: {$all: [{$elemMatch: {b: {$ne: 1}}}]}}", "", "a.b,a.b"); } +/** + * $not within $elemMatch should not attempt to use a sparse index for $exists:false. + */ +TEST(QueryPlannerIXSelectTest, ElemMatchNotExistsShouldNotUseSparseIndex) { + std::vector indices; + auto idxEntry = IndexEntry(BSON("a" << 1)); + idxEntry.sparse = true; + indices.push_back(idxEntry); + std::set expectedIndices; + testRateIndices("{a: {$elemMatch: {$not: {$exists: true}}}}", + "", + kSimpleCollator, + indices, + "", + expectedIndices); +} + +/** + * $in with a null value within $elemMatch can use a sparse index. + */ +TEST(QueryPlannerIXSelectTest, ElemMatchInNullValueShouldUseSparseIndex) { + std::vector indices; + auto idxEntry = IndexEntry(BSON("a" << 1)); + idxEntry.sparse = true; + indices.push_back(idxEntry); + std::set expectedIndices = {0}; + testRateIndices( + "{a: {$elemMatch: {$in: [null]}}}", "", kSimpleCollator, indices, "a", expectedIndices); +} + +/** + * $geo queries within $elemMatch should not use a normal B-tree index. + */ +TEST(QueryPlannerIXSelectTest, ElemMatchGeoShouldNotUseBtreeIndex) { + std::vector indices; + auto idxEntry = IndexEntry(BSON("a" << 1)); + indices.push_back(idxEntry); + std::set expectedIndices; + testRateIndices(R"({a: {$elemMatch: {$geoWithin: {$geometry: {type: 'Polygon', + coordinates: [[[0,0],[0,1],[1,0],[0,0]]]}}}}})", + "", + kSimpleCollator, + indices, + "a", + expectedIndices); +} + +/** + * $eq with a null value within $elemMatch can use a sparse index. + */ +TEST(QueryPlannerIXSelectTest, ElemMatchEqNullValueShouldUseSparseIndex) { + std::vector indices; + auto idxEntry = IndexEntry(BSON("a" << 1)); + idxEntry.sparse = true; + indices.push_back(idxEntry); + std::set expectedIndices = {0}; + testRateIndices( + "{a: {$elemMatch: {$eq: null}}}", "", kSimpleCollator, indices, "a", expectedIndices); +} + +/** + * $elemMatch with multiple children will not use an index if any child is incompatible. + */ +TEST(QueryPlannerIXSelectTest, ElemMatchMultipleChildrenShouldRequireAllToBeCompatible) { + std::vector indices; + auto idxEntry = IndexEntry(BSON("a" << 1)); + idxEntry.sparse = true; + indices.push_back(idxEntry); + std::set expectedIndices; + testRateIndices("{a: {$elemMatch: {$eq: null, $not: {$exists: true}}}}", + "", + kSimpleCollator, + indices, + "", + expectedIndices); +} + /** * If the collator is null, we select the relevant index with a null collator. */ diff --git a/src/mongo/db/query/query_planner_geo_test.cpp b/src/mongo/db/query/query_planner_geo_test.cpp index 9e087444079..0421546a60d 100644 --- a/src/mongo/db/query/query_planner_geo_test.cpp +++ b/src/mongo/db/query/query_planner_geo_test.cpp @@ -1531,4 +1531,66 @@ TEST_F(QueryPlanner2dsphereVersionTest, NegationWithoutGeoPredCannotUseGeoIndex) testMultiple2dsphereIndexVersions(versions, keyPatterns, predicate, solutions); } +TEST_F(QueryPlannerTest, 2dInexactFetchPredicateOverTrailingFieldHandledCorrectly) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + + addIndex(BSON("a" + << "2d" + << "b" + << 1)); + + runQuery(fromjson("{a: {$geoWithin: {$center: [[0, 0], 1]}}, b: {$exists: true}}")); + assertNumSolutions(1U); + assertSolutionExists( + "{fetch: {filter: {a: {$geoWithin: {$center: [[0, 0], 1]}}, b: {$exists: true}}, node: " + "{ixscan: {filter: null, pattern: {a: '2d', b: 1}}}}}"); +} + +TEST_F(QueryPlannerTest, 2dInexactFetchPredicateOverTrailingFieldHandledCorrectlyMultikey) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + + const bool multikey = true; + addIndex(BSON("a" + << "2d" + << "b" + << 1), + multikey); + + runQuery(fromjson("{a: {$geoWithin: {$center: [[0, 0], 1]}}, b: {$exists: true}}")); + assertNumSolutions(1U); + assertSolutionExists( + "{fetch: {filter: {a: {$geoWithin: {$center: [[0, 0], 1]}}, b: {$exists: true}}, node: " + "{ixscan: {filter: null, pattern: {a: '2d', b: 1}}}}}"); +} + +TEST_F(QueryPlannerTest, 2dNearInexactFetchPredicateOverTrailingFieldHandledCorrectly) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + + addIndex(BSON("a" + << "2d" + << "b" + << 1)); + + runQuery(fromjson("{a: {$near: [0, 0]}, b: {$exists: true}}")); + assertNumSolutions(1U); + assertSolutionExists( + "{fetch: {filter: {b: {$exists: true}}, node: {geoNear2d: {a: '2d', b: 1}}}}"); +} + +TEST_F(QueryPlannerTest, 2dNearInexactFetchPredicateOverTrailingFieldMultikey) { + params.options = QueryPlannerParams::NO_TABLE_SCAN; + + const bool multikey = true; + addIndex(BSON("a" + << "2d" + << "b" + << 1), + multikey); + + runQuery(fromjson("{a: {$near: [0, 0]}, b: {$exists: true}}")); + assertNumSolutions(1U); + assertSolutionExists( + "{fetch: {filter: {b: {$exists: true}}, node: {geoNear2d: {a: '2d', b: 1}}}}"); +} + } // namespace diff --git a/src/mongo/db/query/query_planner_text_test.cpp b/src/mongo/db/query/query_planner_text_test.cpp index 5050653292a..cbb7fcdbd39 100644 --- a/src/mongo/db/query/query_planner_text_test.cpp +++ b/src/mongo/db/query/query_planner_text_test.cpp @@ -461,4 +461,77 @@ TEST_F(QueryPlannerTest, SortKeyMetaProjectionWithTextScoreMetaSort) { "{sortKeyGen: {node: {text: {search: 'foo'}}}}}}}}"); } +TEST_F(QueryPlannerTest, PredicatesOverLeadingFieldsWithSharedPathPrefixHandledCorrectly) { + const bool multikey = true; + addIndex(BSON("a.x" << 1 << "a.y" << 1 << "b.x" << 1 << "b.y" << 1 << "_fts" + << "text" + << "_ftsx" + << 1), + multikey); + + runQuery(fromjson("{'a.x': 1, 'a.y': 2, 'b.x': 3, 'b.y': 4, $text: {$search: 'foo'}}")); + + assertNumSolutions(1U); + assertSolutionExists( + "{text: {search: 'foo', prefix: {'a.x': 1, 'a.y': 2, 'b.x': 3, 'b.y': 4}}}"); +} + +TEST_F(QueryPlannerTest, EqualityToArrayOverLeadingFieldHandledCorrectly) { + addIndex(BSON("a" << 1 << "_fts" + << "text" + << "_ftsx" + << 1)); + + runQuery(fromjson("{a: [1, 2, 3], $text: {$search: 'foo'}}")); + + assertNumSolutions(1U); + assertSolutionExists("{text: {search: 'foo', prefix: {a: [1, 2, 3]}}}"); +} + +TEST_F(QueryPlannerTest, EqualityToArrayOverLeadingFieldHandledCorrectlyWithMultikeyTrue) { + const bool multikey = true; + addIndex(BSON("a" << 1 << "_fts" + << "text" + << "_ftsx" + << 1), + multikey); + + runQuery(fromjson("{a: [1, 2, 3], $text: {$search: 'foo'}}")); + + assertNumSolutions(1U); + assertSolutionExists("{text: {search: 'foo', prefix: {a: [1, 2, 3]}}}"); +} + +TEST_F(QueryPlannerTest, InexactFetchPredicateOverTrailingFieldHandledCorrectly) { + addIndex(BSON("a" << 1 << "_fts" + << "text" + << "_ftsx" + << 1 + << "b" + << 1)); + + runQuery(fromjson("{a: 3, $text: {$search: 'foo'}, b: {$exists: true}}")); + + assertNumSolutions(1U); + assertSolutionExists( + "{fetch: {filter: {b: {$exists: true}}, node: {text: {search: 'foo', prefix: {a: 3}}}}}"); +} + +TEST_F(QueryPlannerTest, InexactFetchPredicateOverTrailingFieldHandledCorrectlyMultikeyTrue) { + const bool multikey = true; + addIndex(BSON("a" << 1 << "_fts" + << "text" + << "_ftsx" + << 1 + << "b" + << 1), + multikey); + + runQuery(fromjson("{a: 3, $text: {$search: 'foo'}, b: {$exists: true}}")); + + assertNumSolutions(1U); + assertSolutionExists( + "{fetch: {filter: {b: {$exists: true}}, node: {text: {search: 'foo', prefix: {a: 3}}}}}"); +} + } // namespace diff --git a/src/mongo/db/query/query_solution.h b/src/mongo/db/query/query_solution.h index 7e68378042d..ff02e0df3a5 100644 --- a/src/mongo/db/query/query_solution.h +++ b/src/mongo/db/query/query_solution.h @@ -250,6 +250,13 @@ struct TextNode : public QuerySolutionNode { IndexEntry index; std::unique_ptr ftsQuery; + // The number of fields in the prefix of the text index. For example, if the key pattern is + // + // { a: 1, b: 1, _fts: "text", _ftsx: 1, c: 1 } + // + // then the number of prefix fields is 2, because of "a" and "b". + size_t numPrefixFields = 0u; + // "Prefix" fields of a text index can handle equality predicates. We group them with the // text node while creating the text leaf node and convert them into a BSONObj index prefix // when we finish the text leaf node. diff --git a/src/mongo/db/range_deleter.cpp b/src/mongo/db/range_deleter.cpp index b97d9fae216..e0d1d5c5aa0 100644 --- a/src/mongo/db/range_deleter.cpp +++ b/src/mongo/db/range_deleter.cpp @@ -354,16 +354,13 @@ bool RangeDeleter::deleteNow(OperationContext* txn, taskDetails.stats.queueEndTS = jsTime(); taskDetails.stats.deleteStartTS = jsTime(); - bool result = _env->deleteRange(txn, taskDetails, &taskDetails.stats.deletedDocCount, errMsg); - + bool result = _env->deleteRange(txn, + taskDetails, + &taskDetails.stats.deletedDocCount, + taskDetails.stats.waitForReplDurationMs, + errMsg); taskDetails.stats.deleteEndTS = jsTime(); - if (result) { - taskDetails.stats.waitForReplStartTS = jsTime(); - result = _waitForMajority(txn, errMsg); - taskDetails.stats.waitForReplEndTS = jsTime(); - } - { stdx::lock_guard sl(_queueMutex); _deleteSet.erase(&deleteRange); @@ -485,8 +482,11 @@ void RangeDeleter::doWork() { { auto txn = client->makeOperationContext(); nextTask->stats.deleteStartTS = jsTime(); - bool delResult = - _env->deleteRange(txn.get(), *nextTask, &nextTask->stats.deletedDocCount, &errMsg); + bool delResult = _env->deleteRange(txn.get(), + *nextTask, + &nextTask->stats.deletedDocCount, + nextTask->stats.waitForReplDurationMs, + &errMsg); nextTask->stats.deleteEndTS = jsTime(); if (delResult) { diff --git a/src/mongo/db/range_deleter.h b/src/mongo/db/range_deleter.h index 776f8a825fc..b6acafe3112 100644 --- a/src/mongo/db/range_deleter.h +++ b/src/mongo/db/range_deleter.h @@ -254,6 +254,7 @@ struct DeleteJobStats { Date_t deleteEndTS; Date_t waitForReplStartTS; Date_t waitForReplEndTS; + Milliseconds waitForReplDurationMs; long long int deletedDocCount; @@ -314,6 +315,7 @@ struct RangeDeleterEnv { virtual bool deleteRange(OperationContext* txn, const RangeDeleteEntry& taskDetails, long long int* deletedDocs, + Milliseconds& replWaitDuration, std::string* errMsg) = 0; /** diff --git a/src/mongo/db/range_deleter_db_env.cpp b/src/mongo/db/range_deleter_db_env.cpp index 2f692111c0a..421cafc9692 100644 --- a/src/mongo/db/range_deleter_db_env.cpp +++ b/src/mongo/db/range_deleter_db_env.cpp @@ -61,6 +61,7 @@ using std::string; bool RangeDeleterDBEnv::deleteRange(OperationContext* txn, const RangeDeleteEntry& taskDetails, long long int* deletedDocs, + Milliseconds& replWaitDuration, std::string* errMsg) { const string ns(taskDetails.options.range.ns); const BSONObj inclusiveLower(taskDetails.options.range.minKey); @@ -92,6 +93,7 @@ bool RangeDeleterDBEnv::deleteRange(OperationContext* txn, KeyRange(ns, inclusiveLower, exclusiveUpper, keyPattern), BoundInclusion::kIncludeStartKeyOnly, writeConcern, + replWaitDuration, removeSaverPtr, fromMigrate, onlyRemoveOrphans); diff --git a/src/mongo/db/range_deleter_db_env.h b/src/mongo/db/range_deleter_db_env.h index 0bca8c6618c..b0e5a11d0cc 100644 --- a/src/mongo/db/range_deleter_db_env.h +++ b/src/mongo/db/range_deleter_db_env.h @@ -46,13 +46,16 @@ struct RangeDeleterDBEnv : public RangeDeleterEnv { * Note that secondaryThrottle will be ignored if current process is not part * of a replica set. * - * docsDeleted would contain the number of docs deleted if the deletion was successful. + * deletedDocs would contain the number of docs deleted if the deletion was successful. + * + * Returns time spent waiting for majority replication in replWaitDuration. * * Does not throw Exceptions. */ virtual bool deleteRange(OperationContext* txn, const RangeDeleteEntry& taskDetails, long long int* deletedDocs, + Milliseconds& replWaitDuration, std::string* errMsg); /** diff --git a/src/mongo/db/range_deleter_mock_env.cpp b/src/mongo/db/range_deleter_mock_env.cpp index 1ee01d6a19a..9c50185ba07 100644 --- a/src/mongo/db/range_deleter_mock_env.cpp +++ b/src/mongo/db/range_deleter_mock_env.cpp @@ -104,6 +104,7 @@ DeletedRange RangeDeleterMockEnv::getLastDelete() const { bool RangeDeleterMockEnv::deleteRange(OperationContext* txn, const RangeDeleteEntry& taskDetails, long long int* deletedDocs, + Milliseconds& replWaitDuration, string* errMsg) { { stdx::unique_lock sl(_pauseDeleteMutex); diff --git a/src/mongo/db/range_deleter_mock_env.h b/src/mongo/db/range_deleter_mock_env.h index 307dbe2bd53..5792b24499c 100644 --- a/src/mongo/db/range_deleter_mock_env.h +++ b/src/mongo/db/range_deleter_mock_env.h @@ -127,6 +127,7 @@ public: bool deleteRange(OperationContext* txn, const RangeDeleteEntry& taskDetails, long long int* deletedDocs, + Milliseconds& replWaitDuration, std::string* errMsg); /** diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript index 05dbeff18a0..31f35d384f8 100644 --- a/src/mongo/db/repl/SConscript +++ b/src/mongo/db/repl/SConscript @@ -631,12 +631,12 @@ env.Library('replica_set_messages', 'is_master_response.cpp', 'member_config.cpp', 'old_update_position_args.cpp', + 'repl_set_config.cpp', 'repl_set_heartbeat_args.cpp', 'repl_set_heartbeat_args_v1.cpp', 'repl_set_heartbeat_response.cpp', 'repl_set_html_summary.cpp', 'repl_set_request_votes_args.cpp', - 'repl_set_config.cpp', 'repl_set_tag.cpp', 'update_position_args.cpp', 'last_vote.cpp', diff --git a/src/mongo/db/repl/bgsync.cpp b/src/mongo/db/repl/bgsync.cpp index 2c04c40bf55..fbbf1c56efd 100644 --- a/src/mongo/db/repl/bgsync.cpp +++ b/src/mongo/db/repl/bgsync.cpp @@ -225,15 +225,16 @@ void BackgroundSync::_runProducer() { } // we want to start when we're no longer primary // start() also loads _lastOpTimeFetched, which we know is set from the "if" - auto txn = cc().makeOperationContext(); - if (getState() == ProducerState::Starting) { - start(txn.get()); + { + auto opCtx = cc().makeOperationContext(); + if (getState() == ProducerState::Starting) { + start(opCtx.get()); + } } - - _produce(txn.get()); + _produce(); } -void BackgroundSync::_produce(OperationContext* opCtx) { +void BackgroundSync::_produce() { if (MONGO_FAIL_POINT(stopReplProducer)) { // This log output is used in js tests so please leave it. log() << "bgsync - stopReplProducer fail point " @@ -266,15 +267,18 @@ void BackgroundSync::_produce(OperationContext* opCtx) { } } - auto storageInterface = StorageInterface::get(opCtx); // find a target to sync from the last optime fetched OpTime lastOpTimeFetched; HostAndPort source; HostAndPort oldSource = _syncSourceHost; SyncSourceResolverResponse syncSourceResp; { - const OpTime minValidSaved = storageInterface->getMinValid(opCtx); - + OpTime minValidSaved; + { + auto opCtx = cc().makeOperationContext(); + auto storageInterface = StorageInterface::get(opCtx.get()); + minValidSaved = storageInterface->getMinValid(opCtx.get()); + } stdx::lock_guard lock(_mutex); if (_state != ProducerState::Running) { return; @@ -397,8 +401,12 @@ void BackgroundSync::_produce(OperationContext* opCtx) { // Set the applied point if unset. This is most likely the first time we've established a sync // source since stepping down or otherwise clearing the applied point. We need to set this here, // before the OplogWriter gets a chance to append to the oplog. - if (storageInterface->getAppliedThrough(opCtx).isNull()) { - storageInterface->setAppliedThrough(opCtx, _replCoord->getMyLastAppliedOpTime()); + { + auto opCtx = cc().makeOperationContext(); + auto storageInterface = StorageInterface::get(opCtx.get()); + if (storageInterface->getAppliedThrough(opCtx.get()).isNull()) { + storageInterface->setAppliedThrough(opCtx.get(), _replCoord->getMyLastAppliedOpTime()); + } } // "lastFetched" not used. Already set in _enqueueDocuments. @@ -521,10 +529,18 @@ void BackgroundSync::_produce(OperationContext* opCtx) { } } - OplogInterfaceLocal localOplog(opCtx, rsOplogName); - RollbackSourceImpl rollbackSource(getConnection, source, rsOplogName); - rollback( - opCtx, localOplog, rollbackSource, syncSourceResp.rbid, _replCoord, storageInterface); + { + auto opCtx = cc().makeOperationContext(); + OplogInterfaceLocal localOplog(opCtx.get(), rsOplogName); + RollbackSourceImpl rollbackSource(getConnection, source, rsOplogName); + auto storageInterface = StorageInterface::get(opCtx.get()); + rollback(opCtx.get(), + localOplog, + rollbackSource, + syncSourceResp.rbid, + _replCoord, + storageInterface); + } // Reset the producer to clear the sync source and the last optime fetched. stop(true); diff --git a/src/mongo/db/repl/bgsync.h b/src/mongo/db/repl/bgsync.h index 6d068967723..00f23e59b1d 100644 --- a/src/mongo/db/repl/bgsync.h +++ b/src/mongo/db/repl/bgsync.h @@ -148,7 +148,7 @@ private: void _run(); // Production thread inner loop. void _runProducer(); - void _produce(OperationContext* txn); + void _produce(); /** * Checks current background sync state before pushing operations into blocking queue and diff --git a/src/mongo/db/repl/collection_cloner.cpp b/src/mongo/db/repl/collection_cloner.cpp index 9dc3674efa0..bbaf04ba3dc 100644 --- a/src/mongo/db/repl/collection_cloner.cpp +++ b/src/mongo/db/repl/collection_cloner.cpp @@ -114,7 +114,8 @@ CollectionCloner::CollectionCloner(executor::TaskExecutor* executor, stdx::placeholders::_2, stdx::placeholders::_3), rpc::ServerSelectionMetadata(true, boost::none).toBSON(), - RemoteCommandRequest::kNoTimeout, + RemoteCommandRequest::kNoTimeout /* find network timeout */, + RemoteCommandRequest::kNoTimeout /* getMore network timeout */, RemoteCommandRetryScheduler::makeRetryPolicy( numInitialSyncListIndexesAttempts, executor::RemoteCommandRequest::kNoTimeout, @@ -518,7 +519,8 @@ void CollectionCloner::_beginCollectionCallback(const executor::TaskExecutor::Ca stdx::placeholders::_3, onCompletionGuard), rpc::ServerSelectionMetadata(true, boost::none).toBSON(), - RemoteCommandRequest::kNoTimeout, + RemoteCommandRequest::kNoTimeout /* find network timeout */, + RemoteCommandRequest::kNoTimeout /* getMore network timeout */, RemoteCommandRetryScheduler::makeRetryPolicy( numInitialSyncCollectionFindAttempts.load(), executor::RemoteCommandRequest::kNoTimeout, diff --git a/src/mongo/db/repl/database_cloner.cpp b/src/mongo/db/repl/database_cloner.cpp index 461c4dfe1b7..d0211fdf058 100644 --- a/src/mongo/db/repl/database_cloner.cpp +++ b/src/mongo/db/repl/database_cloner.cpp @@ -114,7 +114,8 @@ DatabaseCloner::DatabaseCloner(executor::TaskExecutor* executor, stdx::placeholders::_2, stdx::placeholders::_3), rpc::ServerSelectionMetadata(true, boost::none).toBSON(), - RemoteCommandRequest::kNoTimeout, + RemoteCommandRequest::kNoTimeout /* find network timeout */, + RemoteCommandRequest::kNoTimeout /* getMore network timeout */, RemoteCommandRetryScheduler::makeRetryPolicy( numInitialSyncListCollectionsAttempts, executor::RemoteCommandRequest::kNoTimeout, diff --git a/src/mongo/db/repl/databases_cloner.cpp b/src/mongo/db/repl/databases_cloner.cpp index 4a18e1be1b8..c9016c322f4 100644 --- a/src/mongo/db/repl/databases_cloner.cpp +++ b/src/mongo/db/repl/databases_cloner.cpp @@ -231,6 +231,44 @@ void DatabasesCloner::setScheduleDbWorkFn_forTest(const CollectionCloner::Schedu _scheduleDbWorkFn = work; } +StatusWith> DatabasesCloner::parseListDatabasesResponse_forTest( + BSONObj dbResponse) { + return _parseListDatabasesResponse(dbResponse); +} + +void DatabasesCloner::setAdminAsFirst_forTest(std::vector& dbsArray) { + _setAdminAsFirst(dbsArray); +} + +StatusWith> DatabasesCloner::_parseListDatabasesResponse( + BSONObj dbResponse) { + if (!dbResponse.hasField("databases")) { + return Status(ErrorCodes::BadValue, + "The 'listDatabases' response does not contain a 'databases' field."); + } + BSONElement response = dbResponse["databases"]; + try { + return response.Array(); + } catch (const MsgAssertionException& e) { + return Status(ErrorCodes::BadValue, + "The 'listDatabases' response is unable to be transformed into an array."); + } +} + +void DatabasesCloner::_setAdminAsFirst(std::vector& dbsArray) { + auto adminIter = std::find_if(dbsArray.begin(), dbsArray.end(), [](BSONElement elem) { + if (!elem.isABSONObj()) { + return false; + } + auto bsonObj = elem.Obj(); + std::string databaseName = bsonObj.getStringField("name"); + return (databaseName == "admin"); + }); + if (adminIter != dbsArray.end()) { + std::iter_swap(adminIter, dbsArray.begin()); + } +} + void DatabasesCloner::_onListDatabaseFinish(const CommandCallbackArgs& cbd) { Status respStatus = cbd.response.status; if (respStatus.isOK()) { @@ -239,24 +277,44 @@ void DatabasesCloner::_onListDatabaseFinish(const CommandCallbackArgs& cbd) { UniqueLock lk(_mutex); if (!respStatus.isOK()) { - LOG(1) << "listDatabases failed: " << respStatus; + LOG(1) << "'listDatabases' failed: " << respStatus; _fail_inlock(&lk, respStatus); return; } - const auto respBSON = cbd.response.data; - // There should not be any cloners yet + // There should not be any cloners yet. invariant(_databaseCloners.size() == 0); - const auto dbsElem = respBSON["databases"].Obj(); - BSONForEach(arrayElement, dbsElem) { + const auto respBSON = cbd.response.data; + + auto databasesArray = _parseListDatabasesResponse(respBSON); + if (!databasesArray.isOK()) { + LOG(1) << "'listDatabases' returned a malformed response: " + << databasesArray.getStatus().toString(); + _fail_inlock(&lk, databasesArray.getStatus()); + return; + } + + auto dbsArray = databasesArray.getValue(); + // Ensure that the 'admin' database is the first element in the array of databases so that it + // will be the first to be cloned. This allows users to authenticate against a database while + // initial sync is occurring. + _setAdminAsFirst(dbsArray); + + for (BSONElement arrayElement : dbsArray) { const BSONObj dbBSON = arrayElement.Obj(); // Check to see if we want to exclude this db from the clone. if (!_includeDbFn(dbBSON)) { - LOG(1) << "excluding db: " << dbBSON; + LOG(1) << "Excluding database from the 'listDatabases' response: " << dbBSON; continue; } + if (!dbBSON.hasField("name")) { + LOG(1) << "Excluding database due to the 'listDatabases' response not containing a " + "'name' field for this entry: " + << dbBSON; + } + const std::string dbName = dbBSON["name"].str(); std::shared_ptr dbCloner{nullptr}; @@ -321,7 +379,6 @@ void DatabasesCloner::_onListDatabaseFinish(const CommandCallbackArgs& cbd) { // add cloner to list. _databaseCloners.push_back(dbCloner); } - if (_databaseCloners.size() == 0) { if (_status.isOK()) { _succeed_inlock(&lk); diff --git a/src/mongo/db/repl/databases_cloner.h b/src/mongo/db/repl/databases_cloner.h index 53cced9ee4f..13f11f1e2a1 100644 --- a/src/mongo/db/repl/databases_cloner.h +++ b/src/mongo/db/repl/databases_cloner.h @@ -34,6 +34,7 @@ #include "mongo/base/disallow_copying.h" #include "mongo/base/status.h" +#include "mongo/base/status_with.h" #include "mongo/bson/bsonobj.h" #include "mongo/client/fetcher.h" #include "mongo/db/namespace_string.h" @@ -104,6 +105,18 @@ public: */ void setScheduleDbWorkFn_forTest(const CollectionCloner::ScheduleDbWorkFn& scheduleDbWorkFn); + /** + * Calls DatabasesCloner::_setAdminAsFirst. + * For testing only. + */ + void setAdminAsFirst_forTest(std::vector& dbsArray); + + /** + * Calls DatabasesCloner::_parseListDatabasesResponse. + * For testing only. + */ + StatusWith> parseListDatabasesResponse_forTest(BSONObj dbResponse); + private: bool _isActive_inlock() const; @@ -135,6 +148,23 @@ private: void _onListDatabaseFinish(const CommandCallbackArgs& cbd); + /** + * Takes a vector of BSONElements and scans for an element that contains a 'name' field with the + * value 'admin'. If found, the element is swapped with the first element in the vector. + * Otherwise, return. + * + * Used to parse the BSONResponse returned by listDatabases. + */ + void _setAdminAsFirst(std::vector& dbsArray); + + /** + * Takes a 'listDatabases' command response and parses the response into a + * vector of BSON elements. + * + * If the input response is malformed, Status ErrorCodes::BadValue will be returned. + */ + StatusWith> _parseListDatabasesResponse(BSONObj dbResponse); + // // All member variables are labeled with one of the following codes indicating the // synchronization rules for accessing them. diff --git a/src/mongo/db/repl/databases_cloner_test.cpp b/src/mongo/db/repl/databases_cloner_test.cpp index 81552201485..e44114e5d58 100644 --- a/src/mongo/db/repl/databases_cloner_test.cpp +++ b/src/mongo/db/repl/databases_cloner_test.cpp @@ -314,6 +314,15 @@ protected: ASSERT_OK(result); }; + std::unique_ptr makeDummyDatabasesCloner() { + return stdx::make_unique(&getStorage(), + &getExecutor(), + &getDbWorkThreadPool(), + HostAndPort{"local:1234"}, + [](const BSONObj&) { return true; }, + [](const Status&) {}); + } + private: executor::ThreadPoolMock::Options makeThreadPoolMockOptions() const override; @@ -450,6 +459,87 @@ TEST_F(DBsClonerTest, StartupReturnsInternalErrorAfterSuccessfulStartup) { ASSERT_TRUE(cloner.isActive()); } +TEST_F(DBsClonerTest, ParseAndSetAdminFirstWhenAdminInListDatabasesResponse) { + const Responses responsesWithAdmin = { + {"listDatabases", fromjson("{ok:1, databases:[{name:'a'}, {name:'aab'}, {name:'admin'}]}")}, + {"listDatabases", fromjson("{ok:1, databases:[{name:'admin'}, {name:'a'}, {name:'b'}]}")}, + }; + std::unique_ptr cloner = makeDummyDatabasesCloner(); + + for (auto&& resp : responsesWithAdmin) { + auto parseResponseStatus = cloner->parseListDatabasesResponse_forTest(resp.second); + ASSERT_TRUE(parseResponseStatus.isOK()); + std::vector dbNamesArray = parseResponseStatus.getValue(); + cloner->setAdminAsFirst_forTest(dbNamesArray); + ASSERT_EQUALS("admin", dbNamesArray[0].Obj().firstElement().str()); + } +} + +TEST_F(DBsClonerTest, ParseAndAttemptSetAdminFirstWhenAdminNotInListDatabasesResponse) { + const Responses responsesWithoutAdmin = { + {"listDatabases", fromjson("{ok:1, databases:[{name:'a'}, {name:'aab'}, {name:'abc'}]}")}, + {"listDatabases", fromjson("{ok:1, databases:[{name:'foo'}, {name:'a'}, {name:'b'}]}")}, + {"listDatabases", fromjson("{ok:1, databases:[{name:1}, {name:2}, {name:3}]}")}, + }; + std::unique_ptr cloner = makeDummyDatabasesCloner(); + + for (auto&& resp : responsesWithoutAdmin) { + auto parseResponseStatus = cloner->parseListDatabasesResponse_forTest(resp.second); + ASSERT_TRUE(parseResponseStatus.isOK()); + std::vector dbNamesArray = parseResponseStatus.getValue(); + std::string expectedResult = dbNamesArray[0].Obj().firstElement().str(); + cloner->setAdminAsFirst_forTest(dbNamesArray); + ASSERT_EQUALS(expectedResult, dbNamesArray[0].Obj().firstElement().str()); + } +} + + +TEST_F(DBsClonerTest, ParseListDatabasesResponseWithMalformedResponses) { + Status expectedResultForNoDatabasesField{ + ErrorCodes::BadValue, + "The 'listDatabases' command response does not contain a databases field."}; + Status expectedResultForNoArrayOfDatabases{ + ErrorCodes::BadValue, + "The 'listDatabases' command response is unable to be transformed into an array."}; + + const Responses responsesWithoutDatabasesField = { + {"listDatabases", fromjson("{ok:1, fake:[{name:'a'}, {name:'aab'}, {name:'foo'}]}")}, + {"listDatabases", fromjson("{ok:1, fake:[{name:'admin'}, {name:'a'}, {name:'b'}]}")}, + }; + + const Responses responsesWithoutArrayOfDatabases = { + {"listDatabases", fromjson("{ok:1, databases:1}")}, + {"listDatabases", fromjson("{ok:1, databases:'abc'}")}, + }; + + const Responses responsesWithInvalidAdminNameField = { + {"listDatabases", fromjson("{ok:1, databases:[{name:'a'}, {name:'aab'}, {fake:'admin'}]}")}, + {"listDatabases", fromjson("{ok:1, databases:[{fake:'admin'}, {name:'a'}, {name:'b'}]}")}, + }; + + std::unique_ptr cloner = makeDummyDatabasesCloner(); + + for (auto&& resp : responsesWithoutDatabasesField) { + auto parseResponseStatus = cloner->parseListDatabasesResponse_forTest(resp.second); + ASSERT_EQ(parseResponseStatus.getStatus(), expectedResultForNoDatabasesField); + } + + for (auto&& resp : responsesWithoutArrayOfDatabases) { + auto parseResponseStatus = cloner->parseListDatabasesResponse_forTest(resp.second); + ASSERT_EQ(parseResponseStatus.getStatus(), expectedResultForNoArrayOfDatabases); + } + + for (auto&& resp : responsesWithInvalidAdminNameField) { + auto parseResponseStatus = cloner->parseListDatabasesResponse_forTest(resp.second); + ASSERT_TRUE(parseResponseStatus.isOK()); + // We expect no elements to be swapped. + std::vector dbNamesArray = parseResponseStatus.getValue(); + std::string expectedResult = dbNamesArray[0].Obj().firstElement().str(); + cloner->setAdminAsFirst_forTest(dbNamesArray); + ASSERT_EQUALS(expectedResult, dbNamesArray[0].Obj().firstElement().str()); + } +} + TEST_F(DBsClonerTest, FailsOnListDatabases) { Status result{Status::OK()}; Status expectedResult{ErrorCodes::BadValue, "foo"}; diff --git a/src/mongo/db/repl/initial_syncer.cpp b/src/mongo/db/repl/initial_syncer.cpp index e9a920bb37b..f3bc1a8acd3 100644 --- a/src/mongo/db/repl/initial_syncer.cpp +++ b/src/mongo/db/repl/initial_syncer.cpp @@ -1189,7 +1189,8 @@ Status InitialSyncer::_scheduleLastOplogEntryFetcher_inlock(Fetcher::CallbackFn query, callback, rpc::ServerSelectionMetadata(true, boost::none).toBSON(), - RemoteCommandRequest::kNoTimeout, + RemoteCommandRequest::kNoTimeout /* find network timeout */, + RemoteCommandRequest::kNoTimeout /* getMore network timeout */, RemoteCommandRetryScheduler::makeRetryPolicy( numInitialSyncOplogFindAttempts, executor::RemoteCommandRequest::kNoTimeout, diff --git a/src/mongo/db/repl/master_slave.cpp b/src/mongo/db/repl/master_slave.cpp index 96cfe58efdc..6929d5be297 100644 --- a/src/mongo/db/repl/master_slave.cpp +++ b/src/mongo/db/repl/master_slave.cpp @@ -168,7 +168,7 @@ BSONObj ReplSource::jsobj() { BSONObjBuilder dbsNextPassBuilder; int n = 0; - for (set::iterator i = addDbNextPass.begin(); i != addDbNextPass.end(); i++) { + for (set::iterator i = addDbNextPass.begin(); i != addDbNextPass.end(); i++) { n++; dbsNextPassBuilder.appendBool(*i, 1); } @@ -177,7 +177,8 @@ BSONObj ReplSource::jsobj() { BSONObjBuilder incompleteCloneDbsBuilder; n = 0; - for (set::iterator i = incompleteCloneDbs.begin(); i != incompleteCloneDbs.end(); i++) { + for (set::iterator i = incompleteCloneDbs.begin(); i != incompleteCloneDbs.end(); + i++) { n++; incompleteCloneDbsBuilder.appendBool(*i, 1); } @@ -188,7 +189,7 @@ BSONObj ReplSource::jsobj() { } void ReplSource::ensureMe(OperationContext* txn) { - string myname = getHostName(); + std::string myname = getHostName(); // local.me is an identifier for a server for getLastError w:2+ bool exists = Helpers::getSingleton(txn, "local.me", _me); @@ -378,10 +379,10 @@ public: } virtual bool run(OperationContext* txn, - const string& ns, + const std::string& ns, BSONObj& cmdObj, int options, - string& errmsg, + std::string& errmsg, BSONObjBuilder& result) { HandshakeArgs handshake; Status status = handshake.initialize(cmdObj); @@ -398,7 +399,7 @@ public: } handshakeCmd; bool replHandshake(DBClientConnection* conn, const OID& myRID) { - string myname = getHostName(); + std::string myname = getHostName(); BSONObjBuilder cmd; cmd.append("handshake", myRID); @@ -450,7 +451,7 @@ void ReplSource::forceResync(OperationContext* txn, const char* requester) { BSONElement e = i.next(); if (e.eoo()) break; - string name = e.embeddedObject().getField("name").valuestr(); + std::string name = e.embeddedObject().getField("name").valuestr(); if (!e.embeddedObject().getBoolField("empty")) { if (name != "local") { if (only.empty() || only == name) { @@ -481,7 +482,7 @@ Status ReplSource::_updateIfDoneWithInitialSync() { return Status::OK(); } -void ReplSource::resyncDrop(OperationContext* txn, const string& dbName) { +void ReplSource::resyncDrop(OperationContext* txn, const std::string& dbName) { log() << "resync: dropping database " << dbName; invariant(txn->lockState()->isW()); @@ -531,13 +532,13 @@ void ReplSource::resync(OperationContext* txn, const std::string& dbName) { static DatabaseIgnorer ___databaseIgnorer; -void DatabaseIgnorer::doIgnoreUntilAfter(const string& db, const Timestamp& futureOplogTime) { +void DatabaseIgnorer::doIgnoreUntilAfter(const std::string& db, const Timestamp& futureOplogTime) { if (futureOplogTime > _ignores[db]) { _ignores[db] = futureOplogTime; } } -bool DatabaseIgnorer::ignoreAt(const string& db, const Timestamp& currentOplogTime) { +bool DatabaseIgnorer::ignoreAt(const std::string& db, const Timestamp& currentOplogTime) { if (_ignores[db].isNull()) { return false; } @@ -627,7 +628,7 @@ bool ReplSource::handleDuplicateDbName(OperationContext* txn, // The database is present on the master and no conflicting databases // are present on the master. Drop any local conflicts. - for (set::const_iterator i = duplicates.begin(); i != duplicates.end(); ++i) { + for (set::const_iterator i = duplicates.begin(); i != duplicates.end(); ++i) { ___databaseIgnorer.doIgnoreUntilAfter(*i, lastTime); incompleteCloneDbs.erase(*i); addDbNextPass.erase(*i); @@ -822,10 +823,10 @@ void ReplSource::_sync_pullOpLog_applyOperation(OperationContext* txn, } void ReplSource::syncToTailOfRemoteLog() { - string _ns = ns(); + std::string _ns = ns(); BSONObjBuilder b; if (!only.empty()) { - b.appendRegex("ns", string("^") + pcrecpp::RE::QuoteMeta(only)); + b.appendRegex("ns", std::string("^") + pcrecpp::RE::QuoteMeta(only)); } BSONObj last = oplogReader.findOne(_ns.c_str(), Query(b.done()).sort(BSON("$natural" << -1))); if (!last.isEmpty()) { @@ -873,7 +874,7 @@ public: */ int ReplSource::_sync_pullOpLog(OperationContext* txn, int& nApplied) { int okResultCode = restartSyncAfterSleep; - string ns = string("local.oplog.$") + sourceName(); + std::string ns = std::string("local.oplog.$") + sourceName(); LOG(2) << "sync_pullOpLog " << ns << " syncedTo:" << syncedTo.toStringLong() << '\n'; bool tailing = true; @@ -893,7 +894,7 @@ int ReplSource::_sync_pullOpLog(OperationContext* txn, int& nApplied) { BSONElement e = i.next(); if (e.eoo()) break; - string name = e.embeddedObject().getField("name").valuestr(); + std::string name = e.embeddedObject().getField("name").valuestr(); if (!e.embeddedObject().getBoolField("empty")) { if (name != "local") { if (only.empty() || only == name) { @@ -917,7 +918,7 @@ int ReplSource::_sync_pullOpLog(OperationContext* txn, int& nApplied) { if (!only.empty()) { // note we may here skip a LOT of data table scanning, a lot of work for the master. // maybe append "\\." here? - query.appendRegex("ns", string("^") + pcrecpp::RE::QuoteMeta(only)); + query.appendRegex("ns", std::string("^") + pcrecpp::RE::QuoteMeta(only)); } BSONObj queryObj = query.done(); // e.g. queryObj = { ts: { $gte: syncedTo } } @@ -936,7 +937,7 @@ int ReplSource::_sync_pullOpLog(OperationContext* txn, int& nApplied) { // show any deferred database creates from a previous pass { - set::iterator i = addDbNextPass.begin(); + set::iterator i = addDbNextPass.begin(); if (i != addDbNextPass.end()) { BSONObjBuilder b; b.append("ns", *i + '.'); @@ -980,7 +981,7 @@ int ReplSource::_sync_pullOpLog(OperationContext* txn, int& nApplied) { BSONObj op = oplogReader.nextSafe(); BSONElement ts = op.getField("ts"); if (ts.type() != Date && ts.type() != bsonTimestamp) { - string err = op.getStringField("$err"); + std::string err = op.getStringField("$err"); if (!err.empty()) { // 13051 is "tailable cursor requested on non capped collection" if (op.getIntField("code") == 13051) { @@ -1148,7 +1149,7 @@ int ReplSource::sync(OperationContext* txn, int& nApplied) { // FIXME Handle cases where this db isn't on default port, or default port is spec'd in // hostName. - if ((string("localhost") == hostName || string("127.0.0.1") == hostName) && + if ((std::string("localhost") == hostName || std::string("127.0.0.1") == hostName) && serverGlobalParams.port == ServerGlobalParams::DefaultDBPort) { log() << "can't sync from self (localhost). sources configuration may be wrong." << endl; sleepsecs(5); @@ -1293,7 +1294,7 @@ static void replMain(OperationContext* txn) { if (s) { stringstream ss; ss << "sleep " << s << " sec before next pass"; - string msg = ss.str(); + std::string msg = ss.str(); if (!serverGlobalParams.quiet) log() << msg << endl; ReplInfo r(msg.c_str()); diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp index 43a1eba5042..76df271ed78 100644 --- a/src/mongo/db/repl/oplog.cpp +++ b/src/mongo/db/repl/oplog.cpp @@ -116,6 +116,11 @@ namespace { // cached copy...so don't rename, drop, etc.!!! Collection* _localOplogCollection = nullptr; +// Specifies whether we abort initial sync when attempting to apply a renameCollection operation. +// If set to true, users risk corrupting their data. This should only be enabled by expert users +// of the server who understand the risks this poses. +MONGO_EXPORT_SERVER_PARAMETER(allowUnsafeRenamesDuringInitialSync, bool, false); + PseudoRandom hashGenerator(std::unique_ptr(SecureRandom::create())->nextInt64()); // Synchronizes the section where a new Timestamp is generated and when it actually @@ -663,6 +668,45 @@ std::map opsMap = { } // namespace +std::pair prepForApplyOpsIndexInsert(const BSONElement& fieldO, + const BSONObj& op, + const NamespaceString& requestNss) { + uassert(ErrorCodes::NoSuchKey, + str::stream() << "Missing expected index spec in field 'o': " << op, + !fieldO.eoo()); + uassert(ErrorCodes::TypeMismatch, + str::stream() << "Expected object for index spec in field 'o': " << op, + fieldO.isABSONObj()); + BSONObj indexSpec = fieldO.embeddedObject(); + + std::string indexNs; + uassertStatusOK(bsonExtractStringField(indexSpec, "ns", &indexNs)); + const NamespaceString indexNss(indexNs); + uassert(ErrorCodes::InvalidNamespace, + str::stream() << "Invalid namespace in index spec: " << op, + indexNss.isValid()); + uassert(ErrorCodes::InvalidNamespace, + str::stream() << "Database name mismatch for database (" << requestNss.db() + << ") while creating index: " + << op, + requestNss.db() == indexNss.db()); + + if (!indexSpec["v"]) { + // If the "v" field isn't present in the index specification, then we assume it is a + // v=1 index from an older version of MongoDB. This is because + // (1) we haven't built v=0 indexes as the default for a long time, and + // (2) the index version has been included in the corresponding oplog entry since + // v=2 indexes were introduced. + BSONObjBuilder bob; + + bob.append("v", static_cast(IndexVersion::kV1)); + bob.appendElements(indexSpec); + + indexSpec = bob.obj(); + } + + return std::make_pair(indexSpec, indexNss); +} // @return failure status if an update should have happened and the document DNE. // See replset initial sync code. Status applyOperation_inlock(OperationContext* txn, @@ -688,6 +732,7 @@ Status applyOperation_inlock(OperationContext* txn, o = fieldO.embeddedObject(); const StringData ns = fieldNs.valueStringData(); + NamespaceString requestNss{ns}; BSONObj o2; if (fieldO2.isABSONObj()) @@ -718,27 +763,11 @@ Status applyOperation_inlock(OperationContext* txn, invariant(*opType != 'c'); // commands are processed in applyCommand_inlock() if (*opType == 'i') { - if (nsToCollectionSubstring(ns) == "system.indexes") { - uassert(ErrorCodes::NoSuchKey, - str::stream() << "Missing expected index spec in field 'o': " << op, - !fieldO.eoo()); - uassert(ErrorCodes::TypeMismatch, - str::stream() << "Expected object for index spec in field 'o': " << op, - fieldO.isABSONObj()); - BSONObj indexSpec = fieldO.embeddedObject(); - - std::string indexNs; - uassertStatusOK(bsonExtractStringField(indexSpec, "ns", &indexNs)); - const NamespaceString indexNss(indexNs); - uassert(ErrorCodes::InvalidNamespace, - str::stream() << "Invalid namespace in index spec: " << op, - indexNss.isValid()); - uassert(ErrorCodes::InvalidNamespace, - str::stream() << "Database name mismatch for database (" - << nsToDatabaseSubstring(ns) - << ") while creating index: " - << op, - nsToDatabaseSubstring(ns) == indexNss.db()); + if (requestNss.isSystemDotIndexes()) { + BSONObj indexSpec; + NamespaceString indexNss; + std::tie(indexSpec, indexNss) = + repl::prepForApplyOpsIndexInsert(fieldO, op, requestNss); // Check if collection exists. auto indexCollection = db->getCollection(indexNss); @@ -749,20 +778,6 @@ Status applyOperation_inlock(OperationContext* txn, opCounters->gotInsert(); - if (!indexSpec["v"]) { - // If the "v" field isn't present in the index specification, then we assume it is a - // v=1 index from an older version of MongoDB. This is because - // (1) we haven't built v=0 indexes as the default for a long time, and - // (2) the index version has been included in the corresponding oplog entry since - // v=2 indexes were introduced. - BSONObjBuilder bob; - - bob.append("v", static_cast(IndexVersion::kV1)); - bob.appendElements(indexSpec); - - indexSpec = bob.obj(); - } - bool relaxIndexConstraints = ReplicationCoordinator::get(txn)->shouldRelaxIndexConstraints(indexNss); if (indexSpec["background"].trueValue()) { @@ -863,13 +878,12 @@ Status applyOperation_inlock(OperationContext* txn, BSONObjBuilder b; b.append(o.getField("_id")); - const NamespaceString requestNs(ns); - UpdateRequest request(requestNs); + UpdateRequest request(requestNss); request.setQuery(b.done()); request.setUpdates(o); request.setUpsert(); - UpdateLifecycleImpl updateLifecycle(requestNs); + UpdateLifecycleImpl updateLifecycle(requestNss); request.setLifecycle(&updateLifecycle); UpdateResult res = update(txn, db, request); @@ -894,13 +908,12 @@ Status applyOperation_inlock(OperationContext* txn, str::stream() << "Failed to apply update due to missing _id: " << op.toString(), updateCriteria.hasField("_id")); - const NamespaceString requestNs(ns); - UpdateRequest request(requestNs); + UpdateRequest request(requestNss); request.setQuery(updateCriteria); request.setUpdates(o); request.setUpsert(upsert); - UpdateLifecycleImpl updateLifecycle(requestNs); + UpdateLifecycleImpl updateLifecycle(requestNss); request.setLifecycle(&updateLifecycle); UpdateResult ur = update(txn, db, request); @@ -954,7 +967,12 @@ Status applyOperation_inlock(OperationContext* txn, o.hasField("_id")); if (opType[1] == 0) { - deleteObjects(txn, collection, ns, o, PlanExecutor::YIELD_MANUAL, /*justOne*/ valueB); + deleteObjects(txn, + collection, + requestNss.ns().c_str(), + o, + PlanExecutor::YIELD_MANUAL, + /*justOne*/ valueB); } else verify(opType[1] == 'b'); // "db" advertisement if (incrementOpsAppliedStats) { @@ -970,16 +988,6 @@ Status applyOperation_inlock(OperationContext* txn, 14825, str::stream() << "error in applyOperation : unknown opType " << *opType); } - // AuthorizationManager's logOp method registers a RecoveryUnit::Change and to do so we need - // to a new WriteUnitOfWork, if we dont have a wrapping unit of work already. If we already - // have a wrapping WUOW, the extra nexting is harmless. The logOp really should have been - // done in the WUOW that did the write, but this won't happen because applyOps turns off - // observers. - WriteUnitOfWork wuow(txn); - getGlobalAuthorizationManager()->logOp( - txn, opType, ns.toString().c_str(), o, fieldO2.isABSONObj() ? &o2 : NULL); - wuow.commit(); - return Status::OK(); } @@ -1021,9 +1029,15 @@ Status applyCommand_inlock(OperationContext* txn, // Applying renameCollection during initial sync might lead to data corruption, so we restart // the initial sync. if (!inSteadyStateReplication && o.firstElementFieldName() == std::string("renameCollection")) { - return Status(ErrorCodes::OplogOperationUnsupported, - str::stream() << "Applying renameCollection not supported in initial sync: " - << redact(op)); + if (!allowUnsafeRenamesDuringInitialSync.load()) { + return Status(ErrorCodes::OplogOperationUnsupported, + str::stream() + << "Applying renameCollection not supported in initial sync: " + << redact(op)); + } + warning() << "allowUnsafeRenamesDuringInitialSync set to true. Applying renameCollection " + "operation during initial sync even though it may lead to data corruption: " + << redact(op); } // Applying commands in repl is done under Global W-lock, so it is safe to not @@ -1071,11 +1085,8 @@ Status applyCommand_inlock(OperationContext* txn, break; } default: - if (_oplogCollectionName == masterSlaveOplogName) { - error() << "Failed command " << redact(o) << " on " << nss.db() - << " with status " << status << " during oplog application"; - } else if (curOpToApply.acceptableErrors.find(status.code()) == - curOpToApply.acceptableErrors.end()) { + if (curOpToApply.acceptableErrors.find(status.code()) == + curOpToApply.acceptableErrors.end()) { error() << "Failed command " << redact(o) << " on " << nss.db() << " with status " << status << " during oplog application"; return status; diff --git a/src/mongo/db/repl/oplog.h b/src/mongo/db/repl/oplog.h index b2078e93d28..f51aadb8dcf 100644 --- a/src/mongo/db/repl/oplog.h +++ b/src/mongo/db/repl/oplog.h @@ -103,6 +103,15 @@ void logOp(OperationContext* txn, void oplogCheckCloseDatabase(OperationContext* txn, Database* db); using IncrementOpsAppliedStatsFn = stdx::function; +/** + * Take the object field of a BSONObj, the BSONObj, and the namespace of + * the operation and perform necessary validation to ensure the BSONObj is a + * properly-formed command to insert into system.indexes. This is only to + * be used for insert operations into system.indexes. It is called via applyOps. + */ +std::pair prepForApplyOpsIndexInsert(const BSONElement& fieldO, + const BSONObj& op, + const NamespaceString& requestNss); /** * Take a non-command op and apply it locally * Used for applying from an oplog diff --git a/src/mongo/db/repl/oplog_fetcher.cpp b/src/mongo/db/repl/oplog_fetcher.cpp index ece0ef266fd..c9b96ab77f7 100644 --- a/src/mongo/db/repl/oplog_fetcher.cpp +++ b/src/mongo/db/repl/oplog_fetcher.cpp @@ -36,6 +36,7 @@ #include "mongo/db/commands/server_status_metric.h" #include "mongo/db/jsobj.h" #include "mongo/db/repl/replication_coordinator.h" +#include "mongo/db/server_parameters.h" #include "mongo/db/stats/timer_stats.h" #include "mongo/rpc/metadata/oplog_query_metadata.h" #include "mongo/rpc/metadata/server_selection_metadata.h" @@ -56,8 +57,12 @@ MONGO_FP_DECLARE(stopReplProducer); namespace { -Seconds kOplogInitialFindMaxTime{60}; -Seconds kOplogQueryNetworkTimeout{65}; // 5 seconds past the find command's 1 minute maxTimeMs +// Number of seconds for the `maxTimeMS` on the initial `find` command. +MONGO_EXPORT_SERVER_PARAMETER(oplogInitialFindMaxSeconds, int, 60); + +// Number of milliseconds to add to the `find` and `getMore` timeouts to calculate the network +// timeout for the requests. +const Milliseconds kNetworkTimeoutBufferMS{5000}; Counter64 readersCreatedStats; ServerStatusMetricField displayReadersCreated("repl.network.readersCreated", @@ -91,17 +96,22 @@ Milliseconds calculateAwaitDataTimeout(const ReplSetConfig& config) { */ BSONObj makeFindCommandObject(const NamespaceString& nss, long long currentTerm, - OpTime lastOpTimeFetched) { + OpTime lastOpTimeFetched, + Milliseconds fetcherMaxTimeMS) { BSONObjBuilder cmdBob; cmdBob.append("find", nss.coll()); cmdBob.append("filter", BSON("ts" << BSON("$gte" << lastOpTimeFetched.getTimestamp()))); cmdBob.append("tailable", true); cmdBob.append("oplogReplay", true); cmdBob.append("awaitData", true); - cmdBob.append("maxTimeMS", durationCount(kOplogInitialFindMaxTime)); + cmdBob.append("maxTimeMS", durationCount(fetcherMaxTimeMS)); if (currentTerm != OpTime::kUninitializedTerm) { cmdBob.append("term", currentTerm); } + if (serverGlobalParams.featureCompatibility.version.load() == + ServerGlobalParams::FeatureCompatibility::Version::k34) { + cmdBob.append("readConcern", BSON("afterOpTime" << lastOpTimeFetched.toBSON())); + } return cmdBob.obj(); } @@ -450,6 +460,14 @@ BSONObj OplogFetcher::getMetadataObject_forTest() const { } Milliseconds OplogFetcher::getAwaitDataTimeout_forTest() const { + return _getGetMoreMaxTime(); +} + +Milliseconds OplogFetcher::_getFindMaxTime() const { + return Milliseconds(oplogInitialFindMaxSeconds.load() * 1000); +} + +Milliseconds OplogFetcher::_getGetMoreMaxTime() const { return _awaitDataTimeout; } @@ -660,7 +678,7 @@ void OplogFetcher::_callback(const Fetcher::QueryResponseStatus& result, getMoreBob->appendElements(makeGetMoreCommandObject(queryResponse.nss, queryResponse.cursorId, lastCommittedWithCurrentTerm, - _awaitDataTimeout)); + _getGetMoreMaxTime())); } void OplogFetcher::_finishCallback(Status status) { @@ -691,10 +709,11 @@ std::unique_ptr OplogFetcher::_makeFetcher(long long currentTerm, _executor, _source, _nss.db().toString(), - makeFindCommandObject(_nss, currentTerm, lastFetchedOpTime), + makeFindCommandObject(_nss, currentTerm, lastFetchedOpTime, _getFindMaxTime()), stdx::bind(&OplogFetcher::_callback, this, stdx::placeholders::_1, stdx::placeholders::_3), _metadataObject, - kOplogQueryNetworkTimeout); + _getFindMaxTime() + kNetworkTimeoutBufferMS, + _getGetMoreMaxTime() + kNetworkTimeoutBufferMS); } bool OplogFetcher::_isShuttingDown() const { diff --git a/src/mongo/db/repl/oplog_fetcher.h b/src/mongo/db/repl/oplog_fetcher.h index ad4fde296b1..54bfbabbf8d 100644 --- a/src/mongo/db/repl/oplog_fetcher.h +++ b/src/mongo/db/repl/oplog_fetcher.h @@ -234,6 +234,16 @@ private: void _finishCallback(Status status); void _finishCallback(Status status, OpTimeWithHash opTimeWithHash); + /** + * Returns how long the `find` command should wait before timing out. + */ + virtual Milliseconds _getFindMaxTime() const; + + /** + * Returns how long the `getMore` command should wait before timing out. + */ + virtual Milliseconds _getGetMoreMaxTime() const; + /** * Creates a new instance of the fetcher to tail the remote oplog starting at the given optime. */ diff --git a/src/mongo/db/repl/repl_client_info.cpp b/src/mongo/db/repl/repl_client_info.cpp index 3e98a0cb9d3..5b77e3aa6f3 100644 --- a/src/mongo/db/repl/repl_client_info.cpp +++ b/src/mongo/db/repl/repl_client_info.cpp @@ -26,6 +26,8 @@ * it in the license file. */ +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kReplication + #include "mongo/platform/basic.h" #include "mongo/db/repl/repl_client_info.h" @@ -36,6 +38,7 @@ #include "mongo/db/operation_context.h" #include "mongo/db/repl/replication_coordinator_global.h" #include "mongo/util/decorable.h" +#include "mongo/util/log.h" namespace mongo { namespace repl { @@ -48,10 +51,23 @@ void ReplClientInfo::setLastOp(const OpTime& ot) { _lastOp = ot; } + void ReplClientInfo::setLastOpToSystemLastOpTime(OperationContext* txn) { ReplicationCoordinator* replCoord = repl::ReplicationCoordinator::get(txn->getServiceContext()); if (replCoord->isReplEnabled() && txn->writesAreReplicated()) { - setLastOp(replCoord->getMyLastAppliedOpTime()); + auto systemOpTime = replCoord->getMyLastAppliedOpTime(); + + // If the system optime has gone backwards, that must mean that there was a rollback. + // This is safe, but the last op for a Client should never go backwards, so just leave + // the last op for this Client as it was. + if (systemOpTime >= _lastOp) { + _lastOp = systemOpTime; + } else { + log() << "Not setting the last OpTime for this Client from " << _lastOp + << " to the current system time of " << systemOpTime + << " as that would be moving the OpTime backwards. This should only happen if " + "there was a rollback recently"; + } } } diff --git a/src/mongo/db/repl/repl_set_config.cpp b/src/mongo/db/repl/repl_set_config.cpp index 1ad2d245ac6..024c3ff04a5 100644 --- a/src/mongo/db/repl/repl_set_config.cpp +++ b/src/mongo/db/repl/repl_set_config.cpp @@ -35,11 +35,19 @@ #include "mongo/bson/util/bson_check.h" #include "mongo/bson/util/bson_extract.h" #include "mongo/db/jsobj.h" +#include "mongo/db/mongod_options.h" #include "mongo/db/server_options.h" +#include "mongo/db/server_parameters.h" #include "mongo/stdx/functional.h" #include "mongo/util/stringutils.h" namespace mongo { +/** + * Dont run any sharding validations. Can not be combined with --configsvr or shardvr. Intended to + * allow restarting config server or shard as an independent replica set. + */ +MONGO_EXPORT_STARTUP_SERVER_PARAMETER(skipShardingConfigurationChecks, bool, false); + namespace repl { const size_t ReplSetConfig::kMaxMembers; @@ -552,7 +560,8 @@ Status ReplSetConfig::validate() const { "servers cannot have a non-zero slaveDelay"); } } - if (serverGlobalParams.clusterRole != ClusterRole::ConfigServer) { + if (serverGlobalParams.clusterRole != ClusterRole::ConfigServer && + !skipShardingConfigurationChecks) { return Status(ErrorCodes::BadValue, "Nodes being used for config servers must be started with the " "--configsvr flag"); diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp index cfc6dbb55e1..05d070c08fe 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp @@ -240,8 +240,11 @@ void ReplicationCoordinatorExternalStateImpl::startSteadyStateReplication( _applierThread->startup(); log() << "Starting replication reporter thread"; invariant(!_syncSourceFeedbackThread); - _syncSourceFeedbackThread.reset(new stdx::thread(stdx::bind( - &SyncSourceFeedback::run, &_syncSourceFeedback, _taskExecutor.get(), _bgSync.get()))); + _syncSourceFeedbackThread.reset(new stdx::thread(stdx::bind(&SyncSourceFeedback::run, + &_syncSourceFeedback, + _taskExecutor.get(), + _bgSync.get(), + replCoord))); } void ReplicationCoordinatorExternalStateImpl::stopDataReplication(OperationContext* txn) { diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp index 3de3d35bb61..ad1a9fd93ba 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl.cpp @@ -2922,6 +2922,26 @@ ReplicationCoordinatorImpl::_setCurrentRSConfig_inlock(const ReplSetConfig& newC const ReplSetConfig oldConfig = _rsConfig; _rsConfig = newConfig; _protVersion.store(_rsConfig.getProtocolVersion()); + + // Warn if running --nojournal and writeConcernMajorityJournalDefault = false + StorageEngine* storageEngine = getGlobalServiceContext()->getGlobalStorageEngine(); + if (storageEngine && !storageEngine->isDurable() && + (newConfig.getWriteConcernMajorityShouldJournal() && + (!oldConfig.isInitialized() || !oldConfig.getWriteConcernMajorityShouldJournal()))) { + log() << startupWarningsLog; + log() << "** WARNING: This replica set is running without journaling enabled but the " + << startupWarningsLog; + log() << "** writeConcernMajorityJournalDefault option to the replica set config " + << startupWarningsLog; + log() << "** is set to true. The writeConcernMajorityJournalDefault " + << startupWarningsLog; + log() << "** option to the replica set config must be set to false " + << startupWarningsLog; + log() << "** or w:majority write concerns will never complete." + << startupWarningsLog; + log() << startupWarningsLog; + } + log() << "New replica set config in use: " << _rsConfig.toBSON() << rsLog; _selfIndex = myIndex; if (_selfIndex >= 0) { diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp index fbd976ec6b7..1dd9bf5b7c3 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp @@ -133,7 +133,9 @@ void ReplicationCoordinatorImpl::_startElectSelfV1() { return; } - log() << "conducting a dry run election to see if we could be elected"; + long long term = _topCoord->getTerm(); + + log() << "conducting a dry run election to see if we could be elected. current term: " << term; _voteRequester.reset(new VoteRequester); // This is necessary because the voteRequester may call directly into winning an @@ -141,12 +143,11 @@ void ReplicationCoordinatorImpl::_startElectSelfV1() { // _mutex again. lk.unlock(); - long long term = _topCoord->getTerm(); StatusWith nextPhaseEvh = _voteRequester->start(&_replExecutor, _rsConfig, _selfIndex, - _topCoord->getTerm(), + term, true, // dry run lastOpTime); if (nextPhaseEvh.getStatus() == ErrorCodes::ShutdownInProgress) { @@ -165,7 +166,8 @@ void ReplicationCoordinatorImpl::_onDryRunComplete(long long originalTerm) { LockGuard lk(_topoMutex); if (_topCoord->getTerm() != originalTerm) { - log() << "not running for primary, we have been superceded already"; + log() << "not running for primary, we have been superseded already during dry run. " + << "original term: " << originalTerm << ", current term: " << _topCoord->getTerm(); return; } @@ -175,23 +177,24 @@ void ReplicationCoordinatorImpl::_onDryRunComplete(long long originalTerm) { log() << "not running for primary, we received insufficient votes"; return; } else if (endResult == VoteRequester::Result::kStaleTerm) { - log() << "not running for primary, we have been superceded already"; + log() << "not running for primary, we have been superseded already"; return; } else if (endResult != VoteRequester::Result::kSuccessfullyElected) { log() << "not running for primary, we received an unexpected problem"; return; } - log() << "dry election run succeeded, running for election"; + long long newTerm = originalTerm + 1; + log() << "dry election run succeeded, running for election in term " << newTerm; // Stepdown is impossible from this term update. TopologyCoordinator::UpdateTermResult updateTermResult; - _updateTerm_incallback(originalTerm + 1, &updateTermResult); + _updateTerm_incallback(newTerm, &updateTermResult); invariant(updateTermResult == TopologyCoordinator::UpdateTermResult::kUpdatedTerm); // Secure our vote for ourself first _topCoord->voteForMyselfV1(); // Store the vote in persistent storage. - LastVote lastVote{originalTerm + 1, _selfIndex}; + LastVote lastVote{newTerm, _selfIndex}; auto cbStatus = _replExecutor.scheduleDBWork( [this, lastVote](const ReplicationExecutor::CallbackArgs& cbData) { @@ -232,12 +235,19 @@ void ReplicationCoordinatorImpl::_startVoteRequester(long long newTerm) { LockGuard lk(_topoMutex); + if (_topCoord->getTerm() != newTerm) { + log() << "not running for primary, we have been superseded already while writing our last " + "vote. election term: " + << newTerm << ", current term: " << _topCoord->getTerm(); + return; + } + const auto lastOpTime = _isDurableStorageEngine() ? getMyLastDurableOpTime() : getMyLastAppliedOpTime(); _voteRequester.reset(new VoteRequester); - StatusWith nextPhaseEvh = _voteRequester->start( - &_replExecutor, _rsConfig, _selfIndex, _topCoord->getTerm(), false, lastOpTime); + StatusWith nextPhaseEvh = + _voteRequester->start(&_replExecutor, _rsConfig, _selfIndex, newTerm, false, lastOpTime); if (nextPhaseEvh.getStatus() == ErrorCodes::ShutdownInProgress) { return; } @@ -249,14 +259,15 @@ void ReplicationCoordinatorImpl::_startVoteRequester(long long newTerm) { lossGuard.dismiss(); } -void ReplicationCoordinatorImpl::_onVoteRequestComplete(long long originalTerm) { +void ReplicationCoordinatorImpl::_onVoteRequestComplete(long long newTerm) { invariant(_voteRequester); LoseElectionGuardV1 lossGuard(this); LockGuard lk(_topoMutex); - if (_topCoord->getTerm() != originalTerm) { - log() << "not becoming primary, we have been superceded already"; + if (_topCoord->getTerm() != newTerm) { + log() << "not becoming primary, we have been superseded already during election. " + << "election term: " << newTerm << ", current term: " << _topCoord->getTerm(); return; } @@ -267,7 +278,7 @@ void ReplicationCoordinatorImpl::_onVoteRequestComplete(long long originalTerm) log() << "not becoming primary, we received insufficient votes"; return; case VoteRequester::Result::kStaleTerm: - log() << "not becoming primary, we have been superceded already"; + log() << "not becoming primary, we have been superseded already"; return; case VoteRequester::Result::kSuccessfullyElected: log() << "election succeeded, assuming primary role in term " << _topCoord->getTerm(); diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp index 6e87f882b48..0f550869546 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp @@ -431,7 +431,7 @@ TEST_F(ReplCoordTest, ElectionFailsWhenDryRunResponseContainsANewerTerm) { getReplCoord()->waitForElectionFinish_forTest(); stopCapturingLogMessages(); ASSERT_EQUALS( - 1, countLogLinesContaining("not running for primary, we have been superceded already")); + 1, countLogLinesContaining("not running for primary, we have been superseded already")); } TEST_F(ReplCoordTest, NodeWillNotStandForElectionDuringHeartbeatReconfig) { @@ -689,7 +689,7 @@ TEST_F(ReplCoordTest, ElectionFailsWhenVoteRequestResponseContainsANewerTerm) { getReplCoord()->waitForElectionFinish_forTest(); stopCapturingLogMessages(); ASSERT_EQUALS(1, - countLogLinesContaining("not becoming primary, we have been superceded already")); + countLogLinesContaining("not becoming primary, we have been superseded already")); } TEST_F(ReplCoordTest, ElectionFailsWhenTermChangesDuringDryRun) { @@ -728,8 +728,9 @@ TEST_F(ReplCoordTest, ElectionFailsWhenTermChangesDuringDryRun) { simulateSuccessfulDryRun(onDryRunRequest); stopCapturingLogMessages(); - ASSERT_EQUALS( - 1, countLogLinesContaining("not running for primary, we have been superceded already")); + ASSERT_EQUALS(1, + countLogLinesContaining( + "not running for primary, we have been superseded already during dry run")); } TEST_F(ReplCoordTest, ElectionFailsWhenTermChangesDuringActualElection) { @@ -784,7 +785,7 @@ TEST_F(ReplCoordTest, ElectionFailsWhenTermChangesDuringActualElection) { getReplCoord()->waitForElectionFinish_forTest(); stopCapturingLogMessages(); ASSERT_EQUALS(1, - countLogLinesContaining("not becoming primary, we have been superceded already")); + countLogLinesContaining("not becoming primary, we have been superseded already")); } class PriorityTakeoverTest : public ReplCoordTest { diff --git a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp index b72d7e7dbcc..66c9799029d 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp @@ -778,18 +778,24 @@ void ReplicationCoordinatorImpl::_scheduleNextLivenessUpdate_inlock() { } auto nextTimeout = earliestDate + _rsConfig.getElectionTimeoutPeriod(); - if (nextTimeout > _replExecutor.now()) { - LOG(3) << "scheduling next check at " << nextTimeout; - auto cbh = _scheduleWorkAt(nextTimeout, - stdx::bind(&ReplicationCoordinatorImpl::_handleLivenessTimeout, - this, - stdx::placeholders::_1)); - if (!cbh) { - return; - } - _handleLivenessTimeoutCbh = cbh; - _earliestMemberId = earliestMemberId; + LOG(3) << "scheduling next check at " << nextTimeout; + + // It is possible we will schedule the next timeout in the past. + // ReplicationExecutor::_scheduleWorkAt() schedules its work immediately if it's given a + // time <= now(). + // If we missed the timeout, it means that on our last check the earliest live member was + // just barely fresh and it has become stale since then. We must schedule another liveness + // check to continue conducting liveness checks and be able to step down from primary if we + // lose contact with a majority of nodes. + auto cbh = _scheduleWorkAt(nextTimeout, + stdx::bind(&ReplicationCoordinatorImpl::_handleLivenessTimeout, + this, + stdx::placeholders::_1)); + if (!cbh) { + return; } + _handleLivenessTimeoutCbh = cbh; + _earliestMemberId = earliestMemberId; } void ReplicationCoordinatorImpl::_cancelAndRescheduleLivenessUpdate_inlock(int updatedMemberId) { diff --git a/src/mongo/db/repl/replication_executor_test.cpp b/src/mongo/db/repl/replication_executor_test.cpp index ba41df46769..2e8a66465de 100644 --- a/src/mongo/db/repl/replication_executor_test.cpp +++ b/src/mongo/db/repl/replication_executor_test.cpp @@ -249,6 +249,21 @@ TEST_F(ReplicationExecutorTest, ScheduleCallbackAtNow) { executor.waitForEvent(finishEvent); } +TEST_F(ReplicationExecutorTest, ScheduleCallbackInPast) { + launchExecutorThread(); + getNet()->exitNetwork(); + + ReplicationExecutor& executor = getReplExecutor(); + auto finishEvent = assertGet(executor.makeEvent()); + auto fn = [&executor, finishEvent](const ReplicationExecutor::CallbackArgs& cbData) { + ASSERT_OK(cbData.status); + executor.signalEvent(finishEvent); + }; + + auto cb = executor.scheduleWorkAt(getNet()->now() - Milliseconds(1000), fn); + executor.waitForEvent(finishEvent); +} + TEST_F(ReplicationExecutorTest, ScheduleCallbackAtAFutureTime) { launchExecutorThread(); getNet()->exitNetwork(); diff --git a/src/mongo/db/repl/sync_source_feedback.cpp b/src/mongo/db/repl/sync_source_feedback.cpp index 4d395e376e6..a82b1553212 100644 --- a/src/mongo/db/repl/sync_source_feedback.cpp +++ b/src/mongo/db/repl/sync_source_feedback.cpp @@ -50,27 +50,23 @@ namespace repl { namespace { /** - * Calculates the keep alive interval based on the current configuration in the replication - * coordinator. + * Calculates the keep alive interval based on the given ReplSetConfig. */ -Milliseconds calculateKeepAliveInterval(OperationContext* txn, stdx::mutex& mtx) { - stdx::lock_guard lock(mtx); - auto replCoord = repl::ReplicationCoordinator::get(txn); - auto rsConfig = replCoord->getConfig(); - auto keepAliveInterval = rsConfig.getElectionTimeoutPeriod() / 2; - return keepAliveInterval; +Milliseconds calculateKeepAliveInterval(const ReplSetConfig& rsConfig) { + return rsConfig.getElectionTimeoutPeriod() / 2; } /** * Returns function to prepare update command */ Reporter::PrepareReplSetUpdatePositionCommandFn makePrepareReplSetUpdatePositionCommandFn( - OperationContext* txn, + ReplicationCoordinator* replCoord, stdx::mutex& mtx, const HostAndPort& syncTarget, BackgroundSync* bgsync) { - return [&mtx, syncTarget, txn, bgsync](ReplicationCoordinator::ReplSetUpdatePositionCommandStyle - commandStyle) -> StatusWith { + return [&mtx, syncTarget, replCoord, bgsync]( + ReplicationCoordinator::ReplSetUpdatePositionCommandStyle + commandStyle) -> StatusWith { auto currentSyncTarget = bgsync->getSyncTarget(); if (currentSyncTarget != syncTarget) { if (currentSyncTarget.empty()) { @@ -86,7 +82,6 @@ Reporter::PrepareReplSetUpdatePositionCommandFn makePrepareReplSetUpdatePosition } } - auto replCoord = repl::ReplicationCoordinator::get(txn); if (replCoord->getMemberState().primary()) { // Primary has no one to send updates to. return Status(ErrorCodes::InvalidSyncSource, @@ -114,7 +109,7 @@ void SyncSourceFeedback::forwardSlaveProgress() { } } -Status SyncSourceFeedback::_updateUpstream(OperationContext* txn, +Status SyncSourceFeedback::_updateUpstream(ReplicationCoordinator* replCoord, BackgroundSync* bgsync, Reporter* reporter) { auto syncTarget = reporter->getTarget(); @@ -139,7 +134,6 @@ Status SyncSourceFeedback::_updateUpstream(OperationContext* txn, } else { // Blacklist sync target for .5 seconds and find a new one. stdx::lock_guard lock(_mtx); - auto replCoord = repl::ReplicationCoordinator::get(txn); const auto blacklistDuration = Milliseconds{500}; const auto until = Date_t::now() + blacklistDuration; log() << "Blacklisting " << syncTarget << " due to error: '" << status << "' for " @@ -161,7 +155,9 @@ void SyncSourceFeedback::shutdown() { _cond.notify_all(); } -void SyncSourceFeedback::run(executor::TaskExecutor* executor, BackgroundSync* bgsync) { +void SyncSourceFeedback::run(executor::TaskExecutor* executor, + BackgroundSync* bgsync, + ReplicationCoordinator* replCoord) { Client::initThread("SyncSourceFeedback"); HostAndPort syncTarget; @@ -170,10 +166,9 @@ void SyncSourceFeedback::run(executor::TaskExecutor* executor, BackgroundSync* b Milliseconds keepAliveInterval(0); while (true) { // breaks once _shutdownSignaled is true - auto txn = cc().makeOperationContext(); if (keepAliveInterval == Milliseconds(0)) { - keepAliveInterval = calculateKeepAliveInterval(txn.get(), _mtx); + keepAliveInterval = calculateKeepAliveInterval(replCoord->getConfig()); } { @@ -189,7 +184,7 @@ void SyncSourceFeedback::run(executor::TaskExecutor* executor, BackgroundSync* b continue; } } - MemberState state = ReplicationCoordinator::get(txn.get())->getMemberState(); + MemberState state = replCoord->getMemberState(); if (!(state.primary() || state.startup())) { break; } @@ -204,7 +199,7 @@ void SyncSourceFeedback::run(executor::TaskExecutor* executor, BackgroundSync* b { stdx::lock_guard lock(_mtx); - MemberState state = ReplicationCoordinator::get(txn.get())->getMemberState(); + MemberState state = replCoord->getMemberState(); if (state.primary() || state.startup()) { continue; } @@ -226,7 +221,7 @@ void SyncSourceFeedback::run(executor::TaskExecutor* executor, BackgroundSync* b // Update keepalive value from config. auto oldKeepAliveInterval = keepAliveInterval; - keepAliveInterval = calculateKeepAliveInterval(txn.get(), _mtx); + keepAliveInterval = calculateKeepAliveInterval(replCoord->getConfig()); if (oldKeepAliveInterval != keepAliveInterval) { LOG(1) << "new syncSourceFeedback keep alive duration = " << keepAliveInterval << " (previously " << oldKeepAliveInterval << ")"; @@ -235,7 +230,7 @@ void SyncSourceFeedback::run(executor::TaskExecutor* executor, BackgroundSync* b Reporter reporter( executor, - makePrepareReplSetUpdatePositionCommandFn(txn.get(), _mtx, syncTarget, bgsync), + makePrepareReplSetUpdatePositionCommandFn(replCoord, _mtx, syncTarget, bgsync), syncTarget, keepAliveInterval); { @@ -250,7 +245,7 @@ void SyncSourceFeedback::run(executor::TaskExecutor* executor, BackgroundSync* b _reporter = nullptr; }); - auto status = _updateUpstream(txn.get(), bgsync, &reporter); + auto status = _updateUpstream(replCoord, bgsync, &reporter); if (!status.isOK()) { LOG(1) << "The replication progress command (replSetUpdatePosition) failed and will be " "retried: " diff --git a/src/mongo/db/repl/sync_source_feedback.h b/src/mongo/db/repl/sync_source_feedback.h index 40c29dc172a..35ed470fc26 100644 --- a/src/mongo/db/repl/sync_source_feedback.h +++ b/src/mongo/db/repl/sync_source_feedback.h @@ -31,6 +31,7 @@ #include "mongo/base/disallow_copying.h" #include "mongo/base/status.h" +#include "mongo/db/repl/replication_coordinator.h" #include "mongo/stdx/condition_variable.h" #include "mongo/stdx/mutex.h" @@ -63,7 +64,9 @@ public: * * Task executor is used to run replSetUpdatePosition command on sync source. */ - void run(executor::TaskExecutor* executor, BackgroundSync* bgsync); + void run(executor::TaskExecutor* executor, + BackgroundSync* bgsync, + ReplicationCoordinator* replCoord); /// Signals the run() method to terminate. void shutdown(); @@ -72,7 +75,9 @@ private: /* Inform the sync target of our current position in the oplog, as well as the positions * of all secondaries chained through us. */ - Status _updateUpstream(OperationContext* txn, BackgroundSync* bgsync, Reporter* reporter); + Status _updateUpstream(ReplicationCoordinator* replCoord, + BackgroundSync* bgsync, + Reporter* reporter); // protects cond, _shutdownSignaled, _keepAliveInterval, and _positionChanged. stdx::mutex _mtx; diff --git a/src/mongo/db/repl/sync_source_resolver.cpp b/src/mongo/db/repl/sync_source_resolver.cpp index 948a471b245..17a706f1b61 100644 --- a/src/mongo/db/repl/sync_source_resolver.cpp +++ b/src/mongo/db/repl/sync_source_resolver.cpp @@ -54,6 +54,7 @@ const Seconds SyncSourceResolver::kFirstOplogEntryEmptyBlacklistDuration(10); const Seconds SyncSourceResolver::kFirstOplogEntryNullTimestampBlacklistDuration(10); const Minutes SyncSourceResolver::kTooStaleBlacklistDuration(1); const Seconds SyncSourceResolver::kNoRequiredOpTimeBlacklistDuration(60); +const int SyncSourceResolver::kUninitializedRollbackId(-1); SyncSourceResolver::SyncSourceResolver(executor::TaskExecutor* taskExecutor, SyncSourceSelector* syncSourceSelector, @@ -175,11 +176,13 @@ std::unique_ptr SyncSourceResolver::_makeFirstOplogEntryFetcher( candidate, earliestOpTimeSeen), rpc::ServerSelectionMetadata(true, boost::none).toBSON(), - kFetcherTimeout); + kFetcherTimeout /* find network timeout */, + kFetcherTimeout /* getMore network timeout */); } std::unique_ptr SyncSourceResolver::_makeRequiredOpTimeFetcher(HostAndPort candidate, - OpTime earliestOpTimeSeen) { + OpTime earliestOpTimeSeen, + int rbid) { // This query is structured so that it is executed on the sync source using the oplog // start hack (oplogReplay=true and $gt/$gte predicate over "ts"). return stdx::make_unique( @@ -193,9 +196,11 @@ std::unique_ptr SyncSourceResolver::_makeRequiredOpTimeFetcher(HostAndP this, stdx::placeholders::_1, candidate, - earliestOpTimeSeen), + earliestOpTimeSeen, + rbid), rpc::ServerSelectionMetadata(true, boost::none).toBSON(), - kFetcherTimeout); + kFetcherTimeout /* find network timeout */, + kFetcherTimeout /* getMore network timeout */); } Status SyncSourceResolver::_scheduleFetcher(std::unique_ptr fetcher) { @@ -205,6 +210,9 @@ Status SyncSourceResolver::_scheduleFetcher(std::unique_ptr fetcher) { // executor. auto status = fetcher->schedule(); if (status.isOK()) { + // Fetcher destruction blocks on all outstanding callbacks. If we are currently in a + // Fetcher-related callback, we can't destroy the Fetcher just yet, so we assign it to a + // temporary unique pointer to allow the destruction to run to completion. _shuttingDownFetcher = std::move(_fetcher); _fetcher = std::move(fetcher); } else { @@ -311,10 +319,26 @@ void SyncSourceResolver::_firstOplogEntryFetcherCallback( return; } - _scheduleRBIDRequest(candidate, earliestOpTimeSeen); + auto status = _scheduleRBIDRequest(candidate, earliestOpTimeSeen); + if (!status.isOK()) { + _finishCallback(status); + } } -void SyncSourceResolver::_scheduleRBIDRequest(HostAndPort candidate, OpTime earliestOpTimeSeen) { +Status SyncSourceResolver::_scheduleRBIDRequest(HostAndPort candidate, OpTime earliestOpTimeSeen) { + // Once a work is scheduled, nothing prevents it finishing. We need the mutex to protect the + // access of member variables after scheduling, because otherwise the scheduled callback could + // finish and allow the destructor to fire before we access the member variables. + stdx::lock_guard lk(_mutex); + if (_state == State::kShuttingDown) { + return Status( + ErrorCodes::CallbackCanceled, + str::stream() + << "sync source resolver shut down while checking rollbackId on candidate: " + << candidate); + } + + invariant(_state == State::kRunning); auto handle = _taskExecutor->scheduleRemoteCommand( {candidate, "admin", BSON("replSetGetRBID" << 1), nullptr, kFetcherTimeout}, stdx::bind(&SyncSourceResolver::_rbidRequestCallback, @@ -324,15 +348,11 @@ void SyncSourceResolver::_scheduleRBIDRequest(HostAndPort candidate, OpTime earl stdx::placeholders::_1)); if (!handle.isOK()) { - _finishCallback(handle.getStatus()); - return; + return handle.getStatus(); } - stdx::lock_guard lk(_mutex); _rbidCommandHandle = std::move(handle.getValue()); - if (_state == State::kShuttingDown) { - _taskExecutor->cancel(_rbidCommandHandle); - } + return Status::OK(); } void SyncSourceResolver::_rbidRequestCallback( @@ -344,10 +364,11 @@ void SyncSourceResolver::_rbidRequestCallback( return; } + int rbid = kUninitializedRollbackId; try { uassertStatusOK(rbidReply.response.status); uassertStatusOK(getStatusFromCommandResult(rbidReply.response.data)); - _rbid = rbidReply.response.data["rbid"].Int(); + rbid = rbidReply.response.data["rbid"].Int(); } catch (const DBException& ex) { const auto until = _taskExecutor->now() + kFetcherErrorBlacklistDuration; log() << "Blacklisting " << candidate << " due to error: '" << ex << "' for " @@ -360,13 +381,15 @@ void SyncSourceResolver::_rbidRequestCallback( if (!_requiredOpTime.isNull()) { // Schedule fetcher to look for '_requiredOpTime' in the remote oplog. // Unittest requires that this kind of failure be handled specially. - auto status = _scheduleFetcher(_makeRequiredOpTimeFetcher(candidate, earliestOpTimeSeen)); + auto status = + _scheduleFetcher(_makeRequiredOpTimeFetcher(candidate, earliestOpTimeSeen, rbid)); if (!status.isOK()) { _finishCallback(status); } return; } - _finishCallback(candidate); + + _finishCallback(candidate, rbid); } Status SyncSourceResolver::_compareRequiredOpTimeWithQueryResponse( @@ -399,7 +422,8 @@ Status SyncSourceResolver::_compareRequiredOpTimeWithQueryResponse( void SyncSourceResolver::_requiredOpTimeFetcherCallback( const StatusWith& queryResult, HostAndPort candidate, - OpTime earliestOpTimeSeen) { + OpTime earliestOpTimeSeen, + int rbid) { if (_isShuttingDown()) { _finishCallback(Status(ErrorCodes::CallbackCanceled, str::stream() << "sync source resolver shut down while looking for " @@ -444,18 +468,18 @@ void SyncSourceResolver::_requiredOpTimeFetcherCallback( return; } - _finishCallback(candidate); + _finishCallback(candidate, rbid); } Status SyncSourceResolver::_chooseAndProbeNextSyncSource(OpTime earliestOpTimeSeen) { auto candidateResult = _chooseNewSyncSource(); if (!candidateResult.isOK()) { - return _finishCallback(candidateResult); + return _finishCallback(candidateResult.getStatus()); } if (candidateResult.getValue().empty()) { if (earliestOpTimeSeen.isNull()) { - return _finishCallback(candidateResult); + return _finishCallback(candidateResult.getValue(), kUninitializedRollbackId); } SyncSourceResolverResponse response; @@ -473,16 +497,22 @@ Status SyncSourceResolver::_chooseAndProbeNextSyncSource(OpTime earliestOpTimeSe return Status::OK(); } -Status SyncSourceResolver::_finishCallback(StatusWith result) { +Status SyncSourceResolver::_finishCallback(HostAndPort hostAndPort, int rbid) { SyncSourceResolverResponse response; - response.syncSourceStatus = std::move(result); - if (response.isOK() && !response.getSyncSource().empty()) { - invariant(_requiredOpTime.isNull() || _rbid); - response.rbid = _rbid; + response.syncSourceStatus = std::move(hostAndPort); + if (rbid != kUninitializedRollbackId) { + response.rbid = rbid; } return _finishCallback(response); } +Status SyncSourceResolver::_finishCallback(Status status) { + invariant(!status.isOK()); + SyncSourceResolverResponse response; + response.syncSourceStatus = std::move(status); + return _finishCallback(response); +} + Status SyncSourceResolver::_finishCallback(const SyncSourceResolverResponse& response) { try { _onCompletion(response); diff --git a/src/mongo/db/repl/sync_source_resolver.h b/src/mongo/db/repl/sync_source_resolver.h index 201658caacf..0f9343cfcdb 100644 --- a/src/mongo/db/repl/sync_source_resolver.h +++ b/src/mongo/db/repl/sync_source_resolver.h @@ -102,6 +102,7 @@ public: static const Seconds kFirstOplogEntryNullTimestampBlacklistDuration; static const Minutes kTooStaleBlacklistDuration; static const Seconds kNoRequiredOpTimeBlacklistDuration; + static const int kUninitializedRollbackId; /** * Callback function to report final status of resolving sync source. @@ -154,7 +155,8 @@ private: * Creates fetcher to check the remote oplog for '_requiredOpTime'. */ std::unique_ptr _makeRequiredOpTimeFetcher(HostAndPort candidate, - OpTime earliestOpTimeSeen); + OpTime earliestOpTimeSeen, + int rbid); /** * Schedules fetcher to read oplog on sync source. @@ -179,7 +181,7 @@ private: /** * Schedules a replSetGetRBID command against the candidate to fetch its current rollback id. */ - void _scheduleRBIDRequest(HostAndPort candidate, OpTime earliestOpTimeSeen); + Status _scheduleRBIDRequest(HostAndPort candidate, OpTime earliestOpTimeSeen); void _rbidRequestCallback(HostAndPort candidate, OpTime earliestOpTimeSeen, const executor::TaskExecutor::RemoteCommandCallbackArgs& rbidReply); @@ -194,7 +196,8 @@ private: */ void _requiredOpTimeFetcherCallback(const StatusWith& queryResult, HostAndPort candidate, - OpTime earliestOpTimeSeen); + OpTime earliestOpTimeSeen, + int rbid); /** * Obtains new sync source candidate and schedules remote command to fetcher first oplog entry. @@ -207,7 +210,8 @@ private: * Invokes completion callback and transitions state to State::kComplete. * Returns result.getStatus(). */ - Status _finishCallback(StatusWith result); + Status _finishCallback(HostAndPort hostAndPort, int rbid); + Status _finishCallback(Status status); Status _finishCallback(const SyncSourceResolverResponse& response); // Executor used to send remote commands to sync source candidates. @@ -229,9 +233,6 @@ private: // resolver via this callback in a SyncSourceResolverResponse struct when the resolver finishes. const OnCompletionFn _onCompletion; - // The rbid we will return to our caller. - int _rbid; - // Protects members of this sync source resolver defined below. mutable stdx::mutex _mutex; mutable stdx::condition_variable _condition; diff --git a/src/mongo/db/repl/sync_tail.cpp b/src/mongo/db/repl/sync_tail.cpp index 30f1e28b619..3f65e3fc6b7 100644 --- a/src/mongo/db/repl/sync_tail.cpp +++ b/src/mongo/db/repl/sync_tail.cpp @@ -448,7 +448,6 @@ void applyOps(std::vector& writerVectors, const MultiApplier::ApplyOperationFn& func, std::vector* statusVector) { invariant(writerVectors.size() == statusVector->size()); - TimerHolder timer(&applyBatchStats); for (size_t i = 0; i < writerVectors.size(); i++) { if (!writerVectors[i].empty()) { writerPool->schedule([&func, &writerVectors, statusVector, i] { @@ -695,32 +694,53 @@ public: } private: + /** + * Calculates batch limit size (in bytes) using the maximum capped collection size of the oplog + * size. + * Batches are limited to 10% of the oplog. + */ + std::size_t _calculateBatchLimitBytes() { + auto opCtx = cc().makeOperationContext(); + auto storageInterface = StorageInterface::get(opCtx.get()); + auto oplogMaxSizeResult = + storageInterface->getOplogMaxSize(opCtx.get(), NamespaceString(rsOplogName)); + auto oplogMaxSize = fassertStatusOK(40301, oplogMaxSizeResult); + return std::min(oplogMaxSize / 10, std::size_t(replBatchLimitBytes)); + } + + /** + * If slaveDelay is enabled, this function calculates the most recent timestamp of any oplog + * entries that can be be returned in a batch. + */ + boost::optional _calculateSlaveDelayLatestTimestamp() { + auto service = cc().getServiceContext(); + auto replCoord = ReplicationCoordinator::get(service); + auto slaveDelay = replCoord->getSlaveDelaySecs(); + if (slaveDelay <= Seconds(0)) { + return {}; + } + auto fastClockSource = service->getFastClockSource(); + return fastClockSource->now() - slaveDelay; + } + void run() { Client::initThread("ReplBatcher"); - const ServiceContext::UniqueOperationContext txnPtr = cc().makeOperationContext(); - OperationContext& txn = *txnPtr; - const auto replCoord = ReplicationCoordinator::get(&txn); - const auto fastClockSource = txn.getServiceContext()->getFastClockSource(); - const auto oplogMaxSize = fassertStatusOK( - 40301, - StorageInterface::get(&txn)->getOplogMaxSize(&txn, NamespaceString(rsOplogName))); - // Batches are limited to 10% of the oplog. BatchLimits batchLimits; - batchLimits.bytes = std::min(oplogMaxSize / 10, size_t(replBatchLimitBytes)); + batchLimits.bytes = _calculateBatchLimitBytes(); while (true) { - const auto slaveDelay = replCoord->getSlaveDelaySecs(); - batchLimits.slaveDelayLatestTimestamp = (slaveDelay > Seconds(0)) - ? (fastClockSource->now() - slaveDelay) - : boost::optional(); + batchLimits.slaveDelayLatestTimestamp = _calculateSlaveDelayLatestTimestamp(); // Check this once per batch since users can change it at runtime. batchLimits.ops = replBatchLimitOperations.load(); OpQueue ops; // tryPopAndWaitForMore adds to ops and returns true when we need to end a batch early. - while (!_syncTail->tryPopAndWaitForMore(&txn, &ops, batchLimits)) { + { + auto opCtx = cc().makeOperationContext(); + while (!_syncTail->tryPopAndWaitForMore(opCtx.get(), &ops, batchLimits)) { + } } if (ops.empty() && !ops.mustShutdown()) { @@ -755,14 +775,15 @@ private: void SyncTail::oplogApplication(ReplicationCoordinator* replCoord) { OpQueueBatcher batcher(this); - const ServiceContext::UniqueOperationContext txnPtr = cc().makeOperationContext(); - OperationContext& txn = *txnPtr; std::unique_ptr finalizer{ getGlobalServiceContext()->getGlobalStorageEngine()->isDurable() ? new ApplyBatchFinalizerForJournal(replCoord) : new ApplyBatchFinalizer(replCoord)}; while (true) { // Exits on message from OpQueueBatcher. + const ServiceContext::UniqueOperationContext txnPtr = cc().makeOperationContext(); + OperationContext& txn = *txnPtr; + // For pausing replication in tests. while (MONGO_FAIL_POINT(rsSyncApplyStop)) { // Tests should not trigger clean shutdown while that failpoint is active. If we @@ -1301,6 +1322,9 @@ StatusWith multiApply(OperationContext* txn, std::vector statusVector(workerPool->getNumThreads(), Status::OK()); { + // Each node records cumulative batch application stats for itself using this timer. + TimerHolder timer(&applyBatchStats); + // We must wait for the all work we've dispatched to complete before leaving this block // because the spawned threads refer to objects on our stack, including writerVectors. std::vector writerVectors(workerPool->getNumThreads()); diff --git a/src/mongo/db/repl/topology_coordinator_impl.cpp b/src/mongo/db/repl/topology_coordinator_impl.cpp index 88337b6a500..f5302008206 100644 --- a/src/mongo/db/repl/topology_coordinator_impl.cpp +++ b/src/mongo/db/repl/topology_coordinator_impl.cpp @@ -36,6 +36,7 @@ #include "mongo/db/audit.h" #include "mongo/db/client.h" +#include "mongo/db/mongod_options.h" #include "mongo/db/operation_context.h" #include "mongo/db/repl/heartbeat_response_action.h" #include "mongo/db/repl/is_master_response.h" @@ -1411,7 +1412,7 @@ bool TopologyCoordinatorImpl::_aMajoritySeemsToBeUp() const { return vUp * 2 > _rsConfig.getTotalVotingMembers(); } -bool TopologyCoordinatorImpl::_canSeeHealthyPrimaryOfEqualOrGreaterPriority( +int TopologyCoordinatorImpl::_findHealthyPrimaryOfEqualOrGreaterPriority( const int candidateIndex) const { const double candidatePriority = _rsConfig.getMemberAt(candidateIndex).getPriority(); for (auto it = _hbdata.begin(); it != _hbdata.end(); ++it) { @@ -1421,11 +1422,11 @@ bool TopologyCoordinatorImpl::_canSeeHealthyPrimaryOfEqualOrGreaterPriority( const int itIndex = indexOfIterator(_hbdata, it); const double priority = _rsConfig.getMemberAt(itIndex).getPriority(); if (itIndex != candidateIndex && priority >= candidatePriority) { - return true; + return itIndex; } } - return false; + return -1; } bool TopologyCoordinatorImpl::_isOpTimeCloseEnoughToLatestToElect( @@ -2232,7 +2233,7 @@ MemberState TopologyCoordinatorImpl::getMemberState() const { } if (_rsConfig.isConfigServer()) { - if (_options.clusterRole != ClusterRole::ConfigServer) { + if (_options.clusterRole != ClusterRole::ConfigServer && !skipShardingConfigurationChecks) { return MemberState::RS_REMOVED; } else { invariant(_storageEngineSupportsReadCommitted != ReadCommittedSupport::kUnknown); @@ -2241,7 +2242,7 @@ MemberState TopologyCoordinatorImpl::getMemberState() const { } } } else { - if (_options.clusterRole == ClusterRole::ConfigServer) { + if (_options.clusterRole == ClusterRole::ConfigServer && !skipShardingConfigurationChecks) { return MemberState::RS_REMOVED; } } @@ -2571,29 +2572,54 @@ void TopologyCoordinatorImpl::processReplSetRequestVotes(const ReplSetRequestVot if (args.getTerm() < _term) { response->setVoteGranted(false); - response->setReason("candidate's term is lower than mine"); + response->setReason(str::stream() << "candidate's term (" << args.getTerm() + << ") is lower than mine (" + << _term + << ")"); } else if (args.getConfigVersion() != _rsConfig.getConfigVersion()) { response->setVoteGranted(false); - response->setReason("candidate's config version differs from mine"); + response->setReason(str::stream() << "candidate's config version (" + << args.getConfigVersion() + << ") differs from mine (" + << _rsConfig.getConfigVersion() + << ")"); } else if (args.getSetName() != _rsConfig.getReplSetName()) { response->setVoteGranted(false); - response->setReason("candidate's set name differs from mine"); + response->setReason(str::stream() << "candidate's set name (" << args.getSetName() + << ") differs from mine (" + << _rsConfig.getReplSetName() + << ")"); } else if (args.getLastDurableOpTime() < lastAppliedOpTime) { response->setVoteGranted(false); - response->setReason("candidate's data is staler than mine"); + response + ->setReason(str::stream() + << "candidate's data is staler than mine. candidate's last applied OpTime: " + << args.getLastDurableOpTime().toString() + << ", my last applied OpTime: " + << lastAppliedOpTime.toString()); } else if (!args.isADryRun() && _lastVote.getTerm() == args.getTerm()) { response->setVoteGranted(false); - response->setReason("already voted for another candidate this term"); - } else if (_selfConfig().isArbiter() && - _canSeeHealthyPrimaryOfEqualOrGreaterPriority(args.getCandidateIndex())) { - response->setVoteGranted(false); - response->setReason("can see a healthy primary of equal or greater priority"); + response->setReason(str::stream() + << "already voted for another candidate (" + << _rsConfig.getMemberAt(_lastVote.getCandidateIndex()).getHostAndPort() + << ") this term (" + << _lastVote.getTerm() + << ")"); } else { - if (!args.isADryRun()) { - _lastVote.setTerm(args.getTerm()); - _lastVote.setCandidateIndex(args.getCandidateIndex()); + int betterPrimary = _findHealthyPrimaryOfEqualOrGreaterPriority(args.getCandidateIndex()); + if (_selfConfig().isArbiter() && betterPrimary >= 0) { + response->setVoteGranted(false); + response->setReason(str::stream() + << "can see a healthy primary (" + << _rsConfig.getMemberAt(betterPrimary).getHostAndPort() + << ") of equal or greater priority"); + } else { + if (!args.isADryRun()) { + _lastVote.setTerm(args.getTerm()); + _lastVote.setCandidateIndex(args.getCandidateIndex()); + } + response->setVoteGranted(true); } - response->setVoteGranted(true); } } diff --git a/src/mongo/db/repl/topology_coordinator_impl.h b/src/mongo/db/repl/topology_coordinator_impl.h index d4b0476cde1..1a03338e3c4 100644 --- a/src/mongo/db/repl/topology_coordinator_impl.h +++ b/src/mongo/db/repl/topology_coordinator_impl.h @@ -310,9 +310,9 @@ private: // Sees if a majority number of votes are held by members who are currently "up" bool _aMajoritySeemsToBeUp() const; - // Returns true if the node can see a healthy primary of equal or greater priority to the - // candidate. - bool _canSeeHealthyPrimaryOfEqualOrGreaterPriority(const int candidateIndex) const; + // Checks if the node can see a healthy primary of equal or greater priority to the + // candidate. If so, returns the index of that node. Otherwise returns -1. + int _findHealthyPrimaryOfEqualOrGreaterPriority(const int candidateIndex) const; // Is otherOpTime close enough (within 10 seconds) to the latest known optime to qualify // for an election diff --git a/src/mongo/db/repl/topology_coordinator_impl_test.cpp b/src/mongo/db/repl/topology_coordinator_impl_test.cpp index e3708a9bf54..1f8d4b56c40 100644 --- a/src/mongo/db/repl/topology_coordinator_impl_test.cpp +++ b/src/mongo/db/repl/topology_coordinator_impl_test.cpp @@ -5912,7 +5912,8 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVotesToTwoDifferentNodesInTheSameTerm) { // different candidate same term, should be a problem getTopoCoord().processReplSetRequestVotes(args2, &response2, lastAppliedOpTime); - ASSERT_EQUALS("already voted for another candidate this term", response2.getReason()); + ASSERT_EQUALS("already voted for another candidate (hself:27017) this term (1)", + response2.getReason()); ASSERT_FALSE(response2.getVoteGranted()); } @@ -6028,7 +6029,8 @@ TEST_F(TopoCoordTest, VoteRequestShouldNotPreventDryRunsForThatTerm) { ReplSetRequestVotesResponse response2; getTopoCoord().processReplSetRequestVotes(args2, &response2, lastAppliedOpTime); - ASSERT_EQUALS("already voted for another candidate this term", response2.getReason()); + ASSERT_EQUALS("already voted for another candidate (hself:27017) this term (1)", + response2.getReason()); ASSERT_FALSE(response2.getVoteGranted()); } @@ -6063,7 +6065,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenReplSetNameDoesNotMatch) { OpTime lastAppliedOpTime; getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); - ASSERT_EQUALS("candidate's set name differs from mine", response.getReason()); + ASSERT_EQUALS("candidate's set name (wrongName) differs from mine (rs0)", response.getReason()); ASSERT_FALSE(response.getVoteGranted()); } @@ -6098,7 +6100,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenConfigVersionDoesNotMatch) { OpTime lastAppliedOpTime; getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); - ASSERT_EQUALS("candidate's config version differs from mine", response.getReason()); + ASSERT_EQUALS("candidate's config version (0) differs from mine (1)", response.getReason()); ASSERT_FALSE(response.getVoteGranted()); } @@ -6145,7 +6147,8 @@ TEST_F(TopoCoordTest, ArbiterDoesNotGrantVoteWhenItCanSeeAHealthyPrimaryOfEqualO OpTime lastAppliedOpTime; getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); - ASSERT_EQUALS("can see a healthy primary of equal or greater priority", response.getReason()); + ASSERT_EQUALS("can see a healthy primary (h2:27017) of equal or greater priority", + response.getReason()); ASSERT_FALSE(response.getVoteGranted()); } @@ -6184,7 +6187,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenTermIsStale) { OpTime lastAppliedOpTime; getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); - ASSERT_EQUALS("candidate's term is lower than mine", response.getReason()); + ASSERT_EQUALS("candidate's term (1) is lower than mine (2)", response.getReason()); ASSERT_EQUALS(2, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); } @@ -6221,7 +6224,12 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenOpTimeIsStale) { OpTime lastAppliedOpTime2 = {Timestamp(20, 0), 0}; getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime2); - ASSERT_EQUALS("candidate's data is staler than mine", response.getReason()); + ASSERT_EQUALS( + str::stream() << "candidate's data is staler than mine. candidate's last applied OpTime: " + << OpTime().toString() + << ", my last applied OpTime: " + << OpTime(Timestamp(20, 0), 0).toString(), + response.getReason()); ASSERT_FALSE(response.getVoteGranted()); } @@ -6281,7 +6289,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenReplSetNameDoesNotMatch) { ReplSetRequestVotesResponse response; getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); - ASSERT_EQUALS("candidate's set name differs from mine", response.getReason()); + ASSERT_EQUALS("candidate's set name (wrongName) differs from mine (rs0)", response.getReason()); ASSERT_EQUALS(1, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); } @@ -6342,7 +6350,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenConfigVersionDoesNotMatch) { ReplSetRequestVotesResponse response; getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); - ASSERT_EQUALS("candidate's config version differs from mine", response.getReason()); + ASSERT_EQUALS("candidate's config version (0) differs from mine (1)", response.getReason()); ASSERT_EQUALS(1, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); } @@ -6402,7 +6410,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenTermIsStale) { ReplSetRequestVotesResponse response; getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); - ASSERT_EQUALS("candidate's term is lower than mine", response.getReason()); + ASSERT_EQUALS("candidate's term (0) is lower than mine (1)", response.getReason()); ASSERT_EQUALS(1, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); } @@ -6525,7 +6533,12 @@ TEST_F(TopoCoordTest, DoNotGrantDryRunVoteWhenOpTimeIsStale) { OpTime lastAppliedOpTime2 = {Timestamp(20, 0), 0}; getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime2); - ASSERT_EQUALS("candidate's data is staler than mine", response.getReason()); + ASSERT_EQUALS( + str::stream() << "candidate's data is staler than mine. candidate's last applied OpTime: " + << OpTime().toString() + << ", my last applied OpTime: " + << OpTime(Timestamp(20, 0), 0).toString(), + response.getReason()); ASSERT_EQUALS(1, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); } diff --git a/src/mongo/db/repl/topology_coordinator_impl_v1_test.cpp b/src/mongo/db/repl/topology_coordinator_impl_v1_test.cpp index 8062208949b..8f60269cd71 100644 --- a/src/mongo/db/repl/topology_coordinator_impl_v1_test.cpp +++ b/src/mongo/db/repl/topology_coordinator_impl_v1_test.cpp @@ -2574,7 +2574,8 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVotesToTwoDifferentNodesInTheSameTerm) { // different candidate same term, should be a problem getTopoCoord().processReplSetRequestVotes(args2, &response2, lastAppliedOpTime); - ASSERT_EQUALS("already voted for another candidate this term", response2.getReason()); + ASSERT_EQUALS("already voted for another candidate (hself:27017) this term (1)", + response2.getReason()); ASSERT_FALSE(response2.getVoteGranted()); } @@ -2674,7 +2675,8 @@ TEST_F(TopoCoordTest, DryRunVoteRequestShouldNotPreventSubsequentDryRunsForThatT ReplSetRequestVotesResponse response4; getTopoCoord().processReplSetRequestVotes(args4, &response4, lastAppliedOpTime); - ASSERT_EQUALS("already voted for another candidate this term", response4.getReason()); + ASSERT_EQUALS("already voted for another candidate (hself:27017) this term (1)", + response4.getReason()); ASSERT_FALSE(response4.getVoteGranted()); } @@ -2732,7 +2734,8 @@ TEST_F(TopoCoordTest, VoteRequestShouldNotPreventDryRunsForThatTerm) { ReplSetRequestVotesResponse response2; getTopoCoord().processReplSetRequestVotes(args2, &response2, lastAppliedOpTime); - ASSERT_EQUALS("already voted for another candidate this term", response2.getReason()); + ASSERT_EQUALS("already voted for another candidate (hself:27017) this term (1)", + response2.getReason()); ASSERT_FALSE(response2.getVoteGranted()); } @@ -2767,7 +2770,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenReplSetNameDoesNotMatch) { OpTime lastAppliedOpTime; getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); - ASSERT_EQUALS("candidate's set name differs from mine", response.getReason()); + ASSERT_EQUALS("candidate's set name (wrongName) differs from mine (rs0)", response.getReason()); ASSERT_FALSE(response.getVoteGranted()); } @@ -2802,7 +2805,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenConfigVersionDoesNotMatch) { OpTime lastAppliedOpTime; getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); - ASSERT_EQUALS("candidate's config version differs from mine", response.getReason()); + ASSERT_EQUALS("candidate's config version (0) differs from mine (1)", response.getReason()); ASSERT_FALSE(response.getVoteGranted()); } @@ -2841,7 +2844,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenTermIsStale) { OpTime lastAppliedOpTime; getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); - ASSERT_EQUALS("candidate's term is lower than mine", response.getReason()); + ASSERT_EQUALS("candidate's term (1) is lower than mine (2)", response.getReason()); ASSERT_EQUALS(2, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); } @@ -2878,7 +2881,12 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenOpTimeIsStale) { OpTime lastAppliedOpTime2 = {Timestamp(20, 0), 0}; getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime2); - ASSERT_EQUALS("candidate's data is staler than mine", response.getReason()); + ASSERT_EQUALS( + str::stream() << "candidate's data is staler than mine. candidate's last applied OpTime: " + << OpTime().toString() + << ", my last applied OpTime: " + << OpTime(Timestamp(20, 0), 0).toString(), + response.getReason()); ASSERT_FALSE(response.getVoteGranted()); } @@ -2938,7 +2946,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenReplSetNameDoesNotMatch) { ReplSetRequestVotesResponse response; getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); - ASSERT_EQUALS("candidate's set name differs from mine", response.getReason()); + ASSERT_EQUALS("candidate's set name (wrongName) differs from mine (rs0)", response.getReason()); ASSERT_EQUALS(1, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); } @@ -2999,7 +3007,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenConfigVersionDoesNotMatch) { ReplSetRequestVotesResponse response; getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); - ASSERT_EQUALS("candidate's config version differs from mine", response.getReason()); + ASSERT_EQUALS("candidate's config version (0) differs from mine (1)", response.getReason()); ASSERT_EQUALS(1, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); } @@ -3059,7 +3067,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenTermIsStale) { ReplSetRequestVotesResponse response; getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); - ASSERT_EQUALS("candidate's term is lower than mine", response.getReason()); + ASSERT_EQUALS("candidate's term (0) is lower than mine (1)", response.getReason()); ASSERT_EQUALS(1, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); } @@ -3182,7 +3190,12 @@ TEST_F(TopoCoordTest, DoNotGrantDryRunVoteWhenOpTimeIsStale) { OpTime lastAppliedOpTime2 = {Timestamp(20, 0), 0}; getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime2); - ASSERT_EQUALS("candidate's data is staler than mine", response.getReason()); + ASSERT_EQUALS( + str::stream() << "candidate's data is staler than mine. candidate's last applied OpTime: " + << OpTime().toString() + << ", my last applied OpTime: " + << OpTime(Timestamp(20, 0), 0).toString(), + response.getReason()); ASSERT_EQUALS(1, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); } diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript index b2abb86c056..b4c8b12a3d2 100644 --- a/src/mongo/db/s/SConscript +++ b/src/mongo/db/s/SConscript @@ -47,7 +47,6 @@ env.Library( source=[ 'active_migrations_registry.cpp', 'chunk_move_write_concern_options.cpp', - 'collection_range_deleter.cpp', 'collection_sharding_state.cpp', 'metadata_manager.cpp', 'migration_chunk_cloner_source.cpp', diff --git a/src/mongo/db/s/balancer/balancer.cpp b/src/mongo/db/s/balancer/balancer.cpp index 36987fb81ec..7e334b8b86e 100644 --- a/src/mongo/db/s/balancer/balancer.cpp +++ b/src/mongo/db/s/balancer/balancer.cpp @@ -243,8 +243,9 @@ void Balancer::waitForBalancerToStop() { void Balancer::joinCurrentRound(OperationContext* opCtx) { stdx::unique_lock scopedLock(_mutex); const auto numRoundsAtStart = _numBalancerRounds; - _condVar.wait(scopedLock, - [&] { return !_inBalancerRound || _numBalancerRounds != numRoundsAtStart; }); + opCtx->waitForConditionOrInterrupt(_condVar, scopedLock, [&] { + return !_inBalancerRound || _numBalancerRounds != numRoundsAtStart; + }); } Status Balancer::rebalanceSingleChunk(OperationContext* opCtx, const ChunkType& chunk) { diff --git a/src/mongo/db/s/balancer/balancer.h b/src/mongo/db/s/balancer/balancer.h index d8847a4ee90..7f601a849e2 100644 --- a/src/mongo/db/s/balancer/balancer.h +++ b/src/mongo/db/s/balancer/balancer.h @@ -110,7 +110,8 @@ public: /** * Potentially blocking method, which will return immediately if the balancer is not running a - * balancer round and will block until the current round completes otherwise. + * balancer round and will block until the current round completes otherwise. If the operation + * context's deadline is exceeded, it will throw an ExceededTimeLimit exception. */ void joinCurrentRound(OperationContext* txn); diff --git a/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.cpp b/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.cpp index 5d96be7004b..2f7c6feb7ec 100644 --- a/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.cpp +++ b/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.cpp @@ -32,7 +32,6 @@ #include "mongo/db/s/balancer/balancer_chunk_selection_policy_impl.h" -#include #include #include "mongo/base/status_with.h" @@ -59,7 +58,7 @@ namespace { /** * Does a linear pass over the information cached in the specified chunk manager and extracts chunk - * distrubution and chunk placement information which is needed by the balancer policy. + * distribution and chunk placement information which is needed by the balancer policy. */ StatusWith createCollectionDistributionStatus( OperationContext* opCtx, const ShardStatisticsVector& allShards, ChunkManager* chunkMgr) { @@ -258,6 +257,7 @@ StatusWith BalancerChunkSelectionPolicyImpl::selectChunksToMo } MigrateInfoVector candidateChunks; + std::set usedShards; for (const auto& coll : collections) { if (coll.getDropped()) { @@ -271,8 +271,8 @@ StatusWith BalancerChunkSelectionPolicyImpl::selectChunksToMo continue; } - auto candidatesStatus = - _getMigrateCandidatesForCollection(opCtx, nss, shardStats, aggressiveBalanceHint); + auto candidatesStatus = _getMigrateCandidatesForCollection( + opCtx, nss, shardStats, aggressiveBalanceHint, &usedShards); if (candidatesStatus == ErrorCodes::NamespaceNotFound) { // Namespace got dropped before we managed to get to it, so just skip it continue; @@ -416,7 +416,8 @@ StatusWith BalancerChunkSelectionPolicyImpl::_getMigrateCandi OperationContext* opCtx, const NamespaceString& nss, const ShardStatisticsVector& shardStats, - bool aggressiveBalanceHint) { + bool aggressiveBalanceHint, + std::set* usedShards) { auto routingInfoStatus = Grid::get(opCtx)->catalogCache()->getShardedCollectionRoutingInfoWithRefresh(opCtx, nss); if (!routingInfoStatus.isOK()) { @@ -475,7 +476,7 @@ StatusWith BalancerChunkSelectionPolicyImpl::_getMigrateCandi } } - return BalancerPolicy::balance(shardStats, distribution, aggressiveBalanceHint); + return BalancerPolicy::balance(shardStats, distribution, aggressiveBalanceHint, usedShards); } } // namespace mongo diff --git a/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.h b/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.h index f010d8c723b..5a1277cd5ea 100644 --- a/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.h +++ b/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.h @@ -67,7 +67,8 @@ private: OperationContext* txn, const NamespaceString& nss, const ShardStatisticsVector& shardStats, - bool aggressiveBalanceHint); + bool aggressiveBalanceHint, + std::set* usedShards); // Source for obtaining cluster statistics. Not owned and must not be destroyed before the // policy object is destroyed. diff --git a/src/mongo/db/s/balancer/balancer_policy.cpp b/src/mongo/db/s/balancer/balancer_policy.cpp index 348d8cec603..f5a0473d93e 100644 --- a/src/mongo/db/s/balancer/balancer_policy.cpp +++ b/src/mongo/db/s/balancer/balancer_policy.cpp @@ -1,30 +1,30 @@ /** -* Copyright (C) 2010 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -* -* As a special exception, the copyright holders give permission to link the -* code of portions of this program with the OpenSSL library under certain -* conditions as described in each individual source file and distribute -* linked combinations including the program with the OpenSSL library. You -* must comply with the GNU Affero General Public License in all respects -* for all of the code used other than as permitted herein. If you modify -* file(s) with this exception, you may extend this exception to your -* version of the file(s), but you are not obligated to do so. If you do not -* wish to do so, delete this exception statement from your version. If you -* delete this exception statement from all source files in the program, -* then also delete it in the license file. -*/ + * Copyright (C) 2018 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ #define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kSharding @@ -32,7 +32,6 @@ #include "mongo/db/s/balancer/balancer_policy.h" -#include "mongo/bson/simple_bsonobj_comparator.h" #include "mongo/s/catalog/type_shard.h" #include "mongo/s/catalog/type_tags.h" #include "mongo/util/log.h" @@ -291,20 +290,17 @@ ShardId BalancerPolicy::_getMostOverloadedShard(const ShardStatisticsVector& sha vector BalancerPolicy::balance(const ShardStatisticsVector& shardStats, const DistributionStatus& distribution, - bool shouldAggressivelyBalance) { + bool shouldAggressivelyBalance, + std::set* usedShards) { vector migrations; - // Set of shards, which have already been used for migrations. Used so we don't return multiple - // migrations for the same shard. - set usedShards; - // 1) Check for shards, which are in draining mode { for (const auto& stat : shardStats) { if (!stat.isDraining) continue; - if (usedShards.count(stat.shardId)) + if (usedShards->count(stat.shardId)) continue; const vector& chunks = distribution.getChunks(stat.shardId); @@ -326,7 +322,7 @@ vector BalancerPolicy::balance(const ShardStatisticsVector& shardSt const string tag = distribution.getTagForChunk(chunk); const ShardId to = - _getLeastLoadedReceiverShard(shardStats, distribution, tag, usedShards); + _getLeastLoadedReceiverShard(shardStats, distribution, tag, *usedShards); if (!to.isValid()) { if (migrations.empty()) { warning() << "Chunk " << redact(chunk.toString()) @@ -337,8 +333,8 @@ vector BalancerPolicy::balance(const ShardStatisticsVector& shardSt invariant(to != stat.shardId); migrations.emplace_back(to, chunk); - invariant(usedShards.insert(stat.shardId).second); - invariant(usedShards.insert(to).second); + invariant(usedShards->insert(stat.shardId).second); + invariant(usedShards->insert(to).second); break; } @@ -352,7 +348,7 @@ vector BalancerPolicy::balance(const ShardStatisticsVector& shardSt // 2) Check for chunks, which are on the wrong shard and must be moved off of it if (!distribution.tags().empty()) { for (const auto& stat : shardStats) { - if (usedShards.count(stat.shardId)) + if (usedShards->count(stat.shardId)) continue; const vector& chunks = distribution.getChunks(stat.shardId); @@ -373,7 +369,7 @@ vector BalancerPolicy::balance(const ShardStatisticsVector& shardSt } const ShardId to = - _getLeastLoadedReceiverShard(shardStats, distribution, tag, usedShards); + _getLeastLoadedReceiverShard(shardStats, distribution, tag, *usedShards); if (!to.isValid()) { if (migrations.empty()) { warning() << "Chunk " << redact(chunk.toString()) << " violates zone " @@ -384,8 +380,8 @@ vector BalancerPolicy::balance(const ShardStatisticsVector& shardSt invariant(to != stat.shardId); migrations.emplace_back(to, chunk); - invariant(usedShards.insert(stat.shardId).second); - invariant(usedShards.insert(to).second); + invariant(usedShards->insert(stat.shardId).second); + invariant(usedShards->insert(to).second); break; } } @@ -434,7 +430,7 @@ vector BalancerPolicy::balance(const ShardStatisticsVector& shardSt idealNumberOfChunksPerShardForTag, imbalanceThreshold, &migrations, - &usedShards)) + usedShards)) ; } diff --git a/src/mongo/db/s/balancer/balancer_policy.h b/src/mongo/db/s/balancer/balancer_policy.h index 63f44c3ca68..afba1db4915 100644 --- a/src/mongo/db/s/balancer/balancer_policy.h +++ b/src/mongo/db/s/balancer/balancer_policy.h @@ -1,39 +1,42 @@ /** -* Copyright (C) 2010 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -* -* As a special exception, the copyright holders give permission to link the -* code of portions of this program with the OpenSSL library under certain -* conditions as described in each individual source file and distribute -* linked combinations including the program with the OpenSSL library. You -* must comply with the GNU Affero General Public License in all respects -* for all of the code used other than as permitted herein. If you modify -* file(s) with this exception, you may extend this exception to your -* version of the file(s), but you are not obligated to do so. If you do not -* wish to do so, delete this exception statement from your version. If you -* delete this exception statement from all source files in the program, -* then also delete it in the license file. -*/ + * Copyright (C) 2018 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ #pragma once +#include +#include + #include "mongo/base/disallow_copying.h" #include "mongo/bson/bsonobj.h" #include "mongo/bson/simple_bsonobj_comparator.h" #include "mongo/db/s/balancer/cluster_statistics.h" #include "mongo/s/catalog/type_chunk.h" -#include "mongo/s/client/shard.h" +#include "mongo/s/shard_id.h" namespace mongo { @@ -179,10 +182,15 @@ public: * * The shouldAggressivelyBalance parameter causes the threshold for chunk could disparity * between shards to be lowered. + * + * The usedShards parameter is in/out and it contains the set of shards, which have already been + * used for migrations. Used so we don't return multiple conflicting migrations for the same + * shard. */ static std::vector balance(const ShardStatisticsVector& shardStats, const DistributionStatus& distribution, - bool shouldAggressivelyBalance); + bool shouldAggressivelyBalance, + std::set* usedShards); /** * Using the specified distribution information, returns a suggested better location for the diff --git a/src/mongo/db/s/balancer/balancer_policy_test.cpp b/src/mongo/db/s/balancer/balancer_policy_test.cpp index de5ceafaa83..f6abd7c8acf 100644 --- a/src/mongo/db/s/balancer/balancer_policy_test.cpp +++ b/src/mongo/db/s/balancer/balancer_policy_test.cpp @@ -110,14 +110,22 @@ std::pair generateCluster( return std::make_pair(std::move(shardStats), std::move(chunkMap)); } +std::vector balanceChunks(const ShardStatisticsVector& shardStats, + const DistributionStatus& distribution, + bool shouldAggressivelyBalance) { + std::set usedShards; + return BalancerPolicy::balance( + shardStats, distribution, shouldAggressivelyBalance, &usedShards); +} + TEST(BalancerPolicy, Basic) { auto cluster = generateCluster( {{ShardStatistics(kShardId0, kNoMaxSize, 4, false, emptyTagSet, emptyShardVersion), 4}, {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}, {ShardStatistics(kShardId2, kNoMaxSize, 3, false, emptyTagSet, emptyShardVersion), 3}}); - const auto migrations(BalancerPolicy::balance( - cluster.first, DistributionStatus(kNamespace, cluster.second), false)); + const auto migrations( + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false)); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId1, migrations[0].to); @@ -131,8 +139,8 @@ TEST(BalancerPolicy, SmallClusterShouldBePerfectlyBalanced) { {ShardStatistics(kShardId1, kNoMaxSize, 2, false, emptyTagSet, emptyShardVersion), 2}, {ShardStatistics(kShardId2, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}}); - const auto migrations(BalancerPolicy::balance( - cluster.first, DistributionStatus(kNamespace, cluster.second), false)); + const auto migrations( + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false)); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId1, migrations[0].from); ASSERT_EQ(kShardId2, migrations[0].to); @@ -146,10 +154,8 @@ TEST(BalancerPolicy, SingleChunkShouldNotMove) { {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}}); ASSERT( - BalancerPolicy::balance(cluster.first, DistributionStatus(kNamespace, cluster.second), true) - .empty()); - ASSERT(BalancerPolicy::balance( - cluster.first, DistributionStatus(kNamespace, cluster.second), false) + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), true).empty()); + ASSERT(balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false) .empty()); } @@ -161,10 +167,8 @@ TEST(BalancerPolicy, BalanceThresholdObeyed) { {ShardStatistics(kShardId3, kNoMaxSize, 1, false, emptyTagSet, emptyShardVersion), 1}}); ASSERT( - BalancerPolicy::balance(cluster.first, DistributionStatus(kNamespace, cluster.second), true) - .empty()); - ASSERT(BalancerPolicy::balance( - cluster.first, DistributionStatus(kNamespace, cluster.second), false) + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), true).empty()); + ASSERT(balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false) .empty()); } @@ -175,8 +179,8 @@ TEST(BalancerPolicy, ParallelBalancing) { {ShardStatistics(kShardId2, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}, {ShardStatistics(kShardId3, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}}); - const auto migrations(BalancerPolicy::balance( - cluster.first, DistributionStatus(kNamespace, cluster.second), false)); + const auto migrations( + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false)); ASSERT_EQ(2U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); @@ -199,8 +203,8 @@ TEST(BalancerPolicy, ParallelBalancingDoesNotPutChunksOnShardsAboveTheOptimal) { {ShardStatistics(kShardId4, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}, {ShardStatistics(kShardId5, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}}); - const auto migrations(BalancerPolicy::balance( - cluster.first, DistributionStatus(kNamespace, cluster.second), false)); + const auto migrations( + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false)); ASSERT_EQ(2U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); @@ -221,8 +225,60 @@ TEST(BalancerPolicy, ParallelBalancingDoesNotMoveChunksFromShardsBelowOptimal) { {ShardStatistics(kShardId2, kNoMaxSize, 5, false, emptyTagSet, emptyShardVersion), 5}, {ShardStatistics(kShardId3, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}}); + const auto migrations( + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false)); + ASSERT_EQ(1U, migrations.size()); + + ASSERT_EQ(kShardId0, migrations[0].from); + ASSERT_EQ(kShardId3, migrations[0].to); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); +} + +TEST(BalancerPolicy, ParallelBalancingNotSchedulingOnInUseSourceShardsWithMoveNecessary) { + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 8, false, emptyTagSet, emptyShardVersion), 8}, + {ShardStatistics(kShardId1, kNoMaxSize, 4, false, emptyTagSet, emptyShardVersion), 4}, + {ShardStatistics(kShardId2, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}, + {ShardStatistics(kShardId3, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}}); + + // Here kShardId0 would have been selected as a donor + std::set usedShards{kShardId0}; + const auto migrations(BalancerPolicy::balance( + cluster.first, DistributionStatus(kNamespace, cluster.second), false, &usedShards)); + ASSERT_EQ(1U, migrations.size()); + + ASSERT_EQ(kShardId1, migrations[0].from); + ASSERT_EQ(kShardId2, migrations[0].to); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMin(), migrations[0].minKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMax(), migrations[0].maxKey); +} + +TEST(BalancerPolicy, ParallelBalancingNotSchedulingOnInUseSourceShardsWithMoveNotNecessary) { + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 12, false, emptyTagSet, emptyShardVersion), 12}, + {ShardStatistics(kShardId1, kNoMaxSize, 4, false, emptyTagSet, emptyShardVersion), 4}, + {ShardStatistics(kShardId2, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}, + {ShardStatistics(kShardId3, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}}); + + // Here kShardId0 would have been selected as a donor + std::set usedShards{kShardId0}; + const auto migrations(BalancerPolicy::balance( + cluster.first, DistributionStatus(kNamespace, cluster.second), false, &usedShards)); + ASSERT_EQ(0U, migrations.size()); +} + +TEST(BalancerPolicy, ParallelBalancingNotSchedulingOnInUseDestinationShards) { + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 4, false, emptyTagSet, emptyShardVersion), 4}, + {ShardStatistics(kShardId1, kNoMaxSize, 4, false, emptyTagSet, emptyShardVersion), 4}, + {ShardStatistics(kShardId2, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}, + {ShardStatistics(kShardId3, kNoMaxSize, 1, false, emptyTagSet, emptyShardVersion), 1}}); + + // Here kShardId2 would have been selected as a recipient + std::set usedShards{kShardId2}; const auto migrations(BalancerPolicy::balance( - cluster.first, DistributionStatus(kNamespace, cluster.second), false)); + cluster.first, DistributionStatus(kNamespace, cluster.second), false, &usedShards)); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); @@ -241,8 +297,8 @@ TEST(BalancerPolicy, JumboChunksNotMoved) { cluster.second[kShardId0][2].setJumbo(true); cluster.second[kShardId0][3].setJumbo(true); - const auto migrations(BalancerPolicy::balance( - cluster.first, DistributionStatus(kNamespace, cluster.second), false)); + const auto migrations( + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false)); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId1, migrations[0].to); @@ -267,8 +323,8 @@ TEST(BalancerPolicy, JumboChunksNotMovedParallel) { cluster.second[kShardId2][2].setJumbo(false); // Only chunk 1 is not jumbo cluster.second[kShardId2][3].setJumbo(true); - const auto migrations(BalancerPolicy::balance( - cluster.first, DistributionStatus(kNamespace, cluster.second), false)); + const auto migrations( + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false)); ASSERT_EQ(2U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); @@ -288,8 +344,8 @@ TEST(BalancerPolicy, DrainingSingleChunk) { {{ShardStatistics(kShardId0, kNoMaxSize, 2, true, emptyTagSet, emptyShardVersion), 1}, {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 5}}); - const auto migrations(BalancerPolicy::balance( - cluster.first, DistributionStatus(kNamespace, cluster.second), false)); + const auto migrations( + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false)); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId1, migrations[0].to); @@ -305,8 +361,8 @@ TEST(BalancerPolicy, DrainingSingleChunkPerShard) { {ShardStatistics(kShardId2, kNoMaxSize, 2, true, emptyTagSet, emptyShardVersion), 1}, {ShardStatistics(kShardId3, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 5}}); - const auto migrations(BalancerPolicy::balance( - cluster.first, DistributionStatus(kNamespace, cluster.second), false)); + const auto migrations( + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false)); ASSERT_EQ(2U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); @@ -326,8 +382,8 @@ TEST(BalancerPolicy, DrainingWithTwoChunksFirstOneSelected) { {{ShardStatistics(kShardId0, kNoMaxSize, 2, true, emptyTagSet, emptyShardVersion), 2}, {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 5}}); - const auto migrations(BalancerPolicy::balance( - cluster.first, DistributionStatus(kNamespace, cluster.second), false)); + const auto migrations( + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false)); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId1, migrations[0].to); @@ -343,8 +399,8 @@ TEST(BalancerPolicy, DrainingMultipleShardsFirstOneSelected) { {ShardStatistics(kShardId1, kNoMaxSize, 5, true, emptyTagSet, emptyShardVersion), 2}, {ShardStatistics(kShardId2, kNoMaxSize, 5, false, emptyTagSet, emptyShardVersion), 16}}); - const auto migrations(BalancerPolicy::balance( - cluster.first, DistributionStatus(kNamespace, cluster.second), false)); + const auto migrations( + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false)); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId2, migrations[0].to); @@ -359,8 +415,8 @@ TEST(BalancerPolicy, DrainingMultipleShardsWontAcceptChunks) { {ShardStatistics(kShardId1, kNoMaxSize, 0, true, emptyTagSet, emptyShardVersion), 0}, {ShardStatistics(kShardId2, kNoMaxSize, 0, true, emptyTagSet, emptyShardVersion), 0}}); - const auto migrations(BalancerPolicy::balance( - cluster.first, DistributionStatus(kNamespace, cluster.second), false)); + const auto migrations( + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false)); ASSERT(migrations.empty()); } @@ -374,7 +430,7 @@ TEST(BalancerPolicy, DrainingSingleAppropriateShardFoundDueToTag) { ASSERT_OK(distribution.addRangeToZone(ZoneRange( cluster.second[kShardId2][0].getMin(), cluster.second[kShardId2][0].getMax(), "LAX"))); - const auto migrations(BalancerPolicy::balance(cluster.first, distribution, false)); + const auto migrations(balanceChunks(cluster.first, distribution, false)); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId2, migrations[0].from); ASSERT_EQ(kShardId1, migrations[0].to); @@ -392,7 +448,7 @@ TEST(BalancerPolicy, DrainingNoAppropriateShardsFoundDueToTag) { ASSERT_OK(distribution.addRangeToZone(ZoneRange( cluster.second[kShardId2][0].getMin(), cluster.second[kShardId2][0].getMax(), "SEA"))); - const auto migrations(BalancerPolicy::balance(cluster.first, distribution, false)); + const auto migrations(balanceChunks(cluster.first, distribution, false)); ASSERT(migrations.empty()); } @@ -403,8 +459,8 @@ TEST(BalancerPolicy, NoBalancingDueToAllNodesEitherDrainingOrMaxedOut) { {ShardStatistics(kShardId1, 1, 1, false, emptyTagSet, emptyShardVersion), 6}, {ShardStatistics(kShardId2, kNoMaxSize, 1, true, emptyTagSet, emptyShardVersion), 1}}); - const auto migrations(BalancerPolicy::balance( - cluster.first, DistributionStatus(kNamespace, cluster.second), false)); + const auto migrations( + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false)); ASSERT(migrations.empty()); } @@ -417,8 +473,8 @@ TEST(BalancerPolicy, BalancerRespectsMaxShardSizeOnlyBalanceToNonMaxed) { {ShardStatistics(kShardId1, kNoMaxSize, 5, false, emptyTagSet, emptyShardVersion), 5}, {ShardStatistics(kShardId2, kNoMaxSize, 10, false, emptyTagSet, emptyShardVersion), 10}}); - const auto migrations(BalancerPolicy::balance( - cluster.first, DistributionStatus(kNamespace, cluster.second), false)); + const auto migrations( + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false)); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId2, migrations[0].from); ASSERT_EQ(kShardId1, migrations[0].to); @@ -435,8 +491,8 @@ TEST(BalancerPolicy, BalancerRespectsMaxShardSizeWhenAllBalanced) { {ShardStatistics(kShardId1, kNoMaxSize, 4, false, emptyTagSet, emptyShardVersion), 4}, {ShardStatistics(kShardId2, kNoMaxSize, 4, false, emptyTagSet, emptyShardVersion), 4}}); - const auto migrations(BalancerPolicy::balance( - cluster.first, DistributionStatus(kNamespace, cluster.second), false)); + const auto migrations( + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false)); ASSERT(migrations.empty()); } @@ -451,7 +507,7 @@ TEST(BalancerPolicy, BalancerRespectsTagsWhenDraining) { ASSERT_OK(distribution.addRangeToZone(ZoneRange(kMinBSONKey, BSON("x" << 7), "a"))); ASSERT_OK(distribution.addRangeToZone(ZoneRange(BSON("x" << 8), kMaxBSONKey, "b"))); - const auto migrations(BalancerPolicy::balance(cluster.first, distribution, false)); + const auto migrations(balanceChunks(cluster.first, distribution, false)); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId1, migrations[0].from); ASSERT_EQ(kShardId0, migrations[0].to); @@ -470,7 +526,7 @@ TEST(BalancerPolicy, BalancerRespectsTagPolicyBeforeImbalance) { DistributionStatus distribution(kNamespace, cluster.second); ASSERT_OK(distribution.addRangeToZone(ZoneRange(kMinBSONKey, BSON("x" << 100), "a"))); - const auto migrations(BalancerPolicy::balance(cluster.first, distribution, false)); + const auto migrations(balanceChunks(cluster.first, distribution, false)); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId2, migrations[0].from); ASSERT_EQ(kShardId0, migrations[0].to); @@ -490,7 +546,7 @@ TEST(BalancerPolicy, BalancerFixesIncorrectTagsWithCrossShardViolationOfTags) { ASSERT_OK(distribution.addRangeToZone(ZoneRange(kMinBSONKey, BSON("x" << 1), "b"))); ASSERT_OK(distribution.addRangeToZone(ZoneRange(BSON("x" << 8), kMaxBSONKey, "a"))); - const auto migrations(BalancerPolicy::balance(cluster.first, distribution, false)); + const auto migrations(balanceChunks(cluster.first, distribution, false)); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId2, migrations[0].to); @@ -508,7 +564,7 @@ TEST(BalancerPolicy, BalancerFixesIncorrectTagsInOtherwiseBalancedCluster) { DistributionStatus distribution(kNamespace, cluster.second); ASSERT_OK(distribution.addRangeToZone(ZoneRange(kMinBSONKey, BSON("x" << 10), "a"))); - const auto migrations(BalancerPolicy::balance(cluster.first, distribution, false)); + const auto migrations(balanceChunks(cluster.first, distribution, false)); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId2, migrations[0].from); ASSERT_EQ(kShardId0, migrations[0].to); @@ -527,7 +583,7 @@ TEST(BalancerPolicy, BalancerFixesIncorrectTagsInOtherwiseBalancedClusterParalle DistributionStatus distribution(kNamespace, cluster.second); ASSERT_OK(distribution.addRangeToZone(ZoneRange(kMinBSONKey, BSON("x" << 20), "a"))); - const auto migrations(BalancerPolicy::balance(cluster.first, distribution, false)); + const auto migrations(balanceChunks(cluster.first, distribution, false)); ASSERT_EQ(2U, migrations.size()); ASSERT_EQ(kShardId2, migrations[0].from); @@ -550,7 +606,7 @@ TEST(BalancerPolicy, BalancerHandlesNoShardsWithTag) { ASSERT_OK( distribution.addRangeToZone(ZoneRange(kMinBSONKey, BSON("x" << 7), "NonExistentZone"))); - ASSERT(BalancerPolicy::balance(cluster.first, distribution, false).empty()); + ASSERT(balanceChunks(cluster.first, distribution, false).empty()); } TEST(DistributionStatus, AddTagRangeOverlap) { diff --git a/src/mongo/db/s/collection_range_deleter.cpp b/src/mongo/db/s/collection_range_deleter.cpp deleted file mode 100644 index 803541d16f9..00000000000 --- a/src/mongo/db/s/collection_range_deleter.cpp +++ /dev/null @@ -1,222 +0,0 @@ -/** - * Copyright (C) 2016 MongoDB Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kSharding - -#include "mongo/platform/basic.h" - -#include "mongo/db/s/collection_range_deleter.h" - -#include - -#include "mongo/db/catalog/collection.h" -#include "mongo/db/client.h" -#include "mongo/db/db_raii.h" -#include "mongo/db/dbhelpers.h" -#include "mongo/db/exec/working_set_common.h" -#include "mongo/db/index/index_descriptor.h" -#include "mongo/db/keypattern.h" -#include "mongo/db/query/internal_plans.h" -#include "mongo/db/query/query_knobs.h" -#include "mongo/db/query/query_planner.h" -#include "mongo/db/repl/repl_client_info.h" -#include "mongo/db/repl/replication_coordinator_global.h" -#include "mongo/db/s/collection_sharding_state.h" -#include "mongo/db/s/sharding_state.h" -#include "mongo/db/write_concern.h" -#include "mongo/executor/task_executor.h" -#include "mongo/util/log.h" -#include "mongo/util/mongoutils/str.h" -#include "mongo/util/scopeguard.h" - -namespace mongo { - -class ChunkRange; -class OldClientWriteContext; - -using CallbackArgs = executor::TaskExecutor::CallbackArgs; -using logger::LogComponent; - -namespace { - -const WriteConcernOptions kMajorityWriteConcern(WriteConcernOptions::kMajority, - WriteConcernOptions::SyncMode::UNSET, - Seconds(60)); - -} // unnamed namespace - -CollectionRangeDeleter::CollectionRangeDeleter(NamespaceString nss) : _nss(std::move(nss)) {} - -void CollectionRangeDeleter::run() { - Client::initThread(getThreadName()); - ON_BLOCK_EXIT([&] { Client::destroy(); }); - auto txn = cc().makeOperationContext().get(); - bool hasNextRangeToClean = cleanupNextRange(txn); - - // If there are more ranges to run, we add back onto the task executor to run again. - if (hasNextRangeToClean) { - auto executor = ShardingState::get(txn)->getRangeDeleterTaskExecutor(); - executor->scheduleWork([this](const CallbackArgs& cbArgs) { run(); }); - } else { - delete this; - } -} - -bool CollectionRangeDeleter::cleanupNextRange(OperationContext* txn) { - int numDocumentsDeleted; - - { - AutoGetCollection autoColl(txn, _nss, MODE_IX); - Collection* collection = autoColl.getCollection(); - if (!collection) { - return false; - } - - CollectionShardingState* shardingState = CollectionShardingState::get(txn, _nss); - MetadataManager& metadataManager = *shardingState->_metadataManager; - - if (!_rangeInProgress && !metadataManager.hasRangesToClean()) { - // Nothing left to do - return false; - } - - if (!_rangeInProgress || !metadataManager.isInRangesToClean(_rangeInProgress.get())) { - // No valid chunk in progress, get a new one - _rangeInProgress = metadataManager.getNextRangeToClean(); - } - - auto metadata = shardingState->getMetadata(); - if (!metadata) { - return false; - } - - numDocumentsDeleted = _doDeletion(txn, collection, metadata->getKeyPattern()); - if (numDocumentsDeleted <= 0) { - metadataManager.removeRangeToClean(_rangeInProgress.get()); - _rangeInProgress = boost::none; - return true; - } - } - - // wait for replication - WriteConcernResult wcResult; - auto currentClientOpTime = repl::ReplClientInfo::forClient(txn->getClient()).getLastOp(); - Status status = waitForWriteConcern(txn, currentClientOpTime, kMajorityWriteConcern, &wcResult); - if (!status.isOK()) { - warning() << "Error when waiting for write concern after removing chunks in " << _nss - << " : " << status.reason(); - } - - return true; -} - -int CollectionRangeDeleter::_doDeletion(OperationContext* txn, - Collection* collection, - const BSONObj& keyPattern) { - invariant(_rangeInProgress); - invariant(collection); - - // The IndexChunk has a keyPattern that may apply to more than one index - we need to - // select the index and get the full index keyPattern here. - const IndexDescriptor* idx = - collection->getIndexCatalog()->findShardKeyPrefixedIndex(txn, keyPattern, false); - if (idx == NULL) { - warning() << "Unable to find shard key index for " << keyPattern.toString() << " in " - << _nss; - return -1; - } - - KeyPattern indexKeyPattern(idx->keyPattern().getOwned()); - - // Extend bounds to match the index we found - const BSONObj& min = - Helpers::toKeyFormat(indexKeyPattern.extendRangeBound(_rangeInProgress->getMin(), false)); - const BSONObj& max = - Helpers::toKeyFormat(indexKeyPattern.extendRangeBound(_rangeInProgress->getMax(), false)); - - LOG(1) << "begin removal of " << min << " to " << max << " in " << _nss; - - auto indexName = idx->indexName(); - IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByName(txn, indexName); - if (!desc) { - warning() << "shard key index with name " << indexName << " on '" << _nss - << "' was dropped"; - return -1; - } - - std::unique_ptr exec( - InternalPlanner::indexScan(txn, - collection, - desc, - min, - max, - BoundInclusion::kIncludeStartKeyOnly, - PlanExecutor::YIELD_MANUAL, - InternalPlanner::FORWARD, - InternalPlanner::IXSCAN_FETCH)); - int numDeleted = 0; - const int maxItersBeforeYield = std::max(static_cast(internalQueryExecYieldIterations), 1); - - while (numDeleted < maxItersBeforeYield) { - RecordId rloc; - BSONObj obj; - PlanExecutor::ExecState state; - state = exec->getNext(&obj, &rloc); - if (PlanExecutor::IS_EOF == state) { - break; - } - - if (PlanExecutor::FAILURE == state || PlanExecutor::DEAD == state) { - warning(LogComponent::kSharding) - << PlanExecutor::statestr(state) << " - cursor error while trying to delete " << min - << " to " << max << " in " << _nss << ": " << WorkingSetCommon::toStatusString(obj) - << ", stats: " << Explain::getWinningPlanStats(exec.get()); - break; - } - - invariant(PlanExecutor::ADVANCED == state); - - WriteUnitOfWork wuow(txn); - - NamespaceString nss(_nss); - if (!repl::getGlobalReplicationCoordinator()->canAcceptWritesFor(nss)) { - warning() << "stepped down from primary while deleting chunk; " - << "orphaning data in " << _nss << " in range [" << min << ", " << max << ")"; - return numDeleted; - } - - OpDebug* const nullOpDebug = nullptr; - collection->deleteDocument(txn, rloc, nullOpDebug, true); - wuow.commit(); - numDeleted++; - } - - return numDeleted; -} - -} // namespace mongo diff --git a/src/mongo/db/s/collection_range_deleter.h b/src/mongo/db/s/collection_range_deleter.h deleted file mode 100644 index 9d2e1ae8cb4..00000000000 --- a/src/mongo/db/s/collection_range_deleter.h +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Copyright (C) 2016 MongoDB Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ -#pragma once - -#include "mongo/base/disallow_copying.h" -#include "mongo/db/namespace_string.h" -#include "mongo/s/catalog/type_chunk.h" - -namespace mongo { - -class BSONObj; -class Collection; -class OperationContext; - -class CollectionRangeDeleter { - MONGO_DISALLOW_COPYING(CollectionRangeDeleter); - -public: - CollectionRangeDeleter(NamespaceString nss); - - /** - * Starts deleting ranges and cleans up this object when it is finished. - */ - void run(); - - /** - * Acquires the collection IX lock and checks whether there are new entries for the collection's - * rangesToClean structure. If there are, deletes some small amount of entries and yields using - * the standard query yielding logic. - * - * Returns true if there are more entries in rangesToClean, false if there is no more progress - * to be made. - */ - bool cleanupNextRange(OperationContext* txn); - -private: - /** - * Performs the deletion of a small amount of entries within the range in progress. - * This function will invariant if called while _rangeInProgress is not set. - * - * Returns the number of documents deleted (0 if deletion is finished), or -1 for error. - */ - int _doDeletion(OperationContext* txn, Collection* collection, const BSONObj& keyPattern); - - NamespaceString _nss; - - // Holds a range for which deletion has begun. If empty, then a new range - // must be requested from rangesToClean - boost::optional _rangeInProgress; -}; - -} // namespace mongo diff --git a/src/mongo/db/s/collection_sharding_state.h b/src/mongo/db/s/collection_sharding_state.h index 2a01bec5567..1abababcc1b 100644 --- a/src/mongo/db/s/collection_sharding_state.h +++ b/src/mongo/db/s/collection_sharding_state.h @@ -168,8 +168,6 @@ public: void onDropCollection(OperationContext* txn, const NamespaceString& collectionName); private: - friend class CollectionRangeDeleter; - /** * Checks whether the shard version of the operation matches that of the collection. * diff --git a/src/mongo/db/s/metadata_manager.cpp b/src/mongo/db/s/metadata_manager.cpp index 28ca20e2f6f..0bba1ff478e 100644 --- a/src/mongo/db/s/metadata_manager.cpp +++ b/src/mongo/db/s/metadata_manager.cpp @@ -34,7 +34,6 @@ #include "mongo/bson/simple_bsonobj_comparator.h" #include "mongo/db/range_arithmetic.h" -#include "mongo/db/s/collection_range_deleter.h" #include "mongo/db/s/sharding_state.h" #include "mongo/stdx/memory.h" #include "mongo/util/log.h" @@ -47,9 +46,8 @@ MetadataManager::MetadataManager(ServiceContext* sc, NamespaceString nss) : _nss(std::move(nss)), _serviceContext(sc), _activeMetadataTracker(stdx::make_unique(nullptr)), - _receivingChunks(SimpleBSONObjComparator::kInstance.makeBSONObjIndexedMap()), - _rangesToClean( - SimpleBSONObjComparator::kInstance.makeBSONObjIndexedMap()) {} + _receivingChunks( + SimpleBSONObjComparator::kInstance.makeBSONObjIndexedMap()) {} MetadataManager::~MetadataManager() { stdx::lock_guard scopedLock(_managerLock); @@ -74,7 +72,6 @@ void MetadataManager::refreshActiveMetadata(std::unique_ptr // collection sharding information regardless of whether the node is sharded or not. if (!remoteMetadata && !_activeMetadataTracker->metadata) { invariant(_receivingChunks.empty()); - invariant(_rangesToClean.empty()); return; } @@ -84,8 +81,6 @@ void MetadataManager::refreshActiveMetadata(std::unique_ptr << _activeMetadataTracker->metadata->toStringBasic() << " as no longer sharded"; _receivingChunks.clear(); - _rangesToClean.clear(); - _setActiveMetadata_inlock(nullptr); return; } @@ -100,8 +95,6 @@ void MetadataManager::refreshActiveMetadata(std::unique_ptr << remoteMetadata->toStringBasic(); invariant(_receivingChunks.empty()); - invariant(_rangesToClean.empty()); - _setActiveMetadata_inlock(std::move(remoteMetadata)); return; } @@ -115,8 +108,6 @@ void MetadataManager::refreshActiveMetadata(std::unique_ptr << remoteMetadata->toStringBasic() << " due to epoch change"; _receivingChunks.clear(); - _rangesToClean.clear(); - _setActiveMetadata_inlock(std::move(remoteMetadata)); return; } @@ -168,9 +159,6 @@ void MetadataManager::refreshActiveMetadata(std::unique_ptr const ChunkRange receivingRange(itRecv->first, itRecv->second.getMaxKey()); _receivingChunks.erase(itRecv); - - // Make sure any potentially partially copied chunks are scheduled to be cleaned up - _addRangeToClean_inlock(receivingRange); } // Need to reset the iterator @@ -207,14 +195,10 @@ void MetadataManager::beginReceive(const ChunkRange& range) { const ChunkRange receivingRange(itRecv->first, itRecv->second.getMaxKey()); _receivingChunks.erase(itRecv); - - // Make sure any potentially partially copied chunks are scheduled to be cleaned up - _addRangeToClean_inlock(receivingRange); } // Need to ensure that the background range deleter task won't delete the range we are about to // receive - _removeRangeToClean_inlock(range, Status::OK()); _receivingChunks.insert( std::make_pair(range.getMin().getOwned(), CachedChunkInfo(range.getMax().getOwned(), ChunkVersion::IGNORED()))); @@ -241,9 +225,6 @@ void MetadataManager::forgetReceive(const ChunkRange& range) { _receivingChunks.erase(it); } - // This is potentially a partially received data, which needs to be cleaned up - _addRangeToClean_inlock(range); - // For compatibility with the current range deleter, update the pending chunks on the collection // metadata to exclude the chunk being received, which was added in beginReceive ChunkType chunk; @@ -343,91 +324,9 @@ ScopedCollectionMetadata::operator bool() const { return _tracker && _tracker->metadata.get(); } -RangeMap MetadataManager::getCopyOfRangesToClean() { - stdx::lock_guard scopedLock(_managerLock); - return _getCopyOfRangesToClean_inlock(); -} - -RangeMap MetadataManager::_getCopyOfRangesToClean_inlock() { - RangeMap ranges = SimpleBSONObjComparator::kInstance.makeBSONObjIndexedMap(); - for (auto it = _rangesToClean.begin(); it != _rangesToClean.end(); ++it) { - ranges.insert(std::make_pair( - it->first, CachedChunkInfo(it->second.getMax(), ChunkVersion::IGNORED()))); - } - return ranges; -} - -std::shared_ptr> MetadataManager::addRangeToClean(const ChunkRange& range) { - stdx::lock_guard scopedLock(_managerLock); - return _addRangeToClean_inlock(range); -} - -std::shared_ptr> MetadataManager::_addRangeToClean_inlock( - const ChunkRange& range) { - // This first invariant currently makes an unnecessary copy, to reuse the - // rangeMapOverlaps helper function. - invariant(!rangeMapOverlaps(_getCopyOfRangesToClean_inlock(), range.getMin(), range.getMax())); - invariant(!rangeMapOverlaps(_receivingChunks, range.getMin(), range.getMax())); - - RangeToCleanDescriptor descriptor(range.getMax().getOwned()); - _rangesToClean.insert(std::make_pair(range.getMin().getOwned(), descriptor)); - - // If _rangesToClean was previously empty, we need to start the collection range deleter - if (_rangesToClean.size() == 1UL) { - ShardingState::get(_serviceContext)->scheduleCleanup(_nss); - } - - return descriptor.getNotification(); -} - -void MetadataManager::removeRangeToClean(const ChunkRange& range, Status deletionStatus) { - stdx::lock_guard scopedLock(_managerLock); - _removeRangeToClean_inlock(range, deletionStatus); -} - -void MetadataManager::_removeRangeToClean_inlock(const ChunkRange& range, Status deletionStatus) { - auto it = _rangesToClean.upper_bound(range.getMin()); - // We want our iterator to point at the greatest value - // that is still less than or equal to range. - if (it != _rangesToClean.begin()) { - --it; - } - - for (; it != _rangesToClean.end() && - SimpleBSONObjComparator::kInstance.evaluate(it->first < range.getMax());) { - if (SimpleBSONObjComparator::kInstance.evaluate(it->second.getMax() <= range.getMin())) { - ++it; - continue; - } - - // There's overlap between *it and range so we remove *it - // and then replace with new ranges. - BSONObj oldMin = it->first; - BSONObj oldMax = it->second.getMax(); - it->second.complete(deletionStatus); - _rangesToClean.erase(it++); - if (SimpleBSONObjComparator::kInstance.evaluate(oldMin < range.getMin())) { - _addRangeToClean_inlock(ChunkRange(oldMin, range.getMin())); - } - - if (SimpleBSONObjComparator::kInstance.evaluate(oldMax > range.getMax())) { - _addRangeToClean_inlock(ChunkRange(range.getMax(), oldMax)); - } - } -} - void MetadataManager::append(BSONObjBuilder* builder) { stdx::lock_guard scopedLock(_managerLock); - BSONArrayBuilder rtcArr(builder->subarrayStart("rangesToClean")); - for (const auto& entry : _rangesToClean) { - BSONObjBuilder obj; - ChunkRange r = ChunkRange(entry.first, entry.second.getMax()); - r.append(&obj); - rtcArr.append(obj.done()); - } - rtcArr.done(); - BSONArrayBuilder pcArr(builder->subarrayStart("pendingChunks")); for (const auto& entry : _receivingChunks) { BSONObjBuilder obj; @@ -447,23 +346,4 @@ void MetadataManager::append(BSONObjBuilder* builder) { amrArr.done(); } -bool MetadataManager::hasRangesToClean() { - stdx::lock_guard scopedLock(_managerLock); - return !_rangesToClean.empty(); -} - -bool MetadataManager::isInRangesToClean(const ChunkRange& range) { - stdx::lock_guard scopedLock(_managerLock); - // For convenience, this line makes an unnecessary copy, to reuse the - // rangeMapContains helper function. - return rangeMapContains(_getCopyOfRangesToClean_inlock(), range.getMin(), range.getMax()); -} - -ChunkRange MetadataManager::getNextRangeToClean() { - stdx::lock_guard scopedLock(_managerLock); - invariant(!_rangesToClean.empty()); - auto it = _rangesToClean.begin(); - return ChunkRange(it->first, it->second.getMax()); -} - } // namespace mongo diff --git a/src/mongo/db/s/metadata_manager.h b/src/mongo/db/s/metadata_manager.h index fa61e9ef2ea..b2e712d7f9f 100644 --- a/src/mongo/db/s/metadata_manager.h +++ b/src/mongo/db/s/metadata_manager.h @@ -84,53 +84,11 @@ public: */ RangeMap getCopyOfReceivingChunks(); - /** - * Adds a new range to be cleaned up. - * The newly introduced range must not overlap with the existing ranges. - */ - std::shared_ptr> addRangeToClean(const ChunkRange& range); - - /** - * Calls removeRangeToClean with Status::OK. - */ - void removeRangeToClean(const ChunkRange& range) { - removeRangeToClean(range, Status::OK()); - } - - /** - * Removes the specified range from the ranges to be cleaned up. - * The specified deletionStatus will be returned to callers waiting - * on whether the deletion succeeded or failed. - */ - void removeRangeToClean(const ChunkRange& range, Status deletionStatus); - - /** - * Gets copy of the set of chunk ranges which are scheduled for cleanup. - * Converts RangeToCleanMap to RangeMap. - */ - RangeMap getCopyOfRangesToClean(); - /** * Appends information on all the chunk ranges in rangesToClean to builder. */ void append(BSONObjBuilder* builder); - /** - * Returns true if _rangesToClean is not empty. - */ - bool hasRangesToClean(); - - /** - * Returns true if the exact range is in _rangesToClean. - */ - bool isInRangesToClean(const ChunkRange& range); - - /** - * Gets and returns, but does not remove, a single ChunkRange from _rangesToClean. - * Should not be called if _rangesToClean is empty: it will hit an invariant. - */ - ChunkRange getNextRangeToClean(); - private: friend class ScopedCollectionMetadata; @@ -193,12 +151,6 @@ private: */ void _removeMetadata_inlock(CollectionMetadataTracker* metadataTracker); - std::shared_ptr> _addRangeToClean_inlock(const ChunkRange& range); - - void _removeRangeToClean_inlock(const ChunkRange& range, Status deletionStatus); - - RangeMap _getCopyOfRangesToClean_inlock(); - void _setActiveMetadata_inlock(std::unique_ptr newMetadata); const NamespaceString _nss; @@ -219,10 +171,6 @@ private: // Chunk ranges which are currently assumed to be transferred to the shard. Indexed by the min // key of the range. RangeMap _receivingChunks; - - // Set of ranges to be deleted. Indexed by the min key of the range. - typedef BSONObjIndexedMap RangeToCleanMap; - RangeToCleanMap _rangesToClean; }; class ScopedCollectionMetadata { diff --git a/src/mongo/db/s/metadata_manager_test.cpp b/src/mongo/db/s/metadata_manager_test.cpp index 19f4fd405ab..207eb4fd37b 100644 --- a/src/mongo/db/s/metadata_manager_test.cpp +++ b/src/mongo/db/s/metadata_manager_test.cpp @@ -50,12 +50,6 @@ using unittest::assertGet; class MetadataManagerTest : public ServiceContextMongoDTest { protected: - void setUp() override { - ServiceContextMongoDTest::setUp(); - ShardingState::get(getServiceContext()) - ->setScheduleCleanupFunctionForTest([](const NamespaceString& nss) {}); - } - static std::unique_ptr makeEmptyMetadata() { const OID epoch = OID::gen(); @@ -106,8 +100,7 @@ TEST_F(MetadataManagerTest, SetAndGetActiveMetadata) { ScopedCollectionMetadata scopedMetadata = manager.getActiveMetadata(manager_ptr); ASSERT_EQ(cmPtr, scopedMetadata.getMetadata()); -}; - +} TEST_F(MetadataManagerTest, ResetActiveMetadata) { manager.refreshActiveMetadata(makeEmptyMetadata()); @@ -124,167 +117,8 @@ TEST_F(MetadataManagerTest, ResetActiveMetadata) { ScopedCollectionMetadata scopedMetadata2 = manager.getActiveMetadata(manager_ptr); ASSERT_EQ(cm2Ptr, scopedMetadata2.getMetadata()); -}; - -TEST_F(MetadataManagerTest, AddAndRemoveRangesToClean) { - ChunkRange cr1 = ChunkRange(BSON("key" << 0), BSON("key" << 10)); - ChunkRange cr2 = ChunkRange(BSON("key" << 10), BSON("key" << 20)); - - manager.addRangeToClean(cr1); - ASSERT_EQ(manager.getCopyOfRangesToClean().size(), 1UL); - manager.removeRangeToClean(cr1); - ASSERT_EQ(manager.getCopyOfRangesToClean().size(), 0UL); - - manager.addRangeToClean(cr1); - manager.addRangeToClean(cr2); - manager.removeRangeToClean(cr1); - ASSERT_EQ(manager.getCopyOfRangesToClean().size(), 1UL); - auto ranges = manager.getCopyOfRangesToClean(); - auto it = ranges.find(cr2.getMin()); - ChunkRange remainingChunk = ChunkRange(it->first, it->second.getMaxKey()); - ASSERT_EQ(remainingChunk.toString(), cr2.toString()); - manager.removeRangeToClean(cr2); -} - -// Tests that a removal in the middle of an existing ChunkRange results in -// two correct chunk ranges. -TEST_F(MetadataManagerTest, RemoveRangeInMiddleOfRange) { - ChunkRange cr1 = ChunkRange(BSON("key" << 0), BSON("key" << 10)); - - manager.addRangeToClean(cr1); - manager.removeRangeToClean(ChunkRange(BSON("key" << 4), BSON("key" << 6))); - ASSERT_EQ(manager.getCopyOfRangesToClean().size(), 2UL); - - auto ranges = manager.getCopyOfRangesToClean(); - auto it = ranges.find(BSON("key" << 0)); - ChunkRange expectedChunk = ChunkRange(BSON("key" << 0), BSON("key" << 4)); - ChunkRange remainingChunk = ChunkRange(it->first, it->second.getMaxKey()); - ASSERT_EQ(remainingChunk.toString(), expectedChunk.toString()); - - it++; - expectedChunk = ChunkRange(BSON("key" << 6), BSON("key" << 10)); - remainingChunk = ChunkRange(it->first, it->second.getMaxKey()); - ASSERT_EQ(remainingChunk.toString(), expectedChunk.toString()); - - manager.removeRangeToClean(cr1); - ASSERT_EQ(manager.getCopyOfRangesToClean().size(), 0UL); -} - -// Tests removals that overlap with just one ChunkRange. -TEST_F(MetadataManagerTest, RemoveRangeWithSingleRangeOverlap) { - ChunkRange cr1 = ChunkRange(BSON("key" << 0), BSON("key" << 10)); - - manager.addRangeToClean(cr1); - manager.removeRangeToClean(ChunkRange(BSON("key" << 0), BSON("key" << 5))); - ASSERT_EQ(manager.getCopyOfRangesToClean().size(), 1UL); - auto ranges = manager.getCopyOfRangesToClean(); - auto it = ranges.find(BSON("key" << 5)); - ChunkRange remainingChunk = ChunkRange(it->first, it->second.getMaxKey()); - ChunkRange expectedChunk = ChunkRange(BSON("key" << 5), BSON("key" << 10)); - ASSERT_EQ(remainingChunk.toString(), expectedChunk.toString()); - - manager.removeRangeToClean(ChunkRange(BSON("key" << 4), BSON("key" << 6))); - ASSERT_EQ(manager.getCopyOfRangesToClean().size(), 1UL); - ranges = manager.getCopyOfRangesToClean(); - it = ranges.find(BSON("key" << 6)); - remainingChunk = ChunkRange(it->first, it->second.getMaxKey()); - expectedChunk = ChunkRange(BSON("key" << 6), BSON("key" << 10)); - ASSERT_EQ(remainingChunk.toString(), expectedChunk.toString()); - - manager.removeRangeToClean(ChunkRange(BSON("key" << 9), BSON("key" << 13))); - ASSERT_EQ(manager.getCopyOfRangesToClean().size(), 1UL); - ranges = manager.getCopyOfRangesToClean(); - it = ranges.find(BSON("key" << 6)); - remainingChunk = ChunkRange(it->first, it->second.getMaxKey()); - expectedChunk = ChunkRange(BSON("key" << 6), BSON("key" << 9)); - ASSERT_EQ(remainingChunk.toString(), expectedChunk.toString()); - - manager.removeRangeToClean(ChunkRange(BSON("key" << 0), BSON("key" << 10))); - ASSERT_EQ(manager.getCopyOfRangesToClean().size(), 0UL); -} - -// Tests removals that overlap with more than one ChunkRange. -TEST_F(MetadataManagerTest, RemoveRangeWithMultipleRangeOverlaps) { - ChunkRange cr1 = ChunkRange(BSON("key" << 0), BSON("key" << 10)); - ChunkRange cr2 = ChunkRange(BSON("key" << 10), BSON("key" << 20)); - ChunkRange cr3 = ChunkRange(BSON("key" << 20), BSON("key" << 30)); - - manager.addRangeToClean(cr1); - manager.addRangeToClean(cr2); - manager.addRangeToClean(cr3); - ASSERT_EQ(manager.getCopyOfRangesToClean().size(), 3UL); - - manager.removeRangeToClean(ChunkRange(BSON("key" << 8), BSON("key" << 22))); - ASSERT_EQ(manager.getCopyOfRangesToClean().size(), 2UL); - auto ranges = manager.getCopyOfRangesToClean(); - auto it = ranges.find(BSON("key" << 0)); - ChunkRange remainingChunk = ChunkRange(it->first, it->second.getMaxKey()); - ChunkRange expectedChunk = ChunkRange(BSON("key" << 0), BSON("key" << 8)); - ASSERT_EQ(remainingChunk.toString(), expectedChunk.toString()); - it++; - remainingChunk = ChunkRange(it->first, it->second.getMaxKey()); - expectedChunk = ChunkRange(BSON("key" << 22), BSON("key" << 30)); - ASSERT_EQ(remainingChunk.toString(), expectedChunk.toString()); - - manager.removeRangeToClean(ChunkRange(BSON("key" << 0), BSON("key" << 30))); - ASSERT_EQ(manager.getCopyOfRangesToClean().size(), 0UL); -} - -TEST_F(MetadataManagerTest, AddAndRemoveRangeNotificationsBlockAndYield) { - manager.refreshActiveMetadata(makeEmptyMetadata()); - - ChunkRange cr1(BSON("key" << 0), BSON("key" << 10)); - auto notification = manager.addRangeToClean(cr1); - manager.removeRangeToClean(cr1, Status::OK()); - ASSERT_OK(notification->get()); - ASSERT_EQ(manager.getCopyOfRangesToClean().size(), 0UL); -} - -TEST_F(MetadataManagerTest, RemoveRangeToCleanCorrectlySetsBadStatus) { - manager.refreshActiveMetadata(makeEmptyMetadata()); - - ChunkRange cr1(BSON("key" << 0), BSON("key" << 10)); - auto notification = manager.addRangeToClean(cr1); - manager.removeRangeToClean(cr1, Status(ErrorCodes::InternalError, "test error")); - ASSERT_NOT_OK(notification->get()); - ASSERT_EQ(manager.getCopyOfRangesToClean().size(), 0UL); -} - -TEST_F(MetadataManagerTest, RemovingSubrangeStillSetsNotificationStatus) { - manager.refreshActiveMetadata(makeEmptyMetadata()); - - ChunkRange cr1(BSON("key" << 0), BSON("key" << 10)); - auto notification = manager.addRangeToClean(cr1); - manager.removeRangeToClean(ChunkRange(BSON("key" << 3), BSON("key" << 7))); - ASSERT_OK(notification->get()); - ASSERT_EQ(manager.getCopyOfRangesToClean().size(), 2UL); - manager.removeRangeToClean(cr1); - ASSERT_EQ(manager.getCopyOfRangesToClean().size(), 0UL); - - notification = manager.addRangeToClean(cr1); - manager.removeRangeToClean(ChunkRange(BSON("key" << 7), BSON("key" << 15))); - ASSERT_OK(notification->get()); - ASSERT_EQ(manager.getCopyOfRangesToClean().size(), 1UL); - manager.removeRangeToClean(cr1); - ASSERT_EQ(manager.getCopyOfRangesToClean().size(), 0UL); } -TEST_F(MetadataManagerTest, NotificationBlocksUntilDeletion) { - manager.refreshActiveMetadata(makeEmptyMetadata()); - - ChunkRange cr1(BSON("key" << 0), BSON("key" << 10)); - auto notification = manager.addRangeToClean(cr1); - auto txn = cc().makeOperationContext().get(); - // Once the new range deleter is set up, this might fail if the range deleter - // deleted cr1 before we got here... - ASSERT_FALSE(notification->waitFor(txn, Milliseconds(0))); - - manager.removeRangeToClean(cr1); - ASSERT_TRUE(notification->waitFor(txn, Milliseconds(0))); - ASSERT_OK(notification->get()); -} - - TEST_F(MetadataManagerTest, RefreshAfterSuccessfulMigrationSinglePending) { manager.refreshActiveMetadata(makeEmptyMetadata()); @@ -417,29 +251,5 @@ TEST_F(MetadataManagerTest, RefreshMetadataAfterDropAndRecreate) { ASSERT_EQ(newVersion, chunkEntry->second.getVersion()); } -// Tests membership functions for _rangesToClean -TEST_F(MetadataManagerTest, RangesToCleanMembership) { - manager.refreshActiveMetadata(makeEmptyMetadata()); - - ASSERT(!manager.hasRangesToClean()); - - ChunkRange cr1 = ChunkRange(BSON("key" << 0), BSON("key" << 10)); - manager.addRangeToClean(cr1); - - ASSERT(manager.hasRangesToClean()); - ASSERT(manager.isInRangesToClean(cr1)); -} - -// Tests that getNextRangeToClean successfully pulls a stored ChunkRange -TEST_F(MetadataManagerTest, GetNextRangeToClean) { - manager.refreshActiveMetadata(makeEmptyMetadata()); - - ChunkRange cr1 = ChunkRange(BSON("key" << 0), BSON("key" << 10)); - manager.addRangeToClean(cr1); - - ChunkRange cr2 = manager.getNextRangeToClean(); - ASSERT_EQ(cr1.toString(), cr2.toString()); -} - } // namespace } // namespace mongo diff --git a/src/mongo/db/s/migration_chunk_cloner_source.h b/src/mongo/db/s/migration_chunk_cloner_source.h index 04cf9e36df2..cb3b10a8ef8 100644 --- a/src/mongo/db/s/migration_chunk_cloner_source.h +++ b/src/mongo/db/s/migration_chunk_cloner_source.h @@ -88,9 +88,13 @@ public: * This must only be called once and no more methods on the cloner must be used afterwards * regardless of whether it succeeds or not. * + * Returns statistics about the move. These are informational only and should not be + * interpreted by the caller for any means other than reporting. + * * NOTE: Must be called without any locks. */ - virtual Status commitClone(OperationContext* txn) = 0; + + virtual StatusWith commitClone(OperationContext* opCtx) = 0; /** * Tells the recipient to abort the clone and cleanup any unused data. This method's diff --git a/src/mongo/db/s/migration_chunk_cloner_source_legacy.cpp b/src/mongo/db/s/migration_chunk_cloner_source_legacy.cpp index a11d74ef587..ea83fa83d0d 100644 --- a/src/mongo/db/s/migration_chunk_cloner_source_legacy.cpp +++ b/src/mongo/db/s/migration_chunk_cloner_source_legacy.cpp @@ -75,9 +75,11 @@ bool isInRange(const BSONObj& obj, BSONObj createRequestWithSessionId(StringData commandName, const NamespaceString& nss, - const MigrationSessionId& sessionId) { + const MigrationSessionId& sessionId, + bool waitForSteadyOrDone = false) { BSONObjBuilder builder; builder.append(commandName, nss.ns()); + builder.append("waitForSteadyOrDone", waitForSteadyOrDone); sessionId.append(&builder); return builder.obj(); } @@ -231,13 +233,8 @@ Status MigrationChunkClonerSourceLegacy::awaitUntilCriticalSectionIsAppropriate( int iteration = 0; while ((Date_t::now() - startTime) < maxTimeToWait) { - // Exponential sleep backoff, up to 1024ms. Don't sleep much on the first few iterations, - // since we want empty chunk migrations to be fast. - sleepmillis(1LL << std::min(iteration, 10)); - iteration++; - auto responseStatus = _callRecipient( - createRequestWithSessionId(kRecvChunkStatus, _args.getNss(), _sessionId)); + createRequestWithSessionId(kRecvChunkStatus, _args.getNss(), _sessionId, true)); if (!responseStatus.isOK()) { return {responseStatus.getStatus().code(), str::stream() @@ -247,6 +244,11 @@ Status MigrationChunkClonerSourceLegacy::awaitUntilCriticalSectionIsAppropriate( const BSONObj& res = responseStatus.getValue(); + if (!res["waited"].boolean()) { + sleepmillis(1LL << std::min(iteration, 10)); + } + iteration++; + stdx::lock_guard sl(_mutex); const std::size_t cloneLocsRemaining = _cloneLocs.size(); @@ -306,7 +308,7 @@ Status MigrationChunkClonerSourceLegacy::awaitUntilCriticalSectionIsAppropriate( return {ErrorCodes::ExceededTimeLimit, "Timed out waiting for the cloner to catch up"}; } -Status MigrationChunkClonerSourceLegacy::commitClone(OperationContext* txn) { +StatusWith MigrationChunkClonerSourceLegacy::commitClone(OperationContext* txn) { invariant(_state == kCloning); invariant(!txn->lockState()->isLocked()); @@ -314,7 +316,7 @@ Status MigrationChunkClonerSourceLegacy::commitClone(OperationContext* txn) { _callRecipient(createRequestWithSessionId(kRecvChunkCommit, _args.getNss(), _sessionId)); if (responseStatus.isOK()) { _cleanup(txn); - return Status::OK(); + return responseStatus; } cancelClone(txn); @@ -576,8 +578,7 @@ Status MigrationChunkClonerSourceLegacy::_storeCurrentLocs(OperationContext* txn if (totalRecs > 0) { avgRecSize = collection->dataSize(txn) / totalRecs; maxRecsWhenFull = _args.getMaxChunkSizeBytes() / avgRecSize; - maxRecsWhenFull = std::min((unsigned long long)(kMaxObjectPerChunk + 1), - 130 * maxRecsWhenFull / 100 /* slack */); + maxRecsWhenFull = 130 * maxRecsWhenFull / 100; // pad some slack } else { avgRecSize = 0; maxRecsWhenFull = kMaxObjectPerChunk + 1; diff --git a/src/mongo/db/s/migration_chunk_cloner_source_legacy.h b/src/mongo/db/s/migration_chunk_cloner_source_legacy.h index c683df2be29..860282a16d0 100644 --- a/src/mongo/db/s/migration_chunk_cloner_source_legacy.h +++ b/src/mongo/db/s/migration_chunk_cloner_source_legacy.h @@ -66,7 +66,7 @@ public: Status awaitUntilCriticalSectionIsAppropriate(OperationContext* txn, Milliseconds maxTimeToWait) override; - Status commitClone(OperationContext* txn) override; + StatusWith commitClone(OperationContext* txn) override; void cancelClone(OperationContext* txn) override; diff --git a/src/mongo/db/s/migration_destination_manager.cpp b/src/mongo/db/s/migration_destination_manager.cpp index 2e748cf3ca9..1c02208ee27 100644 --- a/src/mongo/db/s/migration_destination_manager.cpp +++ b/src/mongo/db/s/migration_destination_manager.cpp @@ -148,10 +148,14 @@ bool opReplicatedEnough(OperationContext* txn, const repl::OpTime& lastOpApplied, const WriteConcernOptions& writeConcern) { WriteConcernResult writeConcernResult; + writeConcernResult.wTimedOut = false; - Status waitForMajorityWriteConcernStatus = + Status majorityStatus = waitForWriteConcern(txn, lastOpApplied, kMajorityWriteConcern, &writeConcernResult); - if (!waitForMajorityWriteConcernStatus.isOK()) { + if (!majorityStatus.isOK()) { + if (!writeConcernResult.wTimedOut) { + uassertStatusOK(majorityStatus); + } return false; } @@ -159,13 +163,16 @@ bool opReplicatedEnough(OperationContext* txn, // write concerns in case the user's write concern is stronger than majority WriteConcernOptions userWriteConcern(writeConcern); userWriteConcern.wTimeout = -1; + writeConcernResult.wTimedOut = false; - Status waitForUserWriteConcernStatus = + Status userStatus = waitForWriteConcern(txn, lastOpApplied, userWriteConcern, &writeConcernResult); - if (!waitForUserWriteConcernStatus.isOK()) { + if (!userStatus.isOK()) { + if (!writeConcernResult.wTimedOut) { + uassertStatusOK(userStatus); + } return false; } - return true; } @@ -220,6 +227,7 @@ MigrationDestinationManager::State MigrationDestinationManager::getState() const void MigrationDestinationManager::setState(State newState) { stdx::lock_guard sl(_mutex); _state = newState; + _stateChangedCV.notify_all(); } bool MigrationDestinationManager::isActive() const { @@ -231,7 +239,21 @@ bool MigrationDestinationManager::_isActive_inlock() const { return _sessionId.is_initialized(); } -void MigrationDestinationManager::report(BSONObjBuilder& b) { +void MigrationDestinationManager::report(BSONObjBuilder& b, + OperationContext* opCtx, + bool waitForSteadyOrDone) { + if (waitForSteadyOrDone) { + stdx::unique_lock lock(_mutex); + try { + opCtx->waitForConditionOrInterruptFor(_stateChangedCV, lock, Seconds(1), [&]() -> bool { + return _state != READY && _state != CLONE && _state != CATCHUP; + }); + } catch (...) { + // Ignoring this error because this is an optional parameter and we catch timeout + // exceptions later. + } + b.append("waited", true); + } stdx::lock_guard sl(_mutex); b.appendBool("active", _sessionId.is_initialized()); @@ -286,6 +308,7 @@ Status MigrationDestinationManager::start(const NamespaceString& nss, invariant(!_scopedRegisterReceiveChunk); _state = READY; + _stateChangedCV.notify_all(); _errmsg = ""; _nss = nss; @@ -336,6 +359,7 @@ bool MigrationDestinationManager::abort(const MigrationSessionId& sessionId) { } _state = ABORT; + _stateChangedCV.notify_all(); _errmsg = "aborted"; return true; @@ -344,6 +368,7 @@ bool MigrationDestinationManager::abort(const MigrationSessionId& sessionId) { void MigrationDestinationManager::abortWithoutSessionIdCheck() { stdx::lock_guard sl(_mutex); _state = ABORT; + _stateChangedCV.notify_all(); _errmsg = "aborted without session id check"; } @@ -368,6 +393,7 @@ bool MigrationDestinationManager::startCommit(const MigrationSessionId& sessionI } _state = COMMIT_START; + _stateChangedCV.notify_all(); const auto deadline = Date_t::now() + Seconds(30); @@ -375,6 +401,8 @@ bool MigrationDestinationManager::startCommit(const MigrationSessionId& sessionI if (stdx::cv_status::timeout == _isActiveCV.wait_until(lock, deadline.toSystemTimePoint())) { _state = FAIL; + _stateChangedCV.notify_all(); + log() << "startCommit never finished!" << migrateLog; return false; } diff --git a/src/mongo/db/s/migration_destination_manager.h b/src/mongo/db/s/migration_destination_manager.h index 0b16202d55e..4bd269f37fd 100644 --- a/src/mongo/db/s/migration_destination_manager.h +++ b/src/mongo/db/s/migration_destination_manager.h @@ -77,7 +77,7 @@ public: /** * Reports the state of the migration manager as a BSON document. */ - void report(BSONObjBuilder& b); + void report(BSONObjBuilder& b, OperationContext* opCtx, bool waitForSteadyOrDone); /** * Returns a report on the active migration, if the migration is active. Otherwise return an @@ -222,6 +222,9 @@ private: State _state{READY}; std::string _errmsg; + + // Condition variable, which is signalled every time the state of the migration changes. + stdx::condition_variable _stateChangedCV; }; } // namespace mongo diff --git a/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp b/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp index a20db670259..ac9752fd96d 100644 --- a/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp +++ b/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp @@ -220,7 +220,9 @@ public: int, string& errmsg, BSONObjBuilder& result) { - ShardingState::get(txn)->migrationDestinationManager()->report(result); + bool waitForSteadyOrDone = cmdObj["waitForSteadyOrDone"].boolean(); + ShardingState::get(txn)->migrationDestinationManager()->report( + result, txn, waitForSteadyOrDone); return true; } @@ -261,12 +263,13 @@ public: int, string& errmsg, BSONObjBuilder& result) { + const MigrationSessionId migrationSessionid( uassertStatusOK(MigrationSessionId::extractFromBSON(cmdObj))); const bool ok = ShardingState::get(txn)->migrationDestinationManager()->startCommit(migrationSessionid); - ShardingState::get(txn)->migrationDestinationManager()->report(result); + ShardingState::get(txn)->migrationDestinationManager()->report(result, txn, false); return ok; } @@ -313,11 +316,11 @@ public: if (migrationSessionIdStatus.isOK()) { const bool ok = mdm->abort(migrationSessionIdStatus.getValue()); - mdm->report(result); + mdm->report(result, txn, false); return ok; } else if (migrationSessionIdStatus == ErrorCodes::NoSuchKey) { mdm->abortWithoutSessionIdCheck(); - mdm->report(result); + mdm->report(result, txn, false); return true; } diff --git a/src/mongo/db/s/migration_source_manager.cpp b/src/mongo/db/s/migration_source_manager.cpp index 0ad841ea56f..b70617deee3 100644 --- a/src/mongo/db/s/migration_source_manager.cpp +++ b/src/mongo/db/s/migration_source_manager.cpp @@ -268,18 +268,18 @@ Status MigrationSourceManager::commitChunkOnRecipient(OperationContext* txn) { auto scopedGuard = MakeGuard([&] { cleanupOnError(txn); }); // Tell the recipient shard to fetch the latest changes. - Status commitCloneStatus = _cloneDriver->commitClone(txn); + auto commitCloneStatus = _cloneDriver->commitClone(txn); if (MONGO_FAIL_POINT(failMigrationCommit) && commitCloneStatus.isOK()) { commitCloneStatus = {ErrorCodes::InternalError, "Failing _recvChunkCommit due to failpoint."}; } - if (!commitCloneStatus.isOK()) { - return {commitCloneStatus.code(), - str::stream() << "commit clone failed due to " << commitCloneStatus.toString()}; + return commitCloneStatus.getStatus(); } + _recipientCloneCounts = commitCloneStatus.getValue()["counts"].Obj().getOwned(); + _state = kCloneCompleted; scopedGuard.Dismiss(); return Status::OK(); @@ -332,9 +332,13 @@ Status MigrationSourceManager::commitChunkMetadataOnConfig(OperationContext* txn ErrorCodes::InternalError, "Failpoint 'migrationCommitNetworkError' generated error"); } - const Status migrationCommitStatus = - (commitChunkMigrationResponse.isOK() ? commitChunkMigrationResponse.getValue().commandStatus - : commitChunkMigrationResponse.getStatus()); + Status migrationCommitStatus = commitChunkMigrationResponse.getStatus(); + if (migrationCommitStatus.isOK()) { + migrationCommitStatus = commitChunkMigrationResponse.getValue().commandStatus; + if (migrationCommitStatus.isOK()) { + migrationCommitStatus = commitChunkMigrationResponse.getValue().writeConcernStatus; + } + } if (!migrationCommitStatus.isOK()) { // Need to get the latest optime in case the refresh request goes to a secondary -- @@ -431,7 +435,9 @@ Status MigrationSourceManager::commitChunkMetadataOnConfig(OperationContext* txn << "from" << _args.getFromShardId() << "to" - << _args.getToShardId()), + << _args.getToShardId() + << "counts" + << _recipientCloneCounts), ShardingCatalogClient::kMajorityWriteConcern); return Status::OK(); diff --git a/src/mongo/db/s/migration_source_manager.h b/src/mongo/db/s/migration_source_manager.h index cb5ce4be792..27d4480a665 100644 --- a/src/mongo/db/s/migration_source_manager.h +++ b/src/mongo/db/s/migration_source_manager.h @@ -233,6 +233,9 @@ private: // callers don't have to hold collection lock in order to wait on it. Available after the // critical section stage has completed. std::shared_ptr> _critSecSignal; + + // The statistics about a chunk migration to be included in moveChunk.commit + BSONObj _recipientCloneCounts; }; } // namespace mongo diff --git a/src/mongo/db/s/sharding_state.cpp b/src/mongo/db/s/sharding_state.cpp index e264c82b57f..02c75e287fe 100644 --- a/src/mongo/db/s/sharding_state.cpp +++ b/src/mongo/db/s/sharding_state.cpp @@ -117,8 +117,7 @@ const std::set ShardingState::_commandsThatInitializeShardingAwaren ShardingState::ShardingState() : _initializationState(static_cast(InitializationState::kNew)), _initializationStatus(Status(ErrorCodes::InternalError, "Uninitialized value")), - _globalInit(&initializeGlobalShardingStateForMongod), - _scheduleWorkFn([](NamespaceString nss) {}) {} + _globalInit(&initializeGlobalShardingStateForMongod) {} ShardingState::~ShardingState() = default; @@ -222,14 +221,6 @@ void ShardingState::setGlobalInitMethodForTest(GlobalInitFunc func) { _globalInit = func; } -void ShardingState::setScheduleCleanupFunctionForTest(RangeDeleterCleanupNotificationFunc fn) { - _scheduleWorkFn = fn; -} - -void ShardingState::scheduleCleanup(const NamespaceString& nss) { - _scheduleWorkFn(nss); -} - Status ShardingState::onStaleShardVersion(OperationContext* txn, const NamespaceString& nss, const ChunkVersion& expectedVersion) { @@ -416,8 +407,6 @@ Status ShardingState::initializeFromShardIdentity(OperationContext* txn, _shardName = shardIdentity.getShardName(); _clusterId = shardIdentity.getClusterId(); - _initializeRangeDeleterTaskExecutor(); - return status; } catch (const DBException& ex) { auto errorStatus = ex.toStatus(); @@ -446,8 +435,6 @@ void ShardingState::_initializeImpl(ConnectionString configSvr, string shardName &ShardRegistry::replicaSetChangeShardRegistryUpdateHook); ReplicaSetMonitor::setAsynchronousConfigChangeHook(&updateShardIdentityConfigStringCB); - _initializeRangeDeleterTaskExecutor(); - _shardName = shardName; } @@ -626,47 +613,76 @@ ChunkVersion ShardingState::_refreshMetadata(OperationContext* txn, const Namesp << " before shard name has been set", shardId.isValid()); - auto newCollectionMetadata = [&]() -> std::unique_ptr { - auto const catalogCache = Grid::get(txn)->catalogCache(); - catalogCache->invalidateShardedCollection(nss); + auto const catalogCache = Grid::get(txn)->catalogCache(); + catalogCache->invalidateShardedCollection(nss); - const auto routingInfo = uassertStatusOK(catalogCache->getCollectionRoutingInfo(txn, nss)); - const auto cm = routingInfo.cm(); - if (!cm) { - return nullptr; - } + auto routingInfo = uassertStatusOK(catalogCache->getCollectionRoutingInfo(txn, nss)); + const auto cm = routingInfo.cm(); - RangeMap shardChunksMap = - SimpleBSONObjComparator::kInstance.makeBSONObjIndexedMap(); + if (!cm) { + // No chunk manager, so unsharded. - for (const auto& chunkMapEntry : cm->chunkMap()) { - const auto& chunk = chunkMapEntry.second; + // Exclusive collection lock needed since we're now changing the metadata + ScopedTransaction transaction(txn, MODE_IX); + AutoGetCollection autoColl(txn, nss, MODE_IX, MODE_X); - if (chunk->getShardId() != shardId) - continue; + auto css = CollectionShardingState::get(txn, nss); + css->refreshMetadata(txn, nullptr); - shardChunksMap.emplace_hint(shardChunksMap.end(), - chunk->getMin(), - CachedChunkInfo(chunk->getMax(), chunk->getLastmod())); - } + return ChunkVersion::UNSHARDED(); + } - return stdx::make_unique(cm->getShardKeyPattern().toBSON(), - cm->getVersion(), - cm->getVersion(shardId), - std::move(shardChunksMap)); - }(); + { + AutoGetCollection autoColl(txn, nss, MODE_IS); + auto css = CollectionShardingState::get(txn, nss); + + // We already have newer version + if (css->getMetadata() && + css->getMetadata()->getCollVersion().epoch() == cm->getVersion().epoch() && + css->getMetadata()->getCollVersion() >= cm->getVersion()) { + LOG(1) << "Skipping refresh of metadata for " << nss << " " + << css->getMetadata()->getCollVersion() << " with an older " << cm->getVersion(); + return css->getMetadata()->getShardVersion(); + } + } // Exclusive collection lock needed since we're now changing the metadata ScopedTransaction transaction(txn, MODE_IX); AutoGetCollection autoColl(txn, nss, MODE_IX, MODE_X); auto css = CollectionShardingState::get(txn, nss); - css->refreshMetadata(txn, std::move(newCollectionMetadata)); - if (!css->getMetadata()) { - return ChunkVersion::UNSHARDED(); + // We already have newer version + if (css->getMetadata() && + css->getMetadata()->getCollVersion().epoch() == cm->getVersion().epoch() && + css->getMetadata()->getCollVersion() >= cm->getVersion()) { + LOG(1) << "Skipping refresh of metadata for " << nss << " " + << css->getMetadata()->getCollVersion() << " with an older " << cm->getVersion(); + return css->getMetadata()->getShardVersion(); + } + + RangeMap shardChunksMap = + SimpleBSONObjComparator::kInstance.makeBSONObjIndexedMap(); + + for (const auto& chunkMapEntry : cm->chunkMap()) { + const auto& chunk = chunkMapEntry.second; + + if (chunk->getShardId() != shardId) + continue; + + shardChunksMap.emplace_hint(shardChunksMap.end(), + chunk->getMin(), + CachedChunkInfo(chunk->getMax(), chunk->getLastmod())); } + std::unique_ptr newCollectionMetadata = + stdx::make_unique(cm->getShardKeyPattern().toBSON(), + cm->getVersion(), + cm->getVersion(shardId), + std::move(shardChunksMap)); + + css->refreshMetadata(txn, std::move(newCollectionMetadata)); + return css->getMetadata()->getShardVersion(); } @@ -756,19 +772,6 @@ Status ShardingState::updateShardIdentityConfigString(OperationContext* txn, return Status::OK(); } -void ShardingState::_initializeRangeDeleterTaskExecutor() { - invariant(!_rangeDeleterTaskExecutor); - auto net = - executor::makeNetworkInterface("NetworkInterfaceCollectionRangeDeleter-TaskExecutor"); - auto netPtr = net.get(); - _rangeDeleterTaskExecutor = stdx::make_unique( - stdx::make_unique(netPtr), std::move(net)); -} - -executor::ThreadPoolTaskExecutor* ShardingState::getRangeDeleterTaskExecutor() { - return _rangeDeleterTaskExecutor.get(); -} - /** * Global free function. */ diff --git a/src/mongo/db/s/sharding_state.h b/src/mongo/db/s/sharding_state.h index e53fee63515..f9e5a507446 100644 --- a/src/mongo/db/s/sharding_state.h +++ b/src/mongo/db/s/sharding_state.h @@ -35,7 +35,6 @@ #include "mongo/bson/oid.h" #include "mongo/db/namespace_string.h" #include "mongo/db/s/active_migrations_registry.h" -#include "mongo/db/s/collection_range_deleter.h" #include "mongo/db/s/migration_destination_manager.h" #include "mongo/executor/task_executor.h" #include "mongo/executor/thread_pool_task_executor.h" @@ -235,24 +234,6 @@ public: */ void setGlobalInitMethodForTest(GlobalInitFunc func); - /** - * Schedules for the range to clean of the given namespace to be deleted. - * Behavior can be modified through setScheduleCleanupFunctionForTest. - */ - void scheduleCleanup(const NamespaceString& nss); - - /** - * Returns a pointer to the collection range deleter task executor. - */ - executor::ThreadPoolTaskExecutor* getRangeDeleterTaskExecutor(); - - /** - * Sets the function used by scheduleWorkOnRangeDeleterTaskExecutor to - * schedule work. Used for mocking the executor for testing. See the ShardingState - * for the default implementation of _scheduleWorkFn. - */ - void setScheduleCleanupFunctionForTest(RangeDeleterCleanupNotificationFunc fn); - /** * If started with --shardsvr, initializes sharding awareness from the shardIdentity document * on disk, if there is one. @@ -348,9 +329,6 @@ private: */ ChunkVersion _refreshMetadata(OperationContext* opCtx, const NamespaceString& nss); - // Initializes a TaskExecutor for cleaning up orphaned ranges - void _initializeRangeDeleterTaskExecutor(); - // Manages the state of the migration recipient shard MigrationDestinationManager _migrationDestManager; @@ -386,13 +364,6 @@ private: // Function for initializing the external sharding state components not owned here. GlobalInitFunc _globalInit; - - // Function for scheduling work on the _rangeDeleterTaskExecutor. - // Used in call to scheduleCleanup(NamespaceString). - RangeDeleterCleanupNotificationFunc _scheduleWorkFn; - - // Task executor for the collection range deleter. - std::unique_ptr _rangeDeleterTaskExecutor; }; } // namespace mongo diff --git a/src/mongo/db/sorter/SConscript b/src/mongo/db/sorter/SConscript index 959f6cdb919..1ceb05088af 100644 --- a/src/mongo/db/sorter/SConscript +++ b/src/mongo/db/sorter/SConscript @@ -5,6 +5,6 @@ sorterEnv.InjectThirdPartyIncludePaths(libraries=['snappy']) sorterEnv.CppUnitTest('sorter_test', 'sorter_test.cpp', LIBDEPS=['$BUILD_DIR/mongo/db/service_context', - '$BUILD_DIR/mongo/db/storage/wiredtiger/storage_wiredtiger_customization_hooks', + '$BUILD_DIR/mongo/db/storage/encryption_hooks', '$BUILD_DIR/mongo/db/storage/storage_options', '$BUILD_DIR/third_party/shim_snappy']) diff --git a/src/mongo/db/sorter/sorter.cpp b/src/mongo/db/sorter/sorter.cpp index 7f6e8866562..0165e0adccc 100644 --- a/src/mongo/db/sorter/sorter.cpp +++ b/src/mongo/db/sorter/sorter.cpp @@ -55,8 +55,8 @@ #include "mongo/config.h" #include "mongo/db/jsobj.h" #include "mongo/db/service_context.h" +#include "mongo/db/storage/encryption_hooks.h" #include "mongo/db/storage/storage_options.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h" #include "mongo/platform/atomic_word.h" #include "mongo/s/mongos_options.h" #include "mongo/util/assert_util.h" @@ -226,15 +226,16 @@ private: read(_buffer.get(), blockSize); massert(16816, "file too short?", !_done); - auto hooks = WiredTigerCustomizationHooks::get(getGlobalServiceContext()); - if (hooks->enabled()) { + auto encryptionHooks = EncryptionHooks::get(getGlobalServiceContext()); + if (encryptionHooks->enabled()) { std::unique_ptr out(new char[blockSize]); size_t outLen; - Status status = hooks->unprotectTmpData(reinterpret_cast(_buffer.get()), - blockSize, - reinterpret_cast(out.get()), - blockSize, - &outLen); + Status status = + encryptionHooks->unprotectTmpData(reinterpret_cast(_buffer.get()), + blockSize, + reinterpret_cast(out.get()), + blockSize, + &outLen); massert(28841, str::stream() << "Failed to unprotect data: " << status.toString(), status.isOK()); @@ -884,16 +885,16 @@ void SortedFileWriter::spill() { } std::unique_ptr out; - auto hooks = WiredTigerCustomizationHooks::get(getGlobalServiceContext()); - if (hooks->enabled()) { - size_t protectedSizeMax = size + hooks->additionalBytesForProtectedBuffer(); + auto encryptionHooks = EncryptionHooks::get(getGlobalServiceContext()); + if (encryptionHooks->enabled()) { + size_t protectedSizeMax = size + encryptionHooks->additionalBytesForProtectedBuffer(); out.reset(new char[protectedSizeMax]); size_t resultLen; - Status status = hooks->protectTmpData(reinterpret_cast(outBuffer), - size, - reinterpret_cast(out.get()), - protectedSizeMax, - &resultLen); + Status status = encryptionHooks->protectTmpData(reinterpret_cast(outBuffer), + size, + reinterpret_cast(out.get()), + protectedSizeMax, + &resultLen); massert(28842, str::stream() << "Failed to compress data: " << status.toString(), status.isOK()); diff --git a/src/mongo/db/stats/range_deleter_server_status.cpp b/src/mongo/db/stats/range_deleter_server_status.cpp index 817ffa444e4..e37143342f0 100644 --- a/src/mongo/db/stats/range_deleter_server_status.cpp +++ b/src/mongo/db/stats/range_deleter_server_status.cpp @@ -84,6 +84,11 @@ public: entryBuilder.append("deleteStart", (*it)->deleteStartTS); entryBuilder.append("deleteEnd", (*it)->deleteEndTS); + const auto waitForReplDurationMs = + durationCount((*it)->waitForReplDurationMs); + if (waitForReplDurationMs > 0) { + entryBuilder.append("waitForReplDurationMs", waitForReplDurationMs); + } if ((*it)->waitForReplEndTS > Date_t()) { entryBuilder.append("waitForReplStart", (*it)->waitForReplStartTS); entryBuilder.append("waitForReplEnd", (*it)->waitForReplEndTS); diff --git a/src/mongo/db/storage/SConscript b/src/mongo/db/storage/SConscript index d65046ed6fa..e887882df87 100644 --- a/src/mongo/db/storage/SConscript +++ b/src/mongo/db/storage/SConscript @@ -11,7 +11,6 @@ env.SConscript( ], ) - env.Library( target='journal_listener', source=[ @@ -41,7 +40,6 @@ env.Library( ], ) - env.Library( target='bson_collection_catalog_entry', source=[ @@ -74,6 +72,19 @@ env.Library( ], ) +env.Library( + target='encryption_hooks', + source= [ + 'encryption_hooks.cpp', + ], + LIBDEPS= ['$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/db/service_context'], + PROGDEPS_DEPENDENTS=[ + '$BUILD_DIR/mongo/mongod', + '$BUILD_DIR/mongo/mongos', + ], + ) + env.Library( target='storage_options', source=[ diff --git a/src/mongo/db/storage/encryption_hooks.cpp b/src/mongo/db/storage/encryption_hooks.cpp new file mode 100644 index 00000000000..9ee9a317dbe --- /dev/null +++ b/src/mongo/db/storage/encryption_hooks.cpp @@ -0,0 +1,96 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/storage/encryption_hooks.h" + +#include + +#include "mongo/base/init.h" +#include "mongo/db/service_context.h" +#include "mongo/db/storage/data_protector.h" +#include "mongo/stdx/memory.h" + +namespace mongo { + +/* Make a EncryptionHooks pointer a decoration on the global ServiceContext */ +MONGO_INITIALIZER_WITH_PREREQUISITES(SetEncryptionHooks, ("SetGlobalEnvironment")) +(InitializerContext* context) { + auto encryptionHooks = stdx::make_unique(); + EncryptionHooks::set(getGlobalServiceContext(), std::move(encryptionHooks)); + + return Status::OK(); +} + +namespace { +const auto getEncryptionHooks = + ServiceContext::declareDecoration>(); +} // namespace + +void EncryptionHooks::set(ServiceContext* service, std::unique_ptr custHooks) { + auto& hooks = getEncryptionHooks(service); + invariant(custHooks); + hooks = std::move(custHooks); +} + +EncryptionHooks* EncryptionHooks::get(ServiceContext* service) { + return getEncryptionHooks(service).get(); +} + +EncryptionHooks::~EncryptionHooks() {} + +bool EncryptionHooks::enabled() const { + return false; +} + +bool EncryptionHooks::restartRequired() { + return false; +} + +std::unique_ptr EncryptionHooks::getDataProtector() { + return std::unique_ptr(); +} + +boost::filesystem::path EncryptionHooks::getProtectedPathSuffix() { + return ""; +} + +Status EncryptionHooks::protectTmpData( + const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) { + return Status(ErrorCodes::InternalError, + "Encryption hooks must be enabled to use preprocessTmpData."); +} + +Status EncryptionHooks::unprotectTmpData( + const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) { + return Status(ErrorCodes::InternalError, + "Encryption hooks must be enabled to use postprocessTmpData."); +} +} // namespace mongo diff --git a/src/mongo/db/storage/encryption_hooks.h b/src/mongo/db/storage/encryption_hooks.h new file mode 100644 index 00000000000..e1c9d553a10 --- /dev/null +++ b/src/mongo/db/storage/encryption_hooks.h @@ -0,0 +1,99 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +#include "mongo/base/disallow_copying.h" +#include "mongo/db/jsobj.h" + +namespace boost { +namespace filesystem { +class path; +} // namespace filesystem +} // namespace boost + +namespace mongo { +class DataProtector; +class ServiceContext; + +class EncryptionHooks { +public: + static void set(ServiceContext* service, std::unique_ptr custHooks); + + static EncryptionHooks* get(ServiceContext* service); + + virtual ~EncryptionHooks(); + + /** + * Returns true if the encryption hooks are enabled. + */ + virtual bool enabled() const; + + /** + * Perform any encryption engine initialization/sanity checking that needs to happen after + * storage engine initialization but before the server starts accepting incoming connections. + * + * Returns true if the server needs to be rebooted because of configuration changes. + */ + virtual bool restartRequired(); + + /** + * Returns the maximum size addition when doing transforming temp data. + */ + size_t additionalBytesForProtectedBuffer() { + return 33; + } + + /** + * Get the data protector object + */ + virtual std::unique_ptr getDataProtector(); + + /** + * Get an implementation specific path suffix to tag files with + */ + virtual boost::filesystem::path getProtectedPathSuffix(); + + /** + * Transform temp data to non-readable form before writing it to disk. + */ + virtual Status protectTmpData( + const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen); + + /** + * Tranforms temp data back to readable form, after reading from disk. + */ + virtual Status unprotectTmpData( + const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen); +}; + +} // namespace mongo diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.cpp b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.cpp index a537a9c63c6..8c816d39f31 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.cpp +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.cpp @@ -556,8 +556,15 @@ void EphemeralForTestRecordStore::temp_cappedTruncateAfter(OperationContext* txn Records::iterator it = inclusive ? _data->records.lower_bound(end) : _data->records.upper_bound(end); while (it != _data->records.end()) { - txn->recoveryUnit()->registerChange(new RemoveChange(txn, _data, it->first, it->second)); - _data->dataSize -= it->second.size; + RecordId id = it->first; + EphemeralForTestRecord record = it->second; + + if (_cappedCallback) { + uassertStatusOK(_cappedCallback->aboutToDeleteCapped(txn, id, record.toRecordData())); + } + + txn->recoveryUnit()->registerChange(new RemoveChange(txn, _data, id, record)); + _data->dataSize -= record.size; _data->records.erase(it++); } } diff --git a/src/mongo/db/storage/key_string.cpp b/src/mongo/db/storage/key_string.cpp index 624c87a2404..fc580ca111d 100644 --- a/src/mongo/db/storage/key_string.cpp +++ b/src/mongo/db/storage/key_string.cpp @@ -1366,7 +1366,8 @@ void toBsonValue(uint8_t ctype, Decimal128 dec(Decimal128::Value{lowbits, highbits}); if (isNegative) dec = dec.negate(); - dec = adjustDecimalExponent(typeBits, dec); + if (dec.isFinite()) + dec = adjustDecimalExponent(typeBits, dec); *stream << dec; break; } @@ -1385,7 +1386,8 @@ void toBsonValue(uint8_t ctype, Decimal128 dec(bin, Decimal128::kRoundTo34Digits, roundAwayFromZero); if (hasDecimalContinuation) dec = readDecimalContinuation(reader, inverted, dec); - dec = adjustDecimalExponent(typeBits, dec); + if (dec.isFinite()) + dec = adjustDecimalExponent(typeBits, dec); *stream << dec; } break; diff --git a/src/mongo/db/storage/key_string_test.cpp b/src/mongo/db/storage/key_string_test.cpp index b29d1b01da8..46a11c2592a 100644 --- a/src/mongo/db/storage/key_string_test.cpp +++ b/src/mongo/db/storage/key_string_test.cpp @@ -596,6 +596,15 @@ const std::vector& getInterestingElements(KeyString::Version version) { // Something that needs multiple bytes of typeBits elements.push_back(BSON("" << BSON_ARRAY("" << BSONSymbol("") << 0 << 0ll << 0.0 << -0.0))); + if (version != KeyString::Version::V0) { + // Something with exceptional typeBits for Decimal + elements.push_back( + BSON("" << BSON_ARRAY("" << BSONSymbol("") << Decimal128::kNegativeInfinity + << Decimal128::kPositiveInfinity + << Decimal128::kPositiveNaN + << Decimal128("0.0000000") + << Decimal128("-0E1000")))); + } // // Interesting numeric cases @@ -605,6 +614,11 @@ const std::vector& getInterestingElements(KeyString::Version version) { elements.push_back(BSON("" << 0ll)); elements.push_back(BSON("" << 0.0)); elements.push_back(BSON("" << -0.0)); + if (version != KeyString::Version::V0) { + Decimal128("0.0.0000000"); + Decimal128("-0E1000"); + } + elements.push_back(BSON("" << std::numeric_limits::quiet_NaN())); elements.push_back(BSON("" << std::numeric_limits::infinity())); elements.push_back(BSON("" << -std::numeric_limits::infinity())); @@ -740,6 +754,11 @@ const std::vector& getInterestingElements(KeyString::Version version) { elements.push_back(BSON("" << Decimal128("4.940656458412465441765687928682214E-324"))); elements.push_back(BSON("" << Decimal128("-4.940656458412465441765687928682214E-324"))); elements.push_back(BSON("" << Decimal128("-4.940656458412465441765687928682213E-324"))); + + // Non-finite values. Note: can't roundtrip negative NaNs, so not testing here. + elements.push_back(BSON("" << Decimal128::kPositiveNaN)); + elements.push_back(BSON("" << Decimal128::kNegativeInfinity)); + elements.push_back(BSON("" << Decimal128::kPositiveInfinity)); } // Tricky double precision number for binary/decimal conversion: very close to a decimal @@ -823,8 +842,36 @@ void testPermutation(KeyString::Version version, } } +namespace { +std::random_device rd; +std::mt19937_64 seedGen(rd()); + +// To be used by perf test for seeding, so that the entire test is repeatable in case of error. +unsigned newSeed() { + unsigned int seed = seedGen(); // Replace by the reported number to repeat test execution. + log() << "Initializing random number generator using seed " << seed; + return seed; +}; + +std::vector thinElements(std::vector elements, + unsigned seed, + size_t maxElements) { + std::mt19937_64 gen(seed); + + if (elements.size() <= maxElements) + return elements; + + log() << "only keeping " << maxElements << " of " << elements.size() + << " elements using random selection"; + std::shuffle(elements.begin(), elements.end(), gen); + elements.resize(maxElements); + return elements; +} +} // namespace + + TEST_F(KeyStringTest, AllPermCompare) { - const std::vector& elements = getInterestingElements(version); + std::vector elements = getInterestingElements(version); for (size_t i = 0; i < elements.size(); i++) { const BSONObj& o = elements[i]; @@ -839,19 +886,23 @@ TEST_F(KeyStringTest, AllPermCompare) { } TEST_F(KeyStringTest, AllPerm2Compare) { -#if !defined(MONGO_CONFIG_OPTIMIZED_BUILD) - log() << "\t\t\tskipping permutation testing on non-optimized build"; - return; -#endif + std::vector baseElements = getInterestingElements(version); + auto seed = newSeed(); - const std::vector& baseElements = getInterestingElements(version); + // Select only a small subset of elements, as the combination is quadratic. + // We want to select two subsets independently, so all combinations will get tested eventually. + // kMaxPermElements is the desired number of elements to pass to testPermutation. + const size_t kMaxPermElements = kDebugBuild ? 100000 : 500000; + size_t maxElements = sqrt(kMaxPermElements); + auto firstElements = thinElements(baseElements, seed, maxElements); + auto secondElements = thinElements(baseElements, seed + 1, maxElements); std::vector elements; - for (size_t i = 0; i < baseElements.size(); i++) { - for (size_t j = 0; j < baseElements.size(); j++) { + for (size_t i = 0; i < firstElements.size(); i++) { + for (size_t j = 0; j < secondElements.size(); j++) { BSONObjBuilder b; - b.appendElements(baseElements[i]); - b.appendElements(baseElements[j]); + b.appendElements(firstElements[i]); + b.appendElements(secondElements[j]); BSONObj o = b.obj(); elements.push_back(o); } @@ -927,6 +978,27 @@ TEST_F(KeyStringTest, NaNs) { ASSERT(std::isnan(toBson(ks2a, ONE_ASCENDING)[""].Double())); ASSERT(std::isnan(toBson(ks1d, ONE_DESCENDING)[""].Double())); ASSERT(std::isnan(toBson(ks2d, ONE_DESCENDING)[""].Double())); + + if (version == KeyString::Version::V0) + return; + + const auto nan3 = Decimal128::kPositiveNaN; + const auto nan4 = Decimal128::kNegativeNaN; + // Since we only output a single NaN, we can only do ROUNDTRIP testing for nan1. + ROUNDTRIP(version, BSON("" << nan3)); + const KeyString ks3a(version, BSON("" << nan3), ONE_ASCENDING); + const KeyString ks3d(version, BSON("" << nan3), ONE_DESCENDING); + + const KeyString ks4a(version, BSON("" << nan4), ONE_ASCENDING); + const KeyString ks4d(version, BSON("" << nan4), ONE_DESCENDING); + + ASSERT_EQ(ks1a, ks4a); + ASSERT_EQ(ks1d, ks4d); + + ASSERT(toBson(ks3a, ONE_ASCENDING)[""].Decimal().isNaN()); + ASSERT(toBson(ks4a, ONE_ASCENDING)[""].Decimal().isNaN()); + ASSERT(toBson(ks3d, ONE_DESCENDING)[""].Decimal().isNaN()); + ASSERT(toBson(ks4d, ONE_DESCENDING)[""].Decimal().isNaN()); } TEST_F(KeyStringTest, NumberOrderLots) { std::vector numbers; @@ -1074,16 +1146,6 @@ const uint64_t kMinPerfMicros = 20 * 1000; const uint64_t kMinPerfSamples = 50 * 1000; typedef std::vector Numbers; -std::random_device rd; -std::mt19937 seedGen(rd()); - -// To be used by perf test for seeding, so that the entire test is repeatable in case of error. -unsigned newSeed() { - unsigned int seed = seedGen(); // Replace by the reported number to repeat test execution. - log() << "Initializing random number generator using seed " << seed; - return seed; -}; - /** * Evaluates ROUNDTRIP on all items in Numbers a sufficient number of times to take at least * kMinPerfMicros microseconds. Logs the elapsed time per ROUNDTRIP evaluation. diff --git a/src/mongo/db/storage/mmap_v1/mmap_v1_engine.cpp b/src/mongo/db/storage/mmap_v1/mmap_v1_engine.cpp index e185afe03c7..71b3ada3ed4 100644 --- a/src/mongo/db/storage/mmap_v1/mmap_v1_engine.cpp +++ b/src/mongo/db/storage/mmap_v1/mmap_v1_engine.cpp @@ -36,6 +36,10 @@ #include #include +#ifdef __linux__ +#include +#endif + #include "mongo/db/mongod_options.h" #include "mongo/db/storage/mmap_v1/data_file_sync.h" #include "mongo/db/storage/mmap_v1/dur.h" diff --git a/src/mongo/db/storage/wiredtiger/SConscript b/src/mongo/db/storage/wiredtiger/SConscript index 9d2a8fc8035..42eb3b4d41d 100644 --- a/src/mongo/db/storage/wiredtiger/SConscript +++ b/src/mongo/db/storage/wiredtiger/SConscript @@ -82,6 +82,7 @@ if wiredtiger: ], LIBDEPS=['storage_wiredtiger_core', 'storage_wiredtiger_customization_hooks', + '$BUILD_DIR/mongo/db/concurrency/lock_manager', '$BUILD_DIR/mongo/db/storage/kv/kv_engine', '$BUILD_DIR/mongo/db/storage/storage_engine_lock_file', '$BUILD_DIR/mongo/db/storage/storage_engine_metadata', diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.cpp index d14b33de5f7..e40d3a58edb 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.cpp @@ -31,12 +31,9 @@ #include "mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h" -#include - #include "mongo/base/init.h" #include "mongo/base/string_data.h" #include "mongo/db/service_context.h" -#include "mongo/db/storage/data_protector.h" #include "mongo/stdx/memory.h" namespace mongo { @@ -44,7 +41,7 @@ namespace mongo { /* Make a WiredTigerCustomizationHooks pointer a decoration on the global ServiceContext */ MONGO_INITIALIZER_WITH_PREREQUISITES(SetWiredTigerCustomizationHooks, ("SetGlobalEnvironment")) (InitializerContext* context) { - auto customizationHooks = stdx::make_unique(); + auto customizationHooks = stdx::make_unique(); WiredTigerCustomizationHooks::set(getGlobalServiceContext(), std::move(customizationHooks)); return Status::OK(); @@ -66,37 +63,14 @@ WiredTigerCustomizationHooks* WiredTigerCustomizationHooks::get(ServiceContext* return getCustomizationHooks(service).get(); } -EmptyWiredTigerCustomizationHooks::~EmptyWiredTigerCustomizationHooks() {} - -bool EmptyWiredTigerCustomizationHooks::enabled() const { - return false; -} +WiredTigerCustomizationHooks::~WiredTigerCustomizationHooks() {} -bool EmptyWiredTigerCustomizationHooks::restartRequired() { +bool WiredTigerCustomizationHooks::enabled() const { return false; } -std::string EmptyWiredTigerCustomizationHooks::getTableCreateConfig(StringData tableName) { - return ""; -} - -std::unique_ptr EmptyWiredTigerCustomizationHooks::getDataProtector() { - return std::unique_ptr(); -} - -boost::filesystem::path EmptyWiredTigerCustomizationHooks::getProtectedPathSuffix() { +std::string WiredTigerCustomizationHooks::getTableCreateConfig(StringData tableName) { return ""; } -Status EmptyWiredTigerCustomizationHooks::protectTmpData( - const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) { - return Status(ErrorCodes::InternalError, - "Customization hooks must be enabled to use preprocessTmpData."); -} - -Status EmptyWiredTigerCustomizationHooks::unprotectTmpData( - const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) { - return Status(ErrorCodes::InternalError, - "Customization hooks must be enabled to use postprocessTmpData."); -} } // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h b/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h index e78995481bd..1ff86a8799e 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h @@ -32,20 +32,11 @@ #include #include -#include "mongo/base/disallow_copying.h" -#include "mongo/db/jsobj.h" - -namespace boost { -namespace filesystem { -class path; -} // namespace filesystem -} // namespace boost - namespace mongo { -class DataProtector; class StringData; class ServiceContext; +// Interface and default implementation for WiredTiger customization hooks class WiredTigerCustomizationHooks { public: static void set(ServiceContext* service, @@ -53,76 +44,18 @@ public: static WiredTigerCustomizationHooks* get(ServiceContext* service); - virtual ~WiredTigerCustomizationHooks() = default; + virtual ~WiredTigerCustomizationHooks(); /** * Returns true if the customization hooks are enabled. */ - virtual bool enabled() const = 0; - - /** - * Perform any encryption engine initialization/sanity checking that needs to happen after - * storage engine initialization but before the server starts accepting incoming connections. - * - * Returns true if the server needs to be rebooted because of configuration changes. - */ - virtual bool restartRequired() = 0; + virtual bool enabled() const; /** * Gets an additional configuration string for the provided table name on a * `WT_SESSION::create` call. */ - virtual std::string getTableCreateConfig(StringData tableName) = 0; - - /** - * Returns the maximum size addition when doing transforming temp data. - */ - size_t additionalBytesForProtectedBuffer() { - return 33; - } - - /** - * Get the data protector object - */ - virtual std::unique_ptr getDataProtector() = 0; - - /** - * Get an implementation specific path suffix to tag files with - */ - virtual boost::filesystem::path getProtectedPathSuffix() = 0; - - /** - * Transform temp data to non-readable form before writing it to disk. - */ - virtual Status protectTmpData( - const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) = 0; - - /** - * Tranforms temp data back to readable form, after reading from disk. - */ - virtual Status unprotectTmpData( - const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) = 0; + virtual std::string getTableCreateConfig(StringData tableName); }; -// Empty default implementation of the abstract class WiredTigerCustomizationHooks -class EmptyWiredTigerCustomizationHooks : public WiredTigerCustomizationHooks { -public: - ~EmptyWiredTigerCustomizationHooks() override; - - bool enabled() const override; - - bool restartRequired() override; - - std::string getTableCreateConfig(StringData tableName) override; - - std::unique_ptr getDataProtector() override; - - boost::filesystem::path getProtectedPathSuffix() override; - - Status protectTmpData( - const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) override; - - Status unprotectTmpData( - const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) override; -}; } // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp index 6331594a57c..ecf3c183996 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp @@ -1102,6 +1102,7 @@ void WiredTigerIndexUnique::_unindex(WT_CURSOR* c, triggerWriteConflictAtPoint(c); return; } + invariantWTOK(ret); WT_ITEM value; invariantWTOK(c->get_value(c, &value)); BufReader br(value.data, value.size); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index f48fff37b0b..aafb52b30c0 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -228,6 +228,7 @@ WiredTigerKVEngine::WiredTigerKVEngine(const std::string& canonicalName, ss << "checkpoint=(wait=" << wiredTigerGlobalOptions.checkpointDelaySecs; ss << ",log_size=2GB),"; ss << "statistics_log=(wait=" << wiredTigerGlobalOptions.statisticsLogDelaySecs << "),"; + ss << "verbose=(recovery_progress),"; } ss << WiredTigerCustomizationHooks::get(getGlobalServiceContext()) ->getTableCreateConfig("system"); @@ -253,6 +254,13 @@ WiredTigerKVEngine::WiredTigerKVEngine(const std::string& canonicalName, msgassertedNoTrace(28718, s.reason()); } invariantWTOK(_conn->close(_conn, NULL)); + // After successful recovery, remove the journal directory. + try { + boost::filesystem::remove_all(journalPath); + } catch (std::exception& e) { + error() << "error removing journal dir " << journalPath.string() << ' ' << e.what(); + throw; + } } // This setting overrides the earlier setting because it is later in the config string. ss << ",log=(enabled=false),"; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp index 0d3cf93a307..7761857bf90 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp @@ -909,7 +909,7 @@ int64_t WiredTigerRecordStore::storageSize(OperationContext* txn, if (_isEphemeral) { return dataSize(txn); } - WiredTigerSession* session = WiredTigerRecoveryUnit::get(txn)->getSession(txn); + WiredTigerSession* session = WiredTigerRecoveryUnit::get(txn)->getSessionNoTxn(txn); StatusWith result = WiredTigerUtil::getStatisticsValueAs(session->getSession(), "statistics:" + getURI(), @@ -1194,7 +1194,9 @@ bool WiredTigerRecordStore::yieldAndAwaitOplogDeletionRequest(OperationContext* // The top-level locks were freed, so also release any potential low-level (storage engine) // locks that might be held. - txn->recoveryUnit()->abandonSnapshot(); + WiredTigerRecoveryUnit* recoveryUnit = (WiredTigerRecoveryUnit*)txn->recoveryUnit(); + recoveryUnit->abandonSnapshot(); + recoveryUnit->beginIdle(); // Wait for an oplog deletion request, or for this record store to have been destroyed. oplogStones->awaitHasExcessStonesOrDead(); @@ -1219,15 +1221,22 @@ void WiredTigerRecordStore::reclaimOplog(OperationContext* txn) { try { WriteUnitOfWork wuow(txn); - WiredTigerCursor startwrap(_uri, _tableId, true, txn); - WT_CURSOR* start = startwrap.get(); - start->set_key(start, _makeKey(_oplogStones->firstRecord)); + WiredTigerCursor cwrap(_uri, _tableId, true, txn); + WT_CURSOR* cursor = cwrap.get(); - WiredTigerCursor endwrap(_uri, _tableId, true, txn); - WT_CURSOR* end = endwrap.get(); - end->set_key(end, _makeKey(stone->lastRecord)); + // The first record in the oplog should be within the truncate range. + int ret = WT_READ_CHECK(cursor->next(cursor)); + invariantWTOK(ret); + int64_t key; + invariantWTOK(cursor->get_key(cursor, &key)); + RecordId firstRecord = _fromKey(key); + if (firstRecord < _oplogStones->firstRecord || firstRecord > stone->lastRecord) { + warning() << "First oplog record " << firstRecord << " is not in truncation range (" + << _oplogStones->firstRecord << ", " << stone->lastRecord << ")"; + } - invariantWTOK(session->truncate(session, nullptr, start, end, nullptr)); + cursor->set_key(cursor, _makeKey(stone->lastRecord)); + invariantWTOK(session->truncate(session, nullptr, nullptr, cursor, nullptr)); _changeNumRecords(txn, -stone->records); _increaseDataSize(txn, -stone->bytes); @@ -1619,7 +1628,7 @@ void WiredTigerRecordStore::appendCustomStats(OperationContext* txn, result->appendIntOrLL("sleepCount", _cappedSleep.load()); result->appendIntOrLL("sleepMS", _cappedSleepMS.load()); } - WiredTigerSession* session = WiredTigerRecoveryUnit::get(txn)->getSession(txn); + WiredTigerSession* session = WiredTigerRecoveryUnit::get(txn)->getSessionNoTxn(txn); WT_SESSION* s = session->getSession(); BSONObjBuilder bob(result->subobjStart(_engineName)); { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp index 66fd5e0ddfa..fe35bd07651 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp @@ -43,6 +43,7 @@ #include "mongo/util/concurrency/ticketholder.h" #include "mongo/util/log.h" #include "mongo/util/mongoutils/str.h" +#include "mongo/util/scopeguard.h" #include "mongo/util/stacktrace.h" namespace mongo { @@ -172,7 +173,12 @@ WiredTigerSession* WiredTigerRecoveryUnit::getSession(OperationContext* opCtx) { WiredTigerSession* WiredTigerRecoveryUnit::getSessionNoTxn(OperationContext* opCtx) { _ensureSession(); - return _session.get(); + WiredTigerSession* session = _session.get(); + + // Dropping the queued idents might block session, which is not desired for fastpath workflow + // like FTDC thread. Disable dropping of queued idents for such sessions. + session->dropQueuedIdentsAtSessionEndAllowed(false); + return session; } void WiredTigerRecoveryUnit::abandonSnapshot() { @@ -249,6 +255,13 @@ void WiredTigerRecoveryUnit::_txnOpen(OperationContext* opCtx) { _active = true; } +void WiredTigerRecoveryUnit::beginIdle() { + // Close all cursors, we don't want to keep any old cached cursors around. + if (_session) { + _session->closeAllCursors(""); + } +} + // --------------------- WiredTigerCursor::WiredTigerCursor(const std::string& uri, diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h index 86d9eece13d..695ad331240 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h @@ -83,6 +83,12 @@ public: WiredTigerSession* getSession(OperationContext* opCtx); + /** + * Enter a period of wait or computation during which there are no WT calls. + * Any non-relevant cached handles can be closed. + */ + void beginIdle(); + /** * Returns a session without starting a new WT txn on the session. Will not close any already * running session. diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_server_status.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_server_status.cpp index 4967dfc2f86..3cb87349fdb 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_server_status.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_server_status.cpp @@ -35,6 +35,7 @@ #include "mongo/base/checked_cast.h" #include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/concurrency/d_concurrency.h" #include "mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h" #include "mongo/db/storage/wiredtiger/wiredtiger_record_store.h" #include "mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h" @@ -56,6 +57,8 @@ bool WiredTigerServerStatusSection::includeByDefault() const { BSONObj WiredTigerServerStatusSection::generateSection(OperationContext* txn, const BSONElement& configElement) const { + Lock::GlobalLock lk(txn->lockState(), LockMode::MODE_IS, UINT_MAX); + // The session does not open a transaction here as one is not needed and opening one would // mean that execution could become blocked when a new transaction cannot be allocated // immediately. diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp index 4fccd06ed6d..9df6ba94651 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp @@ -36,6 +36,7 @@ #include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h" #include "mongo/base/error_codes.h" +#include "mongo/db/server_parameters.h" #include "mongo/db/storage/journal_listener.h" #include "mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h" #include "mongo/db/storage/wiredtiger/wiredtiger_util.h" @@ -46,13 +47,32 @@ namespace mongo { +std::atomic kWiredTigerCursorCacheSize(10000); // NOLINT + +class WiredTigerCursorCacheSize + : public ExportedServerParameter { +public: + WiredTigerCursorCacheSize() + : ExportedServerParameter( + ServerParameterSet::getGlobal(), + "wiredTigerCursorCacheSize", + &kWiredTigerCursorCacheSize) {} + + virtual Status validate(const std::int32_t& potentialNewValue) { + if (potentialNewValue < 0) { + return Status(ErrorCodes::BadValue, + str::stream() + << "wiredTigerCursorCacheSize must be greater than or equal " + << "to 0, but attempted to set to: " + << potentialNewValue); + } + + return Status::OK(); + } +} WiredTigerCursorCacheSizeSetting; + WiredTigerSession::WiredTigerSession(WT_CONNECTION* conn, uint64_t epoch, uint64_t cursorEpoch) - : _epoch(epoch), - _cursorEpoch(cursorEpoch), - _session(NULL), - _cursorGen(0), - _cursorsCached(0), - _cursorsOut(0) { + : _epoch(epoch), _cursorEpoch(cursorEpoch), _session(NULL), _cursorGen(0), _cursorsOut(0) { invariantWTOK(conn->open_session(conn, NULL, "isolation=snapshot", &_session)); } @@ -65,7 +85,6 @@ WiredTigerSession::WiredTigerSession(WT_CONNECTION* conn, _cache(cache), _session(NULL), _cursorGen(0), - _cursorsCached(0), _cursorsOut(0) { invariantWTOK(conn->open_session(conn, NULL, "isolation=snapshot", &_session)); } @@ -83,7 +102,6 @@ WT_CURSOR* WiredTigerSession::getCursor(const std::string& uri, uint64_t id, boo WT_CURSOR* c = i->_cursor; _cursors.erase(i); _cursorsOut++; - _cursorsCached--; return c; } } @@ -107,17 +125,11 @@ void WiredTigerSession::releaseCursor(uint64_t id, WT_CURSOR* cursor) { // Cursors are pushed to the front of the list and removed from the back _cursors.push_front(WiredTigerCachedCursor(id, _cursorGen++, cursor)); - _cursorsCached++; - - // "Old" is defined as not used in the last N**2 operations, if we have N cursors cached. - // The reasoning here is to imagine a workload with N tables performing operations randomly - // across all of them (i.e., each cursor has 1/N chance of used for each operation). We - // would like to cache N cursors in that case, so any given cursor could go N**2 operations - // in between use. - while (_cursorGen - _cursors.back()._gen > 10000) { + + std::uint64_t cursorCacheSize = static_cast(kWiredTigerCursorCacheSize.load()); + while (!_cursors.empty() && _cursorGen - _cursors.back()._gen > cursorCacheSize) { cursor = _cursors.back()._cursor; _cursors.pop_back(); - _cursorsCached--; invariantWTOK(cursor->close(cursor)); } } @@ -125,9 +137,10 @@ void WiredTigerSession::releaseCursor(uint64_t id, WT_CURSOR* cursor) { void WiredTigerSession::closeAllCursors(const std::string& uri) { invariant(_session); + bool all = (uri == ""); for (auto i = _cursors.begin(); i != _cursors.end();) { WT_CURSOR* cursor = i->_cursor; - if (cursor && uri == cursor->uri) { + if (cursor && (all || uri == cursor->uri)) { invariantWTOK(cursor->close(cursor)); i = _cursors.erase(i); } else @@ -344,6 +357,11 @@ void WiredTigerSessionCache::releaseSession(WiredTigerSession* session) { bool returnedToCache = false; uint64_t currentEpoch = _epoch.load(); + bool dropQueuedIdentsAtSessionEnd = session->isDropQueuedIdentsAtSessionEndAllowed(); + + // Reset this session's flag for dropping queued idents to default, before returning it to + // session cache. + session->dropQueuedIdentsAtSessionEndAllowed(true); if (session->_getEpoch() == currentEpoch) { // check outside of lock to reduce contention stdx::lock_guard lock(_cacheLock); @@ -357,7 +375,7 @@ void WiredTigerSessionCache::releaseSession(WiredTigerSession* session) { if (!returnedToCache) delete session; - if (_engine && _engine->haveDropsQueued()) + if (dropQueuedIdentsAtSessionEnd && _engine && _engine->haveDropsQueued()) _engine->dropSomeQueuedIdents(); } diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h index e8fa9811796..34c05a3304b 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.h @@ -99,12 +99,24 @@ public: void closeCursorsForQueuedDrops(WiredTigerKVEngine* engine); + /** + * Closes all cached cursors matching the uri. If the uri is empty, + * all cached cursors are closed. + */ void closeAllCursors(const std::string& uri); int cursorsOut() const { return _cursorsOut; } + bool isDropQueuedIdentsAtSessionEndAllowed() const { + return _dropQueuedIdentsAtSessionEnd; + } + + void dropQueuedIdentsAtSessionEndAllowed(bool dropQueuedIdentsAtSessionEnd) { + _dropQueuedIdentsAtSessionEnd = dropQueuedIdentsAtSessionEnd; + } + static uint64_t genTableId(); /** @@ -134,7 +146,8 @@ private: WT_SESSION* _session; // owned CursorCache _cursors; // owned uint64_t _cursorGen; - int _cursorsCached, _cursorsOut; + int _cursorsOut; + bool _dropQueuedIdentsAtSessionEnd = true; }; /** @@ -174,8 +187,8 @@ public: void closeCursorsForQueuedDrops(); /** - * Closes all cached cursors and ensures that previously opened cursors will be closed on - * release. + * Closes all cached cursors matching the uri. If the uri is empty, + * all cached cursors are closed. */ void closeAllCursors(const std::string& uri); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_size_storer.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_size_storer.cpp index 7783cea4cbf..bab41079a9b 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_size_storer.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_size_storer.cpp @@ -202,7 +202,7 @@ void WiredTigerSizeStorer::syncCache(bool syncToDisk) { WT_SESSION* session = _session.getSession(); invariantWTOK(session->begin_transaction(session, syncToDisk ? "sync=true" : "")); - ScopeGuard rollbacker = MakeGuard(session->rollback_transaction, session, ""); + auto rollbacker = MakeGuard(session->rollback_transaction, session, ""); for (Map::iterator it = myMap.begin(); it != myMap.end(); ++it) { string uriKey = it->first; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_snapshot_manager.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_snapshot_manager.cpp index da0f618f1bd..e6000e62604 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_snapshot_manager.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_snapshot_manager.cpp @@ -38,6 +38,7 @@ #include "mongo/db/storage/wiredtiger/wiredtiger_snapshot_manager.h" #include "mongo/util/log.h" #include "mongo/util/mongoutils/str.h" +#include "mongo/util/scopeguard.h" namespace mongo { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp index f00b4a33804..c0f0907f8da 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp @@ -103,9 +103,13 @@ void WiredTigerUtil::fetchTypeAndSourceURI(OperationContext* opCtx, StatusWith WiredTigerUtil::getMetadata(OperationContext* opCtx, StringData uri) { invariant(opCtx); - WiredTigerCursor curwrap("metadata:create", WiredTigerSession::kMetadataTableId, false, opCtx); - WT_CURSOR* cursor = curwrap.get(); + + auto session = WiredTigerRecoveryUnit::get(opCtx)->getSessionNoTxn(opCtx); + WT_CURSOR* cursor = + session->getCursor("metadata:create", WiredTigerSession::kMetadataTableId, false); invariant(cursor); + ON_BLOCK_EXIT([&] { session->releaseCursor(WiredTigerSession::kMetadataTableId, cursor); }); + std::string strUri = uri.toString(); cursor->set_key(cursor, strUri.c_str()); int ret = cursor->search(cursor); diff --git a/src/mongo/db/views/resolved_view.cpp b/src/mongo/db/views/resolved_view.cpp index d5f4faa7f68..f497cbbb9bd 100644 --- a/src/mongo/db/views/resolved_view.cpp +++ b/src/mongo/db/views/resolved_view.cpp @@ -62,7 +62,27 @@ ResolvedView ResolvedView::fromBSON(BSONObj commandResponseObj) { pipeline.push_back(item.Obj().getOwned()); } - return {ResolvedView(NamespaceString(viewDef["ns"].valueStringData()), pipeline)}; + BSONObj collationSpec; + if (auto collationElt = viewDef["collation"]) { + uassert(40639, + "View definition 'collation' field must be an object", + collationElt.type() == BSONType::Object); + collationSpec = collationElt.embeddedObject().getOwned(); + } + + return {NamespaceString(viewDef["ns"].valueStringData()), + std::move(pipeline), + std::move(collationSpec)}; +} + +BSONObj ResolvedView::toBSON() const { + BSONObjBuilder builder; + builder.append("ns", _namespace.ns()); + builder.append("pipeline", _pipeline); + if (!_defaultCollation.isEmpty()) { + builder.append("collation", _defaultCollation); + } + return builder.obj(); } StatusWith ResolvedView::asExpandedViewAggregation( @@ -104,6 +124,13 @@ StatusWith ResolvedView::asExpandedViewAggregation( aggregationBuilder.append("allowDiskUse", true); } + // Operations on a view must always use the default collation of the view. We must have already + // checked that if the user's request specifies a collation, it matches the collation of the + // view. + if (!_defaultCollation.isEmpty()) { + aggregationBuilder.append("collation", _defaultCollation); + } + return aggregationBuilder.obj(); } diff --git a/src/mongo/db/views/resolved_view.h b/src/mongo/db/views/resolved_view.h index a5016ffc4b3..789576c737f 100644 --- a/src/mongo/db/views/resolved_view.h +++ b/src/mongo/db/views/resolved_view.h @@ -44,8 +44,12 @@ class AggregationRequest; */ class ResolvedView { public: - ResolvedView(const NamespaceString& collectionNs, const std::vector& pipeline) - : _namespace(collectionNs), _pipeline(pipeline) {} + ResolvedView(const NamespaceString& collectionNs, + std::vector pipeline, + BSONObj defaultCollation) + : _namespace(collectionNs), + _pipeline(std::move(pipeline)), + _defaultCollation(std::move(defaultCollation)) {} /** * Returns whether 'commandResponseObj' contains a CommandOnShardedViewNotSupportedOnMongod @@ -55,6 +59,8 @@ public: static ResolvedView fromBSON(BSONObj commandResponseObj); + BSONObj toBSON() const; + /** * Convert an aggregation command on a view to the equivalent command against the views * underlying collection. @@ -70,9 +76,21 @@ public: return _pipeline; } + const BSONObj& getDefaultCollation() const { + return _defaultCollation; + } + private: NamespaceString _namespace; std::vector _pipeline; + + // The default collation associated with this view. An empty object means that the default is + // the simple collation. + // + // Currently all operations which run over a view must use the default collation. This means + // that operations on the view which do not specify a collation inherit the default. Operations + // on the view which specify any other collation fail with a user error. + BSONObj _defaultCollation; }; } // namespace mongo diff --git a/src/mongo/db/views/resolved_view_test.cpp b/src/mongo/db/views/resolved_view_test.cpp index eebcda01db5..d8b95308d45 100644 --- a/src/mongo/db/views/resolved_view_test.cpp +++ b/src/mongo/db/views/resolved_view_test.cpp @@ -33,6 +33,7 @@ #include "mongo/bson/bsonmisc.h" #include "mongo/bson/bsonobj.h" #include "mongo/bson/bsonobjbuilder.h" +#include "mongo/bson/json.h" #include "mongo/db/namespace_string.h" #include "mongo/db/pipeline/aggregation_request.h" #include "mongo/db/views/resolved_view.h" @@ -50,9 +51,10 @@ namespace { const NamespaceString viewNss("testdb.testview"); const NamespaceString backingNss("testdb.testcoll"); const std::vector emptyPipeline; +const BSONObj kSimpleCollation; TEST(ResolvedViewTest, ExpandingCmdObjWithEmptyPipelineOnNoOpViewYieldsEmptyPipeline) { - const ResolvedView resolvedView{backingNss, emptyPipeline}; + const ResolvedView resolvedView{backingNss, emptyPipeline, kSimpleCollation}; BSONObj cmdObj = BSON("aggregate" << viewNss.coll() << "pipeline" << BSONArray()); auto result = resolvedView.asExpandedViewAggregation(cmdObj); @@ -65,7 +67,7 @@ TEST(ResolvedViewTest, ExpandingCmdObjWithEmptyPipelineOnNoOpViewYieldsEmptyPipe TEST(ResolvedViewTest, ExpandingCmdObjWithNonemptyPipelineAppendsToViewPipeline) { std::vector viewPipeline{BSON("skip" << 7)}; - const ResolvedView resolvedView{backingNss, viewPipeline}; + const ResolvedView resolvedView{backingNss, viewPipeline, kSimpleCollation}; BSONObj cmdObj = BSON("aggregate" << viewNss.coll() << "pipeline" << BSON_ARRAY(BSON("limit" << 3))); @@ -80,13 +82,13 @@ TEST(ResolvedViewTest, ExpandingCmdObjWithNonemptyPipelineAppendsToViewPipeline) } TEST(ResolvedViewTest, ExpandingCmdObjFailsIfCmdObjIsNotAValidAggregationCommand) { - const ResolvedView resolvedView{backingNss, emptyPipeline}; + const ResolvedView resolvedView{backingNss, emptyPipeline, kSimpleCollation}; BSONObj badCmdObj = BSON("invalid" << 0); ASSERT_NOT_OK(resolvedView.asExpandedViewAggregation(badCmdObj).getStatus()); } TEST(ResolvedViewTest, ExpandingAggRequestWithEmptyPipelineOnNoOpViewYieldsEmptyPipeline) { - const ResolvedView resolvedView{backingNss, emptyPipeline}; + const ResolvedView resolvedView{backingNss, emptyPipeline, kSimpleCollation}; AggregationRequest aggRequest(viewNss, {}); auto result = resolvedView.asExpandedViewAggregation(aggRequest); @@ -99,7 +101,7 @@ TEST(ResolvedViewTest, ExpandingAggRequestWithEmptyPipelineOnNoOpViewYieldsEmpty TEST(ResolvedViewTest, ExpandingAggRequestWithNonemptyPipelineAppendsToViewPipeline) { std::vector viewPipeline{BSON("skip" << 7)}; - const ResolvedView resolvedView{backingNss, viewPipeline}; + const ResolvedView resolvedView{backingNss, viewPipeline, kSimpleCollation}; std::vector userAggregationPipeline = {BSON("limit" << 3)}; AggregationRequest aggRequest(viewNss, userAggregationPipeline); @@ -115,7 +117,7 @@ TEST(ResolvedViewTest, ExpandingAggRequestWithNonemptyPipelineAppendsToViewPipel } TEST(ResolvedViewTest, ExpandingAggRequestPreservesExplain) { - const ResolvedView resolvedView{backingNss, emptyPipeline}; + const ResolvedView resolvedView{backingNss, emptyPipeline, kSimpleCollation}; AggregationRequest aggRequest(viewNss, {}); aggRequest.setExplain(true); @@ -130,7 +132,7 @@ TEST(ResolvedViewTest, ExpandingAggRequestPreservesExplain) { } TEST(ResolvedViewTest, ExpandingAggRequestPreservesBypassDocumentValidation) { - const ResolvedView resolvedView{backingNss, emptyPipeline}; + const ResolvedView resolvedView{backingNss, emptyPipeline, kSimpleCollation}; AggregationRequest aggRequest(viewNss, {}); aggRequest.setBypassDocumentValidation(true); @@ -145,7 +147,7 @@ TEST(ResolvedViewTest, ExpandingAggRequestPreservesBypassDocumentValidation) { } TEST(ResolvedViewTest, ExpandingAggRequestPreservesAllowDiskUse) { - const ResolvedView resolvedView{backingNss, emptyPipeline}; + const ResolvedView resolvedView{backingNss, emptyPipeline, kSimpleCollation}; AggregationRequest aggRequest(viewNss, {}); aggRequest.setAllowDiskUse(true); @@ -159,6 +161,27 @@ TEST(ResolvedViewTest, ExpandingAggRequestPreservesAllowDiskUse) { ASSERT_BSONOBJ_EQ(result.getValue(), expected); } +TEST(ResolvedViewTest, ExpandingAggRequestPreservesDefaultCollationOfView) { + const ResolvedView resolvedView{backingNss, + emptyPipeline, + BSON("locale" + << "fr_CA")}; + ASSERT_BSONOBJ_EQ(resolvedView.getDefaultCollation(), + BSON("locale" + << "fr_CA")); + AggregationRequest aggRequest(viewNss, {}); + + auto result = resolvedView.asExpandedViewAggregation(aggRequest); + ASSERT_OK(result.getStatus()); + + BSONObj expected = + BSON("aggregate" << backingNss.coll() << "pipeline" << BSONArray() << "cursor" << BSONObj() + << "collation" + << BSON("locale" + << "fr_CA")); + ASSERT_BSONOBJ_EQ(result.getValue(), expected); +} + TEST(ResolvedViewTest, FromBSONFailsIfMissingResolvedView) { BSONObj badCmdResponse = BSON("x" << 1); ASSERT_THROWS_CODE(ResolvedView::fromBSON(badCmdResponse), UserException, 40248); @@ -190,6 +213,13 @@ TEST(ResolvedViewTest, FromBSONFailsOnInvalidPipelineType) { ASSERT_THROWS_CODE(ResolvedView::fromBSON(badCmdResponse), UserException, 40251); } +TEST(ResolvedViewTest, FromBSONFailsOnInvalidCollationType) { + BSONObj badCmdResponse = + BSON("resolvedView" << BSON( + "ns" << backingNss.ns() << "pipeline" << BSONArray() << "collation" << 1)); + ASSERT_THROWS_CODE(ResolvedView::fromBSON(badCmdResponse), AssertionException, 40639); +} + TEST(ResolvedViewTest, FromBSONSuccessfullyParsesEmptyBSONArrayIntoEmptyVector) { BSONObj cmdResponse = BSON("resolvedView" << BSON("ns" << backingNss.ns() << "pipeline" << BSONArray())); @@ -222,6 +252,22 @@ TEST(ResolvedViewTest, FromBSONSuccessfullyParsesPopulatedBSONArrayIntoVector) { SimpleBSONObjComparator::kInstance.makeEqualTo())); } +TEST(ResolvedViewTest, FromBSONSuccessfullyParsesCollation) { + BSONObj cmdResponse = BSON( + "resolvedView" << BSON("ns" << backingNss.ns() << "pipeline" << BSONArray() << "collation" + << BSON("locale" + << "fil"))); + const ResolvedView result = ResolvedView::fromBSON(cmdResponse); + ASSERT_EQ(result.getNamespace(), backingNss); + ASSERT(std::equal(emptyPipeline.begin(), + emptyPipeline.end(), + result.getPipeline().begin(), + SimpleBSONObjComparator::kInstance.makeEqualTo())); + ASSERT_BSONOBJ_EQ(result.getDefaultCollation(), + BSON("locale" + << "fil")); +} + TEST(ResolvedViewTest, IsResolvedViewErrorResponseDetectsKickbackErrorCodeSuccessfully) { BSONObj errorResponse = BSON("ok" << 0 << "code" << ErrorCodes::CommandOnShardedViewNotSupportedOnMongod << "errmsg" @@ -235,5 +281,34 @@ TEST(ResolvedViewTest, IsResolvedViewErrorResponseReportsFalseOnNonKickbackError << "View nesting too deep or view cycle detected"); ASSERT_FALSE(ResolvedView::isResolvedViewErrorResponse(errorResponse)); } + +TEST(ResolvedViewTest, ToBSONSerializesCorrectly) { + const ResolvedView resolvedView{backingNss, + std::vector{BSON("$match" << BSON("x" << 1))}, + BSON("locale" + << "fr_CA")}; + auto serialized = resolvedView.toBSON(); + ASSERT_BSONOBJ_EQ( + serialized, + fromjson( + "{ns: 'testdb.testcoll', pipeline: [{$match: {x: 1}}], collation: {locale: 'fr_CA'}}")); +} + +TEST(ResolvedViewTest, ToBSONOutputCanBeReparsed) { + const ResolvedView resolvedView{backingNss, + emptyPipeline, + BSON("locale" + << "fr_CA")}; + auto serialized = resolvedView.toBSON(); + auto reparsedResolvedView = ResolvedView::fromBSON(BSON("resolvedView" << serialized)); + ASSERT_EQ(reparsedResolvedView.getNamespace(), backingNss); + ASSERT(std::equal(emptyPipeline.begin(), + emptyPipeline.end(), + reparsedResolvedView.getPipeline().begin(), + SimpleBSONObjComparator::kInstance.makeEqualTo())); + ASSERT_BSONOBJ_EQ(reparsedResolvedView.getDefaultCollation(), + BSON("locale" + << "fr_CA")); +} } // namespace } // namespace mongo diff --git a/src/mongo/db/views/view_catalog.cpp b/src/mongo/db/views/view_catalog.cpp index 30cabf24935..d590c509a7e 100644 --- a/src/mongo/db/views/view_catalog.cpp +++ b/src/mongo/db/views/view_catalog.cpp @@ -394,6 +394,7 @@ StatusWith ViewCatalog::resolveView(OperationContext* txn, stdx::lock_guard lk(_mutex); const NamespaceString* resolvedNss = &nss; std::vector resolvedPipeline; + BSONObj collation; for (int i = 0; i < ViewGraph::kMaxViewDepth; i++) { auto view = _lookup_inlock(txn, resolvedNss->ns()); @@ -408,10 +409,13 @@ StatusWith ViewCatalog::resolveView(OperationContext* txn, str::stream() << "View pipeline exceeds maximum size; maximum size is " << ViewGraph::kMaxViewPipelineSizeBytes}; } - return StatusWith({*resolvedNss, resolvedPipeline}); + return StatusWith( + {*resolvedNss, std::move(resolvedPipeline), std::move(collation)}); } resolvedNss = &(view->viewOn()); + collation = view->defaultCollator() ? view->defaultCollator()->getSpec().toBSON() + : CollationSpec::kSimpleSpec; // Prepend the underlying view's pipeline to the current working pipeline. const std::vector& toPrepend = view->pipeline(); @@ -419,7 +423,8 @@ StatusWith ViewCatalog::resolveView(OperationContext* txn, // If the first stage is a $collStats, then we return early with the viewOn namespace. if (toPrepend.size() > 0 && !toPrepend[0]["$collStats"].eoo()) { - return StatusWith({*resolvedNss, resolvedPipeline}); + return StatusWith( + {*resolvedNss, std::move(resolvedPipeline), std::move(collation)}); } } diff --git a/src/mongo/db/views/view_catalog_test.cpp b/src/mongo/db/views/view_catalog_test.cpp index 1ca54f809df..367b4f73b0c 100644 --- a/src/mongo/db/views/view_catalog_test.cpp +++ b/src/mongo/db/views/view_catalog_test.cpp @@ -38,6 +38,8 @@ #include "mongo/bson/bsonobj.h" #include "mongo/bson/bsonobjbuilder.h" #include "mongo/db/namespace_string.h" +#include "mongo/db/operation_context.h" +#include "mongo/db/query/collation/collator_factory_interface.h" #include "mongo/db/query/query_test_service_context.h" #include "mongo/db/server_options.h" #include "mongo/db/service_context_noop.h" @@ -417,6 +419,42 @@ TEST_F(ViewCatalogFixture, ResolveViewCorrectPipeline) { } } +TEST_F(ViewCatalogFixture, ResolveViewCorrectlyExtractsDefaultCollation) { + const NamespaceString view1("db.view1"); + const NamespaceString view2("db.view2"); + const NamespaceString viewOn("db.coll"); + BSONArrayBuilder pipeline1; + BSONArrayBuilder pipeline2; + + pipeline1 << BSON("$match" << BSON("foo" << 1)); + pipeline2 << BSON("$match" << BSON("foo" << 2)); + + BSONObj collation = BSON("locale" + << "mock_reverse_string"); + + ASSERT_OK(viewCatalog.createView(opCtx.get(), view1, viewOn, pipeline1.arr(), collation)); + ASSERT_OK(viewCatalog.createView(opCtx.get(), view2, view1, pipeline2.arr(), collation)); + + auto resolvedView = viewCatalog.resolveView(opCtx.get(), view2); + ASSERT(resolvedView.isOK()); + + ASSERT_EQ(resolvedView.getValue().getNamespace(), viewOn); + + std::vector expected = {BSON("$match" << BSON("foo" << 1)), + BSON("$match" << BSON("foo" << 2))}; + std::vector result = resolvedView.getValue().getPipeline(); + ASSERT_EQ(expected.size(), result.size()); + for (uint32_t i = 0; i < expected.size(); i++) { + ASSERT(SimpleBSONObjComparator::kInstance.evaluate(expected[i] == result[i])); + } + + auto expectedCollation = + CollatorFactoryInterface::get(opCtx->getServiceContext())->makeFromBSON(collation); + ASSERT_OK(expectedCollation.getStatus()); + ASSERT_BSONOBJ_EQ(resolvedView.getValue().getDefaultCollation(), + expectedCollation.getValue()->getSpec().toBSON()); +} + TEST_F(ViewCatalogFixture, InvalidateThenReload) { const NamespaceString viewName("db.view"); const NamespaceString viewOn("db.coll"); diff --git a/src/mongo/db/views/view_sharding_check.cpp b/src/mongo/db/views/view_sharding_check.cpp index cc662ff2549..90835e03a07 100644 --- a/src/mongo/db/views/view_sharding_check.cpp +++ b/src/mongo/db/views/view_sharding_check.cpp @@ -69,11 +69,7 @@ StatusWith ViewShardingCheck::getResolvedViewIfSharded(OperationContext return BSONObj(); } - BSONObjBuilder viewDetailBob; - viewDetailBob.append("ns", sourceNss.ns()); - viewDetailBob.append("pipeline", resolvedView.getValue().getPipeline()); - - return viewDetailBob.obj(); + return resolvedView.getValue().toBSON(); } void ViewShardingCheck::appendShardedViewStatus(const BSONObj& resolvedView, BSONObjBuilder* out) { diff --git a/src/mongo/dbtests/dbhelper_tests.cpp b/src/mongo/dbtests/dbhelper_tests.cpp index 1ed948092c7..7a4d745e764 100644 --- a/src/mongo/dbtests/dbhelper_tests.cpp +++ b/src/mongo/dbtests/dbhelper_tests.cpp @@ -74,8 +74,12 @@ public: KeyRange range(ns, BSON("_id" << _min), BSON("_id" << _max), BSON("_id" << 1)); mongo::WriteConcernOptions dummyWriteConcern; - Helpers::removeRange( - &txn, range, BoundInclusion::kIncludeStartKeyOnly, dummyWriteConcern); + Milliseconds dummyReplWaitDuration; + Helpers::removeRange(&txn, + range, + BoundInclusion::kIncludeStartKeyOnly, + dummyWriteConcern, + dummyReplWaitDuration); } // Check that the expected documents remain. diff --git a/src/mongo/dbtests/jstests.cpp b/src/mongo/dbtests/jstests.cpp index 13e71a0a74a..c0421910c2f 100644 --- a/src/mongo/dbtests/jstests.cpp +++ b/src/mongo/dbtests/jstests.cpp @@ -2388,6 +2388,49 @@ public: } }; +class RequiresOwnedObjects { +public: + void run() { + char buf[] = {5, 0, 0, 0, 0}; + BSONObj unowned(buf); + BSONObj owned = unowned.getOwned(); + + ASSERT(!unowned.isOwned()); + ASSERT(owned.isOwned()); + + // Ensure that by default we can bind owned and unowned + { + unique_ptr s(getGlobalScriptEngine()->newScope()); + s->setObject("unowned", unowned, true); + s->setObject("owned", owned, true); + } + + // After we set the flag, we should only be able to set owned + { + unique_ptr s(getGlobalScriptEngine()->newScope()); + s->requireOwnedObjects(); + s->setObject("owned", owned, true); + + bool threwException = false; + try { + s->setObject("unowned", unowned, true); + } catch (...) { + threwException = true; + + auto status = exceptionToStatus(); + + ASSERT_EQUALS(status.code(), ErrorCodes::BadValue); + } + + ASSERT(threwException); + + // after resetting, we can set unowned's again + s->reset(); + s->setObject("unowned", unowned, true); + } + } +}; + class All : public Suite { public: All() : Suite("js") {} @@ -2444,6 +2487,7 @@ public: add(); add(); + add(); add(); add(); diff --git a/src/mongo/dbtests/rollbacktests.cpp b/src/mongo/dbtests/rollbacktests.cpp index b842eb5a9bc..300d3f46aed 100644 --- a/src/mongo/dbtests/rollbacktests.cpp +++ b/src/mongo/dbtests/rollbacktests.cpp @@ -216,7 +216,7 @@ public: } }; -template +template class RenameCollection { public: void run() { @@ -236,7 +236,8 @@ public: WriteUnitOfWork uow(&txn); ASSERT(!collectionExists(&ctx, source.ns())); ASSERT(!collectionExists(&ctx, target.ns())); - ASSERT_OK(userCreateNS(&txn, ctx.db(), source.ns(), BSONObj(), defaultIndexes)); + auto options = capped ? BSON("capped" << true << "size" << 1000) : BSONObj(); + ASSERT_OK(userCreateNS(&txn, ctx.db(), source.ns(), options, defaultIndexes)); uow.commit(); } ASSERT(collectionExists(&ctx, source.ns())); @@ -263,7 +264,7 @@ public: } }; -template +template class RenameDropTargetCollection { public: void run() { @@ -288,8 +289,9 @@ public: WriteUnitOfWork uow(&txn); ASSERT(!collectionExists(&ctx, source.ns())); ASSERT(!collectionExists(&ctx, target.ns())); - ASSERT_OK(userCreateNS(&txn, ctx.db(), source.ns(), BSONObj(), defaultIndexes)); - ASSERT_OK(userCreateNS(&txn, ctx.db(), target.ns(), BSONObj(), defaultIndexes)); + auto options = capped ? BSON("capped" << true << "size" << 1000) : BSONObj(); + ASSERT_OK(userCreateNS(&txn, ctx.db(), source.ns(), options, defaultIndexes)); + ASSERT_OK(userCreateNS(&txn, ctx.db(), target.ns(), options, defaultIndexes)); insertRecord(&txn, source, sourceDoc); insertRecord(&txn, target, targetDoc); diff --git a/src/mongo/executor/connection_pool.cpp b/src/mongo/executor/connection_pool.cpp index cf8aadd4d3a..9bbf528725e 100644 --- a/src/mongo/executor/connection_pool.cpp +++ b/src/mongo/executor/connection_pool.cpp @@ -60,6 +60,45 @@ namespace executor { */ class ConnectionPool::SpecificPool { public: + /** + * These active client methods must be used whenever entering a specific pool outside of the + * shutdown background task. The presence of an active client will bump a counter on the + * specific pool which will prevent the shutdown thread from deleting it. + * + * The complexity comes from the need to hold a lock when writing to the + * _activeClients param on the specific pool. Because the code beneath the client needs to lock + * and unlock the parent mutex (and can leave unlocked), we want to start the client with the + * lock acquired, move it into the client, then re-acquire to decrement the counter on the way + * out. + * + * It's used like: + * + * pool.runWithActiveClient([](stdx::unique_lock lk){ codeToBeProtected(); }); + */ + template + void runWithActiveClient(Callback&& cb) { + runWithActiveClient(stdx::unique_lock(_parent->_mutex), + std::forward(cb)); + } + + template + void runWithActiveClient(stdx::unique_lock lk, Callback&& cb) { + invariant(lk.owns_lock()); + + _activeClients++; + + const auto guard = MakeGuard([&] { + invariant(!lk.owns_lock()); + stdx::lock_guard lk(_parent->_mutex); + _activeClients--; + }); + + { + decltype(lk) localLk(std::move(lk)); + cb(std::move(localLk)); + } + } + SpecificPool(ConnectionPool* parent, const HostAndPort& hostAndPort); ~SpecificPool(); @@ -149,6 +188,7 @@ private: std::unique_ptr _requestTimer; Date_t _requestTimerExpiration; + size_t _activeClients; size_t _generation; bool _inFulfillRequests; bool _inSpawnConnections; @@ -206,8 +246,11 @@ void ConnectionPool::dropConnections(const HostAndPort& hostAndPort) { if (iter == _pools.end()) return; - iter->second.get()->processFailure( - Status(ErrorCodes::PooledConnectionsDropped, "Pooled connections dropped"), std::move(lk)); + iter->second->runWithActiveClient(std::move(lk), [&](decltype(lk) lk) { + iter->second->processFailure( + Status(ErrorCodes::PooledConnectionsDropped, "Pooled connections dropped"), + std::move(lk)); + }); } void ConnectionPool::get(const HostAndPort& hostAndPort, @@ -229,7 +272,9 @@ void ConnectionPool::get(const HostAndPort& hostAndPort, invariant(pool); - pool->getConnection(hostAndPort, timeout, std::move(lk), std::move(cb)); + pool->runWithActiveClient(std::move(lk), [&](decltype(lk) lk) { + pool->getConnection(hostAndPort, timeout, std::move(lk), std::move(cb)); + }); } void ConnectionPool::appendConnectionStats(ConnectionPoolStats* stats) const { @@ -264,13 +309,16 @@ void ConnectionPool::returnConnection(ConnectionInterface* conn) { invariant(iter != _pools.end()); - iter->second.get()->returnConnection(conn, std::move(lk)); + iter->second->runWithActiveClient(std::move(lk), [&](decltype(lk) lk) { + iter->second->returnConnection(conn, std::move(lk)); + }); } ConnectionPool::SpecificPool::SpecificPool(ConnectionPool* parent, const HostAndPort& hostAndPort) : _parent(parent), _hostAndPort(hostAndPort), _requestTimer(parent->_factory->makeTimer()), + _activeClients(0), _generation(0), _inFulfillRequests(false), _inSpawnConnections(false), @@ -362,47 +410,48 @@ void ConnectionPool::SpecificPool::returnConnection(ConnectionInterface* connPtr // Unlock in case refresh can occur immediately lk.unlock(); - connPtr->refresh(_parent->_options.refreshTimeout, - [this](ConnectionInterface* connPtr, Status status) { - connPtr->indicateUsed(); - - stdx::unique_lock lk(_parent->_mutex); - - auto conn = takeFromProcessingPool(connPtr); - - // If the host and port were dropped, let this lapse - if (conn->getGeneration() != _generation) { - spawnConnections(lk); - return; - } - - // If we're in shutdown, we don't need refreshed connections - if (_state == State::kInShutdown) - return; - - // If the connection refreshed successfully, throw it back in the ready - // pool - if (status.isOK()) { - addToReady(lk, std::move(conn)); - spawnConnections(lk); - return; - } - - // If we've exceeded the time limit, start a new connect, rather than - // failing all operations. We do this because the various callers have - // their own time limit which is unrelated to our internal one. - if (status.code() == ErrorCodes::NetworkInterfaceExceededTimeLimit) { - log() << "Pending connection to host " << _hostAndPort - << " did not complete within the connection timeout," - << " retrying with a new connection;" << openConnections(lk) - << " connections to that host remain open"; - spawnConnections(lk); - return; - } - - // Otherwise pass the failure on through - processFailure(status, std::move(lk)); - }); + connPtr->refresh( + _parent->_options.refreshTimeout, [this](ConnectionInterface* connPtr, Status status) { + connPtr->indicateUsed(); + + runWithActiveClient([&](stdx::unique_lock lk) { + auto conn = takeFromProcessingPool(connPtr); + + // If the host and port were dropped, let this lapse + if (conn->getGeneration() != _generation) { + spawnConnections(lk); + return; + } + + // If we're in shutdown, we don't need refreshed connections + if (_state == State::kInShutdown) + return; + + // If the connection refreshed successfully, throw it back in + // the ready pool + if (status.isOK()) { + addToReady(lk, std::move(conn)); + spawnConnections(lk); + return; + } + + // If we've exceeded the time limit, start a new connect, + // rather than failing all operations. We do this because the + // various callers have their own time limit which is unrelated + // to our internal one. + if (status.code() == ErrorCodes::NetworkInterfaceExceededTimeLimit) { + log() << "Pending connection to host " << _hostAndPort + << " did not complete within the connection timeout," + << " retrying with a new connection;" << openConnections(lk) + << " connections to that host remain open"; + spawnConnections(lk); + return; + } + + // Otherwise pass the failure on through + processFailure(status, std::move(lk)); + }); + }); lk.lock(); } else { // If it's fine as it is, just put it in the ready queue @@ -425,25 +474,25 @@ void ConnectionPool::SpecificPool::addToReady(stdx::unique_lock& lk connPtr->setTimeout(_parent->_options.refreshRequirement, [this, connPtr]() { OwnedConnection conn; - stdx::unique_lock lk(_parent->_mutex); - - if (!_readyPool.count(connPtr)) { - // We've already been checked out. We don't need to refresh - // ourselves. - return; - } + runWithActiveClient([&](stdx::unique_lock lk) { + if (!_readyPool.count(connPtr)) { + // We've already been checked out. We don't need to refresh + // ourselves. + return; + } - conn = takeFromPool(_readyPool, connPtr); + conn = takeFromPool(_readyPool, connPtr); - // If we're in shutdown, we don't need to refresh connections - if (_state == State::kInShutdown) - return; + // If we're in shutdown, we don't need to refresh connections + if (_state == State::kInShutdown) + return; - _checkedOutPool[connPtr] = std::move(conn); + _checkedOutPool[connPtr] = std::move(conn); - connPtr->indicateSuccess(); + connPtr->indicateSuccess(); - returnConnection(connPtr, std::move(lk)); + returnConnection(connPtr, std::move(lk)); + }); }); fulfillRequests(lk); @@ -586,26 +635,26 @@ void ConnectionPool::SpecificPool::spawnConnections(stdx::unique_lock_options.refreshTimeout, [this](ConnectionInterface* connPtr, Status status) { connPtr->indicateUsed(); - stdx::unique_lock lk(_parent->_mutex); - - auto conn = takeFromProcessingPool(connPtr); - - if (conn->getGeneration() != _generation) { - // If the host and port was dropped, let the - // connection lapse - spawnConnections(lk); - } else if (status.isOK()) { - addToReady(lk, std::move(conn)); - spawnConnections(lk); - } else if (status.code() == ErrorCodes::NetworkInterfaceExceededTimeLimit) { - // If we've exceeded the time limit, restart the connect, rather than - // failing all operations. We do this because the various callers - // have their own time limit which is unrelated to our internal one. - spawnConnections(lk); - } else { - // If the setup failed, cascade the failure edge - processFailure(status, std::move(lk)); - } + runWithActiveClient([&](stdx::unique_lock lk) { + auto conn = takeFromProcessingPool(connPtr); + + if (conn->getGeneration() != _generation) { + // If the host and port was dropped, let the + // connection lapse + spawnConnections(lk); + } else if (status.isOK()) { + addToReady(lk, std::move(conn)); + spawnConnections(lk); + } else if (status.code() == ErrorCodes::NetworkInterfaceExceededTimeLimit) { + // If we've exceeded the time limit, restart the connect, rather than + // failing all operations. We do this because the various callers + // have their own time limit which is unrelated to our internal one. + spawnConnections(lk); + } else { + // If the setup failed, cascade the failure edge + processFailure(status, std::move(lk)); + } + }); }); // Note that this assumes that the refreshTimeout is sound for the // setupTimeout @@ -641,7 +690,7 @@ void ConnectionPool::SpecificPool::shutdown() { // If we have processing connections, wait for them to finish or timeout // before shutdown - if (_processingPool.size() || _droppedProcessingPool.size()) { + if (_processingPool.size() || _droppedProcessingPool.size() || _activeClients) { _requestTimer->setTimeout(Seconds(1), [this]() { shutdown(); }); return; @@ -693,27 +742,27 @@ void ConnectionPool::SpecificPool::updateStateInLock() { // We set a timer for the most recent request, then invoke each timed // out request we couldn't service _requestTimer->setTimeout(timeout, [this]() { - stdx::unique_lock lk(_parent->_mutex); - - auto now = _parent->_factory->now(); - - while (_requests.size()) { - auto& x = _requests.top(); - - if (x.first <= now) { - auto cb = std::move(x.second); - _requests.pop(); - - lk.unlock(); - cb(Status(ErrorCodes::NetworkInterfaceExceededTimeLimit, - "Couldn't get a connection within the time limit")); - lk.lock(); - } else { - break; + runWithActiveClient([&](stdx::unique_lock lk) { + auto now = _parent->_factory->now(); + + while (_requests.size()) { + auto& x = _requests.top(); + + if (x.first <= now) { + auto cb = std::move(x.second); + _requests.pop(); + + lk.unlock(); + cb(Status(ErrorCodes::NetworkInterfaceExceededTimeLimit, + "Couldn't get a connection within the time limit")); + lk.lock(); + } else { + break; + } } - } - updateStateInLock(); + updateStateInLock(); + }); }); } else if (_checkedOutPool.size()) { // If we have no requests, but someone's using a connection, we just diff --git a/src/mongo/executor/network_interface_asio_auth.cpp b/src/mongo/executor/network_interface_asio_auth.cpp index 1f2039dbd1e..571a1be1f81 100644 --- a/src/mongo/executor/network_interface_asio_auth.cpp +++ b/src/mongo/executor/network_interface_asio_auth.cpp @@ -38,6 +38,7 @@ #include "mongo/db/auth/authorization_manager_global.h" #include "mongo/db/auth/internal_user_auth.h" #include "mongo/db/commands.h" +#include "mongo/db/commands/feature_compatibility_version_command_parser.h" #include "mongo/db/server_options.h" #include "mongo/db/wire_version.h" #include "mongo/rpc/factory.h" diff --git a/src/mongo/executor/task_executor.h b/src/mongo/executor/task_executor.h index 2d558512f91..ebf7c447ef0 100644 --- a/src/mongo/executor/task_executor.h +++ b/src/mongo/executor/task_executor.h @@ -196,6 +196,8 @@ public: /** * Schedules "work" to be run by the executor no sooner than "when". * + * If "when" is <= now(), then it schedules the "work" to be run ASAP. + * * Returns a handle for waiting on or canceling the callback, or * ErrorCodes::ShutdownInProgress. * diff --git a/src/mongo/executor/task_executor_test_common.cpp b/src/mongo/executor/task_executor_test_common.cpp index 57c12813250..d7a384231b8 100644 --- a/src/mongo/executor/task_executor_test_common.cpp +++ b/src/mongo/executor/task_executor_test_common.cpp @@ -328,14 +328,22 @@ COMMON_EXECUTOR_TEST(ScheduleWorkAt) { Status status1 = getDetectableErrorStatus(); Status status2 = getDetectableErrorStatus(); Status status3 = getDetectableErrorStatus(); + Status status4 = getDetectableErrorStatus(); + const Date_t now = net->now(); const TaskExecutor::CallbackHandle cb1 = unittest::assertGet(executor.scheduleWorkAt( now + Milliseconds(100), stdx::bind(setStatus, stdx::placeholders::_1, &status1))); + const TaskExecutor::CallbackHandle cb4 = unittest::assertGet(executor.scheduleWorkAt( + now - Milliseconds(50), stdx::bind(setStatus, stdx::placeholders::_1, &status4))); unittest::assertGet(executor.scheduleWorkAt( now + Milliseconds(5000), stdx::bind(setStatus, stdx::placeholders::_1, &status3))); const TaskExecutor::CallbackHandle cb2 = unittest::assertGet(executor.scheduleWorkAt( now + Milliseconds(200), stdx::bind(setStatusAndShutdown, stdx::placeholders::_1, &status2))); + + executor.wait(cb4); + ASSERT_OK(status4); + const Date_t startTime = net->now(); net->enterNetwork(); net->runUntil(startTime + Milliseconds(200)); diff --git a/src/mongo/rpc/get_status_from_command_result.cpp b/src/mongo/rpc/get_status_from_command_result.cpp index 2b4adf94aae..13c176f44df 100644 --- a/src/mongo/rpc/get_status_from_command_result.cpp +++ b/src/mongo/rpc/get_status_from_command_result.cpp @@ -30,6 +30,7 @@ #include "mongo/rpc/get_status_from_command_result.h" +#include "mongo/base/error_codes.h" #include "mongo/base/status.h" #include "mongo/bson/util/bson_extract.h" #include "mongo/db/jsobj.h" @@ -40,6 +41,7 @@ namespace mongo { namespace { const std::string kCmdResponseWriteConcernField = "writeConcernError"; +const std::string kCmdResponseWriteErrorsField = "writeErrors"; } // namespace Status getStatusFromCommandResult(const BSONObj& result) { @@ -107,4 +109,45 @@ Status getWriteConcernStatusFromCommandResult(const BSONObj& obj) { return wcError.toStatus(); } +Status getFirstWriteErrorStatusFromCommandResult(const BSONObj& cmdResponse) { + BSONElement writeErrorElem; + auto status = bsonExtractTypedField( + cmdResponse, kCmdResponseWriteErrorsField, BSONType::Array, &writeErrorElem); + if (!status.isOK()) { + if (status == ErrorCodes::NoSuchKey) { + return Status::OK(); + } else { + return status; + } + } + + auto firstWriteErrorElem = writeErrorElem.Obj().firstElement(); + if (!firstWriteErrorElem) { + return Status::OK(); + } + + if (firstWriteErrorElem.type() != BSONType::Object) { + return Status(ErrorCodes::UnsupportedFormat, + str::stream() << "writeErrors should be an array of objects, found " + << typeName(firstWriteErrorElem.type())); + } + + auto firstWriteErrorObj = firstWriteErrorElem.Obj(); + + return Status(ErrorCodes::fromInt(firstWriteErrorObj["code"].Int()), + firstWriteErrorObj["errmsg"].String()); +} + +Status getStatusFromWriteCommandReply(const BSONObj& cmdResponse) { + auto status = getStatusFromCommandResult(cmdResponse); + if (!status.isOK()) { + return status; + } + status = getFirstWriteErrorStatusFromCommandResult(cmdResponse); + if (!status.isOK()) { + return status; + } + return getWriteConcernStatusFromCommandResult(cmdResponse); +} + } // namespace mongo diff --git a/src/mongo/rpc/get_status_from_command_result.h b/src/mongo/rpc/get_status_from_command_result.h index 971805ee088..423b656a0ed 100644 --- a/src/mongo/rpc/get_status_from_command_result.h +++ b/src/mongo/rpc/get_status_from_command_result.h @@ -51,4 +51,17 @@ Status getStatusFromCommandResult(const BSONObj& result); */ Status getWriteConcernStatusFromCommandResult(const BSONObj& cmdResponse); + +/** + * Extracts the first write error from a command response and converts it into a status. This + * ignores all errors after the first and does not preserve the write error index, so it should not + * be used with bulk writes. + */ +Status getFirstWriteErrorStatusFromCommandResult(const BSONObj& cmdResponse); + +/** + * Extracts any type of error from a write command response. + */ +Status getStatusFromWriteCommandReply(const BSONObj& cmdResponse); + } // namespace mongo diff --git a/src/mongo/s/catalog/type_tags.cpp b/src/mongo/s/catalog/type_tags.cpp index 59d28cc1cc4..b0c72053cca 100644 --- a/src/mongo/s/catalog/type_tags.cpp +++ b/src/mongo/s/catalog/type_tags.cpp @@ -34,6 +34,7 @@ #include "mongo/bson/bsonobj.h" #include "mongo/bson/bsonobjbuilder.h" #include "mongo/bson/util/bson_extract.h" +#include "mongo/s/catalog/type_chunk.h" #include "mongo/util/assert_util.h" #include "mongo/util/mongoutils/str.h" @@ -55,8 +56,9 @@ StatusWith TagsType::fromBSON(const BSONObj& source) { { std::string tagsNs; Status status = bsonExtractStringField(source, ns.name(), &tagsNs); - if (!status.isOK()) + if (!status.isOK()) { return status; + } tags._ns = tagsNs; } @@ -64,28 +66,21 @@ StatusWith TagsType::fromBSON(const BSONObj& source) { { std::string tagsTag; Status status = bsonExtractStringField(source, tag.name(), &tagsTag); - if (!status.isOK()) + if (!status.isOK()) { return status; + } tags._tag = tagsTag; } { - BSONElement tagsMinKey; - Status status = bsonExtractTypedField(source, min.name(), Object, &tagsMinKey); - if (!status.isOK()) - return status; - - tags._minKey = tagsMinKey.Obj().getOwned(); - } - - { - BSONElement tagsMaxKey; - Status status = bsonExtractTypedField(source, max.name(), Object, &tagsMaxKey); - if (!status.isOK()) - return status; + auto tagRangeStatus = ChunkRange::fromBSON(source); + if (!tagRangeStatus.isOK()) + return tagRangeStatus.getStatus(); - tags._maxKey = tagsMaxKey.Obj().getOwned(); + const auto tagRange = std::move(tagRangeStatus.getValue()); + tags._minKey = tagRange.getMin().getOwned(); + tags._maxKey = tagRange.getMax().getOwned(); } return tags; diff --git a/src/mongo/s/catalog/type_tags_test.cpp b/src/mongo/s/catalog/type_tags_test.cpp index 78af4d5fe03..16a7c6dc2ee 100644 --- a/src/mongo/s/catalog/type_tags_test.cpp +++ b/src/mongo/s/catalog/type_tags_test.cpp @@ -117,9 +117,8 @@ TEST(TagsType, KeysNotAscending) { BSON(TagsType::tag("tag") << TagsType::ns("test.mycol") << TagsType::min(BSON("a" << 20)) << TagsType::max(BSON("a" << 10))); - StatusWith status = TagsType::fromBSON(obj); - const TagsType& tag = status.getValue(); - ASSERT_EQUALS(ErrorCodes::BadValue, tag.validate()); + StatusWith tagStatus = TagsType::fromBSON(obj); + ASSERT_EQUALS(ErrorCodes::FailedToParse, tagStatus.getStatus()); } TEST(TagsType, BadType) { diff --git a/src/mongo/s/client/shard_remote.cpp b/src/mongo/s/client/shard_remote.cpp index 85e0bc7678b..1e0ea51ed4b 100644 --- a/src/mongo/s/client/shard_remote.cpp +++ b/src/mongo/s/client/shard_remote.cpp @@ -340,7 +340,8 @@ StatusWith ShardRemote::_exhaustiveFindOnConfig( findCmdBuilder.done(), fetcherCallback, _appendMetadataForCommand(txn, readPrefWithMinOpTime), - maxTimeMS); + maxTimeMS /* find network timeout */, + maxTimeMS /* getMore network timeout */); Status scheduleStatus = fetcher.schedule(); if (!scheduleStatus.isOK()) { return scheduleStatus; diff --git a/src/mongo/s/commands/SConscript b/src/mongo/s/commands/SConscript index 800f8fe6005..b70eb831ac4 100644 --- a/src/mongo/s/commands/SConscript +++ b/src/mongo/s/commands/SConscript @@ -83,6 +83,7 @@ env.Library( LIBDEPS=[ '$BUILD_DIR/mongo/db/commands/apply_ops_cmd_common', '$BUILD_DIR/mongo/db/commands/killcursors_common', + '$BUILD_DIR/mongo/db/ftdc/ftdc_server', '$BUILD_DIR/mongo/db/pipeline/aggregation', '$BUILD_DIR/mongo/db/views/views', '$BUILD_DIR/mongo/rpc/client_metadata', diff --git a/src/mongo/s/commands/cluster_ftdc_commands.cpp b/src/mongo/s/commands/cluster_ftdc_commands.cpp index 23903e92984..12021e98f6d 100644 --- a/src/mongo/s/commands/cluster_ftdc_commands.cpp +++ b/src/mongo/s/commands/cluster_ftdc_commands.cpp @@ -67,6 +67,28 @@ public: Status checkAuthForCommand(Client* client, const std::string& dbname, const BSONObj& cmdObj) override { + + if (!AuthorizationSession::get(client)->isAuthorizedForActionsOnResource( + ResourcePattern::forClusterResource(), ActionType::serverStatus)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + + if (!AuthorizationSession::get(client)->isAuthorizedForActionsOnResource( + ResourcePattern::forClusterResource(), ActionType::replSetGetStatus)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + + if (!AuthorizationSession::get(client)->isAuthorizedForActionsOnResource( + ResourcePattern::forClusterResource(), ActionType::connPoolStats)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + + if (!AuthorizationSession::get(client)->isAuthorizedForActionsOnResource( + ResourcePattern::forExactNamespace(NamespaceString("local", "oplog.rs")), + ActionType::collStats)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + return Status::OK(); } @@ -77,9 +99,10 @@ public: std::string& errmsg, BSONObjBuilder& result) override { - errmsg = "getDiagnosticData not allowed through mongos"; + result.append( + "data", FTDCController::get(txn->getServiceContext())->getMostRecentPeriodicDocument()); - return false; + return true; } }; diff --git a/src/mongo/s/commands/cluster_shard_collection_cmd.cpp b/src/mongo/s/commands/cluster_shard_collection_cmd.cpp index f1ba15396a9..eac3e2a24e0 100644 --- a/src/mongo/s/commands/cluster_shard_collection_cmd.cpp +++ b/src/mongo/s/commands/cluster_shard_collection_cmd.cpp @@ -52,6 +52,7 @@ #include "mongo/db/write_concern_options.h" #include "mongo/s/balancer_configuration.h" #include "mongo/s/catalog/sharding_catalog_client.h" +#include "mongo/s/catalog/type_database.h" #include "mongo/s/catalog_cache.h" #include "mongo/s/client/shard_registry.h" #include "mongo/s/commands/cluster_write.h" @@ -186,12 +187,17 @@ public: auto const shardRegistry = Grid::get(opCtx)->shardRegistry(); auto const catalogCache = Grid::get(opCtx)->catalogCache(); - auto dbInfo = uassertStatusOK(catalogCache->getDatabase(opCtx, nss.db())); - - // Ensure sharding is allowed on the database - uassert(ErrorCodes::IllegalOperation, - str::stream() << "sharding not enabled for db " << nss.db(), - dbInfo.shardingEnabled()); + // Ensure sharding is allowed on the database by reading directly from the config server, + // because reading through the cache might produce a stale entry if the "enableSharding" was + // called through a different mongos + { + const auto opTimeWithDbt = + uassertStatusOK(catalogClient->getDatabase(opCtx, nss.db().toString())); + const auto& dbt = opTimeWithDbt.value; + uassert(ErrorCodes::IllegalOperation, + str::stream() << "sharding not enabled for db " << nss.db(), + dbt.getSharded()); + } auto routingInfo = uassertStatusOK(catalogCache->getCollectionRoutingInfo(opCtx, nss)); diff --git a/src/mongo/s/commands/cluster_write.cpp b/src/mongo/s/commands/cluster_write.cpp index b17035993d9..565a1ddc4d7 100644 --- a/src/mongo/s/commands/cluster_write.cpp +++ b/src/mongo/s/commands/cluster_write.cpp @@ -65,28 +65,6 @@ void toBatchError(const Status& status, BatchedCommandResponse* response) { dassert(response->isValid(NULL)); } -/** - * Given a maxChunkSize configuration and the number of chunks in a particular sharded collection, - * returns an optimal chunk size to use in order to achieve a good ratio between number of chunks - * and their size. - */ -uint64_t calculateDesiredChunkSize(uint64_t maxChunkSizeBytes, uint64_t numChunks) { - // Splitting faster in early chunks helps spread out an initial load better - const uint64_t minChunkSize = 1 << 20; // 1 MBytes - - if (numChunks <= 1) { - return 1024; - } else if (numChunks < 3) { - return minChunkSize / 2; - } else if (numChunks < 10) { - return std::max(maxChunkSizeBytes / 4, minChunkSize); - } else if (numChunks < 20) { - return std::max(maxChunkSizeBytes / 2, minChunkSize); - } else { - return maxChunkSizeBytes; - } -} - /** * Returns the split point that will result in one of the chunk having exactly one document. Also * returns an empty document if the split point cannot be determined. @@ -341,8 +319,7 @@ void updateChunkWriteStatsAndSplitIfNeeded(OperationContext* opCtx, const uint64_t chunkBytesWritten = chunk->addBytesWritten(dataWritten); - const uint64_t desiredChunkSize = - calculateDesiredChunkSize(balancerConfig->getMaxChunkSizeBytes(), manager->numChunks()); + const uint64_t desiredChunkSize = balancerConfig->getMaxChunkSizeBytes(); // If this chunk is at either end of the range, trigger auto-split at 10% less data written in // order to trigger the top-chunk optimization. diff --git a/src/mongo/s/move_chunk_request.cpp b/src/mongo/s/move_chunk_request.cpp index 4622f26fd29..104587ef751 100644 --- a/src/mongo/s/move_chunk_request.cpp +++ b/src/mongo/s/move_chunk_request.cpp @@ -195,7 +195,8 @@ bool MoveChunkRequest::operator==(const MoveChunkRequest& other) const { return false; if (_range != other._range) return false; - + if (_waitForDelete != other._waitForDelete) + return false; return true; } diff --git a/src/mongo/s/server.cpp b/src/mongo/s/server.cpp index 9177cccdb73..0b8a9a17651 100644 --- a/src/mongo/s/server.cpp +++ b/src/mongo/s/server.cpp @@ -51,6 +51,7 @@ #include "mongo/db/auth/user_cache_invalidator_job.h" #include "mongo/db/client.h" #include "mongo/db/dbwebserver.h" +#include "mongo/db/ftdc/ftdc_mongos.h" #include "mongo/db/initialize_server_global_state.h" #include "mongo/db/instance.h" #include "mongo/db/lasterror.h" @@ -152,6 +153,9 @@ static void cleanupTask() { if (auto catalog = Grid::get(txn)->catalogClient(txn)) { catalog->shutDown(txn); } + + // Shutdown Full-Time Data Capture + stopMongoSFTDC(); } audit::logShutdown(Client::getCurrent()); @@ -293,6 +297,8 @@ static ExitCode runMongosServer() { web.detach(); } + startMongoSFTDC(); + Status status = getGlobalAuthorizationManager()->initialize(NULL); if (!status.isOK()) { error() << "Initializing authorization data failed: " << status; diff --git a/src/mongo/s/sharding_initialization.cpp b/src/mongo/s/sharding_initialization.cpp index 2bf84b02d18..1c28829a696 100644 --- a/src/mongo/s/sharding_initialization.cpp +++ b/src/mongo/s/sharding_initialization.cpp @@ -182,6 +182,30 @@ Status initializeGlobalShardingState(OperationContext* txn, connPoolOptions.refreshRequirement = Milliseconds(ShardingTaskExecutorPoolRefreshRequirementMS); connPoolOptions.refreshTimeout = Milliseconds(ShardingTaskExecutorPoolRefreshTimeoutMS); + if (connPoolOptions.refreshRequirement <= connPoolOptions.refreshTimeout) { + auto newRefreshTimeout = connPoolOptions.refreshRequirement - Milliseconds(1); + warning() << "ShardingTaskExecutorPoolRefreshRequirementMS (" + << connPoolOptions.refreshRequirement + << ") set below ShardingTaskExecutorPoolRefreshTimeoutMS (" + << connPoolOptions.refreshTimeout + << "). Adjusting ShardingTaskExecutorPoolRefreshTimeoutMS to " + << newRefreshTimeout; + connPoolOptions.refreshTimeout = newRefreshTimeout; + } + + if (connPoolOptions.hostTimeout <= + connPoolOptions.refreshRequirement + connPoolOptions.refreshTimeout) { + auto newHostTimeout = + connPoolOptions.refreshRequirement + connPoolOptions.refreshTimeout + Milliseconds(1); + warning() << "ShardingTaskExecutorPoolHostTimeoutMS (" << connPoolOptions.hostTimeout + << ") set below ShardingTaskExecutorPoolRefreshRequirementMS (" + << connPoolOptions.refreshRequirement + << ") + ShardingTaskExecutorPoolRefreshTimeoutMS (" + << connPoolOptions.refreshTimeout + << "). Adjusting ShardingTaskExecutorPoolHostTimeoutMS to " << newHostTimeout; + connPoolOptions.hostTimeout = newHostTimeout; + } + auto network = executor::makeNetworkInterface("NetworkInterfaceASIO-ShardRegistry", stdx::make_unique(), diff --git a/src/mongo/scripting/deadline_monitor.cpp b/src/mongo/scripting/deadline_monitor.cpp index f75c34a0cc2..5eb0f52e5de 100644 --- a/src/mongo/scripting/deadline_monitor.cpp +++ b/src/mongo/scripting/deadline_monitor.cpp @@ -34,7 +34,7 @@ namespace mongo { -MONGO_EXPORT_SERVER_PARAMETER(scriptingEngineInterruptIntervalMS, int, 0); +MONGO_EXPORT_SERVER_PARAMETER(scriptingEngineInterruptIntervalMS, int, 1000); int getScriptingEngineInterruptInterval() { return scriptingEngineInterruptIntervalMS.load(); diff --git a/src/mongo/scripting/deadline_monitor.h b/src/mongo/scripting/deadline_monitor.h index b3c5007c72d..9081d7366e6 100644 --- a/src/mongo/scripting/deadline_monitor.h +++ b/src/mongo/scripting/deadline_monitor.h @@ -141,10 +141,10 @@ private: const Date_t now = Date_t::now(); const auto interruptInterval = Milliseconds{getScriptingEngineInterruptInterval()}; - if ((interruptInterval.count() > 0) && (now - lastInterruptCycle > interruptInterval)) { + if (now - lastInterruptCycle > interruptInterval) { for (const auto& task : _tasks) { - if (task.second > now) - task.first->interrupt(); + if (task.first->isKillPending()) + task.first->kill(); } lastInterruptCycle = now; } diff --git a/src/mongo/scripting/deadline_monitor_test.cpp b/src/mongo/scripting/deadline_monitor_test.cpp index 71daefbce8f..d802673bd3d 100644 --- a/src/mongo/scripting/deadline_monitor_test.cpp +++ b/src/mongo/scripting/deadline_monitor_test.cpp @@ -73,8 +73,12 @@ public: _group->noteKill(); } void interrupt() {} + const bool isKillPending() { + return killPending; + } TaskGroup* _group; uint64_t _killed; + bool killPending = false; }; // single task expires before stopping the deadline @@ -175,4 +179,13 @@ TEST(DeadlineMonitor, MultipleTasksExpireOrComplete) { } } +TEST(DeadlineMonitor, IsKillPendingKills) { + DeadlineMonitor dm; + TaskGroup group; + Task task(&group); + dm.startDeadline(&task, -1); + task.killPending = true; + group.waitForKillCount(1); + ASSERT(task._killed); +} } // namespace mongo diff --git a/src/mongo/scripting/engine.cpp b/src/mongo/scripting/engine.cpp index 3e9a5c542cf..1848bccfba2 100644 --- a/src/mongo/scripting/engine.cpp +++ b/src/mongo/scripting/engine.cpp @@ -42,6 +42,7 @@ #include "mongo/db/service_context.h" #include "mongo/platform/unordered_set.h" #include "mongo/scripting/dbdirectclient_factory.h" +#include "mongo/util/fail_point_service.h" #include "mongo/util/file.h" #include "mongo/util/log.h" #include "mongo/util/text.h" @@ -55,7 +56,10 @@ using std::unique_ptr; AtomicInt64 Scope::_lastVersion(1); + namespace { + +MONGO_FP_DECLARE(mr_killop_test_fp); // 2 GB is the largest support Javascript file size. const fileofs kMaxJsFileLength = fileofs(2) * 1024 * 1024 * 1024; @@ -231,6 +235,15 @@ void Scope::loadStored(OperationContext* txn, bool ignoreNotConnected) { uassert(10209, str::stream() << "name has to be a string: " << n, n.type() == String); uassert(10210, "value has to be set", v.type() != EOO); + if (MONGO_FAIL_POINT(mr_killop_test_fp)) { + + /* This thread sleep makes the interrupts in the test come in at a time + * where the js misses the interrupt and throw an exception instead of + * being interrupted + */ + stdx::this_thread::sleep_for(stdx::chrono::seconds(1)); + } + try { setElement(n.valuestr(), v, o); thisTime.insert(n.valuestr()); @@ -427,6 +440,9 @@ public: void advanceGeneration() { _real->advanceGeneration(); } + void requireOwnedObjects() override { + _real->requireOwnedObjects(); + } bool isKillPending() const { return _real->isKillPending(); } diff --git a/src/mongo/scripting/engine.h b/src/mongo/scripting/engine.h index 2b2ab0ff14d..e773dec5f60 100644 --- a/src/mongo/scripting/engine.h +++ b/src/mongo/scripting/engine.h @@ -44,7 +44,7 @@ class OperationContext; struct JSFile { const char* name; - const StringData& source; + const StringData source; }; class Scope { @@ -103,6 +103,8 @@ public: virtual void advanceGeneration() = 0; + virtual void requireOwnedObjects() = 0; + virtual ScriptingFunction createFunction(const char* code); /** diff --git a/src/mongo/scripting/mozjs/PosixNSPR.cpp b/src/mongo/scripting/mozjs/PosixNSPR.cpp index ed1a3d5a49d..9054e930443 100644 --- a/src/mongo/scripting/mozjs/PosixNSPR.cpp +++ b/src/mongo/scripting/mozjs/PosixNSPR.cpp @@ -24,6 +24,7 @@ #include "mongo/stdx/chrono.h" #include "mongo/stdx/condition_variable.h" +#include "mongo/stdx/memory.h" #include "mongo/stdx/mutex.h" #include "mongo/stdx/thread.h" #include "mongo/util/concurrency/thread_name.h" @@ -98,9 +99,13 @@ PRThread* PR_CreateThread(PRThreadType type, MOZ_ASSERT(priority == PR_PRIORITY_NORMAL); try { - std::unique_ptr t( - js_new(start, arg, state != PR_UNJOINABLE_THREAD), - js_delete_nonconst); + // We can't use the nspr allocator to allocate this thread, because under asan we + // instrument the allocator so that asan can track the pointers correctly. This + // instrumentation + // requires that pointers be deleted in the same thread that they were allocated in. + // The threads created in PR_CreateThread are not always freed in the same thread + // that they were created in. So, we use the standard allocator here. + auto t = mongo::stdx::make_unique(start, arg, state != PR_UNJOINABLE_THREAD); t->thread() = mongo::stdx::thread(&nspr::Thread::ThreadRoutine, t.get()); @@ -118,7 +123,7 @@ PRStatus PR_JoinThread(PRThread* thread) { try { thread->thread().join(); - js_delete(thread); + delete thread; return PR_SUCCESS; } catch (...) { diff --git a/src/mongo/scripting/mozjs/bson.cpp b/src/mongo/scripting/mozjs/bson.cpp index 5a2ebd0dfed..a2881c44664 100644 --- a/src/mongo/scripting/mozjs/bson.cpp +++ b/src/mongo/scripting/mozjs/bson.cpp @@ -59,13 +59,17 @@ namespace { * the appearance of mutable state on the read/write versions. */ struct BSONHolder { - BSONHolder(const BSONObj& obj, const BSONObj* parent, std::size_t generation, bool ro) + BSONHolder(const BSONObj& obj, const BSONObj* parent, const MozJSImplScope* scope, bool ro) : _obj(obj), - _generation(generation), + _generation(scope->getGeneration()), _isOwned(obj.isOwned() || (parent && parent->isOwned())), _resolved(false), _readOnly(ro), _altered(false) { + uassert( + ErrorCodes::BadValue, + "Attempt to bind an unowned BSON Object to a JS scope marked as requiring ownership", + _isOwned || (!scope->requiresOwnedObjects())); if (parent) { _parent.emplace(*parent); } @@ -107,7 +111,7 @@ void BSONInfo::make( auto scope = getScope(cx); scope->getProto().newObject(obj); - JS_SetPrivate(obj, scope->trackedNew(bson, parent, scope->getGeneration(), ro)); + JS_SetPrivate(obj, scope->trackedNew(bson, parent, scope, ro)); } void BSONInfo::finalize(JSFreeOp* fop, JSObject* obj) { diff --git a/src/mongo/scripting/mozjs/db.cpp b/src/mongo/scripting/mozjs/db.cpp index 7fa2f179241..7aa73337237 100644 --- a/src/mongo/scripting/mozjs/db.cpp +++ b/src/mongo/scripting/mozjs/db.cpp @@ -45,38 +45,17 @@ namespace mozjs { const char* const DBInfo::className = "DB"; -void DBInfo::getProperty(JSContext* cx, - JS::HandleObject obj, - JS::HandleId id, - JS::MutableHandleValue vp) { - // 2nd look into real values, may be cached collection object - if (!vp.isUndefined()) { - auto scope = getScope(cx); - auto opContext = scope->getOpContext(); - - if (opContext && vp.isObject()) { - ObjectWrapper o(cx, vp); - - if (o.hasOwnField(InternedString::_fullName)) { - // need to check every time that the collection did not get sharded - if (haveLocalShardingInfo(opContext, o.getString(InternedString::_fullName))) - uasserted(ErrorCodes::BadValue, "can't use sharded collection from db.eval"); - } - } +void DBInfo::resolve(JSContext* cx, JS::HandleObject obj, JS::HandleId id, bool* resolvedp) { + *resolvedp = false; - return; - } + JS::RootedValue coll(cx); JS::RootedObject parent(cx); if (!JS_GetPrototype(cx, obj, &parent)) uasserted(ErrorCodes::JSInterpreterFailure, "Couldn't get prototype"); ObjectWrapper parentWrapper(cx, parent); - - if (parentWrapper.hasOwnField(id)) { - parentWrapper.getValue(id, vp); - return; - } + ObjectWrapper o(cx, obj); IdWrapper idw(cx, id); @@ -87,21 +66,39 @@ void DBInfo::getProperty(JSContext* cx, if (sname.size() == 0 || sname[0] == '_') { return; } + + // SpiderMonkey will call resolve even for __proto__ so acknowledge it exists. + if (sname == "__proto__"_sd) { + *resolvedp = true; + return; + } + } + + // Check if this exists on the parent, ie. DBCollection::resolve case + if (parentWrapper.hasOwnField(id)) { + parentWrapper.getValue(id, &coll); + + o.defineProperty(id, coll, 0); + + *resolvedp = true; + return; } // no hit, create new collection JS::RootedValue getCollection(cx); parentWrapper.getValue(InternedString::getCollection, &getCollection); + // Check if getCollection has been installed yet + // It is undefined if the user has a db name the same as one of methods/properties of the DB + // object. if (!(getCollection.isObject() && JS_ObjectIsFunction(cx, getCollection.toObjectOrNull()))) { - uasserted(ErrorCodes::BadValue, "getCollection is not a function"); + return; } JS::AutoValueArray<1> args(cx); idw.toValue(args[0]); - JS::RootedValue coll(cx); ObjectWrapper(cx, obj).callMethod(getCollection, args, &coll); uassert(16861, @@ -111,7 +108,7 @@ void DBInfo::getProperty(JSContext* cx, // cache collection for reuse, don't enumerate ObjectWrapper(cx, obj).defineProperty(id, coll, 0); - vp.set(coll); + *resolvedp = true; } void DBInfo::construct(JSContext* cx, JS::CallArgs args) { diff --git a/src/mongo/scripting/mozjs/db.h b/src/mongo/scripting/mozjs/db.h index c752953236f..8f080265ef0 100644 --- a/src/mongo/scripting/mozjs/db.h +++ b/src/mongo/scripting/mozjs/db.h @@ -44,10 +44,7 @@ namespace mozjs { */ struct DBInfo : public BaseInfo { static void construct(JSContext* cx, JS::CallArgs args); - static void getProperty(JSContext* cx, - JS::HandleObject obj, - JS::HandleId id, - JS::MutableHandleValue vp); + static void resolve(JSContext* cx, JS::HandleObject obj, JS::HandleId id, bool* resolvedp); static const char* const className; }; diff --git a/src/mongo/scripting/mozjs/dbcollection.cpp b/src/mongo/scripting/mozjs/dbcollection.cpp index e0f903dd1fc..b1983bd367c 100644 --- a/src/mongo/scripting/mozjs/dbcollection.cpp +++ b/src/mongo/scripting/mozjs/dbcollection.cpp @@ -44,11 +44,11 @@ namespace mozjs { const char* const DBCollectionInfo::className = "DBCollection"; -void DBCollectionInfo::getProperty(JSContext* cx, - JS::HandleObject obj, - JS::HandleId id, - JS::MutableHandleValue vp) { - DBInfo::getProperty(cx, obj, id, vp); +void DBCollectionInfo::resolve(JSContext* cx, + JS::HandleObject obj, + JS::HandleId id, + bool* resolvedp) { + DBInfo::resolve(cx, obj, id, resolvedp); } void DBCollectionInfo::construct(JSContext* cx, JS::CallArgs args) { diff --git a/src/mongo/scripting/mozjs/dbcollection.h b/src/mongo/scripting/mozjs/dbcollection.h index 34be422dfd4..ed3c39ff0f2 100644 --- a/src/mongo/scripting/mozjs/dbcollection.h +++ b/src/mongo/scripting/mozjs/dbcollection.h @@ -44,10 +44,7 @@ namespace mozjs { */ struct DBCollectionInfo : public BaseInfo { static void construct(JSContext* cx, JS::CallArgs args); - static void getProperty(JSContext* cx, - JS::HandleObject obj, - JS::HandleId id, - JS::MutableHandleValue vp); + static void resolve(JSContext* cx, JS::HandleObject obj, JS::HandleId id, bool* resolvedp); static const char* const className; }; diff --git a/src/mongo/scripting/mozjs/dbquery.cpp b/src/mongo/scripting/mozjs/dbquery.cpp index c2543814505..15bd45bde3c 100644 --- a/src/mongo/scripting/mozjs/dbquery.cpp +++ b/src/mongo/scripting/mozjs/dbquery.cpp @@ -105,13 +105,8 @@ void DBQueryInfo::construct(JSContext* cx, JS::CallArgs args) { args.rval().setObjectOrNull(thisv); } -void DBQueryInfo::getProperty(JSContext* cx, - JS::HandleObject obj, - JS::HandleId id, - JS::MutableHandleValue vp) { - if (!vp.isUndefined()) { - return; - } +void DBQueryInfo::resolve(JSContext* cx, JS::HandleObject obj, JS::HandleId id, bool* resolvedp) { + *resolvedp = false; IdWrapper wid(cx, id); @@ -134,9 +129,19 @@ void DBQueryInfo::getProperty(JSContext* cx, args[0].setInt32(wid.toInt32()); - ObjectWrapper(cx, obj).callMethod(arrayAccess, args, vp); - } else { - uasserted(ErrorCodes::BadValue, "arrayAccess is not a function"); + JS::RootedValue vp(cx); + + ObjectWrapper(cx, obj).callMethod(arrayAccess, args, &vp); + + if (!vp.isNullOrUndefined()) { + ObjectWrapper o(cx, obj); + + // Assumes the user won't modify the contents of what DBQuery::arrayAccess returns + // otherwise we need to install a getter. + o.defineProperty(id, vp, 0); + } + + *resolvedp = true; } } diff --git a/src/mongo/scripting/mozjs/dbquery.h b/src/mongo/scripting/mozjs/dbquery.h index dc844c3084c..590185c55ee 100644 --- a/src/mongo/scripting/mozjs/dbquery.h +++ b/src/mongo/scripting/mozjs/dbquery.h @@ -41,10 +41,8 @@ namespace mozjs { */ struct DBQueryInfo : public BaseInfo { static void construct(JSContext* cx, JS::CallArgs args); - static void getProperty(JSContext* cx, - JS::HandleObject obj, - JS::HandleId id, - JS::MutableHandleValue vp); + static void resolve(JSContext* cx, JS::HandleObject obj, JS::HandleId id, bool* resolvedp); + static const char* const className; }; diff --git a/src/mongo/scripting/mozjs/implscope.cpp b/src/mongo/scripting/mozjs/implscope.cpp index c37ebc0cb2a..c31d5b940a4 100644 --- a/src/mongo/scripting/mozjs/implscope.cpp +++ b/src/mongo/scripting/mozjs/implscope.cpp @@ -258,7 +258,6 @@ MozJSImplScope::ASANHandles::ASANHandles() { } MozJSImplScope::ASANHandles::~ASANHandles() { - invariant(_handles.empty()); invariant(kCurrentASANHandles == this); kCurrentASANHandles = nullptr; } @@ -342,6 +341,14 @@ MozJSImplScope::MozRuntime::MozRuntime(const MozJSScriptEngine* engine) { .setIon(true) .setAsyncStack(false) .setNativeRegExp(true); + } else { + JS::RuntimeOptionsRef(_runtime.get()) + .setAsmJS(false) + .setThrowOnAsmJSValidationFailure(false) + .setBaseline(false) + .setIon(false) + .setAsyncStack(false) + .setNativeRegExp(false); } const StackLocator locator; @@ -403,6 +410,7 @@ MozJSImplScope::MozJSImplScope(MozJSScriptEngine* engine) _connectState(ConnectState::Not), _status(Status::OK()), _generation(0), + _requireOwnedObjects(false), _hasOutOfMemoryException(false), _binDataProto(_context), _bsonProto(_context), @@ -491,82 +499,81 @@ void MozJSImplScope::init(const BSONObj* data) { } } -void MozJSImplScope::setNumber(const char* field, double val) { - MozJSEntry entry(this); +template +auto MozJSImplScope::_runSafely(ImplScopeFunction&& functionToRun) -> decltype(functionToRun()) { + try { + MozJSEntry entry(this); + return functionToRun(); + } catch (...) { + _error = _status.reason(); - ObjectWrapper(_context, _global).setNumber(field, val); + // Clear the status state + auto status = std::move(_status); + uassertStatusOK(status); + throw; + } } -void MozJSImplScope::setString(const char* field, StringData val) { - MozJSEntry entry(this); +void MozJSImplScope::setNumber(const char* field, double val) { + _runSafely([this, &field, &val] { ObjectWrapper(_context, _global).setNumber(field, val); }); +} - ObjectWrapper(_context, _global).setString(field, val); +void MozJSImplScope::setString(const char* field, StringData val) { + _runSafely([this, &field, &val] { ObjectWrapper(_context, _global).setString(field, val); }); } void MozJSImplScope::setBoolean(const char* field, bool val) { - MozJSEntry entry(this); - - ObjectWrapper(_context, _global).setBoolean(field, val); + _runSafely([this, &field, &val] { ObjectWrapper(_context, _global).setBoolean(field, val); }); } void MozJSImplScope::setElement(const char* field, const BSONElement& e, const BSONObj& parent) { - MozJSEntry entry(this); + _runSafely([this, &field, &e, &parent] { - ObjectWrapper(_context, _global).setBSONElement(field, e, parent, false); + ObjectWrapper(_context, _global).setBSONElement(field, e, parent, false); + }); } void MozJSImplScope::setObject(const char* field, const BSONObj& obj, bool readOnly) { - MozJSEntry entry(this); + _runSafely([this, &field, &obj, &readOnly] { - ObjectWrapper(_context, _global).setBSON(field, obj, readOnly); + ObjectWrapper(_context, _global).setBSON(field, obj, readOnly); + }); } int MozJSImplScope::type(const char* field) { - MozJSEntry entry(this); - - return ObjectWrapper(_context, _global).type(field); + return _runSafely([this, &field] { return ObjectWrapper(_context, _global).type(field); }); } double MozJSImplScope::getNumber(const char* field) { - MozJSEntry entry(this); - - return ObjectWrapper(_context, _global).getNumber(field); + return _runSafely([this, &field] { return ObjectWrapper(_context, _global).getNumber(field); }); } int MozJSImplScope::getNumberInt(const char* field) { - MozJSEntry entry(this); - - return ObjectWrapper(_context, _global).getNumberInt(field); + return _runSafely( + [this, &field] { return ObjectWrapper(_context, _global).getNumberInt(field); }); } long long MozJSImplScope::getNumberLongLong(const char* field) { - MozJSEntry entry(this); - - return ObjectWrapper(_context, _global).getNumberLongLong(field); + return _runSafely( + [this, &field] { return ObjectWrapper(_context, _global).getNumberLongLong(field); }); } Decimal128 MozJSImplScope::getNumberDecimal(const char* field) { - MozJSEntry entry(this); - - return ObjectWrapper(_context, _global).getNumberDecimal(field); + return _runSafely( + [this, &field] { return ObjectWrapper(_context, _global).getNumberDecimal(field); }); } std::string MozJSImplScope::getString(const char* field) { - MozJSEntry entry(this); - - return ObjectWrapper(_context, _global).getString(field); + return _runSafely([this, &field] { return ObjectWrapper(_context, _global).getString(field); }); } bool MozJSImplScope::getBoolean(const char* field) { - MozJSEntry entry(this); - - return ObjectWrapper(_context, _global).getBoolean(field); + return _runSafely( + [this, &field] { return ObjectWrapper(_context, _global).getBoolean(field); }); } BSONObj MozJSImplScope::getObject(const char* field) { - MozJSEntry entry(this); - - return ObjectWrapper(_context, _global).getObject(field); + return _runSafely([this, &field] { return ObjectWrapper(_context, _global).getObject(field); }); } void MozJSImplScope::newFunction(StringData raw, JS::MutableHandleValue out) { @@ -642,17 +649,15 @@ ScriptingFunction MozJSImplScope::_createFunction(const char* raw) { } void MozJSImplScope::setFunction(const char* field, const char* code) { - MozJSEntry entry(this); - - JS::RootedValue fun(_context); - _MozJSCreateFunction(code, &fun); - ObjectWrapper(_context, _global).setValue(field, fun); + _runSafely([this, &field, &code] { + JS::RootedValue fun(_context); + _MozJSCreateFunction(code, &fun); + ObjectWrapper(_context, _global).setValue(field, fun); + }); } void MozJSImplScope::rename(const char* from, const char* to) { - MozJSEntry entry(this); - - ObjectWrapper(_context, _global).rename(from, to); + _runSafely([this, &from, &to] { ObjectWrapper(_context, _global).rename(from, to); }); } int MozJSImplScope::invoke(ScriptingFunction func, @@ -764,15 +769,15 @@ bool MozJSImplScope::exec(StringData code, } void MozJSImplScope::injectNative(const char* field, NativeFunction func, void* data) { - MozJSEntry entry(this); - - JS::RootedObject obj(_context); + _runSafely([this, &field, &func, &data] { + JS::RootedObject obj(_context); - NativeFunctionInfo::make(_context, &obj, func, data); + NativeFunctionInfo::make(_context, &obj, func, data); - JS::RootedValue value(_context); - value.setObjectOrNull(obj); - ObjectWrapper(_context, _global).setValue(field, value); + JS::RootedValue value(_context); + value.setObjectOrNull(obj); + ObjectWrapper(_context, _global).setValue(field, value); + }); } void MozJSImplScope::gc() { @@ -781,63 +786,65 @@ void MozJSImplScope::gc() { } void MozJSImplScope::localConnectForDbEval(OperationContext* txn, const char* dbName) { - MozJSEntry entry(this); - - if (_connectState == ConnectState::External) - uasserted(12510, "externalSetup already called, can't call localConnect"); - if (_connectState == ConnectState::Local) { - if (_localDBName == dbName) - return; - uasserted(12511, - str::stream() << "localConnect previously called with name " << _localDBName); - } + _runSafely([this, &txn, &dbName] { + if (_connectState == ConnectState::External) + uasserted(12510, "externalSetup already called, can't call localConnect"); + if (_connectState == ConnectState::Local) { + if (_localDBName == dbName) + return; + uasserted(12511, + str::stream() << "localConnect previously called with name " << _localDBName); + } - // NOTE: order is important here. the following methods must be called after - // the above conditional statements. + // NOTE: order is important here. the following methods must be called after + // the above conditional statements. - _connectState = ConnectState::Local; - _localDBName = dbName; + _connectState = ConnectState::Local; + _localDBName = dbName; - loadStored(txn); + loadStored(txn); - // install db access functions in the global object - installDBAccess(); + // install db access functions in the global object + installDBAccess(); - // install the Mongo function object and instantiate the 'db' global - _mongoLocalProto.install(_global); - execCoreFiles(); + // install the Mongo function object and instantiate the 'db' global + _mongoLocalProto.install(_global); + execCoreFiles(); - const char* const makeMongo = "const _mongo = new Mongo()"; - exec(makeMongo, "local connect 2", false, true, true, 0); + const char* const makeMongo = "const _mongo = new Mongo()"; + exec(makeMongo, "local connect 2", false, true, true, 0); - std::string makeDB = str::stream() << "const db = _mongo.getDB(\"" << dbName << "\");"; - exec(makeDB, "local connect 3", false, true, true, 0); + std::string makeDB = str::stream() << "const db = _mongo.getDB(\"" << dbName << "\");"; + exec(makeDB, "local connect 3", false, true, true, 0); + }); } void MozJSImplScope::externalSetup() { - MozJSEntry entry(this); - if (_connectState == ConnectState::External) - return; - if (_connectState == ConnectState::Local) - uasserted(12512, "localConnect already called, can't call externalSetup"); + _runSafely([&] { + if (_connectState == ConnectState::External) + return; + if (_connectState == ConnectState::Local) + uasserted(12512, "localConnect already called, can't call externalSetup"); - // install db access functions in the global object - installDBAccess(); + // install db access functions in the global object + installDBAccess(); - // install thread-related functions (e.g. _threadInject) - installFork(); + // install thread-related functions (e.g. _threadInject) + installFork(); - // install the Mongo function object - _mongoExternalProto.install(_global); - execCoreFiles(); - _connectState = ConnectState::External; + // install the Mongo function object + _mongoExternalProto.install(_global); + execCoreFiles(); + _connectState = ConnectState::External; + }); } void MozJSImplScope::reset() { unregisterOperation(); _pendingKill.store(false); _pendingGC.store(false); + _requireOwnedObjects = false; advanceGeneration(); } @@ -951,6 +958,14 @@ void MozJSImplScope::advanceGeneration() { _generation++; } +void MozJSImplScope::requireOwnedObjects() { + _requireOwnedObjects = true; +} + +bool MozJSImplScope::requiresOwnedObjects() const { + return _requireOwnedObjects; +} + const std::string& MozJSImplScope::getParentStack() const { return _parentStack; } diff --git a/src/mongo/scripting/mozjs/implscope.h b/src/mongo/scripting/mozjs/implscope.h index df37553e181..2de309e87ab 100644 --- a/src/mongo/scripting/mozjs/implscope.h +++ b/src/mongo/scripting/mozjs/implscope.h @@ -309,6 +309,10 @@ public: void advanceGeneration() override; + void requireOwnedObjects() override; + + bool requiresOwnedObjects() const; + JS::HandleId getInternedStringId(InternedString name) { return _internedStrings.getInternedString(name); } @@ -341,6 +345,9 @@ public: }; private: + template + auto _runSafely(ImplScopeFunction&& functionToRun) -> decltype(functionToRun()); + void _MozJSCreateFunction(StringData raw, JS::MutableHandleValue fun); /** @@ -402,6 +409,7 @@ private: Status _status; std::string _parentStack; std::size_t _generation; + bool _requireOwnedObjects; bool _hasOutOfMemoryException; WrapType _binDataProto; diff --git a/src/mongo/scripting/mozjs/mongohelpers.js b/src/mongo/scripting/mozjs/mongohelpers.js index b0b35bb2fe8..d3f743623a3 100644 --- a/src/mongo/scripting/mozjs/mongohelpers.js +++ b/src/mongo/scripting/mozjs/mongohelpers.js @@ -33,6 +33,14 @@ exportToMongoHelpers = { // This function accepts an expression or function body and returns a function definition 'functionExpressionParser': function functionExpressionParser(fnSrc) { + + // Ensure that a provided expression or function body is not terminated with a ';'. + // This ensures we interpret the input as a single expression, rather than a sequence + // of expressions, and can wrap it in parentheses. + while (fnSrc.endsWith(";") || fnSrc != fnSrc.trimRight()) { + fnSrc = fnSrc.slice(0, -1).trimRight(); + } + var parseTree; try { parseTree = this.Reflect.parse(fnSrc); diff --git a/src/mongo/scripting/mozjs/proxyscope.cpp b/src/mongo/scripting/mozjs/proxyscope.cpp index 6a6d20f4448..05697468f07 100644 --- a/src/mongo/scripting/mozjs/proxyscope.cpp +++ b/src/mongo/scripting/mozjs/proxyscope.cpp @@ -122,6 +122,10 @@ void MozJSProxyScope::advanceGeneration() { run([&] { _implScope->advanceGeneration(); }); } +void MozJSProxyScope::requireOwnedObjects() { + run([&] { _implScope->requireOwnedObjects(); }); +} + double MozJSProxyScope::getNumber(const char* field) { double out; run([&] { out = _implScope->getNumber(field); }); diff --git a/src/mongo/scripting/mozjs/proxyscope.h b/src/mongo/scripting/mozjs/proxyscope.h index 451981330a1..4dd69a3ebe9 100644 --- a/src/mongo/scripting/mozjs/proxyscope.h +++ b/src/mongo/scripting/mozjs/proxyscope.h @@ -129,6 +129,8 @@ public: void advanceGeneration() override; + void requireOwnedObjects() override; + double getNumber(const char* field) override; int getNumberInt(const char* field) override; long long getNumberLongLong(const char* field) override; diff --git a/src/mongo/shell/bench.cpp b/src/mongo/shell/bench.cpp index 040002f5c6e..338477ebe56 100644 --- a/src/mongo/shell/bench.cpp +++ b/src/mongo/shell/bench.cpp @@ -674,7 +674,7 @@ void BenchRunWorker::generateLoadOnConnection(DBClientBase* conn) { invariant(bsonTemplateEvaluator.setId(_id) == BsonTemplateEvaluator::StatusSuccess); if (_config->username != "") { - string errmsg; + std::string errmsg; if (!conn->auth("admin", _config->username, _config->password, errmsg)) { uasserted(15931, "Authenticating to connection for _benchThread failed: " + errmsg); } @@ -918,7 +918,7 @@ void BenchRunWorker::generateLoadOnConnection(DBClientBase* conn) { if (!result["err"].eoo() && result["err"].type() == String && (_config->throwGLE || op.throwGLE)) - throw DBException((string) "From benchRun GLE" + + throw DBException((std::string) "From benchRun GLE" + causedBy(result["err"].String()), result["code"].eoo() ? 0 : result["code"].Int()); } @@ -984,7 +984,7 @@ void BenchRunWorker::generateLoadOnConnection(DBClientBase* conn) { if (!result["err"].eoo() && result["err"].type() == String && (_config->throwGLE || op.throwGLE)) - throw DBException((string) "From benchRun GLE" + + throw DBException((std::string) "From benchRun GLE" + causedBy(result["err"].String()), result["code"].eoo() ? 0 : result["code"].Int()); } @@ -1031,7 +1031,7 @@ void BenchRunWorker::generateLoadOnConnection(DBClientBase* conn) { if (!result["err"].eoo() && result["err"].type() == String && (_config->throwGLE || op.throwGLE)) - throw DBException((string) "From benchRun GLE " + + throw DBException((std::string) "From benchRun GLE " + causedBy(result["err"].String()), result["code"].eoo() ? 0 : result["code"].Int()); } @@ -1133,7 +1133,7 @@ void BenchRunWorker::run() { try { std::unique_ptr conn(_config->createConnection()); if (!_config->username.empty()) { - string errmsg; + std::string errmsg; if (!conn->auth("admin", _config->username, _config->password, errmsg)) { uasserted(15932, "Authenticating to connection for benchThread failed: " + errmsg); } @@ -1165,7 +1165,7 @@ void BenchRunner::start() { std::unique_ptr conn(_config->createConnection()); // Must authenticate to admin db in order to run serverStatus command if (_config->username != "") { - string errmsg; + std::string errmsg; if (!conn->auth("admin", _config->username, _config->password, errmsg)) { uasserted( 16704, @@ -1201,7 +1201,7 @@ void BenchRunner::stop() { { std::unique_ptr conn(_config->createConnection()); if (_config->username != "") { - string errmsg; + std::string errmsg; // this can only fail if admin access was revoked since start of run if (!conn->auth("admin", _config->username, _config->password, errmsg)) { uasserted( diff --git a/src/mongo/shell/dbshell.cpp b/src/mongo/shell/dbshell.cpp index 801130d07c5..90c59f0ee55 100644 --- a/src/mongo/shell/dbshell.cpp +++ b/src/mongo/shell/dbshell.cpp @@ -43,6 +43,7 @@ #include "mongo/base/initializer.h" #include "mongo/base/status.h" #include "mongo/client/dbclientinterface.h" +#include "mongo/client/mongo_uri.h" #include "mongo/client/sasl_client_authenticate.h" #include "mongo/db/client.h" #include "mongo/db/log_process_details.h" @@ -215,57 +216,151 @@ void setupSignals() { signal(SIGINT, quitNicely); } -string getURIFromArgs(const std::string& url, const std::string& host, const std::string& port) { - if (host.size() == 0 && port.size() == 0) { - return url.size() == 0 ? kDefaultMongoURL.toString() : url; +string getURIFromArgs(const std::string& arg, const std::string& host, const std::string& port) { + if (host.empty() && arg.empty() && port.empty()) { + // Nothing provided, just play the default. + return kDefaultMongoURL.toString(); } - // The name URL is misleading; really it's just a positional argument that wasn't a file. The - // check for "/" means "this 'URL' is probably a real URL and not the db name (e.g.)". - if (url.find("/") != string::npos) { - cerr << "if a full URI is provided, you cannot also specify host or port" << endl; + if (str::startsWith(arg, "mongodb://") && host.empty() && port.empty()) { + // mongo mongodb://blah + return arg; + } + if (str::startsWith(host, "mongodb://") && arg.empty() && port.empty()) { + // mongo --host mongodb://blah + return host; + } + + // We expect a positional arg to be a plain dbname or plain hostname at this point + // since we have separate host/port args. + if ((arg.find('/') != string::npos) && (host.size() || port.size())) { + cerr << "If a full URI is provided, you cannot also specify --host or --port" << endl; quickExit(-1); } - bool hostEndsInSock = str::endsWith(host, ".sock"); + const auto parseDbHost = [port](const std::string& db, const std::string& host) -> std::string { + // Parse --host as a connection string. + // e.g. rs0/host0:27000,host1:27001 + const auto slashPos = host.find('/'); + const auto hasReplSet = (slashPos > 0) && (slashPos != std::string::npos); + + std::ostringstream ss; + ss << "mongodb://"; + + // Handle each sub-element of the connection string individually. + // Comma separated list of host elements. + // Each host element may be: + // * /unix/domain.sock + // * hostname + // * hostname:port + // If --port is specified and port is included in connection string, + // then they must match exactly. + auto start = hasReplSet ? slashPos + 1 : 0; + while (start < host.size()) { + // Encode each host component. + auto end = host.find(',', start); + if (end == std::string::npos) { + end = host.size(); + } + if ((end - start) == 0) { + // Ignore empty components. + start = end + 1; + continue; + } + + const auto hostElem = host.substr(start, end - start); + if ((hostElem.find('/') != std::string::npos) && str::endsWith(hostElem, ".sock")) { + // Unix domain socket, ignore --port. + ss << uriEncode(hostElem); + + } else { + auto colon = hostElem.find(':'); + if ((colon != std::string::npos) && + (hostElem.find(':', colon + 1) != std::string::npos)) { + // Looks like an IPv6 numeric address. + const auto close = hostElem.find(']'); + if ((hostElem[0] == '[') && (close != std::string::npos)) { + // Encapsulated already. + ss << '[' << uriEncode(hostElem.substr(1, close - 1), ":") << ']'; + colon = hostElem.find(':', close + 1); + } else { + // Not encapsulated yet. + ss << '[' << uriEncode(hostElem, ":") << ']'; + colon = std::string::npos; + } + } else if (colon != std::string::npos) { + // Not IPv6 numeric, but does have a port. + ss << uriEncode(hostElem.substr(0, colon)); + } else { + // Raw hostname/IPv4 without port. + ss << uriEncode(hostElem); + } + + if (colon != std::string::npos) { + // Have a port in our host element, verify it. + const auto myport = hostElem.substr(colon + 1); + if (port.size() && (port != myport)) { + cerr << "connection string bears different port than provided by --port" + << endl; + quickExit(-1); + } + ss << ':' << uriEncode(myport); + } else if (port.size()) { + ss << ':' << uriEncode(port); + } else { + ss << ":27017"; + } + } + start = end + 1; + if (start < host.size()) { + ss << ','; + } + } + + ss << '/' << uriEncode(db); - // If host looks like a full URI (i.e. has a slash and isn't a unix socket) and the other fields - // are empty, then just return host. - std::string::size_type slashPos; - if (url.size() == 0 && port.size() == 0 && - (!hostEndsInSock && ((slashPos = host.find("/")) != string::npos))) { - if (str::startsWith(host, "mongodb://")) { - return host; + if (hasReplSet) { + // Remap included replica set name to URI option + ss << "?replicaSet=" << uriEncode(host.substr(0, slashPos)); } - // If there's a slash in the host field, then it's the replica set name, not a database name - stringstream ss; - ss << "mongodb://" << host.substr(slashPos + 1) - << "/?replicaSet=" << host.substr(0, slashPos); + return ss.str(); - } + }; + + if (host.size()) { + // --host provided, treat it as the connect string and get db from positional arg. + return parseDbHost(arg, host); + } else if (arg.size()) { + // --host missing, but we have a potential host/db positional arg. + const auto slashPos = arg.find('/'); + if (slashPos != std::string::npos) { + // host/db pair. + return parseDbHost(arg.substr(slashPos + 1), arg.substr(0, slashPos)); + } - stringstream ss; - if (host.size() == 0) { - ss << "mongodb://127.0.0.1"; - } else { - if (!str::startsWith(host, "mongodb://")) { - ss << "mongodb://"; + // Compatability formats. + // * Any arg with a dot is assumed to be a hostname or IPv4 numeric address. + // * Any arg with a colon followed by a digit assumed to be host or IP followed by port. + // * Anything else is assumed to be a db. + + if (arg.find('.') != std::string::npos) { + // Assume IPv4 or hostnameish. + return parseDbHost("test", arg); } - ss << host; - } - if (!hostEndsInSock) { - if (port.size() > 0) { - ss << ":" << port; - } else if (host.find(':') == string::npos || str::endsWith(host, "]")) { - // Default the port to 27017 if the host did not provide one (i.e. the host has no - // colons or ends in ']' like an IPv6 address). - ss << ":27017"; + const auto colonPos = arg.find(':'); + if ((colonPos != std::string::npos) && ((colonPos + 1) < arg.size()) && + isdigit(arg[colonPos + 1])) { + // Assume IPv4 or hostname with port. + return parseDbHost("test", arg); } + + // db, assume localhost. + return parseDbHost(arg, "127.0.0.1"); } - ss << "/" << url; - return ss.str(); + // --host empty, position arg empty, fallback on localhost without a dbname. + return parseDbHost("", "127.0.0.1"); } static string OpSymbols = "~!%^&*-+=|:,<>/?."; diff --git a/src/mongo/shell/mongo.js b/src/mongo/shell/mongo.js index 69971609bb8..92fc83b38b9 100644 --- a/src/mongo/shell/mongo.js +++ b/src/mongo/shell/mongo.js @@ -223,6 +223,9 @@ connect = function(url, user, pass) { if (!url.startsWith("mongodb://")) { const colon = url.lastIndexOf(":"); const slash = url.lastIndexOf("/"); + if (url.split("/").length > 1) { + url = url.substring(0, slash).replace(/\//g, "%2F") + url.substring(slash); + } if (slash == 0) { throw Error("Failed to parse mongodb:// URL: " + url); } @@ -235,7 +238,7 @@ connect = function(url, user, pass) { chatty("connecting to: " + url); var m = new Mongo(url); - db = m.getDB(m.defaultDB); + var db = m.getDB(m.defaultDB); if (user && pass) { if (!db.auth(user, pass)) { diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js index 0e05a1bfa4a..9d6a4378009 100644 --- a/src/mongo/shell/replsettest.js +++ b/src/mongo/shell/replsettest.js @@ -427,8 +427,10 @@ var ReplSetTest = function(opts) { var member = {}; member._id = i; - var port = this.ports[i]; - member.host = this.host + ":" + port; + member.host = this.host; + if (!member.host.contains('/')) { + member.host += ":" + this.ports[i]; + } var nodeOpts = this.nodeOptions["n" + i]; if (nodeOpts) { @@ -470,7 +472,7 @@ var ReplSetTest = function(opts) { * * @param options - The options passed to {@link MongoRunner.runMongod} */ - this.startSet = function(options) { + this.startSet = function(options, restart) { print("ReplSetTest starting set"); if (options && options.keyFile) { @@ -482,7 +484,7 @@ var ReplSetTest = function(opts) { var nodes = []; for (var n = 0; n < this.ports.length; n++) { - nodes.push(this.start(n, options)); + nodes.push(this.start(n, options, restart)); } this.nodes = nodes; @@ -1883,7 +1885,11 @@ var ReplSetTest = function(opts) { * Constructor, which instantiates the ReplSetTest object from an existing set. */ function _constructFromExistingSeedNode(seedNode) { - var conf = _replSetGetConfig(new Mongo(seedNode)); + const conn = new Mongo(seedNode); + if (jsTest.options().keyFile) { + self.keyFile = jsTest.options().keyFile; + } + var conf = asCluster(conn, () => _replSetGetConfig(conn)); print('Recreating replica set from config ' + tojson(conf)); var existingNodes = conf.members.map(member => member.host); diff --git a/src/mongo/shell/servers_misc.js b/src/mongo/shell/servers_misc.js index fdb49deb76a..f419428e2d0 100644 --- a/src/mongo/shell/servers_misc.js +++ b/src/mongo/shell/servers_misc.js @@ -214,11 +214,21 @@ function startParallelShell(jsCode, port, noConnect) { var args = ["mongo"]; if (typeof db == "object") { - var hostAndPort = db.getMongo().host.split(':'); - var host = hostAndPort[0]; - args.push("--host", host); - if (!port && hostAndPort.length >= 2) { - var port = hostAndPort[1]; + if (!port) { + // If no port override specified, just passthrough connect string. + args.push("--host", db.getMongo().host); + } else { + // Strip port numbers from connect string. + const uri = new MongoURI(db.getMongo().host); + var connString = uri.servers + .map(function(server) { + return server.host; + }) + .join(','); + if (uri.setName.length > 0) { + connString = uri.setName + '/' + connString; + } + args.push("--host", connString); } } if (port) { diff --git a/src/mongo/shell/shardingtest.js b/src/mongo/shell/shardingtest.js index 9b06dcd89ce..7a0fdae70f3 100644 --- a/src/mongo/shell/shardingtest.js +++ b/src/mongo/shell/shardingtest.js @@ -370,10 +370,14 @@ var ShardingTest = function(params) { } }; - this.stop = function(opts) { + this.stopAllMongos = function(opts) { for (var i = 0; i < this._mongos.length; i++) { this.stopMongos(i, opts); } + }; + + this.stop = function(opts) { + this.stopAllMongos(opts); for (var i = 0; i < this._connections.length; i++) { if (this._rs[i]) { diff --git a/src/mongo/shell/utils.js b/src/mongo/shell/utils.js index 9794a60df4e..148eacc1c89 100644 --- a/src/mongo/shell/utils.js +++ b/src/mongo/shell/utils.js @@ -158,6 +158,18 @@ print.captureAllOutput = function(fn, args) { return res; }; +var indentStr = function(indent, s) { + if (typeof(s) === "undefined") { + s = indent; + indent = 0; + } + if (indent > 0) { + indent = (new Array(indent + 1)).join(" "); + s = indent + s.replace(/\n/g, "\n" + indent); + } + return s; +}; + if (typeof TestData == "undefined") { TestData = undefined; } @@ -230,7 +242,8 @@ jsTestOptions = function() { networkMessageCompressors: TestData.networkMessageCompressors, skipValidationOnInvalidViewDefinitions: TestData.skipValidationOnInvalidViewDefinitions, forceValidationWithFeatureCompatibilityVersion: - TestData.forceValidationWithFeatureCompatibilityVersion + TestData.forceValidationWithFeatureCompatibilityVersion, + skipValidationNamespaces: TestData.skipValidationNamespaces || [], }); } return _jsTestOptions; diff --git a/src/mongo/shell/utils_sh.js b/src/mongo/shell/utils_sh.js index ef8ae2dfa36..6e9c61a32e5 100644 --- a/src/mongo/shell/utils_sh.js +++ b/src/mongo/shell/utils_sh.js @@ -632,6 +632,18 @@ sh.getRecentMigrations = function(configDB) { return result; }; +sh._shardingStatusStr = function(indent, s) { + // convert from logical indentation to actual num of chars + if (indent == 0) { + indent = 0; + } else if (indent == 1) { + indent = 2; + } else { + indent = (indent - 1) * 8; + } + return indentStr(indent, s) + "\n"; +}; + function printShardingStatus(configDB, verbose) { // configDB is a DB object that contains the sharding metadata of interest. // Defaults to the db named "config" on the current connection. @@ -646,15 +658,15 @@ function printShardingStatus(configDB, verbose) { } var raw = ""; - var output = function(s) { - raw += s + "\n"; + var output = function(indent, s) { + raw += sh._shardingStatusStr(indent, s); }; - output("--- Sharding Status --- "); - output(" sharding version: " + tojson(configDB.getCollection("version").findOne())); + output(0, "--- Sharding Status --- "); + output(1, "sharding version: " + tojson(configDB.getCollection("version").findOne())); - output(" shards:"); + output(1, "shards:"); configDB.shards.find().sort({_id: 1}).forEach(function(z) { - output("\t" + tojsononeline(z)); + output(2, tojsononeline(z)); }); // (most recently) active mongoses @@ -672,9 +684,9 @@ function printShardingStatus(configDB, verbose) { } } - output(" " + mongosAdjective + " mongoses:"); + output(1, mongosAdjective + " mongoses:"); if (mostRecentMongosTime === null) { - output("\tnone"); + output(2, "none"); } else { var recentMongosQuery = { ping: { @@ -688,7 +700,7 @@ function printShardingStatus(configDB, verbose) { if (verbose) { configDB.mongos.find(recentMongosQuery).sort({ping: -1}).forEach(function(z) { - output("\t" + tojsononeline(z)); + output(2, tojsononeline(z)); }); } else { configDB.mongos @@ -698,23 +710,23 @@ function printShardingStatus(configDB, verbose) { {$sort: {num: -1}} ]) .forEach(function(z) { - output("\t" + tojson(z._id) + " : " + z.num); + output(2, tojson(z._id) + " : " + z.num); }); } } - output(" autosplit:"); + output(1, "autosplit:"); // Is autosplit currently enabled - output("\tCurrently enabled: " + (sh.getShouldAutoSplit(configDB) ? "yes" : "no")); + output(2, "Currently enabled: " + (sh.getShouldAutoSplit(configDB) ? "yes" : "no")); - output(" balancer:"); + output(1, "balancer:"); // Is the balancer currently enabled - output("\tCurrently enabled: " + (sh.getBalancerState(configDB) ? "yes" : "no")); + output(2, "Currently enabled: " + (sh.getBalancerState(configDB) ? "yes" : "no")); // Is the balancer currently active - output("\tCurrently running: " + (sh.isBalancerRunning(configDB) ? "yes" : "no")); + output(2, "Currently running: " + (sh.isBalancerRunning(configDB) ? "yes" : "no")); // Output details of the current balancer round var balLock = sh.getBalancerLockDetails(configDB); @@ -725,16 +737,17 @@ function printShardingStatus(configDB, verbose) { // Output the balancer window var balSettings = sh.getBalancerWindow(configDB); if (balSettings) { - output("\t\tBalancer active window is set between " + balSettings.start + " and " + - balSettings.stop + " server local time"); + output(3, + "Balancer active window is set between " + balSettings.start + " and " + + balSettings.stop + " server local time"); } // Output the list of active migrations var activeMigrations = sh.getActiveMigrations(configDB); if (activeMigrations.length > 0) { - output("\tCollections with active migrations: "); + output(2, "Collections with active migrations: "); activeMigrations.forEach(function(migration) { - output("\t\t" + migration._id + " started at " + migration.when); + output(3, migration._id + " started at " + migration.when); }); } @@ -755,31 +768,32 @@ function printShardingStatus(configDB, verbose) { // Review config.actionlog for errors var actionReport = sh.getRecentFailedRounds(configDB); // Always print the number of failed rounds - output("\tFailed balancer rounds in last 5 attempts: " + actionReport.count); + output(2, "Failed balancer rounds in last 5 attempts: " + actionReport.count); // Only print the errors if there are any if (actionReport.count > 0) { - output("\tLast reported error: " + actionReport.lastErr); - output("\tTime of Reported error: " + actionReport.lastTime); + output(2, "Last reported error: " + actionReport.lastErr); + output(2, "Time of Reported error: " + actionReport.lastTime); } - output("\tMigration Results for the last 24 hours: "); + output(2, "Migration Results for the last 24 hours: "); var migrations = sh.getRecentMigrations(configDB); if (migrations.length > 0) { migrations.forEach(function(x) { if (x._id === "Success") { - output("\t\t" + x.count + " : " + x._id); + output(3, x.count + " : " + x._id); } else { - output("\t\t" + x.count + " : Failed with error '" + x._id + "', from " + - x.from + " to " + x.to); + output(3, + x.count + " : Failed with error '" + x._id + "', from " + x.from + + " to " + x.to); } }); } else { - output("\t\tNo recent migrations"); + output(3, "No recent migrations"); } } - output(" databases:"); + output(1, "databases:"); configDB.databases.find().sort({name: 1}).forEach(function(db) { var truthy = function(value) { return !!value; @@ -795,20 +809,22 @@ function printShardingStatus(configDB, verbose) { return s; }; - output("\t" + tojsononeline(db, "", true)); + output(2, tojsononeline(db, "", true)); if (db.partitioned) { configDB.collections.find({_id: new RegExp("^" + RegExp.escape(db._id) + "\\.")}) .sort({_id: 1}) .forEach(function(coll) { if (!coll.dropped) { - output("\t\t" + coll._id); - output("\t\t\tshard key: " + tojson(coll.key)); - output("\t\t\tunique: " + truthy(coll.unique) + - nonBooleanNote("unique", coll.unique)); - output("\t\t\tbalancing: " + !truthy(coll.noBalance) + - nonBooleanNote("noBalance", coll.noBalance)); - output("\t\t\tchunks:"); + output(3, coll._id); + output(4, "shard key: " + tojson(coll.key)); + output(4, + "unique: " + truthy(coll.unique) + + nonBooleanNote("unique", coll.unique)); + output(4, + "balancing: " + !truthy(coll.noBalance) + + nonBooleanNote("noBalance", coll.noBalance)); + output(4, "chunks:"); res = configDB.chunks .aggregate({$match: {ns: coll._id}}, @@ -819,26 +835,29 @@ function printShardingStatus(configDB, verbose) { var totalChunks = 0; res.forEach(function(z) { totalChunks += z.nChunks; - output("\t\t\t\t" + z.shard + "\t" + z.nChunks); + output(5, z.shard + "\t" + z.nChunks); }); if (totalChunks < 20 || verbose) { configDB.chunks.find({"ns": coll._id}) .sort({min: 1}) .forEach(function(chunk) { - output("\t\t\t" + tojson(chunk.min) + " -->> " + - tojson(chunk.max) + " on : " + chunk.shard + " " + - tojson(chunk.lastmod) + " " + - (chunk.jumbo ? "jumbo " : "")); + output(4, + tojson(chunk.min) + " -->> " + tojson(chunk.max) + + " on : " + chunk.shard + " " + + tojson(chunk.lastmod) + " " + + (chunk.jumbo ? "jumbo " : "")); }); } else { output( - "\t\t\ttoo many chunks to print, use verbose if you want to force print"); + 4, + "too many chunks to print, use verbose if you want to force print"); } configDB.tags.find({ns: coll._id}).sort({min: 1}).forEach(function(tag) { - output("\t\t\t tag: " + tag.tag + " " + tojson(tag.min) + " -->> " + - tojson(tag.max)); + output(4, + " tag: " + tag.tag + " " + tojson(tag.min) + " -->> " + + tojson(tag.max)); }); } }); @@ -861,29 +880,29 @@ function printShardingSizes(configDB) { } var raw = ""; - var output = function(s) { - raw += s + "\n"; + var output = function(indent, s) { + raw += sh._shardingStatusStr(indent, s); }; - output("--- Sharding Status --- "); - output(" sharding version: " + tojson(configDB.getCollection("version").findOne())); + output(0, "--- Sharding Sizes --- "); + output(1, "sharding version: " + tojson(configDB.getCollection("version").findOne())); - output(" shards:"); + output(1, "shards:"); var shards = {}; configDB.shards.find().forEach(function(z) { shards[z._id] = new Mongo(z.host); - output(" " + tojson(z)); + output(2, tojson(z)); }); var saveDB = db; - output(" databases:"); + output(1, "databases:"); configDB.databases.find().sort({name: 1}).forEach(function(db) { - output("\t" + tojson(db, "", true)); + output(2, tojson(db, "", true)); if (db.partitioned) { configDB.collections.find({_id: new RegExp("^" + RegExp.escape(db._id) + "\.")}) .sort({_id: 1}) .forEach(function(coll) { - output("\t\t" + coll._id + " chunks:"); + output(3, coll._id + " chunks:"); configDB.chunks.find({"ns": coll._id}).sort({min: 1}).forEach(function(chunk) { var mydb = shards[chunk.shard].getDB(db._id); var out = mydb.runCommand({ @@ -895,8 +914,9 @@ function printShardingSizes(configDB) { delete out.millis; delete out.ok; - output("\t\t\t" + tojson(chunk.min) + " -->> " + tojson(chunk.max) + - " on : " + chunk.shard + " " + tojson(out)); + output(4, + tojson(chunk.min) + " -->> " + tojson(chunk.max) + " on : " + + chunk.shard + " " + tojson(out)); }); }); diff --git a/src/mongo/transport/SConscript b/src/mongo/transport/SConscript index 8e414610b2c..e82b6bd28fa 100644 --- a/src/mongo/transport/SConscript +++ b/src/mongo/transport/SConscript @@ -100,7 +100,9 @@ env.CppUnitTest( ], ) -env.Library( +messageCompressorEnv = env.Clone() +messageCompressorEnv.InjectThirdPartyIncludePaths(libraries=['snappy']) +messageCompressorEnv.Library( target='message_compressor', source=[ 'message_compressor_manager.cpp', diff --git a/src/mongo/transport/message_compressor_manager.cpp b/src/mongo/transport/message_compressor_manager.cpp index 669b091f422..9a5b5ca81a9 100644 --- a/src/mongo/transport/message_compressor_manager.cpp +++ b/src/mongo/transport/message_compressor_manager.cpp @@ -51,23 +51,31 @@ struct CompressionHeader { uint8_t compressorId; void serialize(DataRangeCursor* cursor) { - cursor->writeAndAdvance>(originalOpCode); - cursor->writeAndAdvance>(uncompressedSize); - cursor->writeAndAdvance>(compressorId); + uassertStatusOK(cursor->writeAndAdvance>(originalOpCode)); + uassertStatusOK(cursor->writeAndAdvance>(uncompressedSize)); + uassertStatusOK(cursor->writeAndAdvance>(compressorId)); } CompressionHeader(int32_t _opcode, int32_t _size, uint8_t _id) : originalOpCode{_opcode}, uncompressedSize{_size}, compressorId{_id} {} CompressionHeader(ConstDataRangeCursor* cursor) { - originalOpCode = cursor->readAndAdvance>().getValue(); - uncompressedSize = cursor->readAndAdvance>().getValue(); - compressorId = cursor->readAndAdvance>().getValue(); + originalOpCode = _readWithChecking>(cursor); + uncompressedSize = _readWithChecking>(cursor); + compressorId = _readWithChecking>(cursor); } static size_t size() { return sizeof(originalOpCode) + sizeof(uncompressedSize) + sizeof(compressorId); } + +private: + template + T _readWithChecking(ConstDataRangeCursor* cursor) { + auto sw = cursor->readAndAdvance(); + uassertStatusOK(sw.getStatus()); + return sw.getValue(); + } }; } // namespace @@ -124,6 +132,9 @@ StatusWith MessageCompressorManager::compressMessage(const Message& msg StatusWith MessageCompressorManager::decompressMessage(const Message& msg) { auto inputHeader = msg.header(); ConstDataRangeCursor input(inputHeader.data(), inputHeader.data() + inputHeader.dataLen()); + if (input.length() < CompressionHeader::size()) { + return {ErrorCodes::BadValue, "Invalid compressed message header"}; + } CompressionHeader compressionHeader(&input); auto compressor = _registry->getCompressor(compressionHeader.compressorId); @@ -132,7 +143,12 @@ StatusWith MessageCompressorManager::decompressMessage(const Message& m "Compression algorithm specified in message is not available"}; } - auto bufferSize = compressionHeader.uncompressedSize + MsgData::MsgDataHeaderSize; + size_t bufferSize = compressionHeader.uncompressedSize + MsgData::MsgDataHeaderSize; + if (bufferSize > MaxMessageSizeBytes) { + return {ErrorCodes::BadValue, + "Decompressed message would be larger than maximum message size"}; + } + auto outputMessageBuffer = SharedBuffer::allocate(bufferSize); MsgData::View outMessage(outputMessageBuffer.get()); outMessage.setId(inputHeader.getId()); diff --git a/src/mongo/transport/message_compressor_manager_test.cpp b/src/mongo/transport/message_compressor_manager_test.cpp index 383bb1e3260..5c533e45e75 100644 --- a/src/mongo/transport/message_compressor_manager_test.cpp +++ b/src/mongo/transport/message_compressor_manager_test.cpp @@ -26,6 +26,8 @@ * it in the license file. */ +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kNetwork + #include "mongo/platform/basic.h" #include "mongo/bson/bsonobjbuilder.h" @@ -33,7 +35,9 @@ #include "mongo/transport/message_compressor_manager.h" #include "mongo/transport/message_compressor_noop.h" #include "mongo/transport/message_compressor_registry.h" +#include "mongo/transport/message_compressor_snappy.h" #include "mongo/unittest/unittest.h" +#include "mongo/util/log.h" #include "mongo/util/net/message.h" #include @@ -120,6 +124,41 @@ void checkFidelity(const Message& msg, std::unique_ptr co ASSERT_EQ(memcmp(decompressedMsgView.data(), originalView.data(), originalView.dataLen()), 0); } +void checkOverflow(std::unique_ptr compressor) { + // This is our test data that we're going to try to compress/decompress into a buffer that's + // way too small. + const std::string data = + "We embrace reality. We apply high-quality thinking and rigor." + "We have courage in our convictions but work hard to ensure biases " + "or personal beliefs do not get in the way of finding the best solution."; + ConstDataRange input(data.data(), data.size()); + + // This is our tiny buffer that should cause an error. + std::array smallBuffer; + DataRange smallOutput(smallBuffer.data(), smallBuffer.size()); + + // This is a normal sized buffer that we can store a compressed version of our test data safely + std::vector normalBuffer; + normalBuffer.resize(compressor->getMaxCompressedSize(data.size())); + auto sws = compressor->compressData(input, DataRange(normalBuffer.data(), normalBuffer.size())); + ASSERT_OK(sws); + DataRange normalRange = DataRange(normalBuffer.data(), sws.getValue()); + + // Check that compressing the test data into a small buffer fails + ASSERT_NOT_OK(compressor->compressData(input, smallOutput)); + + // Check that decompressing compressed test data into a small buffer fails + ASSERT_NOT_OK(compressor->decompressData(normalRange, smallOutput)); + + // Check that decompressing a valid buffer that's missing data doesn't overflow the + // source buffer. + std::vector scratch; + scratch.resize(data.size()); + ConstDataRange tooSmallRange(normalBuffer.data(), normalBuffer.size() / 2); + ASSERT_NOT_OK( + compressor->decompressData(tooSmallRange, DataRange(scratch.data(), scratch.size()))); +} + Message buildMessage() { const auto data = std::string{"Hello, world!"}; const auto bufferSize = MsgData::MsgDataHeaderSize + data.size(); @@ -179,8 +218,54 @@ TEST(NoopMessageCompressor, Fidelity) { TEST(SnappyMessageCompressor, Fidelity) { auto testMessage = buildMessage(); - checkFidelity(testMessage, stdx::make_unique()); + checkFidelity(testMessage, stdx::make_unique()); +} + +TEST(SnappyMessageCompressor, Overflow) { + checkOverflow(stdx::make_unique()); +} + +TEST(MessageCompressorManager, MessageSizeTooLarge) { + auto registry = buildRegistry(); + MessageCompressorManager compManager(®istry); + + auto badMessageBuffer = SharedBuffer::allocate(128); + MsgData::View badMessage(badMessageBuffer.get()); + badMessage.setId(1); + badMessage.setResponseToMsgId(0); + badMessage.setOperation(dbCompressed); + badMessage.setLen(128); + + DataRangeCursor cursor(badMessage.data(), badMessage.data() + badMessage.dataLen()); + uassertStatusOK(cursor.writeAndAdvance>(dbQuery)); + uassertStatusOK(cursor.writeAndAdvance>(MaxMessageSizeBytes + 1)); + uassertStatusOK( + cursor.writeAndAdvance>(registry.getCompressor("noop")->getId())); + + auto status = compManager.decompressMessage(Message(badMessageBuffer)).getStatus(); + ASSERT_NOT_OK(status); +} + +TEST(MessageCompressorManager, RuntMessage) { + auto registry = buildRegistry(); + MessageCompressorManager compManager(®istry); + + auto badMessageBuffer = SharedBuffer::allocate(128); + MsgData::View badMessage(badMessageBuffer.get()); + badMessage.setId(1); + badMessage.setResponseToMsgId(0); + badMessage.setOperation(dbCompressed); + badMessage.setLen(MsgData::MsgDataHeaderSize + 8); + + // This is a totally bogus compression header of just the orginal opcode + 0 byte uncompressed + // size + DataRangeCursor cursor(badMessage.data(), badMessage.data() + badMessage.dataLen()); + uassertStatusOK(cursor.writeAndAdvance>(dbQuery)); + uassertStatusOK(cursor.writeAndAdvance>(0)); + + auto status = compManager.decompressMessage(Message(badMessageBuffer)).getStatus(); + ASSERT_NOT_OK(status); } -} // namespace mongo } // namespace +} // namespace mongo diff --git a/src/mongo/transport/message_compressor_snappy.cpp b/src/mongo/transport/message_compressor_snappy.cpp index db1e0c9dfca..9d523fca661 100644 --- a/src/mongo/transport/message_compressor_snappy.cpp +++ b/src/mongo/transport/message_compressor_snappy.cpp @@ -30,12 +30,13 @@ #include "mongo/platform/basic.h" +#include "mongo/base/data_range_cursor.h" #include "mongo/base/init.h" #include "mongo/stdx/memory.h" #include "mongo/transport/message_compressor_registry.h" #include "mongo/transport/message_compressor_snappy.h" -#include "third_party/snappy-1.1.3/snappy.h" +#include namespace mongo { @@ -48,7 +49,10 @@ std::size_t SnappyMessageCompressor::getMaxCompressedSize(size_t inputSize) { StatusWith SnappyMessageCompressor::compressData(ConstDataRange input, DataRange output) { - size_t outLength; + size_t outLength = output.length(); + if (output.length() < getMaxCompressedSize(input.length())) { + return {ErrorCodes::BadValue, "Output too small for max size of compressed input"}; + } snappy::RawCompress(input.data(), input.length(), const_cast(output.data()), &outLength); counterHitCompress(input.length(), outLength); @@ -57,10 +61,13 @@ StatusWith SnappyMessageCompressor::compressData(ConstDataRange inp StatusWith SnappyMessageCompressor::decompressData(ConstDataRange input, DataRange output) { - bool ret = - snappy::RawUncompress(input.data(), input.length(), const_cast(output.data())); + size_t expectedLength = 0; + if (!snappy::GetUncompressedLength(input.data(), input.length(), &expectedLength) || + expectedLength != output.length()) { + return {ErrorCodes::BadValue, "Compressed message was invalid or corrupted"}; + } - if (!ret) { + if (!snappy::RawUncompress(input.data(), input.length(), const_cast(output.data()))) { return Status{ErrorCodes::BadValue, "Compressed message was invalid or corrupted"}; } diff --git a/src/mongo/util/SConscript b/src/mongo/util/SConscript index 3346a596763..fe0380c1345 100644 --- a/src/mongo/util/SConscript +++ b/src/mongo/util/SConscript @@ -499,6 +499,17 @@ env.CppUnitTest( ] ) +env.CppUnitTest( + target='producer_consumer_queue_test', + source=[ + 'producer_consumer_queue_test.cpp', + ], + LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/db/service_context', + ] +) + env.CppUnitTest( target='duration_test', source=[ diff --git a/src/mongo/util/assert_util.cpp b/src/mongo/util/assert_util.cpp index 9d40b1f5d62..500a8cbb5eb 100644 --- a/src/mongo/util/assert_util.cpp +++ b/src/mongo/util/assert_util.cpp @@ -232,10 +232,9 @@ NOINLINE_DECL void msgassertedWithLocation(int msgid, const char* msg, const char* file, unsigned line) { - assertionCount.condrollover(++assertionCount.warning); + assertionCount.condrollover(++assertionCount.msg); log() << "Assertion: " << msgid << ":" << redact(msg) << ' ' << file << ' ' << dec << line << endl; - logContext(); throw MsgAssertionException(msgid, msg); } @@ -243,7 +242,7 @@ NOINLINE_DECL void msgassertedNoTraceWithLocation(int msgid, const char* msg, const char* file, unsigned line) { - assertionCount.condrollover(++assertionCount.warning); + assertionCount.condrollover(++assertionCount.msg); log() << "Assertion: " << msgid << ":" << redact(msg) << ' ' << file << ' ' << dec << line << endl; throw MsgAssertionException(msgid, msg); diff --git a/src/mongo/util/cmdline_utils/censor_cmdline.cpp b/src/mongo/util/cmdline_utils/censor_cmdline.cpp index 044de375e56..e903242ac75 100644 --- a/src/mongo/util/cmdline_utils/censor_cmdline.cpp +++ b/src/mongo/util/cmdline_utils/censor_cmdline.cpp @@ -45,6 +45,7 @@ static bool _isPasswordArgument(const char* argumentName) { "net.ssl.clusterPassword", "processManagement.windowsService.servicePassword", "security.kmip.clientCertificatePassword", + "security.ldap.bind.queryPassword", NULL // Last entry sentinel. }; for (const char* const* current = passwordArguments; *current; ++current) { @@ -60,6 +61,7 @@ static bool _isPasswordSwitch(const char* switchName) { "sslClusterPassword", "servicePassword", "kmipClientCertificatePassword", + "ldapQueryPassword", NULL // Last entry sentinel. }; diff --git a/src/mongo/util/concurrency/with_lock.h b/src/mongo/util/concurrency/with_lock.h new file mode 100644 index 00000000000..e1607e361cf --- /dev/null +++ b/src/mongo/util/concurrency/with_lock.h @@ -0,0 +1,111 @@ +/** Copyright 2017 MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include "mongo/stdx/mutex.h" +#include "mongo/util/assert_util.h" + +#include + +namespace mongo { + +/** + * WithLock is an attestation to pass as an argument to functions that must be called only while + * holding a lock, as a rigorous alternative to an unchecked naming convention and/or stern + * comments. It helps prevent a common usage error. + * + * It may be used to modernize code from (something like) this + * + * // Member _mutex MUST be held when calling this: + * void _clobber_inlock(OperationContext* opCtx) { + * _stuff = makeStuff(opCtx); + * } + * + * into + * + * void _clobber(WithLock, OperationContext* opCtx) { + * _stuff = makeStuff(opCtx); + * } + * + * A call to such a function looks like this: + * + * stdx::lock_guard lk(_mutex); + * _clobber(lk, opCtx); // instead of _clobber_inlock(opCtx) + * + * Note that the formal argument need not (and should not) be named unless it is needed to pass + * the attestation along to another function: + * + * void _clobber(WithLock lock, OperationContext* opCtx) { + * _really_clobber(lock, opCtx); + * } + * + */ +struct WithLock { + template + WithLock(stdx::lock_guard const&) noexcept {} + + template + WithLock(stdx::unique_lock const& lock) noexcept { + invariant(lock.owns_lock()); + } + + // Add constructors from any other lock types here. + + // Pass by value is OK. + WithLock(WithLock const&) noexcept {} + WithLock(WithLock&&) noexcept {} + + // No assigning WithLocks. + void operator=(WithLock const&) = delete; + void operator=(WithLock&&) = delete; + + // No moving a lock_guard<> or unique_lock<> in. + template + WithLock(stdx::lock_guard&&) = delete; + template + WithLock(stdx::unique_lock&&) = delete; + + /* + * Produces a WithLock without benefit of any actual lock, for use in cases where a lock is not + * really needed, such as in many (but not all!) constructors. + */ + static WithLock withoutLock() noexcept { + return {}; + } + +private: + WithLock() noexcept = default; +}; + +} // namespace mongo + +namespace std { +// No moving a WithLock: +template <> +mongo::WithLock&& move(mongo::WithLock&&) noexcept = delete; +} // namespace std diff --git a/src/mongo/util/net/hostandport.cpp b/src/mongo/util/net/hostandport.cpp index abd8f2d6ade..4b46938fbdb 100644 --- a/src/mongo/util/net/hostandport.cpp +++ b/src/mongo/util/net/hostandport.cpp @@ -95,7 +95,9 @@ void HostAndPort::append(StringBuilder& ss) const { } else { ss << host(); } - ss << ':' << port(); + if (host().find('/') == std::string::npos) { + ss << ':' << port(); + } } bool HostAndPort::empty() const { diff --git a/src/mongo/util/net/miniwebserver.cpp b/src/mongo/util/net/miniwebserver.cpp index 5f4165d42f4..9fe847168ee 100644 --- a/src/mongo/util/net/miniwebserver.cpp +++ b/src/mongo/util/net/miniwebserver.cpp @@ -45,6 +45,7 @@ namespace mongo { using std::shared_ptr; +using std::string; using std::stringstream; using std::vector; diff --git a/src/mongo/util/net/ssl_manager.cpp b/src/mongo/util/net/ssl_manager.cpp index 187c0c105e2..4bb0befb4b0 100644 --- a/src/mongo/util/net/ssl_manager.cpp +++ b/src/mongo/util/net/ssl_manager.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -171,21 +172,28 @@ const STACK_OF(X509_EXTENSION) * X509_get0_extensions(const X509* peerCert) { * OpenSSL before version 1.1.0 requires applications provide a callback which emits a thread * identifier. This ID is used to store thread specific ERR information. When a thread is * terminated, it must call ERR_remove_state or ERR_remove_thread_state. These functions may - * themselves invoke the application provided callback. + * themselves invoke the application provided callback. These IDs are stored in a hashtable with + * a questionable hash function. They must be uniformly distributed to prevent collisions. */ class SSLThreadInfo { public: static unsigned long getID() { - enforceCleanupOnShutdown(); + struct ThreadLocalState { + bool firstCall; + unsigned long id; + }; + static MONGO_TRIVIALLY_CONSTRUCTIBLE_THREAD_LOCAL ThreadLocalState state{true, 0}; + if (state.firstCall) { + state.id = _idManager.reserveID(); + unsigned long id = state.id; + boost::this_thread::at_thread_exit([id] { + ERR_remove_state(0); + _idManager.releaseID(id); + }); + state.firstCall = false; + } -#ifdef _WIN32 - return GetCurrentThreadId(); -#else - static_assert(sizeof(void*) == sizeof(unsigned long), - "OpenSSL needs the address of a thread-unique object to be castable to" - "unsigned long"); - return reinterpret_cast(&errno); -#endif + return state.id; } static void lockingCallback(int mode, int type, const char* file, int line) { @@ -208,21 +216,39 @@ public: private: SSLThreadInfo() = delete; - // When called, ensures that this thread will, on termination, call ERR_remove_state. - static void enforceCleanupOnShutdown() { - static MONGO_TRIVIALLY_CONSTRUCTIBLE_THREAD_LOCAL bool firstCall = true; - if (firstCall) { - boost::this_thread::at_thread_exit([] { ERR_remove_state(0); }); - firstCall = false; - } - } - // Note: see SERVER-8734 for why we are using a recursive mutex here. // Once the deadlock fix in OpenSSL is incorporated into most distros of // Linux, this can be changed back to a nonrecursive mutex. static std::vector> _mutex; + + class ThreadIDManager { + public: + unsigned long reserveID() { + stdx::unique_lock lock(_idMutex); + if (!_idLast.empty()) { + unsigned long ret = _idLast.top(); + _idLast.pop(); + return ret; + } + return ++_idNext; + } + + void releaseID(unsigned long id) { + stdx::unique_lock lock(_idMutex); + _idLast.push(id); + } + + private: + // Machinery for producing IDs that are unique for the life of a thread. + stdx::mutex _idMutex; // Protects _idNext and _idLast. + unsigned long _idNext = 0; // Stores the next thread ID to use, if none already allocated. + std::stack> + _idLast; // Stores old thread IDs, for reuse. + }; + static ThreadIDManager _idManager; }; std::vector> SSLThreadInfo::_mutex; +SSLThreadInfo::ThreadIDManager SSLThreadInfo::_idManager; // We only want to free SSL_CTX objects if they have been populated. OpenSSL seems to perform this // check before freeing them, but because it does not document this, we should protect ourselves. diff --git a/src/mongo/util/options_parser/options_parser.cpp b/src/mongo/util/options_parser/options_parser.cpp index 5d03910a390..63bf184ec85 100644 --- a/src/mongo/util/options_parser/options_parser.cpp +++ b/src/mongo/util/options_parser/options_parser.cpp @@ -341,6 +341,11 @@ Status YAMLNodeToValue(const YAML::Node& YAMLNode, return Status::OK(); } + if (!YAMLNode.IsScalar() && !YAMLNode.IsNull()) { + return Status(ErrorCodes::BadValue, + str::stream() << "Scalar option '" << key << "' must be a single value"); + } + // Our YAML parser reads everything as a string, so we need to parse it ourselves. std::string stringVal = YAMLNode.Scalar(); return stringToValue(stringVal, type, key, value); diff --git a/src/mongo/util/options_parser/options_parser_test.cpp b/src/mongo/util/options_parser/options_parser_test.cpp index a0d97c512bb..aeb4a875914 100644 --- a/src/mongo/util/options_parser/options_parser_test.cpp +++ b/src/mongo/util/options_parser/options_parser_test.cpp @@ -1834,6 +1834,82 @@ TEST(Parsing, BadConfigFileOption) { ASSERT_NOT_OK(parser.run(testOpts, argv, env_map, &environment)); } +TEST(Parsing, MapForScalarMismatch) { + OptionsParserTester parser; + moe::Environment environment; + moe::OptionSection testOpts; + + testOpts.addOptionChaining("config", "config", moe::Int, "Config file to parse"); + testOpts.addOptionChaining("str", "str", moe::String, ""); + + std::vector argv; + argv.push_back("binaryname"); + argv.push_back("--config"); + argv.push_back("config.json"); + std::map env_map; + + parser.setConfig("config.json", R"cfg({ str: { elem: "val" } })cfg"); + + ASSERT_NOT_OK(parser.run(testOpts, argv, env_map, &environment)); +} + +TEST(Parsing, ScalarForMapMismatch) { + OptionsParserTester parser; + moe::Environment environment; + moe::OptionSection testOpts; + + testOpts.addOptionChaining("config", "config", moe::Int, "Config file to parse"); + testOpts.addOptionChaining("strmap", "strmap", moe::StringMap, ""); + + std::vector argv; + argv.push_back("binaryname"); + argv.push_back("--config"); + argv.push_back("config.json"); + std::map env_map; + + parser.setConfig("config.json", R"cfg({ str: "val" })cfg"); + + ASSERT_NOT_OK(parser.run(testOpts, argv, env_map, &environment)); +} + +TEST(Parsing, ListForScalarMismatch) { + OptionsParserTester parser; + moe::Environment environment; + moe::OptionSection testOpts; + + testOpts.addOptionChaining("config", "config", moe::Int, "Config file to parse"); + testOpts.addOptionChaining("str", "str", moe::String, ""); + + std::vector argv; + argv.push_back("binaryname"); + argv.push_back("--config"); + argv.push_back("config.json"); + std::map env_map; + + parser.setConfig("config.json", R"cfg({ str: ["val"] })cfg"); + + ASSERT_NOT_OK(parser.run(testOpts, argv, env_map, &environment)); +} + +TEST(Parsing, ScalarForListMismatch) { + OptionsParserTester parser; + moe::Environment environment; + moe::OptionSection testOpts; + + testOpts.addOptionChaining("config", "config", moe::Int, "Config file to parse"); + testOpts.addOptionChaining("strlist", "strlist", moe::StringVector, ""); + + std::vector argv; + argv.push_back("binaryname"); + argv.push_back("--config"); + argv.push_back("config.json"); + std::map env_map; + + parser.setConfig("config.json", R"cfg({ str: "val" })cfg"); + + ASSERT_NOT_OK(parser.run(testOpts, argv, env_map, &environment)); +} + TEST(ConfigFromFilesystem, JSONGood) { moe::OptionsParser parser; moe::Environment environment; diff --git a/src/mongo/util/password.cpp b/src/mongo/util/password.cpp index 11c936bac45..4814cbb3b12 100644 --- a/src/mongo/util/password.cpp +++ b/src/mongo/util/password.cpp @@ -46,7 +46,7 @@ namespace mongo { string askPassword() { std::string password; - cout << "Enter password: "; + cerr << "Enter password: "; #ifndef _WIN32 const int stdinfd = 0; termios termio; @@ -102,7 +102,7 @@ string askPassword() { return string(); } #endif - cout << "\n"; + cerr << "\n"; return password; } } diff --git a/src/mongo/util/producer_consumer_queue.h b/src/mongo/util/producer_consumer_queue.h new file mode 100644 index 00000000000..f0e942a7d94 --- /dev/null +++ b/src/mongo/util/producer_consumer_queue.h @@ -0,0 +1,559 @@ +/** + * Copyright (C) 2018 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "mongo/db/operation_context.h" +#include "mongo/stdx/condition_variable.h" +#include "mongo/stdx/mutex.h" +#include "mongo/stdx/type_traits.h" +#include "mongo/util/concurrency/with_lock.h" +#include "mongo/util/scopeguard.h" + +namespace mongo { + +namespace producer_consumer_queue_detail { + +/** + * The default cost function for the producer consumer queue. + * + * By default, all items in the queue have equal weight. + */ +struct DefaultCostFunction { + template + size_t operator()(const T&) const { + return 1; + } +}; + +// Various helpers to tighten down whether the args getting passed are valid interruption args. +// +// Whatever the caller passes in the interruption args, they need to be invocable on one of +// these helpers. std::is_invocable would do the job in C++17 +std::false_type areInterruptionArgsHelper(...) { + return {}; +} + +std::true_type areInterruptionArgsHelper(OperationContext*) { + return {}; +} + +std::true_type areInterruptionArgsHelper(OperationContext*, Date_t) { + return {}; +} + +std::true_type areInterruptionArgsHelper(Milliseconds) { + return {}; +} + +std::true_type areInterruptionArgsHelper(Date_t) { + return {}; +} + +template +auto areInterruptionArgs(U&& u, InterruptionArgs&&... args) + -> decltype(areInterruptionArgsHelper(std::forward(u), + std::forward(args)...)) { + return areInterruptionArgsHelper(std::forward(u), std::forward(args)...); +} + +std::true_type areInterruptionArgs() { + return {}; +} + +} // namespace producer_consumer_queue_detail + +/** + * A bounded, blocking, thread safe, cost parametrizable, single producer, multi-consumer queue. + * + * Properties: + * bounded - the queue can be limited in the number of items it can hold + * blocking - when the queue is full, or has no entries, callers block + * thread safe - the queue can be accessed safely from multiple threads at the same time + * cost parametrizable - the cost of items in the queue need not be equal. I.e. your items could + * be discrete byte buffers and the queue depth measured in bytes, so that + * the queue could hold one large buffer, or many smaller ones + * single producer - Only one thread may push work into the queue + * multi-consumer - Any number of threads may pop work out of the queue + * + * Interruptibility: + * All of the blocking methods on this type allow for 6 kinds of interruptibility. The matrix is + * parameterized by (void|OperationContext*)|(void|Milliseconds|Date_t). These provide different + * kinds of waiting based on whether the method should be interruptible via opCtx, and then + * whether they should timeout via deadline or duration + * + * A contrived example: pcq.pop(opCtx, Minutes(1)) would be warranted if: + * - The caller is blocking on a client thread. (opCtx) + * - The caller needs to act periodically on inactivity. (the duration) + * + * Exceptions include: + * timeouts + * ErrorCodes::ExceededTimeLimit exceptions + * opCtx interrupts + * ErrorCodes::Interrupted exceptions + * closure of queue endpoints + * ErrorCodes::ProducerConsumerQueueEndClosed + * pushes with batches that exceed the max queue size + * ErrorCodes::ProducerConsumerQueueBatchTooLarge + * + * Cost Function: + * The cost function must have a call operator which takes a const T& and returns the cost in + * size_t units. It must be pure across moves for a given T and never return zero. The intent of + * the cost function is to express the kind of bounds the queue provides, rather than to + * specialize behavior for a type. I.e. you should not specialize the default cost function and + * the cost function should always be explicit in the type. + */ +template +class ProducerConsumerQueue { + +public: + // By default the queue depth is unlimited + ProducerConsumerQueue() + : ProducerConsumerQueue(std::numeric_limits::max(), CostFunc{}) {} + + // Or it can be measured in whatever units your size function returns + explicit ProducerConsumerQueue(size_t size) : ProducerConsumerQueue(size, CostFunc{}) {} + + // If your cost function has meaningful state, you may also pass a non-default constructed + // instance + explicit ProducerConsumerQueue(size_t size, CostFunc costFunc) + : _max(size), _costFunc(std::move(costFunc)) {} + + ProducerConsumerQueue(const ProducerConsumerQueue&) = delete; + ProducerConsumerQueue& operator=(const ProducerConsumerQueue&) = delete; + + ProducerConsumerQueue(ProducerConsumerQueue&&) = delete; + ProducerConsumerQueue& operator=(ProducerConsumerQueue&&) = delete; + + ~ProducerConsumerQueue() { + invariant(!_producerWants); + invariant(!_consumers); + } + + // Pushes the passed T into the queue + // + // Leaves T unchanged if an interrupt exception is thrown while waiting for space + template < + typename... InterruptionArgs, + typename = stdx::enable_if_t()...))::value>> + void push(T&& t, InterruptionArgs&&... interruptionArgs) { + _pushRunner([&](stdx::unique_lock& lk) { + auto cost = _invokeCostFunc(t, lk); + uassert(ErrorCodes::ProducerConsumerQueueBatchTooLarge, + str::stream() << "cost of item (" << cost + << ") larger than maximum queue size (" + << _max + << ")", + cost <= _max); + + _waitForSpace(lk, cost, std::forward(interruptionArgs)...); + _push(lk, std::move(t)); + }); + } + + // Pushes all Ts into the queue + // + // Blocks until all of the Ts can be pushed at once + // + // StartIterator must be ForwardIterator + // + // Leaves the values underneath the iterators unchanged if an interrupt exception is thrown + // while waiting for space + // + // Lifecycle methods of T must not throw if you want to use this method, as there's no obvious + // mechanism to see what was and was not pushed if those do throw + template < + typename StartIterator, + typename EndIterator, + typename... InterruptionArgs, + typename = stdx::enable_if_t()...))::value>> + void pushMany(StartIterator start, EndIterator last, InterruptionArgs&&... interruptionArgs) { + return _pushRunner([&](stdx::unique_lock& lk) { + size_t cost = 0; + for (auto iter = start; iter != last; ++iter) { + cost += _invokeCostFunc(*iter, lk); + } + + uassert(ErrorCodes::ProducerConsumerQueueBatchTooLarge, + str::stream() << "cost of items in batch (" << cost + << ") larger than maximum queue size (" + << _max + << ")", + cost <= _max); + + _waitForSpace(lk, cost, std::forward(interruptionArgs)...); + + for (auto iter = start; iter != last; ++iter) { + _push(lk, std::move(*iter)); + } + }); + } + + // Attempts a non-blocking push of a value + // + // Leaves T unchanged if it fails + bool tryPush(T&& t) { + return _pushRunner( + [&](stdx::unique_lock& lk) { return _tryPush(lk, std::move(t)); }); + } + + // Pops one T out of the queue + template < + typename... InterruptionArgs, + typename = stdx::enable_if_t()...))::value>> + T pop(InterruptionArgs&&... interruptionArgs) { + return _popRunner([&](stdx::unique_lock& lk) { + _waitForNonEmpty(lk, std::forward(interruptionArgs)...); + return _pop(lk); + }); + } + + // Waits for at least one item in the queue, then pops items out of the queue until it would + // block + // + // OutputIterator must not throw on move assignment to *iter or popped values may be lost + // TODO: add sfinae to check to enforce + // + // Returns the cost value of the items extracted, along with the updated output iterator + template < + typename OutputIterator, + typename... InterruptionArgs, + typename = stdx::enable_if_t()...))::value>> + std::pair popMany(OutputIterator iterator, + InterruptionArgs&&... interruptionArgs) { + return popManyUpTo(_max, iterator, std::forward(interruptionArgs)...); + } + + // Waits for at least one item in the queue, then pops items out of the queue until it would + // block, or we've exceeded our budget + // + // OutputIterator must not throw on move assignment to *iter or popped values may be lost + // TODO: add sfinae to check to enforce + // + // Returns the cost value of the items extracted, along with the updated output iterator + template < + typename OutputIterator, + typename... InterruptionArgs, + typename = stdx::enable_if_t()...))::value>> + std::pair popManyUpTo(size_t budget, + OutputIterator iterator, + InterruptionArgs&&... interruptionArgs) { + return _popRunner([&](stdx::unique_lock& lk) { + size_t cost = 0; + + _waitForNonEmpty(lk, std::forward(interruptionArgs)...); + + while (auto out = _tryPop(lk)) { + cost += _invokeCostFunc(*out, lk); + *iterator = std::move(*out); + ++iterator; + + if (cost >= budget) { + break; + } + } + + return std::make_pair(cost, iterator); + }); + } + + // Attempts a non-blocking pop of a value + boost::optional tryPop() { + return _popRunner([&](stdx::unique_lock& lk) { return _tryPop(lk); }); + } + + // Closes the producer end. Consumers will continue to consume until the queue is exhausted, at + // which time they will begin to throw with an interruption dbexception + void closeProducerEnd() { + stdx::lock_guard lk(_mutex); + + _producerEndClosed = true; + + _notifyIfNecessary(lk); + } + + // Closes the consumer end. This causes all callers to throw with an interruption dbexception + void closeConsumerEnd() { + stdx::lock_guard lk(_mutex); + + _consumerEndClosed = true; + _producerEndClosed = true; + + _notifyIfNecessary(lk); + } + + // TEST ONLY FUNCTIONS + + // Returns the current depth of the queue in CostFunction units + size_t sizeForTest() const { + stdx::lock_guard lk(_mutex); + + return _current; + } + + // Returns true if the queue is empty + bool emptyForTest() const { + return sizeForTest() == 0; + } + +private: + size_t _invokeCostFunc(const T& t, WithLock) { + auto cost = _costFunc(t); + invariant(cost); + return cost; + } + + void _checkProducerClosed(WithLock) { + uassert( + ErrorCodes::ProducerConsumerQueueEndClosed, "Producer end closed", !_producerEndClosed); + uassert( + ErrorCodes::ProducerConsumerQueueEndClosed, "Consumer end closed", !_consumerEndClosed); + } + + void _checkConsumerClosed(WithLock) { + uassert( + ErrorCodes::ProducerConsumerQueueEndClosed, "Consumer end closed", !_consumerEndClosed); + uassert(ErrorCodes::ProducerConsumerQueueEndClosed, + "Producer end closed and values exhausted", + !(_producerEndClosed && _queue.empty())); + } + + void _notifyIfNecessary(WithLock) { + // If we've closed the consumer end, or if the production end is closed and we've exhausted + // the queue, wake everyone up and get out of here + if (_consumerEndClosed || (_queue.empty() && _producerEndClosed)) { + if (_consumers) { + _condvarConsumer.notify_all(); + } + + if (_producerWants) { + _condvarProducer.notify_one(); + } + + return; + } + + // If a producer is queued, and we have enough space for it to push its work + if (_producerWants && _current + _producerWants <= _max) { + _condvarProducer.notify_one(); + + return; + } + + // If we have consumers and anything in the queue, notify consumers + if (_consumers && _queue.size()) { + _condvarConsumer.notify_one(); + + return; + } + } + + template + auto _pushRunner(Callback&& cb) -> decltype( + cb(std::declval>::type>())) { + stdx::unique_lock lk(_mutex); + + _checkProducerClosed(lk); + + const auto guard = MakeGuard([&] { _notifyIfNecessary(lk); }); + + return cb(lk); + } + + template + auto _popRunner(Callback&& cb) -> decltype( + cb(std::declval>::type>())) { + stdx::unique_lock lk(_mutex); + + _checkConsumerClosed(lk); + + const auto guard = MakeGuard([&] { _notifyIfNecessary(lk); }); + + return cb(lk); + } + + bool _tryPush(WithLock wl, T&& t) { + size_t cost = _invokeCostFunc(t, wl); + if (_current + cost <= _max) { + _queue.emplace(std::move(t)); + _current += cost; + return true; + } + + return false; + } + + void _push(WithLock wl, T&& t) { + size_t cost = _invokeCostFunc(t, wl); + invariant(_current + cost <= _max); + + _queue.emplace(std::move(t)); + _current += cost; + } + + boost::optional _tryPop(WithLock wl) { + boost::optional out; + + if (!_queue.empty()) { + out.emplace(std::move(_queue.front())); + _queue.pop(); + _current -= _invokeCostFunc(*out, wl); + } + + return out; + } + + T _pop(WithLock wl) { + invariant(_queue.size()); + + auto t = std::move(_queue.front()); + _queue.pop(); + + _current -= _invokeCostFunc(t, wl); + + return t; + } + + template + void _waitForSpace(stdx::unique_lock& lk, + size_t cost, + InterruptionArgs&&... interruptionArgs) { + invariant(!_producerWants); + + _producerWants = cost; + const auto guard = MakeGuard([&] { _producerWants = 0; }); + + _waitFor(lk, + _condvarProducer, + [&] { + _checkProducerClosed(lk); + return _current + cost <= _max; + }, + std::forward(interruptionArgs)...); + } + + template + void _waitForNonEmpty(stdx::unique_lock& lk, + InterruptionArgs&&... interruptionArgs) { + + _consumers++; + const auto guard = MakeGuard([&] { _consumers--; }); + + _waitFor(lk, + _condvarConsumer, + [&] { + _checkConsumerClosed(lk); + return _queue.size(); + }, + std::forward(interruptionArgs)...); + } + + template + void _waitFor(stdx::unique_lock& lk, + stdx::condition_variable& condvar, + Callback&& pred, + OperationContext* opCtx) { + opCtx->waitForConditionOrInterrupt(condvar, lk, pred); + } + + template + void _waitFor(stdx::unique_lock& lk, + stdx::condition_variable& condvar, + Callback&& pred) { + condvar.wait(lk, pred); + } + + template + void _waitFor(stdx::unique_lock& lk, + stdx::condition_variable& condvar, + Callback&& pred, + OperationContext* opCtx, + Date_t deadline) { + uassert(ErrorCodes::ExceededTimeLimit, + "exceeded timeout", + opCtx->waitForConditionOrInterruptUntil(condvar, lk, deadline, pred)); + } + + template + void _waitFor(stdx::unique_lock& lk, + stdx::condition_variable& condvar, + Callback&& pred, + Date_t deadline) { + uassert(ErrorCodes::ExceededTimeLimit, + "exceeded timeout", + condvar.wait_until(lk, deadline.toSystemTimePoint(), pred)); + } + + template + void _waitFor(stdx::unique_lock& lk, + stdx::condition_variable& condvar, + Callback&& pred, + Milliseconds duration) { + uassert(ErrorCodes::ExceededTimeLimit, + "exceeded timeout", + condvar.wait_for(lk, duration.toSystemDuration(), pred)); + } + + mutable stdx::mutex _mutex; + stdx::condition_variable _condvarConsumer; + stdx::condition_variable _condvarProducer; + + // Max size of the queue + const size_t _max; + + // User's cost function + CostFunc _costFunc; + + // Current size of the queue + size_t _current = 0; + + std::queue _queue; + + // Counter for consumers in the queue + size_t _consumers = 0; + + // Size of batch the blocking producer wants to insert + size_t _producerWants = 0; + + // Flags that we're shutting down the queue + bool _consumerEndClosed = false; + bool _producerEndClosed = false; +}; + +} // namespace mongo diff --git a/src/mongo/util/producer_consumer_queue_test.cpp b/src/mongo/util/producer_consumer_queue_test.cpp new file mode 100644 index 00000000000..f824b5a63de --- /dev/null +++ b/src/mongo/util/producer_consumer_queue_test.cpp @@ -0,0 +1,531 @@ +/** + * Copyright (C) 2018 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/unittest/unittest.h" + +#include "mongo/util/producer_consumer_queue.h" + +#include "mongo/db/service_context_noop.h" +#include "mongo/stdx/condition_variable.h" +#include "mongo/stdx/memory.h" +#include "mongo/stdx/mutex.h" +#include "mongo/stdx/thread.h" +#include "mongo/util/assert_util.h" + +namespace mongo { + +namespace { + +class ProducerConsumerQueueTest : public unittest::Test { +public: + ProducerConsumerQueueTest() : _serviceCtx(stdx::make_unique()) {} + + template + stdx::thread runThread(StringData name, Callback&& cb) { + return stdx::thread([this, name, cb] { cb(); }); + } + +private: + std::unique_ptr _serviceCtx; +}; + +class MoveOnly { +public: + struct CostFunc { + CostFunc() = default; + explicit CostFunc(size_t val) : val(val) {} + + size_t operator()(const MoveOnly& mo) const { + return val + *mo._val; + } + + const size_t val = 0; + }; + + explicit MoveOnly(int i) : _val(i) {} + + MoveOnly(const MoveOnly&) = delete; + MoveOnly& operator=(const MoveOnly&) = delete; + + MoveOnly(MoveOnly&& other) : _val(other._val) { + other._val.reset(); + } + + MoveOnly& operator=(MoveOnly&& other) { + if (&other == this) { + return *this; + } + + _val = other._val; + other._val.reset(); + + return *this; + } + + bool movedFrom() const { + return !_val; + } + + friend bool operator==(const MoveOnly& lhs, const MoveOnly& rhs) { + return *lhs._val == *rhs._val; + } + + friend bool operator!=(const MoveOnly& lhs, const MoveOnly& rhs) { + return !(lhs == rhs); + } + + friend std::ostream& operator<<(std::ostream& os, const MoveOnly& mo) { + return (os << "MoveOnly(" << *mo._val << ")"); + } + +private: + boost::optional _val; +}; + +TEST_F(ProducerConsumerQueueTest, basicPushPop) { + ProducerConsumerQueue pcq{}; + + runThread("Producer", [&]() { pcq.push(MoveOnly(1)); }).join(); + + ASSERT_EQUALS(pcq.sizeForTest(), 1ul); + + runThread("Consumer", [&]() { ASSERT_EQUALS(pcq.pop(), MoveOnly(1)); }).join(); + + ASSERT_TRUE(pcq.emptyForTest()); +} + +TEST_F(ProducerConsumerQueueTest, closeConsumerEnd) { + ProducerConsumerQueue pcq{1}; + + pcq.push(MoveOnly(1)); + + auto producer = runThread("Producer", [&]() { + ASSERT_THROWS_CODE( + pcq.push(MoveOnly(2)), DBException, ErrorCodes::ProducerConsumerQueueEndClosed); + }); + + ASSERT_EQUALS(pcq.sizeForTest(), 1ul); + + pcq.closeConsumerEnd(); + + ASSERT_THROWS_CODE(pcq.pop(), DBException, ErrorCodes::ProducerConsumerQueueEndClosed); + + producer.join(); +} + +TEST_F(ProducerConsumerQueueTest, closeProducerEndImmediate) { + ProducerConsumerQueue pcq{}; + + pcq.push(MoveOnly(1)); + pcq.closeProducerEnd(); + + runThread("Consumer", [&]() { + ASSERT_EQUALS(pcq.pop(), MoveOnly(1)); + + ASSERT_THROWS_CODE(pcq.pop(), DBException, ErrorCodes::ProducerConsumerQueueEndClosed); + }).join(); +} + +TEST_F(ProducerConsumerQueueTest, closeProducerEndBlocking) { + ProducerConsumerQueue pcq{}; + + auto consumer = runThread("Consumer", [&]() { + ASSERT_THROWS_CODE(pcq.pop(), DBException, ErrorCodes::ProducerConsumerQueueEndClosed); + }); + + pcq.closeProducerEnd(); + + consumer.join(); +} + +TEST_F(ProducerConsumerQueueTest, popsWithTimeout) { + ProducerConsumerQueue pcq{}; + + runThread("Consumer", [&]() { + ASSERT_THROWS_CODE(pcq.pop(Milliseconds(100)), DBException, ErrorCodes::ExceededTimeLimit); + + std::vector vec; + ASSERT_THROWS_CODE(pcq.popMany(std::back_inserter(vec), Milliseconds(100)), + DBException, + ErrorCodes::ExceededTimeLimit); + + ASSERT_THROWS_CODE(pcq.popManyUpTo(1000, std::back_inserter(vec), Milliseconds(100)), + DBException, + ErrorCodes::ExceededTimeLimit); + }).join(); + + ASSERT_EQUALS(pcq.sizeForTest(), 0ul); +} + +TEST_F(ProducerConsumerQueueTest, pushesWithTimeout) { + ProducerConsumerQueue pcq{1}; + + { + MoveOnly mo(1); + pcq.push(std::move(mo)); + ASSERT(mo.movedFrom()); + } + + runThread("Consumer", [&]() { + { + MoveOnly mo(2); + ASSERT_THROWS_CODE(pcq.push(std::move(mo), Milliseconds(100)), + DBException, + ErrorCodes::ExceededTimeLimit); + ASSERT_EQUALS(pcq.sizeForTest(), 1ul); + ASSERT(!mo.movedFrom()); + ASSERT_EQUALS(mo, MoveOnly(2)); + } + + { + std::vector vec; + vec.emplace_back(MoveOnly(2)); + + auto iter = begin(vec); + ASSERT_THROWS_CODE(pcq.pushMany(iter, end(vec), Milliseconds(100)), + DBException, + ErrorCodes::ExceededTimeLimit); + ASSERT_EQUALS(pcq.sizeForTest(), 1ul); + ASSERT(!vec[0].movedFrom()); + ASSERT_EQUALS(vec[0], MoveOnly(2)); + } + }).join(); + + ASSERT_EQUALS(pcq.sizeForTest(), 1ul); +} + +TEST_F(ProducerConsumerQueueTest, basicPushPopWithBlocking) { + ProducerConsumerQueue pcq{}; + + auto consumer = runThread("Consumer", [&]() { ASSERT_EQUALS(pcq.pop(), MoveOnly(1)); }); + + auto producer = runThread("Producer", [&]() { pcq.push(MoveOnly(1)); }); + + consumer.join(); + producer.join(); + + ASSERT_TRUE(pcq.emptyForTest()); +} + +TEST_F(ProducerConsumerQueueTest, multipleStepPushPopWithBlocking) { + ProducerConsumerQueue pcq{1}; + + auto consumer = runThread("Consumer", [&]() { + for (int i = 0; i < 10; ++i) { + ASSERT_EQUALS(pcq.pop(), MoveOnly(i)); + } + }); + + auto producer = runThread("Producer", [&]() { + for (int i = 0; i < 10; ++i) { + pcq.push(MoveOnly(i)); + } + }); + + consumer.join(); + producer.join(); + + ASSERT_TRUE(pcq.emptyForTest()); +} + + +TEST_F(ProducerConsumerQueueTest, pushTooLarge) { + { + ProducerConsumerQueue pcq{1}; + + runThread("Producer", [&]() { + ASSERT_THROWS_CODE( + pcq.push(MoveOnly(2)), DBException, ErrorCodes::ProducerConsumerQueueBatchTooLarge); + }).join(); + } + + { + ProducerConsumerQueue pcq{4}; + + std::vector vec; + vec.push_back(MoveOnly(3)); + vec.push_back(MoveOnly(3)); + + runThread("Producer", [&]() { + ASSERT_THROWS_CODE(pcq.pushMany(begin(vec), end(vec)), + DBException, + ErrorCodes::ProducerConsumerQueueBatchTooLarge); + }).join(); + } +} + +TEST_F(ProducerConsumerQueueTest, pushManyPopWithoutBlocking) { + ProducerConsumerQueue pcq{}; + + runThread("Producer", [&]() { + std::vector vec; + for (int i = 0; i < 10; ++i) { + vec.emplace_back(MoveOnly(i)); + } + + pcq.pushMany(begin(vec), end(vec)); + }).join(); + + runThread("Consumer", [&]() { + for (int i = 0; i < 10; ++i) { + ASSERT_EQUALS(pcq.pop(), MoveOnly(i)); + } + }).join(); + + ASSERT_TRUE(pcq.emptyForTest()); +} + +TEST_F(ProducerConsumerQueueTest, popManyPopWithBlocking) { + ProducerConsumerQueue pcq{2}; + + auto consumer = runThread("Consumer", [&]() { + for (int i = 0; i < 10; i = i + 2) { + std::vector out; + + pcq.popMany(std::back_inserter(out)); + + ASSERT_EQUALS(out.size(), 2ul); + ASSERT_EQUALS(out[0], MoveOnly(i)); + ASSERT_EQUALS(out[1], MoveOnly(i + 1)); + } + }); + + auto producer = runThread("Producer", [&]() { + std::vector vec; + for (int i = 0; i < 10; ++i) { + vec.emplace_back(MoveOnly(i)); + } + + for (auto iter = begin(vec); iter != end(vec); iter += 2) { + pcq.pushMany(iter, iter + 2); + } + }); + + consumer.join(); + producer.join(); + + ASSERT_TRUE(pcq.emptyForTest()); +} + +TEST_F(ProducerConsumerQueueTest, popManyUpToPopWithBlocking) { + ProducerConsumerQueue pcq{4}; + + auto consumer = runThread("Consumer", [&]() { + for (int i = 0; i < 10; i = i + 2) { + std::vector out; + + size_t spent; + std::tie(spent, std::ignore) = pcq.popManyUpTo(2, std::back_inserter(out)); + + ASSERT_EQUALS(spent, 2ul); + ASSERT_EQUALS(out.size(), 2ul); + ASSERT_EQUALS(out[0], MoveOnly(i)); + ASSERT_EQUALS(out[1], MoveOnly(i + 1)); + } + }); + + auto producer = runThread("Producer", [&]() { + std::vector vec; + for (int i = 0; i < 10; ++i) { + vec.emplace_back(MoveOnly(i)); + } + + for (auto iter = begin(vec); iter != end(vec); iter += 2) { + pcq.pushMany(iter, iter + 2); + } + }); + + consumer.join(); + producer.join(); + + ASSERT_TRUE(pcq.emptyForTest()); +} + +TEST_F(ProducerConsumerQueueTest, popManyUpToPopWithBlockingWithSpecialCost) { + ProducerConsumerQueue pcq{}; + + auto consumer = runThread("Consumer", [&]() { + { + std::vector out; + size_t spent; + std::tie(spent, std::ignore) = pcq.popManyUpTo(5, std::back_inserter(out)); + + ASSERT_EQUALS(spent, 6ul); + ASSERT_EQUALS(out.size(), 3ul); + ASSERT_EQUALS(out[0], MoveOnly(1)); + ASSERT_EQUALS(out[1], MoveOnly(2)); + ASSERT_EQUALS(out[2], MoveOnly(3)); + } + + { + std::vector out; + size_t spent; + std::tie(spent, std::ignore) = pcq.popManyUpTo(15, std::back_inserter(out)); + + ASSERT_EQUALS(spent, 9ul); + ASSERT_EQUALS(out.size(), 2ul); + ASSERT_EQUALS(out[0], MoveOnly(4)); + ASSERT_EQUALS(out[1], MoveOnly(5)); + } + }); + + auto producer = runThread("Producer", [&]() { + std::vector vec; + for (int i = 1; i < 6; ++i) { + vec.emplace_back(MoveOnly(i)); + } + + pcq.pushMany(begin(vec), end(vec)); + }); + + consumer.join(); + producer.join(); + + ASSERT_TRUE(pcq.emptyForTest()); +} + +TEST_F(ProducerConsumerQueueTest, singleProducerMultiConsumer) { + ProducerConsumerQueue pcq{}; + + stdx::mutex mutex; + size_t success = 0; + size_t failure = 0; + + std::array threads; + for (auto& thread : threads) { + thread = runThread("Consumer", [&]() { + { + try { + pcq.pop(); + stdx::lock_guard lk(mutex); + success++; + } catch (const DBException& exception) { + ASSERT_EQUALS(exception.getCode(), ErrorCodes::ProducerConsumerQueueEndClosed); + stdx::lock_guard lk(mutex); + failure++; + } + } + }); + } + + pcq.push(MoveOnly(1)); + pcq.push(MoveOnly(2)); + + pcq.closeProducerEnd(); + + for (auto& thread : threads) { + thread.join(); + } + + ASSERT_EQUALS(success, 2ul); + ASSERT_EQUALS(failure, 1ul); + + ASSERT_TRUE(pcq.emptyForTest()); +} + +TEST_F(ProducerConsumerQueueTest, basicTryPop) { + ProducerConsumerQueue pcq{}; + + ASSERT_FALSE(pcq.tryPop()); + ASSERT_TRUE(pcq.tryPush(MoveOnly(1))); + ASSERT_EQUALS(pcq.sizeForTest(), 1ul); + + auto val = pcq.tryPop(); + + ASSERT_FALSE(pcq.tryPop()); + ASSERT_TRUE(val); + ASSERT_EQUALS(*val, MoveOnly(1)); + + ASSERT_TRUE(pcq.emptyForTest()); +} + +TEST_F(ProducerConsumerQueueTest, basicTryPush) { + ProducerConsumerQueue pcq{1}; + + ASSERT_TRUE(pcq.tryPush(MoveOnly(1))); + ASSERT_FALSE(pcq.tryPush(MoveOnly(2))); + + ASSERT_EQUALS(pcq.sizeForTest(), 1ul); + + auto val = pcq.tryPop(); + ASSERT_FALSE(pcq.tryPop()); + ASSERT_TRUE(val); + ASSERT_EQUALS(*val, MoveOnly(1)); + + ASSERT_TRUE(pcq.emptyForTest()); +} + +TEST_F(ProducerConsumerQueueTest, tryPushWithSpecialCost) { + ProducerConsumerQueue pcq{5}; + + ASSERT_TRUE(pcq.tryPush(MoveOnly(1))); + ASSERT_TRUE(pcq.tryPush(MoveOnly(2))); + ASSERT_FALSE(pcq.tryPush(MoveOnly(3))); + + ASSERT_EQUALS(pcq.sizeForTest(), 3ul); + + auto val1 = pcq.tryPop(); + ASSERT_EQUALS(pcq.sizeForTest(), 2ul); + auto val2 = pcq.tryPop(); + ASSERT_EQUALS(pcq.sizeForTest(), 0ul); + ASSERT_FALSE(pcq.tryPop()); + ASSERT_TRUE(val1); + ASSERT_TRUE(val2); + ASSERT_EQUALS(*val1, MoveOnly(1)); + ASSERT_EQUALS(*val2, MoveOnly(2)); + + ASSERT_TRUE(pcq.emptyForTest()); +} + +TEST_F(ProducerConsumerQueueTest, tryPushWithSpecialStatefulCost) { + ProducerConsumerQueue pcq{5, MoveOnly::CostFunc(1)}; + + ASSERT_TRUE(pcq.tryPush(MoveOnly(1))); + ASSERT_TRUE(pcq.tryPush(MoveOnly(2))); + ASSERT_FALSE(pcq.tryPush(MoveOnly(3))); + + ASSERT_EQUALS(pcq.sizeForTest(), 5ul); + + auto val1 = pcq.tryPop(); + ASSERT_EQUALS(pcq.sizeForTest(), 3ul); + auto val2 = pcq.tryPop(); + ASSERT_EQUALS(pcq.sizeForTest(), 0ul); + ASSERT_FALSE(pcq.tryPop()); + ASSERT_TRUE(val1); + ASSERT_TRUE(val2); + ASSERT_EQUALS(*val1, MoveOnly(1)); + ASSERT_EQUALS(*val2, MoveOnly(2)); + + ASSERT_TRUE(pcq.emptyForTest()); +} + +} // namespace + +} // namespace mongo diff --git a/src/mongo/util/tcmalloc_server_status_section.cpp b/src/mongo/util/tcmalloc_server_status_section.cpp index 303076ba784..784f74e8cf8 100644 --- a/src/mongo/util/tcmalloc_server_status_section.cpp +++ b/src/mongo/util/tcmalloc_server_status_section.cpp @@ -38,6 +38,7 @@ #include "mongo/base/init.h" #include "mongo/db/commands/server_status.h" +#include "mongo/db/server_parameters.h" #include "mongo/db/service_context.h" #include "mongo/transport/transport_layer.h" #include "mongo/util/log.h" @@ -55,11 +56,17 @@ const int kManyClients = 40; stdx::mutex tcmallocCleanupLock; +MONGO_EXPORT_SERVER_PARAMETER(tcmallocEnableMarkThreadIdle, bool, true); + /** * Callback to allow TCMalloc to release freed memory to the central list at * favorable times. Ideally would do some milder cleanup or scavenge... */ void threadStateChange() { + if (!tcmallocEnableMarkThreadIdle.load()) { + return; + } + if (getGlobalServiceContext()->getTransportLayer()->sessionStats().numOpenSessions <= kManyClients) return; @@ -156,6 +163,25 @@ public: appendNumericPropertyIfAvailable( sub, "aggressive_memory_decommit", "tcmalloc.aggressive_memory_decommit"); + appendNumericPropertyIfAvailable( + sub, "pageheap_committed_bytes", "tcmalloc.pageheap_committed_bytes"); + appendNumericPropertyIfAvailable( + sub, "pageheap_scavenge_count", "tcmalloc.pageheap_scavenge_count"); + appendNumericPropertyIfAvailable( + sub, "pageheap_commit_count", "tcmalloc.pageheap_commit_count"); + appendNumericPropertyIfAvailable( + sub, "pageheap_total_commit_bytes", "tcmalloc.pageheap_total_commit_bytes"); + appendNumericPropertyIfAvailable( + sub, "pageheap_decommit_count", "tcmalloc.pageheap_decommit_count"); + appendNumericPropertyIfAvailable( + sub, "pageheap_total_decommit_bytes", "tcmalloc.pageheap_total_decommit_bytes"); + appendNumericPropertyIfAvailable( + sub, "pageheap_reserve_count", "tcmalloc.pageheap_reserve_count"); + appendNumericPropertyIfAvailable( + sub, "pageheap_total_reserve_bytes", "tcmalloc.pageheap_total_reserve_bytes"); + appendNumericPropertyIfAvailable( + sub, "spinlock_total_delay_ns", "tcmalloc.spinlock_total_delay_ns"); + #if MONGO_HAVE_GPERFTOOLS_SIZE_CLASS_STATS if (verbosity >= 2) { // Size class information diff --git a/src/third_party/SConscript b/src/third_party/SConscript index 3a820399677..add20816b83 100644 --- a/src/third_party/SConscript +++ b/src/third_party/SConscript @@ -6,7 +6,7 @@ Import("wiredtiger") boostSuffix = "-1.60.0" snappySuffix = '-1.1.3' zlibSuffix = '-1.2.8' -pcreSuffix = "-8.39" +pcreSuffix = "-8.41" mozjsSuffix = '-45' yamlSuffix = '-0.5.3' icuSuffix = '-57.1' diff --git a/src/third_party/gperftools-2.5/src/base/spinlock.cc b/src/third_party/gperftools-2.5/src/base/spinlock.cc index 85ff21ed404..acb01c8537a 100644 --- a/src/third_party/gperftools-2.5/src/base/spinlock.cc +++ b/src/third_party/gperftools-2.5/src/base/spinlock.cc @@ -49,6 +49,47 @@ const base::LinkerInitialized SpinLock::LINKER_INITIALIZED = base::LINKER_INITIALIZED; namespace { + +const int64_t kNanoPerSec = uint64_t(1000) * 1000 * 1000; + +#ifdef _WIN32 +int64_t frequency; + +void InitTimer() { + LARGE_INTEGER large_freq; + QueryPerformanceFrequency(&large_freq); + frequency = large_freq.QuadPart; +} + +int64_t NowMonotonic() { + LARGE_INTEGER time_value; + QueryPerformanceCounter(&time_value); + return time_value.QuadPart; +} + +int64_t TicksToNanos(int64_t timer_value) { + return timer_value * kNanoPerSec / frequency; +} + +#else // _WIN32 +void InitTimer() {} + +int64_t NowMonotonic() { + struct timespec t; + + if (clock_gettime(CLOCK_MONOTONIC, &t)) { + return 0; + } + + return (static_cast(t.tv_sec) * kNanoPerSec) + t.tv_nsec; +} + +int64_t TicksToNanos(int64_t timer_value) { + return timer_value; +} + +#endif // _WIN32 + struct SpinLock_InitHelper { SpinLock_InitHelper() { // On multi-cpu machines, spin for longer before yielding @@ -56,6 +97,8 @@ struct SpinLock_InitHelper { if (GetSystemCPUsCount() > 1) { adaptive_spin_count = 1000; } + + InitTimer(); } }; @@ -69,6 +112,31 @@ inline void SpinlockPause(void) { __asm__ __volatile__("rep; nop" : : ); #endif } + +// The current version of atomic_ops.h lacks base::subtle::Barrier_AtomicIncrement +// so a CAS loop is used instead. +void NoBarrier_AtomicAdd(volatile base::subtle::Atomic64* ptr, + base::subtle::Atomic64 increment) { + base::subtle::Atomic64 base_value = base::subtle::NoBarrier_Load(ptr); + while (true) { + base::subtle::Atomic64 new_value; + base::subtle::NoBarrier_Store(&new_value, base::subtle::NoBarrier_Load(&base_value) + + base::subtle::NoBarrier_Load(&increment)); + + // Swap in the new incremented value. + base::subtle::Atomic64 cas_result = base::subtle::Acquire_CompareAndSwap( + ptr, base_value, new_value); + + // Check if the increment succeeded. + if (cas_result == base_value) { + return; + } + + // If the increment failed, just use the previous value as the value to + // add our increment to. + base_value = cas_result; + }; +} } // unnamed namespace @@ -84,10 +152,13 @@ Atomic32 SpinLock::SpinLoop() { kSpinLockSleeper); } +base::subtle::Atomic64 SpinLock::totalDelayNanos_ = 0; + void SpinLock::SlowLock() { Atomic32 lock_value = SpinLoop(); int lock_wait_call_count = 0; + int64_t start = 0; while (lock_value != kSpinLockFree) { // If the lock is currently held, but not marked as having a sleeper, mark // it as having a sleeper. @@ -115,12 +186,18 @@ void SpinLock::SlowLock() { } // Wait for an OS specific delay. + start = NowMonotonic(); base::internal::SpinLockDelay(&lockword_, lock_value, ++lock_wait_call_count); // Spin again after returning from the wait routine to give this thread // some chance of obtaining the lock. lock_value = SpinLoop(); } + + if (start) { + NoBarrier_AtomicAdd(&totalDelayNanos_, static_cast( + TicksToNanos(NowMonotonic() - start))); + } } void SpinLock::SlowUnlock() { diff --git a/src/third_party/gperftools-2.5/src/base/spinlock.h b/src/third_party/gperftools-2.5/src/base/spinlock.h index 7243aeaaefd..c5ad8f048f6 100644 --- a/src/third_party/gperftools-2.5/src/base/spinlock.h +++ b/src/third_party/gperftools-2.5/src/base/spinlock.h @@ -107,6 +107,10 @@ class LOCKABLE SpinLock { return base::subtle::NoBarrier_Load(&lockword_) != kSpinLockFree; } + static base::subtle::Atomic64 GetTotalDelayNanos() { + return base::subtle::NoBarrier_Load(&totalDelayNanos_); + } + static const base::LinkerInitialized LINKER_INITIALIZED; // backwards compat private: enum { kSpinLockFree = 0 }; @@ -115,6 +119,8 @@ class LOCKABLE SpinLock { volatile Atomic32 lockword_; + static base::subtle::Atomic64 totalDelayNanos_; + void SlowLock(); void SlowUnlock(); Atomic32 SpinLoop(); diff --git a/src/third_party/gperftools-2.5/src/page_heap.cc b/src/third_party/gperftools-2.5/src/page_heap.cc index f52ae2af029..eaa33255f2c 100644 --- a/src/third_party/gperftools-2.5/src/page_heap.cc +++ b/src/third_party/gperftools-2.5/src/page_heap.cc @@ -245,16 +245,22 @@ Span* PageHeap::Split(Span* span, Length n) { } void PageHeap::CommitSpan(Span* span) { + ++stats_.commit_count; + TCMalloc_SystemCommit(reinterpret_cast(span->start << kPageShift), static_cast(span->length << kPageShift)); stats_.committed_bytes += span->length << kPageShift; + stats_.total_commit_bytes += (span->length << kPageShift); } bool PageHeap::DecommitSpan(Span* span) { + ++stats_.decommit_count; + bool rv = TCMalloc_SystemRelease(reinterpret_cast(span->start << kPageShift), static_cast(span->length << kPageShift)); if (rv) { stats_.committed_bytes -= span->length << kPageShift; + stats_.total_decommit_bytes += (span->length << kPageShift); } return rv; @@ -441,6 +447,8 @@ void PageHeap::IncrementalScavenge(Length n) { return; } + ++stats_.scavenge_count; + Length released_pages = ReleaseAtLeastNPages(1); if (released_pages == 0) { @@ -616,9 +624,16 @@ bool PageHeap::GrowHeap(Length n) { ask = actual_size >> kPageShift; RecordGrowth(ask << kPageShift); + ++stats_.reserve_count; + ++stats_.commit_count; + uint64_t old_system_bytes = stats_.system_bytes; stats_.system_bytes += (ask << kPageShift); stats_.committed_bytes += (ask << kPageShift); + + stats_.total_commit_bytes += (ask << kPageShift); + stats_.total_reserve_bytes += (ask << kPageShift); + const PageID p = reinterpret_cast(ptr) >> kPageShift; ASSERT(p > 0); diff --git a/src/third_party/gperftools-2.5/src/page_heap.h b/src/third_party/gperftools-2.5/src/page_heap.h index 18abed1974a..5c6607bbf16 100644 --- a/src/third_party/gperftools-2.5/src/page_heap.h +++ b/src/third_party/gperftools-2.5/src/page_heap.h @@ -143,12 +143,24 @@ class PERFTOOLS_DLL_DECL PageHeap { // Page heap statistics struct Stats { - Stats() : system_bytes(0), free_bytes(0), unmapped_bytes(0), committed_bytes(0) {} + Stats() : system_bytes(0), free_bytes(0), unmapped_bytes(0), committed_bytes(0), + scavenge_count(0), commit_count(0), decommit_count(0), + total_commit_bytes(0), total_decommit_bytes(0), + reserve_count(0), total_reserve_bytes(0) {} uint64_t system_bytes; // Total bytes allocated from system uint64_t free_bytes; // Total bytes on normal freelists uint64_t unmapped_bytes; // Total bytes on returned freelists uint64_t committed_bytes; // Bytes committed, always <= system_bytes_. + uint64_t scavenge_count; // Number of times scavagened flush pages + + uint64_t commit_count; // Number of virtual memory commits + uint64_t total_commit_bytes; // Bytes committed in lifetime of process + uint64_t decommit_count; // Number of virtual memory decommits + uint64_t total_decommit_bytes; // Bytes decommitted in lifetime of process + + uint64_t reserve_count; // Number of virtual memory reserves + uint64_t total_reserve_bytes; // Bytes reserved in lifetime of process }; inline Stats stats() const { return stats_; } diff --git a/src/third_party/gperftools-2.5/src/tcmalloc.cc b/src/third_party/gperftools-2.5/src/tcmalloc.cc index ba6c06e9d89..b573b4a9821 100644 --- a/src/third_party/gperftools-2.5/src/tcmalloc.cc +++ b/src/third_party/gperftools-2.5/src/tcmalloc.cc @@ -703,6 +703,54 @@ class TCMallocImplementation : public MallocExtension { return true; } + if (strcmp(name, "tcmalloc.pageheap_committed_bytes") == 0) { + SpinLockHolder l(Static::pageheap_lock()); + *value = Static::pageheap()->stats().committed_bytes; + return true; + } + + if (strcmp(name, "tcmalloc.pageheap_scavenge_count") == 0) { + SpinLockHolder l(Static::pageheap_lock()); + *value = Static::pageheap()->stats().scavenge_count; + return true; + } + + if (strcmp(name, "tcmalloc.pageheap_commit_count") == 0) { + SpinLockHolder l(Static::pageheap_lock()); + *value = Static::pageheap()->stats().commit_count; + return true; + } + + if (strcmp(name, "tcmalloc.pageheap_total_commit_bytes") == 0) { + SpinLockHolder l(Static::pageheap_lock()); + *value = Static::pageheap()->stats().total_commit_bytes; + return true; + } + + if (strcmp(name, "tcmalloc.pageheap_decommit_count") == 0) { + SpinLockHolder l(Static::pageheap_lock()); + *value = Static::pageheap()->stats().decommit_count; + return true; + } + + if (strcmp(name, "tcmalloc.pageheap_total_decommit_bytes") == 0) { + SpinLockHolder l(Static::pageheap_lock()); + *value = Static::pageheap()->stats().total_decommit_bytes; + return true; + } + + if (strcmp(name, "tcmalloc.pageheap_reserve_count") == 0) { + SpinLockHolder l(Static::pageheap_lock()); + *value = Static::pageheap()->stats().reserve_count; + return true; + } + + if (strcmp(name, "tcmalloc.pageheap_total_reserve_bytes") == 0) { + SpinLockHolder l(Static::pageheap_lock()); + *value = Static::pageheap()->stats().total_reserve_bytes; + return true; + } + if (strcmp(name, "tcmalloc.max_total_thread_cache_bytes") == 0) { SpinLockHolder l(Static::pageheap_lock()); *value = ThreadCache::overall_thread_cache_size(); @@ -721,6 +769,11 @@ class TCMallocImplementation : public MallocExtension { return true; } + if (strcmp(name, "tcmalloc.spinlock_total_delay_ns") == 0) { + *value = SpinLock::GetTotalDelayNanos(); + return true; + } + return false; } diff --git a/src/third_party/pcre-8.39/132html b/src/third_party/pcre-8.39/132html deleted file mode 100755 index e00024970a5..00000000000 --- a/src/third_party/pcre-8.39/132html +++ /dev/null @@ -1,313 +0,0 @@ -#! /usr/bin/perl -w - -# Script to turn PCRE man pages into HTML - - -# Subroutine to handle font changes and other escapes - -sub do_line { -my($s) = $_[0]; - -$s =~ s/ -$s =~ s/>/>/g; -$s =~ s"\\fI(.*?)\\f[RP]"$1"g; -$s =~ s"\\fB(.*?)\\f[RP]"$1"g; -$s =~ s"\\e"\\"g; -$s =~ s/(?<=Copyright )\(c\)/©/g; -$s; -} - -# Subroutine to ensure not in a paragraph - -sub end_para { -if ($inpara) - { - print TEMP "\n" if ($inpre); - print TEMP "

\n"; - } -$inpara = $inpre = 0; -$wrotetext = 0; -} - -# Subroutine to start a new paragraph - -sub new_para { -&end_para(); -print TEMP "

\n"; -$inpara = 1; -} - - -# Main program - -$innf = 0; -$inpara = 0; -$inpre = 0; -$wrotetext = 0; -$toc = 0; -$ref = 1; - -while ($#ARGV >= 0 && $ARGV[0] =~ /^-/) - { - $toc = 1 if $ARGV[0] eq "-toc"; - shift; - } - -# Initial output to STDOUT - -print < - -$ARGV[0] specification - - -

$ARGV[0] man page

-

-Return to the PCRE index page. -

-

-This page is part of the PCRE HTML documentation. It was generated automatically -from the original man page. If there is any nonsense in it, please consult the -man page, in case the conversion went wrong. -
-End - -print "

    \n" if ($toc); - -open(TEMP, ">/tmp/$$") || die "Can't open /tmp/$$ for output\n"; - -while () - { - # Handle lines beginning with a dot - - if (/^\./) - { - # Some of the PCRE man pages used to contain instances of .br. However, - # they should have all been removed because they cause trouble in some - # (other) automated systems that translate man pages to HTML. Complain if - # we find .br or .in (another macro that is deprecated). - - if (/^\.br/ || /^\.in/) - { - print STDERR "\n*** Deprecated macro encountered - rewrite needed\n"; - print STDERR "*** $_\n"; - die "*** Processing abandoned\n"; - } - - # Instead of .br, relevent "literal" sections are enclosed in .nf/.fi. - - elsif (/^\.nf/) - { - $innf = 1; - } - - elsif (/^\.fi/) - { - $innf = 0; - } - - # Handling .sp is subtle. If it is inside a literal section, do nothing if - # the next line is a non literal text line; similarly, if not inside a - # literal section, do nothing if a literal follows, unless we are inside - # a .nf/.ne section. The point being that the 
     and
    that delimit - # literal sections will do the spacing. Always skip if no previous output. - - elsif (/^\.sp/) - { - if ($wrotetext) - { - $_ = ; - if ($inpre) - { - print TEMP "\n" if (/^[\s.]/); - } - else - { - print TEMP "
    \n
    \n" if ($innf || !/^[\s.]/); - } - redo; # Now process the lookahead line we just read - } - } - elsif (/^\.TP/ || /^\.PP/ || /^\.P/) - { - &new_para(); - } - elsif (/^\.SH\s*("?)(.*)\1/) - { - # Ignore the NAME section - if ($2 =~ /^NAME\b/) - { - ; - next; - } - - &end_para(); - my($title) = &do_line($2); - if ($toc) - { - printf("
  • $title\n", - $ref, $ref); - printf TEMP ("
    $title
    \n", - $ref); - $ref++; - } - else - { - print TEMP "
    \n$title\n
    \n"; - } - } - elsif (/^\.SS\s*("?)(.*)\1/) - { - &end_para(); - my($title) = &do_line($2); - print TEMP "
    \n$title\n
    \n"; - } - elsif (/^\.B\s*(.*)/) - { - &new_para() if (!$inpara); - $_ = &do_line($1); - s/"(.*?)"/$1/g; - print TEMP "$_\n"; - $wrotetext = 1; - } - elsif (/^\.I\s*(.*)/) - { - &new_para() if (!$inpara); - $_ = &do_line($1); - s/"(.*?)"/$1/g; - print TEMP "$_\n"; - $wrotetext = 1; - } - - # A comment that starts "HREF" takes the next line as a name that - # is turned into a hyperlink, using the text given, which might be - # in a special font. If it ends in () or (digits) or punctuation, they - # aren't part of the link. - - elsif (/^\.\\"\s*HREF/) - { - $_=; - chomp; - $_ = &do_line($_); - $_ =~ s/\s+$//; - $_ =~ /^(?:<.>)?([^<(]+)(?:\(\))?(?:<\/.>)?(?:\(\d+\))?[.,;:]?$/; - print TEMP "$_\n"; - } - - # A comment that starts "HTML" inserts literal HTML - - elsif (/^\.\\"\s*HTML\s*(.*)/) - { - print TEMP $1; - } - - # A comment that starts < inserts that HTML at the end of the - # *next* input line - so as not to get a newline between them. - - elsif (/^\.\\"\s*(<.*>)/) - { - my($markup) = $1; - $_=; - chomp; - $_ = &do_line($_); - $_ =~ s/\s+$//; - print TEMP "$_$markup\n"; - } - - # A comment that starts JOIN joins the next two lines together, with one - # space between them. Then that line is processed. This is used in some - # displays where two lines are needed for the "man" version. JOINSH works - # the same, except that it assumes this is a shell command, so removes - # continuation backslashes. - - elsif (/^\.\\"\s*JOIN(SH)?/) - { - my($one,$two); - $one = ; - $two = ; - $one =~ s/\s*\\e\s*$// if (defined($1)); - chomp($one); - $two =~ s/^\s+//; - $_ = "$one $two"; - redo; # Process the joined lines - } - - # .EX/.EE are used in the pcredemo page to bracket the entire program, - # which is unmodified except for turning backslash into "\e". - - elsif (/^\.EX\s*$/) - { - print TEMP "
    \n";
-      while ()
-        {
-        last if /^\.EE\s*$/;
-        s/\\e/\\/g;
-        s/&/&/g;
-        s//>/g;
-        print TEMP;
-        }
-      }
-
-    # Ignore anything not recognized
-
-    next;
-    }
-
-  # Line does not begin with a dot. Replace blank lines with new paragraphs
-
-  if (/^\s*$/)
-    {
-    &end_para() if ($wrotetext);
-    next;
-    }
-
-  # Convert fonts changes and output an ordinary line. Ensure that indented
-  # lines are marked as literal.
-
-  $_ = &do_line($_);
-  &new_para() if (!$inpara);
-
-  if (/^\s/)
-    {
-    if (!$inpre)
-      {
-      print TEMP "
    \n";
-      $inpre = 1;
-      }
-    }
-  elsif ($inpre)
-    {
-    print TEMP "
    \n";
-    $inpre = 0;
-    }
-
-  # Add 
     to the end of a non-literal line if we are within .nf/.fi
-
-  $_ .= "
    \n" if (!$inpre && $innf);
-
-  print TEMP;
-  $wrotetext = 1;
-  }
-
-# The TOC, if present, will have been written - terminate it
-
-print "
\n" if ($toc); - -# Copy the remainder to the standard output - -close(TEMP); -open(TEMP, "/tmp/$$") || die "Can't open /tmp/$$ for input\n"; - -print while (); - -print < -Return to the PCRE index page. -

-End - -close(TEMP); -unlink("/tmp/$$"); - -# End diff --git a/src/third_party/pcre-8.39/AUTHORS b/src/third_party/pcre-8.39/AUTHORS deleted file mode 100644 index 342417a8a19..00000000000 --- a/src/third_party/pcre-8.39/AUTHORS +++ /dev/null @@ -1,45 +0,0 @@ -THE MAIN PCRE LIBRARY ---------------------- - -Written by: Philip Hazel -Email local part: ph10 -Email domain: cam.ac.uk - -University of Cambridge Computing Service, -Cambridge, England. - -Copyright (c) 1997-2016 University of Cambridge -All rights reserved - - -PCRE JUST-IN-TIME COMPILATION SUPPORT -------------------------------------- - -Written by: Zoltan Herczeg -Email local part: hzmester -Emain domain: freemail.hu - -Copyright(c) 2010-2016 Zoltan Herczeg -All rights reserved. - - -STACK-LESS JUST-IN-TIME COMPILER --------------------------------- - -Written by: Zoltan Herczeg -Email local part: hzmester -Emain domain: freemail.hu - -Copyright(c) 2009-2016 Zoltan Herczeg -All rights reserved. - - -THE C++ WRAPPER LIBRARY ------------------------ - -Written by: Google Inc. - -Copyright (c) 2007-2012 Google Inc -All rights reserved - -#### diff --git a/src/third_party/pcre-8.39/COPYING b/src/third_party/pcre-8.39/COPYING deleted file mode 100644 index 58eed01b61d..00000000000 --- a/src/third_party/pcre-8.39/COPYING +++ /dev/null @@ -1,5 +0,0 @@ -PCRE LICENCE - -Please see the file LICENCE in the PCRE distribution for licensing details. - -End diff --git a/src/third_party/pcre-8.39/ChangeLog b/src/third_party/pcre-8.39/ChangeLog deleted file mode 100644 index a34f845f8a1..00000000000 --- a/src/third_party/pcre-8.39/ChangeLog +++ /dev/null @@ -1,6010 +0,0 @@ -ChangeLog for PCRE ------------------- - -Note that the PCRE 8.xx series (PCRE1) is now in a bugfix-only state. All -development is happening in the PCRE2 10.xx series. - -Version 8.39 14-June-2016 -------------------------- - -1. If PCRE_AUTO_CALLOUT was set on a pattern that had a (?# comment between - an item and its qualifier (for example, A(?#comment)?B) pcre_compile() - misbehaved. This bug was found by the LLVM fuzzer. - -2. Similar to the above, if an isolated \E was present between an item and its - qualifier when PCRE_AUTO_CALLOUT was set, pcre_compile() misbehaved. This - bug was found by the LLVM fuzzer. - -3. Further to 8.38/46, negated classes such as [^[:^ascii:]\d] were also not - working correctly in UCP mode. - -4. The POSIX wrapper function regexec() crashed if the option REG_STARTEND - was set when the pmatch argument was NULL. It now returns REG_INVARG. - -5. Allow for up to 32-bit numbers in the ordin() function in pcregrep. - -6. An empty \Q\E sequence between an item and its qualifier caused - pcre_compile() to misbehave when auto callouts were enabled. This bug was - found by the LLVM fuzzer. - -7. If a pattern that was compiled with PCRE_EXTENDED started with white - space or a #-type comment that was followed by (?-x), which turns off - PCRE_EXTENDED, and there was no subsequent (?x) to turn it on again, - pcre_compile() assumed that (?-x) applied to the whole pattern and - consequently mis-compiled it. This bug was found by the LLVM fuzzer. - -8. A call of pcre_copy_named_substring() for a named substring whose number - was greater than the space in the ovector could cause a crash. - -9. Yet another buffer overflow bug involved duplicate named groups with a - group that reset capture numbers (compare 8.38/7 below). Once again, I have - just allowed for more memory, even if not needed. (A proper fix is - implemented in PCRE2, but it involves a lot of refactoring.) - -10. pcre_get_substring_list() crashed if the use of \K in a match caused the - start of the match to be earlier than the end. - -11. Migrating appropriate PCRE2 JIT improvements to PCRE. - -12. A pattern such as /(?<=((?C)0))/, which has a callout inside a lookbehind - assertion, caused pcretest to generate incorrect output, and also to read - uninitialized memory (detected by ASAN or valgrind). - -13. A pattern that included (*ACCEPT) in the middle of a sufficiently deeply - nested set of parentheses of sufficient size caused an overflow of the - compiling workspace (which was diagnosed, but of course is not desirable). - -14. And yet another buffer overflow bug involving duplicate named groups, this - time nested, with a nested back reference. Yet again, I have just allowed - for more memory, because anything more needs all the refactoring that has - been done for PCRE2. An example pattern that provoked this bug is: - /((?J)(?'R'(?'R'(?'R'(?'R'(?'R'(?|(\k'R'))))))))/ and the bug was - registered as CVE-2016-1283. - -15. pcretest went into a loop if global matching was requested with an ovector - size less than 2. It now gives an error message. This bug was found by - afl-fuzz. - -16. An invalid pattern fragment such as (?(?C)0 was not diagnosing an error - ("assertion expected") when (?(?C) was not followed by an opening - parenthesis. - -17. Fixed typo ("&&" for "&") in pcre_study(). Fortunately, this could not - actually affect anything, by sheer luck. - -18. Applied Chris Wilson's patch (Bugzilla #1681) to CMakeLists.txt for MSVC - static compilation. - -19. Modified the RunTest script to incorporate a valgrind suppressions file so - that certain errors, provoked by the SSE2 instruction set when JIT is used, - are ignored. - -20. A racing condition is fixed in JIT reported by Mozilla. - -21. Minor code refactor to avoid "array subscript is below array bounds" - compiler warning. - -22. Minor code refactor to avoid "left shift of negative number" warning. - -23. Fix typo causing compile error when 16- or 32-bit JIT is compiled without - UCP support. - -24. Refactor to avoid compiler warnings in pcrecpp.cc. - -25. Refactor to fix a typo in pcre_jit_test.c - -26. Patch to support compiling pcrecpp.cc with Intel compiler. - - -Version 8.38 23-November-2015 ------------------------------ - -1. If a group that contained a recursive back reference also contained a - forward reference subroutine call followed by a non-forward-reference - subroutine call, for example /.((?2)(?R)\1)()/, pcre_compile() failed to - compile correct code, leading to undefined behaviour or an internally - detected error. This bug was discovered by the LLVM fuzzer. - -2. Quantification of certain items (e.g. atomic back references) could cause - incorrect code to be compiled when recursive forward references were - involved. For example, in this pattern: /(?1)()((((((\1++))\x85)+)|))/. - This bug was discovered by the LLVM fuzzer. - -3. A repeated conditional group whose condition was a reference by name caused - a buffer overflow if there was more than one group with the given name. - This bug was discovered by the LLVM fuzzer. - -4. A recursive back reference by name within a group that had the same name as - another group caused a buffer overflow. For example: - /(?J)(?'d'(?'d'\g{d}))/. This bug was discovered by the LLVM fuzzer. - -5. A forward reference by name to a group whose number is the same as the - current group, for example in this pattern: /(?|(\k'Pm')|(?'Pm'))/, caused - a buffer overflow at compile time. This bug was discovered by the LLVM - fuzzer. - -6. A lookbehind assertion within a set of mutually recursive subpatterns could - provoke a buffer overflow. This bug was discovered by the LLVM fuzzer. - -7. Another buffer overflow bug involved duplicate named groups with a - reference between their definition, with a group that reset capture - numbers, for example: /(?J:(?|(?'R')(\k'R')|((?'R'))))/. This has been - fixed by always allowing for more memory, even if not needed. (A proper fix - is implemented in PCRE2, but it involves more refactoring.) - -8. There was no check for integer overflow in subroutine calls such as (?123). - -9. The table entry for \l in EBCDIC environments was incorrect, leading to its - being treated as a literal 'l' instead of causing an error. - -10. There was a buffer overflow if pcre_exec() was called with an ovector of - size 1. This bug was found by american fuzzy lop. - -11. If a non-capturing group containing a conditional group that could match - an empty string was repeated, it was not identified as matching an empty - string itself. For example: /^(?:(?(1)x|)+)+$()/. - -12. In an EBCDIC environment, pcretest was mishandling the escape sequences - \a and \e in test subject lines. - -13. In an EBCDIC environment, \a in a pattern was converted to the ASCII - instead of the EBCDIC value. - -14. The handling of \c in an EBCDIC environment has been revised so that it is - now compatible with the specification in Perl's perlebcdic page. - -15. The EBCDIC character 0x41 is a non-breaking space, equivalent to 0xa0 in - ASCII/Unicode. This has now been added to the list of characters that are - recognized as white space in EBCDIC. - -16. When PCRE was compiled without UCP support, the use of \p and \P gave an - error (correctly) when used outside a class, but did not give an error - within a class. - -17. \h within a class was incorrectly compiled in EBCDIC environments. - -18. A pattern with an unmatched closing parenthesis that contained a backward - assertion which itself contained a forward reference caused buffer - overflow. And example pattern is: /(?=di(?<=(?1))|(?=(.))))/. - -19. JIT should return with error when the compiled pattern requires more stack - space than the maximum. - -20. A possessively repeated conditional group that could match an empty string, - for example, /(?(R))*+/, was incorrectly compiled. - -21. Fix infinite recursion in the JIT compiler when certain patterns such as - /(?:|a|){100}x/ are analysed. - -22. Some patterns with character classes involving [: and \\ were incorrectly - compiled and could cause reading from uninitialized memory or an incorrect - error diagnosis. - -23. Pathological patterns containing many nested occurrences of [: caused - pcre_compile() to run for a very long time. - -24. A conditional group with only one branch has an implicit empty alternative - branch and must therefore be treated as potentially matching an empty - string. - -25. If (?R was followed by - or + incorrect behaviour happened instead of a - diagnostic. - -26. Arrange to give up on finding the minimum matching length for overly - complex patterns. - -27. Similar to (4) above: in a pattern with duplicated named groups and an - occurrence of (?| it is possible for an apparently non-recursive back - reference to become recursive if a later named group with the relevant - number is encountered. This could lead to a buffer overflow. Wen Guanxing - from Venustech ADLAB discovered this bug. - -28. If pcregrep was given the -q option with -c or -l, or when handling a - binary file, it incorrectly wrote output to stdout. - -29. The JIT compiler did not restore the control verb head in case of *THEN - control verbs. This issue was found by Karl Skomski with a custom LLVM - fuzzer. - -30. Error messages for syntax errors following \g and \k were giving inaccurate - offsets in the pattern. - -31. Added a check for integer overflow in conditions (?() and - (?(R). This omission was discovered by Karl Skomski with the LLVM - fuzzer. - -32. Handling recursive references such as (?2) when the reference is to a group - later in the pattern uses code that is very hacked about and error-prone. - It has been re-written for PCRE2. Here in PCRE1, a check has been added to - give an internal error if it is obvious that compiling has gone wrong. - -33. The JIT compiler should not check repeats after a {0,1} repeat byte code. - This issue was found by Karl Skomski with a custom LLVM fuzzer. - -34. The JIT compiler should restore the control chain for empty possessive - repeats. This issue was found by Karl Skomski with a custom LLVM fuzzer. - -35. Match limit check added to JIT recursion. This issue was found by Karl - Skomski with a custom LLVM fuzzer. - -36. Yet another case similar to 27 above has been circumvented by an - unconditional allocation of extra memory. This issue is fixed "properly" in - PCRE2 by refactoring the way references are handled. Wen Guanxing - from Venustech ADLAB discovered this bug. - -37. Fix two assertion fails in JIT. These issues were found by Karl Skomski - with a custom LLVM fuzzer. - -38. Fixed a corner case of range optimization in JIT. - -39. An incorrect error "overran compiling workspace" was given if there were - exactly enough group forward references such that the last one extended - into the workspace safety margin. The next one would have expanded the - workspace. The test for overflow was not including the safety margin. - -40. A match limit issue is fixed in JIT which was found by Karl Skomski - with a custom LLVM fuzzer. - -41. Remove the use of /dev/null in testdata/testinput2, because it doesn't - work under Windows. (Why has it taken so long for anyone to notice?) - -42. In a character class such as [\W\p{Any}] where both a negative-type escape - ("not a word character") and a property escape were present, the property - escape was being ignored. - -43. Fix crash caused by very long (*MARK) or (*THEN) names. - -44. A sequence such as [[:punct:]b] that is, a POSIX character class followed - by a single ASCII character in a class item, was incorrectly compiled in - UCP mode. The POSIX class got lost, but only if the single character - followed it. - -45. [:punct:] in UCP mode was matching some characters in the range 128-255 - that should not have been matched. - -46. If [:^ascii:] or [:^xdigit:] or [:^cntrl:] are present in a non-negated - class, all characters with code points greater than 255 are in the class. - When a Unicode property was also in the class (if PCRE_UCP is set, escapes - such as \w are turned into Unicode properties), wide characters were not - correctly handled, and could fail to match. - - -Version 8.37 28-April-2015 --------------------------- - -1. When an (*ACCEPT) is triggered inside capturing parentheses, it arranges - for those parentheses to be closed with whatever has been captured so far. - However, it was failing to mark any other groups between the hightest - capture so far and the currrent group as "unset". Thus, the ovector for - those groups contained whatever was previously there. An example is the - pattern /(x)|((*ACCEPT))/ when matched against "abcd". - -2. If an assertion condition was quantified with a minimum of zero (an odd - thing to do, but it happened), SIGSEGV or other misbehaviour could occur. - -3. If a pattern in pcretest input had the P (POSIX) modifier followed by an - unrecognized modifier, a crash could occur. - -4. An attempt to do global matching in pcretest with a zero-length ovector - caused a crash. - -5. Fixed a memory leak during matching that could occur for a subpattern - subroutine call (recursive or otherwise) if the number of captured groups - that had to be saved was greater than ten. - -6. Catch a bad opcode during auto-possessification after compiling a bad UTF - string with NO_UTF_CHECK. This is a tidyup, not a bug fix, as passing bad - UTF with NO_UTF_CHECK is documented as having an undefined outcome. - -7. A UTF pattern containing a "not" match of a non-ASCII character and a - subroutine reference could loop at compile time. Example: /[^\xff]((?1))/. - -8. When a pattern is compiled, it remembers the highest back reference so that - when matching, if the ovector is too small, extra memory can be obtained to - use instead. A conditional subpattern whose condition is a check on a - capture having happened, such as, for example in the pattern - /^(?:(a)|b)(?(1)A|B)/, is another kind of back reference, but it was not - setting the highest backreference number. This mattered only if pcre_exec() - was called with an ovector that was too small to hold the capture, and there - was no other kind of back reference (a situation which is probably quite - rare). The effect of the bug was that the condition was always treated as - FALSE when the capture could not be consulted, leading to a incorrect - behaviour by pcre_exec(). This bug has been fixed. - -9. A reference to a duplicated named group (either a back reference or a test - for being set in a conditional) that occurred in a part of the pattern where - PCRE_DUPNAMES was not set caused the amount of memory needed for the pattern - to be incorrectly calculated, leading to overwriting. - -10. A mutually recursive set of back references such as (\2)(\1) caused a - segfault at study time (while trying to find the minimum matching length). - The infinite loop is now broken (with the minimum length unset, that is, - zero). - -11. If an assertion that was used as a condition was quantified with a minimum - of zero, matching went wrong. In particular, if the whole group had - unlimited repetition and could match an empty string, a segfault was - likely. The pattern (?(?=0)?)+ is an example that caused this. Perl allows - assertions to be quantified, but not if they are being used as conditions, - so the above pattern is faulted by Perl. PCRE has now been changed so that - it also rejects such patterns. - -12. A possessive capturing group such as (a)*+ with a minimum repeat of zero - failed to allow the zero-repeat case if pcre2_exec() was called with an - ovector too small to capture the group. - -13. Fixed two bugs in pcretest that were discovered by fuzzing and reported by - Red Hat Product Security: - - (a) A crash if /K and /F were both set with the option to save the compiled - pattern. - - (b) Another crash if the option to print captured substrings in a callout - was combined with setting a null ovector, for example \O\C+ as a subject - string. - -14. A pattern such as "((?2){0,1999}())?", which has a group containing a - forward reference repeated a large (but limited) number of times within a - repeated outer group that has a zero minimum quantifier, caused incorrect - code to be compiled, leading to the error "internal error: - previously-checked referenced subpattern not found" when an incorrect - memory address was read. This bug was reported as "heap overflow", - discovered by Kai Lu of Fortinet's FortiGuard Labs and given the CVE number - CVE-2015-2325. - -23. A pattern such as "((?+1)(\1))/" containing a forward reference subroutine - call within a group that also contained a recursive back reference caused - incorrect code to be compiled. This bug was reported as "heap overflow", - discovered by Kai Lu of Fortinet's FortiGuard Labs, and given the CVE - number CVE-2015-2326. - -24. Computing the size of the JIT read-only data in advance has been a source - of various issues, and new ones are still appear unfortunately. To fix - existing and future issues, size computation is eliminated from the code, - and replaced by on-demand memory allocation. - -25. A pattern such as /(?i)[A-`]/, where characters in the other case are - adjacent to the end of the range, and the range contained characters with - more than one other case, caused incorrect behaviour when compiled in UTF - mode. In that example, the range a-j was left out of the class. - -26. Fix JIT compilation of conditional blocks, which assertion - is converted to (*FAIL). E.g: /(?(?!))/. - -27. The pattern /(?(?!)^)/ caused references to random memory. This bug was - discovered by the LLVM fuzzer. - -28. The assertion (?!) is optimized to (*FAIL). This was not handled correctly - when this assertion was used as a condition, for example (?(?!)a|b). In - pcre2_match() it worked by luck; in pcre2_dfa_match() it gave an incorrect - error about an unsupported item. - -29. For some types of pattern, for example /Z*(|d*){216}/, the auto- - possessification code could take exponential time to complete. A recursion - depth limit of 1000 has been imposed to limit the resources used by this - optimization. - -30. A pattern such as /(*UTF)[\S\V\H]/, which contains a negated special class - such as \S in non-UCP mode, explicit wide characters (> 255) can be ignored - because \S ensures they are all in the class. The code for doing this was - interacting badly with the code for computing the amount of space needed to - compile the pattern, leading to a buffer overflow. This bug was discovered - by the LLVM fuzzer. - -31. A pattern such as /((?2)+)((?1))/ which has mutual recursion nested inside - other kinds of group caused stack overflow at compile time. This bug was - discovered by the LLVM fuzzer. - -32. A pattern such as /(?1)(?#?'){8}(a)/ which had a parenthesized comment - between a subroutine call and its quantifier was incorrectly compiled, - leading to buffer overflow or other errors. This bug was discovered by the - LLVM fuzzer. - -33. The illegal pattern /(?(?.*!.*)?)/ was not being diagnosed as missing an - assertion after (?(. The code was failing to check the character after - (?(?< for the ! or = that would indicate a lookbehind assertion. This bug - was discovered by the LLVM fuzzer. - -34. A pattern such as /X((?2)()*+){2}+/ which has a possessive quantifier with - a fixed maximum following a group that contains a subroutine reference was - incorrectly compiled and could trigger buffer overflow. This bug was - discovered by the LLVM fuzzer. - -35. A mutual recursion within a lookbehind assertion such as (?<=((?2))((?1))) - caused a stack overflow instead of the diagnosis of a non-fixed length - lookbehind assertion. This bug was discovered by the LLVM fuzzer. - -36. The use of \K in a positive lookbehind assertion in a non-anchored pattern - (e.g. /(?<=\Ka)/) could make pcregrep loop. - -37. There was a similar problem to 36 in pcretest for global matches. - -38. If a greedy quantified \X was preceded by \C in UTF mode (e.g. \C\X*), - and a subsequent item in the pattern caused a non-match, backtracking over - the repeated \X did not stop, but carried on past the start of the subject, - causing reference to random memory and/or a segfault. There were also some - other cases where backtracking after \C could crash. This set of bugs was - discovered by the LLVM fuzzer. - -39. The function for finding the minimum length of a matching string could take - a very long time if mutual recursion was present many times in a pattern, - for example, /((?2){73}(?2))((?1))/. A better mutual recursion detection - method has been implemented. This infelicity was discovered by the LLVM - fuzzer. - -40. Static linking against the PCRE library using the pkg-config module was - failing on missing pthread symbols. - - -Version 8.36 26-September-2014 ------------------------------- - -1. Got rid of some compiler warnings in the C++ modules that were shown up by - -Wmissing-field-initializers and -Wunused-parameter. - -2. The tests for quantifiers being too big (greater than 65535) were being - applied after reading the number, and stupidly assuming that integer - overflow would give a negative number. The tests are now applied as the - numbers are read. - -3. Tidy code in pcre_exec.c where two branches that used to be different are - now the same. - -4. The JIT compiler did not generate match limit checks for certain - bracketed expressions with quantifiers. This may lead to exponential - backtracking, instead of returning with PCRE_ERROR_MATCHLIMIT. This - issue should be resolved now. - -5. Fixed an issue, which occures when nested alternatives are optimized - with table jumps. - -6. Inserted two casts and changed some ints to size_t in the light of some - reported 64-bit compiler warnings (Bugzilla 1477). - -7. Fixed a bug concerned with zero-minimum possessive groups that could match - an empty string, which sometimes were behaving incorrectly in the - interpreter (though correctly in the JIT matcher). This pcretest input is - an example: - - '\A(?:[^"]++|"(?:[^"]*+|"")*+")++' - NON QUOTED "QUOT""ED" AFTER "NOT MATCHED - - the interpreter was reporting a match of 'NON QUOTED ' only, whereas the - JIT matcher and Perl both matched 'NON QUOTED "QUOT""ED" AFTER '. The test - for an empty string was breaking the inner loop and carrying on at a lower - level, when possessive repeated groups should always return to a higher - level as they have no backtrack points in them. The empty string test now - occurs at the outer level. - -8. Fixed a bug that was incorrectly auto-possessifying \w+ in the pattern - ^\w+(?>\s*)(?<=\w) which caused it not to match "test test". - -9. Give a compile-time error for \o{} (as Perl does) and for \x{} (which Perl - doesn't). - -10. Change 8.34/15 introduced a bug that caused the amount of memory needed - to hold a pattern to be incorrectly computed (too small) when there were - named back references to duplicated names. This could cause "internal - error: code overflow" or "double free or corruption" or other memory - handling errors. - -11. When named subpatterns had the same prefixes, back references could be - confused. For example, in this pattern: - - /(?Pa)?(?Pb)?(?()c|d)*l/ - - the reference to 'Name' was incorrectly treated as a reference to a - duplicate name. - -12. A pattern such as /^s?c/mi8 where the optional character has more than - one "other case" was incorrectly compiled such that it would only try to - match starting at "c". - -13. When a pattern starting with \s was studied, VT was not included in the - list of possible starting characters; this should have been part of the - 8.34/18 patch. - -14. If a character class started [\Qx]... where x is any character, the class - was incorrectly terminated at the ]. - -15. If a pattern that started with a caseless match for a character with more - than one "other case" was studied, PCRE did not set up the starting code - unit bit map for the list of possible characters. Now it does. This is an - optimization improvement, not a bug fix. - -16. The Unicode data tables have been updated to Unicode 7.0.0. - -17. Fixed a number of memory leaks in pcregrep. - -18. Avoid a compiler warning (from some compilers) for a function call with - a cast that removes "const" from an lvalue by using an intermediate - variable (to which the compiler does not object). - -19. Incorrect code was compiled if a group that contained an internal recursive - back reference was optional (had quantifier with a minimum of zero). This - example compiled incorrect code: /(((a\2)|(a*)\g<-1>))*/ and other examples - caused segmentation faults because of stack overflows at compile time. - -20. A pattern such as /((?(R)a|(?1)))+/, which contains a recursion within a - group that is quantified with an indefinite repeat, caused a compile-time - loop which used up all the system stack and provoked a segmentation fault. - This was not the same bug as 19 above. - -21. Add PCRECPP_EXP_DECL declaration to operator<< in pcre_stringpiece.h. - Patch by Mike Frysinger. - - -Version 8.35 04-April-2014 --------------------------- - -1. A new flag is set, when property checks are present in an XCLASS. - When this flag is not set, PCRE can perform certain optimizations - such as studying these XCLASS-es. - -2. The auto-possessification of character sets were improved: a normal - and an extended character set can be compared now. Furthermore - the JIT compiler optimizes more character set checks. - -3. Got rid of some compiler warnings for potentially uninitialized variables - that show up only when compiled with -O2. - -4. A pattern such as (?=ab\K) that uses \K in an assertion can set the start - of a match later then the end of the match. The pcretest program was not - handling the case sensibly - it was outputting from the start to the next - binary zero. It now reports this situation in a message, and outputs the - text from the end to the start. - -5. Fast forward search is improved in JIT. Instead of the first three - characters, any three characters with fixed position can be searched. - Search order: first, last, middle. - -6. Improve character range checks in JIT. Characters are read by an inprecise - function now, which returns with an unknown value if the character code is - above a certain threshold (e.g: 256). The only limitation is that the value - must be bigger than the threshold as well. This function is useful when - the characters above the threshold are handled in the same way. - -7. The macros whose names start with RAWUCHAR are placeholders for a future - mode in which only the bottom 21 bits of 32-bit data items are used. To - make this more memorable for those maintaining the code, the names have - been changed to start with UCHAR21, and an extensive comment has been added - to their definition. - -8. Add missing (new) files sljitNativeTILEGX.c and sljitNativeTILEGX-encoder.c - to the export list in Makefile.am (they were accidentally omitted from the - 8.34 tarball). - -9. The informational output from pcretest used the phrase "starting byte set" - which is inappropriate for the 16-bit and 32-bit libraries. As the output - for "first char" and "need char" really means "non-UTF-char", I've changed - "byte" to "char", and slightly reworded the output. The documentation about - these values has also been (I hope) clarified. - -10. Another JIT related optimization: use table jumps for selecting the correct - backtracking path, when more than four alternatives are present inside a - bracket. - -11. Empty match is not possible, when the minimum length is greater than zero, - and there is no \K in the pattern. JIT should avoid empty match checks in - such cases. - -12. In a caseless character class with UCP support, when a character with more - than one alternative case was not the first character of a range, not all - the alternative cases were added to the class. For example, s and \x{17f} - are both alternative cases for S: the class [RST] was handled correctly, - but [R-T] was not. - -13. The configure.ac file always checked for pthread support when JIT was - enabled. This is not used in Windows, so I have put this test inside a - check for the presence of windows.h (which was already tested for). - -14. Improve pattern prefix search by a simplified Boyer-Moore algorithm in JIT. - The algorithm provides a way to skip certain starting offsets, and usually - faster than linear prefix searches. - -15. Change 13 for 8.20 updated RunTest to check for the 'fr' locale as well - as for 'fr_FR' and 'french'. For some reason, however, it then used the - Windows-specific input and output files, which have 'french' screwed in. - So this could never have worked. One of the problems with locales is that - they aren't always the same. I have now updated RunTest so that it checks - the output of the locale test (test 3) against three different output - files, and it allows the test to pass if any one of them matches. With luck - this should make the test pass on some versions of Solaris where it was - failing. Because of the uncertainty, the script did not used to stop if - test 3 failed; it now does. If further versions of a French locale ever - come to light, they can now easily be added. - -16. If --with-pcregrep-bufsize was given a non-integer value such as "50K", - there was a message during ./configure, but it did not stop. This now - provokes an error. The invalid example in README has been corrected. - If a value less than the minimum is given, the minimum value has always - been used, but now a warning is given. - -17. If --enable-bsr-anycrlf was set, the special 16/32-bit test failed. This - was a bug in the test system, which is now fixed. Also, the list of various - configurations that are tested for each release did not have one with both - 16/32 bits and --enable-bar-anycrlf. It now does. - -18. pcretest was missing "-C bsr" for displaying the \R default setting. - -19. Little endian PowerPC systems are supported now by the JIT compiler. - -20. The fast forward newline mechanism could enter to an infinite loop on - certain invalid UTF-8 input. Although we don't support these cases - this issue can be fixed by a performance optimization. - -21. Change 33 of 8.34 is not sufficient to ensure stack safety because it does - not take account if existing stack usage. There is now a new global - variable called pcre_stack_guard that can be set to point to an external - function to check stack availability. It is called at the start of - processing every parenthesized group. - -22. A typo in the code meant that in ungreedy mode the max/min qualifier - behaved like a min-possessive qualifier, and, for example, /a{1,3}b/U did - not match "ab". - -23. When UTF was disabled, the JIT program reported some incorrect compile - errors. These messages are silenced now. - -24. Experimental support for ARM-64 and MIPS-64 has been added to the JIT - compiler. - -25. Change all the temporary files used in RunGrepTest to be different to those - used by RunTest so that the tests can be run simultaneously, for example by - "make -j check". - - -Version 8.34 15-December-2013 ------------------------------ - -1. Add pcre[16|32]_jit_free_unused_memory to forcibly free unused JIT - executable memory. Patch inspired by Carsten Klein. - -2. ./configure --enable-coverage defined SUPPORT_GCOV in config.h, although - this macro is never tested and has no effect, because the work to support - coverage involves only compiling and linking options and special targets in - the Makefile. The comment in config.h implied that defining the macro would - enable coverage support, which is totally false. There was also support for - setting this macro in the CMake files (my fault, I just copied it from - configure). SUPPORT_GCOV has now been removed. - -3. Make a small performance improvement in strlen16() and strlen32() in - pcretest. - -4. Change 36 for 8.33 left some unreachable statements in pcre_exec.c, - detected by the Solaris compiler (gcc doesn't seem to be able to diagnose - these cases). There was also one in pcretest.c. - -5. Cleaned up a "may be uninitialized" compiler warning in pcre_exec.c. - -6. In UTF mode, the code for checking whether a group could match an empty - string (which is used for indefinitely repeated groups to allow for - breaking an infinite loop) was broken when the group contained a repeated - negated single-character class with a character that occupied more than one - data item and had a minimum repetition of zero (for example, [^\x{100}]* in - UTF-8 mode). The effect was undefined: the group might or might not be - deemed as matching an empty string, or the program might have crashed. - -7. The code for checking whether a group could match an empty string was not - recognizing that \h, \H, \v, \V, and \R must match a character. - -8. Implemented PCRE_INFO_MATCH_EMPTY, which yields 1 if the pattern can match - an empty string. If it can, pcretest shows this in its information output. - -9. Fixed two related bugs that applied to Unicode extended grapheme clusters - that were repeated with a maximizing qualifier (e.g. \X* or \X{2,5}) when - matched by pcre_exec() without using JIT: - - (a) If the rest of the pattern did not match after a maximal run of - grapheme clusters, the code for backing up to try with fewer of them - did not always back up over a full grapheme when characters that do not - have the modifier quality were involved, e.g. Hangul syllables. - - (b) If the match point in a subject started with modifier character, and - there was no match, the code could incorrectly back up beyond the match - point, and potentially beyond the first character in the subject, - leading to a segfault or an incorrect match result. - -10. A conditional group with an assertion condition could lead to PCRE - recording an incorrect first data item for a match if no other first data - item was recorded. For example, the pattern (?(?=ab)ab) recorded "a" as a - first data item, and therefore matched "ca" after "c" instead of at the - start. - -11. Change 40 for 8.33 (allowing pcregrep to find empty strings) showed up a - bug that caused the command "echo a | ./pcregrep -M '|a'" to loop. - -12. The source of pcregrep now includes z/OS-specific code so that it can be - compiled for z/OS as part of the special z/OS distribution. - -13. Added the -T and -TM options to pcretest. - -14. The code in pcre_compile.c for creating the table of named capturing groups - has been refactored. Instead of creating the table dynamically during the - actual compiling pass, the information is remembered during the pre-compile - pass (on the stack unless there are more than 20 named groups, in which - case malloc() is used) and the whole table is created before the actual - compile happens. This has simplified the code (it is now nearly 150 lines - shorter) and prepared the way for better handling of references to groups - with duplicate names. - -15. A back reference to a named subpattern when there is more than one of the - same name now checks them in the order in which they appear in the pattern. - The first one that is set is used for the reference. Previously only the - first one was inspected. This change makes PCRE more compatible with Perl. - -16. Unicode character properties were updated from Unicode 6.3.0. - -17. The compile-time code for auto-possessification has been refactored, based - on a patch by Zoltan Herczeg. It now happens after instead of during - compilation. The code is cleaner, and more cases are handled. The option - PCRE_NO_AUTO_POSSESS is added for testing purposes, and the -O and /O - options in pcretest are provided to set it. It can also be set by - (*NO_AUTO_POSSESS) at the start of a pattern. - -18. The character VT has been added to the default ("C" locale) set of - characters that match \s and are generally treated as white space, - following this same change in Perl 5.18. There is now no difference between - "Perl space" and "POSIX space". Whether VT is treated as white space in - other locales depends on the locale. - -19. The code for checking named groups as conditions, either for being set or - for being recursed, has been refactored (this is related to 14 and 15 - above). Processing unduplicated named groups should now be as fast at - numerical groups, and processing duplicated groups should be faster than - before. - -20. Two patches to the CMake build system, by Alexander Barkov: - - (1) Replace the "source" command by "." in CMakeLists.txt because - "source" is a bash-ism. - - (2) Add missing HAVE_STDINT_H and HAVE_INTTYPES_H to config-cmake.h.in; - without these the CMake build does not work on Solaris. - -21. Perl has changed its handling of \8 and \9. If there is no previously - encountered capturing group of those numbers, they are treated as the - literal characters 8 and 9 instead of a binary zero followed by the - literals. PCRE now does the same. - -22. Following Perl, added \o{} to specify codepoints in octal, making it - possible to specify values greater than 0777 and also making them - unambiguous. - -23. Perl now gives an error for missing closing braces after \x{... instead of - treating the string as literal. PCRE now does the same. - -24. RunTest used to grumble if an inappropriate test was selected explicitly, - but just skip it when running all tests. This make it awkward to run ranges - of tests when one of them was inappropriate. Now it just skips any - inappropriate tests, as it always did when running all tests. - -25. If PCRE_AUTO_CALLOUT and PCRE_UCP were set for a pattern that contained - character types such as \d or \w, too many callouts were inserted, and the - data that they returned was rubbish. - -26. In UCP mode, \s was not matching two of the characters that Perl matches, - namely NEL (U+0085) and MONGOLIAN VOWEL SEPARATOR (U+180E), though they - were matched by \h. The code has now been refactored so that the lists of - the horizontal and vertical whitespace characters used for \h and \v (which - are defined only in one place) are now also used for \s. - -27. Add JIT support for the 64 bit TileGX architecture. - Patch by Jiong Wang (Tilera Corporation). - -28. Possessive quantifiers for classes (both explicit and automatically - generated) now use special opcodes instead of wrapping in ONCE brackets. - -29. Whereas an item such as A{4}+ ignored the possessivenes of the quantifier - (because it's meaningless), this was not happening when PCRE_CASELESS was - set. Not wrong, but inefficient. - -30. Updated perltest.pl to add /u (force Unicode mode) when /W (use Unicode - properties for \w, \d, etc) is present in a test regex. Otherwise if the - test contains no characters greater than 255, Perl doesn't realise it - should be using Unicode semantics. - -31. Upgraded the handling of the POSIX classes [:graph:], [:print:], and - [:punct:] when PCRE_UCP is set so as to include the same characters as Perl - does in Unicode mode. - -32. Added the "forbid" facility to pcretest so that putting tests into the - wrong test files can sometimes be quickly detected. - -33. There is now a limit (default 250) on the depth of nesting of parentheses. - This limit is imposed to control the amount of system stack used at compile - time. It can be changed at build time by --with-parens-nest-limit=xxx or - the equivalent in CMake. - -34. Character classes such as [A-\d] or [a-[:digit:]] now cause compile-time - errors. Perl warns for these when in warning mode, but PCRE has no facility - for giving warnings. - -35. Change 34 for 8.13 allowed quantifiers on assertions, because Perl does. - However, this was not working for (?!) because it is optimized to (*FAIL), - for which PCRE does not allow quantifiers. The optimization is now disabled - when a quantifier follows (?!). I can't see any use for this, but it makes - things uniform. - -36. Perl no longer allows group names to start with digits, so I have made this - change also in PCRE. It simplifies the code a bit. - -37. In extended mode, Perl ignores spaces before a + that indicates a - possessive quantifier. PCRE allowed a space before the quantifier, but not - before the possessive +. It now does. - -38. The use of \K (reset reported match start) within a repeated possessive - group such as (a\Kb)*+ was not working. - -40. Document that the same character tables must be used at compile time and - run time, and that the facility to pass tables to pcre_exec() and - pcre_dfa_exec() is for use only with saved/restored patterns. - -41. Applied Jeff Trawick's patch CMakeLists.txt, which "provides two new - features for Builds with MSVC: - - 1. Support pcre.rc and/or pcreposix.rc (as is already done for MinGW - builds). The .rc files can be used to set FileDescription and many other - attributes. - - 2. Add an option (-DINSTALL_MSVC_PDB) to enable installation of .pdb files. - This allows higher-level build scripts which want .pdb files to avoid - hard-coding the exact files needed." - -42. Added support for [[:<:]] and [[:>:]] as used in the BSD POSIX library to - mean "start of word" and "end of word", respectively, as a transition aid. - -43. A minimizing repeat of a class containing codepoints greater than 255 in - non-UTF 16-bit or 32-bit modes caused an internal error when PCRE was - compiled to use the heap for recursion. - -44. Got rid of some compiler warnings for unused variables when UTF but not UCP - is configured. - - -Version 8.33 28-May-2013 ------------------------- - -1. Added 'U' to some constants that are compared to unsigned integers, to - avoid compiler signed/unsigned warnings. Added (int) casts to unsigned - variables that are added to signed variables, to ensure the result is - signed and can be negated. - -2. Applied patch by Daniel Richard G for quashing MSVC warnings to the - CMake config files. - -3. Revise the creation of config.h.generic so that all boolean macros are - #undefined, whereas non-boolean macros are #ifndef/#endif-ed. This makes - overriding via -D on the command line possible. - -4. Changing the definition of the variable "op" in pcre_exec.c from pcre_uchar - to unsigned int is reported to make a quite noticeable speed difference in - a specific Windows environment. Testing on Linux did also appear to show - some benefit (and it is clearly not harmful). Also fixed the definition of - Xop which should be unsigned. - -5. Related to (4), changing the definition of the intermediate variable cc - in repeated character loops from pcre_uchar to pcre_uint32 also gave speed - improvements. - -6. Fix forward search in JIT when link size is 3 or greater. Also removed some - unnecessary spaces. - -7. Adjust autogen.sh and configure.ac to lose warnings given by automake 1.12 - and later. - -8. Fix two buffer over read issues in 16 and 32 bit modes. Affects JIT only. - -9. Optimizing fast_forward_start_bits in JIT. - -10. Adding support for callouts in JIT, and fixing some issues revealed - during this work. Namely: - - (a) Unoptimized capturing brackets incorrectly reset on backtrack. - - (b) Minimum length was not checked before the matching is started. - -11. The value of capture_last that is passed to callouts was incorrect in some - cases when there was a capture on one path that was subsequently abandoned - after a backtrack. Also, the capture_last value is now reset after a - recursion, since all captures are also reset in this case. - -12. The interpreter no longer returns the "too many substrings" error in the - case when an overflowing capture is in a branch that is subsequently - abandoned after a backtrack. - -13. In the pathological case when an offset vector of size 2 is used, pcretest - now prints out the matched string after a yield of 0 or 1. - -14. Inlining subpatterns in recursions, when certain conditions are fulfilled. - Only supported by the JIT compiler at the moment. - -15. JIT compiler now supports 32 bit Macs thanks to Lawrence Velazquez. - -16. Partial matches now set offsets[2] to the "bumpalong" value, that is, the - offset of the starting point of the matching process, provided the offsets - vector is large enough. - -17. The \A escape now records a lookbehind value of 1, though its execution - does not actually inspect the previous character. This is to ensure that, - in partial multi-segment matching, at least one character from the old - segment is retained when a new segment is processed. Otherwise, if there - are no lookbehinds in the pattern, \A might match incorrectly at the start - of a new segment. - -18. Added some #ifdef __VMS code into pcretest.c to help VMS implementations. - -19. Redefined some pcre_uchar variables in pcre_exec.c as pcre_uint32; this - gives some modest performance improvement in 8-bit mode. - -20. Added the PCRE-specific property \p{Xuc} for matching characters that can - be expressed in certain programming languages using Universal Character - Names. - -21. Unicode validation has been updated in the light of Unicode Corrigendum #9, - which points out that "non characters" are not "characters that may not - appear in Unicode strings" but rather "characters that are reserved for - internal use and have only local meaning". - -22. When a pattern was compiled with automatic callouts (PCRE_AUTO_CALLOUT) and - there was a conditional group that depended on an assertion, if the - assertion was false, the callout that immediately followed the alternation - in the condition was skipped when pcre_exec() was used for matching. - -23. Allow an explicit callout to be inserted before an assertion that is the - condition for a conditional group, for compatibility with automatic - callouts, which always insert a callout at this point. - -24. In 8.31, (*COMMIT) was confined to within a recursive subpattern. Perl also - confines (*SKIP) and (*PRUNE) in the same way, and this has now been done. - -25. (*PRUNE) is now supported by the JIT compiler. - -26. Fix infinite loop when /(?<=(*SKIP)ac)a/ is matched against aa. - -27. Fix the case where there are two or more SKIPs with arguments that may be - ignored. - -28. (*SKIP) is now supported by the JIT compiler. - -29. (*THEN) is now supported by the JIT compiler. - -30. Update RunTest with additional test selector options. - -31. The way PCRE handles backtracking verbs has been changed in two ways. - - (1) Previously, in something like (*COMMIT)(*SKIP), COMMIT would override - SKIP. Now, PCRE acts on whichever backtracking verb is reached first by - backtracking. In some cases this makes it more Perl-compatible, but Perl's - rather obscure rules do not always do the same thing. - - (2) Previously, backtracking verbs were confined within assertions. This is - no longer the case for positive assertions, except for (*ACCEPT). Again, - this sometimes improves Perl compatibility, and sometimes does not. - -32. A number of tests that were in test 2 because Perl did things differently - have been moved to test 1, because either Perl or PCRE has changed, and - these tests are now compatible. - -32. Backtracking control verbs are now handled in the same way in JIT and - interpreter. - -33. An opening parenthesis in a MARK/PRUNE/SKIP/THEN name in a pattern that - contained a forward subroutine reference caused a compile error. - -34. Auto-detect and optimize limited repetitions in JIT. - -35. Implement PCRE_NEVER_UTF to lock out the use of UTF, in particular, - blocking (*UTF) etc. - -36. In the interpreter, maximizing pattern repetitions for characters and - character types now use tail recursion, which reduces stack usage. - -37. The value of the max lookbehind was not correctly preserved if a compiled - and saved regex was reloaded on a host of different endianness. - -38. Implemented (*LIMIT_MATCH) and (*LIMIT_RECURSION). As part of the extension - of the compiled pattern block, expand the flags field from 16 to 32 bits - because it was almost full. - -39. Try madvise first before posix_madvise. - -40. Change 7 for PCRE 7.9 made it impossible for pcregrep to find empty lines - with a pattern such as ^$. It has taken 4 years for anybody to notice! The - original change locked out all matches of empty strings. This has been - changed so that one match of an empty string per line is recognized. - Subsequent searches on the same line (for colouring or for --only-matching, - for example) do not recognize empty strings. - -41. Applied a user patch to fix a number of spelling mistakes in comments. - -42. Data lines longer than 65536 caused pcretest to crash. - -43. Clarified the data type for length and startoffset arguments for pcre_exec - and pcre_dfa_exec in the function-specific man pages, where they were - explicitly stated to be in bytes, never having been updated. I also added - some clarification to the pcreapi man page. - -44. A call to pcre_dfa_exec() with an output vector size less than 2 caused - a segmentation fault. - - -Version 8.32 30-November-2012 ------------------------------ - -1. Improved JIT compiler optimizations for first character search and single - character iterators. - -2. Supporting IBM XL C compilers for PPC architectures in the JIT compiler. - Patch by Daniel Richard G. - -3. Single character iterator optimizations in the JIT compiler. - -4. Improved JIT compiler optimizations for character ranges. - -5. Rename the "leave" variable names to "quit" to improve WinCE compatibility. - Reported by Giuseppe D'Angelo. - -6. The PCRE_STARTLINE bit, indicating that a match can occur only at the start - of a line, was being set incorrectly in cases where .* appeared inside - atomic brackets at the start of a pattern, or where there was a subsequent - *PRUNE or *SKIP. - -7. Improved instruction cache flush for POWER/PowerPC. - Patch by Daniel Richard G. - -8. Fixed a number of issues in pcregrep, making it more compatible with GNU - grep: - - (a) There is now no limit to the number of patterns to be matched. - - (b) An error is given if a pattern is too long. - - (c) Multiple uses of --exclude, --exclude-dir, --include, and --include-dir - are now supported. - - (d) --exclude-from and --include-from (multiple use) have been added. - - (e) Exclusions and inclusions now apply to all files and directories, not - just to those obtained from scanning a directory recursively. - - (f) Multiple uses of -f and --file-list are now supported. - - (g) In a Windows environment, the default for -d has been changed from - "read" (the GNU grep default) to "skip", because otherwise the presence - of a directory in the file list provokes an error. - - (h) The documentation has been revised and clarified in places. - -9. Improve the matching speed of capturing brackets. - -10. Changed the meaning of \X so that it now matches a Unicode extended - grapheme cluster. - -11. Patch by Daniel Richard G to the autoconf files to add a macro for sorting - out POSIX threads when JIT support is configured. - -12. Added support for PCRE_STUDY_EXTRA_NEEDED. - -13. In the POSIX wrapper regcomp() function, setting re_nsub field in the preg - structure could go wrong in environments where size_t is not the same size - as int. - -14. Applied user-supplied patch to pcrecpp.cc to allow PCRE_NO_UTF8_CHECK to be - set. - -15. The EBCDIC support had decayed; later updates to the code had included - explicit references to (e.g.) \x0a instead of CHAR_LF. There has been a - general tidy up of EBCDIC-related issues, and the documentation was also - not quite right. There is now a test that can be run on ASCII systems to - check some of the EBCDIC-related things (but is it not a full test). - -16. The new PCRE_STUDY_EXTRA_NEEDED option is now used by pcregrep, resulting - in a small tidy to the code. - -17. Fix JIT tests when UTF is disabled and both 8 and 16 bit mode are enabled. - -18. If the --only-matching (-o) option in pcregrep is specified multiple - times, each one causes appropriate output. For example, -o1 -o2 outputs the - substrings matched by the 1st and 2nd capturing parentheses. A separating - string can be specified by --om-separator (default empty). - -19. Improving the first n character searches. - -20. Turn case lists for horizontal and vertical white space into macros so that - they are defined only once. - -21. This set of changes together give more compatible Unicode case-folding - behaviour for characters that have more than one other case when UCP - support is available. - - (a) The Unicode property table now has offsets into a new table of sets of - three or more characters that are case-equivalent. The MultiStage2.py - script that generates these tables (the pcre_ucd.c file) now scans - CaseFolding.txt instead of UnicodeData.txt for character case - information. - - (b) The code for adding characters or ranges of characters to a character - class has been abstracted into a generalized function that also handles - case-independence. In UTF-mode with UCP support, this uses the new data - to handle characters with more than one other case. - - (c) A bug that is fixed as a result of (b) is that codepoints less than 256 - whose other case is greater than 256 are now correctly matched - caselessly. Previously, the high codepoint matched the low one, but not - vice versa. - - (d) The processing of \h, \H, \v, and \ in character classes now makes use - of the new class addition function, using character lists defined as - macros alongside the case definitions of 20 above. - - (e) Caseless back references now work with characters that have more than - one other case. - - (f) General caseless matching of characters with more than one other case - is supported. - -22. Unicode character properties were updated from Unicode 6.2.0 - -23. Improved CMake support under Windows. Patch by Daniel Richard G. - -24. Add support for 32-bit character strings, and UTF-32 - -25. Major JIT compiler update (code refactoring and bugfixing). - Experimental Sparc 32 support is added. - -26. Applied a modified version of Daniel Richard G's patch to create - pcre.h.generic and config.h.generic by "make" instead of in the - PrepareRelease script. - -27. Added a definition for CHAR_NULL (helpful for the z/OS port), and use it in - pcre_compile.c when checking for a zero character. - -28. Introducing a native interface for JIT. Through this interface, the compiled - machine code can be directly executed. The purpose of this interface is to - provide fast pattern matching, so several sanity checks are not performed. - However, feature tests are still performed. The new interface provides - 1.4x speedup compared to the old one. - -29. If pcre_exec() or pcre_dfa_exec() was called with a negative value for - the subject string length, the error given was PCRE_ERROR_BADOFFSET, which - was confusing. There is now a new error PCRE_ERROR_BADLENGTH for this case. - -30. In 8-bit UTF-8 mode, pcretest failed to give an error for data codepoints - greater than 0x7fffffff (which cannot be represented in UTF-8, even under - the "old" RFC 2279). Instead, it ended up passing a negative length to - pcre_exec(). - -31. Add support for GCC's visibility feature to hide internal functions. - -32. Running "pcretest -C pcre8" or "pcretest -C pcre16" gave a spurious error - "unknown -C option" after outputting 0 or 1. - -33. There is now support for generating a code coverage report for the test - suite in environments where gcc is the compiler and lcov is installed. This - is mainly for the benefit of the developers. - -34. If PCRE is built with --enable-valgrind, certain memory regions are marked - unaddressable using valgrind annotations, allowing valgrind to detect - invalid memory accesses. This is mainly for the benefit of the developers. - -25. (*UTF) can now be used to start a pattern in any of the three libraries. - -26. Give configure error if --enable-cpp but no C++ compiler found. - - -Version 8.31 06-July-2012 -------------------------- - -1. Fixing a wrong JIT test case and some compiler warnings. - -2. Removed a bashism from the RunTest script. - -3. Add a cast to pcre_exec.c to fix the warning "unary minus operator applied - to unsigned type, result still unsigned" that was given by an MS compiler - on encountering the code "-sizeof(xxx)". - -4. Partial matching support is added to the JIT compiler. - -5. Fixed several bugs concerned with partial matching of items that consist - of more than one character: - - (a) /^(..)\1/ did not partially match "aba" because checking references was - done on an "all or nothing" basis. This also applied to repeated - references. - - (b) \R did not give a hard partial match if \r was found at the end of the - subject. - - (c) \X did not give a hard partial match after matching one or more - characters at the end of the subject. - - (d) When newline was set to CRLF, a pattern such as /a$/ did not recognize - a partial match for the string "\r". - - (e) When newline was set to CRLF, the metacharacter "." did not recognize - a partial match for a CR character at the end of the subject string. - -6. If JIT is requested using /S++ or -s++ (instead of just /S+ or -s+) when - running pcretest, the text "(JIT)" added to the output whenever JIT is - actually used to run the match. - -7. Individual JIT compile options can be set in pcretest by following -s+[+] - or /S+[+] with a digit between 1 and 7. - -8. OP_NOT now supports any UTF character not just single-byte ones. - -9. (*MARK) control verb is now supported by the JIT compiler. - -10. The command "./RunTest list" lists the available tests without actually - running any of them. (Because I keep forgetting what they all are.) - -11. Add PCRE_INFO_MAXLOOKBEHIND. - -12. Applied a (slightly modified) user-supplied patch that improves performance - when the heap is used for recursion (compiled with --disable-stack-for- - recursion). Instead of malloc and free for each heap frame each time a - logical recursion happens, frames are retained on a chain and re-used where - possible. This sometimes gives as much as 30% improvement. - -13. As documented, (*COMMIT) is now confined to within a recursive subpattern - call. - -14. As documented, (*COMMIT) is now confined to within a positive assertion. - -15. It is now possible to link pcretest with libedit as an alternative to - libreadline. - -16. (*COMMIT) control verb is now supported by the JIT compiler. - -17. The Unicode data tables have been updated to Unicode 6.1.0. - -18. Added --file-list option to pcregrep. - -19. Added binary file support to pcregrep, including the -a, --binary-files, - -I, and --text options. - -20. The madvise function is renamed for posix_madvise for QNX compatibility - reasons. Fixed by Giuseppe D'Angelo. - -21. Fixed a bug for backward assertions with REVERSE 0 in the JIT compiler. - -22. Changed the option for creating symbolic links for 16-bit man pages from - -s to -sf so that re-installing does not cause issues. - -23. Support PCRE_NO_START_OPTIMIZE in JIT as (*MARK) support requires it. - -24. Fixed a very old bug in pcretest that caused errors with restarted DFA - matches in certain environments (the workspace was not being correctly - retained). Also added to pcre_dfa_exec() a simple plausibility check on - some of the workspace data at the beginning of a restart. - -25. \s*\R was auto-possessifying the \s* when it should not, whereas \S*\R - was not doing so when it should - probably a typo introduced by SVN 528 - (change 8.10/14). - -26. When PCRE_UCP was not set, \w+\x{c4} was incorrectly auto-possessifying the - \w+ when the character tables indicated that \x{c4} was a word character. - There were several related cases, all because the tests for doing a table - lookup were testing for characters less than 127 instead of 255. - -27. If a pattern contains capturing parentheses that are not used in a match, - their slots in the ovector are set to -1. For those that are higher than - any matched groups, this happens at the end of processing. In the case when - there were back references that the ovector was too small to contain - (causing temporary malloc'd memory to be used during matching), and the - highest capturing number was not used, memory off the end of the ovector - was incorrectly being set to -1. (It was using the size of the temporary - memory instead of the true size.) - -28. To catch bugs like 27 using valgrind, when pcretest is asked to specify an - ovector size, it uses memory at the end of the block that it has got. - -29. Check for an overlong MARK name and give an error at compile time. The - limit is 255 for the 8-bit library and 65535 for the 16-bit library. - -30. JIT compiler update. - -31. JIT is now supported on jailbroken iOS devices. Thanks for Ruiger - Rill for the patch. - -32. Put spaces around SLJIT_PRINT_D in the JIT compiler. Required by CXX11. - -33. Variable renamings in the PCRE-JIT compiler. No functionality change. - -34. Fixed typos in pcregrep: in two places there was SUPPORT_LIBZ2 instead of - SUPPORT_LIBBZ2. This caused a build problem when bzip2 but not gzip (zlib) - was enabled. - -35. Improve JIT code generation for greedy plus quantifier. - -36. When /((?:a?)*)*c/ or /((?>a?)*)*c/ was matched against "aac", it set group - 1 to "aa" instead of to an empty string. The bug affected repeated groups - that could potentially match an empty string. - -37. Optimizing single character iterators in JIT. - -38. Wide characters specified with \uxxxx in JavaScript mode are now subject to - the same checks as \x{...} characters in non-JavaScript mode. Specifically, - codepoints that are too big for the mode are faulted, and in a UTF mode, - disallowed codepoints are also faulted. - -39. If PCRE was compiled with UTF support, in three places in the DFA - matcher there was code that should only have been obeyed in UTF mode, but - was being obeyed unconditionally. In 8-bit mode this could cause incorrect - processing when bytes with values greater than 127 were present. In 16-bit - mode the bug would be provoked by values in the range 0xfc00 to 0xdc00. In - both cases the values are those that cannot be the first data item in a UTF - character. The three items that might have provoked this were recursions, - possessively repeated groups, and atomic groups. - -40. Ensure that libpcre is explicitly listed in the link commands for pcretest - and pcregrep, because some OS require shared objects to be explicitly - passed to ld, causing the link step to fail if they are not. - -41. There were two incorrect #ifdefs in pcre_study.c, meaning that, in 16-bit - mode, patterns that started with \h* or \R* might be incorrectly matched. - - -Version 8.30 04-February-2012 ------------------------------ - -1. Renamed "isnumber" as "is_a_number" because in some Mac environments this - name is defined in ctype.h. - -2. Fixed a bug in fixed-length calculation for lookbehinds that would show up - only in quite long subpatterns. - -3. Removed the function pcre_info(), which has been obsolete and deprecated - since it was replaced by pcre_fullinfo() in February 2000. - -4. For a non-anchored pattern, if (*SKIP) was given with a name that did not - match a (*MARK), and the match failed at the start of the subject, a - reference to memory before the start of the subject could occur. This bug - was introduced by fix 17 of release 8.21. - -5. A reference to an unset group with zero minimum repetition was giving - totally wrong answers (in non-JavaScript-compatibility mode). For example, - /(another)?(\1?)test/ matched against "hello world test". This bug was - introduced in release 8.13. - -6. Add support for 16-bit character strings (a large amount of work involving - many changes and refactorings). - -7. RunGrepTest failed on msys because \r\n was replaced by whitespace when the - command "pattern=`printf 'xxx\r\njkl'`" was run. The pattern is now taken - from a file. - -8. Ovector size of 2 is also supported by JIT based pcre_exec (the ovector size - rounding is not applied in this particular case). - -9. The invalid Unicode surrogate codepoints U+D800 to U+DFFF are now rejected - if they appear, or are escaped, in patterns. - -10. Get rid of a number of -Wunused-but-set-variable warnings. - -11. The pattern /(?=(*:x))(q|)/ matches an empty string, and returns the mark - "x". The similar pattern /(?=(*:x))((*:y)q|)/ did not return a mark at all. - Oddly, Perl behaves the same way. PCRE has been fixed so that this pattern - also returns the mark "x". This bug applied to capturing parentheses, - non-capturing parentheses, and atomic parentheses. It also applied to some - assertions. - -12. Stephen Kelly's patch to CMakeLists.txt allows it to parse the version - information out of configure.ac instead of relying on pcre.h.generic, which - is not stored in the repository. - -13. Applied Dmitry V. Levin's patch for a more portable method for linking with - -lreadline. - -14. ZH added PCRE_CONFIG_JITTARGET; added its output to pcretest -C. - -15. Applied Graycode's patch to put the top-level frame on the stack rather - than the heap when not using the stack for recursion. This gives a - performance improvement in many cases when recursion is not deep. - -16. Experimental code added to "pcretest -C" to output the stack frame size. - - -Version 8.21 12-Dec-2011 ------------------------- - -1. Updating the JIT compiler. - -2. JIT compiler now supports OP_NCREF, OP_RREF and OP_NRREF. New test cases - are added as well. - -3. Fix cache-flush issue on PowerPC (It is still an experimental JIT port). - PCRE_EXTRA_TABLES is not suported by JIT, and should be checked before - calling _pcre_jit_exec. Some extra comments are added. - -4. (*MARK) settings inside atomic groups that do not contain any capturing - parentheses, for example, (?>a(*:m)), were not being passed out. This bug - was introduced by change 18 for 8.20. - -5. Supporting of \x, \U and \u in JavaScript compatibility mode based on the - ECMA-262 standard. - -6. Lookbehinds such as (?<=a{2}b) that contained a fixed repetition were - erroneously being rejected as "not fixed length" if PCRE_CASELESS was set. - This bug was probably introduced by change 9 of 8.13. - -7. While fixing 6 above, I noticed that a number of other items were being - incorrectly rejected as "not fixed length". This arose partly because newer - opcodes had not been added to the fixed-length checking code. I have (a) - corrected the bug and added tests for these items, and (b) arranged for an - error to occur if an unknown opcode is encountered while checking for fixed - length instead of just assuming "not fixed length". The items that were - rejected were: (*ACCEPT), (*COMMIT), (*FAIL), (*MARK), (*PRUNE), (*SKIP), - (*THEN), \h, \H, \v, \V, and single character negative classes with fixed - repetitions, e.g. [^a]{3}, with and without PCRE_CASELESS. - -8. A possessively repeated conditional subpattern such as (?(?=c)c|d)++ was - being incorrectly compiled and would have given unpredicatble results. - -9. A possessively repeated subpattern with minimum repeat count greater than - one behaved incorrectly. For example, (A){2,}+ behaved as if it was - (A)(A)++ which meant that, after a subsequent mismatch, backtracking into - the first (A) could occur when it should not. - -10. Add a cast and remove a redundant test from the code. - -11. JIT should use pcre_malloc/pcre_free for allocation. - -12. Updated pcre-config so that it no longer shows -L/usr/lib, which seems - best practice nowadays, and helps with cross-compiling. (If the exec_prefix - is anything other than /usr, -L is still shown). - -13. In non-UTF-8 mode, \C is now supported in lookbehinds and DFA matching. - -14. Perl does not support \N without a following name in a [] class; PCRE now - also gives an error. - -15. If a forward reference was repeated with an upper limit of around 2000, - it caused the error "internal error: overran compiling workspace". The - maximum number of forward references (including repeats) was limited by the - internal workspace, and dependent on the LINK_SIZE. The code has been - rewritten so that the workspace expands (via pcre_malloc) if necessary, and - the default depends on LINK_SIZE. There is a new upper limit (for safety) - of around 200,000 forward references. While doing this, I also speeded up - the filling in of repeated forward references. - -16. A repeated forward reference in a pattern such as (a)(?2){2}(.) was - incorrectly expecting the subject to contain another "a" after the start. - -17. When (*SKIP:name) is activated without a corresponding (*MARK:name) earlier - in the match, the SKIP should be ignored. This was not happening; instead - the SKIP was being treated as NOMATCH. For patterns such as - /A(*MARK:A)A+(*SKIP:B)Z|AAC/ this meant that the AAC branch was never - tested. - -18. The behaviour of (*MARK), (*PRUNE), and (*THEN) has been reworked and is - now much more compatible with Perl, in particular in cases where the result - is a non-match for a non-anchored pattern. For example, if - /b(*:m)f|a(*:n)w/ is matched against "abc", the non-match returns the name - "m", where previously it did not return a name. A side effect of this - change is that for partial matches, the last encountered mark name is - returned, as for non matches. A number of tests that were previously not - Perl-compatible have been moved into the Perl-compatible test files. The - refactoring has had the pleasing side effect of removing one argument from - the match() function, thus reducing its stack requirements. - -19. If the /S+ option was used in pcretest to study a pattern using JIT, - subsequent uses of /S (without +) incorrectly behaved like /S+. - -21. Retrieve executable code size support for the JIT compiler and fixing - some warnings. - -22. A caseless match of a UTF-8 character whose other case uses fewer bytes did - not work when the shorter character appeared right at the end of the - subject string. - -23. Added some (int) casts to non-JIT modules to reduce warnings on 64-bit - systems. - -24. Added PCRE_INFO_JITSIZE to pass on the value from (21) above, and also - output it when the /M option is used in pcretest. - -25. The CheckMan script was not being included in the distribution. Also, added - an explicit "perl" to run Perl scripts from the PrepareRelease script - because this is reportedly needed in Windows. - -26. If study data was being save in a file and studying had not found a set of - "starts with" bytes for the pattern, the data written to the file (though - never used) was taken from uninitialized memory and so caused valgrind to - complain. - -27. Updated RunTest.bat as provided by Sheri Pierce. - -28. Fixed a possible uninitialized memory bug in pcre_jit_compile.c. - -29. Computation of memory usage for the table of capturing group names was - giving an unnecessarily large value. - - -Version 8.20 21-Oct-2011 ------------------------- - -1. Change 37 of 8.13 broke patterns like [:a]...[b:] because it thought it had - a POSIX class. After further experiments with Perl, which convinced me that - Perl has bugs and confusions, a closing square bracket is no longer allowed - in a POSIX name. This bug also affected patterns with classes that started - with full stops. - -2. If a pattern such as /(a)b|ac/ is matched against "ac", there is no - captured substring, but while checking the failing first alternative, - substring 1 is temporarily captured. If the output vector supplied to - pcre_exec() was not big enough for this capture, the yield of the function - was still zero ("insufficient space for captured substrings"). This cannot - be totally fixed without adding another stack variable, which seems a lot - of expense for a edge case. However, I have improved the situation in cases - such as /(a)(b)x|abc/ matched against "abc", where the return code - indicates that fewer than the maximum number of slots in the ovector have - been set. - -3. Related to (2) above: when there are more back references in a pattern than - slots in the output vector, pcre_exec() uses temporary memory during - matching, and copies in the captures as far as possible afterwards. It was - using the entire output vector, but this conflicts with the specification - that only 2/3 is used for passing back captured substrings. Now it uses - only the first 2/3, for compatibility. This is, of course, another edge - case. - -4. Zoltan Herczeg's just-in-time compiler support has been integrated into the - main code base, and can be used by building with --enable-jit. When this is - done, pcregrep automatically uses it unless --disable-pcregrep-jit or the - runtime --no-jit option is given. - -5. When the number of matches in a pcre_dfa_exec() run exactly filled the - ovector, the return from the function was zero, implying that there were - other matches that did not fit. The correct "exactly full" value is now - returned. - -6. If a subpattern that was called recursively or as a subroutine contained - (*PRUNE) or any other control that caused it to give a non-standard return, - invalid errors such as "Error -26 (nested recursion at the same subject - position)" or even infinite loops could occur. - -7. If a pattern such as /a(*SKIP)c|b(*ACCEPT)|/ was studied, it stopped - computing the minimum length on reaching *ACCEPT, and so ended up with the - wrong value of 1 rather than 0. Further investigation indicates that - computing a minimum subject length in the presence of *ACCEPT is difficult - (think back references, subroutine calls), and so I have changed the code - so that no minimum is registered for a pattern that contains *ACCEPT. - -8. If (*THEN) was present in the first (true) branch of a conditional group, - it was not handled as intended. [But see 16 below.] - -9. Replaced RunTest.bat and CMakeLists.txt with improved versions provided by - Sheri Pierce. - -10. A pathological pattern such as /(*ACCEPT)a/ was miscompiled, thinking that - the first byte in a match must be "a". - -11. Change 17 for 8.13 increased the recursion depth for patterns like - /a(?:.)*?a/ drastically. I've improved things by remembering whether a - pattern contains any instances of (*THEN). If it does not, the old - optimizations are restored. It would be nice to do this on a per-group - basis, but at the moment that is not feasible. - -12. In some environments, the output of pcretest -C is CRLF terminated. This - broke RunTest's code that checks for the link size. A single white space - character after the value is now allowed for. - -13. RunTest now checks for the "fr" locale as well as for "fr_FR" and "french". - For "fr", it uses the Windows-specific input and output files. - -14. If (*THEN) appeared in a group that was called recursively or as a - subroutine, it did not work as intended. [But see next item.] - -15. Consider the pattern /A (B(*THEN)C) | D/ where A, B, C, and D are complex - pattern fragments (but not containing any | characters). If A and B are - matched, but there is a failure in C so that it backtracks to (*THEN), PCRE - was behaving differently to Perl. PCRE backtracked into A, but Perl goes to - D. In other words, Perl considers parentheses that do not contain any | - characters to be part of a surrounding alternative, whereas PCRE was - treading (B(*THEN)C) the same as (B(*THEN)C|(*FAIL)) -- which Perl handles - differently. PCRE now behaves in the same way as Perl, except in the case - of subroutine/recursion calls such as (?1) which have in any case always - been different (but PCRE had them first :-). - -16. Related to 15 above: Perl does not treat the | in a conditional group as - creating alternatives. Such a group is treated in the same way as an - ordinary group without any | characters when processing (*THEN). PCRE has - been changed to match Perl's behaviour. - -17. If a user had set PCREGREP_COLO(U)R to something other than 1:31, the - RunGrepTest script failed. - -18. Change 22 for version 13 caused atomic groups to use more stack. This is - inevitable for groups that contain captures, but it can lead to a lot of - stack use in large patterns. The old behaviour has been restored for atomic - groups that do not contain any capturing parentheses. - -19. If the PCRE_NO_START_OPTIMIZE option was set for pcre_compile(), it did not - suppress the check for a minimum subject length at run time. (If it was - given to pcre_exec() or pcre_dfa_exec() it did work.) - -20. Fixed an ASCII-dependent infelicity in pcretest that would have made it - fail to work when decoding hex characters in data strings in EBCDIC - environments. - -21. It appears that in at least one Mac OS environment, the isxdigit() function - is implemented as a macro that evaluates to its argument more than once, - contravening the C 90 Standard (I haven't checked a later standard). There - was an instance in pcretest which caused it to go wrong when processing - \x{...} escapes in subject strings. The has been rewritten to avoid using - things like p++ in the argument of isxdigit(). - - -Version 8.13 16-Aug-2011 ------------------------- - -1. The Unicode data tables have been updated to Unicode 6.0.0. - -2. Two minor typos in pcre_internal.h have been fixed. - -3. Added #include to pcre_scanner_unittest.cc, pcrecpp.cc, and - pcrecpp_unittest.cc. They are needed for strcmp(), memset(), and strchr() - in some environments (e.g. Solaris 10/SPARC using Sun Studio 12U2). - -4. There were a number of related bugs in the code for matching backrefences - caselessly in UTF-8 mode when codes for the characters concerned were - different numbers of bytes. For example, U+023A and U+2C65 are an upper - and lower case pair, using 2 and 3 bytes, respectively. The main bugs were: - (a) A reference to 3 copies of a 2-byte code matched only 2 of a 3-byte - code. (b) A reference to 2 copies of a 3-byte code would not match 2 of a - 2-byte code at the end of the subject (it thought there wasn't enough data - left). - -5. Comprehensive information about what went wrong is now returned by - pcre_exec() and pcre_dfa_exec() when the UTF-8 string check fails, as long - as the output vector has at least 2 elements. The offset of the start of - the failing character and a reason code are placed in the vector. - -6. When the UTF-8 string check fails for pcre_compile(), the offset that is - now returned is for the first byte of the failing character, instead of the - last byte inspected. This is an incompatible change, but I hope it is small - enough not to be a problem. It makes the returned offset consistent with - pcre_exec() and pcre_dfa_exec(). - -7. pcretest now gives a text phrase as well as the error number when - pcre_exec() or pcre_dfa_exec() fails; if the error is a UTF-8 check - failure, the offset and reason code are output. - -8. When \R was used with a maximizing quantifier it failed to skip backwards - over a \r\n pair if the subsequent match failed. Instead, it just skipped - back over a single character (\n). This seems wrong (because it treated the - two characters as a single entity when going forwards), conflicts with the - documentation that \R is equivalent to (?>\r\n|\n|...etc), and makes the - behaviour of \R* different to (\R)*, which also seems wrong. The behaviour - has been changed. - -9. Some internal refactoring has changed the processing so that the handling - of the PCRE_CASELESS and PCRE_MULTILINE options is done entirely at compile - time (the PCRE_DOTALL option was changed this way some time ago: version - 7.7 change 16). This has made it possible to abolish the OP_OPT op code, - which was always a bit of a fudge. It also means that there is one less - argument for the match() function, which reduces its stack requirements - slightly. This change also fixes an incompatibility with Perl: the pattern - (?i:([^b]))(?1) should not match "ab", but previously PCRE gave a match. - -10. More internal refactoring has drastically reduced the number of recursive - calls to match() for possessively repeated groups such as (abc)++ when - using pcre_exec(). - -11. While implementing 10, a number of bugs in the handling of groups were - discovered and fixed: - - (?<=(a)+) was not diagnosed as invalid (non-fixed-length lookbehind). - (a|)*(?1) gave a compile-time internal error. - ((a|)+)+ did not notice that the outer group could match an empty string. - (^a|^)+ was not marked as anchored. - (.*a|.*)+ was not marked as matching at start or after a newline. - -12. Yet more internal refactoring has removed another argument from the match() - function. Special calls to this function are now indicated by setting a - value in a variable in the "match data" data block. - -13. Be more explicit in pcre_study() instead of relying on "default" for - opcodes that mean there is no starting character; this means that when new - ones are added and accidentally left out of pcre_study(), testing should - pick them up. - -14. The -s option of pcretest has been documented for ages as being an old - synonym of -m (show memory usage). I have changed it to mean "force study - for every regex", that is, assume /S for every regex. This is similar to -i - and -d etc. It's slightly incompatible, but I'm hoping nobody is still - using it. It makes it easier to run collections of tests with and without - study enabled, and thereby test pcre_study() more easily. All the standard - tests are now run with and without -s (but some patterns can be marked as - "never study" - see 20 below). - -15. When (*ACCEPT) was used in a subpattern that was called recursively, the - restoration of the capturing data to the outer values was not happening - correctly. - -16. If a recursively called subpattern ended with (*ACCEPT) and matched an - empty string, and PCRE_NOTEMPTY was set, pcre_exec() thought the whole - pattern had matched an empty string, and so incorrectly returned a no - match. - -17. There was optimizing code for the last branch of non-capturing parentheses, - and also for the obeyed branch of a conditional subexpression, which used - tail recursion to cut down on stack usage. Unfortunately, now that there is - the possibility of (*THEN) occurring in these branches, tail recursion is - no longer possible because the return has to be checked for (*THEN). These - two optimizations have therefore been removed. [But see 8.20/11 above.] - -18. If a pattern containing \R was studied, it was assumed that \R always - matched two bytes, thus causing the minimum subject length to be - incorrectly computed because \R can also match just one byte. - -19. If a pattern containing (*ACCEPT) was studied, the minimum subject length - was incorrectly computed. - -20. If /S is present twice on a test pattern in pcretest input, it now - *disables* studying, thereby overriding the use of -s on the command line - (see 14 above). This is necessary for one or two tests to keep the output - identical in both cases. - -21. When (*ACCEPT) was used in an assertion that matched an empty string and - PCRE_NOTEMPTY was set, PCRE applied the non-empty test to the assertion. - -22. When an atomic group that contained a capturing parenthesis was - successfully matched, but the branch in which it appeared failed, the - capturing was not being forgotten if a higher numbered group was later - captured. For example, /(?>(a))b|(a)c/ when matching "ac" set capturing - group 1 to "a", when in fact it should be unset. This applied to multi- - branched capturing and non-capturing groups, repeated or not, and also to - positive assertions (capturing in negative assertions does not happen - in PCRE) and also to nested atomic groups. - -23. Add the ++ qualifier feature to pcretest, to show the remainder of the - subject after a captured substring, to make it easier to tell which of a - number of identical substrings has been captured. - -24. The way atomic groups are processed by pcre_exec() has been changed so that - if they are repeated, backtracking one repetition now resets captured - values correctly. For example, if ((?>(a+)b)+aabab) is matched against - "aaaabaaabaabab" the value of captured group 2 is now correctly recorded as - "aaa". Previously, it would have been "a". As part of this code - refactoring, the way recursive calls are handled has also been changed. - -25. If an assertion condition captured any substrings, they were not passed - back unless some other capturing happened later. For example, if - (?(?=(a))a) was matched against "a", no capturing was returned. - -26. When studying a pattern that contained subroutine calls or assertions, - the code for finding the minimum length of a possible match was handling - direct recursions such as (xxx(?1)|yyy) but not mutual recursions (where - group 1 called group 2 while simultaneously a separate group 2 called group - 1). A stack overflow occurred in this case. I have fixed this by limiting - the recursion depth to 10. - -27. Updated RunTest.bat in the distribution to the version supplied by Tom - Fortmann. This supports explicit test numbers on the command line, and has - argument validation and error reporting. - -28. An instance of \X with an unlimited repeat could fail if at any point the - first character it looked at was a mark character. - -29. Some minor code refactoring concerning Unicode properties and scripts - should reduce the stack requirement of match() slightly. - -30. Added the '=' option to pcretest to check the setting of unused capturing - slots at the end of the pattern, which are documented as being -1, but are - not included in the return count. - -31. If \k was not followed by a braced, angle-bracketed, or quoted name, PCRE - compiled something random. Now it gives a compile-time error (as does - Perl). - -32. A *MARK encountered during the processing of a positive assertion is now - recorded and passed back (compatible with Perl). - -33. If --only-matching or --colour was set on a pcregrep call whose pattern - had alternative anchored branches, the search for a second match in a line - was done as if at the line start. Thus, for example, /^01|^02/ incorrectly - matched the line "0102" twice. The same bug affected patterns that started - with a backwards assertion. For example /\b01|\b02/ also matched "0102" - twice. - -34. Previously, PCRE did not allow quantification of assertions. However, Perl - does, and because of capturing effects, quantifying parenthesized - assertions may at times be useful. Quantifiers are now allowed for - parenthesized assertions. - -35. A minor code tidy in pcre_compile() when checking options for \R usage. - -36. \g was being checked for fancy things in a character class, when it should - just be a literal "g". - -37. PCRE was rejecting [:a[:digit:]] whereas Perl was not. It seems that the - appearance of a nested POSIX class supersedes an apparent external class. - For example, [:a[:digit:]b:] matches "a", "b", ":", or a digit. Also, - unescaped square brackets may also appear as part of class names. For - example, [:a[:abc]b:] gives unknown class "[:abc]b:]". PCRE now behaves - more like Perl. (But see 8.20/1 above.) - -38. PCRE was giving an error for \N with a braced quantifier such as {1,} (this - was because it thought it was \N{name}, which is not supported). - -39. Add minix to OS list not supporting the -S option in pcretest. - -40. PCRE tries to detect cases of infinite recursion at compile time, but it - cannot analyze patterns in sufficient detail to catch mutual recursions - such as ((?1))((?2)). There is now a runtime test that gives an error if a - subgroup is called recursively as a subpattern for a second time at the - same position in the subject string. In previous releases this might have - been caught by the recursion limit, or it might have run out of stack. - -41. A pattern such as /(?(R)a+|(?R)b)/ is quite safe, as the recursion can - happen only once. PCRE was, however incorrectly giving a compile time error - "recursive call could loop indefinitely" because it cannot analyze the - pattern in sufficient detail. The compile time test no longer happens when - PCRE is compiling a conditional subpattern, but actual runaway loops are - now caught at runtime (see 40 above). - -42. It seems that Perl allows any characters other than a closing parenthesis - to be part of the NAME in (*MARK:NAME) and other backtracking verbs. PCRE - has been changed to be the same. - -43. Updated configure.ac to put in more quoting round AC_LANG_PROGRAM etc. so - as not to get warnings when autogen.sh is called. Also changed - AC_PROG_LIBTOOL (deprecated) to LT_INIT (the current macro). - -44. To help people who use pcregrep to scan files containing exceedingly long - lines, the following changes have been made: - - (a) The default value of the buffer size parameter has been increased from - 8K to 20K. (The actual buffer used is three times this size.) - - (b) The default can be changed by ./configure --with-pcregrep-bufsize when - PCRE is built. - - (c) A --buffer-size=n option has been added to pcregrep, to allow the size - to be set at run time. - - (d) Numerical values in pcregrep options can be followed by K or M, for - example --buffer-size=50K. - - (e) If a line being scanned overflows pcregrep's buffer, an error is now - given and the return code is set to 2. - -45. Add a pointer to the latest mark to the callout data block. - -46. The pattern /.(*F)/, when applied to "abc" with PCRE_PARTIAL_HARD, gave a - partial match of an empty string instead of no match. This was specific to - the use of ".". - -47. The pattern /f.*/8s, when applied to "for" with PCRE_PARTIAL_HARD, gave a - complete match instead of a partial match. This bug was dependent on both - the PCRE_UTF8 and PCRE_DOTALL options being set. - -48. For a pattern such as /\babc|\bdef/ pcre_study() was failing to set up the - starting byte set, because \b was not being ignored. - - -Version 8.12 15-Jan-2011 ------------------------- - -1. Fixed some typos in the markup of the man pages, and wrote a script that - checks for such things as part of the documentation building process. - -2. On a big-endian 64-bit system, pcregrep did not correctly process the - --match-limit and --recursion-limit options (added for 8.11). In - particular, this made one of the standard tests fail. (The integer value - went into the wrong half of a long int.) - -3. If the --colour option was given to pcregrep with -v (invert match), it - did strange things, either producing crazy output, or crashing. It should, - of course, ignore a request for colour when reporting lines that do not - match. - -4. Another pcregrep bug caused similar problems if --colour was specified with - -M (multiline) and the pattern match finished with a line ending. - -5. In pcregrep, when a pattern that ended with a literal newline sequence was - matched in multiline mode, the following line was shown as part of the - match. This seems wrong, so I have changed it. - -6. Another pcregrep bug in multiline mode, when --colour was specified, caused - the check for further matches in the same line (so they could be coloured) - to overrun the end of the current line. If another match was found, it was - incorrectly shown (and then shown again when found in the next line). - -7. If pcregrep was compiled under Windows, there was a reference to the - function pcregrep_exit() before it was defined. I am assuming this was - the cause of the "error C2371: 'pcregrep_exit' : redefinition;" that was - reported by a user. I've moved the definition above the reference. - - -Version 8.11 10-Dec-2010 ------------------------- - -1. (*THEN) was not working properly if there were untried alternatives prior - to it in the current branch. For example, in ((a|b)(*THEN)(*F)|c..) it - backtracked to try for "b" instead of moving to the next alternative branch - at the same level (in this case, to look for "c"). The Perl documentation - is clear that when (*THEN) is backtracked onto, it goes to the "next - alternative in the innermost enclosing group". - -2. (*COMMIT) was not overriding (*THEN), as it does in Perl. In a pattern - such as (A(*COMMIT)B(*THEN)C|D) any failure after matching A should - result in overall failure. Similarly, (*COMMIT) now overrides (*PRUNE) and - (*SKIP), (*SKIP) overrides (*PRUNE) and (*THEN), and (*PRUNE) overrides - (*THEN). - -3. If \s appeared in a character class, it removed the VT character from - the class, even if it had been included by some previous item, for example - in [\x00-\xff\s]. (This was a bug related to the fact that VT is not part - of \s, but is part of the POSIX "space" class.) - -4. A partial match never returns an empty string (because you can always - match an empty string at the end of the subject); however the checking for - an empty string was starting at the "start of match" point. This has been - changed to the "earliest inspected character" point, because the returned - data for a partial match starts at this character. This means that, for - example, /(?<=abc)def/ gives a partial match for the subject "abc" - (previously it gave "no match"). - -5. Changes have been made to the way PCRE_PARTIAL_HARD affects the matching - of $, \z, \Z, \b, and \B. If the match point is at the end of the string, - previously a full match would be given. However, setting PCRE_PARTIAL_HARD - has an implication that the given string is incomplete (because a partial - match is preferred over a full match). For this reason, these items now - give a partial match in this situation. [Aside: previously, the one case - /t\b/ matched against "cat" with PCRE_PARTIAL_HARD set did return a partial - match rather than a full match, which was wrong by the old rules, but is - now correct.] - -6. There was a bug in the handling of #-introduced comments, recognized when - PCRE_EXTENDED is set, when PCRE_NEWLINE_ANY and PCRE_UTF8 were also set. - If a UTF-8 multi-byte character included the byte 0x85 (e.g. +U0445, whose - UTF-8 encoding is 0xd1,0x85), this was misinterpreted as a newline when - scanning for the end of the comment. (*Character* 0x85 is an "any" newline, - but *byte* 0x85 is not, in UTF-8 mode). This bug was present in several - places in pcre_compile(). - -7. Related to (6) above, when pcre_compile() was skipping #-introduced - comments when looking ahead for named forward references to subpatterns, - the only newline sequence it recognized was NL. It now handles newlines - according to the set newline convention. - -8. SunOS4 doesn't have strerror() or strtoul(); pcregrep dealt with the - former, but used strtoul(), whereas pcretest avoided strtoul() but did not - cater for a lack of strerror(). These oversights have been fixed. - -9. Added --match-limit and --recursion-limit to pcregrep. - -10. Added two casts needed to build with Visual Studio when NO_RECURSE is set. - -11. When the -o option was used, pcregrep was setting a return code of 1, even - when matches were found, and --line-buffered was not being honoured. - -12. Added an optional parentheses number to the -o and --only-matching options - of pcregrep. - -13. Imitating Perl's /g action for multiple matches is tricky when the pattern - can match an empty string. The code to do it in pcretest and pcredemo - needed fixing: - - (a) When the newline convention was "crlf", pcretest got it wrong, skipping - only one byte after an empty string match just before CRLF (this case - just got forgotten; "any" and "anycrlf" were OK). - - (b) The pcretest code also had a bug, causing it to loop forever in UTF-8 - mode when an empty string match preceded an ASCII character followed by - a non-ASCII character. (The code for advancing by one character rather - than one byte was nonsense.) - - (c) The pcredemo.c sample program did not have any code at all to handle - the cases when CRLF is a valid newline sequence. - -14. Neither pcre_exec() nor pcre_dfa_exec() was checking that the value given - as a starting offset was within the subject string. There is now a new - error, PCRE_ERROR_BADOFFSET, which is returned if the starting offset is - negative or greater than the length of the string. In order to test this, - pcretest is extended to allow the setting of negative starting offsets. - -15. In both pcre_exec() and pcre_dfa_exec() the code for checking that the - starting offset points to the beginning of a UTF-8 character was - unnecessarily clumsy. I tidied it up. - -16. Added PCRE_ERROR_SHORTUTF8 to make it possible to distinguish between a - bad UTF-8 sequence and one that is incomplete when using PCRE_PARTIAL_HARD. - -17. Nobody had reported that the --include_dir option, which was added in - release 7.7 should have been called --include-dir (hyphen, not underscore) - for compatibility with GNU grep. I have changed it to --include-dir, but - left --include_dir as an undocumented synonym, and the same for - --exclude-dir, though that is not available in GNU grep, at least as of - release 2.5.4. - -18. At a user's suggestion, the macros GETCHAR and friends (which pick up UTF-8 - characters from a string of bytes) have been redefined so as not to use - loops, in order to improve performance in some environments. At the same - time, I abstracted some of the common code into auxiliary macros to save - repetition (this should not affect the compiled code). - -19. If \c was followed by a multibyte UTF-8 character, bad things happened. A - compile-time error is now given if \c is not followed by an ASCII - character, that is, a byte less than 128. (In EBCDIC mode, the code is - different, and any byte value is allowed.) - -20. Recognize (*NO_START_OPT) at the start of a pattern to set the PCRE_NO_ - START_OPTIMIZE option, which is now allowed at compile time - but just - passed through to pcre_exec() or pcre_dfa_exec(). This makes it available - to pcregrep and other applications that have no direct access to PCRE - options. The new /Y option in pcretest sets this option when calling - pcre_compile(). - -21. Change 18 of release 8.01 broke the use of named subpatterns for recursive - back references. Groups containing recursive back references were forced to - be atomic by that change, but in the case of named groups, the amount of - memory required was incorrectly computed, leading to "Failed: internal - error: code overflow". This has been fixed. - -22. Some patches to pcre_stringpiece.h, pcre_stringpiece_unittest.cc, and - pcretest.c, to avoid build problems in some Borland environments. - - -Version 8.10 25-Jun-2010 ------------------------- - -1. Added support for (*MARK:ARG) and for ARG additions to PRUNE, SKIP, and - THEN. - -2. (*ACCEPT) was not working when inside an atomic group. - -3. Inside a character class, \B is treated as a literal by default, but - faulted if PCRE_EXTRA is set. This mimics Perl's behaviour (the -w option - causes the error). The code is unchanged, but I tidied the documentation. - -4. Inside a character class, PCRE always treated \R and \X as literals, - whereas Perl faults them if its -w option is set. I have changed PCRE so - that it faults them when PCRE_EXTRA is set. - -5. Added support for \N, which always matches any character other than - newline. (It is the same as "." when PCRE_DOTALL is not set.) - -6. When compiling pcregrep with newer versions of gcc which may have - FORTIFY_SOURCE set, several warnings "ignoring return value of 'fwrite', - declared with attribute warn_unused_result" were given. Just casting the - result to (void) does not stop the warnings; a more elaborate fudge is - needed. I've used a macro to implement this. - -7. Minor change to pcretest.c to avoid a compiler warning. - -8. Added four artifical Unicode properties to help with an option to make - \s etc use properties (see next item). The new properties are: Xan - (alphanumeric), Xsp (Perl space), Xps (POSIX space), and Xwd (word). - -9. Added PCRE_UCP to make \b, \d, \s, \w, and certain POSIX character classes - use Unicode properties. (*UCP) at the start of a pattern can be used to set - this option. Modified pcretest to add /W to test this facility. Added - REG_UCP to make it available via the POSIX interface. - -10. Added --line-buffered to pcregrep. - -11. In UTF-8 mode, if a pattern that was compiled with PCRE_CASELESS was - studied, and the match started with a letter with a code point greater than - 127 whose first byte was different to the first byte of the other case of - the letter, the other case of this starting letter was not recognized - (#976). - -12. If a pattern that was studied started with a repeated Unicode property - test, for example, \p{Nd}+, there was the theoretical possibility of - setting up an incorrect bitmap of starting bytes, but fortunately it could - not have actually happened in practice until change 8 above was made (it - added property types that matched character-matching opcodes). - -13. pcre_study() now recognizes \h, \v, and \R when constructing a bit map of - possible starting bytes for non-anchored patterns. - -14. Extended the "auto-possessify" feature of pcre_compile(). It now recognizes - \R, and also a number of cases that involve Unicode properties, both - explicit and implicit when PCRE_UCP is set. - -15. If a repeated Unicode property match (e.g. \p{Lu}*) was used with non-UTF-8 - input, it could crash or give wrong results if characters with values - greater than 0xc0 were present in the subject string. (Detail: it assumed - UTF-8 input when processing these items.) - -16. Added a lot of (int) casts to avoid compiler warnings in systems where - size_t is 64-bit (#991). - -17. Added a check for running out of memory when PCRE is compiled with - --disable-stack-for-recursion (#990). - -18. If the last data line in a file for pcretest does not have a newline on - the end, a newline was missing in the output. - -19. The default pcre_chartables.c file recognizes only ASCII characters (values - less than 128) in its various bitmaps. However, there is a facility for - generating tables according to the current locale when PCRE is compiled. It - turns out that in some environments, 0x85 and 0xa0, which are Unicode space - characters, are recognized by isspace() and therefore were getting set in - these tables, and indeed these tables seem to approximate to ISO 8859. This - caused a problem in UTF-8 mode when pcre_study() was used to create a list - of bytes that can start a match. For \s, it was including 0x85 and 0xa0, - which of course cannot start UTF-8 characters. I have changed the code so - that only real ASCII characters (less than 128) and the correct starting - bytes for UTF-8 encodings are set for characters greater than 127 when in - UTF-8 mode. (When PCRE_UCP is set - see 9 above - the code is different - altogether.) - -20. Added the /T option to pcretest so as to be able to run tests with non- - standard character tables, thus making it possible to include the tests - used for 19 above in the standard set of tests. - -21. A pattern such as (?&t)(?#()(?(DEFINE)(?a)) which has a forward - reference to a subpattern the other side of a comment that contains an - opening parenthesis caused either an internal compiling error, or a - reference to the wrong subpattern. - - -Version 8.02 19-Mar-2010 ------------------------- - -1. The Unicode data tables have been updated to Unicode 5.2.0. - -2. Added the option --libs-cpp to pcre-config, but only when C++ support is - configured. - -3. Updated the licensing terms in the pcregexp.pas file, as agreed with the - original author of that file, following a query about its status. - -4. On systems that do not have stdint.h (e.g. Solaris), check for and include - inttypes.h instead. This fixes a bug that was introduced by change 8.01/8. - -5. A pattern such as (?&t)*+(?(DEFINE)(?.)) which has a possessive - quantifier applied to a forward-referencing subroutine call, could compile - incorrect code or give the error "internal error: previously-checked - referenced subpattern not found". - -6. Both MS Visual Studio and Symbian OS have problems with initializing - variables to point to external functions. For these systems, therefore, - pcre_malloc etc. are now initialized to local functions that call the - relevant global functions. - -7. There were two entries missing in the vectors called coptable and poptable - in pcre_dfa_exec.c. This could lead to memory accesses outsize the vectors. - I've fixed the data, and added a kludgy way of testing at compile time that - the lengths are correct (equal to the number of opcodes). - -8. Following on from 7, I added a similar kludge to check the length of the - eint vector in pcreposix.c. - -9. Error texts for pcre_compile() are held as one long string to avoid too - much relocation at load time. To find a text, the string is searched, - counting zeros. There was no check for running off the end of the string, - which could happen if a new error number was added without updating the - string. - -10. \K gave a compile-time error if it appeared in a lookbehind assersion. - -11. \K was not working if it appeared in an atomic group or in a group that - was called as a "subroutine", or in an assertion. Perl 5.11 documents that - \K is "not well defined" if used in an assertion. PCRE now accepts it if - the assertion is positive, but not if it is negative. - -12. Change 11 fortuitously reduced the size of the stack frame used in the - "match()" function of pcre_exec.c by one pointer. Forthcoming - implementation of support for (*MARK) will need an extra pointer on the - stack; I have reserved it now, so that the stack frame size does not - decrease. - -13. A pattern such as (?P(?P0)|(?P>L2)(?P>L1)) in which the only other - item in branch that calls a recursion is a subroutine call - as in the - second branch in the above example - was incorrectly given the compile- - time error "recursive call could loop indefinitely" because pcre_compile() - was not correctly checking the subroutine for matching a non-empty string. - -14. The checks for overrunning compiling workspace could trigger after an - overrun had occurred. This is a "should never occur" error, but it can be - triggered by pathological patterns such as hundreds of nested parentheses. - The checks now trigger 100 bytes before the end of the workspace. - -15. Fix typo in configure.ac: "srtoq" should be "strtoq". - - -Version 8.01 19-Jan-2010 ------------------------- - -1. If a pattern contained a conditional subpattern with only one branch (in - particular, this includes all (*DEFINE) patterns), a call to pcre_study() - computed the wrong minimum data length (which is of course zero for such - subpatterns). This could cause incorrect "no match" results. - -2. For patterns such as (?i)a(?-i)b|c where an option setting at the start of - the pattern is reset in the first branch, pcre_compile() failed with - "internal error: code overflow at offset...". This happened only when - the reset was to the original external option setting. (An optimization - abstracts leading options settings into an external setting, which was the - cause of this.) - -3. A pattern such as ^(?!a(*SKIP)b) where a negative assertion contained one - of the verbs SKIP, PRUNE, or COMMIT, did not work correctly. When the - assertion pattern did not match (meaning that the assertion was true), it - was incorrectly treated as false if the SKIP had been reached during the - matching. This also applied to assertions used as conditions. - -4. If an item that is not supported by pcre_dfa_exec() was encountered in an - assertion subpattern, including such a pattern used as a condition, - unpredictable results occurred, instead of the error return - PCRE_ERROR_DFA_UITEM. - -5. The C++ GlobalReplace function was not working like Perl for the special - situation when an empty string is matched. It now does the fancy magic - stuff that is necessary. - -6. In pcre_internal.h, obsolete includes to setjmp.h and stdarg.h have been - removed. (These were left over from very, very early versions of PCRE.) - -7. Some cosmetic changes to the code to make life easier when compiling it - as part of something else: - - (a) Change DEBUG to PCRE_DEBUG. - - (b) In pcre_compile(), rename the member of the "branch_chain" structure - called "current" as "current_branch", to prevent a collision with the - Linux macro when compiled as a kernel module. - - (c) In pcre_study(), rename the function set_bit() as set_table_bit(), to - prevent a collision with the Linux macro when compiled as a kernel - module. - -8. In pcre_compile() there are some checks for integer overflows that used to - cast potentially large values to (double). This has been changed to that - when building, a check for int64_t is made, and if it is found, it is used - instead, thus avoiding the use of floating point arithmetic. (There is no - other use of FP in PCRE.) If int64_t is not found, the fallback is to - double. - -9. Added two casts to avoid signed/unsigned warnings from VS Studio Express - 2005 (difference between two addresses compared to an unsigned value). - -10. Change the standard AC_CHECK_LIB test for libbz2 in configure.ac to a - custom one, because of the following reported problem in Windows: - - - libbz2 uses the Pascal calling convention (WINAPI) for the functions - under Win32. - - The standard autoconf AC_CHECK_LIB fails to include "bzlib.h", - therefore missing the function definition. - - The compiler thus generates a "C" signature for the test function. - - The linker fails to find the "C" function. - - PCRE fails to configure if asked to do so against libbz2. - -11. When running libtoolize from libtool-2.2.6b as part of autogen.sh, these - messages were output: - - Consider adding `AC_CONFIG_MACRO_DIR([m4])' to configure.ac and - rerunning libtoolize, to keep the correct libtool macros in-tree. - Consider adding `-I m4' to ACLOCAL_AMFLAGS in Makefile.am. - - I have done both of these things. - -12. Although pcre_dfa_exec() does not use nearly as much stack as pcre_exec() - most of the time, it *can* run out if it is given a pattern that contains a - runaway infinite recursion. I updated the discussion in the pcrestack man - page. - -13. Now that we have gone to the x.xx style of version numbers, the minor - version may start with zero. Using 08 or 09 is a bad idea because users - might check the value of PCRE_MINOR in their code, and 08 or 09 may be - interpreted as invalid octal numbers. I've updated the previous comment in - configure.ac, and also added a check that gives an error if 08 or 09 are - used. - -14. Change 8.00/11 was not quite complete: code had been accidentally omitted, - causing partial matching to fail when the end of the subject matched \W - in a UTF-8 pattern where \W was quantified with a minimum of 3. - -15. There were some discrepancies between the declarations in pcre_internal.h - of _pcre_is_newline(), _pcre_was_newline(), and _pcre_valid_utf8() and - their definitions. The declarations used "const uschar *" and the - definitions used USPTR. Even though USPTR is normally defined as "const - unsigned char *" (and uschar is typedeffed as "unsigned char"), it was - reported that: "This difference in casting confuses some C++ compilers, for - example, SunCC recognizes above declarations as different functions and - generates broken code for hbpcre." I have changed the declarations to use - USPTR. - -16. GNU libtool is named differently on some systems. The autogen.sh script now - tries several variants such as glibtoolize (MacOSX) and libtoolize1x - (FreeBSD). - -17. Applied Craig's patch that fixes an HP aCC compile error in pcre 8.00 - (strtoXX undefined when compiling pcrecpp.cc). The patch contains this - comment: "Figure out how to create a longlong from a string: strtoll and - equivalent. It's not enough to call AC_CHECK_FUNCS: hpux has a strtoll, for - instance, but it only takes 2 args instead of 3!" - -18. A subtle bug concerned with back references has been fixed by a change of - specification, with a corresponding code fix. A pattern such as - ^(xa|=?\1a)+$ which contains a back reference inside the group to which it - refers, was giving matches when it shouldn't. For example, xa=xaaa would - match that pattern. Interestingly, Perl (at least up to 5.11.3) has the - same bug. Such groups have to be quantified to be useful, or contained - inside another quantified group. (If there's no repetition, the reference - can never match.) The problem arises because, having left the group and - moved on to the rest of the pattern, a later failure that backtracks into - the group uses the captured value from the final iteration of the group - rather than the correct earlier one. I have fixed this in PCRE by forcing - any group that contains a reference to itself to be an atomic group; that - is, there cannot be any backtracking into it once it has completed. This is - similar to recursive and subroutine calls. - - -Version 8.00 19-Oct-09 ----------------------- - -1. The table for translating pcre_compile() error codes into POSIX error codes - was out-of-date, and there was no check on the pcre_compile() error code - being within the table. This could lead to an OK return being given in - error. - -2. Changed the call to open a subject file in pcregrep from fopen(pathname, - "r") to fopen(pathname, "rb"), which fixed a problem with some of the tests - in a Windows environment. - -3. The pcregrep --count option prints the count for each file even when it is - zero, as does GNU grep. However, pcregrep was also printing all files when - --files-with-matches was added. Now, when both options are given, it prints - counts only for those files that have at least one match. (GNU grep just - prints the file name in this circumstance, but including the count seems - more useful - otherwise, why use --count?) Also ensured that the - combination -clh just lists non-zero counts, with no names. - -4. The long form of the pcregrep -F option was incorrectly implemented as - --fixed_strings instead of --fixed-strings. This is an incompatible change, - but it seems right to fix it, and I didn't think it was worth preserving - the old behaviour. - -5. The command line items --regex=pattern and --regexp=pattern were not - recognized by pcregrep, which required --regex pattern or --regexp pattern - (with a space rather than an '='). The man page documented the '=' forms, - which are compatible with GNU grep; these now work. - -6. No libpcreposix.pc file was created for pkg-config; there was just - libpcre.pc and libpcrecpp.pc. The omission has been rectified. - -7. Added #ifndef SUPPORT_UCP into the pcre_ucd.c module, to reduce its size - when UCP support is not needed, by modifying the Python script that - generates it from Unicode data files. This should not matter if the module - is correctly used as a library, but I received one complaint about 50K of - unwanted data. My guess is that the person linked everything into his - program rather than using a library. Anyway, it does no harm. - -8. A pattern such as /\x{123}{2,2}+/8 was incorrectly compiled; the trigger - was a minimum greater than 1 for a wide character in a possessive - repetition. The same bug could also affect patterns like /(\x{ff}{0,2})*/8 - which had an unlimited repeat of a nested, fixed maximum repeat of a wide - character. Chaos in the form of incorrect output or a compiling loop could - result. - -9. The restrictions on what a pattern can contain when partial matching is - requested for pcre_exec() have been removed. All patterns can now be - partially matched by this function. In addition, if there are at least two - slots in the offset vector, the offset of the earliest inspected character - for the match and the offset of the end of the subject are set in them when - PCRE_ERROR_PARTIAL is returned. - -10. Partial matching has been split into two forms: PCRE_PARTIAL_SOFT, which is - synonymous with PCRE_PARTIAL, for backwards compatibility, and - PCRE_PARTIAL_HARD, which causes a partial match to supersede a full match, - and may be more useful for multi-segment matching. - -11. Partial matching with pcre_exec() is now more intuitive. A partial match - used to be given if ever the end of the subject was reached; now it is - given only if matching could not proceed because another character was - needed. This makes a difference in some odd cases such as Z(*FAIL) with the - string "Z", which now yields "no match" instead of "partial match". In the - case of pcre_dfa_exec(), "no match" is given if every matching path for the - final character ended with (*FAIL). - -12. Restarting a match using pcre_dfa_exec() after a partial match did not work - if the pattern had a "must contain" character that was already found in the - earlier partial match, unless partial matching was again requested. For - example, with the pattern /dog.(body)?/, the "must contain" character is - "g". If the first part-match was for the string "dog", restarting with - "sbody" failed. This bug has been fixed. - -13. The string returned by pcre_dfa_exec() after a partial match has been - changed so that it starts at the first inspected character rather than the - first character of the match. This makes a difference only if the pattern - starts with a lookbehind assertion or \b or \B (\K is not supported by - pcre_dfa_exec()). It's an incompatible change, but it makes the two - matching functions compatible, and I think it's the right thing to do. - -14. Added a pcredemo man page, created automatically from the pcredemo.c file, - so that the demonstration program is easily available in environments where - PCRE has not been installed from source. - -15. Arranged to add -DPCRE_STATIC to cflags in libpcre.pc, libpcreposix.cp, - libpcrecpp.pc and pcre-config when PCRE is not compiled as a shared - library. - -16. Added REG_UNGREEDY to the pcreposix interface, at the request of a user. - It maps to PCRE_UNGREEDY. It is not, of course, POSIX-compatible, but it - is not the first non-POSIX option to be added. Clearly some people find - these options useful. - -17. If a caller to the POSIX matching function regexec() passes a non-zero - value for nmatch with a NULL value for pmatch, the value of - nmatch is forced to zero. - -18. RunGrepTest did not have a test for the availability of the -u option of - the diff command, as RunTest does. It now checks in the same way as - RunTest, and also checks for the -b option. - -19. If an odd number of negated classes containing just a single character - interposed, within parentheses, between a forward reference to a named - subpattern and the definition of the subpattern, compilation crashed with - an internal error, complaining that it could not find the referenced - subpattern. An example of a crashing pattern is /(?&A)(([^m])(?))/. - [The bug was that it was starting one character too far in when skipping - over the character class, thus treating the ] as data rather than - terminating the class. This meant it could skip too much.] - -20. Added PCRE_NOTEMPTY_ATSTART in order to be able to correctly implement the - /g option in pcretest when the pattern contains \K, which makes it possible - to have an empty string match not at the start, even when the pattern is - anchored. Updated pcretest and pcredemo to use this option. - -21. If the maximum number of capturing subpatterns in a recursion was greater - than the maximum at the outer level, the higher number was returned, but - with unset values at the outer level. The correct (outer level) value is - now given. - -22. If (*ACCEPT) appeared inside capturing parentheses, previous releases of - PCRE did not set those parentheses (unlike Perl). I have now found a way to - make it do so. The string so far is captured, making this feature - compatible with Perl. - -23. The tests have been re-organized, adding tests 11 and 12, to make it - possible to check the Perl 5.10 features against Perl 5.10. - -24. Perl 5.10 allows subroutine calls in lookbehinds, as long as the subroutine - pattern matches a fixed length string. PCRE did not allow this; now it - does. Neither allows recursion. - -25. I finally figured out how to implement a request to provide the minimum - length of subject string that was needed in order to match a given pattern. - (It was back references and recursion that I had previously got hung up - on.) This code has now been added to pcre_study(); it finds a lower bound - to the length of subject needed. It is not necessarily the greatest lower - bound, but using it to avoid searching strings that are too short does give - some useful speed-ups. The value is available to calling programs via - pcre_fullinfo(). - -26. While implementing 25, I discovered to my embarrassment that pcretest had - not been passing the result of pcre_study() to pcre_dfa_exec(), so the - study optimizations had never been tested with that matching function. - Oops. What is worse, even when it was passed study data, there was a bug in - pcre_dfa_exec() that meant it never actually used it. Double oops. There - were also very few tests of studied patterns with pcre_dfa_exec(). - -27. If (?| is used to create subpatterns with duplicate numbers, they are now - allowed to have the same name, even if PCRE_DUPNAMES is not set. However, - on the other side of the coin, they are no longer allowed to have different - names, because these cannot be distinguished in PCRE, and this has caused - confusion. (This is a difference from Perl.) - -28. When duplicate subpattern names are present (necessarily with different - numbers, as required by 27 above), and a test is made by name in a - conditional pattern, either for a subpattern having been matched, or for - recursion in such a pattern, all the associated numbered subpatterns are - tested, and the overall condition is true if the condition is true for any - one of them. This is the way Perl works, and is also more like the way - testing by number works. - - -Version 7.9 11-Apr-09 ---------------------- - -1. When building with support for bzlib/zlib (pcregrep) and/or readline - (pcretest), all targets were linked against these libraries. This included - libpcre, libpcreposix, and libpcrecpp, even though they do not use these - libraries. This caused unwanted dependencies to be created. This problem - has been fixed, and now only pcregrep is linked with bzlib/zlib and only - pcretest is linked with readline. - -2. The "typedef int BOOL" in pcre_internal.h that was included inside the - "#ifndef FALSE" condition by an earlier change (probably 7.8/18) has been - moved outside it again, because FALSE and TRUE are already defined in AIX, - but BOOL is not. - -3. The pcre_config() function was treating the PCRE_MATCH_LIMIT and - PCRE_MATCH_LIMIT_RECURSION values as ints, when they should be long ints. - -4. The pcregrep documentation said spaces were inserted as well as colons (or - hyphens) following file names and line numbers when outputting matching - lines. This is not true; no spaces are inserted. I have also clarified the - wording for the --colour (or --color) option. - -5. In pcregrep, when --colour was used with -o, the list of matching strings - was not coloured; this is different to GNU grep, so I have changed it to be - the same. - -6. When --colo(u)r was used in pcregrep, only the first matching substring in - each matching line was coloured. Now it goes on to look for further matches - of any of the test patterns, which is the same behaviour as GNU grep. - -7. A pattern that could match an empty string could cause pcregrep to loop; it - doesn't make sense to accept an empty string match in pcregrep, so I have - locked it out (using PCRE's PCRE_NOTEMPTY option). By experiment, this - seems to be how GNU grep behaves. [But see later change 40 for release - 8.33.] - -8. The pattern (?(?=.*b)b|^) was incorrectly compiled as "match must be at - start or after a newline", because the conditional assertion was not being - correctly handled. The rule now is that both the assertion and what follows - in the first alternative must satisfy the test. - -9. If auto-callout was enabled in a pattern with a conditional group whose - condition was an assertion, PCRE could crash during matching, both with - pcre_exec() and pcre_dfa_exec(). - -10. The PCRE_DOLLAR_ENDONLY option was not working when pcre_dfa_exec() was - used for matching. - -11. Unicode property support in character classes was not working for - characters (bytes) greater than 127 when not in UTF-8 mode. - -12. Added the -M command line option to pcretest. - -14. Added the non-standard REG_NOTEMPTY option to the POSIX interface. - -15. Added the PCRE_NO_START_OPTIMIZE match-time option. - -16. Added comments and documentation about mis-use of no_arg in the C++ - wrapper. - -17. Implemented support for UTF-8 encoding in EBCDIC environments, a patch - from Martin Jerabek that uses macro names for all relevant character and - string constants. - -18. Added to pcre_internal.h two configuration checks: (a) If both EBCDIC and - SUPPORT_UTF8 are set, give an error; (b) If SUPPORT_UCP is set without - SUPPORT_UTF8, define SUPPORT_UTF8. The "configure" script handles both of - these, but not everybody uses configure. - -19. A conditional group that had only one branch was not being correctly - recognized as an item that could match an empty string. This meant that an - enclosing group might also not be so recognized, causing infinite looping - (and probably a segfault) for patterns such as ^"((?(?=[a])[^"])|b)*"$ - with the subject "ab", where knowledge that the repeated group can match - nothing is needed in order to break the loop. - -20. If a pattern that was compiled with callouts was matched using pcre_dfa_ - exec(), but without supplying a callout function, matching went wrong. - -21. If PCRE_ERROR_MATCHLIMIT occurred during a recursion, there was a memory - leak if the size of the offset vector was greater than 30. When the vector - is smaller, the saved offsets during recursion go onto a local stack - vector, but for larger vectors malloc() is used. It was failing to free - when the recursion yielded PCRE_ERROR_MATCH_LIMIT (or any other "abnormal" - error, in fact). - -22. There was a missing #ifdef SUPPORT_UTF8 round one of the variables in the - heapframe that is used only when UTF-8 support is enabled. This caused no - problem, but was untidy. - -23. Steven Van Ingelgem's patch to CMakeLists.txt to change the name - CMAKE_BINARY_DIR to PROJECT_BINARY_DIR so that it works when PCRE is - included within another project. - -24. Steven Van Ingelgem's patches to add more options to the CMake support, - slightly modified by me: - - (a) PCRE_BUILD_TESTS can be set OFF not to build the tests, including - not building pcregrep. - - (b) PCRE_BUILD_PCREGREP can be see OFF not to build pcregrep, but only - if PCRE_BUILD_TESTS is also set OFF, because the tests use pcregrep. - -25. Forward references, both numeric and by name, in patterns that made use of - duplicate group numbers, could behave incorrectly or give incorrect errors, - because when scanning forward to find the reference group, PCRE was not - taking into account the duplicate group numbers. A pattern such as - ^X(?3)(a)(?|(b)|(q))(Y) is an example. - -26. Changed a few more instances of "const unsigned char *" to USPTR, making - the feature of a custom pointer more persuasive (as requested by a user). - -27. Wrapped the definitions of fileno and isatty for Windows, which appear in - pcretest.c, inside #ifndefs, because it seems they are sometimes already - pre-defined. - -28. Added support for (*UTF8) at the start of a pattern. - -29. Arrange for flags added by the "release type" setting in CMake to be shown - in the configuration summary. - - -Version 7.8 05-Sep-08 ---------------------- - -1. Replaced UCP searching code with optimized version as implemented for Ad - Muncher (http://www.admuncher.com/) by Peter Kankowski. This uses a two- - stage table and inline lookup instead of a function, giving speed ups of 2 - to 5 times on some simple patterns that I tested. Permission was given to - distribute the MultiStage2.py script that generates the tables (it's not in - the tarball, but is in the Subversion repository). - -2. Updated the Unicode datatables to Unicode 5.1.0. This adds yet more - scripts. - -3. Change 12 for 7.7 introduced a bug in pcre_study() when a pattern contained - a group with a zero qualifier. The result of the study could be incorrect, - or the function might crash, depending on the pattern. - -4. Caseless matching was not working for non-ASCII characters in back - references. For example, /(\x{de})\1/8i was not matching \x{de}\x{fe}. - It now works when Unicode Property Support is available. - -5. In pcretest, an escape such as \x{de} in the data was always generating - a UTF-8 string, even in non-UTF-8 mode. Now it generates a single byte in - non-UTF-8 mode. If the value is greater than 255, it gives a warning about - truncation. - -6. Minor bugfix in pcrecpp.cc (change "" == ... to NULL == ...). - -7. Added two (int) casts to pcregrep when printing the difference of two - pointers, in case they are 64-bit values. - -8. Added comments about Mac OS X stack usage to the pcrestack man page and to - test 2 if it fails. - -9. Added PCRE_CALL_CONVENTION just before the names of all exported functions, - and a #define of that name to empty if it is not externally set. This is to - allow users of MSVC to set it if necessary. - -10. The PCRE_EXP_DEFN macro which precedes exported functions was missing from - the convenience functions in the pcre_get.c source file. - -11. An option change at the start of a pattern that had top-level alternatives - could cause overwriting and/or a crash. This command provoked a crash in - some environments: - - printf "/(?i)[\xc3\xa9\xc3\xbd]|[\xc3\xa9\xc3\xbdA]/8\n" | pcretest - - This potential security problem was recorded as CVE-2008-2371. - -12. For a pattern where the match had to start at the beginning or immediately - after a newline (e.g /.*anything/ without the DOTALL flag), pcre_exec() and - pcre_dfa_exec() could read past the end of the passed subject if there was - no match. To help with detecting such bugs (e.g. with valgrind), I modified - pcretest so that it places the subject at the end of its malloc-ed buffer. - -13. The change to pcretest in 12 above threw up a couple more cases when pcre_ - exec() might read past the end of the data buffer in UTF-8 mode. - -14. A similar bug to 7.3/2 existed when the PCRE_FIRSTLINE option was set and - the data contained the byte 0x85 as part of a UTF-8 character within its - first line. This applied both to normal and DFA matching. - -15. Lazy qualifiers were not working in some cases in UTF-8 mode. For example, - /^[^d]*?$/8 failed to match "abc". - -16. Added a missing copyright notice to pcrecpp_internal.h. - -17. Make it more clear in the documentation that values returned from - pcre_exec() in ovector are byte offsets, not character counts. - -18. Tidied a few places to stop certain compilers from issuing warnings. - -19. Updated the Virtual Pascal + BCC files to compile the latest v7.7, as - supplied by Stefan Weber. I made a further small update for 7.8 because - there is a change of source arrangements: the pcre_searchfuncs.c module is - replaced by pcre_ucd.c. - - -Version 7.7 07-May-08 ---------------------- - -1. Applied Craig's patch to sort out a long long problem: "If we can't convert - a string to a long long, pretend we don't even have a long long." This is - done by checking for the strtoq, strtoll, and _strtoi64 functions. - -2. Applied Craig's patch to pcrecpp.cc to restore ABI compatibility with - pre-7.6 versions, which defined a global no_arg variable instead of putting - it in the RE class. (See also #8 below.) - -3. Remove a line of dead code, identified by coverity and reported by Nuno - Lopes. - -4. Fixed two related pcregrep bugs involving -r with --include or --exclude: - - (1) The include/exclude patterns were being applied to the whole pathnames - of files, instead of just to the final components. - - (2) If there was more than one level of directory, the subdirectories were - skipped unless they satisfied the include/exclude conditions. This is - inconsistent with GNU grep (and could even be seen as contrary to the - pcregrep specification - which I improved to make it absolutely clear). - The action now is always to scan all levels of directory, and just - apply the include/exclude patterns to regular files. - -5. Added the --include_dir and --exclude_dir patterns to pcregrep, and used - --exclude_dir in the tests to avoid scanning .svn directories. - -6. Applied Craig's patch to the QuoteMeta function so that it escapes the - NUL character as backslash + 0 rather than backslash + NUL, because PCRE - doesn't support NULs in patterns. - -7. Added some missing "const"s to declarations of static tables in - pcre_compile.c and pcre_dfa_exec.c. - -8. Applied Craig's patch to pcrecpp.cc to fix a problem in OS X that was - caused by fix #2 above. (Subsequently also a second patch to fix the - first patch. And a third patch - this was a messy problem.) - -9. Applied Craig's patch to remove the use of push_back(). - -10. Applied Alan Lehotsky's patch to add REG_STARTEND support to the POSIX - matching function regexec(). - -11. Added support for the Oniguruma syntax \g, \g, \g'name', \g'n', - which, however, unlike Perl's \g{...}, are subroutine calls, not back - references. PCRE supports relative numbers with this syntax (I don't think - Oniguruma does). - -12. Previously, a group with a zero repeat such as (...){0} was completely - omitted from the compiled regex. However, this means that if the group - was called as a subroutine from elsewhere in the pattern, things went wrong - (an internal error was given). Such groups are now left in the compiled - pattern, with a new opcode that causes them to be skipped at execution - time. - -13. Added the PCRE_JAVASCRIPT_COMPAT option. This makes the following changes - to the way PCRE behaves: - - (a) A lone ] character is dis-allowed (Perl treats it as data). - - (b) A back reference to an unmatched subpattern matches an empty string - (Perl fails the current match path). - - (c) A data ] in a character class must be notated as \] because if the - first data character in a class is ], it defines an empty class. (In - Perl it is not possible to have an empty class.) The empty class [] - never matches; it forces failure and is equivalent to (*FAIL) or (?!). - The negative empty class [^] matches any one character, independently - of the DOTALL setting. - -14. A pattern such as /(?2)[]a()b](abc)/ which had a forward reference to a - non-existent subpattern following a character class starting with ']' and - containing () gave an internal compiling error instead of "reference to - non-existent subpattern". Fortunately, when the pattern did exist, the - compiled code was correct. (When scanning forwards to check for the - existence of the subpattern, it was treating the data ']' as terminating - the class, so got the count wrong. When actually compiling, the reference - was subsequently set up correctly.) - -15. The "always fail" assertion (?!) is optimzed to (*FAIL) by pcre_compile; - it was being rejected as not supported by pcre_dfa_exec(), even though - other assertions are supported. I have made pcre_dfa_exec() support - (*FAIL). - -16. The implementation of 13c above involved the invention of a new opcode, - OP_ALLANY, which is like OP_ANY but doesn't check the /s flag. Since /s - cannot be changed at match time, I realized I could make a small - improvement to matching performance by compiling OP_ALLANY instead of - OP_ANY for "." when DOTALL was set, and then removing the runtime tests - on the OP_ANY path. - -17. Compiling pcretest on Windows with readline support failed without the - following two fixes: (1) Make the unistd.h include conditional on - HAVE_UNISTD_H; (2) #define isatty and fileno as _isatty and _fileno. - -18. Changed CMakeLists.txt and cmake/FindReadline.cmake to arrange for the - ncurses library to be included for pcretest when ReadLine support is - requested, but also to allow for it to be overridden. This patch came from - Daniel Bergström. - -19. There was a typo in the file ucpinternal.h where f0_rangeflag was defined - as 0x00f00000 instead of 0x00800000. Luckily, this would not have caused - any errors with the current Unicode tables. Thanks to Peter Kankowski for - spotting this. - - -Version 7.6 28-Jan-08 ---------------------- - -1. A character class containing a very large number of characters with - codepoints greater than 255 (in UTF-8 mode, of course) caused a buffer - overflow. - -2. Patch to cut out the "long long" test in pcrecpp_unittest when - HAVE_LONG_LONG is not defined. - -3. Applied Christian Ehrlicher's patch to update the CMake build files to - bring them up to date and include new features. This patch includes: - - - Fixed PH's badly added libz and libbz2 support. - - Fixed a problem with static linking. - - Added pcredemo. [But later removed - see 7 below.] - - Fixed dftables problem and added an option. - - Added a number of HAVE_XXX tests, including HAVE_WINDOWS_H and - HAVE_LONG_LONG. - - Added readline support for pcretest. - - Added an listing of the option settings after cmake has run. - -4. A user submitted a patch to Makefile that makes it easy to create - "pcre.dll" under mingw when using Configure/Make. I added stuff to - Makefile.am that cause it to include this special target, without - affecting anything else. Note that the same mingw target plus all - the other distribution libraries and programs are now supported - when configuring with CMake (see 6 below) instead of with - Configure/Make. - -5. Applied Craig's patch that moves no_arg into the RE class in the C++ code. - This is an attempt to solve the reported problem "pcrecpp::no_arg is not - exported in the Windows port". It has not yet been confirmed that the patch - solves the problem, but it does no harm. - -6. Applied Sheri's patch to CMakeLists.txt to add NON_STANDARD_LIB_PREFIX and - NON_STANDARD_LIB_SUFFIX for dll names built with mingw when configured - with CMake, and also correct the comment about stack recursion. - -7. Remove the automatic building of pcredemo from the ./configure system and - from CMakeLists.txt. The whole idea of pcredemo.c is that it is an example - of a program that users should build themselves after PCRE is installed, so - building it automatically is not really right. What is more, it gave - trouble in some build environments. - -8. Further tidies to CMakeLists.txt from Sheri and Christian. - - -Version 7.5 10-Jan-08 ---------------------- - -1. Applied a patch from Craig: "This patch makes it possible to 'ignore' - values in parens when parsing an RE using the C++ wrapper." - -2. Negative specials like \S did not work in character classes in UTF-8 mode. - Characters greater than 255 were excluded from the class instead of being - included. - -3. The same bug as (2) above applied to negated POSIX classes such as - [:^space:]. - -4. PCRECPP_STATIC was referenced in pcrecpp_internal.h, but nowhere was it - defined or documented. It seems to have been a typo for PCRE_STATIC, so - I have changed it. - -5. The construct (?&) was not diagnosed as a syntax error (it referenced the - first named subpattern) and a construct such as (?&a) would reference the - first named subpattern whose name started with "a" (in other words, the - length check was missing). Both these problems are fixed. "Subpattern name - expected" is now given for (?&) (a zero-length name), and this patch also - makes it give the same error for \k'' (previously it complained that that - was a reference to a non-existent subpattern). - -6. The erroneous patterns (?+-a) and (?-+a) give different error messages; - this is right because (?- can be followed by option settings as well as by - digits. I have, however, made the messages clearer. - -7. Patterns such as (?(1)a|b) (a pattern that contains fewer subpatterns - than the number used in the conditional) now cause a compile-time error. - This is actually not compatible with Perl, which accepts such patterns, but - treats the conditional as always being FALSE (as PCRE used to), but it - seems to me that giving a diagnostic is better. - -8. Change "alphameric" to the more common word "alphanumeric" in comments - and messages. - -9. Fix two occurrences of "backslash" in comments that should have been - "backspace". - -10. Remove two redundant lines of code that can never be obeyed (their function - was moved elsewhere). - -11. The program that makes PCRE's Unicode character property table had a bug - which caused it to generate incorrect table entries for sequences of - characters that have the same character type, but are in different scripts. - It amalgamated them into a single range, with the script of the first of - them. In other words, some characters were in the wrong script. There were - thirteen such cases, affecting characters in the following ranges: - - U+002b0 - U+002c1 - U+0060c - U+0060d - U+0061e - U+00612 - U+0064b - U+0065e - U+0074d - U+0076d - U+01800 - U+01805 - U+01d00 - U+01d77 - U+01d9b - U+01dbf - U+0200b - U+0200f - U+030fc - U+030fe - U+03260 - U+0327f - U+0fb46 - U+0fbb1 - U+10450 - U+1049d - -12. The -o option (show only the matching part of a line) for pcregrep was not - compatible with GNU grep in that, if there was more than one match in a - line, it showed only the first of them. It now behaves in the same way as - GNU grep. - -13. If the -o and -v options were combined for pcregrep, it printed a blank - line for every non-matching line. GNU grep prints nothing, and pcregrep now - does the same. The return code can be used to tell if there were any - non-matching lines. - -14. Added --file-offsets and --line-offsets to pcregrep. - -15. The pattern (?=something)(?R) was not being diagnosed as a potentially - infinitely looping recursion. The bug was that positive lookaheads were not - being skipped when checking for a possible empty match (negative lookaheads - and both kinds of lookbehind were skipped). - -16. Fixed two typos in the Windows-only code in pcregrep.c, and moved the - inclusion of to before rather than after the definition of - INVALID_FILE_ATTRIBUTES (patch from David Byron). - -17. Specifying a possessive quantifier with a specific limit for a Unicode - character property caused pcre_compile() to compile bad code, which led at - runtime to PCRE_ERROR_INTERNAL (-14). Examples of patterns that caused this - are: /\p{Zl}{2,3}+/8 and /\p{Cc}{2}+/8. It was the possessive "+" that - caused the error; without that there was no problem. - -18. Added --enable-pcregrep-libz and --enable-pcregrep-libbz2. - -19. Added --enable-pcretest-libreadline. - -20. In pcrecpp.cc, the variable 'count' was incremented twice in - RE::GlobalReplace(). As a result, the number of replacements returned was - double what it should be. I removed one of the increments, but Craig sent a - later patch that removed the other one (the right fix) and added unit tests - that check the return values (which was not done before). - -21. Several CMake things: - - (1) Arranged that, when cmake is used on Unix, the libraries end up with - the names libpcre and libpcreposix, not just pcre and pcreposix. - - (2) The above change means that pcretest and pcregrep are now correctly - linked with the newly-built libraries, not previously installed ones. - - (3) Added PCRE_SUPPORT_LIBREADLINE, PCRE_SUPPORT_LIBZ, PCRE_SUPPORT_LIBBZ2. - -22. In UTF-8 mode, with newline set to "any", a pattern such as .*a.*=.b.* - crashed when matching a string such as a\x{2029}b (note that \x{2029} is a - UTF-8 newline character). The key issue is that the pattern starts .*; - this means that the match must be either at the beginning, or after a - newline. The bug was in the code for advancing after a failed match and - checking that the new position followed a newline. It was not taking - account of UTF-8 characters correctly. - -23. PCRE was behaving differently from Perl in the way it recognized POSIX - character classes. PCRE was not treating the sequence [:...:] as a - character class unless the ... were all letters. Perl, however, seems to - allow any characters between [: and :], though of course it rejects as - unknown any "names" that contain non-letters, because all the known class - names consist only of letters. Thus, Perl gives an error for [[:1234:]], - for example, whereas PCRE did not - it did not recognize a POSIX character - class. This seemed a bit dangerous, so the code has been changed to be - closer to Perl. The behaviour is not identical to Perl, because PCRE will - diagnose an unknown class for, for example, [[:l\ower:]] where Perl will - treat it as [[:lower:]]. However, PCRE does now give "unknown" errors where - Perl does, and where it didn't before. - -24. Rewrite so as to remove the single use of %n from pcregrep because in some - Windows environments %n is disabled by default. - - -Version 7.4 21-Sep-07 ---------------------- - -1. Change 7.3/28 was implemented for classes by looking at the bitmap. This - means that a class such as [\s] counted as "explicit reference to CR or - LF". That isn't really right - the whole point of the change was to try to - help when there was an actual mention of one of the two characters. So now - the change happens only if \r or \n (or a literal CR or LF) character is - encountered. - -2. The 32-bit options word was also used for 6 internal flags, but the numbers - of both had grown to the point where there were only 3 bits left. - Fortunately, there was spare space in the data structure, and so I have - moved the internal flags into a new 16-bit field to free up more option - bits. - -3. The appearance of (?J) at the start of a pattern set the DUPNAMES option, - but did not set the internal JCHANGED flag - either of these is enough to - control the way the "get" function works - but the PCRE_INFO_JCHANGED - facility is supposed to tell if (?J) was ever used, so now (?J) at the - start sets both bits. - -4. Added options (at build time, compile time, exec time) to change \R from - matching any Unicode line ending sequence to just matching CR, LF, or CRLF. - -5. doc/pcresyntax.html was missing from the distribution. - -6. Put back the definition of PCRE_ERROR_NULLWSLIMIT, for backward - compatibility, even though it is no longer used. - -7. Added macro for snprintf to pcrecpp_unittest.cc and also for strtoll and - strtoull to pcrecpp.cc to select the available functions in WIN32 when the - windows.h file is present (where different names are used). [This was - reversed later after testing - see 16 below.] - -8. Changed all #include to #include "config.h". There were also - some further cases that I changed to "pcre.h". - -9. When pcregrep was used with the --colour option, it missed the line ending - sequence off the lines that it output. - -10. It was pointed out to me that arrays of string pointers cause lots of - relocations when a shared library is dynamically loaded. A technique of - using a single long string with a table of offsets can drastically reduce - these. I have refactored PCRE in four places to do this. The result is - dramatic: - - Originally: 290 - After changing UCP table: 187 - After changing error message table: 43 - After changing table of "verbs" 36 - After changing table of Posix names 22 - - Thanks to the folks working on Gregex for glib for this insight. - -11. --disable-stack-for-recursion caused compiling to fail unless -enable- - unicode-properties was also set. - -12. Updated the tests so that they work when \R is defaulted to ANYCRLF. - -13. Added checks for ANY and ANYCRLF to pcrecpp.cc where it previously - checked only for CRLF. - -14. Added casts to pcretest.c to avoid compiler warnings. - -15. Added Craig's patch to various pcrecpp modules to avoid compiler warnings. - -16. Added Craig's patch to remove the WINDOWS_H tests, that were not working, - and instead check for _strtoi64 explicitly, and avoid the use of snprintf() - entirely. This removes changes made in 7 above. - -17. The CMake files have been updated, and there is now more information about - building with CMake in the NON-UNIX-USE document. - - -Version 7.3 28-Aug-07 ---------------------- - - 1. In the rejigging of the build system that eventually resulted in 7.1, the - line "#include " was included in pcre_internal.h. The use of angle - brackets there is not right, since it causes compilers to look for an - installed pcre.h, not the version that is in the source that is being - compiled (which of course may be different). I have changed it back to: - - #include "pcre.h" - - I have a vague recollection that the change was concerned with compiling in - different directories, but in the new build system, that is taken care of - by the VPATH setting the Makefile. - - 2. The pattern .*$ when run in not-DOTALL UTF-8 mode with newline=any failed - when the subject happened to end in the byte 0x85 (e.g. if the last - character was \x{1ec5}). *Character* 0x85 is one of the "any" newline - characters but of course it shouldn't be taken as a newline when it is part - of another character. The bug was that, for an unlimited repeat of . in - not-DOTALL UTF-8 mode, PCRE was advancing by bytes rather than by - characters when looking for a newline. - - 3. A small performance improvement in the DOTALL UTF-8 mode .* case. - - 4. Debugging: adjusted the names of opcodes for different kinds of parentheses - in debug output. - - 5. Arrange to use "%I64d" instead of "%lld" and "%I64u" instead of "%llu" for - long printing in the pcrecpp unittest when running under MinGW. - - 6. ESC_K was left out of the EBCDIC table. - - 7. Change 7.0/38 introduced a new limit on the number of nested non-capturing - parentheses; I made it 1000, which seemed large enough. Unfortunately, the - limit also applies to "virtual nesting" when a pattern is recursive, and in - this case 1000 isn't so big. I have been able to remove this limit at the - expense of backing off one optimization in certain circumstances. Normally, - when pcre_exec() would call its internal match() function recursively and - immediately return the result unconditionally, it uses a "tail recursion" - feature to save stack. However, when a subpattern that can match an empty - string has an unlimited repetition quantifier, it no longer makes this - optimization. That gives it a stack frame in which to save the data for - checking that an empty string has been matched. Previously this was taken - from the 1000-entry workspace that had been reserved. So now there is no - explicit limit, but more stack is used. - - 8. Applied Daniel's patches to solve problems with the import/export magic - syntax that is required for Windows, and which was going wrong for the - pcreposix and pcrecpp parts of the library. These were overlooked when this - problem was solved for the main library. - - 9. There were some crude static tests to avoid integer overflow when computing - the size of patterns that contain repeated groups with explicit upper - limits. As the maximum quantifier is 65535, the maximum group length was - set at 30,000 so that the product of these two numbers did not overflow a - 32-bit integer. However, it turns out that people want to use groups that - are longer than 30,000 bytes (though not repeat them that many times). - Change 7.0/17 (the refactoring of the way the pattern size is computed) has - made it possible to implement the integer overflow checks in a much more - dynamic way, which I have now done. The artificial limitation on group - length has been removed - we now have only the limit on the total length of - the compiled pattern, which depends on the LINK_SIZE setting. - -10. Fixed a bug in the documentation for get/copy named substring when - duplicate names are permitted. If none of the named substrings are set, the - functions return PCRE_ERROR_NOSUBSTRING (7); the doc said they returned an - empty string. - -11. Because Perl interprets \Q...\E at a high level, and ignores orphan \E - instances, patterns such as [\Q\E] or [\E] or even [^\E] cause an error, - because the ] is interpreted as the first data character and the - terminating ] is not found. PCRE has been made compatible with Perl in this - regard. Previously, it interpreted [\Q\E] as an empty class, and [\E] could - cause memory overwriting. - -10. Like Perl, PCRE automatically breaks an unlimited repeat after an empty - string has been matched (to stop an infinite loop). It was not recognizing - a conditional subpattern that could match an empty string if that - subpattern was within another subpattern. For example, it looped when - trying to match (((?(1)X|))*) but it was OK with ((?(1)X|)*) where the - condition was not nested. This bug has been fixed. - -12. A pattern like \X?\d or \P{L}?\d in non-UTF-8 mode could cause a backtrack - past the start of the subject in the presence of bytes with the top bit - set, for example "\x8aBCD". - -13. Added Perl 5.10 experimental backtracking controls (*FAIL), (*F), (*PRUNE), - (*SKIP), (*THEN), (*COMMIT), and (*ACCEPT). - -14. Optimized (?!) to (*FAIL). - -15. Updated the test for a valid UTF-8 string to conform to the later RFC 3629. - This restricts code points to be within the range 0 to 0x10FFFF, excluding - the "low surrogate" sequence 0xD800 to 0xDFFF. Previously, PCRE allowed the - full range 0 to 0x7FFFFFFF, as defined by RFC 2279. Internally, it still - does: it's just the validity check that is more restrictive. - -16. Inserted checks for integer overflows during escape sequence (backslash) - processing, and also fixed erroneous offset values for syntax errors during - backslash processing. - -17. Fixed another case of looking too far back in non-UTF-8 mode (cf 12 above) - for patterns like [\PPP\x8a]{1,}\x80 with the subject "A\x80". - -18. An unterminated class in a pattern like (?1)\c[ with a "forward reference" - caused an overrun. - -19. A pattern like (?:[\PPa*]*){8,} which had an "extended class" (one with - something other than just ASCII characters) inside a group that had an - unlimited repeat caused a loop at compile time (while checking to see - whether the group could match an empty string). - -20. Debugging a pattern containing \p or \P could cause a crash. For example, - [\P{Any}] did so. (Error in the code for printing property names.) - -21. An orphan \E inside a character class could cause a crash. - -22. A repeated capturing bracket such as (A)? could cause a wild memory - reference during compilation. - -23. There are several functions in pcre_compile() that scan along a compiled - expression for various reasons (e.g. to see if it's fixed length for look - behind). There were bugs in these functions when a repeated \p or \P was - present in the pattern. These operators have additional parameters compared - with \d, etc, and these were not being taken into account when moving along - the compiled data. Specifically: - - (a) A item such as \p{Yi}{3} in a lookbehind was not treated as fixed - length. - - (b) An item such as \pL+ within a repeated group could cause crashes or - loops. - - (c) A pattern such as \p{Yi}+(\P{Yi}+)(?1) could give an incorrect - "reference to non-existent subpattern" error. - - (d) A pattern like (\P{Yi}{2}\277)? could loop at compile time. - -24. A repeated \S or \W in UTF-8 mode could give wrong answers when multibyte - characters were involved (for example /\S{2}/8g with "A\x{a3}BC"). - -25. Using pcregrep in multiline, inverted mode (-Mv) caused it to loop. - -26. Patterns such as [\P{Yi}A] which include \p or \P and just one other - character were causing crashes (broken optimization). - -27. Patterns such as (\P{Yi}*\277)* (group with possible zero repeat containing - \p or \P) caused a compile-time loop. - -28. More problems have arisen in unanchored patterns when CRLF is a valid line - break. For example, the unstudied pattern [\r\n]A does not match the string - "\r\nA" because change 7.0/46 below moves the current point on by two - characters after failing to match at the start. However, the pattern \nA - *does* match, because it doesn't start till \n, and if [\r\n]A is studied, - the same is true. There doesn't seem any very clean way out of this, but - what I have chosen to do makes the common cases work: PCRE now takes note - of whether there can be an explicit match for \r or \n anywhere in the - pattern, and if so, 7.0/46 no longer applies. As part of this change, - there's a new PCRE_INFO_HASCRORLF option for finding out whether a compiled - pattern has explicit CR or LF references. - -29. Added (*CR) etc for changing newline setting at start of pattern. - - -Version 7.2 19-Jun-07 ---------------------- - - 1. If the fr_FR locale cannot be found for test 3, try the "french" locale, - which is apparently normally available under Windows. - - 2. Re-jig the pcregrep tests with different newline settings in an attempt - to make them independent of the local environment's newline setting. - - 3. Add code to configure.ac to remove -g from the CFLAGS default settings. - - 4. Some of the "internals" tests were previously cut out when the link size - was not 2, because the output contained actual offsets. The recent new - "Z" feature of pcretest means that these can be cut out, making the tests - usable with all link sizes. - - 5. Implemented Stan Switzer's goto replacement for longjmp() when not using - stack recursion. This gives a massive performance boost under BSD, but just - a small improvement under Linux. However, it saves one field in the frame - in all cases. - - 6. Added more features from the forthcoming Perl 5.10: - - (a) (?-n) (where n is a string of digits) is a relative subroutine or - recursion call. It refers to the nth most recently opened parentheses. - - (b) (?+n) is also a relative subroutine call; it refers to the nth next - to be opened parentheses. - - (c) Conditions that refer to capturing parentheses can be specified - relatively, for example, (?(-2)... or (?(+3)... - - (d) \K resets the start of the current match so that everything before - is not part of it. - - (e) \k{name} is synonymous with \k and \k'name' (.NET compatible). - - (f) \g{name} is another synonym - part of Perl 5.10's unification of - reference syntax. - - (g) (?| introduces a group in which the numbering of parentheses in each - alternative starts with the same number. - - (h) \h, \H, \v, and \V match horizontal and vertical whitespace. - - 7. Added two new calls to pcre_fullinfo(): PCRE_INFO_OKPARTIAL and - PCRE_INFO_JCHANGED. - - 8. A pattern such as (.*(.)?)* caused pcre_exec() to fail by either not - terminating or by crashing. Diagnosed by Viktor Griph; it was in the code - for detecting groups that can match an empty string. - - 9. A pattern with a very large number of alternatives (more than several - hundred) was running out of internal workspace during the pre-compile - phase, where pcre_compile() figures out how much memory will be needed. A - bit of new cunning has reduced the workspace needed for groups with - alternatives. The 1000-alternative test pattern now uses 12 bytes of - workspace instead of running out of the 4096 that are available. - -10. Inserted some missing (unsigned int) casts to get rid of compiler warnings. - -11. Applied patch from Google to remove an optimization that didn't quite work. - The report of the bug said: - - pcrecpp::RE("a*").FullMatch("aaa") matches, while - pcrecpp::RE("a*?").FullMatch("aaa") does not, and - pcrecpp::RE("a*?\\z").FullMatch("aaa") does again. - -12. If \p or \P was used in non-UTF-8 mode on a character greater than 127 - it matched the wrong number of bytes. - - -Version 7.1 24-Apr-07 ---------------------- - - 1. Applied Bob Rossi and Daniel G's patches to convert the build system to one - that is more "standard", making use of automake and other Autotools. There - is some re-arrangement of the files and adjustment of comments consequent - on this. - - 2. Part of the patch fixed a problem with the pcregrep tests. The test of -r - for recursive directory scanning broke on some systems because the files - are not scanned in any specific order and on different systems the order - was different. A call to "sort" has been inserted into RunGrepTest for the - approprate test as a short-term fix. In the longer term there may be an - alternative. - - 3. I had an email from Eric Raymond about problems translating some of PCRE's - man pages to HTML (despite the fact that I distribute HTML pages, some - people do their own conversions for various reasons). The problems - concerned the use of low-level troff macros .br and .in. I have therefore - removed all such uses from the man pages (some were redundant, some could - be replaced by .nf/.fi pairs). The 132html script that I use to generate - HTML has been updated to handle .nf/.fi and to complain if it encounters - .br or .in. - - 4. Updated comments in configure.ac that get placed in config.h.in and also - arranged for config.h to be included in the distribution, with the name - config.h.generic, for the benefit of those who have to compile without - Autotools (compare pcre.h, which is now distributed as pcre.h.generic). - - 5. Updated the support (such as it is) for Virtual Pascal, thanks to Stefan - Weber: (1) pcre_internal.h was missing some function renames; (2) updated - makevp.bat for the current PCRE, using the additional files - makevp_c.txt, makevp_l.txt, and pcregexp.pas. - - 6. A Windows user reported a minor discrepancy with test 2, which turned out - to be caused by a trailing space on an input line that had got lost in his - copy. The trailing space was an accident, so I've just removed it. - - 7. Add -Wl,-R... flags in pcre-config.in for *BSD* systems, as I'm told - that is needed. - - 8. Mark ucp_table (in ucptable.h) and ucp_gentype (in pcre_ucp_searchfuncs.c) - as "const" (a) because they are and (b) because it helps the PHP - maintainers who have recently made a script to detect big data structures - in the php code that should be moved to the .rodata section. I remembered - to update Builducptable as well, so it won't revert if ucptable.h is ever - re-created. - - 9. Added some extra #ifdef SUPPORT_UTF8 conditionals into pcretest.c, - pcre_printint.src, pcre_compile.c, pcre_study.c, and pcre_tables.c, in - order to be able to cut out the UTF-8 tables in the latter when UTF-8 - support is not required. This saves 1.5-2K of code, which is important in - some applications. - - Later: more #ifdefs are needed in pcre_ord2utf8.c and pcre_valid_utf8.c - so as not to refer to the tables, even though these functions will never be - called when UTF-8 support is disabled. Otherwise there are problems with a - shared library. - -10. Fixed two bugs in the emulated memmove() function in pcre_internal.h: - - (a) It was defining its arguments as char * instead of void *. - - (b) It was assuming that all moves were upwards in memory; this was true - a long time ago when I wrote it, but is no longer the case. - - The emulated memove() is provided for those environments that have neither - memmove() nor bcopy(). I didn't think anyone used it these days, but that - is clearly not the case, as these two bugs were recently reported. - -11. The script PrepareRelease is now distributed: it calls 132html, CleanTxt, - and Detrail to create the HTML documentation, the .txt form of the man - pages, and it removes trailing spaces from listed files. It also creates - pcre.h.generic and config.h.generic from pcre.h and config.h. In the latter - case, it wraps all the #defines with #ifndefs. This script should be run - before "make dist". - -12. Fixed two fairly obscure bugs concerned with quantified caseless matching - with Unicode property support. - - (a) For a maximizing quantifier, if the two different cases of the - character were of different lengths in their UTF-8 codings (there are - some cases like this - I found 11), and the matching function had to - back up over a mixture of the two cases, it incorrectly assumed they - were both the same length. - - (b) When PCRE was configured to use the heap rather than the stack for - recursion during matching, it was not correctly preserving the data for - the other case of a UTF-8 character when checking ahead for a match - while processing a minimizing repeat. If the check also involved - matching a wide character, but failed, corruption could cause an - erroneous result when trying to check for a repeat of the original - character. - -13. Some tidying changes to the testing mechanism: - - (a) The RunTest script now detects the internal link size and whether there - is UTF-8 and UCP support by running ./pcretest -C instead of relying on - values substituted by "configure". (The RunGrepTest script already did - this for UTF-8.) The configure.ac script no longer substitutes the - relevant variables. - - (b) The debugging options /B and /D in pcretest show the compiled bytecode - with length and offset values. This means that the output is different - for different internal link sizes. Test 2 is skipped for link sizes - other than 2 because of this, bypassing the problem. Unfortunately, - there was also a test in test 3 (the locale tests) that used /B and - failed for link sizes other than 2. Rather than cut the whole test out, - I have added a new /Z option to pcretest that replaces the length and - offset values with spaces. This is now used to make test 3 independent - of link size. (Test 2 will be tidied up later.) - -14. If erroroffset was passed as NULL to pcre_compile, it provoked a - segmentation fault instead of returning the appropriate error message. - -15. In multiline mode when the newline sequence was set to "any", the pattern - ^$ would give a match between the \r and \n of a subject such as "A\r\nB". - This doesn't seem right; it now treats the CRLF combination as the line - ending, and so does not match in that case. It's only a pattern such as ^$ - that would hit this one: something like ^ABC$ would have failed after \r - and then tried again after \r\n. - -16. Changed the comparison command for RunGrepTest from "diff -u" to "diff -ub" - in an attempt to make files that differ only in their line terminators - compare equal. This works on Linux. - -17. Under certain error circumstances pcregrep might try to free random memory - as it exited. This is now fixed, thanks to valgrind. - -19. In pcretest, if the pattern /(?m)^$/g was matched against the string - "abc\r\n\r\n", it found an unwanted second match after the second \r. This - was because its rules for how to advance for /g after matching an empty - string at the end of a line did not allow for this case. They now check for - it specially. - -20. pcretest is supposed to handle patterns and data of any length, by - extending its buffers when necessary. It was getting this wrong when the - buffer for a data line had to be extended. - -21. Added PCRE_NEWLINE_ANYCRLF which is like ANY, but matches only CR, LF, or - CRLF as a newline sequence. - -22. Code for handling Unicode properties in pcre_dfa_exec() wasn't being cut - out by #ifdef SUPPORT_UCP. This did no harm, as it could never be used, but - I have nevertheless tidied it up. - -23. Added some casts to kill warnings from HP-UX ia64 compiler. - -24. Added a man page for pcre-config. - - -Version 7.0 19-Dec-06 ---------------------- - - 1. Fixed a signed/unsigned compiler warning in pcre_compile.c, shown up by - moving to gcc 4.1.1. - - 2. The -S option for pcretest uses setrlimit(); I had omitted to #include - sys/time.h, which is documented as needed for this function. It doesn't - seem to matter on Linux, but it showed up on some releases of OS X. - - 3. It seems that there are systems where bytes whose values are greater than - 127 match isprint() in the "C" locale. The "C" locale should be the - default when a C program starts up. In most systems, only ASCII printing - characters match isprint(). This difference caused the output from pcretest - to vary, making some of the tests fail. I have changed pcretest so that: - - (a) When it is outputting text in the compiled version of a pattern, bytes - other than 32-126 are always shown as hex escapes. - - (b) When it is outputting text that is a matched part of a subject string, - it does the same, unless a different locale has been set for the match - (using the /L modifier). In this case, it uses isprint() to decide. - - 4. Fixed a major bug that caused incorrect computation of the amount of memory - required for a compiled pattern when options that changed within the - pattern affected the logic of the preliminary scan that determines the - length. The relevant options are -x, and -i in UTF-8 mode. The result was - that the computed length was too small. The symptoms of this bug were - either the PCRE error "internal error: code overflow" from pcre_compile(), - or a glibc crash with a message such as "pcretest: free(): invalid next - size (fast)". Examples of patterns that provoked this bug (shown in - pcretest format) are: - - /(?-x: )/x - /(?x)(?-x: \s*#\s*)/ - /((?i)[\x{c0}])/8 - /(?i:[\x{c0}])/8 - - HOWEVER: Change 17 below makes this fix obsolete as the memory computation - is now done differently. - - 5. Applied patches from Google to: (a) add a QuoteMeta function to the C++ - wrapper classes; (b) implement a new function in the C++ scanner that is - more efficient than the old way of doing things because it avoids levels of - recursion in the regex matching; (c) add a paragraph to the documentation - for the FullMatch() function. - - 6. The escape sequence \n was being treated as whatever was defined as - "newline". Not only was this contrary to the documentation, which states - that \n is character 10 (hex 0A), but it also went horribly wrong when - "newline" was defined as CRLF. This has been fixed. - - 7. In pcre_dfa_exec.c the value of an unsigned integer (the variable called c) - was being set to -1 for the "end of line" case (supposedly a value that no - character can have). Though this value is never used (the check for end of - line is "zero bytes in current character"), it caused compiler complaints. - I've changed it to 0xffffffff. - - 8. In pcre_version.c, the version string was being built by a sequence of - C macros that, in the event of PCRE_PRERELEASE being defined as an empty - string (as it is for production releases) called a macro with an empty - argument. The C standard says the result of this is undefined. The gcc - compiler treats it as an empty string (which was what was wanted) but it is - reported that Visual C gives an error. The source has been hacked around to - avoid this problem. - - 9. On the advice of a Windows user, included and in Windows - builds of pcretest, and changed the call to _setmode() to use _O_BINARY - instead of 0x8000. Made all the #ifdefs test both _WIN32 and WIN32 (not all - of them did). - -10. Originally, pcretest opened its input and output without "b"; then I was - told that "b" was needed in some environments, so it was added for release - 5.0 to both the input and output. (It makes no difference on Unix-like - systems.) Later I was told that it is wrong for the input on Windows. I've - now abstracted the modes into two macros, to make it easier to fiddle with - them, and removed "b" from the input mode under Windows. - -11. Added pkgconfig support for the C++ wrapper library, libpcrecpp. - -12. Added -help and --help to pcretest as an official way of being reminded - of the options. - -13. Removed some redundant semicolons after macro calls in pcrecpparg.h.in - and pcrecpp.cc because they annoy compilers at high warning levels. - -14. A bit of tidying/refactoring in pcre_exec.c in the main bumpalong loop. - -15. Fixed an occurrence of == in configure.ac that should have been = (shell - scripts are not C programs :-) and which was not noticed because it works - on Linux. - -16. pcretest is supposed to handle any length of pattern and data line (as one - line or as a continued sequence of lines) by extending its input buffer if - necessary. This feature was broken for very long pattern lines, leading to - a string of junk being passed to pcre_compile() if the pattern was longer - than about 50K. - -17. I have done a major re-factoring of the way pcre_compile() computes the - amount of memory needed for a compiled pattern. Previously, there was code - that made a preliminary scan of the pattern in order to do this. That was - OK when PCRE was new, but as the facilities have expanded, it has become - harder and harder to keep it in step with the real compile phase, and there - have been a number of bugs (see for example, 4 above). I have now found a - cunning way of running the real compile function in a "fake" mode that - enables it to compute how much memory it would need, while actually only - ever using a few hundred bytes of working memory and without too many - tests of the mode. This should make future maintenance and development - easier. A side effect of this work is that the limit of 200 on the nesting - depth of parentheses has been removed (though this was never a serious - limitation, I suspect). However, there is a downside: pcre_compile() now - runs more slowly than before (30% or more, depending on the pattern). I - hope this isn't a big issue. There is no effect on runtime performance. - -18. Fixed a minor bug in pcretest: if a pattern line was not terminated by a - newline (only possible for the last line of a file) and it was a - pattern that set a locale (followed by /Lsomething), pcretest crashed. - -19. Added additional timing features to pcretest. (1) The -tm option now times - matching only, not compiling. (2) Both -t and -tm can be followed, as a - separate command line item, by a number that specifies the number of - repeats to use when timing. The default is 50000; this gives better - precision, but takes uncomfortably long for very large patterns. - -20. Extended pcre_study() to be more clever in cases where a branch of a - subpattern has no definite first character. For example, (a*|b*)[cd] would - previously give no result from pcre_study(). Now it recognizes that the - first character must be a, b, c, or d. - -21. There was an incorrect error "recursive call could loop indefinitely" if - a subpattern (or the entire pattern) that was being tested for matching an - empty string contained only one non-empty item after a nested subpattern. - For example, the pattern (?>\x{100}*)\d(?R) provoked this error - incorrectly, because the \d was being skipped in the check. - -22. The pcretest program now has a new pattern option /B and a command line - option -b, which is equivalent to adding /B to every pattern. This causes - it to show the compiled bytecode, without the additional information that - -d shows. The effect of -d is now the same as -b with -i (and similarly, /D - is the same as /B/I). - -23. A new optimization is now able automatically to treat some sequences such - as a*b as a*+b. More specifically, if something simple (such as a character - or a simple class like \d) has an unlimited quantifier, and is followed by - something that cannot possibly match the quantified thing, the quantifier - is automatically "possessified". - -24. A recursive reference to a subpattern whose number was greater than 39 - went wrong under certain circumstances in UTF-8 mode. This bug could also - have affected the operation of pcre_study(). - -25. Realized that a little bit of performance could be had by replacing - (c & 0xc0) == 0xc0 with c >= 0xc0 when processing UTF-8 characters. - -26. Timing data from pcretest is now shown to 4 decimal places instead of 3. - -27. Possessive quantifiers such as a++ were previously implemented by turning - them into atomic groups such as ($>a+). Now they have their own opcodes, - which improves performance. This includes the automatically created ones - from 23 above. - -28. A pattern such as (?=(\w+))\1: which simulates an atomic group using a - lookahead was broken if it was not anchored. PCRE was mistakenly expecting - the first matched character to be a colon. This applied both to named and - numbered groups. - -29. The ucpinternal.h header file was missing its idempotency #ifdef. - -30. I was sent a "project" file called libpcre.a.dev which I understand makes - building PCRE on Windows easier, so I have included it in the distribution. - -31. There is now a check in pcretest against a ridiculously large number being - returned by pcre_exec() or pcre_dfa_exec(). If this happens in a /g or /G - loop, the loop is abandoned. - -32. Forward references to subpatterns in conditions such as (?(2)...) where - subpattern 2 is defined later cause pcre_compile() to search forwards in - the pattern for the relevant set of parentheses. This search went wrong - when there were unescaped parentheses in a character class, parentheses - escaped with \Q...\E, or parentheses in a #-comment in /x mode. - -33. "Subroutine" calls and backreferences were previously restricted to - referencing subpatterns earlier in the regex. This restriction has now - been removed. - -34. Added a number of extra features that are going to be in Perl 5.10. On the - whole, these are just syntactic alternatives for features that PCRE had - previously implemented using the Python syntax or my own invention. The - other formats are all retained for compatibility. - - (a) Named groups can now be defined as (?...) or (?'name'...) as well - as (?P...). The new forms, as well as being in Perl 5.10, are - also .NET compatible. - - (b) A recursion or subroutine call to a named group can now be defined as - (?&name) as well as (?P>name). - - (c) A backreference to a named group can now be defined as \k or - \k'name' as well as (?P=name). The new forms, as well as being in Perl - 5.10, are also .NET compatible. - - (d) A conditional reference to a named group can now use the syntax - (?() or (?('name') as well as (?(name). - - (e) A "conditional group" of the form (?(DEFINE)...) can be used to define - groups (named and numbered) that are never evaluated inline, but can be - called as "subroutines" from elsewhere. In effect, the DEFINE condition - is always false. There may be only one alternative in such a group. - - (f) A test for recursion can be given as (?(R1).. or (?(R&name)... as well - as the simple (?(R). The condition is true only if the most recent - recursion is that of the given number or name. It does not search out - through the entire recursion stack. - - (g) The escape \gN or \g{N} has been added, where N is a positive or - negative number, specifying an absolute or relative reference. - -35. Tidied to get rid of some further signed/unsigned compiler warnings and - some "unreachable code" warnings. - -36. Updated the Unicode property tables to Unicode version 5.0.0. Amongst other - things, this adds five new scripts. - -37. Perl ignores orphaned \E escapes completely. PCRE now does the same. - There were also incompatibilities regarding the handling of \Q..\E inside - character classes, for example with patterns like [\Qa\E-\Qz\E] where the - hyphen was adjacent to \Q or \E. I hope I've cleared all this up now. - -38. Like Perl, PCRE detects when an indefinitely repeated parenthesized group - matches an empty string, and forcibly breaks the loop. There were bugs in - this code in non-simple cases. For a pattern such as ^(a()*)* matched - against aaaa the result was just "a" rather than "aaaa", for example. Two - separate and independent bugs (that affected different cases) have been - fixed. - -39. Refactored the code to abolish the use of different opcodes for small - capturing bracket numbers. This is a tidy that I avoided doing when I - removed the limit on the number of capturing brackets for 3.5 back in 2001. - The new approach is not only tidier, it makes it possible to reduce the - memory needed to fix the previous bug (38). - -40. Implemented PCRE_NEWLINE_ANY to recognize any of the Unicode newline - sequences (http://unicode.org/unicode/reports/tr18/) as "newline" when - processing dot, circumflex, or dollar metacharacters, or #-comments in /x - mode. - -41. Add \R to match any Unicode newline sequence, as suggested in the Unicode - report. - -42. Applied patch, originally from Ari Pollak, modified by Google, to allow - copy construction and assignment in the C++ wrapper. - -43. Updated pcregrep to support "--newline=any". In the process, I fixed a - couple of bugs that could have given wrong results in the "--newline=crlf" - case. - -44. Added a number of casts and did some reorganization of signed/unsigned int - variables following suggestions from Dair Grant. Also renamed the variable - "this" as "item" because it is a C++ keyword. - -45. Arranged for dftables to add - - #include "pcre_internal.h" - - to pcre_chartables.c because without it, gcc 4.x may remove the array - definition from the final binary if PCRE is built into a static library and - dead code stripping is activated. - -46. For an unanchored pattern, if a match attempt fails at the start of a - newline sequence, and the newline setting is CRLF or ANY, and the next two - characters are CRLF, advance by two characters instead of one. - - -Version 6.7 04-Jul-06 ---------------------- - - 1. In order to handle tests when input lines are enormously long, pcretest has - been re-factored so that it automatically extends its buffers when - necessary. The code is crude, but this _is_ just a test program. The - default size has been increased from 32K to 50K. - - 2. The code in pcre_study() was using the value of the re argument before - testing it for NULL. (Of course, in any sensible call of the function, it - won't be NULL.) - - 3. The memmove() emulation function in pcre_internal.h, which is used on - systems that lack both memmove() and bcopy() - that is, hardly ever - - was missing a "static" storage class specifier. - - 4. When UTF-8 mode was not set, PCRE looped when compiling certain patterns - containing an extended class (one that cannot be represented by a bitmap - because it contains high-valued characters or Unicode property items, e.g. - [\pZ]). Almost always one would set UTF-8 mode when processing such a - pattern, but PCRE should not loop if you do not (it no longer does). - [Detail: two cases were found: (a) a repeated subpattern containing an - extended class; (b) a recursive reference to a subpattern that followed a - previous extended class. It wasn't skipping over the extended class - correctly when UTF-8 mode was not set.] - - 5. A negated single-character class was not being recognized as fixed-length - in lookbehind assertions such as (?<=[^f]), leading to an incorrect - compile error "lookbehind assertion is not fixed length". - - 6. The RunPerlTest auxiliary script was showing an unexpected difference - between PCRE and Perl for UTF-8 tests. It turns out that it is hard to - write a Perl script that can interpret lines of an input file either as - byte characters or as UTF-8, which is what "perltest" was being required to - do for the non-UTF-8 and UTF-8 tests, respectively. Essentially what you - can't do is switch easily at run time between having the "use utf8;" pragma - or not. In the end, I fudged it by using the RunPerlTest script to insert - "use utf8;" explicitly for the UTF-8 tests. - - 7. In multiline (/m) mode, PCRE was matching ^ after a terminating newline at - the end of the subject string, contrary to the documentation and to what - Perl does. This was true of both matching functions. Now it matches only at - the start of the subject and immediately after *internal* newlines. - - 8. A call of pcre_fullinfo() from pcretest to get the option bits was passing - a pointer to an int instead of a pointer to an unsigned long int. This - caused problems on 64-bit systems. - - 9. Applied a patch from the folks at Google to pcrecpp.cc, to fix "another - instance of the 'standard' template library not being so standard". - -10. There was no check on the number of named subpatterns nor the maximum - length of a subpattern name. The product of these values is used to compute - the size of the memory block for a compiled pattern. By supplying a very - long subpattern name and a large number of named subpatterns, the size - computation could be caused to overflow. This is now prevented by limiting - the length of names to 32 characters, and the number of named subpatterns - to 10,000. - -11. Subpatterns that are repeated with specific counts have to be replicated in - the compiled pattern. The size of memory for this was computed from the - length of the subpattern and the repeat count. The latter is limited to - 65535, but there was no limit on the former, meaning that integer overflow - could in principle occur. The compiled length of a repeated subpattern is - now limited to 30,000 bytes in order to prevent this. - -12. Added the optional facility to have named substrings with the same name. - -13. Added the ability to use a named substring as a condition, using the - Python syntax: (?(name)yes|no). This overloads (?(R)... and names that - are numbers (not recommended). Forward references are permitted. - -14. Added forward references in named backreferences (if you see what I mean). - -15. In UTF-8 mode, with the PCRE_DOTALL option set, a quantified dot in the - pattern could run off the end of the subject. For example, the pattern - "(?s)(.{1,5})"8 did this with the subject "ab". - -16. If PCRE_DOTALL or PCRE_MULTILINE were set, pcre_dfa_exec() behaved as if - PCRE_CASELESS was set when matching characters that were quantified with ? - or *. - -17. A character class other than a single negated character that had a minimum - but no maximum quantifier - for example [ab]{6,} - was not handled - correctly by pce_dfa_exec(). It would match only one character. - -18. A valid (though odd) pattern that looked like a POSIX character - class but used an invalid character after [ (for example [[,abc,]]) caused - pcre_compile() to give the error "Failed: internal error: code overflow" or - in some cases to crash with a glibc free() error. This could even happen if - the pattern terminated after [[ but there just happened to be a sequence of - letters, a binary zero, and a closing ] in the memory that followed. - -19. Perl's treatment of octal escapes in the range \400 to \777 has changed - over the years. Originally (before any Unicode support), just the bottom 8 - bits were taken. Thus, for example, \500 really meant \100. Nowadays the - output from "man perlunicode" includes this: - - The regular expression compiler produces polymorphic opcodes. That - is, the pattern adapts to the data and automatically switches to - the Unicode character scheme when presented with Unicode data--or - instead uses a traditional byte scheme when presented with byte - data. - - Sadly, a wide octal escape does not cause a switch, and in a string with - no other multibyte characters, these octal escapes are treated as before. - Thus, in Perl, the pattern /\500/ actually matches \100 but the pattern - /\500|\x{1ff}/ matches \500 or \777 because the whole thing is treated as a - Unicode string. - - I have not perpetrated such confusion in PCRE. Up till now, it took just - the bottom 8 bits, as in old Perl. I have now made octal escapes with - values greater than \377 illegal in non-UTF-8 mode. In UTF-8 mode they - translate to the appropriate multibyte character. - -29. Applied some refactoring to reduce the number of warnings from Microsoft - and Borland compilers. This has included removing the fudge introduced - seven years ago for the OS/2 compiler (see 2.02/2 below) because it caused - a warning about an unused variable. - -21. PCRE has not included VT (character 0x0b) in the set of whitespace - characters since release 4.0, because Perl (from release 5.004) does not. - [Or at least, is documented not to: some releases seem to be in conflict - with the documentation.] However, when a pattern was studied with - pcre_study() and all its branches started with \s, PCRE still included VT - as a possible starting character. Of course, this did no harm; it just - caused an unnecessary match attempt. - -22. Removed a now-redundant internal flag bit that recorded the fact that case - dependency changed within the pattern. This was once needed for "required - byte" processing, but is no longer used. This recovers a now-scarce options - bit. Also moved the least significant internal flag bit to the most- - significant bit of the word, which was not previously used (hangover from - the days when it was an int rather than a uint) to free up another bit for - the future. - -23. Added support for CRLF line endings as well as CR and LF. As well as the - default being selectable at build time, it can now be changed at runtime - via the PCRE_NEWLINE_xxx flags. There are now options for pcregrep to - specify that it is scanning data with non-default line endings. - -24. Changed the definition of CXXLINK to make it agree with the definition of - LINK in the Makefile, by replacing LDFLAGS to CXXFLAGS. - -25. Applied Ian Taylor's patches to avoid using another stack frame for tail - recursions. This makes a big different to stack usage for some patterns. - -26. If a subpattern containing a named recursion or subroutine reference such - as (?P>B) was quantified, for example (xxx(?P>B)){3}, the calculation of - the space required for the compiled pattern went wrong and gave too small a - value. Depending on the environment, this could lead to "Failed: internal - error: code overflow at offset 49" or "glibc detected double free or - corruption" errors. - -27. Applied patches from Google (a) to support the new newline modes and (b) to - advance over multibyte UTF-8 characters in GlobalReplace. - -28. Change free() to pcre_free() in pcredemo.c. Apparently this makes a - difference for some implementation of PCRE in some Windows version. - -29. Added some extra testing facilities to pcretest: - - \q in a data line sets the "match limit" value - \Q in a data line sets the "match recursion limt" value - -S sets the stack size, where is in megabytes - - The -S option isn't available for Windows. - - -Version 6.6 06-Feb-06 ---------------------- - - 1. Change 16(a) for 6.5 broke things, because PCRE_DATA_SCOPE was not defined - in pcreposix.h. I have copied the definition from pcre.h. - - 2. Change 25 for 6.5 broke compilation in a build directory out-of-tree - because pcre.h is no longer a built file. - - 3. Added Jeff Friedl's additional debugging patches to pcregrep. These are - not normally included in the compiled code. - - -Version 6.5 01-Feb-06 ---------------------- - - 1. When using the partial match feature with pcre_dfa_exec(), it was not - anchoring the second and subsequent partial matches at the new starting - point. This could lead to incorrect results. For example, with the pattern - /1234/, partially matching against "123" and then "a4" gave a match. - - 2. Changes to pcregrep: - - (a) All non-match returns from pcre_exec() were being treated as failures - to match the line. Now, unless the error is PCRE_ERROR_NOMATCH, an - error message is output. Some extra information is given for the - PCRE_ERROR_MATCHLIMIT and PCRE_ERROR_RECURSIONLIMIT errors, which are - probably the only errors that are likely to be caused by users (by - specifying a regex that has nested indefinite repeats, for instance). - If there are more than 20 of these errors, pcregrep is abandoned. - - (b) A binary zero was treated as data while matching, but terminated the - output line if it was written out. This has been fixed: binary zeroes - are now no different to any other data bytes. - - (c) Whichever of the LC_ALL or LC_CTYPE environment variables is set is - used to set a locale for matching. The --locale=xxxx long option has - been added (no short equivalent) to specify a locale explicitly on the - pcregrep command, overriding the environment variables. - - (d) When -B was used with -n, some line numbers in the output were one less - than they should have been. - - (e) Added the -o (--only-matching) option. - - (f) If -A or -C was used with -c (count only), some lines of context were - accidentally printed for the final match. - - (g) Added the -H (--with-filename) option. - - (h) The combination of options -rh failed to suppress file names for files - that were found from directory arguments. - - (i) Added the -D (--devices) and -d (--directories) options. - - (j) Added the -F (--fixed-strings) option. - - (k) Allow "-" to be used as a file name for -f as well as for a data file. - - (l) Added the --colo(u)r option. - - (m) Added Jeffrey Friedl's -S testing option, but within #ifdefs so that it - is not present by default. - - 3. A nasty bug was discovered in the handling of recursive patterns, that is, - items such as (?R) or (?1), when the recursion could match a number of - alternatives. If it matched one of the alternatives, but subsequently, - outside the recursion, there was a failure, the code tried to back up into - the recursion. However, because of the way PCRE is implemented, this is not - possible, and the result was an incorrect result from the match. - - In order to prevent this happening, the specification of recursion has - been changed so that all such subpatterns are automatically treated as - atomic groups. Thus, for example, (?R) is treated as if it were (?>(?R)). - - 4. I had overlooked the fact that, in some locales, there are characters for - which isalpha() is true but neither isupper() nor islower() are true. In - the fr_FR locale, for instance, the \xAA and \xBA characters (ordmasculine - and ordfeminine) are like this. This affected the treatment of \w and \W - when they appeared in character classes, but not when they appeared outside - a character class. The bit map for "word" characters is now created - separately from the results of isalnum() instead of just taking it from the - upper, lower, and digit maps. (Plus the underscore character, of course.) - - 5. The above bug also affected the handling of POSIX character classes such as - [[:alpha:]] and [[:alnum:]]. These do not have their own bit maps in PCRE's - permanent tables. Instead, the bit maps for such a class were previously - created as the appropriate unions of the upper, lower, and digit bitmaps. - Now they are created by subtraction from the [[:word:]] class, which has - its own bitmap. - - 6. The [[:blank:]] character class matches horizontal, but not vertical space. - It is created by subtracting the vertical space characters (\x09, \x0a, - \x0b, \x0c) from the [[:space:]] bitmap. Previously, however, the - subtraction was done in the overall bitmap for a character class, meaning - that a class such as [\x0c[:blank:]] was incorrect because \x0c would not - be recognized. This bug has been fixed. - - 7. Patches from the folks at Google: - - (a) pcrecpp.cc: "to handle a corner case that may or may not happen in - real life, but is still worth protecting against". - - (b) pcrecpp.cc: "corrects a bug when negative radixes are used with - regular expressions". - - (c) pcre_scanner.cc: avoid use of std::count() because not all systems - have it. - - (d) Split off pcrecpparg.h from pcrecpp.h and had the former built by - "configure" and the latter not, in order to fix a problem somebody had - with compiling the Arg class on HP-UX. - - (e) Improve the error-handling of the C++ wrapper a little bit. - - (f) New tests for checking recursion limiting. - - 8. The pcre_memmove() function, which is used only if the environment does not - have a standard memmove() function (and is therefore rarely compiled), - contained two bugs: (a) use of int instead of size_t, and (b) it was not - returning a result (though PCRE never actually uses the result). - - 9. In the POSIX regexec() interface, if nmatch is specified as a ridiculously - large number - greater than INT_MAX/(3*sizeof(int)) - REG_ESPACE is - returned instead of calling malloc() with an overflowing number that would - most likely cause subsequent chaos. - -10. The debugging option of pcretest was not showing the NO_AUTO_CAPTURE flag. - -11. The POSIX flag REG_NOSUB is now supported. When a pattern that was compiled - with this option is matched, the nmatch and pmatch options of regexec() are - ignored. - -12. Added REG_UTF8 to the POSIX interface. This is not defined by POSIX, but is - provided in case anyone wants to the the POSIX interface with UTF-8 - strings. - -13. Added CXXLDFLAGS to the Makefile parameters to provide settings only on the - C++ linking (needed for some HP-UX environments). - -14. Avoid compiler warnings in get_ucpname() when compiled without UCP support - (unused parameter) and in the pcre_printint() function (omitted "default" - switch label when the default is to do nothing). - -15. Added some code to make it possible, when PCRE is compiled as a C++ - library, to replace subject pointers for pcre_exec() with a smart pointer - class, thus making it possible to process discontinuous strings. - -16. The two macros PCRE_EXPORT and PCRE_DATA_SCOPE are confusing, and perform - much the same function. They were added by different people who were trying - to make PCRE easy to compile on non-Unix systems. It has been suggested - that PCRE_EXPORT be abolished now that there is more automatic apparatus - for compiling on Windows systems. I have therefore replaced it with - PCRE_DATA_SCOPE. This is set automatically for Windows; if not set it - defaults to "extern" for C or "extern C" for C++, which works fine on - Unix-like systems. It is now possible to override the value of PCRE_DATA_ - SCOPE with something explicit in config.h. In addition: - - (a) pcreposix.h still had just "extern" instead of either of these macros; - I have replaced it with PCRE_DATA_SCOPE. - - (b) Functions such as _pcre_xclass(), which are internal to the library, - but external in the C sense, all had PCRE_EXPORT in their definitions. - This is apparently wrong for the Windows case, so I have removed it. - (It makes no difference on Unix-like systems.) - -17. Added a new limit, MATCH_LIMIT_RECURSION, which limits the depth of nesting - of recursive calls to match(). This is different to MATCH_LIMIT because - that limits the total number of calls to match(), not all of which increase - the depth of recursion. Limiting the recursion depth limits the amount of - stack (or heap if NO_RECURSE is set) that is used. The default can be set - when PCRE is compiled, and changed at run time. A patch from Google adds - this functionality to the C++ interface. - -18. Changes to the handling of Unicode character properties: - - (a) Updated the table to Unicode 4.1.0. - - (b) Recognize characters that are not in the table as "Cn" (undefined). - - (c) I revised the way the table is implemented to a much improved format - which includes recognition of ranges. It now supports the ranges that - are defined in UnicodeData.txt, and it also amalgamates other - characters into ranges. This has reduced the number of entries in the - table from around 16,000 to around 3,000, thus reducing its size - considerably. I realized I did not need to use a tree structure after - all - a binary chop search is just as efficient. Having reduced the - number of entries, I extended their size from 6 bytes to 8 bytes to - allow for more data. - - (d) Added support for Unicode script names via properties such as \p{Han}. - -19. In UTF-8 mode, a backslash followed by a non-Ascii character was not - matching that character. - -20. When matching a repeated Unicode property with a minimum greater than zero, - (for example \pL{2,}), PCRE could look past the end of the subject if it - reached it while seeking the minimum number of characters. This could - happen only if some of the characters were more than one byte long, because - there is a check for at least the minimum number of bytes. - -21. Refactored the implementation of \p and \P so as to be more general, to - allow for more different types of property in future. This has changed the - compiled form incompatibly. Anybody with saved compiled patterns that use - \p or \P will have to recompile them. - -22. Added "Any" and "L&" to the supported property types. - -23. Recognize \x{...} as a code point specifier, even when not in UTF-8 mode, - but give a compile time error if the value is greater than 0xff. - -24. The man pages for pcrepartial, pcreprecompile, and pcre_compile2 were - accidentally not being installed or uninstalled. - -25. The pcre.h file was built from pcre.h.in, but the only changes that were - made were to insert the current release number. This seemed silly, because - it made things harder for people building PCRE on systems that don't run - "configure". I have turned pcre.h into a distributed file, no longer built - by "configure", with the version identification directly included. There is - no longer a pcre.h.in file. - - However, this change necessitated a change to the pcre-config script as - well. It is built from pcre-config.in, and one of the substitutions was the - release number. I have updated configure.ac so that ./configure now finds - the release number by grepping pcre.h. - -26. Added the ability to run the tests under valgrind. - - -Version 6.4 05-Sep-05 ---------------------- - - 1. Change 6.0/10/(l) to pcregrep introduced a bug that caused separator lines - "--" to be printed when multiple files were scanned, even when none of the - -A, -B, or -C options were used. This is not compatible with Gnu grep, so I - consider it to be a bug, and have restored the previous behaviour. - - 2. A couple of code tidies to get rid of compiler warnings. - - 3. The pcretest program used to cheat by referring to symbols in the library - whose names begin with _pcre_. These are internal symbols that are not - really supposed to be visible externally, and in some environments it is - possible to suppress them. The cheating is now confined to including - certain files from the library's source, which is a bit cleaner. - - 4. Renamed pcre.in as pcre.h.in to go with pcrecpp.h.in; it also makes the - file's purpose clearer. - - 5. Reorganized pcre_ucp_findchar(). - - -Version 6.3 15-Aug-05 ---------------------- - - 1. The file libpcre.pc.in did not have general read permission in the tarball. - - 2. There were some problems when building without C++ support: - - (a) If C++ support was not built, "make install" and "make test" still - tried to test it. - - (b) There were problems when the value of CXX was explicitly set. Some - changes have been made to try to fix these, and ... - - (c) --disable-cpp can now be used to explicitly disable C++ support. - - (d) The use of @CPP_OBJ@ directly caused a blank line preceded by a - backslash in a target when C++ was disabled. This confuses some - versions of "make", apparently. Using an intermediate variable solves - this. (Same for CPP_LOBJ.) - - 3. $(LINK_FOR_BUILD) now includes $(CFLAGS_FOR_BUILD) and $(LINK) - (non-Windows) now includes $(CFLAGS) because these flags are sometimes - necessary on certain architectures. - - 4. Added a setting of -export-symbols-regex to the link command to remove - those symbols that are exported in the C sense, but actually are local - within the library, and not documented. Their names all begin with - "_pcre_". This is not a perfect job, because (a) we have to except some - symbols that pcretest ("illegally") uses, and (b) the facility isn't always - available (and never for static libraries). I have made a note to try to - find a way round (a) in the future. - - -Version 6.2 01-Aug-05 ---------------------- - - 1. There was no test for integer overflow of quantifier values. A construction - such as {1111111111111111} would give undefined results. What is worse, if - a minimum quantifier for a parenthesized subpattern overflowed and became - negative, the calculation of the memory size went wrong. This could have - led to memory overwriting. - - 2. Building PCRE using VPATH was broken. Hopefully it is now fixed. - - 3. Added "b" to the 2nd argument of fopen() in dftables.c, for non-Unix-like - operating environments where this matters. - - 4. Applied Giuseppe Maxia's patch to add additional features for controlling - PCRE options from within the C++ wrapper. - - 5. Named capturing subpatterns were not being correctly counted when a pattern - was compiled. This caused two problems: (a) If there were more than 100 - such subpatterns, the calculation of the memory needed for the whole - compiled pattern went wrong, leading to an overflow error. (b) Numerical - back references of the form \12, where the number was greater than 9, were - not recognized as back references, even though there were sufficient - previous subpatterns. - - 6. Two minor patches to pcrecpp.cc in order to allow it to compile on older - versions of gcc, e.g. 2.95.4. - - -Version 6.1 21-Jun-05 ---------------------- - - 1. There was one reference to the variable "posix" in pcretest.c that was not - surrounded by "#if !defined NOPOSIX". - - 2. Make it possible to compile pcretest without DFA support, UTF8 support, or - the cross-check on the old pcre_info() function, for the benefit of the - cut-down version of PCRE that is currently imported into Exim. - - 3. A (silly) pattern starting with (?i)(?-i) caused an internal space - allocation error. I've done the easy fix, which wastes 2 bytes for sensible - patterns that start (?i) but I don't think that matters. The use of (?i) is - just an example; this all applies to the other options as well. - - 4. Since libtool seems to echo the compile commands it is issuing, the output - from "make" can be reduced a bit by putting "@" in front of each libtool - compile command. - - 5. Patch from the folks at Google for configure.in to be a bit more thorough - in checking for a suitable C++ installation before trying to compile the - C++ stuff. This should fix a reported problem when a compiler was present, - but no suitable headers. - - 6. The man pages all had just "PCRE" as their title. I have changed them to - be the relevant file name. I have also arranged that these names are - retained in the file doc/pcre.txt, which is a concatenation in text format - of all the man pages except the little individual ones for each function. - - 7. The NON-UNIX-USE file had not been updated for the different set of source - files that come with release 6. I also added a few comments about the C++ - wrapper. - - -Version 6.0 07-Jun-05 ---------------------- - - 1. Some minor internal re-organization to help with my DFA experiments. - - 2. Some missing #ifdef SUPPORT_UCP conditionals in pcretest and printint that - didn't matter for the library itself when fully configured, but did matter - when compiling without UCP support, or within Exim, where the ucp files are - not imported. - - 3. Refactoring of the library code to split up the various functions into - different source modules. The addition of the new DFA matching code (see - below) to a single monolithic source would have made it really too - unwieldy, quite apart from causing all the code to be include in a - statically linked application, when only some functions are used. This is - relevant even without the DFA addition now that patterns can be compiled in - one application and matched in another. - - The downside of splitting up is that there have to be some external - functions and data tables that are used internally in different modules of - the library but which are not part of the API. These have all had their - names changed to start with "_pcre_" so that they are unlikely to clash - with other external names. - - 4. Added an alternate matching function, pcre_dfa_exec(), which matches using - a different (DFA) algorithm. Although it is slower than the original - function, it does have some advantages for certain types of matching - problem. - - 5. Upgrades to pcretest in order to test the features of pcre_dfa_exec(), - including restarting after a partial match. - - 6. A patch for pcregrep that defines INVALID_FILE_ATTRIBUTES if it is not - defined when compiling for Windows was sent to me. I have put it into the - code, though I have no means of testing or verifying it. - - 7. Added the pcre_refcount() auxiliary function. - - 8. Added the PCRE_FIRSTLINE option. This constrains an unanchored pattern to - match before or at the first newline in the subject string. In pcretest, - the /f option on a pattern can be used to set this. - - 9. A repeated \w when used in UTF-8 mode with characters greater than 256 - would behave wrongly. This has been present in PCRE since release 4.0. - -10. A number of changes to the pcregrep command: - - (a) Refactored how -x works; insert ^(...)$ instead of setting - PCRE_ANCHORED and checking the length, in preparation for adding - something similar for -w. - - (b) Added the -w (match as a word) option. - - (c) Refactored the way lines are read and buffered so as to have more - than one at a time available. - - (d) Implemented a pcregrep test script. - - (e) Added the -M (multiline match) option. This allows patterns to match - over several lines of the subject. The buffering ensures that at least - 8K, or the rest of the document (whichever is the shorter) is available - for matching (and similarly the previous 8K for lookbehind assertions). - - (f) Changed the --help output so that it now says - - -w, --word-regex(p) - - instead of two lines, one with "regex" and the other with "regexp" - because that confused at least one person since the short forms are the - same. (This required a bit of code, as the output is generated - automatically from a table. It wasn't just a text change.) - - (g) -- can be used to terminate pcregrep options if the next thing isn't an - option but starts with a hyphen. Could be a pattern or a path name - starting with a hyphen, for instance. - - (h) "-" can be given as a file name to represent stdin. - - (i) When file names are being printed, "(standard input)" is used for - the standard input, for compatibility with GNU grep. Previously - "" was used. - - (j) The option --label=xxx can be used to supply a name to be used for - stdin when file names are being printed. There is no short form. - - (k) Re-factored the options decoding logic because we are going to add - two more options that take data. Such options can now be given in four - different ways, e.g. "-fname", "-f name", "--file=name", "--file name". - - (l) Added the -A, -B, and -C options for requesting that lines of context - around matches be printed. - - (m) Added the -L option to print the names of files that do not contain - any matching lines, that is, the complement of -l. - - (n) The return code is 2 if any file cannot be opened, but pcregrep does - continue to scan other files. - - (o) The -s option was incorrectly implemented. For compatibility with other - greps, it now suppresses the error message for a non-existent or non- - accessible file (but not the return code). There is a new option called - -q that suppresses the output of matching lines, which was what -s was - previously doing. - - (p) Added --include and --exclude options to specify files for inclusion - and exclusion when recursing. - -11. The Makefile was not using the Autoconf-supported LDFLAGS macro properly. - Hopefully, it now does. - -12. Missing cast in pcre_study(). - -13. Added an "uninstall" target to the makefile. - -14. Replaced "extern" in the function prototypes in Makefile.in with - "PCRE_DATA_SCOPE", which defaults to 'extern' or 'extern "C"' in the Unix - world, but is set differently for Windows. - -15. Added a second compiling function called pcre_compile2(). The only - difference is that it has an extra argument, which is a pointer to an - integer error code. When there is a compile-time failure, this is set - non-zero, in addition to the error test pointer being set to point to an - error message. The new argument may be NULL if no error number is required - (but then you may as well call pcre_compile(), which is now just a - wrapper). This facility is provided because some applications need a - numeric error indication, but it has also enabled me to tidy up the way - compile-time errors are handled in the POSIX wrapper. - -16. Added VPATH=.libs to the makefile; this should help when building with one - prefix path and installing with another. (Or so I'm told by someone who - knows more about this stuff than I do.) - -17. Added a new option, REG_DOTALL, to the POSIX function regcomp(). This - passes PCRE_DOTALL to the pcre_compile() function, making the "." character - match everything, including newlines. This is not POSIX-compatible, but - somebody wanted the feature. From pcretest it can be activated by using - both the P and the s flags. - -18. AC_PROG_LIBTOOL appeared twice in Makefile.in. Removed one. - -19. libpcre.pc was being incorrectly installed as executable. - -20. A couple of places in pcretest check for end-of-line by looking for '\n'; - it now also looks for '\r' so that it will work unmodified on Windows. - -21. Added Google's contributed C++ wrapper to the distribution. - -22. Added some untidy missing memory free() calls in pcretest, to keep - Electric Fence happy when testing. - - - -Version 5.0 13-Sep-04 ---------------------- - - 1. Internal change: literal characters are no longer packed up into items - containing multiple characters in a single byte-string. Each character - is now matched using a separate opcode. However, there may be more than one - byte in the character in UTF-8 mode. - - 2. The pcre_callout_block structure has two new fields: pattern_position and - next_item_length. These contain the offset in the pattern to the next match - item, and its length, respectively. - - 3. The PCRE_AUTO_CALLOUT option for pcre_compile() requests the automatic - insertion of callouts before each pattern item. Added the /C option to - pcretest to make use of this. - - 4. On the advice of a Windows user, the lines - - #if defined(_WIN32) || defined(WIN32) - _setmode( _fileno( stdout ), 0x8000 ); - #endif /* defined(_WIN32) || defined(WIN32) */ - - have been added to the source of pcretest. This apparently does useful - magic in relation to line terminators. - - 5. Changed "r" and "w" in the calls to fopen() in pcretest to "rb" and "wb" - for the benefit of those environments where the "b" makes a difference. - - 6. The icc compiler has the same options as gcc, but "configure" doesn't seem - to know about it. I have put a hack into configure.in that adds in code - to set GCC=yes if CC=icc. This seems to end up at a point in the - generated configure script that is early enough to affect the setting of - compiler options, which is what is needed, but I have no means of testing - whether it really works. (The user who reported this had patched the - generated configure script, which of course I cannot do.) - - LATER: After change 22 below (new libtool files), the configure script - seems to know about icc (and also ecc). Therefore, I have commented out - this hack in configure.in. - - 7. Added support for pkg-config (2 patches were sent in). - - 8. Negated POSIX character classes that used a combination of internal tables - were completely broken. These were [[:^alpha:]], [[:^alnum:]], and - [[:^ascii]]. Typically, they would match almost any characters. The other - POSIX classes were not broken in this way. - - 9. Matching the pattern "\b.*?" against "ab cd", starting at offset 1, failed - to find the match, as PCRE was deluded into thinking that the match had to - start at the start point or following a newline. The same bug applied to - patterns with negative forward assertions or any backward assertions - preceding ".*" at the start, unless the pattern required a fixed first - character. This was a failing pattern: "(?!.bcd).*". The bug is now fixed. - -10. In UTF-8 mode, when moving forwards in the subject after a failed match - starting at the last subject character, bytes beyond the end of the subject - string were read. - -11. Renamed the variable "class" as "classbits" to make life easier for C++ - users. (Previously there was a macro definition, but it apparently wasn't - enough.) - -12. Added the new field "tables" to the extra data so that tables can be passed - in at exec time, or the internal tables can be re-selected. This allows - a compiled regex to be saved and re-used at a later time by a different - program that might have everything at different addresses. - -13. Modified the pcre-config script so that, when run on Solaris, it shows a - -R library as well as a -L library. - -14. The debugging options of pcretest (-d on the command line or D on a - pattern) showed incorrect output for anything following an extended class - that contained multibyte characters and which was followed by a quantifier. - -15. Added optional support for general category Unicode character properties - via the \p, \P, and \X escapes. Unicode property support implies UTF-8 - support. It adds about 90K to the size of the library. The meanings of the - inbuilt class escapes such as \d and \s have NOT been changed. - -16. Updated pcredemo.c to include calls to free() to release the memory for the - compiled pattern. - -17. The generated file chartables.c was being created in the source directory - instead of in the building directory. This caused the build to fail if the - source directory was different from the building directory, and was - read-only. - -18. Added some sample Win commands from Mark Tetrode into the NON-UNIX-USE - file. No doubt somebody will tell me if they don't make sense... Also added - Dan Mooney's comments about building on OpenVMS. - -19. Added support for partial matching via the PCRE_PARTIAL option for - pcre_exec() and the \P data escape in pcretest. - -20. Extended pcretest with 3 new pattern features: - - (i) A pattern option of the form ">rest-of-line" causes pcretest to - write the compiled pattern to the file whose name is "rest-of-line". - This is a straight binary dump of the data, with the saved pointer to - the character tables forced to be NULL. The study data, if any, is - written too. After writing, pcretest reads a new pattern. - - (ii) If, instead of a pattern, ": new target - : new target - : use native compiler - : use native linker - : handle Windows platform correctly - : ditto - : ditto - copy DLL to top builddir before testing - - As part of these changes, -no-undefined was removed again. This was reported - to give trouble on HP-UX 11.0, so getting rid of it seems like a good idea - in any case. - -3. Some tidies to get rid of compiler warnings: - - . In the match_data structure, match_limit was an unsigned long int, whereas - match_call_count was an int. I've made them both unsigned long ints. - - . In pcretest the fact that a const uschar * doesn't automatically cast to - a void * provoked a warning. - - . Turning on some more compiler warnings threw up some "shadow" variables - and a few more missing casts. - -4. If PCRE was complied with UTF-8 support, but called without the PCRE_UTF8 - option, a class that contained a single character with a value between 128 - and 255 (e.g. /[\xFF]/) caused PCRE to crash. - -5. If PCRE was compiled with UTF-8 support, but called without the PCRE_UTF8 - option, a class that contained several characters, but with at least one - whose value was between 128 and 255 caused PCRE to crash. - - -Version 4.1 12-Mar-03 ---------------------- - -1. Compiling with gcc -pedantic found a couple of places where casts were -needed, and a string in dftables.c that was longer than standard compilers are -required to support. - -2. Compiling with Sun's compiler found a few more places where the code could -be tidied up in order to avoid warnings. - -3. The variables for cross-compiling were called HOST_CC and HOST_CFLAGS; the -first of these names is deprecated in the latest Autoconf in favour of the name -CC_FOR_BUILD, because "host" is typically used to mean the system on which the -compiled code will be run. I can't find a reference for HOST_CFLAGS, but by -analogy I have changed it to CFLAGS_FOR_BUILD. - -4. Added -no-undefined to the linking command in the Makefile, because this is -apparently helpful for Windows. To make it work, also added "-L. -lpcre" to the -linking step for the pcreposix library. - -5. PCRE was failing to diagnose the case of two named groups with the same -name. - -6. A problem with one of PCRE's optimizations was discovered. PCRE remembers a -literal character that is needed in the subject for a match, and scans along to -ensure that it is present before embarking on the full matching process. This -saves time in cases of nested unlimited repeats that are never going to match. -Problem: the scan can take a lot of time if the subject is very long (e.g. -megabytes), thus penalizing straightforward matches. It is now done only if the -amount of subject to be scanned is less than 1000 bytes. - -7. A lesser problem with the same optimization is that it was recording the -first character of an anchored pattern as "needed", thus provoking a search -right along the subject, even when the first match of the pattern was going to -fail. The "needed" character is now not set for anchored patterns, unless it -follows something in the pattern that is of non-fixed length. Thus, it still -fulfils its original purpose of finding quick non-matches in cases of nested -unlimited repeats, but isn't used for simple anchored patterns such as /^abc/. - - -Version 4.0 17-Feb-03 ---------------------- - -1. If a comment in an extended regex that started immediately after a meta-item -extended to the end of string, PCRE compiled incorrect data. This could lead to -all kinds of weird effects. Example: /#/ was bad; /()#/ was bad; /a#/ was not. - -2. Moved to autoconf 2.53 and libtool 1.4.2. - -3. Perl 5.8 no longer needs "use utf8" for doing UTF-8 things. Consequently, -the special perltest8 script is no longer needed - all the tests can be run -from a single perltest script. - -4. From 5.004, Perl has not included the VT character (0x0b) in the set defined -by \s. It has now been removed in PCRE. This means it isn't recognized as -whitespace in /x regexes too, which is the same as Perl. Note that the POSIX -class [:space:] *does* include VT, thereby creating a mess. - -5. Added the class [:blank:] (a GNU extension from Perl 5.8) to match only -space and tab. - -6. Perl 5.005 was a long time ago. It's time to amalgamate the tests that use -its new features into the main test script, reducing the number of scripts. - -7. Perl 5.8 has changed the meaning of patterns like /a(?i)b/. Earlier versions -were backward compatible, and made the (?i) apply to the whole pattern, as if -/i were given. Now it behaves more logically, and applies the option setting -only to what follows. PCRE has been changed to follow suit. However, if it -finds options settings right at the start of the pattern, it extracts them into -the global options, as before. Thus, they show up in the info data. - -8. Added support for the \Q...\E escape sequence. Characters in between are -treated as literals. This is slightly different from Perl in that $ and @ are -also handled as literals inside the quotes. In Perl, they will cause variable -interpolation. Note the following examples: - - Pattern PCRE matches Perl matches - - \Qabc$xyz\E abc$xyz abc followed by the contents of $xyz - \Qabc\$xyz\E abc\$xyz abc\$xyz - \Qabc\E\$\Qxyz\E abc$xyz abc$xyz - -For compatibility with Perl, \Q...\E sequences are recognized inside character -classes as well as outside them. - -9. Re-organized 3 code statements in pcretest to avoid "overflow in -floating-point constant arithmetic" warnings from a Microsoft compiler. Added a -(size_t) cast to one statement in pcretest and one in pcreposix to avoid -signed/unsigned warnings. - -10. SunOS4 doesn't have strtoul(). This was used only for unpicking the -o -option for pcretest, so I've replaced it by a simple function that does just -that job. - -11. pcregrep was ending with code 0 instead of 2 for the commands "pcregrep" or -"pcregrep -". - -12. Added "possessive quantifiers" ?+, *+, ++, and {,}+ which come from Sun's -Java package. This provides some syntactic sugar for simple cases of what my -documentation calls "once-only subpatterns". A pattern such as x*+ is the same -as (?>x*). In other words, if what is inside (?>...) is just a single repeated -item, you can use this simplified notation. Note that only makes sense with -greedy quantifiers. Consequently, the use of the possessive quantifier forces -greediness, whatever the setting of the PCRE_UNGREEDY option. - -13. A change of greediness default within a pattern was not taking effect at -the current level for patterns like /(b+(?U)a+)/. It did apply to parenthesized -subpatterns that followed. Patterns like /b+(?U)a+/ worked because the option -was abstracted outside. - -14. PCRE now supports the \G assertion. It is true when the current matching -position is at the start point of the match. This differs from \A when the -starting offset is non-zero. Used with the /g option of pcretest (or similar -code), it works in the same way as it does for Perl's /g option. If all -alternatives of a regex begin with \G, the expression is anchored to the start -match position, and the "anchored" flag is set in the compiled expression. - -15. Some bugs concerning the handling of certain option changes within patterns -have been fixed. These applied to options other than (?ims). For example, -"a(?x: b c )d" did not match "XabcdY" but did match "Xa b c dY". It should have -been the other way round. Some of this was related to change 7 above. - -16. PCRE now gives errors for /[.x.]/ and /[=x=]/ as unsupported POSIX -features, as Perl does. Previously, PCRE gave the warnings only for /[[.x.]]/ -and /[[=x=]]/. PCRE now also gives an error for /[:name:]/ because it supports -POSIX classes only within a class (e.g. /[[:alpha:]]/). - -17. Added support for Perl's \C escape. This matches one byte, even in UTF8 -mode. Unlike ".", it always matches newline, whatever the setting of -PCRE_DOTALL. However, PCRE does not permit \C to appear in lookbehind -assertions. Perl allows it, but it doesn't (in general) work because it can't -calculate the length of the lookbehind. At least, that's the case for Perl -5.8.0 - I've been told they are going to document that it doesn't work in -future. - -18. Added an error diagnosis for escapes that PCRE does not support: these are -\L, \l, \N, \P, \p, \U, \u, and \X. - -19. Although correctly diagnosing a missing ']' in a character class, PCRE was -reading past the end of the pattern in cases such as /[abcd/. - -20. PCRE was getting more memory than necessary for patterns with classes that -contained both POSIX named classes and other characters, e.g. /[[:space:]abc/. - -21. Added some code, conditional on #ifdef VPCOMPAT, to make life easier for -compiling PCRE for use with Virtual Pascal. - -22. Small fix to the Makefile to make it work properly if the build is done -outside the source tree. - -23. Added a new extension: a condition to go with recursion. If a conditional -subpattern starts with (?(R) the "true" branch is used if recursion has -happened, whereas the "false" branch is used only at the top level. - -24. When there was a very long string of literal characters (over 255 bytes -without UTF support, over 250 bytes with UTF support), the computation of how -much memory was required could be incorrect, leading to segfaults or other -strange effects. - -25. PCRE was incorrectly assuming anchoring (either to start of subject or to -start of line for a non-DOTALL pattern) when a pattern started with (.*) and -there was a subsequent back reference to those brackets. This meant that, for -example, /(.*)\d+\1/ failed to match "abc123bc". Unfortunately, it isn't -possible to check for precisely this case. All we can do is abandon the -optimization if .* occurs inside capturing brackets when there are any back -references whatsoever. (See below for a better fix that came later.) - -26. The handling of the optimization for finding the first character of a -non-anchored pattern, and for finding a character that is required later in the -match were failing in some cases. This didn't break the matching; it just -failed to optimize when it could. The way this is done has been re-implemented. - -27. Fixed typo in error message for invalid (?R item (it said "(?p"). - -28. Added a new feature that provides some of the functionality that Perl -provides with (?{...}). The facility is termed a "callout". The way it is done -in PCRE is for the caller to provide an optional function, by setting -pcre_callout to its entry point. Like pcre_malloc and pcre_free, this is a -global variable. By default it is unset, which disables all calling out. To get -the function called, the regex must include (?C) at appropriate points. This -is, in fact, equivalent to (?C0), and any number <= 255 may be given with (?C). -This provides a means of identifying different callout points. When PCRE -reaches such a point in the regex, if pcre_callout has been set, the external -function is called. It is provided with data in a structure called -pcre_callout_block, which is defined in pcre.h. If the function returns 0, -matching continues; if it returns a non-zero value, the match at the current -point fails. However, backtracking will occur if possible. [This was changed -later and other features added - see item 49 below.] - -29. pcretest is upgraded to test the callout functionality. It provides a -callout function that displays information. By default, it shows the start of -the match and the current position in the text. There are some new data escapes -to vary what happens: - - \C+ in addition, show current contents of captured substrings - \C- do not supply a callout function - \C!n return 1 when callout number n is reached - \C!n!m return 1 when callout number n is reached for the mth time - -30. If pcregrep was called with the -l option and just a single file name, it -output "" if a match was found, instead of the file name. - -31. Improve the efficiency of the POSIX API to PCRE. If the number of capturing -slots is less than POSIX_MALLOC_THRESHOLD, use a block on the stack to pass to -pcre_exec(). This saves a malloc/free per call. The default value of -POSIX_MALLOC_THRESHOLD is 10; it can be changed by --with-posix-malloc-threshold -when configuring. - -32. The default maximum size of a compiled pattern is 64K. There have been a -few cases of people hitting this limit. The code now uses macros to handle the -storing of links as offsets within the compiled pattern. It defaults to 2-byte -links, but this can be changed to 3 or 4 bytes by --with-link-size when -configuring. Tests 2 and 5 work only with 2-byte links because they output -debugging information about compiled patterns. - -33. Internal code re-arrangements: - -(a) Moved the debugging function for printing out a compiled regex into - its own source file (printint.c) and used #include to pull it into - pcretest.c and, when DEBUG is defined, into pcre.c, instead of having two - separate copies. - -(b) Defined the list of op-code names for debugging as a macro in - internal.h so that it is next to the definition of the opcodes. - -(c) Defined a table of op-code lengths for simpler skipping along compiled - code. This is again a macro in internal.h so that it is next to the - definition of the opcodes. - -34. Added support for recursive calls to individual subpatterns, along the -lines of Robin Houston's patch (but implemented somewhat differently). - -35. Further mods to the Makefile to help Win32. Also, added code to pcregrep to -allow it to read and process whole directories in Win32. This code was -contributed by Lionel Fourquaux; it has not been tested by me. - -36. Added support for named subpatterns. The Python syntax (?P...) is -used to name a group. Names consist of alphanumerics and underscores, and must -be unique. Back references use the syntax (?P=name) and recursive calls use -(?P>name) which is a PCRE extension to the Python extension. Groups still have -numbers. The function pcre_fullinfo() can be used after compilation to extract -a name/number map. There are three relevant calls: - - PCRE_INFO_NAMEENTRYSIZE yields the size of each entry in the map - PCRE_INFO_NAMECOUNT yields the number of entries - PCRE_INFO_NAMETABLE yields a pointer to the map. - -The map is a vector of fixed-size entries. The size of each entry depends on -the length of the longest name used. The first two bytes of each entry are the -group number, most significant byte first. There follows the corresponding -name, zero terminated. The names are in alphabetical order. - -37. Make the maximum literal string in the compiled code 250 for the non-UTF-8 -case instead of 255. Making it the same both with and without UTF-8 support -means that the same test output works with both. - -38. There was a case of malloc(0) in the POSIX testing code in pcretest. Avoid -calling malloc() with a zero argument. - -39. Change 25 above had to resort to a heavy-handed test for the .* anchoring -optimization. I've improved things by keeping a bitmap of backreferences with -numbers 1-31 so that if .* occurs inside capturing brackets that are not in -fact referenced, the optimization can be applied. It is unlikely that a -relevant occurrence of .* (i.e. one which might indicate anchoring or forcing -the match to follow \n) will appear inside brackets with a number greater than -31, but if it does, any back reference > 31 suppresses the optimization. - -40. Added a new compile-time option PCRE_NO_AUTO_CAPTURE. This has the effect -of disabling numbered capturing parentheses. Any opening parenthesis that is -not followed by ? behaves as if it were followed by ?: but named parentheses -can still be used for capturing (and they will acquire numbers in the usual -way). - -41. Redesigned the return codes from the match() function into yes/no/error so -that errors can be passed back from deep inside the nested calls. A malloc -failure while inside a recursive subpattern call now causes the -PCRE_ERROR_NOMEMORY return instead of quietly going wrong. - -42. It is now possible to set a limit on the number of times the match() -function is called in a call to pcre_exec(). This facility makes it possible to -limit the amount of recursion and backtracking, though not in a directly -obvious way, because the match() function is used in a number of different -circumstances. The count starts from zero for each position in the subject -string (for non-anchored patterns). The default limit is, for compatibility, a -large number, namely 10 000 000. You can change this in two ways: - -(a) When configuring PCRE before making, you can use --with-match-limit=n - to set a default value for the compiled library. - -(b) For each call to pcre_exec(), you can pass a pcre_extra block in which - a different value is set. See 45 below. - -If the limit is exceeded, pcre_exec() returns PCRE_ERROR_MATCHLIMIT. - -43. Added a new function pcre_config(int, void *) to enable run-time extraction -of things that can be changed at compile time. The first argument specifies -what is wanted and the second points to where the information is to be placed. -The current list of available information is: - - PCRE_CONFIG_UTF8 - -The output is an integer that is set to one if UTF-8 support is available; -otherwise it is set to zero. - - PCRE_CONFIG_NEWLINE - -The output is an integer that it set to the value of the code that is used for -newline. It is either LF (10) or CR (13). - - PCRE_CONFIG_LINK_SIZE - -The output is an integer that contains the number of bytes used for internal -linkage in compiled expressions. The value is 2, 3, or 4. See item 32 above. - - PCRE_CONFIG_POSIX_MALLOC_THRESHOLD - -The output is an integer that contains the threshold above which the POSIX -interface uses malloc() for output vectors. See item 31 above. - - PCRE_CONFIG_MATCH_LIMIT - -The output is an unsigned integer that contains the default limit of the number -of match() calls in a pcre_exec() execution. See 42 above. - -44. pcretest has been upgraded by the addition of the -C option. This causes it -to extract all the available output from the new pcre_config() function, and to -output it. The program then exits immediately. - -45. A need has arisen to pass over additional data with calls to pcre_exec() in -order to support additional features. One way would have been to define -pcre_exec2() (for example) with extra arguments, but this would not have been -extensible, and would also have required all calls to the original function to -be mapped to the new one. Instead, I have chosen to extend the mechanism that -is used for passing in "extra" data from pcre_study(). - -The pcre_extra structure is now exposed and defined in pcre.h. It currently -contains the following fields: - - flags a bitmap indicating which of the following fields are set - study_data opaque data from pcre_study() - match_limit a way of specifying a limit on match() calls for a specific - call to pcre_exec() - callout_data data for callouts (see 49 below) - -The flag bits are also defined in pcre.h, and are - - PCRE_EXTRA_STUDY_DATA - PCRE_EXTRA_MATCH_LIMIT - PCRE_EXTRA_CALLOUT_DATA - -The pcre_study() function now returns one of these new pcre_extra blocks, with -the actual study data pointed to by the study_data field, and the -PCRE_EXTRA_STUDY_DATA flag set. This can be passed directly to pcre_exec() as -before. That is, this change is entirely upwards-compatible and requires no -change to existing code. - -If you want to pass in additional data to pcre_exec(), you can either place it -in a pcre_extra block provided by pcre_study(), or create your own pcre_extra -block. - -46. pcretest has been extended to test the PCRE_EXTRA_MATCH_LIMIT feature. If a -data string contains the escape sequence \M, pcretest calls pcre_exec() several -times with different match limits, until it finds the minimum value needed for -pcre_exec() to complete. The value is then output. This can be instructive; for -most simple matches the number is quite small, but for pathological cases it -gets very large very quickly. - -47. There's a new option for pcre_fullinfo() called PCRE_INFO_STUDYSIZE. It -returns the size of the data block pointed to by the study_data field in a -pcre_extra block, that is, the value that was passed as the argument to -pcre_malloc() when PCRE was getting memory in which to place the information -created by pcre_study(). The fourth argument should point to a size_t variable. -pcretest has been extended so that this information is shown after a successful -pcre_study() call when information about the compiled regex is being displayed. - -48. Cosmetic change to Makefile: there's no need to have / after $(DESTDIR) -because what follows is always an absolute path. (Later: it turns out that this -is more than cosmetic for MinGW, because it doesn't like empty path -components.) - -49. Some changes have been made to the callout feature (see 28 above): - -(i) A callout function now has three choices for what it returns: - - 0 => success, carry on matching - > 0 => failure at this point, but backtrack if possible - < 0 => serious error, return this value from pcre_exec() - - Negative values should normally be chosen from the set of PCRE_ERROR_xxx - values. In particular, returning PCRE_ERROR_NOMATCH forces a standard - "match failed" error. The error number PCRE_ERROR_CALLOUT is reserved for - use by callout functions. It will never be used by PCRE itself. - -(ii) The pcre_extra structure (see 45 above) has a void * field called - callout_data, with corresponding flag bit PCRE_EXTRA_CALLOUT_DATA. The - pcre_callout_block structure has a field of the same name. The contents of - the field passed in the pcre_extra structure are passed to the callout - function in the corresponding field in the callout block. This makes it - easier to use the same callout-containing regex from multiple threads. For - testing, the pcretest program has a new data escape - - \C*n pass the number n (may be negative) as callout_data - - If the callout function in pcretest receives a non-zero value as - callout_data, it returns that value. - -50. Makefile wasn't handling CFLAGS properly when compiling dftables. Also, -there were some redundant $(CFLAGS) in commands that are now specified as -$(LINK), which already includes $(CFLAGS). - -51. Extensions to UTF-8 support are listed below. These all apply when (a) PCRE -has been compiled with UTF-8 support *and* pcre_compile() has been compiled -with the PCRE_UTF8 flag. Patterns that are compiled without that flag assume -one-byte characters throughout. Note that case-insensitive matching applies -only to characters whose values are less than 256. PCRE doesn't support the -notion of cases for higher-valued characters. - -(i) A character class whose characters are all within 0-255 is handled as - a bit map, and the map is inverted for negative classes. Previously, a - character > 255 always failed to match such a class; however it should - match if the class was a negative one (e.g. [^ab]). This has been fixed. - -(ii) A negated character class with a single character < 255 is coded as - "not this character" (OP_NOT). This wasn't working properly when the test - character was multibyte, either singly or repeated. - -(iii) Repeats of multibyte characters are now handled correctly in UTF-8 - mode, for example: \x{100}{2,3}. - -(iv) The character escapes \b, \B, \d, \D, \s, \S, \w, and \W (either - singly or repeated) now correctly test multibyte characters. However, - PCRE doesn't recognize any characters with values greater than 255 as - digits, spaces, or word characters. Such characters always match \D, \S, - and \W, and never match \d, \s, or \w. - -(v) Classes may now contain characters and character ranges with values - greater than 255. For example: [ab\x{100}-\x{400}]. - -(vi) pcregrep now has a --utf-8 option (synonym -u) which makes it call - PCRE in UTF-8 mode. - -52. The info request value PCRE_INFO_FIRSTCHAR has been renamed -PCRE_INFO_FIRSTBYTE because it is a byte value. However, the old name is -retained for backwards compatibility. (Note that LASTLITERAL is also a byte -value.) - -53. The single man page has become too large. I have therefore split it up into -a number of separate man pages. These also give rise to individual HTML pages; -these are now put in a separate directory, and there is an index.html page that -lists them all. Some hyperlinking between the pages has been installed. - -54. Added convenience functions for handling named capturing parentheses. - -55. Unknown escapes inside character classes (e.g. [\M]) and escapes that -aren't interpreted therein (e.g. [\C]) are literals in Perl. This is now also -true in PCRE, except when the PCRE_EXTENDED option is set, in which case they -are faulted. - -56. Introduced HOST_CC and HOST_CFLAGS which can be set in the environment when -calling configure. These values are used when compiling the dftables.c program -which is run to generate the source of the default character tables. They -default to the values of CC and CFLAGS. If you are cross-compiling PCRE, -you will need to set these values. - -57. Updated the building process for Windows DLL, as provided by Fred Cox. - - -Version 3.9 02-Jan-02 ---------------------- - -1. A bit of extraneous text had somehow crept into the pcregrep documentation. - -2. If --disable-static was given, the building process failed when trying to -build pcretest and pcregrep. (For some reason it was using libtool to compile -them, which is not right, as they aren't part of the library.) - - -Version 3.8 18-Dec-01 ---------------------- - -1. The experimental UTF-8 code was completely screwed up. It was packing the -bytes in the wrong order. How dumb can you get? - - -Version 3.7 29-Oct-01 ---------------------- - -1. In updating pcretest to check change 1 of version 3.6, I screwed up. -This caused pcretest, when used on the test data, to segfault. Unfortunately, -this didn't happen under Solaris 8, where I normally test things. - -2. The Makefile had to be changed to make it work on BSD systems, where 'make' -doesn't seem to recognize that ./xxx and xxx are the same file. (This entry -isn't in ChangeLog distributed with 3.7 because I forgot when I hastily made -this fix an hour or so after the initial 3.7 release.) - - -Version 3.6 23-Oct-01 ---------------------- - -1. Crashed with /(sens|respons)e and \1ibility/ and "sense and sensibility" if -offsets passed as NULL with zero offset count. - -2. The config.guess and config.sub files had not been updated when I moved to -the latest autoconf. - - -Version 3.5 15-Aug-01 ---------------------- - -1. Added some missing #if !defined NOPOSIX conditionals in pcretest.c that -had been forgotten. - -2. By using declared but undefined structures, we can avoid using "void" -definitions in pcre.h while keeping the internal definitions of the structures -private. - -3. The distribution is now built using autoconf 2.50 and libtool 1.4. From a -user point of view, this means that both static and shared libraries are built -by default, but this can be individually controlled. More of the work of -handling this static/shared cases is now inside libtool instead of PCRE's make -file. - -4. The pcretest utility is now installed along with pcregrep because it is -useful for users (to test regexs) and by doing this, it automatically gets -relinked by libtool. The documentation has been turned into a man page, so -there are now .1, .txt, and .html versions in /doc. - -5. Upgrades to pcregrep: - (i) Added long-form option names like gnu grep. - (ii) Added --help to list all options with an explanatory phrase. - (iii) Added -r, --recursive to recurse into sub-directories. - (iv) Added -f, --file to read patterns from a file. - -6. pcre_exec() was referring to its "code" argument before testing that -argument for NULL (and giving an error if it was NULL). - -7. Upgraded Makefile.in to allow for compiling in a different directory from -the source directory. - -8. Tiny buglet in pcretest: when pcre_fullinfo() was called to retrieve the -options bits, the pointer it was passed was to an int instead of to an unsigned -long int. This mattered only on 64-bit systems. - -9. Fixed typo (3.4/1) in pcre.h again. Sigh. I had changed pcre.h (which is -generated) instead of pcre.in, which it its source. Also made the same change -in several of the .c files. - -10. A new release of gcc defines printf() as a macro, which broke pcretest -because it had an ifdef in the middle of a string argument for printf(). Fixed -by using separate calls to printf(). - -11. Added --enable-newline-is-cr and --enable-newline-is-lf to the configure -script, to force use of CR or LF instead of \n in the source. On non-Unix -systems, the value can be set in config.h. - -12. The limit of 200 on non-capturing parentheses is a _nesting_ limit, not an -absolute limit. Changed the text of the error message to make this clear, and -likewise updated the man page. - -13. The limit of 99 on the number of capturing subpatterns has been removed. -The new limit is 65535, which I hope will not be a "real" limit. - - -Version 3.4 22-Aug-00 ---------------------- - -1. Fixed typo in pcre.h: unsigned const char * changed to const unsigned char *. - -2. Diagnose condition (?(0) as an error instead of crashing on matching. - - -Version 3.3 01-Aug-00 ---------------------- - -1. If an octal character was given, but the value was greater than \377, it -was not getting masked to the least significant bits, as documented. This could -lead to crashes in some systems. - -2. Perl 5.6 (if not earlier versions) accepts classes like [a-\d] and treats -the hyphen as a literal. PCRE used to give an error; it now behaves like Perl. - -3. Added the functions pcre_free_substring() and pcre_free_substring_list(). -These just pass their arguments on to (pcre_free)(), but they are provided -because some uses of PCRE bind it to non-C systems that can call its functions, -but cannot call free() or pcre_free() directly. - -4. Add "make test" as a synonym for "make check". Corrected some comments in -the Makefile. - -5. Add $(DESTDIR)/ in front of all the paths in the "install" target in the -Makefile. - -6. Changed the name of pgrep to pcregrep, because Solaris has introduced a -command called pgrep for grepping around the active processes. - -7. Added the beginnings of support for UTF-8 character strings. - -8. Arranged for the Makefile to pass over the settings of CC, CFLAGS, and -RANLIB to ./ltconfig so that they are used by libtool. I think these are all -the relevant ones. (AR is not passed because ./ltconfig does its own figuring -out for the ar command.) - - -Version 3.2 12-May-00 ---------------------- - -This is purely a bug fixing release. - -1. If the pattern /((Z)+|A)*/ was matched agained ZABCDEFG it matched Z instead -of ZA. This was just one example of several cases that could provoke this bug, -which was introduced by change 9 of version 2.00. The code for breaking -infinite loops after an iteration that matches an empty string was't working -correctly. - -2. The pcretest program was not imitating Perl correctly for the pattern /a*/g -when matched against abbab (for example). After matching an empty string, it -wasn't forcing anchoring when setting PCRE_NOTEMPTY for the next attempt; this -caused it to match further down the string than it should. - -3. The code contained an inclusion of sys/types.h. It isn't clear why this -was there because it doesn't seem to be needed, and it causes trouble on some -systems, as it is not a Standard C header. It has been removed. - -4. Made 4 silly changes to the source to avoid stupid compiler warnings that -were reported on the Macintosh. The changes were from - - while ((c = *(++ptr)) != 0 && c != '\n'); -to - while ((c = *(++ptr)) != 0 && c != '\n') ; - -Totally extraordinary, but if that's what it takes... - -5. PCRE is being used in one environment where neither memmove() nor bcopy() is -available. Added HAVE_BCOPY and an autoconf test for it; if neither -HAVE_MEMMOVE nor HAVE_BCOPY is set, use a built-in emulation function which -assumes the way PCRE uses memmove() (always moving upwards). - -6. PCRE is being used in one environment where strchr() is not available. There -was only one use in pcre.c, and writing it out to avoid strchr() probably gives -faster code anyway. - - -Version 3.1 09-Feb-00 ---------------------- - -The only change in this release is the fixing of some bugs in Makefile.in for -the "install" target: - -(1) It was failing to install pcreposix.h. - -(2) It was overwriting the pcre.3 man page with the pcreposix.3 man page. - - -Version 3.0 01-Feb-00 ---------------------- - -1. Add support for the /+ modifier to perltest (to output $` like it does in -pcretest). - -2. Add support for the /g modifier to perltest. - -3. Fix pcretest so that it behaves even more like Perl for /g when the pattern -matches null strings. - -4. Fix perltest so that it doesn't do unwanted things when fed an empty -pattern. Perl treats empty patterns specially - it reuses the most recent -pattern, which is not what we want. Replace // by /(?#)/ in order to avoid this -effect. - -5. The POSIX interface was broken in that it was just handing over the POSIX -captured string vector to pcre_exec(), but (since release 2.00) PCRE has -required a bigger vector, with some working space on the end. This means that -the POSIX wrapper now has to get and free some memory, and copy the results. - -6. Added some simple autoconf support, placing the test data and the -documentation in separate directories, re-organizing some of the -information files, and making it build pcre-config (a GNU standard). Also added -libtool support for building PCRE as a shared library, which is now the -default. - -7. Got rid of the leading zero in the definition of PCRE_MINOR because 08 and -09 are not valid octal constants. Single digits will be used for minor values -less than 10. - -8. Defined REG_EXTENDED and REG_NOSUB as zero in the POSIX header, so that -existing programs that set these in the POSIX interface can use PCRE without -modification. - -9. Added a new function, pcre_fullinfo() with an extensible interface. It can -return all that pcre_info() returns, plus additional data. The pcre_info() -function is retained for compatibility, but is considered to be obsolete. - -10. Added experimental recursion feature (?R) to handle one common case that -Perl 5.6 will be able to do with (?p{...}). - -11. Added support for POSIX character classes like [:alpha:], which Perl is -adopting. - - -Version 2.08 31-Aug-99 ----------------------- - -1. When startoffset was not zero and the pattern began with ".*", PCRE was not -trying to match at the startoffset position, but instead was moving forward to -the next newline as if a previous match had failed. - -2. pcretest was not making use of PCRE_NOTEMPTY when repeating for /g and /G, -and could get into a loop if a null string was matched other than at the start -of the subject. - -3. Added definitions of PCRE_MAJOR and PCRE_MINOR to pcre.h so the version can -be distinguished at compile time, and for completeness also added PCRE_DATE. - -5. Added Paul Sokolovsky's minor changes to make it easy to compile a Win32 DLL -in GnuWin32 environments. - - -Version 2.07 29-Jul-99 ----------------------- - -1. The documentation is now supplied in plain text form and HTML as well as in -the form of man page sources. - -2. C++ compilers don't like assigning (void *) values to other pointer types. -In particular this affects malloc(). Although there is no problem in Standard -C, I've put in casts to keep C++ compilers happy. - -3. Typo on pcretest.c; a cast of (unsigned char *) in the POSIX regexec() call -should be (const char *). - -4. If NOPOSIX is defined, pcretest.c compiles without POSIX support. This may -be useful for non-Unix systems who don't want to bother with the POSIX stuff. -However, I haven't made this a standard facility. The documentation doesn't -mention it, and the Makefile doesn't support it. - -5. The Makefile now contains an "install" target, with editable destinations at -the top of the file. The pcretest program is not installed. - -6. pgrep -V now gives the PCRE version number and date. - -7. Fixed bug: a zero repetition after a literal string (e.g. /abcde{0}/) was -causing the entire string to be ignored, instead of just the last character. - -8. If a pattern like /"([^\\"]+|\\.)*"/ is applied in the normal way to a -non-matching string, it can take a very, very long time, even for strings of -quite modest length, because of the nested recursion. PCRE now does better in -some of these cases. It does this by remembering the last required literal -character in the pattern, and pre-searching the subject to ensure it is present -before running the real match. In other words, it applies a heuristic to detect -some types of certain failure quickly, and in the above example, if presented -with a string that has no trailing " it gives "no match" very quickly. - -9. A new runtime option PCRE_NOTEMPTY causes null string matches to be ignored; -other alternatives are tried instead. - - -Version 2.06 09-Jun-99 ----------------------- - -1. Change pcretest's output for amount of store used to show just the code -space, because the remainder (the data block) varies in size between 32-bit and -64-bit systems. - -2. Added an extra argument to pcre_exec() to supply an offset in the subject to -start matching at. This allows lookbehinds to work when searching for multiple -occurrences in a string. - -3. Added additional options to pcretest for testing multiple occurrences: - - /+ outputs the rest of the string that follows a match - /g loops for multiple occurrences, using the new startoffset argument - /G loops for multiple occurrences by passing an incremented pointer - -4. PCRE wasn't doing the "first character" optimization for patterns starting -with \b or \B, though it was doing it for other lookbehind assertions. That is, -it wasn't noticing that a match for a pattern such as /\bxyz/ has to start with -the letter 'x'. On long subject strings, this gives a significant speed-up. - - -Version 2.05 21-Apr-99 ----------------------- - -1. Changed the type of magic_number from int to long int so that it works -properly on 16-bit systems. - -2. Fixed a bug which caused patterns starting with .* not to work correctly -when the subject string contained newline characters. PCRE was assuming -anchoring for such patterns in all cases, which is not correct because .* will -not pass a newline unless PCRE_DOTALL is set. It now assumes anchoring only if -DOTALL is set at top level; otherwise it knows that patterns starting with .* -must be retried after every newline in the subject. - - -Version 2.04 18-Feb-99 ----------------------- - -1. For parenthesized subpatterns with repeats whose minimum was zero, the -computation of the store needed to hold the pattern was incorrect (too large). -If such patterns were nested a few deep, this could multiply and become a real -problem. - -2. Added /M option to pcretest to show the memory requirement of a specific -pattern. Made -m a synonym of -s (which does this globally) for compatibility. - -3. Subpatterns of the form (regex){n,m} (i.e. limited maximum) were being -compiled in such a way that the backtracking after subsequent failure was -pessimal. Something like (a){0,3} was compiled as (a)?(a)?(a)? instead of -((a)((a)(a)?)?)? with disastrous performance if the maximum was of any size. - - -Version 2.03 02-Feb-99 ----------------------- - -1. Fixed typo and small mistake in man page. - -2. Added 4th condition (GPL supersedes if conflict) and created separate -LICENCE file containing the conditions. - -3. Updated pcretest so that patterns such as /abc\/def/ work like they do in -Perl, that is the internal \ allows the delimiter to be included in the -pattern. Locked out the use of \ as a delimiter. If \ immediately follows -the final delimiter, add \ to the end of the pattern (to test the error). - -4. Added the convenience functions for extracting substrings after a successful -match. Updated pcretest to make it able to test these functions. - - -Version 2.02 14-Jan-99 ----------------------- - -1. Initialized the working variables associated with each extraction so that -their saving and restoring doesn't refer to uninitialized store. - -2. Put dummy code into study.c in order to trick the optimizer of the IBM C -compiler for OS/2 into generating correct code. Apparently IBM isn't going to -fix the problem. - -3. Pcretest: the timing code wasn't using LOOPREPEAT for timing execution -calls, and wasn't printing the correct value for compiling calls. Increased the -default value of LOOPREPEAT, and the number of significant figures in the -times. - -4. Changed "/bin/rm" in the Makefile to "-rm" so it works on Windows NT. - -5. Renamed "deftables" as "dftables" to get it down to 8 characters, to avoid -a building problem on Windows NT with a FAT file system. - - -Version 2.01 21-Oct-98 ----------------------- - -1. Changed the API for pcre_compile() to allow for the provision of a pointer -to character tables built by pcre_maketables() in the current locale. If NULL -is passed, the default tables are used. - - -Version 2.00 24-Sep-98 ----------------------- - -1. Since the (>?) facility is in Perl 5.005, don't require PCRE_EXTRA to enable -it any more. - -2. Allow quantification of (?>) groups, and make it work correctly. - -3. The first character computation wasn't working for (?>) groups. - -4. Correct the implementation of \Z (it is permitted to match on the \n at the -end of the subject) and add 5.005's \z, which really does match only at the -very end of the subject. - -5. Remove the \X "cut" facility; Perl doesn't have it, and (?> is neater. - -6. Remove the ability to specify CASELESS, MULTILINE, DOTALL, and -DOLLAR_END_ONLY at runtime, to make it possible to implement the Perl 5.005 -localized options. All options to pcre_study() were also removed. - -7. Add other new features from 5.005: - - $(?<= positive lookbehind - $(?a*))*/ (a PCRE_EXTRA facility). - - -Version 1.00 18-Nov-97 ----------------------- - -1. Added compile-time macros to support systems such as SunOS4 which don't have -memmove() or strerror() but have other things that can be used instead. - -2. Arranged that "make clean" removes the executables. - - -Version 0.99 27-Oct-97 ----------------------- - -1. Fixed bug in code for optimizing classes with only one character. It was -initializing a 32-byte map regardless, which could cause it to run off the end -of the memory it had got. - -2. Added, conditional on PCRE_EXTRA, the proposed (?>REGEX) construction. - - -Version 0.98 22-Oct-97 ----------------------- - -1. Fixed bug in code for handling temporary memory usage when there are more -back references than supplied space in the ovector. This could cause segfaults. - - -Version 0.97 21-Oct-97 ----------------------- - -1. Added the \X "cut" facility, conditional on PCRE_EXTRA. - -2. Optimized negated single characters not to use a bit map. - -3. Brought error texts together as macro definitions; clarified some of them; -fixed one that was wrong - it said "range out of order" when it meant "invalid -escape sequence". - -4. Changed some char * arguments to const char *. - -5. Added PCRE_NOTBOL and PCRE_NOTEOL (from POSIX). - -6. Added the POSIX-style API wrapper in pcreposix.a and testing facilities in -pcretest. - - -Version 0.96 16-Oct-97 ----------------------- - -1. Added a simple "pgrep" utility to the distribution. - -2. Fixed an incompatibility with Perl: "{" is now treated as a normal character -unless it appears in one of the precise forms "{ddd}", "{ddd,}", or "{ddd,ddd}" -where "ddd" means "one or more decimal digits". - -3. Fixed serious bug. If a pattern had a back reference, but the call to -pcre_exec() didn't supply a large enough ovector to record the related -identifying subpattern, the match always failed. PCRE now remembers the number -of the largest back reference, and gets some temporary memory in which to save -the offsets during matching if necessary, in order to ensure that -backreferences always work. - -4. Increased the compatibility with Perl in a number of ways: - - (a) . no longer matches \n by default; an option PCRE_DOTALL is provided - to request this handling. The option can be set at compile or exec time. - - (b) $ matches before a terminating newline by default; an option - PCRE_DOLLAR_ENDONLY is provided to override this (but not in multiline - mode). The option can be set at compile or exec time. - - (c) The handling of \ followed by a digit other than 0 is now supposed to be - the same as Perl's. If the decimal number it represents is less than 10 - or there aren't that many previous left capturing parentheses, an octal - escape is read. Inside a character class, it's always an octal escape, - even if it is a single digit. - - (d) An escaped but undefined alphabetic character is taken as a literal, - unless PCRE_EXTRA is set. Currently this just reserves the remaining - escapes. - - (e) {0} is now permitted. (The previous item is removed from the compiled - pattern). - -5. Changed all the names of code files so that the basic parts are no longer -than 10 characters, and abolished the teeny "globals.c" file. - -6. Changed the handling of character classes; they are now done with a 32-byte -bit map always. - -7. Added the -d and /D options to pcretest to make it possible to look at the -internals of compilation without having to recompile pcre. - - -Version 0.95 23-Sep-97 ----------------------- - -1. Fixed bug in pre-pass concerning escaped "normal" characters such as \x5c or -\x20 at the start of a run of normal characters. These were being treated as -real characters, instead of the source characters being re-checked. - - -Version 0.94 18-Sep-97 ----------------------- - -1. The functions are now thread-safe, with the caveat that the global variables -containing pointers to malloc() and free() or alternative functions are the -same for all threads. - -2. Get pcre_study() to generate a bitmap of initial characters for non- -anchored patterns when this is possible, and use it if passed to pcre_exec(). - - -Version 0.93 15-Sep-97 ----------------------- - -1. /(b)|(:+)/ was computing an incorrect first character. - -2. Add pcre_study() to the API and the passing of pcre_extra to pcre_exec(), -but not actually doing anything yet. - -3. Treat "-" characters in classes that cannot be part of ranges as literals, -as Perl does (e.g. [-az] or [az-]). - -4. Set the anchored flag if a branch starts with .* or .*? because that tests -all possible positions. - -5. Split up into different modules to avoid including unneeded functions in a -compiled binary. However, compile and exec are still in one module. The "study" -function is split off. - -6. The character tables are now in a separate module whose source is generated -by an auxiliary program - but can then be edited by hand if required. There are -now no calls to isalnum(), isspace(), isdigit(), isxdigit(), tolower() or -toupper() in the code. - -7. Turn the malloc/free funtions variables into pcre_malloc and pcre_free and -make them global. Abolish the function for setting them, as the caller can now -set them directly. - - -Version 0.92 11-Sep-97 ----------------------- - -1. A repeat with a fixed maximum and a minimum of 1 for an ordinary character -(e.g. /a{1,3}/) was broken (I mis-optimized it). - -2. Caseless matching was not working in character classes if the characters in -the pattern were in upper case. - -3. Make ranges like [W-c] work in the same way as Perl for caseless matching. - -4. Make PCRE_ANCHORED public and accept as a compile option. - -5. Add an options word to pcre_exec() and accept PCRE_ANCHORED and -PCRE_CASELESS at run time. Add escapes \A and \I to pcretest to cause it to -pass them. - -6. Give an error if bad option bits passed at compile or run time. - -7. Add PCRE_MULTILINE at compile and exec time, and (?m) as well. Add \M to -pcretest to cause it to pass that flag. - -8. Add pcre_info(), to get the number of identifying subpatterns, the stored -options, and the first character, if set. - -9. Recognize C+ or C{n,m} where n >= 1 as providing a fixed starting character. - - -Version 0.91 10-Sep-97 ----------------------- - -1. PCRE was failing to diagnose unlimited repeats of subpatterns that could -match the empty string as in /(a*)*/. It was looping and ultimately crashing. - -2. PCRE was looping on encountering an indefinitely repeated back reference to -a subpattern that had matched an empty string, e.g. /(a|)\1*/. It now does what -Perl does - treats the match as successful. - -**** diff --git a/src/third_party/pcre-8.39/CheckMan b/src/third_party/pcre-8.39/CheckMan deleted file mode 100755 index 480d735481e..00000000000 --- a/src/third_party/pcre-8.39/CheckMan +++ /dev/null @@ -1,67 +0,0 @@ -#! /usr/bin/perl - -# A script to scan PCRE's man pages to check for typos in the control -# sequences. I use only a small set of the available repertoire, so it is -# straightforward to check that nothing else has slipped in by mistake. This -# script should be called in the doc directory. - -$yield = 0; - -while (scalar(@ARGV) > 0) - { - $line = 0; - $file = shift @ARGV; - - open (IN, $file) || die "Failed to open $file\n"; - - while () - { - $line++; - if (/^\s*$/) - { - printf "Empty line $line of $file\n"; - $yield = 1; - } - elsif (/^\./) - { - if (!/^\.\s*$| - ^\.B\s+\S| - ^\.TH\s\S| - ^\.SH\s\S| - ^\.SS\s\S| - ^\.TP(?:\s?\d+)?\s*$| - ^\.SM\s*$| - ^\.br\s*$| - ^\.rs\s*$| - ^\.sp\s*$| - ^\.nf\s*$| - ^\.fi\s*$| - ^\.P\s*$| - ^\.PP\s*$| - ^\.\\"(?:\ HREF)?\s*$| - ^\.\\"\sHTML\s\s*$| - ^\.\\"\sHTML\s<\/a>\s*$| - ^\.\\"\s<\/a>\s*$| - ^\.\\"\sJOINSH\s*$| - ^\.\\"\sJOIN\s*$/x - ) - { - printf "Bad control line $line of $file\n"; - $yield = 1; - } - } - else - { - if (/\\[^ef]|\\f[^IBP]/) - { - printf "Bad backslash in line $line of $file\n"; - $yield = 1; - } - } - } - - close(IN); - } - -exit $yield; -# End diff --git a/src/third_party/pcre-8.39/CleanTxt b/src/third_party/pcre-8.39/CleanTxt deleted file mode 100755 index 1f42519c8d7..00000000000 --- a/src/third_party/pcre-8.39/CleanTxt +++ /dev/null @@ -1,113 +0,0 @@ -#! /usr/bin/perl -w - -# Script to take the output of nroff -man and remove all the backspacing and -# the page footers and the screen commands etc so that it is more usefully -# readable online. In fact, in the latest nroff, intermediate footers don't -# seem to be generated any more. - -$blankcount = 0; -$lastwascut = 0; -$firstheader = 1; - -# Input on STDIN; output to STDOUT. - -while () - { - s/\x1b\[\d+m//g; # Remove screen controls "ESC [ number m" - s/.\x8//g; # Remove "char, backspace" - - # Handle header lines. Retain only the first one we encounter, but remove - # the blank line that follows. Any others (e.g. at end of document) and the - # following blank line are dropped. - - if (/^PCRE(\w*)\(([13])\)\s+PCRE\1\(\2\)$/) - { - if ($firstheader) - { - $firstheader = 0; - print; - $lastprinted = $_; - $lastwascut = 0; - } - $_=; # Remove a blank that follows - next; - } - - # Count runs of empty lines - - if (/^\s*$/) - { - $blankcount++; - $lastwascut = 0; - next; - } - - # If a chunk of lines has been cut out (page footer) and the next line - # has a different indentation, put back one blank line. - - if ($lastwascut && $blankcount < 1 && defined($lastprinted)) - { - ($a) = $lastprinted =~ /^(\s*)/; - ($b) = $_ =~ /^(\s*)/; - $blankcount++ if ($a ne $b); - } - - # We get here only when we have a non-blank line in hand. If it was preceded - # by 3 or more blank lines, read the next 3 lines and see if they are blank. - # If so, remove all 7 lines, and remember that we have just done a cut. - - if ($blankcount >= 3) - { - for ($i = 0; $i < 3; $i++) - { - $next[$i] = ; - $next[$i] = "" if !defined $next[$i]; - $next[$i] =~ s/\x1b\[\d+m//g; # Remove screen controls "ESC [ number m" - $next[$i] =~ s/.\x8//g; # Remove "char, backspace" - } - - # Cut out chunks of the form <3 blanks><3 blanks> - - if ($next[0] =~ /^\s*$/ && - $next[1] =~ /^\s*$/ && - $next[2] =~ /^\s*$/) - { - $blankcount -= 3; - $lastwascut = 1; - } - - # Otherwise output the saved blanks, the current, and the next three - # lines. Remember the last printed line. - - else - { - for ($i = 0; $i < $blankcount; $i++) { print "\n"; } - print; - for ($i = 0; $i < 3; $i++) - { - $next[$i] =~ s/.\x8//g; - print $next[$i]; - $lastprinted = $_; - } - $lastwascut = 0; - $blankcount = 0; - } - } - - # This non-blank line is not preceded by 3 or more blank lines. Output - # any blanks there are, and the line. Remember it. Force two blank lines - # before headings. - - else - { - $blankcount = 2 if /^\S/ && !/^Last updated/ && !/^Copyright/ && - defined($lastprinted); - for ($i = 0; $i < $blankcount; $i++) { print "\n"; } - print; - $lastprinted = $_; - $lastwascut = 0; - $blankcount = 0; - } - } - -# End diff --git a/src/third_party/pcre-8.39/Detrail b/src/third_party/pcre-8.39/Detrail deleted file mode 100755 index 1c5c7e9cae1..00000000000 --- a/src/third_party/pcre-8.39/Detrail +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/perl - -# This is a script for removing trailing whitespace from lines in files that -# are listed on the command line. - -# This subroutine does the work for one file. - -sub detrail { -my($file) = $_[0]; -my($changed) = 0; -open(IN, "$file") || die "Can't open $file for input"; -@lines = ; -close(IN); -foreach (@lines) - { - if (/\s+\n$/) - { - s/\s+\n$/\n/; - $changed = 1; - } - } -if ($changed) - { - open(OUT, ">$file") || die "Can't open $file for output"; - print OUT @lines; - close(OUT); - } -} - -# This is the main program - -$, = ""; # Output field separator -for ($i = 0; $i < @ARGV; $i++) { &detrail($ARGV[$i]); } - -# End diff --git a/src/third_party/pcre-8.39/HACKING b/src/third_party/pcre-8.39/HACKING deleted file mode 100644 index 691b7a14e50..00000000000 --- a/src/third_party/pcre-8.39/HACKING +++ /dev/null @@ -1,528 +0,0 @@ -Technical Notes about PCRE --------------------------- - -These are very rough technical notes that record potentially useful information -about PCRE internals. For information about testing PCRE, see the pcretest -documentation and the comment at the head of the RunTest file. - - -Historical note 1 ------------------ - -Many years ago I implemented some regular expression functions to an algorithm -suggested by Martin Richards. These were not Unix-like in form, and were quite -restricted in what they could do by comparison with Perl. The interesting part -about the algorithm was that the amount of space required to hold the compiled -form of an expression was known in advance. The code to apply an expression did -not operate by backtracking, as the original Henry Spencer code and current -Perl code does, but instead checked all possibilities simultaneously by keeping -a list of current states and checking all of them as it advanced through the -subject string. In the terminology of Jeffrey Friedl's book, it was a "DFA -algorithm", though it was not a traditional Finite State Machine (FSM). When -the pattern was all used up, all remaining states were possible matches, and -the one matching the longest subset of the subject string was chosen. This did -not necessarily maximize the individual wild portions of the pattern, as is -expected in Unix and Perl-style regular expressions. - - -Historical note 2 ------------------ - -By contrast, the code originally written by Henry Spencer (which was -subsequently heavily modified for Perl) compiles the expression twice: once in -a dummy mode in order to find out how much store will be needed, and then for -real. (The Perl version probably doesn't do this any more; I'm talking about -the original library.) The execution function operates by backtracking and -maximizing (or, optionally, minimizing in Perl) the amount of the subject that -matches individual wild portions of the pattern. This is an "NFA algorithm" in -Friedl's terminology. - - -OK, here's the real stuff -------------------------- - -For the set of functions that form the "basic" PCRE library (which are -unrelated to those mentioned above), I tried at first to invent an algorithm -that used an amount of store bounded by a multiple of the number of characters -in the pattern, to save on compiling time. However, because of the greater -complexity in Perl regular expressions, I couldn't do this. In any case, a -first pass through the pattern is helpful for other reasons. - - -Support for 16-bit and 32-bit data strings -------------------------------------------- - -From release 8.30, PCRE supports 16-bit as well as 8-bit data strings; and from -release 8.32, PCRE supports 32-bit data strings. The library can be compiled -in any combination of 8-bit, 16-bit or 32-bit modes, creating up to three -different libraries. In the description that follows, the word "short" is used -for a 16-bit data quantity, and the word "unit" is used for a quantity that is -a byte in 8-bit mode, a short in 16-bit mode and a 32-bit word in 32-bit mode. -However, so as not to over-complicate the text, the names of PCRE functions are -given in 8-bit form only. - - -Computing the memory requirement: how it was --------------------------------------------- - -Up to and including release 6.7, PCRE worked by running a very degenerate first -pass to calculate a maximum store size, and then a second pass to do the real -compile - which might use a bit less than the predicted amount of memory. The -idea was that this would turn out faster than the Henry Spencer code because -the first pass is degenerate and the second pass can just store stuff straight -into the vector, which it knows is big enough. - - -Computing the memory requirement: how it is -------------------------------------------- - -By the time I was working on a potential 6.8 release, the degenerate first pass -had become very complicated and hard to maintain. Indeed one of the early -things I did for 6.8 was to fix Yet Another Bug in the memory computation. Then -I had a flash of inspiration as to how I could run the real compile function in -a "fake" mode that enables it to compute how much memory it would need, while -actually only ever using a few hundred bytes of working memory, and without too -many tests of the mode that might slow it down. So I refactored the compiling -functions to work this way. This got rid of about 600 lines of source. It -should make future maintenance and development easier. As this was such a major -change, I never released 6.8, instead upping the number to 7.0 (other quite -major changes were also present in the 7.0 release). - -A side effect of this work was that the previous limit of 200 on the nesting -depth of parentheses was removed. However, there is a downside: pcre_compile() -runs more slowly than before (30% or more, depending on the pattern) because it -is doing a full analysis of the pattern. My hope was that this would not be a -big issue, and in the event, nobody has commented on it. - -At release 8.34, a limit on the nesting depth of parentheses was re-introduced -(default 250, settable at build time) so as to put a limit on the amount of -system stack used by pcre_compile(). This is a safety feature for environments -with small stacks where the patterns are provided by users. - - -Traditional matching function ------------------------------ - -The "traditional", and original, matching function is called pcre_exec(), and -it implements an NFA algorithm, similar to the original Henry Spencer algorithm -and the way that Perl works. This is not surprising, since it is intended to be -as compatible with Perl as possible. This is the function most users of PCRE -will use most of the time. From release 8.20, if PCRE is compiled with -just-in-time (JIT) support, and studying a compiled pattern with JIT is -successful, the JIT code is run instead of the normal pcre_exec() code, but the -result is the same. - - -Supplementary matching function -------------------------------- - -From PCRE 6.0, there is also a supplementary matching function called -pcre_dfa_exec(). This implements a DFA matching algorithm that searches -simultaneously for all possible matches that start at one point in the subject -string. (Going back to my roots: see Historical Note 1 above.) This function -intreprets the same compiled pattern data as pcre_exec(); however, not all the -facilities are available, and those that are do not always work in quite the -same way. See the user documentation for details. - -The algorithm that is used for pcre_dfa_exec() is not a traditional FSM, -because it may have a number of states active at one time. More work would be -needed at compile time to produce a traditional FSM where only one state is -ever active at once. I believe some other regex matchers work this way. JIT -support is not available for this kind of matching. - - -Changeable options ------------------- - -The /i, /m, or /s options (PCRE_CASELESS, PCRE_MULTILINE, PCRE_DOTALL, and some -others) may change in the middle of patterns. From PCRE 8.13, their processing -is handled entirely at compile time by generating different opcodes for the -different settings. The runtime functions do not need to keep track of an -options state any more. - - -Format of compiled patterns ---------------------------- - -The compiled form of a pattern is a vector of unsigned units (bytes in 8-bit -mode, shorts in 16-bit mode, 32-bit words in 32-bit mode), containing items of -variable length. The first unit in an item contains an opcode, and the length -of the item is either implicit in the opcode or contained in the data that -follows it. - -In many cases listed below, LINK_SIZE data values are specified for offsets -within the compiled pattern. LINK_SIZE always specifies a number of bytes. The -default value for LINK_SIZE is 2, but PCRE can be compiled to use 3-byte or -4-byte values for these offsets, although this impairs the performance. (3-byte -LINK_SIZE values are available only in 8-bit mode.) Specifing a LINK_SIZE -larger than 2 is necessary only when patterns whose compiled length is greater -than 64K are going to be processed. In this description, we assume the "normal" -compilation options. Data values that are counts (e.g. quantifiers) are two -bytes long in 8-bit mode (most significant byte first), or one unit in 16-bit -and 32-bit modes. - - -Opcodes with no following data ------------------------------- - -These items are all just one unit long - - OP_END end of pattern - OP_ANY match any one character other than newline - OP_ALLANY match any one character, including newline - OP_ANYBYTE match any single unit, even in UTF-8/16 mode - OP_SOD match start of data: \A - OP_SOM, start of match (subject + offset): \G - OP_SET_SOM, set start of match (\K) - OP_CIRC ^ (start of data) - OP_CIRCM ^ multiline mode (start of data or after newline) - OP_NOT_WORD_BOUNDARY \W - OP_WORD_BOUNDARY \w - OP_NOT_DIGIT \D - OP_DIGIT \d - OP_NOT_HSPACE \H - OP_HSPACE \h - OP_NOT_WHITESPACE \S - OP_WHITESPACE \s - OP_NOT_VSPACE \V - OP_VSPACE \v - OP_NOT_WORDCHAR \W - OP_WORDCHAR \w - OP_EODN match end of data or newline at end: \Z - OP_EOD match end of data: \z - OP_DOLL $ (end of data, or before final newline) - OP_DOLLM $ multiline mode (end of data or before newline) - OP_EXTUNI match an extended Unicode grapheme cluster - OP_ANYNL match any Unicode newline sequence - - OP_ASSERT_ACCEPT ) - OP_ACCEPT ) These are Perl 5.10's "backtracking control - OP_COMMIT ) verbs". If OP_ACCEPT is inside capturing - OP_FAIL ) parentheses, it may be preceded by one or more - OP_PRUNE ) OP_CLOSE, each followed by a count that - OP_SKIP ) indicates which parentheses must be closed. - OP_THEN ) - -OP_ASSERT_ACCEPT is used when (*ACCEPT) is encountered within an assertion. -This ends the assertion, not the entire pattern match. - - -Backtracking control verbs with optional data ---------------------------------------------- - -(*THEN) without an argument generates the opcode OP_THEN and no following data. -OP_MARK is followed by the mark name, preceded by a one-unit length, and -followed by a binary zero. For (*PRUNE), (*SKIP), and (*THEN) with arguments, -the opcodes OP_PRUNE_ARG, OP_SKIP_ARG, and OP_THEN_ARG are used, with the name -following in the same format as OP_MARK. - - -Matching literal characters ---------------------------- - -The OP_CHAR opcode is followed by a single character that is to be matched -casefully. For caseless matching, OP_CHARI is used. In UTF-8 or UTF-16 modes, -the character may be more than one unit long. In UTF-32 mode, characters -are always exactly one unit long. - -If there is only one character in a character class, OP_CHAR or OP_CHARI is -used for a positive class, and OP_NOT or OP_NOTI for a negative one (that is, -for something like [^a]). - - -Repeating single characters ---------------------------- - -The common repeats (*, +, ?), when applied to a single character, use the -following opcodes, which come in caseful and caseless versions: - - Caseful Caseless - OP_STAR OP_STARI - OP_MINSTAR OP_MINSTARI - OP_POSSTAR OP_POSSTARI - OP_PLUS OP_PLUSI - OP_MINPLUS OP_MINPLUSI - OP_POSPLUS OP_POSPLUSI - OP_QUERY OP_QUERYI - OP_MINQUERY OP_MINQUERYI - OP_POSQUERY OP_POSQUERYI - -Each opcode is followed by the character that is to be repeated. In ASCII mode, -these are two-unit items; in UTF-8 or UTF-16 modes, the length is variable; in -UTF-32 mode these are one-unit items. Those with "MIN" in their names are the -minimizing versions. Those with "POS" in their names are possessive versions. -Other repeats make use of these opcodes: - - Caseful Caseless - OP_UPTO OP_UPTOI - OP_MINUPTO OP_MINUPTOI - OP_POSUPTO OP_POSUPTOI - OP_EXACT OP_EXACTI - -Each of these is followed by a count and then the repeated character. OP_UPTO -matches from 0 to the given number. A repeat with a non-zero minimum and a -fixed maximum is coded as an OP_EXACT followed by an OP_UPTO (or OP_MINUPTO or -OPT_POSUPTO). - -Another set of matching repeating opcodes (called OP_NOTSTAR, OP_NOTSTARI, -etc.) are used for repeated, negated, single-character classes such as [^a]*. -The normal single-character opcodes (OP_STAR, etc.) are used for repeated -positive single-character classes. - - -Repeating character types -------------------------- - -Repeats of things like \d are done exactly as for single characters, except -that instead of a character, the opcode for the type is stored in the data -unit. The opcodes are: - - OP_TYPESTAR - OP_TYPEMINSTAR - OP_TYPEPOSSTAR - OP_TYPEPLUS - OP_TYPEMINPLUS - OP_TYPEPOSPLUS - OP_TYPEQUERY - OP_TYPEMINQUERY - OP_TYPEPOSQUERY - OP_TYPEUPTO - OP_TYPEMINUPTO - OP_TYPEPOSUPTO - OP_TYPEEXACT - - -Match by Unicode property -------------------------- - -OP_PROP and OP_NOTPROP are used for positive and negative matches of a -character by testing its Unicode property (the \p and \P escape sequences). -Each is followed by two units that encode the desired property as a type and a -value. The types are a set of #defines of the form PT_xxx, and the values are -enumerations of the form ucp_xx, defined in the ucp.h source file. The value is -relevant only for PT_GC (General Category), PT_PC (Particular Category), and -PT_SC (Script). - -Repeats of these items use the OP_TYPESTAR etc. set of opcodes, followed by -three units: OP_PROP or OP_NOTPROP, and then the desired property type and -value. - - -Character classes ------------------ - -If there is only one character in a class, OP_CHAR or OP_CHARI is used for a -positive class, and OP_NOT or OP_NOTI for a negative one (that is, for -something like [^a]). - -A set of repeating opcodes (called OP_NOTSTAR etc.) are used for repeated, -negated, single-character classes. The normal single-character opcodes -(OP_STAR, etc.) are used for repeated positive single-character classes. - -When there is more than one character in a class, and all the code points are -less than 256, OP_CLASS is used for a positive class, and OP_NCLASS for a -negative one. In either case, the opcode is followed by a 32-byte (16-short, -8-word) bit map containing a 1 bit for every character that is acceptable. The -bits are counted from the least significant end of each unit. In caseless mode, -bits for both cases are set. - -The reason for having both OP_CLASS and OP_NCLASS is so that, in UTF-8/16/32 -mode, subject characters with values greater than 255 can be handled correctly. -For OP_CLASS they do not match, whereas for OP_NCLASS they do. - -For classes containing characters with values greater than 255 or that contain -\p or \P, OP_XCLASS is used. It optionally uses a bit map if any code points -are less than 256, followed by a list of pairs (for a range) and single -characters. In caseless mode, both cases are explicitly listed. - -OP_XCLASS is followed by a unit containing flag bits: XCL_NOT indicates that -this is a negative class, and XCL_MAP indicates that a bit map is present. -There follows the bit map, if XCL_MAP is set, and then a sequence of items -coded as follows: - - XCL_END marks the end of the list - XCL_SINGLE one character follows - XCL_RANGE two characters follow - XCL_PROP a Unicode property (type, value) follows - XCL_NOTPROP a Unicode property (type, value) follows - -If a range starts with a code point less than 256 and ends with one greater -than 256, an XCL_RANGE item is used, without setting any bits in the bit map. -This means that if no other items in the class set bits in the map, a map is -not needed. - - -Back references ---------------- - -OP_REF (caseful) or OP_REFI (caseless) is followed by a count containing the -reference number if the reference is to a unique capturing group (either by -number or by name). When named groups are used, there may be more than one -group with the same name. In this case, a reference by name generates OP_DNREF -or OP_DNREFI. These are followed by two counts: the index (not the byte offset) -in the group name table of the first entry for the requred name, followed by -the number of groups with the same name. - - -Repeating character classes and back references ------------------------------------------------ - -Single-character classes are handled specially (see above). This section -applies to other classes and also to back references. In both cases, the repeat -information follows the base item. The matching code looks at the following -opcode to see if it is one of - - OP_CRSTAR - OP_CRMINSTAR - OP_CRPOSSTAR - OP_CRPLUS - OP_CRMINPLUS - OP_CRPOSPLUS - OP_CRQUERY - OP_CRMINQUERY - OP_CRPOSQUERY - OP_CRRANGE - OP_CRMINRANGE - OP_CRPOSRANGE - -All but the last three are single-unit items, with no data. The others are -followed by the minimum and maximum repeat counts. - - -Brackets and alternation ------------------------- - -A pair of non-capturing round brackets is wrapped round each expression at -compile time, so alternation always happens in the context of brackets. - -[Note for North Americans: "bracket" to some English speakers, including -myself, can be round, square, curly, or pointy. Hence this usage rather than -"parentheses".] - -Non-capturing brackets use the opcode OP_BRA. Originally PCRE was limited to 99 -capturing brackets and it used a different opcode for each one. From release -3.5, the limit was removed by putting the bracket number into the data for -higher-numbered brackets. From release 7.0 all capturing brackets are handled -this way, using the single opcode OP_CBRA. - -A bracket opcode is followed by LINK_SIZE bytes which give the offset to the -next alternative OP_ALT or, if there aren't any branches, to the matching -OP_KET opcode. Each OP_ALT is followed by LINK_SIZE bytes giving the offset to -the next one, or to the OP_KET opcode. For capturing brackets, the bracket -number is a count that immediately follows the offset. - -OP_KET is used for subpatterns that do not repeat indefinitely, and OP_KETRMIN -and OP_KETRMAX are used for indefinite repetitions, minimally or maximally -respectively (see below for possessive repetitions). All three are followed by -LINK_SIZE bytes giving (as a positive number) the offset back to the matching -bracket opcode. - -If a subpattern is quantified such that it is permitted to match zero times, it -is preceded by one of OP_BRAZERO, OP_BRAMINZERO, or OP_SKIPZERO. These are -single-unit opcodes that tell the matcher that skipping the following -subpattern entirely is a valid branch. In the case of the first two, not -skipping the pattern is also valid (greedy and non-greedy). The third is used -when a pattern has the quantifier {0,0}. It cannot be entirely discarded, -because it may be called as a subroutine from elsewhere in the regex. - -A subpattern with an indefinite maximum repetition is replicated in the -compiled data its minimum number of times (or once with OP_BRAZERO if the -minimum is zero), with the final copy terminating with OP_KETRMIN or OP_KETRMAX -as appropriate. - -A subpattern with a bounded maximum repetition is replicated in a nested -fashion up to the maximum number of times, with OP_BRAZERO or OP_BRAMINZERO -before each replication after the minimum, so that, for example, (abc){2,5} is -compiled as (abc)(abc)((abc)((abc)(abc)?)?)?, except that each bracketed group -has the same number. - -When a repeated subpattern has an unbounded upper limit, it is checked to see -whether it could match an empty string. If this is the case, the opcode in the -final replication is changed to OP_SBRA or OP_SCBRA. This tells the matcher -that it needs to check for matching an empty string when it hits OP_KETRMIN or -OP_KETRMAX, and if so, to break the loop. - - -Possessive brackets -------------------- - -When a repeated group (capturing or non-capturing) is marked as possessive by -the "+" notation, e.g. (abc)++, different opcodes are used. Their names all -have POS on the end, e.g. OP_BRAPOS instead of OP_BRA and OP_SCPBRPOS instead -of OP_SCBRA. The end of such a group is marked by OP_KETRPOS. If the minimum -repetition is zero, the group is preceded by OP_BRAPOSZERO. - - -Once-only (atomic) groups -------------------------- - -These are just like other subpatterns, but they start with the opcode -OP_ONCE or OP_ONCE_NC. The former is used when there are no capturing brackets -within the atomic group; the latter when there are. The distinction is needed -for when there is a backtrack to before the group - any captures within the -group must be reset, so it is necessary to retain backtracking points inside -the group even after it is complete in order to do this. When there are no -captures in an atomic group, all the backtracking can be discarded when it is -complete. This is more efficient, and also uses less stack. - -The check for matching an empty string in an unbounded repeat is handled -entirely at runtime, so there are just these two opcodes for atomic groups. - - -Assertions ----------- - -Forward assertions are also just like other subpatterns, but starting with one -of the opcodes OP_ASSERT or OP_ASSERT_NOT. Backward assertions use the opcodes -OP_ASSERTBACK and OP_ASSERTBACK_NOT, and the first opcode inside the assertion -is OP_REVERSE, followed by a count of the number of characters to move back the -pointer in the subject string. In ASCII mode, the count is a number of units, -but in UTF-8/16 mode each character may occupy more than one unit; in UTF-32 -mode each character occupies exactly one unit. A separate count is present in -each alternative of a lookbehind assertion, allowing them to have different -fixed lengths. - - -Conditional subpatterns ------------------------ - -These are like other subpatterns, but they start with the opcode OP_COND, or -OP_SCOND for one that might match an empty string in an unbounded repeat. If -the condition is a back reference, this is stored at the start of the -subpattern using the opcode OP_CREF followed by a count containing the -reference number, provided that the reference is to a unique capturing group. -If the reference was by name and there is more than one group with that name, -OP_DNCREF is used instead. It is followed by two counts: the index in the group -names table, and the number of groups with the same name. - -If the condition is "in recursion" (coded as "(?(R)"), or "in recursion of -group x" (coded as "(?(Rx)"), the group number is stored at the start of the -subpattern using the opcode OP_RREF (with a value of zero for "the whole -pattern") or OP_DNRREF (with data as for OP_DNCREF). For a DEFINE condition, -just the single unit OP_DEF is used (it has no associated data). Otherwise, a -conditional subpattern always starts with one of the assertions. - - -Recursion ---------- - -Recursion either matches the current regex, or some subexpression. The opcode -OP_RECURSE is followed by aLINK_SIZE value that is the offset to the starting -bracket from the start of the whole pattern. From release 6.5, OP_RECURSE is -automatically wrapped inside OP_ONCE brackets, because otherwise some patterns -broke it. OP_RECURSE is also used for "subroutine" calls, even though they are -not strictly a recursion. - - -Callout -------- - -OP_CALLOUT is followed by one unit of data that holds a callout number in the -range 0 to 254 for manual callouts, or 255 for an automatic callout. In both -cases there follows a count giving the offset in the pattern string to the -start of the following item, and another count giving the length of this item. -These values make is possible for pcretest to output useful tracing information -using automatic callouts. - -Philip Hazel -November 2013 diff --git a/src/third_party/pcre-8.39/INSTALL b/src/third_party/pcre-8.39/INSTALL deleted file mode 100644 index 2099840756e..00000000000 --- a/src/third_party/pcre-8.39/INSTALL +++ /dev/null @@ -1,370 +0,0 @@ -Installation Instructions -************************* - -Copyright (C) 1994-1996, 1999-2002, 2004-2013 Free Software Foundation, -Inc. - - Copying and distribution of this file, with or without modification, -are permitted in any medium without royalty provided the copyright -notice and this notice are preserved. This file is offered as-is, -without warranty of any kind. - -Basic Installation -================== - - Briefly, the shell command `./configure && make && make install' -should configure, build, and install this package. The following -more-detailed instructions are generic; see the `README' file for -instructions specific to this package. Some packages provide this -`INSTALL' file but do not implement all of the features documented -below. The lack of an optional feature in a given package is not -necessarily a bug. More recommendations for GNU packages can be found -in *note Makefile Conventions: (standards)Makefile Conventions. - - The `configure' shell script attempts to guess correct values for -various system-dependent variables used during compilation. It uses -those values to create a `Makefile' in each directory of the package. -It may also create one or more `.h' files containing system-dependent -definitions. Finally, it creates a shell script `config.status' that -you can run in the future to recreate the current configuration, and a -file `config.log' containing compiler output (useful mainly for -debugging `configure'). - - It can also use an optional file (typically called `config.cache' -and enabled with `--cache-file=config.cache' or simply `-C') that saves -the results of its tests to speed up reconfiguring. Caching is -disabled by default to prevent problems with accidental use of stale -cache files. - - If you need to do unusual things to compile the package, please try -to figure out how `configure' could check whether to do them, and mail -diffs or instructions to the address given in the `README' so they can -be considered for the next release. If you are using the cache, and at -some point `config.cache' contains results you don't want to keep, you -may remove or edit it. - - The file `configure.ac' (or `configure.in') is used to create -`configure' by a program called `autoconf'. You need `configure.ac' if -you want to change it or regenerate `configure' using a newer version -of `autoconf'. - - The simplest way to compile this package is: - - 1. `cd' to the directory containing the package's source code and type - `./configure' to configure the package for your system. - - Running `configure' might take a while. While running, it prints - some messages telling which features it is checking for. - - 2. Type `make' to compile the package. - - 3. Optionally, type `make check' to run any self-tests that come with - the package, generally using the just-built uninstalled binaries. - - 4. Type `make install' to install the programs and any data files and - documentation. When installing into a prefix owned by root, it is - recommended that the package be configured and built as a regular - user, and only the `make install' phase executed with root - privileges. - - 5. Optionally, type `make installcheck' to repeat any self-tests, but - this time using the binaries in their final installed location. - This target does not install anything. Running this target as a - regular user, particularly if the prior `make install' required - root privileges, verifies that the installation completed - correctly. - - 6. You can remove the program binaries and object files from the - source code directory by typing `make clean'. To also remove the - files that `configure' created (so you can compile the package for - a different kind of computer), type `make distclean'. There is - also a `make maintainer-clean' target, but that is intended mainly - for the package's developers. If you use it, you may have to get - all sorts of other programs in order to regenerate files that came - with the distribution. - - 7. Often, you can also type `make uninstall' to remove the installed - files again. In practice, not all packages have tested that - uninstallation works correctly, even though it is required by the - GNU Coding Standards. - - 8. Some packages, particularly those that use Automake, provide `make - distcheck', which can by used by developers to test that all other - targets like `make install' and `make uninstall' work correctly. - This target is generally not run by end users. - -Compilers and Options -===================== - - Some systems require unusual options for compilation or linking that -the `configure' script does not know about. Run `./configure --help' -for details on some of the pertinent environment variables. - - You can give `configure' initial values for configuration parameters -by setting variables in the command line or in the environment. Here -is an example: - - ./configure CC=c99 CFLAGS=-g LIBS=-lposix - - *Note Defining Variables::, for more details. - -Compiling For Multiple Architectures -==================================== - - You can compile the package for more than one kind of computer at the -same time, by placing the object files for each architecture in their -own directory. To do this, you can use GNU `make'. `cd' to the -directory where you want the object files and executables to go and run -the `configure' script. `configure' automatically checks for the -source code in the directory that `configure' is in and in `..'. This -is known as a "VPATH" build. - - With a non-GNU `make', it is safer to compile the package for one -architecture at a time in the source code directory. After you have -installed the package for one architecture, use `make distclean' before -reconfiguring for another architecture. - - On MacOS X 10.5 and later systems, you can create libraries and -executables that work on multiple system types--known as "fat" or -"universal" binaries--by specifying multiple `-arch' options to the -compiler but only a single `-arch' option to the preprocessor. Like -this: - - ./configure CC="gcc -arch i386 -arch x86_64 -arch ppc -arch ppc64" \ - CXX="g++ -arch i386 -arch x86_64 -arch ppc -arch ppc64" \ - CPP="gcc -E" CXXCPP="g++ -E" - - This is not guaranteed to produce working output in all cases, you -may have to build one architecture at a time and combine the results -using the `lipo' tool if you have problems. - -Installation Names -================== - - By default, `make install' installs the package's commands under -`/usr/local/bin', include files under `/usr/local/include', etc. You -can specify an installation prefix other than `/usr/local' by giving -`configure' the option `--prefix=PREFIX', where PREFIX must be an -absolute file name. - - You can specify separate installation prefixes for -architecture-specific files and architecture-independent files. If you -pass the option `--exec-prefix=PREFIX' to `configure', the package uses -PREFIX as the prefix for installing programs and libraries. -Documentation and other data files still use the regular prefix. - - In addition, if you use an unusual directory layout you can give -options like `--bindir=DIR' to specify different values for particular -kinds of files. Run `configure --help' for a list of the directories -you can set and what kinds of files go in them. In general, the -default for these options is expressed in terms of `${prefix}', so that -specifying just `--prefix' will affect all of the other directory -specifications that were not explicitly provided. - - The most portable way to affect installation locations is to pass the -correct locations to `configure'; however, many packages provide one or -both of the following shortcuts of passing variable assignments to the -`make install' command line to change installation locations without -having to reconfigure or recompile. - - The first method involves providing an override variable for each -affected directory. For example, `make install -prefix=/alternate/directory' will choose an alternate location for all -directory configuration variables that were expressed in terms of -`${prefix}'. Any directories that were specified during `configure', -but not in terms of `${prefix}', must each be overridden at install -time for the entire installation to be relocated. The approach of -makefile variable overrides for each directory variable is required by -the GNU Coding Standards, and ideally causes no recompilation. -However, some platforms have known limitations with the semantics of -shared libraries that end up requiring recompilation when using this -method, particularly noticeable in packages that use GNU Libtool. - - The second method involves providing the `DESTDIR' variable. For -example, `make install DESTDIR=/alternate/directory' will prepend -`/alternate/directory' before all installation names. The approach of -`DESTDIR' overrides is not required by the GNU Coding Standards, and -does not work on platforms that have drive letters. On the other hand, -it does better at avoiding recompilation issues, and works well even -when some directory options were not specified in terms of `${prefix}' -at `configure' time. - -Optional Features -================= - - If the package supports it, you can cause programs to be installed -with an extra prefix or suffix on their names by giving `configure' the -option `--program-prefix=PREFIX' or `--program-suffix=SUFFIX'. - - Some packages pay attention to `--enable-FEATURE' options to -`configure', where FEATURE indicates an optional part of the package. -They may also pay attention to `--with-PACKAGE' options, where PACKAGE -is something like `gnu-as' or `x' (for the X Window System). The -`README' should mention any `--enable-' and `--with-' options that the -package recognizes. - - For packages that use the X Window System, `configure' can usually -find the X include and library files automatically, but if it doesn't, -you can use the `configure' options `--x-includes=DIR' and -`--x-libraries=DIR' to specify their locations. - - Some packages offer the ability to configure how verbose the -execution of `make' will be. For these packages, running `./configure ---enable-silent-rules' sets the default to minimal output, which can be -overridden with `make V=1'; while running `./configure ---disable-silent-rules' sets the default to verbose, which can be -overridden with `make V=0'. - -Particular systems -================== - - On HP-UX, the default C compiler is not ANSI C compatible. If GNU -CC is not installed, it is recommended to use the following options in -order to use an ANSI C compiler: - - ./configure CC="cc -Ae -D_XOPEN_SOURCE=500" - -and if that doesn't work, install pre-built binaries of GCC for HP-UX. - - HP-UX `make' updates targets which have the same time stamps as -their prerequisites, which makes it generally unusable when shipped -generated files such as `configure' are involved. Use GNU `make' -instead. - - On OSF/1 a.k.a. Tru64, some versions of the default C compiler cannot -parse its `' header file. The option `-nodtk' can be used as -a workaround. If GNU CC is not installed, it is therefore recommended -to try - - ./configure CC="cc" - -and if that doesn't work, try - - ./configure CC="cc -nodtk" - - On Solaris, don't put `/usr/ucb' early in your `PATH'. This -directory contains several dysfunctional programs; working variants of -these programs are available in `/usr/bin'. So, if you need `/usr/ucb' -in your `PATH', put it _after_ `/usr/bin'. - - On Haiku, software installed for all users goes in `/boot/common', -not `/usr/local'. It is recommended to use the following options: - - ./configure --prefix=/boot/common - -Specifying the System Type -========================== - - There may be some features `configure' cannot figure out -automatically, but needs to determine by the type of machine the package -will run on. Usually, assuming the package is built to be run on the -_same_ architectures, `configure' can figure that out, but if it prints -a message saying it cannot guess the machine type, give it the -`--build=TYPE' option. TYPE can either be a short name for the system -type, such as `sun4', or a canonical name which has the form: - - CPU-COMPANY-SYSTEM - -where SYSTEM can have one of these forms: - - OS - KERNEL-OS - - See the file `config.sub' for the possible values of each field. If -`config.sub' isn't included in this package, then this package doesn't -need to know the machine type. - - If you are _building_ compiler tools for cross-compiling, you should -use the option `--target=TYPE' to select the type of system they will -produce code for. - - If you want to _use_ a cross compiler, that generates code for a -platform different from the build platform, you should specify the -"host" platform (i.e., that on which the generated programs will -eventually be run) with `--host=TYPE'. - -Sharing Defaults -================ - - If you want to set default values for `configure' scripts to share, -you can create a site shell script called `config.site' that gives -default values for variables like `CC', `cache_file', and `prefix'. -`configure' looks for `PREFIX/share/config.site' if it exists, then -`PREFIX/etc/config.site' if it exists. Or, you can set the -`CONFIG_SITE' environment variable to the location of the site script. -A warning: not all `configure' scripts look for a site script. - -Defining Variables -================== - - Variables not defined in a site shell script can be set in the -environment passed to `configure'. However, some packages may run -configure again during the build, and the customized values of these -variables may be lost. In order to avoid this problem, you should set -them in the `configure' command line, using `VAR=value'. For example: - - ./configure CC=/usr/local2/bin/gcc - -causes the specified `gcc' to be used as the C compiler (unless it is -overridden in the site shell script). - -Unfortunately, this technique does not work for `CONFIG_SHELL' due to -an Autoconf limitation. Until the limitation is lifted, you can use -this workaround: - - CONFIG_SHELL=/bin/bash ./configure CONFIG_SHELL=/bin/bash - -`configure' Invocation -====================== - - `configure' recognizes the following options to control how it -operates. - -`--help' -`-h' - Print a summary of all of the options to `configure', and exit. - -`--help=short' -`--help=recursive' - Print a summary of the options unique to this package's - `configure', and exit. The `short' variant lists options used - only in the top level, while the `recursive' variant lists options - also present in any nested packages. - -`--version' -`-V' - Print the version of Autoconf used to generate the `configure' - script, and exit. - -`--cache-file=FILE' - Enable the cache: use and save the results of the tests in FILE, - traditionally `config.cache'. FILE defaults to `/dev/null' to - disable caching. - -`--config-cache' -`-C' - Alias for `--cache-file=config.cache'. - -`--quiet' -`--silent' -`-q' - Do not print messages saying which checks are being made. To - suppress all normal output, redirect it to `/dev/null' (any error - messages will still be shown). - -`--srcdir=DIR' - Look for the package's source code in directory DIR. Usually - `configure' can determine that directory automatically. - -`--prefix=DIR' - Use DIR as the installation prefix. *note Installation Names:: - for more details, including other options available for fine-tuning - the installation locations. - -`--no-create' -`-n' - Run the configure checks, but stop before creating any output - files. - -`configure' also accepts some other, not widely useful, options. Run -`configure --help' for more details. diff --git a/src/third_party/pcre-8.39/LICENCE b/src/third_party/pcre-8.39/LICENCE deleted file mode 100644 index dd977af971b..00000000000 --- a/src/third_party/pcre-8.39/LICENCE +++ /dev/null @@ -1,93 +0,0 @@ -PCRE LICENCE ------------- - -PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - -Release 8 of PCRE is distributed under the terms of the "BSD" licence, as -specified below. The documentation for PCRE, supplied in the "doc" -directory, is distributed under the same terms as the software itself. The data -in the testdata directory is not copyrighted and is in the public domain. - -The basic library functions are written in C and are freestanding. Also -included in the distribution is a set of C++ wrapper functions, and a -just-in-time compiler that can be used to optimize pattern matching. These -are both optional features that can be omitted when the library is built. - - -THE BASIC LIBRARY FUNCTIONS ---------------------------- - -Written by: Philip Hazel -Email local part: ph10 -Email domain: cam.ac.uk - -University of Cambridge Computing Service, -Cambridge, England. - -Copyright (c) 1997-2016 University of Cambridge -All rights reserved. - - -PCRE JUST-IN-TIME COMPILATION SUPPORT -------------------------------------- - -Written by: Zoltan Herczeg -Email local part: hzmester -Emain domain: freemail.hu - -Copyright(c) 2010-2016 Zoltan Herczeg -All rights reserved. - - -STACK-LESS JUST-IN-TIME COMPILER --------------------------------- - -Written by: Zoltan Herczeg -Email local part: hzmester -Emain domain: freemail.hu - -Copyright(c) 2009-2016 Zoltan Herczeg -All rights reserved. - - -THE C++ WRAPPER FUNCTIONS -------------------------- - -Contributed by: Google Inc. - -Copyright (c) 2007-2012, Google Inc. -All rights reserved. - - -THE "BSD" LICENCE ------------------ - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the name of Google - Inc. nor the names of their contributors may be used to endorse or - promote products derived from this software without specific prior - written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -End diff --git a/src/third_party/pcre-8.39/NEWS b/src/third_party/pcre-8.39/NEWS deleted file mode 100644 index 0ca1bab2a4c..00000000000 --- a/src/third_party/pcre-8.39/NEWS +++ /dev/null @@ -1,725 +0,0 @@ -News about PCRE releases ------------------------- - -Release 8.39 14-June-2016 -------------------------- - -Some appropriate PCRE2 JIT improvements have been retro-fitted to PCRE1. Apart -from that, this is another bug-fix release. Note that this library (now called -PCRE1) is now being maintained for bug fixes only. New projects are advised to -use the new PCRE2 libraries. - - -Release 8.38 23-November-2015 ------------------------------ - -This is bug-fix release. Note that this library (now called PCRE1) is now being -maintained for bug fixes only. New projects are advised to use the new PCRE2 -libraries. - - -Release 8.37 28-April-2015 --------------------------- - -This is bug-fix release. Note that this library (now called PCRE1) is now being -maintained for bug fixes only. New projects are advised to use the new PCRE2 -libraries. - - -Release 8.36 26-September-2014 ------------------------------- - -This is primarily a bug-fix release. However, in addition, the Unicode data -tables have been updated to Unicode 7.0.0. - - -Release 8.35 04-April-2014 --------------------------- - -There have been performance improvements for classes containing non-ASCII -characters and the "auto-possessification" feature has been extended. Other -minor improvements have been implemented and bugs fixed. There is a new callout -feature to enable applications to do detailed stack checks at compile time, to -avoid running out of stack for deeply nested parentheses. The JIT compiler has -been extended with experimental support for ARM-64, MIPS-64, and PPC-LE. - - -Release 8.34 15-December-2013 ------------------------------ - -As well as fixing the inevitable bugs, performance has been improved by -refactoring and extending the amount of "auto-possessification" that PCRE does. -Other notable changes: - -. Implemented PCRE_INFO_MATCH_EMPTY, which yields 1 if the pattern can match - an empty string. If it can, pcretest shows this in its information output. - -. A back reference to a named subpattern when there is more than one of the - same name now checks them in the order in which they appear in the pattern. - The first one that is set is used for the reference. Previously only the - first one was inspected. This change makes PCRE more compatible with Perl. - -. Unicode character properties were updated from Unicode 6.3.0. - -. The character VT has been added to the set of characters that match \s and - are generally treated as white space, following this same change in Perl - 5.18. There is now no difference between "Perl space" and "POSIX space". - -. Perl has changed its handling of \8 and \9. If there is no previously - encountered capturing group of those numbers, they are treated as the - literal characters 8 and 9 instead of a binary zero followed by the - literals. PCRE now does the same. - -. Following Perl, added \o{} to specify codepoints in octal, making it - possible to specify values greater than 0777 and also making them - unambiguous. - -. In UCP mode, \s was not matching two of the characters that Perl matches, - namely NEL (U+0085) and MONGOLIAN VOWEL SEPARATOR (U+180E), though they - were matched by \h. - -. Add JIT support for the 64 bit TileGX architecture. - -. Upgraded the handling of the POSIX classes [:graph:], [:print:], and - [:punct:] when PCRE_UCP is set so as to include the same characters as Perl - does in Unicode mode. - -. Perl no longer allows group names to start with digits, so I have made this - change also in PCRE. - -. Added support for [[:<:]] and [[:>:]] as used in the BSD POSIX library to - mean "start of word" and "end of word", respectively, as a transition aid. - - -Release 8.33 28-May-2013 --------------------------- - -A number of bugs are fixed, and some performance improvements have been made. -There are also some new features, of which these are the most important: - -. The behaviour of the backtracking verbs has been rationalized and - documented in more detail. - -. JIT now supports callouts and all of the backtracking verbs. - -. Unicode validation has been updated in the light of Unicode Corrigendum #9, - which points out that "non characters" are not "characters that may not - appear in Unicode strings" but rather "characters that are reserved for - internal use and have only local meaning". - -. (*LIMIT_MATCH=d) and (*LIMIT_RECURSION=d) have been added so that the - creator of a pattern can specify lower (but not higher) limits for the - matching process. - -. The PCRE_NEVER_UTF option is available to prevent pattern-writers from using - the (*UTF) feature, as this could be a security issue. - - -Release 8.32 30-November-2012 ------------------------------ - -This release fixes a number of bugs, but also has some new features. These are -the highlights: - -. There is now support for 32-bit character strings and UTF-32. Like the - 16-bit support, this is done by compiling a separate 32-bit library. - -. \X now matches a Unicode extended grapheme cluster. - -. Case-independent matching of Unicode characters that have more than one - "other case" now makes all three (or more) characters equivalent. This - applies, for example, to Greek Sigma, which has two lowercase versions. - -. Unicode character properties are updated to Unicode 6.2.0. - -. The EBCDIC support, which had decayed, has had a spring clean. - -. A number of JIT optimizations have been added, which give faster JIT - execution speed. In addition, a new direct interface to JIT execution is - available. This bypasses some of the sanity checks of pcre_exec() to give a - noticeable speed-up. - -. A number of issues in pcregrep have been fixed, making it more compatible - with GNU grep. In particular, --exclude and --include (and variants) apply - to all files now, not just those obtained from scanning a directory - recursively. In Windows environments, the default action for directories is - now "skip" instead of "read" (which provokes an error). - -. If the --only-matching (-o) option in pcregrep is specified multiple - times, each one causes appropriate output. For example, -o1 -o2 outputs the - substrings matched by the 1st and 2nd capturing parentheses. A separating - string can be specified by --om-separator (default empty). - -. When PCRE is built via Autotools using a version of gcc that has the - "visibility" feature, it is used to hide internal library functions that are - not part of the public API. - - -Release 8.31 06-July-2012 -------------------------- - -This is mainly a bug-fixing release, with a small number of developments: - -. The JIT compiler now supports partial matching and the (*MARK) and - (*COMMIT) verbs. - -. PCRE_INFO_MAXLOOKBEHIND can be used to find the longest lookbehind in a - pattern. - -. There should be a performance improvement when using the heap instead of the - stack for recursion. - -. pcregrep can now be linked with libedit as an alternative to libreadline. - -. pcregrep now has a --file-list option where the list of files to scan is - given as a file. - -. pcregrep now recognizes binary files and there are related options. - -. The Unicode tables have been updated to 6.1.0. - -As always, the full list of changes is in the ChangeLog file. - - -Release 8.30 04-February-2012 ------------------------------ - -Release 8.30 introduces a major new feature: support for 16-bit character -strings, compiled as a separate library. There are a few changes to the -8-bit library, in addition to some bug fixes. - -. The pcre_info() function, which has been obsolete for over 10 years, has - been removed. - -. When a compiled pattern was saved to a file and later reloaded on a host - with different endianness, PCRE used automatically to swap the bytes in some - of the data fields. With the advent of the 16-bit library, where more of this - swapping is needed, it is no longer done automatically. Instead, the bad - endianness is detected and a specific error is given. The user can then call - a new function called pcre_pattern_to_host_byte_order() (or an equivalent - 16-bit function) to do the swap. - -. In UTF-8 mode, the values 0xd800 to 0xdfff are not legal Unicode - code points and are now faulted. (They are the so-called "surrogates" - that are reserved for coding high values in UTF-16.) - - -Release 8.21 12-Dec-2011 ------------------------- - -This is almost entirely a bug-fix release. The only new feature is the ability -to obtain the size of the memory used by the JIT compiler. - - -Release 8.20 21-Oct-2011 ------------------------- - -The main change in this release is the inclusion of Zoltan Herczeg's -just-in-time compiler support, which can be accessed by building PCRE with ---enable-jit. Large performance benefits can be had in many situations. 8.20 -also fixes an unfortunate bug that was introduced in 8.13 as well as tidying up -a number of infelicities and differences from Perl. - - -Release 8.13 16-Aug-2011 ------------------------- - -This is mainly a bug-fix release. There has been a lot of internal refactoring. -The Unicode tables have been updated. The only new feature in the library is -the passing of *MARK information to callouts. Some additions have been made to -pcretest to make testing easier and more comprehensive. There is a new option -for pcregrep to adjust its internal buffer size. - - -Release 8.12 15-Jan-2011 ------------------------- - -This release fixes some bugs in pcregrep, one of which caused the tests to fail -on 64-bit big-endian systems. There are no changes to the code of the library. - - -Release 8.11 10-Dec-2010 ------------------------- - -A number of bugs in the library and in pcregrep have been fixed. As always, see -ChangeLog for details. The following are the non-bug-fix changes: - -. Added --match-limit and --recursion-limit to pcregrep. - -. Added an optional parentheses number to the -o and --only-matching options - of pcregrep. - -. Changed the way PCRE_PARTIAL_HARD affects the matching of $, \z, \Z, \b, and - \B. - -. Added PCRE_ERROR_SHORTUTF8 to make it possible to distinguish between a - bad UTF-8 sequence and one that is incomplete when using PCRE_PARTIAL_HARD. - -. Recognize (*NO_START_OPT) at the start of a pattern to set the PCRE_NO_ - START_OPTIMIZE option, which is now allowed at compile time - - -Release 8.10 25-Jun-2010 ------------------------- - -There are two major additions: support for (*MARK) and friends, and the option -PCRE_UCP, which changes the behaviour of \b, \d, \s, and \w (and their -opposites) so that they make use of Unicode properties. There are also a number -of lesser new features, and several bugs have been fixed. A new option, ---line-buffered, has been added to pcregrep, for use when it is connected to -pipes. - - -Release 8.02 19-Mar-2010 ------------------------- - -Another bug-fix release. - - -Release 8.01 19-Jan-2010 ------------------------- - -This is a bug-fix release. Several bugs in the code itself and some bugs and -infelicities in the build system have been fixed. - - -Release 8.00 19-Oct-09 ----------------------- - -Bugs have been fixed in the library and in pcregrep. There are also some -enhancements. Restrictions on patterns used for partial matching have been -removed, extra information is given for partial matches, the partial matching -process has been improved, and an option to make a partial match override a -full match is available. The "study" process has been enhanced by finding a -lower bound matching length. Groups with duplicate numbers may now have -duplicated names without the use of PCRE_DUPNAMES. However, they may not have -different names. The documentation has been revised to reflect these changes. -The version number has been expanded to 3 digits as it is clear that the rate -of change is not slowing down. - - -Release 7.9 11-Apr-09 ---------------------- - -Mostly bugfixes and tidies with just a couple of minor functional additions. - - -Release 7.8 05-Sep-08 ---------------------- - -More bug fixes, plus a performance improvement in Unicode character property -lookup. - - -Release 7.7 07-May-08 ---------------------- - -This is once again mainly a bug-fix release, but there are a couple of new -features. - - -Release 7.6 28-Jan-08 ---------------------- - -The main reason for having this release so soon after 7.5 is because it fixes a -potential buffer overflow problem in pcre_compile() when run in UTF-8 mode. In -addition, the CMake configuration files have been brought up to date. - - -Release 7.5 10-Jan-08 ---------------------- - -This is mainly a bug-fix release. However the ability to link pcregrep with -libz or libbz2 and the ability to link pcretest with libreadline have been -added. Also the --line-offsets and --file-offsets options were added to -pcregrep. - - -Release 7.4 21-Sep-07 ---------------------- - -The only change of specification is the addition of options to control whether -\R matches any Unicode line ending (the default) or just CR, LF, and CRLF. -Otherwise, the changes are bug fixes and a refactoring to reduce the number of -relocations needed in a shared library. There have also been some documentation -updates, in particular, some more information about using CMake to build PCRE -has been added to the NON-UNIX-USE file. - - -Release 7.3 28-Aug-07 ---------------------- - -Most changes are bug fixes. Some that are not: - -1. There is some support for Perl 5.10's experimental "backtracking control - verbs" such as (*PRUNE). - -2. UTF-8 checking is now as per RFC 3629 instead of RFC 2279; this is more - restrictive in the strings it accepts. - -3. Checking for potential integer overflow has been made more dynamic, and as a - consequence there is no longer a hard limit on the size of a subpattern that - has a limited repeat count. - -4. When CRLF is a valid line-ending sequence, pcre_exec() and pcre_dfa_exec() - no longer advance by two characters instead of one when an unanchored match - fails at CRLF if there are explicit CR or LF matches within the pattern. - This gets rid of some anomalous effects that previously occurred. - -5. Some PCRE-specific settings for varying the newline options at the start of - a pattern have been added. - - -Release 7.2 19-Jun-07 ---------------------- - -WARNING: saved patterns that were compiled by earlier versions of PCRE must be -recompiled for use with 7.2 (necessitated by the addition of \K, \h, \H, \v, -and \V). - -Correction to the notes for 7.1: the note about shared libraries for Windows is -wrong. Previously, three libraries were built, but each could function -independently. For example, the pcreposix library also included all the -functions from the basic pcre library. The change is that the three libraries -are no longer independent. They are like the Unix libraries. To use the -pcreposix functions, for example, you need to link with both the pcreposix and -the basic pcre library. - -Some more features from Perl 5.10 have been added: - - (?-n) and (?+n) relative references for recursion and subroutines. - - (?(-n) and (?(+n) relative references as conditions. - - \k{name} and \g{name} are synonyms for \k. - - \K to reset the start of the matched string; for example, (foo)\Kbar - matches bar preceded by foo, but only sets bar as the matched string. - - (?| introduces a group where the capturing parentheses in each alternative - start from the same number; for example, (?|(abc)|(xyz)) sets capturing - parentheses number 1 in both cases. - - \h, \H, \v, \V match horizontal and vertical whitespace, respectively. - - -Release 7.1 24-Apr-07 ---------------------- - -There is only one new feature in this release: a linebreak setting of -PCRE_NEWLINE_ANYCRLF. It is a cut-down version of PCRE_NEWLINE_ANY, which -recognizes only CRLF, CR, and LF as linebreaks. - -A few bugs are fixed (see ChangeLog for details), but the major change is a -complete re-implementation of the build system. This now has full Autotools -support and so is now "standard" in some sense. It should help with compiling -PCRE in a wide variety of environments. - -NOTE: when building shared libraries for Windows, three dlls are now built, -called libpcre, libpcreposix, and libpcrecpp. Previously, everything was -included in a single dll. - -Another important change is that the dftables auxiliary program is no longer -compiled and run at "make" time by default. Instead, a default set of character -tables (assuming ASCII coding) is used. If you want to use dftables to generate -the character tables as previously, add --enable-rebuild-chartables to the -"configure" command. You must do this if you are compiling PCRE to run on a -system that uses EBCDIC code. - -There is a discussion about character tables in the README file. The default is -not to use dftables so that that there is no problem when cross-compiling. - - -Release 7.0 19-Dec-06 ---------------------- - -This release has a new major number because there have been some internal -upheavals to facilitate the addition of new optimizations and other facilities, -and to make subsequent maintenance and extension easier. Compilation is likely -to be a bit slower, but there should be no major effect on runtime performance. -Previously compiled patterns are NOT upwards compatible with this release. If -you have saved compiled patterns from a previous release, you will have to -re-compile them. Important changes that are visible to users are: - -1. The Unicode property tables have been updated to Unicode 5.0.0, which adds - some more scripts. - -2. The option PCRE_NEWLINE_ANY causes PCRE to recognize any Unicode newline - sequence as a newline. - -3. The \R escape matches a single Unicode newline sequence as a single unit. - -4. New features that will appear in Perl 5.10 are now in PCRE. These include - alternative Perl syntax for named parentheses, and Perl syntax for - recursion. - -5. The C++ wrapper interface has been extended by the addition of a - QuoteMeta function and the ability to allow copy construction and - assignment. - -For a complete list of changes, see the ChangeLog file. - - -Release 6.7 04-Jul-06 ---------------------- - -The main additions to this release are the ability to use the same name for -multiple sets of parentheses, and support for CRLF line endings in both the -library and pcregrep (and in pcretest for testing). - -Thanks to Ian Taylor, the stack usage for many kinds of pattern has been -significantly reduced for certain subject strings. - - -Release 6.5 01-Feb-06 ---------------------- - -Important changes in this release: - -1. A number of new features have been added to pcregrep. - -2. The Unicode property tables have been updated to Unicode 4.1.0, and the - supported properties have been extended with script names such as "Arabic", - and the derived properties "Any" and "L&". This has necessitated a change to - the interal format of compiled patterns. Any saved compiled patterns that - use \p or \P must be recompiled. - -3. The specification of recursion in patterns has been changed so that all - recursive subpatterns are automatically treated as atomic groups. Thus, for - example, (?R) is treated as if it were (?>(?R)). This is necessary because - otherwise there are situations where recursion does not work. - -See the ChangeLog for a complete list of changes, which include a number of bug -fixes and tidies. - - -Release 6.0 07-Jun-05 ---------------------- - -The release number has been increased to 6.0 because of the addition of several -major new pieces of functionality. - -A new function, pcre_dfa_exec(), which implements pattern matching using a DFA -algorithm, has been added. This has a number of advantages for certain cases, -though it does run more slowly, and lacks the ability to capture substrings. On -the other hand, it does find all matches, not just the first, and it works -better for partial matching. The pcrematching man page discusses the -differences. - -The pcretest program has been enhanced so that it can make use of the new -pcre_dfa_exec() matching function and the extra features it provides. - -The distribution now includes a C++ wrapper library. This is built -automatically if a C++ compiler is found. The pcrecpp man page discusses this -interface. - -The code itself has been re-organized into many more files, one for each -function, so it no longer requires everything to be linked in when static -linkage is used. As a consequence, some internal functions have had to have -their names exposed. These functions all have names starting with _pcre_. They -are undocumented, and are not intended for use by outside callers. - -The pcregrep program has been enhanced with new functionality such as -multiline-matching and options for output more matching context. See the -ChangeLog for a complete list of changes to the library and the utility -programs. - - -Release 5.0 13-Sep-04 ---------------------- - -The licence under which PCRE is released has been changed to the more -conventional "BSD" licence. - -In the code, some bugs have been fixed, and there are also some major changes -in this release (which is why I've increased the number to 5.0). Some changes -are internal rearrangements, and some provide a number of new facilities. The -new features are: - -1. There's an "automatic callout" feature that inserts callouts before every - item in the regex, and there's a new callout field that gives the position - in the pattern - useful for debugging and tracing. - -2. The extra_data structure can now be used to pass in a set of character - tables at exec time. This is useful if compiled regex are saved and re-used - at a later time when the tables may not be at the same address. If the - default internal tables are used, the pointer saved with the compiled - pattern is now set to NULL, which means that you don't need to do anything - special unless you are using custom tables. - -3. It is possible, with some restrictions on the content of the regex, to - request "partial" matching. A special return code is given if all of the - subject string matched part of the regex. This could be useful for testing - an input field as it is being typed. - -4. There is now some optional support for Unicode character properties, which - means that the patterns items such as \p{Lu} and \X can now be used. Only - the general category properties are supported. If PCRE is compiled with this - support, an additional 90K data structure is include, which increases the - size of the library dramatically. - -5. There is support for saving compiled patterns and re-using them later. - -6. There is support for running regular expressions that were compiled on a - different host with the opposite endianness. - -7. The pcretest program has been extended to accommodate the new features. - -The main internal rearrangement is that sequences of literal characters are no -longer handled as strings. Instead, each character is handled on its own. This -makes some UTF-8 handling easier, and makes the support of partial matching -possible. Compiled patterns containing long literal strings will be larger as a -result of this change; I hope that performance will not be much affected. - - -Release 4.5 01-Dec-03 ---------------------- - -Again mainly a bug-fix and tidying release, with only a couple of new features: - -1. It's possible now to compile PCRE so that it does not use recursive -function calls when matching. Instead it gets memory from the heap. This slows -things down, but may be necessary on systems with limited stacks. - -2. UTF-8 string checking has been tightened to reject overlong sequences and to -check that a starting offset points to the start of a character. Failure of the -latter returns a new error code: PCRE_ERROR_BADUTF8_OFFSET. - -3. PCRE can now be compiled for systems that use EBCDIC code. - - -Release 4.4 21-Aug-03 ---------------------- - -This is mainly a bug-fix and tidying release. The only new feature is that PCRE -checks UTF-8 strings for validity by default. There is an option to suppress -this, just in case anybody wants that teeny extra bit of performance. - - -Releases 4.1 - 4.3 ------------------- - -Sorry, I forgot about updating the NEWS file for these releases. Please take a -look at ChangeLog. - - -Release 4.0 17-Feb-03 ---------------------- - -There have been a lot of changes for the 4.0 release, adding additional -functionality and mending bugs. Below is a list of the highlights of the new -functionality. For full details of these features, please consult the -documentation. For a complete list of changes, see the ChangeLog file. - -1. Support for Perl's \Q...\E escapes. - -2. "Possessive quantifiers" ?+, *+, ++, and {,}+ which come from Sun's Java -package. They provide some syntactic sugar for simple cases of "atomic -grouping". - -3. Support for the \G assertion. It is true when the current matching position -is at the start point of the match. - -4. A new feature that provides some of the functionality that Perl provides -with (?{...}). The facility is termed a "callout". The way it is done in PCRE -is for the caller to provide an optional function, by setting pcre_callout to -its entry point. To get the function called, the regex must include (?C) at -appropriate points. - -5. Support for recursive calls to individual subpatterns. This makes it really -easy to get totally confused. - -6. Support for named subpatterns. The Python syntax (?P...) is used to -name a group. - -7. Several extensions to UTF-8 support; it is now fairly complete. There is an -option for pcregrep to make it operate in UTF-8 mode. - -8. The single man page has been split into a number of separate man pages. -These also give rise to individual HTML pages which are put in a separate -directory. There is an index.html page that lists them all. Some hyperlinking -between the pages has been installed. - - -Release 3.5 15-Aug-01 ---------------------- - -1. The configuring system has been upgraded to use later versions of autoconf -and libtool. By default it builds both a shared and a static library if the OS -supports it. You can use --disable-shared or --disable-static on the configure -command if you want only one of them. - -2. The pcretest utility is now installed along with pcregrep because it is -useful for users (to test regexs) and by doing this, it automatically gets -relinked by libtool. The documentation has been turned into a man page, so -there are now .1, .txt, and .html versions in /doc. - -3. Upgrades to pcregrep: - (i) Added long-form option names like gnu grep. - (ii) Added --help to list all options with an explanatory phrase. - (iii) Added -r, --recursive to recurse into sub-directories. - (iv) Added -f, --file to read patterns from a file. - -4. Added --enable-newline-is-cr and --enable-newline-is-lf to the configure -script, to force use of CR or LF instead of \n in the source. On non-Unix -systems, the value can be set in config.h. - -5. The limit of 200 on non-capturing parentheses is a _nesting_ limit, not an -absolute limit. Changed the text of the error message to make this clear, and -likewise updated the man page. - -6. The limit of 99 on the number of capturing subpatterns has been removed. -The new limit is 65535, which I hope will not be a "real" limit. - - -Release 3.3 01-Aug-00 ---------------------- - -There is some support for UTF-8 character strings. This is incomplete and -experimental. The documentation describes what is and what is not implemented. -Otherwise, this is just a bug-fixing release. - - -Release 3.0 01-Feb-00 ---------------------- - -1. A "configure" script is now used to configure PCRE for Unix systems. It -builds a Makefile, a config.h file, and the pcre-config script. - -2. PCRE is built as a shared library by default. - -3. There is support for POSIX classes such as [:alpha:]. - -5. There is an experimental recursion feature. - ----------------------------------------------------------------------------- - IMPORTANT FOR THOSE UPGRADING FROM VERSIONS BEFORE 2.00 - -Please note that there has been a change in the API such that a larger -ovector is required at matching time, to provide some additional workspace. -The new man page has details. This change was necessary in order to support -some of the new functionality in Perl 5.005. - - IMPORTANT FOR THOSE UPGRADING FROM VERSION 2.00 - -Another (I hope this is the last!) change has been made to the API for the -pcre_compile() function. An additional argument has been added to make it -possible to pass over a pointer to character tables built in the current -locale by pcre_maketables(). To use the default tables, this new argument -should be passed as NULL. - - IMPORTANT FOR THOSE UPGRADING FROM VERSION 2.05 - -Yet another (and again I hope this really is the last) change has been made -to the API for the pcre_exec() function. An additional argument has been -added to make it possible to start the match other than at the start of the -subject string. This is important if there are lookbehinds. The new man -page has the details, but you just want to convert existing programs, all -you need to do is to stick in a new fifth argument to pcre_exec(), with a -value of zero. For example, change - - pcre_exec(pattern, extra, subject, length, options, ovec, ovecsize) -to - pcre_exec(pattern, extra, subject, length, 0, options, ovec, ovecsize) - -**** diff --git a/src/third_party/pcre-8.39/NON-AUTOTOOLS-BUILD b/src/third_party/pcre-8.39/NON-AUTOTOOLS-BUILD deleted file mode 100644 index 3910059106b..00000000000 --- a/src/third_party/pcre-8.39/NON-AUTOTOOLS-BUILD +++ /dev/null @@ -1,772 +0,0 @@ -Building PCRE without using autotools -------------------------------------- - -NOTE: This document relates to PCRE releases that use the original API, with -library names libpcre, libpcre16, and libpcre32. January 2015 saw the first -release of a new API, known as PCRE2, with release numbers starting at 10.00 -and library names libpcre2-8, libpcre2-16, and libpcre2-32. The old libraries -(now called PCRE1) are still being maintained for bug fixes, but there will be -no new development. New projects are advised to use the new PCRE2 libraries. - - -This document contains the following sections: - - General - Generic instructions for the PCRE C library - The C++ wrapper functions - Building for virtual Pascal - Stack size in Windows environments - Linking programs in Windows environments - Calling conventions in Windows environments - Comments about Win32 builds - Building PCRE on Windows with CMake - Use of relative paths with CMake on Windows - Testing with RunTest.bat - Building under Windows CE with Visual Studio 200x - Building under Windows with BCC5.5 - Building using Borland C++ Builder 2007 (CB2007) and higher - Building PCRE on OpenVMS - Building PCRE on Stratus OpenVOS - Building PCRE on native z/OS and z/VM - - -GENERAL - -I (Philip Hazel) have no experience of Windows or VMS sytems and how their -libraries work. The items in the PCRE distribution and Makefile that relate to -anything other than Linux systems are untested by me. - -There are some other comments and files (including some documentation in CHM -format) in the Contrib directory on the FTP site: - - ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/Contrib - -The basic PCRE library consists entirely of code written in Standard C, and so -should compile successfully on any system that has a Standard C compiler and -library. The C++ wrapper functions are a separate issue (see below). - -The PCRE distribution includes a "configure" file for use by the configure/make -(autotools) build system, as found in many Unix-like environments. The README -file contains information about the options for "configure". - -There is also support for CMake, which some users prefer, especially in Windows -environments, though it can also be run in Unix-like environments. See the -section entitled "Building PCRE on Windows with CMake" below. - -Versions of config.h and pcre.h are distributed in the PCRE tarballs under the -names config.h.generic and pcre.h.generic. These are provided for those who -build PCRE without using "configure" or CMake. If you use "configure" or CMake, -the .generic versions are not used. - - -GENERIC INSTRUCTIONS FOR THE PCRE C LIBRARY - -The following are generic instructions for building the PCRE C library "by -hand". If you are going to use CMake, this section does not apply to you; you -can skip ahead to the CMake section. - - (1) Copy or rename the file config.h.generic as config.h, and edit the macro - settings that it contains to whatever is appropriate for your environment. - - In particular, you can alter the definition of the NEWLINE macro to - specify what character(s) you want to be interpreted as line terminators. - In an EBCDIC environment, you MUST change NEWLINE, because its default - value is 10, an ASCII LF. The usual EBCDIC newline character is 21 (0x15, - NL), though in some cases it may be 37 (0x25). - - When you compile any of the PCRE modules, you must specify -DHAVE_CONFIG_H - to your compiler so that config.h is included in the sources. - - An alternative approach is not to edit config.h, but to use -D on the - compiler command line to make any changes that you need to the - configuration options. In this case -DHAVE_CONFIG_H must not be set. - - NOTE: There have been occasions when the way in which certain parameters - in config.h are used has changed between releases. (In the configure/make - world, this is handled automatically.) When upgrading to a new release, - you are strongly advised to review config.h.generic before re-using what - you had previously. - - (2) Copy or rename the file pcre.h.generic as pcre.h. - - (3) EITHER: - Copy or rename file pcre_chartables.c.dist as pcre_chartables.c. - - OR: - Compile dftables.c as a stand-alone program (using -DHAVE_CONFIG_H if - you have set up config.h), and then run it with the single argument - "pcre_chartables.c". This generates a set of standard character tables - and writes them to that file. The tables are generated using the default - C locale for your system. If you want to use a locale that is specified - by LC_xxx environment variables, add the -L option to the dftables - command. You must use this method if you are building on a system that - uses EBCDIC code. - - The tables in pcre_chartables.c are defaults. The caller of PCRE can - specify alternative tables at run time. - - (4) Ensure that you have the following header files: - - pcre_internal.h - ucp.h - - (5) For an 8-bit library, compile the following source files, setting - -DHAVE_CONFIG_H as a compiler option if you have set up config.h with your - configuration, or else use other -D settings to change the configuration - as required. - - pcre_byte_order.c - pcre_chartables.c - pcre_compile.c - pcre_config.c - pcre_dfa_exec.c - pcre_exec.c - pcre_fullinfo.c - pcre_get.c - pcre_globals.c - pcre_jit_compile.c - pcre_maketables.c - pcre_newline.c - pcre_ord2utf8.c - pcre_refcount.c - pcre_string_utils.c - pcre_study.c - pcre_tables.c - pcre_ucd.c - pcre_valid_utf8.c - pcre_version.c - pcre_xclass.c - - Make sure that you include -I. in the compiler command (or equivalent for - an unusual compiler) so that all included PCRE header files are first - sought in the current directory. Otherwise you run the risk of picking up - a previously-installed file from somewhere else. - - Note that you must still compile pcre_jit_compile.c, even if you have not - defined SUPPORT_JIT in config.h, because when JIT support is not - configured, dummy functions are compiled. When JIT support IS configured, - pcre_jit_compile.c #includes sources from the sljit subdirectory, where - there should be 16 files, all of whose names begin with "sljit". - - (6) Now link all the compiled code into an object library in whichever form - your system keeps such libraries. This is the basic PCRE C 8-bit library. - If your system has static and shared libraries, you may have to do this - once for each type. - - (7) If you want to build a 16-bit library (as well as, or instead of the 8-bit - or 32-bit libraries) repeat steps 5-6 with the following files: - - pcre16_byte_order.c - pcre16_chartables.c - pcre16_compile.c - pcre16_config.c - pcre16_dfa_exec.c - pcre16_exec.c - pcre16_fullinfo.c - pcre16_get.c - pcre16_globals.c - pcre16_jit_compile.c - pcre16_maketables.c - pcre16_newline.c - pcre16_ord2utf16.c - pcre16_refcount.c - pcre16_string_utils.c - pcre16_study.c - pcre16_tables.c - pcre16_ucd.c - pcre16_utf16_utils.c - pcre16_valid_utf16.c - pcre16_version.c - pcre16_xclass.c - - (8) If you want to build a 32-bit library (as well as, or instead of the 8-bit - or 16-bit libraries) repeat steps 5-6 with the following files: - - pcre32_byte_order.c - pcre32_chartables.c - pcre32_compile.c - pcre32_config.c - pcre32_dfa_exec.c - pcre32_exec.c - pcre32_fullinfo.c - pcre32_get.c - pcre32_globals.c - pcre32_jit_compile.c - pcre32_maketables.c - pcre32_newline.c - pcre32_ord2utf32.c - pcre32_refcount.c - pcre32_string_utils.c - pcre32_study.c - pcre32_tables.c - pcre32_ucd.c - pcre32_utf32_utils.c - pcre32_valid_utf32.c - pcre32_version.c - pcre32_xclass.c - - (9) If you want to build the POSIX wrapper functions (which apply only to the - 8-bit library), ensure that you have the pcreposix.h file and then compile - pcreposix.c (remembering -DHAVE_CONFIG_H if necessary). Link the result - (on its own) as the pcreposix library. - -(10) The pcretest program can be linked with any combination of the 8-bit, - 16-bit and 32-bit libraries (depending on what you selected in config.h). - Compile pcretest.c and pcre_printint.c (again, don't forget - -DHAVE_CONFIG_H) and link them together with the appropriate library/ies. - If you compiled an 8-bit library, pcretest also needs the pcreposix - wrapper library unless you compiled it with -DNOPOSIX. - -(11) Run pcretest on the testinput files in the testdata directory, and check - that the output matches the corresponding testoutput files. There are - comments about what each test does in the section entitled "Testing PCRE" - in the README file. If you compiled more than one of the 8-bit, 16-bit and - 32-bit libraries, you need to run pcretest with the -16 option to do - 16-bit tests and with the -32 option to do 32-bit tests. - - Some tests are relevant only when certain build-time options are selected. - For example, test 4 is for UTF-8/UTF-16/UTF-32 support, and will not run - if you have built PCRE without it. See the comments at the start of each - testinput file. If you have a suitable Unix-like shell, the RunTest script - will run the appropriate tests for you. The command "RunTest list" will - output a list of all the tests. - - Note that the supplied files are in Unix format, with just LF characters - as line terminators. You may need to edit them to change this if your - system uses a different convention. If you are using Windows, you probably - should use the wintestinput3 file instead of testinput3 (and the - corresponding output file). This is a locale test; wintestinput3 sets the - locale to "french" rather than "fr_FR", and there some minor output - differences. - -(12) If you have built PCRE with SUPPORT_JIT, the JIT features will be tested - by the testdata files. However, you might also like to build and run - the freestanding JIT test program, pcre_jit_test.c. - -(13) If you want to use the pcregrep command, compile and link pcregrep.c; it - uses only the basic 8-bit PCRE library (it does not need the pcreposix - library). - - -THE C++ WRAPPER FUNCTIONS - -The PCRE distribution also contains some C++ wrapper functions and tests, -applicable to the 8-bit library, which were contributed by Google Inc. On a -system that can use "configure" and "make", the functions are automatically -built into a library called pcrecpp. It should be straightforward to compile -the .cc files manually on other systems. The files called xxx_unittest.cc are -test programs for each of the corresponding xxx.cc files. - - -BUILDING FOR VIRTUAL PASCAL - -A script for building PCRE using Borland's C++ compiler for use with VPASCAL -was contributed by Alexander Tokarev. Stefan Weber updated the script and added -additional files. The following files in the distribution are for building PCRE -for use with VP/Borland: makevp_c.txt, makevp_l.txt, makevp.bat, pcregexp.pas. - - -STACK SIZE IN WINDOWS ENVIRONMENTS - -The default processor stack size of 1Mb in some Windows environments is too -small for matching patterns that need much recursion. In particular, test 2 may -fail because of this. Normally, running out of stack causes a crash, but there -have been cases where the test program has just died silently. See your linker -documentation for how to increase stack size if you experience problems. The -Linux default of 8Mb is a reasonable choice for the stack, though even that can -be too small for some pattern/subject combinations. - -PCRE has a compile configuration option to disable the use of stack for -recursion so that heap is used instead. However, pattern matching is -significantly slower when this is done. There is more about stack usage in the -"pcrestack" documentation. - - -LINKING PROGRAMS IN WINDOWS ENVIRONMENTS - -If you want to statically link a program against a PCRE library in the form of -a non-dll .a file, you must define PCRE_STATIC before including pcre.h or -pcrecpp.h, otherwise the pcre_malloc() and pcre_free() exported functions will -be declared __declspec(dllimport), with unwanted results. - - -CALLING CONVENTIONS IN WINDOWS ENVIRONMENTS - -It is possible to compile programs to use different calling conventions using -MSVC. Search the web for "calling conventions" for more information. To make it -easier to change the calling convention for the exported functions in the -PCRE library, the macro PCRE_CALL_CONVENTION is present in all the external -definitions. It can be set externally when compiling (e.g. in CFLAGS). If it is -not set, it defaults to empty; the default calling convention is then used -(which is what is wanted most of the time). - - -COMMENTS ABOUT WIN32 BUILDS (see also "BUILDING PCRE ON WINDOWS WITH CMAKE") - -There are two ways of building PCRE using the "configure, make, make install" -paradigm on Windows systems: using MinGW or using Cygwin. These are not at all -the same thing; they are completely different from each other. There is also -support for building using CMake, which some users find a more straightforward -way of building PCRE under Windows. - -The MinGW home page (http://www.mingw.org/) says this: - - MinGW: A collection of freely available and freely distributable Windows - specific header files and import libraries combined with GNU toolsets that - allow one to produce native Windows programs that do not rely on any - 3rd-party C runtime DLLs. - -The Cygwin home page (http://www.cygwin.com/) says this: - - Cygwin is a Linux-like environment for Windows. It consists of two parts: - - . A DLL (cygwin1.dll) which acts as a Linux API emulation layer providing - substantial Linux API functionality - - . A collection of tools which provide Linux look and feel. - - The Cygwin DLL currently works with all recent, commercially released x86 32 - bit and 64 bit versions of Windows, with the exception of Windows CE. - -On both MinGW and Cygwin, PCRE should build correctly using: - - ./configure && make && make install - -This should create two libraries called libpcre and libpcreposix, and, if you -have enabled building the C++ wrapper, a third one called libpcrecpp. These are -independent libraries: when you link with libpcreposix or libpcrecpp you must -also link with libpcre, which contains the basic functions. (Some earlier -releases of PCRE included the basic libpcre functions in libpcreposix. This no -longer happens.) - -A user submitted a special-purpose patch that makes it easy to create -"pcre.dll" under mingw32 using the "msys" environment. It provides "pcre.dll" -as a special target. If you use this target, no other files are built, and in -particular, the pcretest and pcregrep programs are not built. An example of how -this might be used is: - - ./configure --enable-utf --disable-cpp CFLAGS="-03 -s"; make pcre.dll - -Using Cygwin's compiler generates libraries and executables that depend on -cygwin1.dll. If a library that is generated this way is distributed, -cygwin1.dll has to be distributed as well. Since cygwin1.dll is under the GPL -licence, this forces not only PCRE to be under the GPL, but also the entire -application. A distributor who wants to keep their own code proprietary must -purchase an appropriate Cygwin licence. - -MinGW has no such restrictions. The MinGW compiler generates a library or -executable that can run standalone on Windows without any third party dll or -licensing issues. - -But there is more complication: - -If a Cygwin user uses the -mno-cygwin Cygwin gcc flag, what that really does is -to tell Cygwin's gcc to use the MinGW gcc. Cygwin's gcc is only acting as a -front end to MinGW's gcc (if you install Cygwin's gcc, you get both Cygwin's -gcc and MinGW's gcc). So, a user can: - -. Build native binaries by using MinGW or by getting Cygwin and using - -mno-cygwin. - -. Build binaries that depend on cygwin1.dll by using Cygwin with the normal - compiler flags. - -The test files that are supplied with PCRE are in UNIX format, with LF -characters as line terminators. Unless your PCRE library uses a default newline -option that includes LF as a valid newline, it may be necessary to change the -line terminators in the test files to get some of the tests to work. - - -BUILDING PCRE ON WINDOWS WITH CMAKE - -CMake is an alternative configuration facility that can be used instead of -"configure". CMake creates project files (make files, solution files, etc.) -tailored to numerous development environments, including Visual Studio, -Borland, Msys, MinGW, NMake, and Unix. If possible, use short paths with no -spaces in the names for your CMake installation and your PCRE source and build -directories. - -The following instructions were contributed by a PCRE user. If they are not -followed exactly, errors may occur. In the event that errors do occur, it is -recommended that you delete the CMake cache before attempting to repeat the -CMake build process. In the CMake GUI, the cache can be deleted by selecting -"File > Delete Cache". - -1. Install the latest CMake version available from http://www.cmake.org/, and - ensure that cmake\bin is on your path. - -2. Unzip (retaining folder structure) the PCRE source tree into a source - directory such as C:\pcre. You should ensure your local date and time - is not earlier than the file dates in your source dir if the release is - very new. - -3. Create a new, empty build directory, preferably a subdirectory of the - source dir. For example, C:\pcre\pcre-xx\build. - -4. Run cmake-gui from the Shell envirornment of your build tool, for example, - Msys for Msys/MinGW or Visual Studio Command Prompt for VC/VC++. Do not try - to start Cmake from the Windows Start menu, as this can lead to errors. - -5. Enter C:\pcre\pcre-xx and C:\pcre\pcre-xx\build for the source and build - directories, respectively. - -6. Hit the "Configure" button. - -7. Select the particular IDE / build tool that you are using (Visual - Studio, MSYS makefiles, MinGW makefiles, etc.) - -8. The GUI will then list several configuration options. This is where - you can enable UTF-8 support or other PCRE optional features. - -9. Hit "Configure" again. The adjacent "Generate" button should now be - active. - -10. Hit "Generate". - -11. The build directory should now contain a usable build system, be it a - solution file for Visual Studio, makefiles for MinGW, etc. Exit from - cmake-gui and use the generated build system with your compiler or IDE. - E.g., for MinGW you can run "make", or for Visual Studio, open the PCRE - solution, select the desired configuration (Debug, or Release, etc.) and - build the ALL_BUILD project. - -12. If during configuration with cmake-gui you've elected to build the test - programs, you can execute them by building the test project. E.g., for - MinGW: "make test"; for Visual Studio build the RUN_TESTS project. The - most recent build configuration is targeted by the tests. A summary of - test results is presented. Complete test output is subsequently - available for review in Testing\Temporary under your build dir. - - -USE OF RELATIVE PATHS WITH CMAKE ON WINDOWS - -A PCRE user comments as follows: I thought that others may want to know the -current state of CMAKE_USE_RELATIVE_PATHS support on Windows. Here it is: - --- AdditionalIncludeDirectories is only partially modified (only the - first path - see below) --- Only some of the contained file paths are modified - shown below for - pcre.vcproj --- It properly modifies - -I am sure CMake people can fix that if they want to. Until then one will -need to replace existing absolute paths in project files with relative -paths manually (e.g. from VS) - relative to project file location. I did -just that before being told to try CMAKE_USE_RELATIVE_PATHS. Not a big -deal. - -AdditionalIncludeDirectories="E:\builds\pcre\build;E:\builds\pcre\pcre-7.5;" -AdditionalIncludeDirectories=".;E:\builds\pcre\pcre-7.5;" - -RelativePath="pcre.h" -RelativePath="pcre_chartables.c" -RelativePath="pcre_chartables.c.rule" - - -TESTING WITH RUNTEST.BAT - -If configured with CMake, building the test project ("make test" or building -ALL_TESTS in Visual Studio) creates (and runs) pcre_test.bat (and depending -on your configuration options, possibly other test programs) in the build -directory. Pcre_test.bat runs RunTest.Bat with correct source and exe paths. - -For manual testing with RunTest.bat, provided the build dir is a subdirectory -of the source directory: Open command shell window. Chdir to the location -of your pcretest.exe and pcregrep.exe programs. Call RunTest.bat with -"..\RunTest.Bat" or "..\..\RunTest.bat" as appropriate. - -To run only a particular test with RunTest.Bat provide a test number argument. - -Otherwise: - -1. Copy RunTest.bat into the directory where pcretest.exe and pcregrep.exe - have been created. - -2. Edit RunTest.bat to indentify the full or relative location of - the pcre source (wherein which the testdata folder resides), e.g.: - - set srcdir=C:\pcre\pcre-8.20 - -3. In a Windows command environment, chdir to the location of your bat and - exe programs. - -4. Run RunTest.bat. Test outputs will automatically be compared to expected - results, and discrepancies will be identified in the console output. - -To independently test the just-in-time compiler, run pcre_jit_test.exe. -To test pcrecpp, run pcrecpp_unittest.exe, pcre_stringpiece_unittest.exe and -pcre_scanner_unittest.exe. - - -BUILDING UNDER WINDOWS CE WITH VISUAL STUDIO 200x - -Vincent Richomme sent a zip archive of files to help with this process. They -can be found in the file "pcre-vsbuild.zip" in the Contrib directory of the FTP -site. - - -BUILDING UNDER WINDOWS WITH BCC5.5 - -Michael Roy sent these comments about building PCRE under Windows with BCC5.5: - -Some of the core BCC libraries have a version of PCRE from 1998 built in, which -can lead to pcre_exec() giving an erroneous PCRE_ERROR_NULL from a version -mismatch. I'm including an easy workaround below, if you'd like to include it -in the non-unix instructions: - -When linking a project with BCC5.5, pcre.lib must be included before any of the -libraries cw32.lib, cw32i.lib, cw32mt.lib, and cw32mti.lib on the command line. - - -BUILDING USING BORLAND C++ BUILDER 2007 (CB2007) AND HIGHER - -A PCRE user sent these comments about this environment (see also the comment -from another user that follows them): - -The XE versions of C++ Builder come with a RegularExpressionsCore class which -contain a version of TPerlRegEx. However, direct use of the C PCRE library may -be desirable. - -The default makevp.bat, however, supplied with PCRE builds a version of PCRE -that is not usable with any version of C++ Builder because the compiler ships -with an embedded version of PCRE, version 2.01 from 1998! [See also the note -about BCC5.5 above.] If you want to use PCRE you'll need to rename the -functions (pcre_compile to pcre_compile_bcc, etc) or do as I have done and just -use the 16 bit versions. I'm using std::wstring everywhere anyway. Since the -embedded version of PCRE does not have the 16 bit function names, there is no -conflict. - -Building PCRE using a C++ Builder static library project file (recommended): - -1. Rename or remove pcre.h, pcreposi.h, and pcreposix.h from your C++ Builder -original include path. - -2. Download PCRE from pcre.org and extract to a directory. - -3. Rename pcre_chartables.c.dist to pcre_chartables.c, pcre.h.generic to -pcre.h, and config.h.generic to config.h. - -4. Edit pcre.h and pcre_config.c so that they include config.h. - -5. Edit config.h like so: - -Comment out the following lines: -#define PACKAGE "pcre" -#define PACKAGE_BUGREPORT "" -#define PACKAGE_NAME "PCRE" -#define PACKAGE_STRING "PCRE 8.32" -#define PACKAGE_TARNAME "pcre" -#define PACKAGE_URL "" -#define PACKAGE_VERSION "8.32" - -Add the following lines: -#ifndef SUPPORT_UTF -#define SUPPORT_UTF 100 // any value is fine -#endif - -#ifndef SUPPORT_UCP -#define SUPPORT_UCP 101 // any value is fine -#endif - -#ifndef SUPPORT_UCP -#define SUPPORT_PCRE16 102 // any value is fine -#endif - -#ifndef SUPPORT_UTF8 -#define SUPPORT_UTF8 103 // any value is fine -#endif - -6. Build a C++ Builder project using the IDE. Go to File / New / Other and -choose Static Library. You can name it pcre.cbproj or whatever. Now set your -paths by going to Project / Options. Set the Include path. Do this from the -"Base" option to apply to both Release and Debug builds. Now add the following -files to the project: - -pcre.h -pcre16_byte_order.c -pcre16_chartables.c -pcre16_compile.c -pcre16_config.c -pcre16_dfa_exec.c -pcre16_exec.c -pcre16_fullinfo.c -pcre16_get.c -pcre16_globals.c -pcre16_maketables.c -pcre16_newline.c -pcre16_ord2utf16.c -pcre16_printint.c -pcre16_refcount.c -pcre16_string_utils.c -pcre16_study.c -pcre16_tables.c -pcre16_ucd.c -pcre16_utf16_utils.c -pcre16_valid_utf16.c -pcre16_version.c -pcre16_xclass.c - -//Optional -pcre_version.c - -7. After compiling the .lib file, copy the .lib and header files to a project -you want to use PCRE with. Enjoy. - -Optional ... Building PCRE using the makevp.bat file: - -1. Edit makevp_c.txt and makevp_l.txt and change all the names to the 16 bit -versions. - -2. Edit makevp.bat and set the path to C++ Builder. Run makevp.bat. - -Another PCRE user added this comment: - -Another approach I successfully used for some years with BCB 5 and 6 was to -make sure that include and library paths of PCRE are configured before the -default paths of the IDE in the dialogs where one can manage those paths. -Afterwards one can open the project files using a text editor and manually add -the self created library for pcre itself, pcrecpp doesn't ship with the IDE, in -the library nodes where the IDE manages its own libraries to link against in -front of the IDE-own libraries. This way one can use the default PCRE function -names without getting access violations on runtime. - - - - -BUILDING PCRE ON OPENVMS - -Stephen Hoffman sent the following, in December 2012: - -"Here is a very short write-up on the -OpenVMS port and here - - - -is a zip with the OpenVMS files, and with one modified testing-related PCRE -file." This is a port of PCRE 8.32. - -Earlier, Dan Mooney sent the following comments about building PCRE on OpenVMS. -They relate to an older version of PCRE that used fewer source files, so the -exact commands will need changing. See the current list of source files above. - -"It was quite easy to compile and link the library. I don't have a formal -make file but the attached file [reproduced below] contains the OpenVMS DCL -commands I used to build the library. I had to add #define -POSIX_MALLOC_THRESHOLD 10 to pcre.h since it was not defined anywhere. - -The library was built on: -O/S: HP OpenVMS v7.3-1 -Compiler: Compaq C v6.5-001-48BCD -Linker: vA13-01 - -The test results did not match 100% due to the issues you mention in your -documentation regarding isprint(), iscntrl(), isgraph() and ispunct(). I -modified some of the character tables temporarily and was able to get the -results to match. Tests using the fr locale did not match since I don't have -that locale loaded. The study size was always reported to be 3 less than the -value in the standard test output files." - -========================= -$! This DCL procedure builds PCRE on OpenVMS -$! -$! I followed the instructions in the non-unix-use file in the distribution. -$! -$ COMPILE == "CC/LIST/NOMEMBER_ALIGNMENT/PREFIX_LIBRARY_ENTRIES=ALL_ENTRIES -$ COMPILE DFTABLES.C -$ LINK/EXE=DFTABLES.EXE DFTABLES.OBJ -$ RUN DFTABLES.EXE/OUTPUT=CHARTABLES.C -$ COMPILE MAKETABLES.C -$ COMPILE GET.C -$ COMPILE STUDY.C -$! I had to set POSIX_MALLOC_THRESHOLD to 10 in PCRE.H since the symbol -$! did not seem to be defined anywhere. -$! I edited pcre.h and added #DEFINE SUPPORT_UTF8 to enable UTF8 support. -$ COMPILE PCRE.C -$ LIB/CREATE PCRE MAKETABLES.OBJ, GET.OBJ, STUDY.OBJ, PCRE.OBJ -$! I had to set POSIX_MALLOC_THRESHOLD to 10 in PCRE.H since the symbol -$! did not seem to be defined anywhere. -$ COMPILE PCREPOSIX.C -$ LIB/CREATE PCREPOSIX PCREPOSIX.OBJ -$ COMPILE PCRETEST.C -$ LINK/EXE=PCRETEST.EXE PCRETEST.OBJ, PCRE/LIB, PCREPOSIX/LIB -$! C programs that want access to command line arguments must be -$! defined as a symbol -$ PCRETEST :== "$ SYS$ROADSUSERS:[DMOONEY.REGEXP]PCRETEST.EXE" -$! Arguments must be enclosed in quotes. -$ PCRETEST "-C" -$! Test results: -$! -$! The test results did not match 100%. The functions isprint(), iscntrl(), -$! isgraph() and ispunct() on OpenVMS must not produce the same results -$! as the system that built the test output files provided with the -$! distribution. -$! -$! The study size did not match and was always 3 less on OpenVMS. -$! -$! Locale could not be set to fr -$! -========================= - - -BUILDING PCRE ON STRATUS OPENVOS - -These notes on the port of PCRE to VOS (lightly edited) were supplied by -Ashutosh Warikoo, whose email address has the local part awarikoo and the -domain nse.co.in. The port was for version 7.9 in August 2009. - -1. Building PCRE - -I built pcre on OpenVOS Release 17.0.1at using GNU Tools 3.4a without any -problems. I used the following packages to build PCRE: - - ftp://ftp.stratus.com/pub/vos/posix/ga/posix.save.evf.gz - -Please read and follow the instructions that come with these packages. To start -the build of pcre, from the root of the package type: - - ./build.sh - -2. Installing PCRE - -Once you have successfully built PCRE, login to the SysAdmin group, switch to -the root user, and type - - [ !create_dir (master_disk)>usr --if needed ] - [ !create_dir (master_disk)>usr>local --if needed ] - !gmake install - -This installs PCRE and its man pages into /usr/local. You can add -(master_disk)>usr>local>bin to your command search paths, or if you are in -BASH, add /usr/local/bin to the PATH environment variable. - -4. Restrictions - -This port requires readline library optionally. However during the build I -faced some yet unexplored errors while linking with readline. As it was an -optional component I chose to disable it. - -5. Known Problems - -I ran the test suite, but you will have to be your own judge of whether this -command, and this port, suits your purposes. If you find any problems that -appear to be related to the port itself, please let me know. Please see the -build.log file in the root of the package also. - - -BUILDING PCRE ON NATIVE Z/OS AND Z/VM - -z/OS and z/VM are operating systems for mainframe computers, produced by IBM. -The character code used is EBCDIC, not ASCII or Unicode. In z/OS, UNIX APIs and -applications can be supported through UNIX System Services, and in such an -environment PCRE can be built in the same way as in other systems. However, in -native z/OS (without UNIX System Services) and in z/VM, special ports are -required. For details, please see this web site: - - http://www.zaconsultants.net - -You may download PCRE from WWW.CBTTAPE.ORG, file 882.  Everything, source and -executable, is in EBCDIC and native z/OS file formats and this is the -recommended download site. - -========================== -Last Updated: 25 June 2015 diff --git a/src/third_party/pcre-8.39/NON-UNIX-USE b/src/third_party/pcre-8.39/NON-UNIX-USE deleted file mode 100644 index a25546b6ff5..00000000000 --- a/src/third_party/pcre-8.39/NON-UNIX-USE +++ /dev/null @@ -1,7 +0,0 @@ -Compiling PCRE on non-Unix systems ----------------------------------- - -This has been renamed to better reflect its contents. Please see the file -NON-AUTOTOOLS-BUILD for details of how to build PCRE without using autotools. - -#### diff --git a/src/third_party/pcre-8.39/PrepareRelease b/src/third_party/pcre-8.39/PrepareRelease deleted file mode 100755 index 9891e08d67f..00000000000 --- a/src/third_party/pcre-8.39/PrepareRelease +++ /dev/null @@ -1,258 +0,0 @@ -#/bin/sh - -# Script to prepare the files for building a PCRE release. It does some -# processing of the documentation, detrails files, and creates pcre.h.generic -# and config.h.generic (for use by builders who can't run ./configure). - -# You must run this script before runnning "make dist". If its first argument -# is "doc", it stops after preparing the documentation. There are no other -# arguments. The script makes use of the following files: - -# 132html A Perl script that converts a .1 or .3 man page into HTML. It -# "knows" the relevant troff constructs that are used in the PCRE -# man pages. - -# CheckMan A Perl script that checks man pages for typos in the mark up. - -# CleanTxt A Perl script that cleans up the output of "nroff -man" by -# removing backspaces and other redundant text so as to produce -# a readable .txt file. - -# Detrail A Perl script that removes trailing spaces from files. - -# doc/index.html.src -# A file that is copied as index.html into the doc/html directory -# when the HTML documentation is built. It works like this so that -# doc/html can be deleted and re-created from scratch. - -# README & NON-AUTOTOOLS-BUILD -# These files are copied into the doc/html directory, with .txt -# extensions so that they can by hyperlinked from the HTML -# documentation, because some people just go to the HTML without -# looking for text files. - - -# First, sort out the documentation. Remove pcredemo.3 first because it won't -# pass the markup check (it is created below, using markup that none of the -# other pages use). - -cd doc -echo Processing documentation - -/bin/rm -f pcredemo.3 - -# Check the remaining man pages - -perl ../CheckMan *.1 *.3 -if [ $? != 0 ] ; then exit 1; fi - -# Make Text form of the documentation. It needs some mangling to make it -# tidy for online reading. Concatenate all the .3 stuff, but omit the -# individual function pages. - -cat <pcre.txt ------------------------------------------------------------------------------ -This file contains a concatenation of the PCRE man pages, converted to plain -text format for ease of searching with a text editor, or for use on systems -that do not have a man page processor. The small individual files that give -synopses of each function in the library have not been included. Neither has -the pcredemo program. There are separate text files for the pcregrep and -pcretest commands. ------------------------------------------------------------------------------ - - -End - -echo "Making pcre.txt" -for file in pcre pcre16 pcre32 pcrebuild pcrematching pcreapi pcrecallout \ - pcrecompat pcrepattern pcresyntax pcreunicode pcrejit pcrepartial \ - pcreprecompile pcreperform pcreposix pcrecpp pcresample \ - pcrelimits pcrestack ; do - echo " Processing $file.3" - nroff -c -man $file.3 >$file.rawtxt - perl ../CleanTxt <$file.rawtxt >>pcre.txt - /bin/rm $file.rawtxt - echo "------------------------------------------------------------------------------" >>pcre.txt - if [ "$file" != "pcresample" ] ; then - echo " " >>pcre.txt - echo " " >>pcre.txt - fi -done - -# The three commands -for file in pcretest pcregrep pcre-config ; do - echo Making $file.txt - nroff -c -man $file.1 >$file.rawtxt - perl ../CleanTxt <$file.rawtxt >$file.txt - /bin/rm $file.rawtxt -done - - -# Make pcredemo.3 from the pcredemo.c source file - -echo "Making pcredemo.3" -perl <<"END" >pcredemo.3 - open(IN, "../pcredemo.c") || die "Failed to open pcredemo.c\n"; - open(OUT, ">pcredemo.3") || die "Failed to open pcredemo.3\n"; - print OUT ".\\\" Start example.\n" . - ".de EX\n" . - ". nr mE \\\\n(.f\n" . - ". nf\n" . - ". nh\n" . - ". ft CW\n" . - "..\n" . - ".\n" . - ".\n" . - ".\\\" End example.\n" . - ".de EE\n" . - ". ft \\\\n(mE\n" . - ". fi\n" . - ". hy \\\\n(HY\n" . - "..\n" . - ".\n" . - ".EX\n" ; - while () - { - s/\\/\\e/g; - print OUT; - } - print OUT ".EE\n"; - close(IN); - close(OUT); -END -if [ $? != 0 ] ; then exit 1; fi - - -# Make HTML form of the documentation. - -echo "Making HTML documentation" -/bin/rm html/* -cp index.html.src html/index.html -cp ../README html/README.txt -cp ../NON-AUTOTOOLS-BUILD html/NON-AUTOTOOLS-BUILD.txt - -for file in *.1 ; do - base=`basename $file .1` - echo " Making $base.html" - perl ../132html -toc $base <$file >html/$base.html -done - -# Exclude table of contents for function summaries. It seems that expr -# forces an anchored regex. Also exclude them for small pages that have -# only one section. - -for file in *.3 ; do - base=`basename $file .3` - toc=-toc - if [ `expr $base : '.*_'` -ne 0 ] ; then toc="" ; fi - if [ "$base" = "pcresample" ] || \ - [ "$base" = "pcrestack" ] || \ - [ "$base" = "pcrecompat" ] || \ - [ "$base" = "pcrelimits" ] || \ - [ "$base" = "pcreperform" ] || \ - [ "$base" = "pcreunicode" ] ; then - toc="" - fi - echo " Making $base.html" - perl ../132html $toc $base <$file >html/$base.html - if [ $? != 0 ] ; then exit 1; fi -done - -# End of documentation processing; stop if only documentation required. - -cd .. -echo Documentation done -if [ "$1" = "doc" ] ; then exit; fi - -# These files are detrailed; do not detrail the test data because there may be -# significant trailing spaces. Do not detrail RunTest.bat, because it has CRLF -# line endings and the detrail script removes all trailing white space. The -# configure files are also omitted from the detrailing. We don't bother with -# those pcre[16|32]_xx files that just define COMPILE_PCRE16 and then #include the -# common file, because they aren't going to change. - -files="\ - Makefile.am \ - Makefile.in \ - configure.ac \ - README \ - LICENCE \ - COPYING \ - AUTHORS \ - NEWS \ - NON-UNIX-USE \ - NON-AUTOTOOLS-BUILD \ - INSTALL \ - 132html \ - CleanTxt \ - Detrail \ - ChangeLog \ - CMakeLists.txt \ - RunGrepTest \ - RunTest \ - pcre-config.in \ - libpcre.pc.in \ - libpcre16.pc.in \ - libpcre32.pc.in \ - libpcreposix.pc.in \ - libpcrecpp.pc.in \ - config.h.in \ - pcre_chartables.c.dist \ - pcredemo.c \ - pcregrep.c \ - pcretest.c \ - dftables.c \ - pcreposix.c \ - pcreposix.h \ - pcre.h.in \ - pcre_internal.h \ - pcre_byte_order.c \ - pcre_compile.c \ - pcre_config.c \ - pcre_dfa_exec.c \ - pcre_exec.c \ - pcre_fullinfo.c \ - pcre_get.c \ - pcre_globals.c \ - pcre_jit_compile.c \ - pcre_jit_test.c \ - pcre_maketables.c \ - pcre_newline.c \ - pcre_ord2utf8.c \ - pcre16_ord2utf16.c \ - pcre32_ord2utf32.c \ - pcre_printint.c \ - pcre_refcount.c \ - pcre_string_utils.c \ - pcre_study.c \ - pcre_tables.c \ - pcre_valid_utf8.c \ - pcre_version.c \ - pcre_xclass.c \ - pcre16_utf16_utils.c \ - pcre32_utf32_utils.c \ - pcre16_valid_utf16.c \ - pcre32_valid_utf32.c \ - pcre_scanner.cc \ - pcre_scanner.h \ - pcre_scanner_unittest.cc \ - pcrecpp.cc \ - pcrecpp.h \ - pcrecpparg.h.in \ - pcrecpp_unittest.cc \ - pcre_stringpiece.cc \ - pcre_stringpiece.h.in \ - pcre_stringpiece_unittest.cc \ - perltest.pl \ - ucp.h \ - makevp.bat \ - pcre.def \ - libpcre.def \ - libpcreposix.def" - -echo Detrailing -perl ./Detrail $files doc/p* doc/html/* - -echo Done - -#End diff --git a/src/third_party/pcre-8.39/README b/src/third_party/pcre-8.39/README deleted file mode 100644 index 4887ebf350e..00000000000 --- a/src/third_party/pcre-8.39/README +++ /dev/null @@ -1,1002 +0,0 @@ -README file for PCRE (Perl-compatible regular expression library) ------------------------------------------------------------------ - -NOTE: This set of files relates to PCRE releases that use the original API, -with library names libpcre, libpcre16, and libpcre32. January 2015 saw the -first release of a new API, known as PCRE2, with release numbers starting at -10.00 and library names libpcre2-8, libpcre2-16, and libpcre2-32. The old -libraries (now called PCRE1) are still being maintained for bug fixes, but -there will be no new development. New projects are advised to use the new PCRE2 -libraries. - - -The latest release of PCRE1 is always available in three alternative formats -from: - - ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-xxx.tar.gz - ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-xxx.tar.bz2 - ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-xxx.zip - -There is a mailing list for discussion about the development of PCRE at -pcre-dev@exim.org. You can access the archives and subscribe or manage your -subscription here: - - https://lists.exim.org/mailman/listinfo/pcre-dev - -Please read the NEWS file if you are upgrading from a previous release. -The contents of this README file are: - - The PCRE APIs - Documentation for PCRE - Contributions by users of PCRE - Building PCRE on non-Unix-like systems - Building PCRE without using autotools - Building PCRE using autotools - Retrieving configuration information - Shared libraries - Cross-compiling using autotools - Using HP's ANSI C++ compiler (aCC) - Compiling in Tru64 using native compilers - Using Sun's compilers for Solaris - Using PCRE from MySQL - Making new tarballs - Testing PCRE - Character tables - File manifest - - -The PCRE APIs -------------- - -PCRE is written in C, and it has its own API. There are three sets of -functions, one for the 8-bit library, which processes strings of bytes, one for -the 16-bit library, which processes strings of 16-bit values, and one for the -32-bit library, which processes strings of 32-bit values. The distribution also -includes a set of C++ wrapper functions (see the pcrecpp man page for details), -courtesy of Google Inc., which can be used to call the 8-bit PCRE library from -C++. Other C++ wrappers have been created from time to time. See, for example: -https://github.com/YasserAsmi/regexp, which aims to be simple and similar in -style to the C API. - -The distribution also contains a set of C wrapper functions (again, just for -the 8-bit library) that are based on the POSIX regular expression API (see the -pcreposix man page). These end up in the library called libpcreposix. Note that -this just provides a POSIX calling interface to PCRE; the regular expressions -themselves still follow Perl syntax and semantics. The POSIX API is restricted, -and does not give full access to all of PCRE's facilities. - -The header file for the POSIX-style functions is called pcreposix.h. The -official POSIX name is regex.h, but I did not want to risk possible problems -with existing files of that name by distributing it that way. To use PCRE with -an existing program that uses the POSIX API, pcreposix.h will have to be -renamed or pointed at by a link. - -If you are using the POSIX interface to PCRE and there is already a POSIX regex -library installed on your system, as well as worrying about the regex.h header -file (as mentioned above), you must also take care when linking programs to -ensure that they link with PCRE's libpcreposix library. Otherwise they may pick -up the POSIX functions of the same name from the other library. - -One way of avoiding this confusion is to compile PCRE with the addition of --Dregcomp=PCREregcomp (and similarly for the other POSIX functions) to the -compiler flags (CFLAGS if you are using "configure" -- see below). This has the -effect of renaming the functions so that the names no longer clash. Of course, -you have to do the same thing for your applications, or write them using the -new names. - - -Documentation for PCRE ----------------------- - -If you install PCRE in the normal way on a Unix-like system, you will end up -with a set of man pages whose names all start with "pcre". The one that is just -called "pcre" lists all the others. In addition to these man pages, the PCRE -documentation is supplied in two other forms: - - 1. There are files called doc/pcre.txt, doc/pcregrep.txt, and - doc/pcretest.txt in the source distribution. The first of these is a - concatenation of the text forms of all the section 3 man pages except - the listing of pcredemo.c and those that summarize individual functions. - The other two are the text forms of the section 1 man pages for the - pcregrep and pcretest commands. These text forms are provided for ease of - scanning with text editors or similar tools. They are installed in - /share/doc/pcre, where is the installation prefix - (defaulting to /usr/local). - - 2. A set of files containing all the documentation in HTML form, hyperlinked - in various ways, and rooted in a file called index.html, is distributed in - doc/html and installed in /share/doc/pcre/html. - -Users of PCRE have contributed files containing the documentation for various -releases in CHM format. These can be found in the Contrib directory of the FTP -site (see next section). - - -Contributions by users of PCRE ------------------------------- - -You can find contributions from PCRE users in the directory - - ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/Contrib - -There is a README file giving brief descriptions of what they are. Some are -complete in themselves; others are pointers to URLs containing relevant files. -Some of this material is likely to be well out-of-date. Several of the earlier -contributions provided support for compiling PCRE on various flavours of -Windows (I myself do not use Windows). Nowadays there is more Windows support -in the standard distribution, so these contibutions have been archived. - -A PCRE user maintains downloadable Windows binaries of the pcregrep and -pcretest programs here: - - http://www.rexegg.com/pcregrep-pcretest.html - - -Building PCRE on non-Unix-like systems --------------------------------------- - -For a non-Unix-like system, please read the comments in the file -NON-AUTOTOOLS-BUILD, though if your system supports the use of "configure" and -"make" you may be able to build PCRE using autotools in the same way as for -many Unix-like systems. - -PCRE can also be configured using the GUI facility provided by CMake's -cmake-gui command. This creates Makefiles, solution files, etc. The file -NON-AUTOTOOLS-BUILD has information about CMake. - -PCRE has been compiled on many different operating systems. It should be -straightforward to build PCRE on any system that has a Standard C compiler and -library, because it uses only Standard C functions. - - -Building PCRE without using autotools -------------------------------------- - -The use of autotools (in particular, libtool) is problematic in some -environments, even some that are Unix or Unix-like. See the NON-AUTOTOOLS-BUILD -file for ways of building PCRE without using autotools. - - -Building PCRE using autotools ------------------------------ - -If you are using HP's ANSI C++ compiler (aCC), please see the special note -in the section entitled "Using HP's ANSI C++ compiler (aCC)" below. - -The following instructions assume the use of the widely used "configure; make; -make install" (autotools) process. - -To build PCRE on system that supports autotools, first run the "configure" -command from the PCRE distribution directory, with your current directory set -to the directory where you want the files to be created. This command is a -standard GNU "autoconf" configuration script, for which generic instructions -are supplied in the file INSTALL. - -Most commonly, people build PCRE within its own distribution directory, and in -this case, on many systems, just running "./configure" is sufficient. However, -the usual methods of changing standard defaults are available. For example: - -CFLAGS='-O2 -Wall' ./configure --prefix=/opt/local - -This command specifies that the C compiler should be run with the flags '-O2 --Wall' instead of the default, and that "make install" should install PCRE -under /opt/local instead of the default /usr/local. - -If you want to build in a different directory, just run "configure" with that -directory as current. For example, suppose you have unpacked the PCRE source -into /source/pcre/pcre-xxx, but you want to build it in /build/pcre/pcre-xxx: - -cd /build/pcre/pcre-xxx -/source/pcre/pcre-xxx/configure - -PCRE is written in C and is normally compiled as a C library. However, it is -possible to build it as a C++ library, though the provided building apparatus -does not have any features to support this. - -There are some optional features that can be included or omitted from the PCRE -library. They are also documented in the pcrebuild man page. - -. By default, both shared and static libraries are built. You can change this - by adding one of these options to the "configure" command: - - --disable-shared - --disable-static - - (See also "Shared libraries on Unix-like systems" below.) - -. By default, only the 8-bit library is built. If you add --enable-pcre16 to - the "configure" command, the 16-bit library is also built. If you add - --enable-pcre32 to the "configure" command, the 32-bit library is also built. - If you want only the 16-bit or 32-bit library, use --disable-pcre8 to disable - building the 8-bit library. - -. If you are building the 8-bit library and want to suppress the building of - the C++ wrapper library, you can add --disable-cpp to the "configure" - command. Otherwise, when "configure" is run without --disable-pcre8, it will - try to find a C++ compiler and C++ header files, and if it succeeds, it will - try to build the C++ wrapper. - -. If you want to include support for just-in-time compiling, which can give - large performance improvements on certain platforms, add --enable-jit to the - "configure" command. This support is available only for certain hardware - architectures. If you try to enable it on an unsupported architecture, there - will be a compile time error. - -. When JIT support is enabled, pcregrep automatically makes use of it, unless - you add --disable-pcregrep-jit to the "configure" command. - -. If you want to make use of the support for UTF-8 Unicode character strings in - the 8-bit library, or UTF-16 Unicode character strings in the 16-bit library, - or UTF-32 Unicode character strings in the 32-bit library, you must add - --enable-utf to the "configure" command. Without it, the code for handling - UTF-8, UTF-16 and UTF-8 is not included in the relevant library. Even - when --enable-utf is included, the use of a UTF encoding still has to be - enabled by an option at run time. When PCRE is compiled with this option, its - input can only either be ASCII or UTF-8/16/32, even when running on EBCDIC - platforms. It is not possible to use both --enable-utf and --enable-ebcdic at - the same time. - -. There are no separate options for enabling UTF-8, UTF-16 and UTF-32 - independently because that would allow ridiculous settings such as requesting - UTF-16 support while building only the 8-bit library. However, the option - --enable-utf8 is retained for backwards compatibility with earlier releases - that did not support 16-bit or 32-bit character strings. It is synonymous with - --enable-utf. It is not possible to configure one library with UTF support - and the other without in the same configuration. - -. If, in addition to support for UTF-8/16/32 character strings, you want to - include support for the \P, \p, and \X sequences that recognize Unicode - character properties, you must add --enable-unicode-properties to the - "configure" command. This adds about 30K to the size of the library (in the - form of a property table); only the basic two-letter properties such as Lu - are supported. - -. You can build PCRE to recognize either CR or LF or the sequence CRLF or any - of the preceding, or any of the Unicode newline sequences as indicating the - end of a line. Whatever you specify at build time is the default; the caller - of PCRE can change the selection at run time. The default newline indicator - is a single LF character (the Unix standard). You can specify the default - newline indicator by adding --enable-newline-is-cr or --enable-newline-is-lf - or --enable-newline-is-crlf or --enable-newline-is-anycrlf or - --enable-newline-is-any to the "configure" command, respectively. - - If you specify --enable-newline-is-cr or --enable-newline-is-crlf, some of - the standard tests will fail, because the lines in the test files end with - LF. Even if the files are edited to change the line endings, there are likely - to be some failures. With --enable-newline-is-anycrlf or - --enable-newline-is-any, many tests should succeed, but there may be some - failures. - -. By default, the sequence \R in a pattern matches any Unicode line ending - sequence. This is independent of the option specifying what PCRE considers to - be the end of a line (see above). However, the caller of PCRE can restrict \R - to match only CR, LF, or CRLF. You can make this the default by adding - --enable-bsr-anycrlf to the "configure" command (bsr = "backslash R"). - -. When called via the POSIX interface, PCRE uses malloc() to get additional - storage for processing capturing parentheses if there are more than 10 of - them in a pattern. You can increase this threshold by setting, for example, - - --with-posix-malloc-threshold=20 - - on the "configure" command. - -. PCRE has a counter that limits the depth of nesting of parentheses in a - pattern. This limits the amount of system stack that a pattern uses when it - is compiled. The default is 250, but you can change it by setting, for - example, - - --with-parens-nest-limit=500 - -. PCRE has a counter that can be set to limit the amount of resources it uses - when matching a pattern. If the limit is exceeded during a match, the match - fails. The default is ten million. You can change the default by setting, for - example, - - --with-match-limit=500000 - - on the "configure" command. This is just the default; individual calls to - pcre_exec() can supply their own value. There is more discussion on the - pcreapi man page. - -. There is a separate counter that limits the depth of recursive function calls - during a matching process. This also has a default of ten million, which is - essentially "unlimited". You can change the default by setting, for example, - - --with-match-limit-recursion=500000 - - Recursive function calls use up the runtime stack; running out of stack can - cause programs to crash in strange ways. There is a discussion about stack - sizes in the pcrestack man page. - -. The default maximum compiled pattern size is around 64K. You can increase - this by adding --with-link-size=3 to the "configure" command. In the 8-bit - library, PCRE then uses three bytes instead of two for offsets to different - parts of the compiled pattern. In the 16-bit library, --with-link-size=3 is - the same as --with-link-size=4, which (in both libraries) uses four-byte - offsets. Increasing the internal link size reduces performance. In the 32-bit - library, the only supported link size is 4. - -. You can build PCRE so that its internal match() function that is called from - pcre_exec() does not call itself recursively. Instead, it uses memory blocks - obtained from the heap via the special functions pcre_stack_malloc() and - pcre_stack_free() to save data that would otherwise be saved on the stack. To - build PCRE like this, use - - --disable-stack-for-recursion - - on the "configure" command. PCRE runs more slowly in this mode, but it may be - necessary in environments with limited stack sizes. This applies only to the - normal execution of the pcre_exec() function; if JIT support is being - successfully used, it is not relevant. Equally, it does not apply to - pcre_dfa_exec(), which does not use deeply nested recursion. There is a - discussion about stack sizes in the pcrestack man page. - -. For speed, PCRE uses four tables for manipulating and identifying characters - whose code point values are less than 256. By default, it uses a set of - tables for ASCII encoding that is part of the distribution. If you specify - - --enable-rebuild-chartables - - a program called dftables is compiled and run in the default C locale when - you obey "make". It builds a source file called pcre_chartables.c. If you do - not specify this option, pcre_chartables.c is created as a copy of - pcre_chartables.c.dist. See "Character tables" below for further information. - -. It is possible to compile PCRE for use on systems that use EBCDIC as their - character code (as opposed to ASCII/Unicode) by specifying - - --enable-ebcdic - - This automatically implies --enable-rebuild-chartables (see above). However, - when PCRE is built this way, it always operates in EBCDIC. It cannot support - both EBCDIC and UTF-8/16/32. There is a second option, --enable-ebcdic-nl25, - which specifies that the code value for the EBCDIC NL character is 0x25 - instead of the default 0x15. - -. In environments where valgrind is installed, if you specify - - --enable-valgrind - - PCRE will use valgrind annotations to mark certain memory regions as - unaddressable. This allows it to detect invalid memory accesses, and is - mostly useful for debugging PCRE itself. - -. In environments where the gcc compiler is used and lcov version 1.6 or above - is installed, if you specify - - --enable-coverage - - the build process implements a code coverage report for the test suite. The - report is generated by running "make coverage". If ccache is installed on - your system, it must be disabled when building PCRE for coverage reporting. - You can do this by setting the environment variable CCACHE_DISABLE=1 before - running "make" to build PCRE. There is more information about coverage - reporting in the "pcrebuild" documentation. - -. The pcregrep program currently supports only 8-bit data files, and so - requires the 8-bit PCRE library. It is possible to compile pcregrep to use - libz and/or libbz2, in order to read .gz and .bz2 files (respectively), by - specifying one or both of - - --enable-pcregrep-libz - --enable-pcregrep-libbz2 - - Of course, the relevant libraries must be installed on your system. - -. The default size (in bytes) of the internal buffer used by pcregrep can be - set by, for example: - - --with-pcregrep-bufsize=51200 - - The value must be a plain integer. The default is 20480. - -. It is possible to compile pcretest so that it links with the libreadline - or libedit libraries, by specifying, respectively, - - --enable-pcretest-libreadline or --enable-pcretest-libedit - - If this is done, when pcretest's input is from a terminal, it reads it using - the readline() function. This provides line-editing and history facilities. - Note that libreadline is GPL-licenced, so if you distribute a binary of - pcretest linked in this way, there may be licensing issues. These can be - avoided by linking with libedit (which has a BSD licence) instead. - - Enabling libreadline causes the -lreadline option to be added to the pcretest - build. In many operating environments with a sytem-installed readline - library this is sufficient. However, in some environments (e.g. if an - unmodified distribution version of readline is in use), it may be necessary - to specify something like LIBS="-lncurses" as well. This is because, to quote - the readline INSTALL, "Readline uses the termcap functions, but does not link - with the termcap or curses library itself, allowing applications which link - with readline the to choose an appropriate library." If you get error - messages about missing functions tgetstr, tgetent, tputs, tgetflag, or tgoto, - this is the problem, and linking with the ncurses library should fix it. - -The "configure" script builds the following files for the basic C library: - -. Makefile the makefile that builds the library -. config.h build-time configuration options for the library -. pcre.h the public PCRE header file -. pcre-config script that shows the building settings such as CFLAGS - that were set for "configure" -. libpcre.pc ) data for the pkg-config command -. libpcre16.pc ) -. libpcre32.pc ) -. libpcreposix.pc ) -. libtool script that builds shared and/or static libraries - -Versions of config.h and pcre.h are distributed in the PCRE tarballs under the -names config.h.generic and pcre.h.generic. These are provided for those who -have to built PCRE without using "configure" or CMake. If you use "configure" -or CMake, the .generic versions are not used. - -When building the 8-bit library, if a C++ compiler is found, the following -files are also built: - -. libpcrecpp.pc data for the pkg-config command -. pcrecpparg.h header file for calling PCRE via the C++ wrapper -. pcre_stringpiece.h header for the C++ "stringpiece" functions - -The "configure" script also creates config.status, which is an executable -script that can be run to recreate the configuration, and config.log, which -contains compiler output from tests that "configure" runs. - -Once "configure" has run, you can run "make". This builds the the libraries -libpcre, libpcre16 and/or libpcre32, and a test program called pcretest. If you -enabled JIT support with --enable-jit, a test program called pcre_jit_test is -built as well. - -If the 8-bit library is built, libpcreposix and the pcregrep command are also -built, and if a C++ compiler was found on your system, and you did not disable -it with --disable-cpp, "make" builds the C++ wrapper library, which is called -libpcrecpp, as well as some test programs called pcrecpp_unittest, -pcre_scanner_unittest, and pcre_stringpiece_unittest. - -The command "make check" runs all the appropriate tests. Details of the PCRE -tests are given below in a separate section of this document. - -You can use "make install" to install PCRE into live directories on your -system. The following are installed (file names are all relative to the - that is set when "configure" is run): - - Commands (bin): - pcretest - pcregrep (if 8-bit support is enabled) - pcre-config - - Libraries (lib): - libpcre16 (if 16-bit support is enabled) - libpcre32 (if 32-bit support is enabled) - libpcre (if 8-bit support is enabled) - libpcreposix (if 8-bit support is enabled) - libpcrecpp (if 8-bit and C++ support is enabled) - - Configuration information (lib/pkgconfig): - libpcre16.pc - libpcre32.pc - libpcre.pc - libpcreposix.pc - libpcrecpp.pc (if C++ support is enabled) - - Header files (include): - pcre.h - pcreposix.h - pcre_scanner.h ) - pcre_stringpiece.h ) if C++ support is enabled - pcrecpp.h ) - pcrecpparg.h ) - - Man pages (share/man/man{1,3}): - pcregrep.1 - pcretest.1 - pcre-config.1 - pcre.3 - pcre*.3 (lots more pages, all starting "pcre") - - HTML documentation (share/doc/pcre/html): - index.html - *.html (lots more pages, hyperlinked from index.html) - - Text file documentation (share/doc/pcre): - AUTHORS - COPYING - ChangeLog - LICENCE - NEWS - README - pcre.txt (a concatenation of the man(3) pages) - pcretest.txt the pcretest man page - pcregrep.txt the pcregrep man page - pcre-config.txt the pcre-config man page - -If you want to remove PCRE from your system, you can run "make uninstall". -This removes all the files that "make install" installed. However, it does not -remove any directories, because these are often shared with other programs. - - -Retrieving configuration information ------------------------------------- - -Running "make install" installs the command pcre-config, which can be used to -recall information about the PCRE configuration and installation. For example: - - pcre-config --version - -prints the version number, and - - pcre-config --libs - -outputs information about where the library is installed. This command can be -included in makefiles for programs that use PCRE, saving the programmer from -having to remember too many details. - -The pkg-config command is another system for saving and retrieving information -about installed libraries. Instead of separate commands for each library, a -single command is used. For example: - - pkg-config --cflags pcre - -The data is held in *.pc files that are installed in a directory called -/lib/pkgconfig. - - -Shared libraries ----------------- - -The default distribution builds PCRE as shared libraries and static libraries, -as long as the operating system supports shared libraries. Shared library -support relies on the "libtool" script which is built as part of the -"configure" process. - -The libtool script is used to compile and link both shared and static -libraries. They are placed in a subdirectory called .libs when they are newly -built. The programs pcretest and pcregrep are built to use these uninstalled -libraries (by means of wrapper scripts in the case of shared libraries). When -you use "make install" to install shared libraries, pcregrep and pcretest are -automatically re-built to use the newly installed shared libraries before being -installed themselves. However, the versions left in the build directory still -use the uninstalled libraries. - -To build PCRE using static libraries only you must use --disable-shared when -configuring it. For example: - -./configure --prefix=/usr/gnu --disable-shared - -Then run "make" in the usual way. Similarly, you can use --disable-static to -build only shared libraries. - - -Cross-compiling using autotools -------------------------------- - -You can specify CC and CFLAGS in the normal way to the "configure" command, in -order to cross-compile PCRE for some other host. However, you should NOT -specify --enable-rebuild-chartables, because if you do, the dftables.c source -file is compiled and run on the local host, in order to generate the inbuilt -character tables (the pcre_chartables.c file). This will probably not work, -because dftables.c needs to be compiled with the local compiler, not the cross -compiler. - -When --enable-rebuild-chartables is not specified, pcre_chartables.c is created -by making a copy of pcre_chartables.c.dist, which is a default set of tables -that assumes ASCII code. Cross-compiling with the default tables should not be -a problem. - -If you need to modify the character tables when cross-compiling, you should -move pcre_chartables.c.dist out of the way, then compile dftables.c by hand and -run it on the local host to make a new version of pcre_chartables.c.dist. -Then when you cross-compile PCRE this new version of the tables will be used. - - -Using HP's ANSI C++ compiler (aCC) ----------------------------------- - -Unless C++ support is disabled by specifying the "--disable-cpp" option of the -"configure" script, you must include the "-AA" option in the CXXFLAGS -environment variable in order for the C++ components to compile correctly. - -Also, note that the aCC compiler on PA-RISC platforms may have a defect whereby -needed libraries fail to get included when specifying the "-AA" compiler -option. If you experience unresolved symbols when linking the C++ programs, -use the workaround of specifying the following environment variable prior to -running the "configure" script: - - CXXLDFLAGS="-lstd_v2 -lCsup_v2" - - -Compiling in Tru64 using native compilers ------------------------------------------ - -The following error may occur when compiling with native compilers in the Tru64 -operating system: - - CXX libpcrecpp_la-pcrecpp.lo -cxx: Error: /usr/lib/cmplrs/cxx/V7.1-006/include/cxx/iosfwd, line 58: #error - directive: "cannot include iosfwd -- define __USE_STD_IOSTREAM to - override default - see section 7.1.2 of the C++ Using Guide" -#error "cannot include iosfwd -- define __USE_STD_IOSTREAM to override default -- see section 7.1.2 of the C++ Using Guide" - -This may be followed by other errors, complaining that 'namespace "std" has no -member'. The solution to this is to add the line - -#define __USE_STD_IOSTREAM 1 - -to the config.h file. - - -Using Sun's compilers for Solaris ---------------------------------- - -A user reports that the following configurations work on Solaris 9 sparcv9 and -Solaris 9 x86 (32-bit): - - Solaris 9 sparcv9: ./configure --disable-cpp CC=/bin/cc CFLAGS="-m64 -g" - Solaris 9 x86: ./configure --disable-cpp CC=/bin/cc CFLAGS="-g" - - -Using PCRE from MySQL ---------------------- - -On systems where both PCRE and MySQL are installed, it is possible to make use -of PCRE from within MySQL, as an alternative to the built-in pattern matching. -There is a web page that tells you how to do this: - - http://www.mysqludf.org/lib_mysqludf_preg/index.php - - -Making new tarballs -------------------- - -The command "make dist" creates three PCRE tarballs, in tar.gz, tar.bz2, and -zip formats. The command "make distcheck" does the same, but then does a trial -build of the new distribution to ensure that it works. - -If you have modified any of the man page sources in the doc directory, you -should first run the PrepareRelease script before making a distribution. This -script creates the .txt and HTML forms of the documentation from the man pages. - - -Testing PCRE ------------- - -To test the basic PCRE library on a Unix-like system, run the RunTest script. -There is another script called RunGrepTest that tests the options of the -pcregrep command. If the C++ wrapper library is built, three test programs -called pcrecpp_unittest, pcre_scanner_unittest, and pcre_stringpiece_unittest -are also built. When JIT support is enabled, another test program called -pcre_jit_test is built. - -Both the scripts and all the program tests are run if you obey "make check" or -"make test". For other environments, see the instructions in -NON-AUTOTOOLS-BUILD. - -The RunTest script runs the pcretest test program (which is documented in its -own man page) on each of the relevant testinput files in the testdata -directory, and compares the output with the contents of the corresponding -testoutput files. RunTest uses a file called testtry to hold the main output -from pcretest. Other files whose names begin with "test" are used as working -files in some tests. - -Some tests are relevant only when certain build-time options were selected. For -example, the tests for UTF-8/16/32 support are run only if --enable-utf was -used. RunTest outputs a comment when it skips a test. - -Many of the tests that are not skipped are run up to three times. The second -run forces pcre_study() to be called for all patterns except for a few in some -tests that are marked "never study" (see the pcretest program for how this is -done). If JIT support is available, the non-DFA tests are run a third time, -this time with a forced pcre_study() with the PCRE_STUDY_JIT_COMPILE option. -This testing can be suppressed by putting "nojit" on the RunTest command line. - -The entire set of tests is run once for each of the 8-bit, 16-bit and 32-bit -libraries that are enabled. If you want to run just one set of tests, call -RunTest with either the -8, -16 or -32 option. - -If valgrind is installed, you can run the tests under it by putting "valgrind" -on the RunTest command line. To run pcretest on just one or more specific test -files, give their numbers as arguments to RunTest, for example: - - RunTest 2 7 11 - -You can also specify ranges of tests such as 3-6 or 3- (meaning 3 to the -end), or a number preceded by ~ to exclude a test. For example: - - Runtest 3-15 ~10 - -This runs tests 3 to 15, excluding test 10, and just ~13 runs all the tests -except test 13. Whatever order the arguments are in, the tests are always run -in numerical order. - -You can also call RunTest with the single argument "list" to cause it to output -a list of tests. - -The first test file can be fed directly into the perltest.pl script to check -that Perl gives the same results. The only difference you should see is in the -first few lines, where the Perl version is given instead of the PCRE version. - -The second set of tests check pcre_fullinfo(), pcre_study(), -pcre_copy_substring(), pcre_get_substring(), pcre_get_substring_list(), error -detection, and run-time flags that are specific to PCRE, as well as the POSIX -wrapper API. It also uses the debugging flags to check some of the internals of -pcre_compile(). - -If you build PCRE with a locale setting that is not the standard C locale, the -character tables may be different (see next paragraph). In some cases, this may -cause failures in the second set of tests. For example, in a locale where the -isprint() function yields TRUE for characters in the range 128-255, the use of -[:isascii:] inside a character class defines a different set of characters, and -this shows up in this test as a difference in the compiled code, which is being -listed for checking. Where the comparison test output contains [\x00-\x7f] the -test will contain [\x00-\xff], and similarly in some other cases. This is not a -bug in PCRE. - -The third set of tests checks pcre_maketables(), the facility for building a -set of character tables for a specific locale and using them instead of the -default tables. The tests make use of the "fr_FR" (French) locale. Before -running the test, the script checks for the presence of this locale by running -the "locale" command. If that command fails, or if it doesn't include "fr_FR" -in the list of available locales, the third test cannot be run, and a comment -is output to say why. If running this test produces instances of the error - - ** Failed to set locale "fr_FR" - -in the comparison output, it means that locale is not available on your system, -despite being listed by "locale". This does not mean that PCRE is broken. - -[If you are trying to run this test on Windows, you may be able to get it to -work by changing "fr_FR" to "french" everywhere it occurs. Alternatively, use -RunTest.bat. The version of RunTest.bat included with PCRE 7.4 and above uses -Windows versions of test 2. More info on using RunTest.bat is included in the -document entitled NON-UNIX-USE.] - -The fourth and fifth tests check the UTF-8/16/32 support and error handling and -internal UTF features of PCRE that are not relevant to Perl, respectively. The -sixth and seventh tests do the same for Unicode character properties support. - -The eighth, ninth, and tenth tests check the pcre_dfa_exec() alternative -matching function, in non-UTF-8/16/32 mode, UTF-8/16/32 mode, and UTF-8/16/32 -mode with Unicode property support, respectively. - -The eleventh test checks some internal offsets and code size features; it is -run only when the default "link size" of 2 is set (in other cases the sizes -change) and when Unicode property support is enabled. - -The twelfth test is run only when JIT support is available, and the thirteenth -test is run only when JIT support is not available. They test some JIT-specific -features such as information output from pcretest about JIT compilation. - -The fourteenth, fifteenth, and sixteenth tests are run only in 8-bit mode, and -the seventeenth, eighteenth, and nineteenth tests are run only in 16/32-bit -mode. These are tests that generate different output in the two modes. They are -for general cases, UTF-8/16/32 support, and Unicode property support, -respectively. - -The twentieth test is run only in 16/32-bit mode. It tests some specific -16/32-bit features of the DFA matching engine. - -The twenty-first and twenty-second tests are run only in 16/32-bit mode, when -the link size is set to 2 for the 16-bit library. They test reloading -pre-compiled patterns. - -The twenty-third and twenty-fourth tests are run only in 16-bit mode. They are -for general cases, and UTF-16 support, respectively. - -The twenty-fifth and twenty-sixth tests are run only in 32-bit mode. They are -for general cases, and UTF-32 support, respectively. - - -Character tables ----------------- - -For speed, PCRE uses four tables for manipulating and identifying characters -whose code point values are less than 256. The final argument of the -pcre_compile() function is a pointer to a block of memory containing the -concatenated tables. A call to pcre_maketables() can be used to generate a set -of tables in the current locale. If the final argument for pcre_compile() is -passed as NULL, a set of default tables that is built into the binary is used. - -The source file called pcre_chartables.c contains the default set of tables. By -default, this is created as a copy of pcre_chartables.c.dist, which contains -tables for ASCII coding. However, if --enable-rebuild-chartables is specified -for ./configure, a different version of pcre_chartables.c is built by the -program dftables (compiled from dftables.c), which uses the ANSI C character -handling functions such as isalnum(), isalpha(), isupper(), islower(), etc. to -build the table sources. This means that the default C locale which is set for -your system will control the contents of these default tables. You can change -the default tables by editing pcre_chartables.c and then re-building PCRE. If -you do this, you should take care to ensure that the file does not get -automatically re-generated. The best way to do this is to move -pcre_chartables.c.dist out of the way and replace it with your customized -tables. - -When the dftables program is run as a result of --enable-rebuild-chartables, -it uses the default C locale that is set on your system. It does not pay -attention to the LC_xxx environment variables. In other words, it uses the -system's default locale rather than whatever the compiling user happens to have -set. If you really do want to build a source set of character tables in a -locale that is specified by the LC_xxx variables, you can run the dftables -program by hand with the -L option. For example: - - ./dftables -L pcre_chartables.c.special - -The first two 256-byte tables provide lower casing and case flipping functions, -respectively. The next table consists of three 32-byte bit maps which identify -digits, "word" characters, and white space, respectively. These are used when -building 32-byte bit maps that represent character classes for code points less -than 256. - -The final 256-byte table has bits indicating various character types, as -follows: - - 1 white space character - 2 letter - 4 decimal digit - 8 hexadecimal digit - 16 alphanumeric or '_' - 128 regular expression metacharacter or binary zero - -You should not alter the set of characters that contain the 128 bit, as that -will cause PCRE to malfunction. - - -File manifest -------------- - -The distribution should contain the files listed below. Where a file name is -given as pcre[16|32]_xxx it means that there are three files, one with the name -pcre_xxx, one with the name pcre16_xx, and a third with the name pcre32_xxx. - -(A) Source files of the PCRE library functions and their headers: - - dftables.c auxiliary program for building pcre_chartables.c - when --enable-rebuild-chartables is specified - - pcre_chartables.c.dist a default set of character tables that assume ASCII - coding; used, unless --enable-rebuild-chartables is - specified, by copying to pcre[16]_chartables.c - - pcreposix.c ) - pcre[16|32]_byte_order.c ) - pcre[16|32]_compile.c ) - pcre[16|32]_config.c ) - pcre[16|32]_dfa_exec.c ) - pcre[16|32]_exec.c ) - pcre[16|32]_fullinfo.c ) - pcre[16|32]_get.c ) sources for the functions in the library, - pcre[16|32]_globals.c ) and some internal functions that they use - pcre[16|32]_jit_compile.c ) - pcre[16|32]_maketables.c ) - pcre[16|32]_newline.c ) - pcre[16|32]_refcount.c ) - pcre[16|32]_string_utils.c ) - pcre[16|32]_study.c ) - pcre[16|32]_tables.c ) - pcre[16|32]_ucd.c ) - pcre[16|32]_version.c ) - pcre[16|32]_xclass.c ) - pcre_ord2utf8.c ) - pcre_valid_utf8.c ) - pcre16_ord2utf16.c ) - pcre16_utf16_utils.c ) - pcre16_valid_utf16.c ) - pcre32_utf32_utils.c ) - pcre32_valid_utf32.c ) - - pcre[16|32]_printint.c ) debugging function that is used by pcretest, - ) and can also be #included in pcre_compile() - - pcre.h.in template for pcre.h when built by "configure" - pcreposix.h header for the external POSIX wrapper API - pcre_internal.h header for internal use - sljit/* 16 files that make up the JIT compiler - ucp.h header for Unicode property handling - - config.h.in template for config.h, which is built by "configure" - - pcrecpp.h public header file for the C++ wrapper - pcrecpparg.h.in template for another C++ header file - pcre_scanner.h public header file for C++ scanner functions - pcrecpp.cc ) - pcre_scanner.cc ) source for the C++ wrapper library - - pcre_stringpiece.h.in template for pcre_stringpiece.h, the header for the - C++ stringpiece functions - pcre_stringpiece.cc source for the C++ stringpiece functions - -(B) Source files for programs that use PCRE: - - pcredemo.c simple demonstration of coding calls to PCRE - pcregrep.c source of a grep utility that uses PCRE - pcretest.c comprehensive test program - -(C) Auxiliary files: - - 132html script to turn "man" pages into HTML - AUTHORS information about the author of PCRE - ChangeLog log of changes to the code - CleanTxt script to clean nroff output for txt man pages - Detrail script to remove trailing spaces - HACKING some notes about the internals of PCRE - INSTALL generic installation instructions - LICENCE conditions for the use of PCRE - COPYING the same, using GNU's standard name - Makefile.in ) template for Unix Makefile, which is built by - ) "configure" - Makefile.am ) the automake input that was used to create - ) Makefile.in - NEWS important changes in this release - NON-UNIX-USE the previous name for NON-AUTOTOOLS-BUILD - NON-AUTOTOOLS-BUILD notes on building PCRE without using autotools - PrepareRelease script to make preparations for "make dist" - README this file - RunTest a Unix shell script for running tests - RunGrepTest a Unix shell script for pcregrep tests - aclocal.m4 m4 macros (generated by "aclocal") - config.guess ) files used by libtool, - config.sub ) used only when building a shared library - configure a configuring shell script (built by autoconf) - configure.ac ) the autoconf input that was used to build - ) "configure" and config.h - depcomp ) script to find program dependencies, generated by - ) automake - doc/*.3 man page sources for PCRE - doc/*.1 man page sources for pcregrep and pcretest - doc/index.html.src the base HTML page - doc/html/* HTML documentation - doc/pcre.txt plain text version of the man pages - doc/pcretest.txt plain text documentation of test program - doc/perltest.txt plain text documentation of Perl test program - install-sh a shell script for installing files - libpcre16.pc.in template for libpcre16.pc for pkg-config - libpcre32.pc.in template for libpcre32.pc for pkg-config - libpcre.pc.in template for libpcre.pc for pkg-config - libpcreposix.pc.in template for libpcreposix.pc for pkg-config - libpcrecpp.pc.in template for libpcrecpp.pc for pkg-config - ltmain.sh file used to build a libtool script - missing ) common stub for a few missing GNU programs while - ) installing, generated by automake - mkinstalldirs script for making install directories - perltest.pl Perl test program - pcre-config.in source of script which retains PCRE information - pcre_jit_test.c test program for the JIT compiler - pcrecpp_unittest.cc ) - pcre_scanner_unittest.cc ) test programs for the C++ wrapper - pcre_stringpiece_unittest.cc ) - testdata/testinput* test data for main library tests - testdata/testoutput* expected test results - testdata/grep* input and output for pcregrep tests - testdata/* other supporting test files - -(D) Auxiliary files for cmake support - - cmake/COPYING-CMAKE-SCRIPTS - cmake/FindPackageHandleStandardArgs.cmake - cmake/FindEditline.cmake - cmake/FindReadline.cmake - CMakeLists.txt - config-cmake.h.in - -(E) Auxiliary files for VPASCAL - - makevp.bat - makevp_c.txt - makevp_l.txt - pcregexp.pas - -(F) Auxiliary files for building PCRE "by hand" - - pcre.h.generic ) a version of the public PCRE header file - ) for use in non-"configure" environments - config.h.generic ) a version of config.h for use in non-"configure" - ) environments - -(F) Miscellaneous - - RunTest.bat a script for running tests under Windows - -Philip Hazel -Email local part: ph10 -Email domain: cam.ac.uk -Last updated: 10 February 2015 diff --git a/src/third_party/pcre-8.39/RunGrepTest b/src/third_party/pcre-8.39/RunGrepTest deleted file mode 100755 index 721ec5184ab..00000000000 --- a/src/third_party/pcre-8.39/RunGrepTest +++ /dev/null @@ -1,593 +0,0 @@ -#! /bin/sh - -# Run pcregrep tests. The assumption is that the PCRE tests check the library -# itself. What we are checking here is the file handling and options that are -# supported by pcregrep. This script must be run in the build directory. - -# Set the C locale, so that sort(1) behaves predictably. - -LC_ALL=C -export LC_ALL - -# Remove any non-default colouring and aliases that the caller may have set. - -unset PCREGREP_COLOUR PCREGREP_COLOR -unset cp ls mv rm - -# Remember the current (build) directory, set the program to be tested, and -# valgrind settings when requested. - -builddir=`pwd` -pcregrep=$builddir/pcregrep - -valgrind= -while [ $# -gt 0 ] ; do - case $1 in - valgrind) valgrind="valgrind -q --leak-check=no --smc-check=all";; - *) echo "RunGrepTest: Unknown argument $1"; exit 1;; - esac - shift -done - -echo " " -pcregrep_version=`$pcregrep -V` -if [ "$valgrind" = "" ] ; then - echo "Testing $pcregrep_version" -else - echo "Testing $pcregrep_version using valgrind" -fi - -# Set up a suitable "diff" command for comparison. Some systems have a diff -# that lacks a -u option. Try to deal with this; better do the test for the -b -# option as well. - -cf="diff" -diff -b /dev/null /dev/null 2>/dev/null && cf="diff -b" -diff -u /dev/null /dev/null 2>/dev/null && cf="diff -u" -diff -ub /dev/null /dev/null 2>/dev/null && cf="diff -ub" - -# If this test is being run from "make check", $srcdir will be set. If not, set -# it to the current or parent directory, whichever one contains the test data. -# Subsequently, we run most of the pcregrep tests in the source directory so -# that the file names in the output are always the same. - -if [ -z "$srcdir" -o ! -d "$srcdir/testdata" ] ; then - if [ -d "./testdata" ] ; then - srcdir=. - elif [ -d "../testdata" ] ; then - srcdir=.. - else - echo "Cannot find the testdata directory" - exit 1 - fi -fi - -# Check for the availability of UTF-8 support - -./pcretest -C utf >/dev/null -utf8=$? - -# We need valgrind suppressions when JIT is in use. (This isn't perfect because -# some tests are run with -no-jit, but as PCRE1 is in maintenance only, I have -# not bothered about that.) - -./pcretest -C jit >/dev/null -if [ $? -eq 1 -a "$valgrind" != "" ] ; then - valgrind="$valgrind --suppressions=./testdata/valgrind-jit.supp" -fi - -echo "Testing pcregrep main features" - -echo "---------------------------- Test 1 ------------------------------" >testtrygrep -(cd $srcdir; $valgrind $pcregrep PATTERN ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 2 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep '^PATTERN' ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 3 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -in PATTERN ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 4 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -ic PATTERN ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 5 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -in PATTERN ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 6 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -inh PATTERN ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 7 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -il PATTERN ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 8 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -l PATTERN ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 9 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -q PATTERN ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 10 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -q NEVER-PATTERN ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 11 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -vn pattern ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 12 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -ix pattern ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 13 -----------------------------" >>testtrygrep -echo seventeen >testtemp1grep -(cd $srcdir; $valgrind $pcregrep -f./testdata/greplist -f $builddir/testtemp1grep ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 14 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -w pat ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 15 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep 'abc^*' ./testdata/grepinput) 2>>testtrygrep >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 16 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep abc ./testdata/grepinput ./testdata/nonexistfile) 2>>testtrygrep >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 17 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -M 'the\noutput' ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 18 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -Mn '(the\noutput|dog\.\n--)' ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 19 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -Mix 'Pattern' ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 20 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -Mixn 'complete pair\nof lines' ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 21 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -nA3 'four' ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 22 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -nB3 'four' ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 23 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -C3 'four' ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 24 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -A9 'four' ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 25 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -nB9 'four' ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 26 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -A9 -B9 'four' ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 27 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -A10 'four' ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 28 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -nB10 'four' ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 29 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -C12 -B10 'four' ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 30 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -inB3 'pattern' ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 31 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -inA3 'pattern' ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 32 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -L 'fox' ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 33 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep 'fox' ./testdata/grepnonexist) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 34 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -s 'fox' ./testdata/grepnonexist) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 35 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -L -r --include=grepinputx --include grepinput8 --exclude-dir='^\.' 'fox' ./testdata | sort) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 36 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -L -r --include=grepinput --exclude 'grepinput$' --exclude=grepinput8 --exclude-dir='^\.' 'fox' ./testdata | sort) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 37 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep '^(a+)*\d' ./testdata/grepinput) >>testtrygrep 2>teststderrgrep -echo "RC=$?" >>testtrygrep -echo "======== STDERR ========" >>testtrygrep -cat teststderrgrep >>testtrygrep - -echo "---------------------------- Test 38 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep '>\x00<' ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 39 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -A1 'before the binary zero' ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 40 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -B1 'after the binary zero' ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 41 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -B1 -o '\w+ the binary zero' ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 42 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -B1 -onH '\w+ the binary zero' ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 43 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -on 'before|zero|after' ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 44 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -on -e before -ezero -e after ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 45 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -on -f ./testdata/greplist -e binary ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 46 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -eabc -e '(unclosed' ./testdata/grepinput) 2>>testtrygrep >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 47 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -Fx "AB.VE -elephant" ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 48 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -F "AB.VE -elephant" ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 49 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -F -e DATA -e "AB.VE -elephant" ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 50 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep "^(abc|def|ghi|jkl)" ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 51 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -Mv "brown\sfox" ./testdata/grepinputv) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 52 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --colour=always jumps ./testdata/grepinputv) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 53 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --file-offsets 'before|zero|after' ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 54 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --line-offsets 'before|zero|after' ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 55 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -f./testdata/greplist --color=always ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 56 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -c lazy ./testdata/grepinput*) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 57 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -c -l lazy ./testdata/grepinput*) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 58 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --regex=PATTERN ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 59 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --regexp=PATTERN ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 60 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --regex PATTERN ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 61 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --regexp PATTERN ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 62 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --match-limit=1000 --no-jit -M 'This is a file(.|\R)*file.' ./testdata/grepinput) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 63 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --recursion-limit=1000 --no-jit -M 'This is a file(.|\R)*file.' ./testdata/grepinput) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 64 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -o1 '(?<=PAT)TERN (ap(pear)s)' ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 65 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -o2 '(?<=PAT)TERN (ap(pear)s)' ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 66 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -o3 '(?<=PAT)TERN (ap(pear)s)' ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 67 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -o12 '(?<=PAT)TERN (ap(pear)s)' ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 68 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --only-matching=2 '(?<=PAT)TERN (ap(pear)s)' ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 69 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -vn --colour=always pattern ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 70 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --color=always -M "triple:\t.*\n\n" ./testdata/grepinput3) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 71 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -o "^01|^02|^03" ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 72 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --color=always "^01|^02|^03" ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 73 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -o --colour=always "^01|^02|^03" ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 74 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -o "^01|02|^03" ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 75 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --color=always "^01|02|^03" ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 76 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -o --colour=always "^01|02|^03" ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 77 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -o "^01|^02|03" ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 78 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --color=always "^01|^02|03" ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 79 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -o --colour=always "^01|^02|03" ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 80 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -o "\b01|\b02" ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 81 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --color=always "\\b01|\\b02" ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 82 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -o --colour=always "\\b01|\\b02" ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 83 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --buffer-size=100 "^a" ./testdata/grepinput3) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 84 -----------------------------" >>testtrygrep -echo testdata/grepinput3 >testtemp1grep -(cd $srcdir; $valgrind $pcregrep --file-list ./testdata/grepfilelist --file-list $builddir/testtemp1grep "fox|complete|t7") >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 85 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --file-list=./testdata/grepfilelist "dolor" ./testdata/grepinput3) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 86 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep "dog" ./testdata/grepbinary) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 87 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep "cat" ./testdata/grepbinary) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 88 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -v "cat" ./testdata/grepbinary) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 89 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -I "dog" ./testdata/grepbinary) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 90 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --binary-files=without-match "dog" ./testdata/grepbinary) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 91 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -a "dog" ./testdata/grepbinary) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 92 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --binary-files=text "dog" ./testdata/grepbinary) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 93 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --text "dog" ./testdata/grepbinary) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 94 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -L -r --include=grepinputx --include grepinput8 'fox' ./testdata/grepinput* | sort) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 95 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --file-list ./testdata/grepfilelist --exclude grepinputv "fox|complete") >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 96 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -L -r --include-dir=testdata --exclude '^(?!grepinput)' 'fox' ./test* | sort) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 97 -----------------------------" >>testtrygrep -echo "grepinput$" >testtemp1grep -echo "grepinput8" >>testtemp1grep -(cd $srcdir; $valgrind $pcregrep -L -r --include=grepinput --exclude-from $builddir/testtemp1grep --exclude-dir='^\.' 'fox' ./testdata | sort) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 98 -----------------------------" >>testtrygrep -echo "grepinput$" >testtemp1grep -echo "grepinput8" >>testtemp1grep -(cd $srcdir; $valgrind $pcregrep -L -r --exclude=grepinput3 --include=grepinput --exclude-from $builddir/testtemp1grep --exclude-dir='^\.' 'fox' ./testdata | sort) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 99 -----------------------------" >>testtrygrep -echo "grepinput$" >testtemp1grep -echo "grepinput8" >testtemp2grep -(cd $srcdir; $valgrind $pcregrep -L -r --include grepinput --exclude-from $builddir/testtemp1grep --exclude-from=$builddir/testtemp2grep --exclude-dir='^\.' 'fox' ./testdata | sort) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 100 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -Ho2 --only-matching=1 -o3 '(\w+) binary (\w+)(\.)?' ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 101 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -o3 -Ho2 -o12 --only-matching=1 -o3 --colour=always --om-separator='|' '(\w+) binary (\w+)(\.)?' ./testdata/grepinput) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 102 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -n "^$" ./testdata/grepinput3) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 103 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --only-matching "^$" ./testdata/grepinput3) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 104 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -n --only-matching "^$" ./testdata/grepinput3) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 105 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep --colour=always "ipsum|" ./testdata/grepinput3) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 106 -----------------------------" >>testtrygrep -(cd $srcdir; echo "a" | $valgrind $pcregrep -M "|a" ) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 107 -----------------------------" >>testtrygrep -echo "a" >testtemp1grep -echo "aaaaa" >>testtemp1grep -(cd $srcdir; $valgrind $pcregrep --line-offsets '(?<=\Ka)' $builddir/testtemp1grep) >>testtrygrep 2>&1 -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 108 ------------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -lq PATTERN ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep -echo "RC=$?" >>testtrygrep - -echo "---------------------------- Test 109 -----------------------------" >>testtrygrep -(cd $srcdir; $valgrind $pcregrep -cq lazy ./testdata/grepinput*) >>testtrygrep -echo "RC=$?" >>testtrygrep - -# Now compare the results. - -$cf $srcdir/testdata/grepoutput testtrygrep -if [ $? != 0 ] ; then exit 1; fi - - -# These tests require UTF-8 support - -if [ $utf8 -ne 0 ] ; then - echo "Testing pcregrep UTF-8 features" - - echo "---------------------------- Test U1 ------------------------------" >testtrygrep - (cd $srcdir; $valgrind $pcregrep -n -u --newline=any "^X" ./testdata/grepinput8) >>testtrygrep - echo "RC=$?" >>testtrygrep - - echo "---------------------------- Test U2 ------------------------------" >>testtrygrep - (cd $srcdir; $valgrind $pcregrep -n -u -C 3 --newline=any "Match" ./testdata/grepinput8) >>testtrygrep - echo "RC=$?" >>testtrygrep - - $cf $srcdir/testdata/grepoutput8 testtrygrep - if [ $? != 0 ] ; then exit 1; fi - -else - echo "Skipping pcregrep UTF-8 tests: no UTF-8 support in PCRE library" -fi - - -# We go to some contortions to try to ensure that the tests for the various -# newline settings will work in environments where the normal newline sequence -# is not \n. Do not use exported files, whose line endings might be changed. -# Instead, create an input file using printf so that its contents are exactly -# what we want. Note the messy fudge to get printf to write a string that -# starts with a hyphen. These tests are run in the build directory. - -echo "Testing pcregrep newline settings" -printf "abc\rdef\r\nghi\njkl" >testNinputgrep - -printf "%c--------------------------- Test N1 ------------------------------\r\n" - >testtrygrep -$valgrind $pcregrep -n -N CR "^(abc|def|ghi|jkl)" testNinputgrep >>testtrygrep - -printf "%c--------------------------- Test N2 ------------------------------\r\n" - >>testtrygrep -$valgrind $pcregrep -n --newline=crlf "^(abc|def|ghi|jkl)" testNinputgrep >>testtrygrep - -printf "%c--------------------------- Test N3 ------------------------------\r\n" - >>testtrygrep -pattern=`printf 'def\rjkl'` -$valgrind $pcregrep -n --newline=cr -F "$pattern" testNinputgrep >>testtrygrep - -printf "%c--------------------------- Test N4 ------------------------------\r\n" - >>testtrygrep -$valgrind $pcregrep -n --newline=crlf -F -f $srcdir/testdata/greppatN4 testNinputgrep >>testtrygrep - -printf "%c--------------------------- Test N5 ------------------------------\r\n" - >>testtrygrep -$valgrind $pcregrep -n --newline=any "^(abc|def|ghi|jkl)" testNinputgrep >>testtrygrep - -printf "%c--------------------------- Test N6 ------------------------------\r\n" - >>testtrygrep -$valgrind $pcregrep -n --newline=anycrlf "^(abc|def|ghi|jkl)" testNinputgrep >>testtrygrep - -$cf $srcdir/testdata/grepoutputN testtrygrep -if [ $? != 0 ] ; then exit 1; fi - -exit 0 - -# End diff --git a/src/third_party/pcre-8.39/RunTest b/src/third_party/pcre-8.39/RunTest deleted file mode 100755 index 357a739b75f..00000000000 --- a/src/third_party/pcre-8.39/RunTest +++ /dev/null @@ -1,1014 +0,0 @@ -#! /bin/sh - -############################################################################### -# Run the PCRE tests using the pcretest program. The appropriate tests are -# selected, depending on which build-time options were used. -# -# All tests are now run both with and without -s, to ensure that everything is -# tested with and without studying. However, there are some tests that produce -# different output after studying, typically when we are tracing the actual -# matching process (for example, using auto-callouts). In these few cases, the -# tests are duplicated in the files, one with /S to force studying always, and -# one with /SS to force *not* studying always. The use of -s doesn't then make -# any difference to their output. There is also one test which compiles invalid -# UTF-8 with the UTF-8 check turned off; for this, studying must also be -# disabled with /SS. -# -# When JIT support is available, all appropriate tests are also run with -s+ to -# test (again, almost) everything with studying and the JIT option, unless -# "nojit" is given on the command line. There are also two tests for -# JIT-specific features, one to be run when JIT support is available (unless -# "nojit" is specified), and one when it is not. -# -# Whichever of the 8-, 16- and 32-bit libraries exist are tested. It is also -# possible to select which to test by giving "-8", "-16" or "-32" on the -# command line. -# -# As well as "nojit", "-8", "-16", and "-32", arguments for this script are -# individual test numbers, ranges of tests such as 3-6 or 3- (meaning 3 to the -# end), or a number preceded by ~ to exclude a test. For example, "3-15 ~10" -# runs tests 3 to 15, excluding test 10, and just "~10" runs all the tests -# except test 10. Whatever order the arguments are in, the tests are always run -# in numerical order. -# -# The special argument "3S" runs test 3, stopping if it fails. Test 3 is the -# locale test, and failure usually means there's an issue with the locale -# rather than a bug in PCRE, so normally subsequent tests are run. "3S" is -# useful when you want to debug or update the test. -# -# Inappropriate tests are automatically skipped (with a comment to say so): for -# example, if JIT support is not compiled, test 12 is skipped, whereas if JIT -# support is compiled, test 13 is skipped. -# -# Other arguments can be one of the words "valgrind", "valgrind-log", or "sim" -# followed by an argument to run cross-compiled executables under a simulator, -# for example: -# -# RunTest 3 sim "qemu-arm -s 8388608" -# -# There are two special cases where only one argument is allowed: -# -# If the first and only argument is "ebcdic", the script runs the special -# EBCDIC test that can be useful for checking certain EBCDIC features, even -# when run in an ASCII environment. -# -# If the script is obeyed as "RunTest list", a list of available tests is -# output, but none of them are run. -############################################################################### - -# Define test titles in variables so that they can be output as a list. Some -# of them are modified (e.g. with -8 or -16) when used in the actual tests. - -title1="Test 1: Main functionality (Compatible with Perl >= 5.10)" -title2="Test 2: API, errors, internals, and non-Perl stuff" -title3="Test 3: Locale-specific features" -title4A="Test 4: UTF" -title4B=" support (Compatible with Perl >= 5.10)" -title5="Test 5: API, internals, and non-Perl stuff for UTF" -title6="Test 6: Unicode property support (Compatible with Perl >= 5.10)" -title7="Test 7: API, internals, and non-Perl stuff for Unicode property support" -title8="Test 8: DFA matching main functionality" -title9="Test 9: DFA matching with UTF" -title10="Test 10: DFA matching with Unicode properties" -title11="Test 11: Internal offsets and code size tests" -title12="Test 12: JIT-specific features (when JIT is available)" -title13="Test 13: JIT-specific features (when JIT is not available)" -title14="Test 14: Specials for the basic 8-bit library" -title15="Test 15: Specials for the 8-bit library with UTF-8 support" -title16="Test 16: Specials for the 8-bit library with Unicode propery support" -title17="Test 17: Specials for the basic 16/32-bit library" -title18="Test 18: Specials for the 16/32-bit library with UTF-16/32 support" -title19="Test 19: Specials for the 16/32-bit library with Unicode property support" -title20="Test 20: DFA specials for the basic 16/32-bit library" -title21="Test 21: Reloads for the basic 16/32-bit library" -title22="Test 22: Reloads for the 16/32-bit library with UTF-16/32 support" -title23="Test 23: Specials for the 16-bit library" -title24="Test 24: Specials for the 16-bit library with UTF-16 support" -title25="Test 25: Specials for the 32-bit library" -title26="Test 26: Specials for the 32-bit library with UTF-32 support" - -maxtest=26 - -if [ $# -eq 1 -a "$1" = "list" ]; then - echo $title1 - echo $title2 "(not UTF)" - echo $title3 - echo $title4A $title4B - echo $title5 support - echo $title6 - echo $title7 - echo $title8 - echo $title9 - echo $title10 - echo $title11 - echo $title12 - echo $title13 - echo $title14 - echo $title15 - echo $title16 - echo $title17 - echo $title18 - echo $title19 - echo $title20 - echo $title21 - echo $title22 - echo $title23 - echo $title24 - echo $title25 - echo $title26 - exit 0 -fi - -# Set up a suitable "diff" command for comparison. Some systems -# have a diff that lacks a -u option. Try to deal with this. - -cf="diff" -diff -u /dev/null /dev/null 2>/dev/null && cf="diff -u" - -# Find the test data - -if [ -n "$srcdir" -a -d "$srcdir" ] ; then - testdata="$srcdir/testdata" -elif [ -d "./testdata" ] ; then - testdata=./testdata -elif [ -d "../testdata" ] ; then - testdata=../testdata -else - echo "Cannot find the testdata directory" - exit 1 -fi - - -# ------ Special EBCDIC Test ------- - -if [ $# -eq 1 -a "$1" = "ebcdic" ]; then - ./pcretest -C ebcdic >/dev/null - ebcdic=$? - if [ $ebcdic -ne 1 ] ; then - echo "Cannot run EBCDIC tests: EBCDIC support not compiled" - exit 1 - fi - - for opt in "" "-s" "-dfa" "-s -dfa"; do - ./pcretest -q $opt $testdata/testinputEBC >testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutputEBC testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - if [ "$opt" = "-s" ] ; then echo " OK with study" - elif [ "$opt" = "-dfa" ] ; then echo " OK using DFA" - elif [ "$opt" = "-s -dfa" ] ; then echo " OK using DFA with study" - else echo " OK" - fi - done - -exit 0 -fi - - -# ------ Normal Tests ------ - -# Default values - -arg8= -arg16= -arg32= -nojit= -sim= -skip= -valgrind= -vjs= - -# This is in case the caller has set aliases (as I do - PH) -unset cp ls mv rm - -# Process options and select which tests to run; for those that are explicitly -# requested, check that the necessary optional facilities are available. - -do1=no -do2=no -do3=no -do4=no -do5=no -do6=no -do7=no -do8=no -do9=no -do10=no -do11=no -do12=no -do13=no -do14=no -do15=no -do16=no -do17=no -do18=no -do19=no -do20=no -do21=no -do22=no -do23=no -do24=no -do25=no -do26=no - -while [ $# -gt 0 ] ; do - case $1 in - 1) do1=yes;; - 2) do2=yes;; - 3) do3=yes;; - 4) do4=yes;; - 5) do5=yes;; - 6) do6=yes;; - 7) do7=yes;; - 8) do8=yes;; - 9) do9=yes;; - 10) do10=yes;; - 11) do11=yes;; - 12) do12=yes;; - 13) do13=yes;; - 14) do14=yes;; - 15) do15=yes;; - 16) do16=yes;; - 17) do17=yes;; - 18) do18=yes;; - 19) do19=yes;; - 20) do20=yes;; - 21) do21=yes;; - 22) do22=yes;; - 23) do23=yes;; - 24) do24=yes;; - 25) do25=yes;; - 26) do26=yes;; - -8) arg8=yes;; - -16) arg16=yes;; - -32) arg32=yes;; - nojit) nojit=yes;; - sim) shift; sim=$1;; - valgrind) valgrind="valgrind --tool=memcheck -q --smc-check=all";; - valgrind-log) valgrind="valgrind --tool=memcheck --num-callers=30 --leak-check=no --error-limit=no --smc-check=all --log-file=report.%p ";; - ~*) - if expr "$1" : '~[0-9][0-9]*$' >/dev/null; then - skip="$skip `expr "$1" : '~\([0-9]*\)*$'`" - else - echo "Unknown option or test selector '$1'"; exit 1 - fi - ;; - *-*) - if expr "$1" : '[0-9][0-9]*-[0-9]*$' >/dev/null; then - tf=`expr "$1" : '\([0-9]*\)'` - tt=`expr "$1" : '.*-\([0-9]*\)'` - if [ "$tt" = "" ] ; then tt=$maxtest; fi - if expr \( "$tf" "<" 1 \) \| \( "$tt" ">" "$maxtest" \) >/dev/null; then - echo "Invalid test range '$1'"; exit 1 - fi - while expr "$tf" "<=" "$tt" >/dev/null; do - eval do${tf}=yes - tf=`expr $tf + 1` - done - else - echo "Invalid test range '$1'"; exit 1 - fi - ;; - *) echo "Unknown option or test selector '$1'"; exit 1;; - esac - shift -done - -# Find which optional facilities are available. - -$sim ./pcretest -C linksize >/dev/null -link_size=$? -if [ $link_size -lt 2 ] ; then - echo "Failed to find internal link size" - exit 1 -fi -if [ $link_size -gt 4 ] ; then - echo "Failed to find internal link size" - exit 1 -fi - -# All of 8-bit, 16-bit, and 32-bit character strings may be supported, but only -# one need be. - -$sim ./pcretest -C pcre8 >/dev/null -support8=$? -$sim ./pcretest -C pcre16 >/dev/null -support16=$? -$sim ./pcretest -C pcre32 >/dev/null -support32=$? - -# Initialize all bitsizes skipped - -test8=skip -test16=skip -test32=skip - -# If no bitsize arguments, select all that are available - -if [ "$arg8$arg16$arg32" = "" ] ; then - if [ $support8 -ne 0 ] ; then - test8= - fi - if [ $support16 -ne 0 ] ; then - test16=-16 - fi - if [ $support32 -ne 0 ] ; then - test32=-32 - fi - -# Select requested bit sizes - -else - if [ "$arg8" = yes ] ; then - if [ $support8 -eq 0 ] ; then - echo "Cannot run 8-bit library tests: 8-bit library not compiled" - exit 1 - fi - test8= - fi - if [ "$arg16" = yes ] ; then - if [ $support16 -eq 0 ] ; then - echo "Cannot run 16-bit library tests: 16-bit library not compiled" - exit 1 - fi - test16=-16 - fi - if [ "$arg32" = yes ] ; then - if [ $support32 -eq 0 ] ; then - echo "Cannot run 32-bit library tests: 32-bit library not compiled" - exit 1 - fi - test32=-32 - fi -fi - -# UTF support always applies to all bit sizes if both are supported; we can't -# have UTF-8 support without UTF-16 support (for example). - -$sim ./pcretest -C utf >/dev/null -utf=$? - -$sim ./pcretest -C ucp >/dev/null -ucp=$? - -jitopt= -$sim ./pcretest -C jit >/dev/null -jit=$? -if [ $jit -ne 0 -a "$nojit" != "yes" ] ; then - jitopt=-s+ - if [ "$valgrind" != "" ] ; then - vjs="--suppressions=$testdata/valgrind-jit.supp" - fi -fi - -# If no specific tests were requested, select all. Those that are not -# relevant will be automatically skipped. - -if [ $do1 = no -a $do2 = no -a $do3 = no -a $do4 = no -a \ - $do5 = no -a $do6 = no -a $do7 = no -a $do8 = no -a \ - $do9 = no -a $do10 = no -a $do11 = no -a $do12 = no -a \ - $do13 = no -a $do14 = no -a $do15 = no -a $do16 = no -a \ - $do17 = no -a $do18 = no -a $do19 = no -a $do20 = no -a \ - $do21 = no -a $do22 = no -a $do23 = no -a $do24 = no -a \ - $do25 = no -a $do26 = no ] ; then - do1=yes - do2=yes - do3=yes - do4=yes - do5=yes - do6=yes - do7=yes - do8=yes - do9=yes - do10=yes - do11=yes - do12=yes - do13=yes - do14=yes - do15=yes - do16=yes - do17=yes - do18=yes - do19=yes - do20=yes - do21=yes - do22=yes - do23=yes - do24=yes - do25=yes - do26=yes -fi - -# Handle any explicit skips at this stage, so that an argument list may consist -# only of explicit skips. - -for i in $skip; do eval do$i=no; done - -# Show which release and which test data - -echo "" -echo PCRE C library tests using test data from $testdata -$sim ./pcretest /dev/null - -for bmode in "$test8" "$test16" "$test32"; do - case "$bmode" in - skip) continue;; - -16) if [ "$test8$test32" != "skipskip" ] ; then echo ""; fi - bits=16; echo "---- Testing 16-bit library ----"; echo "";; - -32) if [ "$test8$test16" != "skipskip" ] ; then echo ""; fi - bits=32; echo "---- Testing 32-bit library ----"; echo "";; - *) bits=8; echo "---- Testing 8-bit library ----"; echo "";; - esac - -# Primary test, compatible with JIT and all versions of Perl >= 5.8 - -if [ $do1 = yes ] ; then - echo $title1 - for opt in "" "-s" $jitopt; do - $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput1 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput1 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - if [ "$opt" = "-s" ] ; then echo " OK with study" - elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" - else echo " OK" - fi - done -fi - -# PCRE tests that are not JIT or Perl-compatible: API, errors, internals - -if [ $do2 = yes ] ; then - echo $title2 "(not UTF-$bits)" - for opt in "" "-s" $jitopt; do - $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput2 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput2 testtry - if [ $? != 0 ] ; then exit 1; fi - else - echo " " - echo "** Test 2 requires a lot of stack. If it has crashed with a" - echo "** segmentation fault, it may be that you do not have enough" - echo "** stack available by default. Please see the 'pcrestack' man" - echo "** page for a discussion of PCRE's stack usage." - echo " " - exit 1 - fi - if [ "$opt" = "-s" ] ; then echo " OK with study" - elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" - else echo " OK" - fi - done -fi - -# Locale-specific tests, provided that either the "fr_FR" or the "french" -# locale is available. The former is the Unix-like standard; the latter is -# for Windows. Another possibility is "fr". Unfortunately, different versions -# of the French locale give different outputs for some items. This test passes -# if the output matches any one of the alternative output files. - -if [ $do3 = yes ] ; then - locale -a | grep '^fr_FR$' >/dev/null - if [ $? -eq 0 ] ; then - locale=fr_FR - infile=$testdata/testinput3 - outfile=$testdata/testoutput3 - outfile2=$testdata/testoutput3A - outfile3=$testdata/testoutput3B - else - infile=test3input - outfile=test3output - outfile2=test3outputA - outfile3=test3outputB - locale -a | grep '^french$' >/dev/null - if [ $? -eq 0 ] ; then - locale=french - sed 's/fr_FR/french/' $testdata/testinput3 >test3input - sed 's/fr_FR/french/' $testdata/testoutput3 >test3output - sed 's/fr_FR/french/' $testdata/testoutput3A >test3outputA - sed 's/fr_FR/french/' $testdata/testoutput3B >test3outputB - else - locale -a | grep '^fr$' >/dev/null - if [ $? -eq 0 ] ; then - locale=fr - sed 's/fr_FR/fr/' $testdata/intestinput3 >test3input - sed 's/fr_FR/fr/' $testdata/intestoutput3 >test3output - sed 's/fr_FR/fr/' $testdata/intestoutput3A >test3outputA - sed 's/fr_FR/fr/' $testdata/intestoutput3B >test3outputB - else - locale= - fi - fi - fi - - if [ "$locale" != "" ] ; then - echo $title3 "(using '$locale' locale)" - for opt in "" "-s" $jitopt; do - $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $infile testtry - if [ $? = 0 ] ; then - if $cf $outfile testtry >teststdout || \ - $cf $outfile2 testtry >teststdout || \ - $cf $outfile3 testtry >teststdout - then - if [ "$opt" = "-s" ] ; then echo " OK with study" - elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" - else echo " OK" - fi - else - echo "** Locale test did not run successfully. The output did not match" - echo " $outfile, $outfile2 or $outfile3." - echo " This may mean that there is a problem with the locale settings rather" - echo " than a bug in PCRE." - exit 1 - fi - else exit 1 - fi - done - else - echo "Cannot test locale-specific features - none of the 'fr_FR', 'fr' or" - echo "'french' locales exist, or the \"locale\" command is not available" - echo "to check for them." - echo " " - fi -fi - -# Additional tests for UTF support - -if [ $do4 = yes ] ; then - echo ${title4A}-${bits}${title4B} - if [ $utf -eq 0 ] ; then - echo " Skipped because UTF-$bits support is not available" - else - for opt in "" "-s" $jitopt; do - $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput4 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput4 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - if [ "$opt" = "-s" ] ; then echo " OK with study" - elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" - else echo " OK" - fi - done - fi -fi - -if [ $do5 = yes ] ; then - echo ${title5}-${bits} support - if [ $utf -eq 0 ] ; then - echo " Skipped because UTF-$bits support is not available" - else - for opt in "" "-s" $jitopt; do - $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput5 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput5 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - if [ "$opt" = "-s" ] ; then echo " OK with study" - elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" - else echo " OK" - fi - done - fi -fi - -if [ $do6 = yes ] ; then - echo $title6 - if [ $utf -eq 0 -o $ucp -eq 0 ] ; then - echo " Skipped because Unicode property support is not available" - else - for opt in "" "-s" $jitopt; do - $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput6 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput6 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - if [ "$opt" = "-s" ] ; then echo " OK with study" - elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" - else echo " OK" - fi - done - fi -fi - -# Test non-Perl-compatible Unicode property support - -if [ $do7 = yes ] ; then - echo $title7 - if [ $utf -eq 0 -o $ucp -eq 0 ] ; then - echo " Skipped because Unicode property support is not available" - else - for opt in "" "-s" $jitopt; do - $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput7 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput7 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - if [ "$opt" = "-s" ] ; then echo " OK with study" - elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" - else echo " OK" - fi - done - fi -fi - -# Tests for DFA matching support - -if [ $do8 = yes ] ; then - echo $title8 - for opt in "" "-s"; do - $sim $valgrind ./pcretest -q $bmode $opt -dfa $testdata/testinput8 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput8 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - if [ "$opt" = "-s" ] ; then echo " OK with study" ; else echo " OK"; fi - done -fi - -if [ $do9 = yes ] ; then - echo ${title9}-${bits} - if [ $utf -eq 0 ] ; then - echo " Skipped because UTF-$bits support is not available" - else - for opt in "" "-s"; do - $sim $valgrind ./pcretest -q $bmode $opt -dfa $testdata/testinput9 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput9 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - if [ "$opt" = "-s" ] ; then echo " OK with study" ; else echo " OK"; fi - done - fi -fi - -if [ $do10 = yes ] ; then - echo $title10 - if [ $utf -eq 0 -o $ucp -eq 0 ] ; then - echo " Skipped because Unicode property support is not available" - else - for opt in "" "-s"; do - $sim $valgrind ./pcretest -q $bmode $opt -dfa $testdata/testinput10 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput10 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - if [ "$opt" = "-s" ] ; then echo " OK with study" ; else echo " OK"; fi - done - fi -fi - -# Test of internal offsets and code sizes. This test is run only when there -# is Unicode property support and the link size is 2. The actual tests are -# mostly the same as in some of the above, but in this test we inspect some -# offsets and sizes that require a known link size. This is a doublecheck for -# the maintainer, just in case something changes unexpectely. The output from -# this test is not the same in 8-bit and 16-bit modes. - -if [ $do11 = yes ] ; then - echo $title11 - if [ $link_size -ne 2 ] ; then - echo " Skipped because link size is not 2" - elif [ $ucp -eq 0 ] ; then - echo " Skipped because Unicode property support is not available" - else - for opt in "" "-s"; do - $sim $valgrind ./pcretest -q $bmode $opt $testdata/testinput11 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput11-$bits testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - if [ "$opt" = "-s" ] ; then echo " OK with study" ; else echo " OK"; fi - done - fi -fi - -# Test JIT-specific features when JIT is available - -if [ $do12 = yes ] ; then - echo $title12 - if [ $jit -eq 0 -o "$nojit" = "yes" ] ; then - echo " Skipped because JIT is not available or not usable" - else - $sim $valgrind $vjs ./pcretest -q $bmode $testdata/testinput12 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput12 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - echo " OK" - fi -fi - -# Test JIT-specific features when JIT is not available - -if [ $do13 = yes ] ; then - echo $title13 - if [ $jit -ne 0 ] ; then - echo " Skipped because JIT is available" - else - $sim $valgrind ./pcretest -q $bmode $testdata/testinput13 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput13 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - echo " OK" - fi -fi - -# Tests for 8-bit-specific features - -if [ "$do14" = yes ] ; then - echo $title14 - if [ "$bits" = "16" -o "$bits" = "32" ] ; then - echo " Skipped when running 16/32-bit tests" - else - cp -f $testdata/saved16 testsaved16 - cp -f $testdata/saved32 testsaved32 - for opt in "" "-s" $jitopt; do - $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput14 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput14 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - if [ "$opt" = "-s" ] ; then echo " OK with study" - elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" - else echo " OK" - fi - done - fi -fi - -# Tests for 8-bit-specific features (needs UTF-8 support) - -if [ "$do15" = yes ] ; then - echo $title15 - if [ "$bits" = "16" -o "$bits" = "32" ] ; then - echo " Skipped when running 16/32-bit tests" - elif [ $utf -eq 0 ] ; then - echo " Skipped because UTF-$bits support is not available" - else - for opt in "" "-s" $jitopt; do - $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput15 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput15 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - if [ "$opt" = "-s" ] ; then echo " OK with study" - elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" - else echo " OK" - fi - done - fi -fi - -# Tests for 8-bit-specific features (Unicode property support) - -if [ $do16 = yes ] ; then - echo $title16 - if [ "$bits" = "16" -o "$bits" = "32" ] ; then - echo " Skipped when running 16/32-bit tests" - elif [ $ucp -eq 0 ] ; then - echo " Skipped because Unicode property support is not available" - else - for opt in "" "-s" $jitopt; do - $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput16 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput16 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - if [ "$opt" = "-s" ] ; then echo " OK with study" - elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" - else echo " OK" - fi - done - fi -fi - -# Tests for 16/32-bit-specific features - -if [ $do17 = yes ] ; then - echo $title17 - if [ "$bits" = "8" ] ; then - echo " Skipped when running 8-bit tests" - else - for opt in "" "-s" $jitopt; do - $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput17 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput17 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - if [ "$opt" = "-s" ] ; then echo " OK with study" - elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" - else echo " OK" - fi - done - fi -fi - -# Tests for 16/32-bit-specific features (UTF-16/32 support) - -if [ $do18 = yes ] ; then - echo $title18 - if [ "$bits" = "8" ] ; then - echo " Skipped when running 8-bit tests" - elif [ $utf -eq 0 ] ; then - echo " Skipped because UTF-$bits support is not available" - else - for opt in "" "-s" $jitopt; do - $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput18 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput18-$bits testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - if [ "$opt" = "-s" ] ; then echo " OK with study" - elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" - else echo " OK" - fi - done - fi -fi - -# Tests for 16/32-bit-specific features (Unicode property support) - -if [ $do19 = yes ] ; then - echo $title19 - if [ "$bits" = "8" ] ; then - echo " Skipped when running 8-bit tests" - elif [ $ucp -eq 0 ] ; then - echo " Skipped because Unicode property support is not available" - else - for opt in "" "-s" $jitopt; do - $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput19 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput19 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - if [ "$opt" = "-s" ] ; then echo " OK with study" - elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" - else echo " OK" - fi - done - fi -fi - -# Tests for 16/32-bit-specific features in DFA non-UTF-16/32 mode - -if [ $do20 = yes ] ; then - echo $title20 - if [ "$bits" = "8" ] ; then - echo " Skipped when running 8-bit tests" - else - for opt in "" "-s"; do - $sim $valgrind ./pcretest -q $bmode $opt -dfa $testdata/testinput20 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput20 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - if [ "$opt" = "-s" ] ; then echo " OK with study" - else echo " OK" - fi - done - fi -fi - -# Tests for reloads with 16/32-bit library - -if [ $do21 = yes ] ; then - echo $title21 - if [ "$bits" = "8" ] ; then - echo " Skipped when running 8-bit tests" - elif [ $link_size -ne 2 ] ; then - echo " Skipped because link size is not 2" - else - cp -f $testdata/saved8 testsaved8 - cp -f $testdata/saved16LE-1 testsaved16LE-1 - cp -f $testdata/saved16BE-1 testsaved16BE-1 - cp -f $testdata/saved32LE-1 testsaved32LE-1 - cp -f $testdata/saved32BE-1 testsaved32BE-1 - $sim $valgrind ./pcretest -q $bmode $testdata/testinput21 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput21-$bits testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - echo " OK" - fi -fi - -# Tests for reloads with 16/32-bit library (UTF-16 support) - -if [ $do22 = yes ] ; then - echo $title22 - if [ "$bits" = "8" ] ; then - echo " Skipped when running 8-bit tests" - elif [ $utf -eq 0 ] ; then - echo " Skipped because UTF-$bits support is not available" - elif [ $link_size -ne 2 ] ; then - echo " Skipped because link size is not 2" - else - cp -f $testdata/saved16LE-2 testsaved16LE-2 - cp -f $testdata/saved16BE-2 testsaved16BE-2 - cp -f $testdata/saved32LE-2 testsaved32LE-2 - cp -f $testdata/saved32BE-2 testsaved32BE-2 - $sim $valgrind ./pcretest -q $bmode $testdata/testinput22 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput22-$bits testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - echo " OK" - fi -fi - -if [ $do23 = yes ] ; then - echo $title23 - if [ "$bits" = "8" -o "$bits" = "32" ] ; then - echo " Skipped when running 8/32-bit tests" - else - $sim $valgrind ./pcretest -q $bmode $testdata/testinput23 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput23 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - echo " OK" - fi -fi - -if [ $do24 = yes ] ; then - echo $title24 - if [ "$bits" = "8" -o "$bits" = "32" ] ; then - echo " Skipped when running 8/32-bit tests" - elif [ $utf -eq 0 ] ; then - echo " Skipped because UTF-$bits support is not available" - else - $sim $valgrind ./pcretest -q $bmode $testdata/testinput24 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput24 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - echo " OK" - fi -fi - -if [ $do25 = yes ] ; then - echo $title25 - if [ "$bits" = "8" -o "$bits" = "16" ] ; then - echo " Skipped when running 8/16-bit tests" - else - $sim $valgrind ./pcretest -q $bmode $testdata/testinput25 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput25 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - echo " OK" - fi -fi - -if [ $do26 = yes ] ; then - echo $title26 - if [ "$bits" = "8" -o "$bits" = "16" ] ; then - echo " Skipped when running 8/16-bit tests" - elif [ $utf -eq 0 ] ; then - echo " Skipped because UTF-$bits support is not available" - else - $sim $valgrind ./pcretest -q $bmode $testdata/testinput26 testtry - if [ $? = 0 ] ; then - $cf $testdata/testoutput26 testtry - if [ $? != 0 ] ; then exit 1; fi - else exit 1 - fi - echo " OK" - fi -fi - -# End of loop for 8/16/32-bit tests -done - -# Clean up local working files -rm -f test3input test3output test3outputA testNinput testsaved* teststderr teststdout testtry - -# End diff --git a/src/third_party/pcre-8.39/RunTest.bat b/src/third_party/pcre-8.39/RunTest.bat deleted file mode 100644 index 35d7f71f9e8..00000000000 --- a/src/third_party/pcre-8.39/RunTest.bat +++ /dev/null @@ -1,616 +0,0 @@ -@echo off -@rem This file must use CRLF linebreaks to function properly -@rem and requires both pcretest and pcregrep -@rem This file was originally contributed by Ralf Junker, and touched up by -@rem Daniel Richard G. Tests 10-12 added by Philip H. -@rem Philip H also changed test 3 to use "wintest" files. -@rem -@rem Updated by Tom Fortmann to support explicit test numbers on the command line. -@rem Added argument validation and added error reporting. -@rem -@rem MS Windows batch file to run pcretest on testfiles with the correct -@rem options. -@rem -@rem Sheri Pierce added logic to skip feature dependent tests -@rem tests 4 5 9 15 and 18 require utf support -@rem tests 6 7 10 16 and 19 require ucp support -@rem 11 requires ucp and link size 2 -@rem 12 requires presence of jit support -@rem 13 requires absence of jit support -@rem Sheri P also added override tests for study and jit testing -@rem Zoltan Herczeg added libpcre16 support -@rem Zoltan Herczeg added libpcre32 support - -setlocal enabledelayedexpansion -if [%srcdir%]==[] ( -if exist testdata\ set srcdir=.) -if [%srcdir%]==[] ( -if exist ..\testdata\ set srcdir=..) -if [%srcdir%]==[] ( -if exist ..\..\testdata\ set srcdir=..\..) -if NOT exist %srcdir%\testdata\ ( -Error: echo distribution testdata folder not found! -call :conferror -exit /b 1 -goto :eof -) - -if [%pcretest%]==[] set pcretest=.\pcretest.exe - -echo source dir is %srcdir% -echo pcretest=%pcretest% - -if NOT exist %pcretest% ( -echo Error: %pcretest% not found! -echo. -call :conferror -exit /b 1 -) - -%pcretest% -C linksize >NUL -set link_size=%ERRORLEVEL% -%pcretest% -C pcre8 >NUL -set support8=%ERRORLEVEL% -%pcretest% -C pcre16 >NUL -set support16=%ERRORLEVEL% -%pcretest% -C pcre32 >NUL -set support32=%ERRORLEVEL% -%pcretest% -C utf >NUL -set utf=%ERRORLEVEL% -%pcretest% -C ucp >NUL -set ucp=%ERRORLEVEL% -%pcretest% -C jit >NUL -set jit=%ERRORLEVEL% - -if %support8% EQU 1 ( -if not exist testout8 md testout8 -if not exist testoutstudy8 md testoutstudy8 -if not exist testoutjit8 md testoutjit8 -) - -if %support16% EQU 1 ( -if not exist testout16 md testout16 -if not exist testoutstudy16 md testoutstudy16 -if not exist testoutjit16 md testoutjit16 -) - -if %support16% EQU 1 ( -if not exist testout32 md testout32 -if not exist testoutstudy32 md testoutstudy32 -if not exist testoutjit32 md testoutjit32 -) - -set do1=no -set do2=no -set do3=no -set do4=no -set do5=no -set do6=no -set do7=no -set do8=no -set do9=no -set do10=no -set do11=no -set do12=no -set do13=no -set do14=no -set do15=no -set do16=no -set do17=no -set do18=no -set do19=no -set do20=no -set do21=no -set do22=no -set do23=no -set do24=no -set do25=no -set do26=no -set all=yes - -for %%a in (%*) do ( - set valid=no - for %%v in (1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26) do if %%v == %%a set valid=yes - if "!valid!" == "yes" ( - set do%%a=yes - set all=no -) else ( - echo Invalid test number - %%a! - echo Usage %0 [ test_number ] ... - echo Where test_number is one or more optional test numbers 1 through 26, default is all tests. - exit /b 1 -) -) -set failed="no" - -if "%all%" == "yes" ( - set do1=yes - set do2=yes - set do3=yes - set do4=yes - set do5=yes - set do6=yes - set do7=yes - set do8=yes - set do9=yes - set do10=yes - set do11=yes - set do12=yes - set do13=yes - set do14=yes - set do15=yes - set do16=yes - set do17=yes - set do18=yes - set do19=yes - set do20=yes - set do21=yes - set do22=yes - set do23=yes - set do24=yes - set do25=yes - set do26=yes -) - -@echo RunTest.bat's pcretest output is written to newly created subfolders named -@echo testout, testoutstudy and testoutjit. -@echo. - -set mode= -set bits=8 - -:nextMode -if "%mode%" == "" ( - if %support8% EQU 0 goto modeSkip - echo. - echo ---- Testing 8-bit library ---- - echo. -) -if "%mode%" == "-16" ( - if %support16% EQU 0 goto modeSkip - echo. - echo ---- Testing 16-bit library ---- - echo. -) -if "%mode%" == "-32" ( - if %support32% EQU 0 goto modeSkip - echo. - echo ---- Testing 32-bit library ---- - echo. -) -if "%do1%" == "yes" call :do1 -if "%do2%" == "yes" call :do2 -if "%do3%" == "yes" call :do3 -if "%do4%" == "yes" call :do4 -if "%do5%" == "yes" call :do5 -if "%do6%" == "yes" call :do6 -if "%do7%" == "yes" call :do7 -if "%do8%" == "yes" call :do8 -if "%do9%" == "yes" call :do9 -if "%do10%" == "yes" call :do10 -if "%do11%" == "yes" call :do11 -if "%do12%" == "yes" call :do12 -if "%do13%" == "yes" call :do13 -if "%do14%" == "yes" call :do14 -if "%do15%" == "yes" call :do15 -if "%do16%" == "yes" call :do16 -if "%do17%" == "yes" call :do17 -if "%do18%" == "yes" call :do18 -if "%do19%" == "yes" call :do19 -if "%do20%" == "yes" call :do20 -if "%do21%" == "yes" call :do21 -if "%do22%" == "yes" call :do22 -if "%do23%" == "yes" call :do23 -if "%do24%" == "yes" call :do24 -if "%do25%" == "yes" call :do25 -if "%do26%" == "yes" call :do26 -:modeSkip -if "%mode%" == "" ( - set mode=-16 - set bits=16 - goto nextMode -) -if "%mode%" == "-16" ( - set mode=-32 - set bits=32 - goto nextMode -) - -@rem If mode is -32, testing is finished -if %failed% == "yes" ( -echo In above output, one or more of the various tests failed! -exit /b 1 -) -echo All OK -goto :eof - -:runsub -@rem Function to execute pcretest and compare the output -@rem Arguments are as follows: -@rem -@rem 1 = test number -@rem 2 = outputdir -@rem 3 = test name use double quotes -@rem 4 - 9 = pcretest options - -if [%1] == [] ( - echo Missing test number argument! - exit /b 1 -) - -if [%2] == [] ( - echo Missing outputdir! - exit /b 1 -) - -if [%3] == [] ( - echo Missing test name argument! - exit /b 1 -) - -set testinput=testinput%1 -set testoutput=testoutput%1 -if exist %srcdir%\testdata\win%testinput% ( - set testinput=wintestinput%1 - set testoutput=wintestoutput%1 -) - -echo Test %1: %3 -%pcretest% %mode% %4 %5 %6 %7 %8 %9 %srcdir%\testdata\%testinput% >%2%bits%\%testoutput% -if errorlevel 1 ( - echo. failed executing command-line: - echo. %pcretest% %mode% %4 %5 %6 %7 %8 %9 %srcdir%\testdata\%testinput% ^>%2%bits%\%testoutput% - set failed="yes" - goto :eof -) - -set type= -if [%1]==[11] ( - set type=-%bits% -) -if [%1]==[18] ( - set type=-%bits% -) -if [%1]==[21] ( - set type=-%bits% -) -if [%1]==[22] ( - set type=-%bits% -) - -fc /n %srcdir%\testdata\%testoutput%%type% %2%bits%\%testoutput% >NUL - -if errorlevel 1 ( - echo. failed comparison: fc /n %srcdir%\testdata\%testoutput% %2%bits%\%testoutput% - if [%1]==[2] ( - echo. - echo ** Test 2 requires a lot of stack. PCRE can be configured to - echo ** use heap for recursion. Otherwise, to pass Test 2 - echo ** you generally need to allocate 8 mb stack to PCRE. - echo ** See the 'pcrestack' page for a discussion of PCRE's - echo ** stack usage. - echo. -) - if [%1]==[3] ( - echo. - echo ** Test 3 failure usually means french locale is not - echo ** available on the system, rather than a bug or problem with PCRE. - echo. - goto :eof -) - - set failed="yes" - goto :eof -) - -echo. Passed. -goto :eof - -:do1 -call :runsub 1 testout "Main functionality (Compatible with Perl >= 5.10)" -q -call :runsub 1 testoutstudy "Test with Study Override" -q -s -if %jit% EQU 1 call :runsub 1 testoutjit "Test with JIT Override" -q -s+ -goto :eof - -:do2 - call :runsub 2 testout "API, errors, internals, and non-Perl stuff" -q - call :runsub 2 testoutstudy "Test with Study Override" -q -s - if %jit% EQU 1 call :runsub 2 testoutjit "Test with JIT Override" -q -s+ -goto :eof - -:do3 - call :runsub 3 testout "Locale-specific features" -q - call :runsub 3 testoutstudy "Test with Study Override" -q -s - if %jit% EQU 1 call :runsub 3 testoutjit "Test with JIT Override" -q -s+ -goto :eof - -:do4 -if %utf% EQU 0 ( - echo Test 4 Skipped due to absence of UTF-%bits% support. - goto :eof -) - call :runsub 4 testout "UTF-%bits% support - (Compatible with Perl >= 5.10)" -q - call :runsub 4 testoutstudy "Test with Study Override" -q -s - if %jit% EQU 1 call :runsub 4 testoutjit "Test with JIT Override" -q -s+ -goto :eof - -:do5 -if %utf% EQU 0 ( - echo Test 5 Skipped due to absence of UTF-%bits% support. - goto :eof -) - call :runsub 5 testout "API, internals, and non-Perl stuff for UTF-%bits%" -q - call :runsub 5 testoutstudy "Test with Study Override" -q -s - if %jit% EQU 1 call :runsub 5 testoutjit "Test with JIT Override" -q -s+ -goto :eof - -:do6 -if %ucp% EQU 0 ( - echo Test 6 Skipped due to absence of Unicode property support. - goto :eof -) - call :runsub 6 testout "Unicode property support (Compatible with Perl >= 5.10)" -q - call :runsub 6 testoutstudy "Test with Study Override" -q -s - if %jit% EQU 1 call :runsub 6 testoutjit "Test with JIT Override" -q -s+ -goto :eof - -:do7 -if %ucp% EQU 0 ( - echo Test 7 Skipped due to absence of Unicode property support. - goto :eof -) - call :runsub 7 testout "API, internals, and non-Perl stuff for Unicode property support" -q - call :runsub 7 testoutstudy "Test with Study Override" -q -s - if %jit% EQU 1 call :runsub 7 testoutjit "Test with JIT Override" -q -s+ -goto :eof - -:do8 - call :runsub 8 testout "DFA matching main functionality" -q -dfa - call :runsub 8 testoutstudy "Test with Study Override" -q -dfa -s -goto :eof - -:do9 -if %utf% EQU 0 ( - echo Test 9 Skipped due to absence of UTF-%bits% support. - goto :eof -) - call :runsub 9 testout "DFA matching with UTF-%bits%" -q -dfa - call :runsub 9 testoutstudy "Test with Study Override" -q -dfa -s - goto :eof - -:do10 -if %ucp% EQU 0 ( - echo Test 10 Skipped due to absence of Unicode property support. - goto :eof -) - call :runsub 10 testout "DFA matching with Unicode properties" -q -dfa - call :runsub 10 testoutstudy "Test with Study Override" -q -dfa -s -goto :eof - -:do11 -if NOT %link_size% EQU 2 ( - echo Test 11 Skipped because link size is not 2. - goto :eof -) -if %ucp% EQU 0 ( - echo Test 11 Skipped due to absence of Unicode property support. - goto :eof -) - call :runsub 11 testout "Internal offsets and code size tests" -q - call :runsub 11 testoutstudy "Test with Study Override" -q -s -goto :eof - -:do12 -if %jit% EQU 0 ( - echo Test 12 Skipped due to absence of JIT support. - goto :eof -) - call :runsub 12 testout "JIT-specific features (JIT available)" -q -goto :eof - -:do13 -if %jit% EQU 1 ( - echo Test 13 Skipped due to presence of JIT support. - goto :eof -) - call :runsub 13 testout "JIT-specific features (JIT not available)" -q -goto :eof - -:do14 -if NOT %bits% EQU 8 ( - echo Test 14 Skipped when running 16/32-bit tests. - goto :eof -) - copy /Y %srcdir%\testdata\saved16 testsaved16 - copy /Y %srcdir%\testdata\saved32 testsaved32 - call :runsub 14 testout "Specials for the basic 8-bit library" -q - call :runsub 14 testoutstudy "Test with Study Override" -q -s - if %jit% EQU 1 call :runsub 14 testoutjit "Test with JIT Override" -q -s+ -goto :eof - -:do15 -if NOT %bits% EQU 8 ( - echo Test 15 Skipped when running 16/32-bit tests. - goto :eof -) -if %utf% EQU 0 ( - echo Test 15 Skipped due to absence of UTF-%bits% support. - goto :eof -) - call :runsub 15 testout "Specials for the 8-bit library with UTF-%bits% support" -q - call :runsub 15 testoutstudy "Test with Study Override" -q -s - if %jit% EQU 1 call :runsub 15 testoutjit "Test with JIT Override" -q -s+ -goto :eof - -:do16 -if NOT %bits% EQU 8 ( - echo Test 16 Skipped when running 16/32-bit tests. - goto :eof -) -if %ucp% EQU 0 ( - echo Test 16 Skipped due to absence of Unicode property support. - goto :eof -) - call :runsub 16 testout "Specials for the 8-bit library with Unicode propery support" -q - call :runsub 16 testoutstudy "Test with Study Override" -q -s - if %jit% EQU 1 call :runsub 16 testoutjit "Test with JIT Override" -q -s+ -goto :eof - -:do17 -if %bits% EQU 8 ( - echo Test 17 Skipped when running 8-bit tests. - goto :eof -) - call :runsub 17 testout "Specials for the basic 16/32-bit library" -q - call :runsub 17 testoutstudy "Test with Study Override" -q -s - if %jit% EQU 1 call :runsub 17 testoutjit "Test with JIT Override" -q -s+ -goto :eof - -:do18 -if %bits% EQU 8 ( - echo Test 18 Skipped when running 8-bit tests. - goto :eof -) -if %utf% EQU 0 ( - echo Test 18 Skipped due to absence of UTF-%bits% support. - goto :eof -) - call :runsub 18 testout "Specials for the 16/32-bit library with UTF-%bits% support" -q - call :runsub 18 testoutstudy "Test with Study Override" -q -s - if %jit% EQU 1 call :runsub 18 testoutjit "Test with JIT Override" -q -s+ -goto :eof - -:do19 -if %bits% EQU 8 ( - echo Test 19 Skipped when running 8-bit tests. - goto :eof -) -if %ucp% EQU 0 ( - echo Test 19 Skipped due to absence of Unicode property support. - goto :eof -) - call :runsub 19 testout "Specials for the 16/32-bit library with Unicode property support" -q - call :runsub 19 testoutstudy "Test with Study Override" -q -s - if %jit% EQU 1 call :runsub 19 testoutjit "Test with JIT Override" -q -s+ -goto :eof - -:do20 -if %bits% EQU 8 ( - echo Test 20 Skipped when running 8-bit tests. - goto :eof -) - call :runsub 20 testout "DFA specials for the basic 16/32-bit library" -q -dfa - call :runsub 20 testoutstudy "Test with Study Override" -q -dfa -s -goto :eof - -:do21 -if %bits% EQU 8 ( - echo Test 21 Skipped when running 8-bit tests. - goto :eof -) -if NOT %link_size% EQU 2 ( - echo Test 21 Skipped because link size is not 2. - goto :eof -) -copy /Y %srcdir%\testdata\saved8 testsaved8 -copy /Y %srcdir%\testdata\saved16LE-1 testsaved16LE-1 -copy /Y %srcdir%\testdata\saved16BE-1 testsaved16BE-1 -copy /Y %srcdir%\testdata\saved32LE-1 testsaved32LE-1 -copy /Y %srcdir%\testdata\saved32BE-1 testsaved32BE-1 -call :runsub 21 testout "Reloads for the basic 16/32-bit library" -q -call :runsub 21 testoutstudy "Test with Study Override" -q -s -if %jit% EQU 1 call :runsub 21 testoutjit "Test with JIT Override" -q -s+ -goto :eof - -:do22 -if %bits% EQU 8 ( - echo Test 22 Skipped when running 8-bit tests. - goto :eof -) -if %utf% EQU 0 ( - echo Test 22 Skipped due to absence of UTF-%bits% support. - goto :eof -) -if NOT %link_size% EQU 2 ( - echo Test 22 Skipped because link size is not 2. - goto :eof -) -copy /Y %srcdir%\testdata\saved16LE-2 testsaved16LE-2 -copy /Y %srcdir%\testdata\saved16BE-2 testsaved16BE-2 -copy /Y %srcdir%\testdata\saved32LE-2 testsaved32LE-2 -copy /Y %srcdir%\testdata\saved32BE-2 testsaved32BE-2 -call :runsub 22 testout "Reloads for the 16/32-bit library with UTF-16/32 support" -q -call :runsub 22 testoutstudy "Test with Study Override" -q -s -if %jit% EQU 1 call :runsub 22 testoutjit "Test with JIT Override" -q -s+ -goto :eof - -:do23 -if NOT %bits% EQU 16 ( - echo Test 23 Skipped when running 8/32-bit tests. - goto :eof -) -call :runsub 23 testout "Specials for the 16-bit library" -q -call :runsub 23 testoutstudy "Test with Study Override" -q -s -if %jit% EQU 1 call :runsub 23 testoutjit "Test with JIT Override" -q -s+ -goto :eof - -:do24 -if NOT %bits% EQU 16 ( - echo Test 24 Skipped when running 8/32-bit tests. - goto :eof -) -if %utf% EQU 0 ( - echo Test 24 Skipped due to absence of UTF-%bits% support. - goto :eof -) -call :runsub 24 testout "Specials for the 16-bit library with UTF-16 support" -q -call :runsub 24 testoutstudy "Test with Study Override" -q -s -if %jit% EQU 1 call :runsub 24 testoutjit "Test with JIT Override" -q -s+ -goto :eof - -:do25 -if NOT %bits% EQU 32 ( - echo Test 25 Skipped when running 8/16-bit tests. - goto :eof -) -call :runsub 25 testout "Specials for the 32-bit library" -q -call :runsub 25 testoutstudy "Test with Study Override" -q -s -if %jit% EQU 1 call :runsub 25 testoutjit "Test with JIT Override" -q -s+ -goto :eof - -:do26 -if NOT %bits% EQU 32 ( - echo Test 26 Skipped when running 8/16-bit tests. - goto :eof -) -if %utf% EQU 0 ( - echo Test 26 Skipped due to absence of UTF-%bits% support. - goto :eof -) -call :runsub 26 testout "Specials for the 32-bit library with UTF-32 support" -q -call :runsub 26 testoutstudy "Test with Study Override" -q -s -if %jit% EQU 1 call :runsub 26 testoutjit "Test with JIT Override" -q -s+ -goto :eof - -:conferror -@echo. -@echo Either your build is incomplete or you have a configuration error. -@echo. -@echo If configured with cmake and executed via "make test" or the MSVC "RUN_TESTS" -@echo project, pcre_test.bat defines variables and automatically calls RunTest.bat. -@echo For manual testing of all available features, after configuring with cmake -@echo and building, you can run the built pcre_test.bat. For best results with -@echo cmake builds and tests avoid directories with full path names that include -@echo spaces for source or build. -@echo. -@echo Otherwise, if the build dir is in a subdir of the source dir, testdata needed -@echo for input and verification should be found automatically when (from the -@echo location of the the built exes) you call RunTest.bat. By default RunTest.bat -@echo runs all tests compatible with the linked pcre library but it can be given -@echo a test number as an argument. -@echo. -@echo If the build dir is not under the source dir you can either copy your exes -@echo to the source folder or copy RunTest.bat and the testdata folder to the -@echo location of your built exes and then run RunTest.bat. -@echo. -goto :eof diff --git a/src/third_party/pcre-8.39/SConscript b/src/third_party/pcre-8.39/SConscript deleted file mode 100644 index e7b9a59a550..00000000000 --- a/src/third_party/pcre-8.39/SConscript +++ /dev/null @@ -1,68 +0,0 @@ -# -*- mode: python -*- - -Import("env") - -env = env.Clone() -env.Append( CPPDEFINES=[ "HAVE_CONFIG_H", ] ) - -def removeIfPresent(lst, item): - try: - lst.remove(item) - except ValueError: - pass - -for to_remove in ['-Werror', '-Wall', '-W']: - removeIfPresent(env['CCFLAGS'], to_remove) - -# Directories that include generated config.h for various platforms -# -# Generated via -# AutoTools (non-Windows) -# ./configure --disable-stack-for-recursion --enable-utf --enable-unicode-properties -# --with-match-limit=200000 --with-match-limit-recursion=4000 --enable-shared=no -# CMake (Windows) -# -DPCRE_SUPPORT_PCREGREP_JIT:BOOL="0" -DPCRE_BUILD_TESTS:BOOL="0" -# -DPCRE_POSIX_MALLOC_THRESHOLD:STRING="10" -DPCRE_MATCH_LIMIT_RECURSION:STRING="4000" -# -DPCRE_NO_RECURSE:BOOL="1" -DPCRE_LINK_SIZE:STRING="2" -DPCRE_NEWLINE:STRING="LF" -# -DPCRE_SUPPORT_UNICODE_PROPERTIES:BOOL="1" -DPCREGREP_BUFSIZE:STRING="20480" -# -DPCRE_MATCH_LIMIT:STRING="200000" -DPCRE_PARENS_NEST_LIMIT:STRING="250" -# -DPCRE_SUPPORT_UTF:BOOL="1" -# -if env.TargetOSIs('windows'): - env.Append(CPPPATH=["build_windows"]) -elif env.TargetOSIs('solaris'): - env.Append(CPPPATH=["build_solaris"]) -else: - env.Append(CPPPATH=["build_posix"]) - -env.Library( "pcrecpp", [ - # pcre - "pcre_byte_order.c", - "pcre_compile.c", - "pcre_config.c", - "pcre_dfa_exec.c", - "pcre_exec.c", - "pcre_fullinfo.c", - "pcre_get.c", - "pcre_globals.c", - "pcre_maketables.c", - "pcre_newline.c", - "pcre_ord2utf8.c", - "pcre_refcount.c", - "pcre_string_utils.c", - "pcre_study.c", - "pcre_tables.c", - "pcre_ucd.c", - "pcre_valid_utf8.c", - "pcre_version.c", - "pcre_xclass.c", - # pcre nodist - "pcre_chartables.c", - # pcre cpp - "pcrecpp.cc", - "pcre_scanner.cc", - "pcre_stringpiece.cc", - # pcre posix - "pcreposix.c", - ] ) - diff --git a/src/third_party/pcre-8.39/ar-lib b/src/third_party/pcre-8.39/ar-lib deleted file mode 100755 index 463b9ec0206..00000000000 --- a/src/third_party/pcre-8.39/ar-lib +++ /dev/null @@ -1,270 +0,0 @@ -#! /bin/sh -# Wrapper for Microsoft lib.exe - -me=ar-lib -scriptversion=2012-03-01.08; # UTC - -# Copyright (C) 2010-2014 Free Software Foundation, Inc. -# Written by Peter Rosin . -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# As a special exception to the GNU General Public License, if you -# distribute this file as part of a program that contains a -# configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that program. - -# This file is maintained in Automake, please report -# bugs to or send patches to -# . - - -# func_error message -func_error () -{ - echo "$me: $1" 1>&2 - exit 1 -} - -file_conv= - -# func_file_conv build_file -# Convert a $build file to $host form and store it in $file -# Currently only supports Windows hosts. -func_file_conv () -{ - file=$1 - case $file in - / | /[!/]*) # absolute file, and not a UNC file - if test -z "$file_conv"; then - # lazily determine how to convert abs files - case `uname -s` in - MINGW*) - file_conv=mingw - ;; - CYGWIN*) - file_conv=cygwin - ;; - *) - file_conv=wine - ;; - esac - fi - case $file_conv in - mingw) - file=`cmd //C echo "$file " | sed -e 's/"\(.*\) " *$/\1/'` - ;; - cygwin) - file=`cygpath -m "$file" || echo "$file"` - ;; - wine) - file=`winepath -w "$file" || echo "$file"` - ;; - esac - ;; - esac -} - -# func_at_file at_file operation archive -# Iterate over all members in AT_FILE performing OPERATION on ARCHIVE -# for each of them. -# When interpreting the content of the @FILE, do NOT use func_file_conv, -# since the user would need to supply preconverted file names to -# binutils ar, at least for MinGW. -func_at_file () -{ - operation=$2 - archive=$3 - at_file_contents=`cat "$1"` - eval set x "$at_file_contents" - shift - - for member - do - $AR -NOLOGO $operation:"$member" "$archive" || exit $? - done -} - -case $1 in - '') - func_error "no command. Try '$0 --help' for more information." - ;; - -h | --h*) - cat < header file. */ -/* #undef HAVE_BITS_TYPE_TRAITS_H */ - -/* Define to 1 if you have the header file. */ -#define HAVE_BZLIB_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_DIRENT_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_DLFCN_H 1 - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_EDITLINE_READLINE_H */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_EDIT_READLINE_READLINE_H */ - -/* Define to 1 if you have the header file. */ -#define HAVE_INTTYPES_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_LIMITS_H 1 - -/* Define to 1 if the system has the type `long long'. */ -#define HAVE_LONG_LONG 1 - -/* Define to 1 if you have the `memmove' function. */ -#define HAVE_MEMMOVE 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_MEMORY_H 1 - -/* Define if you have POSIX threads libraries and header files. */ -/* #undef HAVE_PTHREAD */ - -/* Have PTHREAD_PRIO_INHERIT. */ -/* #undef HAVE_PTHREAD_PRIO_INHERIT */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_READLINE_HISTORY_H */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_READLINE_READLINE_H */ - -/* Define to 1 if you have the header file. */ -#define HAVE_STDINT_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDLIB_H 1 - -/* Define to 1 if you have the `strerror' function. */ -#define HAVE_STRERROR 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STRING 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STRINGS_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STRING_H 1 - -/* Define to 1 if you have `strtoimax'. */ -/* #undef HAVE_STRTOIMAX */ - -/* Define to 1 if you have `strtoll'. */ -/* #undef HAVE_STRTOLL */ - -/* Define to 1 if you have `strtoq'. */ -#define HAVE_STRTOQ 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_STAT_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_TYPES_H 1 - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_TYPE_TRAITS_H */ - -/* Define to 1 if you have the header file. */ -#define HAVE_UNISTD_H 1 - -/* Define to 1 if the system has the type `unsigned long long'. */ -#define HAVE_UNSIGNED_LONG_LONG 1 - -/* Define to 1 if the compiler supports simple visibility declarations. */ -#define HAVE_VISIBILITY 1 - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_WINDOWS_H */ - -/* Define to 1 if you have the header file. */ -#define HAVE_ZLIB_H 1 - -/* Define to 1 if you have `_strtoi64'. */ -/* #undef HAVE__STRTOI64 */ - -/* The value of LINK_SIZE determines the number of bytes used to store links - as offsets within the compiled regex. The default is 2, which allows for - compiled patterns up to 64K long. This covers the vast majority of cases. - However, PCRE can also be compiled to use 3 or 4 bytes instead. This allows - for longer patterns in extreme cases. */ -#define LINK_SIZE 2 - -/* Define to the sub-directory where libtool stores uninstalled libraries. */ -#define LT_OBJDIR ".libs/" - -/* The value of MATCH_LIMIT determines the default number of times the - internal match() function can be called during a single execution of - pcre_exec(). There is a runtime interface for setting a different limit. - The limit exists in order to catch runaway regular expressions that take - for ever to determine that they do not match. The default is set very large - so that it does not accidentally catch legitimate cases. */ -#define MATCH_LIMIT 200000 - -/* The above limit applies to all calls of match(), whether or not they - increase the recursion depth. In some environments it is desirable to limit - the depth of recursive calls of match() more strictly, in order to restrict - the maximum amount of stack (or heap, if NO_RECURSE is defined) that is - used. The value of MATCH_LIMIT_RECURSION applies only to recursive calls of - match(). To have any useful effect, it must be less than the value of - MATCH_LIMIT. The default is to use the same value as MATCH_LIMIT. There is - a runtime method for setting a different limit. */ -#define MATCH_LIMIT_RECURSION 4000 - -/* This limit is parameterized just in case anybody ever wants to change it. - Care must be taken if it is increased, because it guards against integer - overflow caused by enormously large patterns. */ -#define MAX_NAME_COUNT 10000 - -/* This limit is parameterized just in case anybody ever wants to change it. - Care must be taken if it is increased, because it guards against integer - overflow caused by enormously large patterns. */ -#define MAX_NAME_SIZE 32 - -/* The value of NEWLINE determines the default newline character sequence. - PCRE client programs can override this by selecting other values at run - time. In ASCII environments, the value can be 10 (LF), 13 (CR), or 3338 - (CRLF); in EBCDIC environments the value can be 21 or 37 (LF), 13 (CR), or - 3349 or 3365 (CRLF) because there are two alternative codepoints (0x15 and - 0x25) that are used as the NL line terminator that is equivalent to ASCII - LF. In both ASCII and EBCDIC environments the value can also be -1 (ANY), - or -2 (ANYCRLF). */ -#define NEWLINE 10 - -/* PCRE uses recursive function calls to handle backtracking while matching. - This can sometimes be a problem on systems that have stacks of limited - size. Define NO_RECURSE to any value to get a version that doesn't use - recursion in the match() function; instead it creates its own stack by - steam using pcre_recurse_malloc() to obtain memory from the heap. For more - detail, see the comments and other stuff just above the match() function. - */ -#define NO_RECURSE /**/ - -/* Name of package */ -#define PACKAGE "pcre" - -/* Define to the address where bug reports for this package should be sent. */ -#define PACKAGE_BUGREPORT "" - -/* Define to the full name of this package. */ -#define PACKAGE_NAME "PCRE" - -/* Define to the full name and version of this package. */ -#define PACKAGE_STRING "PCRE 8.39" - -/* Define to the one symbol short name of this package. */ -#define PACKAGE_TARNAME "pcre" - -/* Define to the home page for this package. */ -#define PACKAGE_URL "" - -/* Define to the version of this package. */ -#define PACKAGE_VERSION "8.39" - -/* The value of PARENS_NEST_LIMIT specifies the maximum depth of nested - parentheses (of any kind) in a pattern. This limits the amount of system - stack that is used while compiling a pattern. */ -#define PARENS_NEST_LIMIT 250 - -/* to make a symbol visible */ -#define PCRECPP_EXP_DECL extern __attribute__ ((visibility ("default"))) - -/* to make a symbol visible */ -#define PCRECPP_EXP_DEFN __attribute__ ((visibility ("default"))) - -/* The value of PCREGREP_BUFSIZE determines the size of buffer used by - pcregrep to hold parts of the file it is searching. This is also the - minimum value. The actual amount of memory used by pcregrep is three times - this number, because it allows for the buffering of "before" and "after" - lines. */ -#define PCREGREP_BUFSIZE 20480 - -/* to make a symbol visible */ -#define PCREPOSIX_EXP_DECL extern __attribute__ ((visibility ("default"))) - -/* to make a symbol visible */ -#define PCREPOSIX_EXP_DEFN extern __attribute__ ((visibility ("default"))) - -/* to make a symbol visible */ -#define PCRE_EXP_DATA_DEFN __attribute__ ((visibility ("default"))) - -/* to make a symbol visible */ -#define PCRE_EXP_DECL extern __attribute__ ((visibility ("default"))) - - -/* If you are compiling for a system other than a Unix-like system or - Win32, and it needs some magic to be inserted before the definition - of a function that is exported by the library, define this macro to - contain the relevant magic. If you do not define this macro, a suitable - __declspec value is used for Windows systems; in other environments - "extern" is used for a C compiler and "extern C" for a C++ compiler. - This macro apears at the start of every exported function that is part - of the external API. It does not appear on functions that are "external" - in the C sense, but which are internal to the library. */ -#define PCRE_EXP_DEFN __attribute__ ((visibility ("default"))) - -/* Define to any value if linking statically (TODO: make nice with Libtool) */ -#define PCRE_STATIC 1 - -/* When calling PCRE via the POSIX interface, additional working storage is - required for holding the pointers to capturing substrings because PCRE - requires three integers per substring, whereas the POSIX interface provides - only two. If the number of expected substrings is small, the wrapper - function uses space on the stack, because this is faster than using - malloc() for each call. The threshold above which the stack is no longer - used is defined by POSIX_MALLOC_THRESHOLD. */ -#define POSIX_MALLOC_THRESHOLD 10 - -/* Define to necessary symbol if this constant uses a non-standard name on - your system. */ -/* #undef PTHREAD_CREATE_JOINABLE */ - -/* Define to 1 if you have the ANSI C header files. */ -#define STDC_HEADERS 1 - -/* Define to any value to enable support for Just-In-Time compiling. */ -/* #undef SUPPORT_JIT */ - -/* Define to any value to allow pcregrep to be linked with libbz2, so that it - is able to handle .bz2 files. */ -/* #undef SUPPORT_LIBBZ2 */ - -/* Define to any value to allow pcretest to be linked with libedit. */ -/* #undef SUPPORT_LIBEDIT */ - -/* Define to any value to allow pcretest to be linked with libreadline. */ -/* #undef SUPPORT_LIBREADLINE */ - -/* Define to any value to allow pcregrep to be linked with libz, so that it is - able to handle .gz files. */ -/* #undef SUPPORT_LIBZ */ - -/* Define to any value to enable the 16 bit PCRE library. */ -/* #undef SUPPORT_PCRE16 */ - -/* Define to any value to enable the 32 bit PCRE library. */ -/* #undef SUPPORT_PCRE32 */ - -/* Define to any value to enable the 8 bit PCRE library. */ -#define SUPPORT_PCRE8 /**/ - -/* Define to any value to enable JIT support in pcregrep. */ -/* #undef SUPPORT_PCREGREP_JIT */ - -/* Define to any value to enable support for Unicode properties. */ -#define SUPPORT_UCP /**/ - -/* Define to any value to enable support for the UTF-8/16/32 Unicode encoding. - This will work even in an EBCDIC environment, but it is incompatible with - the EBCDIC macro. That is, PCRE can support *either* EBCDIC code *or* - ASCII/UTF-8/16/32, but not both at once. */ -#define SUPPORT_UTF /**/ - -/* Define to any value for valgrind support to find invalid memory reads. */ -/* #undef SUPPORT_VALGRIND */ - -/* Version number of package */ -#define VERSION "8.39" - -/* Define to empty if `const' does not conform to ANSI C. */ -/* #undef const */ - -/* Define to the type of a signed integer type of width exactly 64 bits if - such a type exists and the standard includes do not define it. */ -/* #undef int64_t */ - -/* Define to `unsigned int' if does not define. */ -/* #undef size_t */ diff --git a/src/third_party/pcre-8.39/build_solaris/config.h b/src/third_party/pcre-8.39/build_solaris/config.h deleted file mode 100644 index a391fd1cee5..00000000000 --- a/src/third_party/pcre-8.39/build_solaris/config.h +++ /dev/null @@ -1,348 +0,0 @@ -/* config.h. Generated from config.h.in by configure. */ -/* config.h.in. Generated from configure.ac by autoheader. */ - - -/* PCRE is written in Standard C, but there are a few non-standard things it -can cope with, allowing it to run on SunOS4 and other "close to standard" -systems. - -In environments that support the GNU autotools, config.h.in is converted into -config.h by the "configure" script. In environments that use CMake, -config-cmake.in is converted into config.h. If you are going to build PCRE "by -hand" without using "configure" or CMake, you should copy the distributed -config.h.generic to config.h, and edit the macro definitions to be the way you -need them. You must then add -DHAVE_CONFIG_H to all of your compile commands, -so that config.h is included at the start of every source. - -Alternatively, you can avoid editing by using -D on the compiler command line -to set the macro values. In this case, you do not have to set -DHAVE_CONFIG_H, -but if you do, default values will be taken from config.h for non-boolean -macros that are not defined on the command line. - -Boolean macros such as HAVE_STDLIB_H and SUPPORT_PCRE8 should either be defined -(conventionally to 1) for TRUE, and not defined at all for FALSE. All such -macros are listed as a commented #undef in config.h.generic. Macros such as -MATCH_LIMIT, whose actual value is relevant, have defaults defined, but are -surrounded by #ifndef/#endif lines so that the value can be overridden by -D. - -PCRE uses memmove() if HAVE_MEMMOVE is defined; otherwise it uses bcopy() if -HAVE_BCOPY is defined. If your system has neither bcopy() nor memmove(), make -sure both macros are undefined; an emulation function will then be used. */ - -/* By default, the \R escape sequence matches any Unicode line ending - character or sequence of characters. If BSR_ANYCRLF is defined (to any - value), this is changed so that backslash-R matches only CR, LF, or CRLF. - The build-time default can be overridden by the user of PCRE at runtime. */ -/* #undef BSR_ANYCRLF */ - -/* If you are compiling for a system that uses EBCDIC instead of ASCII - character codes, define this macro to any value. You must also edit the - NEWLINE macro below to set a suitable EBCDIC newline, commonly 21 (0x15). - On systems that can use "configure" or CMake to set EBCDIC, NEWLINE is - automatically adjusted. When EBCDIC is set, PCRE assumes that all input - strings are in EBCDIC. If you do not define this macro, PCRE will assume - input strings are ASCII or UTF-8/16/32 Unicode. It is not possible to build - a version of PCRE that supports both EBCDIC and UTF-8/16/32. */ -/* #undef EBCDIC */ - -/* In an EBCDIC environment, define this macro to any value to arrange for the - NL character to be 0x25 instead of the default 0x15. NL plays the role that - LF does in an ASCII/Unicode environment. The value must also be set in the - NEWLINE macro below. On systems that can use "configure" or CMake to set - EBCDIC_NL25, the adjustment of NEWLINE is automatic. */ -/* #undef EBCDIC_NL25 */ - -/* Define to 1 if you have the `bcopy' function. */ -#define HAVE_BCOPY 1 - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_BITS_TYPE_TRAITS_H */ - -/* Define to 1 if you have the header file. */ -#define HAVE_BZLIB_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_DIRENT_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_DLFCN_H 1 - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_EDITLINE_READLINE_H */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_EDIT_READLINE_READLINE_H */ - -/* Define to 1 if you have the header file. */ -#define HAVE_INTTYPES_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_LIMITS_H 1 - -/* Define to 1 if the system has the type `long long'. */ -#define HAVE_LONG_LONG 1 - -/* Define to 1 if you have the `memmove' function. */ -#define HAVE_MEMMOVE 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_MEMORY_H 1 - -/* Define if you have POSIX threads libraries and header files. */ -/* #undef HAVE_PTHREAD */ - -/* Have PTHREAD_PRIO_INHERIT. */ -/* #undef HAVE_PTHREAD_PRIO_INHERIT */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_READLINE_HISTORY_H */ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_READLINE_READLINE_H */ - -/* Define to 1 if you have the header file. */ -#define HAVE_STDINT_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDLIB_H 1 - -/* Define to 1 if you have the `strerror' function. */ -#define HAVE_STRERROR 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STRING 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STRINGS_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STRING_H 1 - -/* Define to 1 if you have `strtoimax'. */ -/* #undef HAVE_STRTOIMAX */ - -/* Define to 1 if you have `strtoll'. */ -#define HAVE_STRTOLL 1 - -/* Define to 1 if you have `strtoq'. */ -/* #undef HAVE_STRTOQ */ - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_STAT_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_TYPES_H 1 - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_TYPE_TRAITS_H */ - -/* Define to 1 if you have the header file. */ -#define HAVE_UNISTD_H 1 - -/* Define to 1 if the system has the type `unsigned long long'. */ -#define HAVE_UNSIGNED_LONG_LONG 1 - -/* Define to 1 if the compiler supports simple visibility declarations. */ -#define HAVE_VISIBILITY 1 - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_WINDOWS_H */ - -/* Define to 1 if you have the header file. */ -#define HAVE_ZLIB_H 1 - -/* Define to 1 if you have `_strtoi64'. */ -/* #undef HAVE__STRTOI64 */ - -/* The value of LINK_SIZE determines the number of bytes used to store links - as offsets within the compiled regex. The default is 2, which allows for - compiled patterns up to 64K long. This covers the vast majority of cases. - However, PCRE can also be compiled to use 3 or 4 bytes instead. This allows - for longer patterns in extreme cases. */ -#define LINK_SIZE 2 - -/* Define to the sub-directory where libtool stores uninstalled libraries. */ -#define LT_OBJDIR ".libs/" - -/* The value of MATCH_LIMIT determines the default number of times the - internal match() function can be called during a single execution of - pcre_exec(). There is a runtime interface for setting a different limit. - The limit exists in order to catch runaway regular expressions that take - for ever to determine that they do not match. The default is set very large - so that it does not accidentally catch legitimate cases. */ -#define MATCH_LIMIT 200000 - -/* The above limit applies to all calls of match(), whether or not they - increase the recursion depth. In some environments it is desirable to limit - the depth of recursive calls of match() more strictly, in order to restrict - the maximum amount of stack (or heap, if NO_RECURSE is defined) that is - used. The value of MATCH_LIMIT_RECURSION applies only to recursive calls of - match(). To have any useful effect, it must be less than the value of - MATCH_LIMIT. The default is to use the same value as MATCH_LIMIT. There is - a runtime method for setting a different limit. */ -#define MATCH_LIMIT_RECURSION 4000 - -/* This limit is parameterized just in case anybody ever wants to change it. - Care must be taken if it is increased, because it guards against integer - overflow caused by enormously large patterns. */ -#define MAX_NAME_COUNT 10000 - -/* This limit is parameterized just in case anybody ever wants to change it. - Care must be taken if it is increased, because it guards against integer - overflow caused by enormously large patterns. */ -#define MAX_NAME_SIZE 32 - -/* The value of NEWLINE determines the default newline character sequence. - PCRE client programs can override this by selecting other values at run - time. In ASCII environments, the value can be 10 (LF), 13 (CR), or 3338 - (CRLF); in EBCDIC environments the value can be 21 or 37 (LF), 13 (CR), or - 3349 or 3365 (CRLF) because there are two alternative codepoints (0x15 and - 0x25) that are used as the NL line terminator that is equivalent to ASCII - LF. In both ASCII and EBCDIC environments the value can also be -1 (ANY), - or -2 (ANYCRLF). */ -#define NEWLINE 10 - -/* PCRE uses recursive function calls to handle backtracking while matching. - This can sometimes be a problem on systems that have stacks of limited - size. Define NO_RECURSE to any value to get a version that doesn't use - recursion in the match() function; instead it creates its own stack by - steam using pcre_recurse_malloc() to obtain memory from the heap. For more - detail, see the comments and other stuff just above the match() function. - */ -#define NO_RECURSE /**/ - -/* Name of package */ -#define PACKAGE "pcre" - -/* Define to the address where bug reports for this package should be sent. */ -#define PACKAGE_BUGREPORT "" - -/* Define to the full name of this package. */ -#define PACKAGE_NAME "PCRE" - -/* Define to the full name and version of this package. */ -#define PACKAGE_STRING "PCRE 8.39" - -/* Define to the one symbol short name of this package. */ -#define PACKAGE_TARNAME "pcre" - -/* Define to the home page for this package. */ -#define PACKAGE_URL "" - -/* Define to the version of this package. */ -#define PACKAGE_VERSION "8.39" - -/* The value of PARENS_NEST_LIMIT specifies the maximum depth of nested - parentheses (of any kind) in a pattern. This limits the amount of system - stack that is used while compiling a pattern. */ -#define PARENS_NEST_LIMIT 250 - -/* to make a symbol visible */ -#define PCRECPP_EXP_DECL extern __attribute__ ((visibility ("default"))) - -/* to make a symbol visible */ -#define PCRECPP_EXP_DEFN __attribute__ ((visibility ("default"))) - -/* The value of PCREGREP_BUFSIZE determines the size of buffer used by - pcregrep to hold parts of the file it is searching. This is also the - minimum value. The actual amount of memory used by pcregrep is three times - this number, because it allows for the buffering of "before" and "after" - lines. */ -#define PCREGREP_BUFSIZE 20480 - -/* to make a symbol visible */ -#define PCREPOSIX_EXP_DECL extern __attribute__ ((visibility ("default"))) - -/* to make a symbol visible */ -#define PCREPOSIX_EXP_DEFN extern __attribute__ ((visibility ("default"))) - -/* to make a symbol visible */ -#define PCRE_EXP_DATA_DEFN __attribute__ ((visibility ("default"))) - -/* to make a symbol visible */ -#define PCRE_EXP_DECL extern __attribute__ ((visibility ("default"))) - - -/* If you are compiling for a system other than a Unix-like system or - Win32, and it needs some magic to be inserted before the definition - of a function that is exported by the library, define this macro to - contain the relevant magic. If you do not define this macro, a suitable - __declspec value is used for Windows systems; in other environments - "extern" is used for a C compiler and "extern C" for a C++ compiler. - This macro apears at the start of every exported function that is part - of the external API. It does not appear on functions that are "external" - in the C sense, but which are internal to the library. */ -#define PCRE_EXP_DEFN __attribute__ ((visibility ("default"))) - -/* Define to any value if linking statically (TODO: make nice with Libtool) */ -#define PCRE_STATIC 1 - -/* When calling PCRE via the POSIX interface, additional working storage is - required for holding the pointers to capturing substrings because PCRE - requires three integers per substring, whereas the POSIX interface provides - only two. If the number of expected substrings is small, the wrapper - function uses space on the stack, because this is faster than using - malloc() for each call. The threshold above which the stack is no longer - used is defined by POSIX_MALLOC_THRESHOLD. */ -#define POSIX_MALLOC_THRESHOLD 10 - -/* Define to necessary symbol if this constant uses a non-standard name on - your system. */ -/* #undef PTHREAD_CREATE_JOINABLE */ - -/* Define to 1 if you have the ANSI C header files. */ -#define STDC_HEADERS 1 - -/* Define to any value to enable support for Just-In-Time compiling. */ -/* #undef SUPPORT_JIT */ - -/* Define to any value to allow pcregrep to be linked with libbz2, so that it - is able to handle .bz2 files. */ -/* #undef SUPPORT_LIBBZ2 */ - -/* Define to any value to allow pcretest to be linked with libedit. */ -/* #undef SUPPORT_LIBEDIT */ - -/* Define to any value to allow pcretest to be linked with libreadline. */ -/* #undef SUPPORT_LIBREADLINE */ - -/* Define to any value to allow pcregrep to be linked with libz, so that it is - able to handle .gz files. */ -/* #undef SUPPORT_LIBZ */ - -/* Define to any value to enable the 16 bit PCRE library. */ -/* #undef SUPPORT_PCRE16 */ - -/* Define to any value to enable the 32 bit PCRE library. */ -/* #undef SUPPORT_PCRE32 */ - -/* Define to any value to enable the 8 bit PCRE library. */ -#define SUPPORT_PCRE8 /**/ - -/* Define to any value to enable JIT support in pcregrep. */ -/* #undef SUPPORT_PCREGREP_JIT */ - -/* Define to any value to enable support for Unicode properties. */ -#define SUPPORT_UCP /**/ - -/* Define to any value to enable support for the UTF-8/16/32 Unicode encoding. - This will work even in an EBCDIC environment, but it is incompatible with - the EBCDIC macro. That is, PCRE can support *either* EBCDIC code *or* - ASCII/UTF-8/16/32, but not both at once. */ -#define SUPPORT_UTF /**/ - -/* Define to any value for valgrind support to find invalid memory reads. */ -/* #undef SUPPORT_VALGRIND */ - -/* Version number of package */ -#define VERSION "8.39" - -/* Define to empty if `const' does not conform to ANSI C. */ -/* #undef const */ - -/* Define to the type of a signed integer type of width exactly 64 bits if - such a type exists and the standard includes do not define it. */ -/* #undef int64_t */ - -/* Define to `unsigned int' if does not define. */ -/* #undef size_t */ diff --git a/src/third_party/pcre-8.39/build_windows/config.h b/src/third_party/pcre-8.39/build_windows/config.h deleted file mode 100755 index 2e8d4ec7b6a..00000000000 --- a/src/third_party/pcre-8.39/build_windows/config.h +++ /dev/null @@ -1,57 +0,0 @@ -/* config.h for CMake builds */ - -/* #undef HAVE_DIRENT_H */ -#define HAVE_SYS_STAT_H 1 -#define HAVE_SYS_TYPES_H 1 -/* #undef HAVE_UNISTD_H */ -#define HAVE_WINDOWS_H 1 -#define HAVE_STDINT_H 1 -#define HAVE_INTTYPES_H 1 - -/* #undef HAVE_TYPE_TRAITS_H */ -/* #undef HAVE_BITS_TYPE_TRAITS_H */ - -/* #undef HAVE_BCOPY */ -#define HAVE_MEMMOVE 1 -#define HAVE_STRERROR 1 -#define HAVE_STRTOLL 1 -/* #undef HAVE_STRTOQ */ -#define HAVE__STRTOI64 1 - -#define PCRE_STATIC 1 - -#define SUPPORT_PCRE8 1 -/* #undef SUPPORT_PCRE16 */ -/* #undef SUPPORT_PCRE32 */ -/* #undef SUPPORT_JIT */ -/* #undef SUPPORT_PCREGREP_JIT */ -#define SUPPORT_UTF 1 -#define SUPPORT_UCP 1 -/* #undef EBCDIC */ -/* #undef EBCDIC_NL25 */ -/* #undef BSR_ANYCRLF */ -#define NO_RECURSE 1 - -#define HAVE_LONG_LONG 1 -#define HAVE_UNSIGNED_LONG_LONG 1 - -/* #undef SUPPORT_LIBBZ2 */ -/* #undef SUPPORT_LIBZ */ -/* #undef SUPPORT_LIBEDIT */ -/* #undef SUPPORT_LIBREADLINE */ - -/* #undef SUPPORT_VALGRIND */ -/* #undef SUPPORT_GCOV */ - -#define NEWLINE 10 -#define POSIX_MALLOC_THRESHOLD 10 -#define LINK_SIZE 2 -#define PARENS_NEST_LIMIT 250 -#define MATCH_LIMIT 200000 -#define MATCH_LIMIT_RECURSION 4000 -#define PCREGREP_BUFSIZE 20480 - -#define MAX_NAME_SIZE 32 -#define MAX_NAME_COUNT 10000 - -/* end config.h for CMake builds */ diff --git a/src/third_party/pcre-8.39/dftables.c b/src/third_party/pcre-8.39/dftables.c deleted file mode 100644 index 1fdc8e0f231..00000000000 --- a/src/third_party/pcre-8.39/dftables.c +++ /dev/null @@ -1,212 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This is a freestanding support program to generate a file containing -character tables for PCRE. The tables are built according to the current -locale. Now that pcre_maketables is a function visible to the outside world, we -make use of its code from here in order to be consistent. */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include -#include -#include - -#include "pcre_internal.h" - -#define DFTABLES /* pcre_maketables.c notices this */ -#include "pcre_maketables.c" - - -int main(int argc, char **argv) -{ -FILE *f; -int i = 1; -const unsigned char *tables; -const unsigned char *base_of_tables; - -/* By default, the default C locale is used rather than what the building user -happens to have set. However, if the -L option is given, set the locale from -the LC_xxx environment variables. */ - -if (argc > 1 && strcmp(argv[1], "-L") == 0) - { - setlocale(LC_ALL, ""); /* Set from environment variables */ - i++; - } - -if (argc < i + 1) - { - fprintf(stderr, "dftables: one filename argument is required\n"); - return 1; - } - -tables = pcre_maketables(); -base_of_tables = tables; - -f = fopen(argv[i], "wb"); -if (f == NULL) - { - fprintf(stderr, "dftables: failed to open %s for writing\n", argv[1]); - return 1; - } - -/* There are several fprintf() calls here, because gcc in pedantic mode -complains about the very long string otherwise. */ - -fprintf(f, - "/*************************************************\n" - "* Perl-Compatible Regular Expressions *\n" - "*************************************************/\n\n" - "/* This file was automatically written by the dftables auxiliary\n" - "program. It contains character tables that are used when no external\n" - "tables are passed to PCRE by the application that calls it. The tables\n" - "are used only for characters whose code values are less than 256.\n\n"); -fprintf(f, - "The following #includes are present because without them gcc 4.x may remove\n" - "the array definition from the final binary if PCRE is built into a static\n" - "library and dead code stripping is activated. This leads to link errors.\n" - "Pulling in the header ensures that the array gets flagged as \"someone\n" - "outside this compilation unit might reference this\" and so it will always\n" - "be supplied to the linker. */\n\n"); - -/* Force config.h in z/OS */ - -#if defined NATIVE_ZOS -fprintf(f, - "/* For z/OS, config.h is forced */\n" - "#ifndef HAVE_CONFIG_H\n" - "#define HAVE_CONFIG_H 1\n" - "#endif\n\n"); -#endif - -fprintf(f, - "#ifdef HAVE_CONFIG_H\n" - "#include \"config.h\"\n" - "#endif\n\n" - "#include \"pcre_internal.h\"\n\n"); - -fprintf(f, - "const pcre_uint8 PRIV(default_tables)[] = {\n\n" - "/* This table is a lower casing table. */\n\n"); - -fprintf(f, " "); -for (i = 0; i < 256; i++) - { - if ((i & 7) == 0 && i != 0) fprintf(f, "\n "); - fprintf(f, "%3d", *tables++); - if (i != 255) fprintf(f, ","); - } -fprintf(f, ",\n\n"); - -fprintf(f, "/* This table is a case flipping table. */\n\n"); - -fprintf(f, " "); -for (i = 0; i < 256; i++) - { - if ((i & 7) == 0 && i != 0) fprintf(f, "\n "); - fprintf(f, "%3d", *tables++); - if (i != 255) fprintf(f, ","); - } -fprintf(f, ",\n\n"); - -fprintf(f, - "/* This table contains bit maps for various character classes.\n" - "Each map is 32 bytes long and the bits run from the least\n" - "significant end of each byte. The classes that have their own\n" - "maps are: space, xdigit, digit, upper, lower, word, graph\n" - "print, punct, and cntrl. Other classes are built from combinations. */\n\n"); - -fprintf(f, " "); -for (i = 0; i < cbit_length; i++) - { - if ((i & 7) == 0 && i != 0) - { - if ((i & 31) == 0) fprintf(f, "\n"); - fprintf(f, "\n "); - } - fprintf(f, "0x%02x", *tables++); - if (i != cbit_length - 1) fprintf(f, ","); - } -fprintf(f, ",\n\n"); - -fprintf(f, - "/* This table identifies various classes of character by individual bits:\n" - " 0x%02x white space character\n" - " 0x%02x letter\n" - " 0x%02x decimal digit\n" - " 0x%02x hexadecimal digit\n" - " 0x%02x alphanumeric or '_'\n" - " 0x%02x regular expression metacharacter or binary zero\n*/\n\n", - ctype_space, ctype_letter, ctype_digit, ctype_xdigit, ctype_word, - ctype_meta); - -fprintf(f, " "); -for (i = 0; i < 256; i++) - { - if ((i & 7) == 0 && i != 0) - { - fprintf(f, " /* "); - if (isprint(i-8)) fprintf(f, " %c -", i-8); - else fprintf(f, "%3d-", i-8); - if (isprint(i-1)) fprintf(f, " %c ", i-1); - else fprintf(f, "%3d", i-1); - fprintf(f, " */\n "); - } - fprintf(f, "0x%02x", *tables++); - if (i != 255) fprintf(f, ","); - } - -fprintf(f, "};/* "); -if (isprint(i-8)) fprintf(f, " %c -", i-8); - else fprintf(f, "%3d-", i-8); -if (isprint(i-1)) fprintf(f, " %c ", i-1); - else fprintf(f, "%3d", i-1); -fprintf(f, " */\n\n/* End of pcre_chartables.c */\n"); - -fclose(f); -free((void *)base_of_tables); -return 0; -} - -/* End of dftables.c */ diff --git a/src/third_party/pcre-8.39/libpcre.pc.in b/src/third_party/pcre-8.39/libpcre.pc.in deleted file mode 100644 index 0a35da87f85..00000000000 --- a/src/third_party/pcre-8.39/libpcre.pc.in +++ /dev/null @@ -1,13 +0,0 @@ -# Package Information for pkg-config - -prefix=@prefix@ -exec_prefix=@exec_prefix@ -libdir=@libdir@ -includedir=@includedir@ - -Name: libpcre -Description: PCRE - Perl compatible regular expressions C library with 8 bit character support -Version: @PACKAGE_VERSION@ -Libs: -L${libdir} -lpcre -Libs.private: @PTHREAD_CFLAGS@ @PTHREAD_LIBS@ -Cflags: -I${includedir} @PCRE_STATIC_CFLAG@ diff --git a/src/third_party/pcre-8.39/libpcre16.pc.in b/src/third_party/pcre-8.39/libpcre16.pc.in deleted file mode 100644 index 080c9dcfe8a..00000000000 --- a/src/third_party/pcre-8.39/libpcre16.pc.in +++ /dev/null @@ -1,13 +0,0 @@ -# Package Information for pkg-config - -prefix=@prefix@ -exec_prefix=@exec_prefix@ -libdir=@libdir@ -includedir=@includedir@ - -Name: libpcre16 -Description: PCRE - Perl compatible regular expressions C library with 16 bit character support -Version: @PACKAGE_VERSION@ -Libs: -L${libdir} -lpcre16 -Libs.private: @PTHREAD_CFLAGS@ @PTHREAD_LIBS@ -Cflags: -I${includedir} @PCRE_STATIC_CFLAG@ diff --git a/src/third_party/pcre-8.39/libpcre32.pc.in b/src/third_party/pcre-8.39/libpcre32.pc.in deleted file mode 100644 index a3ae0e11fcf..00000000000 --- a/src/third_party/pcre-8.39/libpcre32.pc.in +++ /dev/null @@ -1,13 +0,0 @@ -# Package Information for pkg-config - -prefix=@prefix@ -exec_prefix=@exec_prefix@ -libdir=@libdir@ -includedir=@includedir@ - -Name: libpcre32 -Description: PCRE - Perl compatible regular expressions C library with 32 bit character support -Version: @PACKAGE_VERSION@ -Libs: -L${libdir} -lpcre32 -Libs.private: @PTHREAD_CFLAGS@ @PTHREAD_LIBS@ -Cflags: -I${includedir} @PCRE_STATIC_CFLAG@ diff --git a/src/third_party/pcre-8.39/libpcrecpp.pc.in b/src/third_party/pcre-8.39/libpcrecpp.pc.in deleted file mode 100644 index ef006fe47aa..00000000000 --- a/src/third_party/pcre-8.39/libpcrecpp.pc.in +++ /dev/null @@ -1,12 +0,0 @@ -# Package Information for pkg-config - -prefix=@prefix@ -exec_prefix=@exec_prefix@ -libdir=@libdir@ -includedir=@includedir@ - -Name: libpcrecpp -Description: PCRECPP - C++ wrapper for PCRE -Version: @PACKAGE_VERSION@ -Libs: -L${libdir} -lpcre -lpcrecpp -Cflags: -I${includedir} @PCRE_STATIC_CFLAG@ diff --git a/src/third_party/pcre-8.39/libpcreposix.pc.in b/src/third_party/pcre-8.39/libpcreposix.pc.in deleted file mode 100644 index c6c0b0c6c66..00000000000 --- a/src/third_party/pcre-8.39/libpcreposix.pc.in +++ /dev/null @@ -1,13 +0,0 @@ -# Package Information for pkg-config - -prefix=@prefix@ -exec_prefix=@exec_prefix@ -libdir=@libdir@ -includedir=@includedir@ - -Name: libpcreposix -Description: PCREPosix - Posix compatible interface to libpcre -Version: @PACKAGE_VERSION@ -Libs: -L${libdir} -lpcreposix -Cflags: -I${includedir} @PCRE_STATIC_CFLAG@ -Requires.private: libpcre diff --git a/src/third_party/pcre-8.39/makevp.bat b/src/third_party/pcre-8.39/makevp.bat deleted file mode 100644 index 5f795487eb9..00000000000 --- a/src/third_party/pcre-8.39/makevp.bat +++ /dev/null @@ -1,66 +0,0 @@ -:: AH 20-12-06 modified for new PCRE-7.0 and VP/BCC -:: PH 19-03-07 renamed !compile.txt and !linklib.txt as makevp-compile.txt and -:: makevp-linklib.txt -:: PH 26-03-07 re-renamed !compile.txt and !linklib.txt as makevp-c.txt and -:: makevp-l.txt -:: PH 29-03-07 hopefully the final rename to makevp_c and makevp_l -:: AH 27.08.08 updated for new PCRE-7.7 -:: required PCRE.H and CONFIG.H will be generated if not existing - -@echo off -echo. -echo Compiling PCRE with BORLAND C++ for VIRTUAL PASCAL -echo. - -REM This file was contributed by Alexander Tokarev for building PCRE for use -REM with Virtual Pascal. It has not been tested with the latest PCRE release. - -REM This file has been modified and extended to compile with newer PCRE releases -REM by Stefan Weber (Angels Holocaust). - -REM CHANGE THIS FOR YOUR BORLAND C++ COMPILER PATH -SET BORLAND=f:\bcc -REM location of the TASM binaries, if compiling with the -B BCC switch -SET TASM=f:\tasm - -SET PATH=%PATH%;%BORLAND%\bin;%TASM%\bin -SET PCRE_VER=77 -SET COMPILE_DEFAULTS=-DHAVE_CONFIG_H -DPCRE_STATIC -I%BORLAND%\include - -del pcre%PCRE_VER%.lib >nul 2>nul - -:: sh configure - -:: check for needed header files -if not exist pcre.h copy pcre.h.generic pcre.h -if not exist config.h copy config.h.generic config.h - -bcc32 -DDFTABLES %COMPILE_DEFAULTS% -L%BORLAND%\lib dftables.c -IF ERRORLEVEL 1 GOTO ERROR - -:: dftables > chartables.c -dftables pcre_chartables.c - -REM compile and link the PCRE library into lib: option -B for ASM compile works too -bcc32 -a4 -c -RT- -y- -v- -u- -R- -Q- -X -d -fp -ff -P- -O2 -Oc -Ov -3 -w-8004 -w-8064 -w-8065 -w-8012 -UDFTABLES -DVPCOMPAT %COMPILE_DEFAULTS% @makevp_c.txt -IF ERRORLEVEL 1 GOTO ERROR - -tlib %BORLAND%\lib\cw32.lib *calloc *del *strncmp *memcpy *memmove *memset *memcmp *strlen -IF ERRORLEVEL 1 GOTO ERROR -tlib pcre%PCRE_VER%.lib @makevp_l.txt +calloc.obj +del.obj +strncmp.obj +memcpy.obj +memmove.obj +memset.obj +memcmp.obj +strlen.obj -IF ERRORLEVEL 1 GOTO ERROR - -del *.obj *.tds *.bak >nul 2>nul - -echo --- -echo Now the library should be complete. Please check all messages above. -echo Don't care for warnings, it's OK. -goto END - -:ERROR -echo --- -echo Error while compiling PCRE. Aborting... -pause -goto END - -:END diff --git a/src/third_party/pcre-8.39/makevp_c.txt b/src/third_party/pcre-8.39/makevp_c.txt deleted file mode 100644 index 5648115405d..00000000000 --- a/src/third_party/pcre-8.39/makevp_c.txt +++ /dev/null @@ -1,21 +0,0 @@ -pcre_byte_order.c -pcre_chartables.c -pcre_compile.c -pcre_config.c -pcre_dfa_exec.c -pcre_exec.c -pcre_fullinfo.c -pcre_get.c -pcre_globals.c -pcre_jit_compile.c -pcre_maketables.c -pcre_newline.c -pcre_ord2utf8.c -pcre_refcount.c -pcre_string_utils.c -pcre_study.c -pcre_tables.c -pcre_ucd.c -pcre_valid_utf8.c -pcre_version.c -pcre_xclass.c diff --git a/src/third_party/pcre-8.39/makevp_l.txt b/src/third_party/pcre-8.39/makevp_l.txt deleted file mode 100644 index 9b071e0f90a..00000000000 --- a/src/third_party/pcre-8.39/makevp_l.txt +++ /dev/null @@ -1,21 +0,0 @@ -+pcre_byte_order.obj & -+pcre_chartables.obj & -+pcre_compile.obj & -+pcre_config.obj & -+pcre_dfa_exec.obj & -+pcre_exec.obj & -+pcre_fullinfo.obj & -+pcre_get.obj & -+pcre_globals.obj & -+pcre_jit_compile.obj & -+pcre_maketables.obj & -+pcre_newline.obj & -+pcre_ord2utf8.obj & -+pcre_refcount.obj & -+pcre_string_utils.obj & -+pcre_study.obj & -+pcre_tables.obj & -+pcre_ucd.obj & -+pcre_valid_utf8.obj & -+pcre_version.obj & -+pcre_xclass.obj diff --git a/src/third_party/pcre-8.39/pcre-config.in b/src/third_party/pcre-8.39/pcre-config.in deleted file mode 100644 index ac06a3325bc..00000000000 --- a/src/third_party/pcre-8.39/pcre-config.in +++ /dev/null @@ -1,133 +0,0 @@ -#!/bin/sh - -prefix=@prefix@ -exec_prefix=@exec_prefix@ -exec_prefix_set=no - -cflags="[--cflags]" - -if test @enable_cpp@ = yes ; then - libs="[--libs-cpp]" -else - libs= -fi - -if test @enable_pcre16@ = yes ; then - libs="[--libs16] $libs" -fi - -if test @enable_pcre32@ = yes ; then - libs="[--libs32] $libs" -fi - -if test @enable_pcre8@ = yes ; then - libs="[--libs] [--libs-posix] $libs" - cflags="$cflags [--cflags-posix]" -fi - -usage="Usage: pcre-config [--prefix] [--exec-prefix] [--version] $libs $cflags" - -if test $# -eq 0; then - echo "${usage}" 1>&2 - exit 1 -fi - -libR= -case `uname -s` in - *SunOS*) - libR=" -R@libdir@" - ;; - *BSD*) - libR=" -Wl,-R@libdir@" - ;; -esac - -libS= -if test @libdir@ != /usr/lib ; then - libS=-L@libdir@ -fi - -while test $# -gt 0; do - case "$1" in - -*=*) optarg=`echo "$1" | sed 's/[-_a-zA-Z0-9]*=//'` ;; - *) optarg= ;; - esac - - case $1 in - --prefix=*) - prefix=$optarg - if test $exec_prefix_set = no ; then - exec_prefix=$optarg - fi - ;; - --prefix) - echo $prefix - ;; - --exec-prefix=*) - exec_prefix=$optarg - exec_prefix_set=yes - ;; - --exec-prefix) - echo $exec_prefix - ;; - --version) - echo @PACKAGE_VERSION@ - ;; - --cflags) - if test @includedir@ != /usr/include ; then - includes=-I@includedir@ - fi - echo $includes @PCRE_STATIC_CFLAG@ - ;; - --cflags-posix) - if test @enable_pcre8@ = yes ; then - if test @includedir@ != /usr/include ; then - includes=-I@includedir@ - fi - echo $includes @PCRE_STATIC_CFLAG@ - else - echo "${usage}" 1>&2 - fi - ;; - --libs-posix) - if test @enable_pcre8@ = yes ; then - echo $libS$libR -lpcreposix -lpcre - else - echo "${usage}" 1>&2 - fi - ;; - --libs) - if test @enable_pcre8@ = yes ; then - echo $libS$libR -lpcre - else - echo "${usage}" 1>&2 - fi - ;; - --libs16) - if test @enable_pcre16@ = yes ; then - echo $libS$libR -lpcre16 - else - echo "${usage}" 1>&2 - fi - ;; - --libs32) - if test @enable_pcre32@ = yes ; then - echo $libS$libR -lpcre32 - else - echo "${usage}" 1>&2 - fi - ;; - --libs-cpp) - if test @enable_cpp@ = yes ; then - echo $libS$libR -lpcrecpp -lpcre - else - echo "${usage}" 1>&2 - fi - ;; - *) - echo "${usage}" 1>&2 - exit 1 - ;; - esac - shift -done diff --git a/src/third_party/pcre-8.39/pcre.h b/src/third_party/pcre-8.39/pcre.h deleted file mode 100644 index 70559700654..00000000000 --- a/src/third_party/pcre-8.39/pcre.h +++ /dev/null @@ -1,677 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* This is the public header file for the PCRE library, to be #included by -applications that call the PCRE functions. - - Copyright (c) 1997-2014 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -#ifndef _PCRE_H -#define _PCRE_H - -/* The current PCRE version information. */ - -#define PCRE_MAJOR 8 -#define PCRE_MINOR 39 -#define PCRE_PRERELEASE -#define PCRE_DATE 2016-06-14 - -/* When an application links to a PCRE DLL in Windows, the symbols that are -imported have to be identified as such. When building PCRE, the appropriate -export setting is defined in pcre_internal.h, which includes this file. So we -don't change existing definitions of PCRE_EXP_DECL and PCRECPP_EXP_DECL. */ - -#if defined(_WIN32) && !defined(PCRE_STATIC) -# ifndef PCRE_EXP_DECL -# define PCRE_EXP_DECL extern __declspec(dllimport) -# endif -# ifdef __cplusplus -# ifndef PCRECPP_EXP_DECL -# define PCRECPP_EXP_DECL extern __declspec(dllimport) -# endif -# ifndef PCRECPP_EXP_DEFN -# define PCRECPP_EXP_DEFN __declspec(dllimport) -# endif -# endif -#endif - -/* By default, we use the standard "extern" declarations. */ - -#ifndef PCRE_EXP_DECL -# ifdef __cplusplus -# define PCRE_EXP_DECL extern "C" -# else -# define PCRE_EXP_DECL extern -# endif -#endif - -#ifdef __cplusplus -# ifndef PCRECPP_EXP_DECL -# define PCRECPP_EXP_DECL extern -# endif -# ifndef PCRECPP_EXP_DEFN -# define PCRECPP_EXP_DEFN -# endif -#endif - -/* Have to include stdlib.h in order to ensure that size_t is defined; -it is needed here for malloc. */ - -#include - -/* Allow for C++ users */ - -#ifdef __cplusplus -extern "C" { -#endif - -/* Public options. Some are compile-time only, some are run-time only, and some -are both. Most of the compile-time options are saved with the compiled regex so -that they can be inspected during studying (and therefore JIT compiling). Note -that pcre_study() has its own set of options. Originally, all the options -defined here used distinct bits. However, almost all the bits in a 32-bit word -are now used, so in order to conserve them, option bits that were previously -only recognized at matching time (i.e. by pcre_exec() or pcre_dfa_exec()) may -also be used for compile-time options that affect only compiling and are not -relevant for studying or JIT compiling. - -Some options for pcre_compile() change its behaviour but do not affect the -behaviour of the execution functions. Other options are passed through to the -execution functions and affect their behaviour, with or without affecting the -behaviour of pcre_compile(). - -Options that can be passed to pcre_compile() are tagged Cx below, with these -variants: - -C1 Affects compile only -C2 Does not affect compile; affects exec, dfa_exec -C3 Affects compile, exec, dfa_exec -C4 Affects compile, exec, dfa_exec, study -C5 Affects compile, exec, study - -Options that can be set for pcre_exec() and/or pcre_dfa_exec() are flagged with -E and D, respectively. They take precedence over C3, C4, and C5 settings passed -from pcre_compile(). Those that are compatible with JIT execution are flagged -with J. */ - -#define PCRE_CASELESS 0x00000001 /* C1 */ -#define PCRE_MULTILINE 0x00000002 /* C1 */ -#define PCRE_DOTALL 0x00000004 /* C1 */ -#define PCRE_EXTENDED 0x00000008 /* C1 */ -#define PCRE_ANCHORED 0x00000010 /* C4 E D */ -#define PCRE_DOLLAR_ENDONLY 0x00000020 /* C2 */ -#define PCRE_EXTRA 0x00000040 /* C1 */ -#define PCRE_NOTBOL 0x00000080 /* E D J */ -#define PCRE_NOTEOL 0x00000100 /* E D J */ -#define PCRE_UNGREEDY 0x00000200 /* C1 */ -#define PCRE_NOTEMPTY 0x00000400 /* E D J */ -#define PCRE_UTF8 0x00000800 /* C4 ) */ -#define PCRE_UTF16 0x00000800 /* C4 ) Synonyms */ -#define PCRE_UTF32 0x00000800 /* C4 ) */ -#define PCRE_NO_AUTO_CAPTURE 0x00001000 /* C1 */ -#define PCRE_NO_UTF8_CHECK 0x00002000 /* C1 E D J ) */ -#define PCRE_NO_UTF16_CHECK 0x00002000 /* C1 E D J ) Synonyms */ -#define PCRE_NO_UTF32_CHECK 0x00002000 /* C1 E D J ) */ -#define PCRE_AUTO_CALLOUT 0x00004000 /* C1 */ -#define PCRE_PARTIAL_SOFT 0x00008000 /* E D J ) Synonyms */ -#define PCRE_PARTIAL 0x00008000 /* E D J ) */ - -/* This pair use the same bit. */ -#define PCRE_NEVER_UTF 0x00010000 /* C1 ) Overlaid */ -#define PCRE_DFA_SHORTEST 0x00010000 /* D ) Overlaid */ - -/* This pair use the same bit. */ -#define PCRE_NO_AUTO_POSSESS 0x00020000 /* C1 ) Overlaid */ -#define PCRE_DFA_RESTART 0x00020000 /* D ) Overlaid */ - -#define PCRE_FIRSTLINE 0x00040000 /* C3 */ -#define PCRE_DUPNAMES 0x00080000 /* C1 */ -#define PCRE_NEWLINE_CR 0x00100000 /* C3 E D */ -#define PCRE_NEWLINE_LF 0x00200000 /* C3 E D */ -#define PCRE_NEWLINE_CRLF 0x00300000 /* C3 E D */ -#define PCRE_NEWLINE_ANY 0x00400000 /* C3 E D */ -#define PCRE_NEWLINE_ANYCRLF 0x00500000 /* C3 E D */ -#define PCRE_BSR_ANYCRLF 0x00800000 /* C3 E D */ -#define PCRE_BSR_UNICODE 0x01000000 /* C3 E D */ -#define PCRE_JAVASCRIPT_COMPAT 0x02000000 /* C5 */ -#define PCRE_NO_START_OPTIMIZE 0x04000000 /* C2 E D ) Synonyms */ -#define PCRE_NO_START_OPTIMISE 0x04000000 /* C2 E D ) */ -#define PCRE_PARTIAL_HARD 0x08000000 /* E D J */ -#define PCRE_NOTEMPTY_ATSTART 0x10000000 /* E D J */ -#define PCRE_UCP 0x20000000 /* C3 */ - -/* Exec-time and get/set-time error codes */ - -#define PCRE_ERROR_NOMATCH (-1) -#define PCRE_ERROR_NULL (-2) -#define PCRE_ERROR_BADOPTION (-3) -#define PCRE_ERROR_BADMAGIC (-4) -#define PCRE_ERROR_UNKNOWN_OPCODE (-5) -#define PCRE_ERROR_UNKNOWN_NODE (-5) /* For backward compatibility */ -#define PCRE_ERROR_NOMEMORY (-6) -#define PCRE_ERROR_NOSUBSTRING (-7) -#define PCRE_ERROR_MATCHLIMIT (-8) -#define PCRE_ERROR_CALLOUT (-9) /* Never used by PCRE itself */ -#define PCRE_ERROR_BADUTF8 (-10) /* Same for 8/16/32 */ -#define PCRE_ERROR_BADUTF16 (-10) /* Same for 8/16/32 */ -#define PCRE_ERROR_BADUTF32 (-10) /* Same for 8/16/32 */ -#define PCRE_ERROR_BADUTF8_OFFSET (-11) /* Same for 8/16 */ -#define PCRE_ERROR_BADUTF16_OFFSET (-11) /* Same for 8/16 */ -#define PCRE_ERROR_PARTIAL (-12) -#define PCRE_ERROR_BADPARTIAL (-13) -#define PCRE_ERROR_INTERNAL (-14) -#define PCRE_ERROR_BADCOUNT (-15) -#define PCRE_ERROR_DFA_UITEM (-16) -#define PCRE_ERROR_DFA_UCOND (-17) -#define PCRE_ERROR_DFA_UMLIMIT (-18) -#define PCRE_ERROR_DFA_WSSIZE (-19) -#define PCRE_ERROR_DFA_RECURSE (-20) -#define PCRE_ERROR_RECURSIONLIMIT (-21) -#define PCRE_ERROR_NULLWSLIMIT (-22) /* No longer actually used */ -#define PCRE_ERROR_BADNEWLINE (-23) -#define PCRE_ERROR_BADOFFSET (-24) -#define PCRE_ERROR_SHORTUTF8 (-25) -#define PCRE_ERROR_SHORTUTF16 (-25) /* Same for 8/16 */ -#define PCRE_ERROR_RECURSELOOP (-26) -#define PCRE_ERROR_JIT_STACKLIMIT (-27) -#define PCRE_ERROR_BADMODE (-28) -#define PCRE_ERROR_BADENDIANNESS (-29) -#define PCRE_ERROR_DFA_BADRESTART (-30) -#define PCRE_ERROR_JIT_BADOPTION (-31) -#define PCRE_ERROR_BADLENGTH (-32) -#define PCRE_ERROR_UNSET (-33) - -/* Specific error codes for UTF-8 validity checks */ - -#define PCRE_UTF8_ERR0 0 -#define PCRE_UTF8_ERR1 1 -#define PCRE_UTF8_ERR2 2 -#define PCRE_UTF8_ERR3 3 -#define PCRE_UTF8_ERR4 4 -#define PCRE_UTF8_ERR5 5 -#define PCRE_UTF8_ERR6 6 -#define PCRE_UTF8_ERR7 7 -#define PCRE_UTF8_ERR8 8 -#define PCRE_UTF8_ERR9 9 -#define PCRE_UTF8_ERR10 10 -#define PCRE_UTF8_ERR11 11 -#define PCRE_UTF8_ERR12 12 -#define PCRE_UTF8_ERR13 13 -#define PCRE_UTF8_ERR14 14 -#define PCRE_UTF8_ERR15 15 -#define PCRE_UTF8_ERR16 16 -#define PCRE_UTF8_ERR17 17 -#define PCRE_UTF8_ERR18 18 -#define PCRE_UTF8_ERR19 19 -#define PCRE_UTF8_ERR20 20 -#define PCRE_UTF8_ERR21 21 -#define PCRE_UTF8_ERR22 22 /* Unused (was non-character) */ - -/* Specific error codes for UTF-16 validity checks */ - -#define PCRE_UTF16_ERR0 0 -#define PCRE_UTF16_ERR1 1 -#define PCRE_UTF16_ERR2 2 -#define PCRE_UTF16_ERR3 3 -#define PCRE_UTF16_ERR4 4 /* Unused (was non-character) */ - -/* Specific error codes for UTF-32 validity checks */ - -#define PCRE_UTF32_ERR0 0 -#define PCRE_UTF32_ERR1 1 -#define PCRE_UTF32_ERR2 2 /* Unused (was non-character) */ -#define PCRE_UTF32_ERR3 3 - -/* Request types for pcre_fullinfo() */ - -#define PCRE_INFO_OPTIONS 0 -#define PCRE_INFO_SIZE 1 -#define PCRE_INFO_CAPTURECOUNT 2 -#define PCRE_INFO_BACKREFMAX 3 -#define PCRE_INFO_FIRSTBYTE 4 -#define PCRE_INFO_FIRSTCHAR 4 /* For backwards compatibility */ -#define PCRE_INFO_FIRSTTABLE 5 -#define PCRE_INFO_LASTLITERAL 6 -#define PCRE_INFO_NAMEENTRYSIZE 7 -#define PCRE_INFO_NAMECOUNT 8 -#define PCRE_INFO_NAMETABLE 9 -#define PCRE_INFO_STUDYSIZE 10 -#define PCRE_INFO_DEFAULT_TABLES 11 -#define PCRE_INFO_OKPARTIAL 12 -#define PCRE_INFO_JCHANGED 13 -#define PCRE_INFO_HASCRORLF 14 -#define PCRE_INFO_MINLENGTH 15 -#define PCRE_INFO_JIT 16 -#define PCRE_INFO_JITSIZE 17 -#define PCRE_INFO_MAXLOOKBEHIND 18 -#define PCRE_INFO_FIRSTCHARACTER 19 -#define PCRE_INFO_FIRSTCHARACTERFLAGS 20 -#define PCRE_INFO_REQUIREDCHAR 21 -#define PCRE_INFO_REQUIREDCHARFLAGS 22 -#define PCRE_INFO_MATCHLIMIT 23 -#define PCRE_INFO_RECURSIONLIMIT 24 -#define PCRE_INFO_MATCH_EMPTY 25 - -/* Request types for pcre_config(). Do not re-arrange, in order to remain -compatible. */ - -#define PCRE_CONFIG_UTF8 0 -#define PCRE_CONFIG_NEWLINE 1 -#define PCRE_CONFIG_LINK_SIZE 2 -#define PCRE_CONFIG_POSIX_MALLOC_THRESHOLD 3 -#define PCRE_CONFIG_MATCH_LIMIT 4 -#define PCRE_CONFIG_STACKRECURSE 5 -#define PCRE_CONFIG_UNICODE_PROPERTIES 6 -#define PCRE_CONFIG_MATCH_LIMIT_RECURSION 7 -#define PCRE_CONFIG_BSR 8 -#define PCRE_CONFIG_JIT 9 -#define PCRE_CONFIG_UTF16 10 -#define PCRE_CONFIG_JITTARGET 11 -#define PCRE_CONFIG_UTF32 12 -#define PCRE_CONFIG_PARENS_LIMIT 13 - -/* Request types for pcre_study(). Do not re-arrange, in order to remain -compatible. */ - -#define PCRE_STUDY_JIT_COMPILE 0x0001 -#define PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE 0x0002 -#define PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE 0x0004 -#define PCRE_STUDY_EXTRA_NEEDED 0x0008 - -/* Bit flags for the pcre[16|32]_extra structure. Do not re-arrange or redefine -these bits, just add new ones on the end, in order to remain compatible. */ - -#define PCRE_EXTRA_STUDY_DATA 0x0001 -#define PCRE_EXTRA_MATCH_LIMIT 0x0002 -#define PCRE_EXTRA_CALLOUT_DATA 0x0004 -#define PCRE_EXTRA_TABLES 0x0008 -#define PCRE_EXTRA_MATCH_LIMIT_RECURSION 0x0010 -#define PCRE_EXTRA_MARK 0x0020 -#define PCRE_EXTRA_EXECUTABLE_JIT 0x0040 - -/* Types */ - -struct real_pcre; /* declaration; the definition is private */ -typedef struct real_pcre pcre; - -struct real_pcre16; /* declaration; the definition is private */ -typedef struct real_pcre16 pcre16; - -struct real_pcre32; /* declaration; the definition is private */ -typedef struct real_pcre32 pcre32; - -struct real_pcre_jit_stack; /* declaration; the definition is private */ -typedef struct real_pcre_jit_stack pcre_jit_stack; - -struct real_pcre16_jit_stack; /* declaration; the definition is private */ -typedef struct real_pcre16_jit_stack pcre16_jit_stack; - -struct real_pcre32_jit_stack; /* declaration; the definition is private */ -typedef struct real_pcre32_jit_stack pcre32_jit_stack; - -/* If PCRE is compiled with 16 bit character support, PCRE_UCHAR16 must contain -a 16 bit wide signed data type. Otherwise it can be a dummy data type since -pcre16 functions are not implemented. There is a check for this in pcre_internal.h. */ -#ifndef PCRE_UCHAR16 -#define PCRE_UCHAR16 unsigned short -#endif - -#ifndef PCRE_SPTR16 -#define PCRE_SPTR16 const PCRE_UCHAR16 * -#endif - -/* If PCRE is compiled with 32 bit character support, PCRE_UCHAR32 must contain -a 32 bit wide signed data type. Otherwise it can be a dummy data type since -pcre32 functions are not implemented. There is a check for this in pcre_internal.h. */ -#ifndef PCRE_UCHAR32 -#define PCRE_UCHAR32 unsigned int -#endif - -#ifndef PCRE_SPTR32 -#define PCRE_SPTR32 const PCRE_UCHAR32 * -#endif - -/* When PCRE is compiled as a C++ library, the subject pointer type can be -replaced with a custom type. For conventional use, the public interface is a -const char *. */ - -#ifndef PCRE_SPTR -#define PCRE_SPTR const char * -#endif - -/* The structure for passing additional data to pcre_exec(). This is defined in -such as way as to be extensible. Always add new fields at the end, in order to -remain compatible. */ - -typedef struct pcre_extra { - unsigned long int flags; /* Bits for which fields are set */ - void *study_data; /* Opaque data from pcre_study() */ - unsigned long int match_limit; /* Maximum number of calls to match() */ - void *callout_data; /* Data passed back in callouts */ - const unsigned char *tables; /* Pointer to character tables */ - unsigned long int match_limit_recursion; /* Max recursive calls to match() */ - unsigned char **mark; /* For passing back a mark pointer */ - void *executable_jit; /* Contains a pointer to a compiled jit code */ -} pcre_extra; - -/* Same structure as above, but with 16 bit char pointers. */ - -typedef struct pcre16_extra { - unsigned long int flags; /* Bits for which fields are set */ - void *study_data; /* Opaque data from pcre_study() */ - unsigned long int match_limit; /* Maximum number of calls to match() */ - void *callout_data; /* Data passed back in callouts */ - const unsigned char *tables; /* Pointer to character tables */ - unsigned long int match_limit_recursion; /* Max recursive calls to match() */ - PCRE_UCHAR16 **mark; /* For passing back a mark pointer */ - void *executable_jit; /* Contains a pointer to a compiled jit code */ -} pcre16_extra; - -/* Same structure as above, but with 32 bit char pointers. */ - -typedef struct pcre32_extra { - unsigned long int flags; /* Bits for which fields are set */ - void *study_data; /* Opaque data from pcre_study() */ - unsigned long int match_limit; /* Maximum number of calls to match() */ - void *callout_data; /* Data passed back in callouts */ - const unsigned char *tables; /* Pointer to character tables */ - unsigned long int match_limit_recursion; /* Max recursive calls to match() */ - PCRE_UCHAR32 **mark; /* For passing back a mark pointer */ - void *executable_jit; /* Contains a pointer to a compiled jit code */ -} pcre32_extra; - -/* The structure for passing out data via the pcre_callout_function. We use a -structure so that new fields can be added on the end in future versions, -without changing the API of the function, thereby allowing old clients to work -without modification. */ - -typedef struct pcre_callout_block { - int version; /* Identifies version of block */ - /* ------------------------ Version 0 ------------------------------- */ - int callout_number; /* Number compiled into pattern */ - int *offset_vector; /* The offset vector */ - PCRE_SPTR subject; /* The subject being matched */ - int subject_length; /* The length of the subject */ - int start_match; /* Offset to start of this match attempt */ - int current_position; /* Where we currently are in the subject */ - int capture_top; /* Max current capture */ - int capture_last; /* Most recently closed capture */ - void *callout_data; /* Data passed in with the call */ - /* ------------------- Added for Version 1 -------------------------- */ - int pattern_position; /* Offset to next item in the pattern */ - int next_item_length; /* Length of next item in the pattern */ - /* ------------------- Added for Version 2 -------------------------- */ - const unsigned char *mark; /* Pointer to current mark or NULL */ - /* ------------------------------------------------------------------ */ -} pcre_callout_block; - -/* Same structure as above, but with 16 bit char pointers. */ - -typedef struct pcre16_callout_block { - int version; /* Identifies version of block */ - /* ------------------------ Version 0 ------------------------------- */ - int callout_number; /* Number compiled into pattern */ - int *offset_vector; /* The offset vector */ - PCRE_SPTR16 subject; /* The subject being matched */ - int subject_length; /* The length of the subject */ - int start_match; /* Offset to start of this match attempt */ - int current_position; /* Where we currently are in the subject */ - int capture_top; /* Max current capture */ - int capture_last; /* Most recently closed capture */ - void *callout_data; /* Data passed in with the call */ - /* ------------------- Added for Version 1 -------------------------- */ - int pattern_position; /* Offset to next item in the pattern */ - int next_item_length; /* Length of next item in the pattern */ - /* ------------------- Added for Version 2 -------------------------- */ - const PCRE_UCHAR16 *mark; /* Pointer to current mark or NULL */ - /* ------------------------------------------------------------------ */ -} pcre16_callout_block; - -/* Same structure as above, but with 32 bit char pointers. */ - -typedef struct pcre32_callout_block { - int version; /* Identifies version of block */ - /* ------------------------ Version 0 ------------------------------- */ - int callout_number; /* Number compiled into pattern */ - int *offset_vector; /* The offset vector */ - PCRE_SPTR32 subject; /* The subject being matched */ - int subject_length; /* The length of the subject */ - int start_match; /* Offset to start of this match attempt */ - int current_position; /* Where we currently are in the subject */ - int capture_top; /* Max current capture */ - int capture_last; /* Most recently closed capture */ - void *callout_data; /* Data passed in with the call */ - /* ------------------- Added for Version 1 -------------------------- */ - int pattern_position; /* Offset to next item in the pattern */ - int next_item_length; /* Length of next item in the pattern */ - /* ------------------- Added for Version 2 -------------------------- */ - const PCRE_UCHAR32 *mark; /* Pointer to current mark or NULL */ - /* ------------------------------------------------------------------ */ -} pcre32_callout_block; - -/* Indirection for store get and free functions. These can be set to -alternative malloc/free functions if required. Special ones are used in the -non-recursive case for "frames". There is also an optional callout function -that is triggered by the (?) regex item. For Virtual Pascal, these definitions -have to take another form. */ - -#ifndef VPCOMPAT -PCRE_EXP_DECL void *(*pcre_malloc)(size_t); -PCRE_EXP_DECL void (*pcre_free)(void *); -PCRE_EXP_DECL void *(*pcre_stack_malloc)(size_t); -PCRE_EXP_DECL void (*pcre_stack_free)(void *); -PCRE_EXP_DECL int (*pcre_callout)(pcre_callout_block *); -PCRE_EXP_DECL int (*pcre_stack_guard)(void); - -PCRE_EXP_DECL void *(*pcre16_malloc)(size_t); -PCRE_EXP_DECL void (*pcre16_free)(void *); -PCRE_EXP_DECL void *(*pcre16_stack_malloc)(size_t); -PCRE_EXP_DECL void (*pcre16_stack_free)(void *); -PCRE_EXP_DECL int (*pcre16_callout)(pcre16_callout_block *); -PCRE_EXP_DECL int (*pcre16_stack_guard)(void); - -PCRE_EXP_DECL void *(*pcre32_malloc)(size_t); -PCRE_EXP_DECL void (*pcre32_free)(void *); -PCRE_EXP_DECL void *(*pcre32_stack_malloc)(size_t); -PCRE_EXP_DECL void (*pcre32_stack_free)(void *); -PCRE_EXP_DECL int (*pcre32_callout)(pcre32_callout_block *); -PCRE_EXP_DECL int (*pcre32_stack_guard)(void); -#else /* VPCOMPAT */ -PCRE_EXP_DECL void *pcre_malloc(size_t); -PCRE_EXP_DECL void pcre_free(void *); -PCRE_EXP_DECL void *pcre_stack_malloc(size_t); -PCRE_EXP_DECL void pcre_stack_free(void *); -PCRE_EXP_DECL int pcre_callout(pcre_callout_block *); -PCRE_EXP_DECL int pcre_stack_guard(void); - -PCRE_EXP_DECL void *pcre16_malloc(size_t); -PCRE_EXP_DECL void pcre16_free(void *); -PCRE_EXP_DECL void *pcre16_stack_malloc(size_t); -PCRE_EXP_DECL void pcre16_stack_free(void *); -PCRE_EXP_DECL int pcre16_callout(pcre16_callout_block *); -PCRE_EXP_DECL int pcre16_stack_guard(void); - -PCRE_EXP_DECL void *pcre32_malloc(size_t); -PCRE_EXP_DECL void pcre32_free(void *); -PCRE_EXP_DECL void *pcre32_stack_malloc(size_t); -PCRE_EXP_DECL void pcre32_stack_free(void *); -PCRE_EXP_DECL int pcre32_callout(pcre32_callout_block *); -PCRE_EXP_DECL int pcre32_stack_guard(void); -#endif /* VPCOMPAT */ - -/* User defined callback which provides a stack just before the match starts. */ - -typedef pcre_jit_stack *(*pcre_jit_callback)(void *); -typedef pcre16_jit_stack *(*pcre16_jit_callback)(void *); -typedef pcre32_jit_stack *(*pcre32_jit_callback)(void *); - -/* Exported PCRE functions */ - -PCRE_EXP_DECL pcre *pcre_compile(const char *, int, const char **, int *, - const unsigned char *); -PCRE_EXP_DECL pcre16 *pcre16_compile(PCRE_SPTR16, int, const char **, int *, - const unsigned char *); -PCRE_EXP_DECL pcre32 *pcre32_compile(PCRE_SPTR32, int, const char **, int *, - const unsigned char *); -PCRE_EXP_DECL pcre *pcre_compile2(const char *, int, int *, const char **, - int *, const unsigned char *); -PCRE_EXP_DECL pcre16 *pcre16_compile2(PCRE_SPTR16, int, int *, const char **, - int *, const unsigned char *); -PCRE_EXP_DECL pcre32 *pcre32_compile2(PCRE_SPTR32, int, int *, const char **, - int *, const unsigned char *); -PCRE_EXP_DECL int pcre_config(int, void *); -PCRE_EXP_DECL int pcre16_config(int, void *); -PCRE_EXP_DECL int pcre32_config(int, void *); -PCRE_EXP_DECL int pcre_copy_named_substring(const pcre *, const char *, - int *, int, const char *, char *, int); -PCRE_EXP_DECL int pcre16_copy_named_substring(const pcre16 *, PCRE_SPTR16, - int *, int, PCRE_SPTR16, PCRE_UCHAR16 *, int); -PCRE_EXP_DECL int pcre32_copy_named_substring(const pcre32 *, PCRE_SPTR32, - int *, int, PCRE_SPTR32, PCRE_UCHAR32 *, int); -PCRE_EXP_DECL int pcre_copy_substring(const char *, int *, int, int, - char *, int); -PCRE_EXP_DECL int pcre16_copy_substring(PCRE_SPTR16, int *, int, int, - PCRE_UCHAR16 *, int); -PCRE_EXP_DECL int pcre32_copy_substring(PCRE_SPTR32, int *, int, int, - PCRE_UCHAR32 *, int); -PCRE_EXP_DECL int pcre_dfa_exec(const pcre *, const pcre_extra *, - const char *, int, int, int, int *, int , int *, int); -PCRE_EXP_DECL int pcre16_dfa_exec(const pcre16 *, const pcre16_extra *, - PCRE_SPTR16, int, int, int, int *, int , int *, int); -PCRE_EXP_DECL int pcre32_dfa_exec(const pcre32 *, const pcre32_extra *, - PCRE_SPTR32, int, int, int, int *, int , int *, int); -PCRE_EXP_DECL int pcre_exec(const pcre *, const pcre_extra *, PCRE_SPTR, - int, int, int, int *, int); -PCRE_EXP_DECL int pcre16_exec(const pcre16 *, const pcre16_extra *, - PCRE_SPTR16, int, int, int, int *, int); -PCRE_EXP_DECL int pcre32_exec(const pcre32 *, const pcre32_extra *, - PCRE_SPTR32, int, int, int, int *, int); -PCRE_EXP_DECL int pcre_jit_exec(const pcre *, const pcre_extra *, - PCRE_SPTR, int, int, int, int *, int, - pcre_jit_stack *); -PCRE_EXP_DECL int pcre16_jit_exec(const pcre16 *, const pcre16_extra *, - PCRE_SPTR16, int, int, int, int *, int, - pcre16_jit_stack *); -PCRE_EXP_DECL int pcre32_jit_exec(const pcre32 *, const pcre32_extra *, - PCRE_SPTR32, int, int, int, int *, int, - pcre32_jit_stack *); -PCRE_EXP_DECL void pcre_free_substring(const char *); -PCRE_EXP_DECL void pcre16_free_substring(PCRE_SPTR16); -PCRE_EXP_DECL void pcre32_free_substring(PCRE_SPTR32); -PCRE_EXP_DECL void pcre_free_substring_list(const char **); -PCRE_EXP_DECL void pcre16_free_substring_list(PCRE_SPTR16 *); -PCRE_EXP_DECL void pcre32_free_substring_list(PCRE_SPTR32 *); -PCRE_EXP_DECL int pcre_fullinfo(const pcre *, const pcre_extra *, int, - void *); -PCRE_EXP_DECL int pcre16_fullinfo(const pcre16 *, const pcre16_extra *, int, - void *); -PCRE_EXP_DECL int pcre32_fullinfo(const pcre32 *, const pcre32_extra *, int, - void *); -PCRE_EXP_DECL int pcre_get_named_substring(const pcre *, const char *, - int *, int, const char *, const char **); -PCRE_EXP_DECL int pcre16_get_named_substring(const pcre16 *, PCRE_SPTR16, - int *, int, PCRE_SPTR16, PCRE_SPTR16 *); -PCRE_EXP_DECL int pcre32_get_named_substring(const pcre32 *, PCRE_SPTR32, - int *, int, PCRE_SPTR32, PCRE_SPTR32 *); -PCRE_EXP_DECL int pcre_get_stringnumber(const pcre *, const char *); -PCRE_EXP_DECL int pcre16_get_stringnumber(const pcre16 *, PCRE_SPTR16); -PCRE_EXP_DECL int pcre32_get_stringnumber(const pcre32 *, PCRE_SPTR32); -PCRE_EXP_DECL int pcre_get_stringtable_entries(const pcre *, const char *, - char **, char **); -PCRE_EXP_DECL int pcre16_get_stringtable_entries(const pcre16 *, PCRE_SPTR16, - PCRE_UCHAR16 **, PCRE_UCHAR16 **); -PCRE_EXP_DECL int pcre32_get_stringtable_entries(const pcre32 *, PCRE_SPTR32, - PCRE_UCHAR32 **, PCRE_UCHAR32 **); -PCRE_EXP_DECL int pcre_get_substring(const char *, int *, int, int, - const char **); -PCRE_EXP_DECL int pcre16_get_substring(PCRE_SPTR16, int *, int, int, - PCRE_SPTR16 *); -PCRE_EXP_DECL int pcre32_get_substring(PCRE_SPTR32, int *, int, int, - PCRE_SPTR32 *); -PCRE_EXP_DECL int pcre_get_substring_list(const char *, int *, int, - const char ***); -PCRE_EXP_DECL int pcre16_get_substring_list(PCRE_SPTR16, int *, int, - PCRE_SPTR16 **); -PCRE_EXP_DECL int pcre32_get_substring_list(PCRE_SPTR32, int *, int, - PCRE_SPTR32 **); -PCRE_EXP_DECL const unsigned char *pcre_maketables(void); -PCRE_EXP_DECL const unsigned char *pcre16_maketables(void); -PCRE_EXP_DECL const unsigned char *pcre32_maketables(void); -PCRE_EXP_DECL int pcre_refcount(pcre *, int); -PCRE_EXP_DECL int pcre16_refcount(pcre16 *, int); -PCRE_EXP_DECL int pcre32_refcount(pcre32 *, int); -PCRE_EXP_DECL pcre_extra *pcre_study(const pcre *, int, const char **); -PCRE_EXP_DECL pcre16_extra *pcre16_study(const pcre16 *, int, const char **); -PCRE_EXP_DECL pcre32_extra *pcre32_study(const pcre32 *, int, const char **); -PCRE_EXP_DECL void pcre_free_study(pcre_extra *); -PCRE_EXP_DECL void pcre16_free_study(pcre16_extra *); -PCRE_EXP_DECL void pcre32_free_study(pcre32_extra *); -PCRE_EXP_DECL const char *pcre_version(void); -PCRE_EXP_DECL const char *pcre16_version(void); -PCRE_EXP_DECL const char *pcre32_version(void); - -/* Utility functions for byte order swaps. */ -PCRE_EXP_DECL int pcre_pattern_to_host_byte_order(pcre *, pcre_extra *, - const unsigned char *); -PCRE_EXP_DECL int pcre16_pattern_to_host_byte_order(pcre16 *, pcre16_extra *, - const unsigned char *); -PCRE_EXP_DECL int pcre32_pattern_to_host_byte_order(pcre32 *, pcre32_extra *, - const unsigned char *); -PCRE_EXP_DECL int pcre16_utf16_to_host_byte_order(PCRE_UCHAR16 *, - PCRE_SPTR16, int, int *, int); -PCRE_EXP_DECL int pcre32_utf32_to_host_byte_order(PCRE_UCHAR32 *, - PCRE_SPTR32, int, int *, int); - -/* JIT compiler related functions. */ - -PCRE_EXP_DECL pcre_jit_stack *pcre_jit_stack_alloc(int, int); -PCRE_EXP_DECL pcre16_jit_stack *pcre16_jit_stack_alloc(int, int); -PCRE_EXP_DECL pcre32_jit_stack *pcre32_jit_stack_alloc(int, int); -PCRE_EXP_DECL void pcre_jit_stack_free(pcre_jit_stack *); -PCRE_EXP_DECL void pcre16_jit_stack_free(pcre16_jit_stack *); -PCRE_EXP_DECL void pcre32_jit_stack_free(pcre32_jit_stack *); -PCRE_EXP_DECL void pcre_assign_jit_stack(pcre_extra *, - pcre_jit_callback, void *); -PCRE_EXP_DECL void pcre16_assign_jit_stack(pcre16_extra *, - pcre16_jit_callback, void *); -PCRE_EXP_DECL void pcre32_assign_jit_stack(pcre32_extra *, - pcre32_jit_callback, void *); -PCRE_EXP_DECL void pcre_jit_free_unused_memory(void); -PCRE_EXP_DECL void pcre16_jit_free_unused_memory(void); -PCRE_EXP_DECL void pcre32_jit_free_unused_memory(void); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* End of pcre.h */ diff --git a/src/third_party/pcre-8.39/pcre.h.generic b/src/third_party/pcre-8.39/pcre.h.generic deleted file mode 100644 index 70559700654..00000000000 --- a/src/third_party/pcre-8.39/pcre.h.generic +++ /dev/null @@ -1,677 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* This is the public header file for the PCRE library, to be #included by -applications that call the PCRE functions. - - Copyright (c) 1997-2014 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -#ifndef _PCRE_H -#define _PCRE_H - -/* The current PCRE version information. */ - -#define PCRE_MAJOR 8 -#define PCRE_MINOR 39 -#define PCRE_PRERELEASE -#define PCRE_DATE 2016-06-14 - -/* When an application links to a PCRE DLL in Windows, the symbols that are -imported have to be identified as such. When building PCRE, the appropriate -export setting is defined in pcre_internal.h, which includes this file. So we -don't change existing definitions of PCRE_EXP_DECL and PCRECPP_EXP_DECL. */ - -#if defined(_WIN32) && !defined(PCRE_STATIC) -# ifndef PCRE_EXP_DECL -# define PCRE_EXP_DECL extern __declspec(dllimport) -# endif -# ifdef __cplusplus -# ifndef PCRECPP_EXP_DECL -# define PCRECPP_EXP_DECL extern __declspec(dllimport) -# endif -# ifndef PCRECPP_EXP_DEFN -# define PCRECPP_EXP_DEFN __declspec(dllimport) -# endif -# endif -#endif - -/* By default, we use the standard "extern" declarations. */ - -#ifndef PCRE_EXP_DECL -# ifdef __cplusplus -# define PCRE_EXP_DECL extern "C" -# else -# define PCRE_EXP_DECL extern -# endif -#endif - -#ifdef __cplusplus -# ifndef PCRECPP_EXP_DECL -# define PCRECPP_EXP_DECL extern -# endif -# ifndef PCRECPP_EXP_DEFN -# define PCRECPP_EXP_DEFN -# endif -#endif - -/* Have to include stdlib.h in order to ensure that size_t is defined; -it is needed here for malloc. */ - -#include - -/* Allow for C++ users */ - -#ifdef __cplusplus -extern "C" { -#endif - -/* Public options. Some are compile-time only, some are run-time only, and some -are both. Most of the compile-time options are saved with the compiled regex so -that they can be inspected during studying (and therefore JIT compiling). Note -that pcre_study() has its own set of options. Originally, all the options -defined here used distinct bits. However, almost all the bits in a 32-bit word -are now used, so in order to conserve them, option bits that were previously -only recognized at matching time (i.e. by pcre_exec() or pcre_dfa_exec()) may -also be used for compile-time options that affect only compiling and are not -relevant for studying or JIT compiling. - -Some options for pcre_compile() change its behaviour but do not affect the -behaviour of the execution functions. Other options are passed through to the -execution functions and affect their behaviour, with or without affecting the -behaviour of pcre_compile(). - -Options that can be passed to pcre_compile() are tagged Cx below, with these -variants: - -C1 Affects compile only -C2 Does not affect compile; affects exec, dfa_exec -C3 Affects compile, exec, dfa_exec -C4 Affects compile, exec, dfa_exec, study -C5 Affects compile, exec, study - -Options that can be set for pcre_exec() and/or pcre_dfa_exec() are flagged with -E and D, respectively. They take precedence over C3, C4, and C5 settings passed -from pcre_compile(). Those that are compatible with JIT execution are flagged -with J. */ - -#define PCRE_CASELESS 0x00000001 /* C1 */ -#define PCRE_MULTILINE 0x00000002 /* C1 */ -#define PCRE_DOTALL 0x00000004 /* C1 */ -#define PCRE_EXTENDED 0x00000008 /* C1 */ -#define PCRE_ANCHORED 0x00000010 /* C4 E D */ -#define PCRE_DOLLAR_ENDONLY 0x00000020 /* C2 */ -#define PCRE_EXTRA 0x00000040 /* C1 */ -#define PCRE_NOTBOL 0x00000080 /* E D J */ -#define PCRE_NOTEOL 0x00000100 /* E D J */ -#define PCRE_UNGREEDY 0x00000200 /* C1 */ -#define PCRE_NOTEMPTY 0x00000400 /* E D J */ -#define PCRE_UTF8 0x00000800 /* C4 ) */ -#define PCRE_UTF16 0x00000800 /* C4 ) Synonyms */ -#define PCRE_UTF32 0x00000800 /* C4 ) */ -#define PCRE_NO_AUTO_CAPTURE 0x00001000 /* C1 */ -#define PCRE_NO_UTF8_CHECK 0x00002000 /* C1 E D J ) */ -#define PCRE_NO_UTF16_CHECK 0x00002000 /* C1 E D J ) Synonyms */ -#define PCRE_NO_UTF32_CHECK 0x00002000 /* C1 E D J ) */ -#define PCRE_AUTO_CALLOUT 0x00004000 /* C1 */ -#define PCRE_PARTIAL_SOFT 0x00008000 /* E D J ) Synonyms */ -#define PCRE_PARTIAL 0x00008000 /* E D J ) */ - -/* This pair use the same bit. */ -#define PCRE_NEVER_UTF 0x00010000 /* C1 ) Overlaid */ -#define PCRE_DFA_SHORTEST 0x00010000 /* D ) Overlaid */ - -/* This pair use the same bit. */ -#define PCRE_NO_AUTO_POSSESS 0x00020000 /* C1 ) Overlaid */ -#define PCRE_DFA_RESTART 0x00020000 /* D ) Overlaid */ - -#define PCRE_FIRSTLINE 0x00040000 /* C3 */ -#define PCRE_DUPNAMES 0x00080000 /* C1 */ -#define PCRE_NEWLINE_CR 0x00100000 /* C3 E D */ -#define PCRE_NEWLINE_LF 0x00200000 /* C3 E D */ -#define PCRE_NEWLINE_CRLF 0x00300000 /* C3 E D */ -#define PCRE_NEWLINE_ANY 0x00400000 /* C3 E D */ -#define PCRE_NEWLINE_ANYCRLF 0x00500000 /* C3 E D */ -#define PCRE_BSR_ANYCRLF 0x00800000 /* C3 E D */ -#define PCRE_BSR_UNICODE 0x01000000 /* C3 E D */ -#define PCRE_JAVASCRIPT_COMPAT 0x02000000 /* C5 */ -#define PCRE_NO_START_OPTIMIZE 0x04000000 /* C2 E D ) Synonyms */ -#define PCRE_NO_START_OPTIMISE 0x04000000 /* C2 E D ) */ -#define PCRE_PARTIAL_HARD 0x08000000 /* E D J */ -#define PCRE_NOTEMPTY_ATSTART 0x10000000 /* E D J */ -#define PCRE_UCP 0x20000000 /* C3 */ - -/* Exec-time and get/set-time error codes */ - -#define PCRE_ERROR_NOMATCH (-1) -#define PCRE_ERROR_NULL (-2) -#define PCRE_ERROR_BADOPTION (-3) -#define PCRE_ERROR_BADMAGIC (-4) -#define PCRE_ERROR_UNKNOWN_OPCODE (-5) -#define PCRE_ERROR_UNKNOWN_NODE (-5) /* For backward compatibility */ -#define PCRE_ERROR_NOMEMORY (-6) -#define PCRE_ERROR_NOSUBSTRING (-7) -#define PCRE_ERROR_MATCHLIMIT (-8) -#define PCRE_ERROR_CALLOUT (-9) /* Never used by PCRE itself */ -#define PCRE_ERROR_BADUTF8 (-10) /* Same for 8/16/32 */ -#define PCRE_ERROR_BADUTF16 (-10) /* Same for 8/16/32 */ -#define PCRE_ERROR_BADUTF32 (-10) /* Same for 8/16/32 */ -#define PCRE_ERROR_BADUTF8_OFFSET (-11) /* Same for 8/16 */ -#define PCRE_ERROR_BADUTF16_OFFSET (-11) /* Same for 8/16 */ -#define PCRE_ERROR_PARTIAL (-12) -#define PCRE_ERROR_BADPARTIAL (-13) -#define PCRE_ERROR_INTERNAL (-14) -#define PCRE_ERROR_BADCOUNT (-15) -#define PCRE_ERROR_DFA_UITEM (-16) -#define PCRE_ERROR_DFA_UCOND (-17) -#define PCRE_ERROR_DFA_UMLIMIT (-18) -#define PCRE_ERROR_DFA_WSSIZE (-19) -#define PCRE_ERROR_DFA_RECURSE (-20) -#define PCRE_ERROR_RECURSIONLIMIT (-21) -#define PCRE_ERROR_NULLWSLIMIT (-22) /* No longer actually used */ -#define PCRE_ERROR_BADNEWLINE (-23) -#define PCRE_ERROR_BADOFFSET (-24) -#define PCRE_ERROR_SHORTUTF8 (-25) -#define PCRE_ERROR_SHORTUTF16 (-25) /* Same for 8/16 */ -#define PCRE_ERROR_RECURSELOOP (-26) -#define PCRE_ERROR_JIT_STACKLIMIT (-27) -#define PCRE_ERROR_BADMODE (-28) -#define PCRE_ERROR_BADENDIANNESS (-29) -#define PCRE_ERROR_DFA_BADRESTART (-30) -#define PCRE_ERROR_JIT_BADOPTION (-31) -#define PCRE_ERROR_BADLENGTH (-32) -#define PCRE_ERROR_UNSET (-33) - -/* Specific error codes for UTF-8 validity checks */ - -#define PCRE_UTF8_ERR0 0 -#define PCRE_UTF8_ERR1 1 -#define PCRE_UTF8_ERR2 2 -#define PCRE_UTF8_ERR3 3 -#define PCRE_UTF8_ERR4 4 -#define PCRE_UTF8_ERR5 5 -#define PCRE_UTF8_ERR6 6 -#define PCRE_UTF8_ERR7 7 -#define PCRE_UTF8_ERR8 8 -#define PCRE_UTF8_ERR9 9 -#define PCRE_UTF8_ERR10 10 -#define PCRE_UTF8_ERR11 11 -#define PCRE_UTF8_ERR12 12 -#define PCRE_UTF8_ERR13 13 -#define PCRE_UTF8_ERR14 14 -#define PCRE_UTF8_ERR15 15 -#define PCRE_UTF8_ERR16 16 -#define PCRE_UTF8_ERR17 17 -#define PCRE_UTF8_ERR18 18 -#define PCRE_UTF8_ERR19 19 -#define PCRE_UTF8_ERR20 20 -#define PCRE_UTF8_ERR21 21 -#define PCRE_UTF8_ERR22 22 /* Unused (was non-character) */ - -/* Specific error codes for UTF-16 validity checks */ - -#define PCRE_UTF16_ERR0 0 -#define PCRE_UTF16_ERR1 1 -#define PCRE_UTF16_ERR2 2 -#define PCRE_UTF16_ERR3 3 -#define PCRE_UTF16_ERR4 4 /* Unused (was non-character) */ - -/* Specific error codes for UTF-32 validity checks */ - -#define PCRE_UTF32_ERR0 0 -#define PCRE_UTF32_ERR1 1 -#define PCRE_UTF32_ERR2 2 /* Unused (was non-character) */ -#define PCRE_UTF32_ERR3 3 - -/* Request types for pcre_fullinfo() */ - -#define PCRE_INFO_OPTIONS 0 -#define PCRE_INFO_SIZE 1 -#define PCRE_INFO_CAPTURECOUNT 2 -#define PCRE_INFO_BACKREFMAX 3 -#define PCRE_INFO_FIRSTBYTE 4 -#define PCRE_INFO_FIRSTCHAR 4 /* For backwards compatibility */ -#define PCRE_INFO_FIRSTTABLE 5 -#define PCRE_INFO_LASTLITERAL 6 -#define PCRE_INFO_NAMEENTRYSIZE 7 -#define PCRE_INFO_NAMECOUNT 8 -#define PCRE_INFO_NAMETABLE 9 -#define PCRE_INFO_STUDYSIZE 10 -#define PCRE_INFO_DEFAULT_TABLES 11 -#define PCRE_INFO_OKPARTIAL 12 -#define PCRE_INFO_JCHANGED 13 -#define PCRE_INFO_HASCRORLF 14 -#define PCRE_INFO_MINLENGTH 15 -#define PCRE_INFO_JIT 16 -#define PCRE_INFO_JITSIZE 17 -#define PCRE_INFO_MAXLOOKBEHIND 18 -#define PCRE_INFO_FIRSTCHARACTER 19 -#define PCRE_INFO_FIRSTCHARACTERFLAGS 20 -#define PCRE_INFO_REQUIREDCHAR 21 -#define PCRE_INFO_REQUIREDCHARFLAGS 22 -#define PCRE_INFO_MATCHLIMIT 23 -#define PCRE_INFO_RECURSIONLIMIT 24 -#define PCRE_INFO_MATCH_EMPTY 25 - -/* Request types for pcre_config(). Do not re-arrange, in order to remain -compatible. */ - -#define PCRE_CONFIG_UTF8 0 -#define PCRE_CONFIG_NEWLINE 1 -#define PCRE_CONFIG_LINK_SIZE 2 -#define PCRE_CONFIG_POSIX_MALLOC_THRESHOLD 3 -#define PCRE_CONFIG_MATCH_LIMIT 4 -#define PCRE_CONFIG_STACKRECURSE 5 -#define PCRE_CONFIG_UNICODE_PROPERTIES 6 -#define PCRE_CONFIG_MATCH_LIMIT_RECURSION 7 -#define PCRE_CONFIG_BSR 8 -#define PCRE_CONFIG_JIT 9 -#define PCRE_CONFIG_UTF16 10 -#define PCRE_CONFIG_JITTARGET 11 -#define PCRE_CONFIG_UTF32 12 -#define PCRE_CONFIG_PARENS_LIMIT 13 - -/* Request types for pcre_study(). Do not re-arrange, in order to remain -compatible. */ - -#define PCRE_STUDY_JIT_COMPILE 0x0001 -#define PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE 0x0002 -#define PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE 0x0004 -#define PCRE_STUDY_EXTRA_NEEDED 0x0008 - -/* Bit flags for the pcre[16|32]_extra structure. Do not re-arrange or redefine -these bits, just add new ones on the end, in order to remain compatible. */ - -#define PCRE_EXTRA_STUDY_DATA 0x0001 -#define PCRE_EXTRA_MATCH_LIMIT 0x0002 -#define PCRE_EXTRA_CALLOUT_DATA 0x0004 -#define PCRE_EXTRA_TABLES 0x0008 -#define PCRE_EXTRA_MATCH_LIMIT_RECURSION 0x0010 -#define PCRE_EXTRA_MARK 0x0020 -#define PCRE_EXTRA_EXECUTABLE_JIT 0x0040 - -/* Types */ - -struct real_pcre; /* declaration; the definition is private */ -typedef struct real_pcre pcre; - -struct real_pcre16; /* declaration; the definition is private */ -typedef struct real_pcre16 pcre16; - -struct real_pcre32; /* declaration; the definition is private */ -typedef struct real_pcre32 pcre32; - -struct real_pcre_jit_stack; /* declaration; the definition is private */ -typedef struct real_pcre_jit_stack pcre_jit_stack; - -struct real_pcre16_jit_stack; /* declaration; the definition is private */ -typedef struct real_pcre16_jit_stack pcre16_jit_stack; - -struct real_pcre32_jit_stack; /* declaration; the definition is private */ -typedef struct real_pcre32_jit_stack pcre32_jit_stack; - -/* If PCRE is compiled with 16 bit character support, PCRE_UCHAR16 must contain -a 16 bit wide signed data type. Otherwise it can be a dummy data type since -pcre16 functions are not implemented. There is a check for this in pcre_internal.h. */ -#ifndef PCRE_UCHAR16 -#define PCRE_UCHAR16 unsigned short -#endif - -#ifndef PCRE_SPTR16 -#define PCRE_SPTR16 const PCRE_UCHAR16 * -#endif - -/* If PCRE is compiled with 32 bit character support, PCRE_UCHAR32 must contain -a 32 bit wide signed data type. Otherwise it can be a dummy data type since -pcre32 functions are not implemented. There is a check for this in pcre_internal.h. */ -#ifndef PCRE_UCHAR32 -#define PCRE_UCHAR32 unsigned int -#endif - -#ifndef PCRE_SPTR32 -#define PCRE_SPTR32 const PCRE_UCHAR32 * -#endif - -/* When PCRE is compiled as a C++ library, the subject pointer type can be -replaced with a custom type. For conventional use, the public interface is a -const char *. */ - -#ifndef PCRE_SPTR -#define PCRE_SPTR const char * -#endif - -/* The structure for passing additional data to pcre_exec(). This is defined in -such as way as to be extensible. Always add new fields at the end, in order to -remain compatible. */ - -typedef struct pcre_extra { - unsigned long int flags; /* Bits for which fields are set */ - void *study_data; /* Opaque data from pcre_study() */ - unsigned long int match_limit; /* Maximum number of calls to match() */ - void *callout_data; /* Data passed back in callouts */ - const unsigned char *tables; /* Pointer to character tables */ - unsigned long int match_limit_recursion; /* Max recursive calls to match() */ - unsigned char **mark; /* For passing back a mark pointer */ - void *executable_jit; /* Contains a pointer to a compiled jit code */ -} pcre_extra; - -/* Same structure as above, but with 16 bit char pointers. */ - -typedef struct pcre16_extra { - unsigned long int flags; /* Bits for which fields are set */ - void *study_data; /* Opaque data from pcre_study() */ - unsigned long int match_limit; /* Maximum number of calls to match() */ - void *callout_data; /* Data passed back in callouts */ - const unsigned char *tables; /* Pointer to character tables */ - unsigned long int match_limit_recursion; /* Max recursive calls to match() */ - PCRE_UCHAR16 **mark; /* For passing back a mark pointer */ - void *executable_jit; /* Contains a pointer to a compiled jit code */ -} pcre16_extra; - -/* Same structure as above, but with 32 bit char pointers. */ - -typedef struct pcre32_extra { - unsigned long int flags; /* Bits for which fields are set */ - void *study_data; /* Opaque data from pcre_study() */ - unsigned long int match_limit; /* Maximum number of calls to match() */ - void *callout_data; /* Data passed back in callouts */ - const unsigned char *tables; /* Pointer to character tables */ - unsigned long int match_limit_recursion; /* Max recursive calls to match() */ - PCRE_UCHAR32 **mark; /* For passing back a mark pointer */ - void *executable_jit; /* Contains a pointer to a compiled jit code */ -} pcre32_extra; - -/* The structure for passing out data via the pcre_callout_function. We use a -structure so that new fields can be added on the end in future versions, -without changing the API of the function, thereby allowing old clients to work -without modification. */ - -typedef struct pcre_callout_block { - int version; /* Identifies version of block */ - /* ------------------------ Version 0 ------------------------------- */ - int callout_number; /* Number compiled into pattern */ - int *offset_vector; /* The offset vector */ - PCRE_SPTR subject; /* The subject being matched */ - int subject_length; /* The length of the subject */ - int start_match; /* Offset to start of this match attempt */ - int current_position; /* Where we currently are in the subject */ - int capture_top; /* Max current capture */ - int capture_last; /* Most recently closed capture */ - void *callout_data; /* Data passed in with the call */ - /* ------------------- Added for Version 1 -------------------------- */ - int pattern_position; /* Offset to next item in the pattern */ - int next_item_length; /* Length of next item in the pattern */ - /* ------------------- Added for Version 2 -------------------------- */ - const unsigned char *mark; /* Pointer to current mark or NULL */ - /* ------------------------------------------------------------------ */ -} pcre_callout_block; - -/* Same structure as above, but with 16 bit char pointers. */ - -typedef struct pcre16_callout_block { - int version; /* Identifies version of block */ - /* ------------------------ Version 0 ------------------------------- */ - int callout_number; /* Number compiled into pattern */ - int *offset_vector; /* The offset vector */ - PCRE_SPTR16 subject; /* The subject being matched */ - int subject_length; /* The length of the subject */ - int start_match; /* Offset to start of this match attempt */ - int current_position; /* Where we currently are in the subject */ - int capture_top; /* Max current capture */ - int capture_last; /* Most recently closed capture */ - void *callout_data; /* Data passed in with the call */ - /* ------------------- Added for Version 1 -------------------------- */ - int pattern_position; /* Offset to next item in the pattern */ - int next_item_length; /* Length of next item in the pattern */ - /* ------------------- Added for Version 2 -------------------------- */ - const PCRE_UCHAR16 *mark; /* Pointer to current mark or NULL */ - /* ------------------------------------------------------------------ */ -} pcre16_callout_block; - -/* Same structure as above, but with 32 bit char pointers. */ - -typedef struct pcre32_callout_block { - int version; /* Identifies version of block */ - /* ------------------------ Version 0 ------------------------------- */ - int callout_number; /* Number compiled into pattern */ - int *offset_vector; /* The offset vector */ - PCRE_SPTR32 subject; /* The subject being matched */ - int subject_length; /* The length of the subject */ - int start_match; /* Offset to start of this match attempt */ - int current_position; /* Where we currently are in the subject */ - int capture_top; /* Max current capture */ - int capture_last; /* Most recently closed capture */ - void *callout_data; /* Data passed in with the call */ - /* ------------------- Added for Version 1 -------------------------- */ - int pattern_position; /* Offset to next item in the pattern */ - int next_item_length; /* Length of next item in the pattern */ - /* ------------------- Added for Version 2 -------------------------- */ - const PCRE_UCHAR32 *mark; /* Pointer to current mark or NULL */ - /* ------------------------------------------------------------------ */ -} pcre32_callout_block; - -/* Indirection for store get and free functions. These can be set to -alternative malloc/free functions if required. Special ones are used in the -non-recursive case for "frames". There is also an optional callout function -that is triggered by the (?) regex item. For Virtual Pascal, these definitions -have to take another form. */ - -#ifndef VPCOMPAT -PCRE_EXP_DECL void *(*pcre_malloc)(size_t); -PCRE_EXP_DECL void (*pcre_free)(void *); -PCRE_EXP_DECL void *(*pcre_stack_malloc)(size_t); -PCRE_EXP_DECL void (*pcre_stack_free)(void *); -PCRE_EXP_DECL int (*pcre_callout)(pcre_callout_block *); -PCRE_EXP_DECL int (*pcre_stack_guard)(void); - -PCRE_EXP_DECL void *(*pcre16_malloc)(size_t); -PCRE_EXP_DECL void (*pcre16_free)(void *); -PCRE_EXP_DECL void *(*pcre16_stack_malloc)(size_t); -PCRE_EXP_DECL void (*pcre16_stack_free)(void *); -PCRE_EXP_DECL int (*pcre16_callout)(pcre16_callout_block *); -PCRE_EXP_DECL int (*pcre16_stack_guard)(void); - -PCRE_EXP_DECL void *(*pcre32_malloc)(size_t); -PCRE_EXP_DECL void (*pcre32_free)(void *); -PCRE_EXP_DECL void *(*pcre32_stack_malloc)(size_t); -PCRE_EXP_DECL void (*pcre32_stack_free)(void *); -PCRE_EXP_DECL int (*pcre32_callout)(pcre32_callout_block *); -PCRE_EXP_DECL int (*pcre32_stack_guard)(void); -#else /* VPCOMPAT */ -PCRE_EXP_DECL void *pcre_malloc(size_t); -PCRE_EXP_DECL void pcre_free(void *); -PCRE_EXP_DECL void *pcre_stack_malloc(size_t); -PCRE_EXP_DECL void pcre_stack_free(void *); -PCRE_EXP_DECL int pcre_callout(pcre_callout_block *); -PCRE_EXP_DECL int pcre_stack_guard(void); - -PCRE_EXP_DECL void *pcre16_malloc(size_t); -PCRE_EXP_DECL void pcre16_free(void *); -PCRE_EXP_DECL void *pcre16_stack_malloc(size_t); -PCRE_EXP_DECL void pcre16_stack_free(void *); -PCRE_EXP_DECL int pcre16_callout(pcre16_callout_block *); -PCRE_EXP_DECL int pcre16_stack_guard(void); - -PCRE_EXP_DECL void *pcre32_malloc(size_t); -PCRE_EXP_DECL void pcre32_free(void *); -PCRE_EXP_DECL void *pcre32_stack_malloc(size_t); -PCRE_EXP_DECL void pcre32_stack_free(void *); -PCRE_EXP_DECL int pcre32_callout(pcre32_callout_block *); -PCRE_EXP_DECL int pcre32_stack_guard(void); -#endif /* VPCOMPAT */ - -/* User defined callback which provides a stack just before the match starts. */ - -typedef pcre_jit_stack *(*pcre_jit_callback)(void *); -typedef pcre16_jit_stack *(*pcre16_jit_callback)(void *); -typedef pcre32_jit_stack *(*pcre32_jit_callback)(void *); - -/* Exported PCRE functions */ - -PCRE_EXP_DECL pcre *pcre_compile(const char *, int, const char **, int *, - const unsigned char *); -PCRE_EXP_DECL pcre16 *pcre16_compile(PCRE_SPTR16, int, const char **, int *, - const unsigned char *); -PCRE_EXP_DECL pcre32 *pcre32_compile(PCRE_SPTR32, int, const char **, int *, - const unsigned char *); -PCRE_EXP_DECL pcre *pcre_compile2(const char *, int, int *, const char **, - int *, const unsigned char *); -PCRE_EXP_DECL pcre16 *pcre16_compile2(PCRE_SPTR16, int, int *, const char **, - int *, const unsigned char *); -PCRE_EXP_DECL pcre32 *pcre32_compile2(PCRE_SPTR32, int, int *, const char **, - int *, const unsigned char *); -PCRE_EXP_DECL int pcre_config(int, void *); -PCRE_EXP_DECL int pcre16_config(int, void *); -PCRE_EXP_DECL int pcre32_config(int, void *); -PCRE_EXP_DECL int pcre_copy_named_substring(const pcre *, const char *, - int *, int, const char *, char *, int); -PCRE_EXP_DECL int pcre16_copy_named_substring(const pcre16 *, PCRE_SPTR16, - int *, int, PCRE_SPTR16, PCRE_UCHAR16 *, int); -PCRE_EXP_DECL int pcre32_copy_named_substring(const pcre32 *, PCRE_SPTR32, - int *, int, PCRE_SPTR32, PCRE_UCHAR32 *, int); -PCRE_EXP_DECL int pcre_copy_substring(const char *, int *, int, int, - char *, int); -PCRE_EXP_DECL int pcre16_copy_substring(PCRE_SPTR16, int *, int, int, - PCRE_UCHAR16 *, int); -PCRE_EXP_DECL int pcre32_copy_substring(PCRE_SPTR32, int *, int, int, - PCRE_UCHAR32 *, int); -PCRE_EXP_DECL int pcre_dfa_exec(const pcre *, const pcre_extra *, - const char *, int, int, int, int *, int , int *, int); -PCRE_EXP_DECL int pcre16_dfa_exec(const pcre16 *, const pcre16_extra *, - PCRE_SPTR16, int, int, int, int *, int , int *, int); -PCRE_EXP_DECL int pcre32_dfa_exec(const pcre32 *, const pcre32_extra *, - PCRE_SPTR32, int, int, int, int *, int , int *, int); -PCRE_EXP_DECL int pcre_exec(const pcre *, const pcre_extra *, PCRE_SPTR, - int, int, int, int *, int); -PCRE_EXP_DECL int pcre16_exec(const pcre16 *, const pcre16_extra *, - PCRE_SPTR16, int, int, int, int *, int); -PCRE_EXP_DECL int pcre32_exec(const pcre32 *, const pcre32_extra *, - PCRE_SPTR32, int, int, int, int *, int); -PCRE_EXP_DECL int pcre_jit_exec(const pcre *, const pcre_extra *, - PCRE_SPTR, int, int, int, int *, int, - pcre_jit_stack *); -PCRE_EXP_DECL int pcre16_jit_exec(const pcre16 *, const pcre16_extra *, - PCRE_SPTR16, int, int, int, int *, int, - pcre16_jit_stack *); -PCRE_EXP_DECL int pcre32_jit_exec(const pcre32 *, const pcre32_extra *, - PCRE_SPTR32, int, int, int, int *, int, - pcre32_jit_stack *); -PCRE_EXP_DECL void pcre_free_substring(const char *); -PCRE_EXP_DECL void pcre16_free_substring(PCRE_SPTR16); -PCRE_EXP_DECL void pcre32_free_substring(PCRE_SPTR32); -PCRE_EXP_DECL void pcre_free_substring_list(const char **); -PCRE_EXP_DECL void pcre16_free_substring_list(PCRE_SPTR16 *); -PCRE_EXP_DECL void pcre32_free_substring_list(PCRE_SPTR32 *); -PCRE_EXP_DECL int pcre_fullinfo(const pcre *, const pcre_extra *, int, - void *); -PCRE_EXP_DECL int pcre16_fullinfo(const pcre16 *, const pcre16_extra *, int, - void *); -PCRE_EXP_DECL int pcre32_fullinfo(const pcre32 *, const pcre32_extra *, int, - void *); -PCRE_EXP_DECL int pcre_get_named_substring(const pcre *, const char *, - int *, int, const char *, const char **); -PCRE_EXP_DECL int pcre16_get_named_substring(const pcre16 *, PCRE_SPTR16, - int *, int, PCRE_SPTR16, PCRE_SPTR16 *); -PCRE_EXP_DECL int pcre32_get_named_substring(const pcre32 *, PCRE_SPTR32, - int *, int, PCRE_SPTR32, PCRE_SPTR32 *); -PCRE_EXP_DECL int pcre_get_stringnumber(const pcre *, const char *); -PCRE_EXP_DECL int pcre16_get_stringnumber(const pcre16 *, PCRE_SPTR16); -PCRE_EXP_DECL int pcre32_get_stringnumber(const pcre32 *, PCRE_SPTR32); -PCRE_EXP_DECL int pcre_get_stringtable_entries(const pcre *, const char *, - char **, char **); -PCRE_EXP_DECL int pcre16_get_stringtable_entries(const pcre16 *, PCRE_SPTR16, - PCRE_UCHAR16 **, PCRE_UCHAR16 **); -PCRE_EXP_DECL int pcre32_get_stringtable_entries(const pcre32 *, PCRE_SPTR32, - PCRE_UCHAR32 **, PCRE_UCHAR32 **); -PCRE_EXP_DECL int pcre_get_substring(const char *, int *, int, int, - const char **); -PCRE_EXP_DECL int pcre16_get_substring(PCRE_SPTR16, int *, int, int, - PCRE_SPTR16 *); -PCRE_EXP_DECL int pcre32_get_substring(PCRE_SPTR32, int *, int, int, - PCRE_SPTR32 *); -PCRE_EXP_DECL int pcre_get_substring_list(const char *, int *, int, - const char ***); -PCRE_EXP_DECL int pcre16_get_substring_list(PCRE_SPTR16, int *, int, - PCRE_SPTR16 **); -PCRE_EXP_DECL int pcre32_get_substring_list(PCRE_SPTR32, int *, int, - PCRE_SPTR32 **); -PCRE_EXP_DECL const unsigned char *pcre_maketables(void); -PCRE_EXP_DECL const unsigned char *pcre16_maketables(void); -PCRE_EXP_DECL const unsigned char *pcre32_maketables(void); -PCRE_EXP_DECL int pcre_refcount(pcre *, int); -PCRE_EXP_DECL int pcre16_refcount(pcre16 *, int); -PCRE_EXP_DECL int pcre32_refcount(pcre32 *, int); -PCRE_EXP_DECL pcre_extra *pcre_study(const pcre *, int, const char **); -PCRE_EXP_DECL pcre16_extra *pcre16_study(const pcre16 *, int, const char **); -PCRE_EXP_DECL pcre32_extra *pcre32_study(const pcre32 *, int, const char **); -PCRE_EXP_DECL void pcre_free_study(pcre_extra *); -PCRE_EXP_DECL void pcre16_free_study(pcre16_extra *); -PCRE_EXP_DECL void pcre32_free_study(pcre32_extra *); -PCRE_EXP_DECL const char *pcre_version(void); -PCRE_EXP_DECL const char *pcre16_version(void); -PCRE_EXP_DECL const char *pcre32_version(void); - -/* Utility functions for byte order swaps. */ -PCRE_EXP_DECL int pcre_pattern_to_host_byte_order(pcre *, pcre_extra *, - const unsigned char *); -PCRE_EXP_DECL int pcre16_pattern_to_host_byte_order(pcre16 *, pcre16_extra *, - const unsigned char *); -PCRE_EXP_DECL int pcre32_pattern_to_host_byte_order(pcre32 *, pcre32_extra *, - const unsigned char *); -PCRE_EXP_DECL int pcre16_utf16_to_host_byte_order(PCRE_UCHAR16 *, - PCRE_SPTR16, int, int *, int); -PCRE_EXP_DECL int pcre32_utf32_to_host_byte_order(PCRE_UCHAR32 *, - PCRE_SPTR32, int, int *, int); - -/* JIT compiler related functions. */ - -PCRE_EXP_DECL pcre_jit_stack *pcre_jit_stack_alloc(int, int); -PCRE_EXP_DECL pcre16_jit_stack *pcre16_jit_stack_alloc(int, int); -PCRE_EXP_DECL pcre32_jit_stack *pcre32_jit_stack_alloc(int, int); -PCRE_EXP_DECL void pcre_jit_stack_free(pcre_jit_stack *); -PCRE_EXP_DECL void pcre16_jit_stack_free(pcre16_jit_stack *); -PCRE_EXP_DECL void pcre32_jit_stack_free(pcre32_jit_stack *); -PCRE_EXP_DECL void pcre_assign_jit_stack(pcre_extra *, - pcre_jit_callback, void *); -PCRE_EXP_DECL void pcre16_assign_jit_stack(pcre16_extra *, - pcre16_jit_callback, void *); -PCRE_EXP_DECL void pcre32_assign_jit_stack(pcre32_extra *, - pcre32_jit_callback, void *); -PCRE_EXP_DECL void pcre_jit_free_unused_memory(void); -PCRE_EXP_DECL void pcre16_jit_free_unused_memory(void); -PCRE_EXP_DECL void pcre32_jit_free_unused_memory(void); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* End of pcre.h */ diff --git a/src/third_party/pcre-8.39/pcre.h.in b/src/third_party/pcre-8.39/pcre.h.in deleted file mode 100644 index 667a45ed575..00000000000 --- a/src/third_party/pcre-8.39/pcre.h.in +++ /dev/null @@ -1,677 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* This is the public header file for the PCRE library, to be #included by -applications that call the PCRE functions. - - Copyright (c) 1997-2014 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -#ifndef _PCRE_H -#define _PCRE_H - -/* The current PCRE version information. */ - -#define PCRE_MAJOR @PCRE_MAJOR@ -#define PCRE_MINOR @PCRE_MINOR@ -#define PCRE_PRERELEASE @PCRE_PRERELEASE@ -#define PCRE_DATE @PCRE_DATE@ - -/* When an application links to a PCRE DLL in Windows, the symbols that are -imported have to be identified as such. When building PCRE, the appropriate -export setting is defined in pcre_internal.h, which includes this file. So we -don't change existing definitions of PCRE_EXP_DECL and PCRECPP_EXP_DECL. */ - -#if defined(_WIN32) && !defined(PCRE_STATIC) -# ifndef PCRE_EXP_DECL -# define PCRE_EXP_DECL extern __declspec(dllimport) -# endif -# ifdef __cplusplus -# ifndef PCRECPP_EXP_DECL -# define PCRECPP_EXP_DECL extern __declspec(dllimport) -# endif -# ifndef PCRECPP_EXP_DEFN -# define PCRECPP_EXP_DEFN __declspec(dllimport) -# endif -# endif -#endif - -/* By default, we use the standard "extern" declarations. */ - -#ifndef PCRE_EXP_DECL -# ifdef __cplusplus -# define PCRE_EXP_DECL extern "C" -# else -# define PCRE_EXP_DECL extern -# endif -#endif - -#ifdef __cplusplus -# ifndef PCRECPP_EXP_DECL -# define PCRECPP_EXP_DECL extern -# endif -# ifndef PCRECPP_EXP_DEFN -# define PCRECPP_EXP_DEFN -# endif -#endif - -/* Have to include stdlib.h in order to ensure that size_t is defined; -it is needed here for malloc. */ - -#include - -/* Allow for C++ users */ - -#ifdef __cplusplus -extern "C" { -#endif - -/* Public options. Some are compile-time only, some are run-time only, and some -are both. Most of the compile-time options are saved with the compiled regex so -that they can be inspected during studying (and therefore JIT compiling). Note -that pcre_study() has its own set of options. Originally, all the options -defined here used distinct bits. However, almost all the bits in a 32-bit word -are now used, so in order to conserve them, option bits that were previously -only recognized at matching time (i.e. by pcre_exec() or pcre_dfa_exec()) may -also be used for compile-time options that affect only compiling and are not -relevant for studying or JIT compiling. - -Some options for pcre_compile() change its behaviour but do not affect the -behaviour of the execution functions. Other options are passed through to the -execution functions and affect their behaviour, with or without affecting the -behaviour of pcre_compile(). - -Options that can be passed to pcre_compile() are tagged Cx below, with these -variants: - -C1 Affects compile only -C2 Does not affect compile; affects exec, dfa_exec -C3 Affects compile, exec, dfa_exec -C4 Affects compile, exec, dfa_exec, study -C5 Affects compile, exec, study - -Options that can be set for pcre_exec() and/or pcre_dfa_exec() are flagged with -E and D, respectively. They take precedence over C3, C4, and C5 settings passed -from pcre_compile(). Those that are compatible with JIT execution are flagged -with J. */ - -#define PCRE_CASELESS 0x00000001 /* C1 */ -#define PCRE_MULTILINE 0x00000002 /* C1 */ -#define PCRE_DOTALL 0x00000004 /* C1 */ -#define PCRE_EXTENDED 0x00000008 /* C1 */ -#define PCRE_ANCHORED 0x00000010 /* C4 E D */ -#define PCRE_DOLLAR_ENDONLY 0x00000020 /* C2 */ -#define PCRE_EXTRA 0x00000040 /* C1 */ -#define PCRE_NOTBOL 0x00000080 /* E D J */ -#define PCRE_NOTEOL 0x00000100 /* E D J */ -#define PCRE_UNGREEDY 0x00000200 /* C1 */ -#define PCRE_NOTEMPTY 0x00000400 /* E D J */ -#define PCRE_UTF8 0x00000800 /* C4 ) */ -#define PCRE_UTF16 0x00000800 /* C4 ) Synonyms */ -#define PCRE_UTF32 0x00000800 /* C4 ) */ -#define PCRE_NO_AUTO_CAPTURE 0x00001000 /* C1 */ -#define PCRE_NO_UTF8_CHECK 0x00002000 /* C1 E D J ) */ -#define PCRE_NO_UTF16_CHECK 0x00002000 /* C1 E D J ) Synonyms */ -#define PCRE_NO_UTF32_CHECK 0x00002000 /* C1 E D J ) */ -#define PCRE_AUTO_CALLOUT 0x00004000 /* C1 */ -#define PCRE_PARTIAL_SOFT 0x00008000 /* E D J ) Synonyms */ -#define PCRE_PARTIAL 0x00008000 /* E D J ) */ - -/* This pair use the same bit. */ -#define PCRE_NEVER_UTF 0x00010000 /* C1 ) Overlaid */ -#define PCRE_DFA_SHORTEST 0x00010000 /* D ) Overlaid */ - -/* This pair use the same bit. */ -#define PCRE_NO_AUTO_POSSESS 0x00020000 /* C1 ) Overlaid */ -#define PCRE_DFA_RESTART 0x00020000 /* D ) Overlaid */ - -#define PCRE_FIRSTLINE 0x00040000 /* C3 */ -#define PCRE_DUPNAMES 0x00080000 /* C1 */ -#define PCRE_NEWLINE_CR 0x00100000 /* C3 E D */ -#define PCRE_NEWLINE_LF 0x00200000 /* C3 E D */ -#define PCRE_NEWLINE_CRLF 0x00300000 /* C3 E D */ -#define PCRE_NEWLINE_ANY 0x00400000 /* C3 E D */ -#define PCRE_NEWLINE_ANYCRLF 0x00500000 /* C3 E D */ -#define PCRE_BSR_ANYCRLF 0x00800000 /* C3 E D */ -#define PCRE_BSR_UNICODE 0x01000000 /* C3 E D */ -#define PCRE_JAVASCRIPT_COMPAT 0x02000000 /* C5 */ -#define PCRE_NO_START_OPTIMIZE 0x04000000 /* C2 E D ) Synonyms */ -#define PCRE_NO_START_OPTIMISE 0x04000000 /* C2 E D ) */ -#define PCRE_PARTIAL_HARD 0x08000000 /* E D J */ -#define PCRE_NOTEMPTY_ATSTART 0x10000000 /* E D J */ -#define PCRE_UCP 0x20000000 /* C3 */ - -/* Exec-time and get/set-time error codes */ - -#define PCRE_ERROR_NOMATCH (-1) -#define PCRE_ERROR_NULL (-2) -#define PCRE_ERROR_BADOPTION (-3) -#define PCRE_ERROR_BADMAGIC (-4) -#define PCRE_ERROR_UNKNOWN_OPCODE (-5) -#define PCRE_ERROR_UNKNOWN_NODE (-5) /* For backward compatibility */ -#define PCRE_ERROR_NOMEMORY (-6) -#define PCRE_ERROR_NOSUBSTRING (-7) -#define PCRE_ERROR_MATCHLIMIT (-8) -#define PCRE_ERROR_CALLOUT (-9) /* Never used by PCRE itself */ -#define PCRE_ERROR_BADUTF8 (-10) /* Same for 8/16/32 */ -#define PCRE_ERROR_BADUTF16 (-10) /* Same for 8/16/32 */ -#define PCRE_ERROR_BADUTF32 (-10) /* Same for 8/16/32 */ -#define PCRE_ERROR_BADUTF8_OFFSET (-11) /* Same for 8/16 */ -#define PCRE_ERROR_BADUTF16_OFFSET (-11) /* Same for 8/16 */ -#define PCRE_ERROR_PARTIAL (-12) -#define PCRE_ERROR_BADPARTIAL (-13) -#define PCRE_ERROR_INTERNAL (-14) -#define PCRE_ERROR_BADCOUNT (-15) -#define PCRE_ERROR_DFA_UITEM (-16) -#define PCRE_ERROR_DFA_UCOND (-17) -#define PCRE_ERROR_DFA_UMLIMIT (-18) -#define PCRE_ERROR_DFA_WSSIZE (-19) -#define PCRE_ERROR_DFA_RECURSE (-20) -#define PCRE_ERROR_RECURSIONLIMIT (-21) -#define PCRE_ERROR_NULLWSLIMIT (-22) /* No longer actually used */ -#define PCRE_ERROR_BADNEWLINE (-23) -#define PCRE_ERROR_BADOFFSET (-24) -#define PCRE_ERROR_SHORTUTF8 (-25) -#define PCRE_ERROR_SHORTUTF16 (-25) /* Same for 8/16 */ -#define PCRE_ERROR_RECURSELOOP (-26) -#define PCRE_ERROR_JIT_STACKLIMIT (-27) -#define PCRE_ERROR_BADMODE (-28) -#define PCRE_ERROR_BADENDIANNESS (-29) -#define PCRE_ERROR_DFA_BADRESTART (-30) -#define PCRE_ERROR_JIT_BADOPTION (-31) -#define PCRE_ERROR_BADLENGTH (-32) -#define PCRE_ERROR_UNSET (-33) - -/* Specific error codes for UTF-8 validity checks */ - -#define PCRE_UTF8_ERR0 0 -#define PCRE_UTF8_ERR1 1 -#define PCRE_UTF8_ERR2 2 -#define PCRE_UTF8_ERR3 3 -#define PCRE_UTF8_ERR4 4 -#define PCRE_UTF8_ERR5 5 -#define PCRE_UTF8_ERR6 6 -#define PCRE_UTF8_ERR7 7 -#define PCRE_UTF8_ERR8 8 -#define PCRE_UTF8_ERR9 9 -#define PCRE_UTF8_ERR10 10 -#define PCRE_UTF8_ERR11 11 -#define PCRE_UTF8_ERR12 12 -#define PCRE_UTF8_ERR13 13 -#define PCRE_UTF8_ERR14 14 -#define PCRE_UTF8_ERR15 15 -#define PCRE_UTF8_ERR16 16 -#define PCRE_UTF8_ERR17 17 -#define PCRE_UTF8_ERR18 18 -#define PCRE_UTF8_ERR19 19 -#define PCRE_UTF8_ERR20 20 -#define PCRE_UTF8_ERR21 21 -#define PCRE_UTF8_ERR22 22 /* Unused (was non-character) */ - -/* Specific error codes for UTF-16 validity checks */ - -#define PCRE_UTF16_ERR0 0 -#define PCRE_UTF16_ERR1 1 -#define PCRE_UTF16_ERR2 2 -#define PCRE_UTF16_ERR3 3 -#define PCRE_UTF16_ERR4 4 /* Unused (was non-character) */ - -/* Specific error codes for UTF-32 validity checks */ - -#define PCRE_UTF32_ERR0 0 -#define PCRE_UTF32_ERR1 1 -#define PCRE_UTF32_ERR2 2 /* Unused (was non-character) */ -#define PCRE_UTF32_ERR3 3 - -/* Request types for pcre_fullinfo() */ - -#define PCRE_INFO_OPTIONS 0 -#define PCRE_INFO_SIZE 1 -#define PCRE_INFO_CAPTURECOUNT 2 -#define PCRE_INFO_BACKREFMAX 3 -#define PCRE_INFO_FIRSTBYTE 4 -#define PCRE_INFO_FIRSTCHAR 4 /* For backwards compatibility */ -#define PCRE_INFO_FIRSTTABLE 5 -#define PCRE_INFO_LASTLITERAL 6 -#define PCRE_INFO_NAMEENTRYSIZE 7 -#define PCRE_INFO_NAMECOUNT 8 -#define PCRE_INFO_NAMETABLE 9 -#define PCRE_INFO_STUDYSIZE 10 -#define PCRE_INFO_DEFAULT_TABLES 11 -#define PCRE_INFO_OKPARTIAL 12 -#define PCRE_INFO_JCHANGED 13 -#define PCRE_INFO_HASCRORLF 14 -#define PCRE_INFO_MINLENGTH 15 -#define PCRE_INFO_JIT 16 -#define PCRE_INFO_JITSIZE 17 -#define PCRE_INFO_MAXLOOKBEHIND 18 -#define PCRE_INFO_FIRSTCHARACTER 19 -#define PCRE_INFO_FIRSTCHARACTERFLAGS 20 -#define PCRE_INFO_REQUIREDCHAR 21 -#define PCRE_INFO_REQUIREDCHARFLAGS 22 -#define PCRE_INFO_MATCHLIMIT 23 -#define PCRE_INFO_RECURSIONLIMIT 24 -#define PCRE_INFO_MATCH_EMPTY 25 - -/* Request types for pcre_config(). Do not re-arrange, in order to remain -compatible. */ - -#define PCRE_CONFIG_UTF8 0 -#define PCRE_CONFIG_NEWLINE 1 -#define PCRE_CONFIG_LINK_SIZE 2 -#define PCRE_CONFIG_POSIX_MALLOC_THRESHOLD 3 -#define PCRE_CONFIG_MATCH_LIMIT 4 -#define PCRE_CONFIG_STACKRECURSE 5 -#define PCRE_CONFIG_UNICODE_PROPERTIES 6 -#define PCRE_CONFIG_MATCH_LIMIT_RECURSION 7 -#define PCRE_CONFIG_BSR 8 -#define PCRE_CONFIG_JIT 9 -#define PCRE_CONFIG_UTF16 10 -#define PCRE_CONFIG_JITTARGET 11 -#define PCRE_CONFIG_UTF32 12 -#define PCRE_CONFIG_PARENS_LIMIT 13 - -/* Request types for pcre_study(). Do not re-arrange, in order to remain -compatible. */ - -#define PCRE_STUDY_JIT_COMPILE 0x0001 -#define PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE 0x0002 -#define PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE 0x0004 -#define PCRE_STUDY_EXTRA_NEEDED 0x0008 - -/* Bit flags for the pcre[16|32]_extra structure. Do not re-arrange or redefine -these bits, just add new ones on the end, in order to remain compatible. */ - -#define PCRE_EXTRA_STUDY_DATA 0x0001 -#define PCRE_EXTRA_MATCH_LIMIT 0x0002 -#define PCRE_EXTRA_CALLOUT_DATA 0x0004 -#define PCRE_EXTRA_TABLES 0x0008 -#define PCRE_EXTRA_MATCH_LIMIT_RECURSION 0x0010 -#define PCRE_EXTRA_MARK 0x0020 -#define PCRE_EXTRA_EXECUTABLE_JIT 0x0040 - -/* Types */ - -struct real_pcre; /* declaration; the definition is private */ -typedef struct real_pcre pcre; - -struct real_pcre16; /* declaration; the definition is private */ -typedef struct real_pcre16 pcre16; - -struct real_pcre32; /* declaration; the definition is private */ -typedef struct real_pcre32 pcre32; - -struct real_pcre_jit_stack; /* declaration; the definition is private */ -typedef struct real_pcre_jit_stack pcre_jit_stack; - -struct real_pcre16_jit_stack; /* declaration; the definition is private */ -typedef struct real_pcre16_jit_stack pcre16_jit_stack; - -struct real_pcre32_jit_stack; /* declaration; the definition is private */ -typedef struct real_pcre32_jit_stack pcre32_jit_stack; - -/* If PCRE is compiled with 16 bit character support, PCRE_UCHAR16 must contain -a 16 bit wide signed data type. Otherwise it can be a dummy data type since -pcre16 functions are not implemented. There is a check for this in pcre_internal.h. */ -#ifndef PCRE_UCHAR16 -#define PCRE_UCHAR16 unsigned short -#endif - -#ifndef PCRE_SPTR16 -#define PCRE_SPTR16 const PCRE_UCHAR16 * -#endif - -/* If PCRE is compiled with 32 bit character support, PCRE_UCHAR32 must contain -a 32 bit wide signed data type. Otherwise it can be a dummy data type since -pcre32 functions are not implemented. There is a check for this in pcre_internal.h. */ -#ifndef PCRE_UCHAR32 -#define PCRE_UCHAR32 unsigned int -#endif - -#ifndef PCRE_SPTR32 -#define PCRE_SPTR32 const PCRE_UCHAR32 * -#endif - -/* When PCRE is compiled as a C++ library, the subject pointer type can be -replaced with a custom type. For conventional use, the public interface is a -const char *. */ - -#ifndef PCRE_SPTR -#define PCRE_SPTR const char * -#endif - -/* The structure for passing additional data to pcre_exec(). This is defined in -such as way as to be extensible. Always add new fields at the end, in order to -remain compatible. */ - -typedef struct pcre_extra { - unsigned long int flags; /* Bits for which fields are set */ - void *study_data; /* Opaque data from pcre_study() */ - unsigned long int match_limit; /* Maximum number of calls to match() */ - void *callout_data; /* Data passed back in callouts */ - const unsigned char *tables; /* Pointer to character tables */ - unsigned long int match_limit_recursion; /* Max recursive calls to match() */ - unsigned char **mark; /* For passing back a mark pointer */ - void *executable_jit; /* Contains a pointer to a compiled jit code */ -} pcre_extra; - -/* Same structure as above, but with 16 bit char pointers. */ - -typedef struct pcre16_extra { - unsigned long int flags; /* Bits for which fields are set */ - void *study_data; /* Opaque data from pcre_study() */ - unsigned long int match_limit; /* Maximum number of calls to match() */ - void *callout_data; /* Data passed back in callouts */ - const unsigned char *tables; /* Pointer to character tables */ - unsigned long int match_limit_recursion; /* Max recursive calls to match() */ - PCRE_UCHAR16 **mark; /* For passing back a mark pointer */ - void *executable_jit; /* Contains a pointer to a compiled jit code */ -} pcre16_extra; - -/* Same structure as above, but with 32 bit char pointers. */ - -typedef struct pcre32_extra { - unsigned long int flags; /* Bits for which fields are set */ - void *study_data; /* Opaque data from pcre_study() */ - unsigned long int match_limit; /* Maximum number of calls to match() */ - void *callout_data; /* Data passed back in callouts */ - const unsigned char *tables; /* Pointer to character tables */ - unsigned long int match_limit_recursion; /* Max recursive calls to match() */ - PCRE_UCHAR32 **mark; /* For passing back a mark pointer */ - void *executable_jit; /* Contains a pointer to a compiled jit code */ -} pcre32_extra; - -/* The structure for passing out data via the pcre_callout_function. We use a -structure so that new fields can be added on the end in future versions, -without changing the API of the function, thereby allowing old clients to work -without modification. */ - -typedef struct pcre_callout_block { - int version; /* Identifies version of block */ - /* ------------------------ Version 0 ------------------------------- */ - int callout_number; /* Number compiled into pattern */ - int *offset_vector; /* The offset vector */ - PCRE_SPTR subject; /* The subject being matched */ - int subject_length; /* The length of the subject */ - int start_match; /* Offset to start of this match attempt */ - int current_position; /* Where we currently are in the subject */ - int capture_top; /* Max current capture */ - int capture_last; /* Most recently closed capture */ - void *callout_data; /* Data passed in with the call */ - /* ------------------- Added for Version 1 -------------------------- */ - int pattern_position; /* Offset to next item in the pattern */ - int next_item_length; /* Length of next item in the pattern */ - /* ------------------- Added for Version 2 -------------------------- */ - const unsigned char *mark; /* Pointer to current mark or NULL */ - /* ------------------------------------------------------------------ */ -} pcre_callout_block; - -/* Same structure as above, but with 16 bit char pointers. */ - -typedef struct pcre16_callout_block { - int version; /* Identifies version of block */ - /* ------------------------ Version 0 ------------------------------- */ - int callout_number; /* Number compiled into pattern */ - int *offset_vector; /* The offset vector */ - PCRE_SPTR16 subject; /* The subject being matched */ - int subject_length; /* The length of the subject */ - int start_match; /* Offset to start of this match attempt */ - int current_position; /* Where we currently are in the subject */ - int capture_top; /* Max current capture */ - int capture_last; /* Most recently closed capture */ - void *callout_data; /* Data passed in with the call */ - /* ------------------- Added for Version 1 -------------------------- */ - int pattern_position; /* Offset to next item in the pattern */ - int next_item_length; /* Length of next item in the pattern */ - /* ------------------- Added for Version 2 -------------------------- */ - const PCRE_UCHAR16 *mark; /* Pointer to current mark or NULL */ - /* ------------------------------------------------------------------ */ -} pcre16_callout_block; - -/* Same structure as above, but with 32 bit char pointers. */ - -typedef struct pcre32_callout_block { - int version; /* Identifies version of block */ - /* ------------------------ Version 0 ------------------------------- */ - int callout_number; /* Number compiled into pattern */ - int *offset_vector; /* The offset vector */ - PCRE_SPTR32 subject; /* The subject being matched */ - int subject_length; /* The length of the subject */ - int start_match; /* Offset to start of this match attempt */ - int current_position; /* Where we currently are in the subject */ - int capture_top; /* Max current capture */ - int capture_last; /* Most recently closed capture */ - void *callout_data; /* Data passed in with the call */ - /* ------------------- Added for Version 1 -------------------------- */ - int pattern_position; /* Offset to next item in the pattern */ - int next_item_length; /* Length of next item in the pattern */ - /* ------------------- Added for Version 2 -------------------------- */ - const PCRE_UCHAR32 *mark; /* Pointer to current mark or NULL */ - /* ------------------------------------------------------------------ */ -} pcre32_callout_block; - -/* Indirection for store get and free functions. These can be set to -alternative malloc/free functions if required. Special ones are used in the -non-recursive case for "frames". There is also an optional callout function -that is triggered by the (?) regex item. For Virtual Pascal, these definitions -have to take another form. */ - -#ifndef VPCOMPAT -PCRE_EXP_DECL void *(*pcre_malloc)(size_t); -PCRE_EXP_DECL void (*pcre_free)(void *); -PCRE_EXP_DECL void *(*pcre_stack_malloc)(size_t); -PCRE_EXP_DECL void (*pcre_stack_free)(void *); -PCRE_EXP_DECL int (*pcre_callout)(pcre_callout_block *); -PCRE_EXP_DECL int (*pcre_stack_guard)(void); - -PCRE_EXP_DECL void *(*pcre16_malloc)(size_t); -PCRE_EXP_DECL void (*pcre16_free)(void *); -PCRE_EXP_DECL void *(*pcre16_stack_malloc)(size_t); -PCRE_EXP_DECL void (*pcre16_stack_free)(void *); -PCRE_EXP_DECL int (*pcre16_callout)(pcre16_callout_block *); -PCRE_EXP_DECL int (*pcre16_stack_guard)(void); - -PCRE_EXP_DECL void *(*pcre32_malloc)(size_t); -PCRE_EXP_DECL void (*pcre32_free)(void *); -PCRE_EXP_DECL void *(*pcre32_stack_malloc)(size_t); -PCRE_EXP_DECL void (*pcre32_stack_free)(void *); -PCRE_EXP_DECL int (*pcre32_callout)(pcre32_callout_block *); -PCRE_EXP_DECL int (*pcre32_stack_guard)(void); -#else /* VPCOMPAT */ -PCRE_EXP_DECL void *pcre_malloc(size_t); -PCRE_EXP_DECL void pcre_free(void *); -PCRE_EXP_DECL void *pcre_stack_malloc(size_t); -PCRE_EXP_DECL void pcre_stack_free(void *); -PCRE_EXP_DECL int pcre_callout(pcre_callout_block *); -PCRE_EXP_DECL int pcre_stack_guard(void); - -PCRE_EXP_DECL void *pcre16_malloc(size_t); -PCRE_EXP_DECL void pcre16_free(void *); -PCRE_EXP_DECL void *pcre16_stack_malloc(size_t); -PCRE_EXP_DECL void pcre16_stack_free(void *); -PCRE_EXP_DECL int pcre16_callout(pcre16_callout_block *); -PCRE_EXP_DECL int pcre16_stack_guard(void); - -PCRE_EXP_DECL void *pcre32_malloc(size_t); -PCRE_EXP_DECL void pcre32_free(void *); -PCRE_EXP_DECL void *pcre32_stack_malloc(size_t); -PCRE_EXP_DECL void pcre32_stack_free(void *); -PCRE_EXP_DECL int pcre32_callout(pcre32_callout_block *); -PCRE_EXP_DECL int pcre32_stack_guard(void); -#endif /* VPCOMPAT */ - -/* User defined callback which provides a stack just before the match starts. */ - -typedef pcre_jit_stack *(*pcre_jit_callback)(void *); -typedef pcre16_jit_stack *(*pcre16_jit_callback)(void *); -typedef pcre32_jit_stack *(*pcre32_jit_callback)(void *); - -/* Exported PCRE functions */ - -PCRE_EXP_DECL pcre *pcre_compile(const char *, int, const char **, int *, - const unsigned char *); -PCRE_EXP_DECL pcre16 *pcre16_compile(PCRE_SPTR16, int, const char **, int *, - const unsigned char *); -PCRE_EXP_DECL pcre32 *pcre32_compile(PCRE_SPTR32, int, const char **, int *, - const unsigned char *); -PCRE_EXP_DECL pcre *pcre_compile2(const char *, int, int *, const char **, - int *, const unsigned char *); -PCRE_EXP_DECL pcre16 *pcre16_compile2(PCRE_SPTR16, int, int *, const char **, - int *, const unsigned char *); -PCRE_EXP_DECL pcre32 *pcre32_compile2(PCRE_SPTR32, int, int *, const char **, - int *, const unsigned char *); -PCRE_EXP_DECL int pcre_config(int, void *); -PCRE_EXP_DECL int pcre16_config(int, void *); -PCRE_EXP_DECL int pcre32_config(int, void *); -PCRE_EXP_DECL int pcre_copy_named_substring(const pcre *, const char *, - int *, int, const char *, char *, int); -PCRE_EXP_DECL int pcre16_copy_named_substring(const pcre16 *, PCRE_SPTR16, - int *, int, PCRE_SPTR16, PCRE_UCHAR16 *, int); -PCRE_EXP_DECL int pcre32_copy_named_substring(const pcre32 *, PCRE_SPTR32, - int *, int, PCRE_SPTR32, PCRE_UCHAR32 *, int); -PCRE_EXP_DECL int pcre_copy_substring(const char *, int *, int, int, - char *, int); -PCRE_EXP_DECL int pcre16_copy_substring(PCRE_SPTR16, int *, int, int, - PCRE_UCHAR16 *, int); -PCRE_EXP_DECL int pcre32_copy_substring(PCRE_SPTR32, int *, int, int, - PCRE_UCHAR32 *, int); -PCRE_EXP_DECL int pcre_dfa_exec(const pcre *, const pcre_extra *, - const char *, int, int, int, int *, int , int *, int); -PCRE_EXP_DECL int pcre16_dfa_exec(const pcre16 *, const pcre16_extra *, - PCRE_SPTR16, int, int, int, int *, int , int *, int); -PCRE_EXP_DECL int pcre32_dfa_exec(const pcre32 *, const pcre32_extra *, - PCRE_SPTR32, int, int, int, int *, int , int *, int); -PCRE_EXP_DECL int pcre_exec(const pcre *, const pcre_extra *, PCRE_SPTR, - int, int, int, int *, int); -PCRE_EXP_DECL int pcre16_exec(const pcre16 *, const pcre16_extra *, - PCRE_SPTR16, int, int, int, int *, int); -PCRE_EXP_DECL int pcre32_exec(const pcre32 *, const pcre32_extra *, - PCRE_SPTR32, int, int, int, int *, int); -PCRE_EXP_DECL int pcre_jit_exec(const pcre *, const pcre_extra *, - PCRE_SPTR, int, int, int, int *, int, - pcre_jit_stack *); -PCRE_EXP_DECL int pcre16_jit_exec(const pcre16 *, const pcre16_extra *, - PCRE_SPTR16, int, int, int, int *, int, - pcre16_jit_stack *); -PCRE_EXP_DECL int pcre32_jit_exec(const pcre32 *, const pcre32_extra *, - PCRE_SPTR32, int, int, int, int *, int, - pcre32_jit_stack *); -PCRE_EXP_DECL void pcre_free_substring(const char *); -PCRE_EXP_DECL void pcre16_free_substring(PCRE_SPTR16); -PCRE_EXP_DECL void pcre32_free_substring(PCRE_SPTR32); -PCRE_EXP_DECL void pcre_free_substring_list(const char **); -PCRE_EXP_DECL void pcre16_free_substring_list(PCRE_SPTR16 *); -PCRE_EXP_DECL void pcre32_free_substring_list(PCRE_SPTR32 *); -PCRE_EXP_DECL int pcre_fullinfo(const pcre *, const pcre_extra *, int, - void *); -PCRE_EXP_DECL int pcre16_fullinfo(const pcre16 *, const pcre16_extra *, int, - void *); -PCRE_EXP_DECL int pcre32_fullinfo(const pcre32 *, const pcre32_extra *, int, - void *); -PCRE_EXP_DECL int pcre_get_named_substring(const pcre *, const char *, - int *, int, const char *, const char **); -PCRE_EXP_DECL int pcre16_get_named_substring(const pcre16 *, PCRE_SPTR16, - int *, int, PCRE_SPTR16, PCRE_SPTR16 *); -PCRE_EXP_DECL int pcre32_get_named_substring(const pcre32 *, PCRE_SPTR32, - int *, int, PCRE_SPTR32, PCRE_SPTR32 *); -PCRE_EXP_DECL int pcre_get_stringnumber(const pcre *, const char *); -PCRE_EXP_DECL int pcre16_get_stringnumber(const pcre16 *, PCRE_SPTR16); -PCRE_EXP_DECL int pcre32_get_stringnumber(const pcre32 *, PCRE_SPTR32); -PCRE_EXP_DECL int pcre_get_stringtable_entries(const pcre *, const char *, - char **, char **); -PCRE_EXP_DECL int pcre16_get_stringtable_entries(const pcre16 *, PCRE_SPTR16, - PCRE_UCHAR16 **, PCRE_UCHAR16 **); -PCRE_EXP_DECL int pcre32_get_stringtable_entries(const pcre32 *, PCRE_SPTR32, - PCRE_UCHAR32 **, PCRE_UCHAR32 **); -PCRE_EXP_DECL int pcre_get_substring(const char *, int *, int, int, - const char **); -PCRE_EXP_DECL int pcre16_get_substring(PCRE_SPTR16, int *, int, int, - PCRE_SPTR16 *); -PCRE_EXP_DECL int pcre32_get_substring(PCRE_SPTR32, int *, int, int, - PCRE_SPTR32 *); -PCRE_EXP_DECL int pcre_get_substring_list(const char *, int *, int, - const char ***); -PCRE_EXP_DECL int pcre16_get_substring_list(PCRE_SPTR16, int *, int, - PCRE_SPTR16 **); -PCRE_EXP_DECL int pcre32_get_substring_list(PCRE_SPTR32, int *, int, - PCRE_SPTR32 **); -PCRE_EXP_DECL const unsigned char *pcre_maketables(void); -PCRE_EXP_DECL const unsigned char *pcre16_maketables(void); -PCRE_EXP_DECL const unsigned char *pcre32_maketables(void); -PCRE_EXP_DECL int pcre_refcount(pcre *, int); -PCRE_EXP_DECL int pcre16_refcount(pcre16 *, int); -PCRE_EXP_DECL int pcre32_refcount(pcre32 *, int); -PCRE_EXP_DECL pcre_extra *pcre_study(const pcre *, int, const char **); -PCRE_EXP_DECL pcre16_extra *pcre16_study(const pcre16 *, int, const char **); -PCRE_EXP_DECL pcre32_extra *pcre32_study(const pcre32 *, int, const char **); -PCRE_EXP_DECL void pcre_free_study(pcre_extra *); -PCRE_EXP_DECL void pcre16_free_study(pcre16_extra *); -PCRE_EXP_DECL void pcre32_free_study(pcre32_extra *); -PCRE_EXP_DECL const char *pcre_version(void); -PCRE_EXP_DECL const char *pcre16_version(void); -PCRE_EXP_DECL const char *pcre32_version(void); - -/* Utility functions for byte order swaps. */ -PCRE_EXP_DECL int pcre_pattern_to_host_byte_order(pcre *, pcre_extra *, - const unsigned char *); -PCRE_EXP_DECL int pcre16_pattern_to_host_byte_order(pcre16 *, pcre16_extra *, - const unsigned char *); -PCRE_EXP_DECL int pcre32_pattern_to_host_byte_order(pcre32 *, pcre32_extra *, - const unsigned char *); -PCRE_EXP_DECL int pcre16_utf16_to_host_byte_order(PCRE_UCHAR16 *, - PCRE_SPTR16, int, int *, int); -PCRE_EXP_DECL int pcre32_utf32_to_host_byte_order(PCRE_UCHAR32 *, - PCRE_SPTR32, int, int *, int); - -/* JIT compiler related functions. */ - -PCRE_EXP_DECL pcre_jit_stack *pcre_jit_stack_alloc(int, int); -PCRE_EXP_DECL pcre16_jit_stack *pcre16_jit_stack_alloc(int, int); -PCRE_EXP_DECL pcre32_jit_stack *pcre32_jit_stack_alloc(int, int); -PCRE_EXP_DECL void pcre_jit_stack_free(pcre_jit_stack *); -PCRE_EXP_DECL void pcre16_jit_stack_free(pcre16_jit_stack *); -PCRE_EXP_DECL void pcre32_jit_stack_free(pcre32_jit_stack *); -PCRE_EXP_DECL void pcre_assign_jit_stack(pcre_extra *, - pcre_jit_callback, void *); -PCRE_EXP_DECL void pcre16_assign_jit_stack(pcre16_extra *, - pcre16_jit_callback, void *); -PCRE_EXP_DECL void pcre32_assign_jit_stack(pcre32_extra *, - pcre32_jit_callback, void *); -PCRE_EXP_DECL void pcre_jit_free_unused_memory(void); -PCRE_EXP_DECL void pcre16_jit_free_unused_memory(void); -PCRE_EXP_DECL void pcre32_jit_free_unused_memory(void); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* End of pcre.h */ diff --git a/src/third_party/pcre-8.39/pcre16_byte_order.c b/src/third_party/pcre-8.39/pcre16_byte_order.c deleted file mode 100644 index 11d2973a3db..00000000000 --- a/src/third_party/pcre-8.39/pcre16_byte_order.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_byte_order.c" - -/* End of pcre16_byte_order.c */ diff --git a/src/third_party/pcre-8.39/pcre16_chartables.c b/src/third_party/pcre-8.39/pcre16_chartables.c deleted file mode 100644 index 7c0ff35f5e0..00000000000 --- a/src/third_party/pcre-8.39/pcre16_chartables.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_chartables.c" - -/* End of pcre16_chartables.c */ diff --git a/src/third_party/pcre-8.39/pcre16_compile.c b/src/third_party/pcre-8.39/pcre16_compile.c deleted file mode 100644 index e499b670877..00000000000 --- a/src/third_party/pcre-8.39/pcre16_compile.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_compile.c" - -/* End of pcre16_compile.c */ diff --git a/src/third_party/pcre-8.39/pcre16_config.c b/src/third_party/pcre-8.39/pcre16_config.c deleted file mode 100644 index b52138764f6..00000000000 --- a/src/third_party/pcre-8.39/pcre16_config.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_config.c" - -/* End of pcre16_config.c */ diff --git a/src/third_party/pcre-8.39/pcre16_dfa_exec.c b/src/third_party/pcre-8.39/pcre16_dfa_exec.c deleted file mode 100644 index 2ba740e972b..00000000000 --- a/src/third_party/pcre-8.39/pcre16_dfa_exec.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_dfa_exec.c" - -/* End of pcre16_dfa_exec.c */ diff --git a/src/third_party/pcre-8.39/pcre16_exec.c b/src/third_party/pcre-8.39/pcre16_exec.c deleted file mode 100644 index 7417b1770c6..00000000000 --- a/src/third_party/pcre-8.39/pcre16_exec.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_exec.c" - -/* End of pcre16_exec.c */ diff --git a/src/third_party/pcre-8.39/pcre16_fullinfo.c b/src/third_party/pcre-8.39/pcre16_fullinfo.c deleted file mode 100644 index 544dca6ed5c..00000000000 --- a/src/third_party/pcre-8.39/pcre16_fullinfo.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_fullinfo.c" - -/* End of pcre16_fullinfo.c */ diff --git a/src/third_party/pcre-8.39/pcre16_get.c b/src/third_party/pcre-8.39/pcre16_get.c deleted file mode 100644 index 3ded08c622c..00000000000 --- a/src/third_party/pcre-8.39/pcre16_get.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_get.c" - -/* End of pcre16_get.c */ diff --git a/src/third_party/pcre-8.39/pcre16_globals.c b/src/third_party/pcre-8.39/pcre16_globals.c deleted file mode 100644 index a136b3d8c22..00000000000 --- a/src/third_party/pcre-8.39/pcre16_globals.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_globals.c" - -/* End of pcre16_globals.c */ diff --git a/src/third_party/pcre-8.39/pcre16_jit_compile.c b/src/third_party/pcre-8.39/pcre16_jit_compile.c deleted file mode 100644 index ab0cacd7646..00000000000 --- a/src/third_party/pcre-8.39/pcre16_jit_compile.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_jit_compile.c" - -/* End of pcre16_jit_compile.c */ diff --git a/src/third_party/pcre-8.39/pcre16_maketables.c b/src/third_party/pcre-8.39/pcre16_maketables.c deleted file mode 100644 index b1cd1c579d6..00000000000 --- a/src/third_party/pcre-8.39/pcre16_maketables.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_maketables.c" - -/* End of pcre16_maketables.c */ diff --git a/src/third_party/pcre-8.39/pcre16_newline.c b/src/third_party/pcre-8.39/pcre16_newline.c deleted file mode 100644 index 7fe201400f5..00000000000 --- a/src/third_party/pcre-8.39/pcre16_newline.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_newline.c" - -/* End of pcre16_newline.c */ diff --git a/src/third_party/pcre-8.39/pcre16_ord2utf16.c b/src/third_party/pcre-8.39/pcre16_ord2utf16.c deleted file mode 100644 index 8e2ce5ea6c5..00000000000 --- a/src/third_party/pcre-8.39/pcre16_ord2utf16.c +++ /dev/null @@ -1,90 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This file contains a private PCRE function that converts an ordinal -character value into a UTF16 string. */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_internal.h" - -/************************************************* -* Convert character value to UTF-16 * -*************************************************/ - -/* This function takes an integer value in the range 0 - 0x10ffff -and encodes it as a UTF-16 character in 1 to 2 pcre_uchars. - -Arguments: - cvalue the character value - buffer pointer to buffer for result - at least 2 pcre_uchars long - -Returns: number of characters placed in the buffer -*/ - -unsigned int -PRIV(ord2utf)(pcre_uint32 cvalue, pcre_uchar *buffer) -{ -#ifdef SUPPORT_UTF - -if (cvalue <= 0xffff) - { - *buffer = (pcre_uchar)cvalue; - return 1; - } - -cvalue -= 0x10000; -*buffer++ = 0xd800 | (cvalue >> 10); -*buffer = 0xdc00 | (cvalue & 0x3ff); -return 2; - -#else /* SUPPORT_UTF */ -(void)(cvalue); /* Keep compiler happy; this function won't ever be */ -(void)(buffer); /* called when SUPPORT_UTF is not defined. */ -return 0; -#endif /* SUPPORT_UTF */ -} - -/* End of pcre16_ord2utf16.c */ diff --git a/src/third_party/pcre-8.39/pcre16_printint.c b/src/third_party/pcre-8.39/pcre16_printint.c deleted file mode 100644 index 33d8c340200..00000000000 --- a/src/third_party/pcre-8.39/pcre16_printint.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_printint.c" - -/* End of pcre16_printint.c */ diff --git a/src/third_party/pcre-8.39/pcre16_refcount.c b/src/third_party/pcre-8.39/pcre16_refcount.c deleted file mode 100644 index d3d15439737..00000000000 --- a/src/third_party/pcre-8.39/pcre16_refcount.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_refcount.c" - -/* End of pcre16_refcount.c */ diff --git a/src/third_party/pcre-8.39/pcre16_string_utils.c b/src/third_party/pcre-8.39/pcre16_string_utils.c deleted file mode 100644 index 382c40799fb..00000000000 --- a/src/third_party/pcre-8.39/pcre16_string_utils.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_string_utils.c" - -/* End of pcre16_string_utils.c */ diff --git a/src/third_party/pcre-8.39/pcre16_study.c b/src/third_party/pcre-8.39/pcre16_study.c deleted file mode 100644 index f87de081fc4..00000000000 --- a/src/third_party/pcre-8.39/pcre16_study.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_study.c" - -/* End of pcre16_study.c */ diff --git a/src/third_party/pcre-8.39/pcre16_tables.c b/src/third_party/pcre-8.39/pcre16_tables.c deleted file mode 100644 index d84297093a4..00000000000 --- a/src/third_party/pcre-8.39/pcre16_tables.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_tables.c" - -/* End of pcre16_tables.c */ diff --git a/src/third_party/pcre-8.39/pcre16_ucd.c b/src/third_party/pcre-8.39/pcre16_ucd.c deleted file mode 100644 index ee23439a013..00000000000 --- a/src/third_party/pcre-8.39/pcre16_ucd.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_ucd.c" - -/* End of pcre16_ucd.c */ diff --git a/src/third_party/pcre-8.39/pcre16_utf16_utils.c b/src/third_party/pcre-8.39/pcre16_utf16_utils.c deleted file mode 100644 index 49ced0c0b1c..00000000000 --- a/src/third_party/pcre-8.39/pcre16_utf16_utils.c +++ /dev/null @@ -1,130 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This module contains a function for converting any UTF-16 character -strings to host byte order. */ - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_internal.h" - -/************************************************* -* Convert any UTF-16 string to host byte order * -*************************************************/ - -/* This function takes an UTF-16 string and converts -it to host byte order. The length can be explicitly set, -or automatically detected for zero terminated strings. -BOMs can be kept or discarded during the conversion. -Conversion can be done in place (output == input). - -Arguments: - output the output buffer, its size must be greater - or equal than the input string - input any UTF-16 string - length the number of 16-bit units in the input string - can be less than zero for zero terminated strings - host_byte_order - A non-zero value means the input is in host byte - order, which can be dynamically changed by BOMs later. - Initially it contains the starting byte order and returns - with the last byte order so it can be used for stream - processing. It can be NULL, which set the host byte - order mode by default. - keep_boms for a non-zero value, the BOM (0xfeff) characters - are copied as well - -Returns: the number of 16-bit units placed into the output buffer, - including the zero-terminator -*/ - -int -pcre16_utf16_to_host_byte_order(PCRE_UCHAR16 *output, PCRE_SPTR16 input, - int length, int *host_byte_order, int keep_boms) -{ -#ifdef SUPPORT_UTF -/* This function converts any UTF-16 string to host byte order and optionally -removes any Byte Order Marks (BOMS). Returns with the remainig length. */ -int host_bo = host_byte_order != NULL ? *host_byte_order : 1; -pcre_uchar *optr = (pcre_uchar *)output; -const pcre_uchar *iptr = (const pcre_uchar *)input; -const pcre_uchar *end; -/* The c variable must be unsigned. */ -register pcre_uchar c; - -if (length < 0) - length = STRLEN_UC(iptr) + 1; -end = iptr + length; - -while (iptr < end) - { - c = *iptr++; - if (c == 0xfeff || c == 0xfffe) - { - /* Detecting the byte order of the machine is unnecessary, it is - enough to know that the UTF-16 string has the same byte order or not. */ - host_bo = c == 0xfeff; - if (keep_boms != 0) - *optr++ = 0xfeff; - else - length--; - } - else - *optr++ = host_bo ? c : ((c >> 8) | (c << 8)); /* Flip bytes if needed. */ - } -if (host_byte_order != NULL) - *host_byte_order = host_bo; - -#else /* Not SUPPORT_UTF */ -(void)(output); /* Keep picky compilers happy */ -(void)(input); -(void)(keep_boms); -(void)(host_byte_order); -#endif /* SUPPORT_UTF */ -return length; -} - -/* End of pcre16_utf16_utils.c */ diff --git a/src/third_party/pcre-8.39/pcre16_valid_utf16.c b/src/third_party/pcre-8.39/pcre16_valid_utf16.c deleted file mode 100644 index 09076539d09..00000000000 --- a/src/third_party/pcre-8.39/pcre16_valid_utf16.c +++ /dev/null @@ -1,137 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2013 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This module contains an internal function for validating UTF-16 character -strings. */ - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_internal.h" - - -/************************************************* -* Validate a UTF-16 string * -*************************************************/ - -/* This function is called (optionally) at the start of compile or match, to -check that a supposed UTF-16 string is actually valid. The early check means -that subsequent code can assume it is dealing with a valid string. The check -can be turned off for maximum performance, but the consequences of supplying an -invalid string are then undefined. - -From release 8.21 more information about the details of the error are passed -back in the returned value: - -PCRE_UTF16_ERR0 No error -PCRE_UTF16_ERR1 Missing low surrogate at the end of the string -PCRE_UTF16_ERR2 Invalid low surrogate -PCRE_UTF16_ERR3 Isolated low surrogate -PCRE_UTF16_ERR4 Unused (was non-character) - -Arguments: - string points to the string - length length of string, or -1 if the string is zero-terminated - errp pointer to an error position offset variable - -Returns: = 0 if the string is a valid UTF-16 string - > 0 otherwise, setting the offset of the bad character -*/ - -int -PRIV(valid_utf)(PCRE_PUCHAR string, int length, int *erroroffset) -{ -#ifdef SUPPORT_UTF -register PCRE_PUCHAR p; -register pcre_uint32 c; - -if (length < 0) - { - for (p = string; *p != 0; p++); - length = p - string; - } - -for (p = string; length-- > 0; p++) - { - c = *p; - - if ((c & 0xf800) != 0xd800) - { - /* Normal UTF-16 code point. Neither high nor low surrogate. */ - } - else if ((c & 0x0400) == 0) - { - /* High surrogate. Must be a followed by a low surrogate. */ - if (length == 0) - { - *erroroffset = p - string; - return PCRE_UTF16_ERR1; - } - p++; - length--; - if ((*p & 0xfc00) != 0xdc00) - { - *erroroffset = p - string; - return PCRE_UTF16_ERR2; - } - } - else - { - /* Isolated low surrogate. Always an error. */ - *erroroffset = p - string; - return PCRE_UTF16_ERR3; - } - } - -#else /* SUPPORT_UTF */ -(void)(string); /* Keep picky compilers happy */ -(void)(length); -(void)(erroroffset); -#endif /* SUPPORT_UTF */ - -return PCRE_UTF16_ERR0; /* This indicates success */ -} - -/* End of pcre16_valid_utf16.c */ diff --git a/src/third_party/pcre-8.39/pcre16_version.c b/src/third_party/pcre-8.39/pcre16_version.c deleted file mode 100644 index e991b1a8cfd..00000000000 --- a/src/third_party/pcre-8.39/pcre16_version.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_version.c" - -/* End of pcre16_version.c */ diff --git a/src/third_party/pcre-8.39/pcre16_xclass.c b/src/third_party/pcre-8.39/pcre16_xclass.c deleted file mode 100644 index 5aac2a36c68..00000000000 --- a/src/third_party/pcre-8.39/pcre16_xclass.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 16 bit character support. */ -#define COMPILE_PCRE16 - -#include "pcre_xclass.c" - -/* End of pcre16_xclass.c */ diff --git a/src/third_party/pcre-8.39/pcre32_byte_order.c b/src/third_party/pcre-8.39/pcre32_byte_order.c deleted file mode 100644 index 9cf5362730a..00000000000 --- a/src/third_party/pcre-8.39/pcre32_byte_order.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_byte_order.c" - -/* End of pcre32_byte_order.c */ diff --git a/src/third_party/pcre-8.39/pcre32_chartables.c b/src/third_party/pcre-8.39/pcre32_chartables.c deleted file mode 100644 index b5d8c23dbf1..00000000000 --- a/src/third_party/pcre-8.39/pcre32_chartables.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_chartables.c" - -/* End of pcre32_chartables.c */ diff --git a/src/third_party/pcre-8.39/pcre32_compile.c b/src/third_party/pcre-8.39/pcre32_compile.c deleted file mode 100644 index d781eb377e0..00000000000 --- a/src/third_party/pcre-8.39/pcre32_compile.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_compile.c" - -/* End of pcre32_compile.c */ diff --git a/src/third_party/pcre-8.39/pcre32_config.c b/src/third_party/pcre-8.39/pcre32_config.c deleted file mode 100644 index d63f3e9ea23..00000000000 --- a/src/third_party/pcre-8.39/pcre32_config.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_config.c" - -/* End of pcre32_config.c */ diff --git a/src/third_party/pcre-8.39/pcre32_dfa_exec.c b/src/third_party/pcre-8.39/pcre32_dfa_exec.c deleted file mode 100644 index b0bfd34f04d..00000000000 --- a/src/third_party/pcre-8.39/pcre32_dfa_exec.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_dfa_exec.c" - -/* End of pcre32_dfa_exec.c */ diff --git a/src/third_party/pcre-8.39/pcre32_exec.c b/src/third_party/pcre-8.39/pcre32_exec.c deleted file mode 100644 index 8170ed77d35..00000000000 --- a/src/third_party/pcre-8.39/pcre32_exec.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_exec.c" - -/* End of pcre32_exec.c */ diff --git a/src/third_party/pcre-8.39/pcre32_fullinfo.c b/src/third_party/pcre-8.39/pcre32_fullinfo.c deleted file mode 100644 index 6ecc5209a08..00000000000 --- a/src/third_party/pcre-8.39/pcre32_fullinfo.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_fullinfo.c" - -/* End of pcre32_fullinfo.c */ diff --git a/src/third_party/pcre-8.39/pcre32_get.c b/src/third_party/pcre-8.39/pcre32_get.c deleted file mode 100644 index d35deee0cd4..00000000000 --- a/src/third_party/pcre-8.39/pcre32_get.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_get.c" - -/* End of pcre32_get.c */ diff --git a/src/third_party/pcre-8.39/pcre32_globals.c b/src/third_party/pcre-8.39/pcre32_globals.c deleted file mode 100644 index 32e0914ca6d..00000000000 --- a/src/third_party/pcre-8.39/pcre32_globals.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_globals.c" - -/* End of pcre32_globals.c */ diff --git a/src/third_party/pcre-8.39/pcre32_jit_compile.c b/src/third_party/pcre-8.39/pcre32_jit_compile.c deleted file mode 100644 index 2e7c6f97c96..00000000000 --- a/src/third_party/pcre-8.39/pcre32_jit_compile.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_jit_compile.c" - -/* End of pcre32_jit_compile.c */ diff --git a/src/third_party/pcre-8.39/pcre32_maketables.c b/src/third_party/pcre-8.39/pcre32_maketables.c deleted file mode 100644 index 5d1b1c64c96..00000000000 --- a/src/third_party/pcre-8.39/pcre32_maketables.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_maketables.c" - -/* End of pcre32_maketables.c */ diff --git a/src/third_party/pcre-8.39/pcre32_newline.c b/src/third_party/pcre-8.39/pcre32_newline.c deleted file mode 100644 index 7f8d5360cdc..00000000000 --- a/src/third_party/pcre-8.39/pcre32_newline.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_newline.c" - -/* End of pcre32_newline.c */ diff --git a/src/third_party/pcre-8.39/pcre32_ord2utf32.c b/src/third_party/pcre-8.39/pcre32_ord2utf32.c deleted file mode 100644 index 606bcb3d7ba..00000000000 --- a/src/third_party/pcre-8.39/pcre32_ord2utf32.c +++ /dev/null @@ -1,82 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This file contains a private PCRE function that converts an ordinal -character value into a UTF32 string. */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_internal.h" - -/************************************************* -* Convert character value to UTF-32 * -*************************************************/ - -/* This function takes an integer value in the range 0 - 0x10ffff -and encodes it as a UTF-32 character in 1 pcre_uchars. - -Arguments: - cvalue the character value - buffer pointer to buffer for result - at least 1 pcre_uchars long - -Returns: number of characters placed in the buffer -*/ - -unsigned int -PRIV(ord2utf)(pcre_uint32 cvalue, pcre_uchar *buffer) -{ -#ifdef SUPPORT_UTF - -*buffer = (pcre_uchar)cvalue; -return 1; - -#else /* SUPPORT_UTF */ -(void)(cvalue); /* Keep compiler happy; this function won't ever be */ -(void)(buffer); /* called when SUPPORT_UTF is not defined. */ -return 0; -#endif /* SUPPORT_UTF */ -} - -/* End of pcre32_ord2utf32.c */ diff --git a/src/third_party/pcre-8.39/pcre32_printint.c b/src/third_party/pcre-8.39/pcre32_printint.c deleted file mode 100644 index f3fd7b25e2c..00000000000 --- a/src/third_party/pcre-8.39/pcre32_printint.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_printint.c" - -/* End of pcre32_printint.c */ diff --git a/src/third_party/pcre-8.39/pcre32_refcount.c b/src/third_party/pcre-8.39/pcre32_refcount.c deleted file mode 100644 index dbdf432d82a..00000000000 --- a/src/third_party/pcre-8.39/pcre32_refcount.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_refcount.c" - -/* End of pcre32_refcount.c */ diff --git a/src/third_party/pcre-8.39/pcre32_string_utils.c b/src/third_party/pcre-8.39/pcre32_string_utils.c deleted file mode 100644 index e37b3d4805f..00000000000 --- a/src/third_party/pcre-8.39/pcre32_string_utils.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_string_utils.c" - -/* End of pcre32_string_utils.c */ diff --git a/src/third_party/pcre-8.39/pcre32_study.c b/src/third_party/pcre-8.39/pcre32_study.c deleted file mode 100644 index d3a3afed791..00000000000 --- a/src/third_party/pcre-8.39/pcre32_study.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_study.c" - -/* End of pcre32_study.c */ diff --git a/src/third_party/pcre-8.39/pcre32_tables.c b/src/third_party/pcre-8.39/pcre32_tables.c deleted file mode 100644 index 3d94cca33a1..00000000000 --- a/src/third_party/pcre-8.39/pcre32_tables.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_tables.c" - -/* End of pcre32_tables.c */ diff --git a/src/third_party/pcre-8.39/pcre32_ucd.c b/src/third_party/pcre-8.39/pcre32_ucd.c deleted file mode 100644 index befe22d3435..00000000000 --- a/src/third_party/pcre-8.39/pcre32_ucd.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_ucd.c" - -/* End of pcre32_ucd.c */ diff --git a/src/third_party/pcre-8.39/pcre32_utf32_utils.c b/src/third_party/pcre-8.39/pcre32_utf32_utils.c deleted file mode 100644 index f844e237165..00000000000 --- a/src/third_party/pcre-8.39/pcre32_utf32_utils.c +++ /dev/null @@ -1,141 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This module contains a function for converting any UTF-32 character -strings to host byte order. */ - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_internal.h" - -#ifdef SUPPORT_UTF -static pcre_uint32 -swap_uint32(pcre_uint32 value) -{ -return ((value & 0x000000ff) << 24) | - ((value & 0x0000ff00) << 8) | - ((value & 0x00ff0000) >> 8) | - (value >> 24); -} -#endif - - -/************************************************* -* Convert any UTF-32 string to host byte order * -*************************************************/ - -/* This function takes an UTF-32 string and converts -it to host byte order. The length can be explicitly set, -or automatically detected for zero terminated strings. -BOMs can be kept or discarded during the conversion. -Conversion can be done in place (output == input). - -Arguments: - output the output buffer, its size must be greater - or equal than the input string - input any UTF-32 string - length the number of 32-bit units in the input string - can be less than zero for zero terminated strings - host_byte_order - A non-zero value means the input is in host byte - order, which can be dynamically changed by BOMs later. - Initially it contains the starting byte order and returns - with the last byte order so it can be used for stream - processing. It can be NULL, which set the host byte - order mode by default. - keep_boms for a non-zero value, the BOM (0xfeff) characters - are copied as well - -Returns: the number of 32-bit units placed into the output buffer, - including the zero-terminator -*/ - -int -pcre32_utf32_to_host_byte_order(PCRE_UCHAR32 *output, PCRE_SPTR32 input, - int length, int *host_byte_order, int keep_boms) -{ -#ifdef SUPPORT_UTF -/* This function converts any UTF-32 string to host byte order and optionally -removes any Byte Order Marks (BOMS). Returns with the remainig length. */ -int host_bo = host_byte_order != NULL ? *host_byte_order : 1; -pcre_uchar *optr = (pcre_uchar *)output; -const pcre_uchar *iptr = (const pcre_uchar *)input; -const pcre_uchar *end; -/* The c variable must be unsigned. */ -register pcre_uchar c; - -if (length < 0) - end = iptr + STRLEN_UC(iptr) + 1; -else - end = iptr + length; - -while (iptr < end) - { - c = *iptr++; - if (c == 0x0000feffu || c == 0xfffe0000u) - { - /* Detecting the byte order of the machine is unnecessary, it is - enough to know that the UTF-32 string has the same byte order or not. */ - host_bo = c == 0x0000feffu; - if (keep_boms != 0) - *optr++ = 0x0000feffu; - } - else - *optr++ = host_bo ? c : swap_uint32(c); - } -if (host_byte_order != NULL) - *host_byte_order = host_bo; - -#else /* SUPPORT_UTF */ -(void)(output); /* Keep picky compilers happy */ -(void)(input); -(void)(keep_boms); -(void)(host_byte_order); -#endif /* SUPPORT_UTF */ -return length; -} - -/* End of pcre32_utf32_utils.c */ diff --git a/src/third_party/pcre-8.39/pcre32_valid_utf32.c b/src/third_party/pcre-8.39/pcre32_valid_utf32.c deleted file mode 100644 index 94cda1a2c4c..00000000000 --- a/src/third_party/pcre-8.39/pcre32_valid_utf32.c +++ /dev/null @@ -1,124 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2013 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This module contains an internal function for validating UTF-32 character -strings. */ - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_internal.h" - -/************************************************* -* Validate a UTF-32 string * -*************************************************/ - -/* This function is called (optionally) at the start of compile or match, to -check that a supposed UTF-32 string is actually valid. The early check means -that subsequent code can assume it is dealing with a valid string. The check -can be turned off for maximum performance, but the consequences of supplying an -invalid string are then undefined. - -More information about the details of the error are passed -back in the returned value: - -PCRE_UTF32_ERR0 No error -PCRE_UTF32_ERR1 Surrogate character -PCRE_UTF32_ERR2 Unused (was non-character) -PCRE_UTF32_ERR3 Character > 0x10ffff - -Arguments: - string points to the string - length length of string, or -1 if the string is zero-terminated - errp pointer to an error position offset variable - -Returns: = 0 if the string is a valid UTF-32 string - > 0 otherwise, setting the offset of the bad character -*/ - -int -PRIV(valid_utf)(PCRE_PUCHAR string, int length, int *erroroffset) -{ -#ifdef SUPPORT_UTF -register PCRE_PUCHAR p; -register pcre_uchar c; - -if (length < 0) - { - for (p = string; *p != 0; p++); - length = p - string; - } - -for (p = string; length-- > 0; p++) - { - c = *p; - - if ((c & 0xfffff800u) != 0xd800u) - { - /* Normal UTF-32 code point. Neither high nor low surrogate. */ - if (c > 0x10ffffu) - { - *erroroffset = p - string; - return PCRE_UTF32_ERR3; - } - } - else - { - /* A surrogate */ - *erroroffset = p - string; - return PCRE_UTF32_ERR1; - } - } - -#else /* SUPPORT_UTF */ -(void)(string); /* Keep picky compilers happy */ -(void)(length); -(void)(erroroffset); -#endif /* SUPPORT_UTF */ - -return PCRE_UTF32_ERR0; /* This indicates success */ -} - -/* End of pcre32_valid_utf32.c */ diff --git a/src/third_party/pcre-8.39/pcre32_version.c b/src/third_party/pcre-8.39/pcre32_version.c deleted file mode 100644 index fdaad9b0859..00000000000 --- a/src/third_party/pcre-8.39/pcre32_version.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_version.c" - -/* End of pcre32_version.c */ diff --git a/src/third_party/pcre-8.39/pcre32_xclass.c b/src/third_party/pcre-8.39/pcre32_xclass.c deleted file mode 100644 index 5662408ad5f..00000000000 --- a/src/third_party/pcre-8.39/pcre32_xclass.c +++ /dev/null @@ -1,45 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Generate code with 32 bit character support. */ -#define COMPILE_PCRE32 - -#include "pcre_xclass.c" - -/* End of pcre32_xclass.c */ diff --git a/src/third_party/pcre-8.39/pcre_byte_order.c b/src/third_party/pcre-8.39/pcre_byte_order.c deleted file mode 100644 index cf5f12b04ea..00000000000 --- a/src/third_party/pcre-8.39/pcre_byte_order.c +++ /dev/null @@ -1,319 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2014 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This module contains an internal function that tests a compiled pattern to -see if it was compiled with the opposite endianness. If so, it uses an -auxiliary local function to flip the appropriate bytes. */ - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "pcre_internal.h" - - -/************************************************* -* Swap byte functions * -*************************************************/ - -/* The following functions swap the bytes of a pcre_uint16 -and pcre_uint32 value. - -Arguments: - value any number - -Returns: the byte swapped value -*/ - -static pcre_uint32 -swap_uint32(pcre_uint32 value) -{ -return ((value & 0x000000ff) << 24) | - ((value & 0x0000ff00) << 8) | - ((value & 0x00ff0000) >> 8) | - (value >> 24); -} - -static pcre_uint16 -swap_uint16(pcre_uint16 value) -{ -return (value >> 8) | (value << 8); -} - - -/************************************************* -* Test for a byte-flipped compiled regex * -*************************************************/ - -/* This function swaps the bytes of a compiled pattern usually -loaded form the disk. It also sets the tables pointer, which -is likely an invalid pointer after reload. - -Arguments: - argument_re points to the compiled expression - extra_data points to extra data or is NULL - tables points to the character tables or NULL - -Returns: 0 if the swap is successful, negative on error -*/ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DECL int pcre_pattern_to_host_byte_order(pcre *argument_re, - pcre_extra *extra_data, const unsigned char *tables) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DECL int pcre16_pattern_to_host_byte_order(pcre16 *argument_re, - pcre16_extra *extra_data, const unsigned char *tables) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DECL int pcre32_pattern_to_host_byte_order(pcre32 *argument_re, - pcre32_extra *extra_data, const unsigned char *tables) -#endif -{ -REAL_PCRE *re = (REAL_PCRE *)argument_re; -pcre_study_data *study; -#ifndef COMPILE_PCRE8 -pcre_uchar *ptr; -int length; -#if defined SUPPORT_UTF && defined COMPILE_PCRE16 -BOOL utf; -BOOL utf16_char; -#endif /* SUPPORT_UTF && COMPILE_PCRE16 */ -#endif /* !COMPILE_PCRE8 */ - -if (re == NULL) return PCRE_ERROR_NULL; -if (re->magic_number == MAGIC_NUMBER) - { - if ((re->flags & PCRE_MODE) == 0) return PCRE_ERROR_BADMODE; - re->tables = tables; - return 0; - } - -if (re->magic_number != REVERSED_MAGIC_NUMBER) return PCRE_ERROR_BADMAGIC; -if ((swap_uint32(re->flags) & PCRE_MODE) == 0) return PCRE_ERROR_BADMODE; - -re->magic_number = MAGIC_NUMBER; -re->size = swap_uint32(re->size); -re->options = swap_uint32(re->options); -re->flags = swap_uint32(re->flags); -re->limit_match = swap_uint32(re->limit_match); -re->limit_recursion = swap_uint32(re->limit_recursion); - -#if defined COMPILE_PCRE8 || defined COMPILE_PCRE16 -re->first_char = swap_uint16(re->first_char); -re->req_char = swap_uint16(re->req_char); -#elif defined COMPILE_PCRE32 -re->first_char = swap_uint32(re->first_char); -re->req_char = swap_uint32(re->req_char); -#endif - -re->max_lookbehind = swap_uint16(re->max_lookbehind); -re->top_bracket = swap_uint16(re->top_bracket); -re->top_backref = swap_uint16(re->top_backref); -re->name_table_offset = swap_uint16(re->name_table_offset); -re->name_entry_size = swap_uint16(re->name_entry_size); -re->name_count = swap_uint16(re->name_count); -re->ref_count = swap_uint16(re->ref_count); -re->tables = tables; - -if (extra_data != NULL && (extra_data->flags & PCRE_EXTRA_STUDY_DATA) != 0) - { - study = (pcre_study_data *)extra_data->study_data; - study->size = swap_uint32(study->size); - study->flags = swap_uint32(study->flags); - study->minlength = swap_uint32(study->minlength); - } - -#ifndef COMPILE_PCRE8 -ptr = (pcre_uchar *)re + re->name_table_offset; -length = re->name_count * re->name_entry_size; -#if defined SUPPORT_UTF && defined COMPILE_PCRE16 -utf = (re->options & PCRE_UTF16) != 0; -utf16_char = FALSE; -#endif /* SUPPORT_UTF && COMPILE_PCRE16 */ - -while(TRUE) - { - /* Swap previous characters. */ - while (length-- > 0) - { -#if defined COMPILE_PCRE16 - *ptr = swap_uint16(*ptr); -#elif defined COMPILE_PCRE32 - *ptr = swap_uint32(*ptr); -#endif - ptr++; - } -#if defined SUPPORT_UTF && defined COMPILE_PCRE16 - if (utf16_char) - { - if (HAS_EXTRALEN(ptr[-1])) - { - /* We know that there is only one extra character in UTF-16. */ - *ptr = swap_uint16(*ptr); - ptr++; - } - } - utf16_char = FALSE; -#endif /* SUPPORT_UTF */ - - /* Get next opcode. */ - length = 0; -#if defined COMPILE_PCRE16 - *ptr = swap_uint16(*ptr); -#elif defined COMPILE_PCRE32 - *ptr = swap_uint32(*ptr); -#endif - switch (*ptr) - { - case OP_END: - return 0; - -#if defined SUPPORT_UTF && defined COMPILE_PCRE16 - case OP_CHAR: - case OP_CHARI: - case OP_NOT: - case OP_NOTI: - case OP_STAR: - case OP_MINSTAR: - case OP_PLUS: - case OP_MINPLUS: - case OP_QUERY: - case OP_MINQUERY: - case OP_UPTO: - case OP_MINUPTO: - case OP_EXACT: - case OP_POSSTAR: - case OP_POSPLUS: - case OP_POSQUERY: - case OP_POSUPTO: - case OP_STARI: - case OP_MINSTARI: - case OP_PLUSI: - case OP_MINPLUSI: - case OP_QUERYI: - case OP_MINQUERYI: - case OP_UPTOI: - case OP_MINUPTOI: - case OP_EXACTI: - case OP_POSSTARI: - case OP_POSPLUSI: - case OP_POSQUERYI: - case OP_POSUPTOI: - case OP_NOTSTAR: - case OP_NOTMINSTAR: - case OP_NOTPLUS: - case OP_NOTMINPLUS: - case OP_NOTQUERY: - case OP_NOTMINQUERY: - case OP_NOTUPTO: - case OP_NOTMINUPTO: - case OP_NOTEXACT: - case OP_NOTPOSSTAR: - case OP_NOTPOSPLUS: - case OP_NOTPOSQUERY: - case OP_NOTPOSUPTO: - case OP_NOTSTARI: - case OP_NOTMINSTARI: - case OP_NOTPLUSI: - case OP_NOTMINPLUSI: - case OP_NOTQUERYI: - case OP_NOTMINQUERYI: - case OP_NOTUPTOI: - case OP_NOTMINUPTOI: - case OP_NOTEXACTI: - case OP_NOTPOSSTARI: - case OP_NOTPOSPLUSI: - case OP_NOTPOSQUERYI: - case OP_NOTPOSUPTOI: - if (utf) utf16_char = TRUE; -#endif - /* Fall through. */ - - default: - length = PRIV(OP_lengths)[*ptr] - 1; - break; - - case OP_CLASS: - case OP_NCLASS: - /* Skip the character bit map. */ - ptr += 32/sizeof(pcre_uchar); - length = 0; - break; - - case OP_XCLASS: - /* Reverse the size of the XCLASS instance. */ - ptr++; -#if defined COMPILE_PCRE16 - *ptr = swap_uint16(*ptr); -#elif defined COMPILE_PCRE32 - *ptr = swap_uint32(*ptr); -#endif -#ifndef COMPILE_PCRE32 - if (LINK_SIZE > 1) - { - /* LINK_SIZE can be 1 or 2 in 16 bit mode. */ - ptr++; - *ptr = swap_uint16(*ptr); - } -#endif - ptr++; - length = (GET(ptr, -LINK_SIZE)) - (1 + LINK_SIZE + 1); -#if defined COMPILE_PCRE16 - *ptr = swap_uint16(*ptr); -#elif defined COMPILE_PCRE32 - *ptr = swap_uint32(*ptr); -#endif - if ((*ptr & XCL_MAP) != 0) - { - /* Skip the character bit map. */ - ptr += 32/sizeof(pcre_uchar); - length -= 32/sizeof(pcre_uchar); - } - break; - } - ptr++; - } -/* Control should never reach here in 16/32 bit mode. */ -#else /* In 8-bit mode, the pattern does not need to be processed. */ -return 0; -#endif /* !COMPILE_PCRE8 */ -} - -/* End of pcre_byte_order.c */ diff --git a/src/third_party/pcre-8.39/pcre_chartables.c b/src/third_party/pcre-8.39/pcre_chartables.c deleted file mode 100644 index 1e20ec29d05..00000000000 --- a/src/third_party/pcre-8.39/pcre_chartables.c +++ /dev/null @@ -1,198 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* This file contains character tables that are used when no external tables -are passed to PCRE by the application that calls it. The tables are used only -for characters whose code values are less than 256. - -This is a default version of the tables that assumes ASCII encoding. A program -called dftables (which is distributed with PCRE) can be used to build -alternative versions of this file. This is necessary if you are running in an -EBCDIC environment, or if you want to default to a different encoding, for -example ISO-8859-1. When dftables is run, it creates these tables in the -current locale. If PCRE is configured with --enable-rebuild-chartables, this -happens automatically. - -The following #includes are present because without them gcc 4.x may remove the -array definition from the final binary if PCRE is built into a static library -and dead code stripping is activated. This leads to link errors. Pulling in the -header ensures that the array gets flagged as "someone outside this compilation -unit might reference this" and so it will always be supplied to the linker. */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "pcre_internal.h" - -const pcre_uint8 PRIV(default_tables)[] = { - -/* This table is a lower casing table. */ - - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, - 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 38, 39, - 40, 41, 42, 43, 44, 45, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, - 56, 57, 58, 59, 60, 61, 62, 63, - 64, 97, 98, 99,100,101,102,103, - 104,105,106,107,108,109,110,111, - 112,113,114,115,116,117,118,119, - 120,121,122, 91, 92, 93, 94, 95, - 96, 97, 98, 99,100,101,102,103, - 104,105,106,107,108,109,110,111, - 112,113,114,115,116,117,118,119, - 120,121,122,123,124,125,126,127, - 128,129,130,131,132,133,134,135, - 136,137,138,139,140,141,142,143, - 144,145,146,147,148,149,150,151, - 152,153,154,155,156,157,158,159, - 160,161,162,163,164,165,166,167, - 168,169,170,171,172,173,174,175, - 176,177,178,179,180,181,182,183, - 184,185,186,187,188,189,190,191, - 192,193,194,195,196,197,198,199, - 200,201,202,203,204,205,206,207, - 208,209,210,211,212,213,214,215, - 216,217,218,219,220,221,222,223, - 224,225,226,227,228,229,230,231, - 232,233,234,235,236,237,238,239, - 240,241,242,243,244,245,246,247, - 248,249,250,251,252,253,254,255, - -/* This table is a case flipping table. */ - - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, - 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 38, 39, - 40, 41, 42, 43, 44, 45, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, - 56, 57, 58, 59, 60, 61, 62, 63, - 64, 97, 98, 99,100,101,102,103, - 104,105,106,107,108,109,110,111, - 112,113,114,115,116,117,118,119, - 120,121,122, 91, 92, 93, 94, 95, - 96, 65, 66, 67, 68, 69, 70, 71, - 72, 73, 74, 75, 76, 77, 78, 79, - 80, 81, 82, 83, 84, 85, 86, 87, - 88, 89, 90,123,124,125,126,127, - 128,129,130,131,132,133,134,135, - 136,137,138,139,140,141,142,143, - 144,145,146,147,148,149,150,151, - 152,153,154,155,156,157,158,159, - 160,161,162,163,164,165,166,167, - 168,169,170,171,172,173,174,175, - 176,177,178,179,180,181,182,183, - 184,185,186,187,188,189,190,191, - 192,193,194,195,196,197,198,199, - 200,201,202,203,204,205,206,207, - 208,209,210,211,212,213,214,215, - 216,217,218,219,220,221,222,223, - 224,225,226,227,228,229,230,231, - 232,233,234,235,236,237,238,239, - 240,241,242,243,244,245,246,247, - 248,249,250,251,252,253,254,255, - -/* This table contains bit maps for various character classes. Each map is 32 -bytes long and the bits run from the least significant end of each byte. The -classes that have their own maps are: space, xdigit, digit, upper, lower, word, -graph, print, punct, and cntrl. Other classes are built from combinations. */ - - 0x00,0x3e,0x00,0x00,0x01,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, - 0x7e,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0xfe,0xff,0xff,0x07,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0x07, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, - 0xfe,0xff,0xff,0x87,0xfe,0xff,0xff,0x07, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0xff, - 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0xff,0xff,0xff,0xff, - 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0xfe,0xff,0x00,0xfc, - 0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0x78, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x80, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - -/* This table identifies various classes of character by individual bits: - 0x01 white space character - 0x02 letter - 0x04 decimal digit - 0x08 hexadecimal digit - 0x10 alphanumeric or '_' - 0x80 regular expression metacharacter or binary zero -*/ - - 0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 0- 7 */ - 0x00,0x01,0x01,0x01,0x01,0x01,0x00,0x00, /* 8- 15 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 16- 23 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 24- 31 */ - 0x01,0x00,0x00,0x00,0x80,0x00,0x00,0x00, /* - ' */ - 0x80,0x80,0x80,0x80,0x00,0x00,0x80,0x00, /* ( - / */ - 0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c, /* 0 - 7 */ - 0x1c,0x1c,0x00,0x00,0x00,0x00,0x00,0x80, /* 8 - ? */ - 0x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /* @ - G */ - 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* H - O */ - 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* P - W */ - 0x12,0x12,0x12,0x80,0x80,0x00,0x80,0x10, /* X - _ */ - 0x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /* ` - g */ - 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* h - o */ - 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* p - w */ - 0x12,0x12,0x12,0x80,0x80,0x00,0x00,0x00, /* x -127 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 128-135 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 136-143 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 144-151 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 152-159 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 160-167 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 168-175 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 176-183 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 184-191 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 192-199 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 200-207 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 208-215 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 216-223 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 224-231 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 232-239 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 240-247 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00};/* 248-255 */ - -/* End of pcre_chartables.c */ diff --git a/src/third_party/pcre-8.39/pcre_chartables.c.dist b/src/third_party/pcre-8.39/pcre_chartables.c.dist deleted file mode 100644 index 1e20ec29d05..00000000000 --- a/src/third_party/pcre-8.39/pcre_chartables.c.dist +++ /dev/null @@ -1,198 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* This file contains character tables that are used when no external tables -are passed to PCRE by the application that calls it. The tables are used only -for characters whose code values are less than 256. - -This is a default version of the tables that assumes ASCII encoding. A program -called dftables (which is distributed with PCRE) can be used to build -alternative versions of this file. This is necessary if you are running in an -EBCDIC environment, or if you want to default to a different encoding, for -example ISO-8859-1. When dftables is run, it creates these tables in the -current locale. If PCRE is configured with --enable-rebuild-chartables, this -happens automatically. - -The following #includes are present because without them gcc 4.x may remove the -array definition from the final binary if PCRE is built into a static library -and dead code stripping is activated. This leads to link errors. Pulling in the -header ensures that the array gets flagged as "someone outside this compilation -unit might reference this" and so it will always be supplied to the linker. */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "pcre_internal.h" - -const pcre_uint8 PRIV(default_tables)[] = { - -/* This table is a lower casing table. */ - - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, - 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 38, 39, - 40, 41, 42, 43, 44, 45, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, - 56, 57, 58, 59, 60, 61, 62, 63, - 64, 97, 98, 99,100,101,102,103, - 104,105,106,107,108,109,110,111, - 112,113,114,115,116,117,118,119, - 120,121,122, 91, 92, 93, 94, 95, - 96, 97, 98, 99,100,101,102,103, - 104,105,106,107,108,109,110,111, - 112,113,114,115,116,117,118,119, - 120,121,122,123,124,125,126,127, - 128,129,130,131,132,133,134,135, - 136,137,138,139,140,141,142,143, - 144,145,146,147,148,149,150,151, - 152,153,154,155,156,157,158,159, - 160,161,162,163,164,165,166,167, - 168,169,170,171,172,173,174,175, - 176,177,178,179,180,181,182,183, - 184,185,186,187,188,189,190,191, - 192,193,194,195,196,197,198,199, - 200,201,202,203,204,205,206,207, - 208,209,210,211,212,213,214,215, - 216,217,218,219,220,221,222,223, - 224,225,226,227,228,229,230,231, - 232,233,234,235,236,237,238,239, - 240,241,242,243,244,245,246,247, - 248,249,250,251,252,253,254,255, - -/* This table is a case flipping table. */ - - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, - 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 38, 39, - 40, 41, 42, 43, 44, 45, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, - 56, 57, 58, 59, 60, 61, 62, 63, - 64, 97, 98, 99,100,101,102,103, - 104,105,106,107,108,109,110,111, - 112,113,114,115,116,117,118,119, - 120,121,122, 91, 92, 93, 94, 95, - 96, 65, 66, 67, 68, 69, 70, 71, - 72, 73, 74, 75, 76, 77, 78, 79, - 80, 81, 82, 83, 84, 85, 86, 87, - 88, 89, 90,123,124,125,126,127, - 128,129,130,131,132,133,134,135, - 136,137,138,139,140,141,142,143, - 144,145,146,147,148,149,150,151, - 152,153,154,155,156,157,158,159, - 160,161,162,163,164,165,166,167, - 168,169,170,171,172,173,174,175, - 176,177,178,179,180,181,182,183, - 184,185,186,187,188,189,190,191, - 192,193,194,195,196,197,198,199, - 200,201,202,203,204,205,206,207, - 208,209,210,211,212,213,214,215, - 216,217,218,219,220,221,222,223, - 224,225,226,227,228,229,230,231, - 232,233,234,235,236,237,238,239, - 240,241,242,243,244,245,246,247, - 248,249,250,251,252,253,254,255, - -/* This table contains bit maps for various character classes. Each map is 32 -bytes long and the bits run from the least significant end of each byte. The -classes that have their own maps are: space, xdigit, digit, upper, lower, word, -graph, print, punct, and cntrl. Other classes are built from combinations. */ - - 0x00,0x3e,0x00,0x00,0x01,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, - 0x7e,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0xfe,0xff,0xff,0x07,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0x07, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, - 0xfe,0xff,0xff,0x87,0xfe,0xff,0xff,0x07, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0xff, - 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0xff,0xff,0xff,0xff, - 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0xfe,0xff,0x00,0xfc, - 0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0x78, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x80, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - -/* This table identifies various classes of character by individual bits: - 0x01 white space character - 0x02 letter - 0x04 decimal digit - 0x08 hexadecimal digit - 0x10 alphanumeric or '_' - 0x80 regular expression metacharacter or binary zero -*/ - - 0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 0- 7 */ - 0x00,0x01,0x01,0x01,0x01,0x01,0x00,0x00, /* 8- 15 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 16- 23 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 24- 31 */ - 0x01,0x00,0x00,0x00,0x80,0x00,0x00,0x00, /* - ' */ - 0x80,0x80,0x80,0x80,0x00,0x00,0x80,0x00, /* ( - / */ - 0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c, /* 0 - 7 */ - 0x1c,0x1c,0x00,0x00,0x00,0x00,0x00,0x80, /* 8 - ? */ - 0x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /* @ - G */ - 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* H - O */ - 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* P - W */ - 0x12,0x12,0x12,0x80,0x80,0x00,0x80,0x10, /* X - _ */ - 0x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /* ` - g */ - 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* h - o */ - 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* p - w */ - 0x12,0x12,0x12,0x80,0x80,0x00,0x00,0x00, /* x -127 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 128-135 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 136-143 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 144-151 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 152-159 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 160-167 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 168-175 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 176-183 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 184-191 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 192-199 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 200-207 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 208-215 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 216-223 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 224-231 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 232-239 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 240-247 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00};/* 248-255 */ - -/* End of pcre_chartables.c */ diff --git a/src/third_party/pcre-8.39/pcre_compile.c b/src/third_party/pcre-8.39/pcre_compile.c deleted file mode 100644 index 7cd39501230..00000000000 --- a/src/third_party/pcre-8.39/pcre_compile.c +++ /dev/null @@ -1,9749 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2016 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This module contains the external function pcre_compile(), along with -supporting internal functions that are not used by other modules. */ - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#define NLBLOCK cd /* Block containing newline information */ -#define PSSTART start_pattern /* Field containing pattern start */ -#define PSEND end_pattern /* Field containing pattern end */ - -#include "pcre_internal.h" - - -/* When PCRE_DEBUG is defined, we need the pcre(16|32)_printint() function, which -is also used by pcretest. PCRE_DEBUG is not defined when building a production -library. We do not need to select pcre16_printint.c specially, because the -COMPILE_PCREx macro will already be appropriately set. */ - -#ifdef PCRE_DEBUG -/* pcre_printint.c should not include any headers */ -#define PCRE_INCLUDED -#include "pcre_printint.c" -#undef PCRE_INCLUDED -#endif - - -/* Macro for setting individual bits in class bitmaps. */ - -#define SETBIT(a,b) a[(b)/8] |= (1 << ((b)&7)) - -/* Maximum length value to check against when making sure that the integer that -holds the compiled pattern length does not overflow. We make it a bit less than -INT_MAX to allow for adding in group terminating bytes, so that we don't have -to check them every time. */ - -#define OFLOW_MAX (INT_MAX - 20) - -/* Definitions to allow mutual recursion */ - -static int - add_list_to_class(pcre_uint8 *, pcre_uchar **, int, compile_data *, - const pcre_uint32 *, unsigned int); - -static BOOL - compile_regex(int, pcre_uchar **, const pcre_uchar **, int *, BOOL, BOOL, int, int, - pcre_uint32 *, pcre_int32 *, pcre_uint32 *, pcre_int32 *, branch_chain *, - compile_data *, int *); - - - -/************************************************* -* Code parameters and static tables * -*************************************************/ - -/* This value specifies the size of stack workspace that is used during the -first pre-compile phase that determines how much memory is required. The regex -is partly compiled into this space, but the compiled parts are discarded as -soon as they can be, so that hopefully there will never be an overrun. The code -does, however, check for an overrun. The largest amount I've seen used is 218, -so this number is very generous. - -The same workspace is used during the second, actual compile phase for -remembering forward references to groups so that they can be filled in at the -end. Each entry in this list occupies LINK_SIZE bytes, so even when LINK_SIZE -is 4 there is plenty of room for most patterns. However, the memory can get -filled up by repetitions of forward references, for example patterns like -/(?1){0,1999}(b)/, and one user did hit the limit. The code has been changed so -that the workspace is expanded using malloc() in this situation. The value -below is therefore a minimum, and we put a maximum on it for safety. The -minimum is now also defined in terms of LINK_SIZE so that the use of malloc() -kicks in at the same number of forward references in all cases. */ - -#define COMPILE_WORK_SIZE (2048*LINK_SIZE) -#define COMPILE_WORK_SIZE_MAX (100*COMPILE_WORK_SIZE) - -/* This value determines the size of the initial vector that is used for -remembering named groups during the pre-compile. It is allocated on the stack, -but if it is too small, it is expanded using malloc(), in a similar way to the -workspace. The value is the number of slots in the list. */ - -#define NAMED_GROUP_LIST_SIZE 20 - -/* The overrun tests check for a slightly smaller size so that they detect the -overrun before it actually does run off the end of the data block. */ - -#define WORK_SIZE_SAFETY_MARGIN (100) - -/* Private flags added to firstchar and reqchar. */ - -#define REQ_CASELESS (1 << 0) /* Indicates caselessness */ -#define REQ_VARY (1 << 1) /* Reqchar followed non-literal item */ -/* Negative values for the firstchar and reqchar flags */ -#define REQ_UNSET (-2) -#define REQ_NONE (-1) - -/* Repeated character flags. */ - -#define UTF_LENGTH 0x10000000l /* The char contains its length. */ - -/* Table for handling escaped characters in the range '0'-'z'. Positive returns -are simple data values; negative values are for special things like \d and so -on. Zero means further processing is needed (for things like \x), or the escape -is invalid. */ - -#ifndef EBCDIC - -/* This is the "normal" table for ASCII systems or for EBCDIC systems running -in UTF-8 mode. */ - -static const short int escapes[] = { - 0, 0, - 0, 0, - 0, 0, - 0, 0, - 0, 0, - CHAR_COLON, CHAR_SEMICOLON, - CHAR_LESS_THAN_SIGN, CHAR_EQUALS_SIGN, - CHAR_GREATER_THAN_SIGN, CHAR_QUESTION_MARK, - CHAR_COMMERCIAL_AT, -ESC_A, - -ESC_B, -ESC_C, - -ESC_D, -ESC_E, - 0, -ESC_G, - -ESC_H, 0, - 0, -ESC_K, - 0, 0, - -ESC_N, 0, - -ESC_P, -ESC_Q, - -ESC_R, -ESC_S, - 0, 0, - -ESC_V, -ESC_W, - -ESC_X, 0, - -ESC_Z, CHAR_LEFT_SQUARE_BRACKET, - CHAR_BACKSLASH, CHAR_RIGHT_SQUARE_BRACKET, - CHAR_CIRCUMFLEX_ACCENT, CHAR_UNDERSCORE, - CHAR_GRAVE_ACCENT, ESC_a, - -ESC_b, 0, - -ESC_d, ESC_e, - ESC_f, 0, - -ESC_h, 0, - 0, -ESC_k, - 0, 0, - ESC_n, 0, - -ESC_p, 0, - ESC_r, -ESC_s, - ESC_tee, 0, - -ESC_v, -ESC_w, - 0, 0, - -ESC_z -}; - -#else - -/* This is the "abnormal" table for EBCDIC systems without UTF-8 support. */ - -static const short int escapes[] = { -/* 48 */ 0, 0, 0, '.', '<', '(', '+', '|', -/* 50 */ '&', 0, 0, 0, 0, 0, 0, 0, -/* 58 */ 0, 0, '!', '$', '*', ')', ';', '~', -/* 60 */ '-', '/', 0, 0, 0, 0, 0, 0, -/* 68 */ 0, 0, '|', ',', '%', '_', '>', '?', -/* 70 */ 0, 0, 0, 0, 0, 0, 0, 0, -/* 78 */ 0, '`', ':', '#', '@', '\'', '=', '"', -/* 80 */ 0, ESC_a, -ESC_b, 0, -ESC_d, ESC_e, ESC_f, 0, -/* 88 */-ESC_h, 0, 0, '{', 0, 0, 0, 0, -/* 90 */ 0, 0, -ESC_k, 0, 0, ESC_n, 0, -ESC_p, -/* 98 */ 0, ESC_r, 0, '}', 0, 0, 0, 0, -/* A0 */ 0, '~', -ESC_s, ESC_tee, 0,-ESC_v, -ESC_w, 0, -/* A8 */ 0,-ESC_z, 0, 0, 0, '[', 0, 0, -/* B0 */ 0, 0, 0, 0, 0, 0, 0, 0, -/* B8 */ 0, 0, 0, 0, 0, ']', '=', '-', -/* C0 */ '{',-ESC_A, -ESC_B, -ESC_C, -ESC_D,-ESC_E, 0, -ESC_G, -/* C8 */-ESC_H, 0, 0, 0, 0, 0, 0, 0, -/* D0 */ '}', 0, -ESC_K, 0, 0,-ESC_N, 0, -ESC_P, -/* D8 */-ESC_Q,-ESC_R, 0, 0, 0, 0, 0, 0, -/* E0 */ '\\', 0, -ESC_S, 0, 0,-ESC_V, -ESC_W, -ESC_X, -/* E8 */ 0,-ESC_Z, 0, 0, 0, 0, 0, 0, -/* F0 */ 0, 0, 0, 0, 0, 0, 0, 0, -/* F8 */ 0, 0, 0, 0, 0, 0, 0, 0 -}; - -/* We also need a table of characters that may follow \c in an EBCDIC -environment for characters 0-31. */ - -static unsigned char ebcdic_escape_c[] = "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_"; - -#endif - - -/* Table of special "verbs" like (*PRUNE). This is a short table, so it is -searched linearly. Put all the names into a single string, in order to reduce -the number of relocations when a shared library is dynamically linked. The -string is built from string macros so that it works in UTF-8 mode on EBCDIC -platforms. */ - -typedef struct verbitem { - int len; /* Length of verb name */ - int op; /* Op when no arg, or -1 if arg mandatory */ - int op_arg; /* Op when arg present, or -1 if not allowed */ -} verbitem; - -static const char verbnames[] = - "\0" /* Empty name is a shorthand for MARK */ - STRING_MARK0 - STRING_ACCEPT0 - STRING_COMMIT0 - STRING_F0 - STRING_FAIL0 - STRING_PRUNE0 - STRING_SKIP0 - STRING_THEN; - -static const verbitem verbs[] = { - { 0, -1, OP_MARK }, - { 4, -1, OP_MARK }, - { 6, OP_ACCEPT, -1 }, - { 6, OP_COMMIT, -1 }, - { 1, OP_FAIL, -1 }, - { 4, OP_FAIL, -1 }, - { 5, OP_PRUNE, OP_PRUNE_ARG }, - { 4, OP_SKIP, OP_SKIP_ARG }, - { 4, OP_THEN, OP_THEN_ARG } -}; - -static const int verbcount = sizeof(verbs)/sizeof(verbitem); - - -/* Substitutes for [[:<:]] and [[:>:]], which mean start and end of word in -another regex library. */ - -static const pcre_uchar sub_start_of_word[] = { - CHAR_BACKSLASH, CHAR_b, CHAR_LEFT_PARENTHESIS, CHAR_QUESTION_MARK, - CHAR_EQUALS_SIGN, CHAR_BACKSLASH, CHAR_w, CHAR_RIGHT_PARENTHESIS, '\0' }; - -static const pcre_uchar sub_end_of_word[] = { - CHAR_BACKSLASH, CHAR_b, CHAR_LEFT_PARENTHESIS, CHAR_QUESTION_MARK, - CHAR_LESS_THAN_SIGN, CHAR_EQUALS_SIGN, CHAR_BACKSLASH, CHAR_w, - CHAR_RIGHT_PARENTHESIS, '\0' }; - - -/* Tables of names of POSIX character classes and their lengths. The names are -now all in a single string, to reduce the number of relocations when a shared -library is dynamically loaded. The list of lengths is terminated by a zero -length entry. The first three must be alpha, lower, upper, as this is assumed -for handling case independence. The indices for graph, print, and punct are -needed, so identify them. */ - -static const char posix_names[] = - STRING_alpha0 STRING_lower0 STRING_upper0 STRING_alnum0 - STRING_ascii0 STRING_blank0 STRING_cntrl0 STRING_digit0 - STRING_graph0 STRING_print0 STRING_punct0 STRING_space0 - STRING_word0 STRING_xdigit; - -static const pcre_uint8 posix_name_lengths[] = { - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 6, 0 }; - -#define PC_GRAPH 8 -#define PC_PRINT 9 -#define PC_PUNCT 10 - - -/* Table of class bit maps for each POSIX class. Each class is formed from a -base map, with an optional addition or removal of another map. Then, for some -classes, there is some additional tweaking: for [:blank:] the vertical space -characters are removed, and for [:alpha:] and [:alnum:] the underscore -character is removed. The triples in the table consist of the base map offset, -second map offset or -1 if no second map, and a non-negative value for map -addition or a negative value for map subtraction (if there are two maps). The -absolute value of the third field has these meanings: 0 => no tweaking, 1 => -remove vertical space characters, 2 => remove underscore. */ - -static const int posix_class_maps[] = { - cbit_word, cbit_digit, -2, /* alpha */ - cbit_lower, -1, 0, /* lower */ - cbit_upper, -1, 0, /* upper */ - cbit_word, -1, 2, /* alnum - word without underscore */ - cbit_print, cbit_cntrl, 0, /* ascii */ - cbit_space, -1, 1, /* blank - a GNU extension */ - cbit_cntrl, -1, 0, /* cntrl */ - cbit_digit, -1, 0, /* digit */ - cbit_graph, -1, 0, /* graph */ - cbit_print, -1, 0, /* print */ - cbit_punct, -1, 0, /* punct */ - cbit_space, -1, 0, /* space */ - cbit_word, -1, 0, /* word - a Perl extension */ - cbit_xdigit,-1, 0 /* xdigit */ -}; - -/* Table of substitutes for \d etc when PCRE_UCP is set. They are replaced by -Unicode property escapes. */ - -#ifdef SUPPORT_UCP -static const pcre_uchar string_PNd[] = { - CHAR_BACKSLASH, CHAR_P, CHAR_LEFT_CURLY_BRACKET, - CHAR_N, CHAR_d, CHAR_RIGHT_CURLY_BRACKET, '\0' }; -static const pcre_uchar string_pNd[] = { - CHAR_BACKSLASH, CHAR_p, CHAR_LEFT_CURLY_BRACKET, - CHAR_N, CHAR_d, CHAR_RIGHT_CURLY_BRACKET, '\0' }; -static const pcre_uchar string_PXsp[] = { - CHAR_BACKSLASH, CHAR_P, CHAR_LEFT_CURLY_BRACKET, - CHAR_X, CHAR_s, CHAR_p, CHAR_RIGHT_CURLY_BRACKET, '\0' }; -static const pcre_uchar string_pXsp[] = { - CHAR_BACKSLASH, CHAR_p, CHAR_LEFT_CURLY_BRACKET, - CHAR_X, CHAR_s, CHAR_p, CHAR_RIGHT_CURLY_BRACKET, '\0' }; -static const pcre_uchar string_PXwd[] = { - CHAR_BACKSLASH, CHAR_P, CHAR_LEFT_CURLY_BRACKET, - CHAR_X, CHAR_w, CHAR_d, CHAR_RIGHT_CURLY_BRACKET, '\0' }; -static const pcre_uchar string_pXwd[] = { - CHAR_BACKSLASH, CHAR_p, CHAR_LEFT_CURLY_BRACKET, - CHAR_X, CHAR_w, CHAR_d, CHAR_RIGHT_CURLY_BRACKET, '\0' }; - -static const pcre_uchar *substitutes[] = { - string_PNd, /* \D */ - string_pNd, /* \d */ - string_PXsp, /* \S */ /* Xsp is Perl space, but from 8.34, Perl */ - string_pXsp, /* \s */ /* space and POSIX space are the same. */ - string_PXwd, /* \W */ - string_pXwd /* \w */ -}; - -/* The POSIX class substitutes must be in the order of the POSIX class names, -defined above, and there are both positive and negative cases. NULL means no -general substitute of a Unicode property escape (\p or \P). However, for some -POSIX classes (e.g. graph, print, punct) a special property code is compiled -directly. */ - -static const pcre_uchar string_pL[] = { - CHAR_BACKSLASH, CHAR_p, CHAR_LEFT_CURLY_BRACKET, - CHAR_L, CHAR_RIGHT_CURLY_BRACKET, '\0' }; -static const pcre_uchar string_pLl[] = { - CHAR_BACKSLASH, CHAR_p, CHAR_LEFT_CURLY_BRACKET, - CHAR_L, CHAR_l, CHAR_RIGHT_CURLY_BRACKET, '\0' }; -static const pcre_uchar string_pLu[] = { - CHAR_BACKSLASH, CHAR_p, CHAR_LEFT_CURLY_BRACKET, - CHAR_L, CHAR_u, CHAR_RIGHT_CURLY_BRACKET, '\0' }; -static const pcre_uchar string_pXan[] = { - CHAR_BACKSLASH, CHAR_p, CHAR_LEFT_CURLY_BRACKET, - CHAR_X, CHAR_a, CHAR_n, CHAR_RIGHT_CURLY_BRACKET, '\0' }; -static const pcre_uchar string_h[] = { - CHAR_BACKSLASH, CHAR_h, '\0' }; -static const pcre_uchar string_pXps[] = { - CHAR_BACKSLASH, CHAR_p, CHAR_LEFT_CURLY_BRACKET, - CHAR_X, CHAR_p, CHAR_s, CHAR_RIGHT_CURLY_BRACKET, '\0' }; -static const pcre_uchar string_PL[] = { - CHAR_BACKSLASH, CHAR_P, CHAR_LEFT_CURLY_BRACKET, - CHAR_L, CHAR_RIGHT_CURLY_BRACKET, '\0' }; -static const pcre_uchar string_PLl[] = { - CHAR_BACKSLASH, CHAR_P, CHAR_LEFT_CURLY_BRACKET, - CHAR_L, CHAR_l, CHAR_RIGHT_CURLY_BRACKET, '\0' }; -static const pcre_uchar string_PLu[] = { - CHAR_BACKSLASH, CHAR_P, CHAR_LEFT_CURLY_BRACKET, - CHAR_L, CHAR_u, CHAR_RIGHT_CURLY_BRACKET, '\0' }; -static const pcre_uchar string_PXan[] = { - CHAR_BACKSLASH, CHAR_P, CHAR_LEFT_CURLY_BRACKET, - CHAR_X, CHAR_a, CHAR_n, CHAR_RIGHT_CURLY_BRACKET, '\0' }; -static const pcre_uchar string_H[] = { - CHAR_BACKSLASH, CHAR_H, '\0' }; -static const pcre_uchar string_PXps[] = { - CHAR_BACKSLASH, CHAR_P, CHAR_LEFT_CURLY_BRACKET, - CHAR_X, CHAR_p, CHAR_s, CHAR_RIGHT_CURLY_BRACKET, '\0' }; - -static const pcre_uchar *posix_substitutes[] = { - string_pL, /* alpha */ - string_pLl, /* lower */ - string_pLu, /* upper */ - string_pXan, /* alnum */ - NULL, /* ascii */ - string_h, /* blank */ - NULL, /* cntrl */ - string_pNd, /* digit */ - NULL, /* graph */ - NULL, /* print */ - NULL, /* punct */ - string_pXps, /* space */ /* Xps is POSIX space, but from 8.34 */ - string_pXwd, /* word */ /* Perl and POSIX space are the same */ - NULL, /* xdigit */ - /* Negated cases */ - string_PL, /* ^alpha */ - string_PLl, /* ^lower */ - string_PLu, /* ^upper */ - string_PXan, /* ^alnum */ - NULL, /* ^ascii */ - string_H, /* ^blank */ - NULL, /* ^cntrl */ - string_PNd, /* ^digit */ - NULL, /* ^graph */ - NULL, /* ^print */ - NULL, /* ^punct */ - string_PXps, /* ^space */ /* Xps is POSIX space, but from 8.34 */ - string_PXwd, /* ^word */ /* Perl and POSIX space are the same */ - NULL /* ^xdigit */ -}; -#define POSIX_SUBSIZE (sizeof(posix_substitutes) / sizeof(pcre_uchar *)) -#endif - -#define STRING(a) # a -#define XSTRING(s) STRING(s) - -/* The texts of compile-time error messages. These are "char *" because they -are passed to the outside world. Do not ever re-use any error number, because -they are documented. Always add a new error instead. Messages marked DEAD below -are no longer used. This used to be a table of strings, but in order to reduce -the number of relocations needed when a shared library is loaded dynamically, -it is now one long string. We cannot use a table of offsets, because the -lengths of inserts such as XSTRING(MAX_NAME_SIZE) are not known. Instead, we -simply count through to the one we want - this isn't a performance issue -because these strings are used only when there is a compilation error. - -Each substring ends with \0 to insert a null character. This includes the final -substring, so that the whole string ends with \0\0, which can be detected when -counting through. */ - -static const char error_texts[] = - "no error\0" - "\\ at end of pattern\0" - "\\c at end of pattern\0" - "unrecognized character follows \\\0" - "numbers out of order in {} quantifier\0" - /* 5 */ - "number too big in {} quantifier\0" - "missing terminating ] for character class\0" - "invalid escape sequence in character class\0" - "range out of order in character class\0" - "nothing to repeat\0" - /* 10 */ - "internal error: invalid forward reference offset\0" - "internal error: unexpected repeat\0" - "unrecognized character after (? or (?-\0" - "POSIX named classes are supported only within a class\0" - "missing )\0" - /* 15 */ - "reference to non-existent subpattern\0" - "erroffset passed as NULL\0" - "unknown option bit(s) set\0" - "missing ) after comment\0" - "parentheses nested too deeply\0" /** DEAD **/ - /* 20 */ - "regular expression is too large\0" - "failed to get memory\0" - "unmatched parentheses\0" - "internal error: code overflow\0" - "unrecognized character after (?<\0" - /* 25 */ - "lookbehind assertion is not fixed length\0" - "malformed number or name after (?(\0" - "conditional group contains more than two branches\0" - "assertion expected after (?( or (?(?C)\0" - "(?R or (?[+-]digits must be followed by )\0" - /* 30 */ - "unknown POSIX class name\0" - "POSIX collating elements are not supported\0" - "this version of PCRE is compiled without UTF support\0" - "spare error\0" /** DEAD **/ - "character value in \\x{} or \\o{} is too large\0" - /* 35 */ - "invalid condition (?(0)\0" - "\\C not allowed in lookbehind assertion\0" - "PCRE does not support \\L, \\l, \\N{name}, \\U, or \\u\0" - "number after (?C is > 255\0" - "closing ) for (?C expected\0" - /* 40 */ - "recursive call could loop indefinitely\0" - "unrecognized character after (?P\0" - "syntax error in subpattern name (missing terminator)\0" - "two named subpatterns have the same name\0" - "invalid UTF-8 string\0" - /* 45 */ - "support for \\P, \\p, and \\X has not been compiled\0" - "malformed \\P or \\p sequence\0" - "unknown property name after \\P or \\p\0" - "subpattern name is too long (maximum " XSTRING(MAX_NAME_SIZE) " characters)\0" - "too many named subpatterns (maximum " XSTRING(MAX_NAME_COUNT) ")\0" - /* 50 */ - "repeated subpattern is too long\0" /** DEAD **/ - "octal value is greater than \\377 in 8-bit non-UTF-8 mode\0" - "internal error: overran compiling workspace\0" - "internal error: previously-checked referenced subpattern not found\0" - "DEFINE group contains more than one branch\0" - /* 55 */ - "repeating a DEFINE group is not allowed\0" /** DEAD **/ - "inconsistent NEWLINE options\0" - "\\g is not followed by a braced, angle-bracketed, or quoted name/number or by a plain number\0" - "a numbered reference must not be zero\0" - "an argument is not allowed for (*ACCEPT), (*FAIL), or (*COMMIT)\0" - /* 60 */ - "(*VERB) not recognized or malformed\0" - "number is too big\0" - "subpattern name expected\0" - "digit expected after (?+\0" - "] is an invalid data character in JavaScript compatibility mode\0" - /* 65 */ - "different names for subpatterns of the same number are not allowed\0" - "(*MARK) must have an argument\0" - "this version of PCRE is not compiled with Unicode property support\0" -#ifndef EBCDIC - "\\c must be followed by an ASCII character\0" -#else - "\\c must be followed by a letter or one of [\\]^_?\0" -#endif - "\\k is not followed by a braced, angle-bracketed, or quoted name\0" - /* 70 */ - "internal error: unknown opcode in find_fixedlength()\0" - "\\N is not supported in a class\0" - "too many forward references\0" - "disallowed Unicode code point (>= 0xd800 && <= 0xdfff)\0" - "invalid UTF-16 string\0" - /* 75 */ - "name is too long in (*MARK), (*PRUNE), (*SKIP), or (*THEN)\0" - "character value in \\u.... sequence is too large\0" - "invalid UTF-32 string\0" - "setting UTF is disabled by the application\0" - "non-hex character in \\x{} (closing brace missing?)\0" - /* 80 */ - "non-octal character in \\o{} (closing brace missing?)\0" - "missing opening brace after \\o\0" - "parentheses are too deeply nested\0" - "invalid range in character class\0" - "group name must start with a non-digit\0" - /* 85 */ - "parentheses are too deeply nested (stack check)\0" - "digits missing in \\x{} or \\o{}\0" - "regular expression is too complicated\0" - ; - -/* Table to identify digits and hex digits. This is used when compiling -patterns. Note that the tables in chartables are dependent on the locale, and -may mark arbitrary characters as digits - but the PCRE compiling code expects -to handle only 0-9, a-z, and A-Z as digits when compiling. That is why we have -a private table here. It costs 256 bytes, but it is a lot faster than doing -character value tests (at least in some simple cases I timed), and in some -applications one wants PCRE to compile efficiently as well as match -efficiently. - -For convenience, we use the same bit definitions as in chartables: - - 0x04 decimal digit - 0x08 hexadecimal digit - -Then we can use ctype_digit and ctype_xdigit in the code. */ - -/* Using a simple comparison for decimal numbers rather than a memory read -is much faster, and the resulting code is simpler (the compiler turns it -into a subtraction and unsigned comparison). */ - -#define IS_DIGIT(x) ((x) >= CHAR_0 && (x) <= CHAR_9) - -#ifndef EBCDIC - -/* This is the "normal" case, for ASCII systems, and EBCDIC systems running in -UTF-8 mode. */ - -static const pcre_uint8 digitab[] = - { - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 0- 7 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 8- 15 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 16- 23 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 24- 31 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* - ' */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* ( - / */ - 0x0c,0x0c,0x0c,0x0c,0x0c,0x0c,0x0c,0x0c, /* 0 - 7 */ - 0x0c,0x0c,0x00,0x00,0x00,0x00,0x00,0x00, /* 8 - ? */ - 0x00,0x08,0x08,0x08,0x08,0x08,0x08,0x00, /* @ - G */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* H - O */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* P - W */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* X - _ */ - 0x00,0x08,0x08,0x08,0x08,0x08,0x08,0x00, /* ` - g */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* h - o */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* p - w */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* x -127 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 128-135 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 136-143 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 144-151 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 152-159 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 160-167 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 168-175 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 176-183 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 184-191 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 192-199 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 200-207 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 208-215 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 216-223 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 224-231 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 232-239 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 240-247 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00};/* 248-255 */ - -#else - -/* This is the "abnormal" case, for EBCDIC systems not running in UTF-8 mode. */ - -static const pcre_uint8 digitab[] = - { - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 0- 7 0 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 8- 15 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 16- 23 10 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 24- 31 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 32- 39 20 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 40- 47 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 48- 55 30 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 56- 63 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* - 71 40 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 72- | */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* & - 87 50 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 88- 95 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* - -103 60 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 104- ? */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 112-119 70 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 120- " */ - 0x00,0x08,0x08,0x08,0x08,0x08,0x08,0x00, /* 128- g 80 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* h -143 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 144- p 90 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* q -159 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 160- x A0 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* y -175 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* ^ -183 B0 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 184-191 */ - 0x00,0x08,0x08,0x08,0x08,0x08,0x08,0x00, /* { - G C0 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* H -207 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* } - P D0 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* Q -223 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* \ - X E0 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* Y -239 */ - 0x0c,0x0c,0x0c,0x0c,0x0c,0x0c,0x0c,0x0c, /* 0 - 7 F0 */ - 0x0c,0x0c,0x00,0x00,0x00,0x00,0x00,0x00};/* 8 -255 */ - -static const pcre_uint8 ebcdic_chartab[] = { /* chartable partial dup */ - 0x80,0x00,0x00,0x00,0x00,0x01,0x00,0x00, /* 0- 7 */ - 0x00,0x00,0x00,0x00,0x01,0x01,0x00,0x00, /* 8- 15 */ - 0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00, /* 16- 23 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 24- 31 */ - 0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00, /* 32- 39 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 40- 47 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 48- 55 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 56- 63 */ - 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* - 71 */ - 0x00,0x00,0x00,0x80,0x00,0x80,0x80,0x80, /* 72- | */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* & - 87 */ - 0x00,0x00,0x00,0x80,0x80,0x80,0x00,0x00, /* 88- 95 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* - -103 */ - 0x00,0x00,0x00,0x00,0x00,0x10,0x00,0x80, /* 104- ? */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 112-119 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 120- " */ - 0x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /* 128- g */ - 0x12,0x12,0x00,0x00,0x00,0x00,0x00,0x00, /* h -143 */ - 0x00,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* 144- p */ - 0x12,0x12,0x00,0x00,0x00,0x00,0x00,0x00, /* q -159 */ - 0x00,0x00,0x12,0x12,0x12,0x12,0x12,0x12, /* 160- x */ - 0x12,0x12,0x00,0x00,0x00,0x00,0x00,0x00, /* y -175 */ - 0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* ^ -183 */ - 0x00,0x00,0x80,0x00,0x00,0x00,0x00,0x00, /* 184-191 */ - 0x80,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /* { - G */ - 0x12,0x12,0x00,0x00,0x00,0x00,0x00,0x00, /* H -207 */ - 0x00,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* } - P */ - 0x12,0x12,0x00,0x00,0x00,0x00,0x00,0x00, /* Q -223 */ - 0x00,0x00,0x12,0x12,0x12,0x12,0x12,0x12, /* \ - X */ - 0x12,0x12,0x00,0x00,0x00,0x00,0x00,0x00, /* Y -239 */ - 0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c, /* 0 - 7 */ - 0x1c,0x1c,0x00,0x00,0x00,0x00,0x00,0x00};/* 8 -255 */ -#endif - - -/* This table is used to check whether auto-possessification is possible -between adjacent character-type opcodes. The left-hand (repeated) opcode is -used to select the row, and the right-hand opcode is use to select the column. -A value of 1 means that auto-possessification is OK. For example, the second -value in the first row means that \D+\d can be turned into \D++\d. - -The Unicode property types (\P and \p) have to be present to fill out the table -because of what their opcode values are, but the table values should always be -zero because property types are handled separately in the code. The last four -columns apply to items that cannot be repeated, so there is no need to have -rows for them. Note that OP_DIGIT etc. are generated only when PCRE_UCP is -*not* set. When it is set, \d etc. are converted into OP_(NOT_)PROP codes. */ - -#define APTROWS (LAST_AUTOTAB_LEFT_OP - FIRST_AUTOTAB_OP + 1) -#define APTCOLS (LAST_AUTOTAB_RIGHT_OP - FIRST_AUTOTAB_OP + 1) - -static const pcre_uint8 autoposstab[APTROWS][APTCOLS] = { -/* \D \d \S \s \W \w . .+ \C \P \p \R \H \h \V \v \X \Z \z $ $M */ - { 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 }, /* \D */ - { 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1 }, /* \d */ - { 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1 }, /* \S */ - { 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 }, /* \s */ - { 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 }, /* \W */ - { 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1 }, /* \w */ - { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0 }, /* . */ - { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 }, /* .+ */ - { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 }, /* \C */ - { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, /* \P */ - { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, /* \p */ - { 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0 }, /* \R */ - { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0 }, /* \H */ - { 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0 }, /* \h */ - { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0 }, /* \V */ - { 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0 }, /* \v */ - { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 } /* \X */ -}; - - -/* This table is used to check whether auto-possessification is possible -between adjacent Unicode property opcodes (OP_PROP and OP_NOTPROP). The -left-hand (repeated) opcode is used to select the row, and the right-hand -opcode is used to select the column. The values are as follows: - - 0 Always return FALSE (never auto-possessify) - 1 Character groups are distinct (possessify if both are OP_PROP) - 2 Check character categories in the same group (general or particular) - 3 TRUE if the two opcodes are not the same (PROP vs NOTPROP) - - 4 Check left general category vs right particular category - 5 Check right general category vs left particular category - - 6 Left alphanum vs right general category - 7 Left space vs right general category - 8 Left word vs right general category - - 9 Right alphanum vs left general category - 10 Right space vs left general category - 11 Right word vs left general category - - 12 Left alphanum vs right particular category - 13 Left space vs right particular category - 14 Left word vs right particular category - - 15 Right alphanum vs left particular category - 16 Right space vs left particular category - 17 Right word vs left particular category -*/ - -static const pcre_uint8 propposstab[PT_TABSIZE][PT_TABSIZE] = { -/* ANY LAMP GC PC SC ALNUM SPACE PXSPACE WORD CLIST UCNC */ - { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, /* PT_ANY */ - { 0, 3, 0, 0, 0, 3, 1, 1, 0, 0, 0 }, /* PT_LAMP */ - { 0, 0, 2, 4, 0, 9, 10, 10, 11, 0, 0 }, /* PT_GC */ - { 0, 0, 5, 2, 0, 15, 16, 16, 17, 0, 0 }, /* PT_PC */ - { 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0 }, /* PT_SC */ - { 0, 3, 6, 12, 0, 3, 1, 1, 0, 0, 0 }, /* PT_ALNUM */ - { 0, 1, 7, 13, 0, 1, 3, 3, 1, 0, 0 }, /* PT_SPACE */ - { 0, 1, 7, 13, 0, 1, 3, 3, 1, 0, 0 }, /* PT_PXSPACE */ - { 0, 0, 8, 14, 0, 0, 1, 1, 3, 0, 0 }, /* PT_WORD */ - { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, /* PT_CLIST */ - { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3 } /* PT_UCNC */ -}; - -/* This table is used to check whether auto-possessification is possible -between adjacent Unicode property opcodes (OP_PROP and OP_NOTPROP) when one -specifies a general category and the other specifies a particular category. The -row is selected by the general category and the column by the particular -category. The value is 1 if the particular category is not part of the general -category. */ - -static const pcre_uint8 catposstab[7][30] = { -/* Cc Cf Cn Co Cs Ll Lm Lo Lt Lu Mc Me Mn Nd Nl No Pc Pd Pe Pf Pi Po Ps Sc Sk Sm So Zl Zp Zs */ - { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, /* C */ - { 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, /* L */ - { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, /* M */ - { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, /* N */ - { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1 }, /* P */ - { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1 }, /* S */ - { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0 } /* Z */ -}; - -/* This table is used when checking ALNUM, (PX)SPACE, SPACE, and WORD against -a general or particular category. The properties in each row are those -that apply to the character set in question. Duplication means that a little -unnecessary work is done when checking, but this keeps things much simpler -because they can all use the same code. For more details see the comment where -this table is used. - -Note: SPACE and PXSPACE used to be different because Perl excluded VT from -"space", but from Perl 5.18 it's included, so both categories are treated the -same here. */ - -static const pcre_uint8 posspropstab[3][4] = { - { ucp_L, ucp_N, ucp_N, ucp_Nl }, /* ALNUM, 3rd and 4th values redundant */ - { ucp_Z, ucp_Z, ucp_C, ucp_Cc }, /* SPACE and PXSPACE, 2nd value redundant */ - { ucp_L, ucp_N, ucp_P, ucp_Po } /* WORD */ -}; - -/* This table is used when converting repeating opcodes into possessified -versions as a result of an explicit possessive quantifier such as ++. A zero -value means there is no possessified version - in those cases the item in -question must be wrapped in ONCE brackets. The table is truncated at OP_CALLOUT -because all relevant opcodes are less than that. */ - -static const pcre_uint8 opcode_possessify[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0 - 15 */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 16 - 31 */ - - 0, /* NOTI */ - OP_POSSTAR, 0, /* STAR, MINSTAR */ - OP_POSPLUS, 0, /* PLUS, MINPLUS */ - OP_POSQUERY, 0, /* QUERY, MINQUERY */ - OP_POSUPTO, 0, /* UPTO, MINUPTO */ - 0, /* EXACT */ - 0, 0, 0, 0, /* POS{STAR,PLUS,QUERY,UPTO} */ - - OP_POSSTARI, 0, /* STARI, MINSTARI */ - OP_POSPLUSI, 0, /* PLUSI, MINPLUSI */ - OP_POSQUERYI, 0, /* QUERYI, MINQUERYI */ - OP_POSUPTOI, 0, /* UPTOI, MINUPTOI */ - 0, /* EXACTI */ - 0, 0, 0, 0, /* POS{STARI,PLUSI,QUERYI,UPTOI} */ - - OP_NOTPOSSTAR, 0, /* NOTSTAR, NOTMINSTAR */ - OP_NOTPOSPLUS, 0, /* NOTPLUS, NOTMINPLUS */ - OP_NOTPOSQUERY, 0, /* NOTQUERY, NOTMINQUERY */ - OP_NOTPOSUPTO, 0, /* NOTUPTO, NOTMINUPTO */ - 0, /* NOTEXACT */ - 0, 0, 0, 0, /* NOTPOS{STAR,PLUS,QUERY,UPTO} */ - - OP_NOTPOSSTARI, 0, /* NOTSTARI, NOTMINSTARI */ - OP_NOTPOSPLUSI, 0, /* NOTPLUSI, NOTMINPLUSI */ - OP_NOTPOSQUERYI, 0, /* NOTQUERYI, NOTMINQUERYI */ - OP_NOTPOSUPTOI, 0, /* NOTUPTOI, NOTMINUPTOI */ - 0, /* NOTEXACTI */ - 0, 0, 0, 0, /* NOTPOS{STARI,PLUSI,QUERYI,UPTOI} */ - - OP_TYPEPOSSTAR, 0, /* TYPESTAR, TYPEMINSTAR */ - OP_TYPEPOSPLUS, 0, /* TYPEPLUS, TYPEMINPLUS */ - OP_TYPEPOSQUERY, 0, /* TYPEQUERY, TYPEMINQUERY */ - OP_TYPEPOSUPTO, 0, /* TYPEUPTO, TYPEMINUPTO */ - 0, /* TYPEEXACT */ - 0, 0, 0, 0, /* TYPEPOS{STAR,PLUS,QUERY,UPTO} */ - - OP_CRPOSSTAR, 0, /* CRSTAR, CRMINSTAR */ - OP_CRPOSPLUS, 0, /* CRPLUS, CRMINPLUS */ - OP_CRPOSQUERY, 0, /* CRQUERY, CRMINQUERY */ - OP_CRPOSRANGE, 0, /* CRRANGE, CRMINRANGE */ - 0, 0, 0, 0, /* CRPOS{STAR,PLUS,QUERY,RANGE} */ - - 0, 0, 0, /* CLASS, NCLASS, XCLASS */ - 0, 0, /* REF, REFI */ - 0, 0, /* DNREF, DNREFI */ - 0, 0 /* RECURSE, CALLOUT */ -}; - - - -/************************************************* -* Find an error text * -*************************************************/ - -/* The error texts are now all in one long string, to save on relocations. As -some of the text is of unknown length, we can't use a table of offsets. -Instead, just count through the strings. This is not a performance issue -because it happens only when there has been a compilation error. - -Argument: the error number -Returns: pointer to the error string -*/ - -static const char * -find_error_text(int n) -{ -const char *s = error_texts; -for (; n > 0; n--) - { - while (*s++ != CHAR_NULL) {}; - if (*s == CHAR_NULL) return "Error text not found (please report)"; - } -return s; -} - - - -/************************************************* -* Expand the workspace * -*************************************************/ - -/* This function is called during the second compiling phase, if the number of -forward references fills the existing workspace, which is originally a block on -the stack. A larger block is obtained from malloc() unless the ultimate limit -has been reached or the increase will be rather small. - -Argument: pointer to the compile data block -Returns: 0 if all went well, else an error number -*/ - -static int -expand_workspace(compile_data *cd) -{ -pcre_uchar *newspace; -int newsize = cd->workspace_size * 2; - -if (newsize > COMPILE_WORK_SIZE_MAX) newsize = COMPILE_WORK_SIZE_MAX; -if (cd->workspace_size >= COMPILE_WORK_SIZE_MAX || - newsize - cd->workspace_size < WORK_SIZE_SAFETY_MARGIN) - return ERR72; - -newspace = (PUBL(malloc))(IN_UCHARS(newsize)); -if (newspace == NULL) return ERR21; -memcpy(newspace, cd->start_workspace, cd->workspace_size * sizeof(pcre_uchar)); -cd->hwm = (pcre_uchar *)newspace + (cd->hwm - cd->start_workspace); -if (cd->workspace_size > COMPILE_WORK_SIZE) - (PUBL(free))((void *)cd->start_workspace); -cd->start_workspace = newspace; -cd->workspace_size = newsize; -return 0; -} - - - -/************************************************* -* Check for counted repeat * -*************************************************/ - -/* This function is called when a '{' is encountered in a place where it might -start a quantifier. It looks ahead to see if it really is a quantifier or not. -It is only a quantifier if it is one of the forms {ddd} {ddd,} or {ddd,ddd} -where the ddds are digits. - -Arguments: - p pointer to the first char after '{' - -Returns: TRUE or FALSE -*/ - -static BOOL -is_counted_repeat(const pcre_uchar *p) -{ -if (!IS_DIGIT(*p)) return FALSE; -p++; -while (IS_DIGIT(*p)) p++; -if (*p == CHAR_RIGHT_CURLY_BRACKET) return TRUE; - -if (*p++ != CHAR_COMMA) return FALSE; -if (*p == CHAR_RIGHT_CURLY_BRACKET) return TRUE; - -if (!IS_DIGIT(*p)) return FALSE; -p++; -while (IS_DIGIT(*p)) p++; - -return (*p == CHAR_RIGHT_CURLY_BRACKET); -} - - - -/************************************************* -* Handle escapes * -*************************************************/ - -/* This function is called when a \ has been encountered. It either returns a -positive value for a simple escape such as \n, or 0 for a data character which -will be placed in chptr. A backreference to group n is returned as negative n. -When UTF-8 is enabled, a positive value greater than 255 may be returned in -chptr. On entry, ptr is pointing at the \. On exit, it is on the final -character of the escape sequence. - -Arguments: - ptrptr points to the pattern position pointer - chptr points to a returned data character - errorcodeptr points to the errorcode variable - bracount number of previous extracting brackets - options the options bits - isclass TRUE if inside a character class - -Returns: zero => a data character - positive => a special escape sequence - negative => a back reference - on error, errorcodeptr is set -*/ - -static int -check_escape(const pcre_uchar **ptrptr, pcre_uint32 *chptr, int *errorcodeptr, - int bracount, int options, BOOL isclass) -{ -/* PCRE_UTF16 has the same value as PCRE_UTF8. */ -BOOL utf = (options & PCRE_UTF8) != 0; -const pcre_uchar *ptr = *ptrptr + 1; -pcre_uint32 c; -int escape = 0; -int i; - -GETCHARINCTEST(c, ptr); /* Get character value, increment pointer */ -ptr--; /* Set pointer back to the last byte */ - -/* If backslash is at the end of the pattern, it's an error. */ - -if (c == CHAR_NULL) *errorcodeptr = ERR1; - -/* Non-alphanumerics are literals. For digits or letters, do an initial lookup -in a table. A non-zero result is something that can be returned immediately. -Otherwise further processing may be required. */ - -#ifndef EBCDIC /* ASCII/UTF-8 coding */ -/* Not alphanumeric */ -else if (c < CHAR_0 || c > CHAR_z) {} -else if ((i = escapes[c - CHAR_0]) != 0) - { if (i > 0) c = (pcre_uint32)i; else escape = -i; } - -#else /* EBCDIC coding */ -/* Not alphanumeric */ -else if (c < CHAR_a || (!MAX_255(c) || (ebcdic_chartab[c] & 0x0E) == 0)) {} -else if ((i = escapes[c - 0x48]) != 0) { if (i > 0) c = (pcre_uint32)i; else escape = -i; } -#endif - -/* Escapes that need further processing, or are illegal. */ - -else - { - const pcre_uchar *oldptr; - BOOL braced, negated, overflow; - int s; - - switch (c) - { - /* A number of Perl escapes are not handled by PCRE. We give an explicit - error. */ - - case CHAR_l: - case CHAR_L: - *errorcodeptr = ERR37; - break; - - case CHAR_u: - if ((options & PCRE_JAVASCRIPT_COMPAT) != 0) - { - /* In JavaScript, \u must be followed by four hexadecimal numbers. - Otherwise it is a lowercase u letter. */ - if (MAX_255(ptr[1]) && (digitab[ptr[1]] & ctype_xdigit) != 0 - && MAX_255(ptr[2]) && (digitab[ptr[2]] & ctype_xdigit) != 0 - && MAX_255(ptr[3]) && (digitab[ptr[3]] & ctype_xdigit) != 0 - && MAX_255(ptr[4]) && (digitab[ptr[4]] & ctype_xdigit) != 0) - { - c = 0; - for (i = 0; i < 4; ++i) - { - register pcre_uint32 cc = *(++ptr); -#ifndef EBCDIC /* ASCII/UTF-8 coding */ - if (cc >= CHAR_a) cc -= 32; /* Convert to upper case */ - c = (c << 4) + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10)); -#else /* EBCDIC coding */ - if (cc >= CHAR_a && cc <= CHAR_z) cc += 64; /* Convert to upper case */ - c = (c << 4) + cc - ((cc >= CHAR_0)? CHAR_0 : (CHAR_A - 10)); -#endif - } - -#if defined COMPILE_PCRE8 - if (c > (utf ? 0x10ffffU : 0xffU)) -#elif defined COMPILE_PCRE16 - if (c > (utf ? 0x10ffffU : 0xffffU)) -#elif defined COMPILE_PCRE32 - if (utf && c > 0x10ffffU) -#endif - { - *errorcodeptr = ERR76; - } - else if (utf && c >= 0xd800 && c <= 0xdfff) *errorcodeptr = ERR73; - } - } - else - *errorcodeptr = ERR37; - break; - - case CHAR_U: - /* In JavaScript, \U is an uppercase U letter. */ - if ((options & PCRE_JAVASCRIPT_COMPAT) == 0) *errorcodeptr = ERR37; - break; - - /* In a character class, \g is just a literal "g". Outside a character - class, \g must be followed by one of a number of specific things: - - (1) A number, either plain or braced. If positive, it is an absolute - backreference. If negative, it is a relative backreference. This is a Perl - 5.10 feature. - - (2) Perl 5.10 also supports \g{name} as a reference to a named group. This - is part of Perl's movement towards a unified syntax for back references. As - this is synonymous with \k{name}, we fudge it up by pretending it really - was \k. - - (3) For Oniguruma compatibility we also support \g followed by a name or a - number either in angle brackets or in single quotes. However, these are - (possibly recursive) subroutine calls, _not_ backreferences. Just return - the ESC_g code (cf \k). */ - - case CHAR_g: - if (isclass) break; - if (ptr[1] == CHAR_LESS_THAN_SIGN || ptr[1] == CHAR_APOSTROPHE) - { - escape = ESC_g; - break; - } - - /* Handle the Perl-compatible cases */ - - if (ptr[1] == CHAR_LEFT_CURLY_BRACKET) - { - const pcre_uchar *p; - for (p = ptr+2; *p != CHAR_NULL && *p != CHAR_RIGHT_CURLY_BRACKET; p++) - if (*p != CHAR_MINUS && !IS_DIGIT(*p)) break; - if (*p != CHAR_NULL && *p != CHAR_RIGHT_CURLY_BRACKET) - { - escape = ESC_k; - break; - } - braced = TRUE; - ptr++; - } - else braced = FALSE; - - if (ptr[1] == CHAR_MINUS) - { - negated = TRUE; - ptr++; - } - else negated = FALSE; - - /* The integer range is limited by the machine's int representation. */ - s = 0; - overflow = FALSE; - while (IS_DIGIT(ptr[1])) - { - if (s > INT_MAX / 10 - 1) /* Integer overflow */ - { - overflow = TRUE; - break; - } - s = s * 10 + (int)(*(++ptr) - CHAR_0); - } - if (overflow) /* Integer overflow */ - { - while (IS_DIGIT(ptr[1])) - ptr++; - *errorcodeptr = ERR61; - break; - } - - if (braced && *(++ptr) != CHAR_RIGHT_CURLY_BRACKET) - { - *errorcodeptr = ERR57; - break; - } - - if (s == 0) - { - *errorcodeptr = ERR58; - break; - } - - if (negated) - { - if (s > bracount) - { - *errorcodeptr = ERR15; - break; - } - s = bracount - (s - 1); - } - - escape = -s; - break; - - /* The handling of escape sequences consisting of a string of digits - starting with one that is not zero is not straightforward. Perl has changed - over the years. Nowadays \g{} for backreferences and \o{} for octal are - recommended to avoid the ambiguities in the old syntax. - - Outside a character class, the digits are read as a decimal number. If the - number is less than 8 (used to be 10), or if there are that many previous - extracting left brackets, then it is a back reference. Otherwise, up to - three octal digits are read to form an escaped byte. Thus \123 is likely to - be octal 123 (cf \0123, which is octal 012 followed by the literal 3). If - the octal value is greater than 377, the least significant 8 bits are - taken. \8 and \9 are treated as the literal characters 8 and 9. - - Inside a character class, \ followed by a digit is always either a literal - 8 or 9 or an octal number. */ - - case CHAR_1: case CHAR_2: case CHAR_3: case CHAR_4: case CHAR_5: - case CHAR_6: case CHAR_7: case CHAR_8: case CHAR_9: - - if (!isclass) - { - oldptr = ptr; - /* The integer range is limited by the machine's int representation. */ - s = (int)(c -CHAR_0); - overflow = FALSE; - while (IS_DIGIT(ptr[1])) - { - if (s > INT_MAX / 10 - 1) /* Integer overflow */ - { - overflow = TRUE; - break; - } - s = s * 10 + (int)(*(++ptr) - CHAR_0); - } - if (overflow) /* Integer overflow */ - { - while (IS_DIGIT(ptr[1])) - ptr++; - *errorcodeptr = ERR61; - break; - } - if (s < 8 || s <= bracount) /* Check for back reference */ - { - escape = -s; - break; - } - ptr = oldptr; /* Put the pointer back and fall through */ - } - - /* Handle a digit following \ when the number is not a back reference. If - the first digit is 8 or 9, Perl used to generate a binary zero byte and - then treat the digit as a following literal. At least by Perl 5.18 this - changed so as not to insert the binary zero. */ - - if ((c = *ptr) >= CHAR_8) break; - - /* Fall through with a digit less than 8 */ - - /* \0 always starts an octal number, but we may drop through to here with a - larger first octal digit. The original code used just to take the least - significant 8 bits of octal numbers (I think this is what early Perls used - to do). Nowadays we allow for larger numbers in UTF-8 mode and 16-bit mode, - but no more than 3 octal digits. */ - - case CHAR_0: - c -= CHAR_0; - while(i++ < 2 && ptr[1] >= CHAR_0 && ptr[1] <= CHAR_7) - c = c * 8 + *(++ptr) - CHAR_0; -#ifdef COMPILE_PCRE8 - if (!utf && c > 0xff) *errorcodeptr = ERR51; -#endif - break; - - /* \o is a relatively new Perl feature, supporting a more general way of - specifying character codes in octal. The only supported form is \o{ddd}. */ - - case CHAR_o: - if (ptr[1] != CHAR_LEFT_CURLY_BRACKET) *errorcodeptr = ERR81; else - if (ptr[2] == CHAR_RIGHT_CURLY_BRACKET) *errorcodeptr = ERR86; else - { - ptr += 2; - c = 0; - overflow = FALSE; - while (*ptr >= CHAR_0 && *ptr <= CHAR_7) - { - register pcre_uint32 cc = *ptr++; - if (c == 0 && cc == CHAR_0) continue; /* Leading zeroes */ -#ifdef COMPILE_PCRE32 - if (c >= 0x20000000l) { overflow = TRUE; break; } -#endif - c = (c << 3) + cc - CHAR_0 ; -#if defined COMPILE_PCRE8 - if (c > (utf ? 0x10ffffU : 0xffU)) { overflow = TRUE; break; } -#elif defined COMPILE_PCRE16 - if (c > (utf ? 0x10ffffU : 0xffffU)) { overflow = TRUE; break; } -#elif defined COMPILE_PCRE32 - if (utf && c > 0x10ffffU) { overflow = TRUE; break; } -#endif - } - if (overflow) - { - while (*ptr >= CHAR_0 && *ptr <= CHAR_7) ptr++; - *errorcodeptr = ERR34; - } - else if (*ptr == CHAR_RIGHT_CURLY_BRACKET) - { - if (utf && c >= 0xd800 && c <= 0xdfff) *errorcodeptr = ERR73; - } - else *errorcodeptr = ERR80; - } - break; - - /* \x is complicated. In JavaScript, \x must be followed by two hexadecimal - numbers. Otherwise it is a lowercase x letter. */ - - case CHAR_x: - if ((options & PCRE_JAVASCRIPT_COMPAT) != 0) - { - if (MAX_255(ptr[1]) && (digitab[ptr[1]] & ctype_xdigit) != 0 - && MAX_255(ptr[2]) && (digitab[ptr[2]] & ctype_xdigit) != 0) - { - c = 0; - for (i = 0; i < 2; ++i) - { - register pcre_uint32 cc = *(++ptr); -#ifndef EBCDIC /* ASCII/UTF-8 coding */ - if (cc >= CHAR_a) cc -= 32; /* Convert to upper case */ - c = (c << 4) + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10)); -#else /* EBCDIC coding */ - if (cc >= CHAR_a && cc <= CHAR_z) cc += 64; /* Convert to upper case */ - c = (c << 4) + cc - ((cc >= CHAR_0)? CHAR_0 : (CHAR_A - 10)); -#endif - } - } - } /* End JavaScript handling */ - - /* Handle \x in Perl's style. \x{ddd} is a character number which can be - greater than 0xff in utf or non-8bit mode, but only if the ddd are hex - digits. If not, { used to be treated as a data character. However, Perl - seems to read hex digits up to the first non-such, and ignore the rest, so - that, for example \x{zz} matches a binary zero. This seems crazy, so PCRE - now gives an error. */ - - else - { - if (ptr[1] == CHAR_LEFT_CURLY_BRACKET) - { - ptr += 2; - if (*ptr == CHAR_RIGHT_CURLY_BRACKET) - { - *errorcodeptr = ERR86; - break; - } - c = 0; - overflow = FALSE; - while (MAX_255(*ptr) && (digitab[*ptr] & ctype_xdigit) != 0) - { - register pcre_uint32 cc = *ptr++; - if (c == 0 && cc == CHAR_0) continue; /* Leading zeroes */ - -#ifdef COMPILE_PCRE32 - if (c >= 0x10000000l) { overflow = TRUE; break; } -#endif - -#ifndef EBCDIC /* ASCII/UTF-8 coding */ - if (cc >= CHAR_a) cc -= 32; /* Convert to upper case */ - c = (c << 4) + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10)); -#else /* EBCDIC coding */ - if (cc >= CHAR_a && cc <= CHAR_z) cc += 64; /* Convert to upper case */ - c = (c << 4) + cc - ((cc >= CHAR_0)? CHAR_0 : (CHAR_A - 10)); -#endif - -#if defined COMPILE_PCRE8 - if (c > (utf ? 0x10ffffU : 0xffU)) { overflow = TRUE; break; } -#elif defined COMPILE_PCRE16 - if (c > (utf ? 0x10ffffU : 0xffffU)) { overflow = TRUE; break; } -#elif defined COMPILE_PCRE32 - if (utf && c > 0x10ffffU) { overflow = TRUE; break; } -#endif - } - - if (overflow) - { - while (MAX_255(*ptr) && (digitab[*ptr] & ctype_xdigit) != 0) ptr++; - *errorcodeptr = ERR34; - } - - else if (*ptr == CHAR_RIGHT_CURLY_BRACKET) - { - if (utf && c >= 0xd800 && c <= 0xdfff) *errorcodeptr = ERR73; - } - - /* If the sequence of hex digits does not end with '}', give an error. - We used just to recognize this construct and fall through to the normal - \x handling, but nowadays Perl gives an error, which seems much more - sensible, so we do too. */ - - else *errorcodeptr = ERR79; - } /* End of \x{} processing */ - - /* Read a single-byte hex-defined char (up to two hex digits after \x) */ - - else - { - c = 0; - while (i++ < 2 && MAX_255(ptr[1]) && (digitab[ptr[1]] & ctype_xdigit) != 0) - { - pcre_uint32 cc; /* Some compilers don't like */ - cc = *(++ptr); /* ++ in initializers */ -#ifndef EBCDIC /* ASCII/UTF-8 coding */ - if (cc >= CHAR_a) cc -= 32; /* Convert to upper case */ - c = c * 16 + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10)); -#else /* EBCDIC coding */ - if (cc <= CHAR_z) cc += 64; /* Convert to upper case */ - c = c * 16 + cc - ((cc >= CHAR_0)? CHAR_0 : (CHAR_A - 10)); -#endif - } - } /* End of \xdd handling */ - } /* End of Perl-style \x handling */ - break; - - /* For \c, a following letter is upper-cased; then the 0x40 bit is flipped. - An error is given if the byte following \c is not an ASCII character. This - coding is ASCII-specific, but then the whole concept of \cx is - ASCII-specific. (However, an EBCDIC equivalent has now been added.) */ - - case CHAR_c: - c = *(++ptr); - if (c == CHAR_NULL) - { - *errorcodeptr = ERR2; - break; - } -#ifndef EBCDIC /* ASCII/UTF-8 coding */ - if (c > 127) /* Excludes all non-ASCII in either mode */ - { - *errorcodeptr = ERR68; - break; - } - if (c >= CHAR_a && c <= CHAR_z) c -= 32; - c ^= 0x40; -#else /* EBCDIC coding */ - if (c >= CHAR_a && c <= CHAR_z) c += 64; - if (c == CHAR_QUESTION_MARK) - c = ('\\' == 188 && '`' == 74)? 0x5f : 0xff; - else - { - for (i = 0; i < 32; i++) - { - if (c == ebcdic_escape_c[i]) break; - } - if (i < 32) c = i; else *errorcodeptr = ERR68; - } -#endif - break; - - /* PCRE_EXTRA enables extensions to Perl in the matter of escapes. Any - other alphanumeric following \ is an error if PCRE_EXTRA was set; - otherwise, for Perl compatibility, it is a literal. This code looks a bit - odd, but there used to be some cases other than the default, and there may - be again in future, so I haven't "optimized" it. */ - - default: - if ((options & PCRE_EXTRA) != 0) switch(c) - { - default: - *errorcodeptr = ERR3; - break; - } - break; - } - } - -/* Perl supports \N{name} for character names, as well as plain \N for "not -newline". PCRE does not support \N{name}. However, it does support -quantification such as \N{2,3}. */ - -if (escape == ESC_N && ptr[1] == CHAR_LEFT_CURLY_BRACKET && - !is_counted_repeat(ptr+2)) - *errorcodeptr = ERR37; - -/* If PCRE_UCP is set, we change the values for \d etc. */ - -if ((options & PCRE_UCP) != 0 && escape >= ESC_D && escape <= ESC_w) - escape += (ESC_DU - ESC_D); - -/* Set the pointer to the final character before returning. */ - -*ptrptr = ptr; -*chptr = c; -return escape; -} - - - -#ifdef SUPPORT_UCP -/************************************************* -* Handle \P and \p * -*************************************************/ - -/* This function is called after \P or \p has been encountered, provided that -PCRE is compiled with support for Unicode properties. On entry, ptrptr is -pointing at the P or p. On exit, it is pointing at the final character of the -escape sequence. - -Argument: - ptrptr points to the pattern position pointer - negptr points to a boolean that is set TRUE for negation else FALSE - ptypeptr points to an unsigned int that is set to the type value - pdataptr points to an unsigned int that is set to the detailed property value - errorcodeptr points to the error code variable - -Returns: TRUE if the type value was found, or FALSE for an invalid type -*/ - -static BOOL -get_ucp(const pcre_uchar **ptrptr, BOOL *negptr, unsigned int *ptypeptr, - unsigned int *pdataptr, int *errorcodeptr) -{ -pcre_uchar c; -int i, bot, top; -const pcre_uchar *ptr = *ptrptr; -pcre_uchar name[32]; - -c = *(++ptr); -if (c == CHAR_NULL) goto ERROR_RETURN; - -*negptr = FALSE; - -/* \P or \p can be followed by a name in {}, optionally preceded by ^ for -negation. */ - -if (c == CHAR_LEFT_CURLY_BRACKET) - { - if (ptr[1] == CHAR_CIRCUMFLEX_ACCENT) - { - *negptr = TRUE; - ptr++; - } - for (i = 0; i < (int)(sizeof(name) / sizeof(pcre_uchar)) - 1; i++) - { - c = *(++ptr); - if (c == CHAR_NULL) goto ERROR_RETURN; - if (c == CHAR_RIGHT_CURLY_BRACKET) break; - name[i] = c; - } - if (c != CHAR_RIGHT_CURLY_BRACKET) goto ERROR_RETURN; - name[i] = 0; - } - -/* Otherwise there is just one following character */ - -else - { - name[0] = c; - name[1] = 0; - } - -*ptrptr = ptr; - -/* Search for a recognized property name using binary chop */ - -bot = 0; -top = PRIV(utt_size); - -while (bot < top) - { - int r; - i = (bot + top) >> 1; - r = STRCMP_UC_C8(name, PRIV(utt_names) + PRIV(utt)[i].name_offset); - if (r == 0) - { - *ptypeptr = PRIV(utt)[i].type; - *pdataptr = PRIV(utt)[i].value; - return TRUE; - } - if (r > 0) bot = i + 1; else top = i; - } - -*errorcodeptr = ERR47; -*ptrptr = ptr; -return FALSE; - -ERROR_RETURN: -*errorcodeptr = ERR46; -*ptrptr = ptr; -return FALSE; -} -#endif - - - -/************************************************* -* Read repeat counts * -*************************************************/ - -/* Read an item of the form {n,m} and return the values. This is called only -after is_counted_repeat() has confirmed that a repeat-count quantifier exists, -so the syntax is guaranteed to be correct, but we need to check the values. - -Arguments: - p pointer to first char after '{' - minp pointer to int for min - maxp pointer to int for max - returned as -1 if no max - errorcodeptr points to error code variable - -Returns: pointer to '}' on success; - current ptr on error, with errorcodeptr set non-zero -*/ - -static const pcre_uchar * -read_repeat_counts(const pcre_uchar *p, int *minp, int *maxp, int *errorcodeptr) -{ -int min = 0; -int max = -1; - -while (IS_DIGIT(*p)) - { - min = min * 10 + (int)(*p++ - CHAR_0); - if (min > 65535) - { - *errorcodeptr = ERR5; - return p; - } - } - -if (*p == CHAR_RIGHT_CURLY_BRACKET) max = min; else - { - if (*(++p) != CHAR_RIGHT_CURLY_BRACKET) - { - max = 0; - while(IS_DIGIT(*p)) - { - max = max * 10 + (int)(*p++ - CHAR_0); - if (max > 65535) - { - *errorcodeptr = ERR5; - return p; - } - } - if (max < min) - { - *errorcodeptr = ERR4; - return p; - } - } - } - -*minp = min; -*maxp = max; -return p; -} - - - -/************************************************* -* Find first significant op code * -*************************************************/ - -/* This is called by several functions that scan a compiled expression looking -for a fixed first character, or an anchoring op code etc. It skips over things -that do not influence this. For some calls, it makes sense to skip negative -forward and all backward assertions, and also the \b assertion; for others it -does not. - -Arguments: - code pointer to the start of the group - skipassert TRUE if certain assertions are to be skipped - -Returns: pointer to the first significant opcode -*/ - -static const pcre_uchar* -first_significant_code(const pcre_uchar *code, BOOL skipassert) -{ -for (;;) - { - switch ((int)*code) - { - case OP_ASSERT_NOT: - case OP_ASSERTBACK: - case OP_ASSERTBACK_NOT: - if (!skipassert) return code; - do code += GET(code, 1); while (*code == OP_ALT); - code += PRIV(OP_lengths)[*code]; - break; - - case OP_WORD_BOUNDARY: - case OP_NOT_WORD_BOUNDARY: - if (!skipassert) return code; - /* Fall through */ - - case OP_CALLOUT: - case OP_CREF: - case OP_DNCREF: - case OP_RREF: - case OP_DNRREF: - case OP_DEF: - code += PRIV(OP_lengths)[*code]; - break; - - default: - return code; - } - } -/* Control never reaches here */ -} - - - -/************************************************* -* Find the fixed length of a branch * -*************************************************/ - -/* Scan a branch and compute the fixed length of subject that will match it, -if the length is fixed. This is needed for dealing with backward assertions. -In UTF8 mode, the result is in characters rather than bytes. The branch is -temporarily terminated with OP_END when this function is called. - -This function is called when a backward assertion is encountered, so that if it -fails, the error message can point to the correct place in the pattern. -However, we cannot do this when the assertion contains subroutine calls, -because they can be forward references. We solve this by remembering this case -and doing the check at the end; a flag specifies which mode we are running in. - -Arguments: - code points to the start of the pattern (the bracket) - utf TRUE in UTF-8 / UTF-16 / UTF-32 mode - atend TRUE if called when the pattern is complete - cd the "compile data" structure - recurses chain of recurse_check to catch mutual recursion - -Returns: the fixed length, - or -1 if there is no fixed length, - or -2 if \C was encountered (in UTF-8 mode only) - or -3 if an OP_RECURSE item was encountered and atend is FALSE - or -4 if an unknown opcode was encountered (internal error) -*/ - -static int -find_fixedlength(pcre_uchar *code, BOOL utf, BOOL atend, compile_data *cd, - recurse_check *recurses) -{ -int length = -1; -recurse_check this_recurse; -register int branchlength = 0; -register pcre_uchar *cc = code + 1 + LINK_SIZE; - -/* Scan along the opcodes for this branch. If we get to the end of the -branch, check the length against that of the other branches. */ - -for (;;) - { - int d; - pcre_uchar *ce, *cs; - register pcre_uchar op = *cc; - - switch (op) - { - /* We only need to continue for OP_CBRA (normal capturing bracket) and - OP_BRA (normal non-capturing bracket) because the other variants of these - opcodes are all concerned with unlimited repeated groups, which of course - are not of fixed length. */ - - case OP_CBRA: - case OP_BRA: - case OP_ONCE: - case OP_ONCE_NC: - case OP_COND: - d = find_fixedlength(cc + ((op == OP_CBRA)? IMM2_SIZE : 0), utf, atend, cd, - recurses); - if (d < 0) return d; - branchlength += d; - do cc += GET(cc, 1); while (*cc == OP_ALT); - cc += 1 + LINK_SIZE; - break; - - /* Reached end of a branch; if it's a ket it is the end of a nested call. - If it's ALT it is an alternation in a nested call. An ACCEPT is effectively - an ALT. If it is END it's the end of the outer call. All can be handled by - the same code. Note that we must not include the OP_KETRxxx opcodes here, - because they all imply an unlimited repeat. */ - - case OP_ALT: - case OP_KET: - case OP_END: - case OP_ACCEPT: - case OP_ASSERT_ACCEPT: - if (length < 0) length = branchlength; - else if (length != branchlength) return -1; - if (*cc != OP_ALT) return length; - cc += 1 + LINK_SIZE; - branchlength = 0; - break; - - /* A true recursion implies not fixed length, but a subroutine call may - be OK. If the subroutine is a forward reference, we can't deal with - it until the end of the pattern, so return -3. */ - - case OP_RECURSE: - if (!atend) return -3; - cs = ce = (pcre_uchar *)cd->start_code + GET(cc, 1); /* Start subpattern */ - do ce += GET(ce, 1); while (*ce == OP_ALT); /* End subpattern */ - if (cc > cs && cc < ce) return -1; /* Recursion */ - else /* Check for mutual recursion */ - { - recurse_check *r = recurses; - for (r = recurses; r != NULL; r = r->prev) if (r->group == cs) break; - if (r != NULL) return -1; /* Mutual recursion */ - } - this_recurse.prev = recurses; - this_recurse.group = cs; - d = find_fixedlength(cs + IMM2_SIZE, utf, atend, cd, &this_recurse); - if (d < 0) return d; - branchlength += d; - cc += 1 + LINK_SIZE; - break; - - /* Skip over assertive subpatterns */ - - case OP_ASSERT: - case OP_ASSERT_NOT: - case OP_ASSERTBACK: - case OP_ASSERTBACK_NOT: - do cc += GET(cc, 1); while (*cc == OP_ALT); - cc += 1 + LINK_SIZE; - break; - - /* Skip over things that don't match chars */ - - case OP_MARK: - case OP_PRUNE_ARG: - case OP_SKIP_ARG: - case OP_THEN_ARG: - cc += cc[1] + PRIV(OP_lengths)[*cc]; - break; - - case OP_CALLOUT: - case OP_CIRC: - case OP_CIRCM: - case OP_CLOSE: - case OP_COMMIT: - case OP_CREF: - case OP_DEF: - case OP_DNCREF: - case OP_DNRREF: - case OP_DOLL: - case OP_DOLLM: - case OP_EOD: - case OP_EODN: - case OP_FAIL: - case OP_NOT_WORD_BOUNDARY: - case OP_PRUNE: - case OP_REVERSE: - case OP_RREF: - case OP_SET_SOM: - case OP_SKIP: - case OP_SOD: - case OP_SOM: - case OP_THEN: - case OP_WORD_BOUNDARY: - cc += PRIV(OP_lengths)[*cc]; - break; - - /* Handle literal characters */ - - case OP_CHAR: - case OP_CHARI: - case OP_NOT: - case OP_NOTI: - branchlength++; - cc += 2; -#ifdef SUPPORT_UTF - if (utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); -#endif - break; - - /* Handle exact repetitions. The count is already in characters, but we - need to skip over a multibyte character in UTF8 mode. */ - - case OP_EXACT: - case OP_EXACTI: - case OP_NOTEXACT: - case OP_NOTEXACTI: - branchlength += (int)GET2(cc,1); - cc += 2 + IMM2_SIZE; -#ifdef SUPPORT_UTF - if (utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); -#endif - break; - - case OP_TYPEEXACT: - branchlength += GET2(cc,1); - if (cc[1 + IMM2_SIZE] == OP_PROP || cc[1 + IMM2_SIZE] == OP_NOTPROP) - cc += 2; - cc += 1 + IMM2_SIZE + 1; - break; - - /* Handle single-char matchers */ - - case OP_PROP: - case OP_NOTPROP: - cc += 2; - /* Fall through */ - - case OP_HSPACE: - case OP_VSPACE: - case OP_NOT_HSPACE: - case OP_NOT_VSPACE: - case OP_NOT_DIGIT: - case OP_DIGIT: - case OP_NOT_WHITESPACE: - case OP_WHITESPACE: - case OP_NOT_WORDCHAR: - case OP_WORDCHAR: - case OP_ANY: - case OP_ALLANY: - branchlength++; - cc++; - break; - - /* The single-byte matcher isn't allowed. This only happens in UTF-8 mode; - otherwise \C is coded as OP_ALLANY. */ - - case OP_ANYBYTE: - return -2; - - /* Check a class for variable quantification */ - - case OP_CLASS: - case OP_NCLASS: -#if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - case OP_XCLASS: - /* The original code caused an unsigned overflow in 64 bit systems, - so now we use a conditional statement. */ - if (op == OP_XCLASS) - cc += GET(cc, 1); - else - cc += PRIV(OP_lengths)[OP_CLASS]; -#else - cc += PRIV(OP_lengths)[OP_CLASS]; -#endif - - switch (*cc) - { - case OP_CRSTAR: - case OP_CRMINSTAR: - case OP_CRPLUS: - case OP_CRMINPLUS: - case OP_CRQUERY: - case OP_CRMINQUERY: - case OP_CRPOSSTAR: - case OP_CRPOSPLUS: - case OP_CRPOSQUERY: - return -1; - - case OP_CRRANGE: - case OP_CRMINRANGE: - case OP_CRPOSRANGE: - if (GET2(cc,1) != GET2(cc,1+IMM2_SIZE)) return -1; - branchlength += (int)GET2(cc,1); - cc += 1 + 2 * IMM2_SIZE; - break; - - default: - branchlength++; - } - break; - - /* Anything else is variable length */ - - case OP_ANYNL: - case OP_BRAMINZERO: - case OP_BRAPOS: - case OP_BRAPOSZERO: - case OP_BRAZERO: - case OP_CBRAPOS: - case OP_EXTUNI: - case OP_KETRMAX: - case OP_KETRMIN: - case OP_KETRPOS: - case OP_MINPLUS: - case OP_MINPLUSI: - case OP_MINQUERY: - case OP_MINQUERYI: - case OP_MINSTAR: - case OP_MINSTARI: - case OP_MINUPTO: - case OP_MINUPTOI: - case OP_NOTMINPLUS: - case OP_NOTMINPLUSI: - case OP_NOTMINQUERY: - case OP_NOTMINQUERYI: - case OP_NOTMINSTAR: - case OP_NOTMINSTARI: - case OP_NOTMINUPTO: - case OP_NOTMINUPTOI: - case OP_NOTPLUS: - case OP_NOTPLUSI: - case OP_NOTPOSPLUS: - case OP_NOTPOSPLUSI: - case OP_NOTPOSQUERY: - case OP_NOTPOSQUERYI: - case OP_NOTPOSSTAR: - case OP_NOTPOSSTARI: - case OP_NOTPOSUPTO: - case OP_NOTPOSUPTOI: - case OP_NOTQUERY: - case OP_NOTQUERYI: - case OP_NOTSTAR: - case OP_NOTSTARI: - case OP_NOTUPTO: - case OP_NOTUPTOI: - case OP_PLUS: - case OP_PLUSI: - case OP_POSPLUS: - case OP_POSPLUSI: - case OP_POSQUERY: - case OP_POSQUERYI: - case OP_POSSTAR: - case OP_POSSTARI: - case OP_POSUPTO: - case OP_POSUPTOI: - case OP_QUERY: - case OP_QUERYI: - case OP_REF: - case OP_REFI: - case OP_DNREF: - case OP_DNREFI: - case OP_SBRA: - case OP_SBRAPOS: - case OP_SCBRA: - case OP_SCBRAPOS: - case OP_SCOND: - case OP_SKIPZERO: - case OP_STAR: - case OP_STARI: - case OP_TYPEMINPLUS: - case OP_TYPEMINQUERY: - case OP_TYPEMINSTAR: - case OP_TYPEMINUPTO: - case OP_TYPEPLUS: - case OP_TYPEPOSPLUS: - case OP_TYPEPOSQUERY: - case OP_TYPEPOSSTAR: - case OP_TYPEPOSUPTO: - case OP_TYPEQUERY: - case OP_TYPESTAR: - case OP_TYPEUPTO: - case OP_UPTO: - case OP_UPTOI: - return -1; - - /* Catch unrecognized opcodes so that when new ones are added they - are not forgotten, as has happened in the past. */ - - default: - return -4; - } - } -/* Control never gets here */ -} - - - -/************************************************* -* Scan compiled regex for specific bracket * -*************************************************/ - -/* This little function scans through a compiled pattern until it finds a -capturing bracket with the given number, or, if the number is negative, an -instance of OP_REVERSE for a lookbehind. The function is global in the C sense -so that it can be called from pcre_study() when finding the minimum matching -length. - -Arguments: - code points to start of expression - utf TRUE in UTF-8 / UTF-16 / UTF-32 mode - number the required bracket number or negative to find a lookbehind - -Returns: pointer to the opcode for the bracket, or NULL if not found -*/ - -const pcre_uchar * -PRIV(find_bracket)(const pcre_uchar *code, BOOL utf, int number) -{ -for (;;) - { - register pcre_uchar c = *code; - - if (c == OP_END) return NULL; - - /* XCLASS is used for classes that cannot be represented just by a bit - map. This includes negated single high-valued characters. The length in - the table is zero; the actual length is stored in the compiled code. */ - - if (c == OP_XCLASS) code += GET(code, 1); - - /* Handle recursion */ - - else if (c == OP_REVERSE) - { - if (number < 0) return (pcre_uchar *)code; - code += PRIV(OP_lengths)[c]; - } - - /* Handle capturing bracket */ - - else if (c == OP_CBRA || c == OP_SCBRA || - c == OP_CBRAPOS || c == OP_SCBRAPOS) - { - int n = (int)GET2(code, 1+LINK_SIZE); - if (n == number) return (pcre_uchar *)code; - code += PRIV(OP_lengths)[c]; - } - - /* Otherwise, we can get the item's length from the table, except that for - repeated character types, we have to test for \p and \P, which have an extra - two bytes of parameters, and for MARK/PRUNE/SKIP/THEN with an argument, we - must add in its length. */ - - else - { - switch(c) - { - case OP_TYPESTAR: - case OP_TYPEMINSTAR: - case OP_TYPEPLUS: - case OP_TYPEMINPLUS: - case OP_TYPEQUERY: - case OP_TYPEMINQUERY: - case OP_TYPEPOSSTAR: - case OP_TYPEPOSPLUS: - case OP_TYPEPOSQUERY: - if (code[1] == OP_PROP || code[1] == OP_NOTPROP) code += 2; - break; - - case OP_TYPEUPTO: - case OP_TYPEMINUPTO: - case OP_TYPEEXACT: - case OP_TYPEPOSUPTO: - if (code[1 + IMM2_SIZE] == OP_PROP || code[1 + IMM2_SIZE] == OP_NOTPROP) - code += 2; - break; - - case OP_MARK: - case OP_PRUNE_ARG: - case OP_SKIP_ARG: - case OP_THEN_ARG: - code += code[1]; - break; - } - - /* Add in the fixed length from the table */ - - code += PRIV(OP_lengths)[c]; - - /* In UTF-8 mode, opcodes that are followed by a character may be followed by - a multi-byte character. The length in the table is a minimum, so we have to - arrange to skip the extra bytes. */ - -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (utf) switch(c) - { - case OP_CHAR: - case OP_CHARI: - case OP_NOT: - case OP_NOTI: - case OP_EXACT: - case OP_EXACTI: - case OP_NOTEXACT: - case OP_NOTEXACTI: - case OP_UPTO: - case OP_UPTOI: - case OP_NOTUPTO: - case OP_NOTUPTOI: - case OP_MINUPTO: - case OP_MINUPTOI: - case OP_NOTMINUPTO: - case OP_NOTMINUPTOI: - case OP_POSUPTO: - case OP_POSUPTOI: - case OP_NOTPOSUPTO: - case OP_NOTPOSUPTOI: - case OP_STAR: - case OP_STARI: - case OP_NOTSTAR: - case OP_NOTSTARI: - case OP_MINSTAR: - case OP_MINSTARI: - case OP_NOTMINSTAR: - case OP_NOTMINSTARI: - case OP_POSSTAR: - case OP_POSSTARI: - case OP_NOTPOSSTAR: - case OP_NOTPOSSTARI: - case OP_PLUS: - case OP_PLUSI: - case OP_NOTPLUS: - case OP_NOTPLUSI: - case OP_MINPLUS: - case OP_MINPLUSI: - case OP_NOTMINPLUS: - case OP_NOTMINPLUSI: - case OP_POSPLUS: - case OP_POSPLUSI: - case OP_NOTPOSPLUS: - case OP_NOTPOSPLUSI: - case OP_QUERY: - case OP_QUERYI: - case OP_NOTQUERY: - case OP_NOTQUERYI: - case OP_MINQUERY: - case OP_MINQUERYI: - case OP_NOTMINQUERY: - case OP_NOTMINQUERYI: - case OP_POSQUERY: - case OP_POSQUERYI: - case OP_NOTPOSQUERY: - case OP_NOTPOSQUERYI: - if (HAS_EXTRALEN(code[-1])) code += GET_EXTRALEN(code[-1]); - break; - } -#else - (void)(utf); /* Keep compiler happy by referencing function argument */ -#endif - } - } -} - - - -/************************************************* -* Scan compiled regex for recursion reference * -*************************************************/ - -/* This little function scans through a compiled pattern until it finds an -instance of OP_RECURSE. - -Arguments: - code points to start of expression - utf TRUE in UTF-8 / UTF-16 / UTF-32 mode - -Returns: pointer to the opcode for OP_RECURSE, or NULL if not found -*/ - -static const pcre_uchar * -find_recurse(const pcre_uchar *code, BOOL utf) -{ -for (;;) - { - register pcre_uchar c = *code; - if (c == OP_END) return NULL; - if (c == OP_RECURSE) return code; - - /* XCLASS is used for classes that cannot be represented just by a bit - map. This includes negated single high-valued characters. The length in - the table is zero; the actual length is stored in the compiled code. */ - - if (c == OP_XCLASS) code += GET(code, 1); - - /* Otherwise, we can get the item's length from the table, except that for - repeated character types, we have to test for \p and \P, which have an extra - two bytes of parameters, and for MARK/PRUNE/SKIP/THEN with an argument, we - must add in its length. */ - - else - { - switch(c) - { - case OP_TYPESTAR: - case OP_TYPEMINSTAR: - case OP_TYPEPLUS: - case OP_TYPEMINPLUS: - case OP_TYPEQUERY: - case OP_TYPEMINQUERY: - case OP_TYPEPOSSTAR: - case OP_TYPEPOSPLUS: - case OP_TYPEPOSQUERY: - if (code[1] == OP_PROP || code[1] == OP_NOTPROP) code += 2; - break; - - case OP_TYPEPOSUPTO: - case OP_TYPEUPTO: - case OP_TYPEMINUPTO: - case OP_TYPEEXACT: - if (code[1 + IMM2_SIZE] == OP_PROP || code[1 + IMM2_SIZE] == OP_NOTPROP) - code += 2; - break; - - case OP_MARK: - case OP_PRUNE_ARG: - case OP_SKIP_ARG: - case OP_THEN_ARG: - code += code[1]; - break; - } - - /* Add in the fixed length from the table */ - - code += PRIV(OP_lengths)[c]; - - /* In UTF-8 mode, opcodes that are followed by a character may be followed - by a multi-byte character. The length in the table is a minimum, so we have - to arrange to skip the extra bytes. */ - -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (utf) switch(c) - { - case OP_CHAR: - case OP_CHARI: - case OP_NOT: - case OP_NOTI: - case OP_EXACT: - case OP_EXACTI: - case OP_NOTEXACT: - case OP_NOTEXACTI: - case OP_UPTO: - case OP_UPTOI: - case OP_NOTUPTO: - case OP_NOTUPTOI: - case OP_MINUPTO: - case OP_MINUPTOI: - case OP_NOTMINUPTO: - case OP_NOTMINUPTOI: - case OP_POSUPTO: - case OP_POSUPTOI: - case OP_NOTPOSUPTO: - case OP_NOTPOSUPTOI: - case OP_STAR: - case OP_STARI: - case OP_NOTSTAR: - case OP_NOTSTARI: - case OP_MINSTAR: - case OP_MINSTARI: - case OP_NOTMINSTAR: - case OP_NOTMINSTARI: - case OP_POSSTAR: - case OP_POSSTARI: - case OP_NOTPOSSTAR: - case OP_NOTPOSSTARI: - case OP_PLUS: - case OP_PLUSI: - case OP_NOTPLUS: - case OP_NOTPLUSI: - case OP_MINPLUS: - case OP_MINPLUSI: - case OP_NOTMINPLUS: - case OP_NOTMINPLUSI: - case OP_POSPLUS: - case OP_POSPLUSI: - case OP_NOTPOSPLUS: - case OP_NOTPOSPLUSI: - case OP_QUERY: - case OP_QUERYI: - case OP_NOTQUERY: - case OP_NOTQUERYI: - case OP_MINQUERY: - case OP_MINQUERYI: - case OP_NOTMINQUERY: - case OP_NOTMINQUERYI: - case OP_POSQUERY: - case OP_POSQUERYI: - case OP_NOTPOSQUERY: - case OP_NOTPOSQUERYI: - if (HAS_EXTRALEN(code[-1])) code += GET_EXTRALEN(code[-1]); - break; - } -#else - (void)(utf); /* Keep compiler happy by referencing function argument */ -#endif - } - } -} - - - -/************************************************* -* Scan compiled branch for non-emptiness * -*************************************************/ - -/* This function scans through a branch of a compiled pattern to see whether it -can match the empty string or not. It is called from could_be_empty() -below and from compile_branch() when checking for an unlimited repeat of a -group that can match nothing. Note that first_significant_code() skips over -backward and negative forward assertions when its final argument is TRUE. If we -hit an unclosed bracket, we return "empty" - this means we've struck an inner -bracket whose current branch will already have been scanned. - -Arguments: - code points to start of search - endcode points to where to stop - utf TRUE if in UTF-8 / UTF-16 / UTF-32 mode - cd contains pointers to tables etc. - recurses chain of recurse_check to catch mutual recursion - -Returns: TRUE if what is matched could be empty -*/ - -static BOOL -could_be_empty_branch(const pcre_uchar *code, const pcre_uchar *endcode, - BOOL utf, compile_data *cd, recurse_check *recurses) -{ -register pcre_uchar c; -recurse_check this_recurse; - -for (code = first_significant_code(code + PRIV(OP_lengths)[*code], TRUE); - code < endcode; - code = first_significant_code(code + PRIV(OP_lengths)[c], TRUE)) - { - const pcre_uchar *ccode; - - c = *code; - - /* Skip over forward assertions; the other assertions are skipped by - first_significant_code() with a TRUE final argument. */ - - if (c == OP_ASSERT) - { - do code += GET(code, 1); while (*code == OP_ALT); - c = *code; - continue; - } - - /* For a recursion/subroutine call, if its end has been reached, which - implies a backward reference subroutine call, we can scan it. If it's a - forward reference subroutine call, we can't. To detect forward reference - we have to scan up the list that is kept in the workspace. This function is - called only when doing the real compile, not during the pre-compile that - measures the size of the compiled pattern. */ - - if (c == OP_RECURSE) - { - const pcre_uchar *scode = cd->start_code + GET(code, 1); - const pcre_uchar *endgroup = scode; - BOOL empty_branch; - - /* Test for forward reference or uncompleted reference. This is disabled - when called to scan a completed pattern by setting cd->start_workspace to - NULL. */ - - if (cd->start_workspace != NULL) - { - const pcre_uchar *tcode; - for (tcode = cd->start_workspace; tcode < cd->hwm; tcode += LINK_SIZE) - if ((int)GET(tcode, 0) == (int)(code + 1 - cd->start_code)) return TRUE; - if (GET(scode, 1) == 0) return TRUE; /* Unclosed */ - } - - /* If the reference is to a completed group, we need to detect whether this - is a recursive call, as otherwise there will be an infinite loop. If it is - a recursion, just skip over it. Simple recursions are easily detected. For - mutual recursions we keep a chain on the stack. */ - - do endgroup += GET(endgroup, 1); while (*endgroup == OP_ALT); - if (code >= scode && code <= endgroup) continue; /* Simple recursion */ - else - { - recurse_check *r = recurses; - for (r = recurses; r != NULL; r = r->prev) - if (r->group == scode) break; - if (r != NULL) continue; /* Mutual recursion */ - } - - /* Completed reference; scan the referenced group, remembering it on the - stack chain to detect mutual recursions. */ - - empty_branch = FALSE; - this_recurse.prev = recurses; - this_recurse.group = scode; - - do - { - if (could_be_empty_branch(scode, endcode, utf, cd, &this_recurse)) - { - empty_branch = TRUE; - break; - } - scode += GET(scode, 1); - } - while (*scode == OP_ALT); - - if (!empty_branch) return FALSE; /* All branches are non-empty */ - continue; - } - - /* Groups with zero repeats can of course be empty; skip them. */ - - if (c == OP_BRAZERO || c == OP_BRAMINZERO || c == OP_SKIPZERO || - c == OP_BRAPOSZERO) - { - code += PRIV(OP_lengths)[c]; - do code += GET(code, 1); while (*code == OP_ALT); - c = *code; - continue; - } - - /* A nested group that is already marked as "could be empty" can just be - skipped. */ - - if (c == OP_SBRA || c == OP_SBRAPOS || - c == OP_SCBRA || c == OP_SCBRAPOS) - { - do code += GET(code, 1); while (*code == OP_ALT); - c = *code; - continue; - } - - /* For other groups, scan the branches. */ - - if (c == OP_BRA || c == OP_BRAPOS || - c == OP_CBRA || c == OP_CBRAPOS || - c == OP_ONCE || c == OP_ONCE_NC || - c == OP_COND || c == OP_SCOND) - { - BOOL empty_branch; - if (GET(code, 1) == 0) return TRUE; /* Hit unclosed bracket */ - - /* If a conditional group has only one branch, there is a second, implied, - empty branch, so just skip over the conditional, because it could be empty. - Otherwise, scan the individual branches of the group. */ - - if (c == OP_COND && code[GET(code, 1)] != OP_ALT) - code += GET(code, 1); - else - { - empty_branch = FALSE; - do - { - if (!empty_branch && could_be_empty_branch(code, endcode, utf, cd, - recurses)) empty_branch = TRUE; - code += GET(code, 1); - } - while (*code == OP_ALT); - if (!empty_branch) return FALSE; /* All branches are non-empty */ - } - - c = *code; - continue; - } - - /* Handle the other opcodes */ - - switch (c) - { - /* Check for quantifiers after a class. XCLASS is used for classes that - cannot be represented just by a bit map. This includes negated single - high-valued characters. The length in PRIV(OP_lengths)[] is zero; the - actual length is stored in the compiled code, so we must update "code" - here. */ - -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - case OP_XCLASS: - ccode = code += GET(code, 1); - goto CHECK_CLASS_REPEAT; -#endif - - case OP_CLASS: - case OP_NCLASS: - ccode = code + PRIV(OP_lengths)[OP_CLASS]; - -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - CHECK_CLASS_REPEAT: -#endif - - switch (*ccode) - { - case OP_CRSTAR: /* These could be empty; continue */ - case OP_CRMINSTAR: - case OP_CRQUERY: - case OP_CRMINQUERY: - case OP_CRPOSSTAR: - case OP_CRPOSQUERY: - break; - - default: /* Non-repeat => class must match */ - case OP_CRPLUS: /* These repeats aren't empty */ - case OP_CRMINPLUS: - case OP_CRPOSPLUS: - return FALSE; - - case OP_CRRANGE: - case OP_CRMINRANGE: - case OP_CRPOSRANGE: - if (GET2(ccode, 1) > 0) return FALSE; /* Minimum > 0 */ - break; - } - break; - - /* Opcodes that must match a character */ - - case OP_ANY: - case OP_ALLANY: - case OP_ANYBYTE: - - case OP_PROP: - case OP_NOTPROP: - case OP_ANYNL: - - case OP_NOT_HSPACE: - case OP_HSPACE: - case OP_NOT_VSPACE: - case OP_VSPACE: - case OP_EXTUNI: - - case OP_NOT_DIGIT: - case OP_DIGIT: - case OP_NOT_WHITESPACE: - case OP_WHITESPACE: - case OP_NOT_WORDCHAR: - case OP_WORDCHAR: - - case OP_CHAR: - case OP_CHARI: - case OP_NOT: - case OP_NOTI: - - case OP_PLUS: - case OP_PLUSI: - case OP_MINPLUS: - case OP_MINPLUSI: - - case OP_NOTPLUS: - case OP_NOTPLUSI: - case OP_NOTMINPLUS: - case OP_NOTMINPLUSI: - - case OP_POSPLUS: - case OP_POSPLUSI: - case OP_NOTPOSPLUS: - case OP_NOTPOSPLUSI: - - case OP_EXACT: - case OP_EXACTI: - case OP_NOTEXACT: - case OP_NOTEXACTI: - - case OP_TYPEPLUS: - case OP_TYPEMINPLUS: - case OP_TYPEPOSPLUS: - case OP_TYPEEXACT: - - return FALSE; - - /* These are going to continue, as they may be empty, but we have to - fudge the length for the \p and \P cases. */ - - case OP_TYPESTAR: - case OP_TYPEMINSTAR: - case OP_TYPEPOSSTAR: - case OP_TYPEQUERY: - case OP_TYPEMINQUERY: - case OP_TYPEPOSQUERY: - if (code[1] == OP_PROP || code[1] == OP_NOTPROP) code += 2; - break; - - /* Same for these */ - - case OP_TYPEUPTO: - case OP_TYPEMINUPTO: - case OP_TYPEPOSUPTO: - if (code[1 + IMM2_SIZE] == OP_PROP || code[1 + IMM2_SIZE] == OP_NOTPROP) - code += 2; - break; - - /* End of branch */ - - case OP_KET: - case OP_KETRMAX: - case OP_KETRMIN: - case OP_KETRPOS: - case OP_ALT: - return TRUE; - - /* In UTF-8 mode, STAR, MINSTAR, POSSTAR, QUERY, MINQUERY, POSQUERY, UPTO, - MINUPTO, and POSUPTO and their caseless and negative versions may be - followed by a multibyte character. */ - -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - case OP_STAR: - case OP_STARI: - case OP_NOTSTAR: - case OP_NOTSTARI: - - case OP_MINSTAR: - case OP_MINSTARI: - case OP_NOTMINSTAR: - case OP_NOTMINSTARI: - - case OP_POSSTAR: - case OP_POSSTARI: - case OP_NOTPOSSTAR: - case OP_NOTPOSSTARI: - - case OP_QUERY: - case OP_QUERYI: - case OP_NOTQUERY: - case OP_NOTQUERYI: - - case OP_MINQUERY: - case OP_MINQUERYI: - case OP_NOTMINQUERY: - case OP_NOTMINQUERYI: - - case OP_POSQUERY: - case OP_POSQUERYI: - case OP_NOTPOSQUERY: - case OP_NOTPOSQUERYI: - - if (utf && HAS_EXTRALEN(code[1])) code += GET_EXTRALEN(code[1]); - break; - - case OP_UPTO: - case OP_UPTOI: - case OP_NOTUPTO: - case OP_NOTUPTOI: - - case OP_MINUPTO: - case OP_MINUPTOI: - case OP_NOTMINUPTO: - case OP_NOTMINUPTOI: - - case OP_POSUPTO: - case OP_POSUPTOI: - case OP_NOTPOSUPTO: - case OP_NOTPOSUPTOI: - - if (utf && HAS_EXTRALEN(code[1 + IMM2_SIZE])) code += GET_EXTRALEN(code[1 + IMM2_SIZE]); - break; -#endif - - /* MARK, and PRUNE/SKIP/THEN with an argument must skip over the argument - string. */ - - case OP_MARK: - case OP_PRUNE_ARG: - case OP_SKIP_ARG: - case OP_THEN_ARG: - code += code[1]; - break; - - /* None of the remaining opcodes are required to match a character. */ - - default: - break; - } - } - -return TRUE; -} - - - -/************************************************* -* Scan compiled regex for non-emptiness * -*************************************************/ - -/* This function is called to check for left recursive calls. We want to check -the current branch of the current pattern to see if it could match the empty -string. If it could, we must look outwards for branches at other levels, -stopping when we pass beyond the bracket which is the subject of the recursion. -This function is called only during the real compile, not during the -pre-compile. - -Arguments: - code points to start of the recursion - endcode points to where to stop (current RECURSE item) - bcptr points to the chain of current (unclosed) branch starts - utf TRUE if in UTF-8 / UTF-16 / UTF-32 mode - cd pointers to tables etc - -Returns: TRUE if what is matched could be empty -*/ - -static BOOL -could_be_empty(const pcre_uchar *code, const pcre_uchar *endcode, - branch_chain *bcptr, BOOL utf, compile_data *cd) -{ -while (bcptr != NULL && bcptr->current_branch >= code) - { - if (!could_be_empty_branch(bcptr->current_branch, endcode, utf, cd, NULL)) - return FALSE; - bcptr = bcptr->outer; - } -return TRUE; -} - - - -/************************************************* -* Base opcode of repeated opcodes * -*************************************************/ - -/* Returns the base opcode for repeated single character type opcodes. If the -opcode is not a repeated character type, it returns with the original value. - -Arguments: c opcode -Returns: base opcode for the type -*/ - -static pcre_uchar -get_repeat_base(pcre_uchar c) -{ -return (c > OP_TYPEPOSUPTO)? c : - (c >= OP_TYPESTAR)? OP_TYPESTAR : - (c >= OP_NOTSTARI)? OP_NOTSTARI : - (c >= OP_NOTSTAR)? OP_NOTSTAR : - (c >= OP_STARI)? OP_STARI : - OP_STAR; -} - - - -#ifdef SUPPORT_UCP -/************************************************* -* Check a character and a property * -*************************************************/ - -/* This function is called by check_auto_possessive() when a property item -is adjacent to a fixed character. - -Arguments: - c the character - ptype the property type - pdata the data for the type - negated TRUE if it's a negated property (\P or \p{^) - -Returns: TRUE if auto-possessifying is OK -*/ - -static BOOL -check_char_prop(pcre_uint32 c, unsigned int ptype, unsigned int pdata, - BOOL negated) -{ -const pcre_uint32 *p; -const ucd_record *prop = GET_UCD(c); - -switch(ptype) - { - case PT_LAMP: - return (prop->chartype == ucp_Lu || - prop->chartype == ucp_Ll || - prop->chartype == ucp_Lt) == negated; - - case PT_GC: - return (pdata == PRIV(ucp_gentype)[prop->chartype]) == negated; - - case PT_PC: - return (pdata == prop->chartype) == negated; - - case PT_SC: - return (pdata == prop->script) == negated; - - /* These are specials */ - - case PT_ALNUM: - return (PRIV(ucp_gentype)[prop->chartype] == ucp_L || - PRIV(ucp_gentype)[prop->chartype] == ucp_N) == negated; - - /* Perl space used to exclude VT, but from Perl 5.18 it is included, which - means that Perl space and POSIX space are now identical. PCRE was changed - at release 8.34. */ - - case PT_SPACE: /* Perl space */ - case PT_PXSPACE: /* POSIX space */ - switch(c) - { - HSPACE_CASES: - VSPACE_CASES: - return negated; - - default: - return (PRIV(ucp_gentype)[prop->chartype] == ucp_Z) == negated; - } - break; /* Control never reaches here */ - - case PT_WORD: - return (PRIV(ucp_gentype)[prop->chartype] == ucp_L || - PRIV(ucp_gentype)[prop->chartype] == ucp_N || - c == CHAR_UNDERSCORE) == negated; - - case PT_CLIST: - p = PRIV(ucd_caseless_sets) + prop->caseset; - for (;;) - { - if (c < *p) return !negated; - if (c == *p++) return negated; - } - break; /* Control never reaches here */ - } - -return FALSE; -} -#endif /* SUPPORT_UCP */ - - - -/************************************************* -* Fill the character property list * -*************************************************/ - -/* Checks whether the code points to an opcode that can take part in auto- -possessification, and if so, fills a list with its properties. - -Arguments: - code points to start of expression - utf TRUE if in UTF-8 / UTF-16 / UTF-32 mode - fcc points to case-flipping table - list points to output list - list[0] will be filled with the opcode - list[1] will be non-zero if this opcode - can match an empty character string - list[2..7] depends on the opcode - -Returns: points to the start of the next opcode if *code is accepted - NULL if *code is not accepted -*/ - -static const pcre_uchar * -get_chr_property_list(const pcre_uchar *code, BOOL utf, - const pcre_uint8 *fcc, pcre_uint32 *list) -{ -pcre_uchar c = *code; -pcre_uchar base; -const pcre_uchar *end; -pcre_uint32 chr; - -#ifdef SUPPORT_UCP -pcre_uint32 *clist_dest; -const pcre_uint32 *clist_src; -#else -utf = utf; /* Suppress "unused parameter" compiler warning */ -#endif - -list[0] = c; -list[1] = FALSE; -code++; - -if (c >= OP_STAR && c <= OP_TYPEPOSUPTO) - { - base = get_repeat_base(c); - c -= (base - OP_STAR); - - if (c == OP_UPTO || c == OP_MINUPTO || c == OP_EXACT || c == OP_POSUPTO) - code += IMM2_SIZE; - - list[1] = (c != OP_PLUS && c != OP_MINPLUS && c != OP_EXACT && c != OP_POSPLUS); - - switch(base) - { - case OP_STAR: - list[0] = OP_CHAR; - break; - - case OP_STARI: - list[0] = OP_CHARI; - break; - - case OP_NOTSTAR: - list[0] = OP_NOT; - break; - - case OP_NOTSTARI: - list[0] = OP_NOTI; - break; - - case OP_TYPESTAR: - list[0] = *code; - code++; - break; - } - c = list[0]; - } - -switch(c) - { - case OP_NOT_DIGIT: - case OP_DIGIT: - case OP_NOT_WHITESPACE: - case OP_WHITESPACE: - case OP_NOT_WORDCHAR: - case OP_WORDCHAR: - case OP_ANY: - case OP_ALLANY: - case OP_ANYNL: - case OP_NOT_HSPACE: - case OP_HSPACE: - case OP_NOT_VSPACE: - case OP_VSPACE: - case OP_EXTUNI: - case OP_EODN: - case OP_EOD: - case OP_DOLL: - case OP_DOLLM: - return code; - - case OP_CHAR: - case OP_NOT: - GETCHARINCTEST(chr, code); - list[2] = chr; - list[3] = NOTACHAR; - return code; - - case OP_CHARI: - case OP_NOTI: - list[0] = (c == OP_CHARI) ? OP_CHAR : OP_NOT; - GETCHARINCTEST(chr, code); - list[2] = chr; - -#ifdef SUPPORT_UCP - if (chr < 128 || (chr < 256 && !utf)) - list[3] = fcc[chr]; - else - list[3] = UCD_OTHERCASE(chr); -#elif defined SUPPORT_UTF || !defined COMPILE_PCRE8 - list[3] = (chr < 256) ? fcc[chr] : chr; -#else - list[3] = fcc[chr]; -#endif - - /* The othercase might be the same value. */ - - if (chr == list[3]) - list[3] = NOTACHAR; - else - list[4] = NOTACHAR; - return code; - -#ifdef SUPPORT_UCP - case OP_PROP: - case OP_NOTPROP: - if (code[0] != PT_CLIST) - { - list[2] = code[0]; - list[3] = code[1]; - return code + 2; - } - - /* Convert only if we have enough space. */ - - clist_src = PRIV(ucd_caseless_sets) + code[1]; - clist_dest = list + 2; - code += 2; - - do { - if (clist_dest >= list + 8) - { - /* Early return if there is not enough space. This should never - happen, since all clists are shorter than 5 character now. */ - list[2] = code[0]; - list[3] = code[1]; - return code; - } - *clist_dest++ = *clist_src; - } - while(*clist_src++ != NOTACHAR); - - /* All characters are stored. The terminating NOTACHAR - is copied form the clist itself. */ - - list[0] = (c == OP_PROP) ? OP_CHAR : OP_NOT; - return code; -#endif - - case OP_NCLASS: - case OP_CLASS: -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - case OP_XCLASS: - if (c == OP_XCLASS) - end = code + GET(code, 0) - 1; - else -#endif - end = code + 32 / sizeof(pcre_uchar); - - switch(*end) - { - case OP_CRSTAR: - case OP_CRMINSTAR: - case OP_CRQUERY: - case OP_CRMINQUERY: - case OP_CRPOSSTAR: - case OP_CRPOSQUERY: - list[1] = TRUE; - end++; - break; - - case OP_CRPLUS: - case OP_CRMINPLUS: - case OP_CRPOSPLUS: - end++; - break; - - case OP_CRRANGE: - case OP_CRMINRANGE: - case OP_CRPOSRANGE: - list[1] = (GET2(end, 1) == 0); - end += 1 + 2 * IMM2_SIZE; - break; - } - list[2] = (pcre_uint32)(end - code); - return end; - } -return NULL; /* Opcode not accepted */ -} - - - -/************************************************* -* Scan further character sets for match * -*************************************************/ - -/* Checks whether the base and the current opcode have a common character, in -which case the base cannot be possessified. - -Arguments: - code points to the byte code - utf TRUE in UTF-8 / UTF-16 / UTF-32 mode - cd static compile data - base_list the data list of the base opcode - -Returns: TRUE if the auto-possessification is possible -*/ - -static BOOL -compare_opcodes(const pcre_uchar *code, BOOL utf, const compile_data *cd, - const pcre_uint32 *base_list, const pcre_uchar *base_end, int *rec_limit) -{ -pcre_uchar c; -pcre_uint32 list[8]; -const pcre_uint32 *chr_ptr; -const pcre_uint32 *ochr_ptr; -const pcre_uint32 *list_ptr; -const pcre_uchar *next_code; -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 -const pcre_uchar *xclass_flags; -#endif -const pcre_uint8 *class_bitset; -const pcre_uint8 *set1, *set2, *set_end; -pcre_uint32 chr; -BOOL accepted, invert_bits; -BOOL entered_a_group = FALSE; - -if (*rec_limit == 0) return FALSE; ---(*rec_limit); - -/* Note: the base_list[1] contains whether the current opcode has greedy -(represented by a non-zero value) quantifier. This is a different from -other character type lists, which stores here that the character iterator -matches to an empty string (also represented by a non-zero value). */ - -for(;;) - { - /* All operations move the code pointer forward. - Therefore infinite recursions are not possible. */ - - c = *code; - - /* Skip over callouts */ - - if (c == OP_CALLOUT) - { - code += PRIV(OP_lengths)[c]; - continue; - } - - if (c == OP_ALT) - { - do code += GET(code, 1); while (*code == OP_ALT); - c = *code; - } - - switch(c) - { - case OP_END: - case OP_KETRPOS: - /* TRUE only in greedy case. The non-greedy case could be replaced by - an OP_EXACT, but it is probably not worth it. (And note that OP_EXACT - uses more memory, which we cannot get at this stage.) */ - - return base_list[1] != 0; - - case OP_KET: - /* If the bracket is capturing, and referenced by an OP_RECURSE, or - it is an atomic sub-pattern (assert, once, etc.) the non-greedy case - cannot be converted to a possessive form. */ - - if (base_list[1] == 0) return FALSE; - - switch(*(code - GET(code, 1))) - { - case OP_ASSERT: - case OP_ASSERT_NOT: - case OP_ASSERTBACK: - case OP_ASSERTBACK_NOT: - case OP_ONCE: - case OP_ONCE_NC: - /* Atomic sub-patterns and assertions can always auto-possessify their - last iterator. However, if the group was entered as a result of checking - a previous iterator, this is not possible. */ - - return !entered_a_group; - } - - code += PRIV(OP_lengths)[c]; - continue; - - case OP_ONCE: - case OP_ONCE_NC: - case OP_BRA: - case OP_CBRA: - next_code = code + GET(code, 1); - code += PRIV(OP_lengths)[c]; - - while (*next_code == OP_ALT) - { - if (!compare_opcodes(code, utf, cd, base_list, base_end, rec_limit)) - return FALSE; - code = next_code + 1 + LINK_SIZE; - next_code += GET(next_code, 1); - } - - entered_a_group = TRUE; - continue; - - case OP_BRAZERO: - case OP_BRAMINZERO: - - next_code = code + 1; - if (*next_code != OP_BRA && *next_code != OP_CBRA - && *next_code != OP_ONCE && *next_code != OP_ONCE_NC) return FALSE; - - do next_code += GET(next_code, 1); while (*next_code == OP_ALT); - - /* The bracket content will be checked by the - OP_BRA/OP_CBRA case above. */ - next_code += 1 + LINK_SIZE; - if (!compare_opcodes(next_code, utf, cd, base_list, base_end, rec_limit)) - return FALSE; - - code += PRIV(OP_lengths)[c]; - continue; - - default: - break; - } - - /* Check for a supported opcode, and load its properties. */ - - code = get_chr_property_list(code, utf, cd->fcc, list); - if (code == NULL) return FALSE; /* Unsupported */ - - /* If either opcode is a small character list, set pointers for comparing - characters from that list with another list, or with a property. */ - - if (base_list[0] == OP_CHAR) - { - chr_ptr = base_list + 2; - list_ptr = list; - } - else if (list[0] == OP_CHAR) - { - chr_ptr = list + 2; - list_ptr = base_list; - } - - /* Character bitsets can also be compared to certain opcodes. */ - - else if (base_list[0] == OP_CLASS || list[0] == OP_CLASS -#ifdef COMPILE_PCRE8 - /* In 8 bit, non-UTF mode, OP_CLASS and OP_NCLASS are the same. */ - || (!utf && (base_list[0] == OP_NCLASS || list[0] == OP_NCLASS)) -#endif - ) - { -#ifdef COMPILE_PCRE8 - if (base_list[0] == OP_CLASS || (!utf && base_list[0] == OP_NCLASS)) -#else - if (base_list[0] == OP_CLASS) -#endif - { - set1 = (pcre_uint8 *)(base_end - base_list[2]); - list_ptr = list; - } - else - { - set1 = (pcre_uint8 *)(code - list[2]); - list_ptr = base_list; - } - - invert_bits = FALSE; - switch(list_ptr[0]) - { - case OP_CLASS: - case OP_NCLASS: - set2 = (pcre_uint8 *) - ((list_ptr == list ? code : base_end) - list_ptr[2]); - break; - -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - case OP_XCLASS: - xclass_flags = (list_ptr == list ? code : base_end) - list_ptr[2] + LINK_SIZE; - if ((*xclass_flags & XCL_HASPROP) != 0) return FALSE; - if ((*xclass_flags & XCL_MAP) == 0) - { - /* No bits are set for characters < 256. */ - if (list[1] == 0) return TRUE; - /* Might be an empty repeat. */ - continue; - } - set2 = (pcre_uint8 *)(xclass_flags + 1); - break; -#endif - - case OP_NOT_DIGIT: - invert_bits = TRUE; - /* Fall through */ - case OP_DIGIT: - set2 = (pcre_uint8 *)(cd->cbits + cbit_digit); - break; - - case OP_NOT_WHITESPACE: - invert_bits = TRUE; - /* Fall through */ - case OP_WHITESPACE: - set2 = (pcre_uint8 *)(cd->cbits + cbit_space); - break; - - case OP_NOT_WORDCHAR: - invert_bits = TRUE; - /* Fall through */ - case OP_WORDCHAR: - set2 = (pcre_uint8 *)(cd->cbits + cbit_word); - break; - - default: - return FALSE; - } - - /* Because the sets are unaligned, we need - to perform byte comparison here. */ - set_end = set1 + 32; - if (invert_bits) - { - do - { - if ((*set1++ & ~(*set2++)) != 0) return FALSE; - } - while (set1 < set_end); - } - else - { - do - { - if ((*set1++ & *set2++) != 0) return FALSE; - } - while (set1 < set_end); - } - - if (list[1] == 0) return TRUE; - /* Might be an empty repeat. */ - continue; - } - - /* Some property combinations also acceptable. Unicode property opcodes are - processed specially; the rest can be handled with a lookup table. */ - - else - { - pcre_uint32 leftop, rightop; - - leftop = base_list[0]; - rightop = list[0]; - -#ifdef SUPPORT_UCP - accepted = FALSE; /* Always set in non-unicode case. */ - if (leftop == OP_PROP || leftop == OP_NOTPROP) - { - if (rightop == OP_EOD) - accepted = TRUE; - else if (rightop == OP_PROP || rightop == OP_NOTPROP) - { - int n; - const pcre_uint8 *p; - BOOL same = leftop == rightop; - BOOL lisprop = leftop == OP_PROP; - BOOL risprop = rightop == OP_PROP; - BOOL bothprop = lisprop && risprop; - - /* There's a table that specifies how each combination is to be - processed: - 0 Always return FALSE (never auto-possessify) - 1 Character groups are distinct (possessify if both are OP_PROP) - 2 Check character categories in the same group (general or particular) - 3 Return TRUE if the two opcodes are not the same - ... see comments below - */ - - n = propposstab[base_list[2]][list[2]]; - switch(n) - { - case 0: break; - case 1: accepted = bothprop; break; - case 2: accepted = (base_list[3] == list[3]) != same; break; - case 3: accepted = !same; break; - - case 4: /* Left general category, right particular category */ - accepted = risprop && catposstab[base_list[3]][list[3]] == same; - break; - - case 5: /* Right general category, left particular category */ - accepted = lisprop && catposstab[list[3]][base_list[3]] == same; - break; - - /* This code is logically tricky. Think hard before fiddling with it. - The posspropstab table has four entries per row. Each row relates to - one of PCRE's special properties such as ALNUM or SPACE or WORD. - Only WORD actually needs all four entries, but using repeats for the - others means they can all use the same code below. - - The first two entries in each row are Unicode general categories, and - apply always, because all the characters they include are part of the - PCRE character set. The third and fourth entries are a general and a - particular category, respectively, that include one or more relevant - characters. One or the other is used, depending on whether the check - is for a general or a particular category. However, in both cases the - category contains more characters than the specials that are defined - for the property being tested against. Therefore, it cannot be used - in a NOTPROP case. - - Example: the row for WORD contains ucp_L, ucp_N, ucp_P, ucp_Po. - Underscore is covered by ucp_P or ucp_Po. */ - - case 6: /* Left alphanum vs right general category */ - case 7: /* Left space vs right general category */ - case 8: /* Left word vs right general category */ - p = posspropstab[n-6]; - accepted = risprop && lisprop == - (list[3] != p[0] && - list[3] != p[1] && - (list[3] != p[2] || !lisprop)); - break; - - case 9: /* Right alphanum vs left general category */ - case 10: /* Right space vs left general category */ - case 11: /* Right word vs left general category */ - p = posspropstab[n-9]; - accepted = lisprop && risprop == - (base_list[3] != p[0] && - base_list[3] != p[1] && - (base_list[3] != p[2] || !risprop)); - break; - - case 12: /* Left alphanum vs right particular category */ - case 13: /* Left space vs right particular category */ - case 14: /* Left word vs right particular category */ - p = posspropstab[n-12]; - accepted = risprop && lisprop == - (catposstab[p[0]][list[3]] && - catposstab[p[1]][list[3]] && - (list[3] != p[3] || !lisprop)); - break; - - case 15: /* Right alphanum vs left particular category */ - case 16: /* Right space vs left particular category */ - case 17: /* Right word vs left particular category */ - p = posspropstab[n-15]; - accepted = lisprop && risprop == - (catposstab[p[0]][base_list[3]] && - catposstab[p[1]][base_list[3]] && - (base_list[3] != p[3] || !risprop)); - break; - } - } - } - - else -#endif /* SUPPORT_UCP */ - - accepted = leftop >= FIRST_AUTOTAB_OP && leftop <= LAST_AUTOTAB_LEFT_OP && - rightop >= FIRST_AUTOTAB_OP && rightop <= LAST_AUTOTAB_RIGHT_OP && - autoposstab[leftop - FIRST_AUTOTAB_OP][rightop - FIRST_AUTOTAB_OP]; - - if (!accepted) return FALSE; - - if (list[1] == 0) return TRUE; - /* Might be an empty repeat. */ - continue; - } - - /* Control reaches here only if one of the items is a small character list. - All characters are checked against the other side. */ - - do - { - chr = *chr_ptr; - - switch(list_ptr[0]) - { - case OP_CHAR: - ochr_ptr = list_ptr + 2; - do - { - if (chr == *ochr_ptr) return FALSE; - ochr_ptr++; - } - while(*ochr_ptr != NOTACHAR); - break; - - case OP_NOT: - ochr_ptr = list_ptr + 2; - do - { - if (chr == *ochr_ptr) - break; - ochr_ptr++; - } - while(*ochr_ptr != NOTACHAR); - if (*ochr_ptr == NOTACHAR) return FALSE; /* Not found */ - break; - - /* Note that OP_DIGIT etc. are generated only when PCRE_UCP is *not* - set. When it is set, \d etc. are converted into OP_(NOT_)PROP codes. */ - - case OP_DIGIT: - if (chr < 256 && (cd->ctypes[chr] & ctype_digit) != 0) return FALSE; - break; - - case OP_NOT_DIGIT: - if (chr > 255 || (cd->ctypes[chr] & ctype_digit) == 0) return FALSE; - break; - - case OP_WHITESPACE: - if (chr < 256 && (cd->ctypes[chr] & ctype_space) != 0) return FALSE; - break; - - case OP_NOT_WHITESPACE: - if (chr > 255 || (cd->ctypes[chr] & ctype_space) == 0) return FALSE; - break; - - case OP_WORDCHAR: - if (chr < 255 && (cd->ctypes[chr] & ctype_word) != 0) return FALSE; - break; - - case OP_NOT_WORDCHAR: - if (chr > 255 || (cd->ctypes[chr] & ctype_word) == 0) return FALSE; - break; - - case OP_HSPACE: - switch(chr) - { - HSPACE_CASES: return FALSE; - default: break; - } - break; - - case OP_NOT_HSPACE: - switch(chr) - { - HSPACE_CASES: break; - default: return FALSE; - } - break; - - case OP_ANYNL: - case OP_VSPACE: - switch(chr) - { - VSPACE_CASES: return FALSE; - default: break; - } - break; - - case OP_NOT_VSPACE: - switch(chr) - { - VSPACE_CASES: break; - default: return FALSE; - } - break; - - case OP_DOLL: - case OP_EODN: - switch (chr) - { - case CHAR_CR: - case CHAR_LF: - case CHAR_VT: - case CHAR_FF: - case CHAR_NEL: -#ifndef EBCDIC - case 0x2028: - case 0x2029: -#endif /* Not EBCDIC */ - return FALSE; - } - break; - - case OP_EOD: /* Can always possessify before \z */ - break; - -#ifdef SUPPORT_UCP - case OP_PROP: - case OP_NOTPROP: - if (!check_char_prop(chr, list_ptr[2], list_ptr[3], - list_ptr[0] == OP_NOTPROP)) - return FALSE; - break; -#endif - - case OP_NCLASS: - if (chr > 255) return FALSE; - /* Fall through */ - - case OP_CLASS: - if (chr > 255) break; - class_bitset = (pcre_uint8 *) - ((list_ptr == list ? code : base_end) - list_ptr[2]); - if ((class_bitset[chr >> 3] & (1 << (chr & 7))) != 0) return FALSE; - break; - -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - case OP_XCLASS: - if (PRIV(xclass)(chr, (list_ptr == list ? code : base_end) - - list_ptr[2] + LINK_SIZE, utf)) return FALSE; - break; -#endif - - default: - return FALSE; - } - - chr_ptr++; - } - while(*chr_ptr != NOTACHAR); - - /* At least one character must be matched from this opcode. */ - - if (list[1] == 0) return TRUE; - } - -/* Control never reaches here. There used to be a fail-save return FALSE; here, -but some compilers complain about an unreachable statement. */ - -} - - - -/************************************************* -* Scan compiled regex for auto-possession * -*************************************************/ - -/* Replaces single character iterations with their possessive alternatives -if appropriate. This function modifies the compiled opcode! - -Arguments: - code points to start of the byte code - utf TRUE in UTF-8 / UTF-16 / UTF-32 mode - cd static compile data - -Returns: nothing -*/ - -static void -auto_possessify(pcre_uchar *code, BOOL utf, const compile_data *cd) -{ -register pcre_uchar c; -const pcre_uchar *end; -pcre_uchar *repeat_opcode; -pcre_uint32 list[8]; -int rec_limit; - -for (;;) - { - c = *code; - - /* When a pattern with bad UTF-8 encoding is compiled with NO_UTF_CHECK, - it may compile without complaining, but may get into a loop here if the code - pointer points to a bad value. This is, of course a documentated possibility, - when NO_UTF_CHECK is set, so it isn't a bug, but we can detect this case and - just give up on this optimization. */ - - if (c >= OP_TABLE_LENGTH) return; - - if (c >= OP_STAR && c <= OP_TYPEPOSUPTO) - { - c -= get_repeat_base(c) - OP_STAR; - end = (c <= OP_MINUPTO) ? - get_chr_property_list(code, utf, cd->fcc, list) : NULL; - list[1] = c == OP_STAR || c == OP_PLUS || c == OP_QUERY || c == OP_UPTO; - - rec_limit = 1000; - if (end != NULL && compare_opcodes(end, utf, cd, list, end, &rec_limit)) - { - switch(c) - { - case OP_STAR: - *code += OP_POSSTAR - OP_STAR; - break; - - case OP_MINSTAR: - *code += OP_POSSTAR - OP_MINSTAR; - break; - - case OP_PLUS: - *code += OP_POSPLUS - OP_PLUS; - break; - - case OP_MINPLUS: - *code += OP_POSPLUS - OP_MINPLUS; - break; - - case OP_QUERY: - *code += OP_POSQUERY - OP_QUERY; - break; - - case OP_MINQUERY: - *code += OP_POSQUERY - OP_MINQUERY; - break; - - case OP_UPTO: - *code += OP_POSUPTO - OP_UPTO; - break; - - case OP_MINUPTO: - *code += OP_POSUPTO - OP_MINUPTO; - break; - } - } - c = *code; - } - else if (c == OP_CLASS || c == OP_NCLASS || c == OP_XCLASS) - { -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - if (c == OP_XCLASS) - repeat_opcode = code + GET(code, 1); - else -#endif - repeat_opcode = code + 1 + (32 / sizeof(pcre_uchar)); - - c = *repeat_opcode; - if (c >= OP_CRSTAR && c <= OP_CRMINRANGE) - { - /* end must not be NULL. */ - end = get_chr_property_list(code, utf, cd->fcc, list); - - list[1] = (c & 1) == 0; - - rec_limit = 1000; - if (compare_opcodes(end, utf, cd, list, end, &rec_limit)) - { - switch (c) - { - case OP_CRSTAR: - case OP_CRMINSTAR: - *repeat_opcode = OP_CRPOSSTAR; - break; - - case OP_CRPLUS: - case OP_CRMINPLUS: - *repeat_opcode = OP_CRPOSPLUS; - break; - - case OP_CRQUERY: - case OP_CRMINQUERY: - *repeat_opcode = OP_CRPOSQUERY; - break; - - case OP_CRRANGE: - case OP_CRMINRANGE: - *repeat_opcode = OP_CRPOSRANGE; - break; - } - } - } - c = *code; - } - - switch(c) - { - case OP_END: - return; - - case OP_TYPESTAR: - case OP_TYPEMINSTAR: - case OP_TYPEPLUS: - case OP_TYPEMINPLUS: - case OP_TYPEQUERY: - case OP_TYPEMINQUERY: - case OP_TYPEPOSSTAR: - case OP_TYPEPOSPLUS: - case OP_TYPEPOSQUERY: - if (code[1] == OP_PROP || code[1] == OP_NOTPROP) code += 2; - break; - - case OP_TYPEUPTO: - case OP_TYPEMINUPTO: - case OP_TYPEEXACT: - case OP_TYPEPOSUPTO: - if (code[1 + IMM2_SIZE] == OP_PROP || code[1 + IMM2_SIZE] == OP_NOTPROP) - code += 2; - break; - -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - case OP_XCLASS: - code += GET(code, 1); - break; -#endif - - case OP_MARK: - case OP_PRUNE_ARG: - case OP_SKIP_ARG: - case OP_THEN_ARG: - code += code[1]; - break; - } - - /* Add in the fixed length from the table */ - - code += PRIV(OP_lengths)[c]; - - /* In UTF-8 mode, opcodes that are followed by a character may be followed by - a multi-byte character. The length in the table is a minimum, so we have to - arrange to skip the extra bytes. */ - -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (utf) switch(c) - { - case OP_CHAR: - case OP_CHARI: - case OP_NOT: - case OP_NOTI: - case OP_STAR: - case OP_MINSTAR: - case OP_PLUS: - case OP_MINPLUS: - case OP_QUERY: - case OP_MINQUERY: - case OP_UPTO: - case OP_MINUPTO: - case OP_EXACT: - case OP_POSSTAR: - case OP_POSPLUS: - case OP_POSQUERY: - case OP_POSUPTO: - case OP_STARI: - case OP_MINSTARI: - case OP_PLUSI: - case OP_MINPLUSI: - case OP_QUERYI: - case OP_MINQUERYI: - case OP_UPTOI: - case OP_MINUPTOI: - case OP_EXACTI: - case OP_POSSTARI: - case OP_POSPLUSI: - case OP_POSQUERYI: - case OP_POSUPTOI: - case OP_NOTSTAR: - case OP_NOTMINSTAR: - case OP_NOTPLUS: - case OP_NOTMINPLUS: - case OP_NOTQUERY: - case OP_NOTMINQUERY: - case OP_NOTUPTO: - case OP_NOTMINUPTO: - case OP_NOTEXACT: - case OP_NOTPOSSTAR: - case OP_NOTPOSPLUS: - case OP_NOTPOSQUERY: - case OP_NOTPOSUPTO: - case OP_NOTSTARI: - case OP_NOTMINSTARI: - case OP_NOTPLUSI: - case OP_NOTMINPLUSI: - case OP_NOTQUERYI: - case OP_NOTMINQUERYI: - case OP_NOTUPTOI: - case OP_NOTMINUPTOI: - case OP_NOTEXACTI: - case OP_NOTPOSSTARI: - case OP_NOTPOSPLUSI: - case OP_NOTPOSQUERYI: - case OP_NOTPOSUPTOI: - if (HAS_EXTRALEN(code[-1])) code += GET_EXTRALEN(code[-1]); - break; - } -#else - (void)(utf); /* Keep compiler happy by referencing function argument */ -#endif - } -} - - - -/************************************************* -* Check for POSIX class syntax * -*************************************************/ - -/* This function is called when the sequence "[:" or "[." or "[=" is -encountered in a character class. It checks whether this is followed by a -sequence of characters terminated by a matching ":]" or ".]" or "=]". If we -reach an unescaped ']' without the special preceding character, return FALSE. - -Originally, this function only recognized a sequence of letters between the -terminators, but it seems that Perl recognizes any sequence of characters, -though of course unknown POSIX names are subsequently rejected. Perl gives an -"Unknown POSIX class" error for [:f\oo:] for example, where previously PCRE -didn't consider this to be a POSIX class. Likewise for [:1234:]. - -The problem in trying to be exactly like Perl is in the handling of escapes. We -have to be sure that [abc[:x\]pqr] is *not* treated as containing a POSIX -class, but [abc[:x\]pqr:]] is (so that an error can be generated). The code -below handles the special cases \\ and \], but does not try to do any other -escape processing. This makes it different from Perl for cases such as -[:l\ower:] where Perl recognizes it as the POSIX class "lower" but PCRE does -not recognize "l\ower". This is a lesser evil than not diagnosing bad classes -when Perl does, I think. - -A user pointed out that PCRE was rejecting [:a[:digit:]] whereas Perl was not. -It seems that the appearance of a nested POSIX class supersedes an apparent -external class. For example, [:a[:digit:]b:] matches "a", "b", ":", or -a digit. - -In Perl, unescaped square brackets may also appear as part of class names. For -example, [:a[:abc]b:] gives unknown POSIX class "[:abc]b:]". However, for -[:a[:abc]b][b:] it gives unknown POSIX class "[:abc]b][b:]", which does not -seem right at all. PCRE does not allow closing square brackets in POSIX class -names. - -Arguments: - ptr pointer to the initial [ - endptr where to return the end pointer - -Returns: TRUE or FALSE -*/ - -static BOOL -check_posix_syntax(const pcre_uchar *ptr, const pcre_uchar **endptr) -{ -pcre_uchar terminator; /* Don't combine these lines; the Solaris cc */ -terminator = *(++ptr); /* compiler warns about "non-constant" initializer. */ -for (++ptr; *ptr != CHAR_NULL; ptr++) - { - if (*ptr == CHAR_BACKSLASH && - (ptr[1] == CHAR_RIGHT_SQUARE_BRACKET || - ptr[1] == CHAR_BACKSLASH)) - ptr++; - else if ((*ptr == CHAR_LEFT_SQUARE_BRACKET && ptr[1] == terminator) || - *ptr == CHAR_RIGHT_SQUARE_BRACKET) return FALSE; - else if (*ptr == terminator && ptr[1] == CHAR_RIGHT_SQUARE_BRACKET) - { - *endptr = ptr; - return TRUE; - } - } -return FALSE; -} - - - - -/************************************************* -* Check POSIX class name * -*************************************************/ - -/* This function is called to check the name given in a POSIX-style class entry -such as [:alnum:]. - -Arguments: - ptr points to the first letter - len the length of the name - -Returns: a value representing the name, or -1 if unknown -*/ - -static int -check_posix_name(const pcre_uchar *ptr, int len) -{ -const char *pn = posix_names; -register int yield = 0; -while (posix_name_lengths[yield] != 0) - { - if (len == posix_name_lengths[yield] && - STRNCMP_UC_C8(ptr, pn, (unsigned int)len) == 0) return yield; - pn += posix_name_lengths[yield] + 1; - yield++; - } -return -1; -} - - -/************************************************* -* Adjust OP_RECURSE items in repeated group * -*************************************************/ - -/* OP_RECURSE items contain an offset from the start of the regex to the group -that is referenced. This means that groups can be replicated for fixed -repetition simply by copying (because the recursion is allowed to refer to -earlier groups that are outside the current group). However, when a group is -optional (i.e. the minimum quantifier is zero), OP_BRAZERO or OP_SKIPZERO is -inserted before it, after it has been compiled. This means that any OP_RECURSE -items within it that refer to the group itself or any contained groups have to -have their offsets adjusted. That one of the jobs of this function. Before it -is called, the partially compiled regex must be temporarily terminated with -OP_END. - -This function has been extended to cope with forward references for recursions -and subroutine calls. It must check the list of such references for the -group we are dealing with. If it finds that one of the recursions in the -current group is on this list, it does not adjust the value in the reference -(which is a group number). After the group has been scanned, all the offsets in -the forward reference list for the group are adjusted. - -Arguments: - group points to the start of the group - adjust the amount by which the group is to be moved - utf TRUE in UTF-8 / UTF-16 / UTF-32 mode - cd contains pointers to tables etc. - save_hwm_offset the hwm forward reference offset at the start of the group - -Returns: nothing -*/ - -static void -adjust_recurse(pcre_uchar *group, int adjust, BOOL utf, compile_data *cd, - size_t save_hwm_offset) -{ -int offset; -pcre_uchar *hc; -pcre_uchar *ptr = group; - -while ((ptr = (pcre_uchar *)find_recurse(ptr, utf)) != NULL) - { - for (hc = (pcre_uchar *)cd->start_workspace + save_hwm_offset; hc < cd->hwm; - hc += LINK_SIZE) - { - offset = (int)GET(hc, 0); - if (cd->start_code + offset == ptr + 1) break; - } - - /* If we have not found this recursion on the forward reference list, adjust - the recursion's offset if it's after the start of this group. */ - - if (hc >= cd->hwm) - { - offset = (int)GET(ptr, 1); - if (cd->start_code + offset >= group) PUT(ptr, 1, offset + adjust); - } - - ptr += 1 + LINK_SIZE; - } - -/* Now adjust all forward reference offsets for the group. */ - -for (hc = (pcre_uchar *)cd->start_workspace + save_hwm_offset; hc < cd->hwm; - hc += LINK_SIZE) - { - offset = (int)GET(hc, 0); - PUT(hc, 0, offset + adjust); - } -} - - - -/************************************************* -* Insert an automatic callout point * -*************************************************/ - -/* This function is called when the PCRE_AUTO_CALLOUT option is set, to insert -callout points before each pattern item. - -Arguments: - code current code pointer - ptr current pattern pointer - cd pointers to tables etc - -Returns: new code pointer -*/ - -static pcre_uchar * -auto_callout(pcre_uchar *code, const pcre_uchar *ptr, compile_data *cd) -{ -*code++ = OP_CALLOUT; -*code++ = 255; -PUT(code, 0, (int)(ptr - cd->start_pattern)); /* Pattern offset */ -PUT(code, LINK_SIZE, 0); /* Default length */ -return code + 2 * LINK_SIZE; -} - - - -/************************************************* -* Complete a callout item * -*************************************************/ - -/* A callout item contains the length of the next item in the pattern, which -we can't fill in till after we have reached the relevant point. This is used -for both automatic and manual callouts. - -Arguments: - previous_callout points to previous callout item - ptr current pattern pointer - cd pointers to tables etc - -Returns: nothing -*/ - -static void -complete_callout(pcre_uchar *previous_callout, const pcre_uchar *ptr, compile_data *cd) -{ -int length = (int)(ptr - cd->start_pattern - GET(previous_callout, 2)); -PUT(previous_callout, 2 + LINK_SIZE, length); -} - - - -#ifdef SUPPORT_UCP -/************************************************* -* Get othercase range * -*************************************************/ - -/* This function is passed the start and end of a class range, in UTF-8 mode -with UCP support. It searches up the characters, looking for ranges of -characters in the "other" case. Each call returns the next one, updating the -start address. A character with multiple other cases is returned on its own -with a special return value. - -Arguments: - cptr points to starting character value; updated - d end value - ocptr where to put start of othercase range - odptr where to put end of othercase range - -Yield: -1 when no more - 0 when a range is returned - >0 the CASESET offset for char with multiple other cases - in this case, ocptr contains the original -*/ - -static int -get_othercase_range(pcre_uint32 *cptr, pcre_uint32 d, pcre_uint32 *ocptr, - pcre_uint32 *odptr) -{ -pcre_uint32 c, othercase, next; -unsigned int co; - -/* Find the first character that has an other case. If it has multiple other -cases, return its case offset value. */ - -for (c = *cptr; c <= d; c++) - { - if ((co = UCD_CASESET(c)) != 0) - { - *ocptr = c++; /* Character that has the set */ - *cptr = c; /* Rest of input range */ - return (int)co; - } - if ((othercase = UCD_OTHERCASE(c)) != c) break; - } - -if (c > d) return -1; /* Reached end of range */ - -/* Found a character that has a single other case. Search for the end of the -range, which is either the end of the input range, or a character that has zero -or more than one other cases. */ - -*ocptr = othercase; -next = othercase + 1; - -for (++c; c <= d; c++) - { - if ((co = UCD_CASESET(c)) != 0 || UCD_OTHERCASE(c) != next) break; - next++; - } - -*odptr = next - 1; /* End of othercase range */ -*cptr = c; /* Rest of input range */ -return 0; -} -#endif /* SUPPORT_UCP */ - - - -/************************************************* -* Add a character or range to a class * -*************************************************/ - -/* This function packages up the logic of adding a character or range of -characters to a class. The character values in the arguments will be within the -valid values for the current mode (8-bit, 16-bit, UTF, etc). This function is -mutually recursive with the function immediately below. - -Arguments: - classbits the bit map for characters < 256 - uchardptr points to the pointer for extra data - options the options word - cd contains pointers to tables etc. - start start of range character - end end of range character - -Returns: the number of < 256 characters added - the pointer to extra data is updated -*/ - -static int -add_to_class(pcre_uint8 *classbits, pcre_uchar **uchardptr, int options, - compile_data *cd, pcre_uint32 start, pcre_uint32 end) -{ -pcre_uint32 c; -pcre_uint32 classbits_end = (end <= 0xff ? end : 0xff); -int n8 = 0; - -/* If caseless matching is required, scan the range and process alternate -cases. In Unicode, there are 8-bit characters that have alternate cases that -are greater than 255 and vice-versa. Sometimes we can just extend the original -range. */ - -if ((options & PCRE_CASELESS) != 0) - { -#ifdef SUPPORT_UCP - if ((options & PCRE_UTF8) != 0) - { - int rc; - pcre_uint32 oc, od; - - options &= ~PCRE_CASELESS; /* Remove for recursive calls */ - c = start; - - while ((rc = get_othercase_range(&c, end, &oc, &od)) >= 0) - { - /* Handle a single character that has more than one other case. */ - - if (rc > 0) n8 += add_list_to_class(classbits, uchardptr, options, cd, - PRIV(ucd_caseless_sets) + rc, oc); - - /* Do nothing if the other case range is within the original range. */ - - else if (oc >= start && od <= end) continue; - - /* Extend the original range if there is overlap, noting that if oc < c, we - can't have od > end because a subrange is always shorter than the basic - range. Otherwise, use a recursive call to add the additional range. */ - - else if (oc < start && od >= start - 1) start = oc; /* Extend downwards */ - else if (od > end && oc <= end + 1) - { - end = od; /* Extend upwards */ - if (end > classbits_end) classbits_end = (end <= 0xff ? end : 0xff); - } - else n8 += add_to_class(classbits, uchardptr, options, cd, oc, od); - } - } - else -#endif /* SUPPORT_UCP */ - - /* Not UTF-mode, or no UCP */ - - for (c = start; c <= classbits_end; c++) - { - SETBIT(classbits, cd->fcc[c]); - n8++; - } - } - -/* Now handle the original range. Adjust the final value according to the bit -length - this means that the same lists of (e.g.) horizontal spaces can be used -in all cases. */ - -#if defined COMPILE_PCRE8 -#ifdef SUPPORT_UTF - if ((options & PCRE_UTF8) == 0) -#endif - if (end > 0xff) end = 0xff; - -#elif defined COMPILE_PCRE16 -#ifdef SUPPORT_UTF - if ((options & PCRE_UTF16) == 0) -#endif - if (end > 0xffff) end = 0xffff; - -#endif /* COMPILE_PCRE[8|16] */ - -/* Use the bitmap for characters < 256. Otherwise use extra data.*/ - -for (c = start; c <= classbits_end; c++) - { - /* Regardless of start, c will always be <= 255. */ - SETBIT(classbits, c); - n8++; - } - -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 -if (start <= 0xff) start = 0xff + 1; - -if (end >= start) - { - pcre_uchar *uchardata = *uchardptr; -#ifdef SUPPORT_UTF - if ((options & PCRE_UTF8) != 0) /* All UTFs use the same flag bit */ - { - if (start < end) - { - *uchardata++ = XCL_RANGE; - uchardata += PRIV(ord2utf)(start, uchardata); - uchardata += PRIV(ord2utf)(end, uchardata); - } - else if (start == end) - { - *uchardata++ = XCL_SINGLE; - uchardata += PRIV(ord2utf)(start, uchardata); - } - } - else -#endif /* SUPPORT_UTF */ - - /* Without UTF support, character values are constrained by the bit length, - and can only be > 256 for 16-bit and 32-bit libraries. */ - -#ifdef COMPILE_PCRE8 - {} -#else - if (start < end) - { - *uchardata++ = XCL_RANGE; - *uchardata++ = start; - *uchardata++ = end; - } - else if (start == end) - { - *uchardata++ = XCL_SINGLE; - *uchardata++ = start; - } -#endif - - *uchardptr = uchardata; /* Updata extra data pointer */ - } -#endif /* SUPPORT_UTF || !COMPILE_PCRE8 */ - -return n8; /* Number of 8-bit characters */ -} - - - - -/************************************************* -* Add a list of characters to a class * -*************************************************/ - -/* This function is used for adding a list of case-equivalent characters to a -class, and also for adding a list of horizontal or vertical whitespace. If the -list is in order (which it should be), ranges of characters are detected and -handled appropriately. This function is mutually recursive with the function -above. - -Arguments: - classbits the bit map for characters < 256 - uchardptr points to the pointer for extra data - options the options word - cd contains pointers to tables etc. - p points to row of 32-bit values, terminated by NOTACHAR - except character to omit; this is used when adding lists of - case-equivalent characters to avoid including the one we - already know about - -Returns: the number of < 256 characters added - the pointer to extra data is updated -*/ - -static int -add_list_to_class(pcre_uint8 *classbits, pcre_uchar **uchardptr, int options, - compile_data *cd, const pcre_uint32 *p, unsigned int except) -{ -int n8 = 0; -while (p[0] < NOTACHAR) - { - int n = 0; - if (p[0] != except) - { - while(p[n+1] == p[0] + n + 1) n++; - n8 += add_to_class(classbits, uchardptr, options, cd, p[0], p[n]); - } - p += n + 1; - } -return n8; -} - - - -/************************************************* -* Add characters not in a list to a class * -*************************************************/ - -/* This function is used for adding the complement of a list of horizontal or -vertical whitespace to a class. The list must be in order. - -Arguments: - classbits the bit map for characters < 256 - uchardptr points to the pointer for extra data - options the options word - cd contains pointers to tables etc. - p points to row of 32-bit values, terminated by NOTACHAR - -Returns: the number of < 256 characters added - the pointer to extra data is updated -*/ - -static int -add_not_list_to_class(pcre_uint8 *classbits, pcre_uchar **uchardptr, - int options, compile_data *cd, const pcre_uint32 *p) -{ -BOOL utf = (options & PCRE_UTF8) != 0; -int n8 = 0; -if (p[0] > 0) - n8 += add_to_class(classbits, uchardptr, options, cd, 0, p[0] - 1); -while (p[0] < NOTACHAR) - { - while (p[1] == p[0] + 1) p++; - n8 += add_to_class(classbits, uchardptr, options, cd, p[0] + 1, - (p[1] == NOTACHAR) ? (utf ? 0x10ffffu : 0xffffffffu) : p[1] - 1); - p++; - } -return n8; -} - - - -/************************************************* -* Compile one branch * -*************************************************/ - -/* Scan the pattern, compiling it into the a vector. If the options are -changed during the branch, the pointer is used to change the external options -bits. This function is used during the pre-compile phase when we are trying -to find out the amount of memory needed, as well as during the real compile -phase. The value of lengthptr distinguishes the two phases. - -Arguments: - optionsptr pointer to the option bits - codeptr points to the pointer to the current code point - ptrptr points to the current pattern pointer - errorcodeptr points to error code variable - firstcharptr place to put the first required character - firstcharflagsptr place to put the first character flags, or a negative number - reqcharptr place to put the last required character - reqcharflagsptr place to put the last required character flags, or a negative number - bcptr points to current branch chain - cond_depth conditional nesting depth - cd contains pointers to tables etc. - lengthptr NULL during the real compile phase - points to length accumulator during pre-compile phase - -Returns: TRUE on success - FALSE, with *errorcodeptr set non-zero on error -*/ - -static BOOL -compile_branch(int *optionsptr, pcre_uchar **codeptr, - const pcre_uchar **ptrptr, int *errorcodeptr, - pcre_uint32 *firstcharptr, pcre_int32 *firstcharflagsptr, - pcre_uint32 *reqcharptr, pcre_int32 *reqcharflagsptr, - branch_chain *bcptr, int cond_depth, - compile_data *cd, int *lengthptr) -{ -int repeat_type, op_type; -int repeat_min = 0, repeat_max = 0; /* To please picky compilers */ -int bravalue = 0; -int greedy_default, greedy_non_default; -pcre_uint32 firstchar, reqchar; -pcre_int32 firstcharflags, reqcharflags; -pcre_uint32 zeroreqchar, zerofirstchar; -pcre_int32 zeroreqcharflags, zerofirstcharflags; -pcre_int32 req_caseopt, reqvary, tempreqvary; -int options = *optionsptr; /* May change dynamically */ -int after_manual_callout = 0; -int length_prevgroup = 0; -register pcre_uint32 c; -int escape; -register pcre_uchar *code = *codeptr; -pcre_uchar *last_code = code; -pcre_uchar *orig_code = code; -pcre_uchar *tempcode; -BOOL inescq = FALSE; -BOOL groupsetfirstchar = FALSE; -const pcre_uchar *ptr = *ptrptr; -const pcre_uchar *tempptr; -const pcre_uchar *nestptr = NULL; -pcre_uchar *previous = NULL; -pcre_uchar *previous_callout = NULL; -size_t item_hwm_offset = 0; -pcre_uint8 classbits[32]; - -/* We can fish out the UTF-8 setting once and for all into a BOOL, but we -must not do this for other options (e.g. PCRE_EXTENDED) because they may change -dynamically as we process the pattern. */ - -#ifdef SUPPORT_UTF -/* PCRE_UTF[16|32] have the same value as PCRE_UTF8. */ -BOOL utf = (options & PCRE_UTF8) != 0; -#ifndef COMPILE_PCRE32 -pcre_uchar utf_chars[6]; -#endif -#else -BOOL utf = FALSE; -#endif - -/* Helper variables for OP_XCLASS opcode (for characters > 255). We define -class_uchardata always so that it can be passed to add_to_class() always, -though it will not be used in non-UTF 8-bit cases. This avoids having to supply -alternative calls for the different cases. */ - -pcre_uchar *class_uchardata; -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 -BOOL xclass; -pcre_uchar *class_uchardata_base; -#endif - -#ifdef PCRE_DEBUG -if (lengthptr != NULL) DPRINTF((">> start branch\n")); -#endif - -/* Set up the default and non-default settings for greediness */ - -greedy_default = ((options & PCRE_UNGREEDY) != 0); -greedy_non_default = greedy_default ^ 1; - -/* Initialize no first byte, no required byte. REQ_UNSET means "no char -matching encountered yet". It gets changed to REQ_NONE if we hit something that -matches a non-fixed char first char; reqchar just remains unset if we never -find one. - -When we hit a repeat whose minimum is zero, we may have to adjust these values -to take the zero repeat into account. This is implemented by setting them to -zerofirstbyte and zeroreqchar when such a repeat is encountered. The individual -item types that can be repeated set these backoff variables appropriately. */ - -firstchar = reqchar = zerofirstchar = zeroreqchar = 0; -firstcharflags = reqcharflags = zerofirstcharflags = zeroreqcharflags = REQ_UNSET; - -/* The variable req_caseopt contains either the REQ_CASELESS value -or zero, according to the current setting of the caseless flag. The -REQ_CASELESS leaves the lower 28 bit empty. It is added into the -firstchar or reqchar variables to record the case status of the -value. This is used only for ASCII characters. */ - -req_caseopt = ((options & PCRE_CASELESS) != 0)? REQ_CASELESS:0; - -/* Switch on next character until the end of the branch */ - -for (;; ptr++) - { - BOOL negate_class; - BOOL should_flip_negation; - BOOL possessive_quantifier; - BOOL is_quantifier; - BOOL is_recurse; - BOOL reset_bracount; - int class_has_8bitchar; - int class_one_char; -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - BOOL xclass_has_prop; -#endif - int newoptions; - int recno; - int refsign; - int skipbytes; - pcre_uint32 subreqchar, subfirstchar; - pcre_int32 subreqcharflags, subfirstcharflags; - int terminator; - unsigned int mclength; - unsigned int tempbracount; - pcre_uint32 ec; - pcre_uchar mcbuffer[8]; - - /* Come here to restart the loop without advancing the pointer. */ - - REDO_LOOP: - - /* Get next character in the pattern */ - - c = *ptr; - - /* If we are at the end of a nested substitution, revert to the outer level - string. Nesting only happens one level deep. */ - - if (c == CHAR_NULL && nestptr != NULL) - { - ptr = nestptr; - nestptr = NULL; - c = *ptr; - } - - /* If we are in the pre-compile phase, accumulate the length used for the - previous cycle of this loop. */ - - if (lengthptr != NULL) - { -#ifdef PCRE_DEBUG - if (code > cd->hwm) cd->hwm = code; /* High water info */ -#endif - if (code > cd->start_workspace + cd->workspace_size - - WORK_SIZE_SAFETY_MARGIN) /* Check for overrun */ - { - *errorcodeptr = (code >= cd->start_workspace + cd->workspace_size)? - ERR52 : ERR87; - goto FAILED; - } - - /* There is at least one situation where code goes backwards: this is the - case of a zero quantifier after a class (e.g. [ab]{0}). At compile time, - the class is simply eliminated. However, it is created first, so we have to - allow memory for it. Therefore, don't ever reduce the length at this point. - */ - - if (code < last_code) code = last_code; - - /* Paranoid check for integer overflow */ - - if (OFLOW_MAX - *lengthptr < code - last_code) - { - *errorcodeptr = ERR20; - goto FAILED; - } - - *lengthptr += (int)(code - last_code); - DPRINTF(("length=%d added %d c=%c (0x%x)\n", *lengthptr, - (int)(code - last_code), c, c)); - - /* If "previous" is set and it is not at the start of the work space, move - it back to there, in order to avoid filling up the work space. Otherwise, - if "previous" is NULL, reset the current code pointer to the start. */ - - if (previous != NULL) - { - if (previous > orig_code) - { - memmove(orig_code, previous, IN_UCHARS(code - previous)); - code -= previous - orig_code; - previous = orig_code; - } - } - else code = orig_code; - - /* Remember where this code item starts so we can pick up the length - next time round. */ - - last_code = code; - } - - /* In the real compile phase, just check the workspace used by the forward - reference list. */ - - else if (cd->hwm > cd->start_workspace + cd->workspace_size) - { - *errorcodeptr = ERR52; - goto FAILED; - } - - /* If in \Q...\E, check for the end; if not, we have a literal. Otherwise an - isolated \E is ignored. */ - - if (c != CHAR_NULL) - { - if (c == CHAR_BACKSLASH && ptr[1] == CHAR_E) - { - inescq = FALSE; - ptr++; - continue; - } - else if (inescq) - { - if (previous_callout != NULL) - { - if (lengthptr == NULL) /* Don't attempt in pre-compile phase */ - complete_callout(previous_callout, ptr, cd); - previous_callout = NULL; - } - if ((options & PCRE_AUTO_CALLOUT) != 0) - { - previous_callout = code; - code = auto_callout(code, ptr, cd); - } - goto NORMAL_CHAR; - } - - /* Check for the start of a \Q...\E sequence. We must do this here rather - than later in case it is immediately followed by \E, which turns it into a - "do nothing" sequence. */ - - if (c == CHAR_BACKSLASH && ptr[1] == CHAR_Q) - { - inescq = TRUE; - ptr++; - continue; - } - } - - /* In extended mode, skip white space and comments. */ - - if ((options & PCRE_EXTENDED) != 0) - { - const pcre_uchar *wscptr = ptr; - while (MAX_255(c) && (cd->ctypes[c] & ctype_space) != 0) c = *(++ptr); - if (c == CHAR_NUMBER_SIGN) - { - ptr++; - while (*ptr != CHAR_NULL) - { - if (IS_NEWLINE(ptr)) /* For non-fixed-length newline cases, */ - { /* IS_NEWLINE sets cd->nllen. */ - ptr += cd->nllen; - break; - } - ptr++; -#ifdef SUPPORT_UTF - if (utf) FORWARDCHAR(ptr); -#endif - } - } - - /* If we skipped any characters, restart the loop. Otherwise, we didn't see - a comment. */ - - if (ptr > wscptr) goto REDO_LOOP; - } - - /* Skip over (?# comments. We need to do this here because we want to know if - the next thing is a quantifier, and these comments may come between an item - and its quantifier. */ - - if (c == CHAR_LEFT_PARENTHESIS && ptr[1] == CHAR_QUESTION_MARK && - ptr[2] == CHAR_NUMBER_SIGN) - { - ptr += 3; - while (*ptr != CHAR_NULL && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++; - if (*ptr == CHAR_NULL) - { - *errorcodeptr = ERR18; - goto FAILED; - } - continue; - } - - /* See if the next thing is a quantifier. */ - - is_quantifier = - c == CHAR_ASTERISK || c == CHAR_PLUS || c == CHAR_QUESTION_MARK || - (c == CHAR_LEFT_CURLY_BRACKET && is_counted_repeat(ptr+1)); - - /* Fill in length of a previous callout, except when the next thing is a - quantifier or when processing a property substitution string in UCP mode. */ - - if (!is_quantifier && previous_callout != NULL && nestptr == NULL && - after_manual_callout-- <= 0) - { - if (lengthptr == NULL) /* Don't attempt in pre-compile phase */ - complete_callout(previous_callout, ptr, cd); - previous_callout = NULL; - } - - /* Create auto callout, except for quantifiers, or while processing property - strings that are substituted for \w etc in UCP mode. */ - - if ((options & PCRE_AUTO_CALLOUT) != 0 && !is_quantifier && nestptr == NULL) - { - previous_callout = code; - code = auto_callout(code, ptr, cd); - } - - /* Process the next pattern item. */ - - switch(c) - { - /* ===================================================================*/ - case CHAR_NULL: /* The branch terminates at string end */ - case CHAR_VERTICAL_LINE: /* or | or ) */ - case CHAR_RIGHT_PARENTHESIS: - *firstcharptr = firstchar; - *firstcharflagsptr = firstcharflags; - *reqcharptr = reqchar; - *reqcharflagsptr = reqcharflags; - *codeptr = code; - *ptrptr = ptr; - if (lengthptr != NULL) - { - if (OFLOW_MAX - *lengthptr < code - last_code) - { - *errorcodeptr = ERR20; - goto FAILED; - } - *lengthptr += (int)(code - last_code); /* To include callout length */ - DPRINTF((">> end branch\n")); - } - return TRUE; - - - /* ===================================================================*/ - /* Handle single-character metacharacters. In multiline mode, ^ disables - the setting of any following char as a first character. */ - - case CHAR_CIRCUMFLEX_ACCENT: - previous = NULL; - if ((options & PCRE_MULTILINE) != 0) - { - if (firstcharflags == REQ_UNSET) - zerofirstcharflags = firstcharflags = REQ_NONE; - *code++ = OP_CIRCM; - } - else *code++ = OP_CIRC; - break; - - case CHAR_DOLLAR_SIGN: - previous = NULL; - *code++ = ((options & PCRE_MULTILINE) != 0)? OP_DOLLM : OP_DOLL; - break; - - /* There can never be a first char if '.' is first, whatever happens about - repeats. The value of reqchar doesn't change either. */ - - case CHAR_DOT: - if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; - zerofirstchar = firstchar; - zerofirstcharflags = firstcharflags; - zeroreqchar = reqchar; - zeroreqcharflags = reqcharflags; - previous = code; - item_hwm_offset = cd->hwm - cd->start_workspace; - *code++ = ((options & PCRE_DOTALL) != 0)? OP_ALLANY: OP_ANY; - break; - - - /* ===================================================================*/ - /* Character classes. If the included characters are all < 256, we build a - 32-byte bitmap of the permitted characters, except in the special case - where there is only one such character. For negated classes, we build the - map as usual, then invert it at the end. However, we use a different opcode - so that data characters > 255 can be handled correctly. - - If the class contains characters outside the 0-255 range, a different - opcode is compiled. It may optionally have a bit map for characters < 256, - but those above are are explicitly listed afterwards. A flag byte tells - whether the bitmap is present, and whether this is a negated class or not. - - In JavaScript compatibility mode, an isolated ']' causes an error. In - default (Perl) mode, it is treated as a data character. */ - - case CHAR_RIGHT_SQUARE_BRACKET: - if ((cd->external_options & PCRE_JAVASCRIPT_COMPAT) != 0) - { - *errorcodeptr = ERR64; - goto FAILED; - } - goto NORMAL_CHAR; - - /* In another (POSIX) regex library, the ugly syntax [[:<:]] and [[:>:]] is - used for "start of word" and "end of word". As these are otherwise illegal - sequences, we don't break anything by recognizing them. They are replaced - by \b(?=\w) and \b(?<=\w) respectively. Sequences like [a[:<:]] are - erroneous and are handled by the normal code below. */ - - case CHAR_LEFT_SQUARE_BRACKET: - if (STRNCMP_UC_C8(ptr+1, STRING_WEIRD_STARTWORD, 6) == 0) - { - nestptr = ptr + 7; - ptr = sub_start_of_word; - goto REDO_LOOP; - } - - if (STRNCMP_UC_C8(ptr+1, STRING_WEIRD_ENDWORD, 6) == 0) - { - nestptr = ptr + 7; - ptr = sub_end_of_word; - goto REDO_LOOP; - } - - /* Handle a real character class. */ - - previous = code; - item_hwm_offset = cd->hwm - cd->start_workspace; - - /* PCRE supports POSIX class stuff inside a class. Perl gives an error if - they are encountered at the top level, so we'll do that too. */ - - if ((ptr[1] == CHAR_COLON || ptr[1] == CHAR_DOT || - ptr[1] == CHAR_EQUALS_SIGN) && - check_posix_syntax(ptr, &tempptr)) - { - *errorcodeptr = (ptr[1] == CHAR_COLON)? ERR13 : ERR31; - goto FAILED; - } - - /* If the first character is '^', set the negation flag and skip it. Also, - if the first few characters (either before or after ^) are \Q\E or \E we - skip them too. This makes for compatibility with Perl. */ - - negate_class = FALSE; - for (;;) - { - c = *(++ptr); - if (c == CHAR_BACKSLASH) - { - if (ptr[1] == CHAR_E) - ptr++; - else if (STRNCMP_UC_C8(ptr + 1, STR_Q STR_BACKSLASH STR_E, 3) == 0) - ptr += 3; - else - break; - } - else if (!negate_class && c == CHAR_CIRCUMFLEX_ACCENT) - negate_class = TRUE; - else break; - } - - /* Empty classes are allowed in JavaScript compatibility mode. Otherwise, - an initial ']' is taken as a data character -- the code below handles - that. In JS mode, [] must always fail, so generate OP_FAIL, whereas - [^] must match any character, so generate OP_ALLANY. */ - - if (c == CHAR_RIGHT_SQUARE_BRACKET && - (cd->external_options & PCRE_JAVASCRIPT_COMPAT) != 0) - { - *code++ = negate_class? OP_ALLANY : OP_FAIL; - if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; - zerofirstchar = firstchar; - zerofirstcharflags = firstcharflags; - break; - } - - /* If a class contains a negative special such as \S, we need to flip the - negation flag at the end, so that support for characters > 255 works - correctly (they are all included in the class). */ - - should_flip_negation = FALSE; - - /* Extended class (xclass) will be used when characters > 255 - might match. */ - -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - xclass = FALSE; - class_uchardata = code + LINK_SIZE + 2; /* For XCLASS items */ - class_uchardata_base = class_uchardata; /* Save the start */ -#endif - - /* For optimization purposes, we track some properties of the class: - class_has_8bitchar will be non-zero if the class contains at least one < - 256 character; class_one_char will be 1 if the class contains just one - character; xclass_has_prop will be TRUE if unicode property checks - are present in the class. */ - - class_has_8bitchar = 0; - class_one_char = 0; -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - xclass_has_prop = FALSE; -#endif - - /* Initialize the 32-char bit map to all zeros. We build the map in a - temporary bit of memory, in case the class contains fewer than two - 8-bit characters because in that case the compiled code doesn't use the bit - map. */ - - memset(classbits, 0, 32 * sizeof(pcre_uint8)); - - /* Process characters until ] is reached. By writing this as a "do" it - means that an initial ] is taken as a data character. At the start of the - loop, c contains the first byte of the character. */ - - if (c != CHAR_NULL) do - { - const pcre_uchar *oldptr; - -#ifdef SUPPORT_UTF - if (utf && HAS_EXTRALEN(c)) - { /* Braces are required because the */ - GETCHARLEN(c, ptr, ptr); /* macro generates multiple statements */ - } -#endif - -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - /* In the pre-compile phase, accumulate the length of any extra - data and reset the pointer. This is so that very large classes that - contain a zillion > 255 characters no longer overwrite the work space - (which is on the stack). We have to remember that there was XCLASS data, - however. */ - - if (class_uchardata > class_uchardata_base) xclass = TRUE; - - if (lengthptr != NULL && class_uchardata > class_uchardata_base) - { - *lengthptr += (int)(class_uchardata - class_uchardata_base); - class_uchardata = class_uchardata_base; - } -#endif - - /* Inside \Q...\E everything is literal except \E */ - - if (inescq) - { - if (c == CHAR_BACKSLASH && ptr[1] == CHAR_E) /* If we are at \E */ - { - inescq = FALSE; /* Reset literal state */ - ptr++; /* Skip the 'E' */ - continue; /* Carry on with next */ - } - goto CHECK_RANGE; /* Could be range if \E follows */ - } - - /* Handle POSIX class names. Perl allows a negation extension of the - form [:^name:]. A square bracket that doesn't match the syntax is - treated as a literal. We also recognize the POSIX constructions - [.ch.] and [=ch=] ("collating elements") and fault them, as Perl - 5.6 and 5.8 do. */ - - if (c == CHAR_LEFT_SQUARE_BRACKET && - (ptr[1] == CHAR_COLON || ptr[1] == CHAR_DOT || - ptr[1] == CHAR_EQUALS_SIGN) && check_posix_syntax(ptr, &tempptr)) - { - BOOL local_negate = FALSE; - int posix_class, taboffset, tabopt; - register const pcre_uint8 *cbits = cd->cbits; - pcre_uint8 pbits[32]; - - if (ptr[1] != CHAR_COLON) - { - *errorcodeptr = ERR31; - goto FAILED; - } - - ptr += 2; - if (*ptr == CHAR_CIRCUMFLEX_ACCENT) - { - local_negate = TRUE; - should_flip_negation = TRUE; /* Note negative special */ - ptr++; - } - - posix_class = check_posix_name(ptr, (int)(tempptr - ptr)); - if (posix_class < 0) - { - *errorcodeptr = ERR30; - goto FAILED; - } - - /* If matching is caseless, upper and lower are converted to - alpha. This relies on the fact that the class table starts with - alpha, lower, upper as the first 3 entries. */ - - if ((options & PCRE_CASELESS) != 0 && posix_class <= 2) - posix_class = 0; - - /* When PCRE_UCP is set, some of the POSIX classes are converted to - different escape sequences that use Unicode properties \p or \P. Others - that are not available via \p or \P generate XCL_PROP/XCL_NOTPROP - directly. */ - -#ifdef SUPPORT_UCP - if ((options & PCRE_UCP) != 0) - { - unsigned int ptype = 0; - int pc = posix_class + ((local_negate)? POSIX_SUBSIZE/2 : 0); - - /* The posix_substitutes table specifies which POSIX classes can be - converted to \p or \P items. */ - - if (posix_substitutes[pc] != NULL) - { - nestptr = tempptr + 1; - ptr = posix_substitutes[pc] - 1; - continue; - } - - /* There are three other classes that generate special property calls - that are recognized only in an XCLASS. */ - - else switch(posix_class) - { - case PC_GRAPH: - ptype = PT_PXGRAPH; - /* Fall through */ - case PC_PRINT: - if (ptype == 0) ptype = PT_PXPRINT; - /* Fall through */ - case PC_PUNCT: - if (ptype == 0) ptype = PT_PXPUNCT; - *class_uchardata++ = local_negate? XCL_NOTPROP : XCL_PROP; - *class_uchardata++ = ptype; - *class_uchardata++ = 0; - xclass_has_prop = TRUE; - ptr = tempptr + 1; - continue; - - /* For the other POSIX classes (ascii, cntrl, xdigit) we are going - to fall through to the non-UCP case and build a bit map for - characters with code points less than 256. If we are in a negated - POSIX class, characters with code points greater than 255 must - either all match or all not match. In the special case where we - have not yet generated any xclass data, and this is the final item - in the overall class, we need do nothing: later on, the opcode - OP_NCLASS will be used to indicate that characters greater than 255 - are acceptable. If we have already seen an xclass item or one may - follow (we have to assume that it might if this is not the end of - the class), explicitly list all wide codepoints, which will then - either not match or match, depending on whether the class is or is - not negated. */ - - default: - if (local_negate && - (xclass || tempptr[2] != CHAR_RIGHT_SQUARE_BRACKET)) - { - *class_uchardata++ = XCL_RANGE; - class_uchardata += PRIV(ord2utf)(0x100, class_uchardata); - class_uchardata += PRIV(ord2utf)(0x10ffff, class_uchardata); - } - break; - } - } -#endif - /* In the non-UCP case, or when UCP makes no difference, we build the - bit map for the POSIX class in a chunk of local store because we may be - adding and subtracting from it, and we don't want to subtract bits that - may be in the main map already. At the end we or the result into the - bit map that is being built. */ - - posix_class *= 3; - - /* Copy in the first table (always present) */ - - memcpy(pbits, cbits + posix_class_maps[posix_class], - 32 * sizeof(pcre_uint8)); - - /* If there is a second table, add or remove it as required. */ - - taboffset = posix_class_maps[posix_class + 1]; - tabopt = posix_class_maps[posix_class + 2]; - - if (taboffset >= 0) - { - if (tabopt >= 0) - for (c = 0; c < 32; c++) pbits[c] |= cbits[c + taboffset]; - else - for (c = 0; c < 32; c++) pbits[c] &= ~cbits[c + taboffset]; - } - - /* Now see if we need to remove any special characters. An option - value of 1 removes vertical space and 2 removes underscore. */ - - if (tabopt < 0) tabopt = -tabopt; - if (tabopt == 1) pbits[1] &= ~0x3c; - else if (tabopt == 2) pbits[11] &= 0x7f; - - /* Add the POSIX table or its complement into the main table that is - being built and we are done. */ - - if (local_negate) - for (c = 0; c < 32; c++) classbits[c] |= ~pbits[c]; - else - for (c = 0; c < 32; c++) classbits[c] |= pbits[c]; - - ptr = tempptr + 1; - /* Every class contains at least one < 256 character. */ - class_has_8bitchar = 1; - /* Every class contains at least two characters. */ - class_one_char = 2; - continue; /* End of POSIX syntax handling */ - } - - /* Backslash may introduce a single character, or it may introduce one - of the specials, which just set a flag. The sequence \b is a special - case. Inside a class (and only there) it is treated as backspace. We - assume that other escapes have more than one character in them, so - speculatively set both class_has_8bitchar and class_one_char bigger - than one. Unrecognized escapes fall through and are either treated - as literal characters (by default), or are faulted if - PCRE_EXTRA is set. */ - - if (c == CHAR_BACKSLASH) - { - escape = check_escape(&ptr, &ec, errorcodeptr, cd->bracount, options, - TRUE); - if (*errorcodeptr != 0) goto FAILED; - if (escape == 0) c = ec; - else if (escape == ESC_b) c = CHAR_BS; /* \b is backspace in a class */ - else if (escape == ESC_N) /* \N is not supported in a class */ - { - *errorcodeptr = ERR71; - goto FAILED; - } - else if (escape == ESC_Q) /* Handle start of quoted string */ - { - if (ptr[1] == CHAR_BACKSLASH && ptr[2] == CHAR_E) - { - ptr += 2; /* avoid empty string */ - } - else inescq = TRUE; - continue; - } - else if (escape == ESC_E) continue; /* Ignore orphan \E */ - - else - { - register const pcre_uint8 *cbits = cd->cbits; - /* Every class contains at least two < 256 characters. */ - class_has_8bitchar++; - /* Every class contains at least two characters. */ - class_one_char += 2; - - switch (escape) - { -#ifdef SUPPORT_UCP - case ESC_du: /* These are the values given for \d etc */ - case ESC_DU: /* when PCRE_UCP is set. We replace the */ - case ESC_wu: /* escape sequence with an appropriate \p */ - case ESC_WU: /* or \P to test Unicode properties instead */ - case ESC_su: /* of the default ASCII testing. */ - case ESC_SU: - nestptr = ptr; - ptr = substitutes[escape - ESC_DU] - 1; /* Just before substitute */ - class_has_8bitchar--; /* Undo! */ - continue; -#endif - case ESC_d: - for (c = 0; c < 32; c++) classbits[c] |= cbits[c+cbit_digit]; - continue; - - case ESC_D: - should_flip_negation = TRUE; - for (c = 0; c < 32; c++) classbits[c] |= ~cbits[c+cbit_digit]; - continue; - - case ESC_w: - for (c = 0; c < 32; c++) classbits[c] |= cbits[c+cbit_word]; - continue; - - case ESC_W: - should_flip_negation = TRUE; - for (c = 0; c < 32; c++) classbits[c] |= ~cbits[c+cbit_word]; - continue; - - /* Perl 5.004 onwards omitted VT from \s, but restored it at Perl - 5.18. Before PCRE 8.34, we had to preserve the VT bit if it was - previously set by something earlier in the character class. - Luckily, the value of CHAR_VT is 0x0b in both ASCII and EBCDIC, so - we could just adjust the appropriate bit. From PCRE 8.34 we no - longer treat \s and \S specially. */ - - case ESC_s: - for (c = 0; c < 32; c++) classbits[c] |= cbits[c+cbit_space]; - continue; - - case ESC_S: - should_flip_negation = TRUE; - for (c = 0; c < 32; c++) classbits[c] |= ~cbits[c+cbit_space]; - continue; - - /* The rest apply in both UCP and non-UCP cases. */ - - case ESC_h: - (void)add_list_to_class(classbits, &class_uchardata, options, cd, - PRIV(hspace_list), NOTACHAR); - continue; - - case ESC_H: - (void)add_not_list_to_class(classbits, &class_uchardata, options, - cd, PRIV(hspace_list)); - continue; - - case ESC_v: - (void)add_list_to_class(classbits, &class_uchardata, options, cd, - PRIV(vspace_list), NOTACHAR); - continue; - - case ESC_V: - (void)add_not_list_to_class(classbits, &class_uchardata, options, - cd, PRIV(vspace_list)); - continue; - - case ESC_p: - case ESC_P: -#ifdef SUPPORT_UCP - { - BOOL negated; - unsigned int ptype = 0, pdata = 0; - if (!get_ucp(&ptr, &negated, &ptype, &pdata, errorcodeptr)) - goto FAILED; - *class_uchardata++ = ((escape == ESC_p) != negated)? - XCL_PROP : XCL_NOTPROP; - *class_uchardata++ = ptype; - *class_uchardata++ = pdata; - xclass_has_prop = TRUE; - class_has_8bitchar--; /* Undo! */ - continue; - } -#else - *errorcodeptr = ERR45; - goto FAILED; -#endif - /* Unrecognized escapes are faulted if PCRE is running in its - strict mode. By default, for compatibility with Perl, they are - treated as literals. */ - - default: - if ((options & PCRE_EXTRA) != 0) - { - *errorcodeptr = ERR7; - goto FAILED; - } - class_has_8bitchar--; /* Undo the speculative increase. */ - class_one_char -= 2; /* Undo the speculative increase. */ - c = *ptr; /* Get the final character and fall through */ - break; - } - } - - /* Fall through if the escape just defined a single character (c >= 0). - This may be greater than 256. */ - - escape = 0; - - } /* End of backslash handling */ - - /* A character may be followed by '-' to form a range. However, Perl does - not permit ']' to be the end of the range. A '-' character at the end is - treated as a literal. Perl ignores orphaned \E sequences entirely. The - code for handling \Q and \E is messy. */ - - CHECK_RANGE: - while (ptr[1] == CHAR_BACKSLASH && ptr[2] == CHAR_E) - { - inescq = FALSE; - ptr += 2; - } - oldptr = ptr; - - /* Remember if \r or \n were explicitly used */ - - if (c == CHAR_CR || c == CHAR_NL) cd->external_flags |= PCRE_HASCRORLF; - - /* Check for range */ - - if (!inescq && ptr[1] == CHAR_MINUS) - { - pcre_uint32 d; - ptr += 2; - while (*ptr == CHAR_BACKSLASH && ptr[1] == CHAR_E) ptr += 2; - - /* If we hit \Q (not followed by \E) at this point, go into escaped - mode. */ - - while (*ptr == CHAR_BACKSLASH && ptr[1] == CHAR_Q) - { - ptr += 2; - if (*ptr == CHAR_BACKSLASH && ptr[1] == CHAR_E) - { ptr += 2; continue; } - inescq = TRUE; - break; - } - - /* Minus (hyphen) at the end of a class is treated as a literal, so put - back the pointer and jump to handle the character that preceded it. */ - - if (*ptr == CHAR_NULL || (!inescq && *ptr == CHAR_RIGHT_SQUARE_BRACKET)) - { - ptr = oldptr; - goto CLASS_SINGLE_CHARACTER; - } - - /* Otherwise, we have a potential range; pick up the next character */ - -#ifdef SUPPORT_UTF - if (utf) - { /* Braces are required because the */ - GETCHARLEN(d, ptr, ptr); /* macro generates multiple statements */ - } - else -#endif - d = *ptr; /* Not UTF-8 mode */ - - /* The second part of a range can be a single-character escape - sequence, but not any of the other escapes. Perl treats a hyphen as a - literal in such circumstances. However, in Perl's warning mode, a - warning is given, so PCRE now faults it as it is almost certainly a - mistake on the user's part. */ - - if (!inescq) - { - if (d == CHAR_BACKSLASH) - { - int descape; - descape = check_escape(&ptr, &d, errorcodeptr, cd->bracount, options, TRUE); - if (*errorcodeptr != 0) goto FAILED; - - /* 0 means a character was put into d; \b is backspace; any other - special causes an error. */ - - if (descape != 0) - { - if (descape == ESC_b) d = CHAR_BS; else - { - *errorcodeptr = ERR83; - goto FAILED; - } - } - } - - /* A hyphen followed by a POSIX class is treated in the same way. */ - - else if (d == CHAR_LEFT_SQUARE_BRACKET && - (ptr[1] == CHAR_COLON || ptr[1] == CHAR_DOT || - ptr[1] == CHAR_EQUALS_SIGN) && - check_posix_syntax(ptr, &tempptr)) - { - *errorcodeptr = ERR83; - goto FAILED; - } - } - - /* Check that the two values are in the correct order. Optimize - one-character ranges. */ - - if (d < c) - { - *errorcodeptr = ERR8; - goto FAILED; - } - if (d == c) goto CLASS_SINGLE_CHARACTER; /* A few lines below */ - - /* We have found a character range, so single character optimizations - cannot be done anymore. Any value greater than 1 indicates that there - is more than one character. */ - - class_one_char = 2; - - /* Remember an explicit \r or \n, and add the range to the class. */ - - if (d == CHAR_CR || d == CHAR_NL) cd->external_flags |= PCRE_HASCRORLF; - - class_has_8bitchar += - add_to_class(classbits, &class_uchardata, options, cd, c, d); - - continue; /* Go get the next char in the class */ - } - - /* Handle a single character - we can get here for a normal non-escape - char, or after \ that introduces a single character or for an apparent - range that isn't. Only the value 1 matters for class_one_char, so don't - increase it if it is already 2 or more ... just in case there's a class - with a zillion characters in it. */ - - CLASS_SINGLE_CHARACTER: - if (class_one_char < 2) class_one_char++; - - /* If xclass_has_prop is false and class_one_char is 1, we have the first - single character in the class, and there have been no prior ranges, or - XCLASS items generated by escapes. If this is the final character in the - class, we can optimize by turning the item into a 1-character OP_CHAR[I] - if it's positive, or OP_NOT[I] if it's negative. In the positive case, it - can cause firstchar to be set. Otherwise, there can be no first char if - this item is first, whatever repeat count may follow. In the case of - reqchar, save the previous value for reinstating. */ - - if (!inescq && -#ifdef SUPPORT_UCP - !xclass_has_prop && -#endif - class_one_char == 1 && ptr[1] == CHAR_RIGHT_SQUARE_BRACKET) - { - ptr++; - zeroreqchar = reqchar; - zeroreqcharflags = reqcharflags; - - if (negate_class) - { -#ifdef SUPPORT_UCP - int d; -#endif - if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; - zerofirstchar = firstchar; - zerofirstcharflags = firstcharflags; - - /* For caseless UTF-8 mode when UCP support is available, check - whether this character has more than one other case. If so, generate - a special OP_NOTPROP item instead of OP_NOTI. */ - -#ifdef SUPPORT_UCP - if (utf && (options & PCRE_CASELESS) != 0 && - (d = UCD_CASESET(c)) != 0) - { - *code++ = OP_NOTPROP; - *code++ = PT_CLIST; - *code++ = d; - } - else -#endif - /* Char has only one other case, or UCP not available */ - - { - *code++ = ((options & PCRE_CASELESS) != 0)? OP_NOTI: OP_NOT; -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (utf && c > MAX_VALUE_FOR_SINGLE_CHAR) - code += PRIV(ord2utf)(c, code); - else -#endif - *code++ = c; - } - - /* We are finished with this character class */ - - goto END_CLASS; - } - - /* For a single, positive character, get the value into mcbuffer, and - then we can handle this with the normal one-character code. */ - -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (utf && c > MAX_VALUE_FOR_SINGLE_CHAR) - mclength = PRIV(ord2utf)(c, mcbuffer); - else -#endif - { - mcbuffer[0] = c; - mclength = 1; - } - goto ONE_CHAR; - } /* End of 1-char optimization */ - - /* There is more than one character in the class, or an XCLASS item - has been generated. Add this character to the class. */ - - class_has_8bitchar += - add_to_class(classbits, &class_uchardata, options, cd, c, c); - } - - /* Loop until ']' reached. This "while" is the end of the "do" far above. - If we are at the end of an internal nested string, revert to the outer - string. */ - - while (((c = *(++ptr)) != CHAR_NULL || - (nestptr != NULL && - (ptr = nestptr, nestptr = NULL, c = *(++ptr)) != CHAR_NULL)) && - (c != CHAR_RIGHT_SQUARE_BRACKET || inescq)); - - /* Check for missing terminating ']' */ - - if (c == CHAR_NULL) - { - *errorcodeptr = ERR6; - goto FAILED; - } - - /* We will need an XCLASS if data has been placed in class_uchardata. In - the second phase this is a sufficient test. However, in the pre-compile - phase, class_uchardata gets emptied to prevent workspace overflow, so it - only if the very last character in the class needs XCLASS will it contain - anything at this point. For this reason, xclass gets set TRUE above when - uchar_classdata is emptied, and that's why this code is the way it is here - instead of just doing a test on class_uchardata below. */ - -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - if (class_uchardata > class_uchardata_base) xclass = TRUE; -#endif - - /* If this is the first thing in the branch, there can be no first char - setting, whatever the repeat count. Any reqchar setting must remain - unchanged after any kind of repeat. */ - - if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; - zerofirstchar = firstchar; - zerofirstcharflags = firstcharflags; - zeroreqchar = reqchar; - zeroreqcharflags = reqcharflags; - - /* If there are characters with values > 255, we have to compile an - extended class, with its own opcode, unless there was a negated special - such as \S in the class, and PCRE_UCP is not set, because in that case all - characters > 255 are in the class, so any that were explicitly given as - well can be ignored. If (when there are explicit characters > 255 that must - be listed) there are no characters < 256, we can omit the bitmap in the - actual compiled code. */ - -#ifdef SUPPORT_UTF - if (xclass && (xclass_has_prop || !should_flip_negation || - (options & PCRE_UCP) != 0)) -#elif !defined COMPILE_PCRE8 - if (xclass && (xclass_has_prop || !should_flip_negation)) -#endif -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - { - *class_uchardata++ = XCL_END; /* Marks the end of extra data */ - *code++ = OP_XCLASS; - code += LINK_SIZE; - *code = negate_class? XCL_NOT:0; - if (xclass_has_prop) *code |= XCL_HASPROP; - - /* If the map is required, move up the extra data to make room for it; - otherwise just move the code pointer to the end of the extra data. */ - - if (class_has_8bitchar > 0) - { - *code++ |= XCL_MAP; - memmove(code + (32 / sizeof(pcre_uchar)), code, - IN_UCHARS(class_uchardata - code)); - if (negate_class && !xclass_has_prop) - for (c = 0; c < 32; c++) classbits[c] = ~classbits[c]; - memcpy(code, classbits, 32); - code = class_uchardata + (32 / sizeof(pcre_uchar)); - } - else code = class_uchardata; - - /* Now fill in the complete length of the item */ - - PUT(previous, 1, (int)(code - previous)); - break; /* End of class handling */ - } - - /* Even though any XCLASS list is now discarded, we must allow for - its memory. */ - - if (lengthptr != NULL) - *lengthptr += (int)(class_uchardata - class_uchardata_base); -#endif - - /* If there are no characters > 255, or they are all to be included or - excluded, set the opcode to OP_CLASS or OP_NCLASS, depending on whether the - whole class was negated and whether there were negative specials such as \S - (non-UCP) in the class. Then copy the 32-byte map into the code vector, - negating it if necessary. */ - - *code++ = (negate_class == should_flip_negation) ? OP_CLASS : OP_NCLASS; - if (lengthptr == NULL) /* Save time in the pre-compile phase */ - { - if (negate_class) - for (c = 0; c < 32; c++) classbits[c] = ~classbits[c]; - memcpy(code, classbits, 32); - } - code += 32 / sizeof(pcre_uchar); - - END_CLASS: - break; - - - /* ===================================================================*/ - /* Various kinds of repeat; '{' is not necessarily a quantifier, but this - has been tested above. */ - - case CHAR_LEFT_CURLY_BRACKET: - if (!is_quantifier) goto NORMAL_CHAR; - ptr = read_repeat_counts(ptr+1, &repeat_min, &repeat_max, errorcodeptr); - if (*errorcodeptr != 0) goto FAILED; - goto REPEAT; - - case CHAR_ASTERISK: - repeat_min = 0; - repeat_max = -1; - goto REPEAT; - - case CHAR_PLUS: - repeat_min = 1; - repeat_max = -1; - goto REPEAT; - - case CHAR_QUESTION_MARK: - repeat_min = 0; - repeat_max = 1; - - REPEAT: - if (previous == NULL) - { - *errorcodeptr = ERR9; - goto FAILED; - } - - if (repeat_min == 0) - { - firstchar = zerofirstchar; /* Adjust for zero repeat */ - firstcharflags = zerofirstcharflags; - reqchar = zeroreqchar; /* Ditto */ - reqcharflags = zeroreqcharflags; - } - - /* Remember whether this is a variable length repeat */ - - reqvary = (repeat_min == repeat_max)? 0 : REQ_VARY; - - op_type = 0; /* Default single-char op codes */ - possessive_quantifier = FALSE; /* Default not possessive quantifier */ - - /* Save start of previous item, in case we have to move it up in order to - insert something before it. */ - - tempcode = previous; - - /* Before checking for a possessive quantifier, we must skip over - whitespace and comments in extended mode because Perl allows white space at - this point. */ - - if ((options & PCRE_EXTENDED) != 0) - { - const pcre_uchar *p = ptr + 1; - for (;;) - { - while (MAX_255(*p) && (cd->ctypes[*p] & ctype_space) != 0) p++; - if (*p != CHAR_NUMBER_SIGN) break; - p++; - while (*p != CHAR_NULL) - { - if (IS_NEWLINE(p)) /* For non-fixed-length newline cases, */ - { /* IS_NEWLINE sets cd->nllen. */ - p += cd->nllen; - break; - } - p++; -#ifdef SUPPORT_UTF - if (utf) FORWARDCHAR(p); -#endif - } /* Loop for comment characters */ - } /* Loop for multiple comments */ - ptr = p - 1; /* Character before the next significant one. */ - } - - /* If the next character is '+', we have a possessive quantifier. This - implies greediness, whatever the setting of the PCRE_UNGREEDY option. - If the next character is '?' this is a minimizing repeat, by default, - but if PCRE_UNGREEDY is set, it works the other way round. We change the - repeat type to the non-default. */ - - if (ptr[1] == CHAR_PLUS) - { - repeat_type = 0; /* Force greedy */ - possessive_quantifier = TRUE; - ptr++; - } - else if (ptr[1] == CHAR_QUESTION_MARK) - { - repeat_type = greedy_non_default; - ptr++; - } - else repeat_type = greedy_default; - - /* If previous was a recursion call, wrap it in atomic brackets so that - previous becomes the atomic group. All recursions were so wrapped in the - past, but it no longer happens for non-repeated recursions. In fact, the - repeated ones could be re-implemented independently so as not to need this, - but for the moment we rely on the code for repeating groups. */ - - if (*previous == OP_RECURSE) - { - memmove(previous + 1 + LINK_SIZE, previous, IN_UCHARS(1 + LINK_SIZE)); - *previous = OP_ONCE; - PUT(previous, 1, 2 + 2*LINK_SIZE); - previous[2 + 2*LINK_SIZE] = OP_KET; - PUT(previous, 3 + 2*LINK_SIZE, 2 + 2*LINK_SIZE); - code += 2 + 2 * LINK_SIZE; - length_prevgroup = 3 + 3*LINK_SIZE; - - /* When actually compiling, we need to check whether this was a forward - reference, and if so, adjust the offset. */ - - if (lengthptr == NULL && cd->hwm >= cd->start_workspace + LINK_SIZE) - { - int offset = GET(cd->hwm, -LINK_SIZE); - if (offset == previous + 1 - cd->start_code) - PUT(cd->hwm, -LINK_SIZE, offset + 1 + LINK_SIZE); - } - } - - /* Now handle repetition for the different types of item. */ - - /* If previous was a character or negated character match, abolish the item - and generate a repeat item instead. If a char item has a minimum of more - than one, ensure that it is set in reqchar - it might not be if a sequence - such as x{3} is the first thing in a branch because the x will have gone - into firstchar instead. */ - - if (*previous == OP_CHAR || *previous == OP_CHARI - || *previous == OP_NOT || *previous == OP_NOTI) - { - switch (*previous) - { - default: /* Make compiler happy. */ - case OP_CHAR: op_type = OP_STAR - OP_STAR; break; - case OP_CHARI: op_type = OP_STARI - OP_STAR; break; - case OP_NOT: op_type = OP_NOTSTAR - OP_STAR; break; - case OP_NOTI: op_type = OP_NOTSTARI - OP_STAR; break; - } - - /* Deal with UTF characters that take up more than one character. It's - easier to write this out separately than try to macrify it. Use c to - hold the length of the character in bytes, plus UTF_LENGTH to flag that - it's a length rather than a small character. */ - -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (utf && NOT_FIRSTCHAR(code[-1])) - { - pcre_uchar *lastchar = code - 1; - BACKCHAR(lastchar); - c = (int)(code - lastchar); /* Length of UTF-8 character */ - memcpy(utf_chars, lastchar, IN_UCHARS(c)); /* Save the char */ - c |= UTF_LENGTH; /* Flag c as a length */ - } - else -#endif /* SUPPORT_UTF */ - - /* Handle the case of a single charater - either with no UTF support, or - with UTF disabled, or for a single character UTF character. */ - { - c = code[-1]; - if (*previous <= OP_CHARI && repeat_min > 1) - { - reqchar = c; - reqcharflags = req_caseopt | cd->req_varyopt; - } - } - - goto OUTPUT_SINGLE_REPEAT; /* Code shared with single character types */ - } - - /* If previous was a character type match (\d or similar), abolish it and - create a suitable repeat item. The code is shared with single-character - repeats by setting op_type to add a suitable offset into repeat_type. Note - the the Unicode property types will be present only when SUPPORT_UCP is - defined, but we don't wrap the little bits of code here because it just - makes it horribly messy. */ - - else if (*previous < OP_EODN) - { - pcre_uchar *oldcode; - int prop_type, prop_value; - op_type = OP_TYPESTAR - OP_STAR; /* Use type opcodes */ - c = *previous; - - OUTPUT_SINGLE_REPEAT: - if (*previous == OP_PROP || *previous == OP_NOTPROP) - { - prop_type = previous[1]; - prop_value = previous[2]; - } - else prop_type = prop_value = -1; - - oldcode = code; - code = previous; /* Usually overwrite previous item */ - - /* If the maximum is zero then the minimum must also be zero; Perl allows - this case, so we do too - by simply omitting the item altogether. */ - - if (repeat_max == 0) goto END_REPEAT; - - /* Combine the op_type with the repeat_type */ - - repeat_type += op_type; - - /* A minimum of zero is handled either as the special case * or ?, or as - an UPTO, with the maximum given. */ - - if (repeat_min == 0) - { - if (repeat_max == -1) *code++ = OP_STAR + repeat_type; - else if (repeat_max == 1) *code++ = OP_QUERY + repeat_type; - else - { - *code++ = OP_UPTO + repeat_type; - PUT2INC(code, 0, repeat_max); - } - } - - /* A repeat minimum of 1 is optimized into some special cases. If the - maximum is unlimited, we use OP_PLUS. Otherwise, the original item is - left in place and, if the maximum is greater than 1, we use OP_UPTO with - one less than the maximum. */ - - else if (repeat_min == 1) - { - if (repeat_max == -1) - *code++ = OP_PLUS + repeat_type; - else - { - code = oldcode; /* leave previous item in place */ - if (repeat_max == 1) goto END_REPEAT; - *code++ = OP_UPTO + repeat_type; - PUT2INC(code, 0, repeat_max - 1); - } - } - - /* The case {n,n} is just an EXACT, while the general case {n,m} is - handled as an EXACT followed by an UPTO. */ - - else - { - *code++ = OP_EXACT + op_type; /* NB EXACT doesn't have repeat_type */ - PUT2INC(code, 0, repeat_min); - - /* If the maximum is unlimited, insert an OP_STAR. Before doing so, - we have to insert the character for the previous code. For a repeated - Unicode property match, there are two extra bytes that define the - required property. In UTF-8 mode, long characters have their length in - c, with the UTF_LENGTH bit as a flag. */ - - if (repeat_max < 0) - { -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (utf && (c & UTF_LENGTH) != 0) - { - memcpy(code, utf_chars, IN_UCHARS(c & 7)); - code += c & 7; - } - else -#endif - { - *code++ = c; - if (prop_type >= 0) - { - *code++ = prop_type; - *code++ = prop_value; - } - } - *code++ = OP_STAR + repeat_type; - } - - /* Else insert an UPTO if the max is greater than the min, again - preceded by the character, for the previously inserted code. If the - UPTO is just for 1 instance, we can use QUERY instead. */ - - else if (repeat_max != repeat_min) - { -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (utf && (c & UTF_LENGTH) != 0) - { - memcpy(code, utf_chars, IN_UCHARS(c & 7)); - code += c & 7; - } - else -#endif - *code++ = c; - if (prop_type >= 0) - { - *code++ = prop_type; - *code++ = prop_value; - } - repeat_max -= repeat_min; - - if (repeat_max == 1) - { - *code++ = OP_QUERY + repeat_type; - } - else - { - *code++ = OP_UPTO + repeat_type; - PUT2INC(code, 0, repeat_max); - } - } - } - - /* The character or character type itself comes last in all cases. */ - -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (utf && (c & UTF_LENGTH) != 0) - { - memcpy(code, utf_chars, IN_UCHARS(c & 7)); - code += c & 7; - } - else -#endif - *code++ = c; - - /* For a repeated Unicode property match, there are two extra bytes that - define the required property. */ - -#ifdef SUPPORT_UCP - if (prop_type >= 0) - { - *code++ = prop_type; - *code++ = prop_value; - } -#endif - } - - /* If previous was a character class or a back reference, we put the repeat - stuff after it, but just skip the item if the repeat was {0,0}. */ - - else if (*previous == OP_CLASS || *previous == OP_NCLASS || -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - *previous == OP_XCLASS || -#endif - *previous == OP_REF || *previous == OP_REFI || - *previous == OP_DNREF || *previous == OP_DNREFI) - { - if (repeat_max == 0) - { - code = previous; - goto END_REPEAT; - } - - if (repeat_min == 0 && repeat_max == -1) - *code++ = OP_CRSTAR + repeat_type; - else if (repeat_min == 1 && repeat_max == -1) - *code++ = OP_CRPLUS + repeat_type; - else if (repeat_min == 0 && repeat_max == 1) - *code++ = OP_CRQUERY + repeat_type; - else - { - *code++ = OP_CRRANGE + repeat_type; - PUT2INC(code, 0, repeat_min); - if (repeat_max == -1) repeat_max = 0; /* 2-byte encoding for max */ - PUT2INC(code, 0, repeat_max); - } - } - - /* If previous was a bracket group, we may have to replicate it in certain - cases. Note that at this point we can encounter only the "basic" bracket - opcodes such as BRA and CBRA, as this is the place where they get converted - into the more special varieties such as BRAPOS and SBRA. A test for >= - OP_ASSERT and <= OP_COND includes ASSERT, ASSERT_NOT, ASSERTBACK, - ASSERTBACK_NOT, ONCE, ONCE_NC, BRA, BRAPOS, CBRA, CBRAPOS, and COND. - Originally, PCRE did not allow repetition of assertions, but now it does, - for Perl compatibility. */ - - else if (*previous >= OP_ASSERT && *previous <= OP_COND) - { - register int i; - int len = (int)(code - previous); - size_t base_hwm_offset = item_hwm_offset; - pcre_uchar *bralink = NULL; - pcre_uchar *brazeroptr = NULL; - - /* Repeating a DEFINE group is pointless, but Perl allows the syntax, so - we just ignore the repeat. */ - - if (*previous == OP_COND && previous[LINK_SIZE+1] == OP_DEF) - goto END_REPEAT; - - /* There is no sense in actually repeating assertions. The only potential - use of repetition is in cases when the assertion is optional. Therefore, - if the minimum is greater than zero, just ignore the repeat. If the - maximum is not zero or one, set it to 1. */ - - if (*previous < OP_ONCE) /* Assertion */ - { - if (repeat_min > 0) goto END_REPEAT; - if (repeat_max < 0 || repeat_max > 1) repeat_max = 1; - } - - /* The case of a zero minimum is special because of the need to stick - OP_BRAZERO in front of it, and because the group appears once in the - data, whereas in other cases it appears the minimum number of times. For - this reason, it is simplest to treat this case separately, as otherwise - the code gets far too messy. There are several special subcases when the - minimum is zero. */ - - if (repeat_min == 0) - { - /* If the maximum is also zero, we used to just omit the group from the - output altogether, like this: - - ** if (repeat_max == 0) - ** { - ** code = previous; - ** goto END_REPEAT; - ** } - - However, that fails when a group or a subgroup within it is referenced - as a subroutine from elsewhere in the pattern, so now we stick in - OP_SKIPZERO in front of it so that it is skipped on execution. As we - don't have a list of which groups are referenced, we cannot do this - selectively. - - If the maximum is 1 or unlimited, we just have to stick in the BRAZERO - and do no more at this point. However, we do need to adjust any - OP_RECURSE calls inside the group that refer to the group itself or any - internal or forward referenced group, because the offset is from the - start of the whole regex. Temporarily terminate the pattern while doing - this. */ - - if (repeat_max <= 1) /* Covers 0, 1, and unlimited */ - { - *code = OP_END; - adjust_recurse(previous, 1, utf, cd, item_hwm_offset); - memmove(previous + 1, previous, IN_UCHARS(len)); - code++; - if (repeat_max == 0) - { - *previous++ = OP_SKIPZERO; - goto END_REPEAT; - } - brazeroptr = previous; /* Save for possessive optimizing */ - *previous++ = OP_BRAZERO + repeat_type; - } - - /* If the maximum is greater than 1 and limited, we have to replicate - in a nested fashion, sticking OP_BRAZERO before each set of brackets. - The first one has to be handled carefully because it's the original - copy, which has to be moved up. The remainder can be handled by code - that is common with the non-zero minimum case below. We have to - adjust the value or repeat_max, since one less copy is required. Once - again, we may have to adjust any OP_RECURSE calls inside the group. */ - - else - { - int offset; - *code = OP_END; - adjust_recurse(previous, 2 + LINK_SIZE, utf, cd, item_hwm_offset); - memmove(previous + 2 + LINK_SIZE, previous, IN_UCHARS(len)); - code += 2 + LINK_SIZE; - *previous++ = OP_BRAZERO + repeat_type; - *previous++ = OP_BRA; - - /* We chain together the bracket offset fields that have to be - filled in later when the ends of the brackets are reached. */ - - offset = (bralink == NULL)? 0 : (int)(previous - bralink); - bralink = previous; - PUTINC(previous, 0, offset); - } - - repeat_max--; - } - - /* If the minimum is greater than zero, replicate the group as many - times as necessary, and adjust the maximum to the number of subsequent - copies that we need. If we set a first char from the group, and didn't - set a required char, copy the latter from the former. If there are any - forward reference subroutine calls in the group, there will be entries on - the workspace list; replicate these with an appropriate increment. */ - - else - { - if (repeat_min > 1) - { - /* In the pre-compile phase, we don't actually do the replication. We - just adjust the length as if we had. Do some paranoid checks for - potential integer overflow. The INT64_OR_DOUBLE type is a 64-bit - integer type when available, otherwise double. */ - - if (lengthptr != NULL) - { - int delta = (repeat_min - 1)*length_prevgroup; - if ((INT64_OR_DOUBLE)(repeat_min - 1)* - (INT64_OR_DOUBLE)length_prevgroup > - (INT64_OR_DOUBLE)INT_MAX || - OFLOW_MAX - *lengthptr < delta) - { - *errorcodeptr = ERR20; - goto FAILED; - } - *lengthptr += delta; - } - - /* This is compiling for real. If there is a set first byte for - the group, and we have not yet set a "required byte", set it. Make - sure there is enough workspace for copying forward references before - doing the copy. */ - - else - { - if (groupsetfirstchar && reqcharflags < 0) - { - reqchar = firstchar; - reqcharflags = firstcharflags; - } - - for (i = 1; i < repeat_min; i++) - { - pcre_uchar *hc; - size_t this_hwm_offset = cd->hwm - cd->start_workspace; - memcpy(code, previous, IN_UCHARS(len)); - - while (cd->hwm > cd->start_workspace + cd->workspace_size - - WORK_SIZE_SAFETY_MARGIN - - (this_hwm_offset - base_hwm_offset)) - { - *errorcodeptr = expand_workspace(cd); - if (*errorcodeptr != 0) goto FAILED; - } - - for (hc = (pcre_uchar *)cd->start_workspace + base_hwm_offset; - hc < (pcre_uchar *)cd->start_workspace + this_hwm_offset; - hc += LINK_SIZE) - { - PUT(cd->hwm, 0, GET(hc, 0) + len); - cd->hwm += LINK_SIZE; - } - base_hwm_offset = this_hwm_offset; - code += len; - } - } - } - - if (repeat_max > 0) repeat_max -= repeat_min; - } - - /* This code is common to both the zero and non-zero minimum cases. If - the maximum is limited, it replicates the group in a nested fashion, - remembering the bracket starts on a stack. In the case of a zero minimum, - the first one was set up above. In all cases the repeat_max now specifies - the number of additional copies needed. Again, we must remember to - replicate entries on the forward reference list. */ - - if (repeat_max >= 0) - { - /* In the pre-compile phase, we don't actually do the replication. We - just adjust the length as if we had. For each repetition we must add 1 - to the length for BRAZERO and for all but the last repetition we must - add 2 + 2*LINKSIZE to allow for the nesting that occurs. Do some - paranoid checks to avoid integer overflow. The INT64_OR_DOUBLE type is - a 64-bit integer type when available, otherwise double. */ - - if (lengthptr != NULL && repeat_max > 0) - { - int delta = repeat_max * (length_prevgroup + 1 + 2 + 2*LINK_SIZE) - - 2 - 2*LINK_SIZE; /* Last one doesn't nest */ - if ((INT64_OR_DOUBLE)repeat_max * - (INT64_OR_DOUBLE)(length_prevgroup + 1 + 2 + 2*LINK_SIZE) - > (INT64_OR_DOUBLE)INT_MAX || - OFLOW_MAX - *lengthptr < delta) - { - *errorcodeptr = ERR20; - goto FAILED; - } - *lengthptr += delta; - } - - /* This is compiling for real */ - - else for (i = repeat_max - 1; i >= 0; i--) - { - pcre_uchar *hc; - size_t this_hwm_offset = cd->hwm - cd->start_workspace; - - *code++ = OP_BRAZERO + repeat_type; - - /* All but the final copy start a new nesting, maintaining the - chain of brackets outstanding. */ - - if (i != 0) - { - int offset; - *code++ = OP_BRA; - offset = (bralink == NULL)? 0 : (int)(code - bralink); - bralink = code; - PUTINC(code, 0, offset); - } - - memcpy(code, previous, IN_UCHARS(len)); - - /* Ensure there is enough workspace for forward references before - copying them. */ - - while (cd->hwm > cd->start_workspace + cd->workspace_size - - WORK_SIZE_SAFETY_MARGIN - - (this_hwm_offset - base_hwm_offset)) - { - *errorcodeptr = expand_workspace(cd); - if (*errorcodeptr != 0) goto FAILED; - } - - for (hc = (pcre_uchar *)cd->start_workspace + base_hwm_offset; - hc < (pcre_uchar *)cd->start_workspace + this_hwm_offset; - hc += LINK_SIZE) - { - PUT(cd->hwm, 0, GET(hc, 0) + len + ((i != 0)? 2+LINK_SIZE : 1)); - cd->hwm += LINK_SIZE; - } - base_hwm_offset = this_hwm_offset; - code += len; - } - - /* Now chain through the pending brackets, and fill in their length - fields (which are holding the chain links pro tem). */ - - while (bralink != NULL) - { - int oldlinkoffset; - int offset = (int)(code - bralink + 1); - pcre_uchar *bra = code - offset; - oldlinkoffset = GET(bra, 1); - bralink = (oldlinkoffset == 0)? NULL : bralink - oldlinkoffset; - *code++ = OP_KET; - PUTINC(code, 0, offset); - PUT(bra, 1, offset); - } - } - - /* If the maximum is unlimited, set a repeater in the final copy. For - ONCE brackets, that's all we need to do. However, possessively repeated - ONCE brackets can be converted into non-capturing brackets, as the - behaviour of (?:xx)++ is the same as (?>xx)++ and this saves having to - deal with possessive ONCEs specially. - - Otherwise, when we are doing the actual compile phase, check to see - whether this group is one that could match an empty string. If so, - convert the initial operator to the S form (e.g. OP_BRA -> OP_SBRA) so - that runtime checking can be done. [This check is also applied to ONCE - groups at runtime, but in a different way.] - - Then, if the quantifier was possessive and the bracket is not a - conditional, we convert the BRA code to the POS form, and the KET code to - KETRPOS. (It turns out to be convenient at runtime to detect this kind of - subpattern at both the start and at the end.) The use of special opcodes - makes it possible to reduce greatly the stack usage in pcre_exec(). If - the group is preceded by OP_BRAZERO, convert this to OP_BRAPOSZERO. - - Then, if the minimum number of matches is 1 or 0, cancel the possessive - flag so that the default action below, of wrapping everything inside - atomic brackets, does not happen. When the minimum is greater than 1, - there will be earlier copies of the group, and so we still have to wrap - the whole thing. */ - - else - { - pcre_uchar *ketcode = code - 1 - LINK_SIZE; - pcre_uchar *bracode = ketcode - GET(ketcode, 1); - - /* Convert possessive ONCE brackets to non-capturing */ - - if ((*bracode == OP_ONCE || *bracode == OP_ONCE_NC) && - possessive_quantifier) *bracode = OP_BRA; - - /* For non-possessive ONCE brackets, all we need to do is to - set the KET. */ - - if (*bracode == OP_ONCE || *bracode == OP_ONCE_NC) - *ketcode = OP_KETRMAX + repeat_type; - - /* Handle non-ONCE brackets and possessive ONCEs (which have been - converted to non-capturing above). */ - - else - { - /* In the compile phase, check for empty string matching. */ - - if (lengthptr == NULL) - { - pcre_uchar *scode = bracode; - do - { - if (could_be_empty_branch(scode, ketcode, utf, cd, NULL)) - { - *bracode += OP_SBRA - OP_BRA; - break; - } - scode += GET(scode, 1); - } - while (*scode == OP_ALT); - } - - /* A conditional group with only one branch has an implicit empty - alternative branch. */ - - if (*bracode == OP_COND && bracode[GET(bracode,1)] != OP_ALT) - *bracode = OP_SCOND; - - /* Handle possessive quantifiers. */ - - if (possessive_quantifier) - { - /* For COND brackets, we wrap the whole thing in a possessively - repeated non-capturing bracket, because we have not invented POS - versions of the COND opcodes. Because we are moving code along, we - must ensure that any pending recursive references are updated. */ - - if (*bracode == OP_COND || *bracode == OP_SCOND) - { - int nlen = (int)(code - bracode); - *code = OP_END; - adjust_recurse(bracode, 1 + LINK_SIZE, utf, cd, item_hwm_offset); - memmove(bracode + 1 + LINK_SIZE, bracode, IN_UCHARS(nlen)); - code += 1 + LINK_SIZE; - nlen += 1 + LINK_SIZE; - *bracode = (*bracode == OP_COND)? OP_BRAPOS : OP_SBRAPOS; - *code++ = OP_KETRPOS; - PUTINC(code, 0, nlen); - PUT(bracode, 1, nlen); - } - - /* For non-COND brackets, we modify the BRA code and use KETRPOS. */ - - else - { - *bracode += 1; /* Switch to xxxPOS opcodes */ - *ketcode = OP_KETRPOS; - } - - /* If the minimum is zero, mark it as possessive, then unset the - possessive flag when the minimum is 0 or 1. */ - - if (brazeroptr != NULL) *brazeroptr = OP_BRAPOSZERO; - if (repeat_min < 2) possessive_quantifier = FALSE; - } - - /* Non-possessive quantifier */ - - else *ketcode = OP_KETRMAX + repeat_type; - } - } - } - - /* If previous is OP_FAIL, it was generated by an empty class [] in - JavaScript mode. The other ways in which OP_FAIL can be generated, that is - by (*FAIL) or (?!) set previous to NULL, which gives a "nothing to repeat" - error above. We can just ignore the repeat in JS case. */ - - else if (*previous == OP_FAIL) goto END_REPEAT; - - /* Else there's some kind of shambles */ - - else - { - *errorcodeptr = ERR11; - goto FAILED; - } - - /* If the character following a repeat is '+', possessive_quantifier is - TRUE. For some opcodes, there are special alternative opcodes for this - case. For anything else, we wrap the entire repeated item inside OP_ONCE - brackets. Logically, the '+' notation is just syntactic sugar, taken from - Sun's Java package, but the special opcodes can optimize it. - - Some (but not all) possessively repeated subpatterns have already been - completely handled in the code just above. For them, possessive_quantifier - is always FALSE at this stage. Note that the repeated item starts at - tempcode, not at previous, which might be the first part of a string whose - (former) last char we repeated. */ - - if (possessive_quantifier) - { - int len; - - /* Possessifying an EXACT quantifier has no effect, so we can ignore it. - However, QUERY, STAR, or UPTO may follow (for quantifiers such as {5,6}, - {5,}, or {5,10}). We skip over an EXACT item; if the length of what - remains is greater than zero, there's a further opcode that can be - handled. If not, do nothing, leaving the EXACT alone. */ - - switch(*tempcode) - { - case OP_TYPEEXACT: - tempcode += PRIV(OP_lengths)[*tempcode] + - ((tempcode[1 + IMM2_SIZE] == OP_PROP - || tempcode[1 + IMM2_SIZE] == OP_NOTPROP)? 2 : 0); - break; - - /* CHAR opcodes are used for exacts whose count is 1. */ - - case OP_CHAR: - case OP_CHARI: - case OP_NOT: - case OP_NOTI: - case OP_EXACT: - case OP_EXACTI: - case OP_NOTEXACT: - case OP_NOTEXACTI: - tempcode += PRIV(OP_lengths)[*tempcode]; -#ifdef SUPPORT_UTF - if (utf && HAS_EXTRALEN(tempcode[-1])) - tempcode += GET_EXTRALEN(tempcode[-1]); -#endif - break; - - /* For the class opcodes, the repeat operator appears at the end; - adjust tempcode to point to it. */ - - case OP_CLASS: - case OP_NCLASS: - tempcode += 1 + 32/sizeof(pcre_uchar); - break; - -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - case OP_XCLASS: - tempcode += GET(tempcode, 1); - break; -#endif - } - - /* If tempcode is equal to code (which points to the end of the repeated - item), it means we have skipped an EXACT item but there is no following - QUERY, STAR, or UPTO; the value of len will be 0, and we do nothing. In - all other cases, tempcode will be pointing to the repeat opcode, and will - be less than code, so the value of len will be greater than 0. */ - - len = (int)(code - tempcode); - if (len > 0) - { - unsigned int repcode = *tempcode; - - /* There is a table for possessifying opcodes, all of which are less - than OP_CALLOUT. A zero entry means there is no possessified version. - */ - - if (repcode < OP_CALLOUT && opcode_possessify[repcode] > 0) - *tempcode = opcode_possessify[repcode]; - - /* For opcode without a special possessified version, wrap the item in - ONCE brackets. Because we are moving code along, we must ensure that any - pending recursive references are updated. */ - - else - { - *code = OP_END; - adjust_recurse(tempcode, 1 + LINK_SIZE, utf, cd, item_hwm_offset); - memmove(tempcode + 1 + LINK_SIZE, tempcode, IN_UCHARS(len)); - code += 1 + LINK_SIZE; - len += 1 + LINK_SIZE; - tempcode[0] = OP_ONCE; - *code++ = OP_KET; - PUTINC(code, 0, len); - PUT(tempcode, 1, len); - } - } - -#ifdef NEVER - if (len > 0) switch (*tempcode) - { - case OP_STAR: *tempcode = OP_POSSTAR; break; - case OP_PLUS: *tempcode = OP_POSPLUS; break; - case OP_QUERY: *tempcode = OP_POSQUERY; break; - case OP_UPTO: *tempcode = OP_POSUPTO; break; - - case OP_STARI: *tempcode = OP_POSSTARI; break; - case OP_PLUSI: *tempcode = OP_POSPLUSI; break; - case OP_QUERYI: *tempcode = OP_POSQUERYI; break; - case OP_UPTOI: *tempcode = OP_POSUPTOI; break; - - case OP_NOTSTAR: *tempcode = OP_NOTPOSSTAR; break; - case OP_NOTPLUS: *tempcode = OP_NOTPOSPLUS; break; - case OP_NOTQUERY: *tempcode = OP_NOTPOSQUERY; break; - case OP_NOTUPTO: *tempcode = OP_NOTPOSUPTO; break; - - case OP_NOTSTARI: *tempcode = OP_NOTPOSSTARI; break; - case OP_NOTPLUSI: *tempcode = OP_NOTPOSPLUSI; break; - case OP_NOTQUERYI: *tempcode = OP_NOTPOSQUERYI; break; - case OP_NOTUPTOI: *tempcode = OP_NOTPOSUPTOI; break; - - case OP_TYPESTAR: *tempcode = OP_TYPEPOSSTAR; break; - case OP_TYPEPLUS: *tempcode = OP_TYPEPOSPLUS; break; - case OP_TYPEQUERY: *tempcode = OP_TYPEPOSQUERY; break; - case OP_TYPEUPTO: *tempcode = OP_TYPEPOSUPTO; break; - - case OP_CRSTAR: *tempcode = OP_CRPOSSTAR; break; - case OP_CRPLUS: *tempcode = OP_CRPOSPLUS; break; - case OP_CRQUERY: *tempcode = OP_CRPOSQUERY; break; - case OP_CRRANGE: *tempcode = OP_CRPOSRANGE; break; - - /* Because we are moving code along, we must ensure that any - pending recursive references are updated. */ - - default: - *code = OP_END; - adjust_recurse(tempcode, 1 + LINK_SIZE, utf, cd, item_hwm_offset); - memmove(tempcode + 1 + LINK_SIZE, tempcode, IN_UCHARS(len)); - code += 1 + LINK_SIZE; - len += 1 + LINK_SIZE; - tempcode[0] = OP_ONCE; - *code++ = OP_KET; - PUTINC(code, 0, len); - PUT(tempcode, 1, len); - break; - } -#endif - } - - /* In all case we no longer have a previous item. We also set the - "follows varying string" flag for subsequently encountered reqchars if - it isn't already set and we have just passed a varying length item. */ - - END_REPEAT: - previous = NULL; - cd->req_varyopt |= reqvary; - break; - - - /* ===================================================================*/ - /* Start of nested parenthesized sub-expression, or comment or lookahead or - lookbehind or option setting or condition or all the other extended - parenthesis forms. */ - - case CHAR_LEFT_PARENTHESIS: - ptr++; - - /* Now deal with various "verbs" that can be introduced by '*'. */ - - if (ptr[0] == CHAR_ASTERISK && (ptr[1] == ':' - || (MAX_255(ptr[1]) && ((cd->ctypes[ptr[1]] & ctype_letter) != 0)))) - { - int i, namelen; - int arglen = 0; - const char *vn = verbnames; - const pcre_uchar *name = ptr + 1; - const pcre_uchar *arg = NULL; - previous = NULL; - ptr++; - while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_letter) != 0) ptr++; - namelen = (int)(ptr - name); - - /* It appears that Perl allows any characters whatsoever, other than - a closing parenthesis, to appear in arguments, so we no longer insist on - letters, digits, and underscores. */ - - if (*ptr == CHAR_COLON) - { - arg = ++ptr; - while (*ptr != CHAR_NULL && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++; - arglen = (int)(ptr - arg); - if ((unsigned int)arglen > MAX_MARK) - { - *errorcodeptr = ERR75; - goto FAILED; - } - } - - if (*ptr != CHAR_RIGHT_PARENTHESIS) - { - *errorcodeptr = ERR60; - goto FAILED; - } - - /* Scan the table of verb names */ - - for (i = 0; i < verbcount; i++) - { - if (namelen == verbs[i].len && - STRNCMP_UC_C8(name, vn, namelen) == 0) - { - int setverb; - - /* Check for open captures before ACCEPT and convert it to - ASSERT_ACCEPT if in an assertion. */ - - if (verbs[i].op == OP_ACCEPT) - { - open_capitem *oc; - if (arglen != 0) - { - *errorcodeptr = ERR59; - goto FAILED; - } - cd->had_accept = TRUE; - for (oc = cd->open_caps; oc != NULL; oc = oc->next) - { - if (lengthptr != NULL) - { -#ifdef COMPILE_PCRE8 - *lengthptr += 1 + IMM2_SIZE; -#elif defined COMPILE_PCRE16 - *lengthptr += 2 + IMM2_SIZE; -#elif defined COMPILE_PCRE32 - *lengthptr += 4 + IMM2_SIZE; -#endif - } - else - { - *code++ = OP_CLOSE; - PUT2INC(code, 0, oc->number); - } - } - setverb = *code++ = - (cd->assert_depth > 0)? OP_ASSERT_ACCEPT : OP_ACCEPT; - - /* Do not set firstchar after *ACCEPT */ - if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; - } - - /* Handle other cases with/without an argument */ - - else if (arglen == 0) - { - if (verbs[i].op < 0) /* Argument is mandatory */ - { - *errorcodeptr = ERR66; - goto FAILED; - } - setverb = *code++ = verbs[i].op; - } - - else - { - if (verbs[i].op_arg < 0) /* Argument is forbidden */ - { - *errorcodeptr = ERR59; - goto FAILED; - } - setverb = *code++ = verbs[i].op_arg; - if (lengthptr != NULL) /* In pass 1 just add in the length */ - { /* to avoid potential workspace */ - *lengthptr += arglen; /* overflow. */ - *code++ = 0; - } - else - { - *code++ = arglen; - memcpy(code, arg, IN_UCHARS(arglen)); - code += arglen; - } - *code++ = 0; - } - - switch (setverb) - { - case OP_THEN: - case OP_THEN_ARG: - cd->external_flags |= PCRE_HASTHEN; - break; - - case OP_PRUNE: - case OP_PRUNE_ARG: - case OP_SKIP: - case OP_SKIP_ARG: - cd->had_pruneorskip = TRUE; - break; - } - - break; /* Found verb, exit loop */ - } - - vn += verbs[i].len + 1; - } - - if (i < verbcount) continue; /* Successfully handled a verb */ - *errorcodeptr = ERR60; /* Verb not recognized */ - goto FAILED; - } - - /* Initialize for "real" parentheses */ - - newoptions = options; - skipbytes = 0; - bravalue = OP_CBRA; - item_hwm_offset = cd->hwm - cd->start_workspace; - reset_bracount = FALSE; - - /* Deal with the extended parentheses; all are introduced by '?', and the - appearance of any of them means that this is not a capturing group. */ - - if (*ptr == CHAR_QUESTION_MARK) - { - int i, set, unset, namelen; - int *optset; - const pcre_uchar *name; - pcre_uchar *slot; - - switch (*(++ptr)) - { - /* ------------------------------------------------------------ */ - case CHAR_VERTICAL_LINE: /* Reset capture count for each branch */ - reset_bracount = TRUE; - cd->dupgroups = TRUE; /* Record (?| encountered */ - /* Fall through */ - - /* ------------------------------------------------------------ */ - case CHAR_COLON: /* Non-capturing bracket */ - bravalue = OP_BRA; - ptr++; - break; - - - /* ------------------------------------------------------------ */ - case CHAR_LEFT_PARENTHESIS: - bravalue = OP_COND; /* Conditional group */ - tempptr = ptr; - - /* A condition can be an assertion, a number (referring to a numbered - group's having been set), a name (referring to a named group), or 'R', - referring to recursion. R and R&name are also permitted for - recursion tests. - - There are ways of testing a named group: (?(name)) is used by Python; - Perl 5.10 onwards uses (?() or (?('name')). - - There is one unfortunate ambiguity, caused by history. 'R' can be the - recursive thing or the name 'R' (and similarly for 'R' followed by - digits). We look for a name first; if not found, we try the other case. - - For compatibility with auto-callouts, we allow a callout to be - specified before a condition that is an assertion. First, check for the - syntax of a callout; if found, adjust the temporary pointer that is - used to check for an assertion condition. That's all that is needed! */ - - if (ptr[1] == CHAR_QUESTION_MARK && ptr[2] == CHAR_C) - { - for (i = 3;; i++) if (!IS_DIGIT(ptr[i])) break; - if (ptr[i] == CHAR_RIGHT_PARENTHESIS) - tempptr += i + 1; - - /* tempptr should now be pointing to the opening parenthesis of the - assertion condition. */ - - if (*tempptr != CHAR_LEFT_PARENTHESIS) - { - *errorcodeptr = ERR28; - goto FAILED; - } - } - - /* For conditions that are assertions, check the syntax, and then exit - the switch. This will take control down to where bracketed groups, - including assertions, are processed. */ - - if (tempptr[1] == CHAR_QUESTION_MARK && - (tempptr[2] == CHAR_EQUALS_SIGN || - tempptr[2] == CHAR_EXCLAMATION_MARK || - (tempptr[2] == CHAR_LESS_THAN_SIGN && - (tempptr[3] == CHAR_EQUALS_SIGN || - tempptr[3] == CHAR_EXCLAMATION_MARK)))) - { - cd->iscondassert = TRUE; - break; - } - - /* Other conditions use OP_CREF/OP_DNCREF/OP_RREF/OP_DNRREF, and all - need to skip at least 1+IMM2_SIZE bytes at the start of the group. */ - - code[1+LINK_SIZE] = OP_CREF; - skipbytes = 1+IMM2_SIZE; - refsign = -1; /* => not a number */ - namelen = -1; /* => not a name; must set to avoid warning */ - name = NULL; /* Always set to avoid warning */ - recno = 0; /* Always set to avoid warning */ - - /* Check for a test for recursion in a named group. */ - - ptr++; - if (*ptr == CHAR_R && ptr[1] == CHAR_AMPERSAND) - { - terminator = -1; - ptr += 2; - code[1+LINK_SIZE] = OP_RREF; /* Change the type of test */ - } - - /* Check for a test for a named group's having been set, using the Perl - syntax (?() or (?('name'), and also allow for the original PCRE - syntax of (?(name) or for (?(+n), (?(-n), and just (?(n). */ - - else if (*ptr == CHAR_LESS_THAN_SIGN) - { - terminator = CHAR_GREATER_THAN_SIGN; - ptr++; - } - else if (*ptr == CHAR_APOSTROPHE) - { - terminator = CHAR_APOSTROPHE; - ptr++; - } - else - { - terminator = CHAR_NULL; - if (*ptr == CHAR_MINUS || *ptr == CHAR_PLUS) refsign = *ptr++; - else if (IS_DIGIT(*ptr)) refsign = 0; - } - - /* Handle a number */ - - if (refsign >= 0) - { - while (IS_DIGIT(*ptr)) - { - if (recno > INT_MAX / 10 - 1) /* Integer overflow */ - { - while (IS_DIGIT(*ptr)) ptr++; - *errorcodeptr = ERR61; - goto FAILED; - } - recno = recno * 10 + (int)(*ptr - CHAR_0); - ptr++; - } - } - - /* Otherwise we expect to read a name; anything else is an error. When - a name is one of a number of duplicates, a different opcode is used and - it needs more memory. Unfortunately we cannot tell whether a name is a - duplicate in the first pass, so we have to allow for more memory. */ - - else - { - if (IS_DIGIT(*ptr)) - { - *errorcodeptr = ERR84; - goto FAILED; - } - if (!MAX_255(*ptr) || (cd->ctypes[*ptr] & ctype_word) == 0) - { - *errorcodeptr = ERR28; /* Assertion expected */ - goto FAILED; - } - name = ptr++; - while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_word) != 0) - { - ptr++; - } - namelen = (int)(ptr - name); - if (lengthptr != NULL) skipbytes += IMM2_SIZE; - } - - /* Check the terminator */ - - if ((terminator > 0 && *ptr++ != (pcre_uchar)terminator) || - *ptr++ != CHAR_RIGHT_PARENTHESIS) - { - ptr--; /* Error offset */ - *errorcodeptr = ERR26; /* Malformed number or name */ - goto FAILED; - } - - /* Do no further checking in the pre-compile phase. */ - - if (lengthptr != NULL) break; - - /* In the real compile we do the work of looking for the actual - reference. If refsign is not negative, it means we have a number in - recno. */ - - if (refsign >= 0) - { - if (recno <= 0) - { - *errorcodeptr = ERR35; - goto FAILED; - } - if (refsign != 0) recno = (refsign == CHAR_MINUS)? - cd->bracount - recno + 1 : recno + cd->bracount; - if (recno <= 0 || recno > cd->final_bracount) - { - *errorcodeptr = ERR15; - goto FAILED; - } - PUT2(code, 2+LINK_SIZE, recno); - if (recno > cd->top_backref) cd->top_backref = recno; - break; - } - - /* Otherwise look for the name. */ - - slot = cd->name_table; - for (i = 0; i < cd->names_found; i++) - { - if (STRNCMP_UC_UC(name, slot+IMM2_SIZE, namelen) == 0) break; - slot += cd->name_entry_size; - } - - /* Found the named subpattern. If the name is duplicated, add one to - the opcode to change CREF/RREF into DNCREF/DNRREF and insert - appropriate data values. Otherwise, just insert the unique subpattern - number. */ - - if (i < cd->names_found) - { - int offset = i++; - int count = 1; - recno = GET2(slot, 0); /* Number from first found */ - if (recno > cd->top_backref) cd->top_backref = recno; - for (; i < cd->names_found; i++) - { - slot += cd->name_entry_size; - if (STRNCMP_UC_UC(name, slot+IMM2_SIZE, namelen) != 0 || - (slot+IMM2_SIZE)[namelen] != 0) break; - count++; - } - - if (count > 1) - { - PUT2(code, 2+LINK_SIZE, offset); - PUT2(code, 2+LINK_SIZE+IMM2_SIZE, count); - skipbytes += IMM2_SIZE; - code[1+LINK_SIZE]++; - } - else /* Not a duplicated name */ - { - PUT2(code, 2+LINK_SIZE, recno); - } - } - - /* If terminator == CHAR_NULL it means that the name followed directly - after the opening parenthesis [e.g. (?(abc)...] and in this case there - are some further alternatives to try. For the cases where terminator != - CHAR_NULL [things like (?(... or (?('name')... or (?(R&name)... ] - we have now checked all the possibilities, so give an error. */ - - else if (terminator != CHAR_NULL) - { - *errorcodeptr = ERR15; - goto FAILED; - } - - /* Check for (?(R) for recursion. Allow digits after R to specify a - specific group number. */ - - else if (*name == CHAR_R) - { - recno = 0; - for (i = 1; i < namelen; i++) - { - if (!IS_DIGIT(name[i])) - { - *errorcodeptr = ERR15; - goto FAILED; - } - if (recno > INT_MAX / 10 - 1) /* Integer overflow */ - { - *errorcodeptr = ERR61; - goto FAILED; - } - recno = recno * 10 + name[i] - CHAR_0; - } - if (recno == 0) recno = RREF_ANY; - code[1+LINK_SIZE] = OP_RREF; /* Change test type */ - PUT2(code, 2+LINK_SIZE, recno); - } - - /* Similarly, check for the (?(DEFINE) "condition", which is always - false. */ - - else if (namelen == 6 && STRNCMP_UC_C8(name, STRING_DEFINE, 6) == 0) - { - code[1+LINK_SIZE] = OP_DEF; - skipbytes = 1; - } - - /* Reference to an unidentified subpattern. */ - - else - { - *errorcodeptr = ERR15; - goto FAILED; - } - break; - - - /* ------------------------------------------------------------ */ - case CHAR_EQUALS_SIGN: /* Positive lookahead */ - bravalue = OP_ASSERT; - cd->assert_depth += 1; - ptr++; - break; - - /* Optimize (?!) to (*FAIL) unless it is quantified - which is a weird - thing to do, but Perl allows all assertions to be quantified, and when - they contain capturing parentheses there may be a potential use for - this feature. Not that that applies to a quantified (?!) but we allow - it for uniformity. */ - - /* ------------------------------------------------------------ */ - case CHAR_EXCLAMATION_MARK: /* Negative lookahead */ - ptr++; - if (*ptr == CHAR_RIGHT_PARENTHESIS && ptr[1] != CHAR_ASTERISK && - ptr[1] != CHAR_PLUS && ptr[1] != CHAR_QUESTION_MARK && - (ptr[1] != CHAR_LEFT_CURLY_BRACKET || !is_counted_repeat(ptr+2))) - { - *code++ = OP_FAIL; - previous = NULL; - continue; - } - bravalue = OP_ASSERT_NOT; - cd->assert_depth += 1; - break; - - - /* ------------------------------------------------------------ */ - case CHAR_LESS_THAN_SIGN: /* Lookbehind or named define */ - switch (ptr[1]) - { - case CHAR_EQUALS_SIGN: /* Positive lookbehind */ - bravalue = OP_ASSERTBACK; - cd->assert_depth += 1; - ptr += 2; - break; - - case CHAR_EXCLAMATION_MARK: /* Negative lookbehind */ - bravalue = OP_ASSERTBACK_NOT; - cd->assert_depth += 1; - ptr += 2; - break; - - default: /* Could be name define, else bad */ - if (MAX_255(ptr[1]) && (cd->ctypes[ptr[1]] & ctype_word) != 0) - goto DEFINE_NAME; - ptr++; /* Correct offset for error */ - *errorcodeptr = ERR24; - goto FAILED; - } - break; - - - /* ------------------------------------------------------------ */ - case CHAR_GREATER_THAN_SIGN: /* One-time brackets */ - bravalue = OP_ONCE; - ptr++; - break; - - - /* ------------------------------------------------------------ */ - case CHAR_C: /* Callout - may be followed by digits; */ - previous_callout = code; /* Save for later completion */ - after_manual_callout = 1; /* Skip one item before completing */ - *code++ = OP_CALLOUT; - { - int n = 0; - ptr++; - while(IS_DIGIT(*ptr)) - n = n * 10 + *ptr++ - CHAR_0; - if (*ptr != CHAR_RIGHT_PARENTHESIS) - { - *errorcodeptr = ERR39; - goto FAILED; - } - if (n > 255) - { - *errorcodeptr = ERR38; - goto FAILED; - } - *code++ = n; - PUT(code, 0, (int)(ptr - cd->start_pattern + 1)); /* Pattern offset */ - PUT(code, LINK_SIZE, 0); /* Default length */ - code += 2 * LINK_SIZE; - } - previous = NULL; - continue; - - - /* ------------------------------------------------------------ */ - case CHAR_P: /* Python-style named subpattern handling */ - if (*(++ptr) == CHAR_EQUALS_SIGN || - *ptr == CHAR_GREATER_THAN_SIGN) /* Reference or recursion */ - { - is_recurse = *ptr == CHAR_GREATER_THAN_SIGN; - terminator = CHAR_RIGHT_PARENTHESIS; - goto NAMED_REF_OR_RECURSE; - } - else if (*ptr != CHAR_LESS_THAN_SIGN) /* Test for Python-style defn */ - { - *errorcodeptr = ERR41; - goto FAILED; - } - /* Fall through to handle (?P< as (?< is handled */ - - - /* ------------------------------------------------------------ */ - DEFINE_NAME: /* Come here from (?< handling */ - case CHAR_APOSTROPHE: - terminator = (*ptr == CHAR_LESS_THAN_SIGN)? - CHAR_GREATER_THAN_SIGN : CHAR_APOSTROPHE; - name = ++ptr; - if (IS_DIGIT(*ptr)) - { - *errorcodeptr = ERR84; /* Group name must start with non-digit */ - goto FAILED; - } - while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_word) != 0) ptr++; - namelen = (int)(ptr - name); - - /* In the pre-compile phase, do a syntax check, remember the longest - name, and then remember the group in a vector, expanding it if - necessary. Duplicates for the same number are skipped; other duplicates - are checked for validity. In the actual compile, there is nothing to - do. */ - - if (lengthptr != NULL) - { - named_group *ng; - pcre_uint32 number = cd->bracount + 1; - - if (*ptr != (pcre_uchar)terminator) - { - *errorcodeptr = ERR42; - goto FAILED; - } - - if (cd->names_found >= MAX_NAME_COUNT) - { - *errorcodeptr = ERR49; - goto FAILED; - } - - if (namelen + IMM2_SIZE + 1 > cd->name_entry_size) - { - cd->name_entry_size = namelen + IMM2_SIZE + 1; - if (namelen > MAX_NAME_SIZE) - { - *errorcodeptr = ERR48; - goto FAILED; - } - } - - /* Scan the list to check for duplicates. For duplicate names, if the - number is the same, break the loop, which causes the name to be - discarded; otherwise, if DUPNAMES is not set, give an error. - If it is set, allow the name with a different number, but continue - scanning in case this is a duplicate with the same number. For - non-duplicate names, give an error if the number is duplicated. */ - - ng = cd->named_groups; - for (i = 0; i < cd->names_found; i++, ng++) - { - if (namelen == ng->length && - STRNCMP_UC_UC(name, ng->name, namelen) == 0) - { - if (ng->number == number) break; - if ((options & PCRE_DUPNAMES) == 0) - { - *errorcodeptr = ERR43; - goto FAILED; - } - cd->dupnames = TRUE; /* Duplicate names exist */ - } - else if (ng->number == number) - { - *errorcodeptr = ERR65; - goto FAILED; - } - } - - if (i >= cd->names_found) /* Not a duplicate with same number */ - { - /* Increase the list size if necessary */ - - if (cd->names_found >= cd->named_group_list_size) - { - int newsize = cd->named_group_list_size * 2; - named_group *newspace = (PUBL(malloc)) - (newsize * sizeof(named_group)); - - if (newspace == NULL) - { - *errorcodeptr = ERR21; - goto FAILED; - } - - memcpy(newspace, cd->named_groups, - cd->named_group_list_size * sizeof(named_group)); - if (cd->named_group_list_size > NAMED_GROUP_LIST_SIZE) - (PUBL(free))((void *)cd->named_groups); - cd->named_groups = newspace; - cd->named_group_list_size = newsize; - } - - cd->named_groups[cd->names_found].name = name; - cd->named_groups[cd->names_found].length = namelen; - cd->named_groups[cd->names_found].number = number; - cd->names_found++; - } - } - - ptr++; /* Move past > or ' in both passes. */ - goto NUMBERED_GROUP; - - - /* ------------------------------------------------------------ */ - case CHAR_AMPERSAND: /* Perl recursion/subroutine syntax */ - terminator = CHAR_RIGHT_PARENTHESIS; - is_recurse = TRUE; - /* Fall through */ - - /* We come here from the Python syntax above that handles both - references (?P=name) and recursion (?P>name), as well as falling - through from the Perl recursion syntax (?&name). We also come here from - the Perl \k or \k'name' back reference syntax and the \k{name} - .NET syntax, and the Oniguruma \g<...> and \g'...' subroutine syntax. */ - - NAMED_REF_OR_RECURSE: - name = ++ptr; - if (IS_DIGIT(*ptr)) - { - *errorcodeptr = ERR84; /* Group name must start with non-digit */ - goto FAILED; - } - while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_word) != 0) ptr++; - namelen = (int)(ptr - name); - - /* In the pre-compile phase, do a syntax check. We used to just set - a dummy reference number, because it was not used in the first pass. - However, with the change of recursive back references to be atomic, - we have to look for the number so that this state can be identified, as - otherwise the incorrect length is computed. If it's not a backwards - reference, the dummy number will do. */ - - if (lengthptr != NULL) - { - named_group *ng; - recno = 0; - - if (namelen == 0) - { - *errorcodeptr = ERR62; - goto FAILED; - } - if (*ptr != (pcre_uchar)terminator) - { - *errorcodeptr = ERR42; - goto FAILED; - } - if (namelen > MAX_NAME_SIZE) - { - *errorcodeptr = ERR48; - goto FAILED; - } - - /* Count named back references. */ - - if (!is_recurse) cd->namedrefcount++; - - /* We have to allow for a named reference to a duplicated name (this - cannot be determined until the second pass). This needs an extra - 16-bit data item. */ - - *lengthptr += IMM2_SIZE; - - /* If this is a forward reference and we are within a (?|...) group, - the reference may end up as the number of a group which we are - currently inside, that is, it could be a recursive reference. In the - real compile this will be picked up and the reference wrapped with - OP_ONCE to make it atomic, so we must space in case this occurs. */ - - /* In fact, this can happen for a non-forward reference because - another group with the same number might be created later. This - issue is fixed "properly" in PCRE2. As PCRE1 is now in maintenance - only mode, we finesse the bug by allowing more memory always. */ - - *lengthptr += 4 + 4*LINK_SIZE; - - /* It is even worse than that. The current reference may be to an - existing named group with a different number (so apparently not - recursive) but which later on is also attached to a group with the - current number. This can only happen if $(| has been previous - encountered. In that case, we allow yet more memory, just in case. - (Again, this is fixed "properly" in PCRE2. */ - - if (cd->dupgroups) *lengthptr += 4 + 4*LINK_SIZE; - - /* Otherwise, check for recursion here. The name table does not exist - in the first pass; instead we must scan the list of names encountered - so far in order to get the number. If the name is not found, leave - the value of recno as 0 for a forward reference. */ - - /* This patch (removing "else") fixes a problem when a reference is - to multiple identically named nested groups from within the nest. - Once again, it is not the "proper" fix, and it results in an - over-allocation of memory. */ - - /* else */ - { - ng = cd->named_groups; - for (i = 0; i < cd->names_found; i++, ng++) - { - if (namelen == ng->length && - STRNCMP_UC_UC(name, ng->name, namelen) == 0) - { - open_capitem *oc; - recno = ng->number; - if (is_recurse) break; - for (oc = cd->open_caps; oc != NULL; oc = oc->next) - { - if (oc->number == recno) - { - oc->flag = TRUE; - break; - } - } - } - } - } - } - - /* In the real compile, search the name table. We check the name - first, and then check that we have reached the end of the name in the - table. That way, if the name is longer than any in the table, the - comparison will fail without reading beyond the table entry. */ - - else - { - slot = cd->name_table; - for (i = 0; i < cd->names_found; i++) - { - if (STRNCMP_UC_UC(name, slot+IMM2_SIZE, namelen) == 0 && - slot[IMM2_SIZE+namelen] == 0) - break; - slot += cd->name_entry_size; - } - - if (i < cd->names_found) - { - recno = GET2(slot, 0); - } - else - { - *errorcodeptr = ERR15; - goto FAILED; - } - } - - /* In both phases, for recursions, we can now go to the code than - handles numerical recursion. */ - - if (is_recurse) goto HANDLE_RECURSION; - - /* In the second pass we must see if the name is duplicated. If so, we - generate a different opcode. */ - - if (lengthptr == NULL && cd->dupnames) - { - int count = 1; - unsigned int index = i; - pcre_uchar *cslot = slot + cd->name_entry_size; - - for (i++; i < cd->names_found; i++) - { - if (STRCMP_UC_UC(slot + IMM2_SIZE, cslot + IMM2_SIZE) != 0) break; - count++; - cslot += cd->name_entry_size; - } - - if (count > 1) - { - if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; - previous = code; - item_hwm_offset = cd->hwm - cd->start_workspace; - *code++ = ((options & PCRE_CASELESS) != 0)? OP_DNREFI : OP_DNREF; - PUT2INC(code, 0, index); - PUT2INC(code, 0, count); - - /* Process each potentially referenced group. */ - - for (; slot < cslot; slot += cd->name_entry_size) - { - open_capitem *oc; - recno = GET2(slot, 0); - cd->backref_map |= (recno < 32)? (1 << recno) : 1; - if (recno > cd->top_backref) cd->top_backref = recno; - - /* Check to see if this back reference is recursive, that it, it - is inside the group that it references. A flag is set so that the - group can be made atomic. */ - - for (oc = cd->open_caps; oc != NULL; oc = oc->next) - { - if (oc->number == recno) - { - oc->flag = TRUE; - break; - } - } - } - - continue; /* End of back ref handling */ - } - } - - /* First pass, or a non-duplicated name. */ - - goto HANDLE_REFERENCE; - - - /* ------------------------------------------------------------ */ - case CHAR_R: /* Recursion, same as (?0) */ - recno = 0; - if (*(++ptr) != CHAR_RIGHT_PARENTHESIS) - { - *errorcodeptr = ERR29; - goto FAILED; - } - goto HANDLE_RECURSION; - - - /* ------------------------------------------------------------ */ - case CHAR_MINUS: case CHAR_PLUS: /* Recursion or subroutine */ - case CHAR_0: case CHAR_1: case CHAR_2: case CHAR_3: case CHAR_4: - case CHAR_5: case CHAR_6: case CHAR_7: case CHAR_8: case CHAR_9: - { - const pcre_uchar *called; - terminator = CHAR_RIGHT_PARENTHESIS; - - /* Come here from the \g<...> and \g'...' code (Oniguruma - compatibility). However, the syntax has been checked to ensure that - the ... are a (signed) number, so that neither ERR63 nor ERR29 will - be called on this path, nor with the jump to OTHER_CHAR_AFTER_QUERY - ever be taken. */ - - HANDLE_NUMERICAL_RECURSION: - - if ((refsign = *ptr) == CHAR_PLUS) - { - ptr++; - if (!IS_DIGIT(*ptr)) - { - *errorcodeptr = ERR63; - goto FAILED; - } - } - else if (refsign == CHAR_MINUS) - { - if (!IS_DIGIT(ptr[1])) - goto OTHER_CHAR_AFTER_QUERY; - ptr++; - } - - recno = 0; - while(IS_DIGIT(*ptr)) - { - if (recno > INT_MAX / 10 - 1) /* Integer overflow */ - { - while (IS_DIGIT(*ptr)) ptr++; - *errorcodeptr = ERR61; - goto FAILED; - } - recno = recno * 10 + *ptr++ - CHAR_0; - } - - if (*ptr != (pcre_uchar)terminator) - { - *errorcodeptr = ERR29; - goto FAILED; - } - - if (refsign == CHAR_MINUS) - { - if (recno == 0) - { - *errorcodeptr = ERR58; - goto FAILED; - } - recno = cd->bracount - recno + 1; - if (recno <= 0) - { - *errorcodeptr = ERR15; - goto FAILED; - } - } - else if (refsign == CHAR_PLUS) - { - if (recno == 0) - { - *errorcodeptr = ERR58; - goto FAILED; - } - recno += cd->bracount; - } - - /* Come here from code above that handles a named recursion */ - - HANDLE_RECURSION: - - previous = code; - item_hwm_offset = cd->hwm - cd->start_workspace; - called = cd->start_code; - - /* When we are actually compiling, find the bracket that is being - referenced. Temporarily end the regex in case it doesn't exist before - this point. If we end up with a forward reference, first check that - the bracket does occur later so we can give the error (and position) - now. Then remember this forward reference in the workspace so it can - be filled in at the end. */ - - if (lengthptr == NULL) - { - *code = OP_END; - if (recno != 0) - called = PRIV(find_bracket)(cd->start_code, utf, recno); - - /* Forward reference */ - - if (called == NULL) - { - if (recno > cd->final_bracount) - { - *errorcodeptr = ERR15; - goto FAILED; - } - - /* Fudge the value of "called" so that when it is inserted as an - offset below, what it actually inserted is the reference number - of the group. Then remember the forward reference. */ - - called = cd->start_code + recno; - if (cd->hwm >= cd->start_workspace + cd->workspace_size - - WORK_SIZE_SAFETY_MARGIN) - { - *errorcodeptr = expand_workspace(cd); - if (*errorcodeptr != 0) goto FAILED; - } - PUTINC(cd->hwm, 0, (int)(code + 1 - cd->start_code)); - } - - /* If not a forward reference, and the subpattern is still open, - this is a recursive call. We check to see if this is a left - recursion that could loop for ever, and diagnose that case. We - must not, however, do this check if we are in a conditional - subpattern because the condition might be testing for recursion in - a pattern such as /(?(R)a+|(?R)b)/, which is perfectly valid. - Forever loops are also detected at runtime, so those that occur in - conditional subpatterns will be picked up then. */ - - else if (GET(called, 1) == 0 && cond_depth <= 0 && - could_be_empty(called, code, bcptr, utf, cd)) - { - *errorcodeptr = ERR40; - goto FAILED; - } - } - - /* Insert the recursion/subroutine item. It does not have a set first - character (relevant if it is repeated, because it will then be - wrapped with ONCE brackets). */ - - *code = OP_RECURSE; - PUT(code, 1, (int)(called - cd->start_code)); - code += 1 + LINK_SIZE; - groupsetfirstchar = FALSE; - } - - /* Can't determine a first byte now */ - - if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; - continue; - - - /* ------------------------------------------------------------ */ - default: /* Other characters: check option setting */ - OTHER_CHAR_AFTER_QUERY: - set = unset = 0; - optset = &set; - - while (*ptr != CHAR_RIGHT_PARENTHESIS && *ptr != CHAR_COLON) - { - switch (*ptr++) - { - case CHAR_MINUS: optset = &unset; break; - - case CHAR_J: /* Record that it changed in the external options */ - *optset |= PCRE_DUPNAMES; - cd->external_flags |= PCRE_JCHANGED; - break; - - case CHAR_i: *optset |= PCRE_CASELESS; break; - case CHAR_m: *optset |= PCRE_MULTILINE; break; - case CHAR_s: *optset |= PCRE_DOTALL; break; - case CHAR_x: *optset |= PCRE_EXTENDED; break; - case CHAR_U: *optset |= PCRE_UNGREEDY; break; - case CHAR_X: *optset |= PCRE_EXTRA; break; - - default: *errorcodeptr = ERR12; - ptr--; /* Correct the offset */ - goto FAILED; - } - } - - /* Set up the changed option bits, but don't change anything yet. */ - - newoptions = (options | set) & (~unset); - - /* If the options ended with ')' this is not the start of a nested - group with option changes, so the options change at this level. - If we are not at the pattern start, reset the greedy defaults and the - case value for firstchar and reqchar. */ - - if (*ptr == CHAR_RIGHT_PARENTHESIS) - { - greedy_default = ((newoptions & PCRE_UNGREEDY) != 0); - greedy_non_default = greedy_default ^ 1; - req_caseopt = ((newoptions & PCRE_CASELESS) != 0)? REQ_CASELESS:0; - - /* Change options at this level, and pass them back for use - in subsequent branches. */ - - *optionsptr = options = newoptions; - previous = NULL; /* This item can't be repeated */ - continue; /* It is complete */ - } - - /* If the options ended with ':' we are heading into a nested group - with possible change of options. Such groups are non-capturing and are - not assertions of any kind. All we need to do is skip over the ':'; - the newoptions value is handled below. */ - - bravalue = OP_BRA; - ptr++; - } /* End of switch for character following (? */ - } /* End of (? handling */ - - /* Opening parenthesis not followed by '*' or '?'. If PCRE_NO_AUTO_CAPTURE - is set, all unadorned brackets become non-capturing and behave like (?:...) - brackets. */ - - else if ((options & PCRE_NO_AUTO_CAPTURE) != 0) - { - bravalue = OP_BRA; - } - - /* Else we have a capturing group. */ - - else - { - NUMBERED_GROUP: - cd->bracount += 1; - PUT2(code, 1+LINK_SIZE, cd->bracount); - skipbytes = IMM2_SIZE; - } - - /* Process nested bracketed regex. First check for parentheses nested too - deeply. */ - - if ((cd->parens_depth += 1) > PARENS_NEST_LIMIT) - { - *errorcodeptr = ERR82; - goto FAILED; - } - - /* All assertions used not to be repeatable, but this was changed for Perl - compatibility. All kinds can now be repeated except for assertions that are - conditions (Perl also forbids these to be repeated). We copy code into a - non-register variable (tempcode) in order to be able to pass its address - because some compilers complain otherwise. At the start of a conditional - group whose condition is an assertion, cd->iscondassert is set. We unset it - here so as to allow assertions later in the group to be quantified. */ - - if (bravalue >= OP_ASSERT && bravalue <= OP_ASSERTBACK_NOT && - cd->iscondassert) - { - previous = NULL; - cd->iscondassert = FALSE; - } - else - { - previous = code; - item_hwm_offset = cd->hwm - cd->start_workspace; - } - - *code = bravalue; - tempcode = code; - tempreqvary = cd->req_varyopt; /* Save value before bracket */ - tempbracount = cd->bracount; /* Save value before bracket */ - length_prevgroup = 0; /* Initialize for pre-compile phase */ - - if (!compile_regex( - newoptions, /* The complete new option state */ - &tempcode, /* Where to put code (updated) */ - &ptr, /* Input pointer (updated) */ - errorcodeptr, /* Where to put an error message */ - (bravalue == OP_ASSERTBACK || - bravalue == OP_ASSERTBACK_NOT), /* TRUE if back assert */ - reset_bracount, /* True if (?| group */ - skipbytes, /* Skip over bracket number */ - cond_depth + - ((bravalue == OP_COND)?1:0), /* Depth of condition subpatterns */ - &subfirstchar, /* For possible first char */ - &subfirstcharflags, - &subreqchar, /* For possible last char */ - &subreqcharflags, - bcptr, /* Current branch chain */ - cd, /* Tables block */ - (lengthptr == NULL)? NULL : /* Actual compile phase */ - &length_prevgroup /* Pre-compile phase */ - )) - goto FAILED; - - cd->parens_depth -= 1; - - /* If this was an atomic group and there are no capturing groups within it, - generate OP_ONCE_NC instead of OP_ONCE. */ - - if (bravalue == OP_ONCE && cd->bracount <= tempbracount) - *code = OP_ONCE_NC; - - if (bravalue >= OP_ASSERT && bravalue <= OP_ASSERTBACK_NOT) - cd->assert_depth -= 1; - - /* At the end of compiling, code is still pointing to the start of the - group, while tempcode has been updated to point past the end of the group. - The pattern pointer (ptr) is on the bracket. - - If this is a conditional bracket, check that there are no more than - two branches in the group, or just one if it's a DEFINE group. We do this - in the real compile phase, not in the pre-pass, where the whole group may - not be available. */ - - if (bravalue == OP_COND && lengthptr == NULL) - { - pcre_uchar *tc = code; - int condcount = 0; - - do { - condcount++; - tc += GET(tc,1); - } - while (*tc != OP_KET); - - /* A DEFINE group is never obeyed inline (the "condition" is always - false). It must have only one branch. */ - - if (code[LINK_SIZE+1] == OP_DEF) - { - if (condcount > 1) - { - *errorcodeptr = ERR54; - goto FAILED; - } - bravalue = OP_DEF; /* Just a flag to suppress char handling below */ - } - - /* A "normal" conditional group. If there is just one branch, we must not - make use of its firstchar or reqchar, because this is equivalent to an - empty second branch. */ - - else - { - if (condcount > 2) - { - *errorcodeptr = ERR27; - goto FAILED; - } - if (condcount == 1) subfirstcharflags = subreqcharflags = REQ_NONE; - } - } - - /* Error if hit end of pattern */ - - if (*ptr != CHAR_RIGHT_PARENTHESIS) - { - *errorcodeptr = ERR14; - goto FAILED; - } - - /* In the pre-compile phase, update the length by the length of the group, - less the brackets at either end. Then reduce the compiled code to just a - set of non-capturing brackets so that it doesn't use much memory if it is - duplicated by a quantifier.*/ - - if (lengthptr != NULL) - { - if (OFLOW_MAX - *lengthptr < length_prevgroup - 2 - 2*LINK_SIZE) - { - *errorcodeptr = ERR20; - goto FAILED; - } - *lengthptr += length_prevgroup - 2 - 2*LINK_SIZE; - code++; /* This already contains bravalue */ - PUTINC(code, 0, 1 + LINK_SIZE); - *code++ = OP_KET; - PUTINC(code, 0, 1 + LINK_SIZE); - break; /* No need to waste time with special character handling */ - } - - /* Otherwise update the main code pointer to the end of the group. */ - - code = tempcode; - - /* For a DEFINE group, required and first character settings are not - relevant. */ - - if (bravalue == OP_DEF) break; - - /* Handle updating of the required and first characters for other types of - group. Update for normal brackets of all kinds, and conditions with two - branches (see code above). If the bracket is followed by a quantifier with - zero repeat, we have to back off. Hence the definition of zeroreqchar and - zerofirstchar outside the main loop so that they can be accessed for the - back off. */ - - zeroreqchar = reqchar; - zeroreqcharflags = reqcharflags; - zerofirstchar = firstchar; - zerofirstcharflags = firstcharflags; - groupsetfirstchar = FALSE; - - if (bravalue >= OP_ONCE) - { - /* If we have not yet set a firstchar in this branch, take it from the - subpattern, remembering that it was set here so that a repeat of more - than one can replicate it as reqchar if necessary. If the subpattern has - no firstchar, set "none" for the whole branch. In both cases, a zero - repeat forces firstchar to "none". */ - - if (firstcharflags == REQ_UNSET) - { - if (subfirstcharflags >= 0) - { - firstchar = subfirstchar; - firstcharflags = subfirstcharflags; - groupsetfirstchar = TRUE; - } - else firstcharflags = REQ_NONE; - zerofirstcharflags = REQ_NONE; - } - - /* If firstchar was previously set, convert the subpattern's firstchar - into reqchar if there wasn't one, using the vary flag that was in - existence beforehand. */ - - else if (subfirstcharflags >= 0 && subreqcharflags < 0) - { - subreqchar = subfirstchar; - subreqcharflags = subfirstcharflags | tempreqvary; - } - - /* If the subpattern set a required byte (or set a first byte that isn't - really the first byte - see above), set it. */ - - if (subreqcharflags >= 0) - { - reqchar = subreqchar; - reqcharflags = subreqcharflags; - } - } - - /* For a forward assertion, we take the reqchar, if set. This can be - helpful if the pattern that follows the assertion doesn't set a different - char. For example, it's useful for /(?=abcde).+/. We can't set firstchar - for an assertion, however because it leads to incorrect effect for patterns - such as /(?=a)a.+/ when the "real" "a" would then become a reqchar instead - of a firstchar. This is overcome by a scan at the end if there's no - firstchar, looking for an asserted first char. */ - - else if (bravalue == OP_ASSERT && subreqcharflags >= 0) - { - reqchar = subreqchar; - reqcharflags = subreqcharflags; - } - break; /* End of processing '(' */ - - - /* ===================================================================*/ - /* Handle metasequences introduced by \. For ones like \d, the ESC_ values - are arranged to be the negation of the corresponding OP_values in the - default case when PCRE_UCP is not set. For the back references, the values - are negative the reference number. Only back references and those types - that consume a character may be repeated. We can test for values between - ESC_b and ESC_Z for the latter; this may have to change if any new ones are - ever created. */ - - case CHAR_BACKSLASH: - tempptr = ptr; - escape = check_escape(&ptr, &ec, errorcodeptr, cd->bracount, options, FALSE); - if (*errorcodeptr != 0) goto FAILED; - - if (escape == 0) /* The escape coded a single character */ - c = ec; - else - { - /* For metasequences that actually match a character, we disable the - setting of a first character if it hasn't already been set. */ - - if (firstcharflags == REQ_UNSET && escape > ESC_b && escape < ESC_Z) - firstcharflags = REQ_NONE; - - /* Set values to reset to if this is followed by a zero repeat. */ - - zerofirstchar = firstchar; - zerofirstcharflags = firstcharflags; - zeroreqchar = reqchar; - zeroreqcharflags = reqcharflags; - - /* \g or \g'name' is a subroutine call by name and \g or \g'n' - is a subroutine call by number (Oniguruma syntax). In fact, the value - ESC_g is returned only for these cases. So we don't need to check for < - or ' if the value is ESC_g. For the Perl syntax \g{n} the value is - -n, and for the Perl syntax \g{name} the result is ESC_k (as - that is a synonym for a named back reference). */ - - if (escape == ESC_g) - { - const pcre_uchar *p; - pcre_uint32 cf; - - item_hwm_offset = cd->hwm - cd->start_workspace; /* Normally this is set when '(' is read */ - terminator = (*(++ptr) == CHAR_LESS_THAN_SIGN)? - CHAR_GREATER_THAN_SIGN : CHAR_APOSTROPHE; - - /* These two statements stop the compiler for warning about possibly - unset variables caused by the jump to HANDLE_NUMERICAL_RECURSION. In - fact, because we do the check for a number below, the paths that - would actually be in error are never taken. */ - - skipbytes = 0; - reset_bracount = FALSE; - - /* If it's not a signed or unsigned number, treat it as a name. */ - - cf = ptr[1]; - if (cf != CHAR_PLUS && cf != CHAR_MINUS && !IS_DIGIT(cf)) - { - is_recurse = TRUE; - goto NAMED_REF_OR_RECURSE; - } - - /* Signed or unsigned number (cf = ptr[1]) is known to be plus or minus - or a digit. */ - - p = ptr + 2; - while (IS_DIGIT(*p)) p++; - if (*p != (pcre_uchar)terminator) - { - *errorcodeptr = ERR57; - goto FAILED; - } - ptr++; - goto HANDLE_NUMERICAL_RECURSION; - } - - /* \k or \k'name' is a back reference by name (Perl syntax). - We also support \k{name} (.NET syntax). */ - - if (escape == ESC_k) - { - if ((ptr[1] != CHAR_LESS_THAN_SIGN && - ptr[1] != CHAR_APOSTROPHE && ptr[1] != CHAR_LEFT_CURLY_BRACKET)) - { - *errorcodeptr = ERR69; - goto FAILED; - } - is_recurse = FALSE; - terminator = (*(++ptr) == CHAR_LESS_THAN_SIGN)? - CHAR_GREATER_THAN_SIGN : (*ptr == CHAR_APOSTROPHE)? - CHAR_APOSTROPHE : CHAR_RIGHT_CURLY_BRACKET; - goto NAMED_REF_OR_RECURSE; - } - - /* Back references are handled specially; must disable firstchar if - not set to cope with cases like (?=(\w+))\1: which would otherwise set - ':' later. */ - - if (escape < 0) - { - open_capitem *oc; - recno = -escape; - - /* Come here from named backref handling when the reference is to a - single group (i.e. not to a duplicated name. */ - - HANDLE_REFERENCE: - if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; - previous = code; - item_hwm_offset = cd->hwm - cd->start_workspace; - *code++ = ((options & PCRE_CASELESS) != 0)? OP_REFI : OP_REF; - PUT2INC(code, 0, recno); - cd->backref_map |= (recno < 32)? (1 << recno) : 1; - if (recno > cd->top_backref) cd->top_backref = recno; - - /* Check to see if this back reference is recursive, that it, it - is inside the group that it references. A flag is set so that the - group can be made atomic. */ - - for (oc = cd->open_caps; oc != NULL; oc = oc->next) - { - if (oc->number == recno) - { - oc->flag = TRUE; - break; - } - } - } - - /* So are Unicode property matches, if supported. */ - -#ifdef SUPPORT_UCP - else if (escape == ESC_P || escape == ESC_p) - { - BOOL negated; - unsigned int ptype = 0, pdata = 0; - if (!get_ucp(&ptr, &negated, &ptype, &pdata, errorcodeptr)) - goto FAILED; - previous = code; - item_hwm_offset = cd->hwm - cd->start_workspace; - *code++ = ((escape == ESC_p) != negated)? OP_PROP : OP_NOTPROP; - *code++ = ptype; - *code++ = pdata; - } -#else - - /* If Unicode properties are not supported, \X, \P, and \p are not - allowed. */ - - else if (escape == ESC_X || escape == ESC_P || escape == ESC_p) - { - *errorcodeptr = ERR45; - goto FAILED; - } -#endif - - /* For the rest (including \X when Unicode properties are supported), we - can obtain the OP value by negating the escape value in the default - situation when PCRE_UCP is not set. When it *is* set, we substitute - Unicode property tests. Note that \b and \B do a one-character - lookbehind, and \A also behaves as if it does. */ - - else - { - if ((escape == ESC_b || escape == ESC_B || escape == ESC_A) && - cd->max_lookbehind == 0) - cd->max_lookbehind = 1; -#ifdef SUPPORT_UCP - if (escape >= ESC_DU && escape <= ESC_wu) - { - nestptr = ptr + 1; /* Where to resume */ - ptr = substitutes[escape - ESC_DU] - 1; /* Just before substitute */ - } - else -#endif - /* In non-UTF-8 mode, we turn \C into OP_ALLANY instead of OP_ANYBYTE - so that it works in DFA mode and in lookbehinds. */ - - { - previous = (escape > ESC_b && escape < ESC_Z)? code : NULL; - item_hwm_offset = cd->hwm - cd->start_workspace; - *code++ = (!utf && escape == ESC_C)? OP_ALLANY : escape; - } - } - continue; - } - - /* We have a data character whose value is in c. In UTF-8 mode it may have - a value > 127. We set its representation in the length/buffer, and then - handle it as a data character. */ - -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (utf && c > MAX_VALUE_FOR_SINGLE_CHAR) - mclength = PRIV(ord2utf)(c, mcbuffer); - else -#endif - - { - mcbuffer[0] = c; - mclength = 1; - } - goto ONE_CHAR; - - - /* ===================================================================*/ - /* Handle a literal character. It is guaranteed not to be whitespace or # - when the extended flag is set. If we are in a UTF mode, it may be a - multi-unit literal character. */ - - default: - NORMAL_CHAR: - mclength = 1; - mcbuffer[0] = c; - -#ifdef SUPPORT_UTF - if (utf && HAS_EXTRALEN(c)) - ACROSSCHAR(TRUE, ptr[1], mcbuffer[mclength++] = *(++ptr)); -#endif - - /* At this point we have the character's bytes in mcbuffer, and the length - in mclength. When not in UTF-8 mode, the length is always 1. */ - - ONE_CHAR: - previous = code; - item_hwm_offset = cd->hwm - cd->start_workspace; - - /* For caseless UTF-8 mode when UCP support is available, check whether - this character has more than one other case. If so, generate a special - OP_PROP item instead of OP_CHARI. */ - -#ifdef SUPPORT_UCP - if (utf && (options & PCRE_CASELESS) != 0) - { - GETCHAR(c, mcbuffer); - if ((c = UCD_CASESET(c)) != 0) - { - *code++ = OP_PROP; - *code++ = PT_CLIST; - *code++ = c; - if (firstcharflags == REQ_UNSET) - firstcharflags = zerofirstcharflags = REQ_NONE; - break; - } - } -#endif - - /* Caseful matches, or not one of the multicase characters. */ - - *code++ = ((options & PCRE_CASELESS) != 0)? OP_CHARI : OP_CHAR; - for (c = 0; c < mclength; c++) *code++ = mcbuffer[c]; - - /* Remember if \r or \n were seen */ - - if (mcbuffer[0] == CHAR_CR || mcbuffer[0] == CHAR_NL) - cd->external_flags |= PCRE_HASCRORLF; - - /* Set the first and required bytes appropriately. If no previous first - byte, set it from this character, but revert to none on a zero repeat. - Otherwise, leave the firstchar value alone, and don't change it on a zero - repeat. */ - - if (firstcharflags == REQ_UNSET) - { - zerofirstcharflags = REQ_NONE; - zeroreqchar = reqchar; - zeroreqcharflags = reqcharflags; - - /* If the character is more than one byte long, we can set firstchar - only if it is not to be matched caselessly. */ - - if (mclength == 1 || req_caseopt == 0) - { - firstchar = mcbuffer[0] | req_caseopt; - firstchar = mcbuffer[0]; - firstcharflags = req_caseopt; - - if (mclength != 1) - { - reqchar = code[-1]; - reqcharflags = cd->req_varyopt; - } - } - else firstcharflags = reqcharflags = REQ_NONE; - } - - /* firstchar was previously set; we can set reqchar only if the length is - 1 or the matching is caseful. */ - - else - { - zerofirstchar = firstchar; - zerofirstcharflags = firstcharflags; - zeroreqchar = reqchar; - zeroreqcharflags = reqcharflags; - if (mclength == 1 || req_caseopt == 0) - { - reqchar = code[-1]; - reqcharflags = req_caseopt | cd->req_varyopt; - } - } - - break; /* End of literal character handling */ - } - } /* end of big loop */ - - -/* Control never reaches here by falling through, only by a goto for all the -error states. Pass back the position in the pattern so that it can be displayed -to the user for diagnosing the error. */ - -FAILED: -*ptrptr = ptr; -return FALSE; -} - - - -/************************************************* -* Compile sequence of alternatives * -*************************************************/ - -/* On entry, ptr is pointing past the bracket character, but on return it -points to the closing bracket, or vertical bar, or end of string. The code -variable is pointing at the byte into which the BRA operator has been stored. -This function is used during the pre-compile phase when we are trying to find -out the amount of memory needed, as well as during the real compile phase. The -value of lengthptr distinguishes the two phases. - -Arguments: - options option bits, including any changes for this subpattern - codeptr -> the address of the current code pointer - ptrptr -> the address of the current pattern pointer - errorcodeptr -> pointer to error code variable - lookbehind TRUE if this is a lookbehind assertion - reset_bracount TRUE to reset the count for each branch - skipbytes skip this many bytes at start (for brackets and OP_COND) - cond_depth depth of nesting for conditional subpatterns - firstcharptr place to put the first required character - firstcharflagsptr place to put the first character flags, or a negative number - reqcharptr place to put the last required character - reqcharflagsptr place to put the last required character flags, or a negative number - bcptr pointer to the chain of currently open branches - cd points to the data block with tables pointers etc. - lengthptr NULL during the real compile phase - points to length accumulator during pre-compile phase - -Returns: TRUE on success -*/ - -static BOOL -compile_regex(int options, pcre_uchar **codeptr, const pcre_uchar **ptrptr, - int *errorcodeptr, BOOL lookbehind, BOOL reset_bracount, int skipbytes, - int cond_depth, - pcre_uint32 *firstcharptr, pcre_int32 *firstcharflagsptr, - pcre_uint32 *reqcharptr, pcre_int32 *reqcharflagsptr, - branch_chain *bcptr, compile_data *cd, int *lengthptr) -{ -const pcre_uchar *ptr = *ptrptr; -pcre_uchar *code = *codeptr; -pcre_uchar *last_branch = code; -pcre_uchar *start_bracket = code; -pcre_uchar *reverse_count = NULL; -open_capitem capitem; -int capnumber = 0; -pcre_uint32 firstchar, reqchar; -pcre_int32 firstcharflags, reqcharflags; -pcre_uint32 branchfirstchar, branchreqchar; -pcre_int32 branchfirstcharflags, branchreqcharflags; -int length; -unsigned int orig_bracount; -unsigned int max_bracount; -branch_chain bc; -size_t save_hwm_offset; - -/* If set, call the external function that checks for stack availability. */ - -if (PUBL(stack_guard) != NULL && PUBL(stack_guard)()) - { - *errorcodeptr= ERR85; - return FALSE; - } - -/* Miscellaneous initialization */ - -bc.outer = bcptr; -bc.current_branch = code; - -firstchar = reqchar = 0; -firstcharflags = reqcharflags = REQ_UNSET; - -save_hwm_offset = cd->hwm - cd->start_workspace; - -/* Accumulate the length for use in the pre-compile phase. Start with the -length of the BRA and KET and any extra bytes that are required at the -beginning. We accumulate in a local variable to save frequent testing of -lenthptr for NULL. We cannot do this by looking at the value of code at the -start and end of each alternative, because compiled items are discarded during -the pre-compile phase so that the work space is not exceeded. */ - -length = 2 + 2*LINK_SIZE + skipbytes; - -/* WARNING: If the above line is changed for any reason, you must also change -the code that abstracts option settings at the start of the pattern and makes -them global. It tests the value of length for (2 + 2*LINK_SIZE) in the -pre-compile phase to find out whether anything has yet been compiled or not. */ - -/* If this is a capturing subpattern, add to the chain of open capturing items -so that we can detect them if (*ACCEPT) is encountered. This is also used to -detect groups that contain recursive back references to themselves. Note that -only OP_CBRA need be tested here; changing this opcode to one of its variants, -e.g. OP_SCBRAPOS, happens later, after the group has been compiled. */ - -if (*code == OP_CBRA) - { - capnumber = GET2(code, 1 + LINK_SIZE); - capitem.number = capnumber; - capitem.next = cd->open_caps; - capitem.flag = FALSE; - cd->open_caps = &capitem; - } - -/* Offset is set zero to mark that this bracket is still open */ - -PUT(code, 1, 0); -code += 1 + LINK_SIZE + skipbytes; - -/* Loop for each alternative branch */ - -orig_bracount = max_bracount = cd->bracount; -for (;;) - { - /* For a (?| group, reset the capturing bracket count so that each branch - uses the same numbers. */ - - if (reset_bracount) cd->bracount = orig_bracount; - - /* Set up dummy OP_REVERSE if lookbehind assertion */ - - if (lookbehind) - { - *code++ = OP_REVERSE; - reverse_count = code; - PUTINC(code, 0, 0); - length += 1 + LINK_SIZE; - } - - /* Now compile the branch; in the pre-compile phase its length gets added - into the length. */ - - if (!compile_branch(&options, &code, &ptr, errorcodeptr, &branchfirstchar, - &branchfirstcharflags, &branchreqchar, &branchreqcharflags, &bc, - cond_depth, cd, (lengthptr == NULL)? NULL : &length)) - { - *ptrptr = ptr; - return FALSE; - } - - /* Keep the highest bracket count in case (?| was used and some branch - has fewer than the rest. */ - - if (cd->bracount > max_bracount) max_bracount = cd->bracount; - - /* In the real compile phase, there is some post-processing to be done. */ - - if (lengthptr == NULL) - { - /* If this is the first branch, the firstchar and reqchar values for the - branch become the values for the regex. */ - - if (*last_branch != OP_ALT) - { - firstchar = branchfirstchar; - firstcharflags = branchfirstcharflags; - reqchar = branchreqchar; - reqcharflags = branchreqcharflags; - } - - /* If this is not the first branch, the first char and reqchar have to - match the values from all the previous branches, except that if the - previous value for reqchar didn't have REQ_VARY set, it can still match, - and we set REQ_VARY for the regex. */ - - else - { - /* If we previously had a firstchar, but it doesn't match the new branch, - we have to abandon the firstchar for the regex, but if there was - previously no reqchar, it takes on the value of the old firstchar. */ - - if (firstcharflags >= 0 && - (firstcharflags != branchfirstcharflags || firstchar != branchfirstchar)) - { - if (reqcharflags < 0) - { - reqchar = firstchar; - reqcharflags = firstcharflags; - } - firstcharflags = REQ_NONE; - } - - /* If we (now or from before) have no firstchar, a firstchar from the - branch becomes a reqchar if there isn't a branch reqchar. */ - - if (firstcharflags < 0 && branchfirstcharflags >= 0 && branchreqcharflags < 0) - { - branchreqchar = branchfirstchar; - branchreqcharflags = branchfirstcharflags; - } - - /* Now ensure that the reqchars match */ - - if (((reqcharflags & ~REQ_VARY) != (branchreqcharflags & ~REQ_VARY)) || - reqchar != branchreqchar) - reqcharflags = REQ_NONE; - else - { - reqchar = branchreqchar; - reqcharflags |= branchreqcharflags; /* To "or" REQ_VARY */ - } - } - - /* If lookbehind, check that this branch matches a fixed-length string, and - put the length into the OP_REVERSE item. Temporarily mark the end of the - branch with OP_END. If the branch contains OP_RECURSE, the result is -3 - because there may be forward references that we can't check here. Set a - flag to cause another lookbehind check at the end. Why not do it all at the - end? Because common, erroneous checks are picked up here and the offset of - the problem can be shown. */ - - if (lookbehind) - { - int fixed_length; - *code = OP_END; - fixed_length = find_fixedlength(last_branch, (options & PCRE_UTF8) != 0, - FALSE, cd, NULL); - DPRINTF(("fixed length = %d\n", fixed_length)); - if (fixed_length == -3) - { - cd->check_lookbehind = TRUE; - } - else if (fixed_length < 0) - { - *errorcodeptr = (fixed_length == -2)? ERR36 : - (fixed_length == -4)? ERR70: ERR25; - *ptrptr = ptr; - return FALSE; - } - else - { - if (fixed_length > cd->max_lookbehind) - cd->max_lookbehind = fixed_length; - PUT(reverse_count, 0, fixed_length); - } - } - } - - /* Reached end of expression, either ')' or end of pattern. In the real - compile phase, go back through the alternative branches and reverse the chain - of offsets, with the field in the BRA item now becoming an offset to the - first alternative. If there are no alternatives, it points to the end of the - group. The length in the terminating ket is always the length of the whole - bracketed item. Return leaving the pointer at the terminating char. */ - - if (*ptr != CHAR_VERTICAL_LINE) - { - if (lengthptr == NULL) - { - int branch_length = (int)(code - last_branch); - do - { - int prev_length = GET(last_branch, 1); - PUT(last_branch, 1, branch_length); - branch_length = prev_length; - last_branch -= branch_length; - } - while (branch_length > 0); - } - - /* Fill in the ket */ - - *code = OP_KET; - PUT(code, 1, (int)(code - start_bracket)); - code += 1 + LINK_SIZE; - - /* If it was a capturing subpattern, check to see if it contained any - recursive back references. If so, we must wrap it in atomic brackets. - Because we are moving code along, we must ensure that any pending recursive - references are updated. In any event, remove the block from the chain. */ - - if (capnumber > 0) - { - if (cd->open_caps->flag) - { - *code = OP_END; - adjust_recurse(start_bracket, 1 + LINK_SIZE, - (options & PCRE_UTF8) != 0, cd, save_hwm_offset); - memmove(start_bracket + 1 + LINK_SIZE, start_bracket, - IN_UCHARS(code - start_bracket)); - *start_bracket = OP_ONCE; - code += 1 + LINK_SIZE; - PUT(start_bracket, 1, (int)(code - start_bracket)); - *code = OP_KET; - PUT(code, 1, (int)(code - start_bracket)); - code += 1 + LINK_SIZE; - length += 2 + 2*LINK_SIZE; - } - cd->open_caps = cd->open_caps->next; - } - - /* Retain the highest bracket number, in case resetting was used. */ - - cd->bracount = max_bracount; - - /* Set values to pass back */ - - *codeptr = code; - *ptrptr = ptr; - *firstcharptr = firstchar; - *firstcharflagsptr = firstcharflags; - *reqcharptr = reqchar; - *reqcharflagsptr = reqcharflags; - if (lengthptr != NULL) - { - if (OFLOW_MAX - *lengthptr < length) - { - *errorcodeptr = ERR20; - return FALSE; - } - *lengthptr += length; - } - return TRUE; - } - - /* Another branch follows. In the pre-compile phase, we can move the code - pointer back to where it was for the start of the first branch. (That is, - pretend that each branch is the only one.) - - In the real compile phase, insert an ALT node. Its length field points back - to the previous branch while the bracket remains open. At the end the chain - is reversed. It's done like this so that the start of the bracket has a - zero offset until it is closed, making it possible to detect recursion. */ - - if (lengthptr != NULL) - { - code = *codeptr + 1 + LINK_SIZE + skipbytes; - length += 1 + LINK_SIZE; - } - else - { - *code = OP_ALT; - PUT(code, 1, (int)(code - last_branch)); - bc.current_branch = last_branch = code; - code += 1 + LINK_SIZE; - } - - ptr++; - } -/* Control never reaches here */ -} - - - - -/************************************************* -* Check for anchored expression * -*************************************************/ - -/* Try to find out if this is an anchored regular expression. Consider each -alternative branch. If they all start with OP_SOD or OP_CIRC, or with a bracket -all of whose alternatives start with OP_SOD or OP_CIRC (recurse ad lib), then -it's anchored. However, if this is a multiline pattern, then only OP_SOD will -be found, because ^ generates OP_CIRCM in that mode. - -We can also consider a regex to be anchored if OP_SOM starts all its branches. -This is the code for \G, which means "match at start of match position, taking -into account the match offset". - -A branch is also implicitly anchored if it starts with .* and DOTALL is set, -because that will try the rest of the pattern at all possible matching points, -so there is no point trying again.... er .... - -.... except when the .* appears inside capturing parentheses, and there is a -subsequent back reference to those parentheses. We haven't enough information -to catch that case precisely. - -At first, the best we could do was to detect when .* was in capturing brackets -and the highest back reference was greater than or equal to that level. -However, by keeping a bitmap of the first 31 back references, we can catch some -of the more common cases more precisely. - -... A second exception is when the .* appears inside an atomic group, because -this prevents the number of characters it matches from being adjusted. - -Arguments: - code points to start of expression (the bracket) - bracket_map a bitmap of which brackets we are inside while testing; this - handles up to substring 31; after that we just have to take - the less precise approach - cd points to the compile data block - atomcount atomic group level - -Returns: TRUE or FALSE -*/ - -static BOOL -is_anchored(register const pcre_uchar *code, unsigned int bracket_map, - compile_data *cd, int atomcount) -{ -do { - const pcre_uchar *scode = first_significant_code( - code + PRIV(OP_lengths)[*code], FALSE); - register int op = *scode; - - /* Non-capturing brackets */ - - if (op == OP_BRA || op == OP_BRAPOS || - op == OP_SBRA || op == OP_SBRAPOS) - { - if (!is_anchored(scode, bracket_map, cd, atomcount)) return FALSE; - } - - /* Capturing brackets */ - - else if (op == OP_CBRA || op == OP_CBRAPOS || - op == OP_SCBRA || op == OP_SCBRAPOS) - { - int n = GET2(scode, 1+LINK_SIZE); - int new_map = bracket_map | ((n < 32)? (1 << n) : 1); - if (!is_anchored(scode, new_map, cd, atomcount)) return FALSE; - } - - /* Positive forward assertions and conditions */ - - else if (op == OP_ASSERT || op == OP_COND) - { - if (!is_anchored(scode, bracket_map, cd, atomcount)) return FALSE; - } - - /* Atomic groups */ - - else if (op == OP_ONCE || op == OP_ONCE_NC) - { - if (!is_anchored(scode, bracket_map, cd, atomcount + 1)) - return FALSE; - } - - /* .* is not anchored unless DOTALL is set (which generates OP_ALLANY) and - it isn't in brackets that are or may be referenced or inside an atomic - group. */ - - else if ((op == OP_TYPESTAR || op == OP_TYPEMINSTAR || - op == OP_TYPEPOSSTAR)) - { - if (scode[1] != OP_ALLANY || (bracket_map & cd->backref_map) != 0 || - atomcount > 0 || cd->had_pruneorskip) - return FALSE; - } - - /* Check for explicit anchoring */ - - else if (op != OP_SOD && op != OP_SOM && op != OP_CIRC) return FALSE; - - code += GET(code, 1); - } -while (*code == OP_ALT); /* Loop for each alternative */ -return TRUE; -} - - - -/************************************************* -* Check for starting with ^ or .* * -*************************************************/ - -/* This is called to find out if every branch starts with ^ or .* so that -"first char" processing can be done to speed things up in multiline -matching and for non-DOTALL patterns that start with .* (which must start at -the beginning or after \n). As in the case of is_anchored() (see above), we -have to take account of back references to capturing brackets that contain .* -because in that case we can't make the assumption. Also, the appearance of .* -inside atomic brackets or in a pattern that contains *PRUNE or *SKIP does not -count, because once again the assumption no longer holds. - -Arguments: - code points to start of expression (the bracket) - bracket_map a bitmap of which brackets we are inside while testing; this - handles up to substring 31; after that we just have to take - the less precise approach - cd points to the compile data - atomcount atomic group level - -Returns: TRUE or FALSE -*/ - -static BOOL -is_startline(const pcre_uchar *code, unsigned int bracket_map, - compile_data *cd, int atomcount) -{ -do { - const pcre_uchar *scode = first_significant_code( - code + PRIV(OP_lengths)[*code], FALSE); - register int op = *scode; - - /* If we are at the start of a conditional assertion group, *both* the - conditional assertion *and* what follows the condition must satisfy the test - for start of line. Other kinds of condition fail. Note that there may be an - auto-callout at the start of a condition. */ - - if (op == OP_COND) - { - scode += 1 + LINK_SIZE; - if (*scode == OP_CALLOUT) scode += PRIV(OP_lengths)[OP_CALLOUT]; - switch (*scode) - { - case OP_CREF: - case OP_DNCREF: - case OP_RREF: - case OP_DNRREF: - case OP_DEF: - case OP_FAIL: - return FALSE; - - default: /* Assertion */ - if (!is_startline(scode, bracket_map, cd, atomcount)) return FALSE; - do scode += GET(scode, 1); while (*scode == OP_ALT); - scode += 1 + LINK_SIZE; - break; - } - scode = first_significant_code(scode, FALSE); - op = *scode; - } - - /* Non-capturing brackets */ - - if (op == OP_BRA || op == OP_BRAPOS || - op == OP_SBRA || op == OP_SBRAPOS) - { - if (!is_startline(scode, bracket_map, cd, atomcount)) return FALSE; - } - - /* Capturing brackets */ - - else if (op == OP_CBRA || op == OP_CBRAPOS || - op == OP_SCBRA || op == OP_SCBRAPOS) - { - int n = GET2(scode, 1+LINK_SIZE); - int new_map = bracket_map | ((n < 32)? (1 << n) : 1); - if (!is_startline(scode, new_map, cd, atomcount)) return FALSE; - } - - /* Positive forward assertions */ - - else if (op == OP_ASSERT) - { - if (!is_startline(scode, bracket_map, cd, atomcount)) return FALSE; - } - - /* Atomic brackets */ - - else if (op == OP_ONCE || op == OP_ONCE_NC) - { - if (!is_startline(scode, bracket_map, cd, atomcount + 1)) return FALSE; - } - - /* .* means "start at start or after \n" if it isn't in atomic brackets or - brackets that may be referenced, as long as the pattern does not contain - *PRUNE or *SKIP, because these break the feature. Consider, for example, - /.*?a(*PRUNE)b/ with the subject "aab", which matches "ab", i.e. not at the - start of a line. */ - - else if (op == OP_TYPESTAR || op == OP_TYPEMINSTAR || op == OP_TYPEPOSSTAR) - { - if (scode[1] != OP_ANY || (bracket_map & cd->backref_map) != 0 || - atomcount > 0 || cd->had_pruneorskip) - return FALSE; - } - - /* Check for explicit circumflex; anything else gives a FALSE result. Note - in particular that this includes atomic brackets OP_ONCE and OP_ONCE_NC - because the number of characters matched by .* cannot be adjusted inside - them. */ - - else if (op != OP_CIRC && op != OP_CIRCM) return FALSE; - - /* Move on to the next alternative */ - - code += GET(code, 1); - } -while (*code == OP_ALT); /* Loop for each alternative */ -return TRUE; -} - - - -/************************************************* -* Check for asserted fixed first char * -*************************************************/ - -/* During compilation, the "first char" settings from forward assertions are -discarded, because they can cause conflicts with actual literals that follow. -However, if we end up without a first char setting for an unanchored pattern, -it is worth scanning the regex to see if there is an initial asserted first -char. If all branches start with the same asserted char, or with a -non-conditional bracket all of whose alternatives start with the same asserted -char (recurse ad lib), then we return that char, with the flags set to zero or -REQ_CASELESS; otherwise return zero with REQ_NONE in the flags. - -Arguments: - code points to start of expression (the bracket) - flags points to the first char flags, or to REQ_NONE - inassert TRUE if in an assertion - -Returns: the fixed first char, or 0 with REQ_NONE in flags -*/ - -static pcre_uint32 -find_firstassertedchar(const pcre_uchar *code, pcre_int32 *flags, - BOOL inassert) -{ -register pcre_uint32 c = 0; -int cflags = REQ_NONE; - -*flags = REQ_NONE; -do { - pcre_uint32 d; - int dflags; - int xl = (*code == OP_CBRA || *code == OP_SCBRA || - *code == OP_CBRAPOS || *code == OP_SCBRAPOS)? IMM2_SIZE:0; - const pcre_uchar *scode = first_significant_code(code + 1+LINK_SIZE + xl, - TRUE); - register pcre_uchar op = *scode; - - switch(op) - { - default: - return 0; - - case OP_BRA: - case OP_BRAPOS: - case OP_CBRA: - case OP_SCBRA: - case OP_CBRAPOS: - case OP_SCBRAPOS: - case OP_ASSERT: - case OP_ONCE: - case OP_ONCE_NC: - d = find_firstassertedchar(scode, &dflags, op == OP_ASSERT); - if (dflags < 0) - return 0; - if (cflags < 0) { c = d; cflags = dflags; } else if (c != d || cflags != dflags) return 0; - break; - - case OP_EXACT: - scode += IMM2_SIZE; - /* Fall through */ - - case OP_CHAR: - case OP_PLUS: - case OP_MINPLUS: - case OP_POSPLUS: - if (!inassert) return 0; - if (cflags < 0) { c = scode[1]; cflags = 0; } - else if (c != scode[1]) return 0; - break; - - case OP_EXACTI: - scode += IMM2_SIZE; - /* Fall through */ - - case OP_CHARI: - case OP_PLUSI: - case OP_MINPLUSI: - case OP_POSPLUSI: - if (!inassert) return 0; - if (cflags < 0) { c = scode[1]; cflags = REQ_CASELESS; } - else if (c != scode[1]) return 0; - break; - } - - code += GET(code, 1); - } -while (*code == OP_ALT); - -*flags = cflags; -return c; -} - - - -/************************************************* -* Add an entry to the name/number table * -*************************************************/ - -/* This function is called between compiling passes to add an entry to the -name/number table, maintaining alphabetical order. Checking for permitted -and forbidden duplicates has already been done. - -Arguments: - cd the compile data block - name the name to add - length the length of the name - groupno the group number - -Returns: nothing -*/ - -static void -add_name(compile_data *cd, const pcre_uchar *name, int length, - unsigned int groupno) -{ -int i; -pcre_uchar *slot = cd->name_table; - -for (i = 0; i < cd->names_found; i++) - { - int crc = memcmp(name, slot+IMM2_SIZE, IN_UCHARS(length)); - if (crc == 0 && slot[IMM2_SIZE+length] != 0) - crc = -1; /* Current name is a substring */ - - /* Make space in the table and break the loop for an earlier name. For a - duplicate or later name, carry on. We do this for duplicates so that in the - simple case (when ?(| is not used) they are in order of their numbers. In all - cases they are in the order in which they appear in the pattern. */ - - if (crc < 0) - { - memmove(slot + cd->name_entry_size, slot, - IN_UCHARS((cd->names_found - i) * cd->name_entry_size)); - break; - } - - /* Continue the loop for a later or duplicate name */ - - slot += cd->name_entry_size; - } - -PUT2(slot, 0, groupno); -memcpy(slot + IMM2_SIZE, name, IN_UCHARS(length)); -slot[IMM2_SIZE + length] = 0; -cd->names_found++; -} - - - -/************************************************* -* Compile a Regular Expression * -*************************************************/ - -/* This function takes a string and returns a pointer to a block of store -holding a compiled version of the expression. The original API for this -function had no error code return variable; it is retained for backwards -compatibility. The new function is given a new name. - -Arguments: - pattern the regular expression - options various option bits - errorcodeptr pointer to error code variable (pcre_compile2() only) - can be NULL if you don't want a code value - errorptr pointer to pointer to error text - erroroffset ptr offset in pattern where error was detected - tables pointer to character tables or NULL - -Returns: pointer to compiled data block, or NULL on error, - with errorptr and erroroffset set -*/ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN pcre * PCRE_CALL_CONVENTION -pcre_compile(const char *pattern, int options, const char **errorptr, - int *erroroffset, const unsigned char *tables) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN pcre16 * PCRE_CALL_CONVENTION -pcre16_compile(PCRE_SPTR16 pattern, int options, const char **errorptr, - int *erroroffset, const unsigned char *tables) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN pcre32 * PCRE_CALL_CONVENTION -pcre32_compile(PCRE_SPTR32 pattern, int options, const char **errorptr, - int *erroroffset, const unsigned char *tables) -#endif -{ -#if defined COMPILE_PCRE8 -return pcre_compile2(pattern, options, NULL, errorptr, erroroffset, tables); -#elif defined COMPILE_PCRE16 -return pcre16_compile2(pattern, options, NULL, errorptr, erroroffset, tables); -#elif defined COMPILE_PCRE32 -return pcre32_compile2(pattern, options, NULL, errorptr, erroroffset, tables); -#endif -} - - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN pcre * PCRE_CALL_CONVENTION -pcre_compile2(const char *pattern, int options, int *errorcodeptr, - const char **errorptr, int *erroroffset, const unsigned char *tables) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN pcre16 * PCRE_CALL_CONVENTION -pcre16_compile2(PCRE_SPTR16 pattern, int options, int *errorcodeptr, - const char **errorptr, int *erroroffset, const unsigned char *tables) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN pcre32 * PCRE_CALL_CONVENTION -pcre32_compile2(PCRE_SPTR32 pattern, int options, int *errorcodeptr, - const char **errorptr, int *erroroffset, const unsigned char *tables) -#endif -{ -REAL_PCRE *re; -int length = 1; /* For final END opcode */ -pcre_int32 firstcharflags, reqcharflags; -pcre_uint32 firstchar, reqchar; -pcre_uint32 limit_match = PCRE_UINT32_MAX; -pcre_uint32 limit_recursion = PCRE_UINT32_MAX; -int newline; -int errorcode = 0; -int skipatstart = 0; -BOOL utf; -BOOL never_utf = FALSE; -size_t size; -pcre_uchar *code; -const pcre_uchar *codestart; -const pcre_uchar *ptr; -compile_data compile_block; -compile_data *cd = &compile_block; - -/* This space is used for "compiling" into during the first phase, when we are -computing the amount of memory that is needed. Compiled items are thrown away -as soon as possible, so that a fairly large buffer should be sufficient for -this purpose. The same space is used in the second phase for remembering where -to fill in forward references to subpatterns. That may overflow, in which case -new memory is obtained from malloc(). */ - -pcre_uchar cworkspace[COMPILE_WORK_SIZE]; - -/* This vector is used for remembering name groups during the pre-compile. In a -similar way to cworkspace, it can be expanded using malloc() if necessary. */ - -named_group named_groups[NAMED_GROUP_LIST_SIZE]; - -/* Set this early so that early errors get offset 0. */ - -ptr = (const pcre_uchar *)pattern; - -/* We can't pass back an error message if errorptr is NULL; I guess the best we -can do is just return NULL, but we can set a code value if there is a code -pointer. */ - -if (errorptr == NULL) - { - if (errorcodeptr != NULL) *errorcodeptr = 99; - return NULL; - } - -*errorptr = NULL; -if (errorcodeptr != NULL) *errorcodeptr = ERR0; - -/* However, we can give a message for this error */ - -if (erroroffset == NULL) - { - errorcode = ERR16; - goto PCRE_EARLY_ERROR_RETURN2; - } - -*erroroffset = 0; - -/* Set up pointers to the individual character tables */ - -if (tables == NULL) tables = PRIV(default_tables); -cd->lcc = tables + lcc_offset; -cd->fcc = tables + fcc_offset; -cd->cbits = tables + cbits_offset; -cd->ctypes = tables + ctypes_offset; - -/* Check that all undefined public option bits are zero */ - -if ((options & ~PUBLIC_COMPILE_OPTIONS) != 0) - { - errorcode = ERR17; - goto PCRE_EARLY_ERROR_RETURN; - } - -/* If PCRE_NEVER_UTF is set, remember it. */ - -if ((options & PCRE_NEVER_UTF) != 0) never_utf = TRUE; - -/* Check for global one-time settings at the start of the pattern, and remember -the offset for later. */ - -cd->external_flags = 0; /* Initialize here for LIMIT_MATCH/RECURSION */ - -while (ptr[skipatstart] == CHAR_LEFT_PARENTHESIS && - ptr[skipatstart+1] == CHAR_ASTERISK) - { - int newnl = 0; - int newbsr = 0; - -/* For completeness and backward compatibility, (*UTFn) is supported in the -relevant libraries, but (*UTF) is generic and always supported. Note that -PCRE_UTF8 == PCRE_UTF16 == PCRE_UTF32. */ - -#ifdef COMPILE_PCRE8 - if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_UTF8_RIGHTPAR, 5) == 0) - { skipatstart += 7; options |= PCRE_UTF8; continue; } -#endif -#ifdef COMPILE_PCRE16 - if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_UTF16_RIGHTPAR, 6) == 0) - { skipatstart += 8; options |= PCRE_UTF16; continue; } -#endif -#ifdef COMPILE_PCRE32 - if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_UTF32_RIGHTPAR, 6) == 0) - { skipatstart += 8; options |= PCRE_UTF32; continue; } -#endif - - else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_UTF_RIGHTPAR, 4) == 0) - { skipatstart += 6; options |= PCRE_UTF8; continue; } - else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_UCP_RIGHTPAR, 4) == 0) - { skipatstart += 6; options |= PCRE_UCP; continue; } - else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_NO_AUTO_POSSESS_RIGHTPAR, 16) == 0) - { skipatstart += 18; options |= PCRE_NO_AUTO_POSSESS; continue; } - else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_NO_START_OPT_RIGHTPAR, 13) == 0) - { skipatstart += 15; options |= PCRE_NO_START_OPTIMIZE; continue; } - - else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_LIMIT_MATCH_EQ, 12) == 0) - { - pcre_uint32 c = 0; - int p = skipatstart + 14; - while (isdigit(ptr[p])) - { - if (c > PCRE_UINT32_MAX / 10 - 1) break; /* Integer overflow */ - c = c*10 + ptr[p++] - CHAR_0; - } - if (ptr[p++] != CHAR_RIGHT_PARENTHESIS) break; - if (c < limit_match) - { - limit_match = c; - cd->external_flags |= PCRE_MLSET; - } - skipatstart = p; - continue; - } - - else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_LIMIT_RECURSION_EQ, 16) == 0) - { - pcre_uint32 c = 0; - int p = skipatstart + 18; - while (isdigit(ptr[p])) - { - if (c > PCRE_UINT32_MAX / 10 - 1) break; /* Integer overflow check */ - c = c*10 + ptr[p++] - CHAR_0; - } - if (ptr[p++] != CHAR_RIGHT_PARENTHESIS) break; - if (c < limit_recursion) - { - limit_recursion = c; - cd->external_flags |= PCRE_RLSET; - } - skipatstart = p; - continue; - } - - if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_CR_RIGHTPAR, 3) == 0) - { skipatstart += 5; newnl = PCRE_NEWLINE_CR; } - else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_LF_RIGHTPAR, 3) == 0) - { skipatstart += 5; newnl = PCRE_NEWLINE_LF; } - else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_CRLF_RIGHTPAR, 5) == 0) - { skipatstart += 7; newnl = PCRE_NEWLINE_CR + PCRE_NEWLINE_LF; } - else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_ANY_RIGHTPAR, 4) == 0) - { skipatstart += 6; newnl = PCRE_NEWLINE_ANY; } - else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_ANYCRLF_RIGHTPAR, 8) == 0) - { skipatstart += 10; newnl = PCRE_NEWLINE_ANYCRLF; } - - else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_BSR_ANYCRLF_RIGHTPAR, 12) == 0) - { skipatstart += 14; newbsr = PCRE_BSR_ANYCRLF; } - else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_BSR_UNICODE_RIGHTPAR, 12) == 0) - { skipatstart += 14; newbsr = PCRE_BSR_UNICODE; } - - if (newnl != 0) - options = (options & ~PCRE_NEWLINE_BITS) | newnl; - else if (newbsr != 0) - options = (options & ~(PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE)) | newbsr; - else break; - } - -/* PCRE_UTF(16|32) have the same value as PCRE_UTF8. */ -utf = (options & PCRE_UTF8) != 0; -if (utf && never_utf) - { - errorcode = ERR78; - goto PCRE_EARLY_ERROR_RETURN2; - } - -/* Can't support UTF unless PCRE has been compiled to include the code. The -return of an error code from PRIV(valid_utf)() is a new feature, introduced in -release 8.13. It is passed back from pcre_[dfa_]exec(), but at the moment is -not used here. */ - -#ifdef SUPPORT_UTF -if (utf && (options & PCRE_NO_UTF8_CHECK) == 0 && - (errorcode = PRIV(valid_utf)((PCRE_PUCHAR)pattern, -1, erroroffset)) != 0) - { -#if defined COMPILE_PCRE8 - errorcode = ERR44; -#elif defined COMPILE_PCRE16 - errorcode = ERR74; -#elif defined COMPILE_PCRE32 - errorcode = ERR77; -#endif - goto PCRE_EARLY_ERROR_RETURN2; - } -#else -if (utf) - { - errorcode = ERR32; - goto PCRE_EARLY_ERROR_RETURN; - } -#endif - -/* Can't support UCP unless PCRE has been compiled to include the code. */ - -#ifndef SUPPORT_UCP -if ((options & PCRE_UCP) != 0) - { - errorcode = ERR67; - goto PCRE_EARLY_ERROR_RETURN; - } -#endif - -/* Check validity of \R options. */ - -if ((options & (PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE)) == - (PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE)) - { - errorcode = ERR56; - goto PCRE_EARLY_ERROR_RETURN; - } - -/* Handle different types of newline. The three bits give seven cases. The -current code allows for fixed one- or two-byte sequences, plus "any" and -"anycrlf". */ - -switch (options & PCRE_NEWLINE_BITS) - { - case 0: newline = NEWLINE; break; /* Build-time default */ - case PCRE_NEWLINE_CR: newline = CHAR_CR; break; - case PCRE_NEWLINE_LF: newline = CHAR_NL; break; - case PCRE_NEWLINE_CR+ - PCRE_NEWLINE_LF: newline = (CHAR_CR << 8) | CHAR_NL; break; - case PCRE_NEWLINE_ANY: newline = -1; break; - case PCRE_NEWLINE_ANYCRLF: newline = -2; break; - default: errorcode = ERR56; goto PCRE_EARLY_ERROR_RETURN; - } - -if (newline == -2) - { - cd->nltype = NLTYPE_ANYCRLF; - } -else if (newline < 0) - { - cd->nltype = NLTYPE_ANY; - } -else - { - cd->nltype = NLTYPE_FIXED; - if (newline > 255) - { - cd->nllen = 2; - cd->nl[0] = (newline >> 8) & 255; - cd->nl[1] = newline & 255; - } - else - { - cd->nllen = 1; - cd->nl[0] = newline; - } - } - -/* Maximum back reference and backref bitmap. The bitmap records up to 31 back -references to help in deciding whether (.*) can be treated as anchored or not. -*/ - -cd->top_backref = 0; -cd->backref_map = 0; - -/* Reflect pattern for debugging output */ - -DPRINTF(("------------------------------------------------------------------\n")); -#ifdef PCRE_DEBUG -print_puchar(stdout, (PCRE_PUCHAR)pattern); -#endif -DPRINTF(("\n")); - -/* Pretend to compile the pattern while actually just accumulating the length -of memory required. This behaviour is triggered by passing a non-NULL final -argument to compile_regex(). We pass a block of workspace (cworkspace) for it -to compile parts of the pattern into; the compiled code is discarded when it is -no longer needed, so hopefully this workspace will never overflow, though there -is a test for its doing so. */ - -cd->bracount = cd->final_bracount = 0; -cd->names_found = 0; -cd->name_entry_size = 0; -cd->name_table = NULL; -cd->dupnames = FALSE; -cd->dupgroups = FALSE; -cd->namedrefcount = 0; -cd->start_code = cworkspace; -cd->hwm = cworkspace; -cd->iscondassert = FALSE; -cd->start_workspace = cworkspace; -cd->workspace_size = COMPILE_WORK_SIZE; -cd->named_groups = named_groups; -cd->named_group_list_size = NAMED_GROUP_LIST_SIZE; -cd->start_pattern = (const pcre_uchar *)pattern; -cd->end_pattern = (const pcre_uchar *)(pattern + STRLEN_UC((const pcre_uchar *)pattern)); -cd->req_varyopt = 0; -cd->parens_depth = 0; -cd->assert_depth = 0; -cd->max_lookbehind = 0; -cd->external_options = options; -cd->open_caps = NULL; - -/* Now do the pre-compile. On error, errorcode will be set non-zero, so we -don't need to look at the result of the function here. The initial options have -been put into the cd block so that they can be changed if an option setting is -found within the regex right at the beginning. Bringing initial option settings -outside can help speed up starting point checks. */ - -ptr += skipatstart; -code = cworkspace; -*code = OP_BRA; - -(void)compile_regex(cd->external_options, &code, &ptr, &errorcode, FALSE, - FALSE, 0, 0, &firstchar, &firstcharflags, &reqchar, &reqcharflags, NULL, - cd, &length); -if (errorcode != 0) goto PCRE_EARLY_ERROR_RETURN; - -DPRINTF(("end pre-compile: length=%d workspace=%d\n", length, - (int)(cd->hwm - cworkspace))); - -if (length > MAX_PATTERN_SIZE) - { - errorcode = ERR20; - goto PCRE_EARLY_ERROR_RETURN; - } - -/* Compute the size of the data block for storing the compiled pattern. Integer -overflow should no longer be possible because nowadays we limit the maximum -value of cd->names_found and cd->name_entry_size. */ - -size = sizeof(REAL_PCRE) + - (length + cd->names_found * cd->name_entry_size) * sizeof(pcre_uchar); - -/* Get the memory. */ - -re = (REAL_PCRE *)(PUBL(malloc))(size); -if (re == NULL) - { - errorcode = ERR21; - goto PCRE_EARLY_ERROR_RETURN; - } - -/* Put in the magic number, and save the sizes, initial options, internal -flags, and character table pointer. NULL is used for the default character -tables. The nullpad field is at the end; it's there to help in the case when a -regex compiled on a system with 4-byte pointers is run on another with 8-byte -pointers. */ - -re->magic_number = MAGIC_NUMBER; -re->size = (int)size; -re->options = cd->external_options; -re->flags = cd->external_flags; -re->limit_match = limit_match; -re->limit_recursion = limit_recursion; -re->first_char = 0; -re->req_char = 0; -re->name_table_offset = sizeof(REAL_PCRE) / sizeof(pcre_uchar); -re->name_entry_size = cd->name_entry_size; -re->name_count = cd->names_found; -re->ref_count = 0; -re->tables = (tables == PRIV(default_tables))? NULL : tables; -re->nullpad = NULL; -#ifdef COMPILE_PCRE32 -re->dummy = 0; -#else -re->dummy1 = re->dummy2 = re->dummy3 = 0; -#endif - -/* The starting points of the name/number translation table and of the code are -passed around in the compile data block. The start/end pattern and initial -options are already set from the pre-compile phase, as is the name_entry_size -field. Reset the bracket count and the names_found field. Also reset the hwm -field; this time it's used for remembering forward references to subpatterns. -*/ - -cd->final_bracount = cd->bracount; /* Save for checking forward references */ -cd->parens_depth = 0; -cd->assert_depth = 0; -cd->bracount = 0; -cd->max_lookbehind = 0; -cd->name_table = (pcre_uchar *)re + re->name_table_offset; -codestart = cd->name_table + re->name_entry_size * re->name_count; -cd->start_code = codestart; -cd->hwm = (pcre_uchar *)(cd->start_workspace); -cd->iscondassert = FALSE; -cd->req_varyopt = 0; -cd->had_accept = FALSE; -cd->had_pruneorskip = FALSE; -cd->check_lookbehind = FALSE; -cd->open_caps = NULL; - -/* If any named groups were found, create the name/number table from the list -created in the first pass. */ - -if (cd->names_found > 0) - { - int i = cd->names_found; - named_group *ng = cd->named_groups; - cd->names_found = 0; - for (; i > 0; i--, ng++) - add_name(cd, ng->name, ng->length, ng->number); - if (cd->named_group_list_size > NAMED_GROUP_LIST_SIZE) - (PUBL(free))((void *)cd->named_groups); - } - -/* Set up a starting, non-extracting bracket, then compile the expression. On -error, errorcode will be set non-zero, so we don't need to look at the result -of the function here. */ - -ptr = (const pcre_uchar *)pattern + skipatstart; -code = (pcre_uchar *)codestart; -*code = OP_BRA; -(void)compile_regex(re->options, &code, &ptr, &errorcode, FALSE, FALSE, 0, 0, - &firstchar, &firstcharflags, &reqchar, &reqcharflags, NULL, cd, NULL); -re->top_bracket = cd->bracount; -re->top_backref = cd->top_backref; -re->max_lookbehind = cd->max_lookbehind; -re->flags = cd->external_flags | PCRE_MODE; - -if (cd->had_accept) - { - reqchar = 0; /* Must disable after (*ACCEPT) */ - reqcharflags = REQ_NONE; - } - -/* If not reached end of pattern on success, there's an excess bracket. */ - -if (errorcode == 0 && *ptr != CHAR_NULL) errorcode = ERR22; - -/* Fill in the terminating state and check for disastrous overflow, but -if debugging, leave the test till after things are printed out. */ - -*code++ = OP_END; - -#ifndef PCRE_DEBUG -if (code - codestart > length) errorcode = ERR23; -#endif - -#ifdef SUPPORT_VALGRIND -/* If the estimated length exceeds the really used length, mark the extra -allocated memory as unaddressable, so that any out-of-bound reads can be -detected. */ -VALGRIND_MAKE_MEM_NOACCESS(code, (length - (code - codestart)) * sizeof(pcre_uchar)); -#endif - -/* Fill in any forward references that are required. There may be repeated -references; optimize for them, as searching a large regex takes time. */ - -if (cd->hwm > cd->start_workspace) - { - int prev_recno = -1; - const pcre_uchar *groupptr = NULL; - while (errorcode == 0 && cd->hwm > cd->start_workspace) - { - int offset, recno; - cd->hwm -= LINK_SIZE; - offset = GET(cd->hwm, 0); - - /* Check that the hwm handling hasn't gone wrong. This whole area is - rewritten in PCRE2 because there are some obscure cases. */ - - if (offset == 0 || codestart[offset-1] != OP_RECURSE) - { - errorcode = ERR10; - break; - } - - recno = GET(codestart, offset); - if (recno != prev_recno) - { - groupptr = PRIV(find_bracket)(codestart, utf, recno); - prev_recno = recno; - } - if (groupptr == NULL) errorcode = ERR53; - else PUT(((pcre_uchar *)codestart), offset, (int)(groupptr - codestart)); - } - } - -/* If the workspace had to be expanded, free the new memory. Set the pointer to -NULL to indicate that forward references have been filled in. */ - -if (cd->workspace_size > COMPILE_WORK_SIZE) - (PUBL(free))((void *)cd->start_workspace); -cd->start_workspace = NULL; - -/* Give an error if there's back reference to a non-existent capturing -subpattern. */ - -if (errorcode == 0 && re->top_backref > re->top_bracket) errorcode = ERR15; - -/* Unless disabled, check whether any single character iterators can be -auto-possessified. The function overwrites the appropriate opcode values, so -the type of the pointer must be cast. NOTE: the intermediate variable "temp" is -used in this code because at least one compiler gives a warning about loss of -"const" attribute if the cast (pcre_uchar *)codestart is used directly in the -function call. */ - -if (errorcode == 0 && (options & PCRE_NO_AUTO_POSSESS) == 0) - { - pcre_uchar *temp = (pcre_uchar *)codestart; - auto_possessify(temp, utf, cd); - } - -/* If there were any lookbehind assertions that contained OP_RECURSE -(recursions or subroutine calls), a flag is set for them to be checked here, -because they may contain forward references. Actual recursions cannot be fixed -length, but subroutine calls can. It is done like this so that those without -OP_RECURSE that are not fixed length get a diagnosic with a useful offset. The -exceptional ones forgo this. We scan the pattern to check that they are fixed -length, and set their lengths. */ - -if (errorcode == 0 && cd->check_lookbehind) - { - pcre_uchar *cc = (pcre_uchar *)codestart; - - /* Loop, searching for OP_REVERSE items, and process those that do not have - their length set. (Actually, it will also re-process any that have a length - of zero, but that is a pathological case, and it does no harm.) When we find - one, we temporarily terminate the branch it is in while we scan it. */ - - for (cc = (pcre_uchar *)PRIV(find_bracket)(codestart, utf, -1); - cc != NULL; - cc = (pcre_uchar *)PRIV(find_bracket)(cc, utf, -1)) - { - if (GET(cc, 1) == 0) - { - int fixed_length; - pcre_uchar *be = cc - 1 - LINK_SIZE + GET(cc, -LINK_SIZE); - int end_op = *be; - *be = OP_END; - fixed_length = find_fixedlength(cc, (re->options & PCRE_UTF8) != 0, TRUE, - cd, NULL); - *be = end_op; - DPRINTF(("fixed length = %d\n", fixed_length)); - if (fixed_length < 0) - { - errorcode = (fixed_length == -2)? ERR36 : - (fixed_length == -4)? ERR70 : ERR25; - break; - } - if (fixed_length > cd->max_lookbehind) cd->max_lookbehind = fixed_length; - PUT(cc, 1, fixed_length); - } - cc += 1 + LINK_SIZE; - } - } - -/* Failed to compile, or error while post-processing */ - -if (errorcode != 0) - { - (PUBL(free))(re); - PCRE_EARLY_ERROR_RETURN: - *erroroffset = (int)(ptr - (const pcre_uchar *)pattern); - PCRE_EARLY_ERROR_RETURN2: - *errorptr = find_error_text(errorcode); - if (errorcodeptr != NULL) *errorcodeptr = errorcode; - return NULL; - } - -/* If the anchored option was not passed, set the flag if we can determine that -the pattern is anchored by virtue of ^ characters or \A or anything else, such -as starting with non-atomic .* when DOTALL is set and there are no occurrences -of *PRUNE or *SKIP. - -Otherwise, if we know what the first byte has to be, save it, because that -speeds up unanchored matches no end. If not, see if we can set the -PCRE_STARTLINE flag. This is helpful for multiline matches when all branches -start with ^. and also when all branches start with non-atomic .* for -non-DOTALL matches when *PRUNE and SKIP are not present. */ - -if ((re->options & PCRE_ANCHORED) == 0) - { - if (is_anchored(codestart, 0, cd, 0)) re->options |= PCRE_ANCHORED; - else - { - if (firstcharflags < 0) - firstchar = find_firstassertedchar(codestart, &firstcharflags, FALSE); - if (firstcharflags >= 0) /* Remove caseless flag for non-caseable chars */ - { -#if defined COMPILE_PCRE8 - re->first_char = firstchar & 0xff; -#elif defined COMPILE_PCRE16 - re->first_char = firstchar & 0xffff; -#elif defined COMPILE_PCRE32 - re->first_char = firstchar; -#endif - if ((firstcharflags & REQ_CASELESS) != 0) - { -#if defined SUPPORT_UCP && !(defined COMPILE_PCRE8) - /* We ignore non-ASCII first chars in 8 bit mode. */ - if (utf) - { - if (re->first_char < 128) - { - if (cd->fcc[re->first_char] != re->first_char) - re->flags |= PCRE_FCH_CASELESS; - } - else if (UCD_OTHERCASE(re->first_char) != re->first_char) - re->flags |= PCRE_FCH_CASELESS; - } - else -#endif - if (MAX_255(re->first_char) - && cd->fcc[re->first_char] != re->first_char) - re->flags |= PCRE_FCH_CASELESS; - } - - re->flags |= PCRE_FIRSTSET; - } - - else if (is_startline(codestart, 0, cd, 0)) re->flags |= PCRE_STARTLINE; - } - } - -/* For an anchored pattern, we use the "required byte" only if it follows a -variable length item in the regex. Remove the caseless flag for non-caseable -bytes. */ - -if (reqcharflags >= 0 && - ((re->options & PCRE_ANCHORED) == 0 || (reqcharflags & REQ_VARY) != 0)) - { -#if defined COMPILE_PCRE8 - re->req_char = reqchar & 0xff; -#elif defined COMPILE_PCRE16 - re->req_char = reqchar & 0xffff; -#elif defined COMPILE_PCRE32 - re->req_char = reqchar; -#endif - if ((reqcharflags & REQ_CASELESS) != 0) - { -#if defined SUPPORT_UCP && !(defined COMPILE_PCRE8) - /* We ignore non-ASCII first chars in 8 bit mode. */ - if (utf) - { - if (re->req_char < 128) - { - if (cd->fcc[re->req_char] != re->req_char) - re->flags |= PCRE_RCH_CASELESS; - } - else if (UCD_OTHERCASE(re->req_char) != re->req_char) - re->flags |= PCRE_RCH_CASELESS; - } - else -#endif - if (MAX_255(re->req_char) && cd->fcc[re->req_char] != re->req_char) - re->flags |= PCRE_RCH_CASELESS; - } - - re->flags |= PCRE_REQCHSET; - } - -/* Print out the compiled data if debugging is enabled. This is never the -case when building a production library. */ - -#ifdef PCRE_DEBUG -printf("Length = %d top_bracket = %d top_backref = %d\n", - length, re->top_bracket, re->top_backref); - -printf("Options=%08x\n", re->options); - -if ((re->flags & PCRE_FIRSTSET) != 0) - { - pcre_uchar ch = re->first_char; - const char *caseless = - ((re->flags & PCRE_FCH_CASELESS) == 0)? "" : " (caseless)"; - if (PRINTABLE(ch)) printf("First char = %c%s\n", ch, caseless); - else printf("First char = \\x%02x%s\n", ch, caseless); - } - -if ((re->flags & PCRE_REQCHSET) != 0) - { - pcre_uchar ch = re->req_char; - const char *caseless = - ((re->flags & PCRE_RCH_CASELESS) == 0)? "" : " (caseless)"; - if (PRINTABLE(ch)) printf("Req char = %c%s\n", ch, caseless); - else printf("Req char = \\x%02x%s\n", ch, caseless); - } - -#if defined COMPILE_PCRE8 -pcre_printint((pcre *)re, stdout, TRUE); -#elif defined COMPILE_PCRE16 -pcre16_printint((pcre *)re, stdout, TRUE); -#elif defined COMPILE_PCRE32 -pcre32_printint((pcre *)re, stdout, TRUE); -#endif - -/* This check is done here in the debugging case so that the code that -was compiled can be seen. */ - -if (code - codestart > length) - { - (PUBL(free))(re); - *errorptr = find_error_text(ERR23); - *erroroffset = ptr - (pcre_uchar *)pattern; - if (errorcodeptr != NULL) *errorcodeptr = ERR23; - return NULL; - } -#endif /* PCRE_DEBUG */ - -/* Check for a pattern than can match an empty string, so that this information -can be provided to applications. */ - -do - { - if (could_be_empty_branch(codestart, code, utf, cd, NULL)) - { - re->flags |= PCRE_MATCH_EMPTY; - break; - } - codestart += GET(codestart, 1); - } -while (*codestart == OP_ALT); - -#if defined COMPILE_PCRE8 -return (pcre *)re; -#elif defined COMPILE_PCRE16 -return (pcre16 *)re; -#elif defined COMPILE_PCRE32 -return (pcre32 *)re; -#endif -} - -/* End of pcre_compile.c */ diff --git a/src/third_party/pcre-8.39/pcre_config.c b/src/third_party/pcre-8.39/pcre_config.c deleted file mode 100644 index 1cbdd9c960c..00000000000 --- a/src/third_party/pcre-8.39/pcre_config.c +++ /dev/null @@ -1,190 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This module contains the external function pcre_config(). */ - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -/* Keep the original link size. */ -static int real_link_size = LINK_SIZE; - -#include "pcre_internal.h" - - -/************************************************* -* Return info about what features are configured * -*************************************************/ - -/* This function has an extensible interface so that additional items can be -added compatibly. - -Arguments: - what what information is required - where where to put the information - -Returns: 0 if data returned, negative on error -*/ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre_config(int what, void *where) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre16_config(int what, void *where) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre32_config(int what, void *where) -#endif -{ -switch (what) - { - case PCRE_CONFIG_UTF8: -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - *((int *)where) = 0; - return PCRE_ERROR_BADOPTION; -#else -#if defined SUPPORT_UTF - *((int *)where) = 1; -#else - *((int *)where) = 0; -#endif - break; -#endif - - case PCRE_CONFIG_UTF16: -#if defined COMPILE_PCRE8 || defined COMPILE_PCRE32 - *((int *)where) = 0; - return PCRE_ERROR_BADOPTION; -#else -#if defined SUPPORT_UTF - *((int *)where) = 1; -#else - *((int *)where) = 0; -#endif - break; -#endif - - case PCRE_CONFIG_UTF32: -#if defined COMPILE_PCRE8 || defined COMPILE_PCRE16 - *((int *)where) = 0; - return PCRE_ERROR_BADOPTION; -#else -#if defined SUPPORT_UTF - *((int *)where) = 1; -#else - *((int *)where) = 0; -#endif - break; -#endif - - case PCRE_CONFIG_UNICODE_PROPERTIES: -#ifdef SUPPORT_UCP - *((int *)where) = 1; -#else - *((int *)where) = 0; -#endif - break; - - case PCRE_CONFIG_JIT: -#ifdef SUPPORT_JIT - *((int *)where) = 1; -#else - *((int *)where) = 0; -#endif - break; - - case PCRE_CONFIG_JITTARGET: -#ifdef SUPPORT_JIT - *((const char **)where) = PRIV(jit_get_target)(); -#else - *((const char **)where) = NULL; -#endif - break; - - case PCRE_CONFIG_NEWLINE: - *((int *)where) = NEWLINE; - break; - - case PCRE_CONFIG_BSR: -#ifdef BSR_ANYCRLF - *((int *)where) = 1; -#else - *((int *)where) = 0; -#endif - break; - - case PCRE_CONFIG_LINK_SIZE: - *((int *)where) = real_link_size; - break; - - case PCRE_CONFIG_POSIX_MALLOC_THRESHOLD: - *((int *)where) = POSIX_MALLOC_THRESHOLD; - break; - - case PCRE_CONFIG_PARENS_LIMIT: - *((unsigned long int *)where) = PARENS_NEST_LIMIT; - break; - - case PCRE_CONFIG_MATCH_LIMIT: - *((unsigned long int *)where) = MATCH_LIMIT; - break; - - case PCRE_CONFIG_MATCH_LIMIT_RECURSION: - *((unsigned long int *)where) = MATCH_LIMIT_RECURSION; - break; - - case PCRE_CONFIG_STACKRECURSE: -#ifdef NO_RECURSE - *((int *)where) = 0; -#else - *((int *)where) = 1; -#endif - break; - - default: return PCRE_ERROR_BADOPTION; - } - -return 0; -} - -/* End of pcre_config.c */ diff --git a/src/third_party/pcre-8.39/pcre_dfa_exec.c b/src/third_party/pcre-8.39/pcre_dfa_exec.c deleted file mode 100644 index 170ce6a0016..00000000000 --- a/src/third_party/pcre-8.39/pcre_dfa_exec.c +++ /dev/null @@ -1,3674 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language (but see -below for why this module is different). - - Written by Philip Hazel - Copyright (c) 1997-2014 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* This module contains the external function pcre_dfa_exec(), which is an -alternative matching function that uses a sort of DFA algorithm (not a true -FSM). This is NOT Perl-compatible, but it has advantages in certain -applications. */ - - -/* NOTE ABOUT PERFORMANCE: A user of this function sent some code that improved -the performance of his patterns greatly. I could not use it as it stood, as it -was not thread safe, and made assumptions about pattern sizes. Also, it caused -test 7 to loop, and test 9 to crash with a segfault. - -The issue is the check for duplicate states, which is done by a simple linear -search up the state list. (Grep for "duplicate" below to find the code.) For -many patterns, there will never be many states active at one time, so a simple -linear search is fine. In patterns that have many active states, it might be a -bottleneck. The suggested code used an indexing scheme to remember which states -had previously been used for each character, and avoided the linear search when -it knew there was no chance of a duplicate. This was implemented when adding -states to the state lists. - -I wrote some thread-safe, not-limited code to try something similar at the time -of checking for duplicates (instead of when adding states), using index vectors -on the stack. It did give a 13% improvement with one specially constructed -pattern for certain subject strings, but on other strings and on many of the -simpler patterns in the test suite it did worse. The major problem, I think, -was the extra time to initialize the index. This had to be done for each call -of internal_dfa_exec(). (The supplied patch used a static vector, initialized -only once - I suspect this was the cause of the problems with the tests.) - -Overall, I concluded that the gains in some cases did not outweigh the losses -in others, so I abandoned this code. */ - - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#define NLBLOCK md /* Block containing newline information */ -#define PSSTART start_subject /* Field containing processed string start */ -#define PSEND end_subject /* Field containing processed string end */ - -#include "pcre_internal.h" - - -/* For use to indent debugging output */ - -#define SP " " - - -/************************************************* -* Code parameters and static tables * -*************************************************/ - -/* These are offsets that are used to turn the OP_TYPESTAR and friends opcodes -into others, under special conditions. A gap of 20 between the blocks should be -enough. The resulting opcodes don't have to be less than 256 because they are -never stored, so we push them well clear of the normal opcodes. */ - -#define OP_PROP_EXTRA 300 -#define OP_EXTUNI_EXTRA 320 -#define OP_ANYNL_EXTRA 340 -#define OP_HSPACE_EXTRA 360 -#define OP_VSPACE_EXTRA 380 - - -/* This table identifies those opcodes that are followed immediately by a -character that is to be tested in some way. This makes it possible to -centralize the loading of these characters. In the case of Type * etc, the -"character" is the opcode for \D, \d, \S, \s, \W, or \w, which will always be a -small value. Non-zero values in the table are the offsets from the opcode where -the character is to be found. ***NOTE*** If the start of this table is -modified, the three tables that follow must also be modified. */ - -static const pcre_uint8 coptable[] = { - 0, /* End */ - 0, 0, 0, 0, 0, /* \A, \G, \K, \B, \b */ - 0, 0, 0, 0, 0, 0, /* \D, \d, \S, \s, \W, \w */ - 0, 0, 0, /* Any, AllAny, Anybyte */ - 0, 0, /* \P, \p */ - 0, 0, 0, 0, 0, /* \R, \H, \h, \V, \v */ - 0, /* \X */ - 0, 0, 0, 0, 0, 0, /* \Z, \z, $, $M, ^, ^M */ - 1, /* Char */ - 1, /* Chari */ - 1, /* not */ - 1, /* noti */ - /* Positive single-char repeats */ - 1, 1, 1, 1, 1, 1, /* *, *?, +, +?, ?, ?? */ - 1+IMM2_SIZE, 1+IMM2_SIZE, /* upto, minupto */ - 1+IMM2_SIZE, /* exact */ - 1, 1, 1, 1+IMM2_SIZE, /* *+, ++, ?+, upto+ */ - 1, 1, 1, 1, 1, 1, /* *I, *?I, +I, +?I, ?I, ??I */ - 1+IMM2_SIZE, 1+IMM2_SIZE, /* upto I, minupto I */ - 1+IMM2_SIZE, /* exact I */ - 1, 1, 1, 1+IMM2_SIZE, /* *+I, ++I, ?+I, upto+I */ - /* Negative single-char repeats - only for chars < 256 */ - 1, 1, 1, 1, 1, 1, /* NOT *, *?, +, +?, ?, ?? */ - 1+IMM2_SIZE, 1+IMM2_SIZE, /* NOT upto, minupto */ - 1+IMM2_SIZE, /* NOT exact */ - 1, 1, 1, 1+IMM2_SIZE, /* NOT *+, ++, ?+, upto+ */ - 1, 1, 1, 1, 1, 1, /* NOT *I, *?I, +I, +?I, ?I, ??I */ - 1+IMM2_SIZE, 1+IMM2_SIZE, /* NOT upto I, minupto I */ - 1+IMM2_SIZE, /* NOT exact I */ - 1, 1, 1, 1+IMM2_SIZE, /* NOT *+I, ++I, ?+I, upto+I */ - /* Positive type repeats */ - 1, 1, 1, 1, 1, 1, /* Type *, *?, +, +?, ?, ?? */ - 1+IMM2_SIZE, 1+IMM2_SIZE, /* Type upto, minupto */ - 1+IMM2_SIZE, /* Type exact */ - 1, 1, 1, 1+IMM2_SIZE, /* Type *+, ++, ?+, upto+ */ - /* Character class & ref repeats */ - 0, 0, 0, 0, 0, 0, /* *, *?, +, +?, ?, ?? */ - 0, 0, /* CRRANGE, CRMINRANGE */ - 0, 0, 0, 0, /* Possessive *+, ++, ?+, CRPOSRANGE */ - 0, /* CLASS */ - 0, /* NCLASS */ - 0, /* XCLASS - variable length */ - 0, /* REF */ - 0, /* REFI */ - 0, /* DNREF */ - 0, /* DNREFI */ - 0, /* RECURSE */ - 0, /* CALLOUT */ - 0, /* Alt */ - 0, /* Ket */ - 0, /* KetRmax */ - 0, /* KetRmin */ - 0, /* KetRpos */ - 0, /* Reverse */ - 0, /* Assert */ - 0, /* Assert not */ - 0, /* Assert behind */ - 0, /* Assert behind not */ - 0, 0, /* ONCE, ONCE_NC */ - 0, 0, 0, 0, 0, /* BRA, BRAPOS, CBRA, CBRAPOS, COND */ - 0, 0, 0, 0, 0, /* SBRA, SBRAPOS, SCBRA, SCBRAPOS, SCOND */ - 0, 0, /* CREF, DNCREF */ - 0, 0, /* RREF, DNRREF */ - 0, /* DEF */ - 0, 0, 0, /* BRAZERO, BRAMINZERO, BRAPOSZERO */ - 0, 0, 0, /* MARK, PRUNE, PRUNE_ARG */ - 0, 0, 0, 0, /* SKIP, SKIP_ARG, THEN, THEN_ARG */ - 0, 0, 0, 0, /* COMMIT, FAIL, ACCEPT, ASSERT_ACCEPT */ - 0, 0 /* CLOSE, SKIPZERO */ -}; - -/* This table identifies those opcodes that inspect a character. It is used to -remember the fact that a character could have been inspected when the end of -the subject is reached. ***NOTE*** If the start of this table is modified, the -two tables that follow must also be modified. */ - -static const pcre_uint8 poptable[] = { - 0, /* End */ - 0, 0, 0, 1, 1, /* \A, \G, \K, \B, \b */ - 1, 1, 1, 1, 1, 1, /* \D, \d, \S, \s, \W, \w */ - 1, 1, 1, /* Any, AllAny, Anybyte */ - 1, 1, /* \P, \p */ - 1, 1, 1, 1, 1, /* \R, \H, \h, \V, \v */ - 1, /* \X */ - 0, 0, 0, 0, 0, 0, /* \Z, \z, $, $M, ^, ^M */ - 1, /* Char */ - 1, /* Chari */ - 1, /* not */ - 1, /* noti */ - /* Positive single-char repeats */ - 1, 1, 1, 1, 1, 1, /* *, *?, +, +?, ?, ?? */ - 1, 1, 1, /* upto, minupto, exact */ - 1, 1, 1, 1, /* *+, ++, ?+, upto+ */ - 1, 1, 1, 1, 1, 1, /* *I, *?I, +I, +?I, ?I, ??I */ - 1, 1, 1, /* upto I, minupto I, exact I */ - 1, 1, 1, 1, /* *+I, ++I, ?+I, upto+I */ - /* Negative single-char repeats - only for chars < 256 */ - 1, 1, 1, 1, 1, 1, /* NOT *, *?, +, +?, ?, ?? */ - 1, 1, 1, /* NOT upto, minupto, exact */ - 1, 1, 1, 1, /* NOT *+, ++, ?+, upto+ */ - 1, 1, 1, 1, 1, 1, /* NOT *I, *?I, +I, +?I, ?I, ??I */ - 1, 1, 1, /* NOT upto I, minupto I, exact I */ - 1, 1, 1, 1, /* NOT *+I, ++I, ?+I, upto+I */ - /* Positive type repeats */ - 1, 1, 1, 1, 1, 1, /* Type *, *?, +, +?, ?, ?? */ - 1, 1, 1, /* Type upto, minupto, exact */ - 1, 1, 1, 1, /* Type *+, ++, ?+, upto+ */ - /* Character class & ref repeats */ - 1, 1, 1, 1, 1, 1, /* *, *?, +, +?, ?, ?? */ - 1, 1, /* CRRANGE, CRMINRANGE */ - 1, 1, 1, 1, /* Possessive *+, ++, ?+, CRPOSRANGE */ - 1, /* CLASS */ - 1, /* NCLASS */ - 1, /* XCLASS - variable length */ - 0, /* REF */ - 0, /* REFI */ - 0, /* DNREF */ - 0, /* DNREFI */ - 0, /* RECURSE */ - 0, /* CALLOUT */ - 0, /* Alt */ - 0, /* Ket */ - 0, /* KetRmax */ - 0, /* KetRmin */ - 0, /* KetRpos */ - 0, /* Reverse */ - 0, /* Assert */ - 0, /* Assert not */ - 0, /* Assert behind */ - 0, /* Assert behind not */ - 0, 0, /* ONCE, ONCE_NC */ - 0, 0, 0, 0, 0, /* BRA, BRAPOS, CBRA, CBRAPOS, COND */ - 0, 0, 0, 0, 0, /* SBRA, SBRAPOS, SCBRA, SCBRAPOS, SCOND */ - 0, 0, /* CREF, DNCREF */ - 0, 0, /* RREF, DNRREF */ - 0, /* DEF */ - 0, 0, 0, /* BRAZERO, BRAMINZERO, BRAPOSZERO */ - 0, 0, 0, /* MARK, PRUNE, PRUNE_ARG */ - 0, 0, 0, 0, /* SKIP, SKIP_ARG, THEN, THEN_ARG */ - 0, 0, 0, 0, /* COMMIT, FAIL, ACCEPT, ASSERT_ACCEPT */ - 0, 0 /* CLOSE, SKIPZERO */ -}; - -/* These 2 tables allow for compact code for testing for \D, \d, \S, \s, \W, -and \w */ - -static const pcre_uint8 toptable1[] = { - 0, 0, 0, 0, 0, 0, - ctype_digit, ctype_digit, - ctype_space, ctype_space, - ctype_word, ctype_word, - 0, 0 /* OP_ANY, OP_ALLANY */ -}; - -static const pcre_uint8 toptable2[] = { - 0, 0, 0, 0, 0, 0, - ctype_digit, 0, - ctype_space, 0, - ctype_word, 0, - 1, 1 /* OP_ANY, OP_ALLANY */ -}; - - -/* Structure for holding data about a particular state, which is in effect the -current data for an active path through the match tree. It must consist -entirely of ints because the working vector we are passed, and which we put -these structures in, is a vector of ints. */ - -typedef struct stateblock { - int offset; /* Offset to opcode */ - int count; /* Count for repeats */ - int data; /* Some use extra data */ -} stateblock; - -#define INTS_PER_STATEBLOCK (int)(sizeof(stateblock)/sizeof(int)) - - -#ifdef PCRE_DEBUG -/************************************************* -* Print character string * -*************************************************/ - -/* Character string printing function for debugging. - -Arguments: - p points to string - length number of bytes - f where to print - -Returns: nothing -*/ - -static void -pchars(const pcre_uchar *p, int length, FILE *f) -{ -pcre_uint32 c; -while (length-- > 0) - { - if (isprint(c = *(p++))) - fprintf(f, "%c", c); - else - fprintf(f, "\\x{%02x}", c); - } -} -#endif - - - -/************************************************* -* Execute a Regular Expression - DFA engine * -*************************************************/ - -/* This internal function applies a compiled pattern to a subject string, -starting at a given point, using a DFA engine. This function is called from the -external one, possibly multiple times if the pattern is not anchored. The -function calls itself recursively for some kinds of subpattern. - -Arguments: - md the match_data block with fixed information - this_start_code the opening bracket of this subexpression's code - current_subject where we currently are in the subject string - start_offset start offset in the subject string - offsets vector to contain the matching string offsets - offsetcount size of same - workspace vector of workspace - wscount size of same - rlevel function call recursion level - -Returns: > 0 => number of match offset pairs placed in offsets - = 0 => offsets overflowed; longest matches are present - -1 => failed to match - < -1 => some kind of unexpected problem - -The following macros are used for adding states to the two state vectors (one -for the current character, one for the following character). */ - -#define ADD_ACTIVE(x,y) \ - if (active_count++ < wscount) \ - { \ - next_active_state->offset = (x); \ - next_active_state->count = (y); \ - next_active_state++; \ - DPRINTF(("%.*sADD_ACTIVE(%d,%d)\n", rlevel*2-2, SP, (x), (y))); \ - } \ - else return PCRE_ERROR_DFA_WSSIZE - -#define ADD_ACTIVE_DATA(x,y,z) \ - if (active_count++ < wscount) \ - { \ - next_active_state->offset = (x); \ - next_active_state->count = (y); \ - next_active_state->data = (z); \ - next_active_state++; \ - DPRINTF(("%.*sADD_ACTIVE_DATA(%d,%d,%d)\n", rlevel*2-2, SP, (x), (y), (z))); \ - } \ - else return PCRE_ERROR_DFA_WSSIZE - -#define ADD_NEW(x,y) \ - if (new_count++ < wscount) \ - { \ - next_new_state->offset = (x); \ - next_new_state->count = (y); \ - next_new_state++; \ - DPRINTF(("%.*sADD_NEW(%d,%d)\n", rlevel*2-2, SP, (x), (y))); \ - } \ - else return PCRE_ERROR_DFA_WSSIZE - -#define ADD_NEW_DATA(x,y,z) \ - if (new_count++ < wscount) \ - { \ - next_new_state->offset = (x); \ - next_new_state->count = (y); \ - next_new_state->data = (z); \ - next_new_state++; \ - DPRINTF(("%.*sADD_NEW_DATA(%d,%d,%d) line %d\n", rlevel*2-2, SP, \ - (x), (y), (z), __LINE__)); \ - } \ - else return PCRE_ERROR_DFA_WSSIZE - -/* And now, here is the code */ - -static int -internal_dfa_exec( - dfa_match_data *md, - const pcre_uchar *this_start_code, - const pcre_uchar *current_subject, - int start_offset, - int *offsets, - int offsetcount, - int *workspace, - int wscount, - int rlevel) -{ -stateblock *active_states, *new_states, *temp_states; -stateblock *next_active_state, *next_new_state; - -const pcre_uint8 *ctypes, *lcc, *fcc; -const pcre_uchar *ptr; -const pcre_uchar *end_code, *first_op; - -dfa_recursion_info new_recursive; - -int active_count, new_count, match_count; - -/* Some fields in the md block are frequently referenced, so we load them into -independent variables in the hope that this will perform better. */ - -const pcre_uchar *start_subject = md->start_subject; -const pcre_uchar *end_subject = md->end_subject; -const pcre_uchar *start_code = md->start_code; - -#ifdef SUPPORT_UTF -BOOL utf = (md->poptions & PCRE_UTF8) != 0; -#else -BOOL utf = FALSE; -#endif - -BOOL reset_could_continue = FALSE; - -rlevel++; -offsetcount &= (-2); - -wscount -= 2; -wscount = (wscount - (wscount % (INTS_PER_STATEBLOCK * 2))) / - (2 * INTS_PER_STATEBLOCK); - -DPRINTF(("\n%.*s---------------------\n" - "%.*sCall to internal_dfa_exec f=%d\n", - rlevel*2-2, SP, rlevel*2-2, SP, rlevel)); - -ctypes = md->tables + ctypes_offset; -lcc = md->tables + lcc_offset; -fcc = md->tables + fcc_offset; - -match_count = PCRE_ERROR_NOMATCH; /* A negative number */ - -active_states = (stateblock *)(workspace + 2); -next_new_state = new_states = active_states + wscount; -new_count = 0; - -first_op = this_start_code + 1 + LINK_SIZE + - ((*this_start_code == OP_CBRA || *this_start_code == OP_SCBRA || - *this_start_code == OP_CBRAPOS || *this_start_code == OP_SCBRAPOS) - ? IMM2_SIZE:0); - -/* The first thing in any (sub) pattern is a bracket of some sort. Push all -the alternative states onto the list, and find out where the end is. This -makes is possible to use this function recursively, when we want to stop at a -matching internal ket rather than at the end. - -If the first opcode in the first alternative is OP_REVERSE, we are dealing with -a backward assertion. In that case, we have to find out the maximum amount to -move back, and set up each alternative appropriately. */ - -if (*first_op == OP_REVERSE) - { - int max_back = 0; - int gone_back; - - end_code = this_start_code; - do - { - int back = GET(end_code, 2+LINK_SIZE); - if (back > max_back) max_back = back; - end_code += GET(end_code, 1); - } - while (*end_code == OP_ALT); - - /* If we can't go back the amount required for the longest lookbehind - pattern, go back as far as we can; some alternatives may still be viable. */ - -#ifdef SUPPORT_UTF - /* In character mode we have to step back character by character */ - - if (utf) - { - for (gone_back = 0; gone_back < max_back; gone_back++) - { - if (current_subject <= start_subject) break; - current_subject--; - ACROSSCHAR(current_subject > start_subject, *current_subject, current_subject--); - } - } - else -#endif - - /* In byte-mode we can do this quickly. */ - - { - gone_back = (current_subject - max_back < start_subject)? - (int)(current_subject - start_subject) : max_back; - current_subject -= gone_back; - } - - /* Save the earliest consulted character */ - - if (current_subject < md->start_used_ptr) - md->start_used_ptr = current_subject; - - /* Now we can process the individual branches. */ - - end_code = this_start_code; - do - { - int back = GET(end_code, 2+LINK_SIZE); - if (back <= gone_back) - { - int bstate = (int)(end_code - start_code + 2 + 2*LINK_SIZE); - ADD_NEW_DATA(-bstate, 0, gone_back - back); - } - end_code += GET(end_code, 1); - } - while (*end_code == OP_ALT); - } - -/* This is the code for a "normal" subpattern (not a backward assertion). The -start of a whole pattern is always one of these. If we are at the top level, -we may be asked to restart matching from the same point that we reached for a -previous partial match. We still have to scan through the top-level branches to -find the end state. */ - -else - { - end_code = this_start_code; - - /* Restarting */ - - if (rlevel == 1 && (md->moptions & PCRE_DFA_RESTART) != 0) - { - do { end_code += GET(end_code, 1); } while (*end_code == OP_ALT); - new_count = workspace[1]; - if (!workspace[0]) - memcpy(new_states, active_states, new_count * sizeof(stateblock)); - } - - /* Not restarting */ - - else - { - int length = 1 + LINK_SIZE + - ((*this_start_code == OP_CBRA || *this_start_code == OP_SCBRA || - *this_start_code == OP_CBRAPOS || *this_start_code == OP_SCBRAPOS) - ? IMM2_SIZE:0); - do - { - ADD_NEW((int)(end_code - start_code + length), 0); - end_code += GET(end_code, 1); - length = 1 + LINK_SIZE; - } - while (*end_code == OP_ALT); - } - } - -workspace[0] = 0; /* Bit indicating which vector is current */ - -DPRINTF(("%.*sEnd state = %d\n", rlevel*2-2, SP, (int)(end_code - start_code))); - -/* Loop for scanning the subject */ - -ptr = current_subject; -for (;;) - { - int i, j; - int clen, dlen; - pcre_uint32 c, d; - int forced_fail = 0; - BOOL partial_newline = FALSE; - BOOL could_continue = reset_could_continue; - reset_could_continue = FALSE; - - /* Make the new state list into the active state list and empty the - new state list. */ - - temp_states = active_states; - active_states = new_states; - new_states = temp_states; - active_count = new_count; - new_count = 0; - - workspace[0] ^= 1; /* Remember for the restarting feature */ - workspace[1] = active_count; - -#ifdef PCRE_DEBUG - printf("%.*sNext character: rest of subject = \"", rlevel*2-2, SP); - pchars(ptr, STRLEN_UC(ptr), stdout); - printf("\"\n"); - - printf("%.*sActive states: ", rlevel*2-2, SP); - for (i = 0; i < active_count; i++) - printf("%d/%d ", active_states[i].offset, active_states[i].count); - printf("\n"); -#endif - - /* Set the pointers for adding new states */ - - next_active_state = active_states + active_count; - next_new_state = new_states; - - /* Load the current character from the subject outside the loop, as many - different states may want to look at it, and we assume that at least one - will. */ - - if (ptr < end_subject) - { - clen = 1; /* Number of data items in the character */ -#ifdef SUPPORT_UTF - GETCHARLENTEST(c, ptr, clen); -#else - c = *ptr; -#endif /* SUPPORT_UTF */ - } - else - { - clen = 0; /* This indicates the end of the subject */ - c = NOTACHAR; /* This value should never actually be used */ - } - - /* Scan up the active states and act on each one. The result of an action - may be to add more states to the currently active list (e.g. on hitting a - parenthesis) or it may be to put states on the new list, for considering - when we move the character pointer on. */ - - for (i = 0; i < active_count; i++) - { - stateblock *current_state = active_states + i; - BOOL caseless = FALSE; - const pcre_uchar *code; - int state_offset = current_state->offset; - int codevalue, rrc; - int count; - -#ifdef PCRE_DEBUG - printf ("%.*sProcessing state %d c=", rlevel*2-2, SP, state_offset); - if (clen == 0) printf("EOL\n"); - else if (c > 32 && c < 127) printf("'%c'\n", c); - else printf("0x%02x\n", c); -#endif - - /* A negative offset is a special case meaning "hold off going to this - (negated) state until the number of characters in the data field have - been skipped". If the could_continue flag was passed over from a previous - state, arrange for it to passed on. */ - - if (state_offset < 0) - { - if (current_state->data > 0) - { - DPRINTF(("%.*sSkipping this character\n", rlevel*2-2, SP)); - ADD_NEW_DATA(state_offset, current_state->count, - current_state->data - 1); - if (could_continue) reset_could_continue = TRUE; - continue; - } - else - { - current_state->offset = state_offset = -state_offset; - } - } - - /* Check for a duplicate state with the same count, and skip if found. - See the note at the head of this module about the possibility of improving - performance here. */ - - for (j = 0; j < i; j++) - { - if (active_states[j].offset == state_offset && - active_states[j].count == current_state->count) - { - DPRINTF(("%.*sDuplicate state: skipped\n", rlevel*2-2, SP)); - goto NEXT_ACTIVE_STATE; - } - } - - /* The state offset is the offset to the opcode */ - - code = start_code + state_offset; - codevalue = *code; - - /* If this opcode inspects a character, but we are at the end of the - subject, remember the fact for use when testing for a partial match. */ - - if (clen == 0 && poptable[codevalue] != 0) - could_continue = TRUE; - - /* If this opcode is followed by an inline character, load it. It is - tempting to test for the presence of a subject character here, but that - is wrong, because sometimes zero repetitions of the subject are - permitted. - - We also use this mechanism for opcodes such as OP_TYPEPLUS that take an - argument that is not a data character - but is always one byte long because - the values are small. We have to take special action to deal with \P, \p, - \H, \h, \V, \v and \X in this case. To keep the other cases fast, convert - these ones to new opcodes. */ - - if (coptable[codevalue] > 0) - { - dlen = 1; -#ifdef SUPPORT_UTF - if (utf) { GETCHARLEN(d, (code + coptable[codevalue]), dlen); } else -#endif /* SUPPORT_UTF */ - d = code[coptable[codevalue]]; - if (codevalue >= OP_TYPESTAR) - { - switch(d) - { - case OP_ANYBYTE: return PCRE_ERROR_DFA_UITEM; - case OP_NOTPROP: - case OP_PROP: codevalue += OP_PROP_EXTRA; break; - case OP_ANYNL: codevalue += OP_ANYNL_EXTRA; break; - case OP_EXTUNI: codevalue += OP_EXTUNI_EXTRA; break; - case OP_NOT_HSPACE: - case OP_HSPACE: codevalue += OP_HSPACE_EXTRA; break; - case OP_NOT_VSPACE: - case OP_VSPACE: codevalue += OP_VSPACE_EXTRA; break; - default: break; - } - } - } - else - { - dlen = 0; /* Not strictly necessary, but compilers moan */ - d = NOTACHAR; /* if these variables are not set. */ - } - - - /* Now process the individual opcodes */ - - switch (codevalue) - { -/* ========================================================================== */ - /* These cases are never obeyed. This is a fudge that causes a compile- - time error if the vectors coptable or poptable, which are indexed by - opcode, are not the correct length. It seems to be the only way to do - such a check at compile time, as the sizeof() operator does not work - in the C preprocessor. */ - - case OP_TABLE_LENGTH: - case OP_TABLE_LENGTH + - ((sizeof(coptable) == OP_TABLE_LENGTH) && - (sizeof(poptable) == OP_TABLE_LENGTH)): - break; - -/* ========================================================================== */ - /* Reached a closing bracket. If not at the end of the pattern, carry - on with the next opcode. For repeating opcodes, also add the repeat - state. Note that KETRPOS will always be encountered at the end of the - subpattern, because the possessive subpattern repeats are always handled - using recursive calls. Thus, it never adds any new states. - - At the end of the (sub)pattern, unless we have an empty string and - PCRE_NOTEMPTY is set, or PCRE_NOTEMPTY_ATSTART is set and we are at the - start of the subject, save the match data, shifting up all previous - matches so we always have the longest first. */ - - case OP_KET: - case OP_KETRMIN: - case OP_KETRMAX: - case OP_KETRPOS: - if (code != end_code) - { - ADD_ACTIVE(state_offset + 1 + LINK_SIZE, 0); - if (codevalue != OP_KET) - { - ADD_ACTIVE(state_offset - GET(code, 1), 0); - } - } - else - { - if (ptr > current_subject || - ((md->moptions & PCRE_NOTEMPTY) == 0 && - ((md->moptions & PCRE_NOTEMPTY_ATSTART) == 0 || - current_subject > start_subject + md->start_offset))) - { - if (match_count < 0) match_count = (offsetcount >= 2)? 1 : 0; - else if (match_count > 0 && ++match_count * 2 > offsetcount) - match_count = 0; - count = ((match_count == 0)? offsetcount : match_count * 2) - 2; - if (count > 0) memmove(offsets + 2, offsets, count * sizeof(int)); - if (offsetcount >= 2) - { - offsets[0] = (int)(current_subject - start_subject); - offsets[1] = (int)(ptr - start_subject); - DPRINTF(("%.*sSet matched string = \"%.*s\"\n", rlevel*2-2, SP, - offsets[1] - offsets[0], (char *)current_subject)); - } - if ((md->moptions & PCRE_DFA_SHORTEST) != 0) - { - DPRINTF(("%.*sEnd of internal_dfa_exec %d: returning %d\n" - "%.*s---------------------\n\n", rlevel*2-2, SP, rlevel, - match_count, rlevel*2-2, SP)); - return match_count; - } - } - } - break; - -/* ========================================================================== */ - /* These opcodes add to the current list of states without looking - at the current character. */ - - /*-----------------------------------------------------------------*/ - case OP_ALT: - do { code += GET(code, 1); } while (*code == OP_ALT); - ADD_ACTIVE((int)(code - start_code), 0); - break; - - /*-----------------------------------------------------------------*/ - case OP_BRA: - case OP_SBRA: - do - { - ADD_ACTIVE((int)(code - start_code + 1 + LINK_SIZE), 0); - code += GET(code, 1); - } - while (*code == OP_ALT); - break; - - /*-----------------------------------------------------------------*/ - case OP_CBRA: - case OP_SCBRA: - ADD_ACTIVE((int)(code - start_code + 1 + LINK_SIZE + IMM2_SIZE), 0); - code += GET(code, 1); - while (*code == OP_ALT) - { - ADD_ACTIVE((int)(code - start_code + 1 + LINK_SIZE), 0); - code += GET(code, 1); - } - break; - - /*-----------------------------------------------------------------*/ - case OP_BRAZERO: - case OP_BRAMINZERO: - ADD_ACTIVE(state_offset + 1, 0); - code += 1 + GET(code, 2); - while (*code == OP_ALT) code += GET(code, 1); - ADD_ACTIVE((int)(code - start_code + 1 + LINK_SIZE), 0); - break; - - /*-----------------------------------------------------------------*/ - case OP_SKIPZERO: - code += 1 + GET(code, 2); - while (*code == OP_ALT) code += GET(code, 1); - ADD_ACTIVE((int)(code - start_code + 1 + LINK_SIZE), 0); - break; - - /*-----------------------------------------------------------------*/ - case OP_CIRC: - if (ptr == start_subject && (md->moptions & PCRE_NOTBOL) == 0) - { ADD_ACTIVE(state_offset + 1, 0); } - break; - - /*-----------------------------------------------------------------*/ - case OP_CIRCM: - if ((ptr == start_subject && (md->moptions & PCRE_NOTBOL) == 0) || - (ptr != end_subject && WAS_NEWLINE(ptr))) - { ADD_ACTIVE(state_offset + 1, 0); } - break; - - /*-----------------------------------------------------------------*/ - case OP_EOD: - if (ptr >= end_subject) - { - if ((md->moptions & PCRE_PARTIAL_HARD) != 0) - could_continue = TRUE; - else { ADD_ACTIVE(state_offset + 1, 0); } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_SOD: - if (ptr == start_subject) { ADD_ACTIVE(state_offset + 1, 0); } - break; - - /*-----------------------------------------------------------------*/ - case OP_SOM: - if (ptr == start_subject + start_offset) { ADD_ACTIVE(state_offset + 1, 0); } - break; - - -/* ========================================================================== */ - /* These opcodes inspect the next subject character, and sometimes - the previous one as well, but do not have an argument. The variable - clen contains the length of the current character and is zero if we are - at the end of the subject. */ - - /*-----------------------------------------------------------------*/ - case OP_ANY: - if (clen > 0 && !IS_NEWLINE(ptr)) - { - if (ptr + 1 >= md->end_subject && - (md->moptions & (PCRE_PARTIAL_HARD)) != 0 && - NLBLOCK->nltype == NLTYPE_FIXED && - NLBLOCK->nllen == 2 && - c == NLBLOCK->nl[0]) - { - could_continue = partial_newline = TRUE; - } - else - { - ADD_NEW(state_offset + 1, 0); - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_ALLANY: - if (clen > 0) - { ADD_NEW(state_offset + 1, 0); } - break; - - /*-----------------------------------------------------------------*/ - case OP_EODN: - if (clen == 0 && (md->moptions & PCRE_PARTIAL_HARD) != 0) - could_continue = TRUE; - else if (clen == 0 || (IS_NEWLINE(ptr) && ptr == end_subject - md->nllen)) - { ADD_ACTIVE(state_offset + 1, 0); } - break; - - /*-----------------------------------------------------------------*/ - case OP_DOLL: - if ((md->moptions & PCRE_NOTEOL) == 0) - { - if (clen == 0 && (md->moptions & PCRE_PARTIAL_HARD) != 0) - could_continue = TRUE; - else if (clen == 0 || - ((md->poptions & PCRE_DOLLAR_ENDONLY) == 0 && IS_NEWLINE(ptr) && - (ptr == end_subject - md->nllen) - )) - { ADD_ACTIVE(state_offset + 1, 0); } - else if (ptr + 1 >= md->end_subject && - (md->moptions & (PCRE_PARTIAL_HARD|PCRE_PARTIAL_SOFT)) != 0 && - NLBLOCK->nltype == NLTYPE_FIXED && - NLBLOCK->nllen == 2 && - c == NLBLOCK->nl[0]) - { - if ((md->moptions & PCRE_PARTIAL_HARD) != 0) - { - reset_could_continue = TRUE; - ADD_NEW_DATA(-(state_offset + 1), 0, 1); - } - else could_continue = partial_newline = TRUE; - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_DOLLM: - if ((md->moptions & PCRE_NOTEOL) == 0) - { - if (clen == 0 && (md->moptions & PCRE_PARTIAL_HARD) != 0) - could_continue = TRUE; - else if (clen == 0 || - ((md->poptions & PCRE_DOLLAR_ENDONLY) == 0 && IS_NEWLINE(ptr))) - { ADD_ACTIVE(state_offset + 1, 0); } - else if (ptr + 1 >= md->end_subject && - (md->moptions & (PCRE_PARTIAL_HARD|PCRE_PARTIAL_SOFT)) != 0 && - NLBLOCK->nltype == NLTYPE_FIXED && - NLBLOCK->nllen == 2 && - c == NLBLOCK->nl[0]) - { - if ((md->moptions & PCRE_PARTIAL_HARD) != 0) - { - reset_could_continue = TRUE; - ADD_NEW_DATA(-(state_offset + 1), 0, 1); - } - else could_continue = partial_newline = TRUE; - } - } - else if (IS_NEWLINE(ptr)) - { ADD_ACTIVE(state_offset + 1, 0); } - break; - - /*-----------------------------------------------------------------*/ - - case OP_DIGIT: - case OP_WHITESPACE: - case OP_WORDCHAR: - if (clen > 0 && c < 256 && - ((ctypes[c] & toptable1[codevalue]) ^ toptable2[codevalue]) != 0) - { ADD_NEW(state_offset + 1, 0); } - break; - - /*-----------------------------------------------------------------*/ - case OP_NOT_DIGIT: - case OP_NOT_WHITESPACE: - case OP_NOT_WORDCHAR: - if (clen > 0 && (c >= 256 || - ((ctypes[c] & toptable1[codevalue]) ^ toptable2[codevalue]) != 0)) - { ADD_NEW(state_offset + 1, 0); } - break; - - /*-----------------------------------------------------------------*/ - case OP_WORD_BOUNDARY: - case OP_NOT_WORD_BOUNDARY: - { - int left_word, right_word; - - if (ptr > start_subject) - { - const pcre_uchar *temp = ptr - 1; - if (temp < md->start_used_ptr) md->start_used_ptr = temp; -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (utf) { BACKCHAR(temp); } -#endif - GETCHARTEST(d, temp); -#ifdef SUPPORT_UCP - if ((md->poptions & PCRE_UCP) != 0) - { - if (d == '_') left_word = TRUE; else - { - int cat = UCD_CATEGORY(d); - left_word = (cat == ucp_L || cat == ucp_N); - } - } - else -#endif - left_word = d < 256 && (ctypes[d] & ctype_word) != 0; - } - else left_word = FALSE; - - if (clen > 0) - { -#ifdef SUPPORT_UCP - if ((md->poptions & PCRE_UCP) != 0) - { - if (c == '_') right_word = TRUE; else - { - int cat = UCD_CATEGORY(c); - right_word = (cat == ucp_L || cat == ucp_N); - } - } - else -#endif - right_word = c < 256 && (ctypes[c] & ctype_word) != 0; - } - else right_word = FALSE; - - if ((left_word == right_word) == (codevalue == OP_NOT_WORD_BOUNDARY)) - { ADD_ACTIVE(state_offset + 1, 0); } - } - break; - - - /*-----------------------------------------------------------------*/ - /* Check the next character by Unicode property. We will get here only - if the support is in the binary; otherwise a compile-time error occurs. - */ - -#ifdef SUPPORT_UCP - case OP_PROP: - case OP_NOTPROP: - if (clen > 0) - { - BOOL OK; - const pcre_uint32 *cp; - const ucd_record * prop = GET_UCD(c); - switch(code[1]) - { - case PT_ANY: - OK = TRUE; - break; - - case PT_LAMP: - OK = prop->chartype == ucp_Lu || prop->chartype == ucp_Ll || - prop->chartype == ucp_Lt; - break; - - case PT_GC: - OK = PRIV(ucp_gentype)[prop->chartype] == code[2]; - break; - - case PT_PC: - OK = prop->chartype == code[2]; - break; - - case PT_SC: - OK = prop->script == code[2]; - break; - - /* These are specials for combination cases. */ - - case PT_ALNUM: - OK = PRIV(ucp_gentype)[prop->chartype] == ucp_L || - PRIV(ucp_gentype)[prop->chartype] == ucp_N; - break; - - /* Perl space used to exclude VT, but from Perl 5.18 it is included, - which means that Perl space and POSIX space are now identical. PCRE - was changed at release 8.34. */ - - case PT_SPACE: /* Perl space */ - case PT_PXSPACE: /* POSIX space */ - switch(c) - { - HSPACE_CASES: - VSPACE_CASES: - OK = TRUE; - break; - - default: - OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z; - break; - } - break; - - case PT_WORD: - OK = PRIV(ucp_gentype)[prop->chartype] == ucp_L || - PRIV(ucp_gentype)[prop->chartype] == ucp_N || - c == CHAR_UNDERSCORE; - break; - - case PT_CLIST: - cp = PRIV(ucd_caseless_sets) + code[2]; - for (;;) - { - if (c < *cp) { OK = FALSE; break; } - if (c == *cp++) { OK = TRUE; break; } - } - break; - - case PT_UCNC: - OK = c == CHAR_DOLLAR_SIGN || c == CHAR_COMMERCIAL_AT || - c == CHAR_GRAVE_ACCENT || (c >= 0xa0 && c <= 0xd7ff) || - c >= 0xe000; - break; - - /* Should never occur, but keep compilers from grumbling. */ - - default: - OK = codevalue != OP_PROP; - break; - } - - if (OK == (codevalue == OP_PROP)) { ADD_NEW(state_offset + 3, 0); } - } - break; -#endif - - - -/* ========================================================================== */ - /* These opcodes likewise inspect the subject character, but have an - argument that is not a data character. It is one of these opcodes: - OP_ANY, OP_ALLANY, OP_DIGIT, OP_NOT_DIGIT, OP_WHITESPACE, OP_NOT_SPACE, - OP_WORDCHAR, OP_NOT_WORDCHAR. The value is loaded into d. */ - - case OP_TYPEPLUS: - case OP_TYPEMINPLUS: - case OP_TYPEPOSPLUS: - count = current_state->count; /* Already matched */ - if (count > 0) { ADD_ACTIVE(state_offset + 2, 0); } - if (clen > 0) - { - if (d == OP_ANY && ptr + 1 >= md->end_subject && - (md->moptions & (PCRE_PARTIAL_HARD)) != 0 && - NLBLOCK->nltype == NLTYPE_FIXED && - NLBLOCK->nllen == 2 && - c == NLBLOCK->nl[0]) - { - could_continue = partial_newline = TRUE; - } - else if ((c >= 256 && d != OP_DIGIT && d != OP_WHITESPACE && d != OP_WORDCHAR) || - (c < 256 && - (d != OP_ANY || !IS_NEWLINE(ptr)) && - ((ctypes[c] & toptable1[d]) ^ toptable2[d]) != 0)) - { - if (count > 0 && codevalue == OP_TYPEPOSPLUS) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - count++; - ADD_NEW(state_offset, count); - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_TYPEQUERY: - case OP_TYPEMINQUERY: - case OP_TYPEPOSQUERY: - ADD_ACTIVE(state_offset + 2, 0); - if (clen > 0) - { - if (d == OP_ANY && ptr + 1 >= md->end_subject && - (md->moptions & (PCRE_PARTIAL_HARD)) != 0 && - NLBLOCK->nltype == NLTYPE_FIXED && - NLBLOCK->nllen == 2 && - c == NLBLOCK->nl[0]) - { - could_continue = partial_newline = TRUE; - } - else if ((c >= 256 && d != OP_DIGIT && d != OP_WHITESPACE && d != OP_WORDCHAR) || - (c < 256 && - (d != OP_ANY || !IS_NEWLINE(ptr)) && - ((ctypes[c] & toptable1[d]) ^ toptable2[d]) != 0)) - { - if (codevalue == OP_TYPEPOSQUERY) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - ADD_NEW(state_offset + 2, 0); - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_TYPESTAR: - case OP_TYPEMINSTAR: - case OP_TYPEPOSSTAR: - ADD_ACTIVE(state_offset + 2, 0); - if (clen > 0) - { - if (d == OP_ANY && ptr + 1 >= md->end_subject && - (md->moptions & (PCRE_PARTIAL_HARD)) != 0 && - NLBLOCK->nltype == NLTYPE_FIXED && - NLBLOCK->nllen == 2 && - c == NLBLOCK->nl[0]) - { - could_continue = partial_newline = TRUE; - } - else if ((c >= 256 && d != OP_DIGIT && d != OP_WHITESPACE && d != OP_WORDCHAR) || - (c < 256 && - (d != OP_ANY || !IS_NEWLINE(ptr)) && - ((ctypes[c] & toptable1[d]) ^ toptable2[d]) != 0)) - { - if (codevalue == OP_TYPEPOSSTAR) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - ADD_NEW(state_offset, 0); - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_TYPEEXACT: - count = current_state->count; /* Number already matched */ - if (clen > 0) - { - if (d == OP_ANY && ptr + 1 >= md->end_subject && - (md->moptions & (PCRE_PARTIAL_HARD)) != 0 && - NLBLOCK->nltype == NLTYPE_FIXED && - NLBLOCK->nllen == 2 && - c == NLBLOCK->nl[0]) - { - could_continue = partial_newline = TRUE; - } - else if ((c >= 256 && d != OP_DIGIT && d != OP_WHITESPACE && d != OP_WORDCHAR) || - (c < 256 && - (d != OP_ANY || !IS_NEWLINE(ptr)) && - ((ctypes[c] & toptable1[d]) ^ toptable2[d]) != 0)) - { - if (++count >= (int)GET2(code, 1)) - { ADD_NEW(state_offset + 1 + IMM2_SIZE + 1, 0); } - else - { ADD_NEW(state_offset, count); } - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_TYPEUPTO: - case OP_TYPEMINUPTO: - case OP_TYPEPOSUPTO: - ADD_ACTIVE(state_offset + 2 + IMM2_SIZE, 0); - count = current_state->count; /* Number already matched */ - if (clen > 0) - { - if (d == OP_ANY && ptr + 1 >= md->end_subject && - (md->moptions & (PCRE_PARTIAL_HARD)) != 0 && - NLBLOCK->nltype == NLTYPE_FIXED && - NLBLOCK->nllen == 2 && - c == NLBLOCK->nl[0]) - { - could_continue = partial_newline = TRUE; - } - else if ((c >= 256 && d != OP_DIGIT && d != OP_WHITESPACE && d != OP_WORDCHAR) || - (c < 256 && - (d != OP_ANY || !IS_NEWLINE(ptr)) && - ((ctypes[c] & toptable1[d]) ^ toptable2[d]) != 0)) - { - if (codevalue == OP_TYPEPOSUPTO) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - if (++count >= (int)GET2(code, 1)) - { ADD_NEW(state_offset + 2 + IMM2_SIZE, 0); } - else - { ADD_NEW(state_offset, count); } - } - } - break; - -/* ========================================================================== */ - /* These are virtual opcodes that are used when something like - OP_TYPEPLUS has OP_PROP, OP_NOTPROP, OP_ANYNL, or OP_EXTUNI as its - argument. It keeps the code above fast for the other cases. The argument - is in the d variable. */ - -#ifdef SUPPORT_UCP - case OP_PROP_EXTRA + OP_TYPEPLUS: - case OP_PROP_EXTRA + OP_TYPEMINPLUS: - case OP_PROP_EXTRA + OP_TYPEPOSPLUS: - count = current_state->count; /* Already matched */ - if (count > 0) { ADD_ACTIVE(state_offset + 4, 0); } - if (clen > 0) - { - BOOL OK; - const pcre_uint32 *cp; - const ucd_record * prop = GET_UCD(c); - switch(code[2]) - { - case PT_ANY: - OK = TRUE; - break; - - case PT_LAMP: - OK = prop->chartype == ucp_Lu || prop->chartype == ucp_Ll || - prop->chartype == ucp_Lt; - break; - - case PT_GC: - OK = PRIV(ucp_gentype)[prop->chartype] == code[3]; - break; - - case PT_PC: - OK = prop->chartype == code[3]; - break; - - case PT_SC: - OK = prop->script == code[3]; - break; - - /* These are specials for combination cases. */ - - case PT_ALNUM: - OK = PRIV(ucp_gentype)[prop->chartype] == ucp_L || - PRIV(ucp_gentype)[prop->chartype] == ucp_N; - break; - - /* Perl space used to exclude VT, but from Perl 5.18 it is included, - which means that Perl space and POSIX space are now identical. PCRE - was changed at release 8.34. */ - - case PT_SPACE: /* Perl space */ - case PT_PXSPACE: /* POSIX space */ - switch(c) - { - HSPACE_CASES: - VSPACE_CASES: - OK = TRUE; - break; - - default: - OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z; - break; - } - break; - - case PT_WORD: - OK = PRIV(ucp_gentype)[prop->chartype] == ucp_L || - PRIV(ucp_gentype)[prop->chartype] == ucp_N || - c == CHAR_UNDERSCORE; - break; - - case PT_CLIST: - cp = PRIV(ucd_caseless_sets) + code[3]; - for (;;) - { - if (c < *cp) { OK = FALSE; break; } - if (c == *cp++) { OK = TRUE; break; } - } - break; - - case PT_UCNC: - OK = c == CHAR_DOLLAR_SIGN || c == CHAR_COMMERCIAL_AT || - c == CHAR_GRAVE_ACCENT || (c >= 0xa0 && c <= 0xd7ff) || - c >= 0xe000; - break; - - /* Should never occur, but keep compilers from grumbling. */ - - default: - OK = codevalue != OP_PROP; - break; - } - - if (OK == (d == OP_PROP)) - { - if (count > 0 && codevalue == OP_PROP_EXTRA + OP_TYPEPOSPLUS) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - count++; - ADD_NEW(state_offset, count); - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_EXTUNI_EXTRA + OP_TYPEPLUS: - case OP_EXTUNI_EXTRA + OP_TYPEMINPLUS: - case OP_EXTUNI_EXTRA + OP_TYPEPOSPLUS: - count = current_state->count; /* Already matched */ - if (count > 0) { ADD_ACTIVE(state_offset + 2, 0); } - if (clen > 0) - { - int lgb, rgb; - const pcre_uchar *nptr = ptr + clen; - int ncount = 0; - if (count > 0 && codevalue == OP_EXTUNI_EXTRA + OP_TYPEPOSPLUS) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - lgb = UCD_GRAPHBREAK(c); - while (nptr < end_subject) - { - dlen = 1; - if (!utf) d = *nptr; else { GETCHARLEN(d, nptr, dlen); } - rgb = UCD_GRAPHBREAK(d); - if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; - ncount++; - lgb = rgb; - nptr += dlen; - } - count++; - ADD_NEW_DATA(-state_offset, count, ncount); - } - break; -#endif - - /*-----------------------------------------------------------------*/ - case OP_ANYNL_EXTRA + OP_TYPEPLUS: - case OP_ANYNL_EXTRA + OP_TYPEMINPLUS: - case OP_ANYNL_EXTRA + OP_TYPEPOSPLUS: - count = current_state->count; /* Already matched */ - if (count > 0) { ADD_ACTIVE(state_offset + 2, 0); } - if (clen > 0) - { - int ncount = 0; - switch (c) - { - case CHAR_VT: - case CHAR_FF: - case CHAR_NEL: -#ifndef EBCDIC - case 0x2028: - case 0x2029: -#endif /* Not EBCDIC */ - if ((md->moptions & PCRE_BSR_ANYCRLF) != 0) break; - goto ANYNL01; - - case CHAR_CR: - if (ptr + 1 < end_subject && UCHAR21TEST(ptr + 1) == CHAR_LF) ncount = 1; - /* Fall through */ - - ANYNL01: - case CHAR_LF: - if (count > 0 && codevalue == OP_ANYNL_EXTRA + OP_TYPEPOSPLUS) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - count++; - ADD_NEW_DATA(-state_offset, count, ncount); - break; - - default: - break; - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_VSPACE_EXTRA + OP_TYPEPLUS: - case OP_VSPACE_EXTRA + OP_TYPEMINPLUS: - case OP_VSPACE_EXTRA + OP_TYPEPOSPLUS: - count = current_state->count; /* Already matched */ - if (count > 0) { ADD_ACTIVE(state_offset + 2, 0); } - if (clen > 0) - { - BOOL OK; - switch (c) - { - VSPACE_CASES: - OK = TRUE; - break; - - default: - OK = FALSE; - break; - } - - if (OK == (d == OP_VSPACE)) - { - if (count > 0 && codevalue == OP_VSPACE_EXTRA + OP_TYPEPOSPLUS) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - count++; - ADD_NEW_DATA(-state_offset, count, 0); - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_HSPACE_EXTRA + OP_TYPEPLUS: - case OP_HSPACE_EXTRA + OP_TYPEMINPLUS: - case OP_HSPACE_EXTRA + OP_TYPEPOSPLUS: - count = current_state->count; /* Already matched */ - if (count > 0) { ADD_ACTIVE(state_offset + 2, 0); } - if (clen > 0) - { - BOOL OK; - switch (c) - { - HSPACE_CASES: - OK = TRUE; - break; - - default: - OK = FALSE; - break; - } - - if (OK == (d == OP_HSPACE)) - { - if (count > 0 && codevalue == OP_HSPACE_EXTRA + OP_TYPEPOSPLUS) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - count++; - ADD_NEW_DATA(-state_offset, count, 0); - } - } - break; - - /*-----------------------------------------------------------------*/ -#ifdef SUPPORT_UCP - case OP_PROP_EXTRA + OP_TYPEQUERY: - case OP_PROP_EXTRA + OP_TYPEMINQUERY: - case OP_PROP_EXTRA + OP_TYPEPOSQUERY: - count = 4; - goto QS1; - - case OP_PROP_EXTRA + OP_TYPESTAR: - case OP_PROP_EXTRA + OP_TYPEMINSTAR: - case OP_PROP_EXTRA + OP_TYPEPOSSTAR: - count = 0; - - QS1: - - ADD_ACTIVE(state_offset + 4, 0); - if (clen > 0) - { - BOOL OK; - const pcre_uint32 *cp; - const ucd_record * prop = GET_UCD(c); - switch(code[2]) - { - case PT_ANY: - OK = TRUE; - break; - - case PT_LAMP: - OK = prop->chartype == ucp_Lu || prop->chartype == ucp_Ll || - prop->chartype == ucp_Lt; - break; - - case PT_GC: - OK = PRIV(ucp_gentype)[prop->chartype] == code[3]; - break; - - case PT_PC: - OK = prop->chartype == code[3]; - break; - - case PT_SC: - OK = prop->script == code[3]; - break; - - /* These are specials for combination cases. */ - - case PT_ALNUM: - OK = PRIV(ucp_gentype)[prop->chartype] == ucp_L || - PRIV(ucp_gentype)[prop->chartype] == ucp_N; - break; - - /* Perl space used to exclude VT, but from Perl 5.18 it is included, - which means that Perl space and POSIX space are now identical. PCRE - was changed at release 8.34. */ - - case PT_SPACE: /* Perl space */ - case PT_PXSPACE: /* POSIX space */ - switch(c) - { - HSPACE_CASES: - VSPACE_CASES: - OK = TRUE; - break; - - default: - OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z; - break; - } - break; - - case PT_WORD: - OK = PRIV(ucp_gentype)[prop->chartype] == ucp_L || - PRIV(ucp_gentype)[prop->chartype] == ucp_N || - c == CHAR_UNDERSCORE; - break; - - case PT_CLIST: - cp = PRIV(ucd_caseless_sets) + code[3]; - for (;;) - { - if (c < *cp) { OK = FALSE; break; } - if (c == *cp++) { OK = TRUE; break; } - } - break; - - case PT_UCNC: - OK = c == CHAR_DOLLAR_SIGN || c == CHAR_COMMERCIAL_AT || - c == CHAR_GRAVE_ACCENT || (c >= 0xa0 && c <= 0xd7ff) || - c >= 0xe000; - break; - - /* Should never occur, but keep compilers from grumbling. */ - - default: - OK = codevalue != OP_PROP; - break; - } - - if (OK == (d == OP_PROP)) - { - if (codevalue == OP_PROP_EXTRA + OP_TYPEPOSSTAR || - codevalue == OP_PROP_EXTRA + OP_TYPEPOSQUERY) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - ADD_NEW(state_offset + count, 0); - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_EXTUNI_EXTRA + OP_TYPEQUERY: - case OP_EXTUNI_EXTRA + OP_TYPEMINQUERY: - case OP_EXTUNI_EXTRA + OP_TYPEPOSQUERY: - count = 2; - goto QS2; - - case OP_EXTUNI_EXTRA + OP_TYPESTAR: - case OP_EXTUNI_EXTRA + OP_TYPEMINSTAR: - case OP_EXTUNI_EXTRA + OP_TYPEPOSSTAR: - count = 0; - - QS2: - - ADD_ACTIVE(state_offset + 2, 0); - if (clen > 0) - { - int lgb, rgb; - const pcre_uchar *nptr = ptr + clen; - int ncount = 0; - if (codevalue == OP_EXTUNI_EXTRA + OP_TYPEPOSSTAR || - codevalue == OP_EXTUNI_EXTRA + OP_TYPEPOSQUERY) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - lgb = UCD_GRAPHBREAK(c); - while (nptr < end_subject) - { - dlen = 1; - if (!utf) d = *nptr; else { GETCHARLEN(d, nptr, dlen); } - rgb = UCD_GRAPHBREAK(d); - if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; - ncount++; - lgb = rgb; - nptr += dlen; - } - ADD_NEW_DATA(-(state_offset + count), 0, ncount); - } - break; -#endif - - /*-----------------------------------------------------------------*/ - case OP_ANYNL_EXTRA + OP_TYPEQUERY: - case OP_ANYNL_EXTRA + OP_TYPEMINQUERY: - case OP_ANYNL_EXTRA + OP_TYPEPOSQUERY: - count = 2; - goto QS3; - - case OP_ANYNL_EXTRA + OP_TYPESTAR: - case OP_ANYNL_EXTRA + OP_TYPEMINSTAR: - case OP_ANYNL_EXTRA + OP_TYPEPOSSTAR: - count = 0; - - QS3: - ADD_ACTIVE(state_offset + 2, 0); - if (clen > 0) - { - int ncount = 0; - switch (c) - { - case CHAR_VT: - case CHAR_FF: - case CHAR_NEL: -#ifndef EBCDIC - case 0x2028: - case 0x2029: -#endif /* Not EBCDIC */ - if ((md->moptions & PCRE_BSR_ANYCRLF) != 0) break; - goto ANYNL02; - - case CHAR_CR: - if (ptr + 1 < end_subject && UCHAR21TEST(ptr + 1) == CHAR_LF) ncount = 1; - /* Fall through */ - - ANYNL02: - case CHAR_LF: - if (codevalue == OP_ANYNL_EXTRA + OP_TYPEPOSSTAR || - codevalue == OP_ANYNL_EXTRA + OP_TYPEPOSQUERY) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - ADD_NEW_DATA(-(state_offset + (int)count), 0, ncount); - break; - - default: - break; - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_VSPACE_EXTRA + OP_TYPEQUERY: - case OP_VSPACE_EXTRA + OP_TYPEMINQUERY: - case OP_VSPACE_EXTRA + OP_TYPEPOSQUERY: - count = 2; - goto QS4; - - case OP_VSPACE_EXTRA + OP_TYPESTAR: - case OP_VSPACE_EXTRA + OP_TYPEMINSTAR: - case OP_VSPACE_EXTRA + OP_TYPEPOSSTAR: - count = 0; - - QS4: - ADD_ACTIVE(state_offset + 2, 0); - if (clen > 0) - { - BOOL OK; - switch (c) - { - VSPACE_CASES: - OK = TRUE; - break; - - default: - OK = FALSE; - break; - } - if (OK == (d == OP_VSPACE)) - { - if (codevalue == OP_VSPACE_EXTRA + OP_TYPEPOSSTAR || - codevalue == OP_VSPACE_EXTRA + OP_TYPEPOSQUERY) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - ADD_NEW_DATA(-(state_offset + (int)count), 0, 0); - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_HSPACE_EXTRA + OP_TYPEQUERY: - case OP_HSPACE_EXTRA + OP_TYPEMINQUERY: - case OP_HSPACE_EXTRA + OP_TYPEPOSQUERY: - count = 2; - goto QS5; - - case OP_HSPACE_EXTRA + OP_TYPESTAR: - case OP_HSPACE_EXTRA + OP_TYPEMINSTAR: - case OP_HSPACE_EXTRA + OP_TYPEPOSSTAR: - count = 0; - - QS5: - ADD_ACTIVE(state_offset + 2, 0); - if (clen > 0) - { - BOOL OK; - switch (c) - { - HSPACE_CASES: - OK = TRUE; - break; - - default: - OK = FALSE; - break; - } - - if (OK == (d == OP_HSPACE)) - { - if (codevalue == OP_HSPACE_EXTRA + OP_TYPEPOSSTAR || - codevalue == OP_HSPACE_EXTRA + OP_TYPEPOSQUERY) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - ADD_NEW_DATA(-(state_offset + (int)count), 0, 0); - } - } - break; - - /*-----------------------------------------------------------------*/ -#ifdef SUPPORT_UCP - case OP_PROP_EXTRA + OP_TYPEEXACT: - case OP_PROP_EXTRA + OP_TYPEUPTO: - case OP_PROP_EXTRA + OP_TYPEMINUPTO: - case OP_PROP_EXTRA + OP_TYPEPOSUPTO: - if (codevalue != OP_PROP_EXTRA + OP_TYPEEXACT) - { ADD_ACTIVE(state_offset + 1 + IMM2_SIZE + 3, 0); } - count = current_state->count; /* Number already matched */ - if (clen > 0) - { - BOOL OK; - const pcre_uint32 *cp; - const ucd_record * prop = GET_UCD(c); - switch(code[1 + IMM2_SIZE + 1]) - { - case PT_ANY: - OK = TRUE; - break; - - case PT_LAMP: - OK = prop->chartype == ucp_Lu || prop->chartype == ucp_Ll || - prop->chartype == ucp_Lt; - break; - - case PT_GC: - OK = PRIV(ucp_gentype)[prop->chartype] == code[1 + IMM2_SIZE + 2]; - break; - - case PT_PC: - OK = prop->chartype == code[1 + IMM2_SIZE + 2]; - break; - - case PT_SC: - OK = prop->script == code[1 + IMM2_SIZE + 2]; - break; - - /* These are specials for combination cases. */ - - case PT_ALNUM: - OK = PRIV(ucp_gentype)[prop->chartype] == ucp_L || - PRIV(ucp_gentype)[prop->chartype] == ucp_N; - break; - - /* Perl space used to exclude VT, but from Perl 5.18 it is included, - which means that Perl space and POSIX space are now identical. PCRE - was changed at release 8.34. */ - - case PT_SPACE: /* Perl space */ - case PT_PXSPACE: /* POSIX space */ - switch(c) - { - HSPACE_CASES: - VSPACE_CASES: - OK = TRUE; - break; - - default: - OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z; - break; - } - break; - - case PT_WORD: - OK = PRIV(ucp_gentype)[prop->chartype] == ucp_L || - PRIV(ucp_gentype)[prop->chartype] == ucp_N || - c == CHAR_UNDERSCORE; - break; - - case PT_CLIST: - cp = PRIV(ucd_caseless_sets) + code[1 + IMM2_SIZE + 2]; - for (;;) - { - if (c < *cp) { OK = FALSE; break; } - if (c == *cp++) { OK = TRUE; break; } - } - break; - - case PT_UCNC: - OK = c == CHAR_DOLLAR_SIGN || c == CHAR_COMMERCIAL_AT || - c == CHAR_GRAVE_ACCENT || (c >= 0xa0 && c <= 0xd7ff) || - c >= 0xe000; - break; - - /* Should never occur, but keep compilers from grumbling. */ - - default: - OK = codevalue != OP_PROP; - break; - } - - if (OK == (d == OP_PROP)) - { - if (codevalue == OP_PROP_EXTRA + OP_TYPEPOSUPTO) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - if (++count >= (int)GET2(code, 1)) - { ADD_NEW(state_offset + 1 + IMM2_SIZE + 3, 0); } - else - { ADD_NEW(state_offset, count); } - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_EXTUNI_EXTRA + OP_TYPEEXACT: - case OP_EXTUNI_EXTRA + OP_TYPEUPTO: - case OP_EXTUNI_EXTRA + OP_TYPEMINUPTO: - case OP_EXTUNI_EXTRA + OP_TYPEPOSUPTO: - if (codevalue != OP_EXTUNI_EXTRA + OP_TYPEEXACT) - { ADD_ACTIVE(state_offset + 2 + IMM2_SIZE, 0); } - count = current_state->count; /* Number already matched */ - if (clen > 0) - { - int lgb, rgb; - const pcre_uchar *nptr = ptr + clen; - int ncount = 0; - if (codevalue == OP_EXTUNI_EXTRA + OP_TYPEPOSUPTO) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - lgb = UCD_GRAPHBREAK(c); - while (nptr < end_subject) - { - dlen = 1; - if (!utf) d = *nptr; else { GETCHARLEN(d, nptr, dlen); } - rgb = UCD_GRAPHBREAK(d); - if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; - ncount++; - lgb = rgb; - nptr += dlen; - } - if (nptr >= end_subject && (md->moptions & PCRE_PARTIAL_HARD) != 0) - reset_could_continue = TRUE; - if (++count >= (int)GET2(code, 1)) - { ADD_NEW_DATA(-(state_offset + 2 + IMM2_SIZE), 0, ncount); } - else - { ADD_NEW_DATA(-state_offset, count, ncount); } - } - break; -#endif - - /*-----------------------------------------------------------------*/ - case OP_ANYNL_EXTRA + OP_TYPEEXACT: - case OP_ANYNL_EXTRA + OP_TYPEUPTO: - case OP_ANYNL_EXTRA + OP_TYPEMINUPTO: - case OP_ANYNL_EXTRA + OP_TYPEPOSUPTO: - if (codevalue != OP_ANYNL_EXTRA + OP_TYPEEXACT) - { ADD_ACTIVE(state_offset + 2 + IMM2_SIZE, 0); } - count = current_state->count; /* Number already matched */ - if (clen > 0) - { - int ncount = 0; - switch (c) - { - case CHAR_VT: - case CHAR_FF: - case CHAR_NEL: -#ifndef EBCDIC - case 0x2028: - case 0x2029: -#endif /* Not EBCDIC */ - if ((md->moptions & PCRE_BSR_ANYCRLF) != 0) break; - goto ANYNL03; - - case CHAR_CR: - if (ptr + 1 < end_subject && UCHAR21TEST(ptr + 1) == CHAR_LF) ncount = 1; - /* Fall through */ - - ANYNL03: - case CHAR_LF: - if (codevalue == OP_ANYNL_EXTRA + OP_TYPEPOSUPTO) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - if (++count >= (int)GET2(code, 1)) - { ADD_NEW_DATA(-(state_offset + 2 + IMM2_SIZE), 0, ncount); } - else - { ADD_NEW_DATA(-state_offset, count, ncount); } - break; - - default: - break; - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_VSPACE_EXTRA + OP_TYPEEXACT: - case OP_VSPACE_EXTRA + OP_TYPEUPTO: - case OP_VSPACE_EXTRA + OP_TYPEMINUPTO: - case OP_VSPACE_EXTRA + OP_TYPEPOSUPTO: - if (codevalue != OP_VSPACE_EXTRA + OP_TYPEEXACT) - { ADD_ACTIVE(state_offset + 2 + IMM2_SIZE, 0); } - count = current_state->count; /* Number already matched */ - if (clen > 0) - { - BOOL OK; - switch (c) - { - VSPACE_CASES: - OK = TRUE; - break; - - default: - OK = FALSE; - } - - if (OK == (d == OP_VSPACE)) - { - if (codevalue == OP_VSPACE_EXTRA + OP_TYPEPOSUPTO) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - if (++count >= (int)GET2(code, 1)) - { ADD_NEW_DATA(-(state_offset + 2 + IMM2_SIZE), 0, 0); } - else - { ADD_NEW_DATA(-state_offset, count, 0); } - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_HSPACE_EXTRA + OP_TYPEEXACT: - case OP_HSPACE_EXTRA + OP_TYPEUPTO: - case OP_HSPACE_EXTRA + OP_TYPEMINUPTO: - case OP_HSPACE_EXTRA + OP_TYPEPOSUPTO: - if (codevalue != OP_HSPACE_EXTRA + OP_TYPEEXACT) - { ADD_ACTIVE(state_offset + 2 + IMM2_SIZE, 0); } - count = current_state->count; /* Number already matched */ - if (clen > 0) - { - BOOL OK; - switch (c) - { - HSPACE_CASES: - OK = TRUE; - break; - - default: - OK = FALSE; - break; - } - - if (OK == (d == OP_HSPACE)) - { - if (codevalue == OP_HSPACE_EXTRA + OP_TYPEPOSUPTO) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - if (++count >= (int)GET2(code, 1)) - { ADD_NEW_DATA(-(state_offset + 2 + IMM2_SIZE), 0, 0); } - else - { ADD_NEW_DATA(-state_offset, count, 0); } - } - } - break; - -/* ========================================================================== */ - /* These opcodes are followed by a character that is usually compared - to the current subject character; it is loaded into d. We still get - here even if there is no subject character, because in some cases zero - repetitions are permitted. */ - - /*-----------------------------------------------------------------*/ - case OP_CHAR: - if (clen > 0 && c == d) { ADD_NEW(state_offset + dlen + 1, 0); } - break; - - /*-----------------------------------------------------------------*/ - case OP_CHARI: - if (clen == 0) break; - -#ifdef SUPPORT_UTF - if (utf) - { - if (c == d) { ADD_NEW(state_offset + dlen + 1, 0); } else - { - unsigned int othercase; - if (c < 128) - othercase = fcc[c]; - else - /* If we have Unicode property support, we can use it to test the - other case of the character. */ -#ifdef SUPPORT_UCP - othercase = UCD_OTHERCASE(c); -#else - othercase = NOTACHAR; -#endif - - if (d == othercase) { ADD_NEW(state_offset + dlen + 1, 0); } - } - } - else -#endif /* SUPPORT_UTF */ - /* Not UTF mode */ - { - if (TABLE_GET(c, lcc, c) == TABLE_GET(d, lcc, d)) - { ADD_NEW(state_offset + 2, 0); } - } - break; - - -#ifdef SUPPORT_UCP - /*-----------------------------------------------------------------*/ - /* This is a tricky one because it can match more than one character. - Find out how many characters to skip, and then set up a negative state - to wait for them to pass before continuing. */ - - case OP_EXTUNI: - if (clen > 0) - { - int lgb, rgb; - const pcre_uchar *nptr = ptr + clen; - int ncount = 0; - lgb = UCD_GRAPHBREAK(c); - while (nptr < end_subject) - { - dlen = 1; - if (!utf) d = *nptr; else { GETCHARLEN(d, nptr, dlen); } - rgb = UCD_GRAPHBREAK(d); - if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; - ncount++; - lgb = rgb; - nptr += dlen; - } - if (nptr >= end_subject && (md->moptions & PCRE_PARTIAL_HARD) != 0) - reset_could_continue = TRUE; - ADD_NEW_DATA(-(state_offset + 1), 0, ncount); - } - break; -#endif - - /*-----------------------------------------------------------------*/ - /* This is a tricky like EXTUNI because it too can match more than one - character (when CR is followed by LF). In this case, set up a negative - state to wait for one character to pass before continuing. */ - - case OP_ANYNL: - if (clen > 0) switch(c) - { - case CHAR_VT: - case CHAR_FF: - case CHAR_NEL: -#ifndef EBCDIC - case 0x2028: - case 0x2029: -#endif /* Not EBCDIC */ - if ((md->moptions & PCRE_BSR_ANYCRLF) != 0) break; - - case CHAR_LF: - ADD_NEW(state_offset + 1, 0); - break; - - case CHAR_CR: - if (ptr + 1 >= end_subject) - { - ADD_NEW(state_offset + 1, 0); - if ((md->moptions & PCRE_PARTIAL_HARD) != 0) - reset_could_continue = TRUE; - } - else if (UCHAR21TEST(ptr + 1) == CHAR_LF) - { - ADD_NEW_DATA(-(state_offset + 1), 0, 1); - } - else - { - ADD_NEW(state_offset + 1, 0); - } - break; - } - break; - - /*-----------------------------------------------------------------*/ - case OP_NOT_VSPACE: - if (clen > 0) switch(c) - { - VSPACE_CASES: - break; - - default: - ADD_NEW(state_offset + 1, 0); - break; - } - break; - - /*-----------------------------------------------------------------*/ - case OP_VSPACE: - if (clen > 0) switch(c) - { - VSPACE_CASES: - ADD_NEW(state_offset + 1, 0); - break; - - default: - break; - } - break; - - /*-----------------------------------------------------------------*/ - case OP_NOT_HSPACE: - if (clen > 0) switch(c) - { - HSPACE_CASES: - break; - - default: - ADD_NEW(state_offset + 1, 0); - break; - } - break; - - /*-----------------------------------------------------------------*/ - case OP_HSPACE: - if (clen > 0) switch(c) - { - HSPACE_CASES: - ADD_NEW(state_offset + 1, 0); - break; - - default: - break; - } - break; - - /*-----------------------------------------------------------------*/ - /* Match a negated single character casefully. */ - - case OP_NOT: - if (clen > 0 && c != d) { ADD_NEW(state_offset + dlen + 1, 0); } - break; - - /*-----------------------------------------------------------------*/ - /* Match a negated single character caselessly. */ - - case OP_NOTI: - if (clen > 0) - { - unsigned int otherd; -#ifdef SUPPORT_UTF - if (utf && d >= 128) - { -#ifdef SUPPORT_UCP - otherd = UCD_OTHERCASE(d); -#endif /* SUPPORT_UCP */ - } - else -#endif /* SUPPORT_UTF */ - otherd = TABLE_GET(d, fcc, d); - if (c != d && c != otherd) - { ADD_NEW(state_offset + dlen + 1, 0); } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_PLUSI: - case OP_MINPLUSI: - case OP_POSPLUSI: - case OP_NOTPLUSI: - case OP_NOTMINPLUSI: - case OP_NOTPOSPLUSI: - caseless = TRUE; - codevalue -= OP_STARI - OP_STAR; - - /* Fall through */ - case OP_PLUS: - case OP_MINPLUS: - case OP_POSPLUS: - case OP_NOTPLUS: - case OP_NOTMINPLUS: - case OP_NOTPOSPLUS: - count = current_state->count; /* Already matched */ - if (count > 0) { ADD_ACTIVE(state_offset + dlen + 1, 0); } - if (clen > 0) - { - pcre_uint32 otherd = NOTACHAR; - if (caseless) - { -#ifdef SUPPORT_UTF - if (utf && d >= 128) - { -#ifdef SUPPORT_UCP - otherd = UCD_OTHERCASE(d); -#endif /* SUPPORT_UCP */ - } - else -#endif /* SUPPORT_UTF */ - otherd = TABLE_GET(d, fcc, d); - } - if ((c == d || c == otherd) == (codevalue < OP_NOTSTAR)) - { - if (count > 0 && - (codevalue == OP_POSPLUS || codevalue == OP_NOTPOSPLUS)) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - count++; - ADD_NEW(state_offset, count); - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_QUERYI: - case OP_MINQUERYI: - case OP_POSQUERYI: - case OP_NOTQUERYI: - case OP_NOTMINQUERYI: - case OP_NOTPOSQUERYI: - caseless = TRUE; - codevalue -= OP_STARI - OP_STAR; - /* Fall through */ - case OP_QUERY: - case OP_MINQUERY: - case OP_POSQUERY: - case OP_NOTQUERY: - case OP_NOTMINQUERY: - case OP_NOTPOSQUERY: - ADD_ACTIVE(state_offset + dlen + 1, 0); - if (clen > 0) - { - pcre_uint32 otherd = NOTACHAR; - if (caseless) - { -#ifdef SUPPORT_UTF - if (utf && d >= 128) - { -#ifdef SUPPORT_UCP - otherd = UCD_OTHERCASE(d); -#endif /* SUPPORT_UCP */ - } - else -#endif /* SUPPORT_UTF */ - otherd = TABLE_GET(d, fcc, d); - } - if ((c == d || c == otherd) == (codevalue < OP_NOTSTAR)) - { - if (codevalue == OP_POSQUERY || codevalue == OP_NOTPOSQUERY) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - ADD_NEW(state_offset + dlen + 1, 0); - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_STARI: - case OP_MINSTARI: - case OP_POSSTARI: - case OP_NOTSTARI: - case OP_NOTMINSTARI: - case OP_NOTPOSSTARI: - caseless = TRUE; - codevalue -= OP_STARI - OP_STAR; - /* Fall through */ - case OP_STAR: - case OP_MINSTAR: - case OP_POSSTAR: - case OP_NOTSTAR: - case OP_NOTMINSTAR: - case OP_NOTPOSSTAR: - ADD_ACTIVE(state_offset + dlen + 1, 0); - if (clen > 0) - { - pcre_uint32 otherd = NOTACHAR; - if (caseless) - { -#ifdef SUPPORT_UTF - if (utf && d >= 128) - { -#ifdef SUPPORT_UCP - otherd = UCD_OTHERCASE(d); -#endif /* SUPPORT_UCP */ - } - else -#endif /* SUPPORT_UTF */ - otherd = TABLE_GET(d, fcc, d); - } - if ((c == d || c == otherd) == (codevalue < OP_NOTSTAR)) - { - if (codevalue == OP_POSSTAR || codevalue == OP_NOTPOSSTAR) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - ADD_NEW(state_offset, 0); - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_EXACTI: - case OP_NOTEXACTI: - caseless = TRUE; - codevalue -= OP_STARI - OP_STAR; - /* Fall through */ - case OP_EXACT: - case OP_NOTEXACT: - count = current_state->count; /* Number already matched */ - if (clen > 0) - { - pcre_uint32 otherd = NOTACHAR; - if (caseless) - { -#ifdef SUPPORT_UTF - if (utf && d >= 128) - { -#ifdef SUPPORT_UCP - otherd = UCD_OTHERCASE(d); -#endif /* SUPPORT_UCP */ - } - else -#endif /* SUPPORT_UTF */ - otherd = TABLE_GET(d, fcc, d); - } - if ((c == d || c == otherd) == (codevalue < OP_NOTSTAR)) - { - if (++count >= (int)GET2(code, 1)) - { ADD_NEW(state_offset + dlen + 1 + IMM2_SIZE, 0); } - else - { ADD_NEW(state_offset, count); } - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_UPTOI: - case OP_MINUPTOI: - case OP_POSUPTOI: - case OP_NOTUPTOI: - case OP_NOTMINUPTOI: - case OP_NOTPOSUPTOI: - caseless = TRUE; - codevalue -= OP_STARI - OP_STAR; - /* Fall through */ - case OP_UPTO: - case OP_MINUPTO: - case OP_POSUPTO: - case OP_NOTUPTO: - case OP_NOTMINUPTO: - case OP_NOTPOSUPTO: - ADD_ACTIVE(state_offset + dlen + 1 + IMM2_SIZE, 0); - count = current_state->count; /* Number already matched */ - if (clen > 0) - { - pcre_uint32 otherd = NOTACHAR; - if (caseless) - { -#ifdef SUPPORT_UTF - if (utf && d >= 128) - { -#ifdef SUPPORT_UCP - otherd = UCD_OTHERCASE(d); -#endif /* SUPPORT_UCP */ - } - else -#endif /* SUPPORT_UTF */ - otherd = TABLE_GET(d, fcc, d); - } - if ((c == d || c == otherd) == (codevalue < OP_NOTSTAR)) - { - if (codevalue == OP_POSUPTO || codevalue == OP_NOTPOSUPTO) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - if (++count >= (int)GET2(code, 1)) - { ADD_NEW(state_offset + dlen + 1 + IMM2_SIZE, 0); } - else - { ADD_NEW(state_offset, count); } - } - } - break; - - -/* ========================================================================== */ - /* These are the class-handling opcodes */ - - case OP_CLASS: - case OP_NCLASS: - case OP_XCLASS: - { - BOOL isinclass = FALSE; - int next_state_offset; - const pcre_uchar *ecode; - - /* For a simple class, there is always just a 32-byte table, and we - can set isinclass from it. */ - - if (codevalue != OP_XCLASS) - { - ecode = code + 1 + (32 / sizeof(pcre_uchar)); - if (clen > 0) - { - isinclass = (c > 255)? (codevalue == OP_NCLASS) : - ((((pcre_uint8 *)(code + 1))[c/8] & (1 << (c&7))) != 0); - } - } - - /* An extended class may have a table or a list of single characters, - ranges, or both, and it may be positive or negative. There's a - function that sorts all this out. */ - - else - { - ecode = code + GET(code, 1); - if (clen > 0) isinclass = PRIV(xclass)(c, code + 1 + LINK_SIZE, utf); - } - - /* At this point, isinclass is set for all kinds of class, and ecode - points to the byte after the end of the class. If there is a - quantifier, this is where it will be. */ - - next_state_offset = (int)(ecode - start_code); - - switch (*ecode) - { - case OP_CRSTAR: - case OP_CRMINSTAR: - case OP_CRPOSSTAR: - ADD_ACTIVE(next_state_offset + 1, 0); - if (isinclass) - { - if (*ecode == OP_CRPOSSTAR) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - ADD_NEW(state_offset, 0); - } - break; - - case OP_CRPLUS: - case OP_CRMINPLUS: - case OP_CRPOSPLUS: - count = current_state->count; /* Already matched */ - if (count > 0) { ADD_ACTIVE(next_state_offset + 1, 0); } - if (isinclass) - { - if (count > 0 && *ecode == OP_CRPOSPLUS) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - count++; - ADD_NEW(state_offset, count); - } - break; - - case OP_CRQUERY: - case OP_CRMINQUERY: - case OP_CRPOSQUERY: - ADD_ACTIVE(next_state_offset + 1, 0); - if (isinclass) - { - if (*ecode == OP_CRPOSQUERY) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - ADD_NEW(next_state_offset + 1, 0); - } - break; - - case OP_CRRANGE: - case OP_CRMINRANGE: - case OP_CRPOSRANGE: - count = current_state->count; /* Already matched */ - if (count >= (int)GET2(ecode, 1)) - { ADD_ACTIVE(next_state_offset + 1 + 2 * IMM2_SIZE, 0); } - if (isinclass) - { - int max = (int)GET2(ecode, 1 + IMM2_SIZE); - if (*ecode == OP_CRPOSRANGE) - { - active_count--; /* Remove non-match possibility */ - next_active_state--; - } - if (++count >= max && max != 0) /* Max 0 => no limit */ - { ADD_NEW(next_state_offset + 1 + 2 * IMM2_SIZE, 0); } - else - { ADD_NEW(state_offset, count); } - } - break; - - default: - if (isinclass) { ADD_NEW(next_state_offset, 0); } - break; - } - } - break; - -/* ========================================================================== */ - /* These are the opcodes for fancy brackets of various kinds. We have - to use recursion in order to handle them. The "always failing" assertion - (?!) is optimised to OP_FAIL when compiling, so we have to support that, - though the other "backtracking verbs" are not supported. */ - - case OP_FAIL: - forced_fail++; /* Count FAILs for multiple states */ - break; - - case OP_ASSERT: - case OP_ASSERT_NOT: - case OP_ASSERTBACK: - case OP_ASSERTBACK_NOT: - { - int rc; - int local_offsets[2]; - int local_workspace[1000]; - const pcre_uchar *endasscode = code + GET(code, 1); - - while (*endasscode == OP_ALT) endasscode += GET(endasscode, 1); - - rc = internal_dfa_exec( - md, /* static match data */ - code, /* this subexpression's code */ - ptr, /* where we currently are */ - (int)(ptr - start_subject), /* start offset */ - local_offsets, /* offset vector */ - sizeof(local_offsets)/sizeof(int), /* size of same */ - local_workspace, /* workspace vector */ - sizeof(local_workspace)/sizeof(int), /* size of same */ - rlevel); /* function recursion level */ - - if (rc == PCRE_ERROR_DFA_UITEM) return rc; - if ((rc >= 0) == (codevalue == OP_ASSERT || codevalue == OP_ASSERTBACK)) - { ADD_ACTIVE((int)(endasscode + LINK_SIZE + 1 - start_code), 0); } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_COND: - case OP_SCOND: - { - int local_offsets[1000]; - int local_workspace[1000]; - int codelink = GET(code, 1); - int condcode; - - /* Because of the way auto-callout works during compile, a callout item - is inserted between OP_COND and an assertion condition. This does not - happen for the other conditions. */ - - if (code[LINK_SIZE+1] == OP_CALLOUT) - { - rrc = 0; - if (PUBL(callout) != NULL) - { - PUBL(callout_block) cb; - cb.version = 1; /* Version 1 of the callout block */ - cb.callout_number = code[LINK_SIZE+2]; - cb.offset_vector = offsets; -#if defined COMPILE_PCRE8 - cb.subject = (PCRE_SPTR)start_subject; -#elif defined COMPILE_PCRE16 - cb.subject = (PCRE_SPTR16)start_subject; -#elif defined COMPILE_PCRE32 - cb.subject = (PCRE_SPTR32)start_subject; -#endif - cb.subject_length = (int)(end_subject - start_subject); - cb.start_match = (int)(current_subject - start_subject); - cb.current_position = (int)(ptr - start_subject); - cb.pattern_position = GET(code, LINK_SIZE + 3); - cb.next_item_length = GET(code, 3 + 2*LINK_SIZE); - cb.capture_top = 1; - cb.capture_last = -1; - cb.callout_data = md->callout_data; - cb.mark = NULL; /* No (*MARK) support */ - if ((rrc = (*PUBL(callout))(&cb)) < 0) return rrc; /* Abandon */ - } - if (rrc > 0) break; /* Fail this thread */ - code += PRIV(OP_lengths)[OP_CALLOUT]; /* Skip callout data */ - } - - condcode = code[LINK_SIZE+1]; - - /* Back reference conditions and duplicate named recursion conditions - are not supported */ - - if (condcode == OP_CREF || condcode == OP_DNCREF || - condcode == OP_DNRREF) - return PCRE_ERROR_DFA_UCOND; - - /* The DEFINE condition is always false, and the assertion (?!) is - converted to OP_FAIL. */ - - if (condcode == OP_DEF || condcode == OP_FAIL) - { ADD_ACTIVE(state_offset + codelink + LINK_SIZE + 1, 0); } - - /* The only supported version of OP_RREF is for the value RREF_ANY, - which means "test if in any recursion". We can't test for specifically - recursed groups. */ - - else if (condcode == OP_RREF) - { - int value = GET2(code, LINK_SIZE + 2); - if (value != RREF_ANY) return PCRE_ERROR_DFA_UCOND; - if (md->recursive != NULL) - { ADD_ACTIVE(state_offset + LINK_SIZE + 2 + IMM2_SIZE, 0); } - else { ADD_ACTIVE(state_offset + codelink + LINK_SIZE + 1, 0); } - } - - /* Otherwise, the condition is an assertion */ - - else - { - int rc; - const pcre_uchar *asscode = code + LINK_SIZE + 1; - const pcre_uchar *endasscode = asscode + GET(asscode, 1); - - while (*endasscode == OP_ALT) endasscode += GET(endasscode, 1); - - rc = internal_dfa_exec( - md, /* fixed match data */ - asscode, /* this subexpression's code */ - ptr, /* where we currently are */ - (int)(ptr - start_subject), /* start offset */ - local_offsets, /* offset vector */ - sizeof(local_offsets)/sizeof(int), /* size of same */ - local_workspace, /* workspace vector */ - sizeof(local_workspace)/sizeof(int), /* size of same */ - rlevel); /* function recursion level */ - - if (rc == PCRE_ERROR_DFA_UITEM) return rc; - if ((rc >= 0) == - (condcode == OP_ASSERT || condcode == OP_ASSERTBACK)) - { ADD_ACTIVE((int)(endasscode + LINK_SIZE + 1 - start_code), 0); } - else - { ADD_ACTIVE(state_offset + codelink + LINK_SIZE + 1, 0); } - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_RECURSE: - { - dfa_recursion_info *ri; - int local_offsets[1000]; - int local_workspace[1000]; - const pcre_uchar *callpat = start_code + GET(code, 1); - int recno = (callpat == md->start_code)? 0 : - GET2(callpat, 1 + LINK_SIZE); - int rc; - - DPRINTF(("%.*sStarting regex recursion\n", rlevel*2-2, SP)); - - /* Check for repeating a recursion without advancing the subject - pointer. This should catch convoluted mutual recursions. (Some simple - cases are caught at compile time.) */ - - for (ri = md->recursive; ri != NULL; ri = ri->prevrec) - if (recno == ri->group_num && ptr == ri->subject_position) - return PCRE_ERROR_RECURSELOOP; - - /* Remember this recursion and where we started it so as to - catch infinite loops. */ - - new_recursive.group_num = recno; - new_recursive.subject_position = ptr; - new_recursive.prevrec = md->recursive; - md->recursive = &new_recursive; - - rc = internal_dfa_exec( - md, /* fixed match data */ - callpat, /* this subexpression's code */ - ptr, /* where we currently are */ - (int)(ptr - start_subject), /* start offset */ - local_offsets, /* offset vector */ - sizeof(local_offsets)/sizeof(int), /* size of same */ - local_workspace, /* workspace vector */ - sizeof(local_workspace)/sizeof(int), /* size of same */ - rlevel); /* function recursion level */ - - md->recursive = new_recursive.prevrec; /* Done this recursion */ - - DPRINTF(("%.*sReturn from regex recursion: rc=%d\n", rlevel*2-2, SP, - rc)); - - /* Ran out of internal offsets */ - - if (rc == 0) return PCRE_ERROR_DFA_RECURSE; - - /* For each successful matched substring, set up the next state with a - count of characters to skip before trying it. Note that the count is in - characters, not bytes. */ - - if (rc > 0) - { - for (rc = rc*2 - 2; rc >= 0; rc -= 2) - { - int charcount = local_offsets[rc+1] - local_offsets[rc]; -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (utf) - { - const pcre_uchar *p = start_subject + local_offsets[rc]; - const pcre_uchar *pp = start_subject + local_offsets[rc+1]; - while (p < pp) if (NOT_FIRSTCHAR(*p++)) charcount--; - } -#endif - if (charcount > 0) - { - ADD_NEW_DATA(-(state_offset + LINK_SIZE + 1), 0, (charcount - 1)); - } - else - { - ADD_ACTIVE(state_offset + LINK_SIZE + 1, 0); - } - } - } - else if (rc != PCRE_ERROR_NOMATCH) return rc; - } - break; - - /*-----------------------------------------------------------------*/ - case OP_BRAPOS: - case OP_SBRAPOS: - case OP_CBRAPOS: - case OP_SCBRAPOS: - case OP_BRAPOSZERO: - { - int charcount, matched_count; - const pcre_uchar *local_ptr = ptr; - BOOL allow_zero; - - if (codevalue == OP_BRAPOSZERO) - { - allow_zero = TRUE; - codevalue = *(++code); /* Codevalue will be one of above BRAs */ - } - else allow_zero = FALSE; - - /* Loop to match the subpattern as many times as possible as if it were - a complete pattern. */ - - for (matched_count = 0;; matched_count++) - { - int local_offsets[2]; - int local_workspace[1000]; - - int rc = internal_dfa_exec( - md, /* fixed match data */ - code, /* this subexpression's code */ - local_ptr, /* where we currently are */ - (int)(ptr - start_subject), /* start offset */ - local_offsets, /* offset vector */ - sizeof(local_offsets)/sizeof(int), /* size of same */ - local_workspace, /* workspace vector */ - sizeof(local_workspace)/sizeof(int), /* size of same */ - rlevel); /* function recursion level */ - - /* Failed to match */ - - if (rc < 0) - { - if (rc != PCRE_ERROR_NOMATCH) return rc; - break; - } - - /* Matched: break the loop if zero characters matched. */ - - charcount = local_offsets[1] - local_offsets[0]; - if (charcount == 0) break; - local_ptr += charcount; /* Advance temporary position ptr */ - } - - /* At this point we have matched the subpattern matched_count - times, and local_ptr is pointing to the character after the end of the - last match. */ - - if (matched_count > 0 || allow_zero) - { - const pcre_uchar *end_subpattern = code; - int next_state_offset; - - do { end_subpattern += GET(end_subpattern, 1); } - while (*end_subpattern == OP_ALT); - next_state_offset = - (int)(end_subpattern - start_code + LINK_SIZE + 1); - - /* Optimization: if there are no more active states, and there - are no new states yet set up, then skip over the subject string - right here, to save looping. Otherwise, set up the new state to swing - into action when the end of the matched substring is reached. */ - - if (i + 1 >= active_count && new_count == 0) - { - ptr = local_ptr; - clen = 0; - ADD_NEW(next_state_offset, 0); - } - else - { - const pcre_uchar *p = ptr; - const pcre_uchar *pp = local_ptr; - charcount = (int)(pp - p); -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (utf) while (p < pp) if (NOT_FIRSTCHAR(*p++)) charcount--; -#endif - ADD_NEW_DATA(-next_state_offset, 0, (charcount - 1)); - } - } - } - break; - - /*-----------------------------------------------------------------*/ - case OP_ONCE: - case OP_ONCE_NC: - { - int local_offsets[2]; - int local_workspace[1000]; - - int rc = internal_dfa_exec( - md, /* fixed match data */ - code, /* this subexpression's code */ - ptr, /* where we currently are */ - (int)(ptr - start_subject), /* start offset */ - local_offsets, /* offset vector */ - sizeof(local_offsets)/sizeof(int), /* size of same */ - local_workspace, /* workspace vector */ - sizeof(local_workspace)/sizeof(int), /* size of same */ - rlevel); /* function recursion level */ - - if (rc >= 0) - { - const pcre_uchar *end_subpattern = code; - int charcount = local_offsets[1] - local_offsets[0]; - int next_state_offset, repeat_state_offset; - - do { end_subpattern += GET(end_subpattern, 1); } - while (*end_subpattern == OP_ALT); - next_state_offset = - (int)(end_subpattern - start_code + LINK_SIZE + 1); - - /* If the end of this subpattern is KETRMAX or KETRMIN, we must - arrange for the repeat state also to be added to the relevant list. - Calculate the offset, or set -1 for no repeat. */ - - repeat_state_offset = (*end_subpattern == OP_KETRMAX || - *end_subpattern == OP_KETRMIN)? - (int)(end_subpattern - start_code - GET(end_subpattern, 1)) : -1; - - /* If we have matched an empty string, add the next state at the - current character pointer. This is important so that the duplicate - checking kicks in, which is what breaks infinite loops that match an - empty string. */ - - if (charcount == 0) - { - ADD_ACTIVE(next_state_offset, 0); - } - - /* Optimization: if there are no more active states, and there - are no new states yet set up, then skip over the subject string - right here, to save looping. Otherwise, set up the new state to swing - into action when the end of the matched substring is reached. */ - - else if (i + 1 >= active_count && new_count == 0) - { - ptr += charcount; - clen = 0; - ADD_NEW(next_state_offset, 0); - - /* If we are adding a repeat state at the new character position, - we must fudge things so that it is the only current state. - Otherwise, it might be a duplicate of one we processed before, and - that would cause it to be skipped. */ - - if (repeat_state_offset >= 0) - { - next_active_state = active_states; - active_count = 0; - i = -1; - ADD_ACTIVE(repeat_state_offset, 0); - } - } - else - { -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (utf) - { - const pcre_uchar *p = start_subject + local_offsets[0]; - const pcre_uchar *pp = start_subject + local_offsets[1]; - while (p < pp) if (NOT_FIRSTCHAR(*p++)) charcount--; - } -#endif - ADD_NEW_DATA(-next_state_offset, 0, (charcount - 1)); - if (repeat_state_offset >= 0) - { ADD_NEW_DATA(-repeat_state_offset, 0, (charcount - 1)); } - } - } - else if (rc != PCRE_ERROR_NOMATCH) return rc; - } - break; - - -/* ========================================================================== */ - /* Handle callouts */ - - case OP_CALLOUT: - rrc = 0; - if (PUBL(callout) != NULL) - { - PUBL(callout_block) cb; - cb.version = 1; /* Version 1 of the callout block */ - cb.callout_number = code[1]; - cb.offset_vector = offsets; -#if defined COMPILE_PCRE8 - cb.subject = (PCRE_SPTR)start_subject; -#elif defined COMPILE_PCRE16 - cb.subject = (PCRE_SPTR16)start_subject; -#elif defined COMPILE_PCRE32 - cb.subject = (PCRE_SPTR32)start_subject; -#endif - cb.subject_length = (int)(end_subject - start_subject); - cb.start_match = (int)(current_subject - start_subject); - cb.current_position = (int)(ptr - start_subject); - cb.pattern_position = GET(code, 2); - cb.next_item_length = GET(code, 2 + LINK_SIZE); - cb.capture_top = 1; - cb.capture_last = -1; - cb.callout_data = md->callout_data; - cb.mark = NULL; /* No (*MARK) support */ - if ((rrc = (*PUBL(callout))(&cb)) < 0) return rrc; /* Abandon */ - } - if (rrc == 0) - { ADD_ACTIVE(state_offset + PRIV(OP_lengths)[OP_CALLOUT], 0); } - break; - - -/* ========================================================================== */ - default: /* Unsupported opcode */ - return PCRE_ERROR_DFA_UITEM; - } - - NEXT_ACTIVE_STATE: continue; - - } /* End of loop scanning active states */ - - /* We have finished the processing at the current subject character. If no - new states have been set for the next character, we have found all the - matches that we are going to find. If we are at the top level and partial - matching has been requested, check for appropriate conditions. - - The "forced_ fail" variable counts the number of (*F) encountered for the - character. If it is equal to the original active_count (saved in - workspace[1]) it means that (*F) was found on every active state. In this - case we don't want to give a partial match. - - The "could_continue" variable is true if a state could have continued but - for the fact that the end of the subject was reached. */ - - if (new_count <= 0) - { - if (rlevel == 1 && /* Top level, and */ - could_continue && /* Some could go on, and */ - forced_fail != workspace[1] && /* Not all forced fail & */ - ( /* either... */ - (md->moptions & PCRE_PARTIAL_HARD) != 0 /* Hard partial */ - || /* or... */ - ((md->moptions & PCRE_PARTIAL_SOFT) != 0 && /* Soft partial and */ - match_count < 0) /* no matches */ - ) && /* And... */ - ( - partial_newline || /* Either partial NL */ - ( /* or ... */ - ptr >= end_subject && /* End of subject and */ - ptr > md->start_used_ptr) /* Inspected non-empty string */ - ) - ) - match_count = PCRE_ERROR_PARTIAL; - DPRINTF(("%.*sEnd of internal_dfa_exec %d: returning %d\n" - "%.*s---------------------\n\n", rlevel*2-2, SP, rlevel, match_count, - rlevel*2-2, SP)); - break; /* In effect, "return", but see the comment below */ - } - - /* One or more states are active for the next character. */ - - ptr += clen; /* Advance to next subject character */ - } /* Loop to move along the subject string */ - -/* Control gets here from "break" a few lines above. We do it this way because -if we use "return" above, we have compiler trouble. Some compilers warn if -there's nothing here because they think the function doesn't return a value. On -the other hand, if we put a dummy statement here, some more clever compilers -complain that it can't be reached. Sigh. */ - -return match_count; -} - - - - -/************************************************* -* Execute a Regular Expression - DFA engine * -*************************************************/ - -/* This external function applies a compiled re to a subject string using a DFA -engine. This function calls the internal function multiple times if the pattern -is not anchored. - -Arguments: - argument_re points to the compiled expression - extra_data points to extra data or is NULL - subject points to the subject string - length length of subject string (may contain binary zeros) - start_offset where to start in the subject string - options option bits - offsets vector of match offsets - offsetcount size of same - workspace workspace vector - wscount size of same - -Returns: > 0 => number of match offset pairs placed in offsets - = 0 => offsets overflowed; longest matches are present - -1 => failed to match - < -1 => some kind of unexpected problem -*/ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre_dfa_exec(const pcre *argument_re, const pcre_extra *extra_data, - const char *subject, int length, int start_offset, int options, int *offsets, - int offsetcount, int *workspace, int wscount) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre16_dfa_exec(const pcre16 *argument_re, const pcre16_extra *extra_data, - PCRE_SPTR16 subject, int length, int start_offset, int options, int *offsets, - int offsetcount, int *workspace, int wscount) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre32_dfa_exec(const pcre32 *argument_re, const pcre32_extra *extra_data, - PCRE_SPTR32 subject, int length, int start_offset, int options, int *offsets, - int offsetcount, int *workspace, int wscount) -#endif -{ -REAL_PCRE *re = (REAL_PCRE *)argument_re; -dfa_match_data match_block; -dfa_match_data *md = &match_block; -BOOL utf, anchored, startline, firstline; -const pcre_uchar *current_subject, *end_subject; -const pcre_study_data *study = NULL; - -const pcre_uchar *req_char_ptr; -const pcre_uint8 *start_bits = NULL; -BOOL has_first_char = FALSE; -BOOL has_req_char = FALSE; -pcre_uchar first_char = 0; -pcre_uchar first_char2 = 0; -pcre_uchar req_char = 0; -pcre_uchar req_char2 = 0; -int newline; - -/* Plausibility checks */ - -if ((options & ~PUBLIC_DFA_EXEC_OPTIONS) != 0) return PCRE_ERROR_BADOPTION; -if (re == NULL || subject == NULL || workspace == NULL || - (offsets == NULL && offsetcount > 0)) return PCRE_ERROR_NULL; -if (offsetcount < 0) return PCRE_ERROR_BADCOUNT; -if (wscount < 20) return PCRE_ERROR_DFA_WSSIZE; -if (length < 0) return PCRE_ERROR_BADLENGTH; -if (start_offset < 0 || start_offset > length) return PCRE_ERROR_BADOFFSET; - -/* Check that the first field in the block is the magic number. If it is not, -return with PCRE_ERROR_BADMAGIC. However, if the magic number is equal to -REVERSED_MAGIC_NUMBER we return with PCRE_ERROR_BADENDIANNESS, which -means that the pattern is likely compiled with different endianness. */ - -if (re->magic_number != MAGIC_NUMBER) - return re->magic_number == REVERSED_MAGIC_NUMBER? - PCRE_ERROR_BADENDIANNESS:PCRE_ERROR_BADMAGIC; -if ((re->flags & PCRE_MODE) == 0) return PCRE_ERROR_BADMODE; - -/* If restarting after a partial match, do some sanity checks on the contents -of the workspace. */ - -if ((options & PCRE_DFA_RESTART) != 0) - { - if ((workspace[0] & (-2)) != 0 || workspace[1] < 1 || - workspace[1] > (wscount - 2)/INTS_PER_STATEBLOCK) - return PCRE_ERROR_DFA_BADRESTART; - } - -/* Set up study, callout, and table data */ - -md->tables = re->tables; -md->callout_data = NULL; - -if (extra_data != NULL) - { - unsigned long int flags = extra_data->flags; - if ((flags & PCRE_EXTRA_STUDY_DATA) != 0) - study = (const pcre_study_data *)extra_data->study_data; - if ((flags & PCRE_EXTRA_MATCH_LIMIT) != 0) return PCRE_ERROR_DFA_UMLIMIT; - if ((flags & PCRE_EXTRA_MATCH_LIMIT_RECURSION) != 0) - return PCRE_ERROR_DFA_UMLIMIT; - if ((flags & PCRE_EXTRA_CALLOUT_DATA) != 0) - md->callout_data = extra_data->callout_data; - if ((flags & PCRE_EXTRA_TABLES) != 0) - md->tables = extra_data->tables; - } - -/* Set some local values */ - -current_subject = (const pcre_uchar *)subject + start_offset; -end_subject = (const pcre_uchar *)subject + length; -req_char_ptr = current_subject - 1; - -#ifdef SUPPORT_UTF -/* PCRE_UTF(16|32) have the same value as PCRE_UTF8. */ -utf = (re->options & PCRE_UTF8) != 0; -#else -utf = FALSE; -#endif - -anchored = (options & (PCRE_ANCHORED|PCRE_DFA_RESTART)) != 0 || - (re->options & PCRE_ANCHORED) != 0; - -/* The remaining fixed data for passing around. */ - -md->start_code = (const pcre_uchar *)argument_re + - re->name_table_offset + re->name_count * re->name_entry_size; -md->start_subject = (const pcre_uchar *)subject; -md->end_subject = end_subject; -md->start_offset = start_offset; -md->moptions = options; -md->poptions = re->options; - -/* If the BSR option is not set at match time, copy what was set -at compile time. */ - -if ((md->moptions & (PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE)) == 0) - { - if ((re->options & (PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE)) != 0) - md->moptions |= re->options & (PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE); -#ifdef BSR_ANYCRLF - else md->moptions |= PCRE_BSR_ANYCRLF; -#endif - } - -/* Handle different types of newline. The three bits give eight cases. If -nothing is set at run time, whatever was used at compile time applies. */ - -switch ((((options & PCRE_NEWLINE_BITS) == 0)? re->options : (pcre_uint32)options) & - PCRE_NEWLINE_BITS) - { - case 0: newline = NEWLINE; break; /* Compile-time default */ - case PCRE_NEWLINE_CR: newline = CHAR_CR; break; - case PCRE_NEWLINE_LF: newline = CHAR_NL; break; - case PCRE_NEWLINE_CR+ - PCRE_NEWLINE_LF: newline = (CHAR_CR << 8) | CHAR_NL; break; - case PCRE_NEWLINE_ANY: newline = -1; break; - case PCRE_NEWLINE_ANYCRLF: newline = -2; break; - default: return PCRE_ERROR_BADNEWLINE; - } - -if (newline == -2) - { - md->nltype = NLTYPE_ANYCRLF; - } -else if (newline < 0) - { - md->nltype = NLTYPE_ANY; - } -else - { - md->nltype = NLTYPE_FIXED; - if (newline > 255) - { - md->nllen = 2; - md->nl[0] = (newline >> 8) & 255; - md->nl[1] = newline & 255; - } - else - { - md->nllen = 1; - md->nl[0] = newline; - } - } - -/* Check a UTF-8 string if required. Unfortunately there's no way of passing -back the character offset. */ - -#ifdef SUPPORT_UTF -if (utf && (options & PCRE_NO_UTF8_CHECK) == 0) - { - int erroroffset; - int errorcode = PRIV(valid_utf)((pcre_uchar *)subject, length, &erroroffset); - if (errorcode != 0) - { - if (offsetcount >= 2) - { - offsets[0] = erroroffset; - offsets[1] = errorcode; - } -#if defined COMPILE_PCRE8 - return (errorcode <= PCRE_UTF8_ERR5 && (options & PCRE_PARTIAL_HARD) != 0) ? - PCRE_ERROR_SHORTUTF8 : PCRE_ERROR_BADUTF8; -#elif defined COMPILE_PCRE16 - return (errorcode <= PCRE_UTF16_ERR1 && (options & PCRE_PARTIAL_HARD) != 0) ? - PCRE_ERROR_SHORTUTF16 : PCRE_ERROR_BADUTF16; -#elif defined COMPILE_PCRE32 - return PCRE_ERROR_BADUTF32; -#endif - } -#if defined COMPILE_PCRE8 || defined COMPILE_PCRE16 - if (start_offset > 0 && start_offset < length && - NOT_FIRSTCHAR(((PCRE_PUCHAR)subject)[start_offset])) - return PCRE_ERROR_BADUTF8_OFFSET; -#endif - } -#endif - -/* If the exec call supplied NULL for tables, use the inbuilt ones. This -is a feature that makes it possible to save compiled regex and re-use them -in other programs later. */ - -if (md->tables == NULL) md->tables = PRIV(default_tables); - -/* The "must be at the start of a line" flags are used in a loop when finding -where to start. */ - -startline = (re->flags & PCRE_STARTLINE) != 0; -firstline = (re->options & PCRE_FIRSTLINE) != 0; - -/* Set up the first character to match, if available. The first_byte value is -never set for an anchored regular expression, but the anchoring may be forced -at run time, so we have to test for anchoring. The first char may be unset for -an unanchored pattern, of course. If there's no first char and the pattern was -studied, there may be a bitmap of possible first characters. */ - -if (!anchored) - { - if ((re->flags & PCRE_FIRSTSET) != 0) - { - has_first_char = TRUE; - first_char = first_char2 = (pcre_uchar)(re->first_char); - if ((re->flags & PCRE_FCH_CASELESS) != 0) - { - first_char2 = TABLE_GET(first_char, md->tables + fcc_offset, first_char); -#if defined SUPPORT_UCP && !(defined COMPILE_PCRE8) - if (utf && first_char > 127) - first_char2 = UCD_OTHERCASE(first_char); -#endif - } - } - else - { - if (!startline && study != NULL && - (study->flags & PCRE_STUDY_MAPPED) != 0) - start_bits = study->start_bits; - } - } - -/* For anchored or unanchored matches, there may be a "last known required -character" set. */ - -if ((re->flags & PCRE_REQCHSET) != 0) - { - has_req_char = TRUE; - req_char = req_char2 = (pcre_uchar)(re->req_char); - if ((re->flags & PCRE_RCH_CASELESS) != 0) - { - req_char2 = TABLE_GET(req_char, md->tables + fcc_offset, req_char); -#if defined SUPPORT_UCP && !(defined COMPILE_PCRE8) - if (utf && req_char > 127) - req_char2 = UCD_OTHERCASE(req_char); -#endif - } - } - -/* Call the main matching function, looping for a non-anchored regex after a -failed match. If not restarting, perform certain optimizations at the start of -a match. */ - -for (;;) - { - int rc; - - if ((options & PCRE_DFA_RESTART) == 0) - { - const pcre_uchar *save_end_subject = end_subject; - - /* If firstline is TRUE, the start of the match is constrained to the first - line of a multiline string. Implement this by temporarily adjusting - end_subject so that we stop scanning at a newline. If the match fails at - the newline, later code breaks this loop. */ - - if (firstline) - { - PCRE_PUCHAR t = current_subject; -#ifdef SUPPORT_UTF - if (utf) - { - while (t < md->end_subject && !IS_NEWLINE(t)) - { - t++; - ACROSSCHAR(t < end_subject, *t, t++); - } - } - else -#endif - while (t < md->end_subject && !IS_NEWLINE(t)) t++; - end_subject = t; - } - - /* There are some optimizations that avoid running the match if a known - starting point is not found. However, there is an option that disables - these, for testing and for ensuring that all callouts do actually occur. - The option can be set in the regex by (*NO_START_OPT) or passed in - match-time options. */ - - if (((options | re->options) & PCRE_NO_START_OPTIMIZE) == 0) - { - /* Advance to a known first pcre_uchar (i.e. data item) */ - - if (has_first_char) - { - if (first_char != first_char2) - { - pcre_uchar csc; - while (current_subject < end_subject && - (csc = UCHAR21TEST(current_subject)) != first_char && csc != first_char2) - current_subject++; - } - else - while (current_subject < end_subject && - UCHAR21TEST(current_subject) != first_char) - current_subject++; - } - - /* Or to just after a linebreak for a multiline match if possible */ - - else if (startline) - { - if (current_subject > md->start_subject + start_offset) - { -#ifdef SUPPORT_UTF - if (utf) - { - while (current_subject < end_subject && - !WAS_NEWLINE(current_subject)) - { - current_subject++; - ACROSSCHAR(current_subject < end_subject, *current_subject, - current_subject++); - } - } - else -#endif - while (current_subject < end_subject && !WAS_NEWLINE(current_subject)) - current_subject++; - - /* If we have just passed a CR and the newline option is ANY or - ANYCRLF, and we are now at a LF, advance the match position by one - more character. */ - - if (UCHAR21TEST(current_subject - 1) == CHAR_CR && - (md->nltype == NLTYPE_ANY || md->nltype == NLTYPE_ANYCRLF) && - current_subject < end_subject && - UCHAR21TEST(current_subject) == CHAR_NL) - current_subject++; - } - } - - /* Advance to a non-unique first pcre_uchar after study */ - - else if (start_bits != NULL) - { - while (current_subject < end_subject) - { - register pcre_uint32 c = UCHAR21TEST(current_subject); -#ifndef COMPILE_PCRE8 - if (c > 255) c = 255; -#endif - if ((start_bits[c/8] & (1 << (c&7))) != 0) break; - current_subject++; - } - } - } - - /* Restore fudged end_subject */ - - end_subject = save_end_subject; - - /* The following two optimizations are disabled for partial matching or if - disabling is explicitly requested (and of course, by the test above, this - code is not obeyed when restarting after a partial match). */ - - if (((options | re->options) & PCRE_NO_START_OPTIMIZE) == 0 && - (options & (PCRE_PARTIAL_HARD|PCRE_PARTIAL_SOFT)) == 0) - { - /* If the pattern was studied, a minimum subject length may be set. This - is a lower bound; no actual string of that length may actually match the - pattern. Although the value is, strictly, in characters, we treat it as - in pcre_uchar units to avoid spending too much time in this optimization. - */ - - if (study != NULL && (study->flags & PCRE_STUDY_MINLEN) != 0 && - (pcre_uint32)(end_subject - current_subject) < study->minlength) - return PCRE_ERROR_NOMATCH; - - /* If req_char is set, we know that that pcre_uchar must appear in the - subject for the match to succeed. If the first pcre_uchar is set, - req_char must be later in the subject; otherwise the test starts at the - match point. This optimization can save a huge amount of work in patterns - with nested unlimited repeats that aren't going to match. Writing - separate code for cased/caseless versions makes it go faster, as does - using an autoincrement and backing off on a match. - - HOWEVER: when the subject string is very, very long, searching to its end - can take a long time, and give bad performance on quite ordinary - patterns. This showed up when somebody was matching /^C/ on a 32-megabyte - string... so we don't do this when the string is sufficiently long. */ - - if (has_req_char && end_subject - current_subject < REQ_BYTE_MAX) - { - register PCRE_PUCHAR p = current_subject + (has_first_char? 1:0); - - /* We don't need to repeat the search if we haven't yet reached the - place we found it at last time. */ - - if (p > req_char_ptr) - { - if (req_char != req_char2) - { - while (p < end_subject) - { - register pcre_uint32 pp = UCHAR21INCTEST(p); - if (pp == req_char || pp == req_char2) { p--; break; } - } - } - else - { - while (p < end_subject) - { - if (UCHAR21INCTEST(p) == req_char) { p--; break; } - } - } - - /* If we can't find the required pcre_uchar, break the matching loop, - which will cause a return or PCRE_ERROR_NOMATCH. */ - - if (p >= end_subject) break; - - /* If we have found the required pcre_uchar, save the point where we - found it, so that we don't search again next time round the loop if - the start hasn't passed this point yet. */ - - req_char_ptr = p; - } - } - } - } /* End of optimizations that are done when not restarting */ - - /* OK, now we can do the business */ - - md->start_used_ptr = current_subject; - md->recursive = NULL; - - rc = internal_dfa_exec( - md, /* fixed match data */ - md->start_code, /* this subexpression's code */ - current_subject, /* where we currently are */ - start_offset, /* start offset in subject */ - offsets, /* offset vector */ - offsetcount, /* size of same */ - workspace, /* workspace vector */ - wscount, /* size of same */ - 0); /* function recurse level */ - - /* Anything other than "no match" means we are done, always; otherwise, carry - on only if not anchored. */ - - if (rc != PCRE_ERROR_NOMATCH || anchored) - { - if (rc == PCRE_ERROR_PARTIAL && offsetcount >= 2) - { - offsets[0] = (int)(md->start_used_ptr - (PCRE_PUCHAR)subject); - offsets[1] = (int)(end_subject - (PCRE_PUCHAR)subject); - if (offsetcount > 2) - offsets[2] = (int)(current_subject - (PCRE_PUCHAR)subject); - } - return rc; - } - - /* Advance to the next subject character unless we are at the end of a line - and firstline is set. */ - - if (firstline && IS_NEWLINE(current_subject)) break; - current_subject++; -#ifdef SUPPORT_UTF - if (utf) - { - ACROSSCHAR(current_subject < end_subject, *current_subject, - current_subject++); - } -#endif - if (current_subject > end_subject) break; - - /* If we have just passed a CR and we are now at a LF, and the pattern does - not contain any explicit matches for \r or \n, and the newline option is CRLF - or ANY or ANYCRLF, advance the match position by one more character. */ - - if (UCHAR21TEST(current_subject - 1) == CHAR_CR && - current_subject < end_subject && - UCHAR21TEST(current_subject) == CHAR_NL && - (re->flags & PCRE_HASCRORLF) == 0 && - (md->nltype == NLTYPE_ANY || - md->nltype == NLTYPE_ANYCRLF || - md->nllen == 2)) - current_subject++; - - } /* "Bumpalong" loop */ - -return PCRE_ERROR_NOMATCH; -} - -/* End of pcre_dfa_exec.c */ diff --git a/src/third_party/pcre-8.39/pcre_exec.c b/src/third_party/pcre-8.39/pcre_exec.c deleted file mode 100644 index 24b23ca2864..00000000000 --- a/src/third_party/pcre-8.39/pcre_exec.c +++ /dev/null @@ -1,7173 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2014 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* This module contains pcre_exec(), the externally visible function that does -pattern matching using an NFA algorithm, trying to mimic Perl as closely as -possible. There are also some static supporting functions. */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#define NLBLOCK md /* Block containing newline information */ -#define PSSTART start_subject /* Field containing processed string start */ -#define PSEND end_subject /* Field containing processed string end */ - -#include "pcre_internal.h" - -/* Undefine some potentially clashing cpp symbols */ - -#undef min -#undef max - -/* The md->capture_last field uses the lower 16 bits for the last captured -substring (which can never be greater than 65535) and a bit in the top half -to mean "capture vector overflowed". This odd way of doing things was -implemented when it was realized that preserving and restoring the overflow bit -whenever the last capture number was saved/restored made for a neater -interface, and doing it this way saved on (a) another variable, which would -have increased the stack frame size (a big NO-NO in PCRE) and (b) another -separate set of save/restore instructions. The following defines are used in -implementing this. */ - -#define CAPLMASK 0x0000ffff /* The bits used for last_capture */ -#define OVFLMASK 0xffff0000 /* The bits used for the overflow flag */ -#define OVFLBIT 0x00010000 /* The bit that is set for overflow */ - -/* Values for setting in md->match_function_type to indicate two special types -of call to match(). We do it this way to save on using another stack variable, -as stack usage is to be discouraged. */ - -#define MATCH_CONDASSERT 1 /* Called to check a condition assertion */ -#define MATCH_CBEGROUP 2 /* Could-be-empty unlimited repeat group */ - -/* Non-error returns from the match() function. Error returns are externally -defined PCRE_ERROR_xxx codes, which are all negative. */ - -#define MATCH_MATCH 1 -#define MATCH_NOMATCH 0 - -/* Special internal returns from the match() function. Make them sufficiently -negative to avoid the external error codes. */ - -#define MATCH_ACCEPT (-999) -#define MATCH_KETRPOS (-998) -#define MATCH_ONCE (-997) -/* The next 5 must be kept together and in sequence so that a test that checks -for any one of them can use a range. */ -#define MATCH_COMMIT (-996) -#define MATCH_PRUNE (-995) -#define MATCH_SKIP (-994) -#define MATCH_SKIP_ARG (-993) -#define MATCH_THEN (-992) -#define MATCH_BACKTRACK_MAX MATCH_THEN -#define MATCH_BACKTRACK_MIN MATCH_COMMIT - -/* Maximum number of ints of offset to save on the stack for recursive calls. -If the offset vector is bigger, malloc is used. This should be a multiple of 3, -because the offset vector is always a multiple of 3 long. */ - -#define REC_STACK_SAVE_MAX 30 - -/* Min and max values for the common repeats; for the maxima, 0 => infinity */ - -static const char rep_min[] = { 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, }; -static const char rep_max[] = { 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, }; - -#ifdef PCRE_DEBUG -/************************************************* -* Debugging function to print chars * -*************************************************/ - -/* Print a sequence of chars in printable format, stopping at the end of the -subject if the requested. - -Arguments: - p points to characters - length number to print - is_subject TRUE if printing from within md->start_subject - md pointer to matching data block, if is_subject is TRUE - -Returns: nothing -*/ - -static void -pchars(const pcre_uchar *p, int length, BOOL is_subject, match_data *md) -{ -pcre_uint32 c; -BOOL utf = md->utf; -if (is_subject && length > md->end_subject - p) length = md->end_subject - p; -while (length-- > 0) - if (isprint(c = UCHAR21INCTEST(p))) printf("%c", (char)c); else printf("\\x{%02x}", c); -} -#endif - - - -/************************************************* -* Match a back-reference * -*************************************************/ - -/* Normally, if a back reference hasn't been set, the length that is passed is -negative, so the match always fails. However, in JavaScript compatibility mode, -the length passed is zero. Note that in caseless UTF-8 mode, the number of -subject bytes matched may be different to the number of reference bytes. - -Arguments: - offset index into the offset vector - eptr pointer into the subject - length length of reference to be matched (number of bytes) - md points to match data block - caseless TRUE if caseless - -Returns: >= 0 the number of subject bytes matched - -1 no match - -2 partial match; always given if at end subject -*/ - -static int -match_ref(int offset, register PCRE_PUCHAR eptr, int length, match_data *md, - BOOL caseless) -{ -PCRE_PUCHAR eptr_start = eptr; -register PCRE_PUCHAR p = md->start_subject + md->offset_vector[offset]; -#if defined SUPPORT_UTF && defined SUPPORT_UCP -BOOL utf = md->utf; -#endif - -#ifdef PCRE_DEBUG -if (eptr >= md->end_subject) - printf("matching subject "); -else - { - printf("matching subject "); - pchars(eptr, length, TRUE, md); - } -printf(" against backref "); -pchars(p, length, FALSE, md); -printf("\n"); -#endif - -/* Always fail if reference not set (and not JavaScript compatible - in that -case the length is passed as zero). */ - -if (length < 0) return -1; - -/* Separate the caseless case for speed. In UTF-8 mode we can only do this -properly if Unicode properties are supported. Otherwise, we can check only -ASCII characters. */ - -if (caseless) - { -#if defined SUPPORT_UTF && defined SUPPORT_UCP - if (utf) - { - /* Match characters up to the end of the reference. NOTE: the number of - data units matched may differ, because in UTF-8 there are some characters - whose upper and lower case versions code have different numbers of bytes. - For example, U+023A (2 bytes in UTF-8) is the upper case version of U+2C65 - (3 bytes in UTF-8); a sequence of 3 of the former uses 6 bytes, as does a - sequence of two of the latter. It is important, therefore, to check the - length along the reference, not along the subject (earlier code did this - wrong). */ - - PCRE_PUCHAR endptr = p + length; - while (p < endptr) - { - pcre_uint32 c, d; - const ucd_record *ur; - if (eptr >= md->end_subject) return -2; /* Partial match */ - GETCHARINC(c, eptr); - GETCHARINC(d, p); - ur = GET_UCD(d); - if (c != d && c != d + ur->other_case) - { - const pcre_uint32 *pp = PRIV(ucd_caseless_sets) + ur->caseset; - for (;;) - { - if (c < *pp) return -1; - if (c == *pp++) break; - } - } - } - } - else -#endif - - /* The same code works when not in UTF-8 mode and in UTF-8 mode when there - is no UCP support. */ - { - while (length-- > 0) - { - pcre_uint32 cc, cp; - if (eptr >= md->end_subject) return -2; /* Partial match */ - cc = UCHAR21TEST(eptr); - cp = UCHAR21TEST(p); - if (TABLE_GET(cp, md->lcc, cp) != TABLE_GET(cc, md->lcc, cc)) return -1; - p++; - eptr++; - } - } - } - -/* In the caseful case, we can just compare the bytes, whether or not we -are in UTF-8 mode. */ - -else - { - while (length-- > 0) - { - if (eptr >= md->end_subject) return -2; /* Partial match */ - if (UCHAR21INCTEST(p) != UCHAR21INCTEST(eptr)) return -1; - } - } - -return (int)(eptr - eptr_start); -} - - - -/*************************************************************************** -**************************************************************************** - RECURSION IN THE match() FUNCTION - -The match() function is highly recursive, though not every recursive call -increases the recursive depth. Nevertheless, some regular expressions can cause -it to recurse to a great depth. I was writing for Unix, so I just let it call -itself recursively. This uses the stack for saving everything that has to be -saved for a recursive call. On Unix, the stack can be large, and this works -fine. - -It turns out that on some non-Unix-like systems there are problems with -programs that use a lot of stack. (This despite the fact that every last chip -has oodles of memory these days, and techniques for extending the stack have -been known for decades.) So.... - -There is a fudge, triggered by defining NO_RECURSE, which avoids recursive -calls by keeping local variables that need to be preserved in blocks of memory -obtained from malloc() instead instead of on the stack. Macros are used to -achieve this so that the actual code doesn't look very different to what it -always used to. - -The original heap-recursive code used longjmp(). However, it seems that this -can be very slow on some operating systems. Following a suggestion from Stan -Switzer, the use of longjmp() has been abolished, at the cost of having to -provide a unique number for each call to RMATCH. There is no way of generating -a sequence of numbers at compile time in C. I have given them names, to make -them stand out more clearly. - -Crude tests on x86 Linux show a small speedup of around 5-8%. However, on -FreeBSD, avoiding longjmp() more than halves the time taken to run the standard -tests. Furthermore, not using longjmp() means that local dynamic variables -don't have indeterminate values; this has meant that the frame size can be -reduced because the result can be "passed back" by straight setting of the -variable instead of being passed in the frame. -**************************************************************************** -***************************************************************************/ - -/* Numbers for RMATCH calls. When this list is changed, the code at HEAP_RETURN -below must be updated in sync. */ - -enum { RM1=1, RM2, RM3, RM4, RM5, RM6, RM7, RM8, RM9, RM10, - RM11, RM12, RM13, RM14, RM15, RM16, RM17, RM18, RM19, RM20, - RM21, RM22, RM23, RM24, RM25, RM26, RM27, RM28, RM29, RM30, - RM31, RM32, RM33, RM34, RM35, RM36, RM37, RM38, RM39, RM40, - RM41, RM42, RM43, RM44, RM45, RM46, RM47, RM48, RM49, RM50, - RM51, RM52, RM53, RM54, RM55, RM56, RM57, RM58, RM59, RM60, - RM61, RM62, RM63, RM64, RM65, RM66, RM67 }; - -/* These versions of the macros use the stack, as normal. There are debugging -versions and production versions. Note that the "rw" argument of RMATCH isn't -actually used in this definition. */ - -#ifndef NO_RECURSE -#define REGISTER register - -#ifdef PCRE_DEBUG -#define RMATCH(ra,rb,rc,rd,re,rw) \ - { \ - printf("match() called in line %d\n", __LINE__); \ - rrc = match(ra,rb,mstart,rc,rd,re,rdepth+1); \ - printf("to line %d\n", __LINE__); \ - } -#define RRETURN(ra) \ - { \ - printf("match() returned %d from line %d\n", ra, __LINE__); \ - return ra; \ - } -#else -#define RMATCH(ra,rb,rc,rd,re,rw) \ - rrc = match(ra,rb,mstart,rc,rd,re,rdepth+1) -#define RRETURN(ra) return ra -#endif - -#else - - -/* These versions of the macros manage a private stack on the heap. Note that -the "rd" argument of RMATCH isn't actually used in this definition. It's the md -argument of match(), which never changes. */ - -#define REGISTER - -#define RMATCH(ra,rb,rc,rd,re,rw)\ - {\ - heapframe *newframe = frame->Xnextframe;\ - if (newframe == NULL)\ - {\ - newframe = (heapframe *)(PUBL(stack_malloc))(sizeof(heapframe));\ - if (newframe == NULL) RRETURN(PCRE_ERROR_NOMEMORY);\ - newframe->Xnextframe = NULL;\ - frame->Xnextframe = newframe;\ - }\ - frame->Xwhere = rw;\ - newframe->Xeptr = ra;\ - newframe->Xecode = rb;\ - newframe->Xmstart = mstart;\ - newframe->Xoffset_top = rc;\ - newframe->Xeptrb = re;\ - newframe->Xrdepth = frame->Xrdepth + 1;\ - newframe->Xprevframe = frame;\ - frame = newframe;\ - DPRINTF(("restarting from line %d\n", __LINE__));\ - goto HEAP_RECURSE;\ - L_##rw:\ - DPRINTF(("jumped back to line %d\n", __LINE__));\ - } - -#define RRETURN(ra)\ - {\ - heapframe *oldframe = frame;\ - frame = oldframe->Xprevframe;\ - if (frame != NULL)\ - {\ - rrc = ra;\ - goto HEAP_RETURN;\ - }\ - return ra;\ - } - - -/* Structure for remembering the local variables in a private frame */ - -typedef struct heapframe { - struct heapframe *Xprevframe; - struct heapframe *Xnextframe; - - /* Function arguments that may change */ - - PCRE_PUCHAR Xeptr; - const pcre_uchar *Xecode; - PCRE_PUCHAR Xmstart; - int Xoffset_top; - eptrblock *Xeptrb; - unsigned int Xrdepth; - - /* Function local variables */ - - PCRE_PUCHAR Xcallpat; -#ifdef SUPPORT_UTF - PCRE_PUCHAR Xcharptr; -#endif - PCRE_PUCHAR Xdata; - PCRE_PUCHAR Xnext; - PCRE_PUCHAR Xpp; - PCRE_PUCHAR Xprev; - PCRE_PUCHAR Xsaved_eptr; - - recursion_info Xnew_recursive; - - BOOL Xcur_is_word; - BOOL Xcondition; - BOOL Xprev_is_word; - -#ifdef SUPPORT_UCP - int Xprop_type; - unsigned int Xprop_value; - int Xprop_fail_result; - int Xoclength; - pcre_uchar Xocchars[6]; -#endif - - int Xcodelink; - int Xctype; - unsigned int Xfc; - int Xfi; - int Xlength; - int Xmax; - int Xmin; - unsigned int Xnumber; - int Xoffset; - unsigned int Xop; - pcre_int32 Xsave_capture_last; - int Xsave_offset1, Xsave_offset2, Xsave_offset3; - int Xstacksave[REC_STACK_SAVE_MAX]; - - eptrblock Xnewptrb; - - /* Where to jump back to */ - - int Xwhere; - -} heapframe; - -#endif - - -/*************************************************************************** -***************************************************************************/ - - - -/************************************************* -* Match from current position * -*************************************************/ - -/* This function is called recursively in many circumstances. Whenever it -returns a negative (error) response, the outer incarnation must also return the -same response. */ - -/* These macros pack up tests that are used for partial matching, and which -appear several times in the code. We set the "hit end" flag if the pointer is -at the end of the subject and also past the start of the subject (i.e. -something has been matched). For hard partial matching, we then return -immediately. The second one is used when we already know we are past the end of -the subject. */ - -#define CHECK_PARTIAL()\ - if (md->partial != 0 && eptr >= md->end_subject && \ - eptr > md->start_used_ptr) \ - { \ - md->hitend = TRUE; \ - if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); \ - } - -#define SCHECK_PARTIAL()\ - if (md->partial != 0 && eptr > md->start_used_ptr) \ - { \ - md->hitend = TRUE; \ - if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); \ - } - - -/* Performance note: It might be tempting to extract commonly used fields from -the md structure (e.g. utf, end_subject) into individual variables to improve -performance. Tests using gcc on a SPARC disproved this; in the first case, it -made performance worse. - -Arguments: - eptr pointer to current character in subject - ecode pointer to current position in compiled code - mstart pointer to the current match start position (can be modified - by encountering \K) - offset_top current top pointer - md pointer to "static" info for the match - eptrb pointer to chain of blocks containing eptr at start of - brackets - for testing for empty matches - rdepth the recursion depth - -Returns: MATCH_MATCH if matched ) these values are >= 0 - MATCH_NOMATCH if failed to match ) - a negative MATCH_xxx value for PRUNE, SKIP, etc - a negative PCRE_ERROR_xxx value if aborted by an error condition - (e.g. stopped by repeated call or recursion limit) -*/ - -static int -match(REGISTER PCRE_PUCHAR eptr, REGISTER const pcre_uchar *ecode, - PCRE_PUCHAR mstart, int offset_top, match_data *md, eptrblock *eptrb, - unsigned int rdepth) -{ -/* These variables do not need to be preserved over recursion in this function, -so they can be ordinary variables in all cases. Mark some of them with -"register" because they are used a lot in loops. */ - -register int rrc; /* Returns from recursive calls */ -register int i; /* Used for loops not involving calls to RMATCH() */ -register pcre_uint32 c; /* Character values not kept over RMATCH() calls */ -register BOOL utf; /* Local copy of UTF flag for speed */ - -BOOL minimize, possessive; /* Quantifier options */ -BOOL caseless; -int condcode; - -/* When recursion is not being used, all "local" variables that have to be -preserved over calls to RMATCH() are part of a "frame". We set up the top-level -frame on the stack here; subsequent instantiations are obtained from the heap -whenever RMATCH() does a "recursion". See the macro definitions above. Putting -the top-level on the stack rather than malloc-ing them all gives a performance -boost in many cases where there is not much "recursion". */ - -#ifdef NO_RECURSE -heapframe *frame = (heapframe *)md->match_frames_base; - -/* Copy in the original argument variables */ - -frame->Xeptr = eptr; -frame->Xecode = ecode; -frame->Xmstart = mstart; -frame->Xoffset_top = offset_top; -frame->Xeptrb = eptrb; -frame->Xrdepth = rdepth; - -/* This is where control jumps back to to effect "recursion" */ - -HEAP_RECURSE: - -/* Macros make the argument variables come from the current frame */ - -#define eptr frame->Xeptr -#define ecode frame->Xecode -#define mstart frame->Xmstart -#define offset_top frame->Xoffset_top -#define eptrb frame->Xeptrb -#define rdepth frame->Xrdepth - -/* Ditto for the local variables */ - -#ifdef SUPPORT_UTF -#define charptr frame->Xcharptr -#endif -#define callpat frame->Xcallpat -#define codelink frame->Xcodelink -#define data frame->Xdata -#define next frame->Xnext -#define pp frame->Xpp -#define prev frame->Xprev -#define saved_eptr frame->Xsaved_eptr - -#define new_recursive frame->Xnew_recursive - -#define cur_is_word frame->Xcur_is_word -#define condition frame->Xcondition -#define prev_is_word frame->Xprev_is_word - -#ifdef SUPPORT_UCP -#define prop_type frame->Xprop_type -#define prop_value frame->Xprop_value -#define prop_fail_result frame->Xprop_fail_result -#define oclength frame->Xoclength -#define occhars frame->Xocchars -#endif - -#define ctype frame->Xctype -#define fc frame->Xfc -#define fi frame->Xfi -#define length frame->Xlength -#define max frame->Xmax -#define min frame->Xmin -#define number frame->Xnumber -#define offset frame->Xoffset -#define op frame->Xop -#define save_capture_last frame->Xsave_capture_last -#define save_offset1 frame->Xsave_offset1 -#define save_offset2 frame->Xsave_offset2 -#define save_offset3 frame->Xsave_offset3 -#define stacksave frame->Xstacksave - -#define newptrb frame->Xnewptrb - -/* When recursion is being used, local variables are allocated on the stack and -get preserved during recursion in the normal way. In this environment, fi and -i, and fc and c, can be the same variables. */ - -#else /* NO_RECURSE not defined */ -#define fi i -#define fc c - -/* Many of the following variables are used only in small blocks of the code. -My normal style of coding would have declared them within each of those blocks. -However, in order to accommodate the version of this code that uses an external -"stack" implemented on the heap, it is easier to declare them all here, so the -declarations can be cut out in a block. The only declarations within blocks -below are for variables that do not have to be preserved over a recursive call -to RMATCH(). */ - -#ifdef SUPPORT_UTF -const pcre_uchar *charptr; -#endif -const pcre_uchar *callpat; -const pcre_uchar *data; -const pcre_uchar *next; -PCRE_PUCHAR pp; -const pcre_uchar *prev; -PCRE_PUCHAR saved_eptr; - -recursion_info new_recursive; - -BOOL cur_is_word; -BOOL condition; -BOOL prev_is_word; - -#ifdef SUPPORT_UCP -int prop_type; -unsigned int prop_value; -int prop_fail_result; -int oclength; -pcre_uchar occhars[6]; -#endif - -int codelink; -int ctype; -int length; -int max; -int min; -unsigned int number; -int offset; -unsigned int op; -pcre_int32 save_capture_last; -int save_offset1, save_offset2, save_offset3; -int stacksave[REC_STACK_SAVE_MAX]; - -eptrblock newptrb; - -/* There is a special fudge for calling match() in a way that causes it to -measure the size of its basic stack frame when the stack is being used for -recursion. The second argument (ecode) being NULL triggers this behaviour. It -cannot normally ever be NULL. The return is the negated value of the frame -size. */ - -if (ecode == NULL) - { - if (rdepth == 0) - return match((PCRE_PUCHAR)&rdepth, NULL, NULL, 0, NULL, NULL, 1); - else - { - int len = (char *)&rdepth - (char *)eptr; - return (len > 0)? -len : len; - } - } -#endif /* NO_RECURSE */ - -/* To save space on the stack and in the heap frame, I have doubled up on some -of the local variables that are used only in localised parts of the code, but -still need to be preserved over recursive calls of match(). These macros define -the alternative names that are used. */ - -#define allow_zero cur_is_word -#define cbegroup condition -#define code_offset codelink -#define condassert condition -#define matched_once prev_is_word -#define foc number -#define save_mark data - -/* These statements are here to stop the compiler complaining about unitialized -variables. */ - -#ifdef SUPPORT_UCP -prop_value = 0; -prop_fail_result = 0; -#endif - - -/* This label is used for tail recursion, which is used in a few cases even -when NO_RECURSE is not defined, in order to reduce the amount of stack that is -used. Thanks to Ian Taylor for noticing this possibility and sending the -original patch. */ - -TAIL_RECURSE: - -/* OK, now we can get on with the real code of the function. Recursive calls -are specified by the macro RMATCH and RRETURN is used to return. When -NO_RECURSE is *not* defined, these just turn into a recursive call to match() -and a "return", respectively (possibly with some debugging if PCRE_DEBUG is -defined). However, RMATCH isn't like a function call because it's quite a -complicated macro. It has to be used in one particular way. This shouldn't, -however, impact performance when true recursion is being used. */ - -#ifdef SUPPORT_UTF -utf = md->utf; /* Local copy of the flag */ -#else -utf = FALSE; -#endif - -/* First check that we haven't called match() too many times, or that we -haven't exceeded the recursive call limit. */ - -if (md->match_call_count++ >= md->match_limit) RRETURN(PCRE_ERROR_MATCHLIMIT); -if (rdepth >= md->match_limit_recursion) RRETURN(PCRE_ERROR_RECURSIONLIMIT); - -/* At the start of a group with an unlimited repeat that may match an empty -string, the variable md->match_function_type is set to MATCH_CBEGROUP. It is -done this way to save having to use another function argument, which would take -up space on the stack. See also MATCH_CONDASSERT below. - -When MATCH_CBEGROUP is set, add the current subject pointer to the chain of -such remembered pointers, to be checked when we hit the closing ket, in order -to break infinite loops that match no characters. When match() is called in -other circumstances, don't add to the chain. The MATCH_CBEGROUP feature must -NOT be used with tail recursion, because the memory block that is used is on -the stack, so a new one may be required for each match(). */ - -if (md->match_function_type == MATCH_CBEGROUP) - { - newptrb.epb_saved_eptr = eptr; - newptrb.epb_prev = eptrb; - eptrb = &newptrb; - md->match_function_type = 0; - } - -/* Now start processing the opcodes. */ - -for (;;) - { - minimize = possessive = FALSE; - op = *ecode; - - switch(op) - { - case OP_MARK: - md->nomatch_mark = ecode + 2; - md->mark = NULL; /* In case previously set by assertion */ - RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode] + ecode[1], offset_top, md, - eptrb, RM55); - if ((rrc == MATCH_MATCH || rrc == MATCH_ACCEPT) && - md->mark == NULL) md->mark = ecode + 2; - - /* A return of MATCH_SKIP_ARG means that matching failed at SKIP with an - argument, and we must check whether that argument matches this MARK's - argument. It is passed back in md->start_match_ptr (an overloading of that - variable). If it does match, we reset that variable to the current subject - position and return MATCH_SKIP. Otherwise, pass back the return code - unaltered. */ - - else if (rrc == MATCH_SKIP_ARG && - STRCMP_UC_UC_TEST(ecode + 2, md->start_match_ptr) == 0) - { - md->start_match_ptr = eptr; - RRETURN(MATCH_SKIP); - } - RRETURN(rrc); - - case OP_FAIL: - RRETURN(MATCH_NOMATCH); - - case OP_COMMIT: - RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode], offset_top, md, - eptrb, RM52); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - RRETURN(MATCH_COMMIT); - - case OP_PRUNE: - RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode], offset_top, md, - eptrb, RM51); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - RRETURN(MATCH_PRUNE); - - case OP_PRUNE_ARG: - md->nomatch_mark = ecode + 2; - md->mark = NULL; /* In case previously set by assertion */ - RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode] + ecode[1], offset_top, md, - eptrb, RM56); - if ((rrc == MATCH_MATCH || rrc == MATCH_ACCEPT) && - md->mark == NULL) md->mark = ecode + 2; - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - RRETURN(MATCH_PRUNE); - - case OP_SKIP: - RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode], offset_top, md, - eptrb, RM53); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - md->start_match_ptr = eptr; /* Pass back current position */ - RRETURN(MATCH_SKIP); - - /* Note that, for Perl compatibility, SKIP with an argument does NOT set - nomatch_mark. When a pattern match ends with a SKIP_ARG for which there was - not a matching mark, we have to re-run the match, ignoring the SKIP_ARG - that failed and any that precede it (either they also failed, or were not - triggered). To do this, we maintain a count of executed SKIP_ARGs. If a - SKIP_ARG gets to top level, the match is re-run with md->ignore_skip_arg - set to the count of the one that failed. */ - - case OP_SKIP_ARG: - md->skip_arg_count++; - if (md->skip_arg_count <= md->ignore_skip_arg) - { - ecode += PRIV(OP_lengths)[*ecode] + ecode[1]; - break; - } - RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode] + ecode[1], offset_top, md, - eptrb, RM57); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - - /* Pass back the current skip name by overloading md->start_match_ptr and - returning the special MATCH_SKIP_ARG return code. This will either be - caught by a matching MARK, or get to the top, where it causes a rematch - with md->ignore_skip_arg set to the value of md->skip_arg_count. */ - - md->start_match_ptr = ecode + 2; - RRETURN(MATCH_SKIP_ARG); - - /* For THEN (and THEN_ARG) we pass back the address of the opcode, so that - the branch in which it occurs can be determined. Overload the start of - match pointer to do this. */ - - case OP_THEN: - RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode], offset_top, md, - eptrb, RM54); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - md->start_match_ptr = ecode; - RRETURN(MATCH_THEN); - - case OP_THEN_ARG: - md->nomatch_mark = ecode + 2; - md->mark = NULL; /* In case previously set by assertion */ - RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode] + ecode[1], offset_top, - md, eptrb, RM58); - if ((rrc == MATCH_MATCH || rrc == MATCH_ACCEPT) && - md->mark == NULL) md->mark = ecode + 2; - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - md->start_match_ptr = ecode; - RRETURN(MATCH_THEN); - - /* Handle an atomic group that does not contain any capturing parentheses. - This can be handled like an assertion. Prior to 8.13, all atomic groups - were handled this way. In 8.13, the code was changed as below for ONCE, so - that backups pass through the group and thereby reset captured values. - However, this uses a lot more stack, so in 8.20, atomic groups that do not - contain any captures generate OP_ONCE_NC, which can be handled in the old, - less stack intensive way. - - Check the alternative branches in turn - the matching won't pass the KET - for this kind of subpattern. If any one branch matches, we carry on as at - the end of a normal bracket, leaving the subject pointer, but resetting - the start-of-match value in case it was changed by \K. */ - - case OP_ONCE_NC: - prev = ecode; - saved_eptr = eptr; - save_mark = md->mark; - do - { - RMATCH(eptr, ecode + 1 + LINK_SIZE, offset_top, md, eptrb, RM64); - if (rrc == MATCH_MATCH) /* Note: _not_ MATCH_ACCEPT */ - { - mstart = md->start_match_ptr; - break; - } - if (rrc == MATCH_THEN) - { - next = ecode + GET(ecode,1); - if (md->start_match_ptr < next && - (*ecode == OP_ALT || *next == OP_ALT)) - rrc = MATCH_NOMATCH; - } - - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - ecode += GET(ecode,1); - md->mark = save_mark; - } - while (*ecode == OP_ALT); - - /* If hit the end of the group (which could be repeated), fail */ - - if (*ecode != OP_ONCE_NC && *ecode != OP_ALT) RRETURN(MATCH_NOMATCH); - - /* Continue as from after the group, updating the offsets high water - mark, since extracts may have been taken. */ - - do ecode += GET(ecode, 1); while (*ecode == OP_ALT); - - offset_top = md->end_offset_top; - eptr = md->end_match_ptr; - - /* For a non-repeating ket, just continue at this level. This also - happens for a repeating ket if no characters were matched in the group. - This is the forcible breaking of infinite loops as implemented in Perl - 5.005. */ - - if (*ecode == OP_KET || eptr == saved_eptr) - { - ecode += 1+LINK_SIZE; - break; - } - - /* The repeating kets try the rest of the pattern or restart from the - preceding bracket, in the appropriate order. The second "call" of match() - uses tail recursion, to avoid using another stack frame. */ - - if (*ecode == OP_KETRMIN) - { - RMATCH(eptr, ecode + 1 + LINK_SIZE, offset_top, md, eptrb, RM65); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - ecode = prev; - goto TAIL_RECURSE; - } - else /* OP_KETRMAX */ - { - RMATCH(eptr, prev, offset_top, md, eptrb, RM66); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - ecode += 1 + LINK_SIZE; - goto TAIL_RECURSE; - } - /* Control never gets here */ - - /* Handle a capturing bracket, other than those that are possessive with an - unlimited repeat. If there is space in the offset vector, save the current - subject position in the working slot at the top of the vector. We mustn't - change the current values of the data slot, because they may be set from a - previous iteration of this group, and be referred to by a reference inside - the group. A failure to match might occur after the group has succeeded, - if something later on doesn't match. For this reason, we need to restore - the working value and also the values of the final offsets, in case they - were set by a previous iteration of the same bracket. - - If there isn't enough space in the offset vector, treat this as if it were - a non-capturing bracket. Don't worry about setting the flag for the error - case here; that is handled in the code for KET. */ - - case OP_CBRA: - case OP_SCBRA: - number = GET2(ecode, 1+LINK_SIZE); - offset = number << 1; - -#ifdef PCRE_DEBUG - printf("start bracket %d\n", number); - printf("subject="); - pchars(eptr, 16, TRUE, md); - printf("\n"); -#endif - - if (offset < md->offset_max) - { - save_offset1 = md->offset_vector[offset]; - save_offset2 = md->offset_vector[offset+1]; - save_offset3 = md->offset_vector[md->offset_end - number]; - save_capture_last = md->capture_last; - save_mark = md->mark; - - DPRINTF(("saving %d %d %d\n", save_offset1, save_offset2, save_offset3)); - md->offset_vector[md->offset_end - number] = - (int)(eptr - md->start_subject); - - for (;;) - { - if (op >= OP_SBRA) md->match_function_type = MATCH_CBEGROUP; - RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode], offset_top, md, - eptrb, RM1); - if (rrc == MATCH_ONCE) break; /* Backing up through an atomic group */ - - /* If we backed up to a THEN, check whether it is within the current - branch by comparing the address of the THEN that is passed back with - the end of the branch. If it is within the current branch, and the - branch is one of two or more alternatives (it either starts or ends - with OP_ALT), we have reached the limit of THEN's action, so convert - the return code to NOMATCH, which will cause normal backtracking to - happen from now on. Otherwise, THEN is passed back to an outer - alternative. This implements Perl's treatment of parenthesized groups, - where a group not containing | does not affect the current alternative, - that is, (X) is NOT the same as (X|(*F)). */ - - if (rrc == MATCH_THEN) - { - next = ecode + GET(ecode,1); - if (md->start_match_ptr < next && - (*ecode == OP_ALT || *next == OP_ALT)) - rrc = MATCH_NOMATCH; - } - - /* Anything other than NOMATCH is passed back. */ - - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - md->capture_last = save_capture_last; - ecode += GET(ecode, 1); - md->mark = save_mark; - if (*ecode != OP_ALT) break; - } - - DPRINTF(("bracket %d failed\n", number)); - md->offset_vector[offset] = save_offset1; - md->offset_vector[offset+1] = save_offset2; - md->offset_vector[md->offset_end - number] = save_offset3; - - /* At this point, rrc will be one of MATCH_ONCE or MATCH_NOMATCH. */ - - RRETURN(rrc); - } - - /* FALL THROUGH ... Insufficient room for saving captured contents. Treat - as a non-capturing bracket. */ - - /* VVVVVVVVVVVVVVVVVVVVVVVVV */ - /* VVVVVVVVVVVVVVVVVVVVVVVVV */ - - DPRINTF(("insufficient capture room: treat as non-capturing\n")); - - /* VVVVVVVVVVVVVVVVVVVVVVVVV */ - /* VVVVVVVVVVVVVVVVVVVVVVVVV */ - - /* Non-capturing or atomic group, except for possessive with unlimited - repeat and ONCE group with no captures. Loop for all the alternatives. - - When we get to the final alternative within the brackets, we used to return - the result of a recursive call to match() whatever happened so it was - possible to reduce stack usage by turning this into a tail recursion, - except in the case of a possibly empty group. However, now that there is - the possiblity of (*THEN) occurring in the final alternative, this - optimization is no longer always possible. - - We can optimize if we know there are no (*THEN)s in the pattern; at present - this is the best that can be done. - - MATCH_ONCE is returned when the end of an atomic group is successfully - reached, but subsequent matching fails. It passes back up the tree (causing - captured values to be reset) until the original atomic group level is - reached. This is tested by comparing md->once_target with the start of the - group. At this point, the return is converted into MATCH_NOMATCH so that - previous backup points can be taken. */ - - case OP_ONCE: - case OP_BRA: - case OP_SBRA: - DPRINTF(("start non-capturing bracket\n")); - - for (;;) - { - if (op >= OP_SBRA || op == OP_ONCE) - md->match_function_type = MATCH_CBEGROUP; - - /* If this is not a possibly empty group, and there are no (*THEN)s in - the pattern, and this is the final alternative, optimize as described - above. */ - - else if (!md->hasthen && ecode[GET(ecode, 1)] != OP_ALT) - { - ecode += PRIV(OP_lengths)[*ecode]; - goto TAIL_RECURSE; - } - - /* In all other cases, we have to make another call to match(). */ - - save_mark = md->mark; - save_capture_last = md->capture_last; - RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode], offset_top, md, eptrb, - RM2); - - /* See comment in the code for capturing groups above about handling - THEN. */ - - if (rrc == MATCH_THEN) - { - next = ecode + GET(ecode,1); - if (md->start_match_ptr < next && - (*ecode == OP_ALT || *next == OP_ALT)) - rrc = MATCH_NOMATCH; - } - - if (rrc != MATCH_NOMATCH) - { - if (rrc == MATCH_ONCE) - { - const pcre_uchar *scode = ecode; - if (*scode != OP_ONCE) /* If not at start, find it */ - { - while (*scode == OP_ALT) scode += GET(scode, 1); - scode -= GET(scode, 1); - } - if (md->once_target == scode) rrc = MATCH_NOMATCH; - } - RRETURN(rrc); - } - ecode += GET(ecode, 1); - md->mark = save_mark; - if (*ecode != OP_ALT) break; - md->capture_last = save_capture_last; - } - - RRETURN(MATCH_NOMATCH); - - /* Handle possessive capturing brackets with an unlimited repeat. We come - here from BRAZERO with allow_zero set TRUE. The offset_vector values are - handled similarly to the normal case above. However, the matching is - different. The end of these brackets will always be OP_KETRPOS, which - returns MATCH_KETRPOS without going further in the pattern. By this means - we can handle the group by iteration rather than recursion, thereby - reducing the amount of stack needed. */ - - case OP_CBRAPOS: - case OP_SCBRAPOS: - allow_zero = FALSE; - - POSSESSIVE_CAPTURE: - number = GET2(ecode, 1+LINK_SIZE); - offset = number << 1; - -#ifdef PCRE_DEBUG - printf("start possessive bracket %d\n", number); - printf("subject="); - pchars(eptr, 16, TRUE, md); - printf("\n"); -#endif - - if (offset >= md->offset_max) goto POSSESSIVE_NON_CAPTURE; - - matched_once = FALSE; - code_offset = (int)(ecode - md->start_code); - - save_offset1 = md->offset_vector[offset]; - save_offset2 = md->offset_vector[offset+1]; - save_offset3 = md->offset_vector[md->offset_end - number]; - save_capture_last = md->capture_last; - - DPRINTF(("saving %d %d %d\n", save_offset1, save_offset2, save_offset3)); - - /* Each time round the loop, save the current subject position for use - when the group matches. For MATCH_MATCH, the group has matched, so we - restart it with a new subject starting position, remembering that we had - at least one match. For MATCH_NOMATCH, carry on with the alternatives, as - usual. If we haven't matched any alternatives in any iteration, check to - see if a previous iteration matched. If so, the group has matched; - continue from afterwards. Otherwise it has failed; restore the previous - capture values before returning NOMATCH. */ - - for (;;) - { - md->offset_vector[md->offset_end - number] = - (int)(eptr - md->start_subject); - if (op >= OP_SBRA) md->match_function_type = MATCH_CBEGROUP; - RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode], offset_top, md, - eptrb, RM63); - if (rrc == MATCH_KETRPOS) - { - offset_top = md->end_offset_top; - ecode = md->start_code + code_offset; - save_capture_last = md->capture_last; - matched_once = TRUE; - mstart = md->start_match_ptr; /* In case \K changed it */ - if (eptr == md->end_match_ptr) /* Matched an empty string */ - { - do ecode += GET(ecode, 1); while (*ecode == OP_ALT); - break; - } - eptr = md->end_match_ptr; - continue; - } - - /* See comment in the code for capturing groups above about handling - THEN. */ - - if (rrc == MATCH_THEN) - { - next = ecode + GET(ecode,1); - if (md->start_match_ptr < next && - (*ecode == OP_ALT || *next == OP_ALT)) - rrc = MATCH_NOMATCH; - } - - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - md->capture_last = save_capture_last; - ecode += GET(ecode, 1); - if (*ecode != OP_ALT) break; - } - - if (!matched_once) - { - md->offset_vector[offset] = save_offset1; - md->offset_vector[offset+1] = save_offset2; - md->offset_vector[md->offset_end - number] = save_offset3; - } - - if (allow_zero || matched_once) - { - ecode += 1 + LINK_SIZE; - break; - } - - RRETURN(MATCH_NOMATCH); - - /* Non-capturing possessive bracket with unlimited repeat. We come here - from BRAZERO with allow_zero = TRUE. The code is similar to the above, - without the capturing complication. It is written out separately for speed - and cleanliness. */ - - case OP_BRAPOS: - case OP_SBRAPOS: - allow_zero = FALSE; - - POSSESSIVE_NON_CAPTURE: - matched_once = FALSE; - code_offset = (int)(ecode - md->start_code); - save_capture_last = md->capture_last; - - for (;;) - { - if (op >= OP_SBRA) md->match_function_type = MATCH_CBEGROUP; - RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode], offset_top, md, - eptrb, RM48); - if (rrc == MATCH_KETRPOS) - { - offset_top = md->end_offset_top; - ecode = md->start_code + code_offset; - matched_once = TRUE; - mstart = md->start_match_ptr; /* In case \K reset it */ - if (eptr == md->end_match_ptr) /* Matched an empty string */ - { - do ecode += GET(ecode, 1); while (*ecode == OP_ALT); - break; - } - eptr = md->end_match_ptr; - continue; - } - - /* See comment in the code for capturing groups above about handling - THEN. */ - - if (rrc == MATCH_THEN) - { - next = ecode + GET(ecode,1); - if (md->start_match_ptr < next && - (*ecode == OP_ALT || *next == OP_ALT)) - rrc = MATCH_NOMATCH; - } - - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - ecode += GET(ecode, 1); - if (*ecode != OP_ALT) break; - md->capture_last = save_capture_last; - } - - if (matched_once || allow_zero) - { - ecode += 1 + LINK_SIZE; - break; - } - RRETURN(MATCH_NOMATCH); - - /* Control never reaches here. */ - - /* Conditional group: compilation checked that there are no more than two - branches. If the condition is false, skipping the first branch takes us - past the end of the item if there is only one branch, but that's exactly - what we want. */ - - case OP_COND: - case OP_SCOND: - - /* The variable codelink will be added to ecode when the condition is - false, to get to the second branch. Setting it to the offset to the ALT - or KET, then incrementing ecode achieves this effect. We now have ecode - pointing to the condition or callout. */ - - codelink = GET(ecode, 1); /* Offset to the second branch */ - ecode += 1 + LINK_SIZE; /* From this opcode */ - - /* Because of the way auto-callout works during compile, a callout item is - inserted between OP_COND and an assertion condition. */ - - if (*ecode == OP_CALLOUT) - { - if (PUBL(callout) != NULL) - { - PUBL(callout_block) cb; - cb.version = 2; /* Version 1 of the callout block */ - cb.callout_number = ecode[1]; - cb.offset_vector = md->offset_vector; -#if defined COMPILE_PCRE8 - cb.subject = (PCRE_SPTR)md->start_subject; -#elif defined COMPILE_PCRE16 - cb.subject = (PCRE_SPTR16)md->start_subject; -#elif defined COMPILE_PCRE32 - cb.subject = (PCRE_SPTR32)md->start_subject; -#endif - cb.subject_length = (int)(md->end_subject - md->start_subject); - cb.start_match = (int)(mstart - md->start_subject); - cb.current_position = (int)(eptr - md->start_subject); - cb.pattern_position = GET(ecode, 2); - cb.next_item_length = GET(ecode, 2 + LINK_SIZE); - cb.capture_top = offset_top/2; - cb.capture_last = md->capture_last & CAPLMASK; - /* Internal change requires this for API compatibility. */ - if (cb.capture_last == 0) cb.capture_last = -1; - cb.callout_data = md->callout_data; - cb.mark = md->nomatch_mark; - if ((rrc = (*PUBL(callout))(&cb)) > 0) RRETURN(MATCH_NOMATCH); - if (rrc < 0) RRETURN(rrc); - } - - /* Advance ecode past the callout, so it now points to the condition. We - must adjust codelink so that the value of ecode+codelink is unchanged. */ - - ecode += PRIV(OP_lengths)[OP_CALLOUT]; - codelink -= PRIV(OP_lengths)[OP_CALLOUT]; - } - - /* Test the various possible conditions */ - - condition = FALSE; - switch(condcode = *ecode) - { - case OP_RREF: /* Numbered group recursion test */ - if (md->recursive != NULL) /* Not recursing => FALSE */ - { - unsigned int recno = GET2(ecode, 1); /* Recursion group number*/ - condition = (recno == RREF_ANY || recno == md->recursive->group_num); - } - break; - - case OP_DNRREF: /* Duplicate named group recursion test */ - if (md->recursive != NULL) - { - int count = GET2(ecode, 1 + IMM2_SIZE); - pcre_uchar *slot = md->name_table + GET2(ecode, 1) * md->name_entry_size; - while (count-- > 0) - { - unsigned int recno = GET2(slot, 0); - condition = recno == md->recursive->group_num; - if (condition) break; - slot += md->name_entry_size; - } - } - break; - - case OP_CREF: /* Numbered group used test */ - offset = GET2(ecode, 1) << 1; /* Doubled ref number */ - condition = offset < offset_top && md->offset_vector[offset] >= 0; - break; - - case OP_DNCREF: /* Duplicate named group used test */ - { - int count = GET2(ecode, 1 + IMM2_SIZE); - pcre_uchar *slot = md->name_table + GET2(ecode, 1) * md->name_entry_size; - while (count-- > 0) - { - offset = GET2(slot, 0) << 1; - condition = offset < offset_top && md->offset_vector[offset] >= 0; - if (condition) break; - slot += md->name_entry_size; - } - } - break; - - case OP_DEF: /* DEFINE - always false */ - case OP_FAIL: /* From optimized (?!) condition */ - break; - - /* The condition is an assertion. Call match() to evaluate it - setting - md->match_function_type to MATCH_CONDASSERT causes it to stop at the end - of an assertion. */ - - default: - md->match_function_type = MATCH_CONDASSERT; - RMATCH(eptr, ecode, offset_top, md, NULL, RM3); - if (rrc == MATCH_MATCH) - { - if (md->end_offset_top > offset_top) - offset_top = md->end_offset_top; /* Captures may have happened */ - condition = TRUE; - - /* Advance ecode past the assertion to the start of the first branch, - but adjust it so that the general choosing code below works. If the - assertion has a quantifier that allows zero repeats we must skip over - the BRAZERO. This is a lunatic thing to do, but somebody did! */ - - if (*ecode == OP_BRAZERO) ecode++; - ecode += GET(ecode, 1); - while (*ecode == OP_ALT) ecode += GET(ecode, 1); - ecode += 1 + LINK_SIZE - PRIV(OP_lengths)[condcode]; - } - - /* PCRE doesn't allow the effect of (*THEN) to escape beyond an - assertion; it is therefore treated as NOMATCH. Any other return is an - error. */ - - else if (rrc != MATCH_NOMATCH && rrc != MATCH_THEN) - { - RRETURN(rrc); /* Need braces because of following else */ - } - break; - } - - /* Choose branch according to the condition */ - - ecode += condition? PRIV(OP_lengths)[condcode] : codelink; - - /* We are now at the branch that is to be obeyed. As there is only one, we - can use tail recursion to avoid using another stack frame, except when - there is unlimited repeat of a possibly empty group. In the latter case, a - recursive call to match() is always required, unless the second alternative - doesn't exist, in which case we can just plough on. Note that, for - compatibility with Perl, the | in a conditional group is NOT treated as - creating two alternatives. If a THEN is encountered in the branch, it - propagates out to the enclosing alternative (unless nested in a deeper set - of alternatives, of course). */ - - if (condition || ecode[-(1+LINK_SIZE)] == OP_ALT) - { - if (op != OP_SCOND) - { - goto TAIL_RECURSE; - } - - md->match_function_type = MATCH_CBEGROUP; - RMATCH(eptr, ecode, offset_top, md, eptrb, RM49); - RRETURN(rrc); - } - - /* Condition false & no alternative; continue after the group. */ - - else - { - } - break; - - - /* Before OP_ACCEPT there may be any number of OP_CLOSE opcodes, - to close any currently open capturing brackets. */ - - case OP_CLOSE: - number = GET2(ecode, 1); /* Must be less than 65536 */ - offset = number << 1; - -#ifdef PCRE_DEBUG - printf("end bracket %d at *ACCEPT", number); - printf("\n"); -#endif - - md->capture_last = (md->capture_last & OVFLMASK) | number; - if (offset >= md->offset_max) md->capture_last |= OVFLBIT; else - { - md->offset_vector[offset] = - md->offset_vector[md->offset_end - number]; - md->offset_vector[offset+1] = (int)(eptr - md->start_subject); - - /* If this group is at or above the current highwater mark, ensure that - any groups between the current high water mark and this group are marked - unset and then update the high water mark. */ - - if (offset >= offset_top) - { - register int *iptr = md->offset_vector + offset_top; - register int *iend = md->offset_vector + offset; - while (iptr < iend) *iptr++ = -1; - offset_top = offset + 2; - } - } - ecode += 1 + IMM2_SIZE; - break; - - - /* End of the pattern, either real or forced. */ - - case OP_END: - case OP_ACCEPT: - case OP_ASSERT_ACCEPT: - - /* If we have matched an empty string, fail if not in an assertion and not - in a recursion if either PCRE_NOTEMPTY is set, or if PCRE_NOTEMPTY_ATSTART - is set and we have matched at the start of the subject. In both cases, - backtracking will then try other alternatives, if any. */ - - if (eptr == mstart && op != OP_ASSERT_ACCEPT && - md->recursive == NULL && - (md->notempty || - (md->notempty_atstart && - mstart == md->start_subject + md->start_offset))) - RRETURN(MATCH_NOMATCH); - - /* Otherwise, we have a match. */ - - md->end_match_ptr = eptr; /* Record where we ended */ - md->end_offset_top = offset_top; /* and how many extracts were taken */ - md->start_match_ptr = mstart; /* and the start (\K can modify) */ - - /* For some reason, the macros don't work properly if an expression is - given as the argument to RRETURN when the heap is in use. */ - - rrc = (op == OP_END)? MATCH_MATCH : MATCH_ACCEPT; - RRETURN(rrc); - - /* Assertion brackets. Check the alternative branches in turn - the - matching won't pass the KET for an assertion. If any one branch matches, - the assertion is true. Lookbehind assertions have an OP_REVERSE item at the - start of each branch to move the current point backwards, so the code at - this level is identical to the lookahead case. When the assertion is part - of a condition, we want to return immediately afterwards. The caller of - this incarnation of the match() function will have set MATCH_CONDASSERT in - md->match_function type, and one of these opcodes will be the first opcode - that is processed. We use a local variable that is preserved over calls to - match() to remember this case. */ - - case OP_ASSERT: - case OP_ASSERTBACK: - save_mark = md->mark; - if (md->match_function_type == MATCH_CONDASSERT) - { - condassert = TRUE; - md->match_function_type = 0; - } - else condassert = FALSE; - - /* Loop for each branch */ - - do - { - RMATCH(eptr, ecode + 1 + LINK_SIZE, offset_top, md, NULL, RM4); - - /* A match means that the assertion is true; break out of the loop - that matches its alternatives. */ - - if (rrc == MATCH_MATCH || rrc == MATCH_ACCEPT) - { - mstart = md->start_match_ptr; /* In case \K reset it */ - break; - } - - /* If not matched, restore the previous mark setting. */ - - md->mark = save_mark; - - /* See comment in the code for capturing groups above about handling - THEN. */ - - if (rrc == MATCH_THEN) - { - next = ecode + GET(ecode,1); - if (md->start_match_ptr < next && - (*ecode == OP_ALT || *next == OP_ALT)) - rrc = MATCH_NOMATCH; - } - - /* Anything other than NOMATCH causes the entire assertion to fail, - passing back the return code. This includes COMMIT, SKIP, PRUNE and an - uncaptured THEN, which means they take their normal effect. This - consistent approach does not always have exactly the same effect as in - Perl. */ - - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - ecode += GET(ecode, 1); - } - while (*ecode == OP_ALT); /* Continue for next alternative */ - - /* If we have tried all the alternative branches, the assertion has - failed. If not, we broke out after a match. */ - - if (*ecode == OP_KET) RRETURN(MATCH_NOMATCH); - - /* If checking an assertion for a condition, return MATCH_MATCH. */ - - if (condassert) RRETURN(MATCH_MATCH); - - /* Continue from after a successful assertion, updating the offsets high - water mark, since extracts may have been taken during the assertion. */ - - do ecode += GET(ecode,1); while (*ecode == OP_ALT); - ecode += 1 + LINK_SIZE; - offset_top = md->end_offset_top; - continue; - - /* Negative assertion: all branches must fail to match for the assertion to - succeed. */ - - case OP_ASSERT_NOT: - case OP_ASSERTBACK_NOT: - save_mark = md->mark; - if (md->match_function_type == MATCH_CONDASSERT) - { - condassert = TRUE; - md->match_function_type = 0; - } - else condassert = FALSE; - - /* Loop for each alternative branch. */ - - do - { - RMATCH(eptr, ecode + 1 + LINK_SIZE, offset_top, md, NULL, RM5); - md->mark = save_mark; /* Always restore the mark setting */ - - switch(rrc) - { - case MATCH_MATCH: /* A successful match means */ - case MATCH_ACCEPT: /* the assertion has failed. */ - RRETURN(MATCH_NOMATCH); - - case MATCH_NOMATCH: /* Carry on with next branch */ - break; - - /* See comment in the code for capturing groups above about handling - THEN. */ - - case MATCH_THEN: - next = ecode + GET(ecode,1); - if (md->start_match_ptr < next && - (*ecode == OP_ALT || *next == OP_ALT)) - { - rrc = MATCH_NOMATCH; - break; - } - /* Otherwise fall through. */ - - /* COMMIT, SKIP, PRUNE, and an uncaptured THEN cause the whole - assertion to fail to match, without considering any more alternatives. - Failing to match means the assertion is true. This is a consistent - approach, but does not always have the same effect as in Perl. */ - - case MATCH_COMMIT: - case MATCH_SKIP: - case MATCH_SKIP_ARG: - case MATCH_PRUNE: - do ecode += GET(ecode,1); while (*ecode == OP_ALT); - goto NEG_ASSERT_TRUE; /* Break out of alternation loop */ - - /* Anything else is an error */ - - default: - RRETURN(rrc); - } - - /* Continue with next branch */ - - ecode += GET(ecode,1); - } - while (*ecode == OP_ALT); - - /* All branches in the assertion failed to match. */ - - NEG_ASSERT_TRUE: - if (condassert) RRETURN(MATCH_MATCH); /* Condition assertion */ - ecode += 1 + LINK_SIZE; /* Continue with current branch */ - continue; - - /* Move the subject pointer back. This occurs only at the start of - each branch of a lookbehind assertion. If we are too close to the start to - move back, this match function fails. When working with UTF-8 we move - back a number of characters, not bytes. */ - - case OP_REVERSE: -#ifdef SUPPORT_UTF - if (utf) - { - i = GET(ecode, 1); - while (i-- > 0) - { - eptr--; - if (eptr < md->start_subject) RRETURN(MATCH_NOMATCH); - BACKCHAR(eptr); - } - } - else -#endif - - /* No UTF-8 support, or not in UTF-8 mode: count is byte count */ - - { - eptr -= GET(ecode, 1); - if (eptr < md->start_subject) RRETURN(MATCH_NOMATCH); - } - - /* Save the earliest consulted character, then skip to next op code */ - - if (eptr < md->start_used_ptr) md->start_used_ptr = eptr; - ecode += 1 + LINK_SIZE; - break; - - /* The callout item calls an external function, if one is provided, passing - details of the match so far. This is mainly for debugging, though the - function is able to force a failure. */ - - case OP_CALLOUT: - if (PUBL(callout) != NULL) - { - PUBL(callout_block) cb; - cb.version = 2; /* Version 1 of the callout block */ - cb.callout_number = ecode[1]; - cb.offset_vector = md->offset_vector; -#if defined COMPILE_PCRE8 - cb.subject = (PCRE_SPTR)md->start_subject; -#elif defined COMPILE_PCRE16 - cb.subject = (PCRE_SPTR16)md->start_subject; -#elif defined COMPILE_PCRE32 - cb.subject = (PCRE_SPTR32)md->start_subject; -#endif - cb.subject_length = (int)(md->end_subject - md->start_subject); - cb.start_match = (int)(mstart - md->start_subject); - cb.current_position = (int)(eptr - md->start_subject); - cb.pattern_position = GET(ecode, 2); - cb.next_item_length = GET(ecode, 2 + LINK_SIZE); - cb.capture_top = offset_top/2; - cb.capture_last = md->capture_last & CAPLMASK; - /* Internal change requires this for API compatibility. */ - if (cb.capture_last == 0) cb.capture_last = -1; - cb.callout_data = md->callout_data; - cb.mark = md->nomatch_mark; - if ((rrc = (*PUBL(callout))(&cb)) > 0) RRETURN(MATCH_NOMATCH); - if (rrc < 0) RRETURN(rrc); - } - ecode += 2 + 2*LINK_SIZE; - break; - - /* Recursion either matches the current regex, or some subexpression. The - offset data is the offset to the starting bracket from the start of the - whole pattern. (This is so that it works from duplicated subpatterns.) - - The state of the capturing groups is preserved over recursion, and - re-instated afterwards. We don't know how many are started and not yet - finished (offset_top records the completed total) so we just have to save - all the potential data. There may be up to 65535 such values, which is too - large to put on the stack, but using malloc for small numbers seems - expensive. As a compromise, the stack is used when there are no more than - REC_STACK_SAVE_MAX values to store; otherwise malloc is used. - - There are also other values that have to be saved. We use a chained - sequence of blocks that actually live on the stack. Thanks to Robin Houston - for the original version of this logic. It has, however, been hacked around - a lot, so he is not to blame for the current way it works. */ - - case OP_RECURSE: - { - recursion_info *ri; - unsigned int recno; - - callpat = md->start_code + GET(ecode, 1); - recno = (callpat == md->start_code)? 0 : - GET2(callpat, 1 + LINK_SIZE); - - /* Check for repeating a recursion without advancing the subject pointer. - This should catch convoluted mutual recursions. (Some simple cases are - caught at compile time.) */ - - for (ri = md->recursive; ri != NULL; ri = ri->prevrec) - if (recno == ri->group_num && eptr == ri->subject_position) - RRETURN(PCRE_ERROR_RECURSELOOP); - - /* Add to "recursing stack" */ - - new_recursive.group_num = recno; - new_recursive.saved_capture_last = md->capture_last; - new_recursive.subject_position = eptr; - new_recursive.prevrec = md->recursive; - md->recursive = &new_recursive; - - /* Where to continue from afterwards */ - - ecode += 1 + LINK_SIZE; - - /* Now save the offset data */ - - new_recursive.saved_max = md->offset_end; - if (new_recursive.saved_max <= REC_STACK_SAVE_MAX) - new_recursive.offset_save = stacksave; - else - { - new_recursive.offset_save = - (int *)(PUBL(malloc))(new_recursive.saved_max * sizeof(int)); - if (new_recursive.offset_save == NULL) RRETURN(PCRE_ERROR_NOMEMORY); - } - memcpy(new_recursive.offset_save, md->offset_vector, - new_recursive.saved_max * sizeof(int)); - - /* OK, now we can do the recursion. After processing each alternative, - restore the offset data and the last captured value. If there were nested - recursions, md->recursive might be changed, so reset it before looping. - */ - - DPRINTF(("Recursing into group %d\n", new_recursive.group_num)); - cbegroup = (*callpat >= OP_SBRA); - do - { - if (cbegroup) md->match_function_type = MATCH_CBEGROUP; - RMATCH(eptr, callpat + PRIV(OP_lengths)[*callpat], offset_top, - md, eptrb, RM6); - memcpy(md->offset_vector, new_recursive.offset_save, - new_recursive.saved_max * sizeof(int)); - md->capture_last = new_recursive.saved_capture_last; - md->recursive = new_recursive.prevrec; - if (rrc == MATCH_MATCH || rrc == MATCH_ACCEPT) - { - DPRINTF(("Recursion matched\n")); - if (new_recursive.offset_save != stacksave) - (PUBL(free))(new_recursive.offset_save); - - /* Set where we got to in the subject, and reset the start in case - it was changed by \K. This *is* propagated back out of a recursion, - for Perl compatibility. */ - - eptr = md->end_match_ptr; - mstart = md->start_match_ptr; - goto RECURSION_MATCHED; /* Exit loop; end processing */ - } - - /* PCRE does not allow THEN, SKIP, PRUNE or COMMIT to escape beyond a - recursion; they cause a NOMATCH for the entire recursion. These codes - are defined in a range that can be tested for. */ - - if (rrc >= MATCH_BACKTRACK_MIN && rrc <= MATCH_BACKTRACK_MAX) - { - if (new_recursive.offset_save != stacksave) - (PUBL(free))(new_recursive.offset_save); - RRETURN(MATCH_NOMATCH); - } - - /* Any return code other than NOMATCH is an error. */ - - if (rrc != MATCH_NOMATCH) - { - DPRINTF(("Recursion gave error %d\n", rrc)); - if (new_recursive.offset_save != stacksave) - (PUBL(free))(new_recursive.offset_save); - RRETURN(rrc); - } - - md->recursive = &new_recursive; - callpat += GET(callpat, 1); - } - while (*callpat == OP_ALT); - - DPRINTF(("Recursion didn't match\n")); - md->recursive = new_recursive.prevrec; - if (new_recursive.offset_save != stacksave) - (PUBL(free))(new_recursive.offset_save); - RRETURN(MATCH_NOMATCH); - } - - RECURSION_MATCHED: - break; - - /* An alternation is the end of a branch; scan along to find the end of the - bracketed group and go to there. */ - - case OP_ALT: - do ecode += GET(ecode,1); while (*ecode == OP_ALT); - break; - - /* BRAZERO, BRAMINZERO and SKIPZERO occur just before a bracket group, - indicating that it may occur zero times. It may repeat infinitely, or not - at all - i.e. it could be ()* or ()? or even (){0} in the pattern. Brackets - with fixed upper repeat limits are compiled as a number of copies, with the - optional ones preceded by BRAZERO or BRAMINZERO. */ - - case OP_BRAZERO: - next = ecode + 1; - RMATCH(eptr, next, offset_top, md, eptrb, RM10); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - do next += GET(next, 1); while (*next == OP_ALT); - ecode = next + 1 + LINK_SIZE; - break; - - case OP_BRAMINZERO: - next = ecode + 1; - do next += GET(next, 1); while (*next == OP_ALT); - RMATCH(eptr, next + 1+LINK_SIZE, offset_top, md, eptrb, RM11); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - ecode++; - break; - - case OP_SKIPZERO: - next = ecode+1; - do next += GET(next,1); while (*next == OP_ALT); - ecode = next + 1 + LINK_SIZE; - break; - - /* BRAPOSZERO occurs before a possessive bracket group. Don't do anything - here; just jump to the group, with allow_zero set TRUE. */ - - case OP_BRAPOSZERO: - op = *(++ecode); - allow_zero = TRUE; - if (op == OP_CBRAPOS || op == OP_SCBRAPOS) goto POSSESSIVE_CAPTURE; - goto POSSESSIVE_NON_CAPTURE; - - /* End of a group, repeated or non-repeating. */ - - case OP_KET: - case OP_KETRMIN: - case OP_KETRMAX: - case OP_KETRPOS: - prev = ecode - GET(ecode, 1); - - /* If this was a group that remembered the subject start, in order to break - infinite repeats of empty string matches, retrieve the subject start from - the chain. Otherwise, set it NULL. */ - - if (*prev >= OP_SBRA || *prev == OP_ONCE) - { - saved_eptr = eptrb->epb_saved_eptr; /* Value at start of group */ - eptrb = eptrb->epb_prev; /* Backup to previous group */ - } - else saved_eptr = NULL; - - /* If we are at the end of an assertion group or a non-capturing atomic - group, stop matching and return MATCH_MATCH, but record the current high - water mark for use by positive assertions. We also need to record the match - start in case it was changed by \K. */ - - if ((*prev >= OP_ASSERT && *prev <= OP_ASSERTBACK_NOT) || - *prev == OP_ONCE_NC) - { - md->end_match_ptr = eptr; /* For ONCE_NC */ - md->end_offset_top = offset_top; - md->start_match_ptr = mstart; - RRETURN(MATCH_MATCH); /* Sets md->mark */ - } - - /* For capturing groups we have to check the group number back at the start - and if necessary complete handling an extraction by setting the offsets and - bumping the high water mark. Whole-pattern recursion is coded as a recurse - into group 0, so it won't be picked up here. Instead, we catch it when the - OP_END is reached. Other recursion is handled here. We just have to record - the current subject position and start match pointer and give a MATCH - return. */ - - if (*prev == OP_CBRA || *prev == OP_SCBRA || - *prev == OP_CBRAPOS || *prev == OP_SCBRAPOS) - { - number = GET2(prev, 1+LINK_SIZE); - offset = number << 1; - -#ifdef PCRE_DEBUG - printf("end bracket %d", number); - printf("\n"); -#endif - - /* Handle a recursively called group. */ - - if (md->recursive != NULL && md->recursive->group_num == number) - { - md->end_match_ptr = eptr; - md->start_match_ptr = mstart; - RRETURN(MATCH_MATCH); - } - - /* Deal with capturing */ - - md->capture_last = (md->capture_last & OVFLMASK) | number; - if (offset >= md->offset_max) md->capture_last |= OVFLBIT; else - { - /* If offset is greater than offset_top, it means that we are - "skipping" a capturing group, and that group's offsets must be marked - unset. In earlier versions of PCRE, all the offsets were unset at the - start of matching, but this doesn't work because atomic groups and - assertions can cause a value to be set that should later be unset. - Example: matching /(?>(a))b|(a)c/ against "ac". This sets group 1 as - part of the atomic group, but this is not on the final matching path, - so must be unset when 2 is set. (If there is no group 2, there is no - problem, because offset_top will then be 2, indicating no capture.) */ - - if (offset > offset_top) - { - register int *iptr = md->offset_vector + offset_top; - register int *iend = md->offset_vector + offset; - while (iptr < iend) *iptr++ = -1; - } - - /* Now make the extraction */ - - md->offset_vector[offset] = - md->offset_vector[md->offset_end - number]; - md->offset_vector[offset+1] = (int)(eptr - md->start_subject); - if (offset_top <= offset) offset_top = offset + 2; - } - } - - /* OP_KETRPOS is a possessive repeating ket. Remember the current position, - and return the MATCH_KETRPOS. This makes it possible to do the repeats one - at a time from the outer level, thus saving stack. This must precede the - empty string test - in this case that test is done at the outer level. */ - - if (*ecode == OP_KETRPOS) - { - md->start_match_ptr = mstart; /* In case \K reset it */ - md->end_match_ptr = eptr; - md->end_offset_top = offset_top; - RRETURN(MATCH_KETRPOS); - } - - /* For an ordinary non-repeating ket, just continue at this level. This - also happens for a repeating ket if no characters were matched in the - group. This is the forcible breaking of infinite loops as implemented in - Perl 5.005. For a non-repeating atomic group that includes captures, - establish a backup point by processing the rest of the pattern at a lower - level. If this results in a NOMATCH return, pass MATCH_ONCE back to the - original OP_ONCE level, thereby bypassing intermediate backup points, but - resetting any captures that happened along the way. */ - - if (*ecode == OP_KET || eptr == saved_eptr) - { - if (*prev == OP_ONCE) - { - RMATCH(eptr, ecode + 1 + LINK_SIZE, offset_top, md, eptrb, RM12); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - md->once_target = prev; /* Level at which to change to MATCH_NOMATCH */ - RRETURN(MATCH_ONCE); - } - ecode += 1 + LINK_SIZE; /* Carry on at this level */ - break; - } - - /* The normal repeating kets try the rest of the pattern or restart from - the preceding bracket, in the appropriate order. In the second case, we can - use tail recursion to avoid using another stack frame, unless we have an - an atomic group or an unlimited repeat of a group that can match an empty - string. */ - - if (*ecode == OP_KETRMIN) - { - RMATCH(eptr, ecode + 1 + LINK_SIZE, offset_top, md, eptrb, RM7); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (*prev == OP_ONCE) - { - RMATCH(eptr, prev, offset_top, md, eptrb, RM8); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - md->once_target = prev; /* Level at which to change to MATCH_NOMATCH */ - RRETURN(MATCH_ONCE); - } - if (*prev >= OP_SBRA) /* Could match an empty string */ - { - RMATCH(eptr, prev, offset_top, md, eptrb, RM50); - RRETURN(rrc); - } - ecode = prev; - goto TAIL_RECURSE; - } - else /* OP_KETRMAX */ - { - RMATCH(eptr, prev, offset_top, md, eptrb, RM13); - if (rrc == MATCH_ONCE && md->once_target == prev) rrc = MATCH_NOMATCH; - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (*prev == OP_ONCE) - { - RMATCH(eptr, ecode + 1 + LINK_SIZE, offset_top, md, eptrb, RM9); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - md->once_target = prev; - RRETURN(MATCH_ONCE); - } - ecode += 1 + LINK_SIZE; - goto TAIL_RECURSE; - } - /* Control never gets here */ - - /* Not multiline mode: start of subject assertion, unless notbol. */ - - case OP_CIRC: - if (md->notbol && eptr == md->start_subject) RRETURN(MATCH_NOMATCH); - - /* Start of subject assertion */ - - case OP_SOD: - if (eptr != md->start_subject) RRETURN(MATCH_NOMATCH); - ecode++; - break; - - /* Multiline mode: start of subject unless notbol, or after any newline. */ - - case OP_CIRCM: - if (md->notbol && eptr == md->start_subject) RRETURN(MATCH_NOMATCH); - if (eptr != md->start_subject && - (eptr == md->end_subject || !WAS_NEWLINE(eptr))) - RRETURN(MATCH_NOMATCH); - ecode++; - break; - - /* Start of match assertion */ - - case OP_SOM: - if (eptr != md->start_subject + md->start_offset) RRETURN(MATCH_NOMATCH); - ecode++; - break; - - /* Reset the start of match point */ - - case OP_SET_SOM: - mstart = eptr; - ecode++; - break; - - /* Multiline mode: assert before any newline, or before end of subject - unless noteol is set. */ - - case OP_DOLLM: - if (eptr < md->end_subject) - { - if (!IS_NEWLINE(eptr)) - { - if (md->partial != 0 && - eptr + 1 >= md->end_subject && - NLBLOCK->nltype == NLTYPE_FIXED && - NLBLOCK->nllen == 2 && - UCHAR21TEST(eptr) == NLBLOCK->nl[0]) - { - md->hitend = TRUE; - if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); - } - RRETURN(MATCH_NOMATCH); - } - } - else - { - if (md->noteol) RRETURN(MATCH_NOMATCH); - SCHECK_PARTIAL(); - } - ecode++; - break; - - /* Not multiline mode: assert before a terminating newline or before end of - subject unless noteol is set. */ - - case OP_DOLL: - if (md->noteol) RRETURN(MATCH_NOMATCH); - if (!md->endonly) goto ASSERT_NL_OR_EOS; - - /* ... else fall through for endonly */ - - /* End of subject assertion (\z) */ - - case OP_EOD: - if (eptr < md->end_subject) RRETURN(MATCH_NOMATCH); - SCHECK_PARTIAL(); - ecode++; - break; - - /* End of subject or ending \n assertion (\Z) */ - - case OP_EODN: - ASSERT_NL_OR_EOS: - if (eptr < md->end_subject && - (!IS_NEWLINE(eptr) || eptr != md->end_subject - md->nllen)) - { - if (md->partial != 0 && - eptr + 1 >= md->end_subject && - NLBLOCK->nltype == NLTYPE_FIXED && - NLBLOCK->nllen == 2 && - UCHAR21TEST(eptr) == NLBLOCK->nl[0]) - { - md->hitend = TRUE; - if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); - } - RRETURN(MATCH_NOMATCH); - } - - /* Either at end of string or \n before end. */ - - SCHECK_PARTIAL(); - ecode++; - break; - - /* Word boundary assertions */ - - case OP_NOT_WORD_BOUNDARY: - case OP_WORD_BOUNDARY: - { - - /* Find out if the previous and current characters are "word" characters. - It takes a bit more work in UTF-8 mode. Characters > 255 are assumed to - be "non-word" characters. Remember the earliest consulted character for - partial matching. */ - -#ifdef SUPPORT_UTF - if (utf) - { - /* Get status of previous character */ - - if (eptr == md->start_subject) prev_is_word = FALSE; else - { - PCRE_PUCHAR lastptr = eptr - 1; - BACKCHAR(lastptr); - if (lastptr < md->start_used_ptr) md->start_used_ptr = lastptr; - GETCHAR(c, lastptr); -#ifdef SUPPORT_UCP - if (md->use_ucp) - { - if (c == '_') prev_is_word = TRUE; else - { - int cat = UCD_CATEGORY(c); - prev_is_word = (cat == ucp_L || cat == ucp_N); - } - } - else -#endif - prev_is_word = c < 256 && (md->ctypes[c] & ctype_word) != 0; - } - - /* Get status of next character */ - - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - cur_is_word = FALSE; - } - else - { - GETCHAR(c, eptr); -#ifdef SUPPORT_UCP - if (md->use_ucp) - { - if (c == '_') cur_is_word = TRUE; else - { - int cat = UCD_CATEGORY(c); - cur_is_word = (cat == ucp_L || cat == ucp_N); - } - } - else -#endif - cur_is_word = c < 256 && (md->ctypes[c] & ctype_word) != 0; - } - } - else -#endif - - /* Not in UTF-8 mode, but we may still have PCRE_UCP set, and for - consistency with the behaviour of \w we do use it in this case. */ - - { - /* Get status of previous character */ - - if (eptr == md->start_subject) prev_is_word = FALSE; else - { - if (eptr <= md->start_used_ptr) md->start_used_ptr = eptr - 1; -#ifdef SUPPORT_UCP - if (md->use_ucp) - { - c = eptr[-1]; - if (c == '_') prev_is_word = TRUE; else - { - int cat = UCD_CATEGORY(c); - prev_is_word = (cat == ucp_L || cat == ucp_N); - } - } - else -#endif - prev_is_word = MAX_255(eptr[-1]) - && ((md->ctypes[eptr[-1]] & ctype_word) != 0); - } - - /* Get status of next character */ - - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - cur_is_word = FALSE; - } - else -#ifdef SUPPORT_UCP - if (md->use_ucp) - { - c = *eptr; - if (c == '_') cur_is_word = TRUE; else - { - int cat = UCD_CATEGORY(c); - cur_is_word = (cat == ucp_L || cat == ucp_N); - } - } - else -#endif - cur_is_word = MAX_255(*eptr) - && ((md->ctypes[*eptr] & ctype_word) != 0); - } - - /* Now see if the situation is what we want */ - - if ((*ecode++ == OP_WORD_BOUNDARY)? - cur_is_word == prev_is_word : cur_is_word != prev_is_word) - RRETURN(MATCH_NOMATCH); - } - break; - - /* Match any single character type except newline; have to take care with - CRLF newlines and partial matching. */ - - case OP_ANY: - if (IS_NEWLINE(eptr)) RRETURN(MATCH_NOMATCH); - if (md->partial != 0 && - eptr + 1 >= md->end_subject && - NLBLOCK->nltype == NLTYPE_FIXED && - NLBLOCK->nllen == 2 && - UCHAR21TEST(eptr) == NLBLOCK->nl[0]) - { - md->hitend = TRUE; - if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); - } - - /* Fall through */ - - /* Match any single character whatsoever. */ - - case OP_ALLANY: - if (eptr >= md->end_subject) /* DO NOT merge the eptr++ here; it must */ - { /* not be updated before SCHECK_PARTIAL. */ - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - eptr++; -#ifdef SUPPORT_UTF - if (utf) ACROSSCHAR(eptr < md->end_subject, *eptr, eptr++); -#endif - ecode++; - break; - - /* Match a single byte, even in UTF-8 mode. This opcode really does match - any byte, even newline, independent of the setting of PCRE_DOTALL. */ - - case OP_ANYBYTE: - if (eptr >= md->end_subject) /* DO NOT merge the eptr++ here; it must */ - { /* not be updated before SCHECK_PARTIAL. */ - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - eptr++; - ecode++; - break; - - case OP_NOT_DIGIT: - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - if ( -#if defined SUPPORT_UTF || !(defined COMPILE_PCRE8) - c < 256 && -#endif - (md->ctypes[c] & ctype_digit) != 0 - ) - RRETURN(MATCH_NOMATCH); - ecode++; - break; - - case OP_DIGIT: - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - if ( -#if defined SUPPORT_UTF || !(defined COMPILE_PCRE8) - c > 255 || -#endif - (md->ctypes[c] & ctype_digit) == 0 - ) - RRETURN(MATCH_NOMATCH); - ecode++; - break; - - case OP_NOT_WHITESPACE: - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - if ( -#if defined SUPPORT_UTF || !(defined COMPILE_PCRE8) - c < 256 && -#endif - (md->ctypes[c] & ctype_space) != 0 - ) - RRETURN(MATCH_NOMATCH); - ecode++; - break; - - case OP_WHITESPACE: - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - if ( -#if defined SUPPORT_UTF || !(defined COMPILE_PCRE8) - c > 255 || -#endif - (md->ctypes[c] & ctype_space) == 0 - ) - RRETURN(MATCH_NOMATCH); - ecode++; - break; - - case OP_NOT_WORDCHAR: - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - if ( -#if defined SUPPORT_UTF || !(defined COMPILE_PCRE8) - c < 256 && -#endif - (md->ctypes[c] & ctype_word) != 0 - ) - RRETURN(MATCH_NOMATCH); - ecode++; - break; - - case OP_WORDCHAR: - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - if ( -#if defined SUPPORT_UTF || !(defined COMPILE_PCRE8) - c > 255 || -#endif - (md->ctypes[c] & ctype_word) == 0 - ) - RRETURN(MATCH_NOMATCH); - ecode++; - break; - - case OP_ANYNL: - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - switch(c) - { - default: RRETURN(MATCH_NOMATCH); - - case CHAR_CR: - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - } - else if (UCHAR21TEST(eptr) == CHAR_LF) eptr++; - break; - - case CHAR_LF: - break; - - case CHAR_VT: - case CHAR_FF: - case CHAR_NEL: -#ifndef EBCDIC - case 0x2028: - case 0x2029: -#endif /* Not EBCDIC */ - if (md->bsr_anycrlf) RRETURN(MATCH_NOMATCH); - break; - } - ecode++; - break; - - case OP_NOT_HSPACE: - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - switch(c) - { - HSPACE_CASES: RRETURN(MATCH_NOMATCH); /* Byte and multibyte cases */ - default: break; - } - ecode++; - break; - - case OP_HSPACE: - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - switch(c) - { - HSPACE_CASES: break; /* Byte and multibyte cases */ - default: RRETURN(MATCH_NOMATCH); - } - ecode++; - break; - - case OP_NOT_VSPACE: - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - switch(c) - { - VSPACE_CASES: RRETURN(MATCH_NOMATCH); - default: break; - } - ecode++; - break; - - case OP_VSPACE: - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - switch(c) - { - VSPACE_CASES: break; - default: RRETURN(MATCH_NOMATCH); - } - ecode++; - break; - -#ifdef SUPPORT_UCP - /* Check the next character by Unicode property. We will get here only - if the support is in the binary; otherwise a compile-time error occurs. */ - - case OP_PROP: - case OP_NOTPROP: - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - { - const pcre_uint32 *cp; - const ucd_record *prop = GET_UCD(c); - - switch(ecode[1]) - { - case PT_ANY: - if (op == OP_NOTPROP) RRETURN(MATCH_NOMATCH); - break; - - case PT_LAMP: - if ((prop->chartype == ucp_Lu || - prop->chartype == ucp_Ll || - prop->chartype == ucp_Lt) == (op == OP_NOTPROP)) - RRETURN(MATCH_NOMATCH); - break; - - case PT_GC: - if ((ecode[2] != PRIV(ucp_gentype)[prop->chartype]) == (op == OP_PROP)) - RRETURN(MATCH_NOMATCH); - break; - - case PT_PC: - if ((ecode[2] != prop->chartype) == (op == OP_PROP)) - RRETURN(MATCH_NOMATCH); - break; - - case PT_SC: - if ((ecode[2] != prop->script) == (op == OP_PROP)) - RRETURN(MATCH_NOMATCH); - break; - - /* These are specials */ - - case PT_ALNUM: - if ((PRIV(ucp_gentype)[prop->chartype] == ucp_L || - PRIV(ucp_gentype)[prop->chartype] == ucp_N) == (op == OP_NOTPROP)) - RRETURN(MATCH_NOMATCH); - break; - - /* Perl space used to exclude VT, but from Perl 5.18 it is included, - which means that Perl space and POSIX space are now identical. PCRE - was changed at release 8.34. */ - - case PT_SPACE: /* Perl space */ - case PT_PXSPACE: /* POSIX space */ - switch(c) - { - HSPACE_CASES: - VSPACE_CASES: - if (op == OP_NOTPROP) RRETURN(MATCH_NOMATCH); - break; - - default: - if ((PRIV(ucp_gentype)[prop->chartype] == ucp_Z) == - (op == OP_NOTPROP)) RRETURN(MATCH_NOMATCH); - break; - } - break; - - case PT_WORD: - if ((PRIV(ucp_gentype)[prop->chartype] == ucp_L || - PRIV(ucp_gentype)[prop->chartype] == ucp_N || - c == CHAR_UNDERSCORE) == (op == OP_NOTPROP)) - RRETURN(MATCH_NOMATCH); - break; - - case PT_CLIST: - cp = PRIV(ucd_caseless_sets) + ecode[2]; - for (;;) - { - if (c < *cp) - { if (op == OP_PROP) { RRETURN(MATCH_NOMATCH); } else break; } - if (c == *cp++) - { if (op == OP_PROP) break; else { RRETURN(MATCH_NOMATCH); } } - } - break; - - case PT_UCNC: - if ((c == CHAR_DOLLAR_SIGN || c == CHAR_COMMERCIAL_AT || - c == CHAR_GRAVE_ACCENT || (c >= 0xa0 && c <= 0xd7ff) || - c >= 0xe000) == (op == OP_NOTPROP)) - RRETURN(MATCH_NOMATCH); - break; - - /* This should never occur */ - - default: - RRETURN(PCRE_ERROR_INTERNAL); - } - - ecode += 3; - } - break; - - /* Match an extended Unicode sequence. We will get here only if the support - is in the binary; otherwise a compile-time error occurs. */ - - case OP_EXTUNI: - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - else - { - int lgb, rgb; - GETCHARINCTEST(c, eptr); - lgb = UCD_GRAPHBREAK(c); - while (eptr < md->end_subject) - { - int len = 1; - if (!utf) c = *eptr; else { GETCHARLEN(c, eptr, len); } - rgb = UCD_GRAPHBREAK(c); - if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; - lgb = rgb; - eptr += len; - } - } - CHECK_PARTIAL(); - ecode++; - break; -#endif /* SUPPORT_UCP */ - - - /* Match a back reference, possibly repeatedly. Look past the end of the - item to see if there is repeat information following. The code is similar - to that for character classes, but repeated for efficiency. Then obey - similar code to character type repeats - written out again for speed. - However, if the referenced string is the empty string, always treat - it as matched, any number of times (otherwise there could be infinite - loops). If the reference is unset, there are two possibilities: - - (a) In the default, Perl-compatible state, set the length negative; - this ensures that every attempt at a match fails. We can't just fail - here, because of the possibility of quantifiers with zero minima. - - (b) If the JavaScript compatibility flag is set, set the length to zero - so that the back reference matches an empty string. - - Otherwise, set the length to the length of what was matched by the - referenced subpattern. - - The OP_REF and OP_REFI opcodes are used for a reference to a numbered group - or to a non-duplicated named group. For a duplicated named group, OP_DNREF - and OP_DNREFI are used. In this case we must scan the list of groups to - which the name refers, and use the first one that is set. */ - - case OP_DNREF: - case OP_DNREFI: - caseless = op == OP_DNREFI; - { - int count = GET2(ecode, 1+IMM2_SIZE); - pcre_uchar *slot = md->name_table + GET2(ecode, 1) * md->name_entry_size; - ecode += 1 + 2*IMM2_SIZE; - - /* Setting the default length first and initializing 'offset' avoids - compiler warnings in the REF_REPEAT code. */ - - length = (md->jscript_compat)? 0 : -1; - offset = 0; - - while (count-- > 0) - { - offset = GET2(slot, 0) << 1; - if (offset < offset_top && md->offset_vector[offset] >= 0) - { - length = md->offset_vector[offset+1] - md->offset_vector[offset]; - break; - } - slot += md->name_entry_size; - } - } - goto REF_REPEAT; - - case OP_REF: - case OP_REFI: - caseless = op == OP_REFI; - offset = GET2(ecode, 1) << 1; /* Doubled ref number */ - ecode += 1 + IMM2_SIZE; - if (offset >= offset_top || md->offset_vector[offset] < 0) - length = (md->jscript_compat)? 0 : -1; - else - length = md->offset_vector[offset+1] - md->offset_vector[offset]; - - /* Set up for repetition, or handle the non-repeated case */ - - REF_REPEAT: - switch (*ecode) - { - case OP_CRSTAR: - case OP_CRMINSTAR: - case OP_CRPLUS: - case OP_CRMINPLUS: - case OP_CRQUERY: - case OP_CRMINQUERY: - c = *ecode++ - OP_CRSTAR; - minimize = (c & 1) != 0; - min = rep_min[c]; /* Pick up values from tables; */ - max = rep_max[c]; /* zero for max => infinity */ - if (max == 0) max = INT_MAX; - break; - - case OP_CRRANGE: - case OP_CRMINRANGE: - minimize = (*ecode == OP_CRMINRANGE); - min = GET2(ecode, 1); - max = GET2(ecode, 1 + IMM2_SIZE); - if (max == 0) max = INT_MAX; - ecode += 1 + 2 * IMM2_SIZE; - break; - - default: /* No repeat follows */ - if ((length = match_ref(offset, eptr, length, md, caseless)) < 0) - { - if (length == -2) eptr = md->end_subject; /* Partial match */ - CHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - eptr += length; - continue; /* With the main loop */ - } - - /* Handle repeated back references. If the length of the reference is - zero, just continue with the main loop. If the length is negative, it - means the reference is unset in non-Java-compatible mode. If the minimum is - zero, we can continue at the same level without recursion. For any other - minimum, carrying on will result in NOMATCH. */ - - if (length == 0) continue; - if (length < 0 && min == 0) continue; - - /* First, ensure the minimum number of matches are present. We get back - the length of the reference string explicitly rather than passing the - address of eptr, so that eptr can be a register variable. */ - - for (i = 1; i <= min; i++) - { - int slength; - if ((slength = match_ref(offset, eptr, length, md, caseless)) < 0) - { - if (slength == -2) eptr = md->end_subject; /* Partial match */ - CHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - eptr += slength; - } - - /* If min = max, continue at the same level without recursion. - They are not both allowed to be zero. */ - - if (min == max) continue; - - /* If minimizing, keep trying and advancing the pointer */ - - if (minimize) - { - for (fi = min;; fi++) - { - int slength; - RMATCH(eptr, ecode, offset_top, md, eptrb, RM14); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if ((slength = match_ref(offset, eptr, length, md, caseless)) < 0) - { - if (slength == -2) eptr = md->end_subject; /* Partial match */ - CHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - eptr += slength; - } - /* Control never gets here */ - } - - /* If maximizing, find the longest string and work backwards */ - - else - { - pp = eptr; - for (i = min; i < max; i++) - { - int slength; - if ((slength = match_ref(offset, eptr, length, md, caseless)) < 0) - { - /* Can't use CHECK_PARTIAL because we don't want to update eptr in - the soft partial matching case. */ - - if (slength == -2 && md->partial != 0 && - md->end_subject > md->start_used_ptr) - { - md->hitend = TRUE; - if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); - } - break; - } - eptr += slength; - } - - while (eptr >= pp) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM15); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - eptr -= length; - } - RRETURN(MATCH_NOMATCH); - } - /* Control never gets here */ - - /* Match a bit-mapped character class, possibly repeatedly. This op code is - used when all the characters in the class have values in the range 0-255, - and either the matching is caseful, or the characters are in the range - 0-127 when UTF-8 processing is enabled. The only difference between - OP_CLASS and OP_NCLASS occurs when a data character outside the range is - encountered. - - First, look past the end of the item to see if there is repeat information - following. Then obey similar code to character type repeats - written out - again for speed. */ - - case OP_NCLASS: - case OP_CLASS: - { - /* The data variable is saved across frames, so the byte map needs to - be stored there. */ -#define BYTE_MAP ((pcre_uint8 *)data) - data = ecode + 1; /* Save for matching */ - ecode += 1 + (32 / sizeof(pcre_uchar)); /* Advance past the item */ - - switch (*ecode) - { - case OP_CRSTAR: - case OP_CRMINSTAR: - case OP_CRPLUS: - case OP_CRMINPLUS: - case OP_CRQUERY: - case OP_CRMINQUERY: - case OP_CRPOSSTAR: - case OP_CRPOSPLUS: - case OP_CRPOSQUERY: - c = *ecode++ - OP_CRSTAR; - if (c < OP_CRPOSSTAR - OP_CRSTAR) minimize = (c & 1) != 0; - else possessive = TRUE; - min = rep_min[c]; /* Pick up values from tables; */ - max = rep_max[c]; /* zero for max => infinity */ - if (max == 0) max = INT_MAX; - break; - - case OP_CRRANGE: - case OP_CRMINRANGE: - case OP_CRPOSRANGE: - minimize = (*ecode == OP_CRMINRANGE); - possessive = (*ecode == OP_CRPOSRANGE); - min = GET2(ecode, 1); - max = GET2(ecode, 1 + IMM2_SIZE); - if (max == 0) max = INT_MAX; - ecode += 1 + 2 * IMM2_SIZE; - break; - - default: /* No repeat follows */ - min = max = 1; - break; - } - - /* First, ensure the minimum number of matches are present. */ - -#ifdef SUPPORT_UTF - if (utf) - { - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINC(c, eptr); - if (c > 255) - { - if (op == OP_CLASS) RRETURN(MATCH_NOMATCH); - } - else - if ((BYTE_MAP[c/8] & (1 << (c&7))) == 0) RRETURN(MATCH_NOMATCH); - } - } - else -#endif - /* Not UTF mode */ - { - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - c = *eptr++; -#ifndef COMPILE_PCRE8 - if (c > 255) - { - if (op == OP_CLASS) RRETURN(MATCH_NOMATCH); - } - else -#endif - if ((BYTE_MAP[c/8] & (1 << (c&7))) == 0) RRETURN(MATCH_NOMATCH); - } - } - - /* If max == min we can continue with the main loop without the - need to recurse. */ - - if (min == max) continue; - - /* If minimizing, keep testing the rest of the expression and advancing - the pointer while it matches the class. */ - - if (minimize) - { -#ifdef SUPPORT_UTF - if (utf) - { - for (fi = min;; fi++) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM16); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINC(c, eptr); - if (c > 255) - { - if (op == OP_CLASS) RRETURN(MATCH_NOMATCH); - } - else - if ((BYTE_MAP[c/8] & (1 << (c&7))) == 0) RRETURN(MATCH_NOMATCH); - } - } - else -#endif - /* Not UTF mode */ - { - for (fi = min;; fi++) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM17); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - c = *eptr++; -#ifndef COMPILE_PCRE8 - if (c > 255) - { - if (op == OP_CLASS) RRETURN(MATCH_NOMATCH); - } - else -#endif - if ((BYTE_MAP[c/8] & (1 << (c&7))) == 0) RRETURN(MATCH_NOMATCH); - } - } - /* Control never gets here */ - } - - /* If maximizing, find the longest possible run, then work backwards. */ - - else - { - pp = eptr; - -#ifdef SUPPORT_UTF - if (utf) - { - for (i = min; i < max; i++) - { - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLEN(c, eptr, len); - if (c > 255) - { - if (op == OP_CLASS) break; - } - else - if ((BYTE_MAP[c/8] & (1 << (c&7))) == 0) break; - eptr += len; - } - - if (possessive) continue; /* No backtracking */ - - for (;;) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM18); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (eptr-- == pp) break; /* Stop if tried at original pos */ - BACKCHAR(eptr); - } - } - else -#endif - /* Not UTF mode */ - { - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - c = *eptr; -#ifndef COMPILE_PCRE8 - if (c > 255) - { - if (op == OP_CLASS) break; - } - else -#endif - if ((BYTE_MAP[c/8] & (1 << (c&7))) == 0) break; - eptr++; - } - - if (possessive) continue; /* No backtracking */ - - while (eptr >= pp) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM19); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - eptr--; - } - } - - RRETURN(MATCH_NOMATCH); - } -#undef BYTE_MAP - } - /* Control never gets here */ - - - /* Match an extended character class. In the 8-bit library, this opcode is - encountered only when UTF-8 mode mode is supported. In the 16-bit and - 32-bit libraries, codepoints greater than 255 may be encountered even when - UTF is not supported. */ - -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - case OP_XCLASS: - { - data = ecode + 1 + LINK_SIZE; /* Save for matching */ - ecode += GET(ecode, 1); /* Advance past the item */ - - switch (*ecode) - { - case OP_CRSTAR: - case OP_CRMINSTAR: - case OP_CRPLUS: - case OP_CRMINPLUS: - case OP_CRQUERY: - case OP_CRMINQUERY: - case OP_CRPOSSTAR: - case OP_CRPOSPLUS: - case OP_CRPOSQUERY: - c = *ecode++ - OP_CRSTAR; - if (c < OP_CRPOSSTAR - OP_CRSTAR) minimize = (c & 1) != 0; - else possessive = TRUE; - min = rep_min[c]; /* Pick up values from tables; */ - max = rep_max[c]; /* zero for max => infinity */ - if (max == 0) max = INT_MAX; - break; - - case OP_CRRANGE: - case OP_CRMINRANGE: - case OP_CRPOSRANGE: - minimize = (*ecode == OP_CRMINRANGE); - possessive = (*ecode == OP_CRPOSRANGE); - min = GET2(ecode, 1); - max = GET2(ecode, 1 + IMM2_SIZE); - if (max == 0) max = INT_MAX; - ecode += 1 + 2 * IMM2_SIZE; - break; - - default: /* No repeat follows */ - min = max = 1; - break; - } - - /* First, ensure the minimum number of matches are present. */ - - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - if (!PRIV(xclass)(c, data, utf)) RRETURN(MATCH_NOMATCH); - } - - /* If max == min we can continue with the main loop without the - need to recurse. */ - - if (min == max) continue; - - /* If minimizing, keep testing the rest of the expression and advancing - the pointer while it matches the class. */ - - if (minimize) - { - for (fi = min;; fi++) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM20); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - if (!PRIV(xclass)(c, data, utf)) RRETURN(MATCH_NOMATCH); - } - /* Control never gets here */ - } - - /* If maximizing, find the longest possible run, then work backwards. */ - - else - { - pp = eptr; - for (i = min; i < max; i++) - { - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } -#ifdef SUPPORT_UTF - GETCHARLENTEST(c, eptr, len); -#else - c = *eptr; -#endif - if (!PRIV(xclass)(c, data, utf)) break; - eptr += len; - } - - if (possessive) continue; /* No backtracking */ - - for(;;) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM21); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (eptr-- == pp) break; /* Stop if tried at original pos */ -#ifdef SUPPORT_UTF - if (utf) BACKCHAR(eptr); -#endif - } - RRETURN(MATCH_NOMATCH); - } - - /* Control never gets here */ - } -#endif /* End of XCLASS */ - - /* Match a single character, casefully */ - - case OP_CHAR: -#ifdef SUPPORT_UTF - if (utf) - { - length = 1; - ecode++; - GETCHARLEN(fc, ecode, length); - if (length > md->end_subject - eptr) - { - CHECK_PARTIAL(); /* Not SCHECK_PARTIAL() */ - RRETURN(MATCH_NOMATCH); - } - while (length-- > 0) if (*ecode++ != UCHAR21INC(eptr)) RRETURN(MATCH_NOMATCH); - } - else -#endif - /* Not UTF mode */ - { - if (md->end_subject - eptr < 1) - { - SCHECK_PARTIAL(); /* This one can use SCHECK_PARTIAL() */ - RRETURN(MATCH_NOMATCH); - } - if (ecode[1] != *eptr++) RRETURN(MATCH_NOMATCH); - ecode += 2; - } - break; - - /* Match a single character, caselessly. If we are at the end of the - subject, give up immediately. */ - - case OP_CHARI: - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - -#ifdef SUPPORT_UTF - if (utf) - { - length = 1; - ecode++; - GETCHARLEN(fc, ecode, length); - - /* If the pattern character's value is < 128, we have only one byte, and - we know that its other case must also be one byte long, so we can use the - fast lookup table. We know that there is at least one byte left in the - subject. */ - - if (fc < 128) - { - pcre_uint32 cc = UCHAR21(eptr); - if (md->lcc[fc] != TABLE_GET(cc, md->lcc, cc)) RRETURN(MATCH_NOMATCH); - ecode++; - eptr++; - } - - /* Otherwise we must pick up the subject character. Note that we cannot - use the value of "length" to check for sufficient bytes left, because the - other case of the character may have more or fewer bytes. */ - - else - { - pcre_uint32 dc; - GETCHARINC(dc, eptr); - ecode += length; - - /* If we have Unicode property support, we can use it to test the other - case of the character, if there is one. */ - - if (fc != dc) - { -#ifdef SUPPORT_UCP - if (dc != UCD_OTHERCASE(fc)) -#endif - RRETURN(MATCH_NOMATCH); - } - } - } - else -#endif /* SUPPORT_UTF */ - - /* Not UTF mode */ - { - if (TABLE_GET(ecode[1], md->lcc, ecode[1]) - != TABLE_GET(*eptr, md->lcc, *eptr)) RRETURN(MATCH_NOMATCH); - eptr++; - ecode += 2; - } - break; - - /* Match a single character repeatedly. */ - - case OP_EXACT: - case OP_EXACTI: - min = max = GET2(ecode, 1); - ecode += 1 + IMM2_SIZE; - goto REPEATCHAR; - - case OP_POSUPTO: - case OP_POSUPTOI: - possessive = TRUE; - /* Fall through */ - - case OP_UPTO: - case OP_UPTOI: - case OP_MINUPTO: - case OP_MINUPTOI: - min = 0; - max = GET2(ecode, 1); - minimize = *ecode == OP_MINUPTO || *ecode == OP_MINUPTOI; - ecode += 1 + IMM2_SIZE; - goto REPEATCHAR; - - case OP_POSSTAR: - case OP_POSSTARI: - possessive = TRUE; - min = 0; - max = INT_MAX; - ecode++; - goto REPEATCHAR; - - case OP_POSPLUS: - case OP_POSPLUSI: - possessive = TRUE; - min = 1; - max = INT_MAX; - ecode++; - goto REPEATCHAR; - - case OP_POSQUERY: - case OP_POSQUERYI: - possessive = TRUE; - min = 0; - max = 1; - ecode++; - goto REPEATCHAR; - - case OP_STAR: - case OP_STARI: - case OP_MINSTAR: - case OP_MINSTARI: - case OP_PLUS: - case OP_PLUSI: - case OP_MINPLUS: - case OP_MINPLUSI: - case OP_QUERY: - case OP_QUERYI: - case OP_MINQUERY: - case OP_MINQUERYI: - c = *ecode++ - ((op < OP_STARI)? OP_STAR : OP_STARI); - minimize = (c & 1) != 0; - min = rep_min[c]; /* Pick up values from tables; */ - max = rep_max[c]; /* zero for max => infinity */ - if (max == 0) max = INT_MAX; - - /* Common code for all repeated single-character matches. We first check - for the minimum number of characters. If the minimum equals the maximum, we - are done. Otherwise, if minimizing, check the rest of the pattern for a - match; if there isn't one, advance up to the maximum, one character at a - time. - - If maximizing, advance up to the maximum number of matching characters, - until eptr is past the end of the maximum run. If possessive, we are - then done (no backing up). Otherwise, match at this position; anything - other than no match is immediately returned. For nomatch, back up one - character, unless we are matching \R and the last thing matched was - \r\n, in which case, back up two bytes. When we reach the first optional - character position, we can save stack by doing a tail recurse. - - The various UTF/non-UTF and caseful/caseless cases are handled separately, - for speed. */ - - REPEATCHAR: -#ifdef SUPPORT_UTF - if (utf) - { - length = 1; - charptr = ecode; - GETCHARLEN(fc, ecode, length); - ecode += length; - - /* Handle multibyte character matching specially here. There is - support for caseless matching if UCP support is present. */ - - if (length > 1) - { -#ifdef SUPPORT_UCP - pcre_uint32 othercase; - if (op >= OP_STARI && /* Caseless */ - (othercase = UCD_OTHERCASE(fc)) != fc) - oclength = PRIV(ord2utf)(othercase, occhars); - else oclength = 0; -#endif /* SUPPORT_UCP */ - - for (i = 1; i <= min; i++) - { - if (eptr <= md->end_subject - length && - memcmp(eptr, charptr, IN_UCHARS(length)) == 0) eptr += length; -#ifdef SUPPORT_UCP - else if (oclength > 0 && - eptr <= md->end_subject - oclength && - memcmp(eptr, occhars, IN_UCHARS(oclength)) == 0) eptr += oclength; -#endif /* SUPPORT_UCP */ - else - { - CHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - } - - if (min == max) continue; - - if (minimize) - { - for (fi = min;; fi++) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM22); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr <= md->end_subject - length && - memcmp(eptr, charptr, IN_UCHARS(length)) == 0) eptr += length; -#ifdef SUPPORT_UCP - else if (oclength > 0 && - eptr <= md->end_subject - oclength && - memcmp(eptr, occhars, IN_UCHARS(oclength)) == 0) eptr += oclength; -#endif /* SUPPORT_UCP */ - else - { - CHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - } - /* Control never gets here */ - } - - else /* Maximize */ - { - pp = eptr; - for (i = min; i < max; i++) - { - if (eptr <= md->end_subject - length && - memcmp(eptr, charptr, IN_UCHARS(length)) == 0) eptr += length; -#ifdef SUPPORT_UCP - else if (oclength > 0 && - eptr <= md->end_subject - oclength && - memcmp(eptr, occhars, IN_UCHARS(oclength)) == 0) eptr += oclength; -#endif /* SUPPORT_UCP */ - else - { - CHECK_PARTIAL(); - break; - } - } - - if (possessive) continue; /* No backtracking */ - for(;;) - { - if (eptr <= pp) goto TAIL_RECURSE; - RMATCH(eptr, ecode, offset_top, md, eptrb, RM23); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); -#ifdef SUPPORT_UCP - eptr--; - BACKCHAR(eptr); -#else /* without SUPPORT_UCP */ - eptr -= length; -#endif /* SUPPORT_UCP */ - } - } - /* Control never gets here */ - } - - /* If the length of a UTF-8 character is 1, we fall through here, and - obey the code as for non-UTF-8 characters below, though in this case the - value of fc will always be < 128. */ - } - else -#endif /* SUPPORT_UTF */ - /* When not in UTF-8 mode, load a single-byte character. */ - fc = *ecode++; - - /* The value of fc at this point is always one character, though we may - or may not be in UTF mode. The code is duplicated for the caseless and - caseful cases, for speed, since matching characters is likely to be quite - common. First, ensure the minimum number of matches are present. If min = - max, continue at the same level without recursing. Otherwise, if - minimizing, keep trying the rest of the expression and advancing one - matching character if failing, up to the maximum. Alternatively, if - maximizing, find the maximum number of characters and work backwards. */ - - DPRINTF(("matching %c{%d,%d} against subject %.*s\n", fc, min, max, - max, (char *)eptr)); - - if (op >= OP_STARI) /* Caseless */ - { -#ifdef COMPILE_PCRE8 - /* fc must be < 128 if UTF is enabled. */ - foc = md->fcc[fc]; -#else -#ifdef SUPPORT_UTF -#ifdef SUPPORT_UCP - if (utf && fc > 127) - foc = UCD_OTHERCASE(fc); -#else - if (utf && fc > 127) - foc = fc; -#endif /* SUPPORT_UCP */ - else -#endif /* SUPPORT_UTF */ - foc = TABLE_GET(fc, md->fcc, fc); -#endif /* COMPILE_PCRE8 */ - - for (i = 1; i <= min; i++) - { - pcre_uint32 cc; /* Faster than pcre_uchar */ - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - cc = UCHAR21TEST(eptr); - if (fc != cc && foc != cc) RRETURN(MATCH_NOMATCH); - eptr++; - } - if (min == max) continue; - if (minimize) - { - for (fi = min;; fi++) - { - pcre_uint32 cc; /* Faster than pcre_uchar */ - RMATCH(eptr, ecode, offset_top, md, eptrb, RM24); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - cc = UCHAR21TEST(eptr); - if (fc != cc && foc != cc) RRETURN(MATCH_NOMATCH); - eptr++; - } - /* Control never gets here */ - } - else /* Maximize */ - { - pp = eptr; - for (i = min; i < max; i++) - { - pcre_uint32 cc; /* Faster than pcre_uchar */ - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - cc = UCHAR21TEST(eptr); - if (fc != cc && foc != cc) break; - eptr++; - } - if (possessive) continue; /* No backtracking */ - for (;;) - { - if (eptr == pp) goto TAIL_RECURSE; - RMATCH(eptr, ecode, offset_top, md, eptrb, RM25); - eptr--; - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - } - /* Control never gets here */ - } - } - - /* Caseful comparisons (includes all multi-byte characters) */ - - else - { - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - if (fc != UCHAR21INCTEST(eptr)) RRETURN(MATCH_NOMATCH); - } - - if (min == max) continue; - - if (minimize) - { - for (fi = min;; fi++) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM26); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - if (fc != UCHAR21INCTEST(eptr)) RRETURN(MATCH_NOMATCH); - } - /* Control never gets here */ - } - else /* Maximize */ - { - pp = eptr; - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - if (fc != UCHAR21TEST(eptr)) break; - eptr++; - } - if (possessive) continue; /* No backtracking */ - for (;;) - { - if (eptr == pp) goto TAIL_RECURSE; - RMATCH(eptr, ecode, offset_top, md, eptrb, RM27); - eptr--; - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - } - /* Control never gets here */ - } - } - /* Control never gets here */ - - /* Match a negated single one-byte character. The character we are - checking can be multibyte. */ - - case OP_NOT: - case OP_NOTI: - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } -#ifdef SUPPORT_UTF - if (utf) - { - register pcre_uint32 ch, och; - - ecode++; - GETCHARINC(ch, ecode); - GETCHARINC(c, eptr); - - if (op == OP_NOT) - { - if (ch == c) RRETURN(MATCH_NOMATCH); - } - else - { -#ifdef SUPPORT_UCP - if (ch > 127) - och = UCD_OTHERCASE(ch); -#else - if (ch > 127) - och = ch; -#endif /* SUPPORT_UCP */ - else - och = TABLE_GET(ch, md->fcc, ch); - if (ch == c || och == c) RRETURN(MATCH_NOMATCH); - } - } - else -#endif - { - register pcre_uint32 ch = ecode[1]; - c = *eptr++; - if (ch == c || (op == OP_NOTI && TABLE_GET(ch, md->fcc, ch) == c)) - RRETURN(MATCH_NOMATCH); - ecode += 2; - } - break; - - /* Match a negated single one-byte character repeatedly. This is almost a - repeat of the code for a repeated single character, but I haven't found a - nice way of commoning these up that doesn't require a test of the - positive/negative option for each character match. Maybe that wouldn't add - very much to the time taken, but character matching *is* what this is all - about... */ - - case OP_NOTEXACT: - case OP_NOTEXACTI: - min = max = GET2(ecode, 1); - ecode += 1 + IMM2_SIZE; - goto REPEATNOTCHAR; - - case OP_NOTUPTO: - case OP_NOTUPTOI: - case OP_NOTMINUPTO: - case OP_NOTMINUPTOI: - min = 0; - max = GET2(ecode, 1); - minimize = *ecode == OP_NOTMINUPTO || *ecode == OP_NOTMINUPTOI; - ecode += 1 + IMM2_SIZE; - goto REPEATNOTCHAR; - - case OP_NOTPOSSTAR: - case OP_NOTPOSSTARI: - possessive = TRUE; - min = 0; - max = INT_MAX; - ecode++; - goto REPEATNOTCHAR; - - case OP_NOTPOSPLUS: - case OP_NOTPOSPLUSI: - possessive = TRUE; - min = 1; - max = INT_MAX; - ecode++; - goto REPEATNOTCHAR; - - case OP_NOTPOSQUERY: - case OP_NOTPOSQUERYI: - possessive = TRUE; - min = 0; - max = 1; - ecode++; - goto REPEATNOTCHAR; - - case OP_NOTPOSUPTO: - case OP_NOTPOSUPTOI: - possessive = TRUE; - min = 0; - max = GET2(ecode, 1); - ecode += 1 + IMM2_SIZE; - goto REPEATNOTCHAR; - - case OP_NOTSTAR: - case OP_NOTSTARI: - case OP_NOTMINSTAR: - case OP_NOTMINSTARI: - case OP_NOTPLUS: - case OP_NOTPLUSI: - case OP_NOTMINPLUS: - case OP_NOTMINPLUSI: - case OP_NOTQUERY: - case OP_NOTQUERYI: - case OP_NOTMINQUERY: - case OP_NOTMINQUERYI: - c = *ecode++ - ((op >= OP_NOTSTARI)? OP_NOTSTARI: OP_NOTSTAR); - minimize = (c & 1) != 0; - min = rep_min[c]; /* Pick up values from tables; */ - max = rep_max[c]; /* zero for max => infinity */ - if (max == 0) max = INT_MAX; - - /* Common code for all repeated single-byte matches. */ - - REPEATNOTCHAR: - GETCHARINCTEST(fc, ecode); - - /* The code is duplicated for the caseless and caseful cases, for speed, - since matching characters is likely to be quite common. First, ensure the - minimum number of matches are present. If min = max, continue at the same - level without recursing. Otherwise, if minimizing, keep trying the rest of - the expression and advancing one matching character if failing, up to the - maximum. Alternatively, if maximizing, find the maximum number of - characters and work backwards. */ - - DPRINTF(("negative matching %c{%d,%d} against subject %.*s\n", fc, min, max, - max, (char *)eptr)); - - if (op >= OP_NOTSTARI) /* Caseless */ - { -#ifdef SUPPORT_UTF -#ifdef SUPPORT_UCP - if (utf && fc > 127) - foc = UCD_OTHERCASE(fc); -#else - if (utf && fc > 127) - foc = fc; -#endif /* SUPPORT_UCP */ - else -#endif /* SUPPORT_UTF */ - foc = TABLE_GET(fc, md->fcc, fc); - -#ifdef SUPPORT_UTF - if (utf) - { - register pcre_uint32 d; - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINC(d, eptr); - if (fc == d || (unsigned int)foc == d) RRETURN(MATCH_NOMATCH); - } - } - else -#endif /* SUPPORT_UTF */ - /* Not UTF mode */ - { - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - if (fc == *eptr || foc == *eptr) RRETURN(MATCH_NOMATCH); - eptr++; - } - } - - if (min == max) continue; - - if (minimize) - { -#ifdef SUPPORT_UTF - if (utf) - { - register pcre_uint32 d; - for (fi = min;; fi++) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM28); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINC(d, eptr); - if (fc == d || (unsigned int)foc == d) RRETURN(MATCH_NOMATCH); - } - } - else -#endif /*SUPPORT_UTF */ - /* Not UTF mode */ - { - for (fi = min;; fi++) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM29); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - if (fc == *eptr || foc == *eptr) RRETURN(MATCH_NOMATCH); - eptr++; - } - } - /* Control never gets here */ - } - - /* Maximize case */ - - else - { - pp = eptr; - -#ifdef SUPPORT_UTF - if (utf) - { - register pcre_uint32 d; - for (i = min; i < max; i++) - { - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLEN(d, eptr, len); - if (fc == d || (unsigned int)foc == d) break; - eptr += len; - } - if (possessive) continue; /* No backtracking */ - for(;;) - { - if (eptr <= pp) goto TAIL_RECURSE; - RMATCH(eptr, ecode, offset_top, md, eptrb, RM30); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - eptr--; - BACKCHAR(eptr); - } - } - else -#endif /* SUPPORT_UTF */ - /* Not UTF mode */ - { - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - if (fc == *eptr || foc == *eptr) break; - eptr++; - } - if (possessive) continue; /* No backtracking */ - for (;;) - { - if (eptr == pp) goto TAIL_RECURSE; - RMATCH(eptr, ecode, offset_top, md, eptrb, RM31); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - eptr--; - } - } - /* Control never gets here */ - } - } - - /* Caseful comparisons */ - - else - { -#ifdef SUPPORT_UTF - if (utf) - { - register pcre_uint32 d; - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINC(d, eptr); - if (fc == d) RRETURN(MATCH_NOMATCH); - } - } - else -#endif - /* Not UTF mode */ - { - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - if (fc == *eptr++) RRETURN(MATCH_NOMATCH); - } - } - - if (min == max) continue; - - if (minimize) - { -#ifdef SUPPORT_UTF - if (utf) - { - register pcre_uint32 d; - for (fi = min;; fi++) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM32); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINC(d, eptr); - if (fc == d) RRETURN(MATCH_NOMATCH); - } - } - else -#endif - /* Not UTF mode */ - { - for (fi = min;; fi++) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM33); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - if (fc == *eptr++) RRETURN(MATCH_NOMATCH); - } - } - /* Control never gets here */ - } - - /* Maximize case */ - - else - { - pp = eptr; - -#ifdef SUPPORT_UTF - if (utf) - { - register pcre_uint32 d; - for (i = min; i < max; i++) - { - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLEN(d, eptr, len); - if (fc == d) break; - eptr += len; - } - if (possessive) continue; /* No backtracking */ - for(;;) - { - if (eptr <= pp) goto TAIL_RECURSE; - RMATCH(eptr, ecode, offset_top, md, eptrb, RM34); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - eptr--; - BACKCHAR(eptr); - } - } - else -#endif - /* Not UTF mode */ - { - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - if (fc == *eptr) break; - eptr++; - } - if (possessive) continue; /* No backtracking */ - for (;;) - { - if (eptr == pp) goto TAIL_RECURSE; - RMATCH(eptr, ecode, offset_top, md, eptrb, RM35); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - eptr--; - } - } - /* Control never gets here */ - } - } - /* Control never gets here */ - - /* Match a single character type repeatedly; several different opcodes - share code. This is very similar to the code for single characters, but we - repeat it in the interests of efficiency. */ - - case OP_TYPEEXACT: - min = max = GET2(ecode, 1); - minimize = TRUE; - ecode += 1 + IMM2_SIZE; - goto REPEATTYPE; - - case OP_TYPEUPTO: - case OP_TYPEMINUPTO: - min = 0; - max = GET2(ecode, 1); - minimize = *ecode == OP_TYPEMINUPTO; - ecode += 1 + IMM2_SIZE; - goto REPEATTYPE; - - case OP_TYPEPOSSTAR: - possessive = TRUE; - min = 0; - max = INT_MAX; - ecode++; - goto REPEATTYPE; - - case OP_TYPEPOSPLUS: - possessive = TRUE; - min = 1; - max = INT_MAX; - ecode++; - goto REPEATTYPE; - - case OP_TYPEPOSQUERY: - possessive = TRUE; - min = 0; - max = 1; - ecode++; - goto REPEATTYPE; - - case OP_TYPEPOSUPTO: - possessive = TRUE; - min = 0; - max = GET2(ecode, 1); - ecode += 1 + IMM2_SIZE; - goto REPEATTYPE; - - case OP_TYPESTAR: - case OP_TYPEMINSTAR: - case OP_TYPEPLUS: - case OP_TYPEMINPLUS: - case OP_TYPEQUERY: - case OP_TYPEMINQUERY: - c = *ecode++ - OP_TYPESTAR; - minimize = (c & 1) != 0; - min = rep_min[c]; /* Pick up values from tables; */ - max = rep_max[c]; /* zero for max => infinity */ - if (max == 0) max = INT_MAX; - - /* Common code for all repeated single character type matches. Note that - in UTF-8 mode, '.' matches a character of any length, but for the other - character types, the valid characters are all one-byte long. */ - - REPEATTYPE: - ctype = *ecode++; /* Code for the character type */ - -#ifdef SUPPORT_UCP - if (ctype == OP_PROP || ctype == OP_NOTPROP) - { - prop_fail_result = ctype == OP_NOTPROP; - prop_type = *ecode++; - prop_value = *ecode++; - } - else prop_type = -1; -#endif - - /* First, ensure the minimum number of matches are present. Use inline - code for maximizing the speed, and do the type test once at the start - (i.e. keep it out of the loop). Separate the UTF-8 code completely as that - is tidier. Also separate the UCP code, which can be the same for both UTF-8 - and single-bytes. */ - - if (min > 0) - { -#ifdef SUPPORT_UCP - if (prop_type >= 0) - { - switch(prop_type) - { - case PT_ANY: - if (prop_fail_result) RRETURN(MATCH_NOMATCH); - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - } - break; - - case PT_LAMP: - for (i = 1; i <= min; i++) - { - int chartype; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - chartype = UCD_CHARTYPE(c); - if ((chartype == ucp_Lu || - chartype == ucp_Ll || - chartype == ucp_Lt) == prop_fail_result) - RRETURN(MATCH_NOMATCH); - } - break; - - case PT_GC: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - if ((UCD_CATEGORY(c) == prop_value) == prop_fail_result) - RRETURN(MATCH_NOMATCH); - } - break; - - case PT_PC: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - if ((UCD_CHARTYPE(c) == prop_value) == prop_fail_result) - RRETURN(MATCH_NOMATCH); - } - break; - - case PT_SC: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - if ((UCD_SCRIPT(c) == prop_value) == prop_fail_result) - RRETURN(MATCH_NOMATCH); - } - break; - - case PT_ALNUM: - for (i = 1; i <= min; i++) - { - int category; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - category = UCD_CATEGORY(c); - if ((category == ucp_L || category == ucp_N) == prop_fail_result) - RRETURN(MATCH_NOMATCH); - } - break; - - /* Perl space used to exclude VT, but from Perl 5.18 it is included, - which means that Perl space and POSIX space are now identical. PCRE - was changed at release 8.34. */ - - case PT_SPACE: /* Perl space */ - case PT_PXSPACE: /* POSIX space */ - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - switch(c) - { - HSPACE_CASES: - VSPACE_CASES: - if (prop_fail_result) RRETURN(MATCH_NOMATCH); - break; - - default: - if ((UCD_CATEGORY(c) == ucp_Z) == prop_fail_result) - RRETURN(MATCH_NOMATCH); - break; - } - } - break; - - case PT_WORD: - for (i = 1; i <= min; i++) - { - int category; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - category = UCD_CATEGORY(c); - if ((category == ucp_L || category == ucp_N || c == CHAR_UNDERSCORE) - == prop_fail_result) - RRETURN(MATCH_NOMATCH); - } - break; - - case PT_CLIST: - for (i = 1; i <= min; i++) - { - const pcre_uint32 *cp; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - cp = PRIV(ucd_caseless_sets) + prop_value; - for (;;) - { - if (c < *cp) - { if (prop_fail_result) break; else { RRETURN(MATCH_NOMATCH); } } - if (c == *cp++) - { if (prop_fail_result) { RRETURN(MATCH_NOMATCH); } else break; } - } - } - break; - - case PT_UCNC: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - if ((c == CHAR_DOLLAR_SIGN || c == CHAR_COMMERCIAL_AT || - c == CHAR_GRAVE_ACCENT || (c >= 0xa0 && c <= 0xd7ff) || - c >= 0xe000) == prop_fail_result) - RRETURN(MATCH_NOMATCH); - } - break; - - /* This should not occur */ - - default: - RRETURN(PCRE_ERROR_INTERNAL); - } - } - - /* Match extended Unicode sequences. We will get here only if the - support is in the binary; otherwise a compile-time error occurs. */ - - else if (ctype == OP_EXTUNI) - { - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - else - { - int lgb, rgb; - GETCHARINCTEST(c, eptr); - lgb = UCD_GRAPHBREAK(c); - while (eptr < md->end_subject) - { - int len = 1; - if (!utf) c = *eptr; else { GETCHARLEN(c, eptr, len); } - rgb = UCD_GRAPHBREAK(c); - if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; - lgb = rgb; - eptr += len; - } - } - CHECK_PARTIAL(); - } - } - - else -#endif /* SUPPORT_UCP */ - -/* Handle all other cases when the coding is UTF-8 */ - -#ifdef SUPPORT_UTF - if (utf) switch(ctype) - { - case OP_ANY: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - if (IS_NEWLINE(eptr)) RRETURN(MATCH_NOMATCH); - if (md->partial != 0 && - eptr + 1 >= md->end_subject && - NLBLOCK->nltype == NLTYPE_FIXED && - NLBLOCK->nllen == 2 && - UCHAR21(eptr) == NLBLOCK->nl[0]) - { - md->hitend = TRUE; - if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); - } - eptr++; - ACROSSCHAR(eptr < md->end_subject, *eptr, eptr++); - } - break; - - case OP_ALLANY: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - eptr++; - ACROSSCHAR(eptr < md->end_subject, *eptr, eptr++); - } - break; - - case OP_ANYBYTE: - if (eptr > md->end_subject - min) RRETURN(MATCH_NOMATCH); - eptr += min; - break; - - case OP_ANYNL: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINC(c, eptr); - switch(c) - { - default: RRETURN(MATCH_NOMATCH); - - case CHAR_CR: - if (eptr < md->end_subject && UCHAR21(eptr) == CHAR_LF) eptr++; - break; - - case CHAR_LF: - break; - - case CHAR_VT: - case CHAR_FF: - case CHAR_NEL: -#ifndef EBCDIC - case 0x2028: - case 0x2029: -#endif /* Not EBCDIC */ - if (md->bsr_anycrlf) RRETURN(MATCH_NOMATCH); - break; - } - } - break; - - case OP_NOT_HSPACE: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINC(c, eptr); - switch(c) - { - HSPACE_CASES: RRETURN(MATCH_NOMATCH); /* Byte and multibyte cases */ - default: break; - } - } - break; - - case OP_HSPACE: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINC(c, eptr); - switch(c) - { - HSPACE_CASES: break; /* Byte and multibyte cases */ - default: RRETURN(MATCH_NOMATCH); - } - } - break; - - case OP_NOT_VSPACE: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINC(c, eptr); - switch(c) - { - VSPACE_CASES: RRETURN(MATCH_NOMATCH); - default: break; - } - } - break; - - case OP_VSPACE: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINC(c, eptr); - switch(c) - { - VSPACE_CASES: break; - default: RRETURN(MATCH_NOMATCH); - } - } - break; - - case OP_NOT_DIGIT: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINC(c, eptr); - if (c < 128 && (md->ctypes[c] & ctype_digit) != 0) - RRETURN(MATCH_NOMATCH); - } - break; - - case OP_DIGIT: - for (i = 1; i <= min; i++) - { - pcre_uint32 cc; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - cc = UCHAR21(eptr); - if (cc >= 128 || (md->ctypes[cc] & ctype_digit) == 0) - RRETURN(MATCH_NOMATCH); - eptr++; - /* No need to skip more bytes - we know it's a 1-byte character */ - } - break; - - case OP_NOT_WHITESPACE: - for (i = 1; i <= min; i++) - { - pcre_uint32 cc; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - cc = UCHAR21(eptr); - if (cc < 128 && (md->ctypes[cc] & ctype_space) != 0) - RRETURN(MATCH_NOMATCH); - eptr++; - ACROSSCHAR(eptr < md->end_subject, *eptr, eptr++); - } - break; - - case OP_WHITESPACE: - for (i = 1; i <= min; i++) - { - pcre_uint32 cc; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - cc = UCHAR21(eptr); - if (cc >= 128 || (md->ctypes[cc] & ctype_space) == 0) - RRETURN(MATCH_NOMATCH); - eptr++; - /* No need to skip more bytes - we know it's a 1-byte character */ - } - break; - - case OP_NOT_WORDCHAR: - for (i = 1; i <= min; i++) - { - pcre_uint32 cc; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - cc = UCHAR21(eptr); - if (cc < 128 && (md->ctypes[cc] & ctype_word) != 0) - RRETURN(MATCH_NOMATCH); - eptr++; - ACROSSCHAR(eptr < md->end_subject, *eptr, eptr++); - } - break; - - case OP_WORDCHAR: - for (i = 1; i <= min; i++) - { - pcre_uint32 cc; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - cc = UCHAR21(eptr); - if (cc >= 128 || (md->ctypes[cc] & ctype_word) == 0) - RRETURN(MATCH_NOMATCH); - eptr++; - /* No need to skip more bytes - we know it's a 1-byte character */ - } - break; - - default: - RRETURN(PCRE_ERROR_INTERNAL); - } /* End switch(ctype) */ - - else -#endif /* SUPPORT_UTF */ - - /* Code for the non-UTF-8 case for minimum matching of operators other - than OP_PROP and OP_NOTPROP. */ - - switch(ctype) - { - case OP_ANY: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - if (IS_NEWLINE(eptr)) RRETURN(MATCH_NOMATCH); - if (md->partial != 0 && - eptr + 1 >= md->end_subject && - NLBLOCK->nltype == NLTYPE_FIXED && - NLBLOCK->nllen == 2 && - *eptr == NLBLOCK->nl[0]) - { - md->hitend = TRUE; - if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); - } - eptr++; - } - break; - - case OP_ALLANY: - if (eptr > md->end_subject - min) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - eptr += min; - break; - - case OP_ANYBYTE: - if (eptr > md->end_subject - min) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - eptr += min; - break; - - case OP_ANYNL: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - switch(*eptr++) - { - default: RRETURN(MATCH_NOMATCH); - - case CHAR_CR: - if (eptr < md->end_subject && *eptr == CHAR_LF) eptr++; - break; - - case CHAR_LF: - break; - - case CHAR_VT: - case CHAR_FF: - case CHAR_NEL: -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - case 0x2028: - case 0x2029: -#endif - if (md->bsr_anycrlf) RRETURN(MATCH_NOMATCH); - break; - } - } - break; - - case OP_NOT_HSPACE: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - switch(*eptr++) - { - default: break; - HSPACE_BYTE_CASES: -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - HSPACE_MULTIBYTE_CASES: -#endif - RRETURN(MATCH_NOMATCH); - } - } - break; - - case OP_HSPACE: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - switch(*eptr++) - { - default: RRETURN(MATCH_NOMATCH); - HSPACE_BYTE_CASES: -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - HSPACE_MULTIBYTE_CASES: -#endif - break; - } - } - break; - - case OP_NOT_VSPACE: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - switch(*eptr++) - { - VSPACE_BYTE_CASES: -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - VSPACE_MULTIBYTE_CASES: -#endif - RRETURN(MATCH_NOMATCH); - default: break; - } - } - break; - - case OP_VSPACE: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - switch(*eptr++) - { - default: RRETURN(MATCH_NOMATCH); - VSPACE_BYTE_CASES: -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - VSPACE_MULTIBYTE_CASES: -#endif - break; - } - } - break; - - case OP_NOT_DIGIT: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - if (MAX_255(*eptr) && (md->ctypes[*eptr] & ctype_digit) != 0) - RRETURN(MATCH_NOMATCH); - eptr++; - } - break; - - case OP_DIGIT: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - if (!MAX_255(*eptr) || (md->ctypes[*eptr] & ctype_digit) == 0) - RRETURN(MATCH_NOMATCH); - eptr++; - } - break; - - case OP_NOT_WHITESPACE: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - if (MAX_255(*eptr) && (md->ctypes[*eptr] & ctype_space) != 0) - RRETURN(MATCH_NOMATCH); - eptr++; - } - break; - - case OP_WHITESPACE: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - if (!MAX_255(*eptr) || (md->ctypes[*eptr] & ctype_space) == 0) - RRETURN(MATCH_NOMATCH); - eptr++; - } - break; - - case OP_NOT_WORDCHAR: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - if (MAX_255(*eptr) && (md->ctypes[*eptr] & ctype_word) != 0) - RRETURN(MATCH_NOMATCH); - eptr++; - } - break; - - case OP_WORDCHAR: - for (i = 1; i <= min; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - if (!MAX_255(*eptr) || (md->ctypes[*eptr] & ctype_word) == 0) - RRETURN(MATCH_NOMATCH); - eptr++; - } - break; - - default: - RRETURN(PCRE_ERROR_INTERNAL); - } - } - - /* If min = max, continue at the same level without recursing */ - - if (min == max) continue; - - /* If minimizing, we have to test the rest of the pattern before each - subsequent match. Again, separate the UTF-8 case for speed, and also - separate the UCP cases. */ - - if (minimize) - { -#ifdef SUPPORT_UCP - if (prop_type >= 0) - { - switch(prop_type) - { - case PT_ANY: - for (fi = min;; fi++) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM36); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - if (prop_fail_result) RRETURN(MATCH_NOMATCH); - } - /* Control never gets here */ - - case PT_LAMP: - for (fi = min;; fi++) - { - int chartype; - RMATCH(eptr, ecode, offset_top, md, eptrb, RM37); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - chartype = UCD_CHARTYPE(c); - if ((chartype == ucp_Lu || - chartype == ucp_Ll || - chartype == ucp_Lt) == prop_fail_result) - RRETURN(MATCH_NOMATCH); - } - /* Control never gets here */ - - case PT_GC: - for (fi = min;; fi++) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM38); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - if ((UCD_CATEGORY(c) == prop_value) == prop_fail_result) - RRETURN(MATCH_NOMATCH); - } - /* Control never gets here */ - - case PT_PC: - for (fi = min;; fi++) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM39); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - if ((UCD_CHARTYPE(c) == prop_value) == prop_fail_result) - RRETURN(MATCH_NOMATCH); - } - /* Control never gets here */ - - case PT_SC: - for (fi = min;; fi++) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM40); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - if ((UCD_SCRIPT(c) == prop_value) == prop_fail_result) - RRETURN(MATCH_NOMATCH); - } - /* Control never gets here */ - - case PT_ALNUM: - for (fi = min;; fi++) - { - int category; - RMATCH(eptr, ecode, offset_top, md, eptrb, RM59); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - category = UCD_CATEGORY(c); - if ((category == ucp_L || category == ucp_N) == prop_fail_result) - RRETURN(MATCH_NOMATCH); - } - /* Control never gets here */ - - /* Perl space used to exclude VT, but from Perl 5.18 it is included, - which means that Perl space and POSIX space are now identical. PCRE - was changed at release 8.34. */ - - case PT_SPACE: /* Perl space */ - case PT_PXSPACE: /* POSIX space */ - for (fi = min;; fi++) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM61); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - switch(c) - { - HSPACE_CASES: - VSPACE_CASES: - if (prop_fail_result) RRETURN(MATCH_NOMATCH); - break; - - default: - if ((UCD_CATEGORY(c) == ucp_Z) == prop_fail_result) - RRETURN(MATCH_NOMATCH); - break; - } - } - /* Control never gets here */ - - case PT_WORD: - for (fi = min;; fi++) - { - int category; - RMATCH(eptr, ecode, offset_top, md, eptrb, RM62); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - category = UCD_CATEGORY(c); - if ((category == ucp_L || - category == ucp_N || - c == CHAR_UNDERSCORE) - == prop_fail_result) - RRETURN(MATCH_NOMATCH); - } - /* Control never gets here */ - - case PT_CLIST: - for (fi = min;; fi++) - { - const pcre_uint32 *cp; - RMATCH(eptr, ecode, offset_top, md, eptrb, RM67); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - cp = PRIV(ucd_caseless_sets) + prop_value; - for (;;) - { - if (c < *cp) - { if (prop_fail_result) break; else { RRETURN(MATCH_NOMATCH); } } - if (c == *cp++) - { if (prop_fail_result) { RRETURN(MATCH_NOMATCH); } else break; } - } - } - /* Control never gets here */ - - case PT_UCNC: - for (fi = min;; fi++) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM60); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - GETCHARINCTEST(c, eptr); - if ((c == CHAR_DOLLAR_SIGN || c == CHAR_COMMERCIAL_AT || - c == CHAR_GRAVE_ACCENT || (c >= 0xa0 && c <= 0xd7ff) || - c >= 0xe000) == prop_fail_result) - RRETURN(MATCH_NOMATCH); - } - /* Control never gets here */ - - /* This should never occur */ - default: - RRETURN(PCRE_ERROR_INTERNAL); - } - } - - /* Match extended Unicode sequences. We will get here only if the - support is in the binary; otherwise a compile-time error occurs. */ - - else if (ctype == OP_EXTUNI) - { - for (fi = min;; fi++) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM41); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - else - { - int lgb, rgb; - GETCHARINCTEST(c, eptr); - lgb = UCD_GRAPHBREAK(c); - while (eptr < md->end_subject) - { - int len = 1; - if (!utf) c = *eptr; else { GETCHARLEN(c, eptr, len); } - rgb = UCD_GRAPHBREAK(c); - if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; - lgb = rgb; - eptr += len; - } - } - CHECK_PARTIAL(); - } - } - else -#endif /* SUPPORT_UCP */ - -#ifdef SUPPORT_UTF - if (utf) - { - for (fi = min;; fi++) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM42); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - if (ctype == OP_ANY && IS_NEWLINE(eptr)) - RRETURN(MATCH_NOMATCH); - GETCHARINC(c, eptr); - switch(ctype) - { - case OP_ANY: /* This is the non-NL case */ - if (md->partial != 0 && /* Take care with CRLF partial */ - eptr >= md->end_subject && - NLBLOCK->nltype == NLTYPE_FIXED && - NLBLOCK->nllen == 2 && - c == NLBLOCK->nl[0]) - { - md->hitend = TRUE; - if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); - } - break; - - case OP_ALLANY: - case OP_ANYBYTE: - break; - - case OP_ANYNL: - switch(c) - { - default: RRETURN(MATCH_NOMATCH); - case CHAR_CR: - if (eptr < md->end_subject && UCHAR21(eptr) == CHAR_LF) eptr++; - break; - - case CHAR_LF: - break; - - case CHAR_VT: - case CHAR_FF: - case CHAR_NEL: -#ifndef EBCDIC - case 0x2028: - case 0x2029: -#endif /* Not EBCDIC */ - if (md->bsr_anycrlf) RRETURN(MATCH_NOMATCH); - break; - } - break; - - case OP_NOT_HSPACE: - switch(c) - { - HSPACE_CASES: RRETURN(MATCH_NOMATCH); - default: break; - } - break; - - case OP_HSPACE: - switch(c) - { - HSPACE_CASES: break; - default: RRETURN(MATCH_NOMATCH); - } - break; - - case OP_NOT_VSPACE: - switch(c) - { - VSPACE_CASES: RRETURN(MATCH_NOMATCH); - default: break; - } - break; - - case OP_VSPACE: - switch(c) - { - VSPACE_CASES: break; - default: RRETURN(MATCH_NOMATCH); - } - break; - - case OP_NOT_DIGIT: - if (c < 256 && (md->ctypes[c] & ctype_digit) != 0) - RRETURN(MATCH_NOMATCH); - break; - - case OP_DIGIT: - if (c >= 256 || (md->ctypes[c] & ctype_digit) == 0) - RRETURN(MATCH_NOMATCH); - break; - - case OP_NOT_WHITESPACE: - if (c < 256 && (md->ctypes[c] & ctype_space) != 0) - RRETURN(MATCH_NOMATCH); - break; - - case OP_WHITESPACE: - if (c >= 256 || (md->ctypes[c] & ctype_space) == 0) - RRETURN(MATCH_NOMATCH); - break; - - case OP_NOT_WORDCHAR: - if (c < 256 && (md->ctypes[c] & ctype_word) != 0) - RRETURN(MATCH_NOMATCH); - break; - - case OP_WORDCHAR: - if (c >= 256 || (md->ctypes[c] & ctype_word) == 0) - RRETURN(MATCH_NOMATCH); - break; - - default: - RRETURN(PCRE_ERROR_INTERNAL); - } - } - } - else -#endif - /* Not UTF mode */ - { - for (fi = min;; fi++) - { - RMATCH(eptr, ecode, offset_top, md, eptrb, RM43); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - if (fi >= max) RRETURN(MATCH_NOMATCH); - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - RRETURN(MATCH_NOMATCH); - } - if (ctype == OP_ANY && IS_NEWLINE(eptr)) - RRETURN(MATCH_NOMATCH); - c = *eptr++; - switch(ctype) - { - case OP_ANY: /* This is the non-NL case */ - if (md->partial != 0 && /* Take care with CRLF partial */ - eptr >= md->end_subject && - NLBLOCK->nltype == NLTYPE_FIXED && - NLBLOCK->nllen == 2 && - c == NLBLOCK->nl[0]) - { - md->hitend = TRUE; - if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); - } - break; - - case OP_ALLANY: - case OP_ANYBYTE: - break; - - case OP_ANYNL: - switch(c) - { - default: RRETURN(MATCH_NOMATCH); - case CHAR_CR: - if (eptr < md->end_subject && *eptr == CHAR_LF) eptr++; - break; - - case CHAR_LF: - break; - - case CHAR_VT: - case CHAR_FF: - case CHAR_NEL: -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - case 0x2028: - case 0x2029: -#endif - if (md->bsr_anycrlf) RRETURN(MATCH_NOMATCH); - break; - } - break; - - case OP_NOT_HSPACE: - switch(c) - { - default: break; - HSPACE_BYTE_CASES: -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - HSPACE_MULTIBYTE_CASES: -#endif - RRETURN(MATCH_NOMATCH); - } - break; - - case OP_HSPACE: - switch(c) - { - default: RRETURN(MATCH_NOMATCH); - HSPACE_BYTE_CASES: -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - HSPACE_MULTIBYTE_CASES: -#endif - break; - } - break; - - case OP_NOT_VSPACE: - switch(c) - { - default: break; - VSPACE_BYTE_CASES: -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - VSPACE_MULTIBYTE_CASES: -#endif - RRETURN(MATCH_NOMATCH); - } - break; - - case OP_VSPACE: - switch(c) - { - default: RRETURN(MATCH_NOMATCH); - VSPACE_BYTE_CASES: -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - VSPACE_MULTIBYTE_CASES: -#endif - break; - } - break; - - case OP_NOT_DIGIT: - if (MAX_255(c) && (md->ctypes[c] & ctype_digit) != 0) RRETURN(MATCH_NOMATCH); - break; - - case OP_DIGIT: - if (!MAX_255(c) || (md->ctypes[c] & ctype_digit) == 0) RRETURN(MATCH_NOMATCH); - break; - - case OP_NOT_WHITESPACE: - if (MAX_255(c) && (md->ctypes[c] & ctype_space) != 0) RRETURN(MATCH_NOMATCH); - break; - - case OP_WHITESPACE: - if (!MAX_255(c) || (md->ctypes[c] & ctype_space) == 0) RRETURN(MATCH_NOMATCH); - break; - - case OP_NOT_WORDCHAR: - if (MAX_255(c) && (md->ctypes[c] & ctype_word) != 0) RRETURN(MATCH_NOMATCH); - break; - - case OP_WORDCHAR: - if (!MAX_255(c) || (md->ctypes[c] & ctype_word) == 0) RRETURN(MATCH_NOMATCH); - break; - - default: - RRETURN(PCRE_ERROR_INTERNAL); - } - } - } - /* Control never gets here */ - } - - /* If maximizing, it is worth using inline code for speed, doing the type - test once at the start (i.e. keep it out of the loop). Again, keep the - UTF-8 and UCP stuff separate. */ - - else - { - pp = eptr; /* Remember where we started */ - -#ifdef SUPPORT_UCP - if (prop_type >= 0) - { - switch(prop_type) - { - case PT_ANY: - for (i = min; i < max; i++) - { - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLENTEST(c, eptr, len); - if (prop_fail_result) break; - eptr+= len; - } - break; - - case PT_LAMP: - for (i = min; i < max; i++) - { - int chartype; - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLENTEST(c, eptr, len); - chartype = UCD_CHARTYPE(c); - if ((chartype == ucp_Lu || - chartype == ucp_Ll || - chartype == ucp_Lt) == prop_fail_result) - break; - eptr+= len; - } - break; - - case PT_GC: - for (i = min; i < max; i++) - { - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLENTEST(c, eptr, len); - if ((UCD_CATEGORY(c) == prop_value) == prop_fail_result) break; - eptr+= len; - } - break; - - case PT_PC: - for (i = min; i < max; i++) - { - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLENTEST(c, eptr, len); - if ((UCD_CHARTYPE(c) == prop_value) == prop_fail_result) break; - eptr+= len; - } - break; - - case PT_SC: - for (i = min; i < max; i++) - { - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLENTEST(c, eptr, len); - if ((UCD_SCRIPT(c) == prop_value) == prop_fail_result) break; - eptr+= len; - } - break; - - case PT_ALNUM: - for (i = min; i < max; i++) - { - int category; - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLENTEST(c, eptr, len); - category = UCD_CATEGORY(c); - if ((category == ucp_L || category == ucp_N) == prop_fail_result) - break; - eptr+= len; - } - break; - - /* Perl space used to exclude VT, but from Perl 5.18 it is included, - which means that Perl space and POSIX space are now identical. PCRE - was changed at release 8.34. */ - - case PT_SPACE: /* Perl space */ - case PT_PXSPACE: /* POSIX space */ - for (i = min; i < max; i++) - { - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLENTEST(c, eptr, len); - switch(c) - { - HSPACE_CASES: - VSPACE_CASES: - if (prop_fail_result) goto ENDLOOP99; /* Break the loop */ - break; - - default: - if ((UCD_CATEGORY(c) == ucp_Z) == prop_fail_result) - goto ENDLOOP99; /* Break the loop */ - break; - } - eptr+= len; - } - ENDLOOP99: - break; - - case PT_WORD: - for (i = min; i < max; i++) - { - int category; - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLENTEST(c, eptr, len); - category = UCD_CATEGORY(c); - if ((category == ucp_L || category == ucp_N || - c == CHAR_UNDERSCORE) == prop_fail_result) - break; - eptr+= len; - } - break; - - case PT_CLIST: - for (i = min; i < max; i++) - { - const pcre_uint32 *cp; - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLENTEST(c, eptr, len); - cp = PRIV(ucd_caseless_sets) + prop_value; - for (;;) - { - if (c < *cp) - { if (prop_fail_result) break; else goto GOT_MAX; } - if (c == *cp++) - { if (prop_fail_result) goto GOT_MAX; else break; } - } - eptr += len; - } - GOT_MAX: - break; - - case PT_UCNC: - for (i = min; i < max; i++) - { - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLENTEST(c, eptr, len); - if ((c == CHAR_DOLLAR_SIGN || c == CHAR_COMMERCIAL_AT || - c == CHAR_GRAVE_ACCENT || (c >= 0xa0 && c <= 0xd7ff) || - c >= 0xe000) == prop_fail_result) - break; - eptr += len; - } - break; - - default: - RRETURN(PCRE_ERROR_INTERNAL); - } - - /* eptr is now past the end of the maximum run */ - - if (possessive) continue; /* No backtracking */ - for(;;) - { - if (eptr <= pp) goto TAIL_RECURSE; - RMATCH(eptr, ecode, offset_top, md, eptrb, RM44); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - eptr--; - if (utf) BACKCHAR(eptr); - } - } - - /* Match extended Unicode grapheme clusters. We will get here only if the - support is in the binary; otherwise a compile-time error occurs. */ - - else if (ctype == OP_EXTUNI) - { - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - else - { - int lgb, rgb; - GETCHARINCTEST(c, eptr); - lgb = UCD_GRAPHBREAK(c); - while (eptr < md->end_subject) - { - int len = 1; - if (!utf) c = *eptr; else { GETCHARLEN(c, eptr, len); } - rgb = UCD_GRAPHBREAK(c); - if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; - lgb = rgb; - eptr += len; - } - } - CHECK_PARTIAL(); - } - - /* eptr is now past the end of the maximum run */ - - if (possessive) continue; /* No backtracking */ - - /* We use <= pp rather than == pp to detect the start of the run while - backtracking because the use of \C in UTF mode can cause BACKCHAR to - move back past pp. This is just palliative; the use of \C in UTF mode - is fraught with danger. */ - - for(;;) - { - int lgb, rgb; - PCRE_PUCHAR fptr; - - if (eptr <= pp) goto TAIL_RECURSE; /* At start of char run */ - RMATCH(eptr, ecode, offset_top, md, eptrb, RM45); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - - /* Backtracking over an extended grapheme cluster involves inspecting - the previous two characters (if present) to see if a break is - permitted between them. */ - - eptr--; - if (!utf) c = *eptr; else - { - BACKCHAR(eptr); - GETCHAR(c, eptr); - } - rgb = UCD_GRAPHBREAK(c); - - for (;;) - { - if (eptr <= pp) goto TAIL_RECURSE; /* At start of char run */ - fptr = eptr - 1; - if (!utf) c = *fptr; else - { - BACKCHAR(fptr); - GETCHAR(c, fptr); - } - lgb = UCD_GRAPHBREAK(c); - if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; - eptr = fptr; - rgb = lgb; - } - } - } - - else -#endif /* SUPPORT_UCP */ - -#ifdef SUPPORT_UTF - if (utf) - { - switch(ctype) - { - case OP_ANY: - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - if (IS_NEWLINE(eptr)) break; - if (md->partial != 0 && /* Take care with CRLF partial */ - eptr + 1 >= md->end_subject && - NLBLOCK->nltype == NLTYPE_FIXED && - NLBLOCK->nllen == 2 && - UCHAR21(eptr) == NLBLOCK->nl[0]) - { - md->hitend = TRUE; - if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); - } - eptr++; - ACROSSCHAR(eptr < md->end_subject, *eptr, eptr++); - } - break; - - case OP_ALLANY: - if (max < INT_MAX) - { - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - eptr++; - ACROSSCHAR(eptr < md->end_subject, *eptr, eptr++); - } - } - else - { - eptr = md->end_subject; /* Unlimited UTF-8 repeat */ - SCHECK_PARTIAL(); - } - break; - - /* The byte case is the same as non-UTF8 */ - - case OP_ANYBYTE: - c = max - min; - if (c > (unsigned int)(md->end_subject - eptr)) - { - eptr = md->end_subject; - SCHECK_PARTIAL(); - } - else eptr += c; - break; - - case OP_ANYNL: - for (i = min; i < max; i++) - { - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLEN(c, eptr, len); - if (c == CHAR_CR) - { - if (++eptr >= md->end_subject) break; - if (UCHAR21(eptr) == CHAR_LF) eptr++; - } - else - { - if (c != CHAR_LF && - (md->bsr_anycrlf || - (c != CHAR_VT && c != CHAR_FF && c != CHAR_NEL -#ifndef EBCDIC - && c != 0x2028 && c != 0x2029 -#endif /* Not EBCDIC */ - ))) - break; - eptr += len; - } - } - break; - - case OP_NOT_HSPACE: - case OP_HSPACE: - for (i = min; i < max; i++) - { - BOOL gotspace; - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLEN(c, eptr, len); - switch(c) - { - HSPACE_CASES: gotspace = TRUE; break; - default: gotspace = FALSE; break; - } - if (gotspace == (ctype == OP_NOT_HSPACE)) break; - eptr += len; - } - break; - - case OP_NOT_VSPACE: - case OP_VSPACE: - for (i = min; i < max; i++) - { - BOOL gotspace; - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLEN(c, eptr, len); - switch(c) - { - VSPACE_CASES: gotspace = TRUE; break; - default: gotspace = FALSE; break; - } - if (gotspace == (ctype == OP_NOT_VSPACE)) break; - eptr += len; - } - break; - - case OP_NOT_DIGIT: - for (i = min; i < max; i++) - { - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLEN(c, eptr, len); - if (c < 256 && (md->ctypes[c] & ctype_digit) != 0) break; - eptr+= len; - } - break; - - case OP_DIGIT: - for (i = min; i < max; i++) - { - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLEN(c, eptr, len); - if (c >= 256 ||(md->ctypes[c] & ctype_digit) == 0) break; - eptr+= len; - } - break; - - case OP_NOT_WHITESPACE: - for (i = min; i < max; i++) - { - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLEN(c, eptr, len); - if (c < 256 && (md->ctypes[c] & ctype_space) != 0) break; - eptr+= len; - } - break; - - case OP_WHITESPACE: - for (i = min; i < max; i++) - { - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLEN(c, eptr, len); - if (c >= 256 ||(md->ctypes[c] & ctype_space) == 0) break; - eptr+= len; - } - break; - - case OP_NOT_WORDCHAR: - for (i = min; i < max; i++) - { - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLEN(c, eptr, len); - if (c < 256 && (md->ctypes[c] & ctype_word) != 0) break; - eptr+= len; - } - break; - - case OP_WORDCHAR: - for (i = min; i < max; i++) - { - int len = 1; - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - GETCHARLEN(c, eptr, len); - if (c >= 256 || (md->ctypes[c] & ctype_word) == 0) break; - eptr+= len; - } - break; - - default: - RRETURN(PCRE_ERROR_INTERNAL); - } - - if (possessive) continue; /* No backtracking */ - for(;;) - { - if (eptr <= pp) goto TAIL_RECURSE; - RMATCH(eptr, ecode, offset_top, md, eptrb, RM46); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - eptr--; - BACKCHAR(eptr); - if (ctype == OP_ANYNL && eptr > pp && UCHAR21(eptr) == CHAR_NL && - UCHAR21(eptr - 1) == CHAR_CR) eptr--; - } - } - else -#endif /* SUPPORT_UTF */ - /* Not UTF mode */ - { - switch(ctype) - { - case OP_ANY: - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - if (IS_NEWLINE(eptr)) break; - if (md->partial != 0 && /* Take care with CRLF partial */ - eptr + 1 >= md->end_subject && - NLBLOCK->nltype == NLTYPE_FIXED && - NLBLOCK->nllen == 2 && - *eptr == NLBLOCK->nl[0]) - { - md->hitend = TRUE; - if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); - } - eptr++; - } - break; - - case OP_ALLANY: - case OP_ANYBYTE: - c = max - min; - if (c > (unsigned int)(md->end_subject - eptr)) - { - eptr = md->end_subject; - SCHECK_PARTIAL(); - } - else eptr += c; - break; - - case OP_ANYNL: - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - c = *eptr; - if (c == CHAR_CR) - { - if (++eptr >= md->end_subject) break; - if (*eptr == CHAR_LF) eptr++; - } - else - { - if (c != CHAR_LF && (md->bsr_anycrlf || - (c != CHAR_VT && c != CHAR_FF && c != CHAR_NEL -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - && c != 0x2028 && c != 0x2029 -#endif - ))) break; - eptr++; - } - } - break; - - case OP_NOT_HSPACE: - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - switch(*eptr) - { - default: eptr++; break; - HSPACE_BYTE_CASES: -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - HSPACE_MULTIBYTE_CASES: -#endif - goto ENDLOOP00; - } - } - ENDLOOP00: - break; - - case OP_HSPACE: - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - switch(*eptr) - { - default: goto ENDLOOP01; - HSPACE_BYTE_CASES: -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - HSPACE_MULTIBYTE_CASES: -#endif - eptr++; break; - } - } - ENDLOOP01: - break; - - case OP_NOT_VSPACE: - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - switch(*eptr) - { - default: eptr++; break; - VSPACE_BYTE_CASES: -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - VSPACE_MULTIBYTE_CASES: -#endif - goto ENDLOOP02; - } - } - ENDLOOP02: - break; - - case OP_VSPACE: - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - switch(*eptr) - { - default: goto ENDLOOP03; - VSPACE_BYTE_CASES: -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - VSPACE_MULTIBYTE_CASES: -#endif - eptr++; break; - } - } - ENDLOOP03: - break; - - case OP_NOT_DIGIT: - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - if (MAX_255(*eptr) && (md->ctypes[*eptr] & ctype_digit) != 0) break; - eptr++; - } - break; - - case OP_DIGIT: - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - if (!MAX_255(*eptr) || (md->ctypes[*eptr] & ctype_digit) == 0) break; - eptr++; - } - break; - - case OP_NOT_WHITESPACE: - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - if (MAX_255(*eptr) && (md->ctypes[*eptr] & ctype_space) != 0) break; - eptr++; - } - break; - - case OP_WHITESPACE: - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - if (!MAX_255(*eptr) || (md->ctypes[*eptr] & ctype_space) == 0) break; - eptr++; - } - break; - - case OP_NOT_WORDCHAR: - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - if (MAX_255(*eptr) && (md->ctypes[*eptr] & ctype_word) != 0) break; - eptr++; - } - break; - - case OP_WORDCHAR: - for (i = min; i < max; i++) - { - if (eptr >= md->end_subject) - { - SCHECK_PARTIAL(); - break; - } - if (!MAX_255(*eptr) || (md->ctypes[*eptr] & ctype_word) == 0) break; - eptr++; - } - break; - - default: - RRETURN(PCRE_ERROR_INTERNAL); - } - - if (possessive) continue; /* No backtracking */ - for (;;) - { - if (eptr == pp) goto TAIL_RECURSE; - RMATCH(eptr, ecode, offset_top, md, eptrb, RM47); - if (rrc != MATCH_NOMATCH) RRETURN(rrc); - eptr--; - if (ctype == OP_ANYNL && eptr > pp && *eptr == CHAR_LF && - eptr[-1] == CHAR_CR) eptr--; - } - } - - /* Control never gets here */ - } - - /* There's been some horrible disaster. Arrival here can only mean there is - something seriously wrong in the code above or the OP_xxx definitions. */ - - default: - DPRINTF(("Unknown opcode %d\n", *ecode)); - RRETURN(PCRE_ERROR_UNKNOWN_OPCODE); - } - - /* Do not stick any code in here without much thought; it is assumed - that "continue" in the code above comes out to here to repeat the main - loop. */ - - } /* End of main loop */ -/* Control never reaches here */ - - -/* When compiling to use the heap rather than the stack for recursive calls to -match(), the RRETURN() macro jumps here. The number that is saved in -frame->Xwhere indicates which label we actually want to return to. */ - -#ifdef NO_RECURSE -#define LBL(val) case val: goto L_RM##val; -HEAP_RETURN: -switch (frame->Xwhere) - { - LBL( 1) LBL( 2) LBL( 3) LBL( 4) LBL( 5) LBL( 6) LBL( 7) LBL( 8) - LBL( 9) LBL(10) LBL(11) LBL(12) LBL(13) LBL(14) LBL(15) LBL(17) - LBL(19) LBL(24) LBL(25) LBL(26) LBL(27) LBL(29) LBL(31) LBL(33) - LBL(35) LBL(43) LBL(47) LBL(48) LBL(49) LBL(50) LBL(51) LBL(52) - LBL(53) LBL(54) LBL(55) LBL(56) LBL(57) LBL(58) LBL(63) LBL(64) - LBL(65) LBL(66) -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - LBL(20) LBL(21) -#endif -#ifdef SUPPORT_UTF - LBL(16) LBL(18) - LBL(22) LBL(23) LBL(28) LBL(30) - LBL(32) LBL(34) LBL(42) LBL(46) -#ifdef SUPPORT_UCP - LBL(36) LBL(37) LBL(38) LBL(39) LBL(40) LBL(41) LBL(44) LBL(45) - LBL(59) LBL(60) LBL(61) LBL(62) LBL(67) -#endif /* SUPPORT_UCP */ -#endif /* SUPPORT_UTF */ - default: - DPRINTF(("jump error in pcre match: label %d non-existent\n", frame->Xwhere)); - return PCRE_ERROR_INTERNAL; - } -#undef LBL -#endif /* NO_RECURSE */ -} - - -/*************************************************************************** -**************************************************************************** - RECURSION IN THE match() FUNCTION - -Undefine all the macros that were defined above to handle this. */ - -#ifdef NO_RECURSE -#undef eptr -#undef ecode -#undef mstart -#undef offset_top -#undef eptrb -#undef flags - -#undef callpat -#undef charptr -#undef data -#undef next -#undef pp -#undef prev -#undef saved_eptr - -#undef new_recursive - -#undef cur_is_word -#undef condition -#undef prev_is_word - -#undef ctype -#undef length -#undef max -#undef min -#undef number -#undef offset -#undef op -#undef save_capture_last -#undef save_offset1 -#undef save_offset2 -#undef save_offset3 -#undef stacksave - -#undef newptrb - -#endif - -/* These two are defined as macros in both cases */ - -#undef fc -#undef fi - -/*************************************************************************** -***************************************************************************/ - - -#ifdef NO_RECURSE -/************************************************* -* Release allocated heap frames * -*************************************************/ - -/* This function releases all the allocated frames. The base frame is on the -machine stack, and so must not be freed. - -Argument: the address of the base frame -Returns: nothing -*/ - -static void -release_match_heapframes (heapframe *frame_base) -{ -heapframe *nextframe = frame_base->Xnextframe; -while (nextframe != NULL) - { - heapframe *oldframe = nextframe; - nextframe = nextframe->Xnextframe; - (PUBL(stack_free))(oldframe); - } -} -#endif - - -/************************************************* -* Execute a Regular Expression * -*************************************************/ - -/* This function applies a compiled re to a subject string and picks out -portions of the string if it matches. Two elements in the vector are set for -each substring: the offsets to the start and end of the substring. - -Arguments: - argument_re points to the compiled expression - extra_data points to extra data or is NULL - subject points to the subject string - length length of subject string (may contain binary zeros) - start_offset where to start in the subject string - options option bits - offsets points to a vector of ints to be filled in with offsets - offsetcount the number of elements in the vector - -Returns: > 0 => success; value is the number of elements filled in - = 0 => success, but offsets is not big enough - -1 => failed to match - < -1 => some kind of unexpected problem -*/ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre_exec(const pcre *argument_re, const pcre_extra *extra_data, - PCRE_SPTR subject, int length, int start_offset, int options, int *offsets, - int offsetcount) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre16_exec(const pcre16 *argument_re, const pcre16_extra *extra_data, - PCRE_SPTR16 subject, int length, int start_offset, int options, int *offsets, - int offsetcount) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre32_exec(const pcre32 *argument_re, const pcre32_extra *extra_data, - PCRE_SPTR32 subject, int length, int start_offset, int options, int *offsets, - int offsetcount) -#endif -{ -int rc, ocount, arg_offset_max; -int newline; -BOOL using_temporary_offsets = FALSE; -BOOL anchored; -BOOL startline; -BOOL firstline; -BOOL utf; -BOOL has_first_char = FALSE; -BOOL has_req_char = FALSE; -pcre_uchar first_char = 0; -pcre_uchar first_char2 = 0; -pcre_uchar req_char = 0; -pcre_uchar req_char2 = 0; -match_data match_block; -match_data *md = &match_block; -const pcre_uint8 *tables; -const pcre_uint8 *start_bits = NULL; -PCRE_PUCHAR start_match = (PCRE_PUCHAR)subject + start_offset; -PCRE_PUCHAR end_subject; -PCRE_PUCHAR start_partial = NULL; -PCRE_PUCHAR match_partial = NULL; -PCRE_PUCHAR req_char_ptr = start_match - 1; - -const pcre_study_data *study; -const REAL_PCRE *re = (const REAL_PCRE *)argument_re; - -#ifdef NO_RECURSE -heapframe frame_zero; -frame_zero.Xprevframe = NULL; /* Marks the top level */ -frame_zero.Xnextframe = NULL; /* None are allocated yet */ -md->match_frames_base = &frame_zero; -#endif - -/* Check for the special magic call that measures the size of the stack used -per recursive call of match(). Without the funny casting for sizeof, a Windows -compiler gave this error: "unary minus operator applied to unsigned type, -result still unsigned". Hopefully the cast fixes that. */ - -if (re == NULL && extra_data == NULL && subject == NULL && length == -999 && - start_offset == -999) -#ifdef NO_RECURSE - return -((int)sizeof(heapframe)); -#else - return match(NULL, NULL, NULL, 0, NULL, NULL, 0); -#endif - -/* Plausibility checks */ - -if ((options & ~PUBLIC_EXEC_OPTIONS) != 0) return PCRE_ERROR_BADOPTION; -if (re == NULL || subject == NULL || (offsets == NULL && offsetcount > 0)) - return PCRE_ERROR_NULL; -if (offsetcount < 0) return PCRE_ERROR_BADCOUNT; -if (length < 0) return PCRE_ERROR_BADLENGTH; -if (start_offset < 0 || start_offset > length) return PCRE_ERROR_BADOFFSET; - -/* Check that the first field in the block is the magic number. If it is not, -return with PCRE_ERROR_BADMAGIC. However, if the magic number is equal to -REVERSED_MAGIC_NUMBER we return with PCRE_ERROR_BADENDIANNESS, which -means that the pattern is likely compiled with different endianness. */ - -if (re->magic_number != MAGIC_NUMBER) - return re->magic_number == REVERSED_MAGIC_NUMBER? - PCRE_ERROR_BADENDIANNESS:PCRE_ERROR_BADMAGIC; -if ((re->flags & PCRE_MODE) == 0) return PCRE_ERROR_BADMODE; - -/* These two settings are used in the code for checking a UTF-8 string that -follows immediately afterwards. Other values in the md block are used only -during "normal" pcre_exec() processing, not when the JIT support is in use, -so they are set up later. */ - -/* PCRE_UTF16 has the same value as PCRE_UTF8. */ -utf = md->utf = (re->options & PCRE_UTF8) != 0; -md->partial = ((options & PCRE_PARTIAL_HARD) != 0)? 2 : - ((options & PCRE_PARTIAL_SOFT) != 0)? 1 : 0; - -/* Check a UTF-8 string if required. Pass back the character offset and error -code for an invalid string if a results vector is available. */ - -#ifdef SUPPORT_UTF -if (utf && (options & PCRE_NO_UTF8_CHECK) == 0) - { - int erroroffset; - int errorcode = PRIV(valid_utf)((PCRE_PUCHAR)subject, length, &erroroffset); - if (errorcode != 0) - { - if (offsetcount >= 2) - { - offsets[0] = erroroffset; - offsets[1] = errorcode; - } -#if defined COMPILE_PCRE8 - return (errorcode <= PCRE_UTF8_ERR5 && md->partial > 1)? - PCRE_ERROR_SHORTUTF8 : PCRE_ERROR_BADUTF8; -#elif defined COMPILE_PCRE16 - return (errorcode <= PCRE_UTF16_ERR1 && md->partial > 1)? - PCRE_ERROR_SHORTUTF16 : PCRE_ERROR_BADUTF16; -#elif defined COMPILE_PCRE32 - return PCRE_ERROR_BADUTF32; -#endif - } -#if defined COMPILE_PCRE8 || defined COMPILE_PCRE16 - /* Check that a start_offset points to the start of a UTF character. */ - if (start_offset > 0 && start_offset < length && - NOT_FIRSTCHAR(((PCRE_PUCHAR)subject)[start_offset])) - return PCRE_ERROR_BADUTF8_OFFSET; -#endif - } -#endif - -/* If the pattern was successfully studied with JIT support, run the JIT -executable instead of the rest of this function. Most options must be set at -compile time for the JIT code to be usable. Fallback to the normal code path if -an unsupported flag is set. */ - -#ifdef SUPPORT_JIT -if (extra_data != NULL - && (extra_data->flags & (PCRE_EXTRA_EXECUTABLE_JIT | - PCRE_EXTRA_TABLES)) == PCRE_EXTRA_EXECUTABLE_JIT - && extra_data->executable_jit != NULL - && (options & ~PUBLIC_JIT_EXEC_OPTIONS) == 0) - { - rc = PRIV(jit_exec)(extra_data, (const pcre_uchar *)subject, length, - start_offset, options, offsets, offsetcount); - - /* PCRE_ERROR_NULL means that the selected normal or partial matching - mode is not compiled. In this case we simply fallback to interpreter. */ - - if (rc != PCRE_ERROR_JIT_BADOPTION) return rc; - } -#endif - -/* Carry on with non-JIT matching. This information is for finding all the -numbers associated with a given name, for condition testing. */ - -md->name_table = (pcre_uchar *)re + re->name_table_offset; -md->name_count = re->name_count; -md->name_entry_size = re->name_entry_size; - -/* Fish out the optional data from the extra_data structure, first setting -the default values. */ - -study = NULL; -md->match_limit = MATCH_LIMIT; -md->match_limit_recursion = MATCH_LIMIT_RECURSION; -md->callout_data = NULL; - -/* The table pointer is always in native byte order. */ - -tables = re->tables; - -/* The two limit values override the defaults, whatever their value. */ - -if (extra_data != NULL) - { - unsigned long int flags = extra_data->flags; - if ((flags & PCRE_EXTRA_STUDY_DATA) != 0) - study = (const pcre_study_data *)extra_data->study_data; - if ((flags & PCRE_EXTRA_MATCH_LIMIT) != 0) - md->match_limit = extra_data->match_limit; - if ((flags & PCRE_EXTRA_MATCH_LIMIT_RECURSION) != 0) - md->match_limit_recursion = extra_data->match_limit_recursion; - if ((flags & PCRE_EXTRA_CALLOUT_DATA) != 0) - md->callout_data = extra_data->callout_data; - if ((flags & PCRE_EXTRA_TABLES) != 0) tables = extra_data->tables; - } - -/* Limits in the regex override only if they are smaller. */ - -if ((re->flags & PCRE_MLSET) != 0 && re->limit_match < md->match_limit) - md->match_limit = re->limit_match; - -if ((re->flags & PCRE_RLSET) != 0 && - re->limit_recursion < md->match_limit_recursion) - md->match_limit_recursion = re->limit_recursion; - -/* If the exec call supplied NULL for tables, use the inbuilt ones. This -is a feature that makes it possible to save compiled regex and re-use them -in other programs later. */ - -if (tables == NULL) tables = PRIV(default_tables); - -/* Set up other data */ - -anchored = ((re->options | options) & PCRE_ANCHORED) != 0; -startline = (re->flags & PCRE_STARTLINE) != 0; -firstline = (re->options & PCRE_FIRSTLINE) != 0; - -/* The code starts after the real_pcre block and the capture name table. */ - -md->start_code = (const pcre_uchar *)re + re->name_table_offset + - re->name_count * re->name_entry_size; - -md->start_subject = (PCRE_PUCHAR)subject; -md->start_offset = start_offset; -md->end_subject = md->start_subject + length; -end_subject = md->end_subject; - -md->endonly = (re->options & PCRE_DOLLAR_ENDONLY) != 0; -md->use_ucp = (re->options & PCRE_UCP) != 0; -md->jscript_compat = (re->options & PCRE_JAVASCRIPT_COMPAT) != 0; -md->ignore_skip_arg = 0; - -/* Some options are unpacked into BOOL variables in the hope that testing -them will be faster than individual option bits. */ - -md->notbol = (options & PCRE_NOTBOL) != 0; -md->noteol = (options & PCRE_NOTEOL) != 0; -md->notempty = (options & PCRE_NOTEMPTY) != 0; -md->notempty_atstart = (options & PCRE_NOTEMPTY_ATSTART) != 0; - -md->hitend = FALSE; -md->mark = md->nomatch_mark = NULL; /* In case never set */ - -md->recursive = NULL; /* No recursion at top level */ -md->hasthen = (re->flags & PCRE_HASTHEN) != 0; - -md->lcc = tables + lcc_offset; -md->fcc = tables + fcc_offset; -md->ctypes = tables + ctypes_offset; - -/* Handle different \R options. */ - -switch (options & (PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE)) - { - case 0: - if ((re->options & (PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE)) != 0) - md->bsr_anycrlf = (re->options & PCRE_BSR_ANYCRLF) != 0; - else -#ifdef BSR_ANYCRLF - md->bsr_anycrlf = TRUE; -#else - md->bsr_anycrlf = FALSE; -#endif - break; - - case PCRE_BSR_ANYCRLF: - md->bsr_anycrlf = TRUE; - break; - - case PCRE_BSR_UNICODE: - md->bsr_anycrlf = FALSE; - break; - - default: return PCRE_ERROR_BADNEWLINE; - } - -/* Handle different types of newline. The three bits give eight cases. If -nothing is set at run time, whatever was used at compile time applies. */ - -switch ((((options & PCRE_NEWLINE_BITS) == 0)? re->options : - (pcre_uint32)options) & PCRE_NEWLINE_BITS) - { - case 0: newline = NEWLINE; break; /* Compile-time default */ - case PCRE_NEWLINE_CR: newline = CHAR_CR; break; - case PCRE_NEWLINE_LF: newline = CHAR_NL; break; - case PCRE_NEWLINE_CR+ - PCRE_NEWLINE_LF: newline = (CHAR_CR << 8) | CHAR_NL; break; - case PCRE_NEWLINE_ANY: newline = -1; break; - case PCRE_NEWLINE_ANYCRLF: newline = -2; break; - default: return PCRE_ERROR_BADNEWLINE; - } - -if (newline == -2) - { - md->nltype = NLTYPE_ANYCRLF; - } -else if (newline < 0) - { - md->nltype = NLTYPE_ANY; - } -else - { - md->nltype = NLTYPE_FIXED; - if (newline > 255) - { - md->nllen = 2; - md->nl[0] = (newline >> 8) & 255; - md->nl[1] = newline & 255; - } - else - { - md->nllen = 1; - md->nl[0] = newline; - } - } - -/* Partial matching was originally supported only for a restricted set of -regexes; from release 8.00 there are no restrictions, but the bits are still -defined (though never set). So there's no harm in leaving this code. */ - -if (md->partial && (re->flags & PCRE_NOPARTIAL) != 0) - return PCRE_ERROR_BADPARTIAL; - -/* If the expression has got more back references than the offsets supplied can -hold, we get a temporary chunk of working store to use during the matching. -Otherwise, we can use the vector supplied, rounding down its size to a multiple -of 3. */ - -ocount = offsetcount - (offsetcount % 3); -arg_offset_max = (2*ocount)/3; - -if (re->top_backref > 0 && re->top_backref >= ocount/3) - { - ocount = re->top_backref * 3 + 3; - md->offset_vector = (int *)(PUBL(malloc))(ocount * sizeof(int)); - if (md->offset_vector == NULL) return PCRE_ERROR_NOMEMORY; - using_temporary_offsets = TRUE; - DPRINTF(("Got memory to hold back references\n")); - } -else md->offset_vector = offsets; -md->offset_end = ocount; -md->offset_max = (2*ocount)/3; -md->capture_last = 0; - -/* Reset the working variable associated with each extraction. These should -never be used unless previously set, but they get saved and restored, and so we -initialize them to avoid reading uninitialized locations. Also, unset the -offsets for the matched string. This is really just for tidiness with callouts, -in case they inspect these fields. */ - -if (md->offset_vector != NULL) - { - register int *iptr = md->offset_vector + ocount; - register int *iend = iptr - re->top_bracket; - if (iend < md->offset_vector + 2) iend = md->offset_vector + 2; - while (--iptr >= iend) *iptr = -1; - if (offsetcount > 0) md->offset_vector[0] = -1; - if (offsetcount > 1) md->offset_vector[1] = -1; - } - -/* Set up the first character to match, if available. The first_char value is -never set for an anchored regular expression, but the anchoring may be forced -at run time, so we have to test for anchoring. The first char may be unset for -an unanchored pattern, of course. If there's no first char and the pattern was -studied, there may be a bitmap of possible first characters. */ - -if (!anchored) - { - if ((re->flags & PCRE_FIRSTSET) != 0) - { - has_first_char = TRUE; - first_char = first_char2 = (pcre_uchar)(re->first_char); - if ((re->flags & PCRE_FCH_CASELESS) != 0) - { - first_char2 = TABLE_GET(first_char, md->fcc, first_char); -#if defined SUPPORT_UCP && !(defined COMPILE_PCRE8) - if (utf && first_char > 127) - first_char2 = UCD_OTHERCASE(first_char); -#endif - } - } - else - if (!startline && study != NULL && - (study->flags & PCRE_STUDY_MAPPED) != 0) - start_bits = study->start_bits; - } - -/* For anchored or unanchored matches, there may be a "last known required -character" set. */ - -if ((re->flags & PCRE_REQCHSET) != 0) - { - has_req_char = TRUE; - req_char = req_char2 = (pcre_uchar)(re->req_char); - if ((re->flags & PCRE_RCH_CASELESS) != 0) - { - req_char2 = TABLE_GET(req_char, md->fcc, req_char); -#if defined SUPPORT_UCP && !(defined COMPILE_PCRE8) - if (utf && req_char > 127) - req_char2 = UCD_OTHERCASE(req_char); -#endif - } - } - - -/* ==========================================================================*/ - -/* Loop for handling unanchored repeated matching attempts; for anchored regexs -the loop runs just once. */ - -for(;;) - { - PCRE_PUCHAR save_end_subject = end_subject; - PCRE_PUCHAR new_start_match; - - /* If firstline is TRUE, the start of the match is constrained to the first - line of a multiline string. That is, the match must be before or at the first - newline. Implement this by temporarily adjusting end_subject so that we stop - scanning at a newline. If the match fails at the newline, later code breaks - this loop. */ - - if (firstline) - { - PCRE_PUCHAR t = start_match; -#ifdef SUPPORT_UTF - if (utf) - { - while (t < md->end_subject && !IS_NEWLINE(t)) - { - t++; - ACROSSCHAR(t < end_subject, *t, t++); - } - } - else -#endif - while (t < md->end_subject && !IS_NEWLINE(t)) t++; - end_subject = t; - } - - /* There are some optimizations that avoid running the match if a known - starting point is not found, or if a known later character is not present. - However, there is an option that disables these, for testing and for ensuring - that all callouts do actually occur. The option can be set in the regex by - (*NO_START_OPT) or passed in match-time options. */ - - if (((options | re->options) & PCRE_NO_START_OPTIMIZE) == 0) - { - /* Advance to a unique first char if there is one. */ - - if (has_first_char) - { - pcre_uchar smc; - - if (first_char != first_char2) - while (start_match < end_subject && - (smc = UCHAR21TEST(start_match)) != first_char && smc != first_char2) - start_match++; - else - while (start_match < end_subject && UCHAR21TEST(start_match) != first_char) - start_match++; - } - - /* Or to just after a linebreak for a multiline match */ - - else if (startline) - { - if (start_match > md->start_subject + start_offset) - { -#ifdef SUPPORT_UTF - if (utf) - { - while (start_match < end_subject && !WAS_NEWLINE(start_match)) - { - start_match++; - ACROSSCHAR(start_match < end_subject, *start_match, - start_match++); - } - } - else -#endif - while (start_match < end_subject && !WAS_NEWLINE(start_match)) - start_match++; - - /* If we have just passed a CR and the newline option is ANY or ANYCRLF, - and we are now at a LF, advance the match position by one more character. - */ - - if (start_match[-1] == CHAR_CR && - (md->nltype == NLTYPE_ANY || md->nltype == NLTYPE_ANYCRLF) && - start_match < end_subject && - UCHAR21TEST(start_match) == CHAR_NL) - start_match++; - } - } - - /* Or to a non-unique first byte after study */ - - else if (start_bits != NULL) - { - while (start_match < end_subject) - { - register pcre_uint32 c = UCHAR21TEST(start_match); -#ifndef COMPILE_PCRE8 - if (c > 255) c = 255; -#endif - if ((start_bits[c/8] & (1 << (c&7))) != 0) break; - start_match++; - } - } - } /* Starting optimizations */ - - /* Restore fudged end_subject */ - - end_subject = save_end_subject; - - /* The following two optimizations are disabled for partial matching or if - disabling is explicitly requested. */ - - if (((options | re->options) & PCRE_NO_START_OPTIMIZE) == 0 && !md->partial) - { - /* If the pattern was studied, a minimum subject length may be set. This is - a lower bound; no actual string of that length may actually match the - pattern. Although the value is, strictly, in characters, we treat it as - bytes to avoid spending too much time in this optimization. */ - - if (study != NULL && (study->flags & PCRE_STUDY_MINLEN) != 0 && - (pcre_uint32)(end_subject - start_match) < study->minlength) - { - rc = MATCH_NOMATCH; - break; - } - - /* If req_char is set, we know that that character must appear in the - subject for the match to succeed. If the first character is set, req_char - must be later in the subject; otherwise the test starts at the match point. - This optimization can save a huge amount of backtracking in patterns with - nested unlimited repeats that aren't going to match. Writing separate code - for cased/caseless versions makes it go faster, as does using an - autoincrement and backing off on a match. - - HOWEVER: when the subject string is very, very long, searching to its end - can take a long time, and give bad performance on quite ordinary patterns. - This showed up when somebody was matching something like /^\d+C/ on a - 32-megabyte string... so we don't do this when the string is sufficiently - long. */ - - if (has_req_char && end_subject - start_match < REQ_BYTE_MAX) - { - register PCRE_PUCHAR p = start_match + (has_first_char? 1:0); - - /* We don't need to repeat the search if we haven't yet reached the - place we found it at last time. */ - - if (p > req_char_ptr) - { - if (req_char != req_char2) - { - while (p < end_subject) - { - register pcre_uint32 pp = UCHAR21INCTEST(p); - if (pp == req_char || pp == req_char2) { p--; break; } - } - } - else - { - while (p < end_subject) - { - if (UCHAR21INCTEST(p) == req_char) { p--; break; } - } - } - - /* If we can't find the required character, break the matching loop, - forcing a match failure. */ - - if (p >= end_subject) - { - rc = MATCH_NOMATCH; - break; - } - - /* If we have found the required character, save the point where we - found it, so that we don't search again next time round the loop if - the start hasn't passed this character yet. */ - - req_char_ptr = p; - } - } - } - -#ifdef PCRE_DEBUG /* Sigh. Some compilers never learn. */ - printf(">>>> Match against: "); - pchars(start_match, end_subject - start_match, TRUE, md); - printf("\n"); -#endif - - /* OK, we can now run the match. If "hitend" is set afterwards, remember the - first starting point for which a partial match was found. */ - - md->start_match_ptr = start_match; - md->start_used_ptr = start_match; - md->match_call_count = 0; - md->match_function_type = 0; - md->end_offset_top = 0; - md->skip_arg_count = 0; - rc = match(start_match, md->start_code, start_match, 2, md, NULL, 0); - if (md->hitend && start_partial == NULL) - { - start_partial = md->start_used_ptr; - match_partial = start_match; - } - - switch(rc) - { - /* If MATCH_SKIP_ARG reaches this level it means that a MARK that matched - the SKIP's arg was not found. In this circumstance, Perl ignores the SKIP - entirely. The only way we can do that is to re-do the match at the same - point, with a flag to force SKIP with an argument to be ignored. Just - treating this case as NOMATCH does not work because it does not check other - alternatives in patterns such as A(*SKIP:A)B|AC when the subject is AC. */ - - case MATCH_SKIP_ARG: - new_start_match = start_match; - md->ignore_skip_arg = md->skip_arg_count; - break; - - /* SKIP passes back the next starting point explicitly, but if it is no - greater than the match we have just done, treat it as NOMATCH. */ - - case MATCH_SKIP: - if (md->start_match_ptr > start_match) - { - new_start_match = md->start_match_ptr; - break; - } - /* Fall through */ - - /* NOMATCH and PRUNE advance by one character. THEN at this level acts - exactly like PRUNE. Unset ignore SKIP-with-argument. */ - - case MATCH_NOMATCH: - case MATCH_PRUNE: - case MATCH_THEN: - md->ignore_skip_arg = 0; - new_start_match = start_match + 1; -#ifdef SUPPORT_UTF - if (utf) - ACROSSCHAR(new_start_match < end_subject, *new_start_match, - new_start_match++); -#endif - break; - - /* COMMIT disables the bumpalong, but otherwise behaves as NOMATCH. */ - - case MATCH_COMMIT: - rc = MATCH_NOMATCH; - goto ENDLOOP; - - /* Any other return is either a match, or some kind of error. */ - - default: - goto ENDLOOP; - } - - /* Control reaches here for the various types of "no match at this point" - result. Reset the code to MATCH_NOMATCH for subsequent checking. */ - - rc = MATCH_NOMATCH; - - /* If PCRE_FIRSTLINE is set, the match must happen before or at the first - newline in the subject (though it may continue over the newline). Therefore, - if we have just failed to match, starting at a newline, do not continue. */ - - if (firstline && IS_NEWLINE(start_match)) break; - - /* Advance to new matching position */ - - start_match = new_start_match; - - /* Break the loop if the pattern is anchored or if we have passed the end of - the subject. */ - - if (anchored || start_match > end_subject) break; - - /* If we have just passed a CR and we are now at a LF, and the pattern does - not contain any explicit matches for \r or \n, and the newline option is CRLF - or ANY or ANYCRLF, advance the match position by one more character. In - normal matching start_match will aways be greater than the first position at - this stage, but a failed *SKIP can cause a return at the same point, which is - why the first test exists. */ - - if (start_match > (PCRE_PUCHAR)subject + start_offset && - start_match[-1] == CHAR_CR && - start_match < end_subject && - *start_match == CHAR_NL && - (re->flags & PCRE_HASCRORLF) == 0 && - (md->nltype == NLTYPE_ANY || - md->nltype == NLTYPE_ANYCRLF || - md->nllen == 2)) - start_match++; - - md->mark = NULL; /* Reset for start of next match attempt */ - } /* End of for(;;) "bumpalong" loop */ - -/* ==========================================================================*/ - -/* We reach here when rc is not MATCH_NOMATCH, or if one of the stopping -conditions is true: - -(1) The pattern is anchored or the match was failed by (*COMMIT); - -(2) We are past the end of the subject; - -(3) PCRE_FIRSTLINE is set and we have failed to match at a newline, because - this option requests that a match occur at or before the first newline in - the subject. - -When we have a match and the offset vector is big enough to deal with any -backreferences, captured substring offsets will already be set up. In the case -where we had to get some local store to hold offsets for backreference -processing, copy those that we can. In this case there need not be overflow if -certain parts of the pattern were not used, even though there are more -capturing parentheses than vector slots. */ - -ENDLOOP: - -if (rc == MATCH_MATCH || rc == MATCH_ACCEPT) - { - if (using_temporary_offsets) - { - if (arg_offset_max >= 4) - { - memcpy(offsets + 2, md->offset_vector + 2, - (arg_offset_max - 2) * sizeof(int)); - DPRINTF(("Copied offsets from temporary memory\n")); - } - if (md->end_offset_top > arg_offset_max) md->capture_last |= OVFLBIT; - DPRINTF(("Freeing temporary memory\n")); - (PUBL(free))(md->offset_vector); - } - - /* Set the return code to the number of captured strings, or 0 if there were - too many to fit into the vector. */ - - rc = ((md->capture_last & OVFLBIT) != 0 && - md->end_offset_top >= arg_offset_max)? - 0 : md->end_offset_top/2; - - /* If there is space in the offset vector, set any unused pairs at the end of - the pattern to -1 for backwards compatibility. It is documented that this - happens. In earlier versions, the whole set of potential capturing offsets - was set to -1 each time round the loop, but this is handled differently now. - "Gaps" are set to -1 dynamically instead (this fixes a bug). Thus, it is only - those at the end that need unsetting here. We can't just unset them all at - the start of the whole thing because they may get set in one branch that is - not the final matching branch. */ - - if (md->end_offset_top/2 <= re->top_bracket && offsets != NULL) - { - register int *iptr, *iend; - int resetcount = 2 + re->top_bracket * 2; - if (resetcount > offsetcount) resetcount = offsetcount; - iptr = offsets + md->end_offset_top; - iend = offsets + resetcount; - while (iptr < iend) *iptr++ = -1; - } - - /* If there is space, set up the whole thing as substring 0. The value of - md->start_match_ptr might be modified if \K was encountered on the success - matching path. */ - - if (offsetcount < 2) rc = 0; else - { - offsets[0] = (int)(md->start_match_ptr - md->start_subject); - offsets[1] = (int)(md->end_match_ptr - md->start_subject); - } - - /* Return MARK data if requested */ - - if (extra_data != NULL && (extra_data->flags & PCRE_EXTRA_MARK) != 0) - *(extra_data->mark) = (pcre_uchar *)md->mark; - DPRINTF((">>>> returning %d\n", rc)); -#ifdef NO_RECURSE - release_match_heapframes(&frame_zero); -#endif - return rc; - } - -/* Control gets here if there has been an error, or if the overall match -attempt has failed at all permitted starting positions. */ - -if (using_temporary_offsets) - { - DPRINTF(("Freeing temporary memory\n")); - (PUBL(free))(md->offset_vector); - } - -/* For anything other than nomatch or partial match, just return the code. */ - -if (rc != MATCH_NOMATCH && rc != PCRE_ERROR_PARTIAL) - { - DPRINTF((">>>> error: returning %d\n", rc)); -#ifdef NO_RECURSE - release_match_heapframes(&frame_zero); -#endif - return rc; - } - -/* Handle partial matches - disable any mark data */ - -if (match_partial != NULL) - { - DPRINTF((">>>> returning PCRE_ERROR_PARTIAL\n")); - md->mark = NULL; - if (offsetcount > 1) - { - offsets[0] = (int)(start_partial - (PCRE_PUCHAR)subject); - offsets[1] = (int)(end_subject - (PCRE_PUCHAR)subject); - if (offsetcount > 2) - offsets[2] = (int)(match_partial - (PCRE_PUCHAR)subject); - } - rc = PCRE_ERROR_PARTIAL; - } - -/* This is the classic nomatch case */ - -else - { - DPRINTF((">>>> returning PCRE_ERROR_NOMATCH\n")); - rc = PCRE_ERROR_NOMATCH; - } - -/* Return the MARK data if it has been requested. */ - -if (extra_data != NULL && (extra_data->flags & PCRE_EXTRA_MARK) != 0) - *(extra_data->mark) = (pcre_uchar *)md->nomatch_mark; -#ifdef NO_RECURSE - release_match_heapframes(&frame_zero); -#endif -return rc; -} - -/* End of pcre_exec.c */ diff --git a/src/third_party/pcre-8.39/pcre_fullinfo.c b/src/third_party/pcre-8.39/pcre_fullinfo.c deleted file mode 100644 index a6c2ece6ca5..00000000000 --- a/src/third_party/pcre-8.39/pcre_fullinfo.c +++ /dev/null @@ -1,245 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2013 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This module contains the external function pcre_fullinfo(), which returns -information about a compiled pattern. */ - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "pcre_internal.h" - - -/************************************************* -* Return info about compiled pattern * -*************************************************/ - -/* This is a newer "info" function which has an extensible interface so -that additional items can be added compatibly. - -Arguments: - argument_re points to compiled code - extra_data points extra data, or NULL - what what information is required - where where to put the information - -Returns: 0 if data returned, negative on error -*/ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre_fullinfo(const pcre *argument_re, const pcre_extra *extra_data, - int what, void *where) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre16_fullinfo(const pcre16 *argument_re, const pcre16_extra *extra_data, - int what, void *where) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre32_fullinfo(const pcre32 *argument_re, const pcre32_extra *extra_data, - int what, void *where) -#endif -{ -const REAL_PCRE *re = (const REAL_PCRE *)argument_re; -const pcre_study_data *study = NULL; - -if (re == NULL || where == NULL) return PCRE_ERROR_NULL; - -if (extra_data != NULL && (extra_data->flags & PCRE_EXTRA_STUDY_DATA) != 0) - study = (const pcre_study_data *)extra_data->study_data; - -/* Check that the first field in the block is the magic number. If it is not, -return with PCRE_ERROR_BADMAGIC. However, if the magic number is equal to -REVERSED_MAGIC_NUMBER we return with PCRE_ERROR_BADENDIANNESS, which -means that the pattern is likely compiled with different endianness. */ - -if (re->magic_number != MAGIC_NUMBER) - return re->magic_number == REVERSED_MAGIC_NUMBER? - PCRE_ERROR_BADENDIANNESS:PCRE_ERROR_BADMAGIC; - -/* Check that this pattern was compiled in the correct bit mode */ - -if ((re->flags & PCRE_MODE) == 0) return PCRE_ERROR_BADMODE; - -switch (what) - { - case PCRE_INFO_OPTIONS: - *((unsigned long int *)where) = re->options & PUBLIC_COMPILE_OPTIONS; - break; - - case PCRE_INFO_SIZE: - *((size_t *)where) = re->size; - break; - - case PCRE_INFO_STUDYSIZE: - *((size_t *)where) = (study == NULL)? 0 : study->size; - break; - - case PCRE_INFO_JITSIZE: -#ifdef SUPPORT_JIT - *((size_t *)where) = - (extra_data != NULL && - (extra_data->flags & PCRE_EXTRA_EXECUTABLE_JIT) != 0 && - extra_data->executable_jit != NULL)? - PRIV(jit_get_size)(extra_data->executable_jit) : 0; -#else - *((size_t *)where) = 0; -#endif - break; - - case PCRE_INFO_CAPTURECOUNT: - *((int *)where) = re->top_bracket; - break; - - case PCRE_INFO_BACKREFMAX: - *((int *)where) = re->top_backref; - break; - - case PCRE_INFO_FIRSTBYTE: - *((int *)where) = - ((re->flags & PCRE_FIRSTSET) != 0)? (int)re->first_char : - ((re->flags & PCRE_STARTLINE) != 0)? -1 : -2; - break; - - case PCRE_INFO_FIRSTCHARACTER: - *((pcre_uint32 *)where) = - (re->flags & PCRE_FIRSTSET) != 0 ? re->first_char : 0; - break; - - case PCRE_INFO_FIRSTCHARACTERFLAGS: - *((int *)where) = - ((re->flags & PCRE_FIRSTSET) != 0) ? 1 : - ((re->flags & PCRE_STARTLINE) != 0) ? 2 : 0; - break; - - /* Make sure we pass back the pointer to the bit vector in the external - block, not the internal copy (with flipped integer fields). */ - - case PCRE_INFO_FIRSTTABLE: - *((const pcre_uint8 **)where) = - (study != NULL && (study->flags & PCRE_STUDY_MAPPED) != 0)? - ((const pcre_study_data *)extra_data->study_data)->start_bits : NULL; - break; - - case PCRE_INFO_MINLENGTH: - *((int *)where) = - (study != NULL && (study->flags & PCRE_STUDY_MINLEN) != 0)? - (int)(study->minlength) : -1; - break; - - case PCRE_INFO_JIT: - *((int *)where) = extra_data != NULL && - (extra_data->flags & PCRE_EXTRA_EXECUTABLE_JIT) != 0 && - extra_data->executable_jit != NULL; - break; - - case PCRE_INFO_LASTLITERAL: - *((int *)where) = - ((re->flags & PCRE_REQCHSET) != 0)? (int)re->req_char : -1; - break; - - case PCRE_INFO_REQUIREDCHAR: - *((pcre_uint32 *)where) = - ((re->flags & PCRE_REQCHSET) != 0) ? re->req_char : 0; - break; - - case PCRE_INFO_REQUIREDCHARFLAGS: - *((int *)where) = - ((re->flags & PCRE_REQCHSET) != 0); - break; - - case PCRE_INFO_NAMEENTRYSIZE: - *((int *)where) = re->name_entry_size; - break; - - case PCRE_INFO_NAMECOUNT: - *((int *)where) = re->name_count; - break; - - case PCRE_INFO_NAMETABLE: - *((const pcre_uchar **)where) = (const pcre_uchar *)re + re->name_table_offset; - break; - - case PCRE_INFO_DEFAULT_TABLES: - *((const pcre_uint8 **)where) = (const pcre_uint8 *)(PRIV(default_tables)); - break; - - /* From release 8.00 this will always return TRUE because NOPARTIAL is - no longer ever set (the restrictions have been removed). */ - - case PCRE_INFO_OKPARTIAL: - *((int *)where) = (re->flags & PCRE_NOPARTIAL) == 0; - break; - - case PCRE_INFO_JCHANGED: - *((int *)where) = (re->flags & PCRE_JCHANGED) != 0; - break; - - case PCRE_INFO_HASCRORLF: - *((int *)where) = (re->flags & PCRE_HASCRORLF) != 0; - break; - - case PCRE_INFO_MAXLOOKBEHIND: - *((int *)where) = re->max_lookbehind; - break; - - case PCRE_INFO_MATCHLIMIT: - if ((re->flags & PCRE_MLSET) == 0) return PCRE_ERROR_UNSET; - *((pcre_uint32 *)where) = re->limit_match; - break; - - case PCRE_INFO_RECURSIONLIMIT: - if ((re->flags & PCRE_RLSET) == 0) return PCRE_ERROR_UNSET; - *((pcre_uint32 *)where) = re->limit_recursion; - break; - - case PCRE_INFO_MATCH_EMPTY: - *((int *)where) = (re->flags & PCRE_MATCH_EMPTY) != 0; - break; - - default: return PCRE_ERROR_BADOPTION; - } - -return 0; -} - -/* End of pcre_fullinfo.c */ diff --git a/src/third_party/pcre-8.39/pcre_get.c b/src/third_party/pcre-8.39/pcre_get.c deleted file mode 100644 index 9475d5e88cd..00000000000 --- a/src/third_party/pcre-8.39/pcre_get.c +++ /dev/null @@ -1,669 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This module contains some convenience functions for extracting substrings -from the subject string after a regex match has succeeded. The original idea -for these functions came from Scott Wimer. */ - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "pcre_internal.h" - - -/************************************************* -* Find number for named string * -*************************************************/ - -/* This function is used by the get_first_set() function below, as well -as being generally available. It assumes that names are unique. - -Arguments: - code the compiled regex - stringname the name whose number is required - -Returns: the number of the named parentheses, or a negative number - (PCRE_ERROR_NOSUBSTRING) if not found -*/ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre_get_stringnumber(const pcre *code, const char *stringname) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre16_get_stringnumber(const pcre16 *code, PCRE_SPTR16 stringname) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre32_get_stringnumber(const pcre32 *code, PCRE_SPTR32 stringname) -#endif -{ -int rc; -int entrysize; -int top, bot; -pcre_uchar *nametable; - -#ifdef COMPILE_PCRE8 -if ((rc = pcre_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top)) != 0) - return rc; -if (top <= 0) return PCRE_ERROR_NOSUBSTRING; - -if ((rc = pcre_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize)) != 0) - return rc; -if ((rc = pcre_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, &nametable)) != 0) - return rc; -#endif -#ifdef COMPILE_PCRE16 -if ((rc = pcre16_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top)) != 0) - return rc; -if (top <= 0) return PCRE_ERROR_NOSUBSTRING; - -if ((rc = pcre16_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize)) != 0) - return rc; -if ((rc = pcre16_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, &nametable)) != 0) - return rc; -#endif -#ifdef COMPILE_PCRE32 -if ((rc = pcre32_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top)) != 0) - return rc; -if (top <= 0) return PCRE_ERROR_NOSUBSTRING; - -if ((rc = pcre32_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize)) != 0) - return rc; -if ((rc = pcre32_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, &nametable)) != 0) - return rc; -#endif - -bot = 0; -while (top > bot) - { - int mid = (top + bot) / 2; - pcre_uchar *entry = nametable + entrysize*mid; - int c = STRCMP_UC_UC((pcre_uchar *)stringname, - (pcre_uchar *)(entry + IMM2_SIZE)); - if (c == 0) return GET2(entry, 0); - if (c > 0) bot = mid + 1; else top = mid; - } - -return PCRE_ERROR_NOSUBSTRING; -} - - - -/************************************************* -* Find (multiple) entries for named string * -*************************************************/ - -/* This is used by the get_first_set() function below, as well as being -generally available. It is used when duplicated names are permitted. - -Arguments: - code the compiled regex - stringname the name whose entries required - firstptr where to put the pointer to the first entry - lastptr where to put the pointer to the last entry - -Returns: the length of each entry, or a negative number - (PCRE_ERROR_NOSUBSTRING) if not found -*/ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre_get_stringtable_entries(const pcre *code, const char *stringname, - char **firstptr, char **lastptr) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre16_get_stringtable_entries(const pcre16 *code, PCRE_SPTR16 stringname, - PCRE_UCHAR16 **firstptr, PCRE_UCHAR16 **lastptr) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre32_get_stringtable_entries(const pcre32 *code, PCRE_SPTR32 stringname, - PCRE_UCHAR32 **firstptr, PCRE_UCHAR32 **lastptr) -#endif -{ -int rc; -int entrysize; -int top, bot; -pcre_uchar *nametable, *lastentry; - -#ifdef COMPILE_PCRE8 -if ((rc = pcre_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top)) != 0) - return rc; -if (top <= 0) return PCRE_ERROR_NOSUBSTRING; - -if ((rc = pcre_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize)) != 0) - return rc; -if ((rc = pcre_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, &nametable)) != 0) - return rc; -#endif -#ifdef COMPILE_PCRE16 -if ((rc = pcre16_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top)) != 0) - return rc; -if (top <= 0) return PCRE_ERROR_NOSUBSTRING; - -if ((rc = pcre16_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize)) != 0) - return rc; -if ((rc = pcre16_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, &nametable)) != 0) - return rc; -#endif -#ifdef COMPILE_PCRE32 -if ((rc = pcre32_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top)) != 0) - return rc; -if (top <= 0) return PCRE_ERROR_NOSUBSTRING; - -if ((rc = pcre32_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize)) != 0) - return rc; -if ((rc = pcre32_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, &nametable)) != 0) - return rc; -#endif - -lastentry = nametable + entrysize * (top - 1); -bot = 0; -while (top > bot) - { - int mid = (top + bot) / 2; - pcre_uchar *entry = nametable + entrysize*mid; - int c = STRCMP_UC_UC((pcre_uchar *)stringname, - (pcre_uchar *)(entry + IMM2_SIZE)); - if (c == 0) - { - pcre_uchar *first = entry; - pcre_uchar *last = entry; - while (first > nametable) - { - if (STRCMP_UC_UC((pcre_uchar *)stringname, - (pcre_uchar *)(first - entrysize + IMM2_SIZE)) != 0) break; - first -= entrysize; - } - while (last < lastentry) - { - if (STRCMP_UC_UC((pcre_uchar *)stringname, - (pcre_uchar *)(last + entrysize + IMM2_SIZE)) != 0) break; - last += entrysize; - } -#if defined COMPILE_PCRE8 - *firstptr = (char *)first; - *lastptr = (char *)last; -#elif defined COMPILE_PCRE16 - *firstptr = (PCRE_UCHAR16 *)first; - *lastptr = (PCRE_UCHAR16 *)last; -#elif defined COMPILE_PCRE32 - *firstptr = (PCRE_UCHAR32 *)first; - *lastptr = (PCRE_UCHAR32 *)last; -#endif - return entrysize; - } - if (c > 0) bot = mid + 1; else top = mid; - } - -return PCRE_ERROR_NOSUBSTRING; -} - - - -/************************************************* -* Find first set of multiple named strings * -*************************************************/ - -/* This function allows for duplicate names in the table of named substrings. -It returns the number of the first one that was set in a pattern match. - -Arguments: - code the compiled regex - stringname the name of the capturing substring - ovector the vector of matched substrings - stringcount number of captured substrings - -Returns: the number of the first that is set, - or the number of the last one if none are set, - or a negative number on error -*/ - -#if defined COMPILE_PCRE8 -static int -get_first_set(const pcre *code, const char *stringname, int *ovector, - int stringcount) -#elif defined COMPILE_PCRE16 -static int -get_first_set(const pcre16 *code, PCRE_SPTR16 stringname, int *ovector, - int stringcount) -#elif defined COMPILE_PCRE32 -static int -get_first_set(const pcre32 *code, PCRE_SPTR32 stringname, int *ovector, - int stringcount) -#endif -{ -const REAL_PCRE *re = (const REAL_PCRE *)code; -int entrysize; -pcre_uchar *entry; -#if defined COMPILE_PCRE8 -char *first, *last; -#elif defined COMPILE_PCRE16 -PCRE_UCHAR16 *first, *last; -#elif defined COMPILE_PCRE32 -PCRE_UCHAR32 *first, *last; -#endif - -#if defined COMPILE_PCRE8 -if ((re->options & PCRE_DUPNAMES) == 0 && (re->flags & PCRE_JCHANGED) == 0) - return pcre_get_stringnumber(code, stringname); -entrysize = pcre_get_stringtable_entries(code, stringname, &first, &last); -#elif defined COMPILE_PCRE16 -if ((re->options & PCRE_DUPNAMES) == 0 && (re->flags & PCRE_JCHANGED) == 0) - return pcre16_get_stringnumber(code, stringname); -entrysize = pcre16_get_stringtable_entries(code, stringname, &first, &last); -#elif defined COMPILE_PCRE32 -if ((re->options & PCRE_DUPNAMES) == 0 && (re->flags & PCRE_JCHANGED) == 0) - return pcre32_get_stringnumber(code, stringname); -entrysize = pcre32_get_stringtable_entries(code, stringname, &first, &last); -#endif -if (entrysize <= 0) return entrysize; -for (entry = (pcre_uchar *)first; entry <= (pcre_uchar *)last; entry += entrysize) - { - int n = GET2(entry, 0); - if (n < stringcount && ovector[n*2] >= 0) return n; - } -return GET2(entry, 0); -} - - - - -/************************************************* -* Copy captured string to given buffer * -*************************************************/ - -/* This function copies a single captured substring into a given buffer. -Note that we use memcpy() rather than strncpy() in case there are binary zeros -in the string. - -Arguments: - subject the subject string that was matched - ovector pointer to the offsets table - stringcount the number of substrings that were captured - (i.e. the yield of the pcre_exec call, unless - that was zero, in which case it should be 1/3 - of the offset table size) - stringnumber the number of the required substring - buffer where to put the substring - size the size of the buffer - -Returns: if successful: - the length of the copied string, not including the zero - that is put on the end; can be zero - if not successful: - PCRE_ERROR_NOMEMORY (-6) buffer too small - PCRE_ERROR_NOSUBSTRING (-7) no such captured substring -*/ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre_copy_substring(const char *subject, int *ovector, int stringcount, - int stringnumber, char *buffer, int size) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre16_copy_substring(PCRE_SPTR16 subject, int *ovector, int stringcount, - int stringnumber, PCRE_UCHAR16 *buffer, int size) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre32_copy_substring(PCRE_SPTR32 subject, int *ovector, int stringcount, - int stringnumber, PCRE_UCHAR32 *buffer, int size) -#endif -{ -int yield; -if (stringnumber < 0 || stringnumber >= stringcount) - return PCRE_ERROR_NOSUBSTRING; -stringnumber *= 2; -yield = ovector[stringnumber+1] - ovector[stringnumber]; -if (size < yield + 1) return PCRE_ERROR_NOMEMORY; -memcpy(buffer, subject + ovector[stringnumber], IN_UCHARS(yield)); -buffer[yield] = 0; -return yield; -} - - - -/************************************************* -* Copy named captured string to given buffer * -*************************************************/ - -/* This function copies a single captured substring into a given buffer, -identifying it by name. If the regex permits duplicate names, the first -substring that is set is chosen. - -Arguments: - code the compiled regex - subject the subject string that was matched - ovector pointer to the offsets table - stringcount the number of substrings that were captured - (i.e. the yield of the pcre_exec call, unless - that was zero, in which case it should be 1/3 - of the offset table size) - stringname the name of the required substring - buffer where to put the substring - size the size of the buffer - -Returns: if successful: - the length of the copied string, not including the zero - that is put on the end; can be zero - if not successful: - PCRE_ERROR_NOMEMORY (-6) buffer too small - PCRE_ERROR_NOSUBSTRING (-7) no such captured substring -*/ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre_copy_named_substring(const pcre *code, const char *subject, - int *ovector, int stringcount, const char *stringname, - char *buffer, int size) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre16_copy_named_substring(const pcre16 *code, PCRE_SPTR16 subject, - int *ovector, int stringcount, PCRE_SPTR16 stringname, - PCRE_UCHAR16 *buffer, int size) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre32_copy_named_substring(const pcre32 *code, PCRE_SPTR32 subject, - int *ovector, int stringcount, PCRE_SPTR32 stringname, - PCRE_UCHAR32 *buffer, int size) -#endif -{ -int n = get_first_set(code, stringname, ovector, stringcount); -if (n <= 0) return n; -#if defined COMPILE_PCRE8 -return pcre_copy_substring(subject, ovector, stringcount, n, buffer, size); -#elif defined COMPILE_PCRE16 -return pcre16_copy_substring(subject, ovector, stringcount, n, buffer, size); -#elif defined COMPILE_PCRE32 -return pcre32_copy_substring(subject, ovector, stringcount, n, buffer, size); -#endif -} - - - -/************************************************* -* Copy all captured strings to new store * -*************************************************/ - -/* This function gets one chunk of store and builds a list of pointers and all -of the captured substrings in it. A NULL pointer is put on the end of the list. - -Arguments: - subject the subject string that was matched - ovector pointer to the offsets table - stringcount the number of substrings that were captured - (i.e. the yield of the pcre_exec call, unless - that was zero, in which case it should be 1/3 - of the offset table size) - listptr set to point to the list of pointers - -Returns: if successful: 0 - if not successful: - PCRE_ERROR_NOMEMORY (-6) failed to get store -*/ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre_get_substring_list(const char *subject, int *ovector, int stringcount, - const char ***listptr) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre16_get_substring_list(PCRE_SPTR16 subject, int *ovector, int stringcount, - PCRE_SPTR16 **listptr) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre32_get_substring_list(PCRE_SPTR32 subject, int *ovector, int stringcount, - PCRE_SPTR32 **listptr) -#endif -{ -int i; -int size = sizeof(pcre_uchar *); -int double_count = stringcount * 2; -pcre_uchar **stringlist; -pcre_uchar *p; - -for (i = 0; i < double_count; i += 2) - { - size += sizeof(pcre_uchar *) + IN_UCHARS(1); - if (ovector[i+1] > ovector[i]) size += IN_UCHARS(ovector[i+1] - ovector[i]); - } - -stringlist = (pcre_uchar **)(PUBL(malloc))(size); -if (stringlist == NULL) return PCRE_ERROR_NOMEMORY; - -#if defined COMPILE_PCRE8 -*listptr = (const char **)stringlist; -#elif defined COMPILE_PCRE16 -*listptr = (PCRE_SPTR16 *)stringlist; -#elif defined COMPILE_PCRE32 -*listptr = (PCRE_SPTR32 *)stringlist; -#endif -p = (pcre_uchar *)(stringlist + stringcount + 1); - -for (i = 0; i < double_count; i += 2) - { - int len = (ovector[i+1] > ovector[i])? (ovector[i+1] - ovector[i]) : 0; - memcpy(p, subject + ovector[i], IN_UCHARS(len)); - *stringlist++ = p; - p += len; - *p++ = 0; - } - -*stringlist = NULL; -return 0; -} - - - -/************************************************* -* Free store obtained by get_substring_list * -*************************************************/ - -/* This function exists for the benefit of people calling PCRE from non-C -programs that can call its functions, but not free() or (PUBL(free))() -directly. - -Argument: the result of a previous pcre_get_substring_list() -Returns: nothing -*/ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN void PCRE_CALL_CONVENTION -pcre_free_substring_list(const char **pointer) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN void PCRE_CALL_CONVENTION -pcre16_free_substring_list(PCRE_SPTR16 *pointer) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN void PCRE_CALL_CONVENTION -pcre32_free_substring_list(PCRE_SPTR32 *pointer) -#endif -{ -(PUBL(free))((void *)pointer); -} - - - -/************************************************* -* Copy captured string to new store * -*************************************************/ - -/* This function copies a single captured substring into a piece of new -store - -Arguments: - subject the subject string that was matched - ovector pointer to the offsets table - stringcount the number of substrings that were captured - (i.e. the yield of the pcre_exec call, unless - that was zero, in which case it should be 1/3 - of the offset table size) - stringnumber the number of the required substring - stringptr where to put a pointer to the substring - -Returns: if successful: - the length of the string, not including the zero that - is put on the end; can be zero - if not successful: - PCRE_ERROR_NOMEMORY (-6) failed to get store - PCRE_ERROR_NOSUBSTRING (-7) substring not present -*/ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre_get_substring(const char *subject, int *ovector, int stringcount, - int stringnumber, const char **stringptr) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre16_get_substring(PCRE_SPTR16 subject, int *ovector, int stringcount, - int stringnumber, PCRE_SPTR16 *stringptr) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre32_get_substring(PCRE_SPTR32 subject, int *ovector, int stringcount, - int stringnumber, PCRE_SPTR32 *stringptr) -#endif -{ -int yield; -pcre_uchar *substring; -if (stringnumber < 0 || stringnumber >= stringcount) - return PCRE_ERROR_NOSUBSTRING; -stringnumber *= 2; -yield = ovector[stringnumber+1] - ovector[stringnumber]; -substring = (pcre_uchar *)(PUBL(malloc))(IN_UCHARS(yield + 1)); -if (substring == NULL) return PCRE_ERROR_NOMEMORY; -memcpy(substring, subject + ovector[stringnumber], IN_UCHARS(yield)); -substring[yield] = 0; -#if defined COMPILE_PCRE8 -*stringptr = (const char *)substring; -#elif defined COMPILE_PCRE16 -*stringptr = (PCRE_SPTR16)substring; -#elif defined COMPILE_PCRE32 -*stringptr = (PCRE_SPTR32)substring; -#endif -return yield; -} - - - -/************************************************* -* Copy named captured string to new store * -*************************************************/ - -/* This function copies a single captured substring, identified by name, into -new store. If the regex permits duplicate names, the first substring that is -set is chosen. - -Arguments: - code the compiled regex - subject the subject string that was matched - ovector pointer to the offsets table - stringcount the number of substrings that were captured - (i.e. the yield of the pcre_exec call, unless - that was zero, in which case it should be 1/3 - of the offset table size) - stringname the name of the required substring - stringptr where to put the pointer - -Returns: if successful: - the length of the copied string, not including the zero - that is put on the end; can be zero - if not successful: - PCRE_ERROR_NOMEMORY (-6) couldn't get memory - PCRE_ERROR_NOSUBSTRING (-7) no such captured substring -*/ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre_get_named_substring(const pcre *code, const char *subject, - int *ovector, int stringcount, const char *stringname, - const char **stringptr) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre16_get_named_substring(const pcre16 *code, PCRE_SPTR16 subject, - int *ovector, int stringcount, PCRE_SPTR16 stringname, - PCRE_SPTR16 *stringptr) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre32_get_named_substring(const pcre32 *code, PCRE_SPTR32 subject, - int *ovector, int stringcount, PCRE_SPTR32 stringname, - PCRE_SPTR32 *stringptr) -#endif -{ -int n = get_first_set(code, stringname, ovector, stringcount); -if (n <= 0) return n; -#if defined COMPILE_PCRE8 -return pcre_get_substring(subject, ovector, stringcount, n, stringptr); -#elif defined COMPILE_PCRE16 -return pcre16_get_substring(subject, ovector, stringcount, n, stringptr); -#elif defined COMPILE_PCRE32 -return pcre32_get_substring(subject, ovector, stringcount, n, stringptr); -#endif -} - - - - -/************************************************* -* Free store obtained by get_substring * -*************************************************/ - -/* This function exists for the benefit of people calling PCRE from non-C -programs that can call its functions, but not free() or (PUBL(free))() -directly. - -Argument: the result of a previous pcre_get_substring() -Returns: nothing -*/ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN void PCRE_CALL_CONVENTION -pcre_free_substring(const char *pointer) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN void PCRE_CALL_CONVENTION -pcre16_free_substring(PCRE_SPTR16 pointer) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN void PCRE_CALL_CONVENTION -pcre32_free_substring(PCRE_SPTR32 pointer) -#endif -{ -(PUBL(free))((void *)pointer); -} - -/* End of pcre_get.c */ diff --git a/src/third_party/pcre-8.39/pcre_globals.c b/src/third_party/pcre-8.39/pcre_globals.c deleted file mode 100644 index 0f106aa9013..00000000000 --- a/src/third_party/pcre-8.39/pcre_globals.c +++ /dev/null @@ -1,86 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2014 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This module contains global variables that are exported by the PCRE library. -PCRE is thread-clean and doesn't use any global variables in the normal sense. -However, it calls memory allocation and freeing functions via the four -indirections below, and it can optionally do callouts, using the fifth -indirection. These values can be changed by the caller, but are shared between -all threads. - -For MS Visual Studio and Symbian OS, there are problems in initializing these -variables to non-local functions. In these cases, therefore, an indirection via -a local function is used. - -Also, when compiling for Virtual Pascal, things are done differently, and -global variables are not used. */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "pcre_internal.h" - -#if defined _MSC_VER || defined __SYMBIAN32__ -static void* LocalPcreMalloc(size_t aSize) - { - return malloc(aSize); - } -static void LocalPcreFree(void* aPtr) - { - free(aPtr); - } -PCRE_EXP_DATA_DEFN void *(*PUBL(malloc))(size_t) = LocalPcreMalloc; -PCRE_EXP_DATA_DEFN void (*PUBL(free))(void *) = LocalPcreFree; -PCRE_EXP_DATA_DEFN void *(*PUBL(stack_malloc))(size_t) = LocalPcreMalloc; -PCRE_EXP_DATA_DEFN void (*PUBL(stack_free))(void *) = LocalPcreFree; -PCRE_EXP_DATA_DEFN int (*PUBL(callout))(PUBL(callout_block) *) = NULL; -PCRE_EXP_DATA_DEFN int (*PUBL(stack_guard))(void) = NULL; - -#elif !defined VPCOMPAT -PCRE_EXP_DATA_DEFN void *(*PUBL(malloc))(size_t) = malloc; -PCRE_EXP_DATA_DEFN void (*PUBL(free))(void *) = free; -PCRE_EXP_DATA_DEFN void *(*PUBL(stack_malloc))(size_t) = malloc; -PCRE_EXP_DATA_DEFN void (*PUBL(stack_free))(void *) = free; -PCRE_EXP_DATA_DEFN int (*PUBL(callout))(PUBL(callout_block) *) = NULL; -PCRE_EXP_DATA_DEFN int (*PUBL(stack_guard))(void) = NULL; -#endif - -/* End of pcre_globals.c */ diff --git a/src/third_party/pcre-8.39/pcre_internal.h b/src/third_party/pcre-8.39/pcre_internal.h deleted file mode 100644 index 2923b29f82d..00000000000 --- a/src/third_party/pcre-8.39/pcre_internal.h +++ /dev/null @@ -1,2798 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2016 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* This header contains definitions that are shared between the different -modules, but which are not relevant to the exported API. This includes some -functions whose names all begin with "_pcre_", "_pcre16_" or "_pcre32_" -depending on the PRIV macro. */ - -#ifndef PCRE_INTERNAL_H -#define PCRE_INTERNAL_H - -/* Define PCRE_DEBUG to get debugging output on stdout. */ - -#if 0 -#define PCRE_DEBUG -#endif - -/* PCRE is compiled as an 8 bit library if it is not requested otherwise. */ - -#if !defined COMPILE_PCRE16 && !defined COMPILE_PCRE32 -#define COMPILE_PCRE8 -#endif - -/* If SUPPORT_UCP is defined, SUPPORT_UTF must also be defined. The -"configure" script ensures this, but not everybody uses "configure". */ - -#if defined SUPPORT_UCP && !(defined SUPPORT_UTF) -#define SUPPORT_UTF 1 -#endif - -/* We define SUPPORT_UTF if SUPPORT_UTF8 is enabled for compatibility -reasons with existing code. */ - -#if defined SUPPORT_UTF8 && !(defined SUPPORT_UTF) -#define SUPPORT_UTF 1 -#endif - -/* Fixme: SUPPORT_UTF8 should be eventually disappear from the code. -Until then we define it if SUPPORT_UTF is defined. */ - -#if defined SUPPORT_UTF && !(defined SUPPORT_UTF8) -#define SUPPORT_UTF8 1 -#endif - -/* We do not support both EBCDIC and UTF-8/16/32 at the same time. The "configure" -script prevents both being selected, but not everybody uses "configure". */ - -#if defined EBCDIC && defined SUPPORT_UTF -#error The use of both EBCDIC and SUPPORT_UTF is not supported. -#endif - -/* Use a macro for debugging printing, 'cause that eliminates the use of #ifdef -inline, and there are *still* stupid compilers about that don't like indented -pre-processor statements, or at least there were when I first wrote this. After -all, it had only been about 10 years then... - -It turns out that the Mac Debugging.h header also defines the macro DPRINTF, so -be absolutely sure we get our version. */ - -#undef DPRINTF -#ifdef PCRE_DEBUG -#define DPRINTF(p) printf p -#else -#define DPRINTF(p) /* Nothing */ -#endif - - -/* Standard C headers plus the external interface definition. The only time -setjmp and stdarg are used is when NO_RECURSE is set. */ - -#include -#include -#include -#include -#include -#include - -/* Valgrind (memcheck) support */ - -#ifdef SUPPORT_VALGRIND -#include -#endif - -/* When compiling a DLL for Windows, the exported symbols have to be declared -using some MS magic. I found some useful information on this web page: -http://msdn2.microsoft.com/en-us/library/y4h7bcy6(VS.80).aspx. According to the -information there, using __declspec(dllexport) without "extern" we have a -definition; with "extern" we have a declaration. The settings here override the -setting in pcre.h (which is included below); it defines only PCRE_EXP_DECL, -which is all that is needed for applications (they just import the symbols). We -use: - - PCRE_EXP_DECL for declarations - PCRE_EXP_DEFN for definitions of exported functions - PCRE_EXP_DATA_DEFN for definitions of exported variables - -The reason for the two DEFN macros is that in non-Windows environments, one -does not want to have "extern" before variable definitions because it leads to -compiler warnings. So we distinguish between functions and variables. In -Windows, the two should always be the same. - -The reason for wrapping this in #ifndef PCRE_EXP_DECL is so that pcretest, -which is an application, but needs to import this file in order to "peek" at -internals, can #include pcre.h first to get an application's-eye view. - -In principle, people compiling for non-Windows, non-Unix-like (i.e. uncommon, -special-purpose environments) might want to stick other stuff in front of -exported symbols. That's why, in the non-Windows case, we set PCRE_EXP_DEFN and -PCRE_EXP_DATA_DEFN only if they are not already set. */ - -#ifndef PCRE_EXP_DECL -# ifdef _WIN32 -# ifndef PCRE_STATIC -# define PCRE_EXP_DECL extern __declspec(dllexport) -# define PCRE_EXP_DEFN __declspec(dllexport) -# define PCRE_EXP_DATA_DEFN __declspec(dllexport) -# else -# define PCRE_EXP_DECL extern -# define PCRE_EXP_DEFN -# define PCRE_EXP_DATA_DEFN -# endif -# else -# ifdef __cplusplus -# define PCRE_EXP_DECL extern "C" -# else -# define PCRE_EXP_DECL extern -# endif -# ifndef PCRE_EXP_DEFN -# define PCRE_EXP_DEFN PCRE_EXP_DECL -# endif -# ifndef PCRE_EXP_DATA_DEFN -# define PCRE_EXP_DATA_DEFN -# endif -# endif -#endif - -/* When compiling with the MSVC compiler, it is sometimes necessary to include -a "calling convention" before exported function names. (This is secondhand -information; I know nothing about MSVC myself). For example, something like - - void __cdecl function(....) - -might be needed. In order so make this easy, all the exported functions have -PCRE_CALL_CONVENTION just before their names. It is rarely needed; if not -set, we ensure here that it has no effect. */ - -#ifndef PCRE_CALL_CONVENTION -#define PCRE_CALL_CONVENTION -#endif - -/* We need to have types that specify unsigned 8, 16 and 32-bit integers. We -cannot determine these outside the compilation (e.g. by running a program as -part of "configure") because PCRE is often cross-compiled for use on other -systems. Instead we make use of the maximum sizes that are available at -preprocessor time in standard C environments. */ - -typedef unsigned char pcre_uint8; - -#if USHRT_MAX == 65535 -typedef unsigned short pcre_uint16; -typedef short pcre_int16; -#define PCRE_UINT16_MAX USHRT_MAX -#define PCRE_INT16_MAX SHRT_MAX -#elif UINT_MAX == 65535 -typedef unsigned int pcre_uint16; -typedef int pcre_int16; -#define PCRE_UINT16_MAX UINT_MAX -#define PCRE_INT16_MAX INT_MAX -#else -#error Cannot determine a type for 16-bit integers -#endif - -#if UINT_MAX == 4294967295U -typedef unsigned int pcre_uint32; -typedef int pcre_int32; -#define PCRE_UINT32_MAX UINT_MAX -#define PCRE_INT32_MAX INT_MAX -#elif ULONG_MAX == 4294967295UL -typedef unsigned long int pcre_uint32; -typedef long int pcre_int32; -#define PCRE_UINT32_MAX ULONG_MAX -#define PCRE_INT32_MAX LONG_MAX -#else -#error Cannot determine a type for 32-bit integers -#endif - -/* When checking for integer overflow in pcre_compile(), we need to handle -large integers. If a 64-bit integer type is available, we can use that. -Otherwise we have to cast to double, which of course requires floating point -arithmetic. Handle this by defining a macro for the appropriate type. If -stdint.h is available, include it; it may define INT64_MAX. Systems that do not -have stdint.h (e.g. Solaris) may have inttypes.h. The macro int64_t may be set -by "configure". */ - -#if defined HAVE_STDINT_H -#include -#elif defined HAVE_INTTYPES_H -#include -#endif - -#if defined INT64_MAX || defined int64_t -#define INT64_OR_DOUBLE int64_t -#else -#define INT64_OR_DOUBLE double -#endif - -/* All character handling must be done as unsigned characters. Otherwise there -are problems with top-bit-set characters and functions such as isspace(). -However, we leave the interface to the outside world as char * or short *, -because that should make things easier for callers. This character type is -called pcre_uchar. - -The IN_UCHARS macro multiply its argument with the byte size of the current -pcre_uchar type. Useful for memcpy and such operations, whose require the -byte size of their input/output buffers. - -The MAX_255 macro checks whether its pcre_uchar input is less than 256. - -The TABLE_GET macro is designed for accessing elements of tables whose contain -exactly 256 items. When the character is able to contain more than 256 -items, some check is needed before accessing these tables. -*/ - -#if defined COMPILE_PCRE8 - -typedef unsigned char pcre_uchar; -#define IN_UCHARS(x) (x) -#define MAX_255(c) 1 -#define TABLE_GET(c, table, default) ((table)[c]) - -#elif defined COMPILE_PCRE16 - -#if USHRT_MAX != 65535 -/* This is a warning message. Change PCRE_UCHAR16 to a 16 bit data type in -pcre.h(.in) and disable (comment out) this message. */ -#error Warning: PCRE_UCHAR16 is not a 16 bit data type. -#endif - -typedef pcre_uint16 pcre_uchar; -#define UCHAR_SHIFT (1) -#define IN_UCHARS(x) ((x) * 2) -#define MAX_255(c) ((c) <= 255u) -#define TABLE_GET(c, table, default) (MAX_255(c)? ((table)[c]):(default)) - -#elif defined COMPILE_PCRE32 - -typedef pcre_uint32 pcre_uchar; -#define UCHAR_SHIFT (2) -#define IN_UCHARS(x) ((x) * 4) -#define MAX_255(c) ((c) <= 255u) -#define TABLE_GET(c, table, default) (MAX_255(c)? ((table)[c]):(default)) - -#else -#error Unsupported compiling mode -#endif /* COMPILE_PCRE[8|16|32] */ - -/* This is an unsigned int value that no character can ever have. UTF-8 -characters only go up to 0x7fffffff (though Unicode doesn't go beyond -0x0010ffff). */ - -#define NOTACHAR 0xffffffff - -/* PCRE is able to support several different kinds of newline (CR, LF, CRLF, -"any" and "anycrlf" at present). The following macros are used to package up -testing for newlines. NLBLOCK, PSSTART, and PSEND are defined in the various -modules to indicate in which datablock the parameters exist, and what the -start/end of string field names are. */ - -#define NLTYPE_FIXED 0 /* Newline is a fixed length string */ -#define NLTYPE_ANY 1 /* Newline is any Unicode line ending */ -#define NLTYPE_ANYCRLF 2 /* Newline is CR, LF, or CRLF */ - -/* This macro checks for a newline at the given position */ - -#define IS_NEWLINE(p) \ - ((NLBLOCK->nltype != NLTYPE_FIXED)? \ - ((p) < NLBLOCK->PSEND && \ - PRIV(is_newline)((p), NLBLOCK->nltype, NLBLOCK->PSEND, \ - &(NLBLOCK->nllen), utf)) \ - : \ - ((p) <= NLBLOCK->PSEND - NLBLOCK->nllen && \ - UCHAR21TEST(p) == NLBLOCK->nl[0] && \ - (NLBLOCK->nllen == 1 || UCHAR21TEST(p+1) == NLBLOCK->nl[1]) \ - ) \ - ) - -/* This macro checks for a newline immediately preceding the given position */ - -#define WAS_NEWLINE(p) \ - ((NLBLOCK->nltype != NLTYPE_FIXED)? \ - ((p) > NLBLOCK->PSSTART && \ - PRIV(was_newline)((p), NLBLOCK->nltype, NLBLOCK->PSSTART, \ - &(NLBLOCK->nllen), utf)) \ - : \ - ((p) >= NLBLOCK->PSSTART + NLBLOCK->nllen && \ - UCHAR21TEST(p - NLBLOCK->nllen) == NLBLOCK->nl[0] && \ - (NLBLOCK->nllen == 1 || UCHAR21TEST(p - NLBLOCK->nllen + 1) == NLBLOCK->nl[1]) \ - ) \ - ) - -/* When PCRE is compiled as a C++ library, the subject pointer can be replaced -with a custom type. This makes it possible, for example, to allow pcre_exec() -to process subject strings that are discontinuous by using a smart pointer -class. It must always be possible to inspect all of the subject string in -pcre_exec() because of the way it backtracks. Two macros are required in the -normal case, for sign-unspecified and unsigned char pointers. The former is -used for the external interface and appears in pcre.h, which is why its name -must begin with PCRE_. */ - -#ifdef CUSTOM_SUBJECT_PTR -#define PCRE_PUCHAR CUSTOM_SUBJECT_PTR -#else -#define PCRE_PUCHAR const pcre_uchar * -#endif - -/* Include the public PCRE header and the definitions of UCP character property -values. */ - -#include "pcre.h" -#include "ucp.h" - -#ifdef COMPILE_PCRE32 -/* Assert that the public PCRE_UCHAR32 is a 32-bit type */ -typedef int __assert_pcre_uchar32_size[sizeof(PCRE_UCHAR32) == 4 ? 1 : -1]; -#endif - -/* When compiling for use with the Virtual Pascal compiler, these functions -need to have their names changed. PCRE must be compiled with the -DVPCOMPAT -option on the command line. */ - -#ifdef VPCOMPAT -#define strlen(s) _strlen(s) -#define strncmp(s1,s2,m) _strncmp(s1,s2,m) -#define memcmp(s,c,n) _memcmp(s,c,n) -#define memcpy(d,s,n) _memcpy(d,s,n) -#define memmove(d,s,n) _memmove(d,s,n) -#define memset(s,c,n) _memset(s,c,n) -#else /* VPCOMPAT */ - -/* To cope with SunOS4 and other systems that lack memmove() but have bcopy(), -define a macro for memmove() if HAVE_MEMMOVE is false, provided that HAVE_BCOPY -is set. Otherwise, include an emulating function for those systems that have -neither (there some non-Unix environments where this is the case). */ - -#ifndef HAVE_MEMMOVE -#undef memmove /* some systems may have a macro */ -#ifdef HAVE_BCOPY -#define memmove(a, b, c) bcopy(b, a, c) -#else /* HAVE_BCOPY */ -static void * -pcre_memmove(void *d, const void *s, size_t n) -{ -size_t i; -unsigned char *dest = (unsigned char *)d; -const unsigned char *src = (const unsigned char *)s; -if (dest > src) - { - dest += n; - src += n; - for (i = 0; i < n; ++i) *(--dest) = *(--src); - return (void *)dest; - } -else - { - for (i = 0; i < n; ++i) *dest++ = *src++; - return (void *)(dest - n); - } -} -#define memmove(a, b, c) pcre_memmove(a, b, c) -#endif /* not HAVE_BCOPY */ -#endif /* not HAVE_MEMMOVE */ -#endif /* not VPCOMPAT */ - - -/* PCRE keeps offsets in its compiled code as 2-byte quantities (always stored -in big-endian order) by default. These are used, for example, to link from the -start of a subpattern to its alternatives and its end. The use of 2 bytes per -offset limits the size of the compiled regex to around 64K, which is big enough -for almost everybody. However, I received a request for an even bigger limit. -For this reason, and also to make the code easier to maintain, the storing and -loading of offsets from the byte string is now handled by the macros that are -defined here. - -The macros are controlled by the value of LINK_SIZE. This defaults to 2 in -the config.h file, but can be overridden by using -D on the command line. This -is automated on Unix systems via the "configure" command. */ - -#if defined COMPILE_PCRE8 - -#if LINK_SIZE == 2 - -#define PUT(a,n,d) \ - (a[n] = (d) >> 8), \ - (a[(n)+1] = (d) & 255) - -#define GET(a,n) \ - (((a)[n] << 8) | (a)[(n)+1]) - -#define MAX_PATTERN_SIZE (1 << 16) - - -#elif LINK_SIZE == 3 - -#define PUT(a,n,d) \ - (a[n] = (d) >> 16), \ - (a[(n)+1] = (d) >> 8), \ - (a[(n)+2] = (d) & 255) - -#define GET(a,n) \ - (((a)[n] << 16) | ((a)[(n)+1] << 8) | (a)[(n)+2]) - -#define MAX_PATTERN_SIZE (1 << 24) - - -#elif LINK_SIZE == 4 - -#define PUT(a,n,d) \ - (a[n] = (d) >> 24), \ - (a[(n)+1] = (d) >> 16), \ - (a[(n)+2] = (d) >> 8), \ - (a[(n)+3] = (d) & 255) - -#define GET(a,n) \ - (((a)[n] << 24) | ((a)[(n)+1] << 16) | ((a)[(n)+2] << 8) | (a)[(n)+3]) - -/* Keep it positive */ -#define MAX_PATTERN_SIZE (1 << 30) - -#else -#error LINK_SIZE must be either 2, 3, or 4 -#endif - -#elif defined COMPILE_PCRE16 - -#if LINK_SIZE == 2 - -/* Redefine LINK_SIZE as a multiple of sizeof(pcre_uchar) */ -#undef LINK_SIZE -#define LINK_SIZE 1 - -#define PUT(a,n,d) \ - (a[n] = (d)) - -#define GET(a,n) \ - (a[n]) - -#define MAX_PATTERN_SIZE (1 << 16) - -#elif LINK_SIZE == 3 || LINK_SIZE == 4 - -/* Redefine LINK_SIZE as a multiple of sizeof(pcre_uchar) */ -#undef LINK_SIZE -#define LINK_SIZE 2 - -#define PUT(a,n,d) \ - (a[n] = (d) >> 16), \ - (a[(n)+1] = (d) & 65535) - -#define GET(a,n) \ - (((a)[n] << 16) | (a)[(n)+1]) - -/* Keep it positive */ -#define MAX_PATTERN_SIZE (1 << 30) - -#else -#error LINK_SIZE must be either 2, 3, or 4 -#endif - -#elif defined COMPILE_PCRE32 - -/* Only supported LINK_SIZE is 4 */ -/* Redefine LINK_SIZE as a multiple of sizeof(pcre_uchar) */ -#undef LINK_SIZE -#define LINK_SIZE 1 - -#define PUT(a,n,d) \ - (a[n] = (d)) - -#define GET(a,n) \ - (a[n]) - -/* Keep it positive */ -#define MAX_PATTERN_SIZE (1 << 30) - -#else -#error Unsupported compiling mode -#endif /* COMPILE_PCRE[8|16|32] */ - -/* Convenience macro defined in terms of the others */ - -#define PUTINC(a,n,d) PUT(a,n,d), a += LINK_SIZE - - -/* PCRE uses some other 2-byte quantities that do not change when the size of -offsets changes. There are used for repeat counts and for other things such as -capturing parenthesis numbers in back references. */ - -#if defined COMPILE_PCRE8 - -#define IMM2_SIZE 2 - -#define PUT2(a,n,d) \ - a[n] = (d) >> 8; \ - a[(n)+1] = (d) & 255 - -/* For reasons that I do not understand, the expression in this GET2 macro is -treated by gcc as a signed expression, even when a is declared as unsigned. It -seems that any kind of arithmetic results in a signed value. */ - -#define GET2(a,n) \ - (unsigned int)(((a)[n] << 8) | (a)[(n)+1]) - -#elif defined COMPILE_PCRE16 - -#define IMM2_SIZE 1 - -#define PUT2(a,n,d) \ - a[n] = d - -#define GET2(a,n) \ - a[n] - -#elif defined COMPILE_PCRE32 - -#define IMM2_SIZE 1 - -#define PUT2(a,n,d) \ - a[n] = d - -#define GET2(a,n) \ - a[n] - -#else -#error Unsupported compiling mode -#endif /* COMPILE_PCRE[8|16|32] */ - -#define PUT2INC(a,n,d) PUT2(a,n,d), a += IMM2_SIZE - -/* The maximum length of a MARK name is currently one data unit; it may be -changed in future to be a fixed number of bytes or to depend on LINK_SIZE. */ - -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 -#define MAX_MARK ((1u << 16) - 1) -#else -#define MAX_MARK ((1u << 8) - 1) -#endif - -/* There is a proposed future special "UTF-21" mode, in which only the lowest -21 bits of a 32-bit character are interpreted as UTF, with the remaining 11 -high-order bits available to the application for other uses. In preparation for -the future implementation of this mode, there are macros that load a data item -and, if in this special mode, mask it to 21 bits. These macros all have names -starting with UCHAR21. In all other modes, including the normal 32-bit -library, the macros all have the same simple definitions. When the new mode is -implemented, it is expected that these definitions will be varied appropriately -using #ifdef when compiling the library that supports the special mode. */ - -#define UCHAR21(eptr) (*(eptr)) -#define UCHAR21TEST(eptr) (*(eptr)) -#define UCHAR21INC(eptr) (*(eptr)++) -#define UCHAR21INCTEST(eptr) (*(eptr)++) - -/* When UTF encoding is being used, a character is no longer just a single -byte in 8-bit mode or a single short in 16-bit mode. The macros for character -handling generate simple sequences when used in the basic mode, and more -complicated ones for UTF characters. GETCHARLENTEST and other macros are not -used when UTF is not supported. To make sure they can never even appear when -UTF support is omitted, we don't even define them. */ - -#ifndef SUPPORT_UTF - -/* #define MAX_VALUE_FOR_SINGLE_CHAR */ -/* #define HAS_EXTRALEN(c) */ -/* #define GET_EXTRALEN(c) */ -/* #define NOT_FIRSTCHAR(c) */ -#define GETCHAR(c, eptr) c = *eptr; -#define GETCHARTEST(c, eptr) c = *eptr; -#define GETCHARINC(c, eptr) c = *eptr++; -#define GETCHARINCTEST(c, eptr) c = *eptr++; -#define GETCHARLEN(c, eptr, len) c = *eptr; -/* #define GETCHARLENTEST(c, eptr, len) */ -/* #define BACKCHAR(eptr) */ -/* #define FORWARDCHAR(eptr) */ -/* #define ACROSSCHAR(condition, eptr, action) */ - -#else /* SUPPORT_UTF */ - -/* Tests whether the code point needs extra characters to decode. */ - -#define HASUTF8EXTRALEN(c) ((c) >= 0xc0) - -/* Base macro to pick up the remaining bytes of a UTF-8 character, not -advancing the pointer. */ - -#define GETUTF8(c, eptr) \ - { \ - if ((c & 0x20) == 0) \ - c = ((c & 0x1f) << 6) | (eptr[1] & 0x3f); \ - else if ((c & 0x10) == 0) \ - c = ((c & 0x0f) << 12) | ((eptr[1] & 0x3f) << 6) | (eptr[2] & 0x3f); \ - else if ((c & 0x08) == 0) \ - c = ((c & 0x07) << 18) | ((eptr[1] & 0x3f) << 12) | \ - ((eptr[2] & 0x3f) << 6) | (eptr[3] & 0x3f); \ - else if ((c & 0x04) == 0) \ - c = ((c & 0x03) << 24) | ((eptr[1] & 0x3f) << 18) | \ - ((eptr[2] & 0x3f) << 12) | ((eptr[3] & 0x3f) << 6) | \ - (eptr[4] & 0x3f); \ - else \ - c = ((c & 0x01) << 30) | ((eptr[1] & 0x3f) << 24) | \ - ((eptr[2] & 0x3f) << 18) | ((eptr[3] & 0x3f) << 12) | \ - ((eptr[4] & 0x3f) << 6) | (eptr[5] & 0x3f); \ - } - -/* Base macro to pick up the remaining bytes of a UTF-8 character, advancing -the pointer. */ - -#define GETUTF8INC(c, eptr) \ - { \ - if ((c & 0x20) == 0) \ - c = ((c & 0x1f) << 6) | (*eptr++ & 0x3f); \ - else if ((c & 0x10) == 0) \ - { \ - c = ((c & 0x0f) << 12) | ((*eptr & 0x3f) << 6) | (eptr[1] & 0x3f); \ - eptr += 2; \ - } \ - else if ((c & 0x08) == 0) \ - { \ - c = ((c & 0x07) << 18) | ((*eptr & 0x3f) << 12) | \ - ((eptr[1] & 0x3f) << 6) | (eptr[2] & 0x3f); \ - eptr += 3; \ - } \ - else if ((c & 0x04) == 0) \ - { \ - c = ((c & 0x03) << 24) | ((*eptr & 0x3f) << 18) | \ - ((eptr[1] & 0x3f) << 12) | ((eptr[2] & 0x3f) << 6) | \ - (eptr[3] & 0x3f); \ - eptr += 4; \ - } \ - else \ - { \ - c = ((c & 0x01) << 30) | ((*eptr & 0x3f) << 24) | \ - ((eptr[1] & 0x3f) << 18) | ((eptr[2] & 0x3f) << 12) | \ - ((eptr[3] & 0x3f) << 6) | (eptr[4] & 0x3f); \ - eptr += 5; \ - } \ - } - -#if defined COMPILE_PCRE8 - -/* These macros were originally written in the form of loops that used data -from the tables whose names start with PRIV(utf8_table). They were rewritten by -a user so as not to use loops, because in some environments this gives a -significant performance advantage, and it seems never to do any harm. */ - -/* Tells the biggest code point which can be encoded as a single character. */ - -#define MAX_VALUE_FOR_SINGLE_CHAR 127 - -/* Tests whether the code point needs extra characters to decode. */ - -#define HAS_EXTRALEN(c) ((c) >= 0xc0) - -/* Returns with the additional number of characters if IS_MULTICHAR(c) is TRUE. -Otherwise it has an undefined behaviour. */ - -#define GET_EXTRALEN(c) (PRIV(utf8_table4)[(c) & 0x3f]) - -/* Returns TRUE, if the given character is not the first character -of a UTF sequence. */ - -#define NOT_FIRSTCHAR(c) (((c) & 0xc0) == 0x80) - -/* Get the next UTF-8 character, not advancing the pointer. This is called when -we know we are in UTF-8 mode. */ - -#define GETCHAR(c, eptr) \ - c = *eptr; \ - if (c >= 0xc0) GETUTF8(c, eptr); - -/* Get the next UTF-8 character, testing for UTF-8 mode, and not advancing the -pointer. */ - -#define GETCHARTEST(c, eptr) \ - c = *eptr; \ - if (utf && c >= 0xc0) GETUTF8(c, eptr); - -/* Get the next UTF-8 character, advancing the pointer. This is called when we -know we are in UTF-8 mode. */ - -#define GETCHARINC(c, eptr) \ - c = *eptr++; \ - if (c >= 0xc0) GETUTF8INC(c, eptr); - -/* Get the next character, testing for UTF-8 mode, and advancing the pointer. -This is called when we don't know if we are in UTF-8 mode. */ - -#define GETCHARINCTEST(c, eptr) \ - c = *eptr++; \ - if (utf && c >= 0xc0) GETUTF8INC(c, eptr); - -/* Base macro to pick up the remaining bytes of a UTF-8 character, not -advancing the pointer, incrementing the length. */ - -#define GETUTF8LEN(c, eptr, len) \ - { \ - if ((c & 0x20) == 0) \ - { \ - c = ((c & 0x1f) << 6) | (eptr[1] & 0x3f); \ - len++; \ - } \ - else if ((c & 0x10) == 0) \ - { \ - c = ((c & 0x0f) << 12) | ((eptr[1] & 0x3f) << 6) | (eptr[2] & 0x3f); \ - len += 2; \ - } \ - else if ((c & 0x08) == 0) \ - {\ - c = ((c & 0x07) << 18) | ((eptr[1] & 0x3f) << 12) | \ - ((eptr[2] & 0x3f) << 6) | (eptr[3] & 0x3f); \ - len += 3; \ - } \ - else if ((c & 0x04) == 0) \ - { \ - c = ((c & 0x03) << 24) | ((eptr[1] & 0x3f) << 18) | \ - ((eptr[2] & 0x3f) << 12) | ((eptr[3] & 0x3f) << 6) | \ - (eptr[4] & 0x3f); \ - len += 4; \ - } \ - else \ - {\ - c = ((c & 0x01) << 30) | ((eptr[1] & 0x3f) << 24) | \ - ((eptr[2] & 0x3f) << 18) | ((eptr[3] & 0x3f) << 12) | \ - ((eptr[4] & 0x3f) << 6) | (eptr[5] & 0x3f); \ - len += 5; \ - } \ - } - -/* Get the next UTF-8 character, not advancing the pointer, incrementing length -if there are extra bytes. This is called when we know we are in UTF-8 mode. */ - -#define GETCHARLEN(c, eptr, len) \ - c = *eptr; \ - if (c >= 0xc0) GETUTF8LEN(c, eptr, len); - -/* Get the next UTF-8 character, testing for UTF-8 mode, not advancing the -pointer, incrementing length if there are extra bytes. This is called when we -do not know if we are in UTF-8 mode. */ - -#define GETCHARLENTEST(c, eptr, len) \ - c = *eptr; \ - if (utf && c >= 0xc0) GETUTF8LEN(c, eptr, len); - -/* If the pointer is not at the start of a character, move it back until -it is. This is called only in UTF-8 mode - we don't put a test within the macro -because almost all calls are already within a block of UTF-8 only code. */ - -#define BACKCHAR(eptr) while((*eptr & 0xc0) == 0x80) eptr-- - -/* Same as above, just in the other direction. */ -#define FORWARDCHAR(eptr) while((*eptr & 0xc0) == 0x80) eptr++ - -/* Same as above, but it allows a fully customizable form. */ -#define ACROSSCHAR(condition, eptr, action) \ - while((condition) && ((eptr) & 0xc0) == 0x80) action - -#elif defined COMPILE_PCRE16 - -/* Tells the biggest code point which can be encoded as a single character. */ - -#define MAX_VALUE_FOR_SINGLE_CHAR 65535 - -/* Tests whether the code point needs extra characters to decode. */ - -#define HAS_EXTRALEN(c) (((c) & 0xfc00) == 0xd800) - -/* Returns with the additional number of characters if IS_MULTICHAR(c) is TRUE. -Otherwise it has an undefined behaviour. */ - -#define GET_EXTRALEN(c) 1 - -/* Returns TRUE, if the given character is not the first character -of a UTF sequence. */ - -#define NOT_FIRSTCHAR(c) (((c) & 0xfc00) == 0xdc00) - -/* Base macro to pick up the low surrogate of a UTF-16 character, not -advancing the pointer. */ - -#define GETUTF16(c, eptr) \ - { c = (((c & 0x3ff) << 10) | (eptr[1] & 0x3ff)) + 0x10000; } - -/* Get the next UTF-16 character, not advancing the pointer. This is called when -we know we are in UTF-16 mode. */ - -#define GETCHAR(c, eptr) \ - c = *eptr; \ - if ((c & 0xfc00) == 0xd800) GETUTF16(c, eptr); - -/* Get the next UTF-16 character, testing for UTF-16 mode, and not advancing the -pointer. */ - -#define GETCHARTEST(c, eptr) \ - c = *eptr; \ - if (utf && (c & 0xfc00) == 0xd800) GETUTF16(c, eptr); - -/* Base macro to pick up the low surrogate of a UTF-16 character, advancing -the pointer. */ - -#define GETUTF16INC(c, eptr) \ - { c = (((c & 0x3ff) << 10) | (*eptr++ & 0x3ff)) + 0x10000; } - -/* Get the next UTF-16 character, advancing the pointer. This is called when we -know we are in UTF-16 mode. */ - -#define GETCHARINC(c, eptr) \ - c = *eptr++; \ - if ((c & 0xfc00) == 0xd800) GETUTF16INC(c, eptr); - -/* Get the next character, testing for UTF-16 mode, and advancing the pointer. -This is called when we don't know if we are in UTF-16 mode. */ - -#define GETCHARINCTEST(c, eptr) \ - c = *eptr++; \ - if (utf && (c & 0xfc00) == 0xd800) GETUTF16INC(c, eptr); - -/* Base macro to pick up the low surrogate of a UTF-16 character, not -advancing the pointer, incrementing the length. */ - -#define GETUTF16LEN(c, eptr, len) \ - { c = (((c & 0x3ff) << 10) | (eptr[1] & 0x3ff)) + 0x10000; len++; } - -/* Get the next UTF-16 character, not advancing the pointer, incrementing -length if there is a low surrogate. This is called when we know we are in -UTF-16 mode. */ - -#define GETCHARLEN(c, eptr, len) \ - c = *eptr; \ - if ((c & 0xfc00) == 0xd800) GETUTF16LEN(c, eptr, len); - -/* Get the next UTF-816character, testing for UTF-16 mode, not advancing the -pointer, incrementing length if there is a low surrogate. This is called when -we do not know if we are in UTF-16 mode. */ - -#define GETCHARLENTEST(c, eptr, len) \ - c = *eptr; \ - if (utf && (c & 0xfc00) == 0xd800) GETUTF16LEN(c, eptr, len); - -/* If the pointer is not at the start of a character, move it back until -it is. This is called only in UTF-16 mode - we don't put a test within the -macro because almost all calls are already within a block of UTF-16 only -code. */ - -#define BACKCHAR(eptr) if ((*eptr & 0xfc00) == 0xdc00) eptr-- - -/* Same as above, just in the other direction. */ -#define FORWARDCHAR(eptr) if ((*eptr & 0xfc00) == 0xdc00) eptr++ - -/* Same as above, but it allows a fully customizable form. */ -#define ACROSSCHAR(condition, eptr, action) \ - if ((condition) && ((eptr) & 0xfc00) == 0xdc00) action - -#elif defined COMPILE_PCRE32 - -/* These are trivial for the 32-bit library, since all UTF-32 characters fit -into one pcre_uchar unit. */ -#define MAX_VALUE_FOR_SINGLE_CHAR (0x10ffffu) -#define HAS_EXTRALEN(c) (0) -#define GET_EXTRALEN(c) (0) -#define NOT_FIRSTCHAR(c) (0) - -/* Get the next UTF-32 character, not advancing the pointer. This is called when -we know we are in UTF-32 mode. */ - -#define GETCHAR(c, eptr) \ - c = *(eptr); - -/* Get the next UTF-32 character, testing for UTF-32 mode, and not advancing the -pointer. */ - -#define GETCHARTEST(c, eptr) \ - c = *(eptr); - -/* Get the next UTF-32 character, advancing the pointer. This is called when we -know we are in UTF-32 mode. */ - -#define GETCHARINC(c, eptr) \ - c = *((eptr)++); - -/* Get the next character, testing for UTF-32 mode, and advancing the pointer. -This is called when we don't know if we are in UTF-32 mode. */ - -#define GETCHARINCTEST(c, eptr) \ - c = *((eptr)++); - -/* Get the next UTF-32 character, not advancing the pointer, not incrementing -length (since all UTF-32 is of length 1). This is called when we know we are in -UTF-32 mode. */ - -#define GETCHARLEN(c, eptr, len) \ - GETCHAR(c, eptr) - -/* Get the next UTF-32character, testing for UTF-32 mode, not advancing the -pointer, not incrementing the length (since all UTF-32 is of length 1). -This is called when we do not know if we are in UTF-32 mode. */ - -#define GETCHARLENTEST(c, eptr, len) \ - GETCHARTEST(c, eptr) - -/* If the pointer is not at the start of a character, move it back until -it is. This is called only in UTF-32 mode - we don't put a test within the -macro because almost all calls are already within a block of UTF-32 only -code. -These are all no-ops since all UTF-32 characters fit into one pcre_uchar. */ - -#define BACKCHAR(eptr) do { } while (0) - -/* Same as above, just in the other direction. */ -#define FORWARDCHAR(eptr) do { } while (0) - -/* Same as above, but it allows a fully customizable form. */ -#define ACROSSCHAR(condition, eptr, action) do { } while (0) - -#else -#error Unsupported compiling mode -#endif /* COMPILE_PCRE[8|16|32] */ - -#endif /* SUPPORT_UTF */ - -/* Tests for Unicode horizontal and vertical whitespace characters must check a -number of different values. Using a switch statement for this generates the -fastest code (no loop, no memory access), and there are several places in the -interpreter code where this happens. In order to ensure that all the case lists -remain in step, we use macros so that there is only one place where the lists -are defined. - -These values are also required as lists in pcre_compile.c when processing \h, -\H, \v and \V in a character class. The lists are defined in pcre_tables.c, but -macros that define the values are here so that all the definitions are -together. The lists must be in ascending character order, terminated by -NOTACHAR (which is 0xffffffff). - -Any changes should ensure that the various macros are kept in step with each -other. NOTE: The values also appear in pcre_jit_compile.c. */ - -/* ------ ASCII/Unicode environments ------ */ - -#ifndef EBCDIC - -#define HSPACE_LIST \ - CHAR_HT, CHAR_SPACE, CHAR_NBSP, \ - 0x1680, 0x180e, 0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, \ - 0x2006, 0x2007, 0x2008, 0x2009, 0x200A, 0x202f, 0x205f, 0x3000, \ - NOTACHAR - -#define HSPACE_MULTIBYTE_CASES \ - case 0x1680: /* OGHAM SPACE MARK */ \ - case 0x180e: /* MONGOLIAN VOWEL SEPARATOR */ \ - case 0x2000: /* EN QUAD */ \ - case 0x2001: /* EM QUAD */ \ - case 0x2002: /* EN SPACE */ \ - case 0x2003: /* EM SPACE */ \ - case 0x2004: /* THREE-PER-EM SPACE */ \ - case 0x2005: /* FOUR-PER-EM SPACE */ \ - case 0x2006: /* SIX-PER-EM SPACE */ \ - case 0x2007: /* FIGURE SPACE */ \ - case 0x2008: /* PUNCTUATION SPACE */ \ - case 0x2009: /* THIN SPACE */ \ - case 0x200A: /* HAIR SPACE */ \ - case 0x202f: /* NARROW NO-BREAK SPACE */ \ - case 0x205f: /* MEDIUM MATHEMATICAL SPACE */ \ - case 0x3000 /* IDEOGRAPHIC SPACE */ - -#define HSPACE_BYTE_CASES \ - case CHAR_HT: \ - case CHAR_SPACE: \ - case CHAR_NBSP - -#define HSPACE_CASES \ - HSPACE_BYTE_CASES: \ - HSPACE_MULTIBYTE_CASES - -#define VSPACE_LIST \ - CHAR_LF, CHAR_VT, CHAR_FF, CHAR_CR, CHAR_NEL, 0x2028, 0x2029, NOTACHAR - -#define VSPACE_MULTIBYTE_CASES \ - case 0x2028: /* LINE SEPARATOR */ \ - case 0x2029 /* PARAGRAPH SEPARATOR */ - -#define VSPACE_BYTE_CASES \ - case CHAR_LF: \ - case CHAR_VT: \ - case CHAR_FF: \ - case CHAR_CR: \ - case CHAR_NEL - -#define VSPACE_CASES \ - VSPACE_BYTE_CASES: \ - VSPACE_MULTIBYTE_CASES - -/* ------ EBCDIC environments ------ */ - -#else -#define HSPACE_LIST CHAR_HT, CHAR_SPACE, CHAR_NBSP, NOTACHAR - -#define HSPACE_BYTE_CASES \ - case CHAR_HT: \ - case CHAR_SPACE: \ - case CHAR_NBSP - -#define HSPACE_CASES HSPACE_BYTE_CASES - -#ifdef EBCDIC_NL25 -#define VSPACE_LIST \ - CHAR_VT, CHAR_FF, CHAR_CR, CHAR_NEL, CHAR_LF, NOTACHAR -#else -#define VSPACE_LIST \ - CHAR_VT, CHAR_FF, CHAR_CR, CHAR_LF, CHAR_NEL, NOTACHAR -#endif - -#define VSPACE_BYTE_CASES \ - case CHAR_LF: \ - case CHAR_VT: \ - case CHAR_FF: \ - case CHAR_CR: \ - case CHAR_NEL - -#define VSPACE_CASES VSPACE_BYTE_CASES -#endif /* EBCDIC */ - -/* ------ End of whitespace macros ------ */ - - - -/* Private flags containing information about the compiled regex. They used to -live at the top end of the options word, but that got almost full, so they were -moved to a 16-bit flags word - which got almost full, so now they are in a -32-bit flags word. From release 8.00, PCRE_NOPARTIAL is unused, as the -restrictions on partial matching have been lifted. It remains for backwards -compatibility. */ - -#define PCRE_MODE8 0x00000001 /* compiled in 8 bit mode */ -#define PCRE_MODE16 0x00000002 /* compiled in 16 bit mode */ -#define PCRE_MODE32 0x00000004 /* compiled in 32 bit mode */ -#define PCRE_FIRSTSET 0x00000010 /* first_char is set */ -#define PCRE_FCH_CASELESS 0x00000020 /* caseless first char */ -#define PCRE_REQCHSET 0x00000040 /* req_byte is set */ -#define PCRE_RCH_CASELESS 0x00000080 /* caseless requested char */ -#define PCRE_STARTLINE 0x00000100 /* start after \n for multiline */ -#define PCRE_NOPARTIAL 0x00000200 /* can't use partial with this regex */ -#define PCRE_JCHANGED 0x00000400 /* j option used in regex */ -#define PCRE_HASCRORLF 0x00000800 /* explicit \r or \n in pattern */ -#define PCRE_HASTHEN 0x00001000 /* pattern contains (*THEN) */ -#define PCRE_MLSET 0x00002000 /* match limit set by regex */ -#define PCRE_RLSET 0x00004000 /* recursion limit set by regex */ -#define PCRE_MATCH_EMPTY 0x00008000 /* pattern can match empty string */ - -#if defined COMPILE_PCRE8 -#define PCRE_MODE PCRE_MODE8 -#elif defined COMPILE_PCRE16 -#define PCRE_MODE PCRE_MODE16 -#elif defined COMPILE_PCRE32 -#define PCRE_MODE PCRE_MODE32 -#endif -#define PCRE_MODE_MASK (PCRE_MODE8 | PCRE_MODE16 | PCRE_MODE32) - -/* Flags for the "extra" block produced by pcre_study(). */ - -#define PCRE_STUDY_MAPPED 0x0001 /* a map of starting chars exists */ -#define PCRE_STUDY_MINLEN 0x0002 /* a minimum length field exists */ - -/* Masks for identifying the public options that are permitted at compile -time, run time, or study time, respectively. */ - -#define PCRE_NEWLINE_BITS (PCRE_NEWLINE_CR|PCRE_NEWLINE_LF|PCRE_NEWLINE_ANY| \ - PCRE_NEWLINE_ANYCRLF) - -#define PUBLIC_COMPILE_OPTIONS \ - (PCRE_CASELESS|PCRE_EXTENDED|PCRE_ANCHORED|PCRE_MULTILINE| \ - PCRE_DOTALL|PCRE_DOLLAR_ENDONLY|PCRE_EXTRA|PCRE_UNGREEDY|PCRE_UTF8| \ - PCRE_NO_AUTO_CAPTURE|PCRE_NO_AUTO_POSSESS| \ - PCRE_NO_UTF8_CHECK|PCRE_AUTO_CALLOUT|PCRE_FIRSTLINE| \ - PCRE_DUPNAMES|PCRE_NEWLINE_BITS|PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE| \ - PCRE_JAVASCRIPT_COMPAT|PCRE_UCP|PCRE_NO_START_OPTIMIZE|PCRE_NEVER_UTF) - -#define PUBLIC_EXEC_OPTIONS \ - (PCRE_ANCHORED|PCRE_NOTBOL|PCRE_NOTEOL|PCRE_NOTEMPTY|PCRE_NOTEMPTY_ATSTART| \ - PCRE_NO_UTF8_CHECK|PCRE_PARTIAL_HARD|PCRE_PARTIAL_SOFT|PCRE_NEWLINE_BITS| \ - PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE|PCRE_NO_START_OPTIMIZE) - -#define PUBLIC_DFA_EXEC_OPTIONS \ - (PCRE_ANCHORED|PCRE_NOTBOL|PCRE_NOTEOL|PCRE_NOTEMPTY|PCRE_NOTEMPTY_ATSTART| \ - PCRE_NO_UTF8_CHECK|PCRE_PARTIAL_HARD|PCRE_PARTIAL_SOFT|PCRE_DFA_SHORTEST| \ - PCRE_DFA_RESTART|PCRE_NEWLINE_BITS|PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE| \ - PCRE_NO_START_OPTIMIZE) - -#define PUBLIC_STUDY_OPTIONS \ - (PCRE_STUDY_JIT_COMPILE|PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE| \ - PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE|PCRE_STUDY_EXTRA_NEEDED) - -#define PUBLIC_JIT_EXEC_OPTIONS \ - (PCRE_NO_UTF8_CHECK|PCRE_NOTBOL|PCRE_NOTEOL|PCRE_NOTEMPTY|\ - PCRE_NOTEMPTY_ATSTART|PCRE_PARTIAL_SOFT|PCRE_PARTIAL_HARD) - -/* Magic number to provide a small check against being handed junk. */ - -#define MAGIC_NUMBER 0x50435245UL /* 'PCRE' */ - -/* This variable is used to detect a loaded regular expression -in different endianness. */ - -#define REVERSED_MAGIC_NUMBER 0x45524350UL /* 'ERCP' */ - -/* The maximum remaining length of subject we are prepared to search for a -req_byte match. */ - -#define REQ_BYTE_MAX 1000 - -/* Miscellaneous definitions. The #ifndef is to pacify compiler warnings in -environments where these macros are defined elsewhere. Unfortunately, there -is no way to do the same for the typedef. */ - -typedef int BOOL; - -#ifndef FALSE -#define FALSE 0 -#define TRUE 1 -#endif - -/* If PCRE is to support UTF-8 on EBCDIC platforms, we cannot use normal -character constants like '*' because the compiler would emit their EBCDIC code, -which is different from their ASCII/UTF-8 code. Instead we define macros for -the characters so that they always use the ASCII/UTF-8 code when UTF-8 support -is enabled. When UTF-8 support is not enabled, the definitions use character -literals. Both character and string versions of each character are needed, and -there are some longer strings as well. - -This means that, on EBCDIC platforms, the PCRE library can handle either -EBCDIC, or UTF-8, but not both. To support both in the same compiled library -would need different lookups depending on whether PCRE_UTF8 was set or not. -This would make it impossible to use characters in switch/case statements, -which would reduce performance. For a theoretical use (which nobody has asked -for) in a minority area (EBCDIC platforms), this is not sensible. Any -application that did need both could compile two versions of the library, using -macros to give the functions distinct names. */ - -#ifndef SUPPORT_UTF - -/* UTF-8 support is not enabled; use the platform-dependent character literals -so that PCRE works in both ASCII and EBCDIC environments, but only in non-UTF -mode. Newline characters are problematic in EBCDIC. Though it has CR and LF -characters, a common practice has been to use its NL (0x15) character as the -line terminator in C-like processing environments. However, sometimes the LF -(0x25) character is used instead, according to this Unicode document: - -http://unicode.org/standard/reports/tr13/tr13-5.html - -PCRE defaults EBCDIC NL to 0x15, but has a build-time option to select 0x25 -instead. Whichever is *not* chosen is defined as NEL. - -In both ASCII and EBCDIC environments, CHAR_NL and CHAR_LF are synonyms for the -same code point. */ - -#ifdef EBCDIC - -#ifndef EBCDIC_NL25 -#define CHAR_NL '\x15' -#define CHAR_NEL '\x25' -#define STR_NL "\x15" -#define STR_NEL "\x25" -#else -#define CHAR_NL '\x25' -#define CHAR_NEL '\x15' -#define STR_NL "\x25" -#define STR_NEL "\x15" -#endif - -#define CHAR_LF CHAR_NL -#define STR_LF STR_NL - -#define CHAR_ESC '\047' -#define CHAR_DEL '\007' -#define CHAR_NBSP '\x41' -#define STR_ESC "\047" -#define STR_DEL "\007" - -#else /* Not EBCDIC */ - -/* In ASCII/Unicode, linefeed is '\n' and we equate this to NL for -compatibility. NEL is the Unicode newline character; make sure it is -a positive value. */ - -#define CHAR_LF '\n' -#define CHAR_NL CHAR_LF -#define CHAR_NEL ((unsigned char)'\x85') -#define CHAR_ESC '\033' -#define CHAR_DEL '\177' -#define CHAR_NBSP ((unsigned char)'\xa0') - -#define STR_LF "\n" -#define STR_NL STR_LF -#define STR_NEL "\x85" -#define STR_ESC "\033" -#define STR_DEL "\177" - -#endif /* EBCDIC */ - -/* The remaining definitions work in both environments. */ - -#define CHAR_NULL '\0' -#define CHAR_HT '\t' -#define CHAR_VT '\v' -#define CHAR_FF '\f' -#define CHAR_CR '\r' -#define CHAR_BS '\b' -#define CHAR_BEL '\a' - -#define CHAR_SPACE ' ' -#define CHAR_EXCLAMATION_MARK '!' -#define CHAR_QUOTATION_MARK '"' -#define CHAR_NUMBER_SIGN '#' -#define CHAR_DOLLAR_SIGN '$' -#define CHAR_PERCENT_SIGN '%' -#define CHAR_AMPERSAND '&' -#define CHAR_APOSTROPHE '\'' -#define CHAR_LEFT_PARENTHESIS '(' -#define CHAR_RIGHT_PARENTHESIS ')' -#define CHAR_ASTERISK '*' -#define CHAR_PLUS '+' -#define CHAR_COMMA ',' -#define CHAR_MINUS '-' -#define CHAR_DOT '.' -#define CHAR_SLASH '/' -#define CHAR_0 '0' -#define CHAR_1 '1' -#define CHAR_2 '2' -#define CHAR_3 '3' -#define CHAR_4 '4' -#define CHAR_5 '5' -#define CHAR_6 '6' -#define CHAR_7 '7' -#define CHAR_8 '8' -#define CHAR_9 '9' -#define CHAR_COLON ':' -#define CHAR_SEMICOLON ';' -#define CHAR_LESS_THAN_SIGN '<' -#define CHAR_EQUALS_SIGN '=' -#define CHAR_GREATER_THAN_SIGN '>' -#define CHAR_QUESTION_MARK '?' -#define CHAR_COMMERCIAL_AT '@' -#define CHAR_A 'A' -#define CHAR_B 'B' -#define CHAR_C 'C' -#define CHAR_D 'D' -#define CHAR_E 'E' -#define CHAR_F 'F' -#define CHAR_G 'G' -#define CHAR_H 'H' -#define CHAR_I 'I' -#define CHAR_J 'J' -#define CHAR_K 'K' -#define CHAR_L 'L' -#define CHAR_M 'M' -#define CHAR_N 'N' -#define CHAR_O 'O' -#define CHAR_P 'P' -#define CHAR_Q 'Q' -#define CHAR_R 'R' -#define CHAR_S 'S' -#define CHAR_T 'T' -#define CHAR_U 'U' -#define CHAR_V 'V' -#define CHAR_W 'W' -#define CHAR_X 'X' -#define CHAR_Y 'Y' -#define CHAR_Z 'Z' -#define CHAR_LEFT_SQUARE_BRACKET '[' -#define CHAR_BACKSLASH '\\' -#define CHAR_RIGHT_SQUARE_BRACKET ']' -#define CHAR_CIRCUMFLEX_ACCENT '^' -#define CHAR_UNDERSCORE '_' -#define CHAR_GRAVE_ACCENT '`' -#define CHAR_a 'a' -#define CHAR_b 'b' -#define CHAR_c 'c' -#define CHAR_d 'd' -#define CHAR_e 'e' -#define CHAR_f 'f' -#define CHAR_g 'g' -#define CHAR_h 'h' -#define CHAR_i 'i' -#define CHAR_j 'j' -#define CHAR_k 'k' -#define CHAR_l 'l' -#define CHAR_m 'm' -#define CHAR_n 'n' -#define CHAR_o 'o' -#define CHAR_p 'p' -#define CHAR_q 'q' -#define CHAR_r 'r' -#define CHAR_s 's' -#define CHAR_t 't' -#define CHAR_u 'u' -#define CHAR_v 'v' -#define CHAR_w 'w' -#define CHAR_x 'x' -#define CHAR_y 'y' -#define CHAR_z 'z' -#define CHAR_LEFT_CURLY_BRACKET '{' -#define CHAR_VERTICAL_LINE '|' -#define CHAR_RIGHT_CURLY_BRACKET '}' -#define CHAR_TILDE '~' - -#define STR_HT "\t" -#define STR_VT "\v" -#define STR_FF "\f" -#define STR_CR "\r" -#define STR_BS "\b" -#define STR_BEL "\a" - -#define STR_SPACE " " -#define STR_EXCLAMATION_MARK "!" -#define STR_QUOTATION_MARK "\"" -#define STR_NUMBER_SIGN "#" -#define STR_DOLLAR_SIGN "$" -#define STR_PERCENT_SIGN "%" -#define STR_AMPERSAND "&" -#define STR_APOSTROPHE "'" -#define STR_LEFT_PARENTHESIS "(" -#define STR_RIGHT_PARENTHESIS ")" -#define STR_ASTERISK "*" -#define STR_PLUS "+" -#define STR_COMMA "," -#define STR_MINUS "-" -#define STR_DOT "." -#define STR_SLASH "/" -#define STR_0 "0" -#define STR_1 "1" -#define STR_2 "2" -#define STR_3 "3" -#define STR_4 "4" -#define STR_5 "5" -#define STR_6 "6" -#define STR_7 "7" -#define STR_8 "8" -#define STR_9 "9" -#define STR_COLON ":" -#define STR_SEMICOLON ";" -#define STR_LESS_THAN_SIGN "<" -#define STR_EQUALS_SIGN "=" -#define STR_GREATER_THAN_SIGN ">" -#define STR_QUESTION_MARK "?" -#define STR_COMMERCIAL_AT "@" -#define STR_A "A" -#define STR_B "B" -#define STR_C "C" -#define STR_D "D" -#define STR_E "E" -#define STR_F "F" -#define STR_G "G" -#define STR_H "H" -#define STR_I "I" -#define STR_J "J" -#define STR_K "K" -#define STR_L "L" -#define STR_M "M" -#define STR_N "N" -#define STR_O "O" -#define STR_P "P" -#define STR_Q "Q" -#define STR_R "R" -#define STR_S "S" -#define STR_T "T" -#define STR_U "U" -#define STR_V "V" -#define STR_W "W" -#define STR_X "X" -#define STR_Y "Y" -#define STR_Z "Z" -#define STR_LEFT_SQUARE_BRACKET "[" -#define STR_BACKSLASH "\\" -#define STR_RIGHT_SQUARE_BRACKET "]" -#define STR_CIRCUMFLEX_ACCENT "^" -#define STR_UNDERSCORE "_" -#define STR_GRAVE_ACCENT "`" -#define STR_a "a" -#define STR_b "b" -#define STR_c "c" -#define STR_d "d" -#define STR_e "e" -#define STR_f "f" -#define STR_g "g" -#define STR_h "h" -#define STR_i "i" -#define STR_j "j" -#define STR_k "k" -#define STR_l "l" -#define STR_m "m" -#define STR_n "n" -#define STR_o "o" -#define STR_p "p" -#define STR_q "q" -#define STR_r "r" -#define STR_s "s" -#define STR_t "t" -#define STR_u "u" -#define STR_v "v" -#define STR_w "w" -#define STR_x "x" -#define STR_y "y" -#define STR_z "z" -#define STR_LEFT_CURLY_BRACKET "{" -#define STR_VERTICAL_LINE "|" -#define STR_RIGHT_CURLY_BRACKET "}" -#define STR_TILDE "~" - -#define STRING_ACCEPT0 "ACCEPT\0" -#define STRING_COMMIT0 "COMMIT\0" -#define STRING_F0 "F\0" -#define STRING_FAIL0 "FAIL\0" -#define STRING_MARK0 "MARK\0" -#define STRING_PRUNE0 "PRUNE\0" -#define STRING_SKIP0 "SKIP\0" -#define STRING_THEN "THEN" - -#define STRING_alpha0 "alpha\0" -#define STRING_lower0 "lower\0" -#define STRING_upper0 "upper\0" -#define STRING_alnum0 "alnum\0" -#define STRING_ascii0 "ascii\0" -#define STRING_blank0 "blank\0" -#define STRING_cntrl0 "cntrl\0" -#define STRING_digit0 "digit\0" -#define STRING_graph0 "graph\0" -#define STRING_print0 "print\0" -#define STRING_punct0 "punct\0" -#define STRING_space0 "space\0" -#define STRING_word0 "word\0" -#define STRING_xdigit "xdigit" - -#define STRING_DEFINE "DEFINE" -#define STRING_WEIRD_STARTWORD "[:<:]]" -#define STRING_WEIRD_ENDWORD "[:>:]]" - -#define STRING_CR_RIGHTPAR "CR)" -#define STRING_LF_RIGHTPAR "LF)" -#define STRING_CRLF_RIGHTPAR "CRLF)" -#define STRING_ANY_RIGHTPAR "ANY)" -#define STRING_ANYCRLF_RIGHTPAR "ANYCRLF)" -#define STRING_BSR_ANYCRLF_RIGHTPAR "BSR_ANYCRLF)" -#define STRING_BSR_UNICODE_RIGHTPAR "BSR_UNICODE)" -#define STRING_UTF8_RIGHTPAR "UTF8)" -#define STRING_UTF16_RIGHTPAR "UTF16)" -#define STRING_UTF32_RIGHTPAR "UTF32)" -#define STRING_UTF_RIGHTPAR "UTF)" -#define STRING_UCP_RIGHTPAR "UCP)" -#define STRING_NO_AUTO_POSSESS_RIGHTPAR "NO_AUTO_POSSESS)" -#define STRING_NO_START_OPT_RIGHTPAR "NO_START_OPT)" -#define STRING_LIMIT_MATCH_EQ "LIMIT_MATCH=" -#define STRING_LIMIT_RECURSION_EQ "LIMIT_RECURSION=" - -#else /* SUPPORT_UTF */ - -/* UTF-8 support is enabled; always use UTF-8 (=ASCII) character codes. This -works in both modes non-EBCDIC platforms, and on EBCDIC platforms in UTF-8 mode -only. */ - -#define CHAR_HT '\011' -#define CHAR_VT '\013' -#define CHAR_FF '\014' -#define CHAR_CR '\015' -#define CHAR_LF '\012' -#define CHAR_NL CHAR_LF -#define CHAR_NEL ((unsigned char)'\x85') -#define CHAR_BS '\010' -#define CHAR_BEL '\007' -#define CHAR_ESC '\033' -#define CHAR_DEL '\177' - -#define CHAR_NULL '\0' -#define CHAR_SPACE '\040' -#define CHAR_EXCLAMATION_MARK '\041' -#define CHAR_QUOTATION_MARK '\042' -#define CHAR_NUMBER_SIGN '\043' -#define CHAR_DOLLAR_SIGN '\044' -#define CHAR_PERCENT_SIGN '\045' -#define CHAR_AMPERSAND '\046' -#define CHAR_APOSTROPHE '\047' -#define CHAR_LEFT_PARENTHESIS '\050' -#define CHAR_RIGHT_PARENTHESIS '\051' -#define CHAR_ASTERISK '\052' -#define CHAR_PLUS '\053' -#define CHAR_COMMA '\054' -#define CHAR_MINUS '\055' -#define CHAR_DOT '\056' -#define CHAR_SLASH '\057' -#define CHAR_0 '\060' -#define CHAR_1 '\061' -#define CHAR_2 '\062' -#define CHAR_3 '\063' -#define CHAR_4 '\064' -#define CHAR_5 '\065' -#define CHAR_6 '\066' -#define CHAR_7 '\067' -#define CHAR_8 '\070' -#define CHAR_9 '\071' -#define CHAR_COLON '\072' -#define CHAR_SEMICOLON '\073' -#define CHAR_LESS_THAN_SIGN '\074' -#define CHAR_EQUALS_SIGN '\075' -#define CHAR_GREATER_THAN_SIGN '\076' -#define CHAR_QUESTION_MARK '\077' -#define CHAR_COMMERCIAL_AT '\100' -#define CHAR_A '\101' -#define CHAR_B '\102' -#define CHAR_C '\103' -#define CHAR_D '\104' -#define CHAR_E '\105' -#define CHAR_F '\106' -#define CHAR_G '\107' -#define CHAR_H '\110' -#define CHAR_I '\111' -#define CHAR_J '\112' -#define CHAR_K '\113' -#define CHAR_L '\114' -#define CHAR_M '\115' -#define CHAR_N '\116' -#define CHAR_O '\117' -#define CHAR_P '\120' -#define CHAR_Q '\121' -#define CHAR_R '\122' -#define CHAR_S '\123' -#define CHAR_T '\124' -#define CHAR_U '\125' -#define CHAR_V '\126' -#define CHAR_W '\127' -#define CHAR_X '\130' -#define CHAR_Y '\131' -#define CHAR_Z '\132' -#define CHAR_LEFT_SQUARE_BRACKET '\133' -#define CHAR_BACKSLASH '\134' -#define CHAR_RIGHT_SQUARE_BRACKET '\135' -#define CHAR_CIRCUMFLEX_ACCENT '\136' -#define CHAR_UNDERSCORE '\137' -#define CHAR_GRAVE_ACCENT '\140' -#define CHAR_a '\141' -#define CHAR_b '\142' -#define CHAR_c '\143' -#define CHAR_d '\144' -#define CHAR_e '\145' -#define CHAR_f '\146' -#define CHAR_g '\147' -#define CHAR_h '\150' -#define CHAR_i '\151' -#define CHAR_j '\152' -#define CHAR_k '\153' -#define CHAR_l '\154' -#define CHAR_m '\155' -#define CHAR_n '\156' -#define CHAR_o '\157' -#define CHAR_p '\160' -#define CHAR_q '\161' -#define CHAR_r '\162' -#define CHAR_s '\163' -#define CHAR_t '\164' -#define CHAR_u '\165' -#define CHAR_v '\166' -#define CHAR_w '\167' -#define CHAR_x '\170' -#define CHAR_y '\171' -#define CHAR_z '\172' -#define CHAR_LEFT_CURLY_BRACKET '\173' -#define CHAR_VERTICAL_LINE '\174' -#define CHAR_RIGHT_CURLY_BRACKET '\175' -#define CHAR_TILDE '\176' -#define CHAR_NBSP ((unsigned char)'\xa0') - -#define STR_HT "\011" -#define STR_VT "\013" -#define STR_FF "\014" -#define STR_CR "\015" -#define STR_NL "\012" -#define STR_BS "\010" -#define STR_BEL "\007" -#define STR_ESC "\033" -#define STR_DEL "\177" - -#define STR_SPACE "\040" -#define STR_EXCLAMATION_MARK "\041" -#define STR_QUOTATION_MARK "\042" -#define STR_NUMBER_SIGN "\043" -#define STR_DOLLAR_SIGN "\044" -#define STR_PERCENT_SIGN "\045" -#define STR_AMPERSAND "\046" -#define STR_APOSTROPHE "\047" -#define STR_LEFT_PARENTHESIS "\050" -#define STR_RIGHT_PARENTHESIS "\051" -#define STR_ASTERISK "\052" -#define STR_PLUS "\053" -#define STR_COMMA "\054" -#define STR_MINUS "\055" -#define STR_DOT "\056" -#define STR_SLASH "\057" -#define STR_0 "\060" -#define STR_1 "\061" -#define STR_2 "\062" -#define STR_3 "\063" -#define STR_4 "\064" -#define STR_5 "\065" -#define STR_6 "\066" -#define STR_7 "\067" -#define STR_8 "\070" -#define STR_9 "\071" -#define STR_COLON "\072" -#define STR_SEMICOLON "\073" -#define STR_LESS_THAN_SIGN "\074" -#define STR_EQUALS_SIGN "\075" -#define STR_GREATER_THAN_SIGN "\076" -#define STR_QUESTION_MARK "\077" -#define STR_COMMERCIAL_AT "\100" -#define STR_A "\101" -#define STR_B "\102" -#define STR_C "\103" -#define STR_D "\104" -#define STR_E "\105" -#define STR_F "\106" -#define STR_G "\107" -#define STR_H "\110" -#define STR_I "\111" -#define STR_J "\112" -#define STR_K "\113" -#define STR_L "\114" -#define STR_M "\115" -#define STR_N "\116" -#define STR_O "\117" -#define STR_P "\120" -#define STR_Q "\121" -#define STR_R "\122" -#define STR_S "\123" -#define STR_T "\124" -#define STR_U "\125" -#define STR_V "\126" -#define STR_W "\127" -#define STR_X "\130" -#define STR_Y "\131" -#define STR_Z "\132" -#define STR_LEFT_SQUARE_BRACKET "\133" -#define STR_BACKSLASH "\134" -#define STR_RIGHT_SQUARE_BRACKET "\135" -#define STR_CIRCUMFLEX_ACCENT "\136" -#define STR_UNDERSCORE "\137" -#define STR_GRAVE_ACCENT "\140" -#define STR_a "\141" -#define STR_b "\142" -#define STR_c "\143" -#define STR_d "\144" -#define STR_e "\145" -#define STR_f "\146" -#define STR_g "\147" -#define STR_h "\150" -#define STR_i "\151" -#define STR_j "\152" -#define STR_k "\153" -#define STR_l "\154" -#define STR_m "\155" -#define STR_n "\156" -#define STR_o "\157" -#define STR_p "\160" -#define STR_q "\161" -#define STR_r "\162" -#define STR_s "\163" -#define STR_t "\164" -#define STR_u "\165" -#define STR_v "\166" -#define STR_w "\167" -#define STR_x "\170" -#define STR_y "\171" -#define STR_z "\172" -#define STR_LEFT_CURLY_BRACKET "\173" -#define STR_VERTICAL_LINE "\174" -#define STR_RIGHT_CURLY_BRACKET "\175" -#define STR_TILDE "\176" - -#define STRING_ACCEPT0 STR_A STR_C STR_C STR_E STR_P STR_T "\0" -#define STRING_COMMIT0 STR_C STR_O STR_M STR_M STR_I STR_T "\0" -#define STRING_F0 STR_F "\0" -#define STRING_FAIL0 STR_F STR_A STR_I STR_L "\0" -#define STRING_MARK0 STR_M STR_A STR_R STR_K "\0" -#define STRING_PRUNE0 STR_P STR_R STR_U STR_N STR_E "\0" -#define STRING_SKIP0 STR_S STR_K STR_I STR_P "\0" -#define STRING_THEN STR_T STR_H STR_E STR_N - -#define STRING_alpha0 STR_a STR_l STR_p STR_h STR_a "\0" -#define STRING_lower0 STR_l STR_o STR_w STR_e STR_r "\0" -#define STRING_upper0 STR_u STR_p STR_p STR_e STR_r "\0" -#define STRING_alnum0 STR_a STR_l STR_n STR_u STR_m "\0" -#define STRING_ascii0 STR_a STR_s STR_c STR_i STR_i "\0" -#define STRING_blank0 STR_b STR_l STR_a STR_n STR_k "\0" -#define STRING_cntrl0 STR_c STR_n STR_t STR_r STR_l "\0" -#define STRING_digit0 STR_d STR_i STR_g STR_i STR_t "\0" -#define STRING_graph0 STR_g STR_r STR_a STR_p STR_h "\0" -#define STRING_print0 STR_p STR_r STR_i STR_n STR_t "\0" -#define STRING_punct0 STR_p STR_u STR_n STR_c STR_t "\0" -#define STRING_space0 STR_s STR_p STR_a STR_c STR_e "\0" -#define STRING_word0 STR_w STR_o STR_r STR_d "\0" -#define STRING_xdigit STR_x STR_d STR_i STR_g STR_i STR_t - -#define STRING_DEFINE STR_D STR_E STR_F STR_I STR_N STR_E -#define STRING_WEIRD_STARTWORD STR_LEFT_SQUARE_BRACKET STR_COLON STR_LESS_THAN_SIGN STR_COLON STR_RIGHT_SQUARE_BRACKET STR_RIGHT_SQUARE_BRACKET -#define STRING_WEIRD_ENDWORD STR_LEFT_SQUARE_BRACKET STR_COLON STR_GREATER_THAN_SIGN STR_COLON STR_RIGHT_SQUARE_BRACKET STR_RIGHT_SQUARE_BRACKET - -#define STRING_CR_RIGHTPAR STR_C STR_R STR_RIGHT_PARENTHESIS -#define STRING_LF_RIGHTPAR STR_L STR_F STR_RIGHT_PARENTHESIS -#define STRING_CRLF_RIGHTPAR STR_C STR_R STR_L STR_F STR_RIGHT_PARENTHESIS -#define STRING_ANY_RIGHTPAR STR_A STR_N STR_Y STR_RIGHT_PARENTHESIS -#define STRING_ANYCRLF_RIGHTPAR STR_A STR_N STR_Y STR_C STR_R STR_L STR_F STR_RIGHT_PARENTHESIS -#define STRING_BSR_ANYCRLF_RIGHTPAR STR_B STR_S STR_R STR_UNDERSCORE STR_A STR_N STR_Y STR_C STR_R STR_L STR_F STR_RIGHT_PARENTHESIS -#define STRING_BSR_UNICODE_RIGHTPAR STR_B STR_S STR_R STR_UNDERSCORE STR_U STR_N STR_I STR_C STR_O STR_D STR_E STR_RIGHT_PARENTHESIS -#define STRING_UTF8_RIGHTPAR STR_U STR_T STR_F STR_8 STR_RIGHT_PARENTHESIS -#define STRING_UTF16_RIGHTPAR STR_U STR_T STR_F STR_1 STR_6 STR_RIGHT_PARENTHESIS -#define STRING_UTF32_RIGHTPAR STR_U STR_T STR_F STR_3 STR_2 STR_RIGHT_PARENTHESIS -#define STRING_UTF_RIGHTPAR STR_U STR_T STR_F STR_RIGHT_PARENTHESIS -#define STRING_UCP_RIGHTPAR STR_U STR_C STR_P STR_RIGHT_PARENTHESIS -#define STRING_NO_AUTO_POSSESS_RIGHTPAR STR_N STR_O STR_UNDERSCORE STR_A STR_U STR_T STR_O STR_UNDERSCORE STR_P STR_O STR_S STR_S STR_E STR_S STR_S STR_RIGHT_PARENTHESIS -#define STRING_NO_START_OPT_RIGHTPAR STR_N STR_O STR_UNDERSCORE STR_S STR_T STR_A STR_R STR_T STR_UNDERSCORE STR_O STR_P STR_T STR_RIGHT_PARENTHESIS -#define STRING_LIMIT_MATCH_EQ STR_L STR_I STR_M STR_I STR_T STR_UNDERSCORE STR_M STR_A STR_T STR_C STR_H STR_EQUALS_SIGN -#define STRING_LIMIT_RECURSION_EQ STR_L STR_I STR_M STR_I STR_T STR_UNDERSCORE STR_R STR_E STR_C STR_U STR_R STR_S STR_I STR_O STR_N STR_EQUALS_SIGN - -#endif /* SUPPORT_UTF */ - -/* Escape items that are just an encoding of a particular data value. */ - -#ifndef ESC_a -#define ESC_a CHAR_BEL -#endif - -#ifndef ESC_e -#define ESC_e CHAR_ESC -#endif - -#ifndef ESC_f -#define ESC_f CHAR_FF -#endif - -#ifndef ESC_n -#define ESC_n CHAR_LF -#endif - -#ifndef ESC_r -#define ESC_r CHAR_CR -#endif - -/* We can't officially use ESC_t because it is a POSIX reserved identifier -(presumably because of all the others like size_t). */ - -#ifndef ESC_tee -#define ESC_tee CHAR_HT -#endif - -/* Codes for different types of Unicode property */ - -#define PT_ANY 0 /* Any property - matches all chars */ -#define PT_LAMP 1 /* L& - the union of Lu, Ll, Lt */ -#define PT_GC 2 /* Specified general characteristic (e.g. L) */ -#define PT_PC 3 /* Specified particular characteristic (e.g. Lu) */ -#define PT_SC 4 /* Script (e.g. Han) */ -#define PT_ALNUM 5 /* Alphanumeric - the union of L and N */ -#define PT_SPACE 6 /* Perl space - Z plus 9,10,12,13 */ -#define PT_PXSPACE 7 /* POSIX space - Z plus 9,10,11,12,13 */ -#define PT_WORD 8 /* Word - L plus N plus underscore */ -#define PT_CLIST 9 /* Pseudo-property: match character list */ -#define PT_UCNC 10 /* Universal Character nameable character */ -#define PT_TABSIZE 11 /* Size of square table for autopossessify tests */ - -/* The following special properties are used only in XCLASS items, when POSIX -classes are specified and PCRE_UCP is set - in other words, for Unicode -handling of these classes. They are not available via the \p or \P escapes like -those in the above list, and so they do not take part in the autopossessifying -table. */ - -#define PT_PXGRAPH 11 /* [:graph:] - characters that mark the paper */ -#define PT_PXPRINT 12 /* [:print:] - [:graph:] plus non-control spaces */ -#define PT_PXPUNCT 13 /* [:punct:] - punctuation characters */ - -/* Flag bits and data types for the extended class (OP_XCLASS) for classes that -contain characters with values greater than 255. */ - -#define XCL_NOT 0x01 /* Flag: this is a negative class */ -#define XCL_MAP 0x02 /* Flag: a 32-byte map is present */ -#define XCL_HASPROP 0x04 /* Flag: property checks are present. */ - -#define XCL_END 0 /* Marks end of individual items */ -#define XCL_SINGLE 1 /* Single item (one multibyte char) follows */ -#define XCL_RANGE 2 /* A range (two multibyte chars) follows */ -#define XCL_PROP 3 /* Unicode property (2-byte property code follows) */ -#define XCL_NOTPROP 4 /* Unicode inverted property (ditto) */ - -/* These are escaped items that aren't just an encoding of a particular data -value such as \n. They must have non-zero values, as check_escape() returns 0 -for a data character. Also, they must appear in the same order as in the -opcode definitions below, up to ESC_z. There's a dummy for OP_ALLANY because it -corresponds to "." in DOTALL mode rather than an escape sequence. It is also -used for [^] in JavaScript compatibility mode, and for \C in non-utf mode. In -non-DOTALL mode, "." behaves like \N. - -The special values ESC_DU, ESC_du, etc. are used instead of ESC_D, ESC_d, etc. -when PCRE_UCP is set and replacement of \d etc by \p sequences is required. -They must be contiguous, and remain in order so that the replacements can be -looked up from a table. - -Negative numbers are used to encode a backreference (\1, \2, \3, etc.) in -check_escape(). There are two tests in the code for an escape -greater than ESC_b and less than ESC_Z to detect the types that may be -repeated. These are the types that consume characters. If any new escapes are -put in between that don't consume a character, that code will have to change. -*/ - -enum { ESC_A = 1, ESC_G, ESC_K, ESC_B, ESC_b, ESC_D, ESC_d, ESC_S, ESC_s, - ESC_W, ESC_w, ESC_N, ESC_dum, ESC_C, ESC_P, ESC_p, ESC_R, ESC_H, - ESC_h, ESC_V, ESC_v, ESC_X, ESC_Z, ESC_z, - ESC_E, ESC_Q, ESC_g, ESC_k, - ESC_DU, ESC_du, ESC_SU, ESC_su, ESC_WU, ESC_wu }; - - -/********************** Opcode definitions ******************/ - -/****** NOTE NOTE NOTE ****** - -Starting from 1 (i.e. after OP_END), the values up to OP_EOD must correspond in -order to the list of escapes immediately above. Furthermore, values up to -OP_DOLLM must not be changed without adjusting the table called autoposstab in -pcre_compile.c - -Whenever this list is updated, the two macro definitions that follow must be -updated to match. The possessification table called "opcode_possessify" in -pcre_compile.c must also be updated, and also the tables called "coptable" -and "poptable" in pcre_dfa_exec.c. - -****** NOTE NOTE NOTE ******/ - - -/* The values between FIRST_AUTOTAB_OP and LAST_AUTOTAB_RIGHT_OP, inclusive, -are used in a table for deciding whether a repeated character type can be -auto-possessified. */ - -#define FIRST_AUTOTAB_OP OP_NOT_DIGIT -#define LAST_AUTOTAB_LEFT_OP OP_EXTUNI -#define LAST_AUTOTAB_RIGHT_OP OP_DOLLM - -enum { - OP_END, /* 0 End of pattern */ - - /* Values corresponding to backslashed metacharacters */ - - OP_SOD, /* 1 Start of data: \A */ - OP_SOM, /* 2 Start of match (subject + offset): \G */ - OP_SET_SOM, /* 3 Set start of match (\K) */ - OP_NOT_WORD_BOUNDARY, /* 4 \B */ - OP_WORD_BOUNDARY, /* 5 \b */ - OP_NOT_DIGIT, /* 6 \D */ - OP_DIGIT, /* 7 \d */ - OP_NOT_WHITESPACE, /* 8 \S */ - OP_WHITESPACE, /* 9 \s */ - OP_NOT_WORDCHAR, /* 10 \W */ - OP_WORDCHAR, /* 11 \w */ - - OP_ANY, /* 12 Match any character except newline (\N) */ - OP_ALLANY, /* 13 Match any character */ - OP_ANYBYTE, /* 14 Match any byte (\C); different to OP_ANY for UTF-8 */ - OP_NOTPROP, /* 15 \P (not Unicode property) */ - OP_PROP, /* 16 \p (Unicode property) */ - OP_ANYNL, /* 17 \R (any newline sequence) */ - OP_NOT_HSPACE, /* 18 \H (not horizontal whitespace) */ - OP_HSPACE, /* 19 \h (horizontal whitespace) */ - OP_NOT_VSPACE, /* 20 \V (not vertical whitespace) */ - OP_VSPACE, /* 21 \v (vertical whitespace) */ - OP_EXTUNI, /* 22 \X (extended Unicode sequence */ - OP_EODN, /* 23 End of data or \n at end of data (\Z) */ - OP_EOD, /* 24 End of data (\z) */ - - /* Line end assertions */ - - OP_DOLL, /* 25 End of line - not multiline */ - OP_DOLLM, /* 26 End of line - multiline */ - OP_CIRC, /* 27 Start of line - not multiline */ - OP_CIRCM, /* 28 Start of line - multiline */ - - /* Single characters; caseful must precede the caseless ones */ - - OP_CHAR, /* 29 Match one character, casefully */ - OP_CHARI, /* 30 Match one character, caselessly */ - OP_NOT, /* 31 Match one character, not the given one, casefully */ - OP_NOTI, /* 32 Match one character, not the given one, caselessly */ - - /* The following sets of 13 opcodes must always be kept in step because - the offset from the first one is used to generate the others. */ - - /* Repeated characters; caseful must precede the caseless ones */ - - OP_STAR, /* 33 The maximizing and minimizing versions of */ - OP_MINSTAR, /* 34 these six opcodes must come in pairs, with */ - OP_PLUS, /* 35 the minimizing one second. */ - OP_MINPLUS, /* 36 */ - OP_QUERY, /* 37 */ - OP_MINQUERY, /* 38 */ - - OP_UPTO, /* 39 From 0 to n matches of one character, caseful*/ - OP_MINUPTO, /* 40 */ - OP_EXACT, /* 41 Exactly n matches */ - - OP_POSSTAR, /* 42 Possessified star, caseful */ - OP_POSPLUS, /* 43 Possessified plus, caseful */ - OP_POSQUERY, /* 44 Posesssified query, caseful */ - OP_POSUPTO, /* 45 Possessified upto, caseful */ - - /* Repeated characters; caseless must follow the caseful ones */ - - OP_STARI, /* 46 */ - OP_MINSTARI, /* 47 */ - OP_PLUSI, /* 48 */ - OP_MINPLUSI, /* 49 */ - OP_QUERYI, /* 50 */ - OP_MINQUERYI, /* 51 */ - - OP_UPTOI, /* 52 From 0 to n matches of one character, caseless */ - OP_MINUPTOI, /* 53 */ - OP_EXACTI, /* 54 */ - - OP_POSSTARI, /* 55 Possessified star, caseless */ - OP_POSPLUSI, /* 56 Possessified plus, caseless */ - OP_POSQUERYI, /* 57 Posesssified query, caseless */ - OP_POSUPTOI, /* 58 Possessified upto, caseless */ - - /* The negated ones must follow the non-negated ones, and match them */ - /* Negated repeated character, caseful; must precede the caseless ones */ - - OP_NOTSTAR, /* 59 The maximizing and minimizing versions of */ - OP_NOTMINSTAR, /* 60 these six opcodes must come in pairs, with */ - OP_NOTPLUS, /* 61 the minimizing one second. They must be in */ - OP_NOTMINPLUS, /* 62 exactly the same order as those above. */ - OP_NOTQUERY, /* 63 */ - OP_NOTMINQUERY, /* 64 */ - - OP_NOTUPTO, /* 65 From 0 to n matches, caseful */ - OP_NOTMINUPTO, /* 66 */ - OP_NOTEXACT, /* 67 Exactly n matches */ - - OP_NOTPOSSTAR, /* 68 Possessified versions, caseful */ - OP_NOTPOSPLUS, /* 69 */ - OP_NOTPOSQUERY, /* 70 */ - OP_NOTPOSUPTO, /* 71 */ - - /* Negated repeated character, caseless; must follow the caseful ones */ - - OP_NOTSTARI, /* 72 */ - OP_NOTMINSTARI, /* 73 */ - OP_NOTPLUSI, /* 74 */ - OP_NOTMINPLUSI, /* 75 */ - OP_NOTQUERYI, /* 76 */ - OP_NOTMINQUERYI, /* 77 */ - - OP_NOTUPTOI, /* 78 From 0 to n matches, caseless */ - OP_NOTMINUPTOI, /* 79 */ - OP_NOTEXACTI, /* 80 Exactly n matches */ - - OP_NOTPOSSTARI, /* 81 Possessified versions, caseless */ - OP_NOTPOSPLUSI, /* 82 */ - OP_NOTPOSQUERYI, /* 83 */ - OP_NOTPOSUPTOI, /* 84 */ - - /* Character types */ - - OP_TYPESTAR, /* 85 The maximizing and minimizing versions of */ - OP_TYPEMINSTAR, /* 86 these six opcodes must come in pairs, with */ - OP_TYPEPLUS, /* 87 the minimizing one second. These codes must */ - OP_TYPEMINPLUS, /* 88 be in exactly the same order as those above. */ - OP_TYPEQUERY, /* 89 */ - OP_TYPEMINQUERY, /* 90 */ - - OP_TYPEUPTO, /* 91 From 0 to n matches */ - OP_TYPEMINUPTO, /* 92 */ - OP_TYPEEXACT, /* 93 Exactly n matches */ - - OP_TYPEPOSSTAR, /* 94 Possessified versions */ - OP_TYPEPOSPLUS, /* 95 */ - OP_TYPEPOSQUERY, /* 96 */ - OP_TYPEPOSUPTO, /* 97 */ - - /* These are used for character classes and back references; only the - first six are the same as the sets above. */ - - OP_CRSTAR, /* 98 The maximizing and minimizing versions of */ - OP_CRMINSTAR, /* 99 all these opcodes must come in pairs, with */ - OP_CRPLUS, /* 100 the minimizing one second. These codes must */ - OP_CRMINPLUS, /* 101 be in exactly the same order as those above. */ - OP_CRQUERY, /* 102 */ - OP_CRMINQUERY, /* 103 */ - - OP_CRRANGE, /* 104 These are different to the three sets above. */ - OP_CRMINRANGE, /* 105 */ - - OP_CRPOSSTAR, /* 106 Possessified versions */ - OP_CRPOSPLUS, /* 107 */ - OP_CRPOSQUERY, /* 108 */ - OP_CRPOSRANGE, /* 109 */ - - /* End of quantifier opcodes */ - - OP_CLASS, /* 110 Match a character class, chars < 256 only */ - OP_NCLASS, /* 111 Same, but the bitmap was created from a negative - class - the difference is relevant only when a - character > 255 is encountered. */ - OP_XCLASS, /* 112 Extended class for handling > 255 chars within the - class. This does both positive and negative. */ - OP_REF, /* 113 Match a back reference, casefully */ - OP_REFI, /* 114 Match a back reference, caselessly */ - OP_DNREF, /* 115 Match a duplicate name backref, casefully */ - OP_DNREFI, /* 116 Match a duplicate name backref, caselessly */ - OP_RECURSE, /* 117 Match a numbered subpattern (possibly recursive) */ - OP_CALLOUT, /* 118 Call out to external function if provided */ - - OP_ALT, /* 119 Start of alternation */ - OP_KET, /* 120 End of group that doesn't have an unbounded repeat */ - OP_KETRMAX, /* 121 These two must remain together and in this */ - OP_KETRMIN, /* 122 order. They are for groups the repeat for ever. */ - OP_KETRPOS, /* 123 Possessive unlimited repeat. */ - - /* The assertions must come before BRA, CBRA, ONCE, and COND, and the four - asserts must remain in order. */ - - OP_REVERSE, /* 124 Move pointer back - used in lookbehind assertions */ - OP_ASSERT, /* 125 Positive lookahead */ - OP_ASSERT_NOT, /* 126 Negative lookahead */ - OP_ASSERTBACK, /* 127 Positive lookbehind */ - OP_ASSERTBACK_NOT, /* 128 Negative lookbehind */ - - /* ONCE, ONCE_NC, BRA, BRAPOS, CBRA, CBRAPOS, and COND must come immediately - after the assertions, with ONCE first, as there's a test for >= ONCE for a - subpattern that isn't an assertion. The POS versions must immediately follow - the non-POS versions in each case. */ - - OP_ONCE, /* 129 Atomic group, contains captures */ - OP_ONCE_NC, /* 130 Atomic group containing no captures */ - OP_BRA, /* 131 Start of non-capturing bracket */ - OP_BRAPOS, /* 132 Ditto, with unlimited, possessive repeat */ - OP_CBRA, /* 133 Start of capturing bracket */ - OP_CBRAPOS, /* 134 Ditto, with unlimited, possessive repeat */ - OP_COND, /* 135 Conditional group */ - - /* These five must follow the previous five, in the same order. There's a - check for >= SBRA to distinguish the two sets. */ - - OP_SBRA, /* 136 Start of non-capturing bracket, check empty */ - OP_SBRAPOS, /* 137 Ditto, with unlimited, possessive repeat */ - OP_SCBRA, /* 138 Start of capturing bracket, check empty */ - OP_SCBRAPOS, /* 139 Ditto, with unlimited, possessive repeat */ - OP_SCOND, /* 140 Conditional group, check empty */ - - /* The next two pairs must (respectively) be kept together. */ - - OP_CREF, /* 141 Used to hold a capture number as condition */ - OP_DNCREF, /* 142 Used to point to duplicate names as a condition */ - OP_RREF, /* 143 Used to hold a recursion number as condition */ - OP_DNRREF, /* 144 Used to point to duplicate names as a condition */ - OP_DEF, /* 145 The DEFINE condition */ - - OP_BRAZERO, /* 146 These two must remain together and in this */ - OP_BRAMINZERO, /* 147 order. */ - OP_BRAPOSZERO, /* 148 */ - - /* These are backtracking control verbs */ - - OP_MARK, /* 149 always has an argument */ - OP_PRUNE, /* 150 */ - OP_PRUNE_ARG, /* 151 same, but with argument */ - OP_SKIP, /* 152 */ - OP_SKIP_ARG, /* 153 same, but with argument */ - OP_THEN, /* 154 */ - OP_THEN_ARG, /* 155 same, but with argument */ - OP_COMMIT, /* 156 */ - - /* These are forced failure and success verbs */ - - OP_FAIL, /* 157 */ - OP_ACCEPT, /* 158 */ - OP_ASSERT_ACCEPT, /* 159 Used inside assertions */ - OP_CLOSE, /* 160 Used before OP_ACCEPT to close open captures */ - - /* This is used to skip a subpattern with a {0} quantifier */ - - OP_SKIPZERO, /* 161 */ - - /* This is not an opcode, but is used to check that tables indexed by opcode - are the correct length, in order to catch updating errors - there have been - some in the past. */ - - OP_TABLE_LENGTH -}; - -/* *** NOTE NOTE NOTE *** Whenever the list above is updated, the two macro -definitions that follow must also be updated to match. There are also tables -called "opcode_possessify" in pcre_compile.c and "coptable" and "poptable" in -pcre_dfa_exec.c that must be updated. */ - - -/* This macro defines textual names for all the opcodes. These are used only -for debugging, and some of them are only partial names. The macro is referenced -only in pcre_printint.c, which fills out the full names in many cases (and in -some cases doesn't actually use these names at all). */ - -#define OP_NAME_LIST \ - "End", "\\A", "\\G", "\\K", "\\B", "\\b", "\\D", "\\d", \ - "\\S", "\\s", "\\W", "\\w", "Any", "AllAny", "Anybyte", \ - "notprop", "prop", "\\R", "\\H", "\\h", "\\V", "\\v", \ - "extuni", "\\Z", "\\z", \ - "$", "$", "^", "^", "char", "chari", "not", "noti", \ - "*", "*?", "+", "+?", "?", "??", \ - "{", "{", "{", \ - "*+","++", "?+", "{", \ - "*", "*?", "+", "+?", "?", "??", \ - "{", "{", "{", \ - "*+","++", "?+", "{", \ - "*", "*?", "+", "+?", "?", "??", \ - "{", "{", "{", \ - "*+","++", "?+", "{", \ - "*", "*?", "+", "+?", "?", "??", \ - "{", "{", "{", \ - "*+","++", "?+", "{", \ - "*", "*?", "+", "+?", "?", "??", "{", "{", "{", \ - "*+","++", "?+", "{", \ - "*", "*?", "+", "+?", "?", "??", "{", "{", \ - "*+","++", "?+", "{", \ - "class", "nclass", "xclass", "Ref", "Refi", "DnRef", "DnRefi", \ - "Recurse", "Callout", \ - "Alt", "Ket", "KetRmax", "KetRmin", "KetRpos", \ - "Reverse", "Assert", "Assert not", "AssertB", "AssertB not", \ - "Once", "Once_NC", \ - "Bra", "BraPos", "CBra", "CBraPos", \ - "Cond", \ - "SBra", "SBraPos", "SCBra", "SCBraPos", \ - "SCond", \ - "Cond ref", "Cond dnref", "Cond rec", "Cond dnrec", "Cond def", \ - "Brazero", "Braminzero", "Braposzero", \ - "*MARK", "*PRUNE", "*PRUNE", "*SKIP", "*SKIP", \ - "*THEN", "*THEN", "*COMMIT", "*FAIL", \ - "*ACCEPT", "*ASSERT_ACCEPT", \ - "Close", "Skip zero" - - -/* This macro defines the length of fixed length operations in the compiled -regex. The lengths are used when searching for specific things, and also in the -debugging printing of a compiled regex. We use a macro so that it can be -defined close to the definitions of the opcodes themselves. - -As things have been extended, some of these are no longer fixed lenths, but are -minima instead. For example, the length of a single-character repeat may vary -in UTF-8 mode. The code that uses this table must know about such things. */ - -#define OP_LENGTHS \ - 1, /* End */ \ - 1, 1, 1, 1, 1, /* \A, \G, \K, \B, \b */ \ - 1, 1, 1, 1, 1, 1, /* \D, \d, \S, \s, \W, \w */ \ - 1, 1, 1, /* Any, AllAny, Anybyte */ \ - 3, 3, /* \P, \p */ \ - 1, 1, 1, 1, 1, /* \R, \H, \h, \V, \v */ \ - 1, /* \X */ \ - 1, 1, 1, 1, 1, 1, /* \Z, \z, $, $M ^, ^M */ \ - 2, /* Char - the minimum length */ \ - 2, /* Chari - the minimum length */ \ - 2, /* not */ \ - 2, /* noti */ \ - /* Positive single-char repeats ** These are */ \ - 2, 2, 2, 2, 2, 2, /* *, *?, +, +?, ?, ?? ** minima in */ \ - 2+IMM2_SIZE, 2+IMM2_SIZE, /* upto, minupto ** mode */ \ - 2+IMM2_SIZE, /* exact */ \ - 2, 2, 2, 2+IMM2_SIZE, /* *+, ++, ?+, upto+ */ \ - 2, 2, 2, 2, 2, 2, /* *I, *?I, +I, +?I, ?I, ??I ** UTF-8 */ \ - 2+IMM2_SIZE, 2+IMM2_SIZE, /* upto I, minupto I */ \ - 2+IMM2_SIZE, /* exact I */ \ - 2, 2, 2, 2+IMM2_SIZE, /* *+I, ++I, ?+I, upto+I */ \ - /* Negative single-char repeats - only for chars < 256 */ \ - 2, 2, 2, 2, 2, 2, /* NOT *, *?, +, +?, ?, ?? */ \ - 2+IMM2_SIZE, 2+IMM2_SIZE, /* NOT upto, minupto */ \ - 2+IMM2_SIZE, /* NOT exact */ \ - 2, 2, 2, 2+IMM2_SIZE, /* Possessive NOT *, +, ?, upto */ \ - 2, 2, 2, 2, 2, 2, /* NOT *I, *?I, +I, +?I, ?I, ??I */ \ - 2+IMM2_SIZE, 2+IMM2_SIZE, /* NOT upto I, minupto I */ \ - 2+IMM2_SIZE, /* NOT exact I */ \ - 2, 2, 2, 2+IMM2_SIZE, /* Possessive NOT *I, +I, ?I, upto I */ \ - /* Positive type repeats */ \ - 2, 2, 2, 2, 2, 2, /* Type *, *?, +, +?, ?, ?? */ \ - 2+IMM2_SIZE, 2+IMM2_SIZE, /* Type upto, minupto */ \ - 2+IMM2_SIZE, /* Type exact */ \ - 2, 2, 2, 2+IMM2_SIZE, /* Possessive *+, ++, ?+, upto+ */ \ - /* Character class & ref repeats */ \ - 1, 1, 1, 1, 1, 1, /* *, *?, +, +?, ?, ?? */ \ - 1+2*IMM2_SIZE, 1+2*IMM2_SIZE, /* CRRANGE, CRMINRANGE */ \ - 1, 1, 1, 1+2*IMM2_SIZE, /* Possessive *+, ++, ?+, CRPOSRANGE */ \ - 1+(32/sizeof(pcre_uchar)), /* CLASS */ \ - 1+(32/sizeof(pcre_uchar)), /* NCLASS */ \ - 0, /* XCLASS - variable length */ \ - 1+IMM2_SIZE, /* REF */ \ - 1+IMM2_SIZE, /* REFI */ \ - 1+2*IMM2_SIZE, /* DNREF */ \ - 1+2*IMM2_SIZE, /* DNREFI */ \ - 1+LINK_SIZE, /* RECURSE */ \ - 2+2*LINK_SIZE, /* CALLOUT */ \ - 1+LINK_SIZE, /* Alt */ \ - 1+LINK_SIZE, /* Ket */ \ - 1+LINK_SIZE, /* KetRmax */ \ - 1+LINK_SIZE, /* KetRmin */ \ - 1+LINK_SIZE, /* KetRpos */ \ - 1+LINK_SIZE, /* Reverse */ \ - 1+LINK_SIZE, /* Assert */ \ - 1+LINK_SIZE, /* Assert not */ \ - 1+LINK_SIZE, /* Assert behind */ \ - 1+LINK_SIZE, /* Assert behind not */ \ - 1+LINK_SIZE, /* ONCE */ \ - 1+LINK_SIZE, /* ONCE_NC */ \ - 1+LINK_SIZE, /* BRA */ \ - 1+LINK_SIZE, /* BRAPOS */ \ - 1+LINK_SIZE+IMM2_SIZE, /* CBRA */ \ - 1+LINK_SIZE+IMM2_SIZE, /* CBRAPOS */ \ - 1+LINK_SIZE, /* COND */ \ - 1+LINK_SIZE, /* SBRA */ \ - 1+LINK_SIZE, /* SBRAPOS */ \ - 1+LINK_SIZE+IMM2_SIZE, /* SCBRA */ \ - 1+LINK_SIZE+IMM2_SIZE, /* SCBRAPOS */ \ - 1+LINK_SIZE, /* SCOND */ \ - 1+IMM2_SIZE, 1+2*IMM2_SIZE, /* CREF, DNCREF */ \ - 1+IMM2_SIZE, 1+2*IMM2_SIZE, /* RREF, DNRREF */ \ - 1, /* DEF */ \ - 1, 1, 1, /* BRAZERO, BRAMINZERO, BRAPOSZERO */ \ - 3, 1, 3, /* MARK, PRUNE, PRUNE_ARG */ \ - 1, 3, /* SKIP, SKIP_ARG */ \ - 1, 3, /* THEN, THEN_ARG */ \ - 1, 1, 1, 1, /* COMMIT, FAIL, ACCEPT, ASSERT_ACCEPT */ \ - 1+IMM2_SIZE, 1 /* CLOSE, SKIPZERO */ - -/* A magic value for OP_RREF to indicate the "any recursion" condition. */ - -#define RREF_ANY 0xffff - -/* Compile time error code numbers. They are given names so that they can more -easily be tracked. When a new number is added, the table called eint in -pcreposix.c must be updated. */ - -enum { ERR0, ERR1, ERR2, ERR3, ERR4, ERR5, ERR6, ERR7, ERR8, ERR9, - ERR10, ERR11, ERR12, ERR13, ERR14, ERR15, ERR16, ERR17, ERR18, ERR19, - ERR20, ERR21, ERR22, ERR23, ERR24, ERR25, ERR26, ERR27, ERR28, ERR29, - ERR30, ERR31, ERR32, ERR33, ERR34, ERR35, ERR36, ERR37, ERR38, ERR39, - ERR40, ERR41, ERR42, ERR43, ERR44, ERR45, ERR46, ERR47, ERR48, ERR49, - ERR50, ERR51, ERR52, ERR53, ERR54, ERR55, ERR56, ERR57, ERR58, ERR59, - ERR60, ERR61, ERR62, ERR63, ERR64, ERR65, ERR66, ERR67, ERR68, ERR69, - ERR70, ERR71, ERR72, ERR73, ERR74, ERR75, ERR76, ERR77, ERR78, ERR79, - ERR80, ERR81, ERR82, ERR83, ERR84, ERR85, ERR86, ERR87, ERRCOUNT }; - -/* JIT compiling modes. The function list is indexed by them. */ - -enum { JIT_COMPILE, JIT_PARTIAL_SOFT_COMPILE, JIT_PARTIAL_HARD_COMPILE, - JIT_NUMBER_OF_COMPILE_MODES }; - -/* The real format of the start of the pcre block; the index of names and the -code vector run on as long as necessary after the end. We store an explicit -offset to the name table so that if a regex is compiled on one host, saved, and -then run on another where the size of pointers is different, all might still -be well. - -The size of the structure must be a multiple of 8 bytes. For the case of -compiled-on-4 and run-on-8, we include an extra pointer that is always NULL so -that there are an even number of pointers which therefore are a multiple of 8 -bytes. - -It is necessary to fork the struct for the 32 bit library, since it needs to -use pcre_uint32 for first_char and req_char. We can't put an ifdef inside the -typedef because pcretest needs access to the struct of the 8-, 16- and 32-bit -variants. - -*** WARNING *** -When new fields are added to these structures, remember to adjust the code in -pcre_byte_order.c that is concerned with swapping the byte order of the fields -when a compiled regex is reloaded on a host with different endianness. -*** WARNING *** -There is also similar byte-flipping code in pcretest.c, which is used for -testing the byte-flipping features. It must also be kept in step. -*** WARNING *** -*/ - -typedef struct real_pcre8_or_16 { - pcre_uint32 magic_number; - pcre_uint32 size; /* Total that was malloced */ - pcre_uint32 options; /* Public options */ - pcre_uint32 flags; /* Private flags */ - pcre_uint32 limit_match; /* Limit set from regex */ - pcre_uint32 limit_recursion; /* Limit set from regex */ - pcre_uint16 first_char; /* Starting character */ - pcre_uint16 req_char; /* This character must be seen */ - pcre_uint16 max_lookbehind; /* Longest lookbehind (characters) */ - pcre_uint16 top_bracket; /* Highest numbered group */ - pcre_uint16 top_backref; /* Highest numbered back reference */ - pcre_uint16 name_table_offset; /* Offset to name table that follows */ - pcre_uint16 name_entry_size; /* Size of any name items */ - pcre_uint16 name_count; /* Number of name items */ - pcre_uint16 ref_count; /* Reference count */ - pcre_uint16 dummy1; /* To ensure size is a multiple of 8 */ - pcre_uint16 dummy2; /* To ensure size is a multiple of 8 */ - pcre_uint16 dummy3; /* To ensure size is a multiple of 8 */ - const pcre_uint8 *tables; /* Pointer to tables or NULL for std */ - void *nullpad; /* NULL padding */ -} real_pcre8_or_16; - -typedef struct real_pcre8_or_16 real_pcre; -typedef struct real_pcre8_or_16 real_pcre16; - -typedef struct real_pcre32 { - pcre_uint32 magic_number; - pcre_uint32 size; /* Total that was malloced */ - pcre_uint32 options; /* Public options */ - pcre_uint32 flags; /* Private flags */ - pcre_uint32 limit_match; /* Limit set from regex */ - pcre_uint32 limit_recursion; /* Limit set from regex */ - pcre_uint32 first_char; /* Starting character */ - pcre_uint32 req_char; /* This character must be seen */ - pcre_uint16 max_lookbehind; /* Longest lookbehind (characters) */ - pcre_uint16 top_bracket; /* Highest numbered group */ - pcre_uint16 top_backref; /* Highest numbered back reference */ - pcre_uint16 name_table_offset; /* Offset to name table that follows */ - pcre_uint16 name_entry_size; /* Size of any name items */ - pcre_uint16 name_count; /* Number of name items */ - pcre_uint16 ref_count; /* Reference count */ - pcre_uint16 dummy; /* To ensure size is a multiple of 8 */ - const pcre_uint8 *tables; /* Pointer to tables or NULL for std */ - void *nullpad; /* NULL padding */ -} real_pcre32; - -#if defined COMPILE_PCRE8 -#define REAL_PCRE real_pcre -#elif defined COMPILE_PCRE16 -#define REAL_PCRE real_pcre16 -#elif defined COMPILE_PCRE32 -#define REAL_PCRE real_pcre32 -#endif - -/* Assert that the size of REAL_PCRE is divisible by 8 */ -typedef int __assert_real_pcre_size_divisible_8[(sizeof(REAL_PCRE) % 8) == 0 ? 1 : -1]; - -/* Needed in pcretest to access some fields in the real_pcre* structures - * directly. They're unified for 8/16/32 bits since the structs only differ - * after these fields; if that ever changes, need to fork those defines into - * 8/16 and 32 bit versions. */ -#define REAL_PCRE_MAGIC(re) (((REAL_PCRE*)re)->magic_number) -#define REAL_PCRE_SIZE(re) (((REAL_PCRE*)re)->size) -#define REAL_PCRE_OPTIONS(re) (((REAL_PCRE*)re)->options) -#define REAL_PCRE_FLAGS(re) (((REAL_PCRE*)re)->flags) - -/* The format of the block used to store data from pcre_study(). The same -remark (see NOTE above) about extending this structure applies. */ - -typedef struct pcre_study_data { - pcre_uint32 size; /* Total that was malloced */ - pcre_uint32 flags; /* Private flags */ - pcre_uint8 start_bits[32]; /* Starting char bits */ - pcre_uint32 minlength; /* Minimum subject length */ -} pcre_study_data; - -/* Structure for building a chain of open capturing subpatterns during -compiling, so that instructions to close them can be compiled when (*ACCEPT) is -encountered. This is also used to identify subpatterns that contain recursive -back references to themselves, so that they can be made atomic. */ - -typedef struct open_capitem { - struct open_capitem *next; /* Chain link */ - pcre_uint16 number; /* Capture number */ - pcre_uint16 flag; /* Set TRUE if recursive back ref */ -} open_capitem; - -/* Structure for building a list of named groups during the first pass of -compiling. */ - -typedef struct named_group { - const pcre_uchar *name; /* Points to the name in the pattern */ - int length; /* Length of the name */ - pcre_uint32 number; /* Group number */ -} named_group; - -/* Structure for passing "static" information around between the functions -doing the compiling, so that they are thread-safe. */ - -typedef struct compile_data { - const pcre_uint8 *lcc; /* Points to lower casing table */ - const pcre_uint8 *fcc; /* Points to case-flipping table */ - const pcre_uint8 *cbits; /* Points to character type table */ - const pcre_uint8 *ctypes; /* Points to table of type maps */ - const pcre_uchar *start_workspace;/* The start of working space */ - const pcre_uchar *start_code; /* The start of the compiled code */ - const pcre_uchar *start_pattern; /* The start of the pattern */ - const pcre_uchar *end_pattern; /* The end of the pattern */ - pcre_uchar *hwm; /* High watermark of workspace */ - open_capitem *open_caps; /* Chain of open capture items */ - named_group *named_groups; /* Points to vector in pre-compile */ - pcre_uchar *name_table; /* The name/number table */ - int names_found; /* Number of entries so far */ - int name_entry_size; /* Size of each entry */ - int named_group_list_size; /* Number of entries in the list */ - int workspace_size; /* Size of workspace */ - unsigned int bracount; /* Count of capturing parens as we compile */ - int final_bracount; /* Saved value after first pass */ - int max_lookbehind; /* Maximum lookbehind (characters) */ - int top_backref; /* Maximum back reference */ - unsigned int backref_map; /* Bitmap of low back refs */ - unsigned int namedrefcount; /* Number of backreferences by name */ - int parens_depth; /* Depth of nested parentheses */ - int assert_depth; /* Depth of nested assertions */ - pcre_uint32 external_options; /* External (initial) options */ - pcre_uint32 external_flags; /* External flag bits to be set */ - int req_varyopt; /* "After variable item" flag for reqbyte */ - BOOL had_accept; /* (*ACCEPT) encountered */ - BOOL had_pruneorskip; /* (*PRUNE) or (*SKIP) encountered */ - BOOL check_lookbehind; /* Lookbehinds need later checking */ - BOOL dupnames; /* Duplicate names exist */ - BOOL dupgroups; /* Duplicate groups exist: (?| found */ - BOOL iscondassert; /* Next assert is a condition */ - int nltype; /* Newline type */ - int nllen; /* Newline string length */ - pcre_uchar nl[4]; /* Newline string when fixed length */ -} compile_data; - -/* Structure for maintaining a chain of pointers to the currently incomplete -branches, for testing for left recursion while compiling. */ - -typedef struct branch_chain { - struct branch_chain *outer; - pcre_uchar *current_branch; -} branch_chain; - -/* Structure for mutual recursion detection. */ - -typedef struct recurse_check { - struct recurse_check *prev; - const pcre_uchar *group; -} recurse_check; - -/* Structure for items in a linked list that represents an explicit recursive -call within the pattern; used by pcre_exec(). */ - -typedef struct recursion_info { - struct recursion_info *prevrec; /* Previous recursion record (or NULL) */ - unsigned int group_num; /* Number of group that was called */ - int *offset_save; /* Pointer to start of saved offsets */ - int saved_max; /* Number of saved offsets */ - int saved_capture_last; /* Last capture number */ - PCRE_PUCHAR subject_position; /* Position at start of recursion */ -} recursion_info; - -/* A similar structure for pcre_dfa_exec(). */ - -typedef struct dfa_recursion_info { - struct dfa_recursion_info *prevrec; - int group_num; - PCRE_PUCHAR subject_position; -} dfa_recursion_info; - -/* Structure for building a chain of data for holding the values of the subject -pointer at the start of each subpattern, so as to detect when an empty string -has been matched by a subpattern - to break infinite loops; used by -pcre_exec(). */ - -typedef struct eptrblock { - struct eptrblock *epb_prev; - PCRE_PUCHAR epb_saved_eptr; -} eptrblock; - - -/* Structure for passing "static" information around between the functions -doing traditional NFA matching, so that they are thread-safe. */ - -typedef struct match_data { - unsigned long int match_call_count; /* As it says */ - unsigned long int match_limit; /* As it says */ - unsigned long int match_limit_recursion; /* As it says */ - int *offset_vector; /* Offset vector */ - int offset_end; /* One past the end */ - int offset_max; /* The maximum usable for return data */ - int nltype; /* Newline type */ - int nllen; /* Newline string length */ - int name_count; /* Number of names in name table */ - int name_entry_size; /* Size of entry in names table */ - unsigned int skip_arg_count; /* For counting SKIP_ARGs */ - unsigned int ignore_skip_arg; /* For re-run when SKIP arg name not found */ - pcre_uchar *name_table; /* Table of names */ - pcre_uchar nl[4]; /* Newline string when fixed */ - const pcre_uint8 *lcc; /* Points to lower casing table */ - const pcre_uint8 *fcc; /* Points to case-flipping table */ - const pcre_uint8 *ctypes; /* Points to table of type maps */ - BOOL notbol; /* NOTBOL flag */ - BOOL noteol; /* NOTEOL flag */ - BOOL utf; /* UTF-8 / UTF-16 flag */ - BOOL jscript_compat; /* JAVASCRIPT_COMPAT flag */ - BOOL use_ucp; /* PCRE_UCP flag */ - BOOL endonly; /* Dollar not before final \n */ - BOOL notempty; /* Empty string match not wanted */ - BOOL notempty_atstart; /* Empty string match at start not wanted */ - BOOL hitend; /* Hit the end of the subject at some point */ - BOOL bsr_anycrlf; /* \R is just any CRLF, not full Unicode */ - BOOL hasthen; /* Pattern contains (*THEN) */ - const pcre_uchar *start_code; /* For use when recursing */ - PCRE_PUCHAR start_subject; /* Start of the subject string */ - PCRE_PUCHAR end_subject; /* End of the subject string */ - PCRE_PUCHAR start_match_ptr; /* Start of matched string */ - PCRE_PUCHAR end_match_ptr; /* Subject position at end match */ - PCRE_PUCHAR start_used_ptr; /* Earliest consulted character */ - int partial; /* PARTIAL options */ - int end_offset_top; /* Highwater mark at end of match */ - pcre_int32 capture_last; /* Most recent capture number + overflow flag */ - int start_offset; /* The start offset value */ - int match_function_type; /* Set for certain special calls of MATCH() */ - eptrblock *eptrchain; /* Chain of eptrblocks for tail recursions */ - int eptrn; /* Next free eptrblock */ - recursion_info *recursive; /* Linked list of recursion data */ - void *callout_data; /* To pass back to callouts */ - const pcre_uchar *mark; /* Mark pointer to pass back on success */ - const pcre_uchar *nomatch_mark;/* Mark pointer to pass back on failure */ - const pcre_uchar *once_target; /* Where to back up to for atomic groups */ -#ifdef NO_RECURSE - void *match_frames_base; /* For remembering malloc'd frames */ -#endif -} match_data; - -/* A similar structure is used for the same purpose by the DFA matching -functions. */ - -typedef struct dfa_match_data { - const pcre_uchar *start_code; /* Start of the compiled pattern */ - const pcre_uchar *start_subject ; /* Start of the subject string */ - const pcre_uchar *end_subject; /* End of subject string */ - const pcre_uchar *start_used_ptr; /* Earliest consulted character */ - const pcre_uint8 *tables; /* Character tables */ - int start_offset; /* The start offset value */ - int moptions; /* Match options */ - int poptions; /* Pattern options */ - int nltype; /* Newline type */ - int nllen; /* Newline string length */ - pcre_uchar nl[4]; /* Newline string when fixed */ - void *callout_data; /* To pass back to callouts */ - dfa_recursion_info *recursive; /* Linked list of recursion data */ -} dfa_match_data; - -/* Bit definitions for entries in the pcre_ctypes table. */ - -#define ctype_space 0x01 -#define ctype_letter 0x02 -#define ctype_digit 0x04 -#define ctype_xdigit 0x08 -#define ctype_word 0x10 /* alphanumeric or '_' */ -#define ctype_meta 0x80 /* regexp meta char or zero (end pattern) */ - -/* Offsets for the bitmap tables in pcre_cbits. Each table contains a set -of bits for a class map. Some classes are built by combining these tables. */ - -#define cbit_space 0 /* [:space:] or \s */ -#define cbit_xdigit 32 /* [:xdigit:] */ -#define cbit_digit 64 /* [:digit:] or \d */ -#define cbit_upper 96 /* [:upper:] */ -#define cbit_lower 128 /* [:lower:] */ -#define cbit_word 160 /* [:word:] or \w */ -#define cbit_graph 192 /* [:graph:] */ -#define cbit_print 224 /* [:print:] */ -#define cbit_punct 256 /* [:punct:] */ -#define cbit_cntrl 288 /* [:cntrl:] */ -#define cbit_length 320 /* Length of the cbits table */ - -/* Offsets of the various tables from the base tables pointer, and -total length. */ - -#define lcc_offset 0 -#define fcc_offset 256 -#define cbits_offset 512 -#define ctypes_offset (cbits_offset + cbit_length) -#define tables_length (ctypes_offset + 256) - -/* Internal function and data prefixes. */ - -#if defined COMPILE_PCRE8 -#ifndef PUBL -#define PUBL(name) pcre_##name -#endif -#ifndef PRIV -#define PRIV(name) _pcre_##name -#endif -#elif defined COMPILE_PCRE16 -#ifndef PUBL -#define PUBL(name) pcre16_##name -#endif -#ifndef PRIV -#define PRIV(name) _pcre16_##name -#endif -#elif defined COMPILE_PCRE32 -#ifndef PUBL -#define PUBL(name) pcre32_##name -#endif -#ifndef PRIV -#define PRIV(name) _pcre32_##name -#endif -#else -#error Unsupported compiling mode -#endif /* COMPILE_PCRE[8|16|32] */ - -/* Layout of the UCP type table that translates property names into types and -codes. Each entry used to point directly to a name, but to reduce the number of -relocations in shared libraries, it now has an offset into a single string -instead. */ - -typedef struct { - pcre_uint16 name_offset; - pcre_uint16 type; - pcre_uint16 value; -} ucp_type_table; - - -/* Internal shared data tables. These are tables that are used by more than one -of the exported public functions. They have to be "external" in the C sense, -but are not part of the PCRE public API. The data for these tables is in the -pcre_tables.c module. */ - -#ifdef COMPILE_PCRE8 -extern const int PRIV(utf8_table1)[]; -extern const int PRIV(utf8_table1_size); -extern const int PRIV(utf8_table2)[]; -extern const int PRIV(utf8_table3)[]; -extern const pcre_uint8 PRIV(utf8_table4)[]; -#endif /* COMPILE_PCRE8 */ - -extern const char PRIV(utt_names)[]; -extern const ucp_type_table PRIV(utt)[]; -extern const int PRIV(utt_size); - -extern const pcre_uint8 PRIV(OP_lengths)[]; -extern const pcre_uint8 PRIV(default_tables)[]; - -extern const pcre_uint32 PRIV(hspace_list)[]; -extern const pcre_uint32 PRIV(vspace_list)[]; - - -/* Internal shared functions. These are functions that are used by more than -one of the exported public functions. They have to be "external" in the C -sense, but are not part of the PCRE public API. */ - -/* String comparison functions. */ -#if defined COMPILE_PCRE8 - -#define STRCMP_UC_UC(str1, str2) \ - strcmp((char *)(str1), (char *)(str2)) -#define STRCMP_UC_C8(str1, str2) \ - strcmp((char *)(str1), (str2)) -#define STRNCMP_UC_UC(str1, str2, num) \ - strncmp((char *)(str1), (char *)(str2), (num)) -#define STRNCMP_UC_C8(str1, str2, num) \ - strncmp((char *)(str1), (str2), (num)) -#define STRLEN_UC(str) strlen((const char *)str) - -#elif defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - -extern int PRIV(strcmp_uc_uc)(const pcre_uchar *, - const pcre_uchar *); -extern int PRIV(strcmp_uc_c8)(const pcre_uchar *, - const char *); -extern int PRIV(strncmp_uc_uc)(const pcre_uchar *, - const pcre_uchar *, unsigned int num); -extern int PRIV(strncmp_uc_c8)(const pcre_uchar *, - const char *, unsigned int num); -extern unsigned int PRIV(strlen_uc)(const pcre_uchar *str); - -#define STRCMP_UC_UC(str1, str2) \ - PRIV(strcmp_uc_uc)((str1), (str2)) -#define STRCMP_UC_C8(str1, str2) \ - PRIV(strcmp_uc_c8)((str1), (str2)) -#define STRNCMP_UC_UC(str1, str2, num) \ - PRIV(strncmp_uc_uc)((str1), (str2), (num)) -#define STRNCMP_UC_C8(str1, str2, num) \ - PRIV(strncmp_uc_c8)((str1), (str2), (num)) -#define STRLEN_UC(str) PRIV(strlen_uc)(str) - -#endif /* COMPILE_PCRE[8|16|32] */ - -#if defined COMPILE_PCRE8 || defined COMPILE_PCRE16 - -#define STRCMP_UC_UC_TEST(str1, str2) STRCMP_UC_UC(str1, str2) -#define STRCMP_UC_C8_TEST(str1, str2) STRCMP_UC_C8(str1, str2) - -#elif defined COMPILE_PCRE32 - -extern int PRIV(strcmp_uc_uc_utf)(const pcre_uchar *, - const pcre_uchar *); -extern int PRIV(strcmp_uc_c8_utf)(const pcre_uchar *, - const char *); - -#define STRCMP_UC_UC_TEST(str1, str2) \ - (utf ? PRIV(strcmp_uc_uc_utf)((str1), (str2)) : PRIV(strcmp_uc_uc)((str1), (str2))) -#define STRCMP_UC_C8_TEST(str1, str2) \ - (utf ? PRIV(strcmp_uc_c8_utf)((str1), (str2)) : PRIV(strcmp_uc_c8)((str1), (str2))) - -#endif /* COMPILE_PCRE[8|16|32] */ - -extern const pcre_uchar *PRIV(find_bracket)(const pcre_uchar *, BOOL, int); -extern BOOL PRIV(is_newline)(PCRE_PUCHAR, int, PCRE_PUCHAR, - int *, BOOL); -extern unsigned int PRIV(ord2utf)(pcre_uint32, pcre_uchar *); -extern int PRIV(valid_utf)(PCRE_PUCHAR, int, int *); -extern BOOL PRIV(was_newline)(PCRE_PUCHAR, int, PCRE_PUCHAR, - int *, BOOL); -extern BOOL PRIV(xclass)(pcre_uint32, const pcre_uchar *, BOOL); - -#ifdef SUPPORT_JIT -extern void PRIV(jit_compile)(const REAL_PCRE *, - PUBL(extra) *, int); -extern int PRIV(jit_exec)(const PUBL(extra) *, - const pcre_uchar *, int, int, int, int *, int); -extern void PRIV(jit_free)(void *); -extern int PRIV(jit_get_size)(void *); -extern const char* PRIV(jit_get_target)(void); -#endif - -/* Unicode character database (UCD) */ - -typedef struct { - pcre_uint8 script; /* ucp_Arabic, etc. */ - pcre_uint8 chartype; /* ucp_Cc, etc. (general categories) */ - pcre_uint8 gbprop; /* ucp_gbControl, etc. (grapheme break property) */ - pcre_uint8 caseset; /* offset to multichar other cases or zero */ - pcre_int32 other_case; /* offset to other case, or zero if none */ -} ucd_record; - -extern const pcre_uint32 PRIV(ucd_caseless_sets)[]; -extern const ucd_record PRIV(ucd_records)[]; -extern const pcre_uint8 PRIV(ucd_stage1)[]; -extern const pcre_uint16 PRIV(ucd_stage2)[]; -extern const pcre_uint32 PRIV(ucp_gentype)[]; -extern const pcre_uint32 PRIV(ucp_gbtable)[]; -#ifdef SUPPORT_JIT -extern const int PRIV(ucp_typerange)[]; -#endif - -#ifdef SUPPORT_UCP -/* UCD access macros */ - -#define UCD_BLOCK_SIZE 128 -#define GET_UCD(ch) (PRIV(ucd_records) + \ - PRIV(ucd_stage2)[PRIV(ucd_stage1)[(int)(ch) / UCD_BLOCK_SIZE] * \ - UCD_BLOCK_SIZE + (int)(ch) % UCD_BLOCK_SIZE]) - -#define UCD_CHARTYPE(ch) GET_UCD(ch)->chartype -#define UCD_SCRIPT(ch) GET_UCD(ch)->script -#define UCD_CATEGORY(ch) PRIV(ucp_gentype)[UCD_CHARTYPE(ch)] -#define UCD_GRAPHBREAK(ch) GET_UCD(ch)->gbprop -#define UCD_CASESET(ch) GET_UCD(ch)->caseset -#define UCD_OTHERCASE(ch) ((pcre_uint32)((int)ch + (int)(GET_UCD(ch)->other_case))) - -#endif /* SUPPORT_UCP */ - -#endif - -/* End of pcre_internal.h */ diff --git a/src/third_party/pcre-8.39/pcre_jit_compile.c b/src/third_party/pcre-8.39/pcre_jit_compile.c deleted file mode 100644 index 4f15a27ac28..00000000000 --- a/src/third_party/pcre-8.39/pcre_jit_compile.c +++ /dev/null @@ -1,11726 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2013 University of Cambridge - - The machine code generator part (this module) was written by Zoltan Herczeg - Copyright (c) 2010-2013 - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "pcre_internal.h" - -#if defined SUPPORT_JIT - -/* All-in-one: Since we use the JIT compiler only from here, -we just include it. This way we don't need to touch the build -system files. */ - -#define SLJIT_MALLOC(size, allocator_data) (PUBL(malloc))(size) -#define SLJIT_FREE(ptr, allocator_data) (PUBL(free))(ptr) -#define SLJIT_CONFIG_AUTO 1 -#define SLJIT_CONFIG_STATIC 1 -#define SLJIT_VERBOSE 0 -#define SLJIT_DEBUG 0 - -#include "sljit/sljitLir.c" - -#if defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED -#error Unsupported architecture -#endif - -/* Defines for debugging purposes. */ - -/* 1 - Use unoptimized capturing brackets. - 2 - Enable capture_last_ptr (includes option 1). */ -/* #define DEBUG_FORCE_UNOPTIMIZED_CBRAS 2 */ - -/* 1 - Always have a control head. */ -/* #define DEBUG_FORCE_CONTROL_HEAD 1 */ - -/* Allocate memory for the regex stack on the real machine stack. -Fast, but limited size. */ -#define MACHINE_STACK_SIZE 32768 - -/* Growth rate for stack allocated by the OS. Should be the multiply -of page size. */ -#define STACK_GROWTH_RATE 8192 - -/* Enable to check that the allocation could destroy temporaries. */ -#if defined SLJIT_DEBUG && SLJIT_DEBUG -#define DESTROY_REGISTERS 1 -#endif - -/* -Short summary about the backtracking mechanism empolyed by the jit code generator: - -The code generator follows the recursive nature of the PERL compatible regular -expressions. The basic blocks of regular expressions are condition checkers -whose execute different commands depending on the result of the condition check. -The relationship between the operators can be horizontal (concatenation) and -vertical (sub-expression) (See struct backtrack_common for more details). - - 'ab' - 'a' and 'b' regexps are concatenated - 'a+' - 'a' is the sub-expression of the '+' operator - -The condition checkers are boolean (true/false) checkers. Machine code is generated -for the checker itself and for the actions depending on the result of the checker. -The 'true' case is called as the matching path (expected path), and the other is called as -the 'backtrack' path. Branch instructions are expesive for all CPUs, so we avoid taken -branches on the matching path. - - Greedy star operator (*) : - Matching path: match happens. - Backtrack path: match failed. - Non-greedy star operator (*?) : - Matching path: no need to perform a match. - Backtrack path: match is required. - -The following example shows how the code generated for a capturing bracket -with two alternatives. Let A, B, C, D are arbirary regular expressions, and -we have the following regular expression: - - A(B|C)D - -The generated code will be the following: - - A matching path - '(' matching path (pushing arguments to the stack) - B matching path - ')' matching path (pushing arguments to the stack) - D matching path - return with successful match - - D backtrack path - ')' backtrack path (If we arrived from "C" jump to the backtrack of "C") - B backtrack path - C expected path - jump to D matching path - C backtrack path - A backtrack path - - Notice, that the order of backtrack code paths are the opposite of the fast - code paths. In this way the topmost value on the stack is always belong - to the current backtrack code path. The backtrack path must check - whether there is a next alternative. If so, it needs to jump back to - the matching path eventually. Otherwise it needs to clear out its own stack - frame and continue the execution on the backtrack code paths. -*/ - -/* -Saved stack frames: - -Atomic blocks and asserts require reloading the values of private data -when the backtrack mechanism performed. Because of OP_RECURSE, the data -are not necessarly known in compile time, thus we need a dynamic restore -mechanism. - -The stack frames are stored in a chain list, and have the following format: -([ capturing bracket offset ][ start value ][ end value ])+ ... [ 0 ] [ previous head ] - -Thus we can restore the private data to a particular point in the stack. -*/ - -typedef struct jit_arguments { - /* Pointers first. */ - struct sljit_stack *stack; - const pcre_uchar *str; - const pcre_uchar *begin; - const pcre_uchar *end; - int *offsets; - pcre_uchar *uchar_ptr; - pcre_uchar *mark_ptr; - void *callout_data; - /* Everything else after. */ - sljit_u32 limit_match; - int real_offset_count; - int offset_count; - sljit_u8 notbol; - sljit_u8 noteol; - sljit_u8 notempty; - sljit_u8 notempty_atstart; -} jit_arguments; - -typedef struct executable_functions { - void *executable_funcs[JIT_NUMBER_OF_COMPILE_MODES]; - void *read_only_data_heads[JIT_NUMBER_OF_COMPILE_MODES]; - sljit_uw executable_sizes[JIT_NUMBER_OF_COMPILE_MODES]; - PUBL(jit_callback) callback; - void *userdata; - sljit_u32 top_bracket; - sljit_u32 limit_match; -} executable_functions; - -typedef struct jump_list { - struct sljit_jump *jump; - struct jump_list *next; -} jump_list; - -typedef struct stub_list { - struct sljit_jump *start; - struct sljit_label *quit; - struct stub_list *next; -} stub_list; - -typedef struct label_addr_list { - struct sljit_label *label; - sljit_uw *update_addr; - struct label_addr_list *next; -} label_addr_list; - -enum frame_types { - no_frame = -1, - no_stack = -2 -}; - -enum control_types { - type_mark = 0, - type_then_trap = 1 -}; - -typedef int (SLJIT_CALL *jit_function)(jit_arguments *args); - -/* The following structure is the key data type for the recursive -code generator. It is allocated by compile_matchingpath, and contains -the arguments for compile_backtrackingpath. Must be the first member -of its descendants. */ -typedef struct backtrack_common { - /* Concatenation stack. */ - struct backtrack_common *prev; - jump_list *nextbacktracks; - /* Internal stack (for component operators). */ - struct backtrack_common *top; - jump_list *topbacktracks; - /* Opcode pointer. */ - pcre_uchar *cc; -} backtrack_common; - -typedef struct assert_backtrack { - backtrack_common common; - jump_list *condfailed; - /* Less than 0 if a frame is not needed. */ - int framesize; - /* Points to our private memory word on the stack. */ - int private_data_ptr; - /* For iterators. */ - struct sljit_label *matchingpath; -} assert_backtrack; - -typedef struct bracket_backtrack { - backtrack_common common; - /* Where to coninue if an alternative is successfully matched. */ - struct sljit_label *alternative_matchingpath; - /* For rmin and rmax iterators. */ - struct sljit_label *recursive_matchingpath; - /* For greedy ? operator. */ - struct sljit_label *zero_matchingpath; - /* Contains the branches of a failed condition. */ - union { - /* Both for OP_COND, OP_SCOND. */ - jump_list *condfailed; - assert_backtrack *assert; - /* For OP_ONCE. Less than 0 if not needed. */ - int framesize; - } u; - /* Points to our private memory word on the stack. */ - int private_data_ptr; -} bracket_backtrack; - -typedef struct bracketpos_backtrack { - backtrack_common common; - /* Points to our private memory word on the stack. */ - int private_data_ptr; - /* Reverting stack is needed. */ - int framesize; - /* Allocated stack size. */ - int stacksize; -} bracketpos_backtrack; - -typedef struct braminzero_backtrack { - backtrack_common common; - struct sljit_label *matchingpath; -} braminzero_backtrack; - -typedef struct char_iterator_backtrack { - backtrack_common common; - /* Next iteration. */ - struct sljit_label *matchingpath; - union { - jump_list *backtracks; - struct { - unsigned int othercasebit; - pcre_uchar chr; - BOOL enabled; - } charpos; - } u; -} char_iterator_backtrack; - -typedef struct ref_iterator_backtrack { - backtrack_common common; - /* Next iteration. */ - struct sljit_label *matchingpath; -} ref_iterator_backtrack; - -typedef struct recurse_entry { - struct recurse_entry *next; - /* Contains the function entry. */ - struct sljit_label *entry; - /* Collects the calls until the function is not created. */ - jump_list *calls; - /* Points to the starting opcode. */ - sljit_sw start; -} recurse_entry; - -typedef struct recurse_backtrack { - backtrack_common common; - BOOL inlined_pattern; -} recurse_backtrack; - -#define OP_THEN_TRAP OP_TABLE_LENGTH - -typedef struct then_trap_backtrack { - backtrack_common common; - /* If then_trap is not NULL, this structure contains the real - then_trap for the backtracking path. */ - struct then_trap_backtrack *then_trap; - /* Points to the starting opcode. */ - sljit_sw start; - /* Exit point for the then opcodes of this alternative. */ - jump_list *quit; - /* Frame size of the current alternative. */ - int framesize; -} then_trap_backtrack; - -#define MAX_RANGE_SIZE 4 - -typedef struct compiler_common { - /* The sljit ceneric compiler. */ - struct sljit_compiler *compiler; - /* First byte code. */ - pcre_uchar *start; - /* Maps private data offset to each opcode. */ - sljit_s32 *private_data_ptrs; - /* Chain list of read-only data ptrs. */ - void *read_only_data_head; - /* Tells whether the capturing bracket is optimized. */ - sljit_u8 *optimized_cbracket; - /* Tells whether the starting offset is a target of then. */ - sljit_u8 *then_offsets; - /* Current position where a THEN must jump. */ - then_trap_backtrack *then_trap; - /* Starting offset of private data for capturing brackets. */ - sljit_s32 cbra_ptr; - /* Output vector starting point. Must be divisible by 2. */ - sljit_s32 ovector_start; - /* Points to the starting character of the current match. */ - sljit_s32 start_ptr; - /* Last known position of the requested byte. */ - sljit_s32 req_char_ptr; - /* Head of the last recursion. */ - sljit_s32 recursive_head_ptr; - /* First inspected character for partial matching. - (Needed for avoiding zero length partial matches.) */ - sljit_s32 start_used_ptr; - /* Starting pointer for partial soft matches. */ - sljit_s32 hit_start; - /* Pointer of the match end position. */ - sljit_s32 match_end_ptr; - /* Points to the marked string. */ - sljit_s32 mark_ptr; - /* Recursive control verb management chain. */ - sljit_s32 control_head_ptr; - /* Points to the last matched capture block index. */ - sljit_s32 capture_last_ptr; - /* Fast forward skipping byte code pointer. */ - pcre_uchar *fast_forward_bc_ptr; - /* Locals used by fast fail optimization. */ - sljit_s32 fast_fail_start_ptr; - sljit_s32 fast_fail_end_ptr; - - /* Flipped and lower case tables. */ - const sljit_u8 *fcc; - sljit_sw lcc; - /* Mode can be PCRE_STUDY_JIT_COMPILE and others. */ - int mode; - /* TRUE, when minlength is greater than 0. */ - BOOL might_be_empty; - /* \K is found in the pattern. */ - BOOL has_set_som; - /* (*SKIP:arg) is found in the pattern. */ - BOOL has_skip_arg; - /* (*THEN) is found in the pattern. */ - BOOL has_then; - /* (*SKIP) or (*SKIP:arg) is found in lookbehind assertion. */ - BOOL has_skip_in_assert_back; - /* Currently in recurse or negative assert. */ - BOOL local_exit; - /* Currently in a positive assert. */ - BOOL positive_assert; - /* Newline control. */ - int nltype; - sljit_u32 nlmax; - sljit_u32 nlmin; - int newline; - int bsr_nltype; - sljit_u32 bsr_nlmax; - sljit_u32 bsr_nlmin; - /* Dollar endonly. */ - int endonly; - /* Tables. */ - sljit_sw ctypes; - /* Named capturing brackets. */ - pcre_uchar *name_table; - sljit_sw name_count; - sljit_sw name_entry_size; - - /* Labels and jump lists. */ - struct sljit_label *partialmatchlabel; - struct sljit_label *quit_label; - struct sljit_label *forced_quit_label; - struct sljit_label *accept_label; - struct sljit_label *ff_newline_shortcut; - stub_list *stubs; - label_addr_list *label_addrs; - recurse_entry *entries; - recurse_entry *currententry; - jump_list *partialmatch; - jump_list *quit; - jump_list *positive_assert_quit; - jump_list *forced_quit; - jump_list *accept; - jump_list *calllimit; - jump_list *stackalloc; - jump_list *revertframes; - jump_list *wordboundary; - jump_list *anynewline; - jump_list *hspace; - jump_list *vspace; - jump_list *casefulcmp; - jump_list *caselesscmp; - jump_list *reset_match; - BOOL jscript_compat; -#ifdef SUPPORT_UTF - BOOL utf; -#ifdef SUPPORT_UCP - BOOL use_ucp; - jump_list *getucd; -#endif -#ifdef COMPILE_PCRE8 - jump_list *utfreadchar; - jump_list *utfreadchar16; - jump_list *utfreadtype8; -#endif -#endif /* SUPPORT_UTF */ -} compiler_common; - -/* For byte_sequence_compare. */ - -typedef struct compare_context { - int length; - int sourcereg; -#if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED - int ucharptr; - union { - sljit_s32 asint; - sljit_u16 asushort; -#if defined COMPILE_PCRE8 - sljit_u8 asbyte; - sljit_u8 asuchars[4]; -#elif defined COMPILE_PCRE16 - sljit_u16 asuchars[2]; -#elif defined COMPILE_PCRE32 - sljit_u32 asuchars[1]; -#endif - } c; - union { - sljit_s32 asint; - sljit_u16 asushort; -#if defined COMPILE_PCRE8 - sljit_u8 asbyte; - sljit_u8 asuchars[4]; -#elif defined COMPILE_PCRE16 - sljit_u16 asuchars[2]; -#elif defined COMPILE_PCRE32 - sljit_u32 asuchars[1]; -#endif - } oc; -#endif -} compare_context; - -/* Undefine sljit macros. */ -#undef CMP - -/* Used for accessing the elements of the stack. */ -#define STACK(i) ((-(i) - 1) * (int)sizeof(sljit_sw)) - -#define TMP1 SLJIT_R0 -#define TMP2 SLJIT_R2 -#define TMP3 SLJIT_R3 -#define STR_PTR SLJIT_S0 -#define STR_END SLJIT_S1 -#define STACK_TOP SLJIT_R1 -#define STACK_LIMIT SLJIT_S2 -#define COUNT_MATCH SLJIT_S3 -#define ARGUMENTS SLJIT_S4 -#define RETURN_ADDR SLJIT_R4 - -/* Local space layout. */ -/* These two locals can be used by the current opcode. */ -#define LOCALS0 (0 * sizeof(sljit_sw)) -#define LOCALS1 (1 * sizeof(sljit_sw)) -/* Two local variables for possessive quantifiers (char1 cannot use them). */ -#define POSSESSIVE0 (2 * sizeof(sljit_sw)) -#define POSSESSIVE1 (3 * sizeof(sljit_sw)) -/* Max limit of recursions. */ -#define LIMIT_MATCH (4 * sizeof(sljit_sw)) -/* The output vector is stored on the stack, and contains pointers -to characters. The vector data is divided into two groups: the first -group contains the start / end character pointers, and the second is -the start pointers when the end of the capturing group has not yet reached. */ -#define OVECTOR_START (common->ovector_start) -#define OVECTOR(i) (OVECTOR_START + (i) * (sljit_sw)sizeof(sljit_sw)) -#define OVECTOR_PRIV(i) (common->cbra_ptr + (i) * (sljit_sw)sizeof(sljit_sw)) -#define PRIVATE_DATA(cc) (common->private_data_ptrs[(cc) - common->start]) - -#if defined COMPILE_PCRE8 -#define MOV_UCHAR SLJIT_MOV_U8 -#define MOVU_UCHAR SLJIT_MOVU_U8 -#elif defined COMPILE_PCRE16 -#define MOV_UCHAR SLJIT_MOV_U16 -#define MOVU_UCHAR SLJIT_MOVU_U16 -#elif defined COMPILE_PCRE32 -#define MOV_UCHAR SLJIT_MOV_U32 -#define MOVU_UCHAR SLJIT_MOVU_U32 -#else -#error Unsupported compiling mode -#endif - -/* Shortcuts. */ -#define DEFINE_COMPILER \ - struct sljit_compiler *compiler = common->compiler -#define OP1(op, dst, dstw, src, srcw) \ - sljit_emit_op1(compiler, (op), (dst), (dstw), (src), (srcw)) -#define OP2(op, dst, dstw, src1, src1w, src2, src2w) \ - sljit_emit_op2(compiler, (op), (dst), (dstw), (src1), (src1w), (src2), (src2w)) -#define LABEL() \ - sljit_emit_label(compiler) -#define JUMP(type) \ - sljit_emit_jump(compiler, (type)) -#define JUMPTO(type, label) \ - sljit_set_label(sljit_emit_jump(compiler, (type)), (label)) -#define JUMPHERE(jump) \ - sljit_set_label((jump), sljit_emit_label(compiler)) -#define SET_LABEL(jump, label) \ - sljit_set_label((jump), (label)) -#define CMP(type, src1, src1w, src2, src2w) \ - sljit_emit_cmp(compiler, (type), (src1), (src1w), (src2), (src2w)) -#define CMPTO(type, src1, src1w, src2, src2w, label) \ - sljit_set_label(sljit_emit_cmp(compiler, (type), (src1), (src1w), (src2), (src2w)), (label)) -#define OP_FLAGS(op, dst, dstw, src, srcw, type) \ - sljit_emit_op_flags(compiler, (op), (dst), (dstw), (src), (srcw), (type)) -#define GET_LOCAL_BASE(dst, dstw, offset) \ - sljit_get_local_base(compiler, (dst), (dstw), (offset)) - -#define READ_CHAR_MAX 0x7fffffff - -static pcre_uchar *bracketend(pcre_uchar *cc) -{ -SLJIT_ASSERT((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) || (*cc >= OP_ONCE && *cc <= OP_SCOND)); -do cc += GET(cc, 1); while (*cc == OP_ALT); -SLJIT_ASSERT(*cc >= OP_KET && *cc <= OP_KETRPOS); -cc += 1 + LINK_SIZE; -return cc; -} - -static int no_alternatives(pcre_uchar *cc) -{ -int count = 0; -SLJIT_ASSERT((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) || (*cc >= OP_ONCE && *cc <= OP_SCOND)); -do - { - cc += GET(cc, 1); - count++; - } -while (*cc == OP_ALT); -SLJIT_ASSERT(*cc >= OP_KET && *cc <= OP_KETRPOS); -return count; -} - -/* Functions whose might need modification for all new supported opcodes: - next_opcode - check_opcode_types - set_private_data_ptrs - get_framesize - init_frame - get_private_data_copy_length - copy_private_data - compile_matchingpath - compile_backtrackingpath -*/ - -static pcre_uchar *next_opcode(compiler_common *common, pcre_uchar *cc) -{ -SLJIT_UNUSED_ARG(common); -switch(*cc) - { - case OP_SOD: - case OP_SOM: - case OP_SET_SOM: - case OP_NOT_WORD_BOUNDARY: - case OP_WORD_BOUNDARY: - case OP_NOT_DIGIT: - case OP_DIGIT: - case OP_NOT_WHITESPACE: - case OP_WHITESPACE: - case OP_NOT_WORDCHAR: - case OP_WORDCHAR: - case OP_ANY: - case OP_ALLANY: - case OP_NOTPROP: - case OP_PROP: - case OP_ANYNL: - case OP_NOT_HSPACE: - case OP_HSPACE: - case OP_NOT_VSPACE: - case OP_VSPACE: - case OP_EXTUNI: - case OP_EODN: - case OP_EOD: - case OP_CIRC: - case OP_CIRCM: - case OP_DOLL: - case OP_DOLLM: - case OP_CRSTAR: - case OP_CRMINSTAR: - case OP_CRPLUS: - case OP_CRMINPLUS: - case OP_CRQUERY: - case OP_CRMINQUERY: - case OP_CRRANGE: - case OP_CRMINRANGE: - case OP_CRPOSSTAR: - case OP_CRPOSPLUS: - case OP_CRPOSQUERY: - case OP_CRPOSRANGE: - case OP_CLASS: - case OP_NCLASS: - case OP_REF: - case OP_REFI: - case OP_DNREF: - case OP_DNREFI: - case OP_RECURSE: - case OP_CALLOUT: - case OP_ALT: - case OP_KET: - case OP_KETRMAX: - case OP_KETRMIN: - case OP_KETRPOS: - case OP_REVERSE: - case OP_ASSERT: - case OP_ASSERT_NOT: - case OP_ASSERTBACK: - case OP_ASSERTBACK_NOT: - case OP_ONCE: - case OP_ONCE_NC: - case OP_BRA: - case OP_BRAPOS: - case OP_CBRA: - case OP_CBRAPOS: - case OP_COND: - case OP_SBRA: - case OP_SBRAPOS: - case OP_SCBRA: - case OP_SCBRAPOS: - case OP_SCOND: - case OP_CREF: - case OP_DNCREF: - case OP_RREF: - case OP_DNRREF: - case OP_DEF: - case OP_BRAZERO: - case OP_BRAMINZERO: - case OP_BRAPOSZERO: - case OP_PRUNE: - case OP_SKIP: - case OP_THEN: - case OP_COMMIT: - case OP_FAIL: - case OP_ACCEPT: - case OP_ASSERT_ACCEPT: - case OP_CLOSE: - case OP_SKIPZERO: - return cc + PRIV(OP_lengths)[*cc]; - - case OP_CHAR: - case OP_CHARI: - case OP_NOT: - case OP_NOTI: - case OP_STAR: - case OP_MINSTAR: - case OP_PLUS: - case OP_MINPLUS: - case OP_QUERY: - case OP_MINQUERY: - case OP_UPTO: - case OP_MINUPTO: - case OP_EXACT: - case OP_POSSTAR: - case OP_POSPLUS: - case OP_POSQUERY: - case OP_POSUPTO: - case OP_STARI: - case OP_MINSTARI: - case OP_PLUSI: - case OP_MINPLUSI: - case OP_QUERYI: - case OP_MINQUERYI: - case OP_UPTOI: - case OP_MINUPTOI: - case OP_EXACTI: - case OP_POSSTARI: - case OP_POSPLUSI: - case OP_POSQUERYI: - case OP_POSUPTOI: - case OP_NOTSTAR: - case OP_NOTMINSTAR: - case OP_NOTPLUS: - case OP_NOTMINPLUS: - case OP_NOTQUERY: - case OP_NOTMINQUERY: - case OP_NOTUPTO: - case OP_NOTMINUPTO: - case OP_NOTEXACT: - case OP_NOTPOSSTAR: - case OP_NOTPOSPLUS: - case OP_NOTPOSQUERY: - case OP_NOTPOSUPTO: - case OP_NOTSTARI: - case OP_NOTMINSTARI: - case OP_NOTPLUSI: - case OP_NOTMINPLUSI: - case OP_NOTQUERYI: - case OP_NOTMINQUERYI: - case OP_NOTUPTOI: - case OP_NOTMINUPTOI: - case OP_NOTEXACTI: - case OP_NOTPOSSTARI: - case OP_NOTPOSPLUSI: - case OP_NOTPOSQUERYI: - case OP_NOTPOSUPTOI: - cc += PRIV(OP_lengths)[*cc]; -#ifdef SUPPORT_UTF - if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); -#endif - return cc; - - /* Special cases. */ - case OP_TYPESTAR: - case OP_TYPEMINSTAR: - case OP_TYPEPLUS: - case OP_TYPEMINPLUS: - case OP_TYPEQUERY: - case OP_TYPEMINQUERY: - case OP_TYPEUPTO: - case OP_TYPEMINUPTO: - case OP_TYPEEXACT: - case OP_TYPEPOSSTAR: - case OP_TYPEPOSPLUS: - case OP_TYPEPOSQUERY: - case OP_TYPEPOSUPTO: - return cc + PRIV(OP_lengths)[*cc] - 1; - - case OP_ANYBYTE: -#ifdef SUPPORT_UTF - if (common->utf) return NULL; -#endif - return cc + 1; - -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - case OP_XCLASS: - return cc + GET(cc, 1); -#endif - - case OP_MARK: - case OP_PRUNE_ARG: - case OP_SKIP_ARG: - case OP_THEN_ARG: - return cc + 1 + 2 + cc[1]; - - default: - /* All opcodes are supported now! */ - SLJIT_ASSERT_STOP(); - return NULL; - } -} - -static BOOL check_opcode_types(compiler_common *common, pcre_uchar *cc, pcre_uchar *ccend) -{ -int count; -pcre_uchar *slot; -pcre_uchar *assert_back_end = cc - 1; - -/* Calculate important variables (like stack size) and checks whether all opcodes are supported. */ -while (cc < ccend) - { - switch(*cc) - { - case OP_SET_SOM: - common->has_set_som = TRUE; - common->might_be_empty = TRUE; - cc += 1; - break; - - case OP_REF: - case OP_REFI: - common->optimized_cbracket[GET2(cc, 1)] = 0; - cc += 1 + IMM2_SIZE; - break; - - case OP_CBRAPOS: - case OP_SCBRAPOS: - common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] = 0; - cc += 1 + LINK_SIZE + IMM2_SIZE; - break; - - case OP_COND: - case OP_SCOND: - /* Only AUTO_CALLOUT can insert this opcode. We do - not intend to support this case. */ - if (cc[1 + LINK_SIZE] == OP_CALLOUT) - return FALSE; - cc += 1 + LINK_SIZE; - break; - - case OP_CREF: - common->optimized_cbracket[GET2(cc, 1)] = 0; - cc += 1 + IMM2_SIZE; - break; - - case OP_DNREF: - case OP_DNREFI: - case OP_DNCREF: - count = GET2(cc, 1 + IMM2_SIZE); - slot = common->name_table + GET2(cc, 1) * common->name_entry_size; - while (count-- > 0) - { - common->optimized_cbracket[GET2(slot, 0)] = 0; - slot += common->name_entry_size; - } - cc += 1 + 2 * IMM2_SIZE; - break; - - case OP_RECURSE: - /* Set its value only once. */ - if (common->recursive_head_ptr == 0) - { - common->recursive_head_ptr = common->ovector_start; - common->ovector_start += sizeof(sljit_sw); - } - cc += 1 + LINK_SIZE; - break; - - case OP_CALLOUT: - if (common->capture_last_ptr == 0) - { - common->capture_last_ptr = common->ovector_start; - common->ovector_start += sizeof(sljit_sw); - } - cc += 2 + 2 * LINK_SIZE; - break; - - case OP_ASSERTBACK: - slot = bracketend(cc); - if (slot > assert_back_end) - assert_back_end = slot; - cc += 1 + LINK_SIZE; - break; - - case OP_THEN_ARG: - common->has_then = TRUE; - common->control_head_ptr = 1; - /* Fall through. */ - - case OP_PRUNE_ARG: - case OP_MARK: - if (common->mark_ptr == 0) - { - common->mark_ptr = common->ovector_start; - common->ovector_start += sizeof(sljit_sw); - } - cc += 1 + 2 + cc[1]; - break; - - case OP_THEN: - common->has_then = TRUE; - common->control_head_ptr = 1; - cc += 1; - break; - - case OP_SKIP: - if (cc < assert_back_end) - common->has_skip_in_assert_back = TRUE; - cc += 1; - break; - - case OP_SKIP_ARG: - common->control_head_ptr = 1; - common->has_skip_arg = TRUE; - if (cc < assert_back_end) - common->has_skip_in_assert_back = TRUE; - cc += 1 + 2 + cc[1]; - break; - - default: - cc = next_opcode(common, cc); - if (cc == NULL) - return FALSE; - break; - } - } -return TRUE; -} - -static BOOL is_accelerated_repeat(pcre_uchar *cc) -{ -switch(*cc) - { - case OP_TYPESTAR: - case OP_TYPEMINSTAR: - case OP_TYPEPLUS: - case OP_TYPEMINPLUS: - case OP_TYPEPOSSTAR: - case OP_TYPEPOSPLUS: - return (cc[1] != OP_ANYNL && cc[1] != OP_EXTUNI); - - case OP_STAR: - case OP_MINSTAR: - case OP_PLUS: - case OP_MINPLUS: - case OP_POSSTAR: - case OP_POSPLUS: - - case OP_STARI: - case OP_MINSTARI: - case OP_PLUSI: - case OP_MINPLUSI: - case OP_POSSTARI: - case OP_POSPLUSI: - - case OP_NOTSTAR: - case OP_NOTMINSTAR: - case OP_NOTPLUS: - case OP_NOTMINPLUS: - case OP_NOTPOSSTAR: - case OP_NOTPOSPLUS: - - case OP_NOTSTARI: - case OP_NOTMINSTARI: - case OP_NOTPLUSI: - case OP_NOTMINPLUSI: - case OP_NOTPOSSTARI: - case OP_NOTPOSPLUSI: - return TRUE; - - case OP_CLASS: - case OP_NCLASS: -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - case OP_XCLASS: - cc += (*cc == OP_XCLASS) ? GET(cc, 1) : (int)(1 + (32 / sizeof(pcre_uchar))); -#else - cc += (1 + (32 / sizeof(pcre_uchar))); -#endif - - switch(*cc) - { - case OP_CRSTAR: - case OP_CRMINSTAR: - case OP_CRPLUS: - case OP_CRMINPLUS: - case OP_CRPOSSTAR: - case OP_CRPOSPLUS: - return TRUE; - } - break; - } -return FALSE; -} - -static SLJIT_INLINE BOOL detect_fast_forward_skip(compiler_common *common, int *private_data_start) -{ -pcre_uchar *cc = common->start; -pcre_uchar *end; - -/* Skip not repeated brackets. */ -while (TRUE) - { - switch(*cc) - { - case OP_SOD: - case OP_SOM: - case OP_SET_SOM: - case OP_NOT_WORD_BOUNDARY: - case OP_WORD_BOUNDARY: - case OP_EODN: - case OP_EOD: - case OP_CIRC: - case OP_CIRCM: - case OP_DOLL: - case OP_DOLLM: - /* Zero width assertions. */ - cc++; - continue; - } - - if (*cc != OP_BRA && *cc != OP_CBRA) - break; - - end = cc + GET(cc, 1); - if (*end != OP_KET || PRIVATE_DATA(end) != 0) - return FALSE; - if (*cc == OP_CBRA) - { - if (common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0) - return FALSE; - cc += IMM2_SIZE; - } - cc += 1 + LINK_SIZE; - } - -if (is_accelerated_repeat(cc)) - { - common->fast_forward_bc_ptr = cc; - common->private_data_ptrs[(cc + 1) - common->start] = *private_data_start; - *private_data_start += sizeof(sljit_sw); - return TRUE; - } -return FALSE; -} - -static SLJIT_INLINE void detect_fast_fail(compiler_common *common, pcre_uchar *cc, int *private_data_start, sljit_s32 depth) -{ - pcre_uchar *next_alt; - - SLJIT_ASSERT(*cc == OP_BRA || *cc == OP_CBRA); - - if (*cc == OP_CBRA && common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0) - return; - - next_alt = bracketend(cc) - (1 + LINK_SIZE); - if (*next_alt != OP_KET || PRIVATE_DATA(next_alt) != 0) - return; - - do - { - next_alt = cc + GET(cc, 1); - - cc += 1 + LINK_SIZE + ((*cc == OP_CBRA) ? IMM2_SIZE : 0); - - while (TRUE) - { - switch(*cc) - { - case OP_SOD: - case OP_SOM: - case OP_SET_SOM: - case OP_NOT_WORD_BOUNDARY: - case OP_WORD_BOUNDARY: - case OP_EODN: - case OP_EOD: - case OP_CIRC: - case OP_CIRCM: - case OP_DOLL: - case OP_DOLLM: - /* Zero width assertions. */ - cc++; - continue; - } - break; - } - - if (depth > 0 && (*cc == OP_BRA || *cc == OP_CBRA)) - detect_fast_fail(common, cc, private_data_start, depth - 1); - - if (is_accelerated_repeat(cc)) - { - common->private_data_ptrs[(cc + 1) - common->start] = *private_data_start; - - if (common->fast_fail_start_ptr == 0) - common->fast_fail_start_ptr = *private_data_start; - - *private_data_start += sizeof(sljit_sw); - common->fast_fail_end_ptr = *private_data_start; - - if (*private_data_start > SLJIT_MAX_LOCAL_SIZE) - return; - } - - cc = next_alt; - } - while (*cc == OP_ALT); -} - -static int get_class_iterator_size(pcre_uchar *cc) -{ -sljit_u32 min; -sljit_u32 max; -switch(*cc) - { - case OP_CRSTAR: - case OP_CRPLUS: - return 2; - - case OP_CRMINSTAR: - case OP_CRMINPLUS: - case OP_CRQUERY: - case OP_CRMINQUERY: - return 1; - - case OP_CRRANGE: - case OP_CRMINRANGE: - min = GET2(cc, 1); - max = GET2(cc, 1 + IMM2_SIZE); - if (max == 0) - return (*cc == OP_CRRANGE) ? 2 : 1; - max -= min; - if (max > 2) - max = 2; - return max; - - default: - return 0; - } -} - -static BOOL detect_repeat(compiler_common *common, pcre_uchar *begin) -{ -pcre_uchar *end = bracketend(begin); -pcre_uchar *next; -pcre_uchar *next_end; -pcre_uchar *max_end; -pcre_uchar type; -sljit_sw length = end - begin; -int min, max, i; - -/* Detect fixed iterations first. */ -if (end[-(1 + LINK_SIZE)] != OP_KET) - return FALSE; - -/* Already detected repeat. */ -if (common->private_data_ptrs[end - common->start - LINK_SIZE] != 0) - return TRUE; - -next = end; -min = 1; -while (1) - { - if (*next != *begin) - break; - next_end = bracketend(next); - if (next_end - next != length || memcmp(begin, next, IN_UCHARS(length)) != 0) - break; - next = next_end; - min++; - } - -if (min == 2) - return FALSE; - -max = 0; -max_end = next; -if (*next == OP_BRAZERO || *next == OP_BRAMINZERO) - { - type = *next; - while (1) - { - if (next[0] != type || next[1] != OP_BRA || next[2 + LINK_SIZE] != *begin) - break; - next_end = bracketend(next + 2 + LINK_SIZE); - if (next_end - next != (length + 2 + LINK_SIZE) || memcmp(begin, next + 2 + LINK_SIZE, IN_UCHARS(length)) != 0) - break; - next = next_end; - max++; - } - - if (next[0] == type && next[1] == *begin && max >= 1) - { - next_end = bracketend(next + 1); - if (next_end - next == (length + 1) && memcmp(begin, next + 1, IN_UCHARS(length)) == 0) - { - for (i = 0; i < max; i++, next_end += 1 + LINK_SIZE) - if (*next_end != OP_KET) - break; - - if (i == max) - { - common->private_data_ptrs[max_end - common->start - LINK_SIZE] = next_end - max_end; - common->private_data_ptrs[max_end - common->start - LINK_SIZE + 1] = (type == OP_BRAZERO) ? OP_UPTO : OP_MINUPTO; - /* +2 the original and the last. */ - common->private_data_ptrs[max_end - common->start - LINK_SIZE + 2] = max + 2; - if (min == 1) - return TRUE; - min--; - max_end -= (1 + LINK_SIZE) + GET(max_end, -LINK_SIZE); - } - } - } - } - -if (min >= 3) - { - common->private_data_ptrs[end - common->start - LINK_SIZE] = max_end - end; - common->private_data_ptrs[end - common->start - LINK_SIZE + 1] = OP_EXACT; - common->private_data_ptrs[end - common->start - LINK_SIZE + 2] = min; - return TRUE; - } - -return FALSE; -} - -#define CASE_ITERATOR_PRIVATE_DATA_1 \ - case OP_MINSTAR: \ - case OP_MINPLUS: \ - case OP_QUERY: \ - case OP_MINQUERY: \ - case OP_MINSTARI: \ - case OP_MINPLUSI: \ - case OP_QUERYI: \ - case OP_MINQUERYI: \ - case OP_NOTMINSTAR: \ - case OP_NOTMINPLUS: \ - case OP_NOTQUERY: \ - case OP_NOTMINQUERY: \ - case OP_NOTMINSTARI: \ - case OP_NOTMINPLUSI: \ - case OP_NOTQUERYI: \ - case OP_NOTMINQUERYI: - -#define CASE_ITERATOR_PRIVATE_DATA_2A \ - case OP_STAR: \ - case OP_PLUS: \ - case OP_STARI: \ - case OP_PLUSI: \ - case OP_NOTSTAR: \ - case OP_NOTPLUS: \ - case OP_NOTSTARI: \ - case OP_NOTPLUSI: - -#define CASE_ITERATOR_PRIVATE_DATA_2B \ - case OP_UPTO: \ - case OP_MINUPTO: \ - case OP_UPTOI: \ - case OP_MINUPTOI: \ - case OP_NOTUPTO: \ - case OP_NOTMINUPTO: \ - case OP_NOTUPTOI: \ - case OP_NOTMINUPTOI: - -#define CASE_ITERATOR_TYPE_PRIVATE_DATA_1 \ - case OP_TYPEMINSTAR: \ - case OP_TYPEMINPLUS: \ - case OP_TYPEQUERY: \ - case OP_TYPEMINQUERY: - -#define CASE_ITERATOR_TYPE_PRIVATE_DATA_2A \ - case OP_TYPESTAR: \ - case OP_TYPEPLUS: - -#define CASE_ITERATOR_TYPE_PRIVATE_DATA_2B \ - case OP_TYPEUPTO: \ - case OP_TYPEMINUPTO: - -static void set_private_data_ptrs(compiler_common *common, int *private_data_start, pcre_uchar *ccend) -{ -pcre_uchar *cc = common->start; -pcre_uchar *alternative; -pcre_uchar *end = NULL; -int private_data_ptr = *private_data_start; -int space, size, bracketlen; -BOOL repeat_check = TRUE; - -while (cc < ccend) - { - space = 0; - size = 0; - bracketlen = 0; - if (private_data_ptr > SLJIT_MAX_LOCAL_SIZE) - break; - - if (repeat_check && (*cc == OP_ONCE || *cc == OP_ONCE_NC || *cc == OP_BRA || *cc == OP_CBRA || *cc == OP_COND)) - { - if (detect_repeat(common, cc)) - { - /* These brackets are converted to repeats, so no global - based single character repeat is allowed. */ - if (cc >= end) - end = bracketend(cc); - } - } - repeat_check = TRUE; - - switch(*cc) - { - case OP_KET: - if (common->private_data_ptrs[cc + 1 - common->start] != 0) - { - common->private_data_ptrs[cc - common->start] = private_data_ptr; - private_data_ptr += sizeof(sljit_sw); - cc += common->private_data_ptrs[cc + 1 - common->start]; - } - cc += 1 + LINK_SIZE; - break; - - case OP_ASSERT: - case OP_ASSERT_NOT: - case OP_ASSERTBACK: - case OP_ASSERTBACK_NOT: - case OP_ONCE: - case OP_ONCE_NC: - case OP_BRAPOS: - case OP_SBRA: - case OP_SBRAPOS: - case OP_SCOND: - common->private_data_ptrs[cc - common->start] = private_data_ptr; - private_data_ptr += sizeof(sljit_sw); - bracketlen = 1 + LINK_SIZE; - break; - - case OP_CBRAPOS: - case OP_SCBRAPOS: - common->private_data_ptrs[cc - common->start] = private_data_ptr; - private_data_ptr += sizeof(sljit_sw); - bracketlen = 1 + LINK_SIZE + IMM2_SIZE; - break; - - case OP_COND: - /* Might be a hidden SCOND. */ - alternative = cc + GET(cc, 1); - if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN) - { - common->private_data_ptrs[cc - common->start] = private_data_ptr; - private_data_ptr += sizeof(sljit_sw); - } - bracketlen = 1 + LINK_SIZE; - break; - - case OP_BRA: - bracketlen = 1 + LINK_SIZE; - break; - - case OP_CBRA: - case OP_SCBRA: - bracketlen = 1 + LINK_SIZE + IMM2_SIZE; - break; - - case OP_BRAZERO: - case OP_BRAMINZERO: - case OP_BRAPOSZERO: - repeat_check = FALSE; - size = 1; - break; - - CASE_ITERATOR_PRIVATE_DATA_1 - space = 1; - size = -2; - break; - - CASE_ITERATOR_PRIVATE_DATA_2A - space = 2; - size = -2; - break; - - CASE_ITERATOR_PRIVATE_DATA_2B - space = 2; - size = -(2 + IMM2_SIZE); - break; - - CASE_ITERATOR_TYPE_PRIVATE_DATA_1 - space = 1; - size = 1; - break; - - CASE_ITERATOR_TYPE_PRIVATE_DATA_2A - if (cc[1] != OP_ANYNL && cc[1] != OP_EXTUNI) - space = 2; - size = 1; - break; - - case OP_TYPEUPTO: - if (cc[1 + IMM2_SIZE] != OP_ANYNL && cc[1 + IMM2_SIZE] != OP_EXTUNI) - space = 2; - size = 1 + IMM2_SIZE; - break; - - case OP_TYPEMINUPTO: - space = 2; - size = 1 + IMM2_SIZE; - break; - - case OP_CLASS: - case OP_NCLASS: - space = get_class_iterator_size(cc + size); - size = 1 + 32 / sizeof(pcre_uchar); - break; - -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - case OP_XCLASS: - space = get_class_iterator_size(cc + size); - size = GET(cc, 1); - break; -#endif - - default: - cc = next_opcode(common, cc); - SLJIT_ASSERT(cc != NULL); - break; - } - - /* Character iterators, which are not inside a repeated bracket, - gets a private slot instead of allocating it on the stack. */ - if (space > 0 && cc >= end) - { - common->private_data_ptrs[cc - common->start] = private_data_ptr; - private_data_ptr += sizeof(sljit_sw) * space; - } - - if (size != 0) - { - if (size < 0) - { - cc += -size; -#ifdef SUPPORT_UTF - if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); -#endif - } - else - cc += size; - } - - if (bracketlen > 0) - { - if (cc >= end) - { - end = bracketend(cc); - if (end[-1 - LINK_SIZE] == OP_KET) - end = NULL; - } - cc += bracketlen; - } - } -*private_data_start = private_data_ptr; -} - -/* Returns with a frame_types (always < 0) if no need for frame. */ -static int get_framesize(compiler_common *common, pcre_uchar *cc, pcre_uchar *ccend, BOOL recursive, BOOL *needs_control_head) -{ -int length = 0; -int possessive = 0; -BOOL stack_restore = FALSE; -BOOL setsom_found = recursive; -BOOL setmark_found = recursive; -/* The last capture is a local variable even for recursions. */ -BOOL capture_last_found = FALSE; - -#if defined DEBUG_FORCE_CONTROL_HEAD && DEBUG_FORCE_CONTROL_HEAD -SLJIT_ASSERT(common->control_head_ptr != 0); -*needs_control_head = TRUE; -#else -*needs_control_head = FALSE; -#endif - -if (ccend == NULL) - { - ccend = bracketend(cc) - (1 + LINK_SIZE); - if (!recursive && (*cc == OP_CBRAPOS || *cc == OP_SCBRAPOS)) - { - possessive = length = (common->capture_last_ptr != 0) ? 5 : 3; - /* This is correct regardless of common->capture_last_ptr. */ - capture_last_found = TRUE; - } - cc = next_opcode(common, cc); - } - -SLJIT_ASSERT(cc != NULL); -while (cc < ccend) - switch(*cc) - { - case OP_SET_SOM: - SLJIT_ASSERT(common->has_set_som); - stack_restore = TRUE; - if (!setsom_found) - { - length += 2; - setsom_found = TRUE; - } - cc += 1; - break; - - case OP_MARK: - case OP_PRUNE_ARG: - case OP_THEN_ARG: - SLJIT_ASSERT(common->mark_ptr != 0); - stack_restore = TRUE; - if (!setmark_found) - { - length += 2; - setmark_found = TRUE; - } - if (common->control_head_ptr != 0) - *needs_control_head = TRUE; - cc += 1 + 2 + cc[1]; - break; - - case OP_RECURSE: - stack_restore = TRUE; - if (common->has_set_som && !setsom_found) - { - length += 2; - setsom_found = TRUE; - } - if (common->mark_ptr != 0 && !setmark_found) - { - length += 2; - setmark_found = TRUE; - } - if (common->capture_last_ptr != 0 && !capture_last_found) - { - length += 2; - capture_last_found = TRUE; - } - cc += 1 + LINK_SIZE; - break; - - case OP_CBRA: - case OP_CBRAPOS: - case OP_SCBRA: - case OP_SCBRAPOS: - stack_restore = TRUE; - if (common->capture_last_ptr != 0 && !capture_last_found) - { - length += 2; - capture_last_found = TRUE; - } - length += 3; - cc += 1 + LINK_SIZE + IMM2_SIZE; - break; - - case OP_THEN: - stack_restore = TRUE; - if (common->control_head_ptr != 0) - *needs_control_head = TRUE; - cc ++; - break; - - default: - stack_restore = TRUE; - /* Fall through. */ - - case OP_NOT_WORD_BOUNDARY: - case OP_WORD_BOUNDARY: - case OP_NOT_DIGIT: - case OP_DIGIT: - case OP_NOT_WHITESPACE: - case OP_WHITESPACE: - case OP_NOT_WORDCHAR: - case OP_WORDCHAR: - case OP_ANY: - case OP_ALLANY: - case OP_ANYBYTE: - case OP_NOTPROP: - case OP_PROP: - case OP_ANYNL: - case OP_NOT_HSPACE: - case OP_HSPACE: - case OP_NOT_VSPACE: - case OP_VSPACE: - case OP_EXTUNI: - case OP_EODN: - case OP_EOD: - case OP_CIRC: - case OP_CIRCM: - case OP_DOLL: - case OP_DOLLM: - case OP_CHAR: - case OP_CHARI: - case OP_NOT: - case OP_NOTI: - - case OP_EXACT: - case OP_POSSTAR: - case OP_POSPLUS: - case OP_POSQUERY: - case OP_POSUPTO: - - case OP_EXACTI: - case OP_POSSTARI: - case OP_POSPLUSI: - case OP_POSQUERYI: - case OP_POSUPTOI: - - case OP_NOTEXACT: - case OP_NOTPOSSTAR: - case OP_NOTPOSPLUS: - case OP_NOTPOSQUERY: - case OP_NOTPOSUPTO: - - case OP_NOTEXACTI: - case OP_NOTPOSSTARI: - case OP_NOTPOSPLUSI: - case OP_NOTPOSQUERYI: - case OP_NOTPOSUPTOI: - - case OP_TYPEEXACT: - case OP_TYPEPOSSTAR: - case OP_TYPEPOSPLUS: - case OP_TYPEPOSQUERY: - case OP_TYPEPOSUPTO: - - case OP_CLASS: - case OP_NCLASS: - case OP_XCLASS: - case OP_CALLOUT: - - cc = next_opcode(common, cc); - SLJIT_ASSERT(cc != NULL); - break; - } - -/* Possessive quantifiers can use a special case. */ -if (SLJIT_UNLIKELY(possessive == length)) - return stack_restore ? no_frame : no_stack; - -if (length > 0) - return length + 1; -return stack_restore ? no_frame : no_stack; -} - -static void init_frame(compiler_common *common, pcre_uchar *cc, pcre_uchar *ccend, int stackpos, int stacktop, BOOL recursive) -{ -DEFINE_COMPILER; -BOOL setsom_found = recursive; -BOOL setmark_found = recursive; -/* The last capture is a local variable even for recursions. */ -BOOL capture_last_found = FALSE; -int offset; - -/* >= 1 + shortest item size (2) */ -SLJIT_UNUSED_ARG(stacktop); -SLJIT_ASSERT(stackpos >= stacktop + 2); - -stackpos = STACK(stackpos); -if (ccend == NULL) - { - ccend = bracketend(cc) - (1 + LINK_SIZE); - if (recursive || (*cc != OP_CBRAPOS && *cc != OP_SCBRAPOS)) - cc = next_opcode(common, cc); - } - -SLJIT_ASSERT(cc != NULL); -while (cc < ccend) - switch(*cc) - { - case OP_SET_SOM: - SLJIT_ASSERT(common->has_set_som); - if (!setsom_found) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0)); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -OVECTOR(0)); - stackpos += (int)sizeof(sljit_sw); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); - setsom_found = TRUE; - } - cc += 1; - break; - - case OP_MARK: - case OP_PRUNE_ARG: - case OP_THEN_ARG: - SLJIT_ASSERT(common->mark_ptr != 0); - if (!setmark_found) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->mark_ptr); - stackpos += (int)sizeof(sljit_sw); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); - setmark_found = TRUE; - } - cc += 1 + 2 + cc[1]; - break; - - case OP_RECURSE: - if (common->has_set_som && !setsom_found) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0)); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -OVECTOR(0)); - stackpos += (int)sizeof(sljit_sw); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); - setsom_found = TRUE; - } - if (common->mark_ptr != 0 && !setmark_found) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->mark_ptr); - stackpos += (int)sizeof(sljit_sw); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); - setmark_found = TRUE; - } - if (common->capture_last_ptr != 0 && !capture_last_found) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->capture_last_ptr); - stackpos += (int)sizeof(sljit_sw); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); - capture_last_found = TRUE; - } - cc += 1 + LINK_SIZE; - break; - - case OP_CBRA: - case OP_CBRAPOS: - case OP_SCBRA: - case OP_SCBRAPOS: - if (common->capture_last_ptr != 0 && !capture_last_found) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->capture_last_ptr); - stackpos += (int)sizeof(sljit_sw); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); - capture_last_found = TRUE; - } - offset = (GET2(cc, 1 + LINK_SIZE)) << 1; - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, OVECTOR(offset)); - stackpos += (int)sizeof(sljit_sw); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset)); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); - stackpos += (int)sizeof(sljit_sw); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP2, 0); - stackpos += (int)sizeof(sljit_sw); - - cc += 1 + LINK_SIZE + IMM2_SIZE; - break; - - default: - cc = next_opcode(common, cc); - SLJIT_ASSERT(cc != NULL); - break; - } - -OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, 0); -SLJIT_ASSERT(stackpos == STACK(stacktop)); -} - -static SLJIT_INLINE int get_private_data_copy_length(compiler_common *common, pcre_uchar *cc, pcre_uchar *ccend, BOOL needs_control_head) -{ -int private_data_length = needs_control_head ? 3 : 2; -int size; -pcre_uchar *alternative; -/* Calculate the sum of the private machine words. */ -while (cc < ccend) - { - size = 0; - switch(*cc) - { - case OP_KET: - if (PRIVATE_DATA(cc) != 0) - { - private_data_length++; - SLJIT_ASSERT(PRIVATE_DATA(cc + 1) != 0); - cc += PRIVATE_DATA(cc + 1); - } - cc += 1 + LINK_SIZE; - break; - - case OP_ASSERT: - case OP_ASSERT_NOT: - case OP_ASSERTBACK: - case OP_ASSERTBACK_NOT: - case OP_ONCE: - case OP_ONCE_NC: - case OP_BRAPOS: - case OP_SBRA: - case OP_SBRAPOS: - case OP_SCOND: - private_data_length++; - SLJIT_ASSERT(PRIVATE_DATA(cc) != 0); - cc += 1 + LINK_SIZE; - break; - - case OP_CBRA: - case OP_SCBRA: - if (common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0) - private_data_length++; - cc += 1 + LINK_SIZE + IMM2_SIZE; - break; - - case OP_CBRAPOS: - case OP_SCBRAPOS: - private_data_length += 2; - cc += 1 + LINK_SIZE + IMM2_SIZE; - break; - - case OP_COND: - /* Might be a hidden SCOND. */ - alternative = cc + GET(cc, 1); - if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN) - private_data_length++; - cc += 1 + LINK_SIZE; - break; - - CASE_ITERATOR_PRIVATE_DATA_1 - if (PRIVATE_DATA(cc)) - private_data_length++; - cc += 2; -#ifdef SUPPORT_UTF - if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); -#endif - break; - - CASE_ITERATOR_PRIVATE_DATA_2A - if (PRIVATE_DATA(cc)) - private_data_length += 2; - cc += 2; -#ifdef SUPPORT_UTF - if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); -#endif - break; - - CASE_ITERATOR_PRIVATE_DATA_2B - if (PRIVATE_DATA(cc)) - private_data_length += 2; - cc += 2 + IMM2_SIZE; -#ifdef SUPPORT_UTF - if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); -#endif - break; - - CASE_ITERATOR_TYPE_PRIVATE_DATA_1 - if (PRIVATE_DATA(cc)) - private_data_length++; - cc += 1; - break; - - CASE_ITERATOR_TYPE_PRIVATE_DATA_2A - if (PRIVATE_DATA(cc)) - private_data_length += 2; - cc += 1; - break; - - CASE_ITERATOR_TYPE_PRIVATE_DATA_2B - if (PRIVATE_DATA(cc)) - private_data_length += 2; - cc += 1 + IMM2_SIZE; - break; - - case OP_CLASS: - case OP_NCLASS: -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - case OP_XCLASS: - size = (*cc == OP_XCLASS) ? GET(cc, 1) : 1 + 32 / (int)sizeof(pcre_uchar); -#else - size = 1 + 32 / (int)sizeof(pcre_uchar); -#endif - if (PRIVATE_DATA(cc)) - private_data_length += get_class_iterator_size(cc + size); - cc += size; - break; - - default: - cc = next_opcode(common, cc); - SLJIT_ASSERT(cc != NULL); - break; - } - } -SLJIT_ASSERT(cc == ccend); -return private_data_length; -} - -static void copy_private_data(compiler_common *common, pcre_uchar *cc, pcre_uchar *ccend, - BOOL save, int stackptr, int stacktop, BOOL needs_control_head) -{ -DEFINE_COMPILER; -int srcw[2]; -int count, size; -BOOL tmp1next = TRUE; -BOOL tmp1empty = TRUE; -BOOL tmp2empty = TRUE; -pcre_uchar *alternative; -enum { - start, - loop, - end -} status; - -status = save ? start : loop; -stackptr = STACK(stackptr - 2); -stacktop = STACK(stacktop - 1); - -if (!save) - { - stackptr += (needs_control_head ? 2 : 1) * sizeof(sljit_sw); - if (stackptr < stacktop) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), stackptr); - stackptr += sizeof(sljit_sw); - tmp1empty = FALSE; - } - if (stackptr < stacktop) - { - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), stackptr); - stackptr += sizeof(sljit_sw); - tmp2empty = FALSE; - } - /* The tmp1next must be TRUE in either way. */ - } - -do - { - count = 0; - switch(status) - { - case start: - SLJIT_ASSERT(save && common->recursive_head_ptr != 0); - count = 1; - srcw[0] = common->recursive_head_ptr; - if (needs_control_head) - { - SLJIT_ASSERT(common->control_head_ptr != 0); - count = 2; - srcw[1] = common->control_head_ptr; - } - status = loop; - break; - - case loop: - if (cc >= ccend) - { - status = end; - break; - } - - switch(*cc) - { - case OP_KET: - if (PRIVATE_DATA(cc) != 0) - { - count = 1; - srcw[0] = PRIVATE_DATA(cc); - SLJIT_ASSERT(PRIVATE_DATA(cc + 1) != 0); - cc += PRIVATE_DATA(cc + 1); - } - cc += 1 + LINK_SIZE; - break; - - case OP_ASSERT: - case OP_ASSERT_NOT: - case OP_ASSERTBACK: - case OP_ASSERTBACK_NOT: - case OP_ONCE: - case OP_ONCE_NC: - case OP_BRAPOS: - case OP_SBRA: - case OP_SBRAPOS: - case OP_SCOND: - count = 1; - srcw[0] = PRIVATE_DATA(cc); - SLJIT_ASSERT(srcw[0] != 0); - cc += 1 + LINK_SIZE; - break; - - case OP_CBRA: - case OP_SCBRA: - if (common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0) - { - count = 1; - srcw[0] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE)); - } - cc += 1 + LINK_SIZE + IMM2_SIZE; - break; - - case OP_CBRAPOS: - case OP_SCBRAPOS: - count = 2; - srcw[0] = PRIVATE_DATA(cc); - srcw[1] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE)); - SLJIT_ASSERT(srcw[0] != 0 && srcw[1] != 0); - cc += 1 + LINK_SIZE + IMM2_SIZE; - break; - - case OP_COND: - /* Might be a hidden SCOND. */ - alternative = cc + GET(cc, 1); - if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN) - { - count = 1; - srcw[0] = PRIVATE_DATA(cc); - SLJIT_ASSERT(srcw[0] != 0); - } - cc += 1 + LINK_SIZE; - break; - - CASE_ITERATOR_PRIVATE_DATA_1 - if (PRIVATE_DATA(cc)) - { - count = 1; - srcw[0] = PRIVATE_DATA(cc); - } - cc += 2; -#ifdef SUPPORT_UTF - if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); -#endif - break; - - CASE_ITERATOR_PRIVATE_DATA_2A - if (PRIVATE_DATA(cc)) - { - count = 2; - srcw[0] = PRIVATE_DATA(cc); - srcw[1] = PRIVATE_DATA(cc) + sizeof(sljit_sw); - } - cc += 2; -#ifdef SUPPORT_UTF - if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); -#endif - break; - - CASE_ITERATOR_PRIVATE_DATA_2B - if (PRIVATE_DATA(cc)) - { - count = 2; - srcw[0] = PRIVATE_DATA(cc); - srcw[1] = PRIVATE_DATA(cc) + sizeof(sljit_sw); - } - cc += 2 + IMM2_SIZE; -#ifdef SUPPORT_UTF - if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); -#endif - break; - - CASE_ITERATOR_TYPE_PRIVATE_DATA_1 - if (PRIVATE_DATA(cc)) - { - count = 1; - srcw[0] = PRIVATE_DATA(cc); - } - cc += 1; - break; - - CASE_ITERATOR_TYPE_PRIVATE_DATA_2A - if (PRIVATE_DATA(cc)) - { - count = 2; - srcw[0] = PRIVATE_DATA(cc); - srcw[1] = srcw[0] + sizeof(sljit_sw); - } - cc += 1; - break; - - CASE_ITERATOR_TYPE_PRIVATE_DATA_2B - if (PRIVATE_DATA(cc)) - { - count = 2; - srcw[0] = PRIVATE_DATA(cc); - srcw[1] = srcw[0] + sizeof(sljit_sw); - } - cc += 1 + IMM2_SIZE; - break; - - case OP_CLASS: - case OP_NCLASS: -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - case OP_XCLASS: - size = (*cc == OP_XCLASS) ? GET(cc, 1) : 1 + 32 / (int)sizeof(pcre_uchar); -#else - size = 1 + 32 / (int)sizeof(pcre_uchar); -#endif - if (PRIVATE_DATA(cc)) - switch(get_class_iterator_size(cc + size)) - { - case 1: - count = 1; - srcw[0] = PRIVATE_DATA(cc); - break; - - case 2: - count = 2; - srcw[0] = PRIVATE_DATA(cc); - srcw[1] = srcw[0] + sizeof(sljit_sw); - break; - - default: - SLJIT_ASSERT_STOP(); - break; - } - cc += size; - break; - - default: - cc = next_opcode(common, cc); - SLJIT_ASSERT(cc != NULL); - break; - } - break; - - case end: - SLJIT_ASSERT_STOP(); - break; - } - - while (count > 0) - { - count--; - if (save) - { - if (tmp1next) - { - if (!tmp1empty) - { - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP1, 0); - stackptr += sizeof(sljit_sw); - } - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), srcw[count]); - tmp1empty = FALSE; - tmp1next = FALSE; - } - else - { - if (!tmp2empty) - { - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP2, 0); - stackptr += sizeof(sljit_sw); - } - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), srcw[count]); - tmp2empty = FALSE; - tmp1next = TRUE; - } - } - else - { - if (tmp1next) - { - SLJIT_ASSERT(!tmp1empty); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), srcw[count], TMP1, 0); - tmp1empty = stackptr >= stacktop; - if (!tmp1empty) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), stackptr); - stackptr += sizeof(sljit_sw); - } - tmp1next = FALSE; - } - else - { - SLJIT_ASSERT(!tmp2empty); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), srcw[count], TMP2, 0); - tmp2empty = stackptr >= stacktop; - if (!tmp2empty) - { - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), stackptr); - stackptr += sizeof(sljit_sw); - } - tmp1next = TRUE; - } - } - } - } -while (status != end); - -if (save) - { - if (tmp1next) - { - if (!tmp1empty) - { - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP1, 0); - stackptr += sizeof(sljit_sw); - } - if (!tmp2empty) - { - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP2, 0); - stackptr += sizeof(sljit_sw); - } - } - else - { - if (!tmp2empty) - { - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP2, 0); - stackptr += sizeof(sljit_sw); - } - if (!tmp1empty) - { - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP1, 0); - stackptr += sizeof(sljit_sw); - } - } - } -SLJIT_ASSERT(cc == ccend && stackptr == stacktop && (save || (tmp1empty && tmp2empty))); -} - -static SLJIT_INLINE pcre_uchar *set_then_offsets(compiler_common *common, pcre_uchar *cc, sljit_u8 *current_offset) -{ -pcre_uchar *end = bracketend(cc); -BOOL has_alternatives = cc[GET(cc, 1)] == OP_ALT; - -/* Assert captures then. */ -if (*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) - current_offset = NULL; -/* Conditional block does not. */ -if (*cc == OP_COND || *cc == OP_SCOND) - has_alternatives = FALSE; - -cc = next_opcode(common, cc); -if (has_alternatives) - current_offset = common->then_offsets + (cc - common->start); - -while (cc < end) - { - if ((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) || (*cc >= OP_ONCE && *cc <= OP_SCOND)) - cc = set_then_offsets(common, cc, current_offset); - else - { - if (*cc == OP_ALT && has_alternatives) - current_offset = common->then_offsets + (cc + 1 + LINK_SIZE - common->start); - if (*cc >= OP_THEN && *cc <= OP_THEN_ARG && current_offset != NULL) - *current_offset = 1; - cc = next_opcode(common, cc); - } - } - -return end; -} - -#undef CASE_ITERATOR_PRIVATE_DATA_1 -#undef CASE_ITERATOR_PRIVATE_DATA_2A -#undef CASE_ITERATOR_PRIVATE_DATA_2B -#undef CASE_ITERATOR_TYPE_PRIVATE_DATA_1 -#undef CASE_ITERATOR_TYPE_PRIVATE_DATA_2A -#undef CASE_ITERATOR_TYPE_PRIVATE_DATA_2B - -static SLJIT_INLINE BOOL is_powerof2(unsigned int value) -{ -return (value & (value - 1)) == 0; -} - -static SLJIT_INLINE void set_jumps(jump_list *list, struct sljit_label *label) -{ -while (list) - { - /* sljit_set_label is clever enough to do nothing - if either the jump or the label is NULL. */ - SET_LABEL(list->jump, label); - list = list->next; - } -} - -static SLJIT_INLINE void add_jump(struct sljit_compiler *compiler, jump_list **list, struct sljit_jump *jump) -{ -jump_list *list_item = sljit_alloc_memory(compiler, sizeof(jump_list)); -if (list_item) - { - list_item->next = *list; - list_item->jump = jump; - *list = list_item; - } -} - -static void add_stub(compiler_common *common, struct sljit_jump *start) -{ -DEFINE_COMPILER; -stub_list *list_item = sljit_alloc_memory(compiler, sizeof(stub_list)); - -if (list_item) - { - list_item->start = start; - list_item->quit = LABEL(); - list_item->next = common->stubs; - common->stubs = list_item; - } -} - -static void flush_stubs(compiler_common *common) -{ -DEFINE_COMPILER; -stub_list *list_item = common->stubs; - -while (list_item) - { - JUMPHERE(list_item->start); - add_jump(compiler, &common->stackalloc, JUMP(SLJIT_FAST_CALL)); - JUMPTO(SLJIT_JUMP, list_item->quit); - list_item = list_item->next; - } -common->stubs = NULL; -} - -static void add_label_addr(compiler_common *common, sljit_uw *update_addr) -{ -DEFINE_COMPILER; -label_addr_list *label_addr; - -label_addr = sljit_alloc_memory(compiler, sizeof(label_addr_list)); -if (label_addr == NULL) - return; -label_addr->label = LABEL(); -label_addr->update_addr = update_addr; -label_addr->next = common->label_addrs; -common->label_addrs = label_addr; -} - -static SLJIT_INLINE void count_match(compiler_common *common) -{ -DEFINE_COMPILER; - -OP2(SLJIT_SUB | SLJIT_SET_E, COUNT_MATCH, 0, COUNT_MATCH, 0, SLJIT_IMM, 1); -add_jump(compiler, &common->calllimit, JUMP(SLJIT_ZERO)); -} - -static SLJIT_INLINE void allocate_stack(compiler_common *common, int size) -{ -/* May destroy all locals and registers except TMP2. */ -DEFINE_COMPILER; - -SLJIT_ASSERT(size > 0); -OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_sw)); -#ifdef DESTROY_REGISTERS -OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 12345); -OP1(SLJIT_MOV, TMP3, 0, TMP1, 0); -OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0); -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, TMP1, 0); -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP1, 0); -#endif -add_stub(common, CMP(SLJIT_GREATER, STACK_TOP, 0, STACK_LIMIT, 0)); -} - -static SLJIT_INLINE void free_stack(compiler_common *common, int size) -{ -DEFINE_COMPILER; - -SLJIT_ASSERT(size > 0); -OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_sw)); -} - -static sljit_uw * allocate_read_only_data(compiler_common *common, sljit_uw size) -{ -DEFINE_COMPILER; -sljit_uw *result; - -if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) - return NULL; - -result = (sljit_uw *)SLJIT_MALLOC(size + sizeof(sljit_uw), compiler->allocator_data); -if (SLJIT_UNLIKELY(result == NULL)) - { - sljit_set_compiler_memory_error(compiler); - return NULL; - } - -*(void**)result = common->read_only_data_head; -common->read_only_data_head = (void *)result; -return result + 1; -} - -static void free_read_only_data(void *current, void *allocator_data) -{ -void *next; - -SLJIT_UNUSED_ARG(allocator_data); - -while (current != NULL) - { - next = *(void**)current; - SLJIT_FREE(current, allocator_data); - current = next; - } -} - -static SLJIT_INLINE void reset_ovector(compiler_common *common, int length) -{ -DEFINE_COMPILER; -struct sljit_label *loop; -int i; - -/* At this point we can freely use all temporary registers. */ -SLJIT_ASSERT(length > 1); -/* TMP1 returns with begin - 1. */ -OP2(SLJIT_SUB, SLJIT_R0, 0, SLJIT_MEM1(SLJIT_S0), SLJIT_OFFSETOF(jit_arguments, begin), SLJIT_IMM, IN_UCHARS(1)); -if (length < 8) - { - for (i = 1; i < length; i++) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(i), SLJIT_R0, 0); - } -else - { - GET_LOCAL_BASE(SLJIT_R1, 0, OVECTOR_START); - OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_IMM, length - 1); - loop = LABEL(); - OP1(SLJIT_MOVU, SLJIT_MEM1(SLJIT_R1), sizeof(sljit_sw), SLJIT_R0, 0); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, 1); - JUMPTO(SLJIT_NOT_ZERO, loop); - } -} - -static SLJIT_INLINE void reset_fast_fail(compiler_common *common) -{ -DEFINE_COMPILER; -sljit_s32 i; - -SLJIT_ASSERT(common->fast_fail_start_ptr < common->fast_fail_end_ptr); - -OP2(SLJIT_SUB, TMP1, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -for (i = common->fast_fail_start_ptr; i < common->fast_fail_end_ptr; i += sizeof(sljit_sw)) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), i, TMP1, 0); -} - -static SLJIT_INLINE void do_reset_match(compiler_common *common, int length) -{ -DEFINE_COMPILER; -struct sljit_label *loop; -int i; - -SLJIT_ASSERT(length > 1); -/* OVECTOR(1) contains the "string begin - 1" constant. */ -if (length > 2) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1)); -if (length < 8) - { - for (i = 2; i < length; i++) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(i), TMP1, 0); - } -else - { - GET_LOCAL_BASE(TMP2, 0, OVECTOR_START + sizeof(sljit_sw)); - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_IMM, length - 2); - loop = LABEL(); - OP1(SLJIT_MOVU, SLJIT_MEM1(TMP2), sizeof(sljit_sw), TMP1, 0); - OP2(SLJIT_SUB | SLJIT_SET_E, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 1); - JUMPTO(SLJIT_NOT_ZERO, loop); - } - -OP1(SLJIT_MOV, STACK_TOP, 0, ARGUMENTS, 0); -if (common->mark_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, SLJIT_IMM, 0); -if (common->control_head_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0); -OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), SLJIT_OFFSETOF(jit_arguments, stack)); -OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->start_ptr); -OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), SLJIT_OFFSETOF(struct sljit_stack, base)); -} - -static sljit_sw SLJIT_CALL do_search_mark(sljit_sw *current, const pcre_uchar *skip_arg) -{ -while (current != NULL) - { - switch (current[-2]) - { - case type_then_trap: - break; - - case type_mark: - if (STRCMP_UC_UC(skip_arg, (pcre_uchar *)current[-3]) == 0) - return current[-4]; - break; - - default: - SLJIT_ASSERT_STOP(); - break; - } - SLJIT_ASSERT(current > (sljit_sw*)current[-1]); - current = (sljit_sw*)current[-1]; - } -return -1; -} - -static SLJIT_INLINE void copy_ovector(compiler_common *common, int topbracket) -{ -DEFINE_COMPILER; -struct sljit_label *loop; -struct sljit_jump *early_quit; - -/* At this point we can freely use all registers. */ -OP1(SLJIT_MOV, SLJIT_S2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1)); -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(1), STR_PTR, 0); - -OP1(SLJIT_MOV, SLJIT_R0, 0, ARGUMENTS, 0); -if (common->mark_ptr != 0) - OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr); -OP1(SLJIT_MOV_S32, SLJIT_R1, 0, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, offset_count)); -if (common->mark_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, mark_ptr), SLJIT_R2, 0); -OP2(SLJIT_SUB, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, offsets), SLJIT_IMM, sizeof(int)); -OP1(SLJIT_MOV, SLJIT_R0, 0, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, begin)); -GET_LOCAL_BASE(SLJIT_S0, 0, OVECTOR_START); -/* Unlikely, but possible */ -early_quit = CMP(SLJIT_EQUAL, SLJIT_R1, 0, SLJIT_IMM, 0); -loop = LABEL(); -OP2(SLJIT_SUB, SLJIT_S1, 0, SLJIT_MEM1(SLJIT_S0), 0, SLJIT_R0, 0); -OP2(SLJIT_ADD, SLJIT_S0, 0, SLJIT_S0, 0, SLJIT_IMM, sizeof(sljit_sw)); -/* Copy the integer value to the output buffer */ -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 -OP2(SLJIT_ASHR, SLJIT_S1, 0, SLJIT_S1, 0, SLJIT_IMM, UCHAR_SHIFT); -#endif -OP1(SLJIT_MOVU_S32, SLJIT_MEM1(SLJIT_R2), sizeof(int), SLJIT_S1, 0); -OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_R1, 0, SLJIT_R1, 0, SLJIT_IMM, 1); -JUMPTO(SLJIT_NOT_ZERO, loop); -JUMPHERE(early_quit); - -/* Calculate the return value, which is the maximum ovector value. */ -if (topbracket > 1) - { - GET_LOCAL_BASE(SLJIT_R0, 0, OVECTOR_START + topbracket * 2 * sizeof(sljit_sw)); - OP1(SLJIT_MOV, SLJIT_R1, 0, SLJIT_IMM, topbracket + 1); - - /* OVECTOR(0) is never equal to SLJIT_S2. */ - loop = LABEL(); - OP1(SLJIT_MOVU, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_R0), -(2 * (sljit_sw)sizeof(sljit_sw))); - OP2(SLJIT_SUB, SLJIT_R1, 0, SLJIT_R1, 0, SLJIT_IMM, 1); - CMPTO(SLJIT_EQUAL, SLJIT_R2, 0, SLJIT_S2, 0, loop); - OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_R1, 0); - } -else - OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, 1); -} - -static SLJIT_INLINE void return_with_partial_match(compiler_common *common, struct sljit_label *quit) -{ -DEFINE_COMPILER; -struct sljit_jump *jump; - -SLJIT_COMPILE_ASSERT(STR_END == SLJIT_S1, str_end_must_be_saved_reg2); -SLJIT_ASSERT(common->start_used_ptr != 0 && common->start_ptr != 0 - && (common->mode == JIT_PARTIAL_SOFT_COMPILE ? common->hit_start != 0 : common->hit_start == 0)); - -OP1(SLJIT_MOV, SLJIT_R1, 0, ARGUMENTS, 0); -OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_PARTIAL); -OP1(SLJIT_MOV_S32, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_R1), SLJIT_OFFSETOF(jit_arguments, real_offset_count)); -CMPTO(SLJIT_SIG_LESS, SLJIT_R2, 0, SLJIT_IMM, 2, quit); - -/* Store match begin and end. */ -OP1(SLJIT_MOV, SLJIT_S0, 0, SLJIT_MEM1(SLJIT_R1), SLJIT_OFFSETOF(jit_arguments, begin)); -OP1(SLJIT_MOV, SLJIT_R1, 0, SLJIT_MEM1(SLJIT_R1), SLJIT_OFFSETOF(jit_arguments, offsets)); - -jump = CMP(SLJIT_SIG_LESS, SLJIT_R2, 0, SLJIT_IMM, 3); -OP2(SLJIT_SUB, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), common->mode == JIT_PARTIAL_HARD_COMPILE ? common->start_ptr : (common->hit_start + (int)sizeof(sljit_sw)), SLJIT_S0, 0); -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 -OP2(SLJIT_ASHR, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, UCHAR_SHIFT); -#endif -OP1(SLJIT_MOV_S32, SLJIT_MEM1(SLJIT_R1), 2 * sizeof(int), SLJIT_R2, 0); -JUMPHERE(jump); - -OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), common->mode == JIT_PARTIAL_HARD_COMPILE ? common->start_used_ptr : common->hit_start); -OP2(SLJIT_SUB, SLJIT_S1, 0, STR_END, 0, SLJIT_S0, 0); -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 -OP2(SLJIT_ASHR, SLJIT_S1, 0, SLJIT_S1, 0, SLJIT_IMM, UCHAR_SHIFT); -#endif -OP1(SLJIT_MOV_S32, SLJIT_MEM1(SLJIT_R1), sizeof(int), SLJIT_S1, 0); - -OP2(SLJIT_SUB, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_S0, 0); -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 -OP2(SLJIT_ASHR, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, UCHAR_SHIFT); -#endif -OP1(SLJIT_MOV_S32, SLJIT_MEM1(SLJIT_R1), 0, SLJIT_R2, 0); - -JUMPTO(SLJIT_JUMP, quit); -} - -static SLJIT_INLINE void check_start_used_ptr(compiler_common *common) -{ -/* May destroy TMP1. */ -DEFINE_COMPILER; -struct sljit_jump *jump; - -if (common->mode == JIT_PARTIAL_SOFT_COMPILE) - { - /* The value of -1 must be kept for start_used_ptr! */ - OP2(SLJIT_ADD, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, SLJIT_IMM, 1); - /* Jumps if start_used_ptr < STR_PTR, or start_used_ptr == -1. Although overwriting - is not necessary if start_used_ptr == STR_PTR, it does not hurt as well. */ - jump = CMP(SLJIT_LESS_EQUAL, TMP1, 0, STR_PTR, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0); - JUMPHERE(jump); - } -else if (common->mode == JIT_PARTIAL_HARD_COMPILE) - { - jump = CMP(SLJIT_LESS_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0); - JUMPHERE(jump); - } -} - -static SLJIT_INLINE BOOL char_has_othercase(compiler_common *common, pcre_uchar *cc) -{ -/* Detects if the character has an othercase. */ -unsigned int c; - -#ifdef SUPPORT_UTF -if (common->utf) - { - GETCHAR(c, cc); - if (c > 127) - { -#ifdef SUPPORT_UCP - return c != UCD_OTHERCASE(c); -#else - return FALSE; -#endif - } -#ifndef COMPILE_PCRE8 - return common->fcc[c] != c; -#endif - } -else -#endif - c = *cc; -return MAX_255(c) ? common->fcc[c] != c : FALSE; -} - -static SLJIT_INLINE unsigned int char_othercase(compiler_common *common, unsigned int c) -{ -/* Returns with the othercase. */ -#ifdef SUPPORT_UTF -if (common->utf && c > 127) - { -#ifdef SUPPORT_UCP - return UCD_OTHERCASE(c); -#else - return c; -#endif - } -#endif -return TABLE_GET(c, common->fcc, c); -} - -static unsigned int char_get_othercase_bit(compiler_common *common, pcre_uchar *cc) -{ -/* Detects if the character and its othercase has only 1 bit difference. */ -unsigned int c, oc, bit; -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 -int n; -#endif - -#ifdef SUPPORT_UTF -if (common->utf) - { - GETCHAR(c, cc); - if (c <= 127) - oc = common->fcc[c]; - else - { -#ifdef SUPPORT_UCP - oc = UCD_OTHERCASE(c); -#else - oc = c; -#endif - } - } -else - { - c = *cc; - oc = TABLE_GET(c, common->fcc, c); - } -#else -c = *cc; -oc = TABLE_GET(c, common->fcc, c); -#endif - -SLJIT_ASSERT(c != oc); - -bit = c ^ oc; -/* Optimized for English alphabet. */ -if (c <= 127 && bit == 0x20) - return (0 << 8) | 0x20; - -/* Since c != oc, they must have at least 1 bit difference. */ -if (!is_powerof2(bit)) - return 0; - -#if defined COMPILE_PCRE8 - -#ifdef SUPPORT_UTF -if (common->utf && c > 127) - { - n = GET_EXTRALEN(*cc); - while ((bit & 0x3f) == 0) - { - n--; - bit >>= 6; - } - return (n << 8) | bit; - } -#endif /* SUPPORT_UTF */ -return (0 << 8) | bit; - -#elif defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - -#ifdef SUPPORT_UTF -if (common->utf && c > 65535) - { - if (bit >= (1 << 10)) - bit >>= 10; - else - return (bit < 256) ? ((2 << 8) | bit) : ((3 << 8) | (bit >> 8)); - } -#endif /* SUPPORT_UTF */ -return (bit < 256) ? ((0 << 8) | bit) : ((1 << 8) | (bit >> 8)); - -#endif /* COMPILE_PCRE[8|16|32] */ -} - -static void check_partial(compiler_common *common, BOOL force) -{ -/* Checks whether a partial matching is occurred. Does not modify registers. */ -DEFINE_COMPILER; -struct sljit_jump *jump = NULL; - -SLJIT_ASSERT(!force || common->mode != JIT_COMPILE); - -if (common->mode == JIT_COMPILE) - return; - -if (!force) - jump = CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0); -else if (common->mode == JIT_PARTIAL_SOFT_COMPILE) - jump = CMP(SLJIT_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, SLJIT_IMM, -1); - -if (common->mode == JIT_PARTIAL_SOFT_COMPILE) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, 0); -else - { - if (common->partialmatchlabel != NULL) - JUMPTO(SLJIT_JUMP, common->partialmatchlabel); - else - add_jump(compiler, &common->partialmatch, JUMP(SLJIT_JUMP)); - } - -if (jump != NULL) - JUMPHERE(jump); -} - -static void check_str_end(compiler_common *common, jump_list **end_reached) -{ -/* Does not affect registers. Usually used in a tight spot. */ -DEFINE_COMPILER; -struct sljit_jump *jump; - -if (common->mode == JIT_COMPILE) - { - add_jump(compiler, end_reached, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); - return; - } - -jump = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0); -if (common->mode == JIT_PARTIAL_SOFT_COMPILE) - { - add_jump(compiler, end_reached, CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, 0); - add_jump(compiler, end_reached, JUMP(SLJIT_JUMP)); - } -else - { - add_jump(compiler, end_reached, CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0)); - if (common->partialmatchlabel != NULL) - JUMPTO(SLJIT_JUMP, common->partialmatchlabel); - else - add_jump(compiler, &common->partialmatch, JUMP(SLJIT_JUMP)); - } -JUMPHERE(jump); -} - -static void detect_partial_match(compiler_common *common, jump_list **backtracks) -{ -DEFINE_COMPILER; -struct sljit_jump *jump; - -if (common->mode == JIT_COMPILE) - { - add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); - return; - } - -/* Partial matching mode. */ -jump = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0); -add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0)); -if (common->mode == JIT_PARTIAL_SOFT_COMPILE) - { - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, 0); - add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); - } -else - { - if (common->partialmatchlabel != NULL) - JUMPTO(SLJIT_JUMP, common->partialmatchlabel); - else - add_jump(compiler, &common->partialmatch, JUMP(SLJIT_JUMP)); - } -JUMPHERE(jump); -} - -static void peek_char(compiler_common *common, sljit_u32 max) -{ -/* Reads the character into TMP1, keeps STR_PTR. -Does not check STR_END. TMP2 Destroyed. */ -DEFINE_COMPILER; -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 -struct sljit_jump *jump; -#endif - -SLJIT_UNUSED_ARG(max); - -OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 -if (common->utf) - { - if (max < 128) return; - - jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - add_jump(compiler, &common->utfreadchar, JUMP(SLJIT_FAST_CALL)); - OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0); - JUMPHERE(jump); - } -#endif /* SUPPORT_UTF && !COMPILE_PCRE32 */ - -#if defined SUPPORT_UTF && defined COMPILE_PCRE16 -if (common->utf) - { - if (max < 0xd800) return; - - OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800); - jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xdc00 - 0xd800 - 1); - /* TMP2 contains the high surrogate. */ - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x40); - OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 10); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3ff); - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - JUMPHERE(jump); - } -#endif -} - -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 - -static BOOL is_char7_bitset(const sljit_u8 *bitset, BOOL nclass) -{ -/* Tells whether the character codes below 128 are enough -to determine a match. */ -const sljit_u8 value = nclass ? 0xff : 0; -const sljit_u8 *end = bitset + 32; - -bitset += 16; -do - { - if (*bitset++ != value) - return FALSE; - } -while (bitset < end); -return TRUE; -} - -static void read_char7_type(compiler_common *common, BOOL full_read) -{ -/* Reads the precise character type of a character into TMP1, if the character -is less than 128. Otherwise it returns with zero. Does not check STR_END. The -full_read argument tells whether characters above max are accepted or not. */ -DEFINE_COMPILER; -struct sljit_jump *jump; - -SLJIT_ASSERT(common->utf); - -OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), 0); -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - -OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes); - -if (full_read) - { - jump = CMP(SLJIT_LESS, TMP2, 0, SLJIT_IMM, 0xc0); - OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(utf8_table4) - 0xc0); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); - JUMPHERE(jump); - } -} - -#endif /* SUPPORT_UTF && COMPILE_PCRE8 */ - -static void read_char_range(compiler_common *common, sljit_u32 min, sljit_u32 max, BOOL update_str_ptr) -{ -/* Reads the precise value of a character into TMP1, if the character is -between min and max (c >= min && c <= max). Otherwise it returns with a value -outside the range. Does not check STR_END. */ -DEFINE_COMPILER; -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 -struct sljit_jump *jump; -#endif -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 -struct sljit_jump *jump2; -#endif - -SLJIT_UNUSED_ARG(update_str_ptr); -SLJIT_UNUSED_ARG(min); -SLJIT_UNUSED_ARG(max); -SLJIT_ASSERT(min <= max); - -OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 -if (common->utf) - { - if (max < 128 && !update_str_ptr) return; - - jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0); - if (min >= 0x10000) - { - OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xf0); - if (update_str_ptr) - OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - jump2 = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0x7); - OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); - OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); - OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(2)); - if (!update_str_ptr) - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(3)); - OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); - OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - JUMPHERE(jump2); - if (update_str_ptr) - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, RETURN_ADDR, 0); - } - else if (min >= 0x800 && max <= 0xffff) - { - OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xe0); - if (update_str_ptr) - OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - jump2 = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xf); - OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); - if (!update_str_ptr) - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); - OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); - OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - JUMPHERE(jump2); - if (update_str_ptr) - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, RETURN_ADDR, 0); - } - else if (max >= 0x800) - add_jump(compiler, (max < 0x10000) ? &common->utfreadchar16 : &common->utfreadchar, JUMP(SLJIT_FAST_CALL)); - else if (max < 128) - { - OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); - } - else - { - OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - if (!update_str_ptr) - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - else - OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); - OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); - OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - if (update_str_ptr) - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, RETURN_ADDR, 0); - } - JUMPHERE(jump); - } -#endif - -#if defined SUPPORT_UTF && defined COMPILE_PCRE16 -if (common->utf) - { - if (max >= 0x10000) - { - OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800); - jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xdc00 - 0xd800 - 1); - /* TMP2 contains the high surrogate. */ - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x40); - OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 10); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3ff); - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - JUMPHERE(jump); - return; - } - - if (max < 0xd800 && !update_str_ptr) return; - - /* Skip low surrogate if necessary. */ - OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800); - jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xdc00 - 0xd800 - 1); - if (update_str_ptr) - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - if (max >= 0xd800) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0x10000); - JUMPHERE(jump); - } -#endif -} - -static SLJIT_INLINE void read_char(compiler_common *common) -{ -read_char_range(common, 0, READ_CHAR_MAX, TRUE); -} - -static void read_char8_type(compiler_common *common, BOOL update_str_ptr) -{ -/* Reads the character type into TMP1, updates STR_PTR. Does not check STR_END. */ -DEFINE_COMPILER; -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 -struct sljit_jump *jump; -#endif -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 -struct sljit_jump *jump2; -#endif - -SLJIT_UNUSED_ARG(update_str_ptr); - -OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), 0); -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 -if (common->utf) - { - /* This can be an extra read in some situations, but hopefully - it is needed in most cases. */ - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes); - jump = CMP(SLJIT_LESS, TMP2, 0, SLJIT_IMM, 0xc0); - if (!update_str_ptr) - { - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); - OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); - OP2(SLJIT_OR, TMP2, 0, TMP2, 0, TMP1, 0); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); - jump2 = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 255); - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes); - JUMPHERE(jump2); - } - else - add_jump(compiler, &common->utfreadtype8, JUMP(SLJIT_FAST_CALL)); - JUMPHERE(jump); - return; - } -#endif /* SUPPORT_UTF && COMPILE_PCRE8 */ - -#if !defined COMPILE_PCRE8 -/* The ctypes array contains only 256 values. */ -OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); -jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 255); -#endif -OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes); -#if !defined COMPILE_PCRE8 -JUMPHERE(jump); -#endif - -#if defined SUPPORT_UTF && defined COMPILE_PCRE16 -if (common->utf && update_str_ptr) - { - /* Skip low surrogate if necessary. */ - OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xd800); - jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xdc00 - 0xd800 - 1); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - JUMPHERE(jump); - } -#endif /* SUPPORT_UTF && COMPILE_PCRE16 */ -} - -static void skip_char_back(compiler_common *common) -{ -/* Goes one character back. Affects STR_PTR and TMP1. Does not check begin. */ -DEFINE_COMPILER; -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 -#if defined COMPILE_PCRE8 -struct sljit_label *label; - -if (common->utf) - { - label = LABEL(); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), -IN_UCHARS(1)); - OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0); - CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0x80, label); - return; - } -#elif defined COMPILE_PCRE16 -if (common->utf) - { - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), -IN_UCHARS(1)); - OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - /* Skip low surrogate if necessary. */ - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xdc00); - OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); - OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); - OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP1, 0); - return; - } -#endif /* COMPILE_PCRE[8|16] */ -#endif /* SUPPORT_UTF && !COMPILE_PCRE32 */ -OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -} - -static void check_newlinechar(compiler_common *common, int nltype, jump_list **backtracks, BOOL jumpifmatch) -{ -/* Character comes in TMP1. Checks if it is a newline. TMP2 may be destroyed. */ -DEFINE_COMPILER; -struct sljit_jump *jump; - -if (nltype == NLTYPE_ANY) - { - add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL)); - add_jump(compiler, backtracks, JUMP(jumpifmatch ? SLJIT_NOT_ZERO : SLJIT_ZERO)); - } -else if (nltype == NLTYPE_ANYCRLF) - { - if (jumpifmatch) - { - add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR)); - add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL)); - } - else - { - jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL)); - JUMPHERE(jump); - } - } -else - { - SLJIT_ASSERT(nltype == NLTYPE_FIXED && common->newline < 256); - add_jump(compiler, backtracks, CMP(jumpifmatch ? SLJIT_EQUAL : SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, common->newline)); - } -} - -#ifdef SUPPORT_UTF - -#if defined COMPILE_PCRE8 -static void do_utfreadchar(compiler_common *common) -{ -/* Fast decoding a UTF-8 character. TMP1 contains the first byte -of the character (>= 0xc0). Return char value in TMP1, length in TMP2. */ -DEFINE_COMPILER; -struct sljit_jump *jump; - -sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); -OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); -OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); -OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); -OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); -OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - -/* Searching for the first zero. */ -OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); -jump = JUMP(SLJIT_NOT_ZERO); -/* Two byte sequence. */ -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, IN_UCHARS(2)); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); - -JUMPHERE(jump); -OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); -OP2(SLJIT_XOR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x800); -OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); -OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); -OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - -OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x10000); -jump = JUMP(SLJIT_NOT_ZERO); -/* Three byte sequence. */ -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); -OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, IN_UCHARS(3)); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); - -/* Four byte sequence. */ -JUMPHERE(jump); -OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(2)); -OP2(SLJIT_XOR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x10000); -OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(3)); -OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); -OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); -OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, IN_UCHARS(4)); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); -} - -static void do_utfreadchar16(compiler_common *common) -{ -/* Fast decoding a UTF-8 character. TMP1 contains the first byte -of the character (>= 0xc0). Return value in TMP1. */ -DEFINE_COMPILER; -struct sljit_jump *jump; - -sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); -OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); -OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); -OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); -OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); -OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - -/* Searching for the first zero. */ -OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); -jump = JUMP(SLJIT_NOT_ZERO); -/* Two byte sequence. */ -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); - -JUMPHERE(jump); -OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x400); -OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_NOT_ZERO); -/* This code runs only in 8 bit mode. No need to shift the value. */ -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); -OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); -OP2(SLJIT_XOR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x800); -OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); -OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); -OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); -/* Three byte sequence. */ -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); -} - -static void do_utfreadtype8(compiler_common *common) -{ -/* Fast decoding a UTF-8 character type. TMP2 contains the first byte -of the character (>= 0xc0). Return value in TMP1. */ -DEFINE_COMPILER; -struct sljit_jump *jump; -struct sljit_jump *compare; - -sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); - -OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0x20); -jump = JUMP(SLJIT_NOT_ZERO); -/* Two byte sequence. */ -OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x1f); -/* The upper 5 bits are known at this point. */ -compare = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0x3); -OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6); -OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); -OP2(SLJIT_OR, TMP2, 0, TMP2, 0, TMP1, 0); -OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); - -JUMPHERE(compare); -OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); - -/* We only have types for characters less than 256. */ -JUMPHERE(jump); -OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(utf8_table4) - 0xc0); -OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); -} - -#endif /* COMPILE_PCRE8 */ - -#endif /* SUPPORT_UTF */ - -#ifdef SUPPORT_UCP - -/* UCD_BLOCK_SIZE must be 128 (see the assert below). */ -#define UCD_BLOCK_MASK 127 -#define UCD_BLOCK_SHIFT 7 - -static void do_getucd(compiler_common *common) -{ -/* Search the UCD record for the character comes in TMP1. -Returns chartype in TMP1 and UCD offset in TMP2. */ -DEFINE_COMPILER; - -SLJIT_ASSERT(UCD_BLOCK_SIZE == 128 && sizeof(ucd_record) == 8); - -sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); -OP2(SLJIT_LSHR, TMP2, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_SHIFT); -OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_stage1)); -OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_MASK); -OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCD_BLOCK_SHIFT); -OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); -OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_stage2)); -OP1(SLJIT_MOV_U16, TMP2, 0, SLJIT_MEM2(TMP2, TMP1), 1); -OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype)); -OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); -} -#endif - -static SLJIT_INLINE struct sljit_label *mainloop_entry(compiler_common *common, BOOL hascrorlf) -{ -DEFINE_COMPILER; -struct sljit_label *mainloop; -struct sljit_label *newlinelabel = NULL; -struct sljit_jump *start; -struct sljit_jump *end = NULL; -struct sljit_jump *end2 = NULL; -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 -struct sljit_jump *singlechar; -#endif -jump_list *newline = NULL; -BOOL newlinecheck = FALSE; -BOOL readuchar = FALSE; - -if (!(hascrorlf || (common->match_end_ptr != 0)) && - (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF || common->newline > 255)) - newlinecheck = TRUE; - -if (common->match_end_ptr != 0) - { - /* Search for the end of the first line. */ - OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0); - - if (common->nltype == NLTYPE_FIXED && common->newline > 255) - { - mainloop = LABEL(); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - end = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1)); - OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, mainloop); - CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff, mainloop); - JUMPHERE(end); - OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - } - else - { - end = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); - mainloop = LABEL(); - /* Continual stores does not cause data dependency. */ - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, STR_PTR, 0); - read_char_range(common, common->nlmin, common->nlmax, TRUE); - check_newlinechar(common, common->nltype, &newline, TRUE); - CMPTO(SLJIT_LESS, STR_PTR, 0, STR_END, 0, mainloop); - JUMPHERE(end); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, STR_PTR, 0); - set_jumps(newline, LABEL()); - } - - OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0); - } - -start = JUMP(SLJIT_JUMP); - -if (newlinecheck) - { - newlinelabel = LABEL(); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - end = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, common->newline & 0xff); - OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT); -#endif - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); - end2 = JUMP(SLJIT_JUMP); - } - -mainloop = LABEL(); - -/* Increasing the STR_PTR here requires one less jump in the most common case. */ -#ifdef SUPPORT_UTF -if (common->utf) readuchar = TRUE; -#endif -if (newlinecheck) readuchar = TRUE; - -if (readuchar) - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); - -if (newlinecheck) - CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, newlinelabel); - -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 -#if defined COMPILE_PCRE8 -if (common->utf) - { - singlechar = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0); - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); - JUMPHERE(singlechar); - } -#elif defined COMPILE_PCRE16 -if (common->utf) - { - singlechar = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800); - OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); - OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); - JUMPHERE(singlechar); - } -#endif /* COMPILE_PCRE[8|16] */ -#endif /* SUPPORT_UTF && !COMPILE_PCRE32 */ -JUMPHERE(start); - -if (newlinecheck) - { - JUMPHERE(end); - JUMPHERE(end2); - } - -return mainloop; -} - -#define MAX_N_CHARS 16 -#define MAX_DIFF_CHARS 6 - -static SLJIT_INLINE void add_prefix_char(pcre_uchar chr, pcre_uchar *chars) -{ -pcre_uchar i, len; - -len = chars[0]; -if (len == 255) - return; - -if (len == 0) - { - chars[0] = 1; - chars[1] = chr; - return; - } - -for (i = len; i > 0; i--) - if (chars[i] == chr) - return; - -if (len >= MAX_DIFF_CHARS - 1) - { - chars[0] = 255; - return; - } - -len++; -chars[len] = chr; -chars[0] = len; -} - -static int scan_prefix(compiler_common *common, pcre_uchar *cc, pcre_uchar *chars, int max_chars, sljit_u32 *rec_count) -{ -/* Recursive function, which scans prefix literals. */ -BOOL last, any, class, caseless; -int len, repeat, len_save, consumed = 0; -sljit_u32 chr; /* Any unicode character. */ -sljit_u8 *bytes, *bytes_end, byte; -pcre_uchar *alternative, *cc_save, *oc; -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 -pcre_uchar othercase[8]; -#elif defined SUPPORT_UTF && defined COMPILE_PCRE16 -pcre_uchar othercase[2]; -#else -pcre_uchar othercase[1]; -#endif - -repeat = 1; -while (TRUE) - { - if (*rec_count == 0) - return 0; - (*rec_count)--; - - last = TRUE; - any = FALSE; - class = FALSE; - caseless = FALSE; - - switch (*cc) - { - case OP_CHARI: - caseless = TRUE; - case OP_CHAR: - last = FALSE; - cc++; - break; - - case OP_SOD: - case OP_SOM: - case OP_SET_SOM: - case OP_NOT_WORD_BOUNDARY: - case OP_WORD_BOUNDARY: - case OP_EODN: - case OP_EOD: - case OP_CIRC: - case OP_CIRCM: - case OP_DOLL: - case OP_DOLLM: - /* Zero width assertions. */ - cc++; - continue; - - case OP_ASSERT: - case OP_ASSERT_NOT: - case OP_ASSERTBACK: - case OP_ASSERTBACK_NOT: - cc = bracketend(cc); - continue; - - case OP_PLUSI: - case OP_MINPLUSI: - case OP_POSPLUSI: - caseless = TRUE; - case OP_PLUS: - case OP_MINPLUS: - case OP_POSPLUS: - cc++; - break; - - case OP_EXACTI: - caseless = TRUE; - case OP_EXACT: - repeat = GET2(cc, 1); - last = FALSE; - cc += 1 + IMM2_SIZE; - break; - - case OP_QUERYI: - case OP_MINQUERYI: - case OP_POSQUERYI: - caseless = TRUE; - case OP_QUERY: - case OP_MINQUERY: - case OP_POSQUERY: - len = 1; - cc++; -#ifdef SUPPORT_UTF - if (common->utf && HAS_EXTRALEN(*cc)) len += GET_EXTRALEN(*cc); -#endif - max_chars = scan_prefix(common, cc + len, chars, max_chars, rec_count); - if (max_chars == 0) - return consumed; - last = FALSE; - break; - - case OP_KET: - cc += 1 + LINK_SIZE; - continue; - - case OP_ALT: - cc += GET(cc, 1); - continue; - - case OP_ONCE: - case OP_ONCE_NC: - case OP_BRA: - case OP_BRAPOS: - case OP_CBRA: - case OP_CBRAPOS: - alternative = cc + GET(cc, 1); - while (*alternative == OP_ALT) - { - max_chars = scan_prefix(common, alternative + 1 + LINK_SIZE, chars, max_chars, rec_count); - if (max_chars == 0) - return consumed; - alternative += GET(alternative, 1); - } - - if (*cc == OP_CBRA || *cc == OP_CBRAPOS) - cc += IMM2_SIZE; - cc += 1 + LINK_SIZE; - continue; - - case OP_CLASS: -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 - if (common->utf && !is_char7_bitset((const sljit_u8 *)(cc + 1), FALSE)) - return consumed; -#endif - class = TRUE; - break; - - case OP_NCLASS: -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (common->utf) return consumed; -#endif - class = TRUE; - break; - -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - case OP_XCLASS: -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (common->utf) return consumed; -#endif - any = TRUE; - cc += GET(cc, 1); - break; -#endif - - case OP_DIGIT: -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 - if (common->utf && !is_char7_bitset((const sljit_u8 *)common->ctypes - cbit_length + cbit_digit, FALSE)) - return consumed; -#endif - any = TRUE; - cc++; - break; - - case OP_WHITESPACE: -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 - if (common->utf && !is_char7_bitset((const sljit_u8 *)common->ctypes - cbit_length + cbit_space, FALSE)) - return consumed; -#endif - any = TRUE; - cc++; - break; - - case OP_WORDCHAR: -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 - if (common->utf && !is_char7_bitset((const sljit_u8 *)common->ctypes - cbit_length + cbit_word, FALSE)) - return consumed; -#endif - any = TRUE; - cc++; - break; - - case OP_NOT: - case OP_NOTI: - cc++; - /* Fall through. */ - case OP_NOT_DIGIT: - case OP_NOT_WHITESPACE: - case OP_NOT_WORDCHAR: - case OP_ANY: - case OP_ALLANY: -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (common->utf) return consumed; -#endif - any = TRUE; - cc++; - break; - -#ifdef SUPPORT_UTF - case OP_NOTPROP: - case OP_PROP: -#ifndef COMPILE_PCRE32 - if (common->utf) return consumed; -#endif - any = TRUE; - cc += 1 + 2; - break; -#endif - - case OP_TYPEEXACT: - repeat = GET2(cc, 1); - cc += 1 + IMM2_SIZE; - continue; - - case OP_NOTEXACT: - case OP_NOTEXACTI: -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (common->utf) return consumed; -#endif - any = TRUE; - repeat = GET2(cc, 1); - cc += 1 + IMM2_SIZE + 1; - break; - - default: - return consumed; - } - - if (any) - { - do - { - chars[0] = 255; - - consumed++; - if (--max_chars == 0) - return consumed; - chars += MAX_DIFF_CHARS; - } - while (--repeat > 0); - - repeat = 1; - continue; - } - - if (class) - { - bytes = (sljit_u8*) (cc + 1); - cc += 1 + 32 / sizeof(pcre_uchar); - - switch (*cc) - { - case OP_CRSTAR: - case OP_CRMINSTAR: - case OP_CRPOSSTAR: - case OP_CRQUERY: - case OP_CRMINQUERY: - case OP_CRPOSQUERY: - max_chars = scan_prefix(common, cc + 1, chars, max_chars, rec_count); - if (max_chars == 0) - return consumed; - break; - - default: - case OP_CRPLUS: - case OP_CRMINPLUS: - case OP_CRPOSPLUS: - break; - - case OP_CRRANGE: - case OP_CRMINRANGE: - case OP_CRPOSRANGE: - repeat = GET2(cc, 1); - if (repeat <= 0) - return consumed; - break; - } - - do - { - if (bytes[31] & 0x80) - chars[0] = 255; - else if (chars[0] != 255) - { - bytes_end = bytes + 32; - chr = 0; - do - { - byte = *bytes++; - SLJIT_ASSERT((chr & 0x7) == 0); - if (byte == 0) - chr += 8; - else - { - do - { - if ((byte & 0x1) != 0) - add_prefix_char(chr, chars); - byte >>= 1; - chr++; - } - while (byte != 0); - chr = (chr + 7) & ~7; - } - } - while (chars[0] != 255 && bytes < bytes_end); - bytes = bytes_end - 32; - } - - consumed++; - if (--max_chars == 0) - return consumed; - chars += MAX_DIFF_CHARS; - } - while (--repeat > 0); - - switch (*cc) - { - case OP_CRSTAR: - case OP_CRMINSTAR: - case OP_CRPOSSTAR: - return consumed; - - case OP_CRQUERY: - case OP_CRMINQUERY: - case OP_CRPOSQUERY: - cc++; - break; - - case OP_CRRANGE: - case OP_CRMINRANGE: - case OP_CRPOSRANGE: - if (GET2(cc, 1) != GET2(cc, 1 + IMM2_SIZE)) - return consumed; - cc += 1 + 2 * IMM2_SIZE; - break; - } - - repeat = 1; - continue; - } - - len = 1; -#ifdef SUPPORT_UTF - if (common->utf && HAS_EXTRALEN(*cc)) len += GET_EXTRALEN(*cc); -#endif - - if (caseless && char_has_othercase(common, cc)) - { -#ifdef SUPPORT_UTF - if (common->utf) - { - GETCHAR(chr, cc); - if ((int)PRIV(ord2utf)(char_othercase(common, chr), othercase) != len) - return consumed; - } - else -#endif - { - chr = *cc; - othercase[0] = TABLE_GET(chr, common->fcc, chr); - } - } - else - { - caseless = FALSE; - othercase[0] = 0; /* Stops compiler warning - PH */ - } - - len_save = len; - cc_save = cc; - while (TRUE) - { - oc = othercase; - do - { - chr = *cc; - add_prefix_char(*cc, chars); - - if (caseless) - add_prefix_char(*oc, chars); - - len--; - consumed++; - if (--max_chars == 0) - return consumed; - chars += MAX_DIFF_CHARS; - cc++; - oc++; - } - while (len > 0); - - if (--repeat == 0) - break; - - len = len_save; - cc = cc_save; - } - - repeat = 1; - if (last) - return consumed; - } -} - -#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) - -static sljit_s32 character_to_int32(pcre_uchar chr) -{ -sljit_s32 value = (sljit_s32)chr; -#if defined COMPILE_PCRE8 -#define SSE2_COMPARE_TYPE_INDEX 0 -return (value << 24) | (value << 16) | (value << 8) | value; -#elif defined COMPILE_PCRE16 -#define SSE2_COMPARE_TYPE_INDEX 1 -return (value << 16) | value; -#elif defined COMPILE_PCRE32 -#define SSE2_COMPARE_TYPE_INDEX 2 -return value; -#else -#error "Unsupported unit width" -#endif -} - -static SLJIT_INLINE void fast_forward_first_char2_sse2(compiler_common *common, pcre_uchar char1, pcre_uchar char2) -{ -DEFINE_COMPILER; -struct sljit_label *start; -struct sljit_jump *quit[3]; -struct sljit_jump *nomatch; -sljit_u8 instruction[8]; -sljit_s32 tmp1_ind = sljit_get_register_index(TMP1); -sljit_s32 tmp2_ind = sljit_get_register_index(TMP2); -sljit_s32 str_ptr_ind = sljit_get_register_index(STR_PTR); -BOOL load_twice = FALSE; -pcre_uchar bit; - -bit = char1 ^ char2; -if (!is_powerof2(bit)) - bit = 0; - -if ((char1 != char2) && bit == 0) - load_twice = TRUE; - -quit[0] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); - -/* First part (unaligned start) */ - -OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char1 | bit)); - -SLJIT_ASSERT(tmp1_ind < 8 && tmp2_ind == 1); - -/* MOVD xmm, r/m32 */ -instruction[0] = 0x66; -instruction[1] = 0x0f; -instruction[2] = 0x6e; -instruction[3] = 0xc0 | (2 << 3) | tmp1_ind; -sljit_emit_op_custom(compiler, instruction, 4); - -if (char1 != char2) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(bit != 0 ? bit : char2)); - - /* MOVD xmm, r/m32 */ - instruction[3] = 0xc0 | (3 << 3) | tmp1_ind; - sljit_emit_op_custom(compiler, instruction, 4); - } - -/* PSHUFD xmm1, xmm2/m128, imm8 */ -instruction[2] = 0x70; -instruction[3] = 0xc0 | (2 << 3) | 2; -instruction[4] = 0; -sljit_emit_op_custom(compiler, instruction, 5); - -if (char1 != char2) - { - /* PSHUFD xmm1, xmm2/m128, imm8 */ - instruction[3] = 0xc0 | (3 << 3) | 3; - instruction[4] = 0; - sljit_emit_op_custom(compiler, instruction, 5); - } - -OP2(SLJIT_AND, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 0xf); -OP2(SLJIT_AND, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, ~0xf); - -/* MOVDQA xmm1, xmm2/m128 */ -#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) - -if (str_ptr_ind < 8) - { - instruction[2] = 0x6f; - instruction[3] = (0 << 3) | str_ptr_ind; - sljit_emit_op_custom(compiler, instruction, 4); - - if (load_twice) - { - instruction[3] = (1 << 3) | str_ptr_ind; - sljit_emit_op_custom(compiler, instruction, 4); - } - } -else - { - instruction[1] = 0x41; - instruction[2] = 0x0f; - instruction[3] = 0x6f; - instruction[4] = (0 << 3) | (str_ptr_ind & 0x7); - sljit_emit_op_custom(compiler, instruction, 5); - - if (load_twice) - { - instruction[4] = (1 << 3) | str_ptr_ind; - sljit_emit_op_custom(compiler, instruction, 5); - } - instruction[1] = 0x0f; - } - -#else - -instruction[2] = 0x6f; -instruction[3] = (0 << 3) | str_ptr_ind; -sljit_emit_op_custom(compiler, instruction, 4); - -if (load_twice) - { - instruction[3] = (1 << 3) | str_ptr_ind; - sljit_emit_op_custom(compiler, instruction, 4); - } - -#endif - -if (bit != 0) - { - /* POR xmm1, xmm2/m128 */ - instruction[2] = 0xeb; - instruction[3] = 0xc0 | (0 << 3) | 3; - sljit_emit_op_custom(compiler, instruction, 4); - } - -/* PCMPEQB/W/D xmm1, xmm2/m128 */ -instruction[2] = 0x74 + SSE2_COMPARE_TYPE_INDEX; -instruction[3] = 0xc0 | (0 << 3) | 2; -sljit_emit_op_custom(compiler, instruction, 4); - -if (load_twice) - { - instruction[3] = 0xc0 | (1 << 3) | 3; - sljit_emit_op_custom(compiler, instruction, 4); - } - -/* PMOVMSKB reg, xmm */ -instruction[2] = 0xd7; -instruction[3] = 0xc0 | (tmp1_ind << 3) | 0; -sljit_emit_op_custom(compiler, instruction, 4); - -if (load_twice) - { - OP1(SLJIT_MOV, TMP3, 0, TMP2, 0); - instruction[3] = 0xc0 | (tmp2_ind << 3) | 1; - sljit_emit_op_custom(compiler, instruction, 4); - - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - OP1(SLJIT_MOV, TMP2, 0, TMP3, 0); - } - -OP2(SLJIT_ASHR, TMP1, 0, TMP1, 0, TMP2, 0); - -/* BSF r32, r/m32 */ -instruction[0] = 0x0f; -instruction[1] = 0xbc; -instruction[2] = 0xc0 | (tmp1_ind << 3) | tmp1_ind; -sljit_emit_op_custom(compiler, instruction, 3); - -nomatch = JUMP(SLJIT_ZERO); - -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); -quit[1] = JUMP(SLJIT_JUMP); - -JUMPHERE(nomatch); - -start = LABEL(); -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 16); -quit[2] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); - -/* Second part (aligned) */ - -instruction[0] = 0x66; -instruction[1] = 0x0f; - -/* MOVDQA xmm1, xmm2/m128 */ -#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) - -if (str_ptr_ind < 8) - { - instruction[2] = 0x6f; - instruction[3] = (0 << 3) | str_ptr_ind; - sljit_emit_op_custom(compiler, instruction, 4); - - if (load_twice) - { - instruction[3] = (1 << 3) | str_ptr_ind; - sljit_emit_op_custom(compiler, instruction, 4); - } - } -else - { - instruction[1] = 0x41; - instruction[2] = 0x0f; - instruction[3] = 0x6f; - instruction[4] = (0 << 3) | (str_ptr_ind & 0x7); - sljit_emit_op_custom(compiler, instruction, 5); - - if (load_twice) - { - instruction[4] = (1 << 3) | str_ptr_ind; - sljit_emit_op_custom(compiler, instruction, 5); - } - instruction[1] = 0x0f; - } - -#else - -instruction[2] = 0x6f; -instruction[3] = (0 << 3) | str_ptr_ind; -sljit_emit_op_custom(compiler, instruction, 4); - -if (load_twice) - { - instruction[3] = (1 << 3) | str_ptr_ind; - sljit_emit_op_custom(compiler, instruction, 4); - } - -#endif - -if (bit != 0) - { - /* POR xmm1, xmm2/m128 */ - instruction[2] = 0xeb; - instruction[3] = 0xc0 | (0 << 3) | 3; - sljit_emit_op_custom(compiler, instruction, 4); - } - -/* PCMPEQB/W/D xmm1, xmm2/m128 */ -instruction[2] = 0x74 + SSE2_COMPARE_TYPE_INDEX; -instruction[3] = 0xc0 | (0 << 3) | 2; -sljit_emit_op_custom(compiler, instruction, 4); - -if (load_twice) - { - instruction[3] = 0xc0 | (1 << 3) | 3; - sljit_emit_op_custom(compiler, instruction, 4); - } - -/* PMOVMSKB reg, xmm */ -instruction[2] = 0xd7; -instruction[3] = 0xc0 | (tmp1_ind << 3) | 0; -sljit_emit_op_custom(compiler, instruction, 4); - -if (load_twice) - { - instruction[3] = 0xc0 | (tmp2_ind << 3) | 1; - sljit_emit_op_custom(compiler, instruction, 4); - - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); - } - -/* BSF r32, r/m32 */ -instruction[0] = 0x0f; -instruction[1] = 0xbc; -instruction[2] = 0xc0 | (tmp1_ind << 3) | tmp1_ind; -sljit_emit_op_custom(compiler, instruction, 3); - -JUMPTO(SLJIT_ZERO, start); - -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); - -start = LABEL(); -SET_LABEL(quit[0], start); -SET_LABEL(quit[1], start); -SET_LABEL(quit[2], start); -} - -#undef SSE2_COMPARE_TYPE_INDEX - -#endif - -static void fast_forward_first_char2(compiler_common *common, pcre_uchar char1, pcre_uchar char2, sljit_s32 offset) -{ -DEFINE_COMPILER; -struct sljit_label *start; -struct sljit_jump *quit; -struct sljit_jump *found; -pcre_uchar mask; -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 -struct sljit_label *utf_start = NULL; -struct sljit_jump *utf_quit = NULL; -#endif -BOOL has_match_end = (common->match_end_ptr != 0); - -if (offset > 0) - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset)); - -if (has_match_end) - { - OP1(SLJIT_MOV, TMP3, 0, STR_END, 0); - - OP2(SLJIT_ADD, STR_END, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, SLJIT_IMM, IN_UCHARS(offset + 1)); -#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) - if (sljit_x86_is_cmov_available()) - { - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_END, 0, TMP3, 0); - sljit_x86_emit_cmov(compiler, SLJIT_GREATER, STR_END, TMP3, 0); - } -#endif - { - quit = CMP(SLJIT_LESS_EQUAL, STR_END, 0, TMP3, 0); - OP1(SLJIT_MOV, STR_END, 0, TMP3, 0); - JUMPHERE(quit); - } - } - -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 -if (common->utf && offset > 0) - utf_start = LABEL(); -#endif - -#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) - -/* SSE2 accelerated first character search. */ - -if (sljit_x86_is_sse2_available()) - { - fast_forward_first_char2_sse2(common, char1, char2); - - SLJIT_ASSERT(common->mode == JIT_COMPILE || offset == 0); - if (common->mode == JIT_COMPILE) - { - /* In complete mode, we don't need to run a match when STR_PTR == STR_END. */ - SLJIT_ASSERT(common->forced_quit_label == NULL); - OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_NOMATCH); - add_jump(compiler, &common->forced_quit, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); - -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (common->utf && offset > 0) - { - SLJIT_ASSERT(common->mode == JIT_COMPILE); - - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-offset)); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -#if defined COMPILE_PCRE8 - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0); - CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0x80, utf_start); -#elif defined COMPILE_PCRE16 - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); - CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0xdc00, utf_start); -#else -#error "Unknown code width" -#endif - OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - } -#endif - - if (offset > 0) - OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset)); - } - else if (sljit_x86_is_cmov_available()) - { - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_PTR, 0, STR_END, 0); - sljit_x86_emit_cmov(compiler, SLJIT_GREATER_EQUAL, STR_PTR, has_match_end ? SLJIT_MEM1(SLJIT_SP) : STR_END, has_match_end ? common->match_end_ptr : 0); - } - else - { - quit = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0); - OP1(SLJIT_MOV, STR_PTR, 0, has_match_end ? SLJIT_MEM1(SLJIT_SP) : STR_END, has_match_end ? common->match_end_ptr : 0); - JUMPHERE(quit); - } - - if (has_match_end) - OP1(SLJIT_MOV, STR_END, 0, TMP3, 0); - return; - } - -#endif - -quit = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); - -start = LABEL(); -OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); - -if (char1 == char2) - found = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, char1); -else - { - mask = char1 ^ char2; - if (is_powerof2(mask)) - { - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, mask); - found = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, char1 | mask); - } - else - { - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char1); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char2); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - found = JUMP(SLJIT_NOT_ZERO); - } - } - -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -CMPTO(SLJIT_LESS, STR_PTR, 0, STR_END, 0, start); - -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 -if (common->utf && offset > 0) - utf_quit = JUMP(SLJIT_JUMP); -#endif - -JUMPHERE(found); - -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 -if (common->utf && offset > 0) - { - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-offset)); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -#if defined COMPILE_PCRE8 - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0); - CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0x80, utf_start); -#elif defined COMPILE_PCRE16 - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); - CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0xdc00, utf_start); -#else -#error "Unknown code width" -#endif - OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - JUMPHERE(utf_quit); - } -#endif - -JUMPHERE(quit); - -if (has_match_end) - { - quit = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); - if (offset > 0) - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset)); - JUMPHERE(quit); - OP1(SLJIT_MOV, STR_END, 0, TMP3, 0); - } - -if (offset > 0) - OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset)); -} - -static SLJIT_INLINE BOOL fast_forward_first_n_chars(compiler_common *common) -{ -DEFINE_COMPILER; -struct sljit_label *start; -struct sljit_jump *quit; -struct sljit_jump *match; -/* bytes[0] represent the number of characters between 0 -and MAX_N_BYTES - 1, 255 represents any character. */ -pcre_uchar chars[MAX_N_CHARS * MAX_DIFF_CHARS]; -sljit_s32 offset; -pcre_uchar mask; -pcre_uchar *char_set, *char_set_end; -int i, max, from; -int range_right = -1, range_len; -sljit_u8 *update_table = NULL; -BOOL in_range; -sljit_u32 rec_count; - -for (i = 0; i < MAX_N_CHARS; i++) - chars[i * MAX_DIFF_CHARS] = 0; - -rec_count = 10000; -max = scan_prefix(common, common->start, chars, MAX_N_CHARS, &rec_count); - -if (max < 1) - return FALSE; - -in_range = FALSE; -/* Prevent compiler "uninitialized" warning */ -from = 0; -range_len = 4 /* minimum length */ - 1; -for (i = 0; i <= max; i++) - { - if (in_range && (i - from) > range_len && (chars[(i - 1) * MAX_DIFF_CHARS] < 255)) - { - range_len = i - from; - range_right = i - 1; - } - - if (i < max && chars[i * MAX_DIFF_CHARS] < 255) - { - SLJIT_ASSERT(chars[i * MAX_DIFF_CHARS] > 0); - if (!in_range) - { - in_range = TRUE; - from = i; - } - } - else - in_range = FALSE; - } - -if (range_right >= 0) - { - update_table = (sljit_u8 *)allocate_read_only_data(common, 256); - if (update_table == NULL) - return TRUE; - memset(update_table, IN_UCHARS(range_len), 256); - - for (i = 0; i < range_len; i++) - { - char_set = chars + ((range_right - i) * MAX_DIFF_CHARS); - SLJIT_ASSERT(char_set[0] > 0 && char_set[0] < 255); - char_set_end = char_set + char_set[0]; - char_set++; - while (char_set <= char_set_end) - { - if (update_table[(*char_set) & 0xff] > IN_UCHARS(i)) - update_table[(*char_set) & 0xff] = IN_UCHARS(i); - char_set++; - } - } - } - -offset = -1; -/* Scan forward. */ -for (i = 0; i < max; i++) - { - if (offset == -1) - { - if (chars[i * MAX_DIFF_CHARS] <= 2) - offset = i; - } - else if (chars[offset * MAX_DIFF_CHARS] == 2 && chars[i * MAX_DIFF_CHARS] <= 2) - { - if (chars[i * MAX_DIFF_CHARS] == 1) - offset = i; - else - { - mask = chars[offset * MAX_DIFF_CHARS + 1] ^ chars[offset * MAX_DIFF_CHARS + 2]; - if (!is_powerof2(mask)) - { - mask = chars[i * MAX_DIFF_CHARS + 1] ^ chars[i * MAX_DIFF_CHARS + 2]; - if (is_powerof2(mask)) - offset = i; - } - } - } - } - -if (range_right < 0) - { - if (offset < 0) - return FALSE; - SLJIT_ASSERT(chars[offset * MAX_DIFF_CHARS] >= 1 && chars[offset * MAX_DIFF_CHARS] <= 2); - /* Works regardless the value is 1 or 2. */ - mask = chars[offset * MAX_DIFF_CHARS + chars[offset * MAX_DIFF_CHARS]]; - fast_forward_first_char2(common, chars[offset * MAX_DIFF_CHARS + 1], mask, offset); - return TRUE; - } - -if (range_right == offset) - offset = -1; - -SLJIT_ASSERT(offset == -1 || (chars[offset * MAX_DIFF_CHARS] >= 1 && chars[offset * MAX_DIFF_CHARS] <= 2)); - -max -= 1; -SLJIT_ASSERT(max > 0); -if (common->match_end_ptr != 0) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); - OP1(SLJIT_MOV, TMP3, 0, STR_END, 0); - OP2(SLJIT_SUB, STR_END, 0, STR_END, 0, SLJIT_IMM, IN_UCHARS(max)); - quit = CMP(SLJIT_LESS_EQUAL, STR_END, 0, TMP1, 0); - OP1(SLJIT_MOV, STR_END, 0, TMP1, 0); - JUMPHERE(quit); - } -else - OP2(SLJIT_SUB, STR_END, 0, STR_END, 0, SLJIT_IMM, IN_UCHARS(max)); - -SLJIT_ASSERT(range_right >= 0); - -#if !(defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) -OP1(SLJIT_MOV, RETURN_ADDR, 0, SLJIT_IMM, (sljit_sw)update_table); -#endif - -start = LABEL(); -quit = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); - -#if defined COMPILE_PCRE8 || (defined SLJIT_LITTLE_ENDIAN && SLJIT_LITTLE_ENDIAN) -OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(range_right)); -#else -OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(range_right + 1) - 1); -#endif - -#if !(defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) -OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(RETURN_ADDR, TMP1), 0); -#else -OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)update_table); -#endif -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); -CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0, start); - -if (offset >= 0) - { - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(offset)); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - - if (chars[offset * MAX_DIFF_CHARS] == 1) - CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, chars[offset * MAX_DIFF_CHARS + 1], start); - else - { - mask = chars[offset * MAX_DIFF_CHARS + 1] ^ chars[offset * MAX_DIFF_CHARS + 2]; - if (is_powerof2(mask)) - { - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, mask); - CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, chars[offset * MAX_DIFF_CHARS + 1] | mask, start); - } - else - { - match = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, chars[offset * MAX_DIFF_CHARS + 1]); - CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, chars[offset * MAX_DIFF_CHARS + 2], start); - JUMPHERE(match); - } - } - } - -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 -if (common->utf && offset != 0) - { - if (offset < 0) - { - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - } - else - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1)); -#if defined COMPILE_PCRE8 - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0); - CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0x80, start); -#elif defined COMPILE_PCRE16 - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); - CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0xdc00, start); -#else -#error "Unknown code width" -#endif - if (offset < 0) - OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - } -#endif - -if (offset >= 0) - OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - -JUMPHERE(quit); - -if (common->match_end_ptr != 0) - { - if (range_right >= 0) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); - OP1(SLJIT_MOV, STR_END, 0, TMP3, 0); - if (range_right >= 0) - { - quit = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP1, 0); - OP1(SLJIT_MOV, STR_PTR, 0, TMP1, 0); - JUMPHERE(quit); - } - } -else - OP2(SLJIT_ADD, STR_END, 0, STR_END, 0, SLJIT_IMM, IN_UCHARS(max)); -return TRUE; -} - -#undef MAX_N_CHARS -#undef MAX_DIFF_CHARS - -static SLJIT_INLINE void fast_forward_first_char(compiler_common *common, pcre_uchar first_char, BOOL caseless) -{ -pcre_uchar oc; - -oc = first_char; -if (caseless) - { - oc = TABLE_GET(first_char, common->fcc, first_char); -#if defined SUPPORT_UCP && !defined COMPILE_PCRE8 - if (first_char > 127 && common->utf) - oc = UCD_OTHERCASE(first_char); -#endif - } - -fast_forward_first_char2(common, first_char, oc, 0); -} - -static SLJIT_INLINE void fast_forward_newline(compiler_common *common) -{ -DEFINE_COMPILER; -struct sljit_label *loop; -struct sljit_jump *lastchar; -struct sljit_jump *firstchar; -struct sljit_jump *quit; -struct sljit_jump *foundcr = NULL; -struct sljit_jump *notfoundnl; -jump_list *newline = NULL; - -if (common->match_end_ptr != 0) - { - OP1(SLJIT_MOV, TMP3, 0, STR_END, 0); - OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); - } - -if (common->nltype == NLTYPE_FIXED && common->newline > 255) - { - lastchar = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); - OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); - firstchar = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0); - - OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(2)); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, STR_PTR, 0, TMP1, 0); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_GREATER_EQUAL); -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCHAR_SHIFT); -#endif - OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0); - - loop = LABEL(); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - quit = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2)); - OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1)); - CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, loop); - CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff, loop); - - JUMPHERE(quit); - JUMPHERE(firstchar); - JUMPHERE(lastchar); - - if (common->match_end_ptr != 0) - OP1(SLJIT_MOV, STR_END, 0, TMP3, 0); - return; - } - -OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); -OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); -firstchar = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0); -skip_char_back(common); - -loop = LABEL(); -common->ff_newline_shortcut = loop; - -read_char_range(common, common->nlmin, common->nlmax, TRUE); -lastchar = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); -if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF) - foundcr = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR); -check_newlinechar(common, common->nltype, &newline, FALSE); -set_jumps(newline, loop); - -if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF) - { - quit = JUMP(SLJIT_JUMP); - JUMPHERE(foundcr); - notfoundnl = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_NL); - OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT); -#endif - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); - JUMPHERE(notfoundnl); - JUMPHERE(quit); - } -JUMPHERE(lastchar); -JUMPHERE(firstchar); - -if (common->match_end_ptr != 0) - OP1(SLJIT_MOV, STR_END, 0, TMP3, 0); -} - -static BOOL check_class_ranges(compiler_common *common, const sljit_u8 *bits, BOOL nclass, BOOL invert, jump_list **backtracks); - -static SLJIT_INLINE void fast_forward_start_bits(compiler_common *common, const sljit_u8 *start_bits) -{ -DEFINE_COMPILER; -struct sljit_label *start; -struct sljit_jump *quit; -struct sljit_jump *found = NULL; -jump_list *matches = NULL; -#ifndef COMPILE_PCRE8 -struct sljit_jump *jump; -#endif - -if (common->match_end_ptr != 0) - { - OP1(SLJIT_MOV, RETURN_ADDR, 0, STR_END, 0); - OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); - } - -start = LABEL(); -quit = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); -OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); -#ifdef SUPPORT_UTF -if (common->utf) - OP1(SLJIT_MOV, TMP3, 0, TMP1, 0); -#endif - -if (!check_class_ranges(common, start_bits, (start_bits[31] & 0x80) != 0, TRUE, &matches)) - { -#ifndef COMPILE_PCRE8 - jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 255); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 255); - JUMPHERE(jump); -#endif - OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7); - OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3); - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)start_bits); - OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); - found = JUMP(SLJIT_NOT_ZERO); - } - -#ifdef SUPPORT_UTF -if (common->utf) - OP1(SLJIT_MOV, TMP1, 0, TMP3, 0); -#endif -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -#ifdef SUPPORT_UTF -#if defined COMPILE_PCRE8 -if (common->utf) - { - CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0, start); - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); - } -#elif defined COMPILE_PCRE16 -if (common->utf) - { - CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800, start); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800); - OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); - OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); - } -#endif /* COMPILE_PCRE[8|16] */ -#endif /* SUPPORT_UTF */ -JUMPTO(SLJIT_JUMP, start); -if (found != NULL) - JUMPHERE(found); -if (matches != NULL) - set_jumps(matches, LABEL()); -JUMPHERE(quit); - -if (common->match_end_ptr != 0) - OP1(SLJIT_MOV, STR_END, 0, RETURN_ADDR, 0); -} - -static SLJIT_INLINE struct sljit_jump *search_requested_char(compiler_common *common, pcre_uchar req_char, BOOL caseless, BOOL has_firstchar) -{ -DEFINE_COMPILER; -struct sljit_label *loop; -struct sljit_jump *toolong; -struct sljit_jump *alreadyfound; -struct sljit_jump *found; -struct sljit_jump *foundoc = NULL; -struct sljit_jump *notfound; -sljit_u32 oc, bit; - -SLJIT_ASSERT(common->req_char_ptr != 0); -OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->req_char_ptr); -OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, REQ_BYTE_MAX); -toolong = CMP(SLJIT_LESS, TMP1, 0, STR_END, 0); -alreadyfound = CMP(SLJIT_LESS, STR_PTR, 0, TMP2, 0); - -if (has_firstchar) - OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -else - OP1(SLJIT_MOV, TMP1, 0, STR_PTR, 0); - -loop = LABEL(); -notfound = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, STR_END, 0); - -OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(TMP1), 0); -oc = req_char; -if (caseless) - { - oc = TABLE_GET(req_char, common->fcc, req_char); -#if defined SUPPORT_UCP && !(defined COMPILE_PCRE8) - if (req_char > 127 && common->utf) - oc = UCD_OTHERCASE(req_char); -#endif - } -if (req_char == oc) - found = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, req_char); -else - { - bit = req_char ^ oc; - if (is_powerof2(bit)) - { - OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, bit); - found = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, req_char | bit); - } - else - { - found = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, req_char); - foundoc = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, oc); - } - } -OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1)); -JUMPTO(SLJIT_JUMP, loop); - -JUMPHERE(found); -if (foundoc) - JUMPHERE(foundoc); -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->req_char_ptr, TMP1, 0); -JUMPHERE(alreadyfound); -JUMPHERE(toolong); -return notfound; -} - -static void do_revertframes(compiler_common *common) -{ -DEFINE_COMPILER; -struct sljit_jump *jump; -struct sljit_label *mainloop; - -sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); -OP1(SLJIT_MOV, TMP1, 0, STACK_TOP, 0); -GET_LOCAL_BASE(TMP3, 0, 0); - -/* Drop frames until we reach STACK_TOP. */ -mainloop = LABEL(); -OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), 0); -OP2(SLJIT_SUB | SLJIT_SET_S, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0); -jump = JUMP(SLJIT_SIG_LESS_EQUAL); - -OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP3, 0); -OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(TMP1), sizeof(sljit_sw)); -OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), sizeof(sljit_sw), SLJIT_MEM1(TMP1), 2 * sizeof(sljit_sw)); -OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 3 * sizeof(sljit_sw)); -JUMPTO(SLJIT_JUMP, mainloop); - -JUMPHERE(jump); -jump = JUMP(SLJIT_SIG_LESS); -/* End of dropping frames. */ -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); - -JUMPHERE(jump); -OP1(SLJIT_NEG, TMP2, 0, TMP2, 0); -OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP3, 0); -OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(TMP1), sizeof(sljit_sw)); -OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_sw)); -JUMPTO(SLJIT_JUMP, mainloop); -} - -static void check_wordboundary(compiler_common *common) -{ -DEFINE_COMPILER; -struct sljit_jump *skipread; -jump_list *skipread_list = NULL; -#if !(defined COMPILE_PCRE8) || defined SUPPORT_UTF -struct sljit_jump *jump; -#endif - -SLJIT_COMPILE_ASSERT(ctype_word == 0x10, ctype_word_must_be_16); - -sljit_emit_fast_enter(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0); -/* Get type of the previous char, and put it to LOCALS1. */ -OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); -OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, SLJIT_IMM, 0); -skipread = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP1, 0); -skip_char_back(common); -check_start_used_ptr(common); -read_char(common); - -/* Testing char type. */ -#ifdef SUPPORT_UCP -if (common->use_ucp) - { - OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 1); - jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE); - add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL)); - OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); - OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Nd - ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); - JUMPHERE(jump); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP2, 0); - } -else -#endif - { -#ifndef COMPILE_PCRE8 - jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255); -#elif defined SUPPORT_UTF - /* Here LOCALS1 has already been zeroed. */ - jump = NULL; - if (common->utf) - jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255); -#endif /* COMPILE_PCRE8 */ - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), common->ctypes); - OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 4 /* ctype_word */); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP1, 0); -#ifndef COMPILE_PCRE8 - JUMPHERE(jump); -#elif defined SUPPORT_UTF - if (jump != NULL) - JUMPHERE(jump); -#endif /* COMPILE_PCRE8 */ - } -JUMPHERE(skipread); - -OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0); -check_str_end(common, &skipread_list); -peek_char(common, READ_CHAR_MAX); - -/* Testing char type. This is a code duplication. */ -#ifdef SUPPORT_UCP -if (common->use_ucp) - { - OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 1); - jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE); - add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL)); - OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); - OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Nd - ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); - JUMPHERE(jump); - } -else -#endif - { -#ifndef COMPILE_PCRE8 - /* TMP2 may be destroyed by peek_char. */ - OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0); - jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255); -#elif defined SUPPORT_UTF - OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0); - jump = NULL; - if (common->utf) - jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255); -#endif - OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), common->ctypes); - OP2(SLJIT_LSHR, TMP2, 0, TMP2, 0, SLJIT_IMM, 4 /* ctype_word */); - OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 1); -#ifndef COMPILE_PCRE8 - JUMPHERE(jump); -#elif defined SUPPORT_UTF - if (jump != NULL) - JUMPHERE(jump); -#endif /* COMPILE_PCRE8 */ - } -set_jumps(skipread_list, LABEL()); - -OP2(SLJIT_XOR | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); -sljit_emit_fast_return(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0); -} - -static BOOL check_class_ranges(compiler_common *common, const sljit_u8 *bits, BOOL nclass, BOOL invert, jump_list **backtracks) -{ -/* May destroy TMP1. */ -DEFINE_COMPILER; -int ranges[MAX_RANGE_SIZE]; -sljit_u8 bit, cbit, all; -int i, byte, length = 0; - -bit = bits[0] & 0x1; -/* All bits will be zero or one (since bit is zero or one). */ -all = -bit; - -for (i = 0; i < 256; ) - { - byte = i >> 3; - if ((i & 0x7) == 0 && bits[byte] == all) - i += 8; - else - { - cbit = (bits[byte] >> (i & 0x7)) & 0x1; - if (cbit != bit) - { - if (length >= MAX_RANGE_SIZE) - return FALSE; - ranges[length] = i; - length++; - bit = cbit; - all = -cbit; - } - i++; - } - } - -if (((bit == 0) && nclass) || ((bit == 1) && !nclass)) - { - if (length >= MAX_RANGE_SIZE) - return FALSE; - ranges[length] = 256; - length++; - } - -if (length < 0 || length > 4) - return FALSE; - -bit = bits[0] & 0x1; -if (invert) bit ^= 0x1; - -/* No character is accepted. */ -if (length == 0 && bit == 0) - add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); - -switch(length) - { - case 0: - /* When bit != 0, all characters are accepted. */ - return TRUE; - - case 1: - add_jump(compiler, backtracks, CMP(bit == 0 ? SLJIT_LESS : SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, ranges[0])); - return TRUE; - - case 2: - if (ranges[0] + 1 != ranges[1]) - { - OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[0]); - add_jump(compiler, backtracks, CMP(bit != 0 ? SLJIT_LESS : SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, ranges[1] - ranges[0])); - } - else - add_jump(compiler, backtracks, CMP(bit != 0 ? SLJIT_EQUAL : SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[0])); - return TRUE; - - case 3: - if (bit != 0) - { - add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, ranges[2])); - if (ranges[0] + 1 != ranges[1]) - { - OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[0]); - add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, ranges[1] - ranges[0])); - } - else - add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[0])); - return TRUE; - } - - add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, ranges[0])); - if (ranges[1] + 1 != ranges[2]) - { - OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[1]); - add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, ranges[2] - ranges[1])); - } - else - add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[1])); - return TRUE; - - case 4: - if ((ranges[1] - ranges[0]) == (ranges[3] - ranges[2]) - && (ranges[0] | (ranges[2] - ranges[0])) == ranges[2] - && (ranges[1] & (ranges[2] - ranges[0])) == 0 - && is_powerof2(ranges[2] - ranges[0])) - { - SLJIT_ASSERT((ranges[0] & (ranges[2] - ranges[0])) == 0 && (ranges[2] & ranges[3] & (ranges[2] - ranges[0])) != 0); - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[2] - ranges[0]); - if (ranges[2] + 1 != ranges[3]) - { - OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[2]); - add_jump(compiler, backtracks, CMP(bit != 0 ? SLJIT_LESS : SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, ranges[3] - ranges[2])); - } - else - add_jump(compiler, backtracks, CMP(bit != 0 ? SLJIT_EQUAL : SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[2])); - return TRUE; - } - - if (bit != 0) - { - i = 0; - if (ranges[0] + 1 != ranges[1]) - { - OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[0]); - add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, ranges[1] - ranges[0])); - i = ranges[0]; - } - else - add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[0])); - - if (ranges[2] + 1 != ranges[3]) - { - OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[2] - i); - add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, ranges[3] - ranges[2])); - } - else - add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[2] - i)); - return TRUE; - } - - OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[0]); - add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, ranges[3] - ranges[0])); - if (ranges[1] + 1 != ranges[2]) - { - OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[1] - ranges[0]); - add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, ranges[2] - ranges[1])); - } - else - add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[1] - ranges[0])); - return TRUE; - - default: - SLJIT_ASSERT_STOP(); - return FALSE; - } -} - -static void check_anynewline(compiler_common *common) -{ -/* Check whether TMP1 contains a newline character. TMP2 destroyed. */ -DEFINE_COMPILER; - -sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); - -OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0a); -OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a); -OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); -OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a); -#if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 -#ifdef COMPILE_PCRE8 -if (common->utf) - { -#endif - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a); -#ifdef COMPILE_PCRE8 - } -#endif -#endif /* SUPPORT_UTF || COMPILE_PCRE16 || COMPILE_PCRE32 */ -OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); -} - -static void check_hspace(compiler_common *common) -{ -/* Check whether TMP1 contains a newline character. TMP2 destroyed. */ -DEFINE_COMPILER; - -sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); - -OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x09); -OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); -OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x20); -OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); -OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xa0); -#if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 -#ifdef COMPILE_PCRE8 -if (common->utf) - { -#endif - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x1680); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x2000); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x200A - 0x2000); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x202f - 0x2000); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x205f - 0x2000); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x3000 - 0x2000); -#ifdef COMPILE_PCRE8 - } -#endif -#endif /* SUPPORT_UTF || COMPILE_PCRE16 || COMPILE_PCRE32 */ -OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); -} - -static void check_vspace(compiler_common *common) -{ -/* Check whether TMP1 contains a newline character. TMP2 destroyed. */ -DEFINE_COMPILER; - -sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); - -OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0a); -OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a); -OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); -OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a); -#if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 -#ifdef COMPILE_PCRE8 -if (common->utf) - { -#endif - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a); -#ifdef COMPILE_PCRE8 - } -#endif -#endif /* SUPPORT_UTF || COMPILE_PCRE16 || COMPILE_PCRE32 */ -OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); -} - -#define CHAR1 STR_END -#define CHAR2 STACK_TOP - -static void do_casefulcmp(compiler_common *common) -{ -DEFINE_COMPILER; -struct sljit_jump *jump; -struct sljit_label *label; - -sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); -OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0); -OP1(SLJIT_MOV, TMP3, 0, CHAR1, 0); -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, CHAR2, 0); -OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1)); -OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - -label = LABEL(); -OP1(MOVU_UCHAR, CHAR1, 0, SLJIT_MEM1(TMP1), IN_UCHARS(1)); -OP1(MOVU_UCHAR, CHAR2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); -jump = CMP(SLJIT_NOT_EQUAL, CHAR1, 0, CHAR2, 0); -OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1)); -JUMPTO(SLJIT_NOT_ZERO, label); - -JUMPHERE(jump); -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -OP1(SLJIT_MOV, CHAR1, 0, TMP3, 0); -OP1(SLJIT_MOV, CHAR2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); -} - -#define LCC_TABLE STACK_LIMIT - -static void do_caselesscmp(compiler_common *common) -{ -DEFINE_COMPILER; -struct sljit_jump *jump; -struct sljit_label *label; - -sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); -OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0); - -OP1(SLJIT_MOV, TMP3, 0, LCC_TABLE, 0); -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, CHAR1, 0); -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, CHAR2, 0); -OP1(SLJIT_MOV, LCC_TABLE, 0, SLJIT_IMM, common->lcc); -OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1)); -OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - -label = LABEL(); -OP1(MOVU_UCHAR, CHAR1, 0, SLJIT_MEM1(TMP1), IN_UCHARS(1)); -OP1(MOVU_UCHAR, CHAR2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); -#ifndef COMPILE_PCRE8 -jump = CMP(SLJIT_GREATER, CHAR1, 0, SLJIT_IMM, 255); -#endif -OP1(SLJIT_MOV_U8, CHAR1, 0, SLJIT_MEM2(LCC_TABLE, CHAR1), 0); -#ifndef COMPILE_PCRE8 -JUMPHERE(jump); -jump = CMP(SLJIT_GREATER, CHAR2, 0, SLJIT_IMM, 255); -#endif -OP1(SLJIT_MOV_U8, CHAR2, 0, SLJIT_MEM2(LCC_TABLE, CHAR2), 0); -#ifndef COMPILE_PCRE8 -JUMPHERE(jump); -#endif -jump = CMP(SLJIT_NOT_EQUAL, CHAR1, 0, CHAR2, 0); -OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1)); -JUMPTO(SLJIT_NOT_ZERO, label); - -JUMPHERE(jump); -OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -OP1(SLJIT_MOV, LCC_TABLE, 0, TMP3, 0); -OP1(SLJIT_MOV, CHAR1, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); -OP1(SLJIT_MOV, CHAR2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); -sljit_emit_fast_return(compiler, RETURN_ADDR, 0); -} - -#undef LCC_TABLE -#undef CHAR1 -#undef CHAR2 - -#if defined SUPPORT_UTF && defined SUPPORT_UCP - -static const pcre_uchar * SLJIT_CALL do_utf_caselesscmp(pcre_uchar *src1, jit_arguments *args, pcre_uchar *end1) -{ -/* This function would be ineffective to do in JIT level. */ -sljit_u32 c1, c2; -const pcre_uchar *src2 = args->uchar_ptr; -const pcre_uchar *end2 = args->end; -const ucd_record *ur; -const sljit_u32 *pp; - -while (src1 < end1) - { - if (src2 >= end2) - return (pcre_uchar*)1; - GETCHARINC(c1, src1); - GETCHARINC(c2, src2); - ur = GET_UCD(c2); - if (c1 != c2 && c1 != c2 + ur->other_case) - { - pp = PRIV(ucd_caseless_sets) + ur->caseset; - for (;;) - { - if (c1 < *pp) return NULL; - if (c1 == *pp++) break; - } - } - } -return src2; -} - -#endif /* SUPPORT_UTF && SUPPORT_UCP */ - -static pcre_uchar *byte_sequence_compare(compiler_common *common, BOOL caseless, pcre_uchar *cc, - compare_context *context, jump_list **backtracks) -{ -DEFINE_COMPILER; -unsigned int othercasebit = 0; -pcre_uchar *othercasechar = NULL; -#ifdef SUPPORT_UTF -int utflength; -#endif - -if (caseless && char_has_othercase(common, cc)) - { - othercasebit = char_get_othercase_bit(common, cc); - SLJIT_ASSERT(othercasebit); - /* Extracting bit difference info. */ -#if defined COMPILE_PCRE8 - othercasechar = cc + (othercasebit >> 8); - othercasebit &= 0xff; -#elif defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - /* Note that this code only handles characters in the BMP. If there - ever are characters outside the BMP whose othercase differs in only one - bit from itself (there currently are none), this code will need to be - revised for COMPILE_PCRE32. */ - othercasechar = cc + (othercasebit >> 9); - if ((othercasebit & 0x100) != 0) - othercasebit = (othercasebit & 0xff) << 8; - else - othercasebit &= 0xff; -#endif /* COMPILE_PCRE[8|16|32] */ - } - -if (context->sourcereg == -1) - { -#if defined COMPILE_PCRE8 -#if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED - if (context->length >= 4) - OP1(SLJIT_MOV_S32, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length); - else if (context->length >= 2) - OP1(SLJIT_MOV_U16, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length); - else -#endif - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length); -#elif defined COMPILE_PCRE16 -#if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED - if (context->length >= 4) - OP1(SLJIT_MOV_S32, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length); - else -#endif - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length); -#elif defined COMPILE_PCRE32 - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length); -#endif /* COMPILE_PCRE[8|16|32] */ - context->sourcereg = TMP2; - } - -#ifdef SUPPORT_UTF -utflength = 1; -if (common->utf && HAS_EXTRALEN(*cc)) - utflength += GET_EXTRALEN(*cc); - -do - { -#endif - - context->length -= IN_UCHARS(1); -#if (defined SLJIT_UNALIGNED && SLJIT_UNALIGNED) && (defined COMPILE_PCRE8 || defined COMPILE_PCRE16) - - /* Unaligned read is supported. */ - if (othercasebit != 0 && othercasechar == cc) - { - context->c.asuchars[context->ucharptr] = *cc | othercasebit; - context->oc.asuchars[context->ucharptr] = othercasebit; - } - else - { - context->c.asuchars[context->ucharptr] = *cc; - context->oc.asuchars[context->ucharptr] = 0; - } - context->ucharptr++; - -#if defined COMPILE_PCRE8 - if (context->ucharptr >= 4 || context->length == 0 || (context->ucharptr == 2 && context->length == 1)) -#else - if (context->ucharptr >= 2 || context->length == 0) -#endif - { - if (context->length >= 4) - OP1(SLJIT_MOV_S32, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length); - else if (context->length >= 2) - OP1(SLJIT_MOV_U16, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length); -#if defined COMPILE_PCRE8 - else if (context->length >= 1) - OP1(SLJIT_MOV_U8, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length); -#endif /* COMPILE_PCRE8 */ - context->sourcereg = context->sourcereg == TMP1 ? TMP2 : TMP1; - - switch(context->ucharptr) - { - case 4 / sizeof(pcre_uchar): - if (context->oc.asint != 0) - OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, context->oc.asint); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asint | context->oc.asint)); - break; - - case 2 / sizeof(pcre_uchar): - if (context->oc.asushort != 0) - OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, context->oc.asushort); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asushort | context->oc.asushort)); - break; - -#ifdef COMPILE_PCRE8 - case 1: - if (context->oc.asbyte != 0) - OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, context->oc.asbyte); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asbyte | context->oc.asbyte)); - break; -#endif - - default: - SLJIT_ASSERT_STOP(); - break; - } - context->ucharptr = 0; - } - -#else - - /* Unaligned read is unsupported or in 32 bit mode. */ - if (context->length >= 1) - OP1(MOV_UCHAR, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length); - - context->sourcereg = context->sourcereg == TMP1 ? TMP2 : TMP1; - - if (othercasebit != 0 && othercasechar == cc) - { - OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, othercasebit); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, *cc | othercasebit)); - } - else - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, *cc)); - -#endif - - cc++; -#ifdef SUPPORT_UTF - utflength--; - } -while (utflength > 0); -#endif - -return cc; -} - -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - -#define SET_TYPE_OFFSET(value) \ - if ((value) != typeoffset) \ - { \ - if ((value) < typeoffset) \ - OP2(SLJIT_ADD, typereg, 0, typereg, 0, SLJIT_IMM, typeoffset - (value)); \ - else \ - OP2(SLJIT_SUB, typereg, 0, typereg, 0, SLJIT_IMM, (value) - typeoffset); \ - } \ - typeoffset = (value); - -#define SET_CHAR_OFFSET(value) \ - if ((value) != charoffset) \ - { \ - if ((value) < charoffset) \ - OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(charoffset - (value))); \ - else \ - OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)((value) - charoffset)); \ - } \ - charoffset = (value); - -static pcre_uchar *compile_char1_matchingpath(compiler_common *common, pcre_uchar type, pcre_uchar *cc, jump_list **backtracks, BOOL check_str_ptr); - -static void compile_xclass_matchingpath(compiler_common *common, pcre_uchar *cc, jump_list **backtracks) -{ -DEFINE_COMPILER; -jump_list *found = NULL; -jump_list **list = (cc[0] & XCL_NOT) == 0 ? &found : backtracks; -sljit_uw c, charoffset, max = 256, min = READ_CHAR_MAX; -struct sljit_jump *jump = NULL; -pcre_uchar *ccbegin; -int compares, invertcmp, numberofcmps; -#if defined SUPPORT_UTF && (defined COMPILE_PCRE8 || defined COMPILE_PCRE16) -BOOL utf = common->utf; -#endif - -#ifdef SUPPORT_UCP -BOOL needstype = FALSE, needsscript = FALSE, needschar = FALSE; -BOOL charsaved = FALSE; -int typereg = TMP1; -const sljit_u32 *other_cases; -sljit_uw typeoffset; -#endif - -/* Scanning the necessary info. */ -cc++; -ccbegin = cc; -compares = 0; -if (cc[-1] & XCL_MAP) - { - min = 0; - cc += 32 / sizeof(pcre_uchar); - } - -while (*cc != XCL_END) - { - compares++; - if (*cc == XCL_SINGLE) - { - cc ++; - GETCHARINCTEST(c, cc); - if (c > max) max = c; - if (c < min) min = c; -#ifdef SUPPORT_UCP - needschar = TRUE; -#endif - } - else if (*cc == XCL_RANGE) - { - cc ++; - GETCHARINCTEST(c, cc); - if (c < min) min = c; - GETCHARINCTEST(c, cc); - if (c > max) max = c; -#ifdef SUPPORT_UCP - needschar = TRUE; -#endif - } -#ifdef SUPPORT_UCP - else - { - SLJIT_ASSERT(*cc == XCL_PROP || *cc == XCL_NOTPROP); - cc++; - if (*cc == PT_CLIST) - { - other_cases = PRIV(ucd_caseless_sets) + cc[1]; - while (*other_cases != NOTACHAR) - { - if (*other_cases > max) max = *other_cases; - if (*other_cases < min) min = *other_cases; - other_cases++; - } - } - else - { - max = READ_CHAR_MAX; - min = 0; - } - - switch(*cc) - { - case PT_ANY: - /* Any either accepts everything or ignored. */ - if (cc[-1] == XCL_PROP) - { - compile_char1_matchingpath(common, OP_ALLANY, cc, backtracks, FALSE); - if (list == backtracks) - add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); - return; - } - break; - - case PT_LAMP: - case PT_GC: - case PT_PC: - case PT_ALNUM: - needstype = TRUE; - break; - - case PT_SC: - needsscript = TRUE; - break; - - case PT_SPACE: - case PT_PXSPACE: - case PT_WORD: - case PT_PXGRAPH: - case PT_PXPRINT: - case PT_PXPUNCT: - needstype = TRUE; - needschar = TRUE; - break; - - case PT_CLIST: - case PT_UCNC: - needschar = TRUE; - break; - - default: - SLJIT_ASSERT_STOP(); - break; - } - cc += 2; - } -#endif - } -SLJIT_ASSERT(compares > 0); - -/* We are not necessary in utf mode even in 8 bit mode. */ -cc = ccbegin; -read_char_range(common, min, max, (cc[-1] & XCL_NOT) != 0); - -if ((cc[-1] & XCL_HASPROP) == 0) - { - if ((cc[-1] & XCL_MAP) != 0) - { - jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255); - if (!check_class_ranges(common, (const sljit_u8 *)cc, (((const sljit_u8 *)cc)[31] & 0x80) != 0, TRUE, &found)) - { - OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7); - OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3); - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc); - OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); - add_jump(compiler, &found, JUMP(SLJIT_NOT_ZERO)); - } - - add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); - JUMPHERE(jump); - - cc += 32 / sizeof(pcre_uchar); - } - else - { - OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, min); - add_jump(compiler, (cc[-1] & XCL_NOT) == 0 ? backtracks : &found, CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, max - min)); - } - } -else if ((cc[-1] & XCL_MAP) != 0) - { - OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0); -#ifdef SUPPORT_UCP - charsaved = TRUE; -#endif - if (!check_class_ranges(common, (const sljit_u8 *)cc, FALSE, TRUE, list)) - { -#ifdef COMPILE_PCRE8 - jump = NULL; - if (common->utf) -#endif - jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255); - - OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7); - OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3); - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc); - OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); - add_jump(compiler, list, JUMP(SLJIT_NOT_ZERO)); - -#ifdef COMPILE_PCRE8 - if (common->utf) -#endif - JUMPHERE(jump); - } - - OP1(SLJIT_MOV, TMP1, 0, RETURN_ADDR, 0); - cc += 32 / sizeof(pcre_uchar); - } - -#ifdef SUPPORT_UCP -if (needstype || needsscript) - { - if (needschar && !charsaved) - OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0); - - OP2(SLJIT_LSHR, TMP2, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_SHIFT); - OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_stage1)); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_MASK); - OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCD_BLOCK_SHIFT); - OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_stage2)); - OP1(SLJIT_MOV_U16, TMP2, 0, SLJIT_MEM2(TMP2, TMP1), 1); - - /* Before anything else, we deal with scripts. */ - if (needsscript) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, script)); - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3); - - ccbegin = cc; - - while (*cc != XCL_END) - { - if (*cc == XCL_SINGLE) - { - cc ++; - GETCHARINCTEST(c, cc); - } - else if (*cc == XCL_RANGE) - { - cc ++; - GETCHARINCTEST(c, cc); - GETCHARINCTEST(c, cc); - } - else - { - SLJIT_ASSERT(*cc == XCL_PROP || *cc == XCL_NOTPROP); - cc++; - if (*cc == PT_SC) - { - compares--; - invertcmp = (compares == 0 && list != backtracks); - if (cc[-1] == XCL_NOTPROP) - invertcmp ^= 0x1; - jump = CMP(SLJIT_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, (int)cc[1]); - add_jump(compiler, compares > 0 ? list : backtracks, jump); - } - cc += 2; - } - } - - cc = ccbegin; - } - - if (needschar) - { - OP1(SLJIT_MOV, TMP1, 0, RETURN_ADDR, 0); - } - - if (needstype) - { - if (!needschar) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype)); - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3); - } - else - { - OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 3); - OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype)); - typereg = RETURN_ADDR; - } - } - } -#endif - -/* Generating code. */ -charoffset = 0; -numberofcmps = 0; -#ifdef SUPPORT_UCP -typeoffset = 0; -#endif - -while (*cc != XCL_END) - { - compares--; - invertcmp = (compares == 0 && list != backtracks); - jump = NULL; - - if (*cc == XCL_SINGLE) - { - cc ++; - GETCHARINCTEST(c, cc); - - if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE)) - { - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); - OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, numberofcmps == 0 ? SLJIT_UNUSED : TMP2, 0, SLJIT_EQUAL); - numberofcmps++; - } - else if (numberofcmps > 0) - { - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); - numberofcmps = 0; - } - else - { - jump = CMP(SLJIT_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); - numberofcmps = 0; - } - } - else if (*cc == XCL_RANGE) - { - cc ++; - GETCHARINCTEST(c, cc); - SET_CHAR_OFFSET(c); - GETCHARINCTEST(c, cc); - - if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE)) - { - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); - OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, numberofcmps == 0 ? SLJIT_UNUSED : TMP2, 0, SLJIT_LESS_EQUAL); - numberofcmps++; - } - else if (numberofcmps > 0) - { - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); - jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); - numberofcmps = 0; - } - else - { - jump = CMP(SLJIT_LESS_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); - numberofcmps = 0; - } - } -#ifdef SUPPORT_UCP - else - { - SLJIT_ASSERT(*cc == XCL_PROP || *cc == XCL_NOTPROP); - if (*cc == XCL_NOTPROP) - invertcmp ^= 0x1; - cc++; - switch(*cc) - { - case PT_ANY: - if (!invertcmp) - jump = JUMP(SLJIT_JUMP); - break; - - case PT_LAMP: - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - typeoffset); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ll - typeoffset); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lt - typeoffset); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); - break; - - case PT_GC: - c = PRIV(ucp_typerange)[(int)cc[1] * 2]; - SET_TYPE_OFFSET(c); - jump = CMP(SLJIT_LESS_EQUAL ^ invertcmp, typereg, 0, SLJIT_IMM, PRIV(ucp_typerange)[(int)cc[1] * 2 + 1] - c); - break; - - case PT_PC: - jump = CMP(SLJIT_EQUAL ^ invertcmp, typereg, 0, SLJIT_IMM, (int)cc[1] - typeoffset); - break; - - case PT_SC: - compares++; - /* Do nothing. */ - break; - - case PT_SPACE: - case PT_PXSPACE: - SET_CHAR_OFFSET(9); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd - 0x9); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); - - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x9); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e - 0x9); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - - SET_TYPE_OFFSET(ucp_Zl); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Zl); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); - jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); - break; - - case PT_WORD: - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_UNDERSCORE - charoffset)); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); - /* Fall through. */ - - case PT_ALNUM: - SET_TYPE_OFFSET(ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); - OP_FLAGS((*cc == PT_ALNUM) ? SLJIT_MOV : SLJIT_OR, TMP2, 0, (*cc == PT_ALNUM) ? SLJIT_UNUSED : TMP2, 0, SLJIT_LESS_EQUAL); - SET_TYPE_OFFSET(ucp_Nd); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_No - ucp_Nd); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); - jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); - break; - - case PT_CLIST: - other_cases = PRIV(ucd_caseless_sets) + cc[1]; - - /* At least three characters are required. - Otherwise this case would be handled by the normal code path. */ - SLJIT_ASSERT(other_cases[0] != NOTACHAR && other_cases[1] != NOTACHAR && other_cases[2] != NOTACHAR); - SLJIT_ASSERT(other_cases[0] < other_cases[1] && other_cases[1] < other_cases[2]); - - /* Optimizing character pairs, if their difference is power of 2. */ - if (is_powerof2(other_cases[1] ^ other_cases[0])) - { - if (charoffset == 0) - OP2(SLJIT_OR, TMP2, 0, TMP1, 0, SLJIT_IMM, other_cases[1] ^ other_cases[0]); - else - { - OP2(SLJIT_ADD, TMP2, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)charoffset); - OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, other_cases[1] ^ other_cases[0]); - } - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, other_cases[1]); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); - other_cases += 2; - } - else if (is_powerof2(other_cases[2] ^ other_cases[1])) - { - if (charoffset == 0) - OP2(SLJIT_OR, TMP2, 0, TMP1, 0, SLJIT_IMM, other_cases[2] ^ other_cases[1]); - else - { - OP2(SLJIT_ADD, TMP2, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)charoffset); - OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, other_cases[1] ^ other_cases[0]); - } - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, other_cases[2]); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); - - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(other_cases[0] - charoffset)); - OP_FLAGS(SLJIT_OR | ((other_cases[3] == NOTACHAR) ? SLJIT_SET_E : 0), TMP2, 0, TMP2, 0, SLJIT_EQUAL); - - other_cases += 3; - } - else - { - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset)); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); - } - - while (*other_cases != NOTACHAR) - { - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset)); - OP_FLAGS(SLJIT_OR | ((*other_cases == NOTACHAR) ? SLJIT_SET_E : 0), TMP2, 0, TMP2, 0, SLJIT_EQUAL); - } - jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); - break; - - case PT_UCNC: - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_DOLLAR_SIGN - charoffset)); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_COMMERCIAL_AT - charoffset)); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_GRAVE_ACCENT - charoffset)); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - - SET_CHAR_OFFSET(0xa0); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(0xd7ff - charoffset)); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); - SET_CHAR_OFFSET(0); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xe000 - 0); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_GREATER_EQUAL); - jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); - break; - - case PT_PXGRAPH: - /* C and Z groups are the farthest two groups. */ - SET_TYPE_OFFSET(ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_GREATER); - - jump = CMP(SLJIT_NOT_EQUAL, typereg, 0, SLJIT_IMM, ucp_Cf - ucp_Ll); - - /* In case of ucp_Cf, we overwrite the result. */ - SET_CHAR_OFFSET(0x2066); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); - - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e - 0x2066); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - - JUMPHERE(jump); - jump = CMP(SLJIT_ZERO ^ invertcmp, TMP2, 0, SLJIT_IMM, 0); - break; - - case PT_PXPRINT: - /* C and Z groups are the farthest two groups. */ - SET_TYPE_OFFSET(ucp_Ll); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_GREATER); - - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Ll); - OP_FLAGS(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_NOT_EQUAL); - - jump = CMP(SLJIT_NOT_EQUAL, typereg, 0, SLJIT_IMM, ucp_Cf - ucp_Ll); - - /* In case of ucp_Cf, we overwrite the result. */ - SET_CHAR_OFFSET(0x2066); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); - - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066); - OP_FLAGS(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_EQUAL); - - JUMPHERE(jump); - jump = CMP(SLJIT_ZERO ^ invertcmp, TMP2, 0, SLJIT_IMM, 0); - break; - - case PT_PXPUNCT: - SET_TYPE_OFFSET(ucp_Sc); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Sc); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS_EQUAL); - - SET_CHAR_OFFSET(0); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x7f); - OP_FLAGS(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); - - SET_TYPE_OFFSET(ucp_Pc); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ps - ucp_Pc); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_LESS_EQUAL); - jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); - break; - - default: - SLJIT_ASSERT_STOP(); - break; - } - cc += 2; - } -#endif - - if (jump != NULL) - add_jump(compiler, compares > 0 ? list : backtracks, jump); - } - -if (found != NULL) - set_jumps(found, LABEL()); -} - -#undef SET_TYPE_OFFSET -#undef SET_CHAR_OFFSET - -#endif - -static pcre_uchar *compile_simple_assertion_matchingpath(compiler_common *common, pcre_uchar type, pcre_uchar *cc, jump_list **backtracks) -{ -DEFINE_COMPILER; -int length; -struct sljit_jump *jump[4]; -#ifdef SUPPORT_UTF -struct sljit_label *label; -#endif /* SUPPORT_UTF */ - -switch(type) - { - case OP_SOD: - OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, TMP1, 0)); - return cc; - - case OP_SOM: - OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, TMP1, 0)); - return cc; - - case OP_NOT_WORD_BOUNDARY: - case OP_WORD_BOUNDARY: - add_jump(compiler, &common->wordboundary, JUMP(SLJIT_FAST_CALL)); - add_jump(compiler, backtracks, JUMP(type == OP_NOT_WORD_BOUNDARY ? SLJIT_NOT_ZERO : SLJIT_ZERO)); - return cc; - - case OP_EODN: - /* Requires rather complex checks. */ - jump[0] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); - if (common->nltype == NLTYPE_FIXED && common->newline > 255) - { - OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - if (common->mode == JIT_COMPILE) - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, STR_END, 0)); - else - { - jump[1] = CMP(SLJIT_EQUAL, TMP2, 0, STR_END, 0); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0); - OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_UNUSED, 0, SLJIT_LESS); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff); - OP_FLAGS(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_NOT_EQUAL); - add_jump(compiler, backtracks, JUMP(SLJIT_NOT_EQUAL)); - check_partial(common, TRUE); - add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); - JUMPHERE(jump[1]); - } - OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff)); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff)); - } - else if (common->nltype == NLTYPE_FIXED) - { - OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, STR_END, 0)); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, common->newline)); - } - else - { - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - jump[1] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR); - OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); - OP2(SLJIT_SUB | SLJIT_SET_U, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0); - jump[2] = JUMP(SLJIT_GREATER); - add_jump(compiler, backtracks, JUMP(SLJIT_LESS)); - /* Equal. */ - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); - jump[3] = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL); - add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); - - JUMPHERE(jump[1]); - if (common->nltype == NLTYPE_ANYCRLF) - { - OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP2, 0, STR_END, 0)); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL)); - } - else - { - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, STR_PTR, 0); - read_char_range(common, common->nlmin, common->nlmax, TRUE); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, STR_END, 0)); - add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL)); - add_jump(compiler, backtracks, JUMP(SLJIT_ZERO)); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); - } - JUMPHERE(jump[2]); - JUMPHERE(jump[3]); - } - JUMPHERE(jump[0]); - check_partial(common, FALSE); - return cc; - - case OP_EOD: - add_jump(compiler, backtracks, CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0)); - check_partial(common, FALSE); - return cc; - - case OP_DOLL: - OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0); - OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, noteol)); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0)); - - if (!common->endonly) - compile_simple_assertion_matchingpath(common, OP_EODN, cc, backtracks); - else - { - add_jump(compiler, backtracks, CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0)); - check_partial(common, FALSE); - } - return cc; - - case OP_DOLLM: - jump[1] = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0); - OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0); - OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, noteol)); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0)); - check_partial(common, FALSE); - jump[0] = JUMP(SLJIT_JUMP); - JUMPHERE(jump[1]); - - if (common->nltype == NLTYPE_FIXED && common->newline > 255) - { - OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - if (common->mode == JIT_COMPILE) - add_jump(compiler, backtracks, CMP(SLJIT_GREATER, TMP2, 0, STR_END, 0)); - else - { - jump[1] = CMP(SLJIT_LESS_EQUAL, TMP2, 0, STR_END, 0); - /* STR_PTR = STR_END - IN_UCHARS(1) */ - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff)); - check_partial(common, TRUE); - add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); - JUMPHERE(jump[1]); - } - - OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff)); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff)); - } - else - { - peek_char(common, common->nlmax); - check_newlinechar(common, common->nltype, backtracks, FALSE); - } - JUMPHERE(jump[0]); - return cc; - - case OP_CIRC: - OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin)); - add_jump(compiler, backtracks, CMP(SLJIT_GREATER, STR_PTR, 0, TMP1, 0)); - OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, notbol)); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0)); - return cc; - - case OP_CIRCM: - OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin)); - jump[1] = CMP(SLJIT_GREATER, STR_PTR, 0, TMP1, 0); - OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, notbol)); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0)); - jump[0] = JUMP(SLJIT_JUMP); - JUMPHERE(jump[1]); - - add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); - if (common->nltype == NLTYPE_FIXED && common->newline > 255) - { - OP2(SLJIT_SUB, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); - add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP2, 0, TMP1, 0)); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2)); - OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1)); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff)); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff)); - } - else - { - skip_char_back(common); - read_char_range(common, common->nlmin, common->nlmax, TRUE); - check_newlinechar(common, common->nltype, backtracks, FALSE); - } - JUMPHERE(jump[0]); - return cc; - - case OP_REVERSE: - length = GET(cc, 0); - if (length == 0) - return cc + LINK_SIZE; - OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); -#ifdef SUPPORT_UTF - if (common->utf) - { - OP1(SLJIT_MOV, TMP3, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, length); - label = LABEL(); - add_jump(compiler, backtracks, CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP3, 0)); - skip_char_back(common); - OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, TMP2, 0, SLJIT_IMM, 1); - JUMPTO(SLJIT_NOT_ZERO, label); - } - else -#endif - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); - OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(length)); - add_jump(compiler, backtracks, CMP(SLJIT_LESS, STR_PTR, 0, TMP1, 0)); - } - check_start_used_ptr(common); - return cc + LINK_SIZE; - } -SLJIT_ASSERT_STOP(); -return cc; -} - -static pcre_uchar *compile_char1_matchingpath(compiler_common *common, pcre_uchar type, pcre_uchar *cc, jump_list **backtracks, BOOL check_str_ptr) -{ -DEFINE_COMPILER; -int length; -unsigned int c, oc, bit; -compare_context context; -struct sljit_jump *jump[3]; -jump_list *end_list; -#ifdef SUPPORT_UTF -struct sljit_label *label; -#ifdef SUPPORT_UCP -pcre_uchar propdata[5]; -#endif -#endif /* SUPPORT_UTF */ - -switch(type) - { - case OP_NOT_DIGIT: - case OP_DIGIT: - /* Digits are usually 0-9, so it is worth to optimize them. */ - if (check_str_ptr) - detect_partial_match(common, backtracks); -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 - if (common->utf && is_char7_bitset((const sljit_u8 *)common->ctypes - cbit_length + cbit_digit, FALSE)) - read_char7_type(common, type == OP_NOT_DIGIT); - else -#endif - read_char8_type(common, type == OP_NOT_DIGIT); - /* Flip the starting bit in the negative case. */ - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_digit); - add_jump(compiler, backtracks, JUMP(type == OP_DIGIT ? SLJIT_ZERO : SLJIT_NOT_ZERO)); - return cc; - - case OP_NOT_WHITESPACE: - case OP_WHITESPACE: - if (check_str_ptr) - detect_partial_match(common, backtracks); -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 - if (common->utf && is_char7_bitset((const sljit_u8 *)common->ctypes - cbit_length + cbit_space, FALSE)) - read_char7_type(common, type == OP_NOT_WHITESPACE); - else -#endif - read_char8_type(common, type == OP_NOT_WHITESPACE); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_space); - add_jump(compiler, backtracks, JUMP(type == OP_WHITESPACE ? SLJIT_ZERO : SLJIT_NOT_ZERO)); - return cc; - - case OP_NOT_WORDCHAR: - case OP_WORDCHAR: - if (check_str_ptr) - detect_partial_match(common, backtracks); -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 - if (common->utf && is_char7_bitset((const sljit_u8 *)common->ctypes - cbit_length + cbit_word, FALSE)) - read_char7_type(common, type == OP_NOT_WORDCHAR); - else -#endif - read_char8_type(common, type == OP_NOT_WORDCHAR); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_word); - add_jump(compiler, backtracks, JUMP(type == OP_WORDCHAR ? SLJIT_ZERO : SLJIT_NOT_ZERO)); - return cc; - - case OP_ANY: - if (check_str_ptr) - detect_partial_match(common, backtracks); - read_char_range(common, common->nlmin, common->nlmax, TRUE); - if (common->nltype == NLTYPE_FIXED && common->newline > 255) - { - jump[0] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff); - end_list = NULL; - if (common->mode != JIT_PARTIAL_HARD_COMPILE) - add_jump(compiler, &end_list, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); - else - check_str_end(common, &end_list); - - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); - add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, common->newline & 0xff)); - set_jumps(end_list, LABEL()); - JUMPHERE(jump[0]); - } - else - check_newlinechar(common, common->nltype, backtracks, TRUE); - return cc; - - case OP_ALLANY: - if (check_str_ptr) - detect_partial_match(common, backtracks); -#ifdef SUPPORT_UTF - if (common->utf) - { - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); -#if defined COMPILE_PCRE8 || defined COMPILE_PCRE16 -#if defined COMPILE_PCRE8 - jump[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0); - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); -#elif defined COMPILE_PCRE16 - jump[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800); - OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800); - OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_UNUSED, 0, SLJIT_EQUAL); - OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); -#endif - JUMPHERE(jump[0]); -#endif /* COMPILE_PCRE[8|16] */ - return cc; - } -#endif - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - return cc; - - case OP_ANYBYTE: - if (check_str_ptr) - detect_partial_match(common, backtracks); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - return cc; - -#ifdef SUPPORT_UTF -#ifdef SUPPORT_UCP - case OP_NOTPROP: - case OP_PROP: - propdata[0] = XCL_HASPROP; - propdata[1] = type == OP_NOTPROP ? XCL_NOTPROP : XCL_PROP; - propdata[2] = cc[0]; - propdata[3] = cc[1]; - propdata[4] = XCL_END; - if (check_str_ptr) - detect_partial_match(common, backtracks); - compile_xclass_matchingpath(common, propdata, backtracks); - return cc + 2; -#endif -#endif - - case OP_ANYNL: - if (check_str_ptr) - detect_partial_match(common, backtracks); - read_char_range(common, common->bsr_nlmin, common->bsr_nlmax, FALSE); - jump[0] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR); - /* We don't need to handle soft partial matching case. */ - end_list = NULL; - if (common->mode != JIT_PARTIAL_HARD_COMPILE) - add_jump(compiler, &end_list, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); - else - check_str_end(common, &end_list); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); - jump[1] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - jump[2] = JUMP(SLJIT_JUMP); - JUMPHERE(jump[0]); - check_newlinechar(common, common->bsr_nltype, backtracks, FALSE); - set_jumps(end_list, LABEL()); - JUMPHERE(jump[1]); - JUMPHERE(jump[2]); - return cc; - - case OP_NOT_HSPACE: - case OP_HSPACE: - if (check_str_ptr) - detect_partial_match(common, backtracks); - read_char_range(common, 0x9, 0x3000, type == OP_NOT_HSPACE); - add_jump(compiler, &common->hspace, JUMP(SLJIT_FAST_CALL)); - add_jump(compiler, backtracks, JUMP(type == OP_NOT_HSPACE ? SLJIT_NOT_ZERO : SLJIT_ZERO)); - return cc; - - case OP_NOT_VSPACE: - case OP_VSPACE: - if (check_str_ptr) - detect_partial_match(common, backtracks); - read_char_range(common, 0xa, 0x2029, type == OP_NOT_VSPACE); - add_jump(compiler, &common->vspace, JUMP(SLJIT_FAST_CALL)); - add_jump(compiler, backtracks, JUMP(type == OP_NOT_VSPACE ? SLJIT_NOT_ZERO : SLJIT_ZERO)); - return cc; - -#ifdef SUPPORT_UCP - case OP_EXTUNI: - if (check_str_ptr) - detect_partial_match(common, backtracks); - read_char(common); - add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL)); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, gbprop)); - /* Optimize register allocation: use a real register. */ - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STACK_TOP, 0); - OP1(SLJIT_MOV_U8, STACK_TOP, 0, SLJIT_MEM2(TMP1, TMP2), 3); - - label = LABEL(); - jump[0] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); - OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0); - read_char(common); - add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL)); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, gbprop)); - OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM2(TMP1, TMP2), 3); - - OP2(SLJIT_SHL, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 2); - OP1(SLJIT_MOV_U32, TMP1, 0, SLJIT_MEM1(STACK_TOP), (sljit_sw)PRIV(ucp_gbtable)); - OP1(SLJIT_MOV, STACK_TOP, 0, TMP2, 0); - OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); - JUMPTO(SLJIT_NOT_ZERO, label); - - OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0); - JUMPHERE(jump[0]); - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); - - if (common->mode == JIT_PARTIAL_HARD_COMPILE) - { - jump[0] = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0); - /* Since we successfully read a char above, partial matching must occure. */ - check_partial(common, TRUE); - JUMPHERE(jump[0]); - } - return cc; -#endif - - case OP_CHAR: - case OP_CHARI: - length = 1; -#ifdef SUPPORT_UTF - if (common->utf && HAS_EXTRALEN(*cc)) length += GET_EXTRALEN(*cc); -#endif - if (common->mode == JIT_COMPILE && check_str_ptr - && (type == OP_CHAR || !char_has_othercase(common, cc) || char_get_othercase_bit(common, cc) != 0)) - { - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(length)); - add_jump(compiler, backtracks, CMP(SLJIT_GREATER, STR_PTR, 0, STR_END, 0)); - - context.length = IN_UCHARS(length); - context.sourcereg = -1; -#if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED - context.ucharptr = 0; -#endif - return byte_sequence_compare(common, type == OP_CHARI, cc, &context, backtracks); - } - - if (check_str_ptr) - detect_partial_match(common, backtracks); -#ifdef SUPPORT_UTF - if (common->utf) - { - GETCHAR(c, cc); - } - else -#endif - c = *cc; - - if (type == OP_CHAR || !char_has_othercase(common, cc)) - { - read_char_range(common, c, c, FALSE); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, c)); - return cc + length; - } - oc = char_othercase(common, c); - read_char_range(common, c < oc ? c : oc, c > oc ? c : oc, FALSE); - bit = c ^ oc; - if (is_powerof2(bit)) - { - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, bit); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, c | bit)); - return cc + length; - } - jump[0] = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, oc)); - JUMPHERE(jump[0]); - return cc + length; - - case OP_NOT: - case OP_NOTI: - if (check_str_ptr) - detect_partial_match(common, backtracks); - length = 1; -#ifdef SUPPORT_UTF - if (common->utf) - { -#ifdef COMPILE_PCRE8 - c = *cc; - if (c < 128) - { - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); - if (type == OP_NOT || !char_has_othercase(common, cc)) - add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c)); - else - { - /* Since UTF8 code page is fixed, we know that c is in [a-z] or [A-Z] range. */ - OP2(SLJIT_OR, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x20); - add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, c | 0x20)); - } - /* Skip the variable-length character. */ - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - jump[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); - JUMPHERE(jump[0]); - return cc + 1; - } - else -#endif /* COMPILE_PCRE8 */ - { - GETCHARLEN(c, cc, length); - } - } - else -#endif /* SUPPORT_UTF */ - c = *cc; - - if (type == OP_NOT || !char_has_othercase(common, cc)) - { - read_char_range(common, c, c, TRUE); - add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c)); - } - else - { - oc = char_othercase(common, c); - read_char_range(common, c < oc ? c : oc, c > oc ? c : oc, TRUE); - bit = c ^ oc; - if (is_powerof2(bit)) - { - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, bit); - add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c | bit)); - } - else - { - add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c)); - add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, oc)); - } - } - return cc + length; - - case OP_CLASS: - case OP_NCLASS: - if (check_str_ptr) - detect_partial_match(common, backtracks); - -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 - bit = (common->utf && is_char7_bitset((const sljit_u8 *)cc, type == OP_NCLASS)) ? 127 : 255; - read_char_range(common, 0, bit, type == OP_NCLASS); -#else - read_char_range(common, 0, 255, type == OP_NCLASS); -#endif - - if (check_class_ranges(common, (const sljit_u8 *)cc, type == OP_NCLASS, FALSE, backtracks)) - return cc + 32 / sizeof(pcre_uchar); - -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 - jump[0] = NULL; - if (common->utf) - { - jump[0] = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, bit); - if (type == OP_CLASS) - { - add_jump(compiler, backtracks, jump[0]); - jump[0] = NULL; - } - } -#elif !defined COMPILE_PCRE8 - jump[0] = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255); - if (type == OP_CLASS) - { - add_jump(compiler, backtracks, jump[0]); - jump[0] = NULL; - } -#endif /* SUPPORT_UTF && COMPILE_PCRE8 */ - - OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7); - OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3); - OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc); - OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); - OP2(SLJIT_AND | SLJIT_SET_E, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); - add_jump(compiler, backtracks, JUMP(SLJIT_ZERO)); - -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - if (jump[0] != NULL) - JUMPHERE(jump[0]); -#endif - return cc + 32 / sizeof(pcre_uchar); - -#if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - case OP_XCLASS: - if (check_str_ptr) - detect_partial_match(common, backtracks); - compile_xclass_matchingpath(common, cc + LINK_SIZE, backtracks); - return cc + GET(cc, 0) - 1; -#endif - } -SLJIT_ASSERT_STOP(); -return cc; -} - -static SLJIT_INLINE pcre_uchar *compile_charn_matchingpath(compiler_common *common, pcre_uchar *cc, pcre_uchar *ccend, jump_list **backtracks) -{ -/* This function consumes at least one input character. */ -/* To decrease the number of length checks, we try to concatenate the fixed length character sequences. */ -DEFINE_COMPILER; -pcre_uchar *ccbegin = cc; -compare_context context; -int size; - -context.length = 0; -do - { - if (cc >= ccend) - break; - - if (*cc == OP_CHAR) - { - size = 1; -#ifdef SUPPORT_UTF - if (common->utf && HAS_EXTRALEN(cc[1])) - size += GET_EXTRALEN(cc[1]); -#endif - } - else if (*cc == OP_CHARI) - { - size = 1; -#ifdef SUPPORT_UTF - if (common->utf) - { - if (char_has_othercase(common, cc + 1) && char_get_othercase_bit(common, cc + 1) == 0) - size = 0; - else if (HAS_EXTRALEN(cc[1])) - size += GET_EXTRALEN(cc[1]); - } - else -#endif - if (char_has_othercase(common, cc + 1) && char_get_othercase_bit(common, cc + 1) == 0) - size = 0; - } - else - size = 0; - - cc += 1 + size; - context.length += IN_UCHARS(size); - } -while (size > 0 && context.length <= 128); - -cc = ccbegin; -if (context.length > 0) - { - /* We have a fixed-length byte sequence. */ - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, context.length); - add_jump(compiler, backtracks, CMP(SLJIT_GREATER, STR_PTR, 0, STR_END, 0)); - - context.sourcereg = -1; -#if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED - context.ucharptr = 0; -#endif - do cc = byte_sequence_compare(common, *cc == OP_CHARI, cc + 1, &context, backtracks); while (context.length > 0); - return cc; - } - -/* A non-fixed length character will be checked if length == 0. */ -return compile_char1_matchingpath(common, *cc, cc + 1, backtracks, TRUE); -} - -/* Forward definitions. */ -static void compile_matchingpath(compiler_common *, pcre_uchar *, pcre_uchar *, backtrack_common *); -static void compile_backtrackingpath(compiler_common *, struct backtrack_common *); - -#define PUSH_BACKTRACK(size, ccstart, error) \ - do \ - { \ - backtrack = sljit_alloc_memory(compiler, (size)); \ - if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) \ - return error; \ - memset(backtrack, 0, size); \ - backtrack->prev = parent->top; \ - backtrack->cc = (ccstart); \ - parent->top = backtrack; \ - } \ - while (0) - -#define PUSH_BACKTRACK_NOVALUE(size, ccstart) \ - do \ - { \ - backtrack = sljit_alloc_memory(compiler, (size)); \ - if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) \ - return; \ - memset(backtrack, 0, size); \ - backtrack->prev = parent->top; \ - backtrack->cc = (ccstart); \ - parent->top = backtrack; \ - } \ - while (0) - -#define BACKTRACK_AS(type) ((type *)backtrack) - -static void compile_dnref_search(compiler_common *common, pcre_uchar *cc, jump_list **backtracks) -{ -/* The OVECTOR offset goes to TMP2. */ -DEFINE_COMPILER; -int count = GET2(cc, 1 + IMM2_SIZE); -pcre_uchar *slot = common->name_table + GET2(cc, 1) * common->name_entry_size; -unsigned int offset; -jump_list *found = NULL; - -SLJIT_ASSERT(*cc == OP_DNREF || *cc == OP_DNREFI); - -OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1)); - -count--; -while (count-- > 0) - { - offset = GET2(slot, 0) << 1; - GET_LOCAL_BASE(TMP2, 0, OVECTOR(offset)); - add_jump(compiler, &found, CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0)); - slot += common->name_entry_size; - } - -offset = GET2(slot, 0) << 1; -GET_LOCAL_BASE(TMP2, 0, OVECTOR(offset)); -if (backtracks != NULL && !common->jscript_compat) - add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0)); - -set_jumps(found, LABEL()); -} - -static void compile_ref_matchingpath(compiler_common *common, pcre_uchar *cc, jump_list **backtracks, BOOL withchecks, BOOL emptyfail) -{ -DEFINE_COMPILER; -BOOL ref = (*cc == OP_REF || *cc == OP_REFI); -int offset = 0; -struct sljit_jump *jump = NULL; -struct sljit_jump *partial; -struct sljit_jump *nopartial; - -if (ref) - { - offset = GET2(cc, 1) << 1; - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset)); - /* OVECTOR(1) contains the "string begin - 1" constant. */ - if (withchecks && !common->jscript_compat) - add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1))); - } -else - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), 0); - -#if defined SUPPORT_UTF && defined SUPPORT_UCP -if (common->utf && *cc == OP_REFI) - { - SLJIT_ASSERT(TMP1 == SLJIT_R0 && STACK_TOP == SLJIT_R1 && TMP2 == SLJIT_R2); - if (ref) - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); - else - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw)); - - if (withchecks) - jump = CMP(SLJIT_EQUAL, TMP1, 0, TMP2, 0); - - /* Needed to save important temporary registers. */ - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STACK_TOP, 0); - OP1(SLJIT_MOV, SLJIT_R1, 0, ARGUMENTS, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_R1), SLJIT_OFFSETOF(jit_arguments, uchar_ptr), STR_PTR, 0); - sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_utf_caselesscmp)); - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); - if (common->mode == JIT_COMPILE) - add_jump(compiler, backtracks, CMP(SLJIT_LESS_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 1)); - else - { - add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0)); - nopartial = CMP(SLJIT_NOT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 1); - check_partial(common, FALSE); - add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); - JUMPHERE(nopartial); - } - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_RETURN_REG, 0); - } -else -#endif /* SUPPORT_UTF && SUPPORT_UCP */ - { - if (ref) - OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP1, 0); - else - OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw), TMP1, 0); - - if (withchecks) - jump = JUMP(SLJIT_ZERO); - - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); - partial = CMP(SLJIT_GREATER, STR_PTR, 0, STR_END, 0); - if (common->mode == JIT_COMPILE) - add_jump(compiler, backtracks, partial); - - add_jump(compiler, *cc == OP_REF ? &common->casefulcmp : &common->caselesscmp, JUMP(SLJIT_FAST_CALL)); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0)); - - if (common->mode != JIT_COMPILE) - { - nopartial = JUMP(SLJIT_JUMP); - JUMPHERE(partial); - /* TMP2 -= STR_END - STR_PTR */ - OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, STR_PTR, 0); - OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, STR_END, 0); - partial = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, 0); - OP1(SLJIT_MOV, STR_PTR, 0, STR_END, 0); - add_jump(compiler, *cc == OP_REF ? &common->casefulcmp : &common->caselesscmp, JUMP(SLJIT_FAST_CALL)); - add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0)); - JUMPHERE(partial); - check_partial(common, FALSE); - add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); - JUMPHERE(nopartial); - } - } - -if (jump != NULL) - { - if (emptyfail) - add_jump(compiler, backtracks, jump); - else - JUMPHERE(jump); - } -} - -static SLJIT_INLINE pcre_uchar *compile_ref_iterator_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent) -{ -DEFINE_COMPILER; -BOOL ref = (*cc == OP_REF || *cc == OP_REFI); -backtrack_common *backtrack; -pcre_uchar type; -int offset = 0; -struct sljit_label *label; -struct sljit_jump *zerolength; -struct sljit_jump *jump = NULL; -pcre_uchar *ccbegin = cc; -int min = 0, max = 0; -BOOL minimize; - -PUSH_BACKTRACK(sizeof(ref_iterator_backtrack), cc, NULL); - -if (ref) - offset = GET2(cc, 1) << 1; -else - cc += IMM2_SIZE; -type = cc[1 + IMM2_SIZE]; - -SLJIT_COMPILE_ASSERT((OP_CRSTAR & 0x1) == 0, crstar_opcode_must_be_even); -minimize = (type & 0x1) != 0; -switch(type) - { - case OP_CRSTAR: - case OP_CRMINSTAR: - min = 0; - max = 0; - cc += 1 + IMM2_SIZE + 1; - break; - case OP_CRPLUS: - case OP_CRMINPLUS: - min = 1; - max = 0; - cc += 1 + IMM2_SIZE + 1; - break; - case OP_CRQUERY: - case OP_CRMINQUERY: - min = 0; - max = 1; - cc += 1 + IMM2_SIZE + 1; - break; - case OP_CRRANGE: - case OP_CRMINRANGE: - min = GET2(cc, 1 + IMM2_SIZE + 1); - max = GET2(cc, 1 + IMM2_SIZE + 1 + IMM2_SIZE); - cc += 1 + IMM2_SIZE + 1 + 2 * IMM2_SIZE; - break; - default: - SLJIT_ASSERT_STOP(); - break; - } - -if (!minimize) - { - if (min == 0) - { - allocate_stack(common, 2); - if (ref) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset)); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0); - /* Temporary release of STR_PTR. */ - OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); - /* Handles both invalid and empty cases. Since the minimum repeat, - is zero the invalid case is basically the same as an empty case. */ - if (ref) - zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); - else - { - compile_dnref_search(common, ccbegin, NULL); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), 0); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1, TMP2, 0); - zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw)); - } - /* Restore if not zero length. */ - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); - } - else - { - allocate_stack(common, 1); - if (ref) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset)); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); - if (ref) - { - add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1))); - zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); - } - else - { - compile_dnref_search(common, ccbegin, &backtrack->topbacktracks); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), 0); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1, TMP2, 0); - zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw)); - } - } - - if (min > 1 || max > 1) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0, SLJIT_IMM, 0); - - label = LABEL(); - if (!ref) - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1); - compile_ref_matchingpath(common, ccbegin, &backtrack->topbacktracks, FALSE, FALSE); - - if (min > 1 || max > 1) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0); - OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0, TMP1, 0); - if (min > 1) - CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, min, label); - if (max > 1) - { - jump = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, max); - allocate_stack(common, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); - JUMPTO(SLJIT_JUMP, label); - JUMPHERE(jump); - } - } - - if (max == 0) - { - /* Includes min > 1 case as well. */ - allocate_stack(common, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); - JUMPTO(SLJIT_JUMP, label); - } - - JUMPHERE(zerolength); - BACKTRACK_AS(ref_iterator_backtrack)->matchingpath = LABEL(); - - count_match(common); - return cc; - } - -allocate_stack(common, ref ? 2 : 3); -if (ref) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset)); -OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); -if (type != OP_CRMINSTAR) - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0); - -if (min == 0) - { - /* Handles both invalid and empty cases. Since the minimum repeat, - is zero the invalid case is basically the same as an empty case. */ - if (ref) - zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); - else - { - compile_dnref_search(common, ccbegin, NULL); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), 0); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(2), TMP2, 0); - zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw)); - } - /* Length is non-zero, we can match real repeats. */ - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); - jump = JUMP(SLJIT_JUMP); - } -else - { - if (ref) - { - add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1))); - zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); - } - else - { - compile_dnref_search(common, ccbegin, &backtrack->topbacktracks); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), 0); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(2), TMP2, 0); - zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw)); - } - } - -BACKTRACK_AS(ref_iterator_backtrack)->matchingpath = LABEL(); -if (max > 0) - add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, max)); - -if (!ref) - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(2)); -compile_ref_matchingpath(common, ccbegin, &backtrack->topbacktracks, TRUE, TRUE); -OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); - -if (min > 1) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); - OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0); - CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, min, BACKTRACK_AS(ref_iterator_backtrack)->matchingpath); - } -else if (max > 0) - OP2(SLJIT_ADD, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 1); - -if (jump != NULL) - JUMPHERE(jump); -JUMPHERE(zerolength); - -count_match(common); -return cc; -} - -static SLJIT_INLINE pcre_uchar *compile_recurse_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent) -{ -DEFINE_COMPILER; -backtrack_common *backtrack; -recurse_entry *entry = common->entries; -recurse_entry *prev = NULL; -sljit_sw start = GET(cc, 1); -pcre_uchar *start_cc; -BOOL needs_control_head; - -PUSH_BACKTRACK(sizeof(recurse_backtrack), cc, NULL); - -/* Inlining simple patterns. */ -if (get_framesize(common, common->start + start, NULL, TRUE, &needs_control_head) == no_stack) - { - start_cc = common->start + start; - compile_matchingpath(common, next_opcode(common, start_cc), bracketend(start_cc) - (1 + LINK_SIZE), backtrack); - BACKTRACK_AS(recurse_backtrack)->inlined_pattern = TRUE; - return cc + 1 + LINK_SIZE; - } - -while (entry != NULL) - { - if (entry->start == start) - break; - prev = entry; - entry = entry->next; - } - -if (entry == NULL) - { - entry = sljit_alloc_memory(compiler, sizeof(recurse_entry)); - if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) - return NULL; - entry->next = NULL; - entry->entry = NULL; - entry->calls = NULL; - entry->start = start; - - if (prev != NULL) - prev->next = entry; - else - common->entries = entry; - } - -if (common->has_set_som && common->mark_ptr != 0) - { - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0)); - allocate_stack(common, 2); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0); - } -else if (common->has_set_som || common->mark_ptr != 0) - { - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->has_set_som ? (int)(OVECTOR(0)) : common->mark_ptr); - allocate_stack(common, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); - } - -if (entry->entry == NULL) - add_jump(compiler, &entry->calls, JUMP(SLJIT_FAST_CALL)); -else - JUMPTO(SLJIT_FAST_CALL, entry->entry); -/* Leave if the match is failed. */ -add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0)); -return cc + 1 + LINK_SIZE; -} - -static int SLJIT_CALL do_callout(struct jit_arguments *arguments, PUBL(callout_block) *callout_block, pcre_uchar **jit_ovector) -{ -const pcre_uchar *begin = arguments->begin; -int *offset_vector = arguments->offsets; -int offset_count = arguments->offset_count; -int i; - -if (PUBL(callout) == NULL) - return 0; - -callout_block->version = 2; -callout_block->callout_data = arguments->callout_data; - -/* Offsets in subject. */ -callout_block->subject_length = arguments->end - arguments->begin; -callout_block->start_match = (pcre_uchar*)callout_block->subject - arguments->begin; -callout_block->current_position = (pcre_uchar*)callout_block->offset_vector - arguments->begin; -#if defined COMPILE_PCRE8 -callout_block->subject = (PCRE_SPTR)begin; -#elif defined COMPILE_PCRE16 -callout_block->subject = (PCRE_SPTR16)begin; -#elif defined COMPILE_PCRE32 -callout_block->subject = (PCRE_SPTR32)begin; -#endif - -/* Convert and copy the JIT offset vector to the offset_vector array. */ -callout_block->capture_top = 0; -callout_block->offset_vector = offset_vector; -for (i = 2; i < offset_count; i += 2) - { - offset_vector[i] = jit_ovector[i] - begin; - offset_vector[i + 1] = jit_ovector[i + 1] - begin; - if (jit_ovector[i] >= begin) - callout_block->capture_top = i; - } - -callout_block->capture_top = (callout_block->capture_top >> 1) + 1; -if (offset_count > 0) - offset_vector[0] = -1; -if (offset_count > 1) - offset_vector[1] = -1; -return (*PUBL(callout))(callout_block); -} - -/* Aligning to 8 byte. */ -#define CALLOUT_ARG_SIZE \ - (((int)sizeof(PUBL(callout_block)) + 7) & ~7) - -#define CALLOUT_ARG_OFFSET(arg) \ - (-CALLOUT_ARG_SIZE + SLJIT_OFFSETOF(PUBL(callout_block), arg)) - -static SLJIT_INLINE pcre_uchar *compile_callout_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent) -{ -DEFINE_COMPILER; -backtrack_common *backtrack; - -PUSH_BACKTRACK(sizeof(backtrack_common), cc, NULL); - -allocate_stack(common, CALLOUT_ARG_SIZE / sizeof(sljit_sw)); - -SLJIT_ASSERT(common->capture_last_ptr != 0); -OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr); -OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); -OP1(SLJIT_MOV_S32, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(callout_number), SLJIT_IMM, cc[1]); -OP1(SLJIT_MOV_S32, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(capture_last), TMP2, 0); - -/* These pointer sized fields temporarly stores internal variables. */ -OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0)); -OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(offset_vector), STR_PTR, 0); -OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(subject), TMP2, 0); - -if (common->mark_ptr != 0) - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, mark_ptr)); -OP1(SLJIT_MOV_S32, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(pattern_position), SLJIT_IMM, GET(cc, 2)); -OP1(SLJIT_MOV_S32, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(next_item_length), SLJIT_IMM, GET(cc, 2 + LINK_SIZE)); -OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(mark), (common->mark_ptr != 0) ? TMP2 : SLJIT_IMM, 0); - -/* Needed to save important temporary registers. */ -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STACK_TOP, 0); -OP2(SLJIT_SUB, SLJIT_R1, 0, STACK_TOP, 0, SLJIT_IMM, CALLOUT_ARG_SIZE); -GET_LOCAL_BASE(SLJIT_R2, 0, OVECTOR_START); -sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_callout)); -OP1(SLJIT_MOV_S32, SLJIT_RETURN_REG, 0, SLJIT_RETURN_REG, 0); -OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); -free_stack(common, CALLOUT_ARG_SIZE / sizeof(sljit_sw)); - -/* Check return value. */ -OP2(SLJIT_SUB | SLJIT_SET_S, SLJIT_UNUSED, 0, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0); -add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_SIG_GREATER)); -if (common->forced_quit_label == NULL) - add_jump(compiler, &common->forced_quit, JUMP(SLJIT_SIG_LESS)); -else - JUMPTO(SLJIT_SIG_LESS, common->forced_quit_label); -return cc + 2 + 2 * LINK_SIZE; -} - -#undef CALLOUT_ARG_SIZE -#undef CALLOUT_ARG_OFFSET - -static SLJIT_INLINE BOOL assert_needs_str_ptr_saving(pcre_uchar *cc) -{ -while (TRUE) - { - switch (*cc) - { - case OP_NOT_WORD_BOUNDARY: - case OP_WORD_BOUNDARY: - case OP_CIRC: - case OP_CIRCM: - case OP_DOLL: - case OP_DOLLM: - case OP_CALLOUT: - case OP_ALT: - cc += PRIV(OP_lengths)[*cc]; - break; - - case OP_KET: - return FALSE; - - default: - return TRUE; - } - } -} - -static pcre_uchar *compile_assert_matchingpath(compiler_common *common, pcre_uchar *cc, assert_backtrack *backtrack, BOOL conditional) -{ -DEFINE_COMPILER; -int framesize; -int extrasize; -BOOL needs_control_head; -int private_data_ptr; -backtrack_common altbacktrack; -pcre_uchar *ccbegin; -pcre_uchar opcode; -pcre_uchar bra = OP_BRA; -jump_list *tmp = NULL; -jump_list **target = (conditional) ? &backtrack->condfailed : &backtrack->common.topbacktracks; -jump_list **found; -/* Saving previous accept variables. */ -BOOL save_local_exit = common->local_exit; -BOOL save_positive_assert = common->positive_assert; -then_trap_backtrack *save_then_trap = common->then_trap; -struct sljit_label *save_quit_label = common->quit_label; -struct sljit_label *save_accept_label = common->accept_label; -jump_list *save_quit = common->quit; -jump_list *save_positive_assert_quit = common->positive_assert_quit; -jump_list *save_accept = common->accept; -struct sljit_jump *jump; -struct sljit_jump *brajump = NULL; - -/* Assert captures then. */ -common->then_trap = NULL; - -if (*cc == OP_BRAZERO || *cc == OP_BRAMINZERO) - { - SLJIT_ASSERT(!conditional); - bra = *cc; - cc++; - } -private_data_ptr = PRIVATE_DATA(cc); -SLJIT_ASSERT(private_data_ptr != 0); -framesize = get_framesize(common, cc, NULL, FALSE, &needs_control_head); -backtrack->framesize = framesize; -backtrack->private_data_ptr = private_data_ptr; -opcode = *cc; -SLJIT_ASSERT(opcode >= OP_ASSERT && opcode <= OP_ASSERTBACK_NOT); -found = (opcode == OP_ASSERT || opcode == OP_ASSERTBACK) ? &tmp : target; -ccbegin = cc; -cc += GET(cc, 1); - -if (bra == OP_BRAMINZERO) - { - /* This is a braminzero backtrack path. */ - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - free_stack(common, 1); - brajump = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0); - } - -if (framesize < 0) - { - extrasize = 1; - if (bra == OP_BRA && !assert_needs_str_ptr_saving(ccbegin + 1 + LINK_SIZE)) - extrasize = 0; - - if (needs_control_head) - extrasize++; - - if (framesize == no_frame) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0); - - if (extrasize > 0) - allocate_stack(common, extrasize); - - if (needs_control_head) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); - - if (extrasize > 0) - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); - - if (needs_control_head) - { - SLJIT_ASSERT(extrasize == 2); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0); - } - } -else - { - extrasize = needs_control_head ? 3 : 2; - allocate_stack(common, framesize + extrasize); - - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + extrasize) * sizeof(sljit_sw)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP2, 0); - if (needs_control_head) - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); - - if (needs_control_head) - { - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(2), TMP1, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP2, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0); - } - else - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0); - - init_frame(common, ccbegin, NULL, framesize + extrasize - 1, extrasize, FALSE); - } - -memset(&altbacktrack, 0, sizeof(backtrack_common)); -if (opcode == OP_ASSERT_NOT || opcode == OP_ASSERTBACK_NOT) - { - /* Negative assert is stronger than positive assert. */ - common->local_exit = TRUE; - common->quit_label = NULL; - common->quit = NULL; - common->positive_assert = FALSE; - } -else - common->positive_assert = TRUE; -common->positive_assert_quit = NULL; - -while (1) - { - common->accept_label = NULL; - common->accept = NULL; - altbacktrack.top = NULL; - altbacktrack.topbacktracks = NULL; - - if (*ccbegin == OP_ALT && extrasize > 0) - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - - altbacktrack.cc = ccbegin; - compile_matchingpath(common, ccbegin + 1 + LINK_SIZE, cc, &altbacktrack); - if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) - { - if (opcode == OP_ASSERT_NOT || opcode == OP_ASSERTBACK_NOT) - { - common->local_exit = save_local_exit; - common->quit_label = save_quit_label; - common->quit = save_quit; - } - common->positive_assert = save_positive_assert; - common->then_trap = save_then_trap; - common->accept_label = save_accept_label; - common->positive_assert_quit = save_positive_assert_quit; - common->accept = save_accept; - return NULL; - } - common->accept_label = LABEL(); - if (common->accept != NULL) - set_jumps(common->accept, common->accept_label); - - /* Reset stack. */ - if (framesize < 0) - { - if (framesize == no_frame) - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - else if (extrasize > 0) - free_stack(common, extrasize); - - if (needs_control_head) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), 0); - } - else - { - if ((opcode != OP_ASSERT_NOT && opcode != OP_ASSERTBACK_NOT) || conditional) - { - /* We don't need to keep the STR_PTR, only the previous private_data_ptr. */ - OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw)); - if (needs_control_head) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), 0); - } - else - { - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - if (needs_control_head) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), (framesize + 1) * sizeof(sljit_sw)); - add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - } - } - - if (opcode == OP_ASSERT_NOT || opcode == OP_ASSERTBACK_NOT) - { - /* We know that STR_PTR was stored on the top of the stack. */ - if (conditional) - { - if (extrasize > 0) - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), needs_control_head ? sizeof(sljit_sw) : 0); - } - else if (bra == OP_BRAZERO) - { - if (framesize < 0) - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (extrasize - 1) * sizeof(sljit_sw)); - else - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_sw)); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (framesize + extrasize - 1) * sizeof(sljit_sw)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0); - } - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); - } - else if (framesize >= 0) - { - /* For OP_BRA and OP_BRAMINZERO. */ - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_sw)); - } - } - add_jump(compiler, found, JUMP(SLJIT_JUMP)); - - compile_backtrackingpath(common, altbacktrack.top); - if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) - { - if (opcode == OP_ASSERT_NOT || opcode == OP_ASSERTBACK_NOT) - { - common->local_exit = save_local_exit; - common->quit_label = save_quit_label; - common->quit = save_quit; - } - common->positive_assert = save_positive_assert; - common->then_trap = save_then_trap; - common->accept_label = save_accept_label; - common->positive_assert_quit = save_positive_assert_quit; - common->accept = save_accept; - return NULL; - } - set_jumps(altbacktrack.topbacktracks, LABEL()); - - if (*cc != OP_ALT) - break; - - ccbegin = cc; - cc += GET(cc, 1); - } - -if (opcode == OP_ASSERT_NOT || opcode == OP_ASSERTBACK_NOT) - { - SLJIT_ASSERT(common->positive_assert_quit == NULL); - /* Makes the check less complicated below. */ - common->positive_assert_quit = common->quit; - } - -/* None of them matched. */ -if (common->positive_assert_quit != NULL) - { - jump = JUMP(SLJIT_JUMP); - set_jumps(common->positive_assert_quit, LABEL()); - SLJIT_ASSERT(framesize != no_stack); - if (framesize < 0) - OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, extrasize * sizeof(sljit_sw)); - else - { - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + extrasize) * sizeof(sljit_sw)); - } - JUMPHERE(jump); - } - -if (needs_control_head) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(1)); - -if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK) - { - /* Assert is failed. */ - if ((conditional && extrasize > 0) || bra == OP_BRAZERO) - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - - if (framesize < 0) - { - /* The topmost item should be 0. */ - if (bra == OP_BRAZERO) - { - if (extrasize == 2) - free_stack(common, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); - } - else if (extrasize > 0) - free_stack(common, extrasize); - } - else - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(extrasize - 1)); - /* The topmost item should be 0. */ - if (bra == OP_BRAZERO) - { - free_stack(common, framesize + extrasize - 1); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); - } - else - free_stack(common, framesize + extrasize); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0); - } - jump = JUMP(SLJIT_JUMP); - if (bra != OP_BRAZERO) - add_jump(compiler, target, jump); - - /* Assert is successful. */ - set_jumps(tmp, LABEL()); - if (framesize < 0) - { - /* We know that STR_PTR was stored on the top of the stack. */ - if (extrasize > 0) - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (extrasize - 1) * sizeof(sljit_sw)); - - /* Keep the STR_PTR on the top of the stack. */ - if (bra == OP_BRAZERO) - { - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); - if (extrasize == 2) - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); - } - else if (bra == OP_BRAMINZERO) - { - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); - } - } - else - { - if (bra == OP_BRA) - { - /* We don't need to keep the STR_PTR, only the previous private_data_ptr. */ - OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw)); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), (extrasize - 2) * sizeof(sljit_sw)); - } - else - { - /* We don't need to keep the STR_PTR, only the previous private_data_ptr. */ - OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 2) * sizeof(sljit_sw)); - if (extrasize == 2) - { - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - if (bra == OP_BRAMINZERO) - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); - } - else - { - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), 0); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), bra == OP_BRAZERO ? STR_PTR : SLJIT_IMM, 0); - } - } - } - - if (bra == OP_BRAZERO) - { - backtrack->matchingpath = LABEL(); - SET_LABEL(jump, backtrack->matchingpath); - } - else if (bra == OP_BRAMINZERO) - { - JUMPTO(SLJIT_JUMP, backtrack->matchingpath); - JUMPHERE(brajump); - if (framesize >= 0) - { - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), framesize * sizeof(sljit_sw)); - } - set_jumps(backtrack->common.topbacktracks, LABEL()); - } - } -else - { - /* AssertNot is successful. */ - if (framesize < 0) - { - if (extrasize > 0) - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - - if (bra != OP_BRA) - { - if (extrasize == 2) - free_stack(common, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); - } - else if (extrasize > 0) - free_stack(common, extrasize); - } - else - { - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(extrasize - 1)); - /* The topmost item should be 0. */ - if (bra != OP_BRA) - { - free_stack(common, framesize + extrasize - 1); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); - } - else - free_stack(common, framesize + extrasize); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0); - } - - if (bra == OP_BRAZERO) - backtrack->matchingpath = LABEL(); - else if (bra == OP_BRAMINZERO) - { - JUMPTO(SLJIT_JUMP, backtrack->matchingpath); - JUMPHERE(brajump); - } - - if (bra != OP_BRA) - { - SLJIT_ASSERT(found == &backtrack->common.topbacktracks); - set_jumps(backtrack->common.topbacktracks, LABEL()); - backtrack->common.topbacktracks = NULL; - } - } - -if (opcode == OP_ASSERT_NOT || opcode == OP_ASSERTBACK_NOT) - { - common->local_exit = save_local_exit; - common->quit_label = save_quit_label; - common->quit = save_quit; - } -common->positive_assert = save_positive_assert; -common->then_trap = save_then_trap; -common->accept_label = save_accept_label; -common->positive_assert_quit = save_positive_assert_quit; -common->accept = save_accept; -return cc + 1 + LINK_SIZE; -} - -static SLJIT_INLINE void match_once_common(compiler_common *common, pcre_uchar ket, int framesize, int private_data_ptr, BOOL has_alternatives, BOOL needs_control_head) -{ -DEFINE_COMPILER; -int stacksize; - -if (framesize < 0) - { - if (framesize == no_frame) - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - else - { - stacksize = needs_control_head ? 1 : 0; - if (ket != OP_KET || has_alternatives) - stacksize++; - - if (stacksize > 0) - free_stack(common, stacksize); - } - - if (needs_control_head) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), (ket != OP_KET || has_alternatives) ? sizeof(sljit_sw) : 0); - - /* TMP2 which is set here used by OP_KETRMAX below. */ - if (ket == OP_KETRMAX) - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), 0); - else if (ket == OP_KETRMIN) - { - /* Move the STR_PTR to the private_data_ptr. */ - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), 0); - } - } -else - { - stacksize = (ket != OP_KET || has_alternatives) ? 2 : 1; - OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + stacksize) * sizeof(sljit_sw)); - if (needs_control_head) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), 0); - - if (ket == OP_KETRMAX) - { - /* TMP2 which is set here used by OP_KETRMAX below. */ - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - } - } -if (needs_control_head) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, TMP1, 0); -} - -static SLJIT_INLINE int match_capture_common(compiler_common *common, int stacksize, int offset, int private_data_ptr) -{ -DEFINE_COMPILER; - -if (common->capture_last_ptr != 0) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, SLJIT_IMM, offset >> 1); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), TMP1, 0); - stacksize++; - } -if (common->optimized_cbracket[offset >> 1] == 0) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset)); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), TMP1, 0); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize + 1), TMP2, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0); - stacksize += 2; - } -return stacksize; -} - -/* - Handling bracketed expressions is probably the most complex part. - - Stack layout naming characters: - S - Push the current STR_PTR - 0 - Push a 0 (NULL) - A - Push the current STR_PTR. Needed for restoring the STR_PTR - before the next alternative. Not pushed if there are no alternatives. - M - Any values pushed by the current alternative. Can be empty, or anything. - C - Push the previous OVECTOR(i), OVECTOR(i+1) and OVECTOR_PRIV(i) to the stack. - L - Push the previous local (pointed by localptr) to the stack - () - opional values stored on the stack - ()* - optonal, can be stored multiple times - - The following list shows the regular expression templates, their PCRE byte codes - and stack layout supported by pcre-sljit. - - (?:) OP_BRA | OP_KET A M - () OP_CBRA | OP_KET C M - (?:)+ OP_BRA | OP_KETRMAX 0 A M S ( A M S )* - OP_SBRA | OP_KETRMAX 0 L M S ( L M S )* - (?:)+? OP_BRA | OP_KETRMIN 0 A M S ( A M S )* - OP_SBRA | OP_KETRMIN 0 L M S ( L M S )* - ()+ OP_CBRA | OP_KETRMAX 0 C M S ( C M S )* - OP_SCBRA | OP_KETRMAX 0 C M S ( C M S )* - ()+? OP_CBRA | OP_KETRMIN 0 C M S ( C M S )* - OP_SCBRA | OP_KETRMIN 0 C M S ( C M S )* - (?:)? OP_BRAZERO | OP_BRA | OP_KET S ( A M 0 ) - (?:)?? OP_BRAMINZERO | OP_BRA | OP_KET S ( A M 0 ) - ()? OP_BRAZERO | OP_CBRA | OP_KET S ( C M 0 ) - ()?? OP_BRAMINZERO | OP_CBRA | OP_KET S ( C M 0 ) - (?:)* OP_BRAZERO | OP_BRA | OP_KETRMAX S 0 ( A M S )* - OP_BRAZERO | OP_SBRA | OP_KETRMAX S 0 ( L M S )* - (?:)*? OP_BRAMINZERO | OP_BRA | OP_KETRMIN S 0 ( A M S )* - OP_BRAMINZERO | OP_SBRA | OP_KETRMIN S 0 ( L M S )* - ()* OP_BRAZERO | OP_CBRA | OP_KETRMAX S 0 ( C M S )* - OP_BRAZERO | OP_SCBRA | OP_KETRMAX S 0 ( C M S )* - ()*? OP_BRAMINZERO | OP_CBRA | OP_KETRMIN S 0 ( C M S )* - OP_BRAMINZERO | OP_SCBRA | OP_KETRMIN S 0 ( C M S )* - - - Stack layout naming characters: - A - Push the alternative index (starting from 0) on the stack. - Not pushed if there is no alternatives. - M - Any values pushed by the current alternative. Can be empty, or anything. - - The next list shows the possible content of a bracket: - (|) OP_*BRA | OP_ALT ... M A - (?()|) OP_*COND | OP_ALT M A - (?>|) OP_ONCE | OP_ALT ... [stack trace] M A - (?>|) OP_ONCE_NC | OP_ALT ... [stack trace] M A - Or nothing, if trace is unnecessary -*/ - -static pcre_uchar *compile_bracket_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent) -{ -DEFINE_COMPILER; -backtrack_common *backtrack; -pcre_uchar opcode; -int private_data_ptr = 0; -int offset = 0; -int i, stacksize; -int repeat_ptr = 0, repeat_length = 0; -int repeat_type = 0, repeat_count = 0; -pcre_uchar *ccbegin; -pcre_uchar *matchingpath; -pcre_uchar *slot; -pcre_uchar bra = OP_BRA; -pcre_uchar ket; -assert_backtrack *assert; -BOOL has_alternatives; -BOOL needs_control_head = FALSE; -struct sljit_jump *jump; -struct sljit_jump *skip; -struct sljit_label *rmax_label = NULL; -struct sljit_jump *braminzero = NULL; - -PUSH_BACKTRACK(sizeof(bracket_backtrack), cc, NULL); - -if (*cc == OP_BRAZERO || *cc == OP_BRAMINZERO) - { - bra = *cc; - cc++; - opcode = *cc; - } - -opcode = *cc; -ccbegin = cc; -matchingpath = bracketend(cc) - 1 - LINK_SIZE; -ket = *matchingpath; -if (ket == OP_KET && PRIVATE_DATA(matchingpath) != 0) - { - repeat_ptr = PRIVATE_DATA(matchingpath); - repeat_length = PRIVATE_DATA(matchingpath + 1); - repeat_type = PRIVATE_DATA(matchingpath + 2); - repeat_count = PRIVATE_DATA(matchingpath + 3); - SLJIT_ASSERT(repeat_length != 0 && repeat_type != 0 && repeat_count != 0); - if (repeat_type == OP_UPTO) - ket = OP_KETRMAX; - if (repeat_type == OP_MINUPTO) - ket = OP_KETRMIN; - } - -if ((opcode == OP_COND || opcode == OP_SCOND) && cc[1 + LINK_SIZE] == OP_DEF) - { - /* Drop this bracket_backtrack. */ - parent->top = backtrack->prev; - return matchingpath + 1 + LINK_SIZE + repeat_length; - } - -matchingpath = ccbegin + 1 + LINK_SIZE; -SLJIT_ASSERT(ket == OP_KET || ket == OP_KETRMAX || ket == OP_KETRMIN); -SLJIT_ASSERT(!((bra == OP_BRAZERO && ket == OP_KETRMIN) || (bra == OP_BRAMINZERO && ket == OP_KETRMAX))); -cc += GET(cc, 1); - -has_alternatives = *cc == OP_ALT; -if (SLJIT_UNLIKELY(opcode == OP_COND || opcode == OP_SCOND)) - has_alternatives = (*matchingpath == OP_RREF || *matchingpath == OP_DNRREF || *matchingpath == OP_FAIL) ? FALSE : TRUE; - -if (SLJIT_UNLIKELY(opcode == OP_COND) && (*cc == OP_KETRMAX || *cc == OP_KETRMIN)) - opcode = OP_SCOND; -if (SLJIT_UNLIKELY(opcode == OP_ONCE_NC)) - opcode = OP_ONCE; - -if (opcode == OP_CBRA || opcode == OP_SCBRA) - { - /* Capturing brackets has a pre-allocated space. */ - offset = GET2(ccbegin, 1 + LINK_SIZE); - if (common->optimized_cbracket[offset] == 0) - { - private_data_ptr = OVECTOR_PRIV(offset); - offset <<= 1; - } - else - { - offset <<= 1; - private_data_ptr = OVECTOR(offset); - } - BACKTRACK_AS(bracket_backtrack)->private_data_ptr = private_data_ptr; - matchingpath += IMM2_SIZE; - } -else if (opcode == OP_ONCE || opcode == OP_SBRA || opcode == OP_SCOND) - { - /* Other brackets simply allocate the next entry. */ - private_data_ptr = PRIVATE_DATA(ccbegin); - SLJIT_ASSERT(private_data_ptr != 0); - BACKTRACK_AS(bracket_backtrack)->private_data_ptr = private_data_ptr; - if (opcode == OP_ONCE) - BACKTRACK_AS(bracket_backtrack)->u.framesize = get_framesize(common, ccbegin, NULL, FALSE, &needs_control_head); - } - -/* Instructions before the first alternative. */ -stacksize = 0; -if (ket == OP_KETRMAX || (ket == OP_KETRMIN && bra != OP_BRAMINZERO)) - stacksize++; -if (bra == OP_BRAZERO) - stacksize++; - -if (stacksize > 0) - allocate_stack(common, stacksize); - -stacksize = 0; -if (ket == OP_KETRMAX || (ket == OP_KETRMIN && bra != OP_BRAMINZERO)) - { - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0); - stacksize++; - } - -if (bra == OP_BRAZERO) - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0); - -if (bra == OP_BRAMINZERO) - { - /* This is a backtrack path! (Since the try-path of OP_BRAMINZERO matches to the empty string) */ - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - if (ket != OP_KETRMIN) - { - free_stack(common, 1); - braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0); - } - else - { - if (opcode == OP_ONCE || opcode >= OP_SBRA) - { - jump = CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); - /* Nothing stored during the first run. */ - skip = JUMP(SLJIT_JUMP); - JUMPHERE(jump); - /* Checking zero-length iteration. */ - if (opcode != OP_ONCE || BACKTRACK_AS(bracket_backtrack)->u.framesize < 0) - { - /* When we come from outside, private_data_ptr contains the previous STR_PTR. */ - braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - } - else - { - /* Except when the whole stack frame must be saved. */ - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), (BACKTRACK_AS(bracket_backtrack)->u.framesize + 1) * sizeof(sljit_sw)); - } - JUMPHERE(skip); - } - else - { - jump = CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); - JUMPHERE(jump); - } - } - } - -if (repeat_type != 0) - { - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, repeat_count); - if (repeat_type == OP_EXACT) - rmax_label = LABEL(); - } - -if (ket == OP_KETRMIN) - BACKTRACK_AS(bracket_backtrack)->recursive_matchingpath = LABEL(); - -if (ket == OP_KETRMAX) - { - rmax_label = LABEL(); - if (has_alternatives && opcode != OP_ONCE && opcode < OP_SBRA && repeat_type == 0) - BACKTRACK_AS(bracket_backtrack)->alternative_matchingpath = rmax_label; - } - -/* Handling capturing brackets and alternatives. */ -if (opcode == OP_ONCE) - { - stacksize = 0; - if (needs_control_head) - { - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); - stacksize++; - } - - if (BACKTRACK_AS(bracket_backtrack)->u.framesize < 0) - { - /* Neither capturing brackets nor recursions are found in the block. */ - if (ket == OP_KETRMIN) - { - stacksize += 2; - if (!needs_control_head) - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - } - else - { - if (BACKTRACK_AS(bracket_backtrack)->u.framesize == no_frame) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0); - if (ket == OP_KETRMAX || has_alternatives) - stacksize++; - } - - if (stacksize > 0) - allocate_stack(common, stacksize); - - stacksize = 0; - if (needs_control_head) - { - stacksize++; - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); - } - - if (ket == OP_KETRMIN) - { - if (needs_control_head) - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0); - if (BACKTRACK_AS(bracket_backtrack)->u.framesize == no_frame) - OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, needs_control_head ? (2 * sizeof(sljit_sw)) : sizeof(sljit_sw)); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize + 1), TMP2, 0); - } - else if (ket == OP_KETRMAX || has_alternatives) - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0); - } - else - { - if (ket != OP_KET || has_alternatives) - stacksize++; - - stacksize += BACKTRACK_AS(bracket_backtrack)->u.framesize + 1; - allocate_stack(common, stacksize); - - if (needs_control_head) - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); - - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - OP2(SLJIT_SUB, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw)); - - stacksize = needs_control_head ? 1 : 0; - if (ket != OP_KET || has_alternatives) - { - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP2, 0); - stacksize++; - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), TMP1, 0); - } - else - { - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP2, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), TMP1, 0); - } - init_frame(common, ccbegin, NULL, BACKTRACK_AS(bracket_backtrack)->u.framesize + stacksize, stacksize + 1, FALSE); - } - } -else if (opcode == OP_CBRA || opcode == OP_SCBRA) - { - /* Saving the previous values. */ - if (common->optimized_cbracket[offset >> 1] != 0) - { - SLJIT_ASSERT(private_data_ptr == OVECTOR(offset)); - allocate_stack(common, 2); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr + sizeof(sljit_sw)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STR_PTR, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP2, 0); - } - else - { - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - allocate_stack(common, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STR_PTR, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); - } - } -else if (opcode == OP_SBRA || opcode == OP_SCOND) - { - /* Saving the previous value. */ - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - allocate_stack(common, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STR_PTR, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); - } -else if (has_alternatives) - { - /* Pushing the starting string pointer. */ - allocate_stack(common, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); - } - -/* Generating code for the first alternative. */ -if (opcode == OP_COND || opcode == OP_SCOND) - { - if (*matchingpath == OP_CREF) - { - SLJIT_ASSERT(has_alternatives); - add_jump(compiler, &(BACKTRACK_AS(bracket_backtrack)->u.condfailed), - CMP(SLJIT_EQUAL, SLJIT_MEM1(SLJIT_SP), OVECTOR(GET2(matchingpath, 1) << 1), SLJIT_MEM1(SLJIT_SP), OVECTOR(1))); - matchingpath += 1 + IMM2_SIZE; - } - else if (*matchingpath == OP_DNCREF) - { - SLJIT_ASSERT(has_alternatives); - - i = GET2(matchingpath, 1 + IMM2_SIZE); - slot = common->name_table + GET2(matchingpath, 1) * common->name_entry_size; - OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1)); - OP2(SLJIT_SUB | SLJIT_SET_E, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(GET2(slot, 0) << 1), TMP1, 0); - slot += common->name_entry_size; - i--; - while (i-- > 0) - { - OP2(SLJIT_SUB, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(GET2(slot, 0) << 1), TMP1, 0); - OP2(SLJIT_OR | SLJIT_SET_E, TMP2, 0, TMP2, 0, STR_PTR, 0); - slot += common->name_entry_size; - } - OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0); - add_jump(compiler, &(BACKTRACK_AS(bracket_backtrack)->u.condfailed), JUMP(SLJIT_ZERO)); - matchingpath += 1 + 2 * IMM2_SIZE; - } - else if (*matchingpath == OP_RREF || *matchingpath == OP_DNRREF || *matchingpath == OP_FAIL) - { - /* Never has other case. */ - BACKTRACK_AS(bracket_backtrack)->u.condfailed = NULL; - SLJIT_ASSERT(!has_alternatives); - - if (*matchingpath == OP_FAIL) - stacksize = 0; - if (*matchingpath == OP_RREF) - { - stacksize = GET2(matchingpath, 1); - if (common->currententry == NULL) - stacksize = 0; - else if (stacksize == RREF_ANY) - stacksize = 1; - else if (common->currententry->start == 0) - stacksize = stacksize == 0; - else - stacksize = stacksize == (int)GET2(common->start, common->currententry->start + 1 + LINK_SIZE); - - if (stacksize != 0) - matchingpath += 1 + IMM2_SIZE; - } - else - { - if (common->currententry == NULL || common->currententry->start == 0) - stacksize = 0; - else - { - stacksize = GET2(matchingpath, 1 + IMM2_SIZE); - slot = common->name_table + GET2(matchingpath, 1) * common->name_entry_size; - i = (int)GET2(common->start, common->currententry->start + 1 + LINK_SIZE); - while (stacksize > 0) - { - if ((int)GET2(slot, 0) == i) - break; - slot += common->name_entry_size; - stacksize--; - } - } - - if (stacksize != 0) - matchingpath += 1 + 2 * IMM2_SIZE; - } - - /* The stacksize == 0 is a common "else" case. */ - if (stacksize == 0) - { - if (*cc == OP_ALT) - { - matchingpath = cc + 1 + LINK_SIZE; - cc += GET(cc, 1); - } - else - matchingpath = cc; - } - } - else - { - SLJIT_ASSERT(has_alternatives && *matchingpath >= OP_ASSERT && *matchingpath <= OP_ASSERTBACK_NOT); - /* Similar code as PUSH_BACKTRACK macro. */ - assert = sljit_alloc_memory(compiler, sizeof(assert_backtrack)); - if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) - return NULL; - memset(assert, 0, sizeof(assert_backtrack)); - assert->common.cc = matchingpath; - BACKTRACK_AS(bracket_backtrack)->u.assert = assert; - matchingpath = compile_assert_matchingpath(common, matchingpath, assert, TRUE); - } - } - -compile_matchingpath(common, matchingpath, cc, backtrack); -if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) - return NULL; - -if (opcode == OP_ONCE) - match_once_common(common, ket, BACKTRACK_AS(bracket_backtrack)->u.framesize, private_data_ptr, has_alternatives, needs_control_head); - -stacksize = 0; -if (repeat_type == OP_MINUPTO) - { - /* We need to preserve the counter. TMP2 will be used below. */ - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), repeat_ptr); - stacksize++; - } -if (ket != OP_KET || bra != OP_BRA) - stacksize++; -if (offset != 0) - { - if (common->capture_last_ptr != 0) - stacksize++; - if (common->optimized_cbracket[offset >> 1] == 0) - stacksize += 2; - } -if (has_alternatives && opcode != OP_ONCE) - stacksize++; - -if (stacksize > 0) - allocate_stack(common, stacksize); - -stacksize = 0; -if (repeat_type == OP_MINUPTO) - { - /* TMP2 was set above. */ - OP2(SLJIT_SUB, SLJIT_MEM1(STACK_TOP), STACK(stacksize), TMP2, 0, SLJIT_IMM, 1); - stacksize++; - } - -if (ket != OP_KET || bra != OP_BRA) - { - if (ket != OP_KET) - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0); - else - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0); - stacksize++; - } - -if (offset != 0) - stacksize = match_capture_common(common, stacksize, offset, private_data_ptr); - -if (has_alternatives) - { - if (opcode != OP_ONCE) - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0); - if (ket != OP_KETRMAX) - BACKTRACK_AS(bracket_backtrack)->alternative_matchingpath = LABEL(); - } - -/* Must be after the matchingpath label. */ -if (offset != 0 && common->optimized_cbracket[offset >> 1] != 0) - { - SLJIT_ASSERT(private_data_ptr == OVECTOR(offset + 0)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0); - } - -if (ket == OP_KETRMAX) - { - if (repeat_type != 0) - { - if (has_alternatives) - BACKTRACK_AS(bracket_backtrack)->alternative_matchingpath = LABEL(); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1); - JUMPTO(SLJIT_NOT_ZERO, rmax_label); - /* Drop STR_PTR for greedy plus quantifier. */ - if (opcode != OP_ONCE) - free_stack(common, 1); - } - else if (opcode == OP_ONCE || opcode >= OP_SBRA) - { - if (has_alternatives) - BACKTRACK_AS(bracket_backtrack)->alternative_matchingpath = LABEL(); - /* Checking zero-length iteration. */ - if (opcode != OP_ONCE) - { - CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STR_PTR, 0, rmax_label); - /* Drop STR_PTR for greedy plus quantifier. */ - if (bra != OP_BRAZERO) - free_stack(common, 1); - } - else - /* TMP2 must contain the starting STR_PTR. */ - CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, STR_PTR, 0, rmax_label); - } - else - JUMPTO(SLJIT_JUMP, rmax_label); - BACKTRACK_AS(bracket_backtrack)->recursive_matchingpath = LABEL(); - } - -if (repeat_type == OP_EXACT) - { - count_match(common); - OP2(SLJIT_SUB | SLJIT_SET_E, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1); - JUMPTO(SLJIT_NOT_ZERO, rmax_label); - } -else if (repeat_type == OP_UPTO) - { - /* We need to preserve the counter. */ - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), repeat_ptr); - allocate_stack(common, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); - } - -if (bra == OP_BRAZERO) - BACKTRACK_AS(bracket_backtrack)->zero_matchingpath = LABEL(); - -if (bra == OP_BRAMINZERO) - { - /* This is a backtrack path! (From the viewpoint of OP_BRAMINZERO) */ - JUMPTO(SLJIT_JUMP, ((braminzero_backtrack *)parent)->matchingpath); - if (braminzero != NULL) - { - JUMPHERE(braminzero); - /* We need to release the end pointer to perform the - backtrack for the zero-length iteration. When - framesize is < 0, OP_ONCE will do the release itself. */ - if (opcode == OP_ONCE && BACKTRACK_AS(bracket_backtrack)->u.framesize >= 0) - { - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - } - else if (ket == OP_KETRMIN && opcode != OP_ONCE) - free_stack(common, 1); - } - /* Continue to the normal backtrack. */ - } - -if ((ket != OP_KET && bra != OP_BRAMINZERO) || bra == OP_BRAZERO) - count_match(common); - -/* Skip the other alternatives. */ -while (*cc == OP_ALT) - cc += GET(cc, 1); -cc += 1 + LINK_SIZE; - -if (opcode == OP_ONCE) - { - /* We temporarily encode the needs_control_head in the lowest bit. - Note: on the target architectures of SLJIT the ((x << 1) >> 1) returns - the same value for small signed numbers (including negative numbers). */ - BACKTRACK_AS(bracket_backtrack)->u.framesize = (BACKTRACK_AS(bracket_backtrack)->u.framesize << 1) | (needs_control_head ? 1 : 0); - } -return cc + repeat_length; -} - -static pcre_uchar *compile_bracketpos_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent) -{ -DEFINE_COMPILER; -backtrack_common *backtrack; -pcre_uchar opcode; -int private_data_ptr; -int cbraprivptr = 0; -BOOL needs_control_head; -int framesize; -int stacksize; -int offset = 0; -BOOL zero = FALSE; -pcre_uchar *ccbegin = NULL; -int stack; /* Also contains the offset of control head. */ -struct sljit_label *loop = NULL; -struct jump_list *emptymatch = NULL; - -PUSH_BACKTRACK(sizeof(bracketpos_backtrack), cc, NULL); -if (*cc == OP_BRAPOSZERO) - { - zero = TRUE; - cc++; - } - -opcode = *cc; -private_data_ptr = PRIVATE_DATA(cc); -SLJIT_ASSERT(private_data_ptr != 0); -BACKTRACK_AS(bracketpos_backtrack)->private_data_ptr = private_data_ptr; -switch(opcode) - { - case OP_BRAPOS: - case OP_SBRAPOS: - ccbegin = cc + 1 + LINK_SIZE; - break; - - case OP_CBRAPOS: - case OP_SCBRAPOS: - offset = GET2(cc, 1 + LINK_SIZE); - /* This case cannot be optimized in the same was as - normal capturing brackets. */ - SLJIT_ASSERT(common->optimized_cbracket[offset] == 0); - cbraprivptr = OVECTOR_PRIV(offset); - offset <<= 1; - ccbegin = cc + 1 + LINK_SIZE + IMM2_SIZE; - break; - - default: - SLJIT_ASSERT_STOP(); - break; - } - -framesize = get_framesize(common, cc, NULL, FALSE, &needs_control_head); -BACKTRACK_AS(bracketpos_backtrack)->framesize = framesize; -if (framesize < 0) - { - if (offset != 0) - { - stacksize = 2; - if (common->capture_last_ptr != 0) - stacksize++; - } - else - stacksize = 1; - - if (needs_control_head) - stacksize++; - if (!zero) - stacksize++; - - BACKTRACK_AS(bracketpos_backtrack)->stacksize = stacksize; - allocate_stack(common, stacksize); - if (framesize == no_frame) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0); - - stack = 0; - if (offset != 0) - { - stack = 2; - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset)); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0); - if (common->capture_last_ptr != 0) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP2, 0); - if (needs_control_head) - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); - if (common->capture_last_ptr != 0) - { - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(2), TMP1, 0); - stack = 3; - } - } - else - { - if (needs_control_head) - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); - stack = 1; - } - - if (needs_control_head) - stack++; - if (!zero) - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stack), SLJIT_IMM, 1); - if (needs_control_head) - { - stack--; - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stack), TMP2, 0); - } - } -else - { - stacksize = framesize + 1; - if (!zero) - stacksize++; - if (needs_control_head) - stacksize++; - if (offset == 0) - stacksize++; - BACKTRACK_AS(bracketpos_backtrack)->stacksize = stacksize; - - allocate_stack(common, stacksize); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - if (needs_control_head) - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); - OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, -STACK(stacksize - 1)); - - stack = 0; - if (!zero) - { - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 1); - stack = 1; - } - if (needs_control_head) - { - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stack), TMP2, 0); - stack++; - } - if (offset == 0) - { - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stack), STR_PTR, 0); - stack++; - } - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stack), TMP1, 0); - init_frame(common, cc, NULL, stacksize - 1, stacksize - framesize, FALSE); - stack -= 1 + (offset == 0); - } - -if (offset != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), cbraprivptr, STR_PTR, 0); - -loop = LABEL(); -while (*cc != OP_KETRPOS) - { - backtrack->top = NULL; - backtrack->topbacktracks = NULL; - cc += GET(cc, 1); - - compile_matchingpath(common, ccbegin, cc, backtrack); - if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) - return NULL; - - if (framesize < 0) - { - if (framesize == no_frame) - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - - if (offset != 0) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), cbraprivptr); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), cbraprivptr, STR_PTR, 0); - if (common->capture_last_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, SLJIT_IMM, offset >> 1); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0); - } - else - { - if (opcode == OP_SBRAPOS) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); - } - - /* Even if the match is empty, we need to reset the control head. */ - if (needs_control_head) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(stack)); - - if (opcode == OP_SBRAPOS || opcode == OP_SCBRAPOS) - add_jump(compiler, &emptymatch, CMP(SLJIT_EQUAL, TMP1, 0, STR_PTR, 0)); - - if (!zero) - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0); - } - else - { - if (offset != 0) - { - OP2(SLJIT_ADD, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, stacksize * sizeof(sljit_sw)); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), cbraprivptr); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), cbraprivptr, STR_PTR, 0); - if (common->capture_last_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, SLJIT_IMM, offset >> 1); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0); - } - else - { - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - OP2(SLJIT_ADD, STACK_TOP, 0, TMP2, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw)); - if (opcode == OP_SBRAPOS) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_sw)); - OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_sw), STR_PTR, 0); - } - - /* Even if the match is empty, we need to reset the control head. */ - if (needs_control_head) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(stack)); - - if (opcode == OP_SBRAPOS || opcode == OP_SCBRAPOS) - add_jump(compiler, &emptymatch, CMP(SLJIT_EQUAL, TMP1, 0, STR_PTR, 0)); - - if (!zero) - { - if (framesize < 0) - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0); - else - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); - } - } - - JUMPTO(SLJIT_JUMP, loop); - flush_stubs(common); - - compile_backtrackingpath(common, backtrack->top); - if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) - return NULL; - set_jumps(backtrack->topbacktracks, LABEL()); - - if (framesize < 0) - { - if (offset != 0) - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), cbraprivptr); - else - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - } - else - { - if (offset != 0) - { - /* Last alternative. */ - if (*cc == OP_KETRPOS) - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), cbraprivptr); - } - else - { - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP2), (framesize + 1) * sizeof(sljit_sw)); - } - } - - if (*cc == OP_KETRPOS) - break; - ccbegin = cc + 1 + LINK_SIZE; - } - -/* We don't have to restore the control head in case of a failed match. */ - -backtrack->topbacktracks = NULL; -if (!zero) - { - if (framesize < 0) - add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0)); - else /* TMP2 is set to [private_data_ptr] above. */ - add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(TMP2), (stacksize - 1) * sizeof(sljit_sw), SLJIT_IMM, 0)); - } - -/* None of them matched. */ -set_jumps(emptymatch, LABEL()); -count_match(common); -return cc + 1 + LINK_SIZE; -} - -static SLJIT_INLINE pcre_uchar *get_iterator_parameters(compiler_common *common, pcre_uchar *cc, pcre_uchar *opcode, pcre_uchar *type, sljit_u32 *max, sljit_u32 *exact, pcre_uchar **end) -{ -int class_len; - -*opcode = *cc; -*exact = 0; - -if (*opcode >= OP_STAR && *opcode <= OP_POSUPTO) - { - cc++; - *type = OP_CHAR; - } -else if (*opcode >= OP_STARI && *opcode <= OP_POSUPTOI) - { - cc++; - *type = OP_CHARI; - *opcode -= OP_STARI - OP_STAR; - } -else if (*opcode >= OP_NOTSTAR && *opcode <= OP_NOTPOSUPTO) - { - cc++; - *type = OP_NOT; - *opcode -= OP_NOTSTAR - OP_STAR; - } -else if (*opcode >= OP_NOTSTARI && *opcode <= OP_NOTPOSUPTOI) - { - cc++; - *type = OP_NOTI; - *opcode -= OP_NOTSTARI - OP_STAR; - } -else if (*opcode >= OP_TYPESTAR && *opcode <= OP_TYPEPOSUPTO) - { - cc++; - *opcode -= OP_TYPESTAR - OP_STAR; - *type = OP_END; - } -else - { - SLJIT_ASSERT(*opcode == OP_CLASS || *opcode == OP_NCLASS || *opcode == OP_XCLASS); - *type = *opcode; - cc++; - class_len = (*type < OP_XCLASS) ? (int)(1 + (32 / sizeof(pcre_uchar))) : GET(cc, 0); - *opcode = cc[class_len - 1]; - - if (*opcode >= OP_CRSTAR && *opcode <= OP_CRMINQUERY) - { - *opcode -= OP_CRSTAR - OP_STAR; - *end = cc + class_len; - - if (*opcode == OP_PLUS || *opcode == OP_MINPLUS) - { - *exact = 1; - *opcode -= OP_PLUS - OP_STAR; - } - } - else if (*opcode >= OP_CRPOSSTAR && *opcode <= OP_CRPOSQUERY) - { - *opcode -= OP_CRPOSSTAR - OP_POSSTAR; - *end = cc + class_len; - - if (*opcode == OP_POSPLUS) - { - *exact = 1; - *opcode = OP_POSSTAR; - } - } - else - { - SLJIT_ASSERT(*opcode == OP_CRRANGE || *opcode == OP_CRMINRANGE || *opcode == OP_CRPOSRANGE); - *max = GET2(cc, (class_len + IMM2_SIZE)); - *exact = GET2(cc, class_len); - - if (*max == 0) - { - if (*opcode == OP_CRPOSRANGE) - *opcode = OP_POSSTAR; - else - *opcode -= OP_CRRANGE - OP_STAR; - } - else - { - *max -= *exact; - if (*max == 0) - *opcode = OP_EXACT; - else if (*max == 1) - { - if (*opcode == OP_CRPOSRANGE) - *opcode = OP_POSQUERY; - else - *opcode -= OP_CRRANGE - OP_QUERY; - } - else - { - if (*opcode == OP_CRPOSRANGE) - *opcode = OP_POSUPTO; - else - *opcode -= OP_CRRANGE - OP_UPTO; - } - } - *end = cc + class_len + 2 * IMM2_SIZE; - } - return cc; - } - -switch(*opcode) - { - case OP_EXACT: - *exact = GET2(cc, 0); - cc += IMM2_SIZE; - break; - - case OP_PLUS: - case OP_MINPLUS: - *exact = 1; - *opcode -= OP_PLUS - OP_STAR; - break; - - case OP_POSPLUS: - *exact = 1; - *opcode = OP_POSSTAR; - break; - - case OP_UPTO: - case OP_MINUPTO: - case OP_POSUPTO: - *max = GET2(cc, 0); - cc += IMM2_SIZE; - break; - } - -if (*type == OP_END) - { - *type = *cc; - *end = next_opcode(common, cc); - cc++; - return cc; - } - -*end = cc + 1; -#ifdef SUPPORT_UTF -if (common->utf && HAS_EXTRALEN(*cc)) *end += GET_EXTRALEN(*cc); -#endif -return cc; -} - -static pcre_uchar *compile_iterator_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent) -{ -DEFINE_COMPILER; -backtrack_common *backtrack; -pcre_uchar opcode; -pcre_uchar type; -sljit_u32 max = 0, exact; -BOOL fast_fail; -sljit_s32 fast_str_ptr; -BOOL charpos_enabled; -pcre_uchar charpos_char; -unsigned int charpos_othercasebit; -pcre_uchar *end; -jump_list *no_match = NULL; -jump_list *no_char1_match = NULL; -struct sljit_jump *jump = NULL; -struct sljit_label *label; -int private_data_ptr = PRIVATE_DATA(cc); -int base = (private_data_ptr == 0) ? SLJIT_MEM1(STACK_TOP) : SLJIT_MEM1(SLJIT_SP); -int offset0 = (private_data_ptr == 0) ? STACK(0) : private_data_ptr; -int offset1 = (private_data_ptr == 0) ? STACK(1) : private_data_ptr + (int)sizeof(sljit_sw); -int tmp_base, tmp_offset; - -PUSH_BACKTRACK(sizeof(char_iterator_backtrack), cc, NULL); - -fast_str_ptr = PRIVATE_DATA(cc + 1); -fast_fail = TRUE; - -SLJIT_ASSERT(common->fast_forward_bc_ptr == NULL || fast_str_ptr == 0 || cc == common->fast_forward_bc_ptr); - -if (cc == common->fast_forward_bc_ptr) - fast_fail = FALSE; -else if (common->fast_fail_start_ptr == 0) - fast_str_ptr = 0; - -SLJIT_ASSERT(common->fast_forward_bc_ptr != NULL || fast_str_ptr == 0 - || (fast_str_ptr >= common->fast_fail_start_ptr && fast_str_ptr <= common->fast_fail_end_ptr)); - -cc = get_iterator_parameters(common, cc, &opcode, &type, &max, &exact, &end); - -if (type != OP_EXTUNI) - { - tmp_base = TMP3; - tmp_offset = 0; - } -else - { - tmp_base = SLJIT_MEM1(SLJIT_SP); - tmp_offset = POSSESSIVE0; - } - -if (fast_fail && fast_str_ptr != 0) - add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), fast_str_ptr)); - -/* Handle fixed part first. */ -if (exact > 1) - { - SLJIT_ASSERT(fast_str_ptr == 0); - if (common->mode == JIT_COMPILE -#ifdef SUPPORT_UTF - && !common->utf -#endif - ) - { - OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(exact)); - add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_GREATER, TMP1, 0, STR_END, 0)); - OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, exact); - label = LABEL(); - compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, FALSE); - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); - JUMPTO(SLJIT_NOT_ZERO, label); - } - else - { - OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, exact); - label = LABEL(); - compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, TRUE); - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); - JUMPTO(SLJIT_NOT_ZERO, label); - } - } -else if (exact == 1) - compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, TRUE); - -switch(opcode) - { - case OP_STAR: - case OP_UPTO: - SLJIT_ASSERT(fast_str_ptr == 0 || opcode == OP_STAR); - - if (type == OP_ANYNL || type == OP_EXTUNI) - { - SLJIT_ASSERT(private_data_ptr == 0); - SLJIT_ASSERT(fast_str_ptr == 0); - - allocate_stack(common, 2); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0); - - if (opcode == OP_UPTO) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0, SLJIT_IMM, max); - - label = LABEL(); - compile_char1_matchingpath(common, type, cc, &BACKTRACK_AS(char_iterator_backtrack)->u.backtracks, TRUE); - if (opcode == OP_UPTO) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0); - OP2(SLJIT_SUB | SLJIT_SET_E, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); - jump = JUMP(SLJIT_ZERO); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0, TMP1, 0); - } - - /* We cannot use TMP3 because of this allocate_stack. */ - allocate_stack(common, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); - JUMPTO(SLJIT_JUMP, label); - if (jump != NULL) - JUMPHERE(jump); - } - else - { - charpos_enabled = FALSE; - charpos_char = 0; - charpos_othercasebit = 0; - - if ((type != OP_CHAR && type != OP_CHARI) && (*end == OP_CHAR || *end == OP_CHARI)) - { - charpos_enabled = TRUE; -#ifdef SUPPORT_UTF - charpos_enabled = !common->utf || !HAS_EXTRALEN(end[1]); -#endif - if (charpos_enabled && *end == OP_CHARI && char_has_othercase(common, end + 1)) - { - charpos_othercasebit = char_get_othercase_bit(common, end + 1); - if (charpos_othercasebit == 0) - charpos_enabled = FALSE; - } - - if (charpos_enabled) - { - charpos_char = end[1]; - /* Consumpe the OP_CHAR opcode. */ - end += 2; -#if defined COMPILE_PCRE8 - SLJIT_ASSERT((charpos_othercasebit >> 8) == 0); -#elif defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - SLJIT_ASSERT((charpos_othercasebit >> 9) == 0); - if ((charpos_othercasebit & 0x100) != 0) - charpos_othercasebit = (charpos_othercasebit & 0xff) << 8; -#endif - if (charpos_othercasebit != 0) - charpos_char |= charpos_othercasebit; - - BACKTRACK_AS(char_iterator_backtrack)->u.charpos.enabled = TRUE; - BACKTRACK_AS(char_iterator_backtrack)->u.charpos.chr = charpos_char; - BACKTRACK_AS(char_iterator_backtrack)->u.charpos.othercasebit = charpos_othercasebit; - } - } - - if (charpos_enabled) - { - if (opcode == OP_UPTO) - OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max + 1); - - /* Search the first instance of charpos_char. */ - jump = JUMP(SLJIT_JUMP); - label = LABEL(); - if (opcode == OP_UPTO) - { - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); - add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_ZERO)); - } - compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, FALSE); - if (fast_str_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); - JUMPHERE(jump); - - detect_partial_match(common, &backtrack->topbacktracks); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - if (charpos_othercasebit != 0) - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, charpos_othercasebit); - CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, charpos_char, label); - - if (private_data_ptr == 0) - allocate_stack(common, 2); - OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); - OP1(SLJIT_MOV, base, offset1, STR_PTR, 0); - if (opcode == OP_UPTO) - { - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); - add_jump(compiler, &no_match, JUMP(SLJIT_ZERO)); - } - - /* Search the last instance of charpos_char. */ - label = LABEL(); - compile_char1_matchingpath(common, type, cc, &no_match, FALSE); - if (fast_str_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); - detect_partial_match(common, &no_match); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); - if (charpos_othercasebit != 0) - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, charpos_othercasebit); - if (opcode == OP_STAR) - { - CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, charpos_char, label); - OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); - } - else - { - jump = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, charpos_char); - OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); - JUMPHERE(jump); - } - - if (opcode == OP_UPTO) - { - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); - JUMPTO(SLJIT_NOT_ZERO, label); - } - else - JUMPTO(SLJIT_JUMP, label); - - set_jumps(no_match, LABEL()); - OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); - } -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - else if (common->utf) - { - if (private_data_ptr == 0) - allocate_stack(common, 2); - - OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); - OP1(SLJIT_MOV, base, offset1, STR_PTR, 0); - - if (opcode == OP_UPTO) - OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max); - - label = LABEL(); - compile_char1_matchingpath(common, type, cc, &no_match, TRUE); - OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); - - if (opcode == OP_UPTO) - { - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); - JUMPTO(SLJIT_NOT_ZERO, label); - } - else - JUMPTO(SLJIT_JUMP, label); - - set_jumps(no_match, LABEL()); - OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); - if (fast_str_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); - } -#endif - else - { - if (private_data_ptr == 0) - allocate_stack(common, 2); - - OP1(SLJIT_MOV, base, offset1, STR_PTR, 0); - if (opcode == OP_UPTO) - OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max); - - label = LABEL(); - detect_partial_match(common, &no_match); - compile_char1_matchingpath(common, type, cc, &no_char1_match, FALSE); - if (opcode == OP_UPTO) - { - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); - JUMPTO(SLJIT_NOT_ZERO, label); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - } - else - JUMPTO(SLJIT_JUMP, label); - - set_jumps(no_char1_match, LABEL()); - OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - set_jumps(no_match, LABEL()); - OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); - if (fast_str_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); - } - } - BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL(); - break; - - case OP_MINSTAR: - if (private_data_ptr == 0) - allocate_stack(common, 1); - OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); - BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL(); - if (fast_str_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); - break; - - case OP_MINUPTO: - SLJIT_ASSERT(fast_str_ptr == 0); - if (private_data_ptr == 0) - allocate_stack(common, 2); - OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); - OP1(SLJIT_MOV, base, offset1, SLJIT_IMM, max + 1); - BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL(); - break; - - case OP_QUERY: - case OP_MINQUERY: - SLJIT_ASSERT(fast_str_ptr == 0); - if (private_data_ptr == 0) - allocate_stack(common, 1); - OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); - if (opcode == OP_QUERY) - compile_char1_matchingpath(common, type, cc, &BACKTRACK_AS(char_iterator_backtrack)->u.backtracks, TRUE); - BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL(); - break; - - case OP_EXACT: - break; - - case OP_POSSTAR: -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (common->utf) - { - OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0); - label = LABEL(); - compile_char1_matchingpath(common, type, cc, &no_match, TRUE); - OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0); - JUMPTO(SLJIT_JUMP, label); - set_jumps(no_match, LABEL()); - OP1(SLJIT_MOV, STR_PTR, 0, tmp_base, tmp_offset); - if (fast_str_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); - break; - } -#endif - label = LABEL(); - detect_partial_match(common, &no_match); - compile_char1_matchingpath(common, type, cc, &no_char1_match, FALSE); - JUMPTO(SLJIT_JUMP, label); - set_jumps(no_char1_match, LABEL()); - OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - set_jumps(no_match, LABEL()); - if (fast_str_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); - break; - - case OP_POSUPTO: - SLJIT_ASSERT(fast_str_ptr == 0); -#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 - if (common->utf) - { - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1, STR_PTR, 0); - OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max); - label = LABEL(); - compile_char1_matchingpath(common, type, cc, &no_match, TRUE); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1, STR_PTR, 0); - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); - JUMPTO(SLJIT_NOT_ZERO, label); - set_jumps(no_match, LABEL()); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1); - break; - } -#endif - OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max); - label = LABEL(); - detect_partial_match(common, &no_match); - compile_char1_matchingpath(common, type, cc, &no_char1_match, FALSE); - OP2(SLJIT_SUB | SLJIT_SET_E, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); - JUMPTO(SLJIT_NOT_ZERO, label); - OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - set_jumps(no_char1_match, LABEL()); - OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - set_jumps(no_match, LABEL()); - break; - - case OP_POSQUERY: - SLJIT_ASSERT(fast_str_ptr == 0); - OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0); - compile_char1_matchingpath(common, type, cc, &no_match, TRUE); - OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0); - set_jumps(no_match, LABEL()); - OP1(SLJIT_MOV, STR_PTR, 0, tmp_base, tmp_offset); - break; - - default: - SLJIT_ASSERT_STOP(); - break; - } - -count_match(common); -return end; -} - -static SLJIT_INLINE pcre_uchar *compile_fail_accept_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent) -{ -DEFINE_COMPILER; -backtrack_common *backtrack; - -PUSH_BACKTRACK(sizeof(backtrack_common), cc, NULL); - -if (*cc == OP_FAIL) - { - add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_JUMP)); - return cc + 1; - } - -if (*cc == OP_ASSERT_ACCEPT || common->currententry != NULL || !common->might_be_empty) - { - /* No need to check notempty conditions. */ - if (common->accept_label == NULL) - add_jump(compiler, &common->accept, JUMP(SLJIT_JUMP)); - else - JUMPTO(SLJIT_JUMP, common->accept_label); - return cc + 1; - } - -if (common->accept_label == NULL) - add_jump(compiler, &common->accept, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0))); -else - CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0), common->accept_label); -OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); -OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty)); -add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0)); -OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty_atstart)); -if (common->accept_label == NULL) - add_jump(compiler, &common->accept, CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, 0)); -else - CMPTO(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, 0, common->accept_label); -OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); -if (common->accept_label == NULL) - add_jump(compiler, &common->accept, CMP(SLJIT_NOT_EQUAL, TMP2, 0, STR_PTR, 0)); -else - CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, STR_PTR, 0, common->accept_label); -add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_JUMP)); -return cc + 1; -} - -static SLJIT_INLINE pcre_uchar *compile_close_matchingpath(compiler_common *common, pcre_uchar *cc) -{ -DEFINE_COMPILER; -int offset = GET2(cc, 1); -BOOL optimized_cbracket = common->optimized_cbracket[offset] != 0; - -/* Data will be discarded anyway... */ -if (common->currententry != NULL) - return cc + 1 + IMM2_SIZE; - -if (!optimized_cbracket) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR_PRIV(offset)); -offset <<= 1; -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0); -if (!optimized_cbracket) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0); -return cc + 1 + IMM2_SIZE; -} - -static SLJIT_INLINE pcre_uchar *compile_control_verb_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent) -{ -DEFINE_COMPILER; -backtrack_common *backtrack; -pcre_uchar opcode = *cc; -pcre_uchar *ccend = cc + 1; - -if (opcode == OP_PRUNE_ARG || opcode == OP_SKIP_ARG || opcode == OP_THEN_ARG) - ccend += 2 + cc[1]; - -PUSH_BACKTRACK(sizeof(backtrack_common), cc, NULL); - -if (opcode == OP_SKIP) - { - allocate_stack(common, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); - return ccend; - } - -if (opcode == OP_PRUNE_ARG || opcode == OP_THEN_ARG) - { - OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_sw)(cc + 2)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, TMP2, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, mark_ptr), TMP2, 0); - } - -return ccend; -} - -static pcre_uchar then_trap_opcode[1] = { OP_THEN_TRAP }; - -static SLJIT_INLINE void compile_then_trap_matchingpath(compiler_common *common, pcre_uchar *cc, pcre_uchar *ccend, backtrack_common *parent) -{ -DEFINE_COMPILER; -backtrack_common *backtrack; -BOOL needs_control_head; -int size; - -PUSH_BACKTRACK_NOVALUE(sizeof(then_trap_backtrack), cc); -common->then_trap = BACKTRACK_AS(then_trap_backtrack); -BACKTRACK_AS(then_trap_backtrack)->common.cc = then_trap_opcode; -BACKTRACK_AS(then_trap_backtrack)->start = (sljit_sw)(cc - common->start); -BACKTRACK_AS(then_trap_backtrack)->framesize = get_framesize(common, cc, ccend, FALSE, &needs_control_head); - -size = BACKTRACK_AS(then_trap_backtrack)->framesize; -size = 3 + (size < 0 ? 0 : size); - -OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); -allocate_stack(common, size); -if (size > 3) - OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, STACK_TOP, 0, SLJIT_IMM, (size - 3) * sizeof(sljit_sw)); -else - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, STACK_TOP, 0); -OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(size - 1), SLJIT_IMM, BACKTRACK_AS(then_trap_backtrack)->start); -OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(size - 2), SLJIT_IMM, type_then_trap); -OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(size - 3), TMP2, 0); - -size = BACKTRACK_AS(then_trap_backtrack)->framesize; -if (size >= 0) - init_frame(common, cc, ccend, size - 1, 0, FALSE); -} - -static void compile_matchingpath(compiler_common *common, pcre_uchar *cc, pcre_uchar *ccend, backtrack_common *parent) -{ -DEFINE_COMPILER; -backtrack_common *backtrack; -BOOL has_then_trap = FALSE; -then_trap_backtrack *save_then_trap = NULL; - -SLJIT_ASSERT(*ccend == OP_END || (*ccend >= OP_ALT && *ccend <= OP_KETRPOS)); - -if (common->has_then && common->then_offsets[cc - common->start] != 0) - { - SLJIT_ASSERT(*ccend != OP_END && common->control_head_ptr != 0); - has_then_trap = TRUE; - save_then_trap = common->then_trap; - /* Tail item on backtrack. */ - compile_then_trap_matchingpath(common, cc, ccend, parent); - } - -while (cc < ccend) - { - switch(*cc) - { - case OP_SOD: - case OP_SOM: - case OP_NOT_WORD_BOUNDARY: - case OP_WORD_BOUNDARY: - case OP_EODN: - case OP_EOD: - case OP_DOLL: - case OP_DOLLM: - case OP_CIRC: - case OP_CIRCM: - case OP_REVERSE: - cc = compile_simple_assertion_matchingpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks); - break; - - case OP_NOT_DIGIT: - case OP_DIGIT: - case OP_NOT_WHITESPACE: - case OP_WHITESPACE: - case OP_NOT_WORDCHAR: - case OP_WORDCHAR: - case OP_ANY: - case OP_ALLANY: - case OP_ANYBYTE: - case OP_NOTPROP: - case OP_PROP: - case OP_ANYNL: - case OP_NOT_HSPACE: - case OP_HSPACE: - case OP_NOT_VSPACE: - case OP_VSPACE: - case OP_EXTUNI: - case OP_NOT: - case OP_NOTI: - cc = compile_char1_matchingpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, TRUE); - break; - - case OP_SET_SOM: - PUSH_BACKTRACK_NOVALUE(sizeof(backtrack_common), cc); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0)); - allocate_stack(common, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(0), STR_PTR, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); - cc++; - break; - - case OP_CHAR: - case OP_CHARI: - if (common->mode == JIT_COMPILE) - cc = compile_charn_matchingpath(common, cc, ccend, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks); - else - cc = compile_char1_matchingpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, TRUE); - break; - - case OP_STAR: - case OP_MINSTAR: - case OP_PLUS: - case OP_MINPLUS: - case OP_QUERY: - case OP_MINQUERY: - case OP_UPTO: - case OP_MINUPTO: - case OP_EXACT: - case OP_POSSTAR: - case OP_POSPLUS: - case OP_POSQUERY: - case OP_POSUPTO: - case OP_STARI: - case OP_MINSTARI: - case OP_PLUSI: - case OP_MINPLUSI: - case OP_QUERYI: - case OP_MINQUERYI: - case OP_UPTOI: - case OP_MINUPTOI: - case OP_EXACTI: - case OP_POSSTARI: - case OP_POSPLUSI: - case OP_POSQUERYI: - case OP_POSUPTOI: - case OP_NOTSTAR: - case OP_NOTMINSTAR: - case OP_NOTPLUS: - case OP_NOTMINPLUS: - case OP_NOTQUERY: - case OP_NOTMINQUERY: - case OP_NOTUPTO: - case OP_NOTMINUPTO: - case OP_NOTEXACT: - case OP_NOTPOSSTAR: - case OP_NOTPOSPLUS: - case OP_NOTPOSQUERY: - case OP_NOTPOSUPTO: - case OP_NOTSTARI: - case OP_NOTMINSTARI: - case OP_NOTPLUSI: - case OP_NOTMINPLUSI: - case OP_NOTQUERYI: - case OP_NOTMINQUERYI: - case OP_NOTUPTOI: - case OP_NOTMINUPTOI: - case OP_NOTEXACTI: - case OP_NOTPOSSTARI: - case OP_NOTPOSPLUSI: - case OP_NOTPOSQUERYI: - case OP_NOTPOSUPTOI: - case OP_TYPESTAR: - case OP_TYPEMINSTAR: - case OP_TYPEPLUS: - case OP_TYPEMINPLUS: - case OP_TYPEQUERY: - case OP_TYPEMINQUERY: - case OP_TYPEUPTO: - case OP_TYPEMINUPTO: - case OP_TYPEEXACT: - case OP_TYPEPOSSTAR: - case OP_TYPEPOSPLUS: - case OP_TYPEPOSQUERY: - case OP_TYPEPOSUPTO: - cc = compile_iterator_matchingpath(common, cc, parent); - break; - - case OP_CLASS: - case OP_NCLASS: - if (cc[1 + (32 / sizeof(pcre_uchar))] >= OP_CRSTAR && cc[1 + (32 / sizeof(pcre_uchar))] <= OP_CRPOSRANGE) - cc = compile_iterator_matchingpath(common, cc, parent); - else - cc = compile_char1_matchingpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, TRUE); - break; - -#if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - case OP_XCLASS: - if (*(cc + GET(cc, 1)) >= OP_CRSTAR && *(cc + GET(cc, 1)) <= OP_CRPOSRANGE) - cc = compile_iterator_matchingpath(common, cc, parent); - else - cc = compile_char1_matchingpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, TRUE); - break; -#endif - - case OP_REF: - case OP_REFI: - if (cc[1 + IMM2_SIZE] >= OP_CRSTAR && cc[1 + IMM2_SIZE] <= OP_CRPOSRANGE) - cc = compile_ref_iterator_matchingpath(common, cc, parent); - else - { - compile_ref_matchingpath(common, cc, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, TRUE, FALSE); - cc += 1 + IMM2_SIZE; - } - break; - - case OP_DNREF: - case OP_DNREFI: - if (cc[1 + 2 * IMM2_SIZE] >= OP_CRSTAR && cc[1 + 2 * IMM2_SIZE] <= OP_CRPOSRANGE) - cc = compile_ref_iterator_matchingpath(common, cc, parent); - else - { - compile_dnref_search(common, cc, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks); - compile_ref_matchingpath(common, cc, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, TRUE, FALSE); - cc += 1 + 2 * IMM2_SIZE; - } - break; - - case OP_RECURSE: - cc = compile_recurse_matchingpath(common, cc, parent); - break; - - case OP_CALLOUT: - cc = compile_callout_matchingpath(common, cc, parent); - break; - - case OP_ASSERT: - case OP_ASSERT_NOT: - case OP_ASSERTBACK: - case OP_ASSERTBACK_NOT: - PUSH_BACKTRACK_NOVALUE(sizeof(assert_backtrack), cc); - cc = compile_assert_matchingpath(common, cc, BACKTRACK_AS(assert_backtrack), FALSE); - break; - - case OP_BRAMINZERO: - PUSH_BACKTRACK_NOVALUE(sizeof(braminzero_backtrack), cc); - cc = bracketend(cc + 1); - if (*(cc - 1 - LINK_SIZE) != OP_KETRMIN) - { - allocate_stack(common, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); - } - else - { - allocate_stack(common, 2); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), STR_PTR, 0); - } - BACKTRACK_AS(braminzero_backtrack)->matchingpath = LABEL(); - count_match(common); - break; - - case OP_ONCE: - case OP_ONCE_NC: - case OP_BRA: - case OP_CBRA: - case OP_COND: - case OP_SBRA: - case OP_SCBRA: - case OP_SCOND: - cc = compile_bracket_matchingpath(common, cc, parent); - break; - - case OP_BRAZERO: - if (cc[1] > OP_ASSERTBACK_NOT) - cc = compile_bracket_matchingpath(common, cc, parent); - else - { - PUSH_BACKTRACK_NOVALUE(sizeof(assert_backtrack), cc); - cc = compile_assert_matchingpath(common, cc, BACKTRACK_AS(assert_backtrack), FALSE); - } - break; - - case OP_BRAPOS: - case OP_CBRAPOS: - case OP_SBRAPOS: - case OP_SCBRAPOS: - case OP_BRAPOSZERO: - cc = compile_bracketpos_matchingpath(common, cc, parent); - break; - - case OP_MARK: - PUSH_BACKTRACK_NOVALUE(sizeof(backtrack_common), cc); - SLJIT_ASSERT(common->mark_ptr != 0); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr); - allocate_stack(common, common->has_skip_arg ? 5 : 1); - OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(common->has_skip_arg ? 4 : 0), TMP2, 0); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_sw)(cc + 2)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, TMP2, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, mark_ptr), TMP2, 0); - if (common->has_skip_arg) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, STACK_TOP, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, type_mark); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(2), SLJIT_IMM, (sljit_sw)(cc + 2)); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(3), STR_PTR, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0); - } - cc += 1 + 2 + cc[1]; - break; - - case OP_PRUNE: - case OP_PRUNE_ARG: - case OP_SKIP: - case OP_SKIP_ARG: - case OP_THEN: - case OP_THEN_ARG: - case OP_COMMIT: - cc = compile_control_verb_matchingpath(common, cc, parent); - break; - - case OP_FAIL: - case OP_ACCEPT: - case OP_ASSERT_ACCEPT: - cc = compile_fail_accept_matchingpath(common, cc, parent); - break; - - case OP_CLOSE: - cc = compile_close_matchingpath(common, cc); - break; - - case OP_SKIPZERO: - cc = bracketend(cc + 1); - break; - - default: - SLJIT_ASSERT_STOP(); - return; - } - if (cc == NULL) - return; - } - -if (has_then_trap) - { - /* Head item on backtrack. */ - PUSH_BACKTRACK_NOVALUE(sizeof(then_trap_backtrack), cc); - BACKTRACK_AS(then_trap_backtrack)->common.cc = then_trap_opcode; - BACKTRACK_AS(then_trap_backtrack)->then_trap = common->then_trap; - common->then_trap = save_then_trap; - } -SLJIT_ASSERT(cc == ccend); -} - -#undef PUSH_BACKTRACK -#undef PUSH_BACKTRACK_NOVALUE -#undef BACKTRACK_AS - -#define COMPILE_BACKTRACKINGPATH(current) \ - do \ - { \ - compile_backtrackingpath(common, (current)); \ - if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) \ - return; \ - } \ - while (0) - -#define CURRENT_AS(type) ((type *)current) - -static void compile_iterator_backtrackingpath(compiler_common *common, struct backtrack_common *current) -{ -DEFINE_COMPILER; -pcre_uchar *cc = current->cc; -pcre_uchar opcode; -pcre_uchar type; -sljit_u32 max = 0, exact; -struct sljit_label *label = NULL; -struct sljit_jump *jump = NULL; -jump_list *jumplist = NULL; -pcre_uchar *end; -int private_data_ptr = PRIVATE_DATA(cc); -int base = (private_data_ptr == 0) ? SLJIT_MEM1(STACK_TOP) : SLJIT_MEM1(SLJIT_SP); -int offset0 = (private_data_ptr == 0) ? STACK(0) : private_data_ptr; -int offset1 = (private_data_ptr == 0) ? STACK(1) : private_data_ptr + (int)sizeof(sljit_sw); - -cc = get_iterator_parameters(common, cc, &opcode, &type, &max, &exact, &end); - -switch(opcode) - { - case OP_STAR: - case OP_UPTO: - if (type == OP_ANYNL || type == OP_EXTUNI) - { - SLJIT_ASSERT(private_data_ptr == 0); - set_jumps(CURRENT_AS(char_iterator_backtrack)->u.backtracks, LABEL()); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - free_stack(common, 1); - CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(char_iterator_backtrack)->matchingpath); - } - else - { - if (CURRENT_AS(char_iterator_backtrack)->u.charpos.enabled) - { - OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); - OP1(SLJIT_MOV, TMP2, 0, base, offset1); - OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); - - jump = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0); - label = LABEL(); - OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1)); - OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); - if (CURRENT_AS(char_iterator_backtrack)->u.charpos.othercasebit != 0) - OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, CURRENT_AS(char_iterator_backtrack)->u.charpos.othercasebit); - CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CURRENT_AS(char_iterator_backtrack)->u.charpos.chr, CURRENT_AS(char_iterator_backtrack)->matchingpath); - skip_char_back(common); - CMPTO(SLJIT_GREATER, STR_PTR, 0, TMP2, 0, label); - } - else - { - OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); - jump = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, base, offset1); - skip_char_back(common); - OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); - JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath); - } - JUMPHERE(jump); - if (private_data_ptr == 0) - free_stack(common, 2); - } - break; - - case OP_MINSTAR: - OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); - compile_char1_matchingpath(common, type, cc, &jumplist, TRUE); - OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); - JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath); - set_jumps(jumplist, LABEL()); - if (private_data_ptr == 0) - free_stack(common, 1); - break; - - case OP_MINUPTO: - OP1(SLJIT_MOV, TMP1, 0, base, offset1); - OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); - OP2(SLJIT_SUB | SLJIT_SET_E, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); - add_jump(compiler, &jumplist, JUMP(SLJIT_ZERO)); - - OP1(SLJIT_MOV, base, offset1, TMP1, 0); - compile_char1_matchingpath(common, type, cc, &jumplist, TRUE); - OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); - JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath); - - set_jumps(jumplist, LABEL()); - if (private_data_ptr == 0) - free_stack(common, 2); - break; - - case OP_QUERY: - OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); - OP1(SLJIT_MOV, base, offset0, SLJIT_IMM, 0); - CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(char_iterator_backtrack)->matchingpath); - jump = JUMP(SLJIT_JUMP); - set_jumps(CURRENT_AS(char_iterator_backtrack)->u.backtracks, LABEL()); - OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); - OP1(SLJIT_MOV, base, offset0, SLJIT_IMM, 0); - JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath); - JUMPHERE(jump); - if (private_data_ptr == 0) - free_stack(common, 1); - break; - - case OP_MINQUERY: - OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); - OP1(SLJIT_MOV, base, offset0, SLJIT_IMM, 0); - jump = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0); - compile_char1_matchingpath(common, type, cc, &jumplist, TRUE); - JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath); - set_jumps(jumplist, LABEL()); - JUMPHERE(jump); - if (private_data_ptr == 0) - free_stack(common, 1); - break; - - case OP_EXACT: - case OP_POSSTAR: - case OP_POSQUERY: - case OP_POSUPTO: - break; - - default: - SLJIT_ASSERT_STOP(); - break; - } - -set_jumps(current->topbacktracks, LABEL()); -} - -static SLJIT_INLINE void compile_ref_iterator_backtrackingpath(compiler_common *common, struct backtrack_common *current) -{ -DEFINE_COMPILER; -pcre_uchar *cc = current->cc; -BOOL ref = (*cc == OP_REF || *cc == OP_REFI); -pcre_uchar type; - -type = cc[ref ? 1 + IMM2_SIZE : 1 + 2 * IMM2_SIZE]; - -if ((type & 0x1) == 0) - { - /* Maximize case. */ - set_jumps(current->topbacktracks, LABEL()); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - free_stack(common, 1); - CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(ref_iterator_backtrack)->matchingpath); - return; - } - -OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); -CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(ref_iterator_backtrack)->matchingpath); -set_jumps(current->topbacktracks, LABEL()); -free_stack(common, ref ? 2 : 3); -} - -static SLJIT_INLINE void compile_recurse_backtrackingpath(compiler_common *common, struct backtrack_common *current) -{ -DEFINE_COMPILER; - -if (CURRENT_AS(recurse_backtrack)->inlined_pattern) - compile_backtrackingpath(common, current->top); -set_jumps(current->topbacktracks, LABEL()); -if (CURRENT_AS(recurse_backtrack)->inlined_pattern) - return; - -if (common->has_set_som && common->mark_ptr != 0) - { - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); - free_stack(common, 2); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(0), TMP2, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, TMP1, 0); - } -else if (common->has_set_som || common->mark_ptr != 0) - { - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - free_stack(common, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->has_set_som ? (int)(OVECTOR(0)) : common->mark_ptr, TMP2, 0); - } -} - -static void compile_assert_backtrackingpath(compiler_common *common, struct backtrack_common *current) -{ -DEFINE_COMPILER; -pcre_uchar *cc = current->cc; -pcre_uchar bra = OP_BRA; -struct sljit_jump *brajump = NULL; - -SLJIT_ASSERT(*cc != OP_BRAMINZERO); -if (*cc == OP_BRAZERO) - { - bra = *cc; - cc++; - } - -if (bra == OP_BRAZERO) - { - SLJIT_ASSERT(current->topbacktracks == NULL); - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - } - -if (CURRENT_AS(assert_backtrack)->framesize < 0) - { - set_jumps(current->topbacktracks, LABEL()); - - if (bra == OP_BRAZERO) - { - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); - CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(assert_backtrack)->matchingpath); - free_stack(common, 1); - } - return; - } - -if (bra == OP_BRAZERO) - { - if (*cc == OP_ASSERT_NOT || *cc == OP_ASSERTBACK_NOT) - { - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); - CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(assert_backtrack)->matchingpath); - free_stack(common, 1); - return; - } - free_stack(common, 1); - brajump = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0); - } - -if (*cc == OP_ASSERT || *cc == OP_ASSERTBACK) - { - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(assert_backtrack)->private_data_ptr); - add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(assert_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(assert_backtrack)->framesize * sizeof(sljit_sw)); - - set_jumps(current->topbacktracks, LABEL()); - } -else - set_jumps(current->topbacktracks, LABEL()); - -if (bra == OP_BRAZERO) - { - /* We know there is enough place on the stack. */ - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); - JUMPTO(SLJIT_JUMP, CURRENT_AS(assert_backtrack)->matchingpath); - JUMPHERE(brajump); - } -} - -static void compile_bracket_backtrackingpath(compiler_common *common, struct backtrack_common *current) -{ -DEFINE_COMPILER; -int opcode, stacksize, alt_count, alt_max; -int offset = 0; -int private_data_ptr = CURRENT_AS(bracket_backtrack)->private_data_ptr; -int repeat_ptr = 0, repeat_type = 0, repeat_count = 0; -pcre_uchar *cc = current->cc; -pcre_uchar *ccbegin; -pcre_uchar *ccprev; -pcre_uchar bra = OP_BRA; -pcre_uchar ket; -assert_backtrack *assert; -sljit_uw *next_update_addr = NULL; -BOOL has_alternatives; -BOOL needs_control_head = FALSE; -struct sljit_jump *brazero = NULL; -struct sljit_jump *alt1 = NULL; -struct sljit_jump *alt2 = NULL; -struct sljit_jump *once = NULL; -struct sljit_jump *cond = NULL; -struct sljit_label *rmin_label = NULL; -struct sljit_label *exact_label = NULL; - -if (*cc == OP_BRAZERO || *cc == OP_BRAMINZERO) - { - bra = *cc; - cc++; - } - -opcode = *cc; -ccbegin = bracketend(cc) - 1 - LINK_SIZE; -ket = *ccbegin; -if (ket == OP_KET && PRIVATE_DATA(ccbegin) != 0) - { - repeat_ptr = PRIVATE_DATA(ccbegin); - repeat_type = PRIVATE_DATA(ccbegin + 2); - repeat_count = PRIVATE_DATA(ccbegin + 3); - SLJIT_ASSERT(repeat_type != 0 && repeat_count != 0); - if (repeat_type == OP_UPTO) - ket = OP_KETRMAX; - if (repeat_type == OP_MINUPTO) - ket = OP_KETRMIN; - } -ccbegin = cc; -cc += GET(cc, 1); -has_alternatives = *cc == OP_ALT; -if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND)) - has_alternatives = (ccbegin[1 + LINK_SIZE] >= OP_ASSERT && ccbegin[1 + LINK_SIZE] <= OP_ASSERTBACK_NOT) || CURRENT_AS(bracket_backtrack)->u.condfailed != NULL; -if (opcode == OP_CBRA || opcode == OP_SCBRA) - offset = (GET2(ccbegin, 1 + LINK_SIZE)) << 1; -if (SLJIT_UNLIKELY(opcode == OP_COND) && (*cc == OP_KETRMAX || *cc == OP_KETRMIN)) - opcode = OP_SCOND; -if (SLJIT_UNLIKELY(opcode == OP_ONCE_NC)) - opcode = OP_ONCE; - -alt_max = has_alternatives ? no_alternatives(ccbegin) : 0; - -/* Decoding the needs_control_head in framesize. */ -if (opcode == OP_ONCE) - { - needs_control_head = (CURRENT_AS(bracket_backtrack)->u.framesize & 0x1) != 0; - CURRENT_AS(bracket_backtrack)->u.framesize >>= 1; - } - -if (ket != OP_KET && repeat_type != 0) - { - /* TMP1 is used in OP_KETRMIN below. */ - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - free_stack(common, 1); - if (repeat_type == OP_UPTO) - OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_SP), repeat_ptr, TMP1, 0, SLJIT_IMM, 1); - else - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), repeat_ptr, TMP1, 0); - } - -if (ket == OP_KETRMAX) - { - if (bra == OP_BRAZERO) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - free_stack(common, 1); - brazero = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0); - } - } -else if (ket == OP_KETRMIN) - { - if (bra != OP_BRAMINZERO) - { - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - if (repeat_type != 0) - { - /* TMP1 was set a few lines above. */ - CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0, CURRENT_AS(bracket_backtrack)->recursive_matchingpath); - /* Drop STR_PTR for non-greedy plus quantifier. */ - if (opcode != OP_ONCE) - free_stack(common, 1); - } - else if (opcode >= OP_SBRA || opcode == OP_ONCE) - { - /* Checking zero-length iteration. */ - if (opcode != OP_ONCE || CURRENT_AS(bracket_backtrack)->u.framesize < 0) - CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, CURRENT_AS(bracket_backtrack)->recursive_matchingpath); - else - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), (CURRENT_AS(bracket_backtrack)->u.framesize + 1) * sizeof(sljit_sw), CURRENT_AS(bracket_backtrack)->recursive_matchingpath); - } - /* Drop STR_PTR for non-greedy plus quantifier. */ - if (opcode != OP_ONCE) - free_stack(common, 1); - } - else - JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_backtrack)->recursive_matchingpath); - } - rmin_label = LABEL(); - if (repeat_type != 0) - OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1); - } -else if (bra == OP_BRAZERO) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - free_stack(common, 1); - brazero = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0); - } -else if (repeat_type == OP_EXACT) - { - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1); - exact_label = LABEL(); - } - -if (offset != 0) - { - if (common->capture_last_ptr != 0) - { - SLJIT_ASSERT(common->optimized_cbracket[offset >> 1] == 0); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, TMP1, 0); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(2)); - free_stack(common, 3); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP2, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP1, 0); - } - else if (common->optimized_cbracket[offset >> 1] == 0) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); - free_stack(common, 2); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP2, 0); - } - } - -if (SLJIT_UNLIKELY(opcode == OP_ONCE)) - { - if (CURRENT_AS(bracket_backtrack)->u.framesize >= 0) - { - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - } - once = JUMP(SLJIT_JUMP); - } -else if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND)) - { - if (has_alternatives) - { - /* Always exactly one alternative. */ - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - free_stack(common, 1); - - alt_max = 2; - alt1 = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, sizeof(sljit_uw)); - } - } -else if (has_alternatives) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - free_stack(common, 1); - - if (alt_max > 4) - { - /* Table jump if alt_max is greater than 4. */ - next_update_addr = allocate_read_only_data(common, alt_max * sizeof(sljit_uw)); - if (SLJIT_UNLIKELY(next_update_addr == NULL)) - return; - sljit_emit_ijump(compiler, SLJIT_JUMP, SLJIT_MEM1(TMP1), (sljit_sw)next_update_addr); - add_label_addr(common, next_update_addr++); - } - else - { - if (alt_max == 4) - alt2 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_uw)); - alt1 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, sizeof(sljit_uw)); - } - } - -COMPILE_BACKTRACKINGPATH(current->top); -if (current->topbacktracks) - set_jumps(current->topbacktracks, LABEL()); - -if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND)) - { - /* Conditional block always has at most one alternative. */ - if (ccbegin[1 + LINK_SIZE] >= OP_ASSERT && ccbegin[1 + LINK_SIZE] <= OP_ASSERTBACK_NOT) - { - SLJIT_ASSERT(has_alternatives); - assert = CURRENT_AS(bracket_backtrack)->u.assert; - if (assert->framesize >= 0 && (ccbegin[1 + LINK_SIZE] == OP_ASSERT || ccbegin[1 + LINK_SIZE] == OP_ASSERTBACK)) - { - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr); - add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), assert->framesize * sizeof(sljit_sw)); - } - cond = JUMP(SLJIT_JUMP); - set_jumps(CURRENT_AS(bracket_backtrack)->u.assert->condfailed, LABEL()); - } - else if (CURRENT_AS(bracket_backtrack)->u.condfailed != NULL) - { - SLJIT_ASSERT(has_alternatives); - cond = JUMP(SLJIT_JUMP); - set_jumps(CURRENT_AS(bracket_backtrack)->u.condfailed, LABEL()); - } - else - SLJIT_ASSERT(!has_alternatives); - } - -if (has_alternatives) - { - alt_count = sizeof(sljit_uw); - do - { - current->top = NULL; - current->topbacktracks = NULL; - current->nextbacktracks = NULL; - /* Conditional blocks always have an additional alternative, even if it is empty. */ - if (*cc == OP_ALT) - { - ccprev = cc + 1 + LINK_SIZE; - cc += GET(cc, 1); - if (opcode != OP_COND && opcode != OP_SCOND) - { - if (opcode != OP_ONCE) - { - if (private_data_ptr != 0) - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); - else - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - } - else - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(needs_control_head ? 1 : 0)); - } - compile_matchingpath(common, ccprev, cc, current); - if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) - return; - } - - /* Instructions after the current alternative is successfully matched. */ - /* There is a similar code in compile_bracket_matchingpath. */ - if (opcode == OP_ONCE) - match_once_common(common, ket, CURRENT_AS(bracket_backtrack)->u.framesize, private_data_ptr, has_alternatives, needs_control_head); - - stacksize = 0; - if (repeat_type == OP_MINUPTO) - { - /* We need to preserve the counter. TMP2 will be used below. */ - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), repeat_ptr); - stacksize++; - } - if (ket != OP_KET || bra != OP_BRA) - stacksize++; - if (offset != 0) - { - if (common->capture_last_ptr != 0) - stacksize++; - if (common->optimized_cbracket[offset >> 1] == 0) - stacksize += 2; - } - if (opcode != OP_ONCE) - stacksize++; - - if (stacksize > 0) - allocate_stack(common, stacksize); - - stacksize = 0; - if (repeat_type == OP_MINUPTO) - { - /* TMP2 was set above. */ - OP2(SLJIT_SUB, SLJIT_MEM1(STACK_TOP), STACK(stacksize), TMP2, 0, SLJIT_IMM, 1); - stacksize++; - } - - if (ket != OP_KET || bra != OP_BRA) - { - if (ket != OP_KET) - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0); - else - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0); - stacksize++; - } - - if (offset != 0) - stacksize = match_capture_common(common, stacksize, offset, private_data_ptr); - - if (opcode != OP_ONCE) - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, alt_count); - - if (offset != 0 && ket == OP_KETRMAX && common->optimized_cbracket[offset >> 1] != 0) - { - /* If ket is not OP_KETRMAX, this code path is executed after the jump to alternative_matchingpath. */ - SLJIT_ASSERT(private_data_ptr == OVECTOR(offset + 0)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0); - } - - JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_backtrack)->alternative_matchingpath); - - if (opcode != OP_ONCE) - { - if (alt_max > 4) - add_label_addr(common, next_update_addr++); - else - { - if (alt_count != 2 * sizeof(sljit_uw)) - { - JUMPHERE(alt1); - if (alt_max == 3 && alt_count == sizeof(sljit_uw)) - alt2 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_uw)); - } - else - { - JUMPHERE(alt2); - if (alt_max == 4) - alt1 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 3 * sizeof(sljit_uw)); - } - } - alt_count += sizeof(sljit_uw); - } - - COMPILE_BACKTRACKINGPATH(current->top); - if (current->topbacktracks) - set_jumps(current->topbacktracks, LABEL()); - SLJIT_ASSERT(!current->nextbacktracks); - } - while (*cc == OP_ALT); - - if (cond != NULL) - { - SLJIT_ASSERT(opcode == OP_COND || opcode == OP_SCOND); - assert = CURRENT_AS(bracket_backtrack)->u.assert; - if ((ccbegin[1 + LINK_SIZE] == OP_ASSERT_NOT || ccbegin[1 + LINK_SIZE] == OP_ASSERTBACK_NOT) && assert->framesize >= 0) - { - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr); - add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), assert->framesize * sizeof(sljit_sw)); - } - JUMPHERE(cond); - } - - /* Free the STR_PTR. */ - if (private_data_ptr == 0) - free_stack(common, 1); - } - -if (offset != 0) - { - /* Using both tmp register is better for instruction scheduling. */ - if (common->optimized_cbracket[offset >> 1] != 0) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); - free_stack(common, 2); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP2, 0); - } - else - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - free_stack(common, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0); - } - } -else if (opcode == OP_SBRA || opcode == OP_SCOND) - { - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(0)); - free_stack(common, 1); - } -else if (opcode == OP_ONCE) - { - cc = ccbegin + GET(ccbegin, 1); - stacksize = needs_control_head ? 1 : 0; - - if (CURRENT_AS(bracket_backtrack)->u.framesize >= 0) - { - /* Reset head and drop saved frame. */ - stacksize += CURRENT_AS(bracket_backtrack)->u.framesize + ((ket != OP_KET || *cc == OP_ALT) ? 2 : 1); - } - else if (ket == OP_KETRMAX || (*cc == OP_ALT && ket != OP_KETRMIN)) - { - /* The STR_PTR must be released. */ - stacksize++; - } - - if (stacksize > 0) - free_stack(common, stacksize); - - JUMPHERE(once); - /* Restore previous private_data_ptr */ - if (CURRENT_AS(bracket_backtrack)->u.framesize >= 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(bracket_backtrack)->u.framesize * sizeof(sljit_sw)); - else if (ket == OP_KETRMIN) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); - /* See the comment below. */ - free_stack(common, 2); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0); - } - } - -if (repeat_type == OP_EXACT) - { - OP2(SLJIT_ADD, TMP1, 0, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), repeat_ptr, TMP1, 0); - CMPTO(SLJIT_LESS_EQUAL, TMP1, 0, SLJIT_IMM, repeat_count, exact_label); - } -else if (ket == OP_KETRMAX) - { - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - if (bra != OP_BRAZERO) - free_stack(common, 1); - - CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(bracket_backtrack)->recursive_matchingpath); - if (bra == OP_BRAZERO) - { - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); - JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_backtrack)->zero_matchingpath); - JUMPHERE(brazero); - free_stack(common, 1); - } - } -else if (ket == OP_KETRMIN) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - - /* OP_ONCE removes everything in case of a backtrack, so we don't - need to explicitly release the STR_PTR. The extra release would - affect badly the free_stack(2) above. */ - if (opcode != OP_ONCE) - free_stack(common, 1); - CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0, rmin_label); - if (opcode == OP_ONCE) - free_stack(common, bra == OP_BRAMINZERO ? 2 : 1); - else if (bra == OP_BRAMINZERO) - free_stack(common, 1); - } -else if (bra == OP_BRAZERO) - { - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_backtrack)->zero_matchingpath); - JUMPHERE(brazero); - } -} - -static SLJIT_INLINE void compile_bracketpos_backtrackingpath(compiler_common *common, struct backtrack_common *current) -{ -DEFINE_COMPILER; -int offset; -struct sljit_jump *jump; - -if (CURRENT_AS(bracketpos_backtrack)->framesize < 0) - { - if (*current->cc == OP_CBRAPOS || *current->cc == OP_SCBRAPOS) - { - offset = (GET2(current->cc, 1 + LINK_SIZE)) << 1; - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0); - if (common->capture_last_ptr != 0) - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(2)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP2, 0); - if (common->capture_last_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, TMP1, 0); - } - set_jumps(current->topbacktracks, LABEL()); - free_stack(common, CURRENT_AS(bracketpos_backtrack)->stacksize); - return; - } - -OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(bracketpos_backtrack)->private_data_ptr); -add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - -if (current->topbacktracks) - { - jump = JUMP(SLJIT_JUMP); - set_jumps(current->topbacktracks, LABEL()); - /* Drop the stack frame. */ - free_stack(common, CURRENT_AS(bracketpos_backtrack)->stacksize); - JUMPHERE(jump); - } -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(bracketpos_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), CURRENT_AS(bracketpos_backtrack)->framesize * sizeof(sljit_sw)); -} - -static SLJIT_INLINE void compile_braminzero_backtrackingpath(compiler_common *common, struct backtrack_common *current) -{ -assert_backtrack backtrack; - -current->top = NULL; -current->topbacktracks = NULL; -current->nextbacktracks = NULL; -if (current->cc[1] > OP_ASSERTBACK_NOT) - { - /* Manual call of compile_bracket_matchingpath and compile_bracket_backtrackingpath. */ - compile_bracket_matchingpath(common, current->cc, current); - compile_bracket_backtrackingpath(common, current->top); - } -else - { - memset(&backtrack, 0, sizeof(backtrack)); - backtrack.common.cc = current->cc; - backtrack.matchingpath = CURRENT_AS(braminzero_backtrack)->matchingpath; - /* Manual call of compile_assert_matchingpath. */ - compile_assert_matchingpath(common, current->cc, &backtrack, FALSE); - } -SLJIT_ASSERT(!current->nextbacktracks && !current->topbacktracks); -} - -static SLJIT_INLINE void compile_control_verb_backtrackingpath(compiler_common *common, struct backtrack_common *current) -{ -DEFINE_COMPILER; -pcre_uchar opcode = *current->cc; -struct sljit_label *loop; -struct sljit_jump *jump; - -if (opcode == OP_THEN || opcode == OP_THEN_ARG) - { - if (common->then_trap != NULL) - { - SLJIT_ASSERT(common->control_head_ptr != 0); - - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, type_then_trap); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, common->then_trap->start); - jump = JUMP(SLJIT_JUMP); - - loop = LABEL(); - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), -(int)sizeof(sljit_sw)); - JUMPHERE(jump); - CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), -(int)(2 * sizeof(sljit_sw)), TMP1, 0, loop); - CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), -(int)(3 * sizeof(sljit_sw)), TMP2, 0, loop); - add_jump(compiler, &common->then_trap->quit, JUMP(SLJIT_JUMP)); - return; - } - else if (common->positive_assert) - { - add_jump(compiler, &common->positive_assert_quit, JUMP(SLJIT_JUMP)); - return; - } - } - -if (common->local_exit) - { - if (common->quit_label == NULL) - add_jump(compiler, &common->quit, JUMP(SLJIT_JUMP)); - else - JUMPTO(SLJIT_JUMP, common->quit_label); - return; - } - -if (opcode == OP_SKIP_ARG) - { - SLJIT_ASSERT(common->control_head_ptr != 0); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STACK_TOP, 0); - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_IMM, (sljit_sw)(current->cc + 2)); - sljit_emit_ijump(compiler, SLJIT_CALL2, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_search_mark)); - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); - - OP1(SLJIT_MOV, STR_PTR, 0, TMP1, 0); - add_jump(compiler, &common->reset_match, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, -1)); - return; - } - -if (opcode == OP_SKIP) - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); -else - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_IMM, 0); -add_jump(compiler, &common->reset_match, JUMP(SLJIT_JUMP)); -} - -static SLJIT_INLINE void compile_then_trap_backtrackingpath(compiler_common *common, struct backtrack_common *current) -{ -DEFINE_COMPILER; -struct sljit_jump *jump; -int size; - -if (CURRENT_AS(then_trap_backtrack)->then_trap) - { - common->then_trap = CURRENT_AS(then_trap_backtrack)->then_trap; - return; - } - -size = CURRENT_AS(then_trap_backtrack)->framesize; -size = 3 + (size < 0 ? 0 : size); - -OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(size - 3)); -free_stack(common, size); -jump = JUMP(SLJIT_JUMP); - -set_jumps(CURRENT_AS(then_trap_backtrack)->quit, LABEL()); -/* STACK_TOP is set by THEN. */ -if (CURRENT_AS(then_trap_backtrack)->framesize >= 0) - add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); -OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); -free_stack(common, 3); - -JUMPHERE(jump); -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, TMP1, 0); -} - -static void compile_backtrackingpath(compiler_common *common, struct backtrack_common *current) -{ -DEFINE_COMPILER; -then_trap_backtrack *save_then_trap = common->then_trap; - -while (current) - { - if (current->nextbacktracks != NULL) - set_jumps(current->nextbacktracks, LABEL()); - switch(*current->cc) - { - case OP_SET_SOM: - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - free_stack(common, 1); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(0), TMP1, 0); - break; - - case OP_STAR: - case OP_MINSTAR: - case OP_PLUS: - case OP_MINPLUS: - case OP_QUERY: - case OP_MINQUERY: - case OP_UPTO: - case OP_MINUPTO: - case OP_EXACT: - case OP_POSSTAR: - case OP_POSPLUS: - case OP_POSQUERY: - case OP_POSUPTO: - case OP_STARI: - case OP_MINSTARI: - case OP_PLUSI: - case OP_MINPLUSI: - case OP_QUERYI: - case OP_MINQUERYI: - case OP_UPTOI: - case OP_MINUPTOI: - case OP_EXACTI: - case OP_POSSTARI: - case OP_POSPLUSI: - case OP_POSQUERYI: - case OP_POSUPTOI: - case OP_NOTSTAR: - case OP_NOTMINSTAR: - case OP_NOTPLUS: - case OP_NOTMINPLUS: - case OP_NOTQUERY: - case OP_NOTMINQUERY: - case OP_NOTUPTO: - case OP_NOTMINUPTO: - case OP_NOTEXACT: - case OP_NOTPOSSTAR: - case OP_NOTPOSPLUS: - case OP_NOTPOSQUERY: - case OP_NOTPOSUPTO: - case OP_NOTSTARI: - case OP_NOTMINSTARI: - case OP_NOTPLUSI: - case OP_NOTMINPLUSI: - case OP_NOTQUERYI: - case OP_NOTMINQUERYI: - case OP_NOTUPTOI: - case OP_NOTMINUPTOI: - case OP_NOTEXACTI: - case OP_NOTPOSSTARI: - case OP_NOTPOSPLUSI: - case OP_NOTPOSQUERYI: - case OP_NOTPOSUPTOI: - case OP_TYPESTAR: - case OP_TYPEMINSTAR: - case OP_TYPEPLUS: - case OP_TYPEMINPLUS: - case OP_TYPEQUERY: - case OP_TYPEMINQUERY: - case OP_TYPEUPTO: - case OP_TYPEMINUPTO: - case OP_TYPEEXACT: - case OP_TYPEPOSSTAR: - case OP_TYPEPOSPLUS: - case OP_TYPEPOSQUERY: - case OP_TYPEPOSUPTO: - case OP_CLASS: - case OP_NCLASS: -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - case OP_XCLASS: -#endif - compile_iterator_backtrackingpath(common, current); - break; - - case OP_REF: - case OP_REFI: - case OP_DNREF: - case OP_DNREFI: - compile_ref_iterator_backtrackingpath(common, current); - break; - - case OP_RECURSE: - compile_recurse_backtrackingpath(common, current); - break; - - case OP_ASSERT: - case OP_ASSERT_NOT: - case OP_ASSERTBACK: - case OP_ASSERTBACK_NOT: - compile_assert_backtrackingpath(common, current); - break; - - case OP_ONCE: - case OP_ONCE_NC: - case OP_BRA: - case OP_CBRA: - case OP_COND: - case OP_SBRA: - case OP_SCBRA: - case OP_SCOND: - compile_bracket_backtrackingpath(common, current); - break; - - case OP_BRAZERO: - if (current->cc[1] > OP_ASSERTBACK_NOT) - compile_bracket_backtrackingpath(common, current); - else - compile_assert_backtrackingpath(common, current); - break; - - case OP_BRAPOS: - case OP_CBRAPOS: - case OP_SBRAPOS: - case OP_SCBRAPOS: - case OP_BRAPOSZERO: - compile_bracketpos_backtrackingpath(common, current); - break; - - case OP_BRAMINZERO: - compile_braminzero_backtrackingpath(common, current); - break; - - case OP_MARK: - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(common->has_skip_arg ? 4 : 0)); - if (common->has_skip_arg) - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - free_stack(common, common->has_skip_arg ? 5 : 1); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, TMP1, 0); - if (common->has_skip_arg) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, TMP2, 0); - break; - - case OP_THEN: - case OP_THEN_ARG: - case OP_PRUNE: - case OP_PRUNE_ARG: - case OP_SKIP: - case OP_SKIP_ARG: - compile_control_verb_backtrackingpath(common, current); - break; - - case OP_COMMIT: - if (!common->local_exit) - OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_NOMATCH); - if (common->quit_label == NULL) - add_jump(compiler, &common->quit, JUMP(SLJIT_JUMP)); - else - JUMPTO(SLJIT_JUMP, common->quit_label); - break; - - case OP_CALLOUT: - case OP_FAIL: - case OP_ACCEPT: - case OP_ASSERT_ACCEPT: - set_jumps(current->topbacktracks, LABEL()); - break; - - case OP_THEN_TRAP: - /* A virtual opcode for then traps. */ - compile_then_trap_backtrackingpath(common, current); - break; - - default: - SLJIT_ASSERT_STOP(); - break; - } - current = current->prev; - } -common->then_trap = save_then_trap; -} - -static SLJIT_INLINE void compile_recurse(compiler_common *common) -{ -DEFINE_COMPILER; -pcre_uchar *cc = common->start + common->currententry->start; -pcre_uchar *ccbegin = cc + 1 + LINK_SIZE + (*cc == OP_BRA ? 0 : IMM2_SIZE); -pcre_uchar *ccend = bracketend(cc) - (1 + LINK_SIZE); -BOOL needs_control_head; -int framesize = get_framesize(common, cc, NULL, TRUE, &needs_control_head); -int private_data_size = get_private_data_copy_length(common, ccbegin, ccend, needs_control_head); -int alternativesize; -BOOL needs_frame; -backtrack_common altbacktrack; -struct sljit_jump *jump; - -/* Recurse captures then. */ -common->then_trap = NULL; - -SLJIT_ASSERT(*cc == OP_BRA || *cc == OP_CBRA || *cc == OP_CBRAPOS || *cc == OP_SCBRA || *cc == OP_SCBRAPOS); -needs_frame = framesize >= 0; -if (!needs_frame) - framesize = 0; -alternativesize = *(cc + GET(cc, 1)) == OP_ALT ? 1 : 0; - -SLJIT_ASSERT(common->currententry->entry == NULL && common->recursive_head_ptr != 0); -common->currententry->entry = LABEL(); -set_jumps(common->currententry->calls, common->currententry->entry); - -sljit_emit_fast_enter(compiler, TMP2, 0); -count_match(common); -allocate_stack(common, private_data_size + framesize + alternativesize); -OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(private_data_size + framesize + alternativesize - 1), TMP2, 0); -copy_private_data(common, ccbegin, ccend, TRUE, private_data_size + framesize + alternativesize, framesize + alternativesize, needs_control_head); -if (needs_control_head) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0); -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr, STACK_TOP, 0); -if (needs_frame) - init_frame(common, cc, NULL, framesize + alternativesize - 1, alternativesize, TRUE); - -if (alternativesize > 0) - OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); - -memset(&altbacktrack, 0, sizeof(backtrack_common)); -common->quit_label = NULL; -common->accept_label = NULL; -common->quit = NULL; -common->accept = NULL; -altbacktrack.cc = ccbegin; -cc += GET(cc, 1); -while (1) - { - altbacktrack.top = NULL; - altbacktrack.topbacktracks = NULL; - - if (altbacktrack.cc != ccbegin) - OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); - - compile_matchingpath(common, altbacktrack.cc, cc, &altbacktrack); - if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) - return; - - add_jump(compiler, &common->accept, JUMP(SLJIT_JUMP)); - - compile_backtrackingpath(common, altbacktrack.top); - if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) - return; - set_jumps(altbacktrack.topbacktracks, LABEL()); - - if (*cc != OP_ALT) - break; - - altbacktrack.cc = cc + 1 + LINK_SIZE; - cc += GET(cc, 1); - } - -/* None of them matched. */ -OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 0); -jump = JUMP(SLJIT_JUMP); - -if (common->quit != NULL) - { - set_jumps(common->quit, LABEL()); - OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr); - if (needs_frame) - { - OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); - add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); - } - OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 0); - common->quit = NULL; - add_jump(compiler, &common->quit, JUMP(SLJIT_JUMP)); - } - -set_jumps(common->accept, LABEL()); -OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr); -if (needs_frame) - { - OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); - add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); - OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); - } -OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 1); - -JUMPHERE(jump); -if (common->quit != NULL) - set_jumps(common->quit, LABEL()); -copy_private_data(common, ccbegin, ccend, FALSE, private_data_size + framesize + alternativesize, framesize + alternativesize, needs_control_head); -free_stack(common, private_data_size + framesize + alternativesize); -if (needs_control_head) - { - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), 2 * sizeof(sljit_sw)); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), sizeof(sljit_sw)); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr, TMP1, 0); - OP1(SLJIT_MOV, TMP1, 0, TMP3, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, TMP2, 0); - } -else - { - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), sizeof(sljit_sw)); - OP1(SLJIT_MOV, TMP1, 0, TMP3, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr, TMP2, 0); - } -sljit_emit_fast_return(compiler, SLJIT_MEM1(STACK_TOP), 0); -} - -#undef COMPILE_BACKTRACKINGPATH -#undef CURRENT_AS - -void -PRIV(jit_compile)(const REAL_PCRE *re, PUBL(extra) *extra, int mode) -{ -struct sljit_compiler *compiler; -backtrack_common rootbacktrack; -compiler_common common_data; -compiler_common *common = &common_data; -const sljit_u8 *tables = re->tables; -pcre_study_data *study; -int private_data_size; -pcre_uchar *ccend; -executable_functions *functions; -void *executable_func; -sljit_uw executable_size; -sljit_uw total_length; -label_addr_list *label_addr; -struct sljit_label *mainloop_label = NULL; -struct sljit_label *continue_match_label; -struct sljit_label *empty_match_found_label = NULL; -struct sljit_label *empty_match_backtrack_label = NULL; -struct sljit_label *reset_match_label; -struct sljit_label *quit_label; -struct sljit_jump *jump; -struct sljit_jump *minlength_check_failed = NULL; -struct sljit_jump *reqbyte_notfound = NULL; -struct sljit_jump *empty_match = NULL; - -SLJIT_ASSERT((extra->flags & PCRE_EXTRA_STUDY_DATA) != 0); -study = extra->study_data; - -if (!tables) - tables = PRIV(default_tables); - -memset(&rootbacktrack, 0, sizeof(backtrack_common)); -memset(common, 0, sizeof(compiler_common)); -rootbacktrack.cc = (pcre_uchar *)re + re->name_table_offset + re->name_count * re->name_entry_size; - -common->start = rootbacktrack.cc; -common->read_only_data_head = NULL; -common->fcc = tables + fcc_offset; -common->lcc = (sljit_sw)(tables + lcc_offset); -common->mode = mode; -common->might_be_empty = study->minlength == 0; -common->nltype = NLTYPE_FIXED; -switch(re->options & PCRE_NEWLINE_BITS) - { - case 0: - /* Compile-time default */ - switch(NEWLINE) - { - case -1: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANY; break; - case -2: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANYCRLF; break; - default: common->newline = NEWLINE; break; - } - break; - case PCRE_NEWLINE_CR: common->newline = CHAR_CR; break; - case PCRE_NEWLINE_LF: common->newline = CHAR_NL; break; - case PCRE_NEWLINE_CR+ - PCRE_NEWLINE_LF: common->newline = (CHAR_CR << 8) | CHAR_NL; break; - case PCRE_NEWLINE_ANY: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANY; break; - case PCRE_NEWLINE_ANYCRLF: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANYCRLF; break; - default: return; - } -common->nlmax = READ_CHAR_MAX; -common->nlmin = 0; -if ((re->options & PCRE_BSR_ANYCRLF) != 0) - common->bsr_nltype = NLTYPE_ANYCRLF; -else if ((re->options & PCRE_BSR_UNICODE) != 0) - common->bsr_nltype = NLTYPE_ANY; -else - { -#ifdef BSR_ANYCRLF - common->bsr_nltype = NLTYPE_ANYCRLF; -#else - common->bsr_nltype = NLTYPE_ANY; -#endif - } -common->bsr_nlmax = READ_CHAR_MAX; -common->bsr_nlmin = 0; -common->endonly = (re->options & PCRE_DOLLAR_ENDONLY) != 0; -common->ctypes = (sljit_sw)(tables + ctypes_offset); -common->name_table = ((pcre_uchar *)re) + re->name_table_offset; -common->name_count = re->name_count; -common->name_entry_size = re->name_entry_size; -common->jscript_compat = (re->options & PCRE_JAVASCRIPT_COMPAT) != 0; -#ifdef SUPPORT_UTF -/* PCRE_UTF[16|32] have the same value as PCRE_UTF8. */ -common->utf = (re->options & PCRE_UTF8) != 0; -#ifdef SUPPORT_UCP -common->use_ucp = (re->options & PCRE_UCP) != 0; -#endif -if (common->utf) - { - if (common->nltype == NLTYPE_ANY) - common->nlmax = 0x2029; - else if (common->nltype == NLTYPE_ANYCRLF) - common->nlmax = (CHAR_CR > CHAR_NL) ? CHAR_CR : CHAR_NL; - else - { - /* We only care about the first newline character. */ - common->nlmax = common->newline & 0xff; - } - - if (common->nltype == NLTYPE_FIXED) - common->nlmin = common->newline & 0xff; - else - common->nlmin = (CHAR_CR < CHAR_NL) ? CHAR_CR : CHAR_NL; - - if (common->bsr_nltype == NLTYPE_ANY) - common->bsr_nlmax = 0x2029; - else - common->bsr_nlmax = (CHAR_CR > CHAR_NL) ? CHAR_CR : CHAR_NL; - common->bsr_nlmin = (CHAR_CR < CHAR_NL) ? CHAR_CR : CHAR_NL; - } -#endif /* SUPPORT_UTF */ -ccend = bracketend(common->start); - -/* Calculate the local space size on the stack. */ -common->ovector_start = LIMIT_MATCH + sizeof(sljit_sw); -common->optimized_cbracket = (sljit_u8 *)SLJIT_MALLOC(re->top_bracket + 1, compiler->allocator_data); -if (!common->optimized_cbracket) - return; -#if defined DEBUG_FORCE_UNOPTIMIZED_CBRAS && DEBUG_FORCE_UNOPTIMIZED_CBRAS == 1 -memset(common->optimized_cbracket, 0, re->top_bracket + 1); -#else -memset(common->optimized_cbracket, 1, re->top_bracket + 1); -#endif - -SLJIT_ASSERT(*common->start == OP_BRA && ccend[-(1 + LINK_SIZE)] == OP_KET); -#if defined DEBUG_FORCE_UNOPTIMIZED_CBRAS && DEBUG_FORCE_UNOPTIMIZED_CBRAS == 2 -common->capture_last_ptr = common->ovector_start; -common->ovector_start += sizeof(sljit_sw); -#endif -if (!check_opcode_types(common, common->start, ccend)) - { - SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data); - return; - } - -/* Checking flags and updating ovector_start. */ -if (mode == JIT_COMPILE && (re->flags & PCRE_REQCHSET) != 0 && (re->options & PCRE_NO_START_OPTIMIZE) == 0) - { - common->req_char_ptr = common->ovector_start; - common->ovector_start += sizeof(sljit_sw); - } -if (mode != JIT_COMPILE) - { - common->start_used_ptr = common->ovector_start; - common->ovector_start += sizeof(sljit_sw); - if (mode == JIT_PARTIAL_SOFT_COMPILE) - { - common->hit_start = common->ovector_start; - common->ovector_start += 2 * sizeof(sljit_sw); - } - } -if ((re->options & PCRE_FIRSTLINE) != 0) - { - common->match_end_ptr = common->ovector_start; - common->ovector_start += sizeof(sljit_sw); - } -#if defined DEBUG_FORCE_CONTROL_HEAD && DEBUG_FORCE_CONTROL_HEAD -common->control_head_ptr = 1; -#endif -if (common->control_head_ptr != 0) - { - common->control_head_ptr = common->ovector_start; - common->ovector_start += sizeof(sljit_sw); - } -if (common->has_set_som) - { - /* Saving the real start pointer is necessary. */ - common->start_ptr = common->ovector_start; - common->ovector_start += sizeof(sljit_sw); - } - -/* Aligning ovector to even number of sljit words. */ -if ((common->ovector_start & sizeof(sljit_sw)) != 0) - common->ovector_start += sizeof(sljit_sw); - -if (common->start_ptr == 0) - common->start_ptr = OVECTOR(0); - -/* Capturing brackets cannot be optimized if callouts are allowed. */ -if (common->capture_last_ptr != 0) - memset(common->optimized_cbracket, 0, re->top_bracket + 1); - -SLJIT_ASSERT(!(common->req_char_ptr != 0 && common->start_used_ptr != 0)); -common->cbra_ptr = OVECTOR_START + (re->top_bracket + 1) * 2 * sizeof(sljit_sw); - -total_length = ccend - common->start; -common->private_data_ptrs = (sljit_s32 *)SLJIT_MALLOC(total_length * (sizeof(sljit_s32) + (common->has_then ? 1 : 0)), compiler->allocator_data); -if (!common->private_data_ptrs) - { - SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data); - return; - } -memset(common->private_data_ptrs, 0, total_length * sizeof(sljit_s32)); - -private_data_size = common->cbra_ptr + (re->top_bracket + 1) * sizeof(sljit_sw); -set_private_data_ptrs(common, &private_data_size, ccend); -if ((re->options & PCRE_ANCHORED) == 0 && (re->options & PCRE_NO_START_OPTIMIZE) == 0) - { - if (!detect_fast_forward_skip(common, &private_data_size) && !common->has_skip_in_assert_back) - detect_fast_fail(common, common->start, &private_data_size, 4); - } - -SLJIT_ASSERT(common->fast_fail_start_ptr <= common->fast_fail_end_ptr); - -if (private_data_size > SLJIT_MAX_LOCAL_SIZE) - { - SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data); - SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data); - return; - } - -if (common->has_then) - { - common->then_offsets = (sljit_u8 *)(common->private_data_ptrs + total_length); - memset(common->then_offsets, 0, total_length); - set_then_offsets(common, common->start, NULL); - } - -compiler = sljit_create_compiler(NULL); -if (!compiler) - { - SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data); - SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data); - return; - } -common->compiler = compiler; - -/* Main pcre_jit_exec entry. */ -sljit_emit_enter(compiler, 0, 1, 5, 5, 0, 0, private_data_size); - -/* Register init. */ -reset_ovector(common, (re->top_bracket + 1) * 2); -if (common->req_char_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->req_char_ptr, SLJIT_R0, 0); - -OP1(SLJIT_MOV, ARGUMENTS, 0, SLJIT_S0, 0); -OP1(SLJIT_MOV, TMP1, 0, SLJIT_S0, 0); -OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); -OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, end)); -OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack)); -OP1(SLJIT_MOV_U32, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, limit_match)); -OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(struct sljit_stack, base)); -OP1(SLJIT_MOV, STACK_LIMIT, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(struct sljit_stack, limit)); -OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LIMIT_MATCH, TMP1, 0); - -if (common->fast_fail_start_ptr < common->fast_fail_end_ptr) - reset_fast_fail(common); - -if (mode == JIT_PARTIAL_SOFT_COMPILE) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, -1); -if (common->mark_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, SLJIT_IMM, 0); -if (common->control_head_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0); - -/* Main part of the matching */ -if ((re->options & PCRE_ANCHORED) == 0) - { - mainloop_label = mainloop_entry(common, (re->flags & PCRE_HASCRORLF) != 0); - continue_match_label = LABEL(); - /* Forward search if possible. */ - if ((re->options & PCRE_NO_START_OPTIMIZE) == 0) - { - if (mode == JIT_COMPILE && fast_forward_first_n_chars(common)) - ; - else if ((re->flags & PCRE_FIRSTSET) != 0) - fast_forward_first_char(common, (pcre_uchar)re->first_char, (re->flags & PCRE_FCH_CASELESS) != 0); - else if ((re->flags & PCRE_STARTLINE) != 0) - fast_forward_newline(common); - else if (study != NULL && (study->flags & PCRE_STUDY_MAPPED) != 0) - fast_forward_start_bits(common, study->start_bits); - } - } -else - continue_match_label = LABEL(); - -if (mode == JIT_COMPILE && study->minlength > 0 && (re->options & PCRE_NO_START_OPTIMIZE) == 0) - { - OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_NOMATCH); - OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(study->minlength)); - minlength_check_failed = CMP(SLJIT_GREATER, TMP2, 0, STR_END, 0); - } -if (common->req_char_ptr != 0) - reqbyte_notfound = search_requested_char(common, (pcre_uchar)re->req_char, (re->flags & PCRE_RCH_CASELESS) != 0, (re->flags & PCRE_FIRSTSET) != 0); - -/* Store the current STR_PTR in OVECTOR(0). */ -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(0), STR_PTR, 0); -/* Copy the limit of allowed recursions. */ -OP1(SLJIT_MOV, COUNT_MATCH, 0, SLJIT_MEM1(SLJIT_SP), LIMIT_MATCH); -if (common->capture_last_ptr != 0) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, SLJIT_IMM, -1); -if (common->fast_forward_bc_ptr != NULL) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), PRIVATE_DATA(common->fast_forward_bc_ptr + 1), STR_PTR, 0); - -if (common->start_ptr != OVECTOR(0)) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_ptr, STR_PTR, 0); - -/* Copy the beginning of the string. */ -if (mode == JIT_PARTIAL_SOFT_COMPILE) - { - jump = CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, -1); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start + sizeof(sljit_sw), STR_PTR, 0); - JUMPHERE(jump); - } -else if (mode == JIT_PARTIAL_HARD_COMPILE) - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0); - -compile_matchingpath(common, common->start, ccend, &rootbacktrack); -if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) - { - sljit_free_compiler(compiler); - SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data); - SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data); - free_read_only_data(common->read_only_data_head, compiler->allocator_data); - return; - } - -if (common->might_be_empty) - { - empty_match = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0)); - empty_match_found_label = LABEL(); - } - -common->accept_label = LABEL(); -if (common->accept != NULL) - set_jumps(common->accept, common->accept_label); - -/* This means we have a match. Update the ovector. */ -copy_ovector(common, re->top_bracket + 1); -common->quit_label = common->forced_quit_label = LABEL(); -if (common->quit != NULL) - set_jumps(common->quit, common->quit_label); -if (common->forced_quit != NULL) - set_jumps(common->forced_quit, common->forced_quit_label); -if (minlength_check_failed != NULL) - SET_LABEL(minlength_check_failed, common->forced_quit_label); -sljit_emit_return(compiler, SLJIT_MOV, SLJIT_RETURN_REG, 0); - -if (mode != JIT_COMPILE) - { - common->partialmatchlabel = LABEL(); - set_jumps(common->partialmatch, common->partialmatchlabel); - return_with_partial_match(common, common->quit_label); - } - -if (common->might_be_empty) - empty_match_backtrack_label = LABEL(); -compile_backtrackingpath(common, rootbacktrack.top); -if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) - { - sljit_free_compiler(compiler); - SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data); - SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data); - free_read_only_data(common->read_only_data_head, compiler->allocator_data); - return; - } - -SLJIT_ASSERT(rootbacktrack.prev == NULL); -reset_match_label = LABEL(); - -if (mode == JIT_PARTIAL_SOFT_COMPILE) - { - /* Update hit_start only in the first time. */ - jump = CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, 0); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, SLJIT_IMM, -1); - OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, TMP1, 0); - JUMPHERE(jump); - } - -/* Check we have remaining characters. */ -if ((re->options & PCRE_ANCHORED) == 0 && (re->options & PCRE_FIRSTLINE) != 0) - { - SLJIT_ASSERT(common->match_end_ptr != 0); - OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); - } - -OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), - (common->fast_forward_bc_ptr != NULL) ? (PRIVATE_DATA(common->fast_forward_bc_ptr + 1)) : common->start_ptr); - -if ((re->options & PCRE_ANCHORED) == 0) - { - if (common->ff_newline_shortcut != NULL) - { - if ((re->options & PCRE_FIRSTLINE) == 0) - CMPTO(SLJIT_LESS, STR_PTR, 0, STR_END, 0, common->ff_newline_shortcut); - /* There cannot be more newlines here. */ - } - else - CMPTO(SLJIT_LESS, STR_PTR, 0, ((re->options & PCRE_FIRSTLINE) == 0) ? STR_END : TMP1, 0, mainloop_label); - } - -/* No more remaining characters. */ -if (reqbyte_notfound != NULL) - JUMPHERE(reqbyte_notfound); - -if (mode == JIT_PARTIAL_SOFT_COMPILE) - CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, -1, common->partialmatchlabel); - -OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_NOMATCH); -JUMPTO(SLJIT_JUMP, common->quit_label); - -flush_stubs(common); - -if (common->might_be_empty) - { - JUMPHERE(empty_match); - OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); - OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty)); - CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0, empty_match_backtrack_label); - OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty_atstart)); - CMPTO(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, 0, empty_match_found_label); - OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); - CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, STR_PTR, 0, empty_match_found_label); - JUMPTO(SLJIT_JUMP, empty_match_backtrack_label); - } - -common->fast_forward_bc_ptr = NULL; -common->fast_fail_start_ptr = 0; -common->fast_fail_end_ptr = 0; -common->currententry = common->entries; -common->local_exit = TRUE; -quit_label = common->quit_label; -while (common->currententry != NULL) - { - /* Might add new entries. */ - compile_recurse(common); - if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) - { - sljit_free_compiler(compiler); - SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data); - SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data); - free_read_only_data(common->read_only_data_head, compiler->allocator_data); - return; - } - flush_stubs(common); - common->currententry = common->currententry->next; - } -common->local_exit = FALSE; -common->quit_label = quit_label; - -/* Allocating stack, returns with PCRE_ERROR_JIT_STACKLIMIT if fails. */ -/* This is a (really) rare case. */ -set_jumps(common->stackalloc, LABEL()); -/* RETURN_ADDR is not a saved register. */ -sljit_emit_fast_enter(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0); -OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP2, 0); -OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); -OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack)); -OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, top), STACK_TOP, 0); -OP2(SLJIT_ADD, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit), SLJIT_IMM, STACK_GROWTH_RATE); - -sljit_emit_ijump(compiler, SLJIT_CALL2, SLJIT_IMM, SLJIT_FUNC_OFFSET(sljit_stack_resize)); -jump = CMP(SLJIT_NOT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0); -OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); -OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack)); -OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, top)); -OP1(SLJIT_MOV, STACK_LIMIT, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit)); -OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); -sljit_emit_fast_return(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0); - -/* Allocation failed. */ -JUMPHERE(jump); -/* We break the return address cache here, but this is a really rare case. */ -OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_JIT_STACKLIMIT); -JUMPTO(SLJIT_JUMP, common->quit_label); - -/* Call limit reached. */ -set_jumps(common->calllimit, LABEL()); -OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_MATCHLIMIT); -JUMPTO(SLJIT_JUMP, common->quit_label); - -if (common->revertframes != NULL) - { - set_jumps(common->revertframes, LABEL()); - do_revertframes(common); - } -if (common->wordboundary != NULL) - { - set_jumps(common->wordboundary, LABEL()); - check_wordboundary(common); - } -if (common->anynewline != NULL) - { - set_jumps(common->anynewline, LABEL()); - check_anynewline(common); - } -if (common->hspace != NULL) - { - set_jumps(common->hspace, LABEL()); - check_hspace(common); - } -if (common->vspace != NULL) - { - set_jumps(common->vspace, LABEL()); - check_vspace(common); - } -if (common->casefulcmp != NULL) - { - set_jumps(common->casefulcmp, LABEL()); - do_casefulcmp(common); - } -if (common->caselesscmp != NULL) - { - set_jumps(common->caselesscmp, LABEL()); - do_caselesscmp(common); - } -if (common->reset_match != NULL) - { - set_jumps(common->reset_match, LABEL()); - do_reset_match(common, (re->top_bracket + 1) * 2); - CMPTO(SLJIT_GREATER, STR_PTR, 0, TMP1, 0, continue_match_label); - OP1(SLJIT_MOV, STR_PTR, 0, TMP1, 0); - JUMPTO(SLJIT_JUMP, reset_match_label); - } -#ifdef SUPPORT_UTF -#ifdef COMPILE_PCRE8 -if (common->utfreadchar != NULL) - { - set_jumps(common->utfreadchar, LABEL()); - do_utfreadchar(common); - } -if (common->utfreadchar16 != NULL) - { - set_jumps(common->utfreadchar16, LABEL()); - do_utfreadchar16(common); - } -if (common->utfreadtype8 != NULL) - { - set_jumps(common->utfreadtype8, LABEL()); - do_utfreadtype8(common); - } -#endif /* COMPILE_PCRE8 */ -#endif /* SUPPORT_UTF */ -#ifdef SUPPORT_UCP -if (common->getucd != NULL) - { - set_jumps(common->getucd, LABEL()); - do_getucd(common); - } -#endif - -SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data); -SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data); - -executable_func = sljit_generate_code(compiler); -executable_size = sljit_get_generated_code_size(compiler); -label_addr = common->label_addrs; -while (label_addr != NULL) - { - *label_addr->update_addr = sljit_get_label_addr(label_addr->label); - label_addr = label_addr->next; - } -sljit_free_compiler(compiler); -if (executable_func == NULL) - { - free_read_only_data(common->read_only_data_head, compiler->allocator_data); - return; - } - -/* Reuse the function descriptor if possible. */ -if ((extra->flags & PCRE_EXTRA_EXECUTABLE_JIT) != 0 && extra->executable_jit != NULL) - functions = (executable_functions *)extra->executable_jit; -else - { - /* Note: If your memory-checker has flagged the allocation below as a - * memory leak, it is probably because you either forgot to call - * pcre_free_study() (or pcre16_free_study()) on the pcre_extra (or - * pcre16_extra) object, or you called said function after having - * cleared the PCRE_EXTRA_EXECUTABLE_JIT bit from the "flags" field - * of the object. (The function will only free the JIT data if the - * bit remains set, as the bit indicates that the pointer to the data - * is valid.) - */ - functions = SLJIT_MALLOC(sizeof(executable_functions), compiler->allocator_data); - if (functions == NULL) - { - /* This case is highly unlikely since we just recently - freed a lot of memory. Not impossible though. */ - sljit_free_code(executable_func); - free_read_only_data(common->read_only_data_head, compiler->allocator_data); - return; - } - memset(functions, 0, sizeof(executable_functions)); - functions->top_bracket = (re->top_bracket + 1) * 2; - functions->limit_match = (re->flags & PCRE_MLSET) != 0 ? re->limit_match : 0; - extra->executable_jit = functions; - extra->flags |= PCRE_EXTRA_EXECUTABLE_JIT; - } - -functions->executable_funcs[mode] = executable_func; -functions->read_only_data_heads[mode] = common->read_only_data_head; -functions->executable_sizes[mode] = executable_size; -} - -static SLJIT_NOINLINE int jit_machine_stack_exec(jit_arguments *arguments, void *executable_func) -{ -union { - void *executable_func; - jit_function call_executable_func; -} convert_executable_func; -sljit_u8 local_space[MACHINE_STACK_SIZE]; -struct sljit_stack local_stack; - -local_stack.top = (sljit_sw)&local_space; -local_stack.base = local_stack.top; -local_stack.limit = local_stack.base + MACHINE_STACK_SIZE; -local_stack.max_limit = local_stack.limit; -arguments->stack = &local_stack; -convert_executable_func.executable_func = executable_func; -return convert_executable_func.call_executable_func(arguments); -} - -int -PRIV(jit_exec)(const PUBL(extra) *extra_data, const pcre_uchar *subject, - int length, int start_offset, int options, int *offsets, int offset_count) -{ -executable_functions *functions = (executable_functions *)extra_data->executable_jit; -union { - void *executable_func; - jit_function call_executable_func; -} convert_executable_func; -jit_arguments arguments; -int max_offset_count; -int retval; -int mode = JIT_COMPILE; - -if ((options & PCRE_PARTIAL_HARD) != 0) - mode = JIT_PARTIAL_HARD_COMPILE; -else if ((options & PCRE_PARTIAL_SOFT) != 0) - mode = JIT_PARTIAL_SOFT_COMPILE; - -if (functions->executable_funcs[mode] == NULL) - return PCRE_ERROR_JIT_BADOPTION; - -/* Sanity checks should be handled by pcre_exec. */ -arguments.str = subject + start_offset; -arguments.begin = subject; -arguments.end = subject + length; -arguments.mark_ptr = NULL; -/* JIT decreases this value less frequently than the interpreter. */ -arguments.limit_match = ((extra_data->flags & PCRE_EXTRA_MATCH_LIMIT) == 0) ? MATCH_LIMIT : (sljit_u32)(extra_data->match_limit); -if (functions->limit_match != 0 && functions->limit_match < arguments.limit_match) - arguments.limit_match = functions->limit_match; -arguments.notbol = (options & PCRE_NOTBOL) != 0; -arguments.noteol = (options & PCRE_NOTEOL) != 0; -arguments.notempty = (options & PCRE_NOTEMPTY) != 0; -arguments.notempty_atstart = (options & PCRE_NOTEMPTY_ATSTART) != 0; -arguments.offsets = offsets; -arguments.callout_data = (extra_data->flags & PCRE_EXTRA_CALLOUT_DATA) != 0 ? extra_data->callout_data : NULL; -arguments.real_offset_count = offset_count; - -/* pcre_exec() rounds offset_count to a multiple of 3, and then uses only 2/3 of -the output vector for storing captured strings, with the remainder used as -workspace. We don't need the workspace here. For compatibility, we limit the -number of captured strings in the same way as pcre_exec(), so that the user -gets the same result with and without JIT. */ - -if (offset_count != 2) - offset_count = ((offset_count - (offset_count % 3)) * 2) / 3; -max_offset_count = functions->top_bracket; -if (offset_count > max_offset_count) - offset_count = max_offset_count; -arguments.offset_count = offset_count; - -if (functions->callback) - arguments.stack = (struct sljit_stack *)functions->callback(functions->userdata); -else - arguments.stack = (struct sljit_stack *)functions->userdata; - -if (arguments.stack == NULL) - retval = jit_machine_stack_exec(&arguments, functions->executable_funcs[mode]); -else - { - convert_executable_func.executable_func = functions->executable_funcs[mode]; - retval = convert_executable_func.call_executable_func(&arguments); - } - -if (retval * 2 > offset_count) - retval = 0; -if ((extra_data->flags & PCRE_EXTRA_MARK) != 0) - *(extra_data->mark) = arguments.mark_ptr; - -return retval; -} - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre_jit_exec(const pcre *argument_re, const pcre_extra *extra_data, - PCRE_SPTR subject, int length, int start_offset, int options, - int *offsets, int offset_count, pcre_jit_stack *stack) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre16_jit_exec(const pcre16 *argument_re, const pcre16_extra *extra_data, - PCRE_SPTR16 subject, int length, int start_offset, int options, - int *offsets, int offset_count, pcre16_jit_stack *stack) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre32_jit_exec(const pcre32 *argument_re, const pcre32_extra *extra_data, - PCRE_SPTR32 subject, int length, int start_offset, int options, - int *offsets, int offset_count, pcre32_jit_stack *stack) -#endif -{ -pcre_uchar *subject_ptr = (pcre_uchar *)subject; -executable_functions *functions = (executable_functions *)extra_data->executable_jit; -union { - void *executable_func; - jit_function call_executable_func; -} convert_executable_func; -jit_arguments arguments; -int max_offset_count; -int retval; -int mode = JIT_COMPILE; - -SLJIT_UNUSED_ARG(argument_re); - -/* Plausibility checks */ -if ((options & ~PUBLIC_JIT_EXEC_OPTIONS) != 0) return PCRE_ERROR_JIT_BADOPTION; - -if ((options & PCRE_PARTIAL_HARD) != 0) - mode = JIT_PARTIAL_HARD_COMPILE; -else if ((options & PCRE_PARTIAL_SOFT) != 0) - mode = JIT_PARTIAL_SOFT_COMPILE; - -if (functions->executable_funcs[mode] == NULL) - return PCRE_ERROR_JIT_BADOPTION; - -/* Sanity checks should be handled by pcre_exec. */ -arguments.stack = (struct sljit_stack *)stack; -arguments.str = subject_ptr + start_offset; -arguments.begin = subject_ptr; -arguments.end = subject_ptr + length; -arguments.mark_ptr = NULL; -/* JIT decreases this value less frequently than the interpreter. */ -arguments.limit_match = ((extra_data->flags & PCRE_EXTRA_MATCH_LIMIT) == 0) ? MATCH_LIMIT : (sljit_u32)(extra_data->match_limit); -if (functions->limit_match != 0 && functions->limit_match < arguments.limit_match) - arguments.limit_match = functions->limit_match; -arguments.notbol = (options & PCRE_NOTBOL) != 0; -arguments.noteol = (options & PCRE_NOTEOL) != 0; -arguments.notempty = (options & PCRE_NOTEMPTY) != 0; -arguments.notempty_atstart = (options & PCRE_NOTEMPTY_ATSTART) != 0; -arguments.offsets = offsets; -arguments.callout_data = (extra_data->flags & PCRE_EXTRA_CALLOUT_DATA) != 0 ? extra_data->callout_data : NULL; -arguments.real_offset_count = offset_count; - -/* pcre_exec() rounds offset_count to a multiple of 3, and then uses only 2/3 of -the output vector for storing captured strings, with the remainder used as -workspace. We don't need the workspace here. For compatibility, we limit the -number of captured strings in the same way as pcre_exec(), so that the user -gets the same result with and without JIT. */ - -if (offset_count != 2) - offset_count = ((offset_count - (offset_count % 3)) * 2) / 3; -max_offset_count = functions->top_bracket; -if (offset_count > max_offset_count) - offset_count = max_offset_count; -arguments.offset_count = offset_count; - -convert_executable_func.executable_func = functions->executable_funcs[mode]; -retval = convert_executable_func.call_executable_func(&arguments); - -if (retval * 2 > offset_count) - retval = 0; -if ((extra_data->flags & PCRE_EXTRA_MARK) != 0) - *(extra_data->mark) = arguments.mark_ptr; - -return retval; -} - -void -PRIV(jit_free)(void *executable_funcs) -{ -int i; -executable_functions *functions = (executable_functions *)executable_funcs; -for (i = 0; i < JIT_NUMBER_OF_COMPILE_MODES; i++) - { - if (functions->executable_funcs[i] != NULL) - sljit_free_code(functions->executable_funcs[i]); - free_read_only_data(functions->read_only_data_heads[i], NULL); - } -SLJIT_FREE(functions, compiler->allocator_data); -} - -int -PRIV(jit_get_size)(void *executable_funcs) -{ -int i; -sljit_uw size = 0; -sljit_uw *executable_sizes = ((executable_functions *)executable_funcs)->executable_sizes; -for (i = 0; i < JIT_NUMBER_OF_COMPILE_MODES; i++) - size += executable_sizes[i]; -return (int)size; -} - -const char* -PRIV(jit_get_target)(void) -{ -return sljit_get_platform_name(); -} - -#if defined COMPILE_PCRE8 -PCRE_EXP_DECL pcre_jit_stack * -pcre_jit_stack_alloc(int startsize, int maxsize) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DECL pcre16_jit_stack * -pcre16_jit_stack_alloc(int startsize, int maxsize) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DECL pcre32_jit_stack * -pcre32_jit_stack_alloc(int startsize, int maxsize) -#endif -{ -if (startsize < 1 || maxsize < 1) - return NULL; -if (startsize > maxsize) - startsize = maxsize; -startsize = (startsize + STACK_GROWTH_RATE - 1) & ~(STACK_GROWTH_RATE - 1); -maxsize = (maxsize + STACK_GROWTH_RATE - 1) & ~(STACK_GROWTH_RATE - 1); -return (PUBL(jit_stack)*)sljit_allocate_stack(startsize, maxsize, NULL); -} - -#if defined COMPILE_PCRE8 -PCRE_EXP_DECL void -pcre_jit_stack_free(pcre_jit_stack *stack) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DECL void -pcre16_jit_stack_free(pcre16_jit_stack *stack) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DECL void -pcre32_jit_stack_free(pcre32_jit_stack *stack) -#endif -{ -sljit_free_stack((struct sljit_stack *)stack, NULL); -} - -#if defined COMPILE_PCRE8 -PCRE_EXP_DECL void -pcre_assign_jit_stack(pcre_extra *extra, pcre_jit_callback callback, void *userdata) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DECL void -pcre16_assign_jit_stack(pcre16_extra *extra, pcre16_jit_callback callback, void *userdata) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DECL void -pcre32_assign_jit_stack(pcre32_extra *extra, pcre32_jit_callback callback, void *userdata) -#endif -{ -executable_functions *functions; -if (extra != NULL && - (extra->flags & PCRE_EXTRA_EXECUTABLE_JIT) != 0 && - extra->executable_jit != NULL) - { - functions = (executable_functions *)extra->executable_jit; - functions->callback = callback; - functions->userdata = userdata; - } -} - -#if defined COMPILE_PCRE8 -PCRE_EXP_DECL void -pcre_jit_free_unused_memory(void) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DECL void -pcre16_jit_free_unused_memory(void) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DECL void -pcre32_jit_free_unused_memory(void) -#endif -{ -sljit_free_unused_memory_exec(); -} - -#else /* SUPPORT_JIT */ - -/* These are dummy functions to avoid linking errors when JIT support is not -being compiled. */ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DECL pcre_jit_stack * -pcre_jit_stack_alloc(int startsize, int maxsize) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DECL pcre16_jit_stack * -pcre16_jit_stack_alloc(int startsize, int maxsize) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DECL pcre32_jit_stack * -pcre32_jit_stack_alloc(int startsize, int maxsize) -#endif -{ -(void)startsize; -(void)maxsize; -return NULL; -} - -#if defined COMPILE_PCRE8 -PCRE_EXP_DECL void -pcre_jit_stack_free(pcre_jit_stack *stack) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DECL void -pcre16_jit_stack_free(pcre16_jit_stack *stack) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DECL void -pcre32_jit_stack_free(pcre32_jit_stack *stack) -#endif -{ -(void)stack; -} - -#if defined COMPILE_PCRE8 -PCRE_EXP_DECL void -pcre_assign_jit_stack(pcre_extra *extra, pcre_jit_callback callback, void *userdata) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DECL void -pcre16_assign_jit_stack(pcre16_extra *extra, pcre16_jit_callback callback, void *userdata) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DECL void -pcre32_assign_jit_stack(pcre32_extra *extra, pcre32_jit_callback callback, void *userdata) -#endif -{ -(void)extra; -(void)callback; -(void)userdata; -} - -#if defined COMPILE_PCRE8 -PCRE_EXP_DECL void -pcre_jit_free_unused_memory(void) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DECL void -pcre16_jit_free_unused_memory(void) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DECL void -pcre32_jit_free_unused_memory(void) -#endif -{ -} - -#endif - -/* End of pcre_jit_compile.c */ diff --git a/src/third_party/pcre-8.39/pcre_jit_test.c b/src/third_party/pcre-8.39/pcre_jit_test.c deleted file mode 100644 index 9b61ec000fa..00000000000 --- a/src/third_party/pcre-8.39/pcre_jit_test.c +++ /dev/null @@ -1,1764 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Main Library written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - - This JIT compiler regression test program was written by Zoltan Herczeg - Copyright (c) 2010-2012 - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include -#include "pcre.h" - - -#include "pcre_internal.h" - -/* - Letter characters: - \xe6\x92\xad = 0x64ad = 25773 (kanji) - Non-letter characters: - \xc2\xa1 = 0xa1 = (Inverted Exclamation Mark) - \xf3\xa9\xb7\x80 = 0xe9dc0 = 957888 - \xed\xa0\x80 = 55296 = 0xd800 (Invalid UTF character) - \xed\xb0\x80 = 56320 = 0xdc00 (Invalid UTF character) - Newlines: - \xc2\x85 = 0x85 = 133 (NExt Line = NEL) - \xe2\x80\xa8 = 0x2028 = 8232 (Line Separator) - Othercase pairs: - \xc3\xa9 = 0xe9 = 233 (e') - \xc3\x89 = 0xc9 = 201 (E') - \xc3\xa1 = 0xe1 = 225 (a') - \xc3\x81 = 0xc1 = 193 (A') - \x53 = 0x53 = S - \x73 = 0x73 = s - \xc5\xbf = 0x17f = 383 (long S) - \xc8\xba = 0x23a = 570 - \xe2\xb1\xa5 = 0x2c65 = 11365 - \xe1\xbd\xb8 = 0x1f78 = 8056 - \xe1\xbf\xb8 = 0x1ff8 = 8184 - \xf0\x90\x90\x80 = 0x10400 = 66560 - \xf0\x90\x90\xa8 = 0x10428 = 66600 - \xc7\x84 = 0x1c4 = 452 - \xc7\x85 = 0x1c5 = 453 - \xc7\x86 = 0x1c6 = 454 - Caseless sets: - ucp_Armenian - \x{531}-\x{556} -> \x{561}-\x{586} - ucp_Coptic - \x{2c80}-\x{2ce3} -> caseless: XOR 0x1 - ucp_Latin - \x{ff21}-\x{ff3a} -> \x{ff41]-\x{ff5a} - - Mark property: - \xcc\x8d = 0x30d = 781 - Special: - \xc2\x80 = 0x80 = 128 (lowest 2 byte character) - \xdf\xbf = 0x7ff = 2047 (highest 2 byte character) - \xe0\xa0\x80 = 0x800 = 2048 (lowest 2 byte character) - \xef\xbf\xbf = 0xffff = 65535 (highest 3 byte character) - \xf0\x90\x80\x80 = 0x10000 = 65536 (lowest 4 byte character) - \xf4\x8f\xbf\xbf = 0x10ffff = 1114111 (highest allowed utf character) -*/ - -static int regression_tests(void); - -int main(void) -{ - int jit = 0; -#if defined SUPPORT_PCRE8 - pcre_config(PCRE_CONFIG_JIT, &jit); -#elif defined SUPPORT_PCRE16 - pcre16_config(PCRE_CONFIG_JIT, &jit); -#elif defined SUPPORT_PCRE32 - pcre32_config(PCRE_CONFIG_JIT, &jit); -#endif - if (!jit) { - printf("JIT must be enabled to run pcre_jit_test\n"); - return 1; - } - return regression_tests(); -} - -/* --------------------------------------------------------------------------------------- */ - -#if !(defined SUPPORT_PCRE8) && !(defined SUPPORT_PCRE16) && !(defined SUPPORT_PCRE32) -#error SUPPORT_PCRE8 or SUPPORT_PCRE16 or SUPPORT_PCRE32 must be defined -#endif - -#define MUA (PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANYCRLF) -#define MUAP (PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANYCRLF | PCRE_UCP) -#define CMUA (PCRE_CASELESS | PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANYCRLF) -#define CMUAP (PCRE_CASELESS | PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANYCRLF | PCRE_UCP) -#define MA (PCRE_MULTILINE | PCRE_NEWLINE_ANYCRLF) -#define MAP (PCRE_MULTILINE | PCRE_NEWLINE_ANYCRLF | PCRE_UCP) -#define CMA (PCRE_CASELESS | PCRE_MULTILINE | PCRE_NEWLINE_ANYCRLF) - -#define OFFSET_MASK 0x00ffff -#define F_NO8 0x010000 -#define F_NO16 0x020000 -#define F_NO32 0x020000 -#define F_NOMATCH 0x040000 -#define F_DIFF 0x080000 -#define F_FORCECONV 0x100000 -#define F_PROPERTY 0x200000 -#define F_STUDY 0x400000 - -struct regression_test_case { - int flags; - int start_offset; - const char *pattern; - const char *input; -}; - -static struct regression_test_case regression_test_cases[] = { - /* Constant strings. */ - { MUA, 0, "AbC", "AbAbC" }, - { MUA, 0, "ACCEPT", "AACACCACCEACCEPACCEPTACCEPTT" }, - { CMUA, 0, "aA#\xc3\xa9\xc3\x81", "aA#Aa#\xc3\x89\xc3\xa1" }, - { MA, 0, "[^a]", "aAbB" }, - { CMA, 0, "[^m]", "mMnN" }, - { MA, 0, "a[^b][^#]", "abacd" }, - { CMA, 0, "A[^B][^E]", "abacd" }, - { CMUA, 0, "[^x][^#]", "XxBll" }, - { MUA, 0, "[^a]", "aaa\xc3\xa1#Ab" }, - { CMUA, 0, "[^A]", "aA\xe6\x92\xad" }, - { MUA, 0, "\\W(\\W)?\\w", "\r\n+bc" }, - { MUA, 0, "\\W(\\W)?\\w", "\n\r+bc" }, - { MUA, 0, "\\W(\\W)?\\w", "\r\r+bc" }, - { MUA, 0, "\\W(\\W)?\\w", "\n\n+bc" }, - { MUA, 0, "[axd]", "sAXd" }, - { CMUA, 0, "[axd]", "sAXd" }, - { CMUA, 0 | F_NOMATCH, "[^axd]", "DxA" }, - { MUA, 0, "[a-dA-C]", "\xe6\x92\xad\xc3\xa9.B" }, - { MUA, 0, "[^a-dA-C]", "\xe6\x92\xad\xc3\xa9" }, - { CMUA, 0, "[^\xc3\xa9]", "\xc3\xa9\xc3\x89." }, - { MUA, 0, "[^\xc3\xa9]", "\xc3\xa9\xc3\x89." }, - { MUA, 0, "[^a]", "\xc2\x80[]" }, - { CMUA, 0, "\xf0\x90\x90\xa7", "\xf0\x90\x91\x8f" }, - { CMA, 0, "1a2b3c4", "1a2B3c51A2B3C4" }, - { PCRE_CASELESS, 0, "\xff#a", "\xff#\xff\xfe##\xff#A" }, - { PCRE_CASELESS, 0, "\xfe", "\xff\xfc#\xfe\xfe" }, - { PCRE_CASELESS, 0, "a1", "Aa1" }, - { MA, 0, "\\Ca", "cda" }, - { CMA, 0, "\\Ca", "CDA" }, - { MA, 0 | F_NOMATCH, "\\Cx", "cda" }, - { CMA, 0 | F_NOMATCH, "\\Cx", "CDA" }, - { CMUAP, 0, "\xf0\x90\x90\x80\xf0\x90\x90\xa8", "\xf0\x90\x90\xa8\xf0\x90\x90\x80" }, - { CMUAP, 0, "\xf0\x90\x90\x80{2}", "\xf0\x90\x90\x80#\xf0\x90\x90\xa8\xf0\x90\x90\x80" }, - { CMUAP, 0, "\xf0\x90\x90\xa8{2}", "\xf0\x90\x90\x80#\xf0\x90\x90\xa8\xf0\x90\x90\x80" }, - { CMUAP, 0, "\xe1\xbd\xb8\xe1\xbf\xb8", "\xe1\xbf\xb8\xe1\xbd\xb8" }, - { MA, 0, "[3-57-9]", "5" }, - - /* Assertions. */ - { MUA, 0, "\\b[^A]", "A_B#" }, - { MA, 0 | F_NOMATCH, "\\b\\W", "\n*" }, - { MUA, 0, "\\B[^,]\\b[^s]\\b", "#X" }, - { MAP, 0, "\\B", "_\xa1" }, - { MAP, 0, "\\b_\\b[,A]\\B", "_," }, - { MUAP, 0, "\\b", "\xe6\x92\xad!" }, - { MUAP, 0, "\\B", "_\xc2\xa1\xc3\xa1\xc2\x85" }, - { MUAP, 0, "\\b[^A]\\B[^c]\\b[^_]\\B", "_\xc3\xa1\xe2\x80\xa8" }, - { MUAP, 0, "\\b\\w+\\B", "\xc3\x89\xc2\xa1\xe6\x92\xad\xc3\x81\xc3\xa1" }, - { MUA, 0 | F_NOMATCH, "\\b.", "\xcd\xbe" }, - { CMUAP, 0, "\\By", "\xf0\x90\x90\xa8y" }, - { MA, 0 | F_NOMATCH, "\\R^", "\n" }, - { MA, 1 | F_NOMATCH, "^", "\n" }, - { 0, 0, "^ab", "ab" }, - { 0, 0 | F_NOMATCH, "^ab", "aab" }, - { PCRE_MULTILINE | PCRE_NEWLINE_CRLF, 0, "^a", "\r\raa\n\naa\r\naa" }, - { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANYCRLF, 0, "^-", "\xe2\x80\xa8--\xc2\x85-\r\n-" }, - { PCRE_MULTILINE | PCRE_NEWLINE_ANY, 0, "^-", "a--b--\x85--" }, - { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANY, 0, "^-", "a--\xe2\x80\xa8--" }, - { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANY, 0, "^-", "a--\xc2\x85--" }, - { 0, 0, "ab$", "ab" }, - { 0, 0 | F_NOMATCH, "ab$", "abab\n\n" }, - { PCRE_DOLLAR_ENDONLY, 0 | F_NOMATCH, "ab$", "abab\r\n" }, - { PCRE_MULTILINE | PCRE_NEWLINE_CRLF, 0, "a$", "\r\raa\n\naa\r\naa" }, - { PCRE_MULTILINE | PCRE_NEWLINE_ANY, 0, "a$", "aaa" }, - { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANYCRLF, 0, "#$", "#\xc2\x85###\r#" }, - { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANY, 0, "#$", "#\xe2\x80\xa9" }, - { PCRE_NOTBOL | PCRE_NEWLINE_ANY, 0 | F_NOMATCH, "^a", "aa\naa" }, - { PCRE_NOTBOL | PCRE_MULTILINE | PCRE_NEWLINE_ANY, 0, "^a", "aa\naa" }, - { PCRE_NOTEOL | PCRE_NEWLINE_ANY, 0 | F_NOMATCH, "a$", "aa\naa" }, - { PCRE_NOTEOL | PCRE_NEWLINE_ANY, 0 | F_NOMATCH, "a$", "aa\r\n" }, - { PCRE_UTF8 | PCRE_DOLLAR_ENDONLY | PCRE_NEWLINE_ANY, 0 | F_PROPERTY, "\\p{Any}{2,}$", "aa\r\n" }, - { PCRE_NOTEOL | PCRE_MULTILINE | PCRE_NEWLINE_ANY, 0, "a$", "aa\naa" }, - { PCRE_NEWLINE_CR, 0, ".\\Z", "aaa" }, - { PCRE_NEWLINE_CR | PCRE_UTF8, 0, "a\\Z", "aaa\r" }, - { PCRE_NEWLINE_CR, 0, ".\\Z", "aaa\n" }, - { PCRE_NEWLINE_CRLF, 0, ".\\Z", "aaa\r" }, - { PCRE_NEWLINE_CRLF | PCRE_UTF8, 0, ".\\Z", "aaa\n" }, - { PCRE_NEWLINE_CRLF, 0, ".\\Z", "aaa\r\n" }, - { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".\\Z", "aaa" }, - { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".\\Z", "aaa\r" }, - { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".\\Z", "aaa\n" }, - { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".\\Z", "aaa\r\n" }, - { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".\\Z", "aaa\xe2\x80\xa8" }, - { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".\\Z", "aaa" }, - { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".\\Z", "aaa\r" }, - { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".\\Z", "aaa\n" }, - { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".\\Z", "aaa\r\n" }, - { PCRE_NEWLINE_ANY | PCRE_UTF8, 0, ".\\Z", "aaa\xc2\x85" }, - { PCRE_NEWLINE_ANY | PCRE_UTF8, 0, ".\\Z", "aaa\xe2\x80\xa8" }, - { MA, 0, "\\Aa", "aaa" }, - { MA, 1 | F_NOMATCH, "\\Aa", "aaa" }, - { MA, 1, "\\Ga", "aaa" }, - { MA, 1 | F_NOMATCH, "\\Ga", "aba" }, - { MA, 0, "a\\z", "aaa" }, - { MA, 0 | F_NOMATCH, "a\\z", "aab" }, - - /* Brackets and alternatives. */ - { MUA, 0, "(ab|bb|cd)", "bacde" }, - { MUA, 0, "(?:ab|a)(bc|c)", "ababc" }, - { MUA, 0, "((ab|(cc))|(bb)|(?:cd|efg))", "abac" }, - { CMUA, 0, "((aB|(Cc))|(bB)|(?:cd|EFg))", "AcCe" }, - { MUA, 0, "((ab|(cc))|(bb)|(?:cd|ebg))", "acebebg" }, - { MUA, 0, "(?:(a)|(?:b))(cc|(?:d|e))(a|b)k", "accabdbbccbk" }, - { MUA, 0, "\xc7\x82|\xc6\x82", "\xf1\x83\x82\x82\xc7\x82\xc7\x83" }, - { MUA, 0, "=\xc7\x82|#\xc6\x82", "\xf1\x83\x82\x82=\xc7\x82\xc7\x83" }, - { MUA, 0, "\xc7\x82\xc7\x83|\xc6\x82\xc6\x82", "\xf1\x83\x82\x82\xc7\x82\xc7\x83" }, - { MUA, 0, "\xc6\x82\xc6\x82|\xc7\x83\xc7\x83|\xc8\x84\xc8\x84", "\xf1\x83\x82\x82\xc8\x84\xc8\x84" }, - - /* Greedy and non-greedy ? operators. */ - { MUA, 0, "(?:a)?a", "laab" }, - { CMUA, 0, "(A)?A", "llaab" }, - { MUA, 0, "(a)?\?a", "aab" }, /* ?? is the prefix of trygraphs in GCC. */ - { MUA, 0, "(a)?a", "manm" }, - { CMUA, 0, "(a|b)?\?d((?:e)?)", "ABABdx" }, - { MUA, 0, "(a|b)?\?d((?:e)?)", "abcde" }, - { MUA, 0, "((?:ab)?\?g|b(?:g(nn|d)?\?)?)?\?(?:n)?m", "abgnbgnnbgdnmm" }, - - /* Greedy and non-greedy + operators */ - { MUA, 0, "(aa)+aa", "aaaaaaa" }, - { MUA, 0, "(aa)+?aa", "aaaaaaa" }, - { MUA, 0, "(?:aba|ab|a)+l", "ababamababal" }, - { MUA, 0, "(?:aba|ab|a)+?l", "ababamababal" }, - { MUA, 0, "(a(?:bc|cb|b|c)+?|ss)+e", "accssabccbcacbccbbXaccssabccbcacbccbbe" }, - { MUA, 0, "(a(?:bc|cb|b|c)+|ss)+?e", "accssabccbcacbccbbXaccssabccbcacbccbbe" }, - { MUA, 0, "(?:(b(c)+?)+)?\?(?:(bc)+|(cb)+)+(?:m)+", "bccbcccbcbccbcbPbccbcccbcbccbcbmmn" }, - - /* Greedy and non-greedy * operators */ - { CMUA, 0, "(?:AA)*AB", "aaaaaaamaaaaaaab" }, - { MUA, 0, "(?:aa)*?ab", "aaaaaaamaaaaaaab" }, - { MUA, 0, "(aa|ab)*ab", "aaabaaab" }, - { CMUA, 0, "(aa|Ab)*?aB", "aaabaaab" }, - { MUA, 0, "(a|b)*(?:a)*(?:b)*m", "abbbaaababanabbbaaababamm" }, - { MUA, 0, "(a|b)*?(?:a)*?(?:b)*?m", "abbbaaababanabbbaaababamm" }, - { MA, 0, "a(a(\\1*)a|(b)b+){0}a", "aa" }, - { MA, 0, "((?:a|)*){0}a", "a" }, - - /* Combining ? + * operators */ - { MUA, 0, "((bm)+)?\?(?:a)*(bm)+n|((am)+?)?(?:a)+(am)*n", "bmbmabmamaaamambmaman" }, - { MUA, 0, "(((ab)?cd)*ef)+g", "abcdcdefcdefefmabcdcdefcdefefgg" }, - { MUA, 0, "(((ab)?\?cd)*?ef)+?g", "abcdcdefcdefefmabcdcdefcdefefgg" }, - { MUA, 0, "(?:(ab)?c|(?:ab)+?d)*g", "ababcdccababddg" }, - { MUA, 0, "(?:(?:ab)?\?c|(ab)+d)*?g", "ababcdccababddg" }, - - /* Single character iterators. */ - { MUA, 0, "(a+aab)+aaaab", "aaaabcaaaabaabcaabcaaabaaaab" }, - { MUA, 0, "(a*a*aab)+x", "aaaaabaabaaabmaabx" }, - { MUA, 0, "(a*?(b|ab)a*?)+x", "aaaabcxbbaabaacbaaabaabax" }, - { MUA, 0, "(a+(ab|ad)a+)+x", "aaabaaaadaabaaabaaaadaaax" }, - { MUA, 0, "(a?(a)a?)+(aaa)", "abaaabaaaaaaaa" }, - { MUA, 0, "(a?\?(a)a?\?)+(b)", "aaaacaaacaacacbaaab" }, - { MUA, 0, "(a{0,4}(b))+d", "aaaaaabaabcaaaaabaaaaabd" }, - { MUA, 0, "(a{0,4}?[^b])+d+(a{0,4}[^b])d+", "aaaaadaaaacaadddaaddd" }, - { MUA, 0, "(ba{2})+c", "baabaaabacbaabaac" }, - { MUA, 0, "(a*+bc++)+", "aaabbcaaabcccab" }, - { MUA, 0, "(a?+[^b])+", "babaacacb" }, - { MUA, 0, "(a{0,3}+b)(a{0,3}+b)(a{0,3}+)[^c]", "abaabaaacbaabaaaac" }, - { CMUA, 0, "([a-c]+[d-f]+?)+?g", "aBdacdehAbDaFgA" }, - { CMUA, 0, "[c-f]+k", "DemmFke" }, - { MUA, 0, "([DGH]{0,4}M)+", "GGDGHDGMMHMDHHGHM" }, - { MUA, 0, "([a-c]{4,}s)+", "abasabbasbbaabsbba" }, - { CMUA, 0, "[ace]{3,7}", "AcbDAcEEcEd" }, - { CMUA, 0, "[ace]{3,7}?", "AcbDAcEEcEd" }, - { CMUA, 0, "[ace]{3,}", "AcbDAcEEcEd" }, - { CMUA, 0, "[ace]{3,}?", "AcbDAcEEcEd" }, - { MUA, 0, "[ckl]{2,}?g", "cdkkmlglglkcg" }, - { CMUA, 0, "[ace]{5}?", "AcCebDAcEEcEd" }, - { MUA, 0, "([AbC]{3,5}?d)+", "BACaAbbAEAACCbdCCbdCCAAbb" }, - { MUA, 0, "([^ab]{0,}s){2}", "abaabcdsABamsDDs" }, - { MUA, 0, "\\b\\w+\\B", "x,a_cd" }, - { MUAP, 0, "\\b[^\xc2\xa1]+\\B", "\xc3\x89\xc2\xa1\xe6\x92\xad\xc3\x81\xc3\xa1" }, - { CMUA, 0, "[^b]+(a*)([^c]?d{3})", "aaaaddd" }, - { CMUAP, 0, "\xe1\xbd\xb8{2}", "\xe1\xbf\xb8#\xe1\xbf\xb8\xe1\xbd\xb8" }, - { CMUA, 0, "[^\xf0\x90\x90\x80]{2,4}@", "\xf0\x90\x90\xa8\xf0\x90\x90\x80###\xf0\x90\x90\x80@@@" }, - { CMUA, 0, "[^\xe1\xbd\xb8][^\xc3\xa9]", "\xe1\xbd\xb8\xe1\xbf\xb8\xc3\xa9\xc3\x89#" }, - { MUA, 0, "[^\xe1\xbd\xb8][^\xc3\xa9]", "\xe1\xbd\xb8\xe1\xbf\xb8\xc3\xa9\xc3\x89#" }, - { MUA, 0, "[^\xe1\xbd\xb8]{3,}?", "##\xe1\xbd\xb8#\xe1\xbd\xb8#\xc3\x89#\xe1\xbd\xb8" }, - { MUA, 0, "\\d+123", "987654321,01234" }, - { MUA, 0, "abcd*|\\w+xy", "aaaaa,abxyz" }, - { MUA, 0, "(?:abc|((?:amc|\\b\\w*xy)))", "aaaaa,abxyz" }, - { MUA, 0, "a(?R)|([a-z]++)#", ".abcd.abcd#."}, - { MUA, 0, "a(?R)|([a-z]++)#", ".abcd.mbcd#."}, - { MUA, 0, ".[ab]*.", "xx" }, - { MUA, 0, ".[ab]*a", "xxa" }, - { MUA, 0, ".[ab]?.", "xx" }, - - /* Bracket repeats with limit. */ - { MUA, 0, "(?:(ab){2}){5}M", "abababababababababababM" }, - { MUA, 0, "(?:ab|abab){1,5}M", "abababababababababababM" }, - { MUA, 0, "(?>ab|abab){1,5}M", "abababababababababababM" }, - { MUA, 0, "(?:ab|abab){1,5}?M", "abababababababababababM" }, - { MUA, 0, "(?>ab|abab){1,5}?M", "abababababababababababM" }, - { MUA, 0, "(?:(ab){1,4}?){1,3}?M", "abababababababababababababM" }, - { MUA, 0, "(?:(ab){1,4}){1,3}abababababababababababM", "ababababababababababababM" }, - { MUA, 0 | F_NOMATCH, "(?:(ab){1,4}){1,3}abababababababababababM", "abababababababababababM" }, - { MUA, 0, "(ab){4,6}?M", "abababababababM" }, - - /* Basic character sets. */ - { MUA, 0, "(?:\\s)+(?:\\S)+", "ab \t\xc3\xa9\xe6\x92\xad " }, - { MUA, 0, "(\\w)*(k)(\\W)?\?", "abcdef abck11" }, - { MUA, 0, "\\((\\d)+\\)\\D", "a() (83 (8)2 (9)ab" }, - { MUA, 0, "\\w(\\s|(?:\\d)*,)+\\w\\wb", "a 5, 4,, bb 5, 4,, aab" }, - { MUA, 0, "(\\v+)(\\V+)", "\x0e\xc2\x85\xe2\x80\xa8\x0b\x09\xe2\x80\xa9" }, - { MUA, 0, "(\\h+)(\\H+)", "\xe2\x80\xa8\xe2\x80\x80\x20\xe2\x80\x8a\xe2\x81\x9f\xe3\x80\x80\x09\x20\xc2\xa0\x0a" }, - { MUA, 0, "x[bcef]+", "xaxdxecbfg" }, - { MUA, 0, "x[bcdghij]+", "xaxexfxdgbjk" }, - { MUA, 0, "x[^befg]+", "xbxexacdhg" }, - { MUA, 0, "x[^bcdl]+", "xlxbxaekmd" }, - { MUA, 0, "x[^bcdghi]+", "xbxdxgxaefji" }, - { MUA, 0, "x[B-Fb-f]+", "xaxAxgxbfBFG" }, - { CMUA, 0, "\\x{e9}+", "#\xf0\x90\x90\xa8\xc3\xa8\xc3\xa9\xc3\x89\xc3\x88" }, - { CMUA, 0, "[^\\x{e9}]+", "\xc3\xa9#\xf0\x90\x90\xa8\xc3\xa8\xc3\x88\xc3\x89" }, - { MUA, 0, "[\\x02\\x7e]+", "\xc3\x81\xe1\xbf\xb8\xf0\x90\x90\xa8\x01\x02\x7e\x7f" }, - { MUA, 0, "[^\\x02\\x7e]+", "\x02\xc3\x81\xe1\xbf\xb8\xf0\x90\x90\xa8\x01\x7f\x7e" }, - { MUA, 0, "[\\x{81}-\\x{7fe}]+", "#\xe1\xbf\xb8\xf0\x90\x90\xa8\xc2\x80\xc2\x81\xdf\xbe\xdf\xbf" }, - { MUA, 0, "[^\\x{81}-\\x{7fe}]+", "\xc2\x81#\xe1\xbf\xb8\xf0\x90\x90\xa8\xc2\x80\xdf\xbf\xdf\xbe" }, - { MUA, 0, "[\\x{801}-\\x{fffe}]+", "#\xc3\xa9\xf0\x90\x90\x80\xe0\xa0\x80\xe0\xa0\x81\xef\xbf\xbe\xef\xbf\xbf" }, - { MUA, 0, "[^\\x{801}-\\x{fffe}]+", "\xe0\xa0\x81#\xc3\xa9\xf0\x90\x90\x80\xe0\xa0\x80\xef\xbf\xbf\xef\xbf\xbe" }, - { MUA, 0, "[\\x{10001}-\\x{10fffe}]+", "#\xc3\xa9\xe2\xb1\xa5\xf0\x90\x80\x80\xf0\x90\x80\x81\xf4\x8f\xbf\xbe\xf4\x8f\xbf\xbf" }, - { MUA, 0, "[^\\x{10001}-\\x{10fffe}]+", "\xf0\x90\x80\x81#\xc3\xa9\xe2\xb1\xa5\xf0\x90\x80\x80\xf4\x8f\xbf\xbf\xf4\x8f\xbf\xbe" }, - - /* Unicode properties. */ - { MUAP, 0, "[1-5\xc3\xa9\\w]", "\xc3\xa1_" }, - { MUAP, 0 | F_PROPERTY, "[\xc3\x81\\p{Ll}]", "A_\xc3\x89\xc3\xa1" }, - { MUAP, 0, "[\\Wd-h_x-z]+", "a\xc2\xa1#_yhzdxi" }, - { MUAP, 0 | F_NOMATCH | F_PROPERTY, "[\\P{Any}]", "abc" }, - { MUAP, 0 | F_NOMATCH | F_PROPERTY, "[^\\p{Any}]", "abc" }, - { MUAP, 0 | F_NOMATCH | F_PROPERTY, "[\\P{Any}\xc3\xa1-\xc3\xa8]", "abc" }, - { MUAP, 0 | F_NOMATCH | F_PROPERTY, "[^\\p{Any}\xc3\xa1-\xc3\xa8]", "abc" }, - { MUAP, 0 | F_NOMATCH | F_PROPERTY, "[\xc3\xa1-\xc3\xa8\\P{Any}]", "abc" }, - { MUAP, 0 | F_NOMATCH | F_PROPERTY, "[^\xc3\xa1-\xc3\xa8\\p{Any}]", "abc" }, - { MUAP, 0 | F_PROPERTY, "[\xc3\xa1-\xc3\xa8\\p{Any}]", "abc" }, - { MUAP, 0 | F_PROPERTY, "[^\xc3\xa1-\xc3\xa8\\P{Any}]", "abc" }, - { MUAP, 0, "[b-\xc3\xa9\\s]", "a\xc\xe6\x92\xad" }, - { CMUAP, 0, "[\xc2\x85-\xc2\x89\xc3\x89]", "\xc2\x84\xc3\xa9" }, - { MUAP, 0, "[^b-d^&\\s]{3,}", "db^ !a\xe2\x80\xa8_ae" }, - { MUAP, 0 | F_PROPERTY, "[^\\S\\P{Any}][\\sN]{1,3}[\\P{N}]{4}", "\xe2\x80\xaa\xa N\x9\xc3\xa9_0" }, - { MUA, 0 | F_PROPERTY, "[^\\P{L}\x9!D-F\xa]{2,3}", "\x9,.DF\xa.CG\xc3\x81" }, - { CMUAP, 0, "[\xc3\xa1-\xc3\xa9_\xe2\x80\xa0-\xe2\x80\xaf]{1,5}[^\xe2\x80\xa0-\xe2\x80\xaf]", "\xc2\xa1\xc3\x89\xc3\x89\xe2\x80\xaf_\xe2\x80\xa0" }, - { MUAP, 0 | F_PROPERTY, "[\xc3\xa2-\xc3\xa6\xc3\x81-\xc3\x84\xe2\x80\xa8-\xe2\x80\xa9\xe6\x92\xad\\p{Zs}]{2,}", "\xe2\x80\xa7\xe2\x80\xa9\xe6\x92\xad \xe6\x92\xae" }, - { MUAP, 0 | F_PROPERTY, "[\\P{L&}]{2}[^\xc2\x85-\xc2\x89\\p{Ll}\\p{Lu}]{2}", "\xc3\xa9\xe6\x92\xad.a\xe6\x92\xad|\xc2\x8a#" }, - { PCRE_UCP, 0, "[a-b\\s]{2,5}[^a]", "AB baaa" }, - - /* Possible empty brackets. */ - { MUA, 0, "(?:|ab||bc|a)+d", "abcxabcabd" }, - { MUA, 0, "(|ab||bc|a)+d", "abcxabcabd" }, - { MUA, 0, "(?:|ab||bc|a)*d", "abcxabcabd" }, - { MUA, 0, "(|ab||bc|a)*d", "abcxabcabd" }, - { MUA, 0, "(?:|ab||bc|a)+?d", "abcxabcabd" }, - { MUA, 0, "(|ab||bc|a)+?d", "abcxabcabd" }, - { MUA, 0, "(?:|ab||bc|a)*?d", "abcxabcabd" }, - { MUA, 0, "(|ab||bc|a)*?d", "abcxabcabd" }, - { MUA, 0, "(((a)*?|(?:ba)+)+?|(?:|c|ca)*)*m", "abaacaccabacabalabaacaccabacabamm" }, - { MUA, 0, "(?:((?:a)*|(ba)+?)+|(|c|ca)*?)*?m", "abaacaccabacabalabaacaccabacabamm" }, - - /* Start offset. */ - { MUA, 3, "(\\d|(?:\\w)*\\w)+", "0ac01Hb" }, - { MUA, 4 | F_NOMATCH, "(\\w\\W\\w)+", "ab#d" }, - { MUA, 2 | F_NOMATCH, "(\\w\\W\\w)+", "ab#d" }, - { MUA, 1, "(\\w\\W\\w)+", "ab#d" }, - - /* Newline. */ - { PCRE_MULTILINE | PCRE_NEWLINE_CRLF, 0, "\\W{0,2}[^#]{3}", "\r\n#....." }, - { PCRE_MULTILINE | PCRE_NEWLINE_CR, 0, "\\W{0,2}[^#]{3}", "\r\n#....." }, - { PCRE_MULTILINE | PCRE_NEWLINE_CRLF, 0, "\\W{1,3}[^#]", "\r\n##...." }, - { MUA | PCRE_NO_UTF8_CHECK, 1, "^.a", "\n\x80\nxa" }, - { MUA, 1, "^", "\r\n" }, - { PCRE_MULTILINE | PCRE_NEWLINE_CRLF, 1 | F_NOMATCH, "^", "\r\n" }, - { PCRE_MULTILINE | PCRE_NEWLINE_CRLF, 1, "^", "\r\na" }, - - /* Any character except newline or any newline. */ - { PCRE_NEWLINE_CRLF, 0, ".", "\r" }, - { PCRE_NEWLINE_CRLF | PCRE_UTF8, 0, ".(.).", "a\xc3\xa1\r\n\n\r\r" }, - { PCRE_NEWLINE_ANYCRLF, 0, ".(.)", "a\rb\nc\r\n\xc2\x85\xe2\x80\xa8" }, - { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".(.)", "a\rb\nc\r\n\xc2\x85\xe2\x80\xa8" }, - { PCRE_NEWLINE_ANY | PCRE_UTF8, 0, "(.).", "a\rb\nc\r\n\xc2\x85\xe2\x80\xa9$de" }, - { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0 | F_NOMATCH, ".(.).", "\xe2\x80\xa8\nb\r" }, - { PCRE_NEWLINE_ANY, 0, "(.)(.)", "#\x85#\r#\n#\r\n#\x84" }, - { PCRE_NEWLINE_ANY | PCRE_UTF8, 0, "(.+)#", "#\rMn\xc2\x85#\n###" }, - { PCRE_BSR_ANYCRLF, 0, "\\R", "\r" }, - { PCRE_BSR_ANYCRLF, 0, "\\R", "\x85#\r\n#" }, - { PCRE_BSR_UNICODE | PCRE_UTF8, 0, "\\R", "ab\xe2\x80\xa8#c" }, - { PCRE_BSR_UNICODE | PCRE_UTF8, 0, "\\R", "ab\r\nc" }, - { PCRE_NEWLINE_CRLF | PCRE_BSR_UNICODE | PCRE_UTF8, 0, "(\\R.)+", "\xc2\x85\r\n#\xe2\x80\xa8\n\r\n\r" }, - { MUA, 0 | F_NOMATCH, "\\R+", "ab" }, - { MUA, 0, "\\R+", "ab\r\n\r" }, - { MUA, 0, "\\R*", "ab\r\n\r" }, - { MUA, 0, "\\R*", "\r\n\r" }, - { MUA, 0, "\\R{2,4}", "\r\nab\r\r" }, - { MUA, 0, "\\R{2,4}", "\r\nab\n\n\n\r\r\r" }, - { MUA, 0, "\\R{2,}", "\r\nab\n\n\n\r\r\r" }, - { MUA, 0, "\\R{0,3}", "\r\n\r\n\r\n\r\n\r\n" }, - { MUA, 0 | F_NOMATCH, "\\R+\\R\\R", "\r\n\r\n" }, - { MUA, 0, "\\R+\\R\\R", "\r\r\r" }, - { MUA, 0, "\\R*\\R\\R", "\n\r" }, - { MUA, 0 | F_NOMATCH, "\\R{2,4}\\R\\R", "\r\r\r" }, - { MUA, 0, "\\R{2,4}\\R\\R", "\r\r\r\r" }, - - /* Atomic groups (no fallback from "next" direction). */ - { MUA, 0 | F_NOMATCH, "(?>ab)ab", "bab" }, - { MUA, 0 | F_NOMATCH, "(?>(ab))ab", "bab" }, - { MUA, 0, "(?>ab)+abc(?>de)*def(?>gh)?ghe(?>ij)+?k(?>lm)*?n(?>op)?\?op", - "bababcdedefgheijijklmlmnop" }, - { MUA, 0, "(?>a(b)+a|(ab)?\?(b))an", "abban" }, - { MUA, 0, "(?>ab+a|(?:ab)?\?b)an", "abban" }, - { MUA, 0, "((?>ab|ad|)*?)(?>|c)*abad", "abababcababad" }, - { MUA, 0, "(?>(aa|b|)*+(?>(##)|###)*d|(aa)(?>(baa)?)m)", "aabaa#####da" }, - { MUA, 0, "((?>a|)+?)b", "aaacaaab" }, - { MUA, 0, "(?>x|)*$", "aaa" }, - { MUA, 0, "(?>(x)|)*$", "aaa" }, - { MUA, 0, "(?>x|())*$", "aaa" }, - { MUA, 0, "((?>[cxy]a|[a-d])*?)b", "aaa+ aaab" }, - { MUA, 0, "((?>[cxy](a)|[a-d])*?)b", "aaa+ aaab" }, - { MUA, 0, "(?>((?>(a+))))bab|(?>((?>(a+))))bb", "aaaabaaabaabab" }, - { MUA, 0, "(?>(?>a+))bab|(?>(?>a+))bb", "aaaabaaabaabab" }, - { MUA, 0, "(?>(a)c|(?>(c)|(a))a)b*?bab", "aaaabaaabaabab" }, - { MUA, 0, "(?>ac|(?>c|a)a)b*?bab", "aaaabaaabaabab" }, - { MUA, 0, "(?>(b)b|(a))*b(?>(c)|d)?x", "ababcaaabdbx" }, - { MUA, 0, "(?>bb|a)*b(?>c|d)?x", "ababcaaabdbx" }, - { MUA, 0, "(?>(bb)|a)*b(?>c|(d))?x", "ababcaaabdbx" }, - { MUA, 0, "(?>(a))*?(?>(a))+?(?>(a))??x", "aaaaaacccaaaaabax" }, - { MUA, 0, "(?>a)*?(?>a)+?(?>a)??x", "aaaaaacccaaaaabax" }, - { MUA, 0, "(?>(a)|)*?(?>(a)|)+?(?>(a)|)??x", "aaaaaacccaaaaabax" }, - { MUA, 0, "(?>a|)*?(?>a|)+?(?>a|)??x", "aaaaaacccaaaaabax" }, - { MUA, 0, "(?>a(?>(a{0,2}))*?b|aac)+b", "aaaaaaacaaaabaaaaacaaaabaacaaabb" }, - { CMA, 0, "(?>((?>a{32}|b+|(a*))?(?>c+|d*)?\?)+e)+?f", "aaccebbdde bbdaaaccebbdee bbdaaaccebbdeef" }, - { MUA, 0, "(?>(?:(?>aa|a||x)+?b|(?>aa|a||(x))+?c)?(?>[ad]{0,2})*?d)+d", "aaacdbaabdcabdbaaacd aacaabdbdcdcaaaadaabcbaadd" }, - { MUA, 0, "(?>(?:(?>aa|a||(x))+?b|(?>aa|a||x)+?c)?(?>[ad]{0,2})*?d)+d", "aaacdbaabdcabdbaaacd aacaabdbdcdcaaaadaabcbaadd" }, - { MUA, 0 | F_PROPERTY, "\\X", "\xcc\x8d\xcc\x8d" }, - { MUA, 0 | F_PROPERTY, "\\X", "\xcc\x8d\xcc\x8d#\xcc\x8d\xcc\x8d" }, - { MUA, 0 | F_PROPERTY, "\\X+..", "\xcc\x8d#\xcc\x8d#\xcc\x8d\xcc\x8d" }, - { MUA, 0 | F_PROPERTY, "\\X{2,4}", "abcdef" }, - { MUA, 0 | F_PROPERTY, "\\X{2,4}?", "abcdef" }, - { MUA, 0 | F_NOMATCH | F_PROPERTY, "\\X{2,4}..", "#\xcc\x8d##" }, - { MUA, 0 | F_PROPERTY, "\\X{2,4}..", "#\xcc\x8d#\xcc\x8d##" }, - { MUA, 0, "(c(ab)?+ab)+", "cabcababcab" }, - { MUA, 0, "(?>(a+)b)+aabab", "aaaabaaabaabab" }, - - /* Possessive quantifiers. */ - { MUA, 0, "(?:a|b)++m", "mababbaaxababbaam" }, - { MUA, 0, "(?:a|b)*+m", "mababbaaxababbaam" }, - { MUA, 0, "(?:a|b)*+m", "ababbaaxababbaam" }, - { MUA, 0, "(a|b)++m", "mababbaaxababbaam" }, - { MUA, 0, "(a|b)*+m", "mababbaaxababbaam" }, - { MUA, 0, "(a|b)*+m", "ababbaaxababbaam" }, - { MUA, 0, "(a|b(*ACCEPT))++m", "maaxab" }, - { MUA, 0, "(?:b*)++m", "bxbbxbbbxm" }, - { MUA, 0, "(?:b*)++m", "bxbbxbbbxbbm" }, - { MUA, 0, "(?:b*)*+m", "bxbbxbbbxm" }, - { MUA, 0, "(?:b*)*+m", "bxbbxbbbxbbm" }, - { MUA, 0, "(b*)++m", "bxbbxbbbxm" }, - { MUA, 0, "(b*)++m", "bxbbxbbbxbbm" }, - { MUA, 0, "(b*)*+m", "bxbbxbbbxm" }, - { MUA, 0, "(b*)*+m", "bxbbxbbbxbbm" }, - { MUA, 0, "(?:a|(b))++m", "mababbaaxababbaam" }, - { MUA, 0, "(?:(a)|b)*+m", "mababbaaxababbaam" }, - { MUA, 0, "(?:(a)|(b))*+m", "ababbaaxababbaam" }, - { MUA, 0, "(a|(b))++m", "mababbaaxababbaam" }, - { MUA, 0, "((a)|b)*+m", "mababbaaxababbaam" }, - { MUA, 0, "((a)|(b))*+m", "ababbaaxababbaam" }, - { MUA, 0, "(a|(b)(*ACCEPT))++m", "maaxab" }, - { MUA, 0, "(?:(b*))++m", "bxbbxbbbxm" }, - { MUA, 0, "(?:(b*))++m", "bxbbxbbbxbbm" }, - { MUA, 0, "(?:(b*))*+m", "bxbbxbbbxm" }, - { MUA, 0, "(?:(b*))*+m", "bxbbxbbbxbbm" }, - { MUA, 0, "((b*))++m", "bxbbxbbbxm" }, - { MUA, 0, "((b*))++m", "bxbbxbbbxbbm" }, - { MUA, 0, "((b*))*+m", "bxbbxbbbxm" }, - { MUA, 0, "((b*))*+m", "bxbbxbbbxbbm" }, - { MUA, 0 | F_NOMATCH, "(?>(b{2,4}))(?:(?:(aa|c))++m|(?:(aa|c))+n)", "bbaacaaccaaaacxbbbmbn" }, - { MUA, 0, "((?:b)++a)+(cd)*+m", "bbababbacdcdnbbababbacdcdm" }, - { MUA, 0, "((?:(b))++a)+((c)d)*+m", "bbababbacdcdnbbababbacdcdm" }, - { MUA, 0, "(?:(?:(?:ab)*+k)++(?:n(?:cd)++)*+)*+m", "ababkkXababkkabkncXababkkabkncdcdncdXababkkabkncdcdncdkkabkncdXababkkabkncdcdncdkkabkncdm" }, - { MUA, 0, "(?:((ab)*+(k))++(n(?:c(d))++)*+)*+m", "ababkkXababkkabkncXababkkabkncdcdncdXababkkabkncdcdncdkkabkncdXababkkabkncdcdncdkkabkncdm" }, - - /* Back references. */ - { MUA, 0, "(aa|bb)(\\1*)(ll|)(\\3*)bbbbbbc", "aaaaaabbbbbbbbc" }, - { CMUA, 0, "(aa|bb)(\\1+)(ll|)(\\3+)bbbbbbc", "bBbbBbCbBbbbBbbcbbBbbbBBbbC" }, - { CMA, 0, "(a{2,4})\\1", "AaAaaAaA" }, - { MUA, 0, "(aa|bb)(\\1?)aa(\\1?)(ll|)(\\4+)bbc", "aaaaaaaabbaabbbbaabbbbc" }, - { MUA, 0, "(aa|bb)(\\1{0,5})(ll|)(\\3{0,5})cc", "bbxxbbbbxxaaaaaaaaaaaaaaaacc" }, - { MUA, 0, "(aa|bb)(\\1{3,5})(ll|)(\\3{3,5})cc", "bbbbbbbbbbbbaaaaaaccbbbbbbbbbbbbbbcc" }, - { MUA, 0, "(aa|bb)(\\1{3,})(ll|)(\\3{3,})cc", "bbbbbbbbbbbbaaaaaaccbbbbbbbbbbbbbbcc" }, - { MUA, 0, "(\\w+)b(\\1+)c", "GabGaGaDbGaDGaDc" }, - { MUA, 0, "(?:(aa)|b)\\1?b", "bb" }, - { CMUA, 0, "(aa|bb)(\\1*?)aa(\\1+?)", "bBBbaaAAaaAAaa" }, - { MUA, 0, "(aa|bb)(\\1*?)(dd|)cc(\\3+?)", "aaaaaccdd" }, - { CMUA, 0, "(?:(aa|bb)(\\1?\?)cc){2}(\\1?\?)", "aAaABBbbAAaAcCaAcCaA" }, - { MUA, 0, "(?:(aa|bb)(\\1{3,5}?)){2}(dd|)(\\3{3,5}?)", "aaaaaabbbbbbbbbbaaaaaaaaaaaaaa" }, - { CMA, 0, "(?:(aa|bb)(\\1{3,}?)){2}(dd|)(\\3{3,}?)", "aaaaaabbbbbbbbbbaaaaaaaaaaaaaa" }, - { MUA, 0, "(?:(aa|bb)(\\1{0,3}?)){2}(dd|)(\\3{0,3}?)b(\\1{0,3}?)(\\1{0,3})", "aaaaaaaaaaaaaaabaaaaa" }, - { MUA, 0, "(a(?:\\1|)a){3}b", "aaaaaaaaaaab" }, - { MA, 0, "(a?)b(\\1\\1*\\1+\\1?\\1*?\\1+?\\1??\\1*+\\1++\\1?+\\1{4}\\1{3,5}\\1{4,}\\1{0,5}\\1{3,5}?\\1{4,}?\\1{0,5}?\\1{3,5}+\\1{4,}+\\1{0,5}+#){2}d", "bb#b##d" }, - { MUAP, 0 | F_PROPERTY, "(\\P{N})\\1{2,}", ".www." }, - { MUAP, 0 | F_PROPERTY, "(\\P{N})\\1{0,2}", "wwwww." }, - { MUAP, 0 | F_PROPERTY, "(\\P{N})\\1{1,2}ww", "wwww" }, - { MUAP, 0 | F_PROPERTY, "(\\P{N})\\1{1,2}ww", "wwwww" }, - { PCRE_UCP, 0 | F_PROPERTY, "(\\P{N})\\1{2,}", ".www." }, - { CMUAP, 0, "(\xf0\x90\x90\x80)\\1", "\xf0\x90\x90\xa8\xf0\x90\x90\xa8" }, - { MUA | PCRE_DUPNAMES, 0 | F_NOMATCH, "\\k{1,3}(?aa)(?bb)", "aabb" }, - { MUA | PCRE_DUPNAMES | PCRE_JAVASCRIPT_COMPAT, 0, "\\k{1,3}(?aa)(?bb)", "aabb" }, - { MUA | PCRE_DUPNAMES | PCRE_JAVASCRIPT_COMPAT, 0, "\\k*(?aa)(?bb)", "aabb" }, - { MUA | PCRE_DUPNAMES, 0, "(?aa)(?bb)\\k{0,3}aaaaaa", "aabbaaaaaa" }, - { MUA | PCRE_DUPNAMES, 0, "(?aa)(?bb)\\k{2,5}bb", "aabbaaaabb" }, - { MUA | PCRE_DUPNAMES, 0, "(?:(?aa)|(?bb))\\k{0,3}m", "aaaaaaaabbbbaabbbbm" }, - { MUA | PCRE_DUPNAMES, 0 | F_NOMATCH, "\\k{1,3}?(?aa)(?bb)", "aabb" }, - { MUA | PCRE_DUPNAMES | PCRE_JAVASCRIPT_COMPAT, 0, "\\k{1,3}?(?aa)(?bb)", "aabb" }, - { MUA | PCRE_DUPNAMES, 0, "\\k*?(?aa)(?bb)", "aabb" }, - { MUA | PCRE_DUPNAMES, 0, "(?:(?aa)|(?bb))\\k{0,3}?m", "aaaaaabbbbbbaabbbbbbbbbbm" }, - { MUA | PCRE_DUPNAMES, 0, "(?:(?aa)|(?bb))\\k*?m", "aaaaaabbbbbbaabbbbbbbbbbm" }, - { MUA | PCRE_DUPNAMES, 0, "(?:(?aa)|(?bb))\\k{2,3}?", "aaaabbbbaaaabbbbbbbbbb" }, - { CMUA | PCRE_DUPNAMES, 0, "(?:(?AA)|(?BB))\\k{0,3}M", "aaaaaaaabbbbaabbbbm" }, - { CMUA | PCRE_DUPNAMES, 0, "(?:(?AA)|(?BB))\\k{1,3}M", "aaaaaaaabbbbaabbbbm" }, - { CMUA | PCRE_DUPNAMES, 0, "(?:(?AA)|(?BB))\\k{0,3}?M", "aaaaaabbbbbbaabbbbbbbbbbm" }, - { CMUA | PCRE_DUPNAMES, 0, "(?:(?AA)|(?BB))\\k{2,3}?", "aaaabbbbaaaabbbbbbbbbb" }, - - /* Assertions. */ - { MUA, 0, "(?=xx|yy|zz)\\w{4}", "abczzdefg" }, - { MUA, 0, "(?=((\\w+)b){3}|ab)", "dbbbb ab" }, - { MUA, 0, "(?!ab|bc|cd)[a-z]{2}", "Xabcdef" }, - { MUA, 0, "(?<=aaa|aa|a)a", "aaa" }, - { MUA, 2, "(?<=aaa|aa|a)a", "aaa" }, - { MA, 0, "(?<=aaa|aa|a)a", "aaa" }, - { MA, 2, "(?<=aaa|aa|a)a", "aaa" }, - { MUA, 0, "(\\d{2})(?!\\w+c|(((\\w?)m){2}n)+|\\1)", "x5656" }, - { MUA, 0, "((?=((\\d{2,6}\\w){2,}))\\w{5,20}K){2,}", "567v09708K12l00M00 567v09708K12l00M00K45K" }, - { MUA, 0, "(?=(?:(?=\\S+a)\\w*(b)){3})\\w+\\d", "bba bbab nbbkba nbbkba0kl" }, - { MUA, 0, "(?>a(?>(b+))a(?=(..)))*?k", "acabbcabbaabacabaabbakk" }, - { MUA, 0, "((?(?=(a))a)+k)", "bbak" }, - { MUA, 0, "((?(?=a)a)+k)", "bbak" }, - { MUA, 0 | F_NOMATCH, "(?=(?>(a))m)amk", "a k" }, - { MUA, 0 | F_NOMATCH, "(?!(?>(a))m)amk", "a k" }, - { MUA, 0 | F_NOMATCH, "(?>(?=(a))am)amk", "a k" }, - { MUA, 0, "(?=(?>a|(?=(?>(b+))a|c)[a-c]+)*?m)[a-cm]+k", "aaam bbam baaambaam abbabba baaambaamk" }, - { MUA, 0, "(?> ?\?\\b(?(?=\\w{1,4}(a))m)\\w{0,8}bc){2,}?", "bca ssbc mabd ssbc mabc" }, - { MUA, 0, "(?:(?=ab)?[^n][^n])+m", "ababcdabcdcdabnababcdabcdcdabm" }, - { MUA, 0, "(?:(?=a(b))?[^n][^n])+m", "ababcdabcdcdabnababcdabcdcdabm" }, - { MUA, 0, "(?:(?=.(.))??\\1.)+m", "aabbbcbacccanaabbbcbacccam" }, - { MUA, 0, "(?:(?=.)??[a-c])+m", "abacdcbacacdcaccam" }, - { MUA, 0, "((?!a)?(?!([^a]))?)+$", "acbab" }, - { MUA, 0, "((?!a)?\?(?!([^a]))?\?)+$", "acbab" }, - { MUA, 0, "a(?=(?C)\\B)b", "ab" }, - { MUA, 0, "a(?!(?C)\\B)bb|ab", "abb" }, - { MUA, 0, "a(?=\\b|(?C)\\B)b", "ab" }, - { MUA, 0, "a(?!\\b|(?C)\\B)bb|ab", "abb" }, - { MUA, 0, "c(?(?=(?C)\\B)ab|a)", "cab" }, - { MUA, 0, "c(?(?!(?C)\\B)ab|a)", "cab" }, - { MUA, 0, "c(?(?=\\b|(?C)\\B)ab|a)", "cab" }, - { MUA, 0, "c(?(?!\\b|(?C)\\B)ab|a)", "cab" }, - { MUA, 0, "a(?=)b", "ab" }, - { MUA, 0 | F_NOMATCH, "a(?!)b", "ab" }, - - /* Not empty, ACCEPT, FAIL */ - { MUA | PCRE_NOTEMPTY, 0 | F_NOMATCH, "a*", "bcx" }, - { MUA | PCRE_NOTEMPTY, 0, "a*", "bcaad" }, - { MUA | PCRE_NOTEMPTY, 0, "a*?", "bcaad" }, - { MUA | PCRE_NOTEMPTY_ATSTART, 0, "a*", "bcaad" }, - { MUA, 0, "a(*ACCEPT)b", "ab" }, - { MUA | PCRE_NOTEMPTY, 0 | F_NOMATCH, "a*(*ACCEPT)b", "bcx" }, - { MUA | PCRE_NOTEMPTY, 0, "a*(*ACCEPT)b", "bcaad" }, - { MUA | PCRE_NOTEMPTY, 0, "a*?(*ACCEPT)b", "bcaad" }, - { MUA | PCRE_NOTEMPTY, 0 | F_NOMATCH, "(?:z|a*(*ACCEPT)b)", "bcx" }, - { MUA | PCRE_NOTEMPTY, 0, "(?:z|a*(*ACCEPT)b)", "bcaad" }, - { MUA | PCRE_NOTEMPTY, 0, "(?:z|a*?(*ACCEPT)b)", "bcaad" }, - { MUA | PCRE_NOTEMPTY_ATSTART, 0, "a*(*ACCEPT)b", "bcx" }, - { MUA | PCRE_NOTEMPTY_ATSTART, 0 | F_NOMATCH, "a*(*ACCEPT)b", "" }, - { MUA, 0, "((a(*ACCEPT)b))", "ab" }, - { MUA, 0, "(a(*FAIL)a|a)", "aaa" }, - { MUA, 0, "(?=ab(*ACCEPT)b)a", "ab" }, - { MUA, 0, "(?=(?:x|ab(*ACCEPT)b))", "ab" }, - { MUA, 0, "(?=(a(b(*ACCEPT)b)))a", "ab" }, - { MUA | PCRE_NOTEMPTY, 0, "(?=a*(*ACCEPT))c", "c" }, - - /* Conditional blocks. */ - { MUA, 0, "(?(?=(a))a|b)+k", "ababbalbbadabak" }, - { MUA, 0, "(?(?!(b))a|b)+k", "ababbalbbadabak" }, - { MUA, 0, "(?(?=a)a|b)+k", "ababbalbbadabak" }, - { MUA, 0, "(?(?!b)a|b)+k", "ababbalbbadabak" }, - { MUA, 0, "(?(?=(a))a*|b*)+k", "ababbalbbadabak" }, - { MUA, 0, "(?(?!(b))a*|b*)+k", "ababbalbbadabak" }, - { MUA, 0, "(?(?!(b))(?:aaaaaa|a)|(?:bbbbbb|b))+aaaak", "aaaaaaaaaaaaaa bbbbbbbbbbbbbbb aaaaaaak" }, - { MUA, 0, "(?(?!b)(?:aaaaaa|a)|(?:bbbbbb|b))+aaaak", "aaaaaaaaaaaaaa bbbbbbbbbbbbbbb aaaaaaak" }, - { MUA, 0 | F_DIFF, "(?(?!(b))(?:aaaaaa|a)|(?:bbbbbb|b))+bbbbk", "aaaaaaaaaaaaaa bbbbbbbbbbbbbbb bbbbbbbk" }, - { MUA, 0, "(?(?!b)(?:aaaaaa|a)|(?:bbbbbb|b))+bbbbk", "aaaaaaaaaaaaaa bbbbbbbbbbbbbbb bbbbbbbk" }, - { MUA, 0, "(?(?=a)a*|b*)+k", "ababbalbbadabak" }, - { MUA, 0, "(?(?!b)a*|b*)+k", "ababbalbbadabak" }, - { MUA, 0, "(?(?=a)ab)", "a" }, - { MUA, 0, "(?(?a)?(?Pb)?(?(Name)c|d)*l", "bc ddd abccabccl" }, - { MUA, 0, "(?Pa)?(?Pb)?(?(Name)c|d)+?dd", "bcabcacdb bdddd" }, - { MUA, 0, "(?Pa)?(?Pb)?(?(Name)c|d)+l", "ababccddabdbccd abcccl" }, - { MUA, 0, "((?:a|aa)(?(1)aaa))x", "aax" }, - { MUA, 0, "(?(?!)a|b)", "ab" }, - { MUA, 0, "(?(?!)a)", "ab" }, - { MUA, 0 | F_NOMATCH, "(?(?!)a|b)", "ac" }, - - /* Set start of match. */ - { MUA, 0, "(?:\\Ka)*aaaab", "aaaaaaaa aaaaaaabb" }, - { MUA, 0, "(?>\\Ka\\Ka)*aaaab", "aaaaaaaa aaaaaaaaaabb" }, - { MUA, 0, "a+\\K(?<=\\Gaa)a", "aaaaaa" }, - { MUA | PCRE_NOTEMPTY, 0 | F_NOMATCH, "a\\K(*ACCEPT)b", "aa" }, - { MUA | PCRE_NOTEMPTY_ATSTART, 0, "a\\K(*ACCEPT)b", "aa" }, - - /* First line. */ - { MUA | PCRE_FIRSTLINE, 0 | F_PROPERTY, "\\p{Any}a", "bb\naaa" }, - { MUA | PCRE_FIRSTLINE, 0 | F_NOMATCH | F_PROPERTY, "\\p{Any}a", "bb\r\naaa" }, - { MUA | PCRE_FIRSTLINE, 0, "(?<=a)", "a" }, - { MUA | PCRE_FIRSTLINE, 0 | F_NOMATCH, "[^a][^b]", "ab" }, - { MUA | PCRE_FIRSTLINE, 0 | F_NOMATCH, "a", "\na" }, - { MUA | PCRE_FIRSTLINE, 0 | F_NOMATCH, "[abc]", "\na" }, - { MUA | PCRE_FIRSTLINE, 0 | F_NOMATCH, "^a", "\na" }, - { MUA | PCRE_FIRSTLINE, 0 | F_NOMATCH, "^(?<=\n)", "\na" }, - { MUA | PCRE_FIRSTLINE, 0, "\xf0\x90\x90\x80", "\xf0\x90\x90\x80" }, - { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANY | PCRE_FIRSTLINE, 0 | F_NOMATCH, "#", "\xc2\x85#" }, - { PCRE_MULTILINE | PCRE_NEWLINE_ANY | PCRE_FIRSTLINE, 0 | F_NOMATCH, "#", "\x85#" }, - { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANY | PCRE_FIRSTLINE, 0 | F_NOMATCH, "^#", "\xe2\x80\xa8#" }, - { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_CRLF | PCRE_FIRSTLINE, 0 | F_PROPERTY, "\\p{Any}", "\r\na" }, - { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_CRLF | PCRE_FIRSTLINE, 0, ".", "\r" }, - { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_CRLF | PCRE_FIRSTLINE, 0, "a", "\ra" }, - { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_CRLF | PCRE_FIRSTLINE, 0 | F_NOMATCH, "ba", "bbb\r\nba" }, - { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_CRLF | PCRE_FIRSTLINE, 0 | F_NOMATCH | F_PROPERTY, "\\p{Any}{4}|a", "\r\na" }, - { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_CRLF | PCRE_FIRSTLINE, 1, ".", "\r\n" }, - { PCRE_FIRSTLINE | PCRE_NEWLINE_LF | PCRE_DOTALL, 0 | F_NOMATCH, "ab.", "ab" }, - { MUA | PCRE_FIRSTLINE, 1 | F_NOMATCH, "^[a-d0-9]", "\nxx\nd" }, - { PCRE_NEWLINE_ANY | PCRE_FIRSTLINE | PCRE_DOTALL, 0, "....a", "012\n0a" }, - - /* Recurse. */ - { MUA, 0, "(a)(?1)", "aa" }, - { MUA, 0, "((a))(?1)", "aa" }, - { MUA, 0, "(b|a)(?1)", "aa" }, - { MUA, 0, "(b|(a))(?1)", "aa" }, - { MUA, 0 | F_NOMATCH, "((a)(b)(?:a*))(?1)", "aba" }, - { MUA, 0, "((a)(b)(?:a*))(?1)", "abab" }, - { MUA, 0, "((a+)c(?2))b(?1)", "aacaabaca" }, - { MUA, 0, "((?2)b|(a)){2}(?1)", "aabab" }, - { MUA, 0, "(?1)(a)*+(?2)(b(?1))", "aababa" }, - { MUA, 0, "(?1)(((a(*ACCEPT)))b)", "axaa" }, - { MUA, 0, "(?1)(?(DEFINE) (((ac(*ACCEPT)))b) )", "akaac" }, - { MUA, 0, "(a+)b(?1)b\\1", "abaaabaaaaa" }, - { MUA, 0 | F_NOMATCH, "(?(DEFINE)(aa|a))(?1)ab", "aab" }, - { MUA, 0, "(?(DEFINE)(a\\Kb))(?1)+ababc", "abababxabababc" }, - { MUA, 0, "(a\\Kb)(?1)+ababc", "abababxababababc" }, - { MUA, 0 | F_NOMATCH, "(a\\Kb)(?1)+ababc", "abababxababababxc" }, - { MUA, 0, "b|<(?R)*>", "<" }, - { MUA, 0, "(a\\K){0}(?:(?1)b|ac)", "ac" }, - { MUA, 0, "(?(DEFINE)(a(?2)|b)(b(?1)|(a)))(?:(?1)|(?2))m", "ababababnababababaam" }, - { MUA, 0, "(a)((?(R)a|b))(?2)", "aabbabaa" }, - { MUA, 0, "(a)((?(R2)a|b))(?2)", "aabbabaa" }, - { MUA, 0, "(a)((?(R1)a|b))(?2)", "ababba" }, - { MUA, 0, "(?(R0)aa|bb(?R))", "abba aabb bbaa" }, - { MUA, 0, "((?(R)(?:aaaa|a)|(?:(aaaa)|(a)))+)(?1)$", "aaaaaaaaaa aaaa" }, - { MUA, 0, "(?Pa(?(R&Name)a|b))(?1)", "aab abb abaa" }, - { MUA, 0, "((?(R)a|(?1)){3})", "XaaaaaaaaaX" }, - { MUA, 0, "((?:(?(R)a|(?1))){3})", "XaaaaaaaaaX" }, - { MUA, 0, "((?(R)a|(?1)){1,3})aaaaaa", "aaaaaaaaXaaaaaaaaa" }, - { MUA, 0, "((?(R)a|(?1)){1,3}?)M", "aaaM" }, - - /* 16 bit specific tests. */ - { CMA, 0 | F_FORCECONV, "\xc3\xa1", "\xc3\x81\xc3\xa1" }, - { CMA, 0 | F_FORCECONV, "\xe1\xbd\xb8", "\xe1\xbf\xb8\xe1\xbd\xb8" }, - { CMA, 0 | F_FORCECONV, "[\xc3\xa1]", "\xc3\x81\xc3\xa1" }, - { CMA, 0 | F_FORCECONV, "[\xe1\xbd\xb8]", "\xe1\xbf\xb8\xe1\xbd\xb8" }, - { CMA, 0 | F_FORCECONV, "[a-\xed\xb0\x80]", "A" }, - { CMA, 0 | F_NO8 | F_FORCECONV, "[a-\\x{dc00}]", "B" }, - { CMA, 0 | F_NO8 | F_NOMATCH | F_FORCECONV, "[b-\\x{dc00}]", "a" }, - { CMA, 0 | F_NO8 | F_FORCECONV, "\xed\xa0\x80\\x{d800}\xed\xb0\x80\\x{dc00}", "\xed\xa0\x80\xed\xa0\x80\xed\xb0\x80\xed\xb0\x80" }, - { CMA, 0 | F_NO8 | F_FORCECONV, "[\xed\xa0\x80\\x{d800}]{1,2}?[\xed\xb0\x80\\x{dc00}]{1,2}?#", "\xed\xa0\x80\xed\xa0\x80\xed\xb0\x80\xed\xb0\x80#" }, - { CMA, 0 | F_FORCECONV, "[\xed\xa0\x80\xed\xb0\x80#]{0,3}(?<=\xed\xb0\x80.)", "\xed\xa0\x80#\xed\xa0\x80##\xed\xb0\x80\xed\xa0\x80" }, - { CMA, 0 | F_FORCECONV, "[\xed\xa0\x80-\xed\xb3\xbf]", "\xed\x9f\xbf\xed\xa0\x83" }, - { CMA, 0 | F_FORCECONV, "[\xed\xa0\x80-\xed\xb3\xbf]", "\xed\xb4\x80\xed\xb3\xb0" }, - { CMA, 0 | F_NO8 | F_FORCECONV, "[\\x{d800}-\\x{dcff}]", "\xed\x9f\xbf\xed\xa0\x83" }, - { CMA, 0 | F_NO8 | F_FORCECONV, "[\\x{d800}-\\x{dcff}]", "\xed\xb4\x80\xed\xb3\xb0" }, - { CMA, 0 | F_FORCECONV, "[\xed\xa0\x80-\xef\xbf\xbf]+[\x1-\xed\xb0\x80]+#", "\xed\xa0\x85\xc3\x81\xed\xa0\x85\xef\xbf\xb0\xc2\x85\xed\xa9\x89#" }, - { CMA, 0 | F_FORCECONV, "[\xed\xa0\x80][\xed\xb0\x80]{2,}", "\xed\xa0\x80\xed\xb0\x80\xed\xa0\x80\xed\xb0\x80\xed\xb0\x80\xed\xb0\x80" }, - { MA, 0 | F_FORCECONV, "[^\xed\xb0\x80]{3,}?", "##\xed\xb0\x80#\xed\xb0\x80#\xc3\x89#\xed\xb0\x80" }, - { MA, 0 | F_NO8 | F_FORCECONV, "[^\\x{dc00}]{3,}?", "##\xed\xb0\x80#\xed\xb0\x80#\xc3\x89#\xed\xb0\x80" }, - { CMA, 0 | F_FORCECONV, ".\\B.", "\xed\xa0\x80\xed\xb0\x80" }, - { CMA, 0 | F_FORCECONV, "\\D+(?:\\d+|.)\\S+(?:\\s+|.)\\W+(?:\\w+|.)\xed\xa0\x80\xed\xa0\x80", "\xed\xa0\x80\xed\xa0\x80\xed\xa0\x80\xed\xa0\x80\xed\xa0\x80\xed\xa0\x80\xed\xa0\x80\xed\xa0\x80" }, - { CMA, 0 | F_FORCECONV, "\\d*\\s*\\w*\xed\xa0\x80\xed\xa0\x80", "\xed\xa0\x80\xed\xa0\x80" }, - { CMA, 0 | F_FORCECONV | F_NOMATCH, "\\d*?\\D*?\\s*?\\S*?\\w*?\\W*?##", "\xed\xa0\x80\xed\xa0\x80\xed\xa0\x80\xed\xa0\x80#" }, - { CMA | PCRE_EXTENDED, 0 | F_FORCECONV, "\xed\xa0\x80 \xed\xb0\x80 !", "\xed\xa0\x80\xed\xb0\x80!" }, - { CMA, 0 | F_FORCECONV, "\xed\xa0\x80+#[^#]+\xed\xa0\x80", "\xed\xa0\x80#a\xed\xa0\x80" }, - { CMA, 0 | F_FORCECONV, "(\xed\xa0\x80+)#\\1", "\xed\xa0\x80\xed\xa0\x80#\xed\xa0\x80\xed\xa0\x80" }, - { PCRE_MULTILINE | PCRE_NEWLINE_ANY, 0 | F_NO8 | F_FORCECONV, "^-", "a--\xe2\x80\xa8--" }, - { PCRE_BSR_UNICODE, 0 | F_NO8 | F_FORCECONV, "\\R", "ab\xe2\x80\xa8" }, - { 0, 0 | F_NO8 | F_FORCECONV, "\\v", "ab\xe2\x80\xa9" }, - { 0, 0 | F_NO8 | F_FORCECONV, "\\h", "ab\xe1\xa0\x8e" }, - { 0, 0 | F_NO8 | F_FORCECONV, "\\v+?\\V+?#", "\xe2\x80\xa9\xe2\x80\xa9\xef\xbf\xbf\xef\xbf\xbf#" }, - { 0, 0 | F_NO8 | F_FORCECONV, "\\h+?\\H+?#", "\xe1\xa0\x8e\xe1\xa0\x8e\xef\xbf\xbf\xef\xbf\xbf#" }, - - /* Partial matching. */ - { MUA | PCRE_PARTIAL_SOFT, 0, "ab", "a" }, - { MUA | PCRE_PARTIAL_SOFT, 0, "ab|a", "a" }, - { MUA | PCRE_PARTIAL_HARD, 0, "ab|a", "a" }, - { MUA | PCRE_PARTIAL_SOFT, 0, "\\b#", "a" }, - { MUA | PCRE_PARTIAL_SOFT, 0, "(?<=a)b", "a" }, - { MUA | PCRE_PARTIAL_SOFT, 0, "abc|(?<=xxa)bc", "xxab" }, - { MUA | PCRE_PARTIAL_SOFT, 0, "a\\B", "a" }, - { MUA | PCRE_PARTIAL_HARD, 0, "a\\b", "a" }, - - /* (*MARK) verb. */ - { MUA, 0, "a(*MARK:aa)a", "ababaa" }, - { MUA, 0 | F_NOMATCH, "a(*:aa)a", "abab" }, - { MUA, 0, "a(*:aa)(b(*:bb)b|bc)", "abc" }, - { MUA, 0 | F_NOMATCH, "a(*:1)x|b(*:2)y", "abc" }, - { MUA, 0, "(?>a(*:aa))b|ac", "ac" }, - { MUA, 0, "(?(DEFINE)(a(*:aa)))(?1)", "a" }, - { MUA, 0 | F_NOMATCH, "(?(DEFINE)((a)(*:aa)))(?1)b", "aa" }, - { MUA, 0, "(?(DEFINE)(a(*:aa)))a(?1)b|aac", "aac" }, - { MUA, 0, "(a(*:aa)){0}(?:b(?1)b|c)+c", "babbab cc" }, - { MUA, 0, "(a(*:aa)){0}(?:b(?1)b)+", "babba" }, - { MUA, 0 | F_NOMATCH | F_STUDY, "(a(*:aa)){0}(?:b(?1)b)+", "ba" }, - { MUA, 0, "(a\\K(*:aa)){0}(?:b(?1)b|c)+c", "babbab cc" }, - { MUA, 0, "(a\\K(*:aa)){0}(?:b(?1)b)+", "babba" }, - { MUA, 0 | F_NOMATCH | F_STUDY, "(a\\K(*:aa)){0}(?:b(?1)b)+", "ba" }, - { MUA, 0 | F_NOMATCH | F_STUDY, "(*:mark)m", "a" }, - - /* (*COMMIT) verb. */ - { MUA, 0 | F_NOMATCH, "a(*COMMIT)b", "ac" }, - { MUA, 0, "aa(*COMMIT)b", "xaxaab" }, - { MUA, 0 | F_NOMATCH, "a(*COMMIT)(*:msg)b|ac", "ac" }, - { MUA, 0 | F_NOMATCH, "(a(*COMMIT)b)++", "abac" }, - { MUA, 0 | F_NOMATCH, "((a)(*COMMIT)b)++", "abac" }, - { MUA, 0 | F_NOMATCH, "(?=a(*COMMIT)b)ab|ad", "ad" }, - - /* (*PRUNE) verb. */ - { MUA, 0, "aa\\K(*PRUNE)b", "aaab" }, - { MUA, 0, "aa(*PRUNE:bb)b|a", "aa" }, - { MUA, 0, "(a)(a)(*PRUNE)b|(a)", "aa" }, - { MUA, 0, "(a)(a)(a)(a)(a)(a)(a)(a)(*PRUNE)b|(a)", "aaaaaaaa" }, - { MUA | PCRE_PARTIAL_SOFT, 0, "a(*PRUNE)a|", "a" }, - { MUA | PCRE_PARTIAL_SOFT, 0, "a(*PRUNE)a|m", "a" }, - { MUA, 0 | F_NOMATCH, "(?=a(*PRUNE)b)ab|ad", "ad" }, - { MUA, 0, "a(*COMMIT)(*PRUNE)d|bc", "abc" }, - { MUA, 0, "(?=a(*COMMIT)b)a(*PRUNE)c|bc", "abc" }, - { MUA, 0 | F_NOMATCH, "(*COMMIT)(?=a(*COMMIT)b)a(*PRUNE)c|bc", "abc" }, - { MUA, 0, "(?=(a)(*COMMIT)b)a(*PRUNE)c|bc", "abc" }, - { MUA, 0 | F_NOMATCH, "(*COMMIT)(?=(a)(*COMMIT)b)a(*PRUNE)c|bc", "abc" }, - { MUA, 0, "(a(*COMMIT)b){0}a(?1)(*PRUNE)c|bc", "abc" }, - { MUA, 0 | F_NOMATCH, "(a(*COMMIT)b){0}a(*COMMIT)(?1)(*PRUNE)c|bc", "abc" }, - { MUA, 0, "(a(*COMMIT)b)++(*PRUNE)d|c", "ababc" }, - { MUA, 0 | F_NOMATCH, "(*COMMIT)(a(*COMMIT)b)++(*PRUNE)d|c", "ababc" }, - { MUA, 0, "((a)(*COMMIT)b)++(*PRUNE)d|c", "ababc" }, - { MUA, 0 | F_NOMATCH, "(*COMMIT)((a)(*COMMIT)b)++(*PRUNE)d|c", "ababc" }, - { MUA, 0, "(?>a(*COMMIT)b)*abab(*PRUNE)d|ba", "ababab" }, - { MUA, 0 | F_NOMATCH, "(*COMMIT)(?>a(*COMMIT)b)*abab(*PRUNE)d|ba", "ababab" }, - { MUA, 0, "(?>a(*COMMIT)b)+abab(*PRUNE)d|ba", "ababab" }, - { MUA, 0 | F_NOMATCH, "(*COMMIT)(?>a(*COMMIT)b)+abab(*PRUNE)d|ba", "ababab" }, - { MUA, 0, "(?>a(*COMMIT)b)?ab(*PRUNE)d|ba", "aba" }, - { MUA, 0 | F_NOMATCH, "(*COMMIT)(?>a(*COMMIT)b)?ab(*PRUNE)d|ba", "aba" }, - { MUA, 0, "(?>a(*COMMIT)b)*?n(*PRUNE)d|ba", "abababn" }, - { MUA, 0 | F_NOMATCH, "(*COMMIT)(?>a(*COMMIT)b)*?n(*PRUNE)d|ba", "abababn" }, - { MUA, 0, "(?>a(*COMMIT)b)+?n(*PRUNE)d|ba", "abababn" }, - { MUA, 0 | F_NOMATCH, "(*COMMIT)(?>a(*COMMIT)b)+?n(*PRUNE)d|ba", "abababn" }, - { MUA, 0, "(?>a(*COMMIT)b)??n(*PRUNE)d|bn", "abn" }, - { MUA, 0 | F_NOMATCH, "(*COMMIT)(?>a(*COMMIT)b)??n(*PRUNE)d|bn", "abn" }, - - /* (*SKIP) verb. */ - { MUA, 0 | F_NOMATCH, "(?=a(*SKIP)b)ab|ad", "ad" }, - { MUA, 0, "(\\w+(*SKIP)#)", "abcd,xyz#," }, - { MUA, 0, "\\w+(*SKIP)#|mm", "abcd,xyz#," }, - { MUA, 0 | F_NOMATCH, "b+(?<=(*SKIP)#c)|b+", "#bbb" }, - - /* (*THEN) verb. */ - { MUA, 0, "((?:a(*THEN)|aab)(*THEN)c|a+)+m", "aabcaabcaabcaabcnacm" }, - { MUA, 0 | F_NOMATCH, "((?:a(*THEN)|aab)(*THEN)c|a+)+m", "aabcm" }, - { MUA, 0, "((?:a(*THEN)|aab)c|a+)+m", "aabcaabcnmaabcaabcm" }, - { MUA, 0, "((?:a|aab)(*THEN)c|a+)+m", "aam" }, - { MUA, 0, "((?:a(*COMMIT)|aab)(*THEN)c|a+)+m", "aam" }, - { MUA, 0, "(?(?=a(*THEN)b)ab|ad)", "ad" }, - { MUA, 0, "(?(?!a(*THEN)b)ad|add)", "add" }, - { MUA, 0 | F_NOMATCH, "(?(?=a)a(*THEN)b|ad)", "ad" }, - { MUA, 0, "(?!(?(?=a)ab|b(*THEN)d))bn|bnn", "bnn" }, - - /* Deep recursion. */ - { MUA, 0, "((((?:(?:(?:\\w)+)?)*|(?>\\w)+?)+|(?>\\w)?\?)*)?\\s", "aaaaa+ " }, - { MUA, 0, "(?:((?:(?:(?:\\w*?)+)??|(?>\\w)?|\\w*+)*)+)+?\\s", "aa+ " }, - { MUA, 0, "((a?)+)+b", "aaaaaaaaaaaa b" }, - - /* Deep recursion: Stack limit reached. */ - { MA, 0 | F_NOMATCH, "a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?aaaaaaaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaaaaaaa" }, - { MA, 0 | F_NOMATCH, "(?:a+)+b", "aaaaaaaaaaaaaaaaaaaaaaaa b" }, - { MA, 0 | F_NOMATCH, "(?:a+?)+?b", "aaaaaaaaaaaaaaaaaaaaaaaa b" }, - { MA, 0 | F_NOMATCH, "(?:a*)*b", "aaaaaaaaaaaaaaaaaaaaaaaa b" }, - { MA, 0 | F_NOMATCH, "(?:a*?)*?b", "aaaaaaaaaaaaaaaaaaaaaaaa b" }, - - { 0, 0, NULL, NULL } -}; - -static const unsigned char *tables(int mode) -{ - /* The purpose of this function to allow valgrind - for reporting invalid reads and writes. */ - static unsigned char *tables_copy; - const char *errorptr; - int erroroffset; - unsigned char *default_tables; -#if defined SUPPORT_PCRE8 - pcre *regex; - char null_str[1] = { 0 }; -#elif defined SUPPORT_PCRE16 - pcre16 *regex; - PCRE_UCHAR16 null_str[1] = { 0 }; -#elif defined SUPPORT_PCRE32 - pcre32 *regex; - PCRE_UCHAR32 null_str[1] = { 0 }; -#endif - - if (mode) { - if (tables_copy) - free(tables_copy); - tables_copy = NULL; - return NULL; - } - - if (tables_copy) - return tables_copy; - - default_tables = NULL; -#if defined SUPPORT_PCRE8 - regex = pcre_compile(null_str, 0, &errorptr, &erroroffset, NULL); - if (regex) { - pcre_fullinfo(regex, NULL, PCRE_INFO_DEFAULT_TABLES, &default_tables); - pcre_free(regex); - } -#elif defined SUPPORT_PCRE16 - regex = pcre16_compile(null_str, 0, &errorptr, &erroroffset, NULL); - if (regex) { - pcre16_fullinfo(regex, NULL, PCRE_INFO_DEFAULT_TABLES, &default_tables); - pcre16_free(regex); - } -#elif defined SUPPORT_PCRE32 - regex = pcre32_compile(null_str, 0, &errorptr, &erroroffset, NULL); - if (regex) { - pcre32_fullinfo(regex, NULL, PCRE_INFO_DEFAULT_TABLES, &default_tables); - pcre32_free(regex); - } -#endif - /* Shouldn't ever happen. */ - if (!default_tables) - return NULL; - - /* Unfortunately this value cannot get from pcre_fullinfo. - Since this is a test program, this is acceptable at the moment. */ - tables_copy = (unsigned char *)malloc(1088); - if (!tables_copy) - return NULL; - - memcpy(tables_copy, default_tables, 1088); - return tables_copy; -} - -#ifdef SUPPORT_PCRE8 -static pcre_jit_stack* callback8(void *arg) -{ - return (pcre_jit_stack *)arg; -} -#endif - -#ifdef SUPPORT_PCRE16 -static pcre16_jit_stack* callback16(void *arg) -{ - return (pcre16_jit_stack *)arg; -} -#endif - -#ifdef SUPPORT_PCRE32 -static pcre32_jit_stack* callback32(void *arg) -{ - return (pcre32_jit_stack *)arg; -} -#endif - -#ifdef SUPPORT_PCRE8 -static pcre_jit_stack *stack8; - -static pcre_jit_stack *getstack8(void) -{ - if (!stack8) - stack8 = pcre_jit_stack_alloc(1, 1024 * 1024); - return stack8; -} - -static void setstack8(pcre_extra *extra) -{ - if (!extra) { - if (stack8) - pcre_jit_stack_free(stack8); - stack8 = NULL; - return; - } - - pcre_assign_jit_stack(extra, callback8, getstack8()); -} -#endif /* SUPPORT_PCRE8 */ - -#ifdef SUPPORT_PCRE16 -static pcre16_jit_stack *stack16; - -static pcre16_jit_stack *getstack16(void) -{ - if (!stack16) - stack16 = pcre16_jit_stack_alloc(1, 1024 * 1024); - return stack16; -} - -static void setstack16(pcre16_extra *extra) -{ - if (!extra) { - if (stack16) - pcre16_jit_stack_free(stack16); - stack16 = NULL; - return; - } - - pcre16_assign_jit_stack(extra, callback16, getstack16()); -} -#endif /* SUPPORT_PCRE16 */ - -#ifdef SUPPORT_PCRE32 -static pcre32_jit_stack *stack32; - -static pcre32_jit_stack *getstack32(void) -{ - if (!stack32) - stack32 = pcre32_jit_stack_alloc(1, 1024 * 1024); - return stack32; -} - -static void setstack32(pcre32_extra *extra) -{ - if (!extra) { - if (stack32) - pcre32_jit_stack_free(stack32); - stack32 = NULL; - return; - } - - pcre32_assign_jit_stack(extra, callback32, getstack32()); -} -#endif /* SUPPORT_PCRE32 */ - -#ifdef SUPPORT_PCRE16 - -static int convert_utf8_to_utf16(const char *input, PCRE_UCHAR16 *output, int *offsetmap, int max_length) -{ - unsigned char *iptr = (unsigned char*)input; - PCRE_UCHAR16 *optr = output; - unsigned int c; - - if (max_length == 0) - return 0; - - while (*iptr && max_length > 1) { - c = 0; - if (offsetmap) - *offsetmap++ = (int)(iptr - (unsigned char*)input); - - if (*iptr < 0xc0) - c = *iptr++; - else if (!(*iptr & 0x20)) { - c = ((iptr[0] & 0x1f) << 6) | (iptr[1] & 0x3f); - iptr += 2; - } else if (!(*iptr & 0x10)) { - c = ((iptr[0] & 0x0f) << 12) | ((iptr[1] & 0x3f) << 6) | (iptr[2] & 0x3f); - iptr += 3; - } else if (!(*iptr & 0x08)) { - c = ((iptr[0] & 0x07) << 18) | ((iptr[1] & 0x3f) << 12) | ((iptr[2] & 0x3f) << 6) | (iptr[3] & 0x3f); - iptr += 4; - } - - if (c < 65536) { - *optr++ = c; - max_length--; - } else if (max_length <= 2) { - *optr = '\0'; - return (int)(optr - output); - } else { - c -= 0x10000; - *optr++ = 0xd800 | ((c >> 10) & 0x3ff); - *optr++ = 0xdc00 | (c & 0x3ff); - max_length -= 2; - if (offsetmap) - offsetmap++; - } - } - if (offsetmap) - *offsetmap = (int)(iptr - (unsigned char*)input); - *optr = '\0'; - return (int)(optr - output); -} - -static int copy_char8_to_char16(const char *input, PCRE_UCHAR16 *output, int max_length) -{ - unsigned char *iptr = (unsigned char*)input; - PCRE_UCHAR16 *optr = output; - - if (max_length == 0) - return 0; - - while (*iptr && max_length > 1) { - *optr++ = *iptr++; - max_length--; - } - *optr = '\0'; - return (int)(optr - output); -} - -#define REGTEST_MAX_LENGTH16 4096 -static PCRE_UCHAR16 regtest_buf16[REGTEST_MAX_LENGTH16]; -static int regtest_offsetmap16[REGTEST_MAX_LENGTH16]; - -#endif /* SUPPORT_PCRE16 */ - -#ifdef SUPPORT_PCRE32 - -static int convert_utf8_to_utf32(const char *input, PCRE_UCHAR32 *output, int *offsetmap, int max_length) -{ - unsigned char *iptr = (unsigned char*)input; - PCRE_UCHAR32 *optr = output; - unsigned int c; - - if (max_length == 0) - return 0; - - while (*iptr && max_length > 1) { - c = 0; - if (offsetmap) - *offsetmap++ = (int)(iptr - (unsigned char*)input); - - if (*iptr < 0xc0) - c = *iptr++; - else if (!(*iptr & 0x20)) { - c = ((iptr[0] & 0x1f) << 6) | (iptr[1] & 0x3f); - iptr += 2; - } else if (!(*iptr & 0x10)) { - c = ((iptr[0] & 0x0f) << 12) | ((iptr[1] & 0x3f) << 6) | (iptr[2] & 0x3f); - iptr += 3; - } else if (!(*iptr & 0x08)) { - c = ((iptr[0] & 0x07) << 18) | ((iptr[1] & 0x3f) << 12) | ((iptr[2] & 0x3f) << 6) | (iptr[3] & 0x3f); - iptr += 4; - } - - *optr++ = c; - max_length--; - } - if (offsetmap) - *offsetmap = (int)(iptr - (unsigned char*)input); - *optr = 0; - return (int)(optr - output); -} - -static int copy_char8_to_char32(const char *input, PCRE_UCHAR32 *output, int max_length) -{ - unsigned char *iptr = (unsigned char*)input; - PCRE_UCHAR32 *optr = output; - - if (max_length == 0) - return 0; - - while (*iptr && max_length > 1) { - *optr++ = *iptr++; - max_length--; - } - *optr = '\0'; - return (int)(optr - output); -} - -#define REGTEST_MAX_LENGTH32 4096 -static PCRE_UCHAR32 regtest_buf32[REGTEST_MAX_LENGTH32]; -static int regtest_offsetmap32[REGTEST_MAX_LENGTH32]; - -#endif /* SUPPORT_PCRE32 */ - -static int check_ascii(const char *input) -{ - const unsigned char *ptr = (unsigned char *)input; - while (*ptr) { - if (*ptr > 127) - return 0; - ptr++; - } - return 1; -} - -static int regression_tests(void) -{ - struct regression_test_case *current = regression_test_cases; - const char *error; - char *cpu_info; - int i, err_offs; - int is_successful, is_ascii; - int total = 0; - int successful = 0; - int successful_row = 0; - int counter = 0; - int study_mode; - int utf = 0, ucp = 0; - int disabled_flags = 0; -#ifdef SUPPORT_PCRE8 - pcre *re8; - pcre_extra *extra8; - pcre_extra dummy_extra8; - int ovector8_1[32]; - int ovector8_2[32]; - int return_value8[2]; - unsigned char *mark8_1, *mark8_2; -#endif -#ifdef SUPPORT_PCRE16 - pcre16 *re16; - pcre16_extra *extra16; - pcre16_extra dummy_extra16; - int ovector16_1[32]; - int ovector16_2[32]; - int return_value16[2]; - PCRE_UCHAR16 *mark16_1, *mark16_2; - int length16; -#endif -#ifdef SUPPORT_PCRE32 - pcre32 *re32; - pcre32_extra *extra32; - pcre32_extra dummy_extra32; - int ovector32_1[32]; - int ovector32_2[32]; - int return_value32[2]; - PCRE_UCHAR32 *mark32_1, *mark32_2; - int length32; -#endif - - /* This test compares the behaviour of interpreter and JIT. Although disabling - utf or ucp may make tests fail, if the pcre_exec result is the SAME, it is - still considered successful from pcre_jit_test point of view. */ - -#if defined SUPPORT_PCRE8 - pcre_config(PCRE_CONFIG_JITTARGET, &cpu_info); -#elif defined SUPPORT_PCRE16 - pcre16_config(PCRE_CONFIG_JITTARGET, &cpu_info); -#elif defined SUPPORT_PCRE32 - pcre32_config(PCRE_CONFIG_JITTARGET, &cpu_info); -#endif - - printf("Running JIT regression tests\n"); - printf(" target CPU of SLJIT compiler: %s\n", cpu_info); - -#if defined SUPPORT_PCRE8 - pcre_config(PCRE_CONFIG_UTF8, &utf); - pcre_config(PCRE_CONFIG_UNICODE_PROPERTIES, &ucp); -#elif defined SUPPORT_PCRE16 - pcre16_config(PCRE_CONFIG_UTF16, &utf); - pcre16_config(PCRE_CONFIG_UNICODE_PROPERTIES, &ucp); -#elif defined SUPPORT_PCRE32 - pcre32_config(PCRE_CONFIG_UTF32, &utf); - pcre32_config(PCRE_CONFIG_UNICODE_PROPERTIES, &ucp); -#endif - - if (!utf) - disabled_flags |= PCRE_UTF8 | PCRE_UTF16 | PCRE_UTF32; - if (!ucp) - disabled_flags |= PCRE_UCP; -#ifdef SUPPORT_PCRE8 - printf(" in 8 bit mode with UTF-8 %s and ucp %s:\n", utf ? "enabled" : "disabled", ucp ? "enabled" : "disabled"); -#endif -#ifdef SUPPORT_PCRE16 - printf(" in 16 bit mode with UTF-16 %s and ucp %s:\n", utf ? "enabled" : "disabled", ucp ? "enabled" : "disabled"); -#endif -#ifdef SUPPORT_PCRE32 - printf(" in 32 bit mode with UTF-32 %s and ucp %s:\n", utf ? "enabled" : "disabled", ucp ? "enabled" : "disabled"); -#endif - - while (current->pattern) { - /* printf("\nPattern: %s :\n", current->pattern); */ - total++; - is_ascii = 0; - if (!(current->start_offset & F_PROPERTY)) - is_ascii = check_ascii(current->pattern) && check_ascii(current->input); - - if (current->flags & PCRE_PARTIAL_SOFT) - study_mode = PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE; - else if (current->flags & PCRE_PARTIAL_HARD) - study_mode = PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE; - else - study_mode = PCRE_STUDY_JIT_COMPILE; - error = NULL; -#ifdef SUPPORT_PCRE8 - re8 = NULL; - if (!(current->start_offset & F_NO8)) - re8 = pcre_compile(current->pattern, - current->flags & ~(PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | disabled_flags), - &error, &err_offs, tables(0)); - - extra8 = NULL; - if (re8) { - error = NULL; - extra8 = pcre_study(re8, study_mode, &error); - if (!extra8) { - printf("\n8 bit: Cannot study pattern: %s\n", current->pattern); - pcre_free(re8); - re8 = NULL; - } - else if (!(extra8->flags & PCRE_EXTRA_EXECUTABLE_JIT)) { - printf("\n8 bit: JIT compiler does not support: %s\n", current->pattern); - pcre_free_study(extra8); - pcre_free(re8); - re8 = NULL; - } - extra8->flags |= PCRE_EXTRA_MARK; - } else if (((utf && ucp) || is_ascii) && !(current->start_offset & F_NO8)) - printf("\n8 bit: Cannot compile pattern \"%s\": %s\n", current->pattern, error); -#endif -#ifdef SUPPORT_PCRE16 - if ((current->flags & PCRE_UTF16) || (current->start_offset & F_FORCECONV)) - convert_utf8_to_utf16(current->pattern, regtest_buf16, NULL, REGTEST_MAX_LENGTH16); - else - copy_char8_to_char16(current->pattern, regtest_buf16, REGTEST_MAX_LENGTH16); - - re16 = NULL; - if (!(current->start_offset & F_NO16)) - re16 = pcre16_compile(regtest_buf16, - current->flags & ~(PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | disabled_flags), - &error, &err_offs, tables(0)); - - extra16 = NULL; - if (re16) { - error = NULL; - extra16 = pcre16_study(re16, study_mode, &error); - if (!extra16) { - printf("\n16 bit: Cannot study pattern: %s\n", current->pattern); - pcre16_free(re16); - re16 = NULL; - } - else if (!(extra16->flags & PCRE_EXTRA_EXECUTABLE_JIT)) { - printf("\n16 bit: JIT compiler does not support: %s\n", current->pattern); - pcre16_free_study(extra16); - pcre16_free(re16); - re16 = NULL; - } - extra16->flags |= PCRE_EXTRA_MARK; - } else if (((utf && ucp) || is_ascii) && !(current->start_offset & F_NO16)) - printf("\n16 bit: Cannot compile pattern \"%s\": %s\n", current->pattern, error); -#endif -#ifdef SUPPORT_PCRE32 - if ((current->flags & PCRE_UTF32) || (current->start_offset & F_FORCECONV)) - convert_utf8_to_utf32(current->pattern, regtest_buf32, NULL, REGTEST_MAX_LENGTH32); - else - copy_char8_to_char32(current->pattern, regtest_buf32, REGTEST_MAX_LENGTH32); - - re32 = NULL; - if (!(current->start_offset & F_NO32)) - re32 = pcre32_compile(regtest_buf32, - current->flags & ~(PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | disabled_flags), - &error, &err_offs, tables(0)); - - extra32 = NULL; - if (re32) { - error = NULL; - extra32 = pcre32_study(re32, study_mode, &error); - if (!extra32) { - printf("\n32 bit: Cannot study pattern: %s\n", current->pattern); - pcre32_free(re32); - re32 = NULL; - } - if (!(extra32->flags & PCRE_EXTRA_EXECUTABLE_JIT)) { - printf("\n32 bit: JIT compiler does not support: %s\n", current->pattern); - pcre32_free_study(extra32); - pcre32_free(re32); - re32 = NULL; - } - extra32->flags |= PCRE_EXTRA_MARK; - } else if (((utf && ucp) || is_ascii) && !(current->start_offset & F_NO32)) - printf("\n32 bit: Cannot compile pattern \"%s\": %s\n", current->pattern, error); -#endif - - counter++; - if ((counter & 0x3) != 0) { -#ifdef SUPPORT_PCRE8 - setstack8(NULL); -#endif -#ifdef SUPPORT_PCRE16 - setstack16(NULL); -#endif -#ifdef SUPPORT_PCRE32 - setstack32(NULL); -#endif - } - -#ifdef SUPPORT_PCRE8 - return_value8[0] = -1000; - return_value8[1] = -1000; - for (i = 0; i < 32; ++i) - ovector8_1[i] = -2; - for (i = 0; i < 32; ++i) - ovector8_2[i] = -2; - if (re8) { - mark8_1 = NULL; - mark8_2 = NULL; - extra8->mark = &mark8_1; - - if ((counter & 0x1) != 0) { - setstack8(extra8); - return_value8[0] = pcre_exec(re8, extra8, current->input, strlen(current->input), current->start_offset & OFFSET_MASK, - current->flags & (PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | PCRE_NO_UTF8_CHECK), ovector8_1, 32); - } else - return_value8[0] = pcre_jit_exec(re8, extra8, current->input, strlen(current->input), current->start_offset & OFFSET_MASK, - current->flags & (PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | PCRE_NO_UTF8_CHECK), ovector8_1, 32, getstack8()); - memset(&dummy_extra8, 0, sizeof(pcre_extra)); - dummy_extra8.flags = PCRE_EXTRA_MARK; - if (current->start_offset & F_STUDY) { - dummy_extra8.flags |= PCRE_EXTRA_STUDY_DATA; - dummy_extra8.study_data = extra8->study_data; - } - dummy_extra8.mark = &mark8_2; - return_value8[1] = pcre_exec(re8, &dummy_extra8, current->input, strlen(current->input), current->start_offset & OFFSET_MASK, - current->flags & (PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | PCRE_NO_UTF8_CHECK), ovector8_2, 32); - } -#endif - -#ifdef SUPPORT_PCRE16 - return_value16[0] = -1000; - return_value16[1] = -1000; - for (i = 0; i < 32; ++i) - ovector16_1[i] = -2; - for (i = 0; i < 32; ++i) - ovector16_2[i] = -2; - if (re16) { - mark16_1 = NULL; - mark16_2 = NULL; - if ((current->flags & PCRE_UTF16) || (current->start_offset & F_FORCECONV)) - length16 = convert_utf8_to_utf16(current->input, regtest_buf16, regtest_offsetmap16, REGTEST_MAX_LENGTH16); - else - length16 = copy_char8_to_char16(current->input, regtest_buf16, REGTEST_MAX_LENGTH16); - extra16->mark = &mark16_1; - if ((counter & 0x1) != 0) { - setstack16(extra16); - return_value16[0] = pcre16_exec(re16, extra16, regtest_buf16, length16, current->start_offset & OFFSET_MASK, - current->flags & (PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | PCRE_NO_UTF8_CHECK), ovector16_1, 32); - } else - return_value16[0] = pcre16_jit_exec(re16, extra16, regtest_buf16, length16, current->start_offset & OFFSET_MASK, - current->flags & (PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | PCRE_NO_UTF8_CHECK), ovector16_1, 32, getstack16()); - memset(&dummy_extra16, 0, sizeof(pcre16_extra)); - dummy_extra16.flags = PCRE_EXTRA_MARK; - if (current->start_offset & F_STUDY) { - dummy_extra16.flags |= PCRE_EXTRA_STUDY_DATA; - dummy_extra16.study_data = extra16->study_data; - } - dummy_extra16.mark = &mark16_2; - return_value16[1] = pcre16_exec(re16, &dummy_extra16, regtest_buf16, length16, current->start_offset & OFFSET_MASK, - current->flags & (PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | PCRE_NO_UTF8_CHECK), ovector16_2, 32); - } -#endif - -#ifdef SUPPORT_PCRE32 - return_value32[0] = -1000; - return_value32[1] = -1000; - for (i = 0; i < 32; ++i) - ovector32_1[i] = -2; - for (i = 0; i < 32; ++i) - ovector32_2[i] = -2; - if (re32) { - mark32_1 = NULL; - mark32_2 = NULL; - if ((current->flags & PCRE_UTF32) || (current->start_offset & F_FORCECONV)) - length32 = convert_utf8_to_utf32(current->input, regtest_buf32, regtest_offsetmap32, REGTEST_MAX_LENGTH32); - else - length32 = copy_char8_to_char32(current->input, regtest_buf32, REGTEST_MAX_LENGTH32); - extra32->mark = &mark32_1; - if ((counter & 0x1) != 0) { - setstack32(extra32); - return_value32[0] = pcre32_exec(re32, extra32, regtest_buf32, length32, current->start_offset & OFFSET_MASK, - current->flags & (PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | PCRE_NO_UTF8_CHECK), ovector32_1, 32); - } else - return_value32[0] = pcre32_jit_exec(re32, extra32, regtest_buf32, length32, current->start_offset & OFFSET_MASK, - current->flags & (PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | PCRE_NO_UTF8_CHECK), ovector32_1, 32, getstack32()); - memset(&dummy_extra32, 0, sizeof(pcre32_extra)); - dummy_extra32.flags = PCRE_EXTRA_MARK; - if (current->start_offset & F_STUDY) { - dummy_extra32.flags |= PCRE_EXTRA_STUDY_DATA; - dummy_extra32.study_data = extra32->study_data; - } - dummy_extra32.mark = &mark32_2; - return_value32[1] = pcre32_exec(re32, &dummy_extra32, regtest_buf32, length32, current->start_offset & OFFSET_MASK, - current->flags & (PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | PCRE_NO_UTF8_CHECK), ovector32_2, 32); - } -#endif - - /* printf("[%d-%d-%d|%d-%d|%d-%d|%d-%d]%s", - return_value8[0], return_value16[0], return_value32[0], - ovector8_1[0], ovector8_1[1], - ovector16_1[0], ovector16_1[1], - ovector32_1[0], ovector32_1[1], - (current->flags & PCRE_CASELESS) ? "C" : ""); */ - - /* If F_DIFF is set, just run the test, but do not compare the results. - Segfaults can still be captured. */ - - is_successful = 1; - if (!(current->start_offset & F_DIFF)) { -#if defined SUPPORT_UTF && ((defined(SUPPORT_PCRE8) + defined(SUPPORT_PCRE16) + defined(SUPPORT_PCRE32)) >= 2) - if (!(current->start_offset & F_FORCECONV)) { - int return_value; - - /* All results must be the same. */ -#ifdef SUPPORT_PCRE8 - if ((return_value = return_value8[0]) != return_value8[1]) { - printf("\n8 bit: Return value differs(J8:%d,I8:%d): [%d] '%s' @ '%s'\n", - return_value8[0], return_value8[1], total, current->pattern, current->input); - is_successful = 0; - } else -#endif -#ifdef SUPPORT_PCRE16 - if ((return_value = return_value16[0]) != return_value16[1]) { - printf("\n16 bit: Return value differs(J16:%d,I16:%d): [%d] '%s' @ '%s'\n", - return_value16[0], return_value16[1], total, current->pattern, current->input); - is_successful = 0; - } else -#endif -#ifdef SUPPORT_PCRE32 - if ((return_value = return_value32[0]) != return_value32[1]) { - printf("\n32 bit: Return value differs(J32:%d,I32:%d): [%d] '%s' @ '%s'\n", - return_value32[0], return_value32[1], total, current->pattern, current->input); - is_successful = 0; - } else -#endif -#if defined SUPPORT_PCRE8 && defined SUPPORT_PCRE16 - if (return_value8[0] != return_value16[0]) { - printf("\n8 and 16 bit: Return value differs(J8:%d,J16:%d): [%d] '%s' @ '%s'\n", - return_value8[0], return_value16[0], - total, current->pattern, current->input); - is_successful = 0; - } else -#endif -#if defined SUPPORT_PCRE8 && defined SUPPORT_PCRE32 - if (return_value8[0] != return_value32[0]) { - printf("\n8 and 32 bit: Return value differs(J8:%d,J32:%d): [%d] '%s' @ '%s'\n", - return_value8[0], return_value32[0], - total, current->pattern, current->input); - is_successful = 0; - } else -#endif -#if defined SUPPORT_PCRE16 && defined SUPPORT_PCRE32 - if (return_value16[0] != return_value32[0]) { - printf("\n16 and 32 bit: Return value differs(J16:%d,J32:%d): [%d] '%s' @ '%s'\n", - return_value16[0], return_value32[0], - total, current->pattern, current->input); - is_successful = 0; - } else -#endif - if (return_value >= 0 || return_value == PCRE_ERROR_PARTIAL) { - if (return_value == PCRE_ERROR_PARTIAL) { - return_value = 2; - } else { - return_value *= 2; - } -#ifdef SUPPORT_PCRE8 - return_value8[0] = return_value; -#endif -#ifdef SUPPORT_PCRE16 - return_value16[0] = return_value; -#endif -#ifdef SUPPORT_PCRE32 - return_value32[0] = return_value; -#endif - /* Transform back the results. */ - if (current->flags & PCRE_UTF8) { -#ifdef SUPPORT_PCRE16 - for (i = 0; i < return_value; ++i) { - if (ovector16_1[i] >= 0) - ovector16_1[i] = regtest_offsetmap16[ovector16_1[i]]; - if (ovector16_2[i] >= 0) - ovector16_2[i] = regtest_offsetmap16[ovector16_2[i]]; - } -#endif -#ifdef SUPPORT_PCRE32 - for (i = 0; i < return_value; ++i) { - if (ovector32_1[i] >= 0) - ovector32_1[i] = regtest_offsetmap32[ovector32_1[i]]; - if (ovector32_2[i] >= 0) - ovector32_2[i] = regtest_offsetmap32[ovector32_2[i]]; - } -#endif - } - - for (i = 0; i < return_value; ++i) { -#if defined SUPPORT_PCRE8 && defined SUPPORT_PCRE16 - if (ovector8_1[i] != ovector8_2[i] || ovector8_1[i] != ovector16_1[i] || ovector8_1[i] != ovector16_2[i]) { - printf("\n8 and 16 bit: Ovector[%d] value differs(J8:%d,I8:%d,J16:%d,I16:%d): [%d] '%s' @ '%s' \n", - i, ovector8_1[i], ovector8_2[i], ovector16_1[i], ovector16_2[i], - total, current->pattern, current->input); - is_successful = 0; - } -#endif -#if defined SUPPORT_PCRE8 && defined SUPPORT_PCRE32 - if (ovector8_1[i] != ovector8_2[i] || ovector8_1[i] != ovector32_1[i] || ovector8_1[i] != ovector32_2[i]) { - printf("\n8 and 32 bit: Ovector[%d] value differs(J8:%d,I8:%d,J32:%d,I32:%d): [%d] '%s' @ '%s' \n", - i, ovector8_1[i], ovector8_2[i], ovector32_1[i], ovector32_2[i], - total, current->pattern, current->input); - is_successful = 0; - } -#endif -#if defined SUPPORT_PCRE16 && defined SUPPORT_PCRE32 - if (ovector16_1[i] != ovector16_2[i] || ovector16_1[i] != ovector32_1[i] || ovector16_1[i] != ovector32_2[i]) { - printf("\n16 and 32 bit: Ovector[%d] value differs(J16:%d,I16:%d,J32:%d,I32:%d): [%d] '%s' @ '%s' \n", - i, ovector16_1[i], ovector16_2[i], ovector32_1[i], ovector32_2[i], - total, current->pattern, current->input); - is_successful = 0; - } -#endif - } - } - } else -#endif /* more than one of SUPPORT_PCRE8, SUPPORT_PCRE16 and SUPPORT_PCRE32 */ - { - /* Only the 8 bit and 16 bit results must be equal. */ -#ifdef SUPPORT_PCRE8 - if (return_value8[0] != return_value8[1]) { - printf("\n8 bit: Return value differs(%d:%d): [%d] '%s' @ '%s'\n", - return_value8[0], return_value8[1], total, current->pattern, current->input); - is_successful = 0; - } else if (return_value8[0] >= 0 || return_value8[0] == PCRE_ERROR_PARTIAL) { - if (return_value8[0] == PCRE_ERROR_PARTIAL) - return_value8[0] = 2; - else - return_value8[0] *= 2; - - for (i = 0; i < return_value8[0]; ++i) - if (ovector8_1[i] != ovector8_2[i]) { - printf("\n8 bit: Ovector[%d] value differs(%d:%d): [%d] '%s' @ '%s'\n", - i, ovector8_1[i], ovector8_2[i], total, current->pattern, current->input); - is_successful = 0; - } - } -#endif - -#ifdef SUPPORT_PCRE16 - if (return_value16[0] != return_value16[1]) { - printf("\n16 bit: Return value differs(%d:%d): [%d] '%s' @ '%s'\n", - return_value16[0], return_value16[1], total, current->pattern, current->input); - is_successful = 0; - } else if (return_value16[0] >= 0 || return_value16[0] == PCRE_ERROR_PARTIAL) { - if (return_value16[0] == PCRE_ERROR_PARTIAL) - return_value16[0] = 2; - else - return_value16[0] *= 2; - - for (i = 0; i < return_value16[0]; ++i) - if (ovector16_1[i] != ovector16_2[i]) { - printf("\n16 bit: Ovector[%d] value differs(%d:%d): [%d] '%s' @ '%s'\n", - i, ovector16_1[i], ovector16_2[i], total, current->pattern, current->input); - is_successful = 0; - } - } -#endif - -#ifdef SUPPORT_PCRE32 - if (return_value32[0] != return_value32[1]) { - printf("\n32 bit: Return value differs(%d:%d): [%d] '%s' @ '%s'\n", - return_value32[0], return_value32[1], total, current->pattern, current->input); - is_successful = 0; - } else if (return_value32[0] >= 0 || return_value32[0] == PCRE_ERROR_PARTIAL) { - if (return_value32[0] == PCRE_ERROR_PARTIAL) - return_value32[0] = 2; - else - return_value32[0] *= 2; - - for (i = 0; i < return_value32[0]; ++i) - if (ovector32_1[i] != ovector32_2[i]) { - printf("\n32 bit: Ovector[%d] value differs(%d:%d): [%d] '%s' @ '%s'\n", - i, ovector32_1[i], ovector32_2[i], total, current->pattern, current->input); - is_successful = 0; - } - } -#endif - } - } - - if (is_successful) { -#ifdef SUPPORT_PCRE8 - if (!(current->start_offset & F_NO8) && ((utf && ucp) || is_ascii)) { - if (return_value8[0] < 0 && !(current->start_offset & F_NOMATCH)) { - printf("8 bit: Test should match: [%d] '%s' @ '%s'\n", - total, current->pattern, current->input); - is_successful = 0; - } - - if (return_value8[0] >= 0 && (current->start_offset & F_NOMATCH)) { - printf("8 bit: Test should not match: [%d] '%s' @ '%s'\n", - total, current->pattern, current->input); - is_successful = 0; - } - } -#endif -#ifdef SUPPORT_PCRE16 - if (!(current->start_offset & F_NO16) && ((utf && ucp) || is_ascii)) { - if (return_value16[0] < 0 && !(current->start_offset & F_NOMATCH)) { - printf("16 bit: Test should match: [%d] '%s' @ '%s'\n", - total, current->pattern, current->input); - is_successful = 0; - } - - if (return_value16[0] >= 0 && (current->start_offset & F_NOMATCH)) { - printf("16 bit: Test should not match: [%d] '%s' @ '%s'\n", - total, current->pattern, current->input); - is_successful = 0; - } - } -#endif -#ifdef SUPPORT_PCRE32 - if (!(current->start_offset & F_NO32) && ((utf && ucp) || is_ascii)) { - if (return_value32[0] < 0 && !(current->start_offset & F_NOMATCH)) { - printf("32 bit: Test should match: [%d] '%s' @ '%s'\n", - total, current->pattern, current->input); - is_successful = 0; - } - - if (return_value32[0] >= 0 && (current->start_offset & F_NOMATCH)) { - printf("32 bit: Test should not match: [%d] '%s' @ '%s'\n", - total, current->pattern, current->input); - is_successful = 0; - } - } -#endif - } - - if (is_successful) { -#ifdef SUPPORT_PCRE8 - if (mark8_1 != mark8_2) { - printf("8 bit: Mark value mismatch: [%d] '%s' @ '%s'\n", - total, current->pattern, current->input); - is_successful = 0; - } -#endif -#ifdef SUPPORT_PCRE16 - if (mark16_1 != mark16_2) { - printf("16 bit: Mark value mismatch: [%d] '%s' @ '%s'\n", - total, current->pattern, current->input); - is_successful = 0; - } -#endif -#ifdef SUPPORT_PCRE32 - if (mark32_1 != mark32_2) { - printf("32 bit: Mark value mismatch: [%d] '%s' @ '%s'\n", - total, current->pattern, current->input); - is_successful = 0; - } -#endif - } - -#ifdef SUPPORT_PCRE8 - if (re8) { - pcre_free_study(extra8); - pcre_free(re8); - } -#endif -#ifdef SUPPORT_PCRE16 - if (re16) { - pcre16_free_study(extra16); - pcre16_free(re16); - } -#endif -#ifdef SUPPORT_PCRE32 - if (re32) { - pcre32_free_study(extra32); - pcre32_free(re32); - } -#endif - - if (is_successful) { - successful++; - successful_row++; - printf("."); - if (successful_row >= 60) { - successful_row = 0; - printf("\n"); - } - } else - successful_row = 0; - - fflush(stdout); - current++; - } - tables(1); -#ifdef SUPPORT_PCRE8 - setstack8(NULL); -#endif -#ifdef SUPPORT_PCRE16 - setstack16(NULL); -#endif -#ifdef SUPPORT_PCRE32 - setstack32(NULL); -#endif - - if (total == successful) { - printf("\nAll JIT regression tests are successfully passed.\n"); - return 0; - } else { - printf("\nSuccessful test ratio: %d%% (%d failed)\n", successful * 100 / total, total - successful); - return 1; - } -} - -/* End of pcre_jit_test.c */ diff --git a/src/third_party/pcre-8.39/pcre_maketables.c b/src/third_party/pcre-8.39/pcre_maketables.c deleted file mode 100644 index a44a6eaa905..00000000000 --- a/src/third_party/pcre-8.39/pcre_maketables.c +++ /dev/null @@ -1,156 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This module contains the external function pcre_maketables(), which builds -character tables for PCRE in the current locale. The file is compiled on its -own as part of the PCRE library. However, it is also included in the -compilation of dftables.c, in which case the macro DFTABLES is defined. */ - - -#ifndef DFTABLES -# ifdef HAVE_CONFIG_H -# include "config.h" -# endif -# include "pcre_internal.h" -#endif - - -/************************************************* -* Create PCRE character tables * -*************************************************/ - -/* This function builds a set of character tables for use by PCRE and returns -a pointer to them. They are build using the ctype functions, and consequently -their contents will depend upon the current locale setting. When compiled as -part of the library, the store is obtained via PUBL(malloc)(), but when -compiled inside dftables, use malloc(). - -Arguments: none -Returns: pointer to the contiguous block of data -*/ - -#if defined COMPILE_PCRE8 -const unsigned char * -pcre_maketables(void) -#elif defined COMPILE_PCRE16 -const unsigned char * -pcre16_maketables(void) -#elif defined COMPILE_PCRE32 -const unsigned char * -pcre32_maketables(void) -#endif -{ -unsigned char *yield, *p; -int i; - -#ifndef DFTABLES -yield = (unsigned char*)(PUBL(malloc))(tables_length); -#else -yield = (unsigned char*)malloc(tables_length); -#endif - -if (yield == NULL) return NULL; -p = yield; - -/* First comes the lower casing table */ - -for (i = 0; i < 256; i++) *p++ = tolower(i); - -/* Next the case-flipping table */ - -for (i = 0; i < 256; i++) *p++ = islower(i)? toupper(i) : tolower(i); - -/* Then the character class tables. Don't try to be clever and save effort on -exclusive ones - in some locales things may be different. - -Note that the table for "space" includes everything "isspace" gives, including -VT in the default locale. This makes it work for the POSIX class [:space:]. -From release 8.34 is is also correct for Perl space, because Perl added VT at -release 5.18. - -Note also that it is possible for a character to be alnum or alpha without -being lower or upper, such as "male and female ordinals" (\xAA and \xBA) in the -fr_FR locale (at least under Debian Linux's locales as of 12/2005). So we must -test for alnum specially. */ - -memset(p, 0, cbit_length); -for (i = 0; i < 256; i++) - { - if (isdigit(i)) p[cbit_digit + i/8] |= 1 << (i&7); - if (isupper(i)) p[cbit_upper + i/8] |= 1 << (i&7); - if (islower(i)) p[cbit_lower + i/8] |= 1 << (i&7); - if (isalnum(i)) p[cbit_word + i/8] |= 1 << (i&7); - if (i == '_') p[cbit_word + i/8] |= 1 << (i&7); - if (isspace(i)) p[cbit_space + i/8] |= 1 << (i&7); - if (isxdigit(i))p[cbit_xdigit + i/8] |= 1 << (i&7); - if (isgraph(i)) p[cbit_graph + i/8] |= 1 << (i&7); - if (isprint(i)) p[cbit_print + i/8] |= 1 << (i&7); - if (ispunct(i)) p[cbit_punct + i/8] |= 1 << (i&7); - if (iscntrl(i)) p[cbit_cntrl + i/8] |= 1 << (i&7); - } -p += cbit_length; - -/* Finally, the character type table. In this, we used to exclude VT from the -white space chars, because Perl didn't recognize it as such for \s and for -comments within regexes. However, Perl changed at release 5.18, so PCRE changed -at release 8.34. */ - -for (i = 0; i < 256; i++) - { - int x = 0; - if (isspace(i)) x += ctype_space; - if (isalpha(i)) x += ctype_letter; - if (isdigit(i)) x += ctype_digit; - if (isxdigit(i)) x += ctype_xdigit; - if (isalnum(i) || i == '_') x += ctype_word; - - /* Note: strchr includes the terminating zero in the characters it considers. - In this instance, that is ok because we want binary zero to be flagged as a - meta-character, which in this sense is any character that terminates a run - of data characters. */ - - if (strchr("\\*+?{^.$|()[", i) != 0) x += ctype_meta; - *p++ = x; - } - -return yield; -} - -/* End of pcre_maketables.c */ diff --git a/src/third_party/pcre-8.39/pcre_newline.c b/src/third_party/pcre-8.39/pcre_newline.c deleted file mode 100644 index b8f5a4de19c..00000000000 --- a/src/third_party/pcre-8.39/pcre_newline.c +++ /dev/null @@ -1,210 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This module contains internal functions for testing newlines when more than -one kind of newline is to be recognized. When a newline is found, its length is -returned. In principle, we could implement several newline "types", each -referring to a different set of newline characters. At present, PCRE supports -only NLTYPE_FIXED, which gets handled without these functions, NLTYPE_ANYCRLF, -and NLTYPE_ANY. The full list of Unicode newline characters is taken from -http://unicode.org/unicode/reports/tr18/. */ - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "pcre_internal.h" - - - -/************************************************* -* Check for newline at given position * -*************************************************/ - -/* It is guaranteed that the initial value of ptr is less than the end of the -string that is being processed. - -Arguments: - ptr pointer to possible newline - type the newline type - endptr pointer to the end of the string - lenptr where to return the length - utf TRUE if in utf mode - -Returns: TRUE or FALSE -*/ - -BOOL -PRIV(is_newline)(PCRE_PUCHAR ptr, int type, PCRE_PUCHAR endptr, int *lenptr, - BOOL utf) -{ -pcre_uint32 c; -(void)utf; -#ifdef SUPPORT_UTF -if (utf) - { - GETCHAR(c, ptr); - } -else -#endif /* SUPPORT_UTF */ - c = *ptr; - -/* Note that this function is called only for ANY or ANYCRLF. */ - -if (type == NLTYPE_ANYCRLF) switch(c) - { - case CHAR_LF: *lenptr = 1; return TRUE; - case CHAR_CR: *lenptr = (ptr < endptr - 1 && ptr[1] == CHAR_LF)? 2 : 1; - return TRUE; - default: return FALSE; - } - -/* NLTYPE_ANY */ - -else switch(c) - { -#ifdef EBCDIC - case CHAR_NEL: -#endif - case CHAR_LF: - case CHAR_VT: - case CHAR_FF: *lenptr = 1; return TRUE; - - case CHAR_CR: - *lenptr = (ptr < endptr - 1 && ptr[1] == CHAR_LF)? 2 : 1; - return TRUE; - -#ifndef EBCDIC -#ifdef COMPILE_PCRE8 - case CHAR_NEL: *lenptr = utf? 2 : 1; return TRUE; - case 0x2028: /* LS */ - case 0x2029: *lenptr = 3; return TRUE; /* PS */ -#else /* COMPILE_PCRE16 || COMPILE_PCRE32 */ - case CHAR_NEL: - case 0x2028: /* LS */ - case 0x2029: *lenptr = 1; return TRUE; /* PS */ -#endif /* COMPILE_PCRE8 */ -#endif /* Not EBCDIC */ - - default: return FALSE; - } -} - - - -/************************************************* -* Check for newline at previous position * -*************************************************/ - -/* It is guaranteed that the initial value of ptr is greater than the start of -the string that is being processed. - -Arguments: - ptr pointer to possible newline - type the newline type - startptr pointer to the start of the string - lenptr where to return the length - utf TRUE if in utf mode - -Returns: TRUE or FALSE -*/ - -BOOL -PRIV(was_newline)(PCRE_PUCHAR ptr, int type, PCRE_PUCHAR startptr, int *lenptr, - BOOL utf) -{ -pcre_uint32 c; -(void)utf; -ptr--; -#ifdef SUPPORT_UTF -if (utf) - { - BACKCHAR(ptr); - GETCHAR(c, ptr); - } -else -#endif /* SUPPORT_UTF */ - c = *ptr; - -/* Note that this function is called only for ANY or ANYCRLF. */ - -if (type == NLTYPE_ANYCRLF) switch(c) - { - case CHAR_LF: - *lenptr = (ptr > startptr && ptr[-1] == CHAR_CR)? 2 : 1; - return TRUE; - - case CHAR_CR: *lenptr = 1; return TRUE; - default: return FALSE; - } - -/* NLTYPE_ANY */ - -else switch(c) - { - case CHAR_LF: - *lenptr = (ptr > startptr && ptr[-1] == CHAR_CR)? 2 : 1; - return TRUE; - -#ifdef EBCDIC - case CHAR_NEL: -#endif - case CHAR_VT: - case CHAR_FF: - case CHAR_CR: *lenptr = 1; return TRUE; - -#ifndef EBCDIC -#ifdef COMPILE_PCRE8 - case CHAR_NEL: *lenptr = utf? 2 : 1; return TRUE; - case 0x2028: /* LS */ - case 0x2029: *lenptr = 3; return TRUE; /* PS */ -#else /* COMPILE_PCRE16 || COMPILE_PCRE32 */ - case CHAR_NEL: - case 0x2028: /* LS */ - case 0x2029: *lenptr = 1; return TRUE; /* PS */ -#endif /* COMPILE_PCRE8 */ -#endif /* NotEBCDIC */ - - default: return FALSE; - } -} - -/* End of pcre_newline.c */ diff --git a/src/third_party/pcre-8.39/pcre_ord2utf8.c b/src/third_party/pcre-8.39/pcre_ord2utf8.c deleted file mode 100644 index 95f1beb963e..00000000000 --- a/src/third_party/pcre-8.39/pcre_ord2utf8.c +++ /dev/null @@ -1,94 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This file contains a private PCRE function that converts an ordinal -character value into a UTF8 string. */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#define COMPILE_PCRE8 - -#include "pcre_internal.h" - -/************************************************* -* Convert character value to UTF-8 * -*************************************************/ - -/* This function takes an integer value in the range 0 - 0x10ffff -and encodes it as a UTF-8 character in 1 to 4 pcre_uchars. - -Arguments: - cvalue the character value - buffer pointer to buffer for result - at least 6 pcre_uchars long - -Returns: number of characters placed in the buffer -*/ - -unsigned -int -PRIV(ord2utf)(pcre_uint32 cvalue, pcre_uchar *buffer) -{ -#ifdef SUPPORT_UTF - -register int i, j; - -for (i = 0; i < PRIV(utf8_table1_size); i++) - if ((int)cvalue <= PRIV(utf8_table1)[i]) break; -buffer += i; -for (j = i; j > 0; j--) - { - *buffer-- = 0x80 | (cvalue & 0x3f); - cvalue >>= 6; - } -*buffer = PRIV(utf8_table2)[i] | cvalue; -return i + 1; - -#else - -(void)(cvalue); /* Keep compiler happy; this function won't ever be */ -(void)(buffer); /* called when SUPPORT_UTF is not defined. */ -return 0; - -#endif -} - -/* End of pcre_ord2utf8.c */ diff --git a/src/third_party/pcre-8.39/pcre_printint.c b/src/third_party/pcre-8.39/pcre_printint.c deleted file mode 100644 index 60dcb55efbf..00000000000 --- a/src/third_party/pcre-8.39/pcre_printint.c +++ /dev/null @@ -1,834 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This module contains a PCRE private debugging function for printing out the -internal form of a compiled regular expression, along with some supporting -local functions. This source file is used in two places: - -(1) It is #included by pcre_compile.c when it is compiled in debugging mode -(PCRE_DEBUG defined in pcre_internal.h). It is not included in production -compiles. In this case PCRE_INCLUDED is defined. - -(2) It is also compiled separately and linked with pcretest.c, which can be -asked to print out a compiled regex for debugging purposes. */ - -#ifndef PCRE_INCLUDED - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -/* For pcretest program. */ -#define PRIV(name) name - -/* We have to include pcre_internal.h because we need the internal info for -displaying the results of pcre_study() and we also need to know about the -internal macros, structures, and other internal data values; pcretest has -"inside information" compared to a program that strictly follows the PCRE API. - -Although pcre_internal.h does itself include pcre.h, we explicitly include it -here before pcre_internal.h so that the PCRE_EXP_xxx macros get set -appropriately for an application, not for building PCRE. */ - -#include "pcre.h" -#include "pcre_internal.h" - -/* These are the funtions that are contained within. It doesn't seem worth -having a separate .h file just for this. */ - -#endif /* PCRE_INCLUDED */ - -#ifdef PCRE_INCLUDED -static /* Keep the following function as private. */ -#endif - -#if defined COMPILE_PCRE8 -void pcre_printint(pcre *external_re, FILE *f, BOOL print_lengths); -#elif defined COMPILE_PCRE16 -void pcre16_printint(pcre *external_re, FILE *f, BOOL print_lengths); -#elif defined COMPILE_PCRE32 -void pcre32_printint(pcre *external_re, FILE *f, BOOL print_lengths); -#endif - -/* Macro that decides whether a character should be output as a literal or in -hexadecimal. We don't use isprint() because that can vary from system to system -(even without the use of locales) and we want the output always to be the same, -for testing purposes. */ - -#ifdef EBCDIC -#define PRINTABLE(c) ((c) >= 64 && (c) < 255) -#else -#define PRINTABLE(c) ((c) >= 32 && (c) < 127) -#endif - -/* The table of operator names. */ - -static const char *priv_OP_names[] = { OP_NAME_LIST }; - -/* This table of operator lengths is not actually used by the working code, -but its size is needed for a check that ensures it is the correct size for the -number of opcodes (thus catching update omissions). */ - -static const pcre_uint8 priv_OP_lengths[] = { OP_LENGTHS }; - - - -/************************************************* -* Print single- or multi-byte character * -*************************************************/ - -static unsigned int -print_char(FILE *f, pcre_uchar *ptr, BOOL utf) -{ -pcre_uint32 c = *ptr; - -#ifndef SUPPORT_UTF - -(void)utf; /* Avoid compiler warning */ -if (PRINTABLE(c)) fprintf(f, "%c", (char)c); -else if (c <= 0x80) fprintf(f, "\\x%02x", c); -else fprintf(f, "\\x{%x}", c); -return 0; - -#else - -#if defined COMPILE_PCRE8 - -if (!utf || (c & 0xc0) != 0xc0) - { - if (PRINTABLE(c)) fprintf(f, "%c", (char)c); - else if (c < 0x80) fprintf(f, "\\x%02x", c); - else fprintf(f, "\\x{%02x}", c); - return 0; - } -else - { - int i; - int a = PRIV(utf8_table4)[c & 0x3f]; /* Number of additional bytes */ - int s = 6*a; - c = (c & PRIV(utf8_table3)[a]) << s; - for (i = 1; i <= a; i++) - { - /* This is a check for malformed UTF-8; it should only occur if the sanity - check has been turned off. Rather than swallow random bytes, just stop if - we hit a bad one. Print it with \X instead of \x as an indication. */ - - if ((ptr[i] & 0xc0) != 0x80) - { - fprintf(f, "\\X{%x}", c); - return i - 1; - } - - /* The byte is OK */ - - s -= 6; - c |= (ptr[i] & 0x3f) << s; - } - fprintf(f, "\\x{%x}", c); - return a; - } - -#elif defined COMPILE_PCRE16 - -if (!utf || (c & 0xfc00) != 0xd800) - { - if (PRINTABLE(c)) fprintf(f, "%c", (char)c); - else if (c <= 0x80) fprintf(f, "\\x%02x", c); - else fprintf(f, "\\x{%02x}", c); - return 0; - } -else - { - /* This is a check for malformed UTF-16; it should only occur if the sanity - check has been turned off. Rather than swallow a low surrogate, just stop if - we hit a bad one. Print it with \X instead of \x as an indication. */ - - if ((ptr[1] & 0xfc00) != 0xdc00) - { - fprintf(f, "\\X{%x}", c); - return 0; - } - - c = (((c & 0x3ff) << 10) | (ptr[1] & 0x3ff)) + 0x10000; - fprintf(f, "\\x{%x}", c); - return 1; - } - -#elif defined COMPILE_PCRE32 - -if (!utf || (c & 0xfffff800u) != 0xd800u) - { - if (PRINTABLE(c)) fprintf(f, "%c", (char)c); - else if (c <= 0x80) fprintf(f, "\\x%02x", c); - else fprintf(f, "\\x{%x}", c); - return 0; - } -else - { - /* This is a check for malformed UTF-32; it should only occur if the sanity - check has been turned off. Rather than swallow a surrogate, just stop if - we hit one. Print it with \X instead of \x as an indication. */ - fprintf(f, "\\X{%x}", c); - return 0; - } - -#endif /* COMPILE_PCRE[8|16|32] */ - -#endif /* SUPPORT_UTF */ -} - -/************************************************* -* Print uchar string (regardless of utf) * -*************************************************/ - -static void -print_puchar(FILE *f, PCRE_PUCHAR ptr) -{ -while (*ptr != '\0') - { - register pcre_uint32 c = *ptr++; - if (PRINTABLE(c)) fprintf(f, "%c", c); else fprintf(f, "\\x{%x}", c); - } -} - -/************************************************* -* Find Unicode property name * -*************************************************/ - -static const char * -get_ucpname(unsigned int ptype, unsigned int pvalue) -{ -#ifdef SUPPORT_UCP -int i; -for (i = PRIV(utt_size) - 1; i >= 0; i--) - { - if (ptype == PRIV(utt)[i].type && pvalue == PRIV(utt)[i].value) break; - } -return (i >= 0)? PRIV(utt_names) + PRIV(utt)[i].name_offset : "??"; -#else -/* It gets harder and harder to shut off unwanted compiler warnings. */ -ptype = ptype * pvalue; -return (ptype == pvalue)? "??" : "??"; -#endif -} - - -/************************************************* -* Print Unicode property value * -*************************************************/ - -/* "Normal" properties can be printed from tables. The PT_CLIST property is a -pseudo-property that contains a pointer to a list of case-equivalent -characters. This is used only when UCP support is available and UTF mode is -selected. It should never occur otherwise, but just in case it does, have -something ready to print. */ - -static void -print_prop(FILE *f, pcre_uchar *code, const char *before, const char *after) -{ -if (code[1] != PT_CLIST) - { - fprintf(f, "%s%s %s%s", before, priv_OP_names[*code], get_ucpname(code[1], - code[2]), after); - } -else - { - const char *not = (*code == OP_PROP)? "" : "not "; -#ifndef SUPPORT_UCP - fprintf(f, "%s%sclist %d%s", before, not, code[2], after); -#else - const pcre_uint32 *p = PRIV(ucd_caseless_sets) + code[2]; - fprintf (f, "%s%sclist", before, not); - while (*p < NOTACHAR) fprintf(f, " %04x", *p++); - fprintf(f, "%s", after); -#endif - } -} - - - - -/************************************************* -* Print compiled regex * -*************************************************/ - -/* Make this function work for a regex with integers either byte order. -However, we assume that what we are passed is a compiled regex. The -print_lengths flag controls whether offsets and lengths of items are printed. -They can be turned off from pcretest so that automatic tests on bytecode can be -written that do not depend on the value of LINK_SIZE. */ - -#ifdef PCRE_INCLUDED -static /* Keep the following function as private. */ -#endif -#if defined COMPILE_PCRE8 -void -pcre_printint(pcre *external_re, FILE *f, BOOL print_lengths) -#elif defined COMPILE_PCRE16 -void -pcre16_printint(pcre *external_re, FILE *f, BOOL print_lengths) -#elif defined COMPILE_PCRE32 -void -pcre32_printint(pcre *external_re, FILE *f, BOOL print_lengths) -#endif -{ -REAL_PCRE *re = (REAL_PCRE *)external_re; -pcre_uchar *codestart, *code; -BOOL utf; - -unsigned int options = re->options; -int offset = re->name_table_offset; -int count = re->name_count; -int size = re->name_entry_size; - -if (re->magic_number != MAGIC_NUMBER) - { - offset = ((offset << 8) & 0xff00) | ((offset >> 8) & 0xff); - count = ((count << 8) & 0xff00) | ((count >> 8) & 0xff); - size = ((size << 8) & 0xff00) | ((size >> 8) & 0xff); - options = ((options << 24) & 0xff000000) | - ((options << 8) & 0x00ff0000) | - ((options >> 8) & 0x0000ff00) | - ((options >> 24) & 0x000000ff); - } - -code = codestart = (pcre_uchar *)re + offset + count * size; -/* PCRE_UTF(16|32) have the same value as PCRE_UTF8. */ -utf = (options & PCRE_UTF8) != 0; - -for(;;) - { - pcre_uchar *ccode; - const char *flag = " "; - pcre_uint32 c; - unsigned int extra = 0; - - if (print_lengths) - fprintf(f, "%3d ", (int)(code - codestart)); - else - fprintf(f, " "); - - switch(*code) - { -/* ========================================================================== */ - /* These cases are never obeyed. This is a fudge that causes a compile- - time error if the vectors OP_names or OP_lengths, which are indexed - by opcode, are not the correct length. It seems to be the only way to do - such a check at compile time, as the sizeof() operator does not work in - the C preprocessor. */ - - case OP_TABLE_LENGTH: - case OP_TABLE_LENGTH + - ((sizeof(priv_OP_names)/sizeof(const char *) == OP_TABLE_LENGTH) && - (sizeof(priv_OP_lengths) == OP_TABLE_LENGTH)): - break; -/* ========================================================================== */ - - case OP_END: - fprintf(f, " %s\n", priv_OP_names[*code]); - fprintf(f, "------------------------------------------------------------------\n"); - return; - - case OP_CHAR: - fprintf(f, " "); - do - { - code++; - code += 1 + print_char(f, code, utf); - } - while (*code == OP_CHAR); - fprintf(f, "\n"); - continue; - - case OP_CHARI: - fprintf(f, " /i "); - do - { - code++; - code += 1 + print_char(f, code, utf); - } - while (*code == OP_CHARI); - fprintf(f, "\n"); - continue; - - case OP_CBRA: - case OP_CBRAPOS: - case OP_SCBRA: - case OP_SCBRAPOS: - if (print_lengths) fprintf(f, "%3d ", GET(code, 1)); - else fprintf(f, " "); - fprintf(f, "%s %d", priv_OP_names[*code], GET2(code, 1+LINK_SIZE)); - break; - - case OP_BRA: - case OP_BRAPOS: - case OP_SBRA: - case OP_SBRAPOS: - case OP_KETRMAX: - case OP_KETRMIN: - case OP_KETRPOS: - case OP_ALT: - case OP_KET: - case OP_ASSERT: - case OP_ASSERT_NOT: - case OP_ASSERTBACK: - case OP_ASSERTBACK_NOT: - case OP_ONCE: - case OP_ONCE_NC: - case OP_COND: - case OP_SCOND: - case OP_REVERSE: - if (print_lengths) fprintf(f, "%3d ", GET(code, 1)); - else fprintf(f, " "); - fprintf(f, "%s", priv_OP_names[*code]); - break; - - case OP_CLOSE: - fprintf(f, " %s %d", priv_OP_names[*code], GET2(code, 1)); - break; - - case OP_CREF: - fprintf(f, "%3d %s", GET2(code,1), priv_OP_names[*code]); - break; - - case OP_DNCREF: - { - pcre_uchar *entry = (pcre_uchar *)re + offset + (GET2(code, 1) * size) + - IMM2_SIZE; - fprintf(f, " %s Cond ref <", flag); - print_puchar(f, entry); - fprintf(f, ">%d", GET2(code, 1 + IMM2_SIZE)); - } - break; - - case OP_RREF: - c = GET2(code, 1); - if (c == RREF_ANY) - fprintf(f, " Cond recurse any"); - else - fprintf(f, " Cond recurse %d", c); - break; - - case OP_DNRREF: - { - pcre_uchar *entry = (pcre_uchar *)re + offset + (GET2(code, 1) * size) + - IMM2_SIZE; - fprintf(f, " %s Cond recurse <", flag); - print_puchar(f, entry); - fprintf(f, ">%d", GET2(code, 1 + IMM2_SIZE)); - } - break; - - case OP_DEF: - fprintf(f, " Cond def"); - break; - - case OP_STARI: - case OP_MINSTARI: - case OP_POSSTARI: - case OP_PLUSI: - case OP_MINPLUSI: - case OP_POSPLUSI: - case OP_QUERYI: - case OP_MINQUERYI: - case OP_POSQUERYI: - flag = "/i"; - /* Fall through */ - case OP_STAR: - case OP_MINSTAR: - case OP_POSSTAR: - case OP_PLUS: - case OP_MINPLUS: - case OP_POSPLUS: - case OP_QUERY: - case OP_MINQUERY: - case OP_POSQUERY: - case OP_TYPESTAR: - case OP_TYPEMINSTAR: - case OP_TYPEPOSSTAR: - case OP_TYPEPLUS: - case OP_TYPEMINPLUS: - case OP_TYPEPOSPLUS: - case OP_TYPEQUERY: - case OP_TYPEMINQUERY: - case OP_TYPEPOSQUERY: - fprintf(f, " %s ", flag); - if (*code >= OP_TYPESTAR) - { - if (code[1] == OP_PROP || code[1] == OP_NOTPROP) - { - print_prop(f, code + 1, "", " "); - extra = 2; - } - else fprintf(f, "%s", priv_OP_names[code[1]]); - } - else extra = print_char(f, code+1, utf); - fprintf(f, "%s", priv_OP_names[*code]); - break; - - case OP_EXACTI: - case OP_UPTOI: - case OP_MINUPTOI: - case OP_POSUPTOI: - flag = "/i"; - /* Fall through */ - case OP_EXACT: - case OP_UPTO: - case OP_MINUPTO: - case OP_POSUPTO: - fprintf(f, " %s ", flag); - extra = print_char(f, code + 1 + IMM2_SIZE, utf); - fprintf(f, "{"); - if (*code != OP_EXACT && *code != OP_EXACTI) fprintf(f, "0,"); - fprintf(f, "%d}", GET2(code,1)); - if (*code == OP_MINUPTO || *code == OP_MINUPTOI) fprintf(f, "?"); - else if (*code == OP_POSUPTO || *code == OP_POSUPTOI) fprintf(f, "+"); - break; - - case OP_TYPEEXACT: - case OP_TYPEUPTO: - case OP_TYPEMINUPTO: - case OP_TYPEPOSUPTO: - if (code[1 + IMM2_SIZE] == OP_PROP || code[1 + IMM2_SIZE] == OP_NOTPROP) - { - print_prop(f, code + IMM2_SIZE + 1, " ", " "); - extra = 2; - } - else fprintf(f, " %s", priv_OP_names[code[1 + IMM2_SIZE]]); - fprintf(f, "{"); - if (*code != OP_TYPEEXACT) fprintf(f, "0,"); - fprintf(f, "%d}", GET2(code,1)); - if (*code == OP_TYPEMINUPTO) fprintf(f, "?"); - else if (*code == OP_TYPEPOSUPTO) fprintf(f, "+"); - break; - - case OP_NOTI: - flag = "/i"; - /* Fall through */ - case OP_NOT: - fprintf(f, " %s [^", flag); - extra = print_char(f, code + 1, utf); - fprintf(f, "]"); - break; - - case OP_NOTSTARI: - case OP_NOTMINSTARI: - case OP_NOTPOSSTARI: - case OP_NOTPLUSI: - case OP_NOTMINPLUSI: - case OP_NOTPOSPLUSI: - case OP_NOTQUERYI: - case OP_NOTMINQUERYI: - case OP_NOTPOSQUERYI: - flag = "/i"; - /* Fall through */ - - case OP_NOTSTAR: - case OP_NOTMINSTAR: - case OP_NOTPOSSTAR: - case OP_NOTPLUS: - case OP_NOTMINPLUS: - case OP_NOTPOSPLUS: - case OP_NOTQUERY: - case OP_NOTMINQUERY: - case OP_NOTPOSQUERY: - fprintf(f, " %s [^", flag); - extra = print_char(f, code + 1, utf); - fprintf(f, "]%s", priv_OP_names[*code]); - break; - - case OP_NOTEXACTI: - case OP_NOTUPTOI: - case OP_NOTMINUPTOI: - case OP_NOTPOSUPTOI: - flag = "/i"; - /* Fall through */ - - case OP_NOTEXACT: - case OP_NOTUPTO: - case OP_NOTMINUPTO: - case OP_NOTPOSUPTO: - fprintf(f, " %s [^", flag); - extra = print_char(f, code + 1 + IMM2_SIZE, utf); - fprintf(f, "]{"); - if (*code != OP_NOTEXACT && *code != OP_NOTEXACTI) fprintf(f, "0,"); - fprintf(f, "%d}", GET2(code,1)); - if (*code == OP_NOTMINUPTO || *code == OP_NOTMINUPTOI) fprintf(f, "?"); - else - if (*code == OP_NOTPOSUPTO || *code == OP_NOTPOSUPTOI) fprintf(f, "+"); - break; - - case OP_RECURSE: - if (print_lengths) fprintf(f, "%3d ", GET(code, 1)); - else fprintf(f, " "); - fprintf(f, "%s", priv_OP_names[*code]); - break; - - case OP_REFI: - flag = "/i"; - /* Fall through */ - case OP_REF: - fprintf(f, " %s \\%d", flag, GET2(code,1)); - ccode = code + priv_OP_lengths[*code]; - goto CLASS_REF_REPEAT; - - case OP_DNREFI: - flag = "/i"; - /* Fall through */ - case OP_DNREF: - { - pcre_uchar *entry = (pcre_uchar *)re + offset + (GET2(code, 1) * size) + - IMM2_SIZE; - fprintf(f, " %s \\k<", flag); - print_puchar(f, entry); - fprintf(f, ">%d", GET2(code, 1 + IMM2_SIZE)); - } - ccode = code + priv_OP_lengths[*code]; - goto CLASS_REF_REPEAT; - - case OP_CALLOUT: - fprintf(f, " %s %d %d %d", priv_OP_names[*code], code[1], GET(code,2), - GET(code, 2 + LINK_SIZE)); - break; - - case OP_PROP: - case OP_NOTPROP: - print_prop(f, code, " ", ""); - break; - - /* OP_XCLASS cannot occur in 8-bit, non-UTF mode. However, there's no harm - in having this code always here, and it makes it less messy without all - those #ifdefs. */ - - case OP_CLASS: - case OP_NCLASS: - case OP_XCLASS: - { - int i; - unsigned int min, max; - BOOL printmap; - BOOL invertmap = FALSE; - pcre_uint8 *map; - pcre_uint8 inverted_map[32]; - - fprintf(f, " ["); - - if (*code == OP_XCLASS) - { - extra = GET(code, 1); - ccode = code + LINK_SIZE + 1; - printmap = (*ccode & XCL_MAP) != 0; - if ((*ccode & XCL_NOT) != 0) - { - invertmap = (*ccode & XCL_HASPROP) == 0; - fprintf(f, "^"); - } - ccode++; - } - else - { - printmap = TRUE; - ccode = code + 1; - } - - /* Print a bit map */ - - if (printmap) - { - map = (pcre_uint8 *)ccode; - if (invertmap) - { - for (i = 0; i < 32; i++) inverted_map[i] = ~map[i]; - map = inverted_map; - } - - for (i = 0; i < 256; i++) - { - if ((map[i/8] & (1 << (i&7))) != 0) - { - int j; - for (j = i+1; j < 256; j++) - if ((map[j/8] & (1 << (j&7))) == 0) break; - if (i == '-' || i == ']') fprintf(f, "\\"); - if (PRINTABLE(i)) fprintf(f, "%c", i); - else fprintf(f, "\\x%02x", i); - if (--j > i) - { - if (j != i + 1) fprintf(f, "-"); - if (j == '-' || j == ']') fprintf(f, "\\"); - if (PRINTABLE(j)) fprintf(f, "%c", j); - else fprintf(f, "\\x%02x", j); - } - i = j; - } - } - ccode += 32 / sizeof(pcre_uchar); - } - - /* For an XCLASS there is always some additional data */ - - if (*code == OP_XCLASS) - { - pcre_uchar ch; - while ((ch = *ccode++) != XCL_END) - { - BOOL not = FALSE; - const char *notch = ""; - - switch(ch) - { - case XCL_NOTPROP: - not = TRUE; - notch = "^"; - /* Fall through */ - - case XCL_PROP: - { - unsigned int ptype = *ccode++; - unsigned int pvalue = *ccode++; - - switch(ptype) - { - case PT_PXGRAPH: - fprintf(f, "[:%sgraph:]", notch); - break; - - case PT_PXPRINT: - fprintf(f, "[:%sprint:]", notch); - break; - - case PT_PXPUNCT: - fprintf(f, "[:%spunct:]", notch); - break; - - default: - fprintf(f, "\\%c{%s}", (not? 'P':'p'), - get_ucpname(ptype, pvalue)); - break; - } - } - break; - - default: - ccode += 1 + print_char(f, ccode, utf); - if (ch == XCL_RANGE) - { - fprintf(f, "-"); - ccode += 1 + print_char(f, ccode, utf); - } - break; - } - } - } - - /* Indicate a non-UTF class which was created by negation */ - - fprintf(f, "]%s", (*code == OP_NCLASS)? " (neg)" : ""); - - /* Handle repeats after a class or a back reference */ - - CLASS_REF_REPEAT: - switch(*ccode) - { - case OP_CRSTAR: - case OP_CRMINSTAR: - case OP_CRPLUS: - case OP_CRMINPLUS: - case OP_CRQUERY: - case OP_CRMINQUERY: - case OP_CRPOSSTAR: - case OP_CRPOSPLUS: - case OP_CRPOSQUERY: - fprintf(f, "%s", priv_OP_names[*ccode]); - extra += priv_OP_lengths[*ccode]; - break; - - case OP_CRRANGE: - case OP_CRMINRANGE: - case OP_CRPOSRANGE: - min = GET2(ccode,1); - max = GET2(ccode,1 + IMM2_SIZE); - if (max == 0) fprintf(f, "{%u,}", min); - else fprintf(f, "{%u,%u}", min, max); - if (*ccode == OP_CRMINRANGE) fprintf(f, "?"); - else if (*ccode == OP_CRPOSRANGE) fprintf(f, "+"); - extra += priv_OP_lengths[*ccode]; - break; - - /* Do nothing if it's not a repeat; this code stops picky compilers - warning about the lack of a default code path. */ - - default: - break; - } - } - break; - - case OP_MARK: - case OP_PRUNE_ARG: - case OP_SKIP_ARG: - case OP_THEN_ARG: - fprintf(f, " %s ", priv_OP_names[*code]); - print_puchar(f, code + 2); - extra += code[1]; - break; - - case OP_THEN: - fprintf(f, " %s", priv_OP_names[*code]); - break; - - case OP_CIRCM: - case OP_DOLLM: - flag = "/m"; - /* Fall through */ - - /* Anything else is just an item with no data, but possibly a flag. */ - - default: - fprintf(f, " %s %s", flag, priv_OP_names[*code]); - break; - } - - code += priv_OP_lengths[*code] + extra; - fprintf(f, "\n"); - } -} - -/* End of pcre_printint.src */ diff --git a/src/third_party/pcre-8.39/pcre_refcount.c b/src/third_party/pcre-8.39/pcre_refcount.c deleted file mode 100644 index 79efa90f216..00000000000 --- a/src/third_party/pcre-8.39/pcre_refcount.c +++ /dev/null @@ -1,92 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This module contains the external function pcre_refcount(), which is an -auxiliary function that can be used to maintain a reference count in a compiled -pattern data block. This might be helpful in applications where the block is -shared by different users. */ - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "pcre_internal.h" - - -/************************************************* -* Maintain reference count * -*************************************************/ - -/* The reference count is a 16-bit field, initialized to zero. It is not -possible to transfer a non-zero count from one host to a different host that -has a different byte order - though I can't see why anyone in their right mind -would ever want to do that! - -Arguments: - argument_re points to compiled code - adjust value to add to the count - -Returns: the (possibly updated) count value (a non-negative number), or - a negative error number -*/ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre_refcount(pcre *argument_re, int adjust) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre16_refcount(pcre16 *argument_re, int adjust) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN int PCRE_CALL_CONVENTION -pcre32_refcount(pcre32 *argument_re, int adjust) -#endif -{ -REAL_PCRE *re = (REAL_PCRE *)argument_re; -if (re == NULL) return PCRE_ERROR_NULL; -if (re->magic_number != MAGIC_NUMBER) return PCRE_ERROR_BADMAGIC; -if ((re->flags & PCRE_MODE) == 0) return PCRE_ERROR_BADMODE; -re->ref_count = (-adjust > re->ref_count)? 0 : - (adjust + re->ref_count > 65535)? 65535 : - re->ref_count + adjust; -return re->ref_count; -} - -/* End of pcre_refcount.c */ diff --git a/src/third_party/pcre-8.39/pcre_scanner.cc b/src/third_party/pcre-8.39/pcre_scanner.cc deleted file mode 100644 index 6be2be6829b..00000000000 --- a/src/third_party/pcre-8.39/pcre_scanner.cc +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Author: Sanjay Ghemawat - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include - -#include "pcrecpp_internal.h" -#include "pcre_scanner.h" - -using std::vector; - -namespace pcrecpp { - -Scanner::Scanner() - : data_(), - input_(data_), - skip_(NULL), - should_skip_(false), - skip_repeat_(false), - save_comments_(false), - comments_(NULL), - comments_offset_(0) { -} - -Scanner::Scanner(const string& in) - : data_(in), - input_(data_), - skip_(NULL), - should_skip_(false), - skip_repeat_(false), - save_comments_(false), - comments_(NULL), - comments_offset_(0) { -} - -Scanner::~Scanner() { - delete skip_; - delete comments_; -} - -void Scanner::SetSkipExpression(const char* re) { - delete skip_; - if (re != NULL) { - skip_ = new RE(re); - should_skip_ = true; - skip_repeat_ = true; - ConsumeSkip(); - } else { - skip_ = NULL; - should_skip_ = false; - skip_repeat_ = false; - } -} - -void Scanner::Skip(const char* re) { - delete skip_; - if (re != NULL) { - skip_ = new RE(re); - should_skip_ = true; - skip_repeat_ = false; - ConsumeSkip(); - } else { - skip_ = NULL; - should_skip_ = false; - skip_repeat_ = false; - } -} - -void Scanner::DisableSkip() { - assert(skip_ != NULL); - should_skip_ = false; -} - -void Scanner::EnableSkip() { - assert(skip_ != NULL); - should_skip_ = true; - ConsumeSkip(); -} - -int Scanner::LineNumber() const { - // TODO: Make it more efficient by keeping track of the last point - // where we computed line numbers and counting newlines since then. - // We could use std:count, but not all systems have it. :-( - int count = 1; - for (const char* p = data_.data(); p < input_.data(); ++p) - if (*p == '\n') - ++count; - return count; -} - -int Scanner::Offset() const { - return (int)(input_.data() - data_.c_str()); -} - -bool Scanner::LookingAt(const RE& re) const { - int consumed; - return re.DoMatch(input_, RE::ANCHOR_START, &consumed, 0, 0); -} - - -bool Scanner::Consume(const RE& re, - const Arg& arg0, - const Arg& arg1, - const Arg& arg2) { - const bool result = re.Consume(&input_, arg0, arg1, arg2); - if (result && should_skip_) ConsumeSkip(); - return result; -} - -// helper function to consume *skip_ and honour save_comments_ -void Scanner::ConsumeSkip() { - const char* start_data = input_.data(); - while (skip_->Consume(&input_)) { - if (!skip_repeat_) { - // Only one skip allowed. - break; - } - } - if (save_comments_) { - if (comments_ == NULL) { - comments_ = new vector; - } - // already pointing one past end, so no need to +1 - int length = (int)(input_.data() - start_data); - if (length > 0) { - comments_->push_back(StringPiece(start_data, length)); - } - } -} - - -void Scanner::GetComments(int start, int end, vector *ranges) { - // short circuit out if we've not yet initialized comments_ - // (e.g., when save_comments is false) - if (!comments_) { - return; - } - // TODO: if we guarantee that comments_ will contain StringPieces - // that are ordered by their start, then we can do a binary search - // for the first StringPiece at or past start and then scan for the - // ones contained in the range, quit early (use equal_range or - // lower_bound) - for (vector::const_iterator it = comments_->begin(); - it != comments_->end(); ++it) { - if ((it->data() >= data_.c_str() + start && - it->data() + it->size() <= data_.c_str() + end)) { - ranges->push_back(*it); - } - } -} - - -void Scanner::GetNextComments(vector *ranges) { - // short circuit out if we've not yet initialized comments_ - // (e.g., when save_comments is false) - if (!comments_) { - return; - } - for (vector::const_iterator it = - comments_->begin() + comments_offset_; - it != comments_->end(); ++it) { - ranges->push_back(*it); - ++comments_offset_; - } -} - -} // namespace pcrecpp diff --git a/src/third_party/pcre-8.39/pcre_scanner.h b/src/third_party/pcre-8.39/pcre_scanner.h deleted file mode 100644 index 5617e4515cb..00000000000 --- a/src/third_party/pcre-8.39/pcre_scanner.h +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Author: Sanjay Ghemawat -// -// Regular-expression based scanner for parsing an input stream. -// -// Example 1: parse a sequence of "var = number" entries from input: -// -// Scanner scanner(input); -// string var; -// int number; -// scanner.SetSkipExpression("\\s+"); // Skip any white space we encounter -// while (scanner.Consume("(\\w+) = (\\d+)", &var, &number)) { -// ...; -// } - -#ifndef _PCRE_SCANNER_H -#define _PCRE_SCANNER_H - -#include -#include -#include - -#include -#include - -namespace pcrecpp { - -class PCRECPP_EXP_DEFN Scanner { - public: - Scanner(); - explicit Scanner(const std::string& input); - ~Scanner(); - - // Return current line number. The returned line-number is - // one-based. I.e. it returns 1 + the number of consumed newlines. - // - // Note: this method may be slow. It may take time proportional to - // the size of the input. - int LineNumber() const; - - // Return the byte-offset that the scanner is looking in the - // input data; - int Offset() const; - - // Return true iff the start of the remaining input matches "re" - bool LookingAt(const RE& re) const; - - // Return true iff all of the following are true - // a. the start of the remaining input matches "re", - // b. if any arguments are supplied, matched sub-patterns can be - // parsed and stored into the arguments. - // If it returns true, it skips over the matched input and any - // following input that matches the "skip" regular expression. - bool Consume(const RE& re, - const Arg& arg0 = RE::no_arg, - const Arg& arg1 = RE::no_arg, - const Arg& arg2 = RE::no_arg - // TODO: Allow more arguments? - ); - - // Set the "skip" regular expression. If after consuming some data, - // a prefix of the input matches this RE, it is automatically - // skipped. For example, a programming language scanner would use - // a skip RE that matches white space and comments. - // - // scanner.SetSkipExpression("\\s+|//.*|/[*](.|\n)*?[*]/"); - // - // Skipping repeats as long as it succeeds. We used to let people do - // this by writing "(...)*" in the regular expression, but that added - // up to lots of recursive calls within the pcre library, so now we - // control repetition explicitly via the function call API. - // - // You can pass NULL for "re" if you do not want any data to be skipped. - void Skip(const char* re); // DEPRECATED; does *not* repeat - void SetSkipExpression(const char* re); - - // Temporarily pause "skip"ing. This - // Skip("Foo"); code ; DisableSkip(); code; EnableSkip() - // is similar to - // Skip("Foo"); code ; Skip(NULL); code ; Skip("Foo"); - // but avoids creating/deleting new RE objects. - void DisableSkip(); - - // Reenable previously paused skipping. Any prefix of the input - // that matches the skip pattern is immediately dropped. - void EnableSkip(); - - /***** Special wrappers around SetSkip() for some common idioms *****/ - - // Arranges to skip whitespace, C comments, C++ comments. - // The overall RE is a disjunction of the following REs: - // \\s whitespace - // //.*\n C++ comment - // /[*](.|\n)*?[*]/ C comment (x*? means minimal repetitions of x) - // We get repetition via the semantics of SetSkipExpression, not by using * - void SkipCXXComments() { - SetSkipExpression("\\s|//.*\n|/[*](?:\n|.)*?[*]/"); - } - - void set_save_comments(bool comments) { - save_comments_ = comments; - } - - bool save_comments() { - return save_comments_; - } - - // Append to vector ranges the comments found in the - // byte range [start,end] (inclusive) of the input data. - // Only comments that were extracted entirely within that - // range are returned: no range splitting of atomically-extracted - // comments is performed. - void GetComments(int start, int end, std::vector *ranges); - - // Append to vector ranges the comments added - // since the last time this was called. This - // functionality is provided for efficiency when - // interleaving scanning with parsing. - void GetNextComments(std::vector *ranges); - - private: - std::string data_; // All the input data - StringPiece input_; // Unprocessed input - RE* skip_; // If non-NULL, RE for skipping input - bool should_skip_; // If true, use skip_ - bool skip_repeat_; // If true, repeat skip_ as long as it works - bool save_comments_; // If true, aggregate the skip expression - - // the skipped comments - // TODO: later consider requiring that the StringPieces be added - // in order by their start position - std::vector *comments_; - - // the offset into comments_ that has been returned by GetNextComments - int comments_offset_; - - // helper function to consume *skip_ and honour - // save_comments_ - void ConsumeSkip(); -}; - -} // namespace pcrecpp - -#endif /* _PCRE_SCANNER_H */ diff --git a/src/third_party/pcre-8.39/pcre_scanner_unittest.cc b/src/third_party/pcre-8.39/pcre_scanner_unittest.cc deleted file mode 100644 index c00312c4f63..00000000000 --- a/src/third_party/pcre-8.39/pcre_scanner_unittest.cc +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Author: Greg J. Badros -// -// Unittest for scanner, especially GetNextComments and GetComments() -// functionality. - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include /* for strchr */ -#include -#include - -#include "pcrecpp.h" -#include "pcre_stringpiece.h" -#include "pcre_scanner.h" - -#define FLAGS_unittest_stack_size 49152 - -// Dies with a fatal error if the two values are not equal. -#define CHECK_EQ(a, b) do { \ - if ( (a) != (b) ) { \ - fprintf(stderr, "%s:%d: Check failed because %s != %s\n", \ - __FILE__, __LINE__, #a, #b); \ - exit(1); \ - } \ -} while (0) - -using std::vector; -using pcrecpp::StringPiece; -using pcrecpp::Scanner; - -static void TestScanner() { - const char input[] = "\n" - "alpha = 1; // this sets alpha\n" - "bravo = 2; // bravo is set here\n" - "gamma = 33; /* and here is gamma */\n"; - - const char *re = "(\\w+) = (\\d+);"; - - Scanner s(input); - string var; - int number; - s.SkipCXXComments(); - s.set_save_comments(true); - vector comments; - - s.Consume(re, &var, &number); - CHECK_EQ(var, "alpha"); - CHECK_EQ(number, 1); - CHECK_EQ(s.LineNumber(), 3); - s.GetNextComments(&comments); - CHECK_EQ(comments.size(), 1); - CHECK_EQ(comments[0].as_string(), " // this sets alpha\n"); - comments.resize(0); - - s.Consume(re, &var, &number); - CHECK_EQ(var, "bravo"); - CHECK_EQ(number, 2); - s.GetNextComments(&comments); - CHECK_EQ(comments.size(), 1); - CHECK_EQ(comments[0].as_string(), " // bravo is set here\n"); - comments.resize(0); - - s.Consume(re, &var, &number); - CHECK_EQ(var, "gamma"); - CHECK_EQ(number, 33); - s.GetNextComments(&comments); - CHECK_EQ(comments.size(), 1); - CHECK_EQ(comments[0].as_string(), " /* and here is gamma */\n"); - comments.resize(0); - - s.GetComments(0, sizeof(input), &comments); - CHECK_EQ(comments.size(), 3); - CHECK_EQ(comments[0].as_string(), " // this sets alpha\n"); - CHECK_EQ(comments[1].as_string(), " // bravo is set here\n"); - CHECK_EQ(comments[2].as_string(), " /* and here is gamma */\n"); - comments.resize(0); - - s.GetComments(0, (int)(strchr(input, '/') - input), &comments); - CHECK_EQ(comments.size(), 0); - comments.resize(0); - - s.GetComments((int)(strchr(input, '/') - input - 1), sizeof(input), - &comments); - CHECK_EQ(comments.size(), 3); - CHECK_EQ(comments[0].as_string(), " // this sets alpha\n"); - CHECK_EQ(comments[1].as_string(), " // bravo is set here\n"); - CHECK_EQ(comments[2].as_string(), " /* and here is gamma */\n"); - comments.resize(0); - - s.GetComments((int)(strchr(input, '/') - input - 1), - (int)(strchr(input + 1, '\n') - input + 1), &comments); - CHECK_EQ(comments.size(), 1); - CHECK_EQ(comments[0].as_string(), " // this sets alpha\n"); - comments.resize(0); -} - -static void TestBigComment() { - string input; - for (int i = 0; i < 1024; ++i) { - char buf[1024]; // definitely big enough - sprintf(buf, " # Comment %d\n", i); - input += buf; - } - input += "name = value;\n"; - - Scanner s(input.c_str()); - s.SetSkipExpression("\\s+|#.*\n"); - - string name; - string value; - s.Consume("(\\w+) = (\\w+);", &name, &value); - CHECK_EQ(name, "name"); - CHECK_EQ(value, "value"); -} - -// TODO: also test scanner and big-comment in a thread with a -// small stack size - -int main(int argc, char** argv) { - (void)argc; - (void)argv; - TestScanner(); - TestBigComment(); - - // Done - printf("OK\n"); - - return 0; -} diff --git a/src/third_party/pcre-8.39/pcre_string_utils.c b/src/third_party/pcre-8.39/pcre_string_utils.c deleted file mode 100644 index 25eacc85073..00000000000 --- a/src/third_party/pcre-8.39/pcre_string_utils.c +++ /dev/null @@ -1,211 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2014 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This module contains internal functions for comparing and finding the length -of strings for different data item sizes. */ - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "pcre_internal.h" - -#ifndef COMPILE_PCRE8 - -/************************************************* -* Compare string utilities * -*************************************************/ - -/* The following two functions compares two strings. Basically a strcmp -for non 8 bit characters. - -Arguments: - str1 first string - str2 second string - -Returns: 0 if both string are equal (like strcmp), 1 otherwise -*/ - -int -PRIV(strcmp_uc_uc)(const pcre_uchar *str1, const pcre_uchar *str2) -{ -pcre_uchar c1; -pcre_uchar c2; - -while (*str1 != '\0' || *str2 != '\0') - { - c1 = *str1++; - c2 = *str2++; - if (c1 != c2) - return ((c1 > c2) << 1) - 1; - } -/* Both length and characters must be equal. */ -return 0; -} - -#ifdef COMPILE_PCRE32 - -int -PRIV(strcmp_uc_uc_utf)(const pcre_uchar *str1, const pcre_uchar *str2) -{ -pcre_uchar c1; -pcre_uchar c2; - -while (*str1 != '\0' || *str2 != '\0') - { - c1 = UCHAR21INC(str1); - c2 = UCHAR21INC(str2); - if (c1 != c2) - return ((c1 > c2) << 1) - 1; - } -/* Both length and characters must be equal. */ -return 0; -} - -#endif /* COMPILE_PCRE32 */ - -int -PRIV(strcmp_uc_c8)(const pcre_uchar *str1, const char *str2) -{ -const pcre_uint8 *ustr2 = (pcre_uint8 *)str2; -pcre_uchar c1; -pcre_uchar c2; - -while (*str1 != '\0' || *ustr2 != '\0') - { - c1 = *str1++; - c2 = (pcre_uchar)*ustr2++; - if (c1 != c2) - return ((c1 > c2) << 1) - 1; - } -/* Both length and characters must be equal. */ -return 0; -} - -#ifdef COMPILE_PCRE32 - -int -PRIV(strcmp_uc_c8_utf)(const pcre_uchar *str1, const char *str2) -{ -const pcre_uint8 *ustr2 = (pcre_uint8 *)str2; -pcre_uchar c1; -pcre_uchar c2; - -while (*str1 != '\0' || *ustr2 != '\0') - { - c1 = UCHAR21INC(str1); - c2 = (pcre_uchar)*ustr2++; - if (c1 != c2) - return ((c1 > c2) << 1) - 1; - } -/* Both length and characters must be equal. */ -return 0; -} - -#endif /* COMPILE_PCRE32 */ - -/* The following two functions compares two, fixed length -strings. Basically an strncmp for non 8 bit characters. - -Arguments: - str1 first string - str2 second string - num size of the string - -Returns: 0 if both string are equal (like strcmp), 1 otherwise -*/ - -int -PRIV(strncmp_uc_uc)(const pcre_uchar *str1, const pcre_uchar *str2, unsigned int num) -{ -pcre_uchar c1; -pcre_uchar c2; - -while (num-- > 0) - { - c1 = *str1++; - c2 = *str2++; - if (c1 != c2) - return ((c1 > c2) << 1) - 1; - } -/* Both length and characters must be equal. */ -return 0; -} - -int -PRIV(strncmp_uc_c8)(const pcre_uchar *str1, const char *str2, unsigned int num) -{ -const pcre_uint8 *ustr2 = (pcre_uint8 *)str2; -pcre_uchar c1; -pcre_uchar c2; - -while (num-- > 0) - { - c1 = *str1++; - c2 = (pcre_uchar)*ustr2++; - if (c1 != c2) - return ((c1 > c2) << 1) - 1; - } -/* Both length and characters must be equal. */ -return 0; -} - -/* The following function returns with the length of -a zero terminated string. Basically an strlen for non 8 bit characters. - -Arguments: - str string - -Returns: length of the string -*/ - -unsigned int -PRIV(strlen_uc)(const pcre_uchar *str) -{ -unsigned int len = 0; -while (*str++ != 0) - len++; -return len; -} - -#endif /* !COMPILE_PCRE8 */ - -/* End of pcre_string_utils.c */ diff --git a/src/third_party/pcre-8.39/pcre_stringpiece.cc b/src/third_party/pcre-8.39/pcre_stringpiece.cc deleted file mode 100644 index 67c0f1fc0e5..00000000000 --- a/src/third_party/pcre-8.39/pcre_stringpiece.cc +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Author: wilsonh@google.com (Wilson Hsieh) -// - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include "pcrecpp_internal.h" -#include "pcre_stringpiece.h" - -std::ostream& operator<<(std::ostream& o, const pcrecpp::StringPiece& piece) { - return (o << piece.as_string()); -} diff --git a/src/third_party/pcre-8.39/pcre_stringpiece.h b/src/third_party/pcre-8.39/pcre_stringpiece.h deleted file mode 100644 index cc3dc42963a..00000000000 --- a/src/third_party/pcre-8.39/pcre_stringpiece.h +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Author: Sanjay Ghemawat -// -// A string like object that points into another piece of memory. -// Useful for providing an interface that allows clients to easily -// pass in either a "const char*" or a "string". -// -// Arghh! I wish C++ literals were automatically of type "string". - -#ifndef _PCRE_STRINGPIECE_H -#define _PCRE_STRINGPIECE_H - -#include -#include -#include // for ostream forward-declaration - -#if 0 -#define HAVE_TYPE_TRAITS -#include -#elif 0 -#define HAVE_TYPE_TRAITS -#include -#endif - -#include - -using std::memcmp; -using std::strlen; -using std::string; - -namespace pcrecpp { - -class PCRECPP_EXP_DEFN StringPiece { - private: - const char* ptr_; - int length_; - - public: - // We provide non-explicit singleton constructors so users can pass - // in a "const char*" or a "string" wherever a "StringPiece" is - // expected. - StringPiece() - : ptr_(NULL), length_(0) { } - StringPiece(const char* str) - : ptr_(str), length_(static_cast(strlen(ptr_))) { } - StringPiece(const unsigned char* str) - : ptr_(reinterpret_cast(str)), - length_(static_cast(strlen(ptr_))) { } - StringPiece(const string& str) - : ptr_(str.data()), length_(static_cast(str.size())) { } - StringPiece(const char* offset, int len) - : ptr_(offset), length_(len) { } - - // data() may return a pointer to a buffer with embedded NULs, and the - // returned buffer may or may not be null terminated. Therefore it is - // typically a mistake to pass data() to a routine that expects a NUL - // terminated string. Use "as_string().c_str()" if you really need to do - // this. Or better yet, change your routine so it does not rely on NUL - // termination. - const char* data() const { return ptr_; } - int size() const { return length_; } - bool empty() const { return length_ == 0; } - - void clear() { ptr_ = NULL; length_ = 0; } - void set(const char* buffer, int len) { ptr_ = buffer; length_ = len; } - void set(const char* str) { - ptr_ = str; - length_ = static_cast(strlen(str)); - } - void set(const void* buffer, int len) { - ptr_ = reinterpret_cast(buffer); - length_ = len; - } - - char operator[](int i) const { return ptr_[i]; } - - void remove_prefix(int n) { - ptr_ += n; - length_ -= n; - } - - void remove_suffix(int n) { - length_ -= n; - } - - bool operator==(const StringPiece& x) const { - return ((length_ == x.length_) && - (memcmp(ptr_, x.ptr_, length_) == 0)); - } - bool operator!=(const StringPiece& x) const { - return !(*this == x); - } - -#define STRINGPIECE_BINARY_PREDICATE(cmp,auxcmp) \ - bool operator cmp (const StringPiece& x) const { \ - int r = memcmp(ptr_, x.ptr_, length_ < x.length_ ? length_ : x.length_); \ - return ((r auxcmp 0) || ((r == 0) && (length_ cmp x.length_))); \ - } - STRINGPIECE_BINARY_PREDICATE(<, <); - STRINGPIECE_BINARY_PREDICATE(<=, <); - STRINGPIECE_BINARY_PREDICATE(>=, >); - STRINGPIECE_BINARY_PREDICATE(>, >); -#undef STRINGPIECE_BINARY_PREDICATE - - int compare(const StringPiece& x) const { - int r = memcmp(ptr_, x.ptr_, length_ < x.length_ ? length_ : x.length_); - if (r == 0) { - if (length_ < x.length_) r = -1; - else if (length_ > x.length_) r = +1; - } - return r; - } - - string as_string() const { - return string(data(), size()); - } - - void CopyToString(string* target) const { - target->assign(ptr_, length_); - } - - // Does "this" start with "x" - bool starts_with(const StringPiece& x) const { - return ((length_ >= x.length_) && (memcmp(ptr_, x.ptr_, x.length_) == 0)); - } -}; - -} // namespace pcrecpp - -// ------------------------------------------------------------------ -// Functions used to create STL containers that use StringPiece -// Remember that a StringPiece's lifetime had better be less than -// that of the underlying string or char*. If it is not, then you -// cannot safely store a StringPiece into an STL container -// ------------------------------------------------------------------ - -#ifdef HAVE_TYPE_TRAITS -// This makes vector really fast for some STL implementations -template<> struct __type_traits { - typedef __true_type has_trivial_default_constructor; - typedef __true_type has_trivial_copy_constructor; - typedef __true_type has_trivial_assignment_operator; - typedef __true_type has_trivial_destructor; - typedef __true_type is_POD_type; -}; -#endif - -// allow StringPiece to be logged -PCRECPP_EXP_DECL std::ostream& operator<<(std::ostream& o, - const pcrecpp::StringPiece& piece); - -#endif /* _PCRE_STRINGPIECE_H */ diff --git a/src/third_party/pcre-8.39/pcre_stringpiece.h.in b/src/third_party/pcre-8.39/pcre_stringpiece.h.in deleted file mode 100644 index eb25826b453..00000000000 --- a/src/third_party/pcre-8.39/pcre_stringpiece.h.in +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Author: Sanjay Ghemawat -// -// A string like object that points into another piece of memory. -// Useful for providing an interface that allows clients to easily -// pass in either a "const char*" or a "string". -// -// Arghh! I wish C++ literals were automatically of type "string". - -#ifndef _PCRE_STRINGPIECE_H -#define _PCRE_STRINGPIECE_H - -#include -#include -#include // for ostream forward-declaration - -#if @pcre_have_type_traits@ -#define HAVE_TYPE_TRAITS -#include -#elif @pcre_have_bits_type_traits@ -#define HAVE_TYPE_TRAITS -#include -#endif - -#include - -using std::memcmp; -using std::strlen; -using std::string; - -namespace pcrecpp { - -class PCRECPP_EXP_DEFN StringPiece { - private: - const char* ptr_; - int length_; - - public: - // We provide non-explicit singleton constructors so users can pass - // in a "const char*" or a "string" wherever a "StringPiece" is - // expected. - StringPiece() - : ptr_(NULL), length_(0) { } - StringPiece(const char* str) - : ptr_(str), length_(static_cast(strlen(ptr_))) { } - StringPiece(const unsigned char* str) - : ptr_(reinterpret_cast(str)), - length_(static_cast(strlen(ptr_))) { } - StringPiece(const string& str) - : ptr_(str.data()), length_(static_cast(str.size())) { } - StringPiece(const char* offset, int len) - : ptr_(offset), length_(len) { } - - // data() may return a pointer to a buffer with embedded NULs, and the - // returned buffer may or may not be null terminated. Therefore it is - // typically a mistake to pass data() to a routine that expects a NUL - // terminated string. Use "as_string().c_str()" if you really need to do - // this. Or better yet, change your routine so it does not rely on NUL - // termination. - const char* data() const { return ptr_; } - int size() const { return length_; } - bool empty() const { return length_ == 0; } - - void clear() { ptr_ = NULL; length_ = 0; } - void set(const char* buffer, int len) { ptr_ = buffer; length_ = len; } - void set(const char* str) { - ptr_ = str; - length_ = static_cast(strlen(str)); - } - void set(const void* buffer, int len) { - ptr_ = reinterpret_cast(buffer); - length_ = len; - } - - char operator[](int i) const { return ptr_[i]; } - - void remove_prefix(int n) { - ptr_ += n; - length_ -= n; - } - - void remove_suffix(int n) { - length_ -= n; - } - - bool operator==(const StringPiece& x) const { - return ((length_ == x.length_) && - (memcmp(ptr_, x.ptr_, length_) == 0)); - } - bool operator!=(const StringPiece& x) const { - return !(*this == x); - } - -#define STRINGPIECE_BINARY_PREDICATE(cmp,auxcmp) \ - bool operator cmp (const StringPiece& x) const { \ - int r = memcmp(ptr_, x.ptr_, length_ < x.length_ ? length_ : x.length_); \ - return ((r auxcmp 0) || ((r == 0) && (length_ cmp x.length_))); \ - } - STRINGPIECE_BINARY_PREDICATE(<, <); - STRINGPIECE_BINARY_PREDICATE(<=, <); - STRINGPIECE_BINARY_PREDICATE(>=, >); - STRINGPIECE_BINARY_PREDICATE(>, >); -#undef STRINGPIECE_BINARY_PREDICATE - - int compare(const StringPiece& x) const { - int r = memcmp(ptr_, x.ptr_, length_ < x.length_ ? length_ : x.length_); - if (r == 0) { - if (length_ < x.length_) r = -1; - else if (length_ > x.length_) r = +1; - } - return r; - } - - string as_string() const { - return string(data(), size()); - } - - void CopyToString(string* target) const { - target->assign(ptr_, length_); - } - - // Does "this" start with "x" - bool starts_with(const StringPiece& x) const { - return ((length_ >= x.length_) && (memcmp(ptr_, x.ptr_, x.length_) == 0)); - } -}; - -} // namespace pcrecpp - -// ------------------------------------------------------------------ -// Functions used to create STL containers that use StringPiece -// Remember that a StringPiece's lifetime had better be less than -// that of the underlying string or char*. If it is not, then you -// cannot safely store a StringPiece into an STL container -// ------------------------------------------------------------------ - -#ifdef HAVE_TYPE_TRAITS -// This makes vector really fast for some STL implementations -template<> struct __type_traits { - typedef __true_type has_trivial_default_constructor; - typedef __true_type has_trivial_copy_constructor; - typedef __true_type has_trivial_assignment_operator; - typedef __true_type has_trivial_destructor; - typedef __true_type is_POD_type; -}; -#endif - -// allow StringPiece to be logged -PCRECPP_EXP_DECL std::ostream& operator<<(std::ostream& o, - const pcrecpp::StringPiece& piece); - -#endif /* _PCRE_STRINGPIECE_H */ diff --git a/src/third_party/pcre-8.39/pcre_stringpiece_unittest.cc b/src/third_party/pcre-8.39/pcre_stringpiece_unittest.cc deleted file mode 100644 index 1c4759da3b0..00000000000 --- a/src/third_party/pcre-8.39/pcre_stringpiece_unittest.cc +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2003 and onwards Google Inc. -// Author: Sanjay Ghemawat - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include -#include // for make_pair - -#include "pcrecpp.h" -#include "pcre_stringpiece.h" - -// CHECK dies with a fatal error if condition is not true. It is *not* -// controlled by NDEBUG, so the check will be executed regardless of -// compilation mode. Therefore, it is safe to do things like: -// CHECK(fp->Write(x) == 4) -#define CHECK(condition) do { \ - if (!(condition)) { \ - fprintf(stderr, "%s:%d: Check failed: %s\n", \ - __FILE__, __LINE__, #condition); \ - exit(1); \ - } \ -} while (0) - -using pcrecpp::StringPiece; - -static void CheckSTLComparator() { - string s1("foo"); - string s2("bar"); - string s3("baz"); - - StringPiece p1(s1); - StringPiece p2(s2); - StringPiece p3(s3); - - typedef std::map TestMap; - TestMap map; - - map.insert(std::make_pair(p1, 0)); - map.insert(std::make_pair(p2, 1)); - map.insert(std::make_pair(p3, 2)); - - CHECK(map.size() == 3); - - TestMap::const_iterator iter = map.begin(); - CHECK(iter->second == 1); - ++iter; - CHECK(iter->second == 2); - ++iter; - CHECK(iter->second == 0); - ++iter; - CHECK(iter == map.end()); - - TestMap::iterator new_iter = map.find("zot"); - CHECK(new_iter == map.end()); - - new_iter = map.find("bar"); - CHECK(new_iter != map.end()); - - map.erase(new_iter); - CHECK(map.size() == 2); - - iter = map.begin(); - CHECK(iter->second == 2); - ++iter; - CHECK(iter->second == 0); - ++iter; - CHECK(iter == map.end()); -} - -static void CheckComparisonOperators() { -#define CMP_Y(op, x, y) \ - CHECK( (StringPiece((x)) op StringPiece((y)))); \ - CHECK( (StringPiece((x)).compare(StringPiece((y))) op 0)) - -#define CMP_N(op, x, y) \ - CHECK(!(StringPiece((x)) op StringPiece((y)))); \ - CHECK(!(StringPiece((x)).compare(StringPiece((y))) op 0)) - - CMP_Y(==, "", ""); - CMP_Y(==, "a", "a"); - CMP_Y(==, "aa", "aa"); - CMP_N(==, "a", ""); - CMP_N(==, "", "a"); - CMP_N(==, "a", "b"); - CMP_N(==, "a", "aa"); - CMP_N(==, "aa", "a"); - - CMP_N(!=, "", ""); - CMP_N(!=, "a", "a"); - CMP_N(!=, "aa", "aa"); - CMP_Y(!=, "a", ""); - CMP_Y(!=, "", "a"); - CMP_Y(!=, "a", "b"); - CMP_Y(!=, "a", "aa"); - CMP_Y(!=, "aa", "a"); - - CMP_Y(<, "a", "b"); - CMP_Y(<, "a", "aa"); - CMP_Y(<, "aa", "b"); - CMP_Y(<, "aa", "bb"); - CMP_N(<, "a", "a"); - CMP_N(<, "b", "a"); - CMP_N(<, "aa", "a"); - CMP_N(<, "b", "aa"); - CMP_N(<, "bb", "aa"); - - CMP_Y(<=, "a", "a"); - CMP_Y(<=, "a", "b"); - CMP_Y(<=, "a", "aa"); - CMP_Y(<=, "aa", "b"); - CMP_Y(<=, "aa", "bb"); - CMP_N(<=, "b", "a"); - CMP_N(<=, "aa", "a"); - CMP_N(<=, "b", "aa"); - CMP_N(<=, "bb", "aa"); - - CMP_N(>=, "a", "b"); - CMP_N(>=, "a", "aa"); - CMP_N(>=, "aa", "b"); - CMP_N(>=, "aa", "bb"); - CMP_Y(>=, "a", "a"); - CMP_Y(>=, "b", "a"); - CMP_Y(>=, "aa", "a"); - CMP_Y(>=, "b", "aa"); - CMP_Y(>=, "bb", "aa"); - - CMP_N(>, "a", "a"); - CMP_N(>, "a", "b"); - CMP_N(>, "a", "aa"); - CMP_N(>, "aa", "b"); - CMP_N(>, "aa", "bb"); - CMP_Y(>, "b", "a"); - CMP_Y(>, "aa", "a"); - CMP_Y(>, "b", "aa"); - CMP_Y(>, "bb", "aa"); - -#undef CMP_Y -#undef CMP_N -} - -int main(int argc, char** argv) { - (void)argc; - (void)argv; - CheckComparisonOperators(); - CheckSTLComparator(); - - printf("OK\n"); - return 0; -} diff --git a/src/third_party/pcre-8.39/pcre_study.c b/src/third_party/pcre-8.39/pcre_study.c deleted file mode 100644 index d9d4960d84e..00000000000 --- a/src/third_party/pcre-8.39/pcre_study.c +++ /dev/null @@ -1,1686 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This module contains the external function pcre_study(), along with local -supporting functions. */ - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "pcre_internal.h" - -#define SET_BIT(c) start_bits[c/8] |= (1 << (c&7)) - -/* Returns from set_start_bits() */ - -enum { SSB_FAIL, SSB_DONE, SSB_CONTINUE, SSB_UNKNOWN }; - - - -/************************************************* -* Find the minimum subject length for a group * -*************************************************/ - -/* Scan a parenthesized group and compute the minimum length of subject that -is needed to match it. This is a lower bound; it does not mean there is a -string of that length that matches. In UTF8 mode, the result is in characters -rather than bytes. - -Arguments: - re compiled pattern block - code pointer to start of group (the bracket) - startcode pointer to start of the whole pattern's code - options the compiling options - recurses chain of recurse_check to catch mutual recursion - countptr pointer to call count (to catch over complexity) - -Returns: the minimum length - -1 if \C in UTF-8 mode or (*ACCEPT) was encountered - -2 internal error (missing capturing bracket) - -3 internal error (opcode not listed) -*/ - -static int -find_minlength(const REAL_PCRE *re, const pcre_uchar *code, - const pcre_uchar *startcode, int options, recurse_check *recurses, - int *countptr) -{ -int length = -1; -/* PCRE_UTF16 has the same value as PCRE_UTF8. */ -BOOL utf = (options & PCRE_UTF8) != 0; -BOOL had_recurse = FALSE; -recurse_check this_recurse; -register int branchlength = 0; -register pcre_uchar *cc = (pcre_uchar *)code + 1 + LINK_SIZE; - -if ((*countptr)++ > 1000) return -1; /* too complex */ - -if (*code == OP_CBRA || *code == OP_SCBRA || - *code == OP_CBRAPOS || *code == OP_SCBRAPOS) cc += IMM2_SIZE; - -/* Scan along the opcodes for this branch. If we get to the end of the -branch, check the length against that of the other branches. */ - -for (;;) - { - int d, min; - pcre_uchar *cs, *ce; - register pcre_uchar op = *cc; - - switch (op) - { - case OP_COND: - case OP_SCOND: - - /* If there is only one branch in a condition, the implied branch has zero - length, so we don't add anything. This covers the DEFINE "condition" - automatically. */ - - cs = cc + GET(cc, 1); - if (*cs != OP_ALT) - { - cc = cs + 1 + LINK_SIZE; - break; - } - - /* Otherwise we can fall through and treat it the same as any other - subpattern. */ - - case OP_CBRA: - case OP_SCBRA: - case OP_BRA: - case OP_SBRA: - case OP_CBRAPOS: - case OP_SCBRAPOS: - case OP_BRAPOS: - case OP_SBRAPOS: - case OP_ONCE: - case OP_ONCE_NC: - d = find_minlength(re, cc, startcode, options, recurses, countptr); - if (d < 0) return d; - branchlength += d; - do cc += GET(cc, 1); while (*cc == OP_ALT); - cc += 1 + LINK_SIZE; - break; - - /* ACCEPT makes things far too complicated; we have to give up. */ - - case OP_ACCEPT: - case OP_ASSERT_ACCEPT: - return -1; - - /* Reached end of a branch; if it's a ket it is the end of a nested - call. If it's ALT it is an alternation in a nested call. If it is END it's - the end of the outer call. All can be handled by the same code. If an - ACCEPT was previously encountered, use the length that was in force at that - time, and pass back the shortest ACCEPT length. */ - - case OP_ALT: - case OP_KET: - case OP_KETRMAX: - case OP_KETRMIN: - case OP_KETRPOS: - case OP_END: - if (length < 0 || (!had_recurse && branchlength < length)) - length = branchlength; - if (op != OP_ALT) return length; - cc += 1 + LINK_SIZE; - branchlength = 0; - had_recurse = FALSE; - break; - - /* Skip over assertive subpatterns */ - - case OP_ASSERT: - case OP_ASSERT_NOT: - case OP_ASSERTBACK: - case OP_ASSERTBACK_NOT: - do cc += GET(cc, 1); while (*cc == OP_ALT); - /* Fall through */ - - /* Skip over things that don't match chars */ - - case OP_REVERSE: - case OP_CREF: - case OP_DNCREF: - case OP_RREF: - case OP_DNRREF: - case OP_DEF: - case OP_CALLOUT: - case OP_SOD: - case OP_SOM: - case OP_EOD: - case OP_EODN: - case OP_CIRC: - case OP_CIRCM: - case OP_DOLL: - case OP_DOLLM: - case OP_NOT_WORD_BOUNDARY: - case OP_WORD_BOUNDARY: - cc += PRIV(OP_lengths)[*cc]; - break; - - /* Skip over a subpattern that has a {0} or {0,x} quantifier */ - - case OP_BRAZERO: - case OP_BRAMINZERO: - case OP_BRAPOSZERO: - case OP_SKIPZERO: - cc += PRIV(OP_lengths)[*cc]; - do cc += GET(cc, 1); while (*cc == OP_ALT); - cc += 1 + LINK_SIZE; - break; - - /* Handle literal characters and + repetitions */ - - case OP_CHAR: - case OP_CHARI: - case OP_NOT: - case OP_NOTI: - case OP_PLUS: - case OP_PLUSI: - case OP_MINPLUS: - case OP_MINPLUSI: - case OP_POSPLUS: - case OP_POSPLUSI: - case OP_NOTPLUS: - case OP_NOTPLUSI: - case OP_NOTMINPLUS: - case OP_NOTMINPLUSI: - case OP_NOTPOSPLUS: - case OP_NOTPOSPLUSI: - branchlength++; - cc += 2; -#ifdef SUPPORT_UTF - if (utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); -#endif - break; - - case OP_TYPEPLUS: - case OP_TYPEMINPLUS: - case OP_TYPEPOSPLUS: - branchlength++; - cc += (cc[1] == OP_PROP || cc[1] == OP_NOTPROP)? 4 : 2; - break; - - /* Handle exact repetitions. The count is already in characters, but we - need to skip over a multibyte character in UTF8 mode. */ - - case OP_EXACT: - case OP_EXACTI: - case OP_NOTEXACT: - case OP_NOTEXACTI: - branchlength += GET2(cc,1); - cc += 2 + IMM2_SIZE; -#ifdef SUPPORT_UTF - if (utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); -#endif - break; - - case OP_TYPEEXACT: - branchlength += GET2(cc,1); - cc += 2 + IMM2_SIZE + ((cc[1 + IMM2_SIZE] == OP_PROP - || cc[1 + IMM2_SIZE] == OP_NOTPROP)? 2 : 0); - break; - - /* Handle single-char non-literal matchers */ - - case OP_PROP: - case OP_NOTPROP: - cc += 2; - /* Fall through */ - - case OP_NOT_DIGIT: - case OP_DIGIT: - case OP_NOT_WHITESPACE: - case OP_WHITESPACE: - case OP_NOT_WORDCHAR: - case OP_WORDCHAR: - case OP_ANY: - case OP_ALLANY: - case OP_EXTUNI: - case OP_HSPACE: - case OP_NOT_HSPACE: - case OP_VSPACE: - case OP_NOT_VSPACE: - branchlength++; - cc++; - break; - - /* "Any newline" might match two characters, but it also might match just - one. */ - - case OP_ANYNL: - branchlength += 1; - cc++; - break; - - /* The single-byte matcher means we can't proceed in UTF-8 mode. (In - non-UTF-8 mode \C will actually be turned into OP_ALLANY, so won't ever - appear, but leave the code, just in case.) */ - - case OP_ANYBYTE: -#ifdef SUPPORT_UTF - if (utf) return -1; -#endif - branchlength++; - cc++; - break; - - /* For repeated character types, we have to test for \p and \P, which have - an extra two bytes of parameters. */ - - case OP_TYPESTAR: - case OP_TYPEMINSTAR: - case OP_TYPEQUERY: - case OP_TYPEMINQUERY: - case OP_TYPEPOSSTAR: - case OP_TYPEPOSQUERY: - if (cc[1] == OP_PROP || cc[1] == OP_NOTPROP) cc += 2; - cc += PRIV(OP_lengths)[op]; - break; - - case OP_TYPEUPTO: - case OP_TYPEMINUPTO: - case OP_TYPEPOSUPTO: - if (cc[1 + IMM2_SIZE] == OP_PROP - || cc[1 + IMM2_SIZE] == OP_NOTPROP) cc += 2; - cc += PRIV(OP_lengths)[op]; - break; - - /* Check a class for variable quantification */ - - case OP_CLASS: - case OP_NCLASS: -#if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - case OP_XCLASS: - /* The original code caused an unsigned overflow in 64 bit systems, - so now we use a conditional statement. */ - if (op == OP_XCLASS) - cc += GET(cc, 1); - else - cc += PRIV(OP_lengths)[OP_CLASS]; -#else - cc += PRIV(OP_lengths)[OP_CLASS]; -#endif - - switch (*cc) - { - case OP_CRPLUS: - case OP_CRMINPLUS: - case OP_CRPOSPLUS: - branchlength++; - /* Fall through */ - - case OP_CRSTAR: - case OP_CRMINSTAR: - case OP_CRQUERY: - case OP_CRMINQUERY: - case OP_CRPOSSTAR: - case OP_CRPOSQUERY: - cc++; - break; - - case OP_CRRANGE: - case OP_CRMINRANGE: - case OP_CRPOSRANGE: - branchlength += GET2(cc,1); - cc += 1 + 2 * IMM2_SIZE; - break; - - default: - branchlength++; - break; - } - break; - - /* Backreferences and subroutine calls are treated in the same way: we find - the minimum length for the subpattern. A recursion, however, causes an - a flag to be set that causes the length of this branch to be ignored. The - logic is that a recursion can only make sense if there is another - alternation that stops the recursing. That will provide the minimum length - (when no recursion happens). A backreference within the group that it is - referencing behaves in the same way. - - If PCRE_JAVASCRIPT_COMPAT is set, a backreference to an unset bracket - matches an empty string (by default it causes a matching failure), so in - that case we must set the minimum length to zero. */ - - case OP_DNREF: /* Duplicate named pattern back reference */ - case OP_DNREFI: - if ((options & PCRE_JAVASCRIPT_COMPAT) == 0) - { - int count = GET2(cc, 1+IMM2_SIZE); - pcre_uchar *slot = (pcre_uchar *)re + - re->name_table_offset + GET2(cc, 1) * re->name_entry_size; - d = INT_MAX; - while (count-- > 0) - { - ce = cs = (pcre_uchar *)PRIV(find_bracket)(startcode, utf, GET2(slot, 0)); - if (cs == NULL) return -2; - do ce += GET(ce, 1); while (*ce == OP_ALT); - if (cc > cs && cc < ce) /* Simple recursion */ - { - d = 0; - had_recurse = TRUE; - break; - } - else - { - recurse_check *r = recurses; - for (r = recurses; r != NULL; r = r->prev) if (r->group == cs) break; - if (r != NULL) /* Mutual recursion */ - { - d = 0; - had_recurse = TRUE; - break; - } - else - { - int dd; - this_recurse.prev = recurses; - this_recurse.group = cs; - dd = find_minlength(re, cs, startcode, options, &this_recurse, - countptr); - if (dd < d) d = dd; - } - } - slot += re->name_entry_size; - } - } - else d = 0; - cc += 1 + 2*IMM2_SIZE; - goto REPEAT_BACK_REFERENCE; - - case OP_REF: /* Single back reference */ - case OP_REFI: - if ((options & PCRE_JAVASCRIPT_COMPAT) == 0) - { - ce = cs = (pcre_uchar *)PRIV(find_bracket)(startcode, utf, GET2(cc, 1)); - if (cs == NULL) return -2; - do ce += GET(ce, 1); while (*ce == OP_ALT); - if (cc > cs && cc < ce) /* Simple recursion */ - { - d = 0; - had_recurse = TRUE; - } - else - { - recurse_check *r = recurses; - for (r = recurses; r != NULL; r = r->prev) if (r->group == cs) break; - if (r != NULL) /* Mutual recursion */ - { - d = 0; - had_recurse = TRUE; - } - else - { - this_recurse.prev = recurses; - this_recurse.group = cs; - d = find_minlength(re, cs, startcode, options, &this_recurse, - countptr); - } - } - } - else d = 0; - cc += 1 + IMM2_SIZE; - - /* Handle repeated back references */ - - REPEAT_BACK_REFERENCE: - switch (*cc) - { - case OP_CRSTAR: - case OP_CRMINSTAR: - case OP_CRQUERY: - case OP_CRMINQUERY: - case OP_CRPOSSTAR: - case OP_CRPOSQUERY: - min = 0; - cc++; - break; - - case OP_CRPLUS: - case OP_CRMINPLUS: - case OP_CRPOSPLUS: - min = 1; - cc++; - break; - - case OP_CRRANGE: - case OP_CRMINRANGE: - case OP_CRPOSRANGE: - min = GET2(cc, 1); - cc += 1 + 2 * IMM2_SIZE; - break; - - default: - min = 1; - break; - } - - branchlength += min * d; - break; - - /* We can easily detect direct recursion, but not mutual recursion. This is - caught by a recursion depth count. */ - - case OP_RECURSE: - cs = ce = (pcre_uchar *)startcode + GET(cc, 1); - do ce += GET(ce, 1); while (*ce == OP_ALT); - if (cc > cs && cc < ce) /* Simple recursion */ - had_recurse = TRUE; - else - { - recurse_check *r = recurses; - for (r = recurses; r != NULL; r = r->prev) if (r->group == cs) break; - if (r != NULL) /* Mutual recursion */ - had_recurse = TRUE; - else - { - this_recurse.prev = recurses; - this_recurse.group = cs; - branchlength += find_minlength(re, cs, startcode, options, - &this_recurse, countptr); - } - } - cc += 1 + LINK_SIZE; - break; - - /* Anything else does not or need not match a character. We can get the - item's length from the table, but for those that can match zero occurrences - of a character, we must take special action for UTF-8 characters. As it - happens, the "NOT" versions of these opcodes are used at present only for - ASCII characters, so they could be omitted from this list. However, in - future that may change, so we include them here so as not to leave a - gotcha for a future maintainer. */ - - case OP_UPTO: - case OP_UPTOI: - case OP_NOTUPTO: - case OP_NOTUPTOI: - case OP_MINUPTO: - case OP_MINUPTOI: - case OP_NOTMINUPTO: - case OP_NOTMINUPTOI: - case OP_POSUPTO: - case OP_POSUPTOI: - case OP_NOTPOSUPTO: - case OP_NOTPOSUPTOI: - - case OP_STAR: - case OP_STARI: - case OP_NOTSTAR: - case OP_NOTSTARI: - case OP_MINSTAR: - case OP_MINSTARI: - case OP_NOTMINSTAR: - case OP_NOTMINSTARI: - case OP_POSSTAR: - case OP_POSSTARI: - case OP_NOTPOSSTAR: - case OP_NOTPOSSTARI: - - case OP_QUERY: - case OP_QUERYI: - case OP_NOTQUERY: - case OP_NOTQUERYI: - case OP_MINQUERY: - case OP_MINQUERYI: - case OP_NOTMINQUERY: - case OP_NOTMINQUERYI: - case OP_POSQUERY: - case OP_POSQUERYI: - case OP_NOTPOSQUERY: - case OP_NOTPOSQUERYI: - - cc += PRIV(OP_lengths)[op]; -#ifdef SUPPORT_UTF - if (utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); -#endif - break; - - /* Skip these, but we need to add in the name length. */ - - case OP_MARK: - case OP_PRUNE_ARG: - case OP_SKIP_ARG: - case OP_THEN_ARG: - cc += PRIV(OP_lengths)[op] + cc[1]; - break; - - /* The remaining opcodes are just skipped over. */ - - case OP_CLOSE: - case OP_COMMIT: - case OP_FAIL: - case OP_PRUNE: - case OP_SET_SOM: - case OP_SKIP: - case OP_THEN: - cc += PRIV(OP_lengths)[op]; - break; - - /* This should not occur: we list all opcodes explicitly so that when - new ones get added they are properly considered. */ - - default: - return -3; - } - } -/* Control never gets here */ -} - - - -/************************************************* -* Set a bit and maybe its alternate case * -*************************************************/ - -/* Given a character, set its first byte's bit in the table, and also the -corresponding bit for the other version of a letter if we are caseless. In -UTF-8 mode, for characters greater than 127, we can only do the caseless thing -when Unicode property support is available. - -Arguments: - start_bits points to the bit map - p points to the character - caseless the caseless flag - cd the block with char table pointers - utf TRUE for UTF-8 / UTF-16 / UTF-32 mode - -Returns: pointer after the character -*/ - -static const pcre_uchar * -set_table_bit(pcre_uint8 *start_bits, const pcre_uchar *p, BOOL caseless, - compile_data *cd, BOOL utf) -{ -pcre_uint32 c = *p; - -#ifdef COMPILE_PCRE8 -SET_BIT(c); - -#ifdef SUPPORT_UTF -if (utf && c > 127) - { - GETCHARINC(c, p); -#ifdef SUPPORT_UCP - if (caseless) - { - pcre_uchar buff[6]; - c = UCD_OTHERCASE(c); - (void)PRIV(ord2utf)(c, buff); - SET_BIT(buff[0]); - } -#endif /* Not SUPPORT_UCP */ - return p; - } -#else /* Not SUPPORT_UTF */ -(void)(utf); /* Stops warning for unused parameter */ -#endif /* SUPPORT_UTF */ - -/* Not UTF-8 mode, or character is less than 127. */ - -if (caseless && (cd->ctypes[c] & ctype_letter) != 0) SET_BIT(cd->fcc[c]); -return p + 1; -#endif /* COMPILE_PCRE8 */ - -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 -if (c > 0xff) - { - c = 0xff; - caseless = FALSE; - } -SET_BIT(c); - -#ifdef SUPPORT_UTF -if (utf && c > 127) - { - GETCHARINC(c, p); -#ifdef SUPPORT_UCP - if (caseless) - { - c = UCD_OTHERCASE(c); - if (c > 0xff) - c = 0xff; - SET_BIT(c); - } -#endif /* SUPPORT_UCP */ - return p; - } -#else /* Not SUPPORT_UTF */ -(void)(utf); /* Stops warning for unused parameter */ -#endif /* SUPPORT_UTF */ - -if (caseless && (cd->ctypes[c] & ctype_letter) != 0) SET_BIT(cd->fcc[c]); -return p + 1; -#endif -} - - - -/************************************************* -* Set bits for a positive character type * -*************************************************/ - -/* This function sets starting bits for a character type. In UTF-8 mode, we can -only do a direct setting for bytes less than 128, as otherwise there can be -confusion with bytes in the middle of UTF-8 characters. In a "traditional" -environment, the tables will only recognize ASCII characters anyway, but in at -least one Windows environment, some higher bytes bits were set in the tables. -So we deal with that case by considering the UTF-8 encoding. - -Arguments: - start_bits the starting bitmap - cbit type the type of character wanted - table_limit 32 for non-UTF-8; 16 for UTF-8 - cd the block with char table pointers - -Returns: nothing -*/ - -static void -set_type_bits(pcre_uint8 *start_bits, int cbit_type, unsigned int table_limit, - compile_data *cd) -{ -register pcre_uint32 c; -for (c = 0; c < table_limit; c++) start_bits[c] |= cd->cbits[c+cbit_type]; -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 -if (table_limit == 32) return; -for (c = 128; c < 256; c++) - { - if ((cd->cbits[c/8] & (1 << (c&7))) != 0) - { - pcre_uchar buff[6]; - (void)PRIV(ord2utf)(c, buff); - SET_BIT(buff[0]); - } - } -#endif -} - - -/************************************************* -* Set bits for a negative character type * -*************************************************/ - -/* This function sets starting bits for a negative character type such as \D. -In UTF-8 mode, we can only do a direct setting for bytes less than 128, as -otherwise there can be confusion with bytes in the middle of UTF-8 characters. -Unlike in the positive case, where we can set appropriate starting bits for -specific high-valued UTF-8 characters, in this case we have to set the bits for -all high-valued characters. The lowest is 0xc2, but we overkill by starting at -0xc0 (192) for simplicity. - -Arguments: - start_bits the starting bitmap - cbit type the type of character wanted - table_limit 32 for non-UTF-8; 16 for UTF-8 - cd the block with char table pointers - -Returns: nothing -*/ - -static void -set_nottype_bits(pcre_uint8 *start_bits, int cbit_type, unsigned int table_limit, - compile_data *cd) -{ -register pcre_uint32 c; -for (c = 0; c < table_limit; c++) start_bits[c] |= ~cd->cbits[c+cbit_type]; -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 -if (table_limit != 32) for (c = 24; c < 32; c++) start_bits[c] = 0xff; -#endif -} - - - -/************************************************* -* Create bitmap of starting bytes * -*************************************************/ - -/* This function scans a compiled unanchored expression recursively and -attempts to build a bitmap of the set of possible starting bytes. As time goes -by, we may be able to get more clever at doing this. The SSB_CONTINUE return is -useful for parenthesized groups in patterns such as (a*)b where the group -provides some optional starting bytes but scanning must continue at the outer -level to find at least one mandatory byte. At the outermost level, this -function fails unless the result is SSB_DONE. - -Arguments: - code points to an expression - start_bits points to a 32-byte table, initialized to 0 - utf TRUE if in UTF-8 / UTF-16 / UTF-32 mode - cd the block with char table pointers - -Returns: SSB_FAIL => Failed to find any starting bytes - SSB_DONE => Found mandatory starting bytes - SSB_CONTINUE => Found optional starting bytes - SSB_UNKNOWN => Hit an unrecognized opcode -*/ - -static int -set_start_bits(const pcre_uchar *code, pcre_uint8 *start_bits, BOOL utf, - compile_data *cd) -{ -register pcre_uint32 c; -int yield = SSB_DONE; -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 -int table_limit = utf? 16:32; -#else -int table_limit = 32; -#endif - -#if 0 -/* ========================================================================= */ -/* The following comment and code was inserted in January 1999. In May 2006, -when it was observed to cause compiler warnings about unused values, I took it -out again. If anybody is still using OS/2, they will have to put it back -manually. */ - -/* This next statement and the later reference to dummy are here in order to -trick the optimizer of the IBM C compiler for OS/2 into generating correct -code. Apparently IBM isn't going to fix the problem, and we would rather not -disable optimization (in this module it actually makes a big difference, and -the pcre module can use all the optimization it can get). */ - -volatile int dummy; -/* ========================================================================= */ -#endif - -do - { - BOOL try_next = TRUE; - const pcre_uchar *tcode = code + 1 + LINK_SIZE; - - if (*code == OP_CBRA || *code == OP_SCBRA || - *code == OP_CBRAPOS || *code == OP_SCBRAPOS) tcode += IMM2_SIZE; - - while (try_next) /* Loop for items in this branch */ - { - int rc; - - switch(*tcode) - { - /* If we reach something we don't understand, it means a new opcode has - been created that hasn't been added to this code. Hopefully this problem - will be discovered during testing. */ - - default: - return SSB_UNKNOWN; - - /* Fail for a valid opcode that implies no starting bits. */ - - case OP_ACCEPT: - case OP_ASSERT_ACCEPT: - case OP_ALLANY: - case OP_ANY: - case OP_ANYBYTE: - case OP_CIRC: - case OP_CIRCM: - case OP_CLOSE: - case OP_COMMIT: - case OP_COND: - case OP_CREF: - case OP_DEF: - case OP_DNCREF: - case OP_DNREF: - case OP_DNREFI: - case OP_DNRREF: - case OP_DOLL: - case OP_DOLLM: - case OP_END: - case OP_EOD: - case OP_EODN: - case OP_EXTUNI: - case OP_FAIL: - case OP_MARK: - case OP_NOT: - case OP_NOTEXACT: - case OP_NOTEXACTI: - case OP_NOTI: - case OP_NOTMINPLUS: - case OP_NOTMINPLUSI: - case OP_NOTMINQUERY: - case OP_NOTMINQUERYI: - case OP_NOTMINSTAR: - case OP_NOTMINSTARI: - case OP_NOTMINUPTO: - case OP_NOTMINUPTOI: - case OP_NOTPLUS: - case OP_NOTPLUSI: - case OP_NOTPOSPLUS: - case OP_NOTPOSPLUSI: - case OP_NOTPOSQUERY: - case OP_NOTPOSQUERYI: - case OP_NOTPOSSTAR: - case OP_NOTPOSSTARI: - case OP_NOTPOSUPTO: - case OP_NOTPOSUPTOI: - case OP_NOTPROP: - case OP_NOTQUERY: - case OP_NOTQUERYI: - case OP_NOTSTAR: - case OP_NOTSTARI: - case OP_NOTUPTO: - case OP_NOTUPTOI: - case OP_NOT_HSPACE: - case OP_NOT_VSPACE: - case OP_PRUNE: - case OP_PRUNE_ARG: - case OP_RECURSE: - case OP_REF: - case OP_REFI: - case OP_REVERSE: - case OP_RREF: - case OP_SCOND: - case OP_SET_SOM: - case OP_SKIP: - case OP_SKIP_ARG: - case OP_SOD: - case OP_SOM: - case OP_THEN: - case OP_THEN_ARG: - return SSB_FAIL; - - /* A "real" property test implies no starting bits, but the fake property - PT_CLIST identifies a list of characters. These lists are short, as they - are used for characters with more than one "other case", so there is no - point in recognizing them for OP_NOTPROP. */ - - case OP_PROP: - if (tcode[1] != PT_CLIST) return SSB_FAIL; - { - const pcre_uint32 *p = PRIV(ucd_caseless_sets) + tcode[2]; - while ((c = *p++) < NOTACHAR) - { -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 - if (utf) - { - pcre_uchar buff[6]; - (void)PRIV(ord2utf)(c, buff); - c = buff[0]; - } -#endif - if (c > 0xff) SET_BIT(0xff); else SET_BIT(c); - } - } - try_next = FALSE; - break; - - /* We can ignore word boundary tests. */ - - case OP_WORD_BOUNDARY: - case OP_NOT_WORD_BOUNDARY: - tcode++; - break; - - /* If we hit a bracket or a positive lookahead assertion, recurse to set - bits from within the subpattern. If it can't find anything, we have to - give up. If it finds some mandatory character(s), we are done for this - branch. Otherwise, carry on scanning after the subpattern. */ - - case OP_BRA: - case OP_SBRA: - case OP_CBRA: - case OP_SCBRA: - case OP_BRAPOS: - case OP_SBRAPOS: - case OP_CBRAPOS: - case OP_SCBRAPOS: - case OP_ONCE: - case OP_ONCE_NC: - case OP_ASSERT: - rc = set_start_bits(tcode, start_bits, utf, cd); - if (rc == SSB_FAIL || rc == SSB_UNKNOWN) return rc; - if (rc == SSB_DONE) try_next = FALSE; else - { - do tcode += GET(tcode, 1); while (*tcode == OP_ALT); - tcode += 1 + LINK_SIZE; - } - break; - - /* If we hit ALT or KET, it means we haven't found anything mandatory in - this branch, though we might have found something optional. For ALT, we - continue with the next alternative, but we have to arrange that the final - result from subpattern is SSB_CONTINUE rather than SSB_DONE. For KET, - return SSB_CONTINUE: if this is the top level, that indicates failure, - but after a nested subpattern, it causes scanning to continue. */ - - case OP_ALT: - yield = SSB_CONTINUE; - try_next = FALSE; - break; - - case OP_KET: - case OP_KETRMAX: - case OP_KETRMIN: - case OP_KETRPOS: - return SSB_CONTINUE; - - /* Skip over callout */ - - case OP_CALLOUT: - tcode += 2 + 2*LINK_SIZE; - break; - - /* Skip over lookbehind and negative lookahead assertions */ - - case OP_ASSERT_NOT: - case OP_ASSERTBACK: - case OP_ASSERTBACK_NOT: - do tcode += GET(tcode, 1); while (*tcode == OP_ALT); - tcode += 1 + LINK_SIZE; - break; - - /* BRAZERO does the bracket, but carries on. */ - - case OP_BRAZERO: - case OP_BRAMINZERO: - case OP_BRAPOSZERO: - rc = set_start_bits(++tcode, start_bits, utf, cd); - if (rc == SSB_FAIL || rc == SSB_UNKNOWN) return rc; -/* ========================================================================= - See the comment at the head of this function concerning the next line, - which was an old fudge for the benefit of OS/2. - dummy = 1; - ========================================================================= */ - do tcode += GET(tcode,1); while (*tcode == OP_ALT); - tcode += 1 + LINK_SIZE; - break; - - /* SKIPZERO skips the bracket. */ - - case OP_SKIPZERO: - tcode++; - do tcode += GET(tcode,1); while (*tcode == OP_ALT); - tcode += 1 + LINK_SIZE; - break; - - /* Single-char * or ? sets the bit and tries the next item */ - - case OP_STAR: - case OP_MINSTAR: - case OP_POSSTAR: - case OP_QUERY: - case OP_MINQUERY: - case OP_POSQUERY: - tcode = set_table_bit(start_bits, tcode + 1, FALSE, cd, utf); - break; - - case OP_STARI: - case OP_MINSTARI: - case OP_POSSTARI: - case OP_QUERYI: - case OP_MINQUERYI: - case OP_POSQUERYI: - tcode = set_table_bit(start_bits, tcode + 1, TRUE, cd, utf); - break; - - /* Single-char upto sets the bit and tries the next */ - - case OP_UPTO: - case OP_MINUPTO: - case OP_POSUPTO: - tcode = set_table_bit(start_bits, tcode + 1 + IMM2_SIZE, FALSE, cd, utf); - break; - - case OP_UPTOI: - case OP_MINUPTOI: - case OP_POSUPTOI: - tcode = set_table_bit(start_bits, tcode + 1 + IMM2_SIZE, TRUE, cd, utf); - break; - - /* At least one single char sets the bit and stops */ - - case OP_EXACT: - tcode += IMM2_SIZE; - /* Fall through */ - case OP_CHAR: - case OP_PLUS: - case OP_MINPLUS: - case OP_POSPLUS: - (void)set_table_bit(start_bits, tcode + 1, FALSE, cd, utf); - try_next = FALSE; - break; - - case OP_EXACTI: - tcode += IMM2_SIZE; - /* Fall through */ - case OP_CHARI: - case OP_PLUSI: - case OP_MINPLUSI: - case OP_POSPLUSI: - (void)set_table_bit(start_bits, tcode + 1, TRUE, cd, utf); - try_next = FALSE; - break; - - /* Special spacing and line-terminating items. These recognize specific - lists of characters. The difference between VSPACE and ANYNL is that the - latter can match the two-character CRLF sequence, but that is not - relevant for finding the first character, so their code here is - identical. */ - - case OP_HSPACE: - SET_BIT(CHAR_HT); - SET_BIT(CHAR_SPACE); -#ifdef SUPPORT_UTF - if (utf) - { -#ifdef COMPILE_PCRE8 - SET_BIT(0xC2); /* For U+00A0 */ - SET_BIT(0xE1); /* For U+1680, U+180E */ - SET_BIT(0xE2); /* For U+2000 - U+200A, U+202F, U+205F */ - SET_BIT(0xE3); /* For U+3000 */ -#elif defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - SET_BIT(0xA0); - SET_BIT(0xFF); /* For characters > 255 */ -#endif /* COMPILE_PCRE[8|16|32] */ - } - else -#endif /* SUPPORT_UTF */ - { -#ifndef EBCDIC - SET_BIT(0xA0); -#endif /* Not EBCDIC */ -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - SET_BIT(0xFF); /* For characters > 255 */ -#endif /* COMPILE_PCRE[16|32] */ - } - try_next = FALSE; - break; - - case OP_ANYNL: - case OP_VSPACE: - SET_BIT(CHAR_LF); - SET_BIT(CHAR_VT); - SET_BIT(CHAR_FF); - SET_BIT(CHAR_CR); -#ifdef SUPPORT_UTF - if (utf) - { -#ifdef COMPILE_PCRE8 - SET_BIT(0xC2); /* For U+0085 */ - SET_BIT(0xE2); /* For U+2028, U+2029 */ -#elif defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - SET_BIT(CHAR_NEL); - SET_BIT(0xFF); /* For characters > 255 */ -#endif /* COMPILE_PCRE[8|16|32] */ - } - else -#endif /* SUPPORT_UTF */ - { - SET_BIT(CHAR_NEL); -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - SET_BIT(0xFF); /* For characters > 255 */ -#endif - } - try_next = FALSE; - break; - - /* Single character types set the bits and stop. Note that if PCRE_UCP - is set, we do not see these op codes because \d etc are converted to - properties. Therefore, these apply in the case when only characters less - than 256 are recognized to match the types. */ - - case OP_NOT_DIGIT: - set_nottype_bits(start_bits, cbit_digit, table_limit, cd); - try_next = FALSE; - break; - - case OP_DIGIT: - set_type_bits(start_bits, cbit_digit, table_limit, cd); - try_next = FALSE; - break; - - /* The cbit_space table has vertical tab as whitespace; we no longer - have to play fancy tricks because Perl added VT to its whitespace at - release 5.18. PCRE added it at release 8.34. */ - - case OP_NOT_WHITESPACE: - set_nottype_bits(start_bits, cbit_space, table_limit, cd); - try_next = FALSE; - break; - - case OP_WHITESPACE: - set_type_bits(start_bits, cbit_space, table_limit, cd); - try_next = FALSE; - break; - - case OP_NOT_WORDCHAR: - set_nottype_bits(start_bits, cbit_word, table_limit, cd); - try_next = FALSE; - break; - - case OP_WORDCHAR: - set_type_bits(start_bits, cbit_word, table_limit, cd); - try_next = FALSE; - break; - - /* One or more character type fudges the pointer and restarts, knowing - it will hit a single character type and stop there. */ - - case OP_TYPEPLUS: - case OP_TYPEMINPLUS: - case OP_TYPEPOSPLUS: - tcode++; - break; - - case OP_TYPEEXACT: - tcode += 1 + IMM2_SIZE; - break; - - /* Zero or more repeats of character types set the bits and then - try again. */ - - case OP_TYPEUPTO: - case OP_TYPEMINUPTO: - case OP_TYPEPOSUPTO: - tcode += IMM2_SIZE; /* Fall through */ - - case OP_TYPESTAR: - case OP_TYPEMINSTAR: - case OP_TYPEPOSSTAR: - case OP_TYPEQUERY: - case OP_TYPEMINQUERY: - case OP_TYPEPOSQUERY: - switch(tcode[1]) - { - default: - case OP_ANY: - case OP_ALLANY: - return SSB_FAIL; - - case OP_HSPACE: - SET_BIT(CHAR_HT); - SET_BIT(CHAR_SPACE); -#ifdef SUPPORT_UTF - if (utf) - { -#ifdef COMPILE_PCRE8 - SET_BIT(0xC2); /* For U+00A0 */ - SET_BIT(0xE1); /* For U+1680, U+180E */ - SET_BIT(0xE2); /* For U+2000 - U+200A, U+202F, U+205F */ - SET_BIT(0xE3); /* For U+3000 */ -#elif defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - SET_BIT(0xA0); - SET_BIT(0xFF); /* For characters > 255 */ -#endif /* COMPILE_PCRE[8|16|32] */ - } - else -#endif /* SUPPORT_UTF */ -#ifndef EBCDIC - SET_BIT(0xA0); -#endif /* Not EBCDIC */ - break; - - case OP_ANYNL: - case OP_VSPACE: - SET_BIT(CHAR_LF); - SET_BIT(CHAR_VT); - SET_BIT(CHAR_FF); - SET_BIT(CHAR_CR); -#ifdef SUPPORT_UTF - if (utf) - { -#ifdef COMPILE_PCRE8 - SET_BIT(0xC2); /* For U+0085 */ - SET_BIT(0xE2); /* For U+2028, U+2029 */ -#elif defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - SET_BIT(CHAR_NEL); - SET_BIT(0xFF); /* For characters > 255 */ -#endif /* COMPILE_PCRE16 */ - } - else -#endif /* SUPPORT_UTF */ - SET_BIT(CHAR_NEL); - break; - - case OP_NOT_DIGIT: - set_nottype_bits(start_bits, cbit_digit, table_limit, cd); - break; - - case OP_DIGIT: - set_type_bits(start_bits, cbit_digit, table_limit, cd); - break; - - /* The cbit_space table has vertical tab as whitespace; we no longer - have to play fancy tricks because Perl added VT to its whitespace at - release 5.18. PCRE added it at release 8.34. */ - - case OP_NOT_WHITESPACE: - set_nottype_bits(start_bits, cbit_space, table_limit, cd); - break; - - case OP_WHITESPACE: - set_type_bits(start_bits, cbit_space, table_limit, cd); - break; - - case OP_NOT_WORDCHAR: - set_nottype_bits(start_bits, cbit_word, table_limit, cd); - break; - - case OP_WORDCHAR: - set_type_bits(start_bits, cbit_word, table_limit, cd); - break; - } - - tcode += 2; - break; - - /* Character class where all the information is in a bit map: set the - bits and either carry on or not, according to the repeat count. If it was - a negative class, and we are operating with UTF-8 characters, any byte - with a value >= 0xc4 is a potentially valid starter because it starts a - character with a value > 255. */ - -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - case OP_XCLASS: - if ((tcode[1 + LINK_SIZE] & XCL_HASPROP) != 0) - return SSB_FAIL; - /* All bits are set. */ - if ((tcode[1 + LINK_SIZE] & XCL_MAP) == 0 && (tcode[1 + LINK_SIZE] & XCL_NOT) != 0) - return SSB_FAIL; -#endif - /* Fall through */ - - case OP_NCLASS: -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 - if (utf) - { - start_bits[24] |= 0xf0; /* Bits for 0xc4 - 0xc8 */ - memset(start_bits+25, 0xff, 7); /* Bits for 0xc9 - 0xff */ - } -#endif -#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 - SET_BIT(0xFF); /* For characters > 255 */ -#endif - /* Fall through */ - - case OP_CLASS: - { - pcre_uint8 *map; -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - map = NULL; - if (*tcode == OP_XCLASS) - { - if ((tcode[1 + LINK_SIZE] & XCL_MAP) != 0) - map = (pcre_uint8 *)(tcode + 1 + LINK_SIZE + 1); - tcode += GET(tcode, 1); - } - else -#endif - { - tcode++; - map = (pcre_uint8 *)tcode; - tcode += 32 / sizeof(pcre_uchar); - } - - /* In UTF-8 mode, the bits in a bit map correspond to character - values, not to byte values. However, the bit map we are constructing is - for byte values. So we have to do a conversion for characters whose - value is > 127. In fact, there are only two possible starting bytes for - characters in the range 128 - 255. */ - -#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 - if (map != NULL) -#endif - { -#if defined SUPPORT_UTF && defined COMPILE_PCRE8 - if (utf) - { - for (c = 0; c < 16; c++) start_bits[c] |= map[c]; - for (c = 128; c < 256; c++) - { - if ((map[c/8] & (1 << (c&7))) != 0) - { - int d = (c >> 6) | 0xc0; /* Set bit for this starter */ - start_bits[d/8] |= (1 << (d&7)); /* and then skip on to the */ - c = (c & 0xc0) + 0x40 - 1; /* next relevant character. */ - } - } - } - else -#endif - { - /* In non-UTF-8 mode, the two bit maps are completely compatible. */ - for (c = 0; c < 32; c++) start_bits[c] |= map[c]; - } - } - - /* Advance past the bit map, and act on what follows. For a zero - minimum repeat, continue; otherwise stop processing. */ - - switch (*tcode) - { - case OP_CRSTAR: - case OP_CRMINSTAR: - case OP_CRQUERY: - case OP_CRMINQUERY: - case OP_CRPOSSTAR: - case OP_CRPOSQUERY: - tcode++; - break; - - case OP_CRRANGE: - case OP_CRMINRANGE: - case OP_CRPOSRANGE: - if (GET2(tcode, 1) == 0) tcode += 1 + 2 * IMM2_SIZE; - else try_next = FALSE; - break; - - default: - try_next = FALSE; - break; - } - } - break; /* End of bitmap class handling */ - - } /* End of switch */ - } /* End of try_next loop */ - - code += GET(code, 1); /* Advance to next branch */ - } -while (*code == OP_ALT); -return yield; -} - - - - - -/************************************************* -* Study a compiled expression * -*************************************************/ - -/* This function is handed a compiled expression that it must study to produce -information that will speed up the matching. It returns a pcre[16]_extra block -which then gets handed back to pcre_exec(). - -Arguments: - re points to the compiled expression - options contains option bits - errorptr points to where to place error messages; - set NULL unless error - -Returns: pointer to a pcre[16]_extra block, with study_data filled in and - the appropriate flags set; - NULL on error or if no optimization possible -*/ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN pcre_extra * PCRE_CALL_CONVENTION -pcre_study(const pcre *external_re, int options, const char **errorptr) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN pcre16_extra * PCRE_CALL_CONVENTION -pcre16_study(const pcre16 *external_re, int options, const char **errorptr) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN pcre32_extra * PCRE_CALL_CONVENTION -pcre32_study(const pcre32 *external_re, int options, const char **errorptr) -#endif -{ -int min; -int count = 0; -BOOL bits_set = FALSE; -pcre_uint8 start_bits[32]; -PUBL(extra) *extra = NULL; -pcre_study_data *study; -const pcre_uint8 *tables; -pcre_uchar *code; -compile_data compile_block; -const REAL_PCRE *re = (const REAL_PCRE *)external_re; - - -*errorptr = NULL; - -if (re == NULL || re->magic_number != MAGIC_NUMBER) - { - *errorptr = "argument is not a compiled regular expression"; - return NULL; - } - -if ((re->flags & PCRE_MODE) == 0) - { -#if defined COMPILE_PCRE8 - *errorptr = "argument not compiled in 8 bit mode"; -#elif defined COMPILE_PCRE16 - *errorptr = "argument not compiled in 16 bit mode"; -#elif defined COMPILE_PCRE32 - *errorptr = "argument not compiled in 32 bit mode"; -#endif - return NULL; - } - -if ((options & ~PUBLIC_STUDY_OPTIONS) != 0) - { - *errorptr = "unknown or incorrect option bit(s) set"; - return NULL; - } - -code = (pcre_uchar *)re + re->name_table_offset + - (re->name_count * re->name_entry_size); - -/* For an anchored pattern, or an unanchored pattern that has a first char, or -a multiline pattern that matches only at "line starts", there is no point in -seeking a list of starting bytes. */ - -if ((re->options & PCRE_ANCHORED) == 0 && - (re->flags & (PCRE_FIRSTSET|PCRE_STARTLINE)) == 0) - { - int rc; - - /* Set the character tables in the block that is passed around */ - - tables = re->tables; - -#if defined COMPILE_PCRE8 - if (tables == NULL) - (void)pcre_fullinfo(external_re, NULL, PCRE_INFO_DEFAULT_TABLES, - (void *)(&tables)); -#elif defined COMPILE_PCRE16 - if (tables == NULL) - (void)pcre16_fullinfo(external_re, NULL, PCRE_INFO_DEFAULT_TABLES, - (void *)(&tables)); -#elif defined COMPILE_PCRE32 - if (tables == NULL) - (void)pcre32_fullinfo(external_re, NULL, PCRE_INFO_DEFAULT_TABLES, - (void *)(&tables)); -#endif - - compile_block.lcc = tables + lcc_offset; - compile_block.fcc = tables + fcc_offset; - compile_block.cbits = tables + cbits_offset; - compile_block.ctypes = tables + ctypes_offset; - - /* See if we can find a fixed set of initial characters for the pattern. */ - - memset(start_bits, 0, 32 * sizeof(pcre_uint8)); - rc = set_start_bits(code, start_bits, (re->options & PCRE_UTF8) != 0, - &compile_block); - bits_set = rc == SSB_DONE; - if (rc == SSB_UNKNOWN) - { - *errorptr = "internal error: opcode not recognized"; - return NULL; - } - } - -/* Find the minimum length of subject string. */ - -switch(min = find_minlength(re, code, code, re->options, NULL, &count)) - { - case -2: *errorptr = "internal error: missing capturing bracket"; return NULL; - case -3: *errorptr = "internal error: opcode not recognized"; return NULL; - default: break; - } - -/* If a set of starting bytes has been identified, or if the minimum length is -greater than zero, or if JIT optimization has been requested, or if -PCRE_STUDY_EXTRA_NEEDED is set, get a pcre[16]_extra block and a -pcre_study_data block. The study data is put in the latter, which is pointed to -by the former, which may also get additional data set later by the calling -program. At the moment, the size of pcre_study_data is fixed. We nevertheless -save it in a field for returning via the pcre_fullinfo() function so that if it -becomes variable in the future, we don't have to change that code. */ - -if (bits_set || min > 0 || (options & ( -#ifdef SUPPORT_JIT - PCRE_STUDY_JIT_COMPILE | PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE | - PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE | -#endif - PCRE_STUDY_EXTRA_NEEDED)) != 0) - { - extra = (PUBL(extra) *)(PUBL(malloc)) - (sizeof(PUBL(extra)) + sizeof(pcre_study_data)); - if (extra == NULL) - { - *errorptr = "failed to get memory"; - return NULL; - } - - study = (pcre_study_data *)((char *)extra + sizeof(PUBL(extra))); - extra->flags = PCRE_EXTRA_STUDY_DATA; - extra->study_data = study; - - study->size = sizeof(pcre_study_data); - study->flags = 0; - - /* Set the start bits always, to avoid unset memory errors if the - study data is written to a file, but set the flag only if any of the bits - are set, to save time looking when none are. */ - - if (bits_set) - { - study->flags |= PCRE_STUDY_MAPPED; - memcpy(study->start_bits, start_bits, sizeof(start_bits)); - } - else memset(study->start_bits, 0, 32 * sizeof(pcre_uint8)); - -#ifdef PCRE_DEBUG - if (bits_set) - { - pcre_uint8 *ptr = start_bits; - int i; - - printf("Start bits:\n"); - for (i = 0; i < 32; i++) - printf("%3d: %02x%s", i * 8, *ptr++, ((i + 1) & 0x7) != 0? " " : "\n"); - } -#endif - - /* Always set the minlength value in the block, because the JIT compiler - makes use of it. However, don't set the bit unless the length is greater than - zero - the interpretive pcre_exec() and pcre_dfa_exec() needn't waste time - checking the zero case. */ - - if (min > 0) - { - study->flags |= PCRE_STUDY_MINLEN; - study->minlength = min; - } - else study->minlength = 0; - - /* If JIT support was compiled and requested, attempt the JIT compilation. - If no starting bytes were found, and the minimum length is zero, and JIT - compilation fails, abandon the extra block and return NULL, unless - PCRE_STUDY_EXTRA_NEEDED is set. */ - -#ifdef SUPPORT_JIT - extra->executable_jit = NULL; - if ((options & PCRE_STUDY_JIT_COMPILE) != 0) - PRIV(jit_compile)(re, extra, JIT_COMPILE); - if ((options & PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE) != 0) - PRIV(jit_compile)(re, extra, JIT_PARTIAL_SOFT_COMPILE); - if ((options & PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE) != 0) - PRIV(jit_compile)(re, extra, JIT_PARTIAL_HARD_COMPILE); - - if (study->flags == 0 && (extra->flags & PCRE_EXTRA_EXECUTABLE_JIT) == 0 && - (options & PCRE_STUDY_EXTRA_NEEDED) == 0) - { -#if defined COMPILE_PCRE8 - pcre_free_study(extra); -#elif defined COMPILE_PCRE16 - pcre16_free_study(extra); -#elif defined COMPILE_PCRE32 - pcre32_free_study(extra); -#endif - extra = NULL; - } -#endif - } - -return extra; -} - - -/************************************************* -* Free the study data * -*************************************************/ - -/* This function frees the memory that was obtained by pcre_study(). - -Argument: a pointer to the pcre[16]_extra block -Returns: nothing -*/ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN void -pcre_free_study(pcre_extra *extra) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN void -pcre16_free_study(pcre16_extra *extra) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN void -pcre32_free_study(pcre32_extra *extra) -#endif -{ -if (extra == NULL) - return; -#ifdef SUPPORT_JIT -if ((extra->flags & PCRE_EXTRA_EXECUTABLE_JIT) != 0 && - extra->executable_jit != NULL) - PRIV(jit_free)(extra->executable_jit); -#endif -PUBL(free)(extra); -} - -/* End of pcre_study.c */ diff --git a/src/third_party/pcre-8.39/pcre_tables.c b/src/third_party/pcre-8.39/pcre_tables.c deleted file mode 100644 index 4960af57c4d..00000000000 --- a/src/third_party/pcre-8.39/pcre_tables.c +++ /dev/null @@ -1,727 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -#ifndef PCRE_INCLUDED - -/* This module contains some fixed tables that are used by more than one of the -PCRE code modules. The tables are also #included by the pcretest program, which -uses macros to change their names from _pcre_xxx to xxxx, thereby avoiding name -clashes with the library. */ - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "pcre_internal.h" - -#endif /* PCRE_INCLUDED */ - -/* Table of sizes for the fixed-length opcodes. It's defined in a macro so that -the definition is next to the definition of the opcodes in pcre_internal.h. */ - -const pcre_uint8 PRIV(OP_lengths)[] = { OP_LENGTHS }; - -/* Tables of horizontal and vertical whitespace characters, suitable for -adding to classes. */ - -const pcre_uint32 PRIV(hspace_list)[] = { HSPACE_LIST }; -const pcre_uint32 PRIV(vspace_list)[] = { VSPACE_LIST }; - - - -/************************************************* -* Tables for UTF-8 support * -*************************************************/ - -/* These are the breakpoints for different numbers of bytes in a UTF-8 -character. */ - -#if (defined SUPPORT_UTF && defined COMPILE_PCRE8) \ - || (defined PCRE_INCLUDED && (defined SUPPORT_PCRE16 || defined SUPPORT_PCRE32)) - -/* These tables are also required by pcretest in 16- or 32-bit mode. */ - -const int PRIV(utf8_table1)[] = - { 0x7f, 0x7ff, 0xffff, 0x1fffff, 0x3ffffff, 0x7fffffff}; - -const int PRIV(utf8_table1_size) = sizeof(PRIV(utf8_table1)) / sizeof(int); - -/* These are the indicator bits and the mask for the data bits to set in the -first byte of a character, indexed by the number of additional bytes. */ - -const int PRIV(utf8_table2)[] = { 0, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc}; -const int PRIV(utf8_table3)[] = { 0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01}; - -/* Table of the number of extra bytes, indexed by the first byte masked with -0x3f. The highest number for a valid UTF-8 first byte is in fact 0x3d. */ - -const pcre_uint8 PRIV(utf8_table4)[] = { - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, - 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 }; - -#endif /* (SUPPORT_UTF && COMPILE_PCRE8) || (PCRE_INCLUDED && SUPPORT_PCRE[16|32])*/ - -#ifdef SUPPORT_UTF - -/* Table to translate from particular type value to the general value. */ - -const pcre_uint32 PRIV(ucp_gentype)[] = { - ucp_C, ucp_C, ucp_C, ucp_C, ucp_C, /* Cc, Cf, Cn, Co, Cs */ - ucp_L, ucp_L, ucp_L, ucp_L, ucp_L, /* Ll, Lu, Lm, Lo, Lt */ - ucp_M, ucp_M, ucp_M, /* Mc, Me, Mn */ - ucp_N, ucp_N, ucp_N, /* Nd, Nl, No */ - ucp_P, ucp_P, ucp_P, ucp_P, ucp_P, /* Pc, Pd, Pe, Pf, Pi */ - ucp_P, ucp_P, /* Ps, Po */ - ucp_S, ucp_S, ucp_S, ucp_S, /* Sc, Sk, Sm, So */ - ucp_Z, ucp_Z, ucp_Z /* Zl, Zp, Zs */ -}; - -/* This table encodes the rules for finding the end of an extended grapheme -cluster. Every code point has a grapheme break property which is one of the -ucp_gbXX values defined in ucp.h. The 2-dimensional table is indexed by the -properties of two adjacent code points. The left property selects a word from -the table, and the right property selects a bit from that word like this: - - ucp_gbtable[left-property] & (1 << right-property) - -The value is non-zero if a grapheme break is NOT permitted between the relevant -two code points. The breaking rules are as follows: - -1. Break at the start and end of text (pretty obviously). - -2. Do not break between a CR and LF; otherwise, break before and after - controls. - -3. Do not break Hangul syllable sequences, the rules for which are: - - L may be followed by L, V, LV or LVT - LV or V may be followed by V or T - LVT or T may be followed by T - -4. Do not break before extending characters. - -The next two rules are only for extended grapheme clusters (but that's what we -are implementing). - -5. Do not break before SpacingMarks. - -6. Do not break after Prepend characters. - -7. Otherwise, break everywhere. -*/ - -const pcre_uint32 PRIV(ucp_gbtable[]) = { - (1< 0x10ffff is not permitted -PCRE_UTF8_ERR14 3-byte character with value 0xd000-0xdfff is not permitted -PCRE_UTF8_ERR15 Overlong 2-byte sequence -PCRE_UTF8_ERR16 Overlong 3-byte sequence -PCRE_UTF8_ERR17 Overlong 4-byte sequence -PCRE_UTF8_ERR18 Overlong 5-byte sequence (won't ever occur) -PCRE_UTF8_ERR19 Overlong 6-byte sequence (won't ever occur) -PCRE_UTF8_ERR20 Isolated 0x80 byte (not within UTF-8 character) -PCRE_UTF8_ERR21 Byte with the illegal value 0xfe or 0xff -PCRE_UTF8_ERR22 Unused (was non-character) - -Arguments: - string points to the string - length length of string, or -1 if the string is zero-terminated - errp pointer to an error position offset variable - -Returns: = 0 if the string is a valid UTF-8 string - > 0 otherwise, setting the offset of the bad character -*/ - -int -PRIV(valid_utf)(PCRE_PUCHAR string, int length, int *erroroffset) -{ -#ifdef SUPPORT_UTF -register PCRE_PUCHAR p; - -if (length < 0) - { - for (p = string; *p != 0; p++); - length = (int)(p - string); - } - -for (p = string; length-- > 0; p++) - { - register pcre_uchar ab, c, d; - - c = *p; - if (c < 128) continue; /* ASCII character */ - - if (c < 0xc0) /* Isolated 10xx xxxx byte */ - { - *erroroffset = (int)(p - string); - return PCRE_UTF8_ERR20; - } - - if (c >= 0xfe) /* Invalid 0xfe or 0xff bytes */ - { - *erroroffset = (int)(p - string); - return PCRE_UTF8_ERR21; - } - - ab = PRIV(utf8_table4)[c & 0x3f]; /* Number of additional bytes */ - if (length < ab) - { - *erroroffset = (int)(p - string); /* Missing bytes */ - return ab - length; /* Codes ERR1 to ERR5 */ - } - length -= ab; /* Length remaining */ - - /* Check top bits in the second byte */ - - if (((d = *(++p)) & 0xc0) != 0x80) - { - *erroroffset = (int)(p - string) - 1; - return PCRE_UTF8_ERR6; - } - - /* For each length, check that the remaining bytes start with the 0x80 bit - set and not the 0x40 bit. Then check for an overlong sequence, and for the - excluded range 0xd800 to 0xdfff. */ - - switch (ab) - { - /* 2-byte character. No further bytes to check for 0x80. Check first byte - for for xx00 000x (overlong sequence). */ - - case 1: if ((c & 0x3e) == 0) - { - *erroroffset = (int)(p - string) - 1; - return PCRE_UTF8_ERR15; - } - break; - - /* 3-byte character. Check third byte for 0x80. Then check first 2 bytes - for 1110 0000, xx0x xxxx (overlong sequence) or - 1110 1101, 1010 xxxx (0xd800 - 0xdfff) */ - - case 2: - if ((*(++p) & 0xc0) != 0x80) /* Third byte */ - { - *erroroffset = (int)(p - string) - 2; - return PCRE_UTF8_ERR7; - } - if (c == 0xe0 && (d & 0x20) == 0) - { - *erroroffset = (int)(p - string) - 2; - return PCRE_UTF8_ERR16; - } - if (c == 0xed && d >= 0xa0) - { - *erroroffset = (int)(p - string) - 2; - return PCRE_UTF8_ERR14; - } - break; - - /* 4-byte character. Check 3rd and 4th bytes for 0x80. Then check first 2 - bytes for for 1111 0000, xx00 xxxx (overlong sequence), then check for a - character greater than 0x0010ffff (f4 8f bf bf) */ - - case 3: - if ((*(++p) & 0xc0) != 0x80) /* Third byte */ - { - *erroroffset = (int)(p - string) - 2; - return PCRE_UTF8_ERR7; - } - if ((*(++p) & 0xc0) != 0x80) /* Fourth byte */ - { - *erroroffset = (int)(p - string) - 3; - return PCRE_UTF8_ERR8; - } - if (c == 0xf0 && (d & 0x30) == 0) - { - *erroroffset = (int)(p - string) - 3; - return PCRE_UTF8_ERR17; - } - if (c > 0xf4 || (c == 0xf4 && d > 0x8f)) - { - *erroroffset = (int)(p - string) - 3; - return PCRE_UTF8_ERR13; - } - break; - - /* 5-byte and 6-byte characters are not allowed by RFC 3629, and will be - rejected by the length test below. However, we do the appropriate tests - here so that overlong sequences get diagnosed, and also in case there is - ever an option for handling these larger code points. */ - - /* 5-byte character. Check 3rd, 4th, and 5th bytes for 0x80. Then check for - 1111 1000, xx00 0xxx */ - - case 4: - if ((*(++p) & 0xc0) != 0x80) /* Third byte */ - { - *erroroffset = (int)(p - string) - 2; - return PCRE_UTF8_ERR7; - } - if ((*(++p) & 0xc0) != 0x80) /* Fourth byte */ - { - *erroroffset = (int)(p - string) - 3; - return PCRE_UTF8_ERR8; - } - if ((*(++p) & 0xc0) != 0x80) /* Fifth byte */ - { - *erroroffset = (int)(p - string) - 4; - return PCRE_UTF8_ERR9; - } - if (c == 0xf8 && (d & 0x38) == 0) - { - *erroroffset = (int)(p - string) - 4; - return PCRE_UTF8_ERR18; - } - break; - - /* 6-byte character. Check 3rd-6th bytes for 0x80. Then check for - 1111 1100, xx00 00xx. */ - - case 5: - if ((*(++p) & 0xc0) != 0x80) /* Third byte */ - { - *erroroffset = (int)(p - string) - 2; - return PCRE_UTF8_ERR7; - } - if ((*(++p) & 0xc0) != 0x80) /* Fourth byte */ - { - *erroroffset = (int)(p - string) - 3; - return PCRE_UTF8_ERR8; - } - if ((*(++p) & 0xc0) != 0x80) /* Fifth byte */ - { - *erroroffset = (int)(p - string) - 4; - return PCRE_UTF8_ERR9; - } - if ((*(++p) & 0xc0) != 0x80) /* Sixth byte */ - { - *erroroffset = (int)(p - string) - 5; - return PCRE_UTF8_ERR10; - } - if (c == 0xfc && (d & 0x3c) == 0) - { - *erroroffset = (int)(p - string) - 5; - return PCRE_UTF8_ERR19; - } - break; - } - - /* Character is valid under RFC 2279, but 4-byte and 5-byte characters are - excluded by RFC 3629. The pointer p is currently at the last byte of the - character. */ - - if (ab > 3) - { - *erroroffset = (int)(p - string) - ab; - return (ab == 4)? PCRE_UTF8_ERR11 : PCRE_UTF8_ERR12; - } - } - -#else /* Not SUPPORT_UTF */ -(void)(string); /* Keep picky compilers happy */ -(void)(length); -(void)(erroroffset); -#endif - -return PCRE_UTF8_ERR0; /* This indicates success */ -} - -/* End of pcre_valid_utf8.c */ diff --git a/src/third_party/pcre-8.39/pcre_version.c b/src/third_party/pcre-8.39/pcre_version.c deleted file mode 100644 index ae86ff28bc8..00000000000 --- a/src/third_party/pcre-8.39/pcre_version.c +++ /dev/null @@ -1,98 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This module contains the external function pcre_version(), which returns a -string that identifies the PCRE version that is in use. */ - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "pcre_internal.h" - - -/************************************************* -* Return version string * -*************************************************/ - -/* These macros are the standard way of turning unquoted text into C strings. -They allow macros like PCRE_MAJOR to be defined without quotes, which is -convenient for user programs that want to test its value. */ - -#define STRING(a) # a -#define XSTRING(s) STRING(s) - -/* A problem turned up with PCRE_PRERELEASE, which is defined empty for -production releases. Originally, it was used naively in this code: - - return XSTRING(PCRE_MAJOR) - "." XSTRING(PCRE_MINOR) - XSTRING(PCRE_PRERELEASE) - " " XSTRING(PCRE_DATE); - -However, when PCRE_PRERELEASE is empty, this leads to an attempted expansion of -STRING(). The C standard states: "If (before argument substitution) any -argument consists of no preprocessing tokens, the behavior is undefined." It -turns out the gcc treats this case as a single empty string - which is what we -really want - but Visual C grumbles about the lack of an argument for the -macro. Unfortunately, both are within their rights. To cope with both ways of -handling this, I had resort to some messy hackery that does a test at run time. -I could find no way of detecting that a macro is defined as an empty string at -pre-processor time. This hack uses a standard trick for avoiding calling -the STRING macro with an empty argument when doing the test. */ - -#if defined COMPILE_PCRE8 -PCRE_EXP_DEFN const char * PCRE_CALL_CONVENTION -pcre_version(void) -#elif defined COMPILE_PCRE16 -PCRE_EXP_DEFN const char * PCRE_CALL_CONVENTION -pcre16_version(void) -#elif defined COMPILE_PCRE32 -PCRE_EXP_DEFN const char * PCRE_CALL_CONVENTION -pcre32_version(void) -#endif -{ -return (XSTRING(Z PCRE_PRERELEASE)[1] == 0)? - XSTRING(PCRE_MAJOR.PCRE_MINOR PCRE_DATE) : - XSTRING(PCRE_MAJOR.PCRE_MINOR) XSTRING(PCRE_PRERELEASE PCRE_DATE); -} - -/* End of pcre_version.c */ diff --git a/src/third_party/pcre-8.39/pcre_xclass.c b/src/third_party/pcre-8.39/pcre_xclass.c deleted file mode 100644 index ef759a589a6..00000000000 --- a/src/third_party/pcre-8.39/pcre_xclass.c +++ /dev/null @@ -1,268 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2013 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This module contains an internal function that is used to match an extended -class. It is used by both pcre_exec() and pcre_def_exec(). */ - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "pcre_internal.h" - - -/************************************************* -* Match character against an XCLASS * -*************************************************/ - -/* This function is called to match a character against an extended class that -might contain values > 255 and/or Unicode properties. - -Arguments: - c the character - data points to the flag byte of the XCLASS data - -Returns: TRUE if character matches, else FALSE -*/ - -BOOL -PRIV(xclass)(pcre_uint32 c, const pcre_uchar *data, BOOL utf) -{ -pcre_uchar t; -BOOL negated = (*data & XCL_NOT) != 0; - -(void)utf; -#ifdef COMPILE_PCRE8 -/* In 8 bit mode, this must always be TRUE. Help the compiler to know that. */ -utf = TRUE; -#endif - -/* Character values < 256 are matched against a bitmap, if one is present. If -not, we still carry on, because there may be ranges that start below 256 in the -additional data. */ - -if (c < 256) - { - if ((*data & XCL_HASPROP) == 0) - { - if ((*data & XCL_MAP) == 0) return negated; - return (((pcre_uint8 *)(data + 1))[c/8] & (1 << (c&7))) != 0; - } - if ((*data & XCL_MAP) != 0 && - (((pcre_uint8 *)(data + 1))[c/8] & (1 << (c&7))) != 0) - return !negated; /* char found */ - } - -/* First skip the bit map if present. Then match against the list of Unicode -properties or large chars or ranges that end with a large char. We won't ever -encounter XCL_PROP or XCL_NOTPROP when UCP support is not compiled. */ - -if ((*data++ & XCL_MAP) != 0) data += 32 / sizeof(pcre_uchar); - -while ((t = *data++) != XCL_END) - { - pcre_uint32 x, y; - if (t == XCL_SINGLE) - { -#ifdef SUPPORT_UTF - if (utf) - { - GETCHARINC(x, data); /* macro generates multiple statements */ - } - else -#endif - x = *data++; - if (c == x) return !negated; - } - else if (t == XCL_RANGE) - { -#ifdef SUPPORT_UTF - if (utf) - { - GETCHARINC(x, data); /* macro generates multiple statements */ - GETCHARINC(y, data); /* macro generates multiple statements */ - } - else -#endif - { - x = *data++; - y = *data++; - } - if (c >= x && c <= y) return !negated; - } - -#ifdef SUPPORT_UCP - else /* XCL_PROP & XCL_NOTPROP */ - { - const ucd_record *prop = GET_UCD(c); - BOOL isprop = t == XCL_PROP; - - switch(*data) - { - case PT_ANY: - if (isprop) return !negated; - break; - - case PT_LAMP: - if ((prop->chartype == ucp_Lu || prop->chartype == ucp_Ll || - prop->chartype == ucp_Lt) == isprop) return !negated; - break; - - case PT_GC: - if ((data[1] == PRIV(ucp_gentype)[prop->chartype]) == isprop) - return !negated; - break; - - case PT_PC: - if ((data[1] == prop->chartype) == isprop) return !negated; - break; - - case PT_SC: - if ((data[1] == prop->script) == isprop) return !negated; - break; - - case PT_ALNUM: - if ((PRIV(ucp_gentype)[prop->chartype] == ucp_L || - PRIV(ucp_gentype)[prop->chartype] == ucp_N) == isprop) - return !negated; - break; - - /* Perl space used to exclude VT, but from Perl 5.18 it is included, - which means that Perl space and POSIX space are now identical. PCRE - was changed at release 8.34. */ - - case PT_SPACE: /* Perl space */ - case PT_PXSPACE: /* POSIX space */ - switch(c) - { - HSPACE_CASES: - VSPACE_CASES: - if (isprop) return !negated; - break; - - default: - if ((PRIV(ucp_gentype)[prop->chartype] == ucp_Z) == isprop) - return !negated; - break; - } - break; - - case PT_WORD: - if ((PRIV(ucp_gentype)[prop->chartype] == ucp_L || - PRIV(ucp_gentype)[prop->chartype] == ucp_N || c == CHAR_UNDERSCORE) - == isprop) - return !negated; - break; - - case PT_UCNC: - if (c < 0xa0) - { - if ((c == CHAR_DOLLAR_SIGN || c == CHAR_COMMERCIAL_AT || - c == CHAR_GRAVE_ACCENT) == isprop) - return !negated; - } - else - { - if ((c < 0xd800 || c > 0xdfff) == isprop) - return !negated; - } - break; - - /* The following three properties can occur only in an XCLASS, as there - is no \p or \P coding for them. */ - - /* Graphic character. Implement this as not Z (space or separator) and - not C (other), except for Cf (format) with a few exceptions. This seems - to be what Perl does. The exceptional characters are: - - U+061C Arabic Letter Mark - U+180E Mongolian Vowel Separator - U+2066 - U+2069 Various "isolate"s - */ - - case PT_PXGRAPH: - if ((PRIV(ucp_gentype)[prop->chartype] != ucp_Z && - (PRIV(ucp_gentype)[prop->chartype] != ucp_C || - (prop->chartype == ucp_Cf && - c != 0x061c && c != 0x180e && (c < 0x2066 || c > 0x2069)) - )) == isprop) - return !negated; - break; - - /* Printable character: same as graphic, with the addition of Zs, i.e. - not Zl and not Zp, and U+180E. */ - - case PT_PXPRINT: - if ((prop->chartype != ucp_Zl && - prop->chartype != ucp_Zp && - (PRIV(ucp_gentype)[prop->chartype] != ucp_C || - (prop->chartype == ucp_Cf && - c != 0x061c && (c < 0x2066 || c > 0x2069)) - )) == isprop) - return !negated; - break; - - /* Punctuation: all Unicode punctuation, plus ASCII characters that - Unicode treats as symbols rather than punctuation, for Perl - compatibility (these are $+<=>^`|~). */ - - case PT_PXPUNCT: - if ((PRIV(ucp_gentype)[prop->chartype] == ucp_P || - (c < 128 && PRIV(ucp_gentype)[prop->chartype] == ucp_S)) == isprop) - return !negated; - break; - - /* This should never occur, but compilers may mutter if there is no - default. */ - - default: - return FALSE; - } - - data += 2; - } -#endif /* SUPPORT_UCP */ - } - -return negated; /* char did not match */ -} - -/* End of pcre_xclass.c */ diff --git a/src/third_party/pcre-8.39/pcrecpp.cc b/src/third_party/pcre-8.39/pcrecpp.cc deleted file mode 100644 index d09c9abc516..00000000000 --- a/src/third_party/pcre-8.39/pcrecpp.cc +++ /dev/null @@ -1,924 +0,0 @@ -// Copyright (c) 2010, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Author: Sanjay Ghemawat - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include -#include -#include /* for SHRT_MIN, USHRT_MAX, etc */ -#include /* for memcpy */ -#include -#include -#include -#include - -#include "pcrecpp_internal.h" -#include "pcre.h" -#include "pcrecpp.h" -#include "pcre_stringpiece.h" - - -namespace pcrecpp { - -// Maximum number of args we can set -static const int kMaxArgs = 16; -static const int kVecSize = (1 + kMaxArgs) * 3; // results + PCRE workspace - -// Special object that stands-in for no argument -Arg RE::no_arg((void*)NULL); - -// This is for ABI compatibility with old versions of pcre (pre-7.6), -// which defined a global no_arg variable instead of putting it in the -// RE class. This works on GCC >= 3, at least. It definitely works -// for ELF, but may not for other object formats (Mach-O, for -// instance, does not support aliases.) We could probably have a more -// inclusive test if we ever needed it. (Note that not only the -// __attribute__ syntax, but also __USER_LABEL_PREFIX__, are -// gnu-specific.) -#if defined(__GNUC__) && __GNUC__ >= 3 && defined(__ELF__) && !defined(__INTEL_COMPILER) -# define ULP_AS_STRING(x) ULP_AS_STRING_INTERNAL(x) -# define ULP_AS_STRING_INTERNAL(x) #x -# define USER_LABEL_PREFIX_STR ULP_AS_STRING(__USER_LABEL_PREFIX__) -extern Arg no_arg - __attribute__((alias(USER_LABEL_PREFIX_STR "_ZN7pcrecpp2RE6no_argE"))); -#endif - -// If a regular expression has no error, its error_ field points here -static const string empty_string; - -// If the user doesn't ask for any options, we just use this one -static RE_Options default_options; - -void RE::Init(const string& pat, const RE_Options* options) { - pattern_ = pat; - if (options == NULL) { - options_ = default_options; - } else { - options_ = *options; - } - error_ = &empty_string; - re_full_ = NULL; - re_partial_ = NULL; - - re_partial_ = Compile(UNANCHORED); - if (re_partial_ != NULL) { - re_full_ = Compile(ANCHOR_BOTH); - } -} - -void RE::Cleanup() { - if (re_full_ != NULL) (*pcre_free)(re_full_); - if (re_partial_ != NULL) (*pcre_free)(re_partial_); - if (error_ != &empty_string) delete error_; -} - - -RE::~RE() { - Cleanup(); -} - - -pcre* RE::Compile(Anchor anchor) { - // First, convert RE_Options into pcre options - int pcre_options = 0; - pcre_options = options_.all_options(); - - // Special treatment for anchoring. This is needed because at - // runtime pcre only provides an option for anchoring at the - // beginning of a string (unless you use offset). - // - // There are three types of anchoring we want: - // UNANCHORED Compile the original pattern, and use - // a pcre unanchored match. - // ANCHOR_START Compile the original pattern, and use - // a pcre anchored match. - // ANCHOR_BOTH Tack a "\z" to the end of the original pattern - // and use a pcre anchored match. - - const char* compile_error; - int eoffset; - pcre* re; - if (anchor != ANCHOR_BOTH) { - re = pcre_compile(pattern_.c_str(), pcre_options, - &compile_error, &eoffset, NULL); - } else { - // Tack a '\z' at the end of RE. Parenthesize it first so that - // the '\z' applies to all top-level alternatives in the regexp. - string wrapped = "(?:"; // A non-counting grouping operator - wrapped += pattern_; - wrapped += ")\\z"; - re = pcre_compile(wrapped.c_str(), pcre_options, - &compile_error, &eoffset, NULL); - } - if (re == NULL) { - if (error_ == &empty_string) error_ = new string(compile_error); - } - return re; -} - -/***** Matching interfaces *****/ - -bool RE::FullMatch(const StringPiece& text, - const Arg& ptr1, - const Arg& ptr2, - const Arg& ptr3, - const Arg& ptr4, - const Arg& ptr5, - const Arg& ptr6, - const Arg& ptr7, - const Arg& ptr8, - const Arg& ptr9, - const Arg& ptr10, - const Arg& ptr11, - const Arg& ptr12, - const Arg& ptr13, - const Arg& ptr14, - const Arg& ptr15, - const Arg& ptr16) const { - const Arg* args[kMaxArgs]; - int n = 0; - if (&ptr1 == &no_arg) { goto done; } args[n++] = &ptr1; - if (&ptr2 == &no_arg) { goto done; } args[n++] = &ptr2; - if (&ptr3 == &no_arg) { goto done; } args[n++] = &ptr3; - if (&ptr4 == &no_arg) { goto done; } args[n++] = &ptr4; - if (&ptr5 == &no_arg) { goto done; } args[n++] = &ptr5; - if (&ptr6 == &no_arg) { goto done; } args[n++] = &ptr6; - if (&ptr7 == &no_arg) { goto done; } args[n++] = &ptr7; - if (&ptr8 == &no_arg) { goto done; } args[n++] = &ptr8; - if (&ptr9 == &no_arg) { goto done; } args[n++] = &ptr9; - if (&ptr10 == &no_arg) { goto done; } args[n++] = &ptr10; - if (&ptr11 == &no_arg) { goto done; } args[n++] = &ptr11; - if (&ptr12 == &no_arg) { goto done; } args[n++] = &ptr12; - if (&ptr13 == &no_arg) { goto done; } args[n++] = &ptr13; - if (&ptr14 == &no_arg) { goto done; } args[n++] = &ptr14; - if (&ptr15 == &no_arg) { goto done; } args[n++] = &ptr15; - if (&ptr16 == &no_arg) { goto done; } args[n++] = &ptr16; - done: - - int consumed; - int vec[kVecSize]; - return DoMatchImpl(text, ANCHOR_BOTH, &consumed, args, n, vec, kVecSize); -} - -bool RE::PartialMatch(const StringPiece& text, - const Arg& ptr1, - const Arg& ptr2, - const Arg& ptr3, - const Arg& ptr4, - const Arg& ptr5, - const Arg& ptr6, - const Arg& ptr7, - const Arg& ptr8, - const Arg& ptr9, - const Arg& ptr10, - const Arg& ptr11, - const Arg& ptr12, - const Arg& ptr13, - const Arg& ptr14, - const Arg& ptr15, - const Arg& ptr16) const { - const Arg* args[kMaxArgs]; - int n = 0; - if (&ptr1 == &no_arg) { goto done; } args[n++] = &ptr1; - if (&ptr2 == &no_arg) { goto done; } args[n++] = &ptr2; - if (&ptr3 == &no_arg) { goto done; } args[n++] = &ptr3; - if (&ptr4 == &no_arg) { goto done; } args[n++] = &ptr4; - if (&ptr5 == &no_arg) { goto done; } args[n++] = &ptr5; - if (&ptr6 == &no_arg) { goto done; } args[n++] = &ptr6; - if (&ptr7 == &no_arg) { goto done; } args[n++] = &ptr7; - if (&ptr8 == &no_arg) { goto done; } args[n++] = &ptr8; - if (&ptr9 == &no_arg) { goto done; } args[n++] = &ptr9; - if (&ptr10 == &no_arg) { goto done; } args[n++] = &ptr10; - if (&ptr11 == &no_arg) { goto done; } args[n++] = &ptr11; - if (&ptr12 == &no_arg) { goto done; } args[n++] = &ptr12; - if (&ptr13 == &no_arg) { goto done; } args[n++] = &ptr13; - if (&ptr14 == &no_arg) { goto done; } args[n++] = &ptr14; - if (&ptr15 == &no_arg) { goto done; } args[n++] = &ptr15; - if (&ptr16 == &no_arg) { goto done; } args[n++] = &ptr16; - done: - - int consumed; - int vec[kVecSize]; - return DoMatchImpl(text, UNANCHORED, &consumed, args, n, vec, kVecSize); -} - -bool RE::Consume(StringPiece* input, - const Arg& ptr1, - const Arg& ptr2, - const Arg& ptr3, - const Arg& ptr4, - const Arg& ptr5, - const Arg& ptr6, - const Arg& ptr7, - const Arg& ptr8, - const Arg& ptr9, - const Arg& ptr10, - const Arg& ptr11, - const Arg& ptr12, - const Arg& ptr13, - const Arg& ptr14, - const Arg& ptr15, - const Arg& ptr16) const { - const Arg* args[kMaxArgs]; - int n = 0; - if (&ptr1 == &no_arg) { goto done; } args[n++] = &ptr1; - if (&ptr2 == &no_arg) { goto done; } args[n++] = &ptr2; - if (&ptr3 == &no_arg) { goto done; } args[n++] = &ptr3; - if (&ptr4 == &no_arg) { goto done; } args[n++] = &ptr4; - if (&ptr5 == &no_arg) { goto done; } args[n++] = &ptr5; - if (&ptr6 == &no_arg) { goto done; } args[n++] = &ptr6; - if (&ptr7 == &no_arg) { goto done; } args[n++] = &ptr7; - if (&ptr8 == &no_arg) { goto done; } args[n++] = &ptr8; - if (&ptr9 == &no_arg) { goto done; } args[n++] = &ptr9; - if (&ptr10 == &no_arg) { goto done; } args[n++] = &ptr10; - if (&ptr11 == &no_arg) { goto done; } args[n++] = &ptr11; - if (&ptr12 == &no_arg) { goto done; } args[n++] = &ptr12; - if (&ptr13 == &no_arg) { goto done; } args[n++] = &ptr13; - if (&ptr14 == &no_arg) { goto done; } args[n++] = &ptr14; - if (&ptr15 == &no_arg) { goto done; } args[n++] = &ptr15; - if (&ptr16 == &no_arg) { goto done; } args[n++] = &ptr16; - done: - - int consumed; - int vec[kVecSize]; - if (DoMatchImpl(*input, ANCHOR_START, &consumed, - args, n, vec, kVecSize)) { - input->remove_prefix(consumed); - return true; - } else { - return false; - } -} - -bool RE::FindAndConsume(StringPiece* input, - const Arg& ptr1, - const Arg& ptr2, - const Arg& ptr3, - const Arg& ptr4, - const Arg& ptr5, - const Arg& ptr6, - const Arg& ptr7, - const Arg& ptr8, - const Arg& ptr9, - const Arg& ptr10, - const Arg& ptr11, - const Arg& ptr12, - const Arg& ptr13, - const Arg& ptr14, - const Arg& ptr15, - const Arg& ptr16) const { - const Arg* args[kMaxArgs]; - int n = 0; - if (&ptr1 == &no_arg) { goto done; } args[n++] = &ptr1; - if (&ptr2 == &no_arg) { goto done; } args[n++] = &ptr2; - if (&ptr3 == &no_arg) { goto done; } args[n++] = &ptr3; - if (&ptr4 == &no_arg) { goto done; } args[n++] = &ptr4; - if (&ptr5 == &no_arg) { goto done; } args[n++] = &ptr5; - if (&ptr6 == &no_arg) { goto done; } args[n++] = &ptr6; - if (&ptr7 == &no_arg) { goto done; } args[n++] = &ptr7; - if (&ptr8 == &no_arg) { goto done; } args[n++] = &ptr8; - if (&ptr9 == &no_arg) { goto done; } args[n++] = &ptr9; - if (&ptr10 == &no_arg) { goto done; } args[n++] = &ptr10; - if (&ptr11 == &no_arg) { goto done; } args[n++] = &ptr11; - if (&ptr12 == &no_arg) { goto done; } args[n++] = &ptr12; - if (&ptr13 == &no_arg) { goto done; } args[n++] = &ptr13; - if (&ptr14 == &no_arg) { goto done; } args[n++] = &ptr14; - if (&ptr15 == &no_arg) { goto done; } args[n++] = &ptr15; - if (&ptr16 == &no_arg) { goto done; } args[n++] = &ptr16; - done: - - int consumed; - int vec[kVecSize]; - if (DoMatchImpl(*input, UNANCHORED, &consumed, - args, n, vec, kVecSize)) { - input->remove_prefix(consumed); - return true; - } else { - return false; - } -} - -bool RE::Replace(const StringPiece& rewrite, - string *str) const { - int vec[kVecSize]; - int matches = TryMatch(*str, 0, UNANCHORED, true, vec, kVecSize); - if (matches == 0) - return false; - - string s; - if (!Rewrite(&s, rewrite, *str, vec, matches)) - return false; - - assert(vec[0] >= 0); - assert(vec[1] >= 0); - str->replace(vec[0], vec[1] - vec[0], s); - return true; -} - -// Returns PCRE_NEWLINE_CRLF, PCRE_NEWLINE_CR, or PCRE_NEWLINE_LF. -// Note that PCRE_NEWLINE_CRLF is defined to be P_N_CR | P_N_LF. -// Modified by PH to add PCRE_NEWLINE_ANY and PCRE_NEWLINE_ANYCRLF. - -static int NewlineMode(int pcre_options) { - // TODO: if we can make it threadsafe, cache this var - int newline_mode = 0; - /* if (newline_mode) return newline_mode; */ // do this once it's cached - if (pcre_options & (PCRE_NEWLINE_CRLF|PCRE_NEWLINE_CR|PCRE_NEWLINE_LF| - PCRE_NEWLINE_ANY|PCRE_NEWLINE_ANYCRLF)) { - newline_mode = (pcre_options & - (PCRE_NEWLINE_CRLF|PCRE_NEWLINE_CR|PCRE_NEWLINE_LF| - PCRE_NEWLINE_ANY|PCRE_NEWLINE_ANYCRLF)); - } else { - int newline; - pcre_config(PCRE_CONFIG_NEWLINE, &newline); - if (newline == 10) - newline_mode = PCRE_NEWLINE_LF; - else if (newline == 13) - newline_mode = PCRE_NEWLINE_CR; - else if (newline == 3338) - newline_mode = PCRE_NEWLINE_CRLF; - else if (newline == -1) - newline_mode = PCRE_NEWLINE_ANY; - else if (newline == -2) - newline_mode = PCRE_NEWLINE_ANYCRLF; - else - assert(NULL == "Unexpected return value from pcre_config(NEWLINE)"); - } - return newline_mode; -} - -int RE::GlobalReplace(const StringPiece& rewrite, - string *str) const { - int count = 0; - int vec[kVecSize]; - string out; - int start = 0; - bool last_match_was_empty_string = false; - - while (start <= static_cast(str->length())) { - // If the previous match was for the empty string, we shouldn't - // just match again: we'll match in the same way and get an - // infinite loop. Instead, we do the match in a special way: - // anchored -- to force another try at the same position -- - // and with a flag saying that this time, ignore empty matches. - // If this special match returns, that means there's a non-empty - // match at this position as well, and we can continue. If not, - // we do what perl does, and just advance by one. - // Notice that perl prints '@@@' for this; - // perl -le '$_ = "aa"; s/b*|aa/@/g; print' - int matches; - if (last_match_was_empty_string) { - matches = TryMatch(*str, start, ANCHOR_START, false, vec, kVecSize); - if (matches <= 0) { - int matchend = start + 1; // advance one character. - // If the current char is CR and we're in CRLF mode, skip LF too. - // Note it's better to call pcre_fullinfo() than to examine - // all_options(), since options_ could have changed bewteen - // compile-time and now, but this is simpler and safe enough. - // Modified by PH to add ANY and ANYCRLF. - if (matchend < static_cast(str->length()) && - (*str)[start] == '\r' && (*str)[matchend] == '\n' && - (NewlineMode(options_.all_options()) == PCRE_NEWLINE_CRLF || - NewlineMode(options_.all_options()) == PCRE_NEWLINE_ANY || - NewlineMode(options_.all_options()) == PCRE_NEWLINE_ANYCRLF)) { - matchend++; - } - // We also need to advance more than one char if we're in utf8 mode. -#ifdef SUPPORT_UTF8 - if (options_.utf8()) { - while (matchend < static_cast(str->length()) && - ((*str)[matchend] & 0xc0) == 0x80) - matchend++; - } -#endif - if (start < static_cast(str->length())) - out.append(*str, start, matchend - start); - start = matchend; - last_match_was_empty_string = false; - continue; - } - } else { - matches = TryMatch(*str, start, UNANCHORED, true, vec, kVecSize); - if (matches <= 0) - break; - } - int matchstart = vec[0], matchend = vec[1]; - assert(matchstart >= start); - assert(matchend >= matchstart); - out.append(*str, start, matchstart - start); - Rewrite(&out, rewrite, *str, vec, matches); - start = matchend; - count++; - last_match_was_empty_string = (matchstart == matchend); - } - - if (count == 0) - return 0; - - if (start < static_cast(str->length())) - out.append(*str, start, str->length() - start); - swap(out, *str); - return count; -} - -bool RE::Extract(const StringPiece& rewrite, - const StringPiece& text, - string *out) const { - int vec[kVecSize]; - int matches = TryMatch(text, 0, UNANCHORED, true, vec, kVecSize); - if (matches == 0) - return false; - out->erase(); - return Rewrite(out, rewrite, text, vec, matches); -} - -/*static*/ string RE::QuoteMeta(const StringPiece& unquoted) { - string result; - - // Escape any ascii character not in [A-Za-z_0-9]. - // - // Note that it's legal to escape a character even if it has no - // special meaning in a regular expression -- so this function does - // that. (This also makes it identical to the perl function of the - // same name; see `perldoc -f quotemeta`.) The one exception is - // escaping NUL: rather than doing backslash + NUL, like perl does, - // we do '\0', because pcre itself doesn't take embedded NUL chars. - for (int ii = 0; ii < unquoted.size(); ++ii) { - // Note that using 'isalnum' here raises the benchmark time from - // 32ns to 58ns: - if (unquoted[ii] == '\0') { - result += "\\0"; - } else if ((unquoted[ii] < 'a' || unquoted[ii] > 'z') && - (unquoted[ii] < 'A' || unquoted[ii] > 'Z') && - (unquoted[ii] < '0' || unquoted[ii] > '9') && - unquoted[ii] != '_' && - // If this is the part of a UTF8 or Latin1 character, we need - // to copy this byte without escaping. Experimentally this is - // what works correctly with the regexp library. - !(unquoted[ii] & 128)) { - result += '\\'; - result += unquoted[ii]; - } else { - result += unquoted[ii]; - } - } - - return result; -} - -/***** Actual matching and rewriting code *****/ - -int RE::TryMatch(const StringPiece& text, - int startpos, - Anchor anchor, - bool empty_ok, - int *vec, - int vecsize) const { - pcre* re = (anchor == ANCHOR_BOTH) ? re_full_ : re_partial_; - if (re == NULL) { - //fprintf(stderr, "Matching against invalid re: %s\n", error_->c_str()); - return 0; - } - - pcre_extra extra = { 0, 0, 0, 0, 0, 0, 0, 0 }; - if (options_.match_limit() > 0) { - extra.flags |= PCRE_EXTRA_MATCH_LIMIT; - extra.match_limit = options_.match_limit(); - } - if (options_.match_limit_recursion() > 0) { - extra.flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION; - extra.match_limit_recursion = options_.match_limit_recursion(); - } - - // int options = 0; - // Changed by PH as a result of bugzilla #1288 - int options = (options_.all_options() & PCRE_NO_UTF8_CHECK); - - if (anchor != UNANCHORED) - options |= PCRE_ANCHORED; - if (!empty_ok) - options |= PCRE_NOTEMPTY; - - int rc = pcre_exec(re, // The regular expression object - &extra, - (text.data() == NULL) ? "" : text.data(), - text.size(), - startpos, - options, - vec, - vecsize); - - // Handle errors - if (rc == PCRE_ERROR_NOMATCH) { - return 0; - } else if (rc < 0) { - //fprintf(stderr, "Unexpected return code: %d when matching '%s'\n", - // re, pattern_.c_str()); - return 0; - } else if (rc == 0) { - // pcre_exec() returns 0 as a special case when the number of - // capturing subpatterns exceeds the size of the vector. - // When this happens, there is a match and the output vector - // is filled, but we miss out on the positions of the extra subpatterns. - rc = vecsize / 2; - } - - return rc; -} - -bool RE::DoMatchImpl(const StringPiece& text, - Anchor anchor, - int* consumed, - const Arg* const* args, - int n, - int* vec, - int vecsize) const { - assert((1 + n) * 3 <= vecsize); // results + PCRE workspace - int matches = TryMatch(text, 0, anchor, true, vec, vecsize); - assert(matches >= 0); // TryMatch never returns negatives - if (matches == 0) - return false; - - *consumed = vec[1]; - - if (n == 0 || args == NULL) { - // We are not interested in results - return true; - } - - if (NumberOfCapturingGroups() < n) { - // RE has fewer capturing groups than number of arg pointers passed in - return false; - } - - // If we got here, we must have matched the whole pattern. - // We do not need (can not do) any more checks on the value of 'matches' here - // -- see the comment for TryMatch. - for (int i = 0; i < n; i++) { - const int start = vec[2*(i+1)]; - const int limit = vec[2*(i+1)+1]; - if (!args[i]->Parse(text.data() + start, limit-start)) { - // TODO: Should we indicate what the error was? - return false; - } - } - - return true; -} - -bool RE::DoMatch(const StringPiece& text, - Anchor anchor, - int* consumed, - const Arg* const args[], - int n) const { - assert(n >= 0); - size_t const vecsize = (1 + n) * 3; // results + PCRE workspace - // (as for kVecSize) - int space[21]; // use stack allocation for small vecsize (common case) - int* vec = vecsize <= 21 ? space : new int[vecsize]; - bool retval = DoMatchImpl(text, anchor, consumed, args, n, vec, (int)vecsize); - if (vec != space) delete [] vec; - return retval; -} - -bool RE::Rewrite(string *out, const StringPiece &rewrite, - const StringPiece &text, int *vec, int veclen) const { - for (const char *s = rewrite.data(), *end = s + rewrite.size(); - s < end; s++) { - int c = *s; - if (c == '\\') { - c = *++s; - if (isdigit(c)) { - int n = (c - '0'); - if (n >= veclen) { - //fprintf(stderr, requested group %d in regexp %.*s\n", - // n, rewrite.size(), rewrite.data()); - return false; - } - int start = vec[2 * n]; - if (start >= 0) - out->append(text.data() + start, vec[2 * n + 1] - start); - } else if (c == '\\') { - *out += '\\'; - } else { - //fprintf(stderr, "invalid rewrite pattern: %.*s\n", - // rewrite.size(), rewrite.data()); - return false; - } - } else { - *out += c; - } - } - return true; -} - -// Return the number of capturing subpatterns, or -1 if the -// regexp wasn't valid on construction. -int RE::NumberOfCapturingGroups() const { - if (re_partial_ == NULL) return -1; - - int result; - int pcre_retval = pcre_fullinfo(re_partial_, // The regular expression object - NULL, // We did not study the pattern - PCRE_INFO_CAPTURECOUNT, - &result); - assert(pcre_retval == 0); - return result; -} - -/***** Parsers for various types *****/ - -bool Arg::parse_null(const char* str, int n, void* dest) { - (void)str; - (void)n; - // We fail if somebody asked us to store into a non-NULL void* pointer - return (dest == NULL); -} - -bool Arg::parse_string(const char* str, int n, void* dest) { - if (dest == NULL) return true; - reinterpret_cast(dest)->assign(str, n); - return true; -} - -bool Arg::parse_stringpiece(const char* str, int n, void* dest) { - if (dest == NULL) return true; - reinterpret_cast(dest)->set(str, n); - return true; -} - -bool Arg::parse_char(const char* str, int n, void* dest) { - if (n != 1) return false; - if (dest == NULL) return true; - *(reinterpret_cast(dest)) = str[0]; - return true; -} - -bool Arg::parse_uchar(const char* str, int n, void* dest) { - if (n != 1) return false; - if (dest == NULL) return true; - *(reinterpret_cast(dest)) = str[0]; - return true; -} - -// Largest number spec that we are willing to parse -static const int kMaxNumberLength = 32; - -// REQUIRES "buf" must have length at least kMaxNumberLength+1 -// REQUIRES "n > 0" -// Copies "str" into "buf" and null-terminates if necessary. -// Returns one of: -// a. "str" if no termination is needed -// b. "buf" if the string was copied and null-terminated -// c. "" if the input was invalid and has no hope of being parsed -static const char* TerminateNumber(char* buf, const char* str, int n) { - if ((n > 0) && isspace(*str)) { - // We are less forgiving than the strtoxxx() routines and do not - // allow leading spaces. - return ""; - } - - // See if the character right after the input text may potentially - // look like a digit. - if (isdigit(str[n]) || - ((str[n] >= 'a') && (str[n] <= 'f')) || - ((str[n] >= 'A') && (str[n] <= 'F'))) { - if (n > kMaxNumberLength) return ""; // Input too big to be a valid number - memcpy(buf, str, n); - buf[n] = '\0'; - return buf; - } else { - // We can parse right out of the supplied string, so return it. - return str; - } -} - -bool Arg::parse_long_radix(const char* str, - int n, - void* dest, - int radix) { - if (n == 0) return false; - char buf[kMaxNumberLength+1]; - str = TerminateNumber(buf, str, n); - char* end; - errno = 0; - long r = strtol(str, &end, radix); - if (end != str + n) return false; // Leftover junk - if (errno) return false; - if (dest == NULL) return true; - *(reinterpret_cast(dest)) = r; - return true; -} - -bool Arg::parse_ulong_radix(const char* str, - int n, - void* dest, - int radix) { - if (n == 0) return false; - char buf[kMaxNumberLength+1]; - str = TerminateNumber(buf, str, n); - if (str[0] == '-') return false; // strtoul() on a negative number?! - char* end; - errno = 0; - unsigned long r = strtoul(str, &end, radix); - if (end != str + n) return false; // Leftover junk - if (errno) return false; - if (dest == NULL) return true; - *(reinterpret_cast(dest)) = r; - return true; -} - -bool Arg::parse_short_radix(const char* str, - int n, - void* dest, - int radix) { - long r; - if (!parse_long_radix(str, n, &r, radix)) return false; // Could not parse - if (r < SHRT_MIN || r > SHRT_MAX) return false; // Out of range - if (dest == NULL) return true; - *(reinterpret_cast(dest)) = static_cast(r); - return true; -} - -bool Arg::parse_ushort_radix(const char* str, - int n, - void* dest, - int radix) { - unsigned long r; - if (!parse_ulong_radix(str, n, &r, radix)) return false; // Could not parse - if (r > USHRT_MAX) return false; // Out of range - if (dest == NULL) return true; - *(reinterpret_cast(dest)) = static_cast(r); - return true; -} - -bool Arg::parse_int_radix(const char* str, - int n, - void* dest, - int radix) { - long r; - if (!parse_long_radix(str, n, &r, radix)) return false; // Could not parse - if (r < INT_MIN || r > INT_MAX) return false; // Out of range - if (dest == NULL) return true; - *(reinterpret_cast(dest)) = r; - return true; -} - -bool Arg::parse_uint_radix(const char* str, - int n, - void* dest, - int radix) { - unsigned long r; - if (!parse_ulong_radix(str, n, &r, radix)) return false; // Could not parse - if (r > UINT_MAX) return false; // Out of range - if (dest == NULL) return true; - *(reinterpret_cast(dest)) = r; - return true; -} - -bool Arg::parse_longlong_radix(const char* str, - int n, - void* dest, - int radix) { -#ifndef HAVE_LONG_LONG - return false; -#else - if (n == 0) return false; - char buf[kMaxNumberLength+1]; - str = TerminateNumber(buf, str, n); - char* end; - errno = 0; -#if defined HAVE_STRTOQ - long long r = strtoq(str, &end, radix); -#elif defined HAVE_STRTOLL - long long r = strtoll(str, &end, radix); -#elif defined HAVE__STRTOI64 - long long r = _strtoi64(str, &end, radix); -#elif defined HAVE_STRTOIMAX - long long r = strtoimax(str, &end, radix); -#else -#error parse_longlong_radix: cannot convert input to a long-long -#endif - if (end != str + n) return false; // Leftover junk - if (errno) return false; - if (dest == NULL) return true; - *(reinterpret_cast(dest)) = r; - return true; -#endif /* HAVE_LONG_LONG */ -} - -bool Arg::parse_ulonglong_radix(const char* str, - int n, - void* dest, - int radix) { -#ifndef HAVE_UNSIGNED_LONG_LONG - return false; -#else - if (n == 0) return false; - char buf[kMaxNumberLength+1]; - str = TerminateNumber(buf, str, n); - if (str[0] == '-') return false; // strtoull() on a negative number?! - char* end; - errno = 0; -#if defined HAVE_STRTOQ - unsigned long long r = strtouq(str, &end, radix); -#elif defined HAVE_STRTOLL - unsigned long long r = strtoull(str, &end, radix); -#elif defined HAVE__STRTOI64 - unsigned long long r = _strtoui64(str, &end, radix); -#elif defined HAVE_STRTOIMAX - unsigned long long r = strtoumax(str, &end, radix); -#else -#error parse_ulonglong_radix: cannot convert input to a long-long -#endif - if (end != str + n) return false; // Leftover junk - if (errno) return false; - if (dest == NULL) return true; - *(reinterpret_cast(dest)) = r; - return true; -#endif /* HAVE_UNSIGNED_LONG_LONG */ -} - -bool Arg::parse_double(const char* str, int n, void* dest) { - if (n == 0) return false; - static const int kMaxLength = 200; - char buf[kMaxLength]; - if (n >= kMaxLength) return false; - memcpy(buf, str, n); - buf[n] = '\0'; - errno = 0; - char* end; - double r = strtod(buf, &end); - if (end != buf + n) return false; // Leftover junk - if (errno) return false; - if (dest == NULL) return true; - *(reinterpret_cast(dest)) = r; - return true; -} - -bool Arg::parse_float(const char* str, int n, void* dest) { - double r; - if (!parse_double(str, n, &r)) return false; - if (dest == NULL) return true; - *(reinterpret_cast(dest)) = static_cast(r); - return true; -} - - -#define DEFINE_INTEGER_PARSERS(name) \ - bool Arg::parse_##name(const char* str, int n, void* dest) { \ - return parse_##name##_radix(str, n, dest, 10); \ - } \ - bool Arg::parse_##name##_hex(const char* str, int n, void* dest) { \ - return parse_##name##_radix(str, n, dest, 16); \ - } \ - bool Arg::parse_##name##_octal(const char* str, int n, void* dest) { \ - return parse_##name##_radix(str, n, dest, 8); \ - } \ - bool Arg::parse_##name##_cradix(const char* str, int n, void* dest) { \ - return parse_##name##_radix(str, n, dest, 0); \ - } - -DEFINE_INTEGER_PARSERS(short) /* */ -DEFINE_INTEGER_PARSERS(ushort) /* */ -DEFINE_INTEGER_PARSERS(int) /* Don't use semicolons after these */ -DEFINE_INTEGER_PARSERS(uint) /* statements because they can cause */ -DEFINE_INTEGER_PARSERS(long) /* compiler warnings if the checking */ -DEFINE_INTEGER_PARSERS(ulong) /* level is turned up high enough. */ -DEFINE_INTEGER_PARSERS(longlong) /* */ -DEFINE_INTEGER_PARSERS(ulonglong) /* */ - -#undef DEFINE_INTEGER_PARSERS - -} // namespace pcrecpp diff --git a/src/third_party/pcre-8.39/pcrecpp.h b/src/third_party/pcre-8.39/pcrecpp.h deleted file mode 100644 index 3e594b0d439..00000000000 --- a/src/third_party/pcre-8.39/pcrecpp.h +++ /dev/null @@ -1,710 +0,0 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Author: Sanjay Ghemawat -// Support for PCRE_XXX modifiers added by Giuseppe Maxia, July 2005 - -#ifndef _PCRECPP_H -#define _PCRECPP_H - -// C++ interface to the pcre regular-expression library. RE supports -// Perl-style regular expressions (with extensions like \d, \w, \s, -// ...). -// -// ----------------------------------------------------------------------- -// REGEXP SYNTAX: -// -// This module is part of the pcre library and hence supports its syntax -// for regular expressions. -// -// The syntax is pretty similar to Perl's. For those not familiar -// with Perl's regular expressions, here are some examples of the most -// commonly used extensions: -// -// "hello (\\w+) world" -- \w matches a "word" character -// "version (\\d+)" -- \d matches a digit -// "hello\\s+world" -- \s matches any whitespace character -// "\\b(\\w+)\\b" -- \b matches empty string at a word boundary -// "(?i)hello" -- (?i) turns on case-insensitive matching -// "/\\*(.*?)\\*/" -- .*? matches . minimum no. of times possible -// -// ----------------------------------------------------------------------- -// MATCHING INTERFACE: -// -// The "FullMatch" operation checks that supplied text matches a -// supplied pattern exactly. -// -// Example: successful match -// pcrecpp::RE re("h.*o"); -// re.FullMatch("hello"); -// -// Example: unsuccessful match (requires full match): -// pcrecpp::RE re("e"); -// !re.FullMatch("hello"); -// -// Example: creating a temporary RE object: -// pcrecpp::RE("h.*o").FullMatch("hello"); -// -// You can pass in a "const char*" or a "string" for "text". The -// examples below tend to use a const char*. -// -// You can, as in the different examples above, store the RE object -// explicitly in a variable or use a temporary RE object. The -// examples below use one mode or the other arbitrarily. Either -// could correctly be used for any of these examples. -// -// ----------------------------------------------------------------------- -// MATCHING WITH SUB-STRING EXTRACTION: -// -// You can supply extra pointer arguments to extract matched subpieces. -// -// Example: extracts "ruby" into "s" and 1234 into "i" -// int i; -// string s; -// pcrecpp::RE re("(\\w+):(\\d+)"); -// re.FullMatch("ruby:1234", &s, &i); -// -// Example: does not try to extract any extra sub-patterns -// re.FullMatch("ruby:1234", &s); -// -// Example: does not try to extract into NULL -// re.FullMatch("ruby:1234", NULL, &i); -// -// Example: integer overflow causes failure -// !re.FullMatch("ruby:1234567891234", NULL, &i); -// -// Example: fails because there aren't enough sub-patterns: -// !pcrecpp::RE("\\w+:\\d+").FullMatch("ruby:1234", &s); -// -// Example: fails because string cannot be stored in integer -// !pcrecpp::RE("(.*)").FullMatch("ruby", &i); -// -// The provided pointer arguments can be pointers to any scalar numeric -// type, or one of -// string (matched piece is copied to string) -// StringPiece (StringPiece is mutated to point to matched piece) -// T (where "bool T::ParseFrom(const char*, int)" exists) -// NULL (the corresponding matched sub-pattern is not copied) -// -// CAVEAT: An optional sub-pattern that does not exist in the matched -// string is assigned the empty string. Therefore, the following will -// return false (because the empty string is not a valid number): -// int number; -// pcrecpp::RE::FullMatch("abc", "[a-z]+(\\d+)?", &number); -// -// ----------------------------------------------------------------------- -// DO_MATCH -// -// The matching interface supports at most 16 arguments per call. -// If you need more, consider using the more general interface -// pcrecpp::RE::DoMatch(). See pcrecpp.h for the signature for DoMatch. -// -// ----------------------------------------------------------------------- -// PARTIAL MATCHES -// -// You can use the "PartialMatch" operation when you want the pattern -// to match any substring of the text. -// -// Example: simple search for a string: -// pcrecpp::RE("ell").PartialMatch("hello"); -// -// Example: find first number in a string: -// int number; -// pcrecpp::RE re("(\\d+)"); -// re.PartialMatch("x*100 + 20", &number); -// assert(number == 100); -// -// ----------------------------------------------------------------------- -// UTF-8 AND THE MATCHING INTERFACE: -// -// By default, pattern and text are plain text, one byte per character. -// The UTF8 flag, passed to the constructor, causes both pattern -// and string to be treated as UTF-8 text, still a byte stream but -// potentially multiple bytes per character. In practice, the text -// is likelier to be UTF-8 than the pattern, but the match returned -// may depend on the UTF8 flag, so always use it when matching -// UTF8 text. E.g., "." will match one byte normally but with UTF8 -// set may match up to three bytes of a multi-byte character. -// -// Example: -// pcrecpp::RE_Options options; -// options.set_utf8(); -// pcrecpp::RE re(utf8_pattern, options); -// re.FullMatch(utf8_string); -// -// Example: using the convenience function UTF8(): -// pcrecpp::RE re(utf8_pattern, pcrecpp::UTF8()); -// re.FullMatch(utf8_string); -// -// NOTE: The UTF8 option is ignored if pcre was not configured with the -// --enable-utf8 flag. -// -// ----------------------------------------------------------------------- -// PASSING MODIFIERS TO THE REGULAR EXPRESSION ENGINE -// -// PCRE defines some modifiers to change the behavior of the regular -// expression engine. -// The C++ wrapper defines an auxiliary class, RE_Options, as a vehicle -// to pass such modifiers to a RE class. -// -// Currently, the following modifiers are supported -// -// modifier description Perl corresponding -// -// PCRE_CASELESS case insensitive match /i -// PCRE_MULTILINE multiple lines match /m -// PCRE_DOTALL dot matches newlines /s -// PCRE_DOLLAR_ENDONLY $ matches only at end N/A -// PCRE_EXTRA strict escape parsing N/A -// PCRE_EXTENDED ignore whitespaces /x -// PCRE_UTF8 handles UTF8 chars built-in -// PCRE_UNGREEDY reverses * and *? N/A -// PCRE_NO_AUTO_CAPTURE disables matching parens N/A (*) -// -// (For a full account on how each modifier works, please check the -// PCRE API reference manual). -// -// (*) Both Perl and PCRE allow non matching parentheses by means of the -// "?:" modifier within the pattern itself. e.g. (?:ab|cd) does not -// capture, while (ab|cd) does. -// -// For each modifier, there are two member functions whose name is made -// out of the modifier in lowercase, without the "PCRE_" prefix. For -// instance, PCRE_CASELESS is handled by -// bool caseless(), -// which returns true if the modifier is set, and -// RE_Options & set_caseless(bool), -// which sets or unsets the modifier. -// -// Moreover, PCRE_EXTRA_MATCH_LIMIT can be accessed through the -// set_match_limit() and match_limit() member functions. -// Setting match_limit to a non-zero value will limit the executation of -// pcre to keep it from doing bad things like blowing the stack or taking -// an eternity to return a result. A value of 5000 is good enough to stop -// stack blowup in a 2MB thread stack. Setting match_limit to zero will -// disable match limiting. Alternately, you can set match_limit_recursion() -// which uses PCRE_EXTRA_MATCH_LIMIT_RECURSION to limit how much pcre -// recurses. match_limit() caps the number of matches pcre does; -// match_limit_recrusion() caps the depth of recursion. -// -// Normally, to pass one or more modifiers to a RE class, you declare -// a RE_Options object, set the appropriate options, and pass this -// object to a RE constructor. Example: -// -// RE_options opt; -// opt.set_caseless(true); -// -// if (RE("HELLO", opt).PartialMatch("hello world")) ... -// -// RE_options has two constructors. The default constructor takes no -// arguments and creates a set of flags that are off by default. -// -// The optional parameter 'option_flags' is to facilitate transfer -// of legacy code from C programs. This lets you do -// RE(pattern, RE_Options(PCRE_CASELESS|PCRE_MULTILINE)).PartialMatch(str); -// -// But new code is better off doing -// RE(pattern, -// RE_Options().set_caseless(true).set_multiline(true)).PartialMatch(str); -// (See below) -// -// If you are going to pass one of the most used modifiers, there are some -// convenience functions that return a RE_Options class with the -// appropriate modifier already set: -// CASELESS(), UTF8(), MULTILINE(), DOTALL(), EXTENDED() -// -// If you need to set several options at once, and you don't want to go -// through the pains of declaring a RE_Options object and setting several -// options, there is a parallel method that give you such ability on the -// fly. You can concatenate several set_xxxxx member functions, since each -// of them returns a reference to its class object. e.g.: to pass -// PCRE_CASELESS, PCRE_EXTENDED, and PCRE_MULTILINE to a RE with one -// statement, you may write -// -// RE(" ^ xyz \\s+ .* blah$", RE_Options() -// .set_caseless(true) -// .set_extended(true) -// .set_multiline(true)).PartialMatch(sometext); -// -// ----------------------------------------------------------------------- -// SCANNING TEXT INCREMENTALLY -// -// The "Consume" operation may be useful if you want to repeatedly -// match regular expressions at the front of a string and skip over -// them as they match. This requires use of the "StringPiece" type, -// which represents a sub-range of a real string. Like RE, StringPiece -// is defined in the pcrecpp namespace. -// -// Example: read lines of the form "var = value" from a string. -// string contents = ...; // Fill string somehow -// pcrecpp::StringPiece input(contents); // Wrap in a StringPiece -// -// string var; -// int value; -// pcrecpp::RE re("(\\w+) = (\\d+)\n"); -// while (re.Consume(&input, &var, &value)) { -// ...; -// } -// -// Each successful call to "Consume" will set "var/value", and also -// advance "input" so it points past the matched text. -// -// The "FindAndConsume" operation is similar to "Consume" but does not -// anchor your match at the beginning of the string. For example, you -// could extract all words from a string by repeatedly calling -// pcrecpp::RE("(\\w+)").FindAndConsume(&input, &word) -// -// ----------------------------------------------------------------------- -// PARSING HEX/OCTAL/C-RADIX NUMBERS -// -// By default, if you pass a pointer to a numeric value, the -// corresponding text is interpreted as a base-10 number. You can -// instead wrap the pointer with a call to one of the operators Hex(), -// Octal(), or CRadix() to interpret the text in another base. The -// CRadix operator interprets C-style "0" (base-8) and "0x" (base-16) -// prefixes, but defaults to base-10. -// -// Example: -// int a, b, c, d; -// pcrecpp::RE re("(.*) (.*) (.*) (.*)"); -// re.FullMatch("100 40 0100 0x40", -// pcrecpp::Octal(&a), pcrecpp::Hex(&b), -// pcrecpp::CRadix(&c), pcrecpp::CRadix(&d)); -// will leave 64 in a, b, c, and d. -// -// ----------------------------------------------------------------------- -// REPLACING PARTS OF STRINGS -// -// You can replace the first match of "pattern" in "str" with -// "rewrite". Within "rewrite", backslash-escaped digits (\1 to \9) -// can be used to insert text matching corresponding parenthesized -// group from the pattern. \0 in "rewrite" refers to the entire -// matching text. E.g., -// -// string s = "yabba dabba doo"; -// pcrecpp::RE("b+").Replace("d", &s); -// -// will leave "s" containing "yada dabba doo". The result is true if -// the pattern matches and a replacement occurs, or false otherwise. -// -// GlobalReplace() is like Replace(), except that it replaces all -// occurrences of the pattern in the string with the rewrite. -// Replacements are not subject to re-matching. E.g., -// -// string s = "yabba dabba doo"; -// pcrecpp::RE("b+").GlobalReplace("d", &s); -// -// will leave "s" containing "yada dada doo". It returns the number -// of replacements made. -// -// Extract() is like Replace(), except that if the pattern matches, -// "rewrite" is copied into "out" (an additional argument) with -// substitutions. The non-matching portions of "text" are ignored. -// Returns true iff a match occurred and the extraction happened -// successfully. If no match occurs, the string is left unaffected. - - -#include -#include -#include // defines the Arg class -// This isn't technically needed here, but we include it -// anyway so folks who include pcrecpp.h don't have to. -#include - -namespace pcrecpp { - -#define PCRE_SET_OR_CLEAR(b, o) \ - if (b) all_options_ |= (o); else all_options_ &= ~(o); \ - return *this - -#define PCRE_IS_SET(o) \ - (all_options_ & o) == o - -/***** Compiling regular expressions: the RE class *****/ - -// RE_Options allow you to set options to be passed along to pcre, -// along with other options we put on top of pcre. -// Only 9 modifiers, plus match_limit and match_limit_recursion, -// are supported now. -class PCRECPP_EXP_DEFN RE_Options { - public: - // constructor - RE_Options() : match_limit_(0), match_limit_recursion_(0), all_options_(0) {} - - // alternative constructor. - // To facilitate transfer of legacy code from C programs - // - // This lets you do - // RE(pattern, RE_Options(PCRE_CASELESS|PCRE_MULTILINE)).PartialMatch(str); - // But new code is better off doing - // RE(pattern, - // RE_Options().set_caseless(true).set_multiline(true)).PartialMatch(str); - RE_Options(int option_flags) : match_limit_(0), match_limit_recursion_(0), - all_options_(option_flags) {} - // we're fine with the default destructor, copy constructor, etc. - - // accessors and mutators - int match_limit() const { return match_limit_; }; - RE_Options &set_match_limit(int limit) { - match_limit_ = limit; - return *this; - } - - int match_limit_recursion() const { return match_limit_recursion_; }; - RE_Options &set_match_limit_recursion(int limit) { - match_limit_recursion_ = limit; - return *this; - } - - bool caseless() const { - return PCRE_IS_SET(PCRE_CASELESS); - } - RE_Options &set_caseless(bool x) { - PCRE_SET_OR_CLEAR(x, PCRE_CASELESS); - } - - bool multiline() const { - return PCRE_IS_SET(PCRE_MULTILINE); - } - RE_Options &set_multiline(bool x) { - PCRE_SET_OR_CLEAR(x, PCRE_MULTILINE); - } - - bool dotall() const { - return PCRE_IS_SET(PCRE_DOTALL); - } - RE_Options &set_dotall(bool x) { - PCRE_SET_OR_CLEAR(x, PCRE_DOTALL); - } - - bool extended() const { - return PCRE_IS_SET(PCRE_EXTENDED); - } - RE_Options &set_extended(bool x) { - PCRE_SET_OR_CLEAR(x, PCRE_EXTENDED); - } - - bool dollar_endonly() const { - return PCRE_IS_SET(PCRE_DOLLAR_ENDONLY); - } - RE_Options &set_dollar_endonly(bool x) { - PCRE_SET_OR_CLEAR(x, PCRE_DOLLAR_ENDONLY); - } - - bool extra() const { - return PCRE_IS_SET(PCRE_EXTRA); - } - RE_Options &set_extra(bool x) { - PCRE_SET_OR_CLEAR(x, PCRE_EXTRA); - } - - bool ungreedy() const { - return PCRE_IS_SET(PCRE_UNGREEDY); - } - RE_Options &set_ungreedy(bool x) { - PCRE_SET_OR_CLEAR(x, PCRE_UNGREEDY); - } - - bool utf8() const { - return PCRE_IS_SET(PCRE_UTF8); - } - RE_Options &set_utf8(bool x) { - PCRE_SET_OR_CLEAR(x, PCRE_UTF8); - } - - bool no_auto_capture() const { - return PCRE_IS_SET(PCRE_NO_AUTO_CAPTURE); - } - RE_Options &set_no_auto_capture(bool x) { - PCRE_SET_OR_CLEAR(x, PCRE_NO_AUTO_CAPTURE); - } - - RE_Options &set_all_options(int opt) { - all_options_ = opt; - return *this; - } - int all_options() const { - return all_options_ ; - } - - // TODO: add other pcre flags - - private: - int match_limit_; - int match_limit_recursion_; - int all_options_; -}; - -// These functions return some common RE_Options -static inline RE_Options UTF8() { - return RE_Options().set_utf8(true); -} - -static inline RE_Options CASELESS() { - return RE_Options().set_caseless(true); -} -static inline RE_Options MULTILINE() { - return RE_Options().set_multiline(true); -} - -static inline RE_Options DOTALL() { - return RE_Options().set_dotall(true); -} - -static inline RE_Options EXTENDED() { - return RE_Options().set_extended(true); -} - -// Interface for regular expression matching. Also corresponds to a -// pre-compiled regular expression. An "RE" object is safe for -// concurrent use by multiple threads. -class PCRECPP_EXP_DEFN RE { - public: - // We provide implicit conversions from strings so that users can - // pass in a string or a "const char*" wherever an "RE" is expected. - RE(const string& pat) { Init(pat, NULL); } - RE(const string& pat, const RE_Options& option) { Init(pat, &option); } - RE(const char* pat) { Init(pat, NULL); } - RE(const char* pat, const RE_Options& option) { Init(pat, &option); } - RE(const unsigned char* pat) { - Init(reinterpret_cast(pat), NULL); - } - RE(const unsigned char* pat, const RE_Options& option) { - Init(reinterpret_cast(pat), &option); - } - - // Copy constructor & assignment - note that these are expensive - // because they recompile the expression. - RE(const RE& re) { Init(re.pattern_, &re.options_); } - const RE& operator=(const RE& re) { - if (this != &re) { - Cleanup(); - - // This is the code that originally came from Google - // Init(re.pattern_.c_str(), &re.options_); - - // This is the replacement from Ari Pollak - Init(re.pattern_, &re.options_); - } - return *this; - } - - - ~RE(); - - // The string specification for this RE. E.g. - // RE re("ab*c?d+"); - // re.pattern(); // "ab*c?d+" - const string& pattern() const { return pattern_; } - - // If RE could not be created properly, returns an error string. - // Else returns the empty string. - const string& error() const { return *error_; } - - /***** The useful part: the matching interface *****/ - - // This is provided so one can do pattern.ReplaceAll() just as - // easily as ReplaceAll(pattern-text, ....) - - bool FullMatch(const StringPiece& text, - const Arg& ptr1 = no_arg, - const Arg& ptr2 = no_arg, - const Arg& ptr3 = no_arg, - const Arg& ptr4 = no_arg, - const Arg& ptr5 = no_arg, - const Arg& ptr6 = no_arg, - const Arg& ptr7 = no_arg, - const Arg& ptr8 = no_arg, - const Arg& ptr9 = no_arg, - const Arg& ptr10 = no_arg, - const Arg& ptr11 = no_arg, - const Arg& ptr12 = no_arg, - const Arg& ptr13 = no_arg, - const Arg& ptr14 = no_arg, - const Arg& ptr15 = no_arg, - const Arg& ptr16 = no_arg) const; - - bool PartialMatch(const StringPiece& text, - const Arg& ptr1 = no_arg, - const Arg& ptr2 = no_arg, - const Arg& ptr3 = no_arg, - const Arg& ptr4 = no_arg, - const Arg& ptr5 = no_arg, - const Arg& ptr6 = no_arg, - const Arg& ptr7 = no_arg, - const Arg& ptr8 = no_arg, - const Arg& ptr9 = no_arg, - const Arg& ptr10 = no_arg, - const Arg& ptr11 = no_arg, - const Arg& ptr12 = no_arg, - const Arg& ptr13 = no_arg, - const Arg& ptr14 = no_arg, - const Arg& ptr15 = no_arg, - const Arg& ptr16 = no_arg) const; - - bool Consume(StringPiece* input, - const Arg& ptr1 = no_arg, - const Arg& ptr2 = no_arg, - const Arg& ptr3 = no_arg, - const Arg& ptr4 = no_arg, - const Arg& ptr5 = no_arg, - const Arg& ptr6 = no_arg, - const Arg& ptr7 = no_arg, - const Arg& ptr8 = no_arg, - const Arg& ptr9 = no_arg, - const Arg& ptr10 = no_arg, - const Arg& ptr11 = no_arg, - const Arg& ptr12 = no_arg, - const Arg& ptr13 = no_arg, - const Arg& ptr14 = no_arg, - const Arg& ptr15 = no_arg, - const Arg& ptr16 = no_arg) const; - - bool FindAndConsume(StringPiece* input, - const Arg& ptr1 = no_arg, - const Arg& ptr2 = no_arg, - const Arg& ptr3 = no_arg, - const Arg& ptr4 = no_arg, - const Arg& ptr5 = no_arg, - const Arg& ptr6 = no_arg, - const Arg& ptr7 = no_arg, - const Arg& ptr8 = no_arg, - const Arg& ptr9 = no_arg, - const Arg& ptr10 = no_arg, - const Arg& ptr11 = no_arg, - const Arg& ptr12 = no_arg, - const Arg& ptr13 = no_arg, - const Arg& ptr14 = no_arg, - const Arg& ptr15 = no_arg, - const Arg& ptr16 = no_arg) const; - - bool Replace(const StringPiece& rewrite, - string *str) const; - - int GlobalReplace(const StringPiece& rewrite, - string *str) const; - - bool Extract(const StringPiece &rewrite, - const StringPiece &text, - string *out) const; - - // Escapes all potentially meaningful regexp characters in - // 'unquoted'. The returned string, used as a regular expression, - // will exactly match the original string. For example, - // 1.5-2.0? - // may become: - // 1\.5\-2\.0\? - // Note QuoteMeta behaves the same as perl's QuoteMeta function, - // *except* that it escapes the NUL character (\0) as backslash + 0, - // rather than backslash + NUL. - static string QuoteMeta(const StringPiece& unquoted); - - - /***** Generic matching interface *****/ - - // Type of match (TODO: Should be restructured as part of RE_Options) - enum Anchor { - UNANCHORED, // No anchoring - ANCHOR_START, // Anchor at start only - ANCHOR_BOTH // Anchor at start and end - }; - - // General matching routine. Stores the length of the match in - // "*consumed" if successful. - bool DoMatch(const StringPiece& text, - Anchor anchor, - int* consumed, - const Arg* const* args, int n) const; - - // Return the number of capturing subpatterns, or -1 if the - // regexp wasn't valid on construction. - int NumberOfCapturingGroups() const; - - // The default value for an argument, to indicate the end of the argument - // list. This must be used only in optional argument defaults. It should NOT - // be passed explicitly. Some people have tried to use it like this: - // - // FullMatch(x, y, &z, no_arg, &w); - // - // This is a mistake, and will not work. - static Arg no_arg; - - private: - - void Init(const string& pattern, const RE_Options* options); - void Cleanup(); - - // Match against "text", filling in "vec" (up to "vecsize" * 2/3) with - // pairs of integers for the beginning and end positions of matched - // text. The first pair corresponds to the entire matched text; - // subsequent pairs correspond, in order, to parentheses-captured - // matches. Returns the number of pairs (one more than the number of - // the last subpattern with a match) if matching was successful - // and zero if the match failed. - // I.e. for RE("(foo)|(bar)|(baz)") it will return 2, 3, and 4 when matching - // against "foo", "bar", and "baz" respectively. - // When matching RE("(foo)|hello") against "hello", it will return 1. - // But the values for all subpattern are filled in into "vec". - int TryMatch(const StringPiece& text, - int startpos, - Anchor anchor, - bool empty_ok, - int *vec, - int vecsize) const; - - // Append the "rewrite" string, with backslash subsitutions from "text" - // and "vec", to string "out". - bool Rewrite(string *out, - const StringPiece& rewrite, - const StringPiece& text, - int *vec, - int veclen) const; - - // internal implementation for DoMatch - bool DoMatchImpl(const StringPiece& text, - Anchor anchor, - int* consumed, - const Arg* const args[], - int n, - int* vec, - int vecsize) const; - - // Compile the regexp for the specified anchoring mode - pcre* Compile(Anchor anchor); - - string pattern_; - RE_Options options_; - pcre* re_full_; // For full matches - pcre* re_partial_; // For partial matches - const string* error_; // Error indicator (or points to empty string) -}; - -} // namespace pcrecpp - -#endif /* _PCRECPP_H */ diff --git a/src/third_party/pcre-8.39/pcrecpp_internal.h b/src/third_party/pcre-8.39/pcrecpp_internal.h deleted file mode 100644 index 827f9e04e2a..00000000000 --- a/src/third_party/pcre-8.39/pcrecpp_internal.h +++ /dev/null @@ -1,71 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* -Copyright (c) 2005, Google Inc. -All rights reserved. - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -#ifndef PCRECPP_INTERNAL_H -#define PCRECPP_INTERNAL_H - -/* When compiling a DLL for Windows, the exported symbols have to be declared -using some MS magic. I found some useful information on this web page: -http://msdn2.microsoft.com/en-us/library/y4h7bcy6(VS.80).aspx. According to the -information there, using __declspec(dllexport) without "extern" we have a -definition; with "extern" we have a declaration. The settings here override the -setting in pcre.h. We use: - - PCRECPP_EXP_DECL for declarations - PCRECPP_EXP_DEFN for definitions of exported functions - -*/ - -#ifndef PCRECPP_EXP_DECL -# ifdef _WIN32 -# ifndef PCRE_STATIC -# define PCRECPP_EXP_DECL extern __declspec(dllexport) -# define PCRECPP_EXP_DEFN __declspec(dllexport) -# else -# define PCRECPP_EXP_DECL extern -# define PCRECPP_EXP_DEFN -# endif -# else -# define PCRECPP_EXP_DECL extern -# define PCRECPP_EXP_DEFN -# endif -#endif - -#endif /* PCRECPP_INTERNAL_H */ - -/* End of pcrecpp_internal.h */ diff --git a/src/third_party/pcre-8.39/pcrecpp_unittest.cc b/src/third_party/pcre-8.39/pcrecpp_unittest.cc deleted file mode 100644 index 92cae8fbea5..00000000000 --- a/src/third_party/pcre-8.39/pcrecpp_unittest.cc +++ /dev/null @@ -1,1291 +0,0 @@ -// -*- coding: utf-8 -*- -// -// Copyright (c) 2005 - 2010, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Author: Sanjay Ghemawat -// -// TODO: Test extractions for PartialMatch/Consume - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include /* for memset and strcmp */ -#include -#include -#include "pcrecpp.h" - -using pcrecpp::StringPiece; -using pcrecpp::RE; -using pcrecpp::RE_Options; -using pcrecpp::Hex; -using pcrecpp::Octal; -using pcrecpp::CRadix; - -static bool VERBOSE_TEST = false; - -// CHECK dies with a fatal error if condition is not true. It is *not* -// controlled by NDEBUG, so the check will be executed regardless of -// compilation mode. Therefore, it is safe to do things like: -// CHECK_EQ(fp->Write(x), 4) -#define CHECK(condition) do { \ - if (!(condition)) { \ - fprintf(stderr, "%s:%d: Check failed: %s\n", \ - __FILE__, __LINE__, #condition); \ - exit(1); \ - } \ -} while (0) - -#define CHECK_EQ(a, b) CHECK(a == b) - -static void Timing1(int num_iters) { - // Same pattern lots of times - RE pattern("ruby:\\d+"); - StringPiece p("ruby:1234"); - for (int j = num_iters; j > 0; j--) { - CHECK(pattern.FullMatch(p)); - } -} - -static void Timing2(int num_iters) { - // Same pattern lots of times - RE pattern("ruby:(\\d+)"); - int i; - for (int j = num_iters; j > 0; j--) { - CHECK(pattern.FullMatch("ruby:1234", &i)); - CHECK_EQ(i, 1234); - } -} - -static void Timing3(int num_iters) { - string text_string; - for (int j = num_iters; j > 0; j--) { - text_string += "this is another line\n"; - } - - RE line_matcher(".*\n"); - string line; - StringPiece text(text_string); - int counter = 0; - while (line_matcher.Consume(&text)) { - counter++; - } - printf("Matched %d lines\n", counter); -} - -#if 0 // uncomment this if you have a way of defining VirtualProcessSize() - -static void LeakTest() { - // Check for memory leaks - unsigned long long initial_size = 0; - for (int i = 0; i < 100000; i++) { - if (i == 50000) { - initial_size = VirtualProcessSize(); - printf("Size after 50000: %llu\n", initial_size); - } - char buf[100]; // definitely big enough - sprintf(buf, "pat%09d", i); - RE newre(buf); - } - uint64 final_size = VirtualProcessSize(); - printf("Size after 100000: %llu\n", final_size); - const double growth = double(final_size - initial_size) / final_size; - printf("Growth: %0.2f%%", growth * 100); - CHECK(growth < 0.02); // Allow < 2% growth -} - -#endif - -static void RadixTests() { - printf("Testing hex\n"); - -#define CHECK_HEX(type, value) \ - do { \ - type v; \ - CHECK(RE("([0-9a-fA-F]+)[uUlL]*").FullMatch(#value, Hex(&v))); \ - CHECK_EQ(v, 0x ## value); \ - CHECK(RE("([0-9a-fA-FxX]+)[uUlL]*").FullMatch("0x" #value, CRadix(&v))); \ - CHECK_EQ(v, 0x ## value); \ - } while(0) - - CHECK_HEX(short, 2bad); - CHECK_HEX(unsigned short, 2badU); - CHECK_HEX(int, dead); - CHECK_HEX(unsigned int, deadU); - CHECK_HEX(long, 7eadbeefL); - CHECK_HEX(unsigned long, deadbeefUL); -#ifdef HAVE_LONG_LONG - CHECK_HEX(long long, 12345678deadbeefLL); -#endif -#ifdef HAVE_UNSIGNED_LONG_LONG - CHECK_HEX(unsigned long long, cafebabedeadbeefULL); -#endif - -#undef CHECK_HEX - - printf("Testing octal\n"); - -#define CHECK_OCTAL(type, value) \ - do { \ - type v; \ - CHECK(RE("([0-7]+)[uUlL]*").FullMatch(#value, Octal(&v))); \ - CHECK_EQ(v, 0 ## value); \ - CHECK(RE("([0-9a-fA-FxX]+)[uUlL]*").FullMatch("0" #value, CRadix(&v))); \ - CHECK_EQ(v, 0 ## value); \ - } while(0) - - CHECK_OCTAL(short, 77777); - CHECK_OCTAL(unsigned short, 177777U); - CHECK_OCTAL(int, 17777777777); - CHECK_OCTAL(unsigned int, 37777777777U); - CHECK_OCTAL(long, 17777777777L); - CHECK_OCTAL(unsigned long, 37777777777UL); -#ifdef HAVE_LONG_LONG - CHECK_OCTAL(long long, 777777777777777777777LL); -#endif -#ifdef HAVE_UNSIGNED_LONG_LONG - CHECK_OCTAL(unsigned long long, 1777777777777777777777ULL); -#endif - -#undef CHECK_OCTAL - - printf("Testing decimal\n"); - -#define CHECK_DECIMAL(type, value) \ - do { \ - type v; \ - CHECK(RE("(-?[0-9]+)[uUlL]*").FullMatch(#value, &v)); \ - CHECK_EQ(v, value); \ - CHECK(RE("(-?[0-9a-fA-FxX]+)[uUlL]*").FullMatch(#value, CRadix(&v))); \ - CHECK_EQ(v, value); \ - } while(0) - - CHECK_DECIMAL(short, -1); - CHECK_DECIMAL(unsigned short, 9999); - CHECK_DECIMAL(int, -1000); - CHECK_DECIMAL(unsigned int, 12345U); - CHECK_DECIMAL(long, -10000000L); - CHECK_DECIMAL(unsigned long, 3083324652U); -#ifdef HAVE_LONG_LONG - CHECK_DECIMAL(long long, -100000000000000LL); -#endif -#ifdef HAVE_UNSIGNED_LONG_LONG - CHECK_DECIMAL(unsigned long long, 1234567890987654321ULL); -#endif - -#undef CHECK_DECIMAL - -} - -static void TestReplace() { - printf("Testing Replace\n"); - - struct ReplaceTest { - const char *regexp; - const char *rewrite; - const char *original; - const char *single; - const char *global; - int global_count; // the expected return value from ReplaceAll - }; - static const ReplaceTest tests[] = { - { "(qu|[b-df-hj-np-tv-z]*)([a-z]+)", - "\\2\\1ay", - "the quick brown fox jumps over the lazy dogs.", - "ethay quick brown fox jumps over the lazy dogs.", - "ethay ickquay ownbray oxfay umpsjay overay ethay azylay ogsday.", - 9 }, - { "\\w+", - "\\0-NOSPAM", - "paul.haahr@google.com", - "paul-NOSPAM.haahr@google.com", - "paul-NOSPAM.haahr-NOSPAM@google-NOSPAM.com-NOSPAM", - 4 }, - { "^", - "(START)", - "foo", - "(START)foo", - "(START)foo", - 1 }, - { "^", - "(START)", - "", - "(START)", - "(START)", - 1 }, - { "$", - "(END)", - "", - "(END)", - "(END)", - 1 }, - { "b", - "bb", - "ababababab", - "abbabababab", - "abbabbabbabbabb", - 5 }, - { "b", - "bb", - "bbbbbb", - "bbbbbbb", - "bbbbbbbbbbbb", - 6 }, - { "b+", - "bb", - "bbbbbb", - "bb", - "bb", - 1 }, - { "b*", - "bb", - "bbbbbb", - "bb", - "bbbb", - 2 }, - { "b*", - "bb", - "aaaaa", - "bbaaaaa", - "bbabbabbabbabbabb", - 6 }, - { "b*", - "bb", - "aa\naa\n", - "bbaa\naa\n", - "bbabbabb\nbbabbabb\nbb", - 7 }, - { "b*", - "bb", - "aa\raa\r", - "bbaa\raa\r", - "bbabbabb\rbbabbabb\rbb", - 7 }, - { "b*", - "bb", - "aa\r\naa\r\n", - "bbaa\r\naa\r\n", - "bbabbabb\r\nbbabbabb\r\nbb", - 7 }, - // Check empty-string matching (it's tricky!) - { "aa|b*", - "@", - "aa", - "@", - "@@", - 2 }, - { "b*|aa", - "@", - "aa", - "@aa", - "@@@", - 3 }, -#ifdef SUPPORT_UTF8 - { "b*", - "bb", - "\xE3\x83\x9B\xE3\x83\xBC\xE3\x83\xA0\xE3\x81\xB8", // utf8 - "bb\xE3\x83\x9B\xE3\x83\xBC\xE3\x83\xA0\xE3\x81\xB8", - "bb\xE3\x83\x9B""bb""\xE3\x83\xBC""bb""\xE3\x83\xA0""bb""\xE3\x81\xB8""bb", - 5 }, - { "b*", - "bb", - "\xE3\x83\x9B\r\n\xE3\x83\xBC\r\xE3\x83\xA0\n\xE3\x81\xB8\r\n", // utf8 - "bb\xE3\x83\x9B\r\n\xE3\x83\xBC\r\xE3\x83\xA0\n\xE3\x81\xB8\r\n", - ("bb\xE3\x83\x9B""bb\r\nbb""\xE3\x83\xBC""bb\rbb""\xE3\x83\xA0" - "bb\nbb""\xE3\x81\xB8""bb\r\nbb"), - 9 }, -#endif - { "", NULL, NULL, NULL, NULL, 0 } - }; - -#ifdef SUPPORT_UTF8 - const bool support_utf8 = true; -#else - const bool support_utf8 = false; -#endif - - for (const ReplaceTest *t = tests; t->original != NULL; ++t) { - RE re(t->regexp, RE_Options(PCRE_NEWLINE_CRLF).set_utf8(support_utf8)); - assert(re.error().empty()); - string one(t->original); - CHECK(re.Replace(t->rewrite, &one)); - CHECK_EQ(one, t->single); - string all(t->original); - const int replace_count = re.GlobalReplace(t->rewrite, &all); - CHECK_EQ(all, t->global); - CHECK_EQ(replace_count, t->global_count); - } - - // One final test: test \r\n replacement when we're not in CRLF mode - { - RE re("b*", RE_Options(PCRE_NEWLINE_CR).set_utf8(support_utf8)); - assert(re.error().empty()); - string all("aa\r\naa\r\n"); - CHECK_EQ(re.GlobalReplace("bb", &all), 9); - CHECK_EQ(all, string("bbabbabb\rbb\nbbabbabb\rbb\nbb")); - } - { - RE re("b*", RE_Options(PCRE_NEWLINE_LF).set_utf8(support_utf8)); - assert(re.error().empty()); - string all("aa\r\naa\r\n"); - CHECK_EQ(re.GlobalReplace("bb", &all), 9); - CHECK_EQ(all, string("bbabbabb\rbb\nbbabbabb\rbb\nbb")); - } - // TODO: test what happens when no PCRE_NEWLINE_* flag is set. - // Alas, the answer depends on how pcre was compiled. -} - -static void TestExtract() { - printf("Testing Extract\n"); - - string s; - - CHECK(RE("(.*)@([^.]*)").Extract("\\2!\\1", "boris@kremvax.ru", &s)); - CHECK_EQ(s, "kremvax!boris"); - - // check the RE interface as well - CHECK(RE(".*").Extract("'\\0'", "foo", &s)); - CHECK_EQ(s, "'foo'"); - CHECK(!RE("bar").Extract("'\\0'", "baz", &s)); - CHECK_EQ(s, "'foo'"); -} - -static void TestConsume() { - printf("Testing Consume\n"); - - string word; - - string s(" aaa b!@#$@#$cccc"); - StringPiece input(s); - - RE r("\\s*(\\w+)"); // matches a word, possibly proceeded by whitespace - CHECK(r.Consume(&input, &word)); - CHECK_EQ(word, "aaa"); - CHECK(r.Consume(&input, &word)); - CHECK_EQ(word, "b"); - CHECK(! r.Consume(&input, &word)); -} - -static void TestFindAndConsume() { - printf("Testing FindAndConsume\n"); - - string word; - - string s(" aaa b!@#$@#$cccc"); - StringPiece input(s); - - RE r("(\\w+)"); // matches a word - CHECK(r.FindAndConsume(&input, &word)); - CHECK_EQ(word, "aaa"); - CHECK(r.FindAndConsume(&input, &word)); - CHECK_EQ(word, "b"); - CHECK(r.FindAndConsume(&input, &word)); - CHECK_EQ(word, "cccc"); - CHECK(! r.FindAndConsume(&input, &word)); -} - -static void TestMatchNumberPeculiarity() { - printf("Testing match-number peculiarity\n"); - - string word1; - string word2; - string word3; - - RE r("(foo)|(bar)|(baz)"); - CHECK(r.PartialMatch("foo", &word1, &word2, &word3)); - CHECK_EQ(word1, "foo"); - CHECK_EQ(word2, ""); - CHECK_EQ(word3, ""); - CHECK(r.PartialMatch("bar", &word1, &word2, &word3)); - CHECK_EQ(word1, ""); - CHECK_EQ(word2, "bar"); - CHECK_EQ(word3, ""); - CHECK(r.PartialMatch("baz", &word1, &word2, &word3)); - CHECK_EQ(word1, ""); - CHECK_EQ(word2, ""); - CHECK_EQ(word3, "baz"); - CHECK(!r.PartialMatch("f", &word1, &word2, &word3)); - - string a; - CHECK(RE("(foo)|hello").FullMatch("hello", &a)); - CHECK_EQ(a, ""); -} - -static void TestRecursion() { - printf("Testing recursion\n"); - - // Get one string that passes (sometimes), one that never does. - string text_good("abcdefghijk"); - string text_bad("acdefghijkl"); - - // According to pcretest, matching text_good against (\w+)*b - // requires match_limit of at least 8192, and match_recursion_limit - // of at least 37. - - RE_Options options_ml; - options_ml.set_match_limit(8192); - RE re("(\\w+)*b", options_ml); - CHECK(re.PartialMatch(text_good) == true); - CHECK(re.PartialMatch(text_bad) == false); - CHECK(re.FullMatch(text_good) == false); - CHECK(re.FullMatch(text_bad) == false); - - options_ml.set_match_limit(1024); - RE re2("(\\w+)*b", options_ml); - CHECK(re2.PartialMatch(text_good) == false); // because of match_limit - CHECK(re2.PartialMatch(text_bad) == false); - CHECK(re2.FullMatch(text_good) == false); - CHECK(re2.FullMatch(text_bad) == false); - - RE_Options options_mlr; - options_mlr.set_match_limit_recursion(50); - RE re3("(\\w+)*b", options_mlr); - CHECK(re3.PartialMatch(text_good) == true); - CHECK(re3.PartialMatch(text_bad) == false); - CHECK(re3.FullMatch(text_good) == false); - CHECK(re3.FullMatch(text_bad) == false); - - options_mlr.set_match_limit_recursion(10); - RE re4("(\\w+)*b", options_mlr); - CHECK(re4.PartialMatch(text_good) == false); - CHECK(re4.PartialMatch(text_bad) == false); - CHECK(re4.FullMatch(text_good) == false); - CHECK(re4.FullMatch(text_bad) == false); -} - -// A meta-quoted string, interpreted as a pattern, should always match -// the original unquoted string. -static void TestQuoteMeta(string unquoted, RE_Options options = RE_Options()) { - string quoted = RE::QuoteMeta(unquoted); - RE re(quoted, options); - CHECK(re.FullMatch(unquoted)); -} - -// A string containing meaningful regexp characters, which is then meta- -// quoted, should not generally match a string the unquoted string does. -static void NegativeTestQuoteMeta(string unquoted, string should_not_match, - RE_Options options = RE_Options()) { - string quoted = RE::QuoteMeta(unquoted); - RE re(quoted, options); - CHECK(!re.FullMatch(should_not_match)); -} - -// Tests that quoted meta characters match their original strings, -// and that a few things that shouldn't match indeed do not. -static void TestQuotaMetaSimple() { - TestQuoteMeta("foo"); - TestQuoteMeta("foo.bar"); - TestQuoteMeta("foo\\.bar"); - TestQuoteMeta("[1-9]"); - TestQuoteMeta("1.5-2.0?"); - TestQuoteMeta("\\d"); - TestQuoteMeta("Who doesn't like ice cream?"); - TestQuoteMeta("((a|b)c?d*e+[f-h]i)"); - TestQuoteMeta("((?!)xxx).*yyy"); - TestQuoteMeta("(["); - TestQuoteMeta(string("foo\0bar", 7)); -} - -static void TestQuoteMetaSimpleNegative() { - NegativeTestQuoteMeta("foo", "bar"); - NegativeTestQuoteMeta("...", "bar"); - NegativeTestQuoteMeta("\\.", "."); - NegativeTestQuoteMeta("\\.", ".."); - NegativeTestQuoteMeta("(a)", "a"); - NegativeTestQuoteMeta("(a|b)", "a"); - NegativeTestQuoteMeta("(a|b)", "(a)"); - NegativeTestQuoteMeta("(a|b)", "a|b"); - NegativeTestQuoteMeta("[0-9]", "0"); - NegativeTestQuoteMeta("[0-9]", "0-9"); - NegativeTestQuoteMeta("[0-9]", "[9]"); - NegativeTestQuoteMeta("((?!)xxx)", "xxx"); -} - -static void TestQuoteMetaLatin1() { - TestQuoteMeta("3\xb2 = 9"); -} - -static void TestQuoteMetaUtf8() { -#ifdef SUPPORT_UTF8 - TestQuoteMeta("Pl\xc3\xa1\x63ido Domingo", pcrecpp::UTF8()); - TestQuoteMeta("xyz", pcrecpp::UTF8()); // No fancy utf8 - TestQuoteMeta("\xc2\xb0", pcrecpp::UTF8()); // 2-byte utf8 (degree symbol) - TestQuoteMeta("27\xc2\xb0 degrees", pcrecpp::UTF8()); // As a middle character - TestQuoteMeta("\xe2\x80\xb3", pcrecpp::UTF8()); // 3-byte utf8 (double prime) - TestQuoteMeta("\xf0\x9d\x85\x9f", pcrecpp::UTF8()); // 4-byte utf8 (music note) - TestQuoteMeta("27\xc2\xb0"); // Interpreted as Latin-1, but should still work - NegativeTestQuoteMeta("27\xc2\xb0", // 2-byte utf (degree symbol) - "27\\\xc2\\\xb0", - pcrecpp::UTF8()); -#endif -} - -static void TestQuoteMetaAll() { - printf("Testing QuoteMeta\n"); - TestQuotaMetaSimple(); - TestQuoteMetaSimpleNegative(); - TestQuoteMetaLatin1(); - TestQuoteMetaUtf8(); -} - -// -// Options tests contributed by -// Giuseppe Maxia, CTO, Stardata s.r.l. -// July 2005 -// -static void GetOneOptionResult( - const char *option_name, - const char *regex, - const char *str, - RE_Options options, - bool full, - string expected) { - - printf("Testing Option <%s>\n", option_name); - if(VERBOSE_TEST) - printf("/%s/ finds \"%s\" within \"%s\" \n", - regex, - expected.c_str(), - str); - string captured(""); - if (full) - RE(regex,options).FullMatch(str, &captured); - else - RE(regex,options).PartialMatch(str, &captured); - CHECK_EQ(captured, expected); -} - -static void TestOneOption( - const char *option_name, - const char *regex, - const char *str, - RE_Options options, - bool full, - bool assertive = true) { - - printf("Testing Option <%s>\n", option_name); - if (VERBOSE_TEST) - printf("'%s' %s /%s/ \n", - str, - (assertive? "matches" : "doesn't match"), - regex); - if (assertive) { - if (full) - CHECK(RE(regex,options).FullMatch(str)); - else - CHECK(RE(regex,options).PartialMatch(str)); - } else { - if (full) - CHECK(!RE(regex,options).FullMatch(str)); - else - CHECK(!RE(regex,options).PartialMatch(str)); - } -} - -static void Test_CASELESS() { - RE_Options options; - RE_Options options2; - - options.set_caseless(true); - TestOneOption("CASELESS (class)", "HELLO", "hello", options, false); - TestOneOption("CASELESS (class2)", "HELLO", "hello", options2.set_caseless(true), false); - TestOneOption("CASELESS (class)", "^[A-Z]+$", "Hello", options, false); - - TestOneOption("CASELESS (function)", "HELLO", "hello", pcrecpp::CASELESS(), false); - TestOneOption("CASELESS (function)", "^[A-Z]+$", "Hello", pcrecpp::CASELESS(), false); - options.set_caseless(false); - TestOneOption("no CASELESS", "HELLO", "hello", options, false, false); -} - -static void Test_MULTILINE() { - RE_Options options; - RE_Options options2; - const char *str = "HELLO\n" "cruel\n" "world\n"; - - options.set_multiline(true); - TestOneOption("MULTILINE (class)", "^cruel$", str, options, false); - TestOneOption("MULTILINE (class2)", "^cruel$", str, options2.set_multiline(true), false); - TestOneOption("MULTILINE (function)", "^cruel$", str, pcrecpp::MULTILINE(), false); - options.set_multiline(false); - TestOneOption("no MULTILINE", "^cruel$", str, options, false, false); -} - -static void Test_DOTALL() { - RE_Options options; - RE_Options options2; - const char *str = "HELLO\n" "cruel\n" "world"; - - options.set_dotall(true); - TestOneOption("DOTALL (class)", "HELLO.*world", str, options, true); - TestOneOption("DOTALL (class2)", "HELLO.*world", str, options2.set_dotall(true), true); - TestOneOption("DOTALL (function)", "HELLO.*world", str, pcrecpp::DOTALL(), true); - options.set_dotall(false); - TestOneOption("no DOTALL", "HELLO.*world", str, options, true, false); -} - -static void Test_DOLLAR_ENDONLY() { - RE_Options options; - RE_Options options2; - const char *str = "HELLO world\n"; - - TestOneOption("no DOLLAR_ENDONLY", "world$", str, options, false); - options.set_dollar_endonly(true); - TestOneOption("DOLLAR_ENDONLY 1", "world$", str, options, false, false); - TestOneOption("DOLLAR_ENDONLY 2", "world$", str, options2.set_dollar_endonly(true), false, false); -} - -static void Test_EXTRA() { - RE_Options options; - const char *str = "HELLO"; - - options.set_extra(true); - TestOneOption("EXTRA 1", "\\HELL\\O", str, options, true, false ); - TestOneOption("EXTRA 2", "\\HELL\\O", str, RE_Options().set_extra(true), true, false ); - options.set_extra(false); - TestOneOption("no EXTRA", "\\HELL\\O", str, options, true ); -} - -static void Test_EXTENDED() { - RE_Options options; - RE_Options options2; - const char *str = "HELLO world"; - - options.set_extended(true); - TestOneOption("EXTENDED (class)", "HELLO world", str, options, false, false); - TestOneOption("EXTENDED (class2)", "HELLO world", str, options2.set_extended(true), false, false); - TestOneOption("EXTENDED (class)", - "^ HE L{2} O " - "\\s+ " - "\\w+ $ ", - str, - options, - false); - - TestOneOption("EXTENDED (function)", "HELLO world", str, pcrecpp::EXTENDED(), false, false); - TestOneOption("EXTENDED (function)", - "^ HE L{2} O " - "\\s+ " - "\\w+ $ ", - str, - pcrecpp::EXTENDED(), - false); - - options.set_extended(false); - TestOneOption("no EXTENDED", "HELLO world", str, options, false); -} - -static void Test_NO_AUTO_CAPTURE() { - RE_Options options; - const char *str = "HELLO world"; - string captured; - - printf("Testing Option \n"); - if (VERBOSE_TEST) - printf("parentheses capture text\n"); - RE re("(world|universe)$", options); - CHECK(re.Extract("\\1", str , &captured)); - CHECK_EQ(captured, "world"); - options.set_no_auto_capture(true); - printf("testing Option \n"); - if (VERBOSE_TEST) - printf("parentheses do not capture text\n"); - re.Extract("\\1",str, &captured ); - CHECK_EQ(captured, "world"); -} - -static void Test_UNGREEDY() { - RE_Options options; - const char *str = "HELLO, 'this' is the 'world'"; - - options.set_ungreedy(true); - GetOneOptionResult("UNGREEDY 1", "('.*')", str, options, false, "'this'" ); - GetOneOptionResult("UNGREEDY 2", "('.*')", str, RE_Options().set_ungreedy(true), false, "'this'" ); - GetOneOptionResult("UNGREEDY", "('.*?')", str, options, false, "'this' is the 'world'" ); - - options.set_ungreedy(false); - GetOneOptionResult("no UNGREEDY", "('.*')", str, options, false, "'this' is the 'world'" ); - GetOneOptionResult("no UNGREEDY", "('.*?')", str, options, false, "'this'" ); -} - -static void Test_all_options() { - const char *str = "HELLO\n" "cruel\n" "world"; - RE_Options options; - options.set_all_options(PCRE_CASELESS | PCRE_DOTALL); - - TestOneOption("all_options (CASELESS|DOTALL)", "^hello.*WORLD", str , options, false); - options.set_all_options(0); - TestOneOption("all_options (0)", "^hello.*WORLD", str , options, false, false); - options.set_all_options(PCRE_MULTILINE | PCRE_EXTENDED); - - TestOneOption("all_options (MULTILINE|EXTENDED)", " ^ c r u e l $ ", str, options, false); - TestOneOption("all_options (MULTILINE|EXTENDED) with constructor", - " ^ c r u e l $ ", - str, - RE_Options(PCRE_MULTILINE | PCRE_EXTENDED), - false); - - TestOneOption("all_options (MULTILINE|EXTENDED) with concatenation", - " ^ c r u e l $ ", - str, - RE_Options() - .set_multiline(true) - .set_extended(true), - false); - - options.set_all_options(0); - TestOneOption("all_options (0)", "^ c r u e l $", str, options, false, false); - -} - -static void TestOptions() { - printf("Testing Options\n"); - Test_CASELESS(); - Test_MULTILINE(); - Test_DOTALL(); - Test_DOLLAR_ENDONLY(); - Test_EXTENDED(); - Test_NO_AUTO_CAPTURE(); - Test_UNGREEDY(); - Test_EXTRA(); - Test_all_options(); -} - -static void TestConstructors() { - printf("Testing constructors\n"); - - RE_Options options; - options.set_dotall(true); - const char *str = "HELLO\n" "cruel\n" "world"; - - RE orig("HELLO.*world", options); - CHECK(orig.FullMatch(str)); - - RE copy1(orig); - CHECK(copy1.FullMatch(str)); - - RE copy2("not a match"); - CHECK(!copy2.FullMatch(str)); - copy2 = copy1; - CHECK(copy2.FullMatch(str)); - copy2 = orig; - CHECK(copy2.FullMatch(str)); - - // Make sure when we assign to ourselves, nothing bad happens - orig = orig; - copy1 = copy1; - copy2 = copy2; - CHECK(orig.FullMatch(str)); - CHECK(copy1.FullMatch(str)); - CHECK(copy2.FullMatch(str)); -} - -int main(int argc, char** argv) { - // Treat any flag as --help - if (argc > 1 && argv[1][0] == '-') { - printf("Usage: %s [timing1|timing2|timing3 num-iters]\n" - " If 'timingX ###' is specified, run the given timing test\n" - " with the given number of iterations, rather than running\n" - " the default corectness test.\n", argv[0]); - return 0; - } - - if (argc > 1) { - if ( argc == 2 || atoi(argv[2]) == 0) { - printf("timing mode needs a num-iters argument\n"); - return 1; - } - if (!strcmp(argv[1], "timing1")) - Timing1(atoi(argv[2])); - else if (!strcmp(argv[1], "timing2")) - Timing2(atoi(argv[2])); - else if (!strcmp(argv[1], "timing3")) - Timing3(atoi(argv[2])); - else - printf("Unknown argument '%s'\n", argv[1]); - return 0; - } - - printf("PCRE C++ wrapper tests\n"); - printf("Testing FullMatch\n"); - - int i; - string s; - - /***** FullMatch with no args *****/ - - CHECK(RE("h.*o").FullMatch("hello")); - CHECK(!RE("h.*o").FullMatch("othello")); // Must be anchored at front - CHECK(!RE("h.*o").FullMatch("hello!")); // Must be anchored at end - CHECK(RE("a*").FullMatch("aaaa")); // Fullmatch with normal op - CHECK(RE("a*?").FullMatch("aaaa")); // Fullmatch with nongreedy op - CHECK(RE("a*?\\z").FullMatch("aaaa")); // Two unusual ops - - /***** FullMatch with args *****/ - - // Zero-arg - CHECK(RE("\\d+").FullMatch("1001")); - - // Single-arg - CHECK(RE("(\\d+)").FullMatch("1001", &i)); - CHECK_EQ(i, 1001); - CHECK(RE("(-?\\d+)").FullMatch("-123", &i)); - CHECK_EQ(i, -123); - CHECK(!RE("()\\d+").FullMatch("10", &i)); - CHECK(!RE("(\\d+)").FullMatch("1234567890123456789012345678901234567890", - &i)); - - // Digits surrounding integer-arg - CHECK(RE("1(\\d*)4").FullMatch("1234", &i)); - CHECK_EQ(i, 23); - CHECK(RE("(\\d)\\d+").FullMatch("1234", &i)); - CHECK_EQ(i, 1); - CHECK(RE("(-\\d)\\d+").FullMatch("-1234", &i)); - CHECK_EQ(i, -1); - CHECK(RE("(\\d)").PartialMatch("1234", &i)); - CHECK_EQ(i, 1); - CHECK(RE("(-\\d)").PartialMatch("-1234", &i)); - CHECK_EQ(i, -1); - - // String-arg - CHECK(RE("h(.*)o").FullMatch("hello", &s)); - CHECK_EQ(s, string("ell")); - - // StringPiece-arg - StringPiece sp; - CHECK(RE("(\\w+):(\\d+)").FullMatch("ruby:1234", &sp, &i)); - CHECK_EQ(sp.size(), 4); - CHECK(memcmp(sp.data(), "ruby", 4) == 0); - CHECK_EQ(i, 1234); - - // Multi-arg - CHECK(RE("(\\w+):(\\d+)").FullMatch("ruby:1234", &s, &i)); - CHECK_EQ(s, string("ruby")); - CHECK_EQ(i, 1234); - - // Ignore non-void* NULL arg - CHECK(RE("he(.*)lo").FullMatch("hello", (char*)NULL)); - CHECK(RE("h(.*)o").FullMatch("hello", (string*)NULL)); - CHECK(RE("h(.*)o").FullMatch("hello", (StringPiece*)NULL)); - CHECK(RE("(.*)").FullMatch("1234", (int*)NULL)); -#ifdef HAVE_LONG_LONG - CHECK(RE("(.*)").FullMatch("1234567890123456", (long long*)NULL)); -#endif - CHECK(RE("(.*)").FullMatch("123.4567890123456", (double*)NULL)); - CHECK(RE("(.*)").FullMatch("123.4567890123456", (float*)NULL)); - - // Fail on non-void* NULL arg if the match doesn't parse for the given type. - CHECK(!RE("h(.*)lo").FullMatch("hello", &s, (char*)NULL)); - CHECK(!RE("(.*)").FullMatch("hello", (int*)NULL)); - CHECK(!RE("(.*)").FullMatch("1234567890123456", (int*)NULL)); - CHECK(!RE("(.*)").FullMatch("hello", (double*)NULL)); - CHECK(!RE("(.*)").FullMatch("hello", (float*)NULL)); - - // Ignored arg - CHECK(RE("(\\w+)(:)(\\d+)").FullMatch("ruby:1234", &s, (void*)NULL, &i)); - CHECK_EQ(s, string("ruby")); - CHECK_EQ(i, 1234); - - // Type tests - { - char c; - CHECK(RE("(H)ello").FullMatch("Hello", &c)); - CHECK_EQ(c, 'H'); - } - { - unsigned char c; - CHECK(RE("(H)ello").FullMatch("Hello", &c)); - CHECK_EQ(c, static_cast('H')); - } - { - short v; - CHECK(RE("(-?\\d+)").FullMatch("100", &v)); CHECK_EQ(v, 100); - CHECK(RE("(-?\\d+)").FullMatch("-100", &v)); CHECK_EQ(v, -100); - CHECK(RE("(-?\\d+)").FullMatch("32767", &v)); CHECK_EQ(v, 32767); - CHECK(RE("(-?\\d+)").FullMatch("-32768", &v)); CHECK_EQ(v, -32768); - CHECK(!RE("(-?\\d+)").FullMatch("-32769", &v)); - CHECK(!RE("(-?\\d+)").FullMatch("32768", &v)); - } - { - unsigned short v; - CHECK(RE("(\\d+)").FullMatch("100", &v)); CHECK_EQ(v, 100); - CHECK(RE("(\\d+)").FullMatch("32767", &v)); CHECK_EQ(v, 32767); - CHECK(RE("(\\d+)").FullMatch("65535", &v)); CHECK_EQ(v, 65535); - CHECK(!RE("(\\d+)").FullMatch("65536", &v)); - } - { - int v; - static const int max_value = 0x7fffffff; - static const int min_value = -max_value - 1; - CHECK(RE("(-?\\d+)").FullMatch("100", &v)); CHECK_EQ(v, 100); - CHECK(RE("(-?\\d+)").FullMatch("-100", &v)); CHECK_EQ(v, -100); - CHECK(RE("(-?\\d+)").FullMatch("2147483647", &v)); CHECK_EQ(v, max_value); - CHECK(RE("(-?\\d+)").FullMatch("-2147483648", &v)); CHECK_EQ(v, min_value); - CHECK(!RE("(-?\\d+)").FullMatch("-2147483649", &v)); - CHECK(!RE("(-?\\d+)").FullMatch("2147483648", &v)); - } - { - unsigned int v; - static const unsigned int max_value = 0xfffffffful; - CHECK(RE("(\\d+)").FullMatch("100", &v)); CHECK_EQ(v, 100); - CHECK(RE("(\\d+)").FullMatch("4294967295", &v)); CHECK_EQ(v, max_value); - CHECK(!RE("(\\d+)").FullMatch("4294967296", &v)); - } -#ifdef HAVE_LONG_LONG -# if defined(__MINGW__) || defined(__MINGW32__) -# define LLD "%I64d" -# define LLU "%I64u" -# else -# define LLD "%lld" -# define LLU "%llu" -# endif - { - long long v; - static const long long max_value = 0x7fffffffffffffffLL; - static const long long min_value = -max_value - 1; - char buf[32]; // definitely big enough for a long long - - CHECK(RE("(-?\\d+)").FullMatch("100", &v)); CHECK_EQ(v, 100); - CHECK(RE("(-?\\d+)").FullMatch("-100",&v)); CHECK_EQ(v, -100); - - sprintf(buf, LLD, max_value); - CHECK(RE("(-?\\d+)").FullMatch(buf,&v)); CHECK_EQ(v, max_value); - - sprintf(buf, LLD, min_value); - CHECK(RE("(-?\\d+)").FullMatch(buf,&v)); CHECK_EQ(v, min_value); - - sprintf(buf, LLD, max_value); - assert(buf[strlen(buf)-1] != '9'); - buf[strlen(buf)-1]++; - CHECK(!RE("(-?\\d+)").FullMatch(buf, &v)); - - sprintf(buf, LLD, min_value); - assert(buf[strlen(buf)-1] != '9'); - buf[strlen(buf)-1]++; - CHECK(!RE("(-?\\d+)").FullMatch(buf, &v)); - } -#endif -#if defined HAVE_UNSIGNED_LONG_LONG && defined HAVE_LONG_LONG - { - unsigned long long v; - long long v2; - static const unsigned long long max_value = 0xffffffffffffffffULL; - char buf[32]; // definitely big enough for a unsigned long long - - CHECK(RE("(-?\\d+)").FullMatch("100",&v)); CHECK_EQ(v, 100); - CHECK(RE("(-?\\d+)").FullMatch("-100",&v2)); CHECK_EQ(v2, -100); - - sprintf(buf, LLU, max_value); - CHECK(RE("(-?\\d+)").FullMatch(buf,&v)); CHECK_EQ(v, max_value); - - assert(buf[strlen(buf)-1] != '9'); - buf[strlen(buf)-1]++; - CHECK(!RE("(-?\\d+)").FullMatch(buf, &v)); - } -#endif - { - float v; - CHECK(RE("(.*)").FullMatch("100", &v)); - CHECK(RE("(.*)").FullMatch("-100.", &v)); - CHECK(RE("(.*)").FullMatch("1e23", &v)); - } - { - double v; - CHECK(RE("(.*)").FullMatch("100", &v)); - CHECK(RE("(.*)").FullMatch("-100.", &v)); - CHECK(RE("(.*)").FullMatch("1e23", &v)); - } - - // Check that matching is fully anchored - CHECK(!RE("(\\d+)").FullMatch("x1001", &i)); - CHECK(!RE("(\\d+)").FullMatch("1001x", &i)); - CHECK(RE("x(\\d+)").FullMatch("x1001", &i)); CHECK_EQ(i, 1001); - CHECK(RE("(\\d+)x").FullMatch("1001x", &i)); CHECK_EQ(i, 1001); - - // Braces - CHECK(RE("[0-9a-f+.-]{5,}").FullMatch("0abcd")); - CHECK(RE("[0-9a-f+.-]{5,}").FullMatch("0abcde")); - CHECK(!RE("[0-9a-f+.-]{5,}").FullMatch("0abc")); - - // Complicated RE - CHECK(RE("foo|bar|[A-Z]").FullMatch("foo")); - CHECK(RE("foo|bar|[A-Z]").FullMatch("bar")); - CHECK(RE("foo|bar|[A-Z]").FullMatch("X")); - CHECK(!RE("foo|bar|[A-Z]").FullMatch("XY")); - - // Check full-match handling (needs '$' tacked on internally) - CHECK(RE("fo|foo").FullMatch("fo")); - CHECK(RE("fo|foo").FullMatch("foo")); - CHECK(RE("fo|foo$").FullMatch("fo")); - CHECK(RE("fo|foo$").FullMatch("foo")); - CHECK(RE("foo$").FullMatch("foo")); - CHECK(!RE("foo\\$").FullMatch("foo$bar")); - CHECK(!RE("fo|bar").FullMatch("fox")); - - // Uncomment the following if we change the handling of '$' to - // prevent it from matching a trailing newline - if (false) { - // Check that we don't get bitten by pcre's special handling of a - // '\n' at the end of the string matching '$' - CHECK(!RE("foo$").PartialMatch("foo\n")); - } - - // Number of args - int a[16]; - CHECK(RE("").FullMatch("")); - - memset(a, 0, sizeof(0)); - CHECK(RE("(\\d){1}").FullMatch("1", - &a[0])); - CHECK_EQ(a[0], 1); - - memset(a, 0, sizeof(0)); - CHECK(RE("(\\d)(\\d)").FullMatch("12", - &a[0], &a[1])); - CHECK_EQ(a[0], 1); - CHECK_EQ(a[1], 2); - - memset(a, 0, sizeof(0)); - CHECK(RE("(\\d)(\\d)(\\d)").FullMatch("123", - &a[0], &a[1], &a[2])); - CHECK_EQ(a[0], 1); - CHECK_EQ(a[1], 2); - CHECK_EQ(a[2], 3); - - memset(a, 0, sizeof(0)); - CHECK(RE("(\\d)(\\d)(\\d)(\\d)").FullMatch("1234", - &a[0], &a[1], &a[2], &a[3])); - CHECK_EQ(a[0], 1); - CHECK_EQ(a[1], 2); - CHECK_EQ(a[2], 3); - CHECK_EQ(a[3], 4); - - memset(a, 0, sizeof(0)); - CHECK(RE("(\\d)(\\d)(\\d)(\\d)(\\d)").FullMatch("12345", - &a[0], &a[1], &a[2], - &a[3], &a[4])); - CHECK_EQ(a[0], 1); - CHECK_EQ(a[1], 2); - CHECK_EQ(a[2], 3); - CHECK_EQ(a[3], 4); - CHECK_EQ(a[4], 5); - - memset(a, 0, sizeof(0)); - CHECK(RE("(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)").FullMatch("123456", - &a[0], &a[1], &a[2], - &a[3], &a[4], &a[5])); - CHECK_EQ(a[0], 1); - CHECK_EQ(a[1], 2); - CHECK_EQ(a[2], 3); - CHECK_EQ(a[3], 4); - CHECK_EQ(a[4], 5); - CHECK_EQ(a[5], 6); - - memset(a, 0, sizeof(0)); - CHECK(RE("(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)").FullMatch("1234567", - &a[0], &a[1], &a[2], &a[3], - &a[4], &a[5], &a[6])); - CHECK_EQ(a[0], 1); - CHECK_EQ(a[1], 2); - CHECK_EQ(a[2], 3); - CHECK_EQ(a[3], 4); - CHECK_EQ(a[4], 5); - CHECK_EQ(a[5], 6); - CHECK_EQ(a[6], 7); - - memset(a, 0, sizeof(0)); - CHECK(RE("(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)" - "(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)").FullMatch( - "1234567890123456", - &a[0], &a[1], &a[2], &a[3], - &a[4], &a[5], &a[6], &a[7], - &a[8], &a[9], &a[10], &a[11], - &a[12], &a[13], &a[14], &a[15])); - CHECK_EQ(a[0], 1); - CHECK_EQ(a[1], 2); - CHECK_EQ(a[2], 3); - CHECK_EQ(a[3], 4); - CHECK_EQ(a[4], 5); - CHECK_EQ(a[5], 6); - CHECK_EQ(a[6], 7); - CHECK_EQ(a[7], 8); - CHECK_EQ(a[8], 9); - CHECK_EQ(a[9], 0); - CHECK_EQ(a[10], 1); - CHECK_EQ(a[11], 2); - CHECK_EQ(a[12], 3); - CHECK_EQ(a[13], 4); - CHECK_EQ(a[14], 5); - CHECK_EQ(a[15], 6); - - /***** PartialMatch *****/ - - printf("Testing PartialMatch\n"); - - CHECK(RE("h.*o").PartialMatch("hello")); - CHECK(RE("h.*o").PartialMatch("othello")); - CHECK(RE("h.*o").PartialMatch("hello!")); - CHECK(RE("((((((((((((((((((((x))))))))))))))))))))").PartialMatch("x")); - - /***** other tests *****/ - - RadixTests(); - TestReplace(); - TestExtract(); - TestConsume(); - TestFindAndConsume(); - TestQuoteMetaAll(); - TestMatchNumberPeculiarity(); - - // Check the pattern() accessor - { - const string kPattern = "http://([^/]+)/.*"; - const RE re(kPattern); - CHECK_EQ(kPattern, re.pattern()); - } - - // Check RE error field. - { - RE re("foo"); - CHECK(re.error().empty()); // Must have no error - } - -#ifdef SUPPORT_UTF8 - // Check UTF-8 handling - { - printf("Testing UTF-8 handling\n"); - - // Three Japanese characters (nihongo) - const unsigned char utf8_string[] = { - 0xe6, 0x97, 0xa5, // 65e5 - 0xe6, 0x9c, 0xac, // 627c - 0xe8, 0xaa, 0x9e, // 8a9e - 0 - }; - const unsigned char utf8_pattern[] = { - '.', - 0xe6, 0x9c, 0xac, // 627c - '.', - 0 - }; - - // Both should match in either mode, bytes or UTF-8 - RE re_test1("........."); - CHECK(re_test1.FullMatch(utf8_string)); - RE re_test2("...", pcrecpp::UTF8()); - CHECK(re_test2.FullMatch(utf8_string)); - - // Check that '.' matches one byte or UTF-8 character - // according to the mode. - string ss; - RE re_test3("(.)"); - CHECK(re_test3.PartialMatch(utf8_string, &ss)); - CHECK_EQ(ss, string("\xe6")); - RE re_test4("(.)", pcrecpp::UTF8()); - CHECK(re_test4.PartialMatch(utf8_string, &ss)); - CHECK_EQ(ss, string("\xe6\x97\xa5")); - - // Check that string matches itself in either mode - RE re_test5(utf8_string); - CHECK(re_test5.FullMatch(utf8_string)); - RE re_test6(utf8_string, pcrecpp::UTF8()); - CHECK(re_test6.FullMatch(utf8_string)); - - // Check that pattern matches string only in UTF8 mode - RE re_test7(utf8_pattern); - CHECK(!re_test7.FullMatch(utf8_string)); - RE re_test8(utf8_pattern, pcrecpp::UTF8()); - CHECK(re_test8.FullMatch(utf8_string)); - } - - // Check that ungreedy, UTF8 regular expressions don't match when they - // oughtn't -- see bug 82246. - { - // This code always worked. - const char* pattern = "\\w+X"; - const string target = "a aX"; - RE match_sentence(pattern); - RE match_sentence_re(pattern, pcrecpp::UTF8()); - - CHECK(!match_sentence.FullMatch(target)); - CHECK(!match_sentence_re.FullMatch(target)); - } - - { - const char* pattern = "(?U)\\w+X"; - const string target = "a aX"; - RE match_sentence(pattern); - RE match_sentence_re(pattern, pcrecpp::UTF8()); - - CHECK(!match_sentence.FullMatch(target)); - CHECK(!match_sentence_re.FullMatch(target)); - } -#endif /* def SUPPORT_UTF8 */ - - printf("Testing error reporting\n"); - - { RE re("a\\1"); CHECK(!re.error().empty()); } - { - RE re("a[x"); - CHECK(!re.error().empty()); - } - { - RE re("a[z-a]"); - CHECK(!re.error().empty()); - } - { - RE re("a[[:foobar:]]"); - CHECK(!re.error().empty()); - } - { - RE re("a(b"); - CHECK(!re.error().empty()); - } - { - RE re("a\\"); - CHECK(!re.error().empty()); - } - - // Test that recursion is stopped - TestRecursion(); - - // Test Options - if (getenv("VERBOSE_TEST") != NULL) - VERBOSE_TEST = true; - TestOptions(); - - // Test the constructors - TestConstructors(); - - // Done - printf("OK\n"); - - return 0; -} diff --git a/src/third_party/pcre-8.39/pcrecpparg.h b/src/third_party/pcre-8.39/pcrecpparg.h deleted file mode 100644 index b4f9c3f4989..00000000000 --- a/src/third_party/pcre-8.39/pcrecpparg.h +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Author: Sanjay Ghemawat - -#ifndef _PCRECPPARG_H -#define _PCRECPPARG_H - -#include // for NULL -#include - -#include - -namespace pcrecpp { - -class StringPiece; - -// Hex/Octal/Binary? - -// Special class for parsing into objects that define a ParseFrom() method -template -class _RE_MatchObject { - public: - static inline bool Parse(const char* str, int n, void* dest) { - if (dest == NULL) return true; - T* object = reinterpret_cast(dest); - return object->ParseFrom(str, n); - } -}; - -class PCRECPP_EXP_DEFN Arg { - public: - // Empty constructor so we can declare arrays of Arg - Arg(); - - // Constructor specially designed for NULL arguments - Arg(void*); - - typedef bool (*Parser)(const char* str, int n, void* dest); - -// Type-specific parsers -#define PCRE_MAKE_PARSER(type,name) \ - Arg(type* p) : arg_(p), parser_(name) { } \ - Arg(type* p, Parser parser) : arg_(p), parser_(parser) { } - - - PCRE_MAKE_PARSER(char, parse_char); - PCRE_MAKE_PARSER(unsigned char, parse_uchar); - PCRE_MAKE_PARSER(short, parse_short); - PCRE_MAKE_PARSER(unsigned short, parse_ushort); - PCRE_MAKE_PARSER(int, parse_int); - PCRE_MAKE_PARSER(unsigned int, parse_uint); - PCRE_MAKE_PARSER(long, parse_long); - PCRE_MAKE_PARSER(unsigned long, parse_ulong); -#if 1 - PCRE_MAKE_PARSER(long long, parse_longlong); -#endif -#if 1 - PCRE_MAKE_PARSER(unsigned long long, parse_ulonglong); -#endif - PCRE_MAKE_PARSER(float, parse_float); - PCRE_MAKE_PARSER(double, parse_double); - PCRE_MAKE_PARSER(std::string, parse_string); - PCRE_MAKE_PARSER(StringPiece, parse_stringpiece); - -#undef PCRE_MAKE_PARSER - - // Generic constructor - template Arg(T*, Parser parser); - // Generic constructor template - template Arg(T* p) - : arg_(p), parser_(_RE_MatchObject::Parse) { - } - - // Parse the data - bool Parse(const char* str, int n) const; - - private: - void* arg_; - Parser parser_; - - static bool parse_null (const char* str, int n, void* dest); - static bool parse_char (const char* str, int n, void* dest); - static bool parse_uchar (const char* str, int n, void* dest); - static bool parse_float (const char* str, int n, void* dest); - static bool parse_double (const char* str, int n, void* dest); - static bool parse_string (const char* str, int n, void* dest); - static bool parse_stringpiece (const char* str, int n, void* dest); - -#define PCRE_DECLARE_INTEGER_PARSER(name) \ - private: \ - static bool parse_ ## name(const char* str, int n, void* dest); \ - static bool parse_ ## name ## _radix( \ - const char* str, int n, void* dest, int radix); \ - public: \ - static bool parse_ ## name ## _hex(const char* str, int n, void* dest); \ - static bool parse_ ## name ## _octal(const char* str, int n, void* dest); \ - static bool parse_ ## name ## _cradix(const char* str, int n, void* dest) - - PCRE_DECLARE_INTEGER_PARSER(short); - PCRE_DECLARE_INTEGER_PARSER(ushort); - PCRE_DECLARE_INTEGER_PARSER(int); - PCRE_DECLARE_INTEGER_PARSER(uint); - PCRE_DECLARE_INTEGER_PARSER(long); - PCRE_DECLARE_INTEGER_PARSER(ulong); - PCRE_DECLARE_INTEGER_PARSER(longlong); - PCRE_DECLARE_INTEGER_PARSER(ulonglong); - -#undef PCRE_DECLARE_INTEGER_PARSER -}; - -inline Arg::Arg() : arg_(NULL), parser_(parse_null) { } -inline Arg::Arg(void* p) : arg_(p), parser_(parse_null) { } - -inline bool Arg::Parse(const char* str, int n) const { - return (*parser_)(str, n, arg_); -} - -// This part of the parser, appropriate only for ints, deals with bases -#define MAKE_INTEGER_PARSER(type, name) \ - inline Arg Hex(type* ptr) { \ - return Arg(ptr, Arg::parse_ ## name ## _hex); } \ - inline Arg Octal(type* ptr) { \ - return Arg(ptr, Arg::parse_ ## name ## _octal); } \ - inline Arg CRadix(type* ptr) { \ - return Arg(ptr, Arg::parse_ ## name ## _cradix); } - -MAKE_INTEGER_PARSER(short, short) /* */ -MAKE_INTEGER_PARSER(unsigned short, ushort) /* */ -MAKE_INTEGER_PARSER(int, int) /* Don't use semicolons */ -MAKE_INTEGER_PARSER(unsigned int, uint) /* after these statement */ -MAKE_INTEGER_PARSER(long, long) /* because they can cause */ -MAKE_INTEGER_PARSER(unsigned long, ulong) /* compiler warnings if */ -#if 1 /* the checking level is */ -MAKE_INTEGER_PARSER(long long, longlong) /* turned up high enough. */ -#endif /* */ -#if 1 /* */ -MAKE_INTEGER_PARSER(unsigned long long, ulonglong) /* */ -#endif - -#undef PCRE_IS_SET -#undef PCRE_SET_OR_CLEAR -#undef MAKE_INTEGER_PARSER - -} // namespace pcrecpp - - -#endif /* _PCRECPPARG_H */ diff --git a/src/third_party/pcre-8.39/pcrecpparg.h.in b/src/third_party/pcre-8.39/pcrecpparg.h.in deleted file mode 100644 index 61bcab5402c..00000000000 --- a/src/third_party/pcre-8.39/pcrecpparg.h.in +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Author: Sanjay Ghemawat - -#ifndef _PCRECPPARG_H -#define _PCRECPPARG_H - -#include // for NULL -#include - -#include - -namespace pcrecpp { - -class StringPiece; - -// Hex/Octal/Binary? - -// Special class for parsing into objects that define a ParseFrom() method -template -class _RE_MatchObject { - public: - static inline bool Parse(const char* str, int n, void* dest) { - if (dest == NULL) return true; - T* object = reinterpret_cast(dest); - return object->ParseFrom(str, n); - } -}; - -class PCRECPP_EXP_DEFN Arg { - public: - // Empty constructor so we can declare arrays of Arg - Arg(); - - // Constructor specially designed for NULL arguments - Arg(void*); - - typedef bool (*Parser)(const char* str, int n, void* dest); - -// Type-specific parsers -#define PCRE_MAKE_PARSER(type,name) \ - Arg(type* p) : arg_(p), parser_(name) { } \ - Arg(type* p, Parser parser) : arg_(p), parser_(parser) { } - - - PCRE_MAKE_PARSER(char, parse_char); - PCRE_MAKE_PARSER(unsigned char, parse_uchar); - PCRE_MAKE_PARSER(short, parse_short); - PCRE_MAKE_PARSER(unsigned short, parse_ushort); - PCRE_MAKE_PARSER(int, parse_int); - PCRE_MAKE_PARSER(unsigned int, parse_uint); - PCRE_MAKE_PARSER(long, parse_long); - PCRE_MAKE_PARSER(unsigned long, parse_ulong); -#if @pcre_have_long_long@ - PCRE_MAKE_PARSER(long long, parse_longlong); -#endif -#if @pcre_have_ulong_long@ - PCRE_MAKE_PARSER(unsigned long long, parse_ulonglong); -#endif - PCRE_MAKE_PARSER(float, parse_float); - PCRE_MAKE_PARSER(double, parse_double); - PCRE_MAKE_PARSER(std::string, parse_string); - PCRE_MAKE_PARSER(StringPiece, parse_stringpiece); - -#undef PCRE_MAKE_PARSER - - // Generic constructor - template Arg(T*, Parser parser); - // Generic constructor template - template Arg(T* p) - : arg_(p), parser_(_RE_MatchObject::Parse) { - } - - // Parse the data - bool Parse(const char* str, int n) const; - - private: - void* arg_; - Parser parser_; - - static bool parse_null (const char* str, int n, void* dest); - static bool parse_char (const char* str, int n, void* dest); - static bool parse_uchar (const char* str, int n, void* dest); - static bool parse_float (const char* str, int n, void* dest); - static bool parse_double (const char* str, int n, void* dest); - static bool parse_string (const char* str, int n, void* dest); - static bool parse_stringpiece (const char* str, int n, void* dest); - -#define PCRE_DECLARE_INTEGER_PARSER(name) \ - private: \ - static bool parse_ ## name(const char* str, int n, void* dest); \ - static bool parse_ ## name ## _radix( \ - const char* str, int n, void* dest, int radix); \ - public: \ - static bool parse_ ## name ## _hex(const char* str, int n, void* dest); \ - static bool parse_ ## name ## _octal(const char* str, int n, void* dest); \ - static bool parse_ ## name ## _cradix(const char* str, int n, void* dest) - - PCRE_DECLARE_INTEGER_PARSER(short); - PCRE_DECLARE_INTEGER_PARSER(ushort); - PCRE_DECLARE_INTEGER_PARSER(int); - PCRE_DECLARE_INTEGER_PARSER(uint); - PCRE_DECLARE_INTEGER_PARSER(long); - PCRE_DECLARE_INTEGER_PARSER(ulong); - PCRE_DECLARE_INTEGER_PARSER(longlong); - PCRE_DECLARE_INTEGER_PARSER(ulonglong); - -#undef PCRE_DECLARE_INTEGER_PARSER -}; - -inline Arg::Arg() : arg_(NULL), parser_(parse_null) { } -inline Arg::Arg(void* p) : arg_(p), parser_(parse_null) { } - -inline bool Arg::Parse(const char* str, int n) const { - return (*parser_)(str, n, arg_); -} - -// This part of the parser, appropriate only for ints, deals with bases -#define MAKE_INTEGER_PARSER(type, name) \ - inline Arg Hex(type* ptr) { \ - return Arg(ptr, Arg::parse_ ## name ## _hex); } \ - inline Arg Octal(type* ptr) { \ - return Arg(ptr, Arg::parse_ ## name ## _octal); } \ - inline Arg CRadix(type* ptr) { \ - return Arg(ptr, Arg::parse_ ## name ## _cradix); } - -MAKE_INTEGER_PARSER(short, short) /* */ -MAKE_INTEGER_PARSER(unsigned short, ushort) /* */ -MAKE_INTEGER_PARSER(int, int) /* Don't use semicolons */ -MAKE_INTEGER_PARSER(unsigned int, uint) /* after these statement */ -MAKE_INTEGER_PARSER(long, long) /* because they can cause */ -MAKE_INTEGER_PARSER(unsigned long, ulong) /* compiler warnings if */ -#if @pcre_have_long_long@ /* the checking level is */ -MAKE_INTEGER_PARSER(long long, longlong) /* turned up high enough. */ -#endif /* */ -#if @pcre_have_ulong_long@ /* */ -MAKE_INTEGER_PARSER(unsigned long long, ulonglong) /* */ -#endif - -#undef PCRE_IS_SET -#undef PCRE_SET_OR_CLEAR -#undef MAKE_INTEGER_PARSER - -} // namespace pcrecpp - - -#endif /* _PCRECPPARG_H */ diff --git a/src/third_party/pcre-8.39/pcredemo.c b/src/third_party/pcre-8.39/pcredemo.c deleted file mode 100644 index 946aba45cdc..00000000000 --- a/src/third_party/pcre-8.39/pcredemo.c +++ /dev/null @@ -1,406 +0,0 @@ -/************************************************* -* PCRE DEMONSTRATION PROGRAM * -*************************************************/ - -/* This is a demonstration program to illustrate the most straightforward ways -of calling the PCRE regular expression library from a C program. See the -pcresample documentation for a short discussion ("man pcresample" if you have -the PCRE man pages installed). - -In Unix-like environments, if PCRE is installed in your standard system -libraries, you should be able to compile this program using this command: - -gcc -Wall pcredemo.c -lpcre -o pcredemo - -If PCRE is not installed in a standard place, it is likely to be installed with -support for the pkg-config mechanism. If you have pkg-config, you can compile -this program using this command: - -gcc -Wall pcredemo.c `pkg-config --cflags --libs libpcre` -o pcredemo - -If you do not have pkg-config, you may have to use this: - -gcc -Wall pcredemo.c -I/usr/local/include -L/usr/local/lib \ - -R/usr/local/lib -lpcre -o pcredemo - -Replace "/usr/local/include" and "/usr/local/lib" with wherever the include and -library files for PCRE are installed on your system. Only some operating -systems (e.g. Solaris) use the -R option. - -Building under Windows: - -If you want to statically link this program against a non-dll .a file, you must -define PCRE_STATIC before including pcre.h, otherwise the pcre_malloc() and -pcre_free() exported functions will be declared __declspec(dllimport), with -unwanted results. So in this environment, uncomment the following line. */ - -/* #define PCRE_STATIC */ - -#include -#include -#include - -#define OVECCOUNT 30 /* should be a multiple of 3 */ - - -int main(int argc, char **argv) -{ -pcre *re; -const char *error; -char *pattern; -char *subject; -unsigned char *name_table; -unsigned int option_bits; -int erroffset; -int find_all; -int crlf_is_newline; -int namecount; -int name_entry_size; -int ovector[OVECCOUNT]; -int subject_length; -int rc, i; -int utf8; - - -/************************************************************************** -* First, sort out the command line. There is only one possible option at * -* the moment, "-g" to request repeated matching to find all occurrences, * -* like Perl's /g option. We set the variable find_all to a non-zero value * -* if the -g option is present. Apart from that, there must be exactly two * -* arguments. * -**************************************************************************/ - -find_all = 0; -for (i = 1; i < argc; i++) - { - if (strcmp(argv[i], "-g") == 0) find_all = 1; - else break; - } - -/* After the options, we require exactly two arguments, which are the pattern, -and the subject string. */ - -if (argc - i != 2) - { - printf("Two arguments required: a regex and a subject string\n"); - return 1; - } - -pattern = argv[i]; -subject = argv[i+1]; -subject_length = (int)strlen(subject); - - -/************************************************************************* -* Now we are going to compile the regular expression pattern, and handle * -* and errors that are detected. * -*************************************************************************/ - -re = pcre_compile( - pattern, /* the pattern */ - 0, /* default options */ - &error, /* for error message */ - &erroffset, /* for error offset */ - NULL); /* use default character tables */ - -/* Compilation failed: print the error message and exit */ - -if (re == NULL) - { - printf("PCRE compilation failed at offset %d: %s\n", erroffset, error); - return 1; - } - - -/************************************************************************* -* If the compilation succeeded, we call PCRE again, in order to do a * -* pattern match against the subject string. This does just ONE match. If * -* further matching is needed, it will be done below. * -*************************************************************************/ - -rc = pcre_exec( - re, /* the compiled pattern */ - NULL, /* no extra data - we didn't study the pattern */ - subject, /* the subject string */ - subject_length, /* the length of the subject */ - 0, /* start at offset 0 in the subject */ - 0, /* default options */ - ovector, /* output vector for substring information */ - OVECCOUNT); /* number of elements in the output vector */ - -/* Matching failed: handle error cases */ - -if (rc < 0) - { - switch(rc) - { - case PCRE_ERROR_NOMATCH: printf("No match\n"); break; - /* - Handle other special cases if you like - */ - default: printf("Matching error %d\n", rc); break; - } - pcre_free(re); /* Release memory used for the compiled pattern */ - return 1; - } - -/* Match succeded */ - -printf("\nMatch succeeded at offset %d\n", ovector[0]); - - -/************************************************************************* -* We have found the first match within the subject string. If the output * -* vector wasn't big enough, say so. Then output any substrings that were * -* captured. * -*************************************************************************/ - -/* The output vector wasn't big enough */ - -if (rc == 0) - { - rc = OVECCOUNT/3; - printf("ovector only has room for %d captured substrings\n", rc - 1); - } - -/* Show substrings stored in the output vector by number. Obviously, in a real -application you might want to do things other than print them. */ - -for (i = 0; i < rc; i++) - { - char *substring_start = subject + ovector[2*i]; - int substring_length = ovector[2*i+1] - ovector[2*i]; - printf("%2d: %.*s\n", i, substring_length, substring_start); - } - - -/************************************************************************** -* That concludes the basic part of this demonstration program. We have * -* compiled a pattern, and performed a single match. The code that follows * -* shows first how to access named substrings, and then how to code for * -* repeated matches on the same subject. * -**************************************************************************/ - -/* See if there are any named substrings, and if so, show them by name. First -we have to extract the count of named parentheses from the pattern. */ - -(void)pcre_fullinfo( - re, /* the compiled pattern */ - NULL, /* no extra data - we didn't study the pattern */ - PCRE_INFO_NAMECOUNT, /* number of named substrings */ - &namecount); /* where to put the answer */ - -if (namecount <= 0) printf("No named substrings\n"); else - { - unsigned char *tabptr; - printf("Named substrings\n"); - - /* Before we can access the substrings, we must extract the table for - translating names to numbers, and the size of each entry in the table. */ - - (void)pcre_fullinfo( - re, /* the compiled pattern */ - NULL, /* no extra data - we didn't study the pattern */ - PCRE_INFO_NAMETABLE, /* address of the table */ - &name_table); /* where to put the answer */ - - (void)pcre_fullinfo( - re, /* the compiled pattern */ - NULL, /* no extra data - we didn't study the pattern */ - PCRE_INFO_NAMEENTRYSIZE, /* size of each entry in the table */ - &name_entry_size); /* where to put the answer */ - - /* Now we can scan the table and, for each entry, print the number, the name, - and the substring itself. */ - - tabptr = name_table; - for (i = 0; i < namecount; i++) - { - int n = (tabptr[0] << 8) | tabptr[1]; - printf("(%d) %*s: %.*s\n", n, name_entry_size - 3, tabptr + 2, - ovector[2*n+1] - ovector[2*n], subject + ovector[2*n]); - tabptr += name_entry_size; - } - } - - -/************************************************************************* -* If the "-g" option was given on the command line, we want to continue * -* to search for additional matches in the subject string, in a similar * -* way to the /g option in Perl. This turns out to be trickier than you * -* might think because of the possibility of matching an empty string. * -* What happens is as follows: * -* * -* If the previous match was NOT for an empty string, we can just start * -* the next match at the end of the previous one. * -* * -* If the previous match WAS for an empty string, we can't do that, as it * -* would lead to an infinite loop. Instead, a special call of pcre_exec() * -* is made with the PCRE_NOTEMPTY_ATSTART and PCRE_ANCHORED flags set. * -* The first of these tells PCRE that an empty string at the start of the * -* subject is not a valid match; other possibilities must be tried. The * -* second flag restricts PCRE to one match attempt at the initial string * -* position. If this match succeeds, an alternative to the empty string * -* match has been found, and we can print it and proceed round the loop, * -* advancing by the length of whatever was found. If this match does not * -* succeed, we still stay in the loop, advancing by just one character. * -* In UTF-8 mode, which can be set by (*UTF8) in the pattern, this may be * -* more than one byte. * -* * -* However, there is a complication concerned with newlines. When the * -* newline convention is such that CRLF is a valid newline, we must * -* advance by two characters rather than one. The newline convention can * -* be set in the regex by (*CR), etc.; if not, we must find the default. * -*************************************************************************/ - -if (!find_all) /* Check for -g */ - { - pcre_free(re); /* Release the memory used for the compiled pattern */ - return 0; /* Finish unless -g was given */ - } - -/* Before running the loop, check for UTF-8 and whether CRLF is a valid newline -sequence. First, find the options with which the regex was compiled; extract -the UTF-8 state, and mask off all but the newline options. */ - -(void)pcre_fullinfo(re, NULL, PCRE_INFO_OPTIONS, &option_bits); -utf8 = option_bits & PCRE_UTF8; -option_bits &= PCRE_NEWLINE_CR|PCRE_NEWLINE_LF|PCRE_NEWLINE_CRLF| - PCRE_NEWLINE_ANY|PCRE_NEWLINE_ANYCRLF; - -/* If no newline options were set, find the default newline convention from the -build configuration. */ - -if (option_bits == 0) - { - int d; - (void)pcre_config(PCRE_CONFIG_NEWLINE, &d); - /* Note that these values are always the ASCII ones, even in - EBCDIC environments. CR = 13, NL = 10. */ - option_bits = (d == 13)? PCRE_NEWLINE_CR : - (d == 10)? PCRE_NEWLINE_LF : - (d == (13<<8 | 10))? PCRE_NEWLINE_CRLF : - (d == -2)? PCRE_NEWLINE_ANYCRLF : - (d == -1)? PCRE_NEWLINE_ANY : 0; - } - -/* See if CRLF is a valid newline sequence. */ - -crlf_is_newline = - option_bits == PCRE_NEWLINE_ANY || - option_bits == PCRE_NEWLINE_CRLF || - option_bits == PCRE_NEWLINE_ANYCRLF; - -/* Loop for second and subsequent matches */ - -for (;;) - { - int options = 0; /* Normally no options */ - int start_offset = ovector[1]; /* Start at end of previous match */ - - /* If the previous match was for an empty string, we are finished if we are - at the end of the subject. Otherwise, arrange to run another match at the - same point to see if a non-empty match can be found. */ - - if (ovector[0] == ovector[1]) - { - if (ovector[0] == subject_length) break; - options = PCRE_NOTEMPTY_ATSTART | PCRE_ANCHORED; - } - - /* Run the next matching operation */ - - rc = pcre_exec( - re, /* the compiled pattern */ - NULL, /* no extra data - we didn't study the pattern */ - subject, /* the subject string */ - subject_length, /* the length of the subject */ - start_offset, /* starting offset in the subject */ - options, /* options */ - ovector, /* output vector for substring information */ - OVECCOUNT); /* number of elements in the output vector */ - - /* This time, a result of NOMATCH isn't an error. If the value in "options" - is zero, it just means we have found all possible matches, so the loop ends. - Otherwise, it means we have failed to find a non-empty-string match at a - point where there was a previous empty-string match. In this case, we do what - Perl does: advance the matching position by one character, and continue. We - do this by setting the "end of previous match" offset, because that is picked - up at the top of the loop as the point at which to start again. - - There are two complications: (a) When CRLF is a valid newline sequence, and - the current position is just before it, advance by an extra byte. (b) - Otherwise we must ensure that we skip an entire UTF-8 character if we are in - UTF-8 mode. */ - - if (rc == PCRE_ERROR_NOMATCH) - { - if (options == 0) break; /* All matches found */ - ovector[1] = start_offset + 1; /* Advance one byte */ - if (crlf_is_newline && /* If CRLF is newline & */ - start_offset < subject_length - 1 && /* we are at CRLF, */ - subject[start_offset] == '\r' && - subject[start_offset + 1] == '\n') - ovector[1] += 1; /* Advance by one more. */ - else if (utf8) /* Otherwise, ensure we */ - { /* advance a whole UTF-8 */ - while (ovector[1] < subject_length) /* character. */ - { - if ((subject[ovector[1]] & 0xc0) != 0x80) break; - ovector[1] += 1; - } - } - continue; /* Go round the loop again */ - } - - /* Other matching errors are not recoverable. */ - - if (rc < 0) - { - printf("Matching error %d\n", rc); - pcre_free(re); /* Release memory used for the compiled pattern */ - return 1; - } - - /* Match succeded */ - - printf("\nMatch succeeded again at offset %d\n", ovector[0]); - - /* The match succeeded, but the output vector wasn't big enough. */ - - if (rc == 0) - { - rc = OVECCOUNT/3; - printf("ovector only has room for %d captured substrings\n", rc - 1); - } - - /* As before, show substrings stored in the output vector by number, and then - also any named substrings. */ - - for (i = 0; i < rc; i++) - { - char *substring_start = subject + ovector[2*i]; - int substring_length = ovector[2*i+1] - ovector[2*i]; - printf("%2d: %.*s\n", i, substring_length, substring_start); - } - - if (namecount <= 0) printf("No named substrings\n"); else - { - unsigned char *tabptr = name_table; - printf("Named substrings\n"); - for (i = 0; i < namecount; i++) - { - int n = (tabptr[0] << 8) | tabptr[1]; - printf("(%d) %*s: %.*s\n", n, name_entry_size - 3, tabptr + 2, - ovector[2*n+1] - ovector[2*n], subject + ovector[2*n]); - tabptr += name_entry_size; - } - } - } /* End of loop to find second and subsequent matches */ - -printf("\n"); -pcre_free(re); /* Release memory used for the compiled pattern */ -return 0; -} - -/* End of pcredemo.c */ diff --git a/src/third_party/pcre-8.39/pcregexp.pas b/src/third_party/pcre-8.39/pcregexp.pas deleted file mode 100644 index bb2b3da8f3d..00000000000 --- a/src/third_party/pcre-8.39/pcregexp.pas +++ /dev/null @@ -1,845 +0,0 @@ -{ - pcRegExp - Perl compatible regular expressions for Virtual Pascal - (c) 2001 Peter S. Voronov aka Chem O'Dun - - Based on PCRE library interface unit for Virtual Pascal. - (c) 2001 Alexander Tokarev - - The current PCRE version is: 3.7 - - This software may be distributed under the terms of the modified BSD license - Copyright (c) 2001, Alexander Tokarev - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of the nor the names of its contributors - may be used to endorse or promote products derived from this software without - specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - The PCRE library is written by: Philip Hazel - Copyright (c) 1997-2004 University of Cambridge - - AngelsHolocaust 4-11-04 updated to use version v5.0 - (INFO: this is regex-directed, NFA) - AH: 9-11-04 - pcre_free: removed var, pcre already gives the ptr, now - everything works as it should (no more crashes) - -> removed CheckRegExp because pcre handles errors perfectly - 10-11-04 - added pcError (errorhandling), pcInit - 13-11-04 - removed the ErrorPos = 0 check -> always print erroroffset - 17-10-05 - support for \1-\9 backreferences in TpcRegExp.GetReplStr - 17-02-06 - added RunTimeOptions: caller can set options while searching - 19-02-06 - added SearchOfs(): let PCRE use the complete string and offset - into the string itself - 20-12-06 - support for version 7.0 - 27.08.08 - support for v7.7 -} - -{$H+} {$DEFINE PCRE_3_7} {$DEFINE PCRE_5_0} {$DEFINE PCRE_7_0} {$DEFINE PCRE_7_7} - -Unit pcregexp; - -Interface - -uses objects; - -Type - PpcRegExp = ^TpcRegExp; -// TpcRegExp = object - TpcRegExp = object(TObject) - MatchesCount: integer; - RegExpC, RegExpExt : Pointer; - Matches:Pointer; - RegExp: shortstring; - SourceLen: integer; - PartialMatch : boolean; - Error : boolean; - ErrorMsg : Pchar; - ErrorPos : integer; - RunTimeOptions: Integer; // options which can be set by the caller - constructor Init(const ARegExp : shortstring; AOptions : integer; ALocale : Pointer); - function Search(AStr: Pchar; ALen : longint) : boolean; virtual; - function SearchNext( AStr: Pchar; ALen : longint) : boolean; virtual; - function SearchOfs ( AStr: Pchar; ALen, AOfs : longint) : boolean; virtual; - function MatchSub(ANom: integer; var Pos, Len : longint) : boolean; virtual; - function MatchFull(var Pos, Len : longint) : boolean; virtual; - function GetSubStr(ANom: integer; AStr: Pchar) : string; virtual; - function GetFullStr(AStr: Pchar) : string; virtual; - function GetReplStr(AStr: Pchar; const ARepl: string) : string; virtual; - function GetPreSubStr(AStr: Pchar) : string; virtual; - function GetPostSubStr(AStr: Pchar) : string; virtual; - function ErrorStr : string; virtual; - destructor Done; virtual; - end; - - function pcGrepMatch(WildCard, aStr: string; AOptions:integer; ALocale : Pointer): Boolean; - function pcGrepSub(WildCard, aStr, aRepl: string; AOptions:integer; ALocale : Pointer): string; - - function pcFastGrepMatch(WildCard, aStr: string): Boolean; - function pcFastGrepSub(WildCard, aStr, aRepl: string): string; - -{$IFDEF PCRE_5_0} - function pcGetVersion : pchar; -{$ENDIF} - - function pcError (var pRegExp : Pointer) : Boolean; - function pcInit (const Pattern: Shortstring; CaseSens: Boolean) : Pointer; - -Const { Options } - PCRE_CASELESS = $0001; - PCRE_MULTILINE = $0002; - PCRE_DOTALL = $0004; - PCRE_EXTENDED = $0008; - PCRE_ANCHORED = $0010; - PCRE_DOLLAR_ENDONLY = $0020; - PCRE_EXTRA = $0040; - PCRE_NOTBOL = $0080; - PCRE_NOTEOL = $0100; - PCRE_UNGREEDY = $0200; - PCRE_NOTEMPTY = $0400; -{$IFDEF PCRE_5_0} - PCRE_UTF8 = $0800; - PCRE_NO_AUTO_CAPTURE = $1000; - PCRE_NO_UTF8_CHECK = $2000; - PCRE_AUTO_CALLOUT = $4000; - PCRE_PARTIAL = $8000; -{$ENDIF} -{$IFDEF PCRE_7_0} - PCRE_DFA_SHORTEST = $00010000; - PCRE_DFA_RESTART = $00020000; - PCRE_FIRSTLINE = $00040000; - PCRE_DUPNAMES = $00080000; - PCRE_NEWLINE_CR = $00100000; - PCRE_NEWLINE_LF = $00200000; - PCRE_NEWLINE_CRLF = $00300000; - PCRE_NEWLINE_ANY = $00400000; - PCRE_NEWLINE_ANYCRLF = $00500000; - - PCRE_NEWLINE_BITS = PCRE_NEWLINE_CR or PCRE_NEWLINE_LF or PCRE_NEWLINE_ANY; - -{$ENDIF} -{$IFDEF PCRE_7_7} - PCRE_BSR_ANYCRLF = $00800000; - PCRE_BSR_UNICODE = $01000000; - PCRE_JAVASCRIPT_COMPAT= $02000000; -{$ENDIF} - - PCRE_COMPILE_ALLOWED_OPTIONS = PCRE_ANCHORED + PCRE_AUTO_CALLOUT + PCRE_CASELESS + - PCRE_DOLLAR_ENDONLY + PCRE_DOTALL + PCRE_EXTENDED + - PCRE_EXTRA + PCRE_MULTILINE + PCRE_NO_AUTO_CAPTURE + - PCRE_UNGREEDY + PCRE_UTF8 + PCRE_NO_UTF8_CHECK - {$IFDEF PCRE_7_0} - + PCRE_DUPNAMES + PCRE_FIRSTLINE + PCRE_NEWLINE_BITS - {$ENDIF} - {$IFDEF PCRE_7_7} - + PCRE_BSR_ANYCRLF + PCRE_BSR_UNICODE + PCRE_JAVASCRIPT_COMPAT - {$ENDIF} - ; - - PCRE_EXEC_ALLOWED_OPTIONS = PCRE_ANCHORED + PCRE_NOTBOL + PCRE_NOTEOL + - PCRE_NOTEMPTY + PCRE_NO_UTF8_CHECK + PCRE_PARTIAL - {$IFDEF PCRE_7_0} - + PCRE_NEWLINE_BITS - {$ENDIF} - {$IFDEF PCRE_7_7} - + PCRE_BSR_ANYCRLF + PCRE_BSR_UNICODE - {$ENDIF} - ; - -{$IFDEF PCRE_7_0} - PCRE_DFA_EXEC_ALLOWED_OPTIONS = PCRE_ANCHORED + PCRE_NOTBOL + PCRE_NOTEOL + - PCRE_NOTEMPTY + PCRE_NO_UTF8_CHECK + PCRE_PARTIAL + - PCRE_DFA_SHORTEST + PCRE_DFA_RESTART + - PCRE_NEWLINE_BITS - {$IFDEF PCRE_7_7} - + PCRE_BSR_ANYCRLF + PCRE_BSR_UNICODE - {$ENDIF} - ; -{$ENDIF} - -{ Exec-time and get/set-time error codes } - PCRE_ERROR_NOMATCH = -1; - PCRE_ERROR_NULL = -2; - PCRE_ERROR_BADOPTION = -3; - PCRE_ERROR_BADMAGIC = -4; - PCRE_ERROR_UNKNOWN_MODE = -5; - PCRE_ERROR_NOMEMORY = -6; - PCRE_ERROR_NOSUBSTRING = -7; -{$IFDEF PCRE_5_0} - PCRE_ERROR_MATCHLIMIT = -8; - PCRE_ERROR_CALLOUT = -9; { Never used by PCRE itself } - PCRE_ERROR_BADUTF8 = -10; - PCRE_ERROR_BADUTF8_OFFSET = -11; - PCRE_ERROR_PARTIAL = -12; - PCRE_ERROR_BADPARTIAL = -13; - PCRE_ERROR_INTERNAL = -14; - PCRE_ERROR_BADCOUNT = -15; -{$ENDIF} -{$IFDEF PCRE_7_0} - PCRE_ERROR_DFA_UITEM = -16; - PCRE_ERROR_DFA_UCOND = -17; - PCRE_ERROR_DFA_UMLIMIT = -18; - PCRE_ERROR_DFA_WSSIZE = -19; - PCRE_ERROR_DFA_RECURSE = -20; - PCRE_ERROR_RECURSIONLIMIT = -21; - PCRE_ERROR_NULLWSLIMIT = -22; - PCRE_ERROR_BADNEWLINE = -23; -{$ENDIF} - -{ Request types for pcre_fullinfo() } - - PCRE_INFO_OPTIONS = 0; - PCRE_INFO_SIZE = 1; - PCRE_INFO_CAPTURECOUNT = 2; - PCRE_INFO_BACKREFMAX = 3; - PCRE_INFO_FIRSTBYTE = 4; - PCRE_INFO_FIRSTCHAR = 4; { For backwards compatibility } - PCRE_INFO_FIRSTTABLE = 5; -{$IFDEF PCRE_5_0} - PCRE_INFO_LASTLITERAL = 6; - PCRE_INFO_NAMEENTRYSIZE = 7; - PCRE_INFO_NAMECOUNT = 8; - PCRE_INFO_NAMETABLE = 9; - PCRE_INFO_STUDYSIZE = 10; - PCRE_INFO_DEFAULT_TABLES = 11; -{$ENDIF PCRE_5_0} -{$IFDEF PCRE_7_7} - PCRE_INFO_OKPARTIAL = 12; - PCRE_INFO_JCHANGED = 13; - PCRE_INFO_HASCRORLF = 14; -{$ENDIF} - -{ Request types for pcre_config() } -{$IFDEF PCRE_5_0} - PCRE_CONFIG_UTF8 = 0; - PCRE_CONFIG_NEWLINE = 1; - PCRE_CONFIG_LINK_SIZE = 2; - PCRE_CONFIG_POSIX_MALLOC_THRESHOLD = 3; - PCRE_CONFIG_MATCH_LIMIT = 4; - PCRE_CONFIG_STACKRECURSE = 5; - PCRE_CONFIG_UNICODE_PROPERTIES = 6; -{$ENDIF PCRE_5_0} -{$IFDEF PCRE_7_0} - PCRE_CONFIG_MATCH_LIMIT_RECURSION = 7; -{$ENDIF} -{$IFDEF PCRE_7_7} - PCRE_CONFIG_BSR = 8; -{$ENDIF} - -{ Bit flags for the pcre_extra structure } -{$IFDEF PCRE_5_0} - PCRE_EXTRA_STUDY_DATA = $0001; - PCRE_EXTRA_MATCH_LIMIT = $0002; - PCRE_EXTRA_CALLOUT_DATA = $0004; - PCRE_EXTRA_TABLES = $0008; -{$ENDIF PCRE_5_0} -{$IFDEF PCRE_7_0} - PCRE_EXTRA_MATCH_LIMIT_RECURSION = $0010; -{$ENDIF} - -Const -// DefaultOptions : integer = 0; - DefaultLocaleTable : pointer = nil; - -{$IFDEF PCRE_5_0} -{ The structure for passing additional data to pcre_exec(). This is defined in -such as way as to be extensible. Always add new fields at the end, in order to -remain compatible. } - -type ppcre_extra = ^tpcre_extra; - tpcre_extra = record - flags : longint; { Bits for which fields are set } - study_data : pointer; { Opaque data from pcre_study() } - match_limit : longint; { Maximum number of calls to match() } - callout_data : pointer; { Data passed back in callouts } - tables : pointer; { Pointer to character tables } - match_limit_recursion: longint; { Max recursive calls to match() } - end; - -type ppcre_callout_block = ^pcre_callout_block; - pcre_callout_block = record - version, - (* ------------------------ Version 0 ------------------------------- *) - callout_number : integer; - offset_vector : pointer; - subject : pchar; - subject_length, start_match, current_position, capture_top, - capture_last : integer; - callout_data : pointer; - (* ------------------- Added for Version 1 -------------------------- *) - pattern_position, next_item_length : integer; - end; -{$ENDIF PCRE_5_0} - -{$OrgName+} -{$IFDEF VIRTUALPASCAL} {&Cdecl+} {$ENDIF VIRTUALPASCAL} - - { local replacement of external pcre memory management functions } - function pcre_malloc( size : integer ) : pointer; - procedure pcre_free( {var} p : pointer ); -{$IFDEF PCRE_5_0} - const pcre_stack_malloc: function ( size : integer ): pointer = pcre_malloc; - pcre_stack_free: procedure ( {var} p : pointer ) = pcre_free; - function pcre_callout(var p : ppcre_callout_block) : integer; -{$ENDIF PCRE_5_0} -{$IFDEF VIRTUALPASCAL} {&Cdecl-} {$ENDIF VIRTUALPASCAL} - -Implementation - -Uses strings, collect, messages, dnapp, commands, advance0, stringsx - {$IFDEF VIRTUALPASCAL} ,vpsyslow {$ENDIF VIRTUALPASCAL}; - -Const - MAGIC_NUMBER = $50435245; { 'PCRE' } - MAX_MATCHES = 90; { changed in 3.5 version; should be divisible by 3, was 64} - -Type - PMatchArray = ^TMatchArray; - TMatchArray = array[0..( MAX_MATCHES * 3 )] of integer; - - PRegExpCollection = ^TRegExpCollection; - TRegExpCollection = object(TSortedCollection) - MaxRegExp : integer; - SearchRegExp : shortstring; - CompareModeInsert : boolean; - constructor Init(AMaxRegExp:integer); - procedure FreeItem(P: Pointer); virtual; - function Compare(P1, P2: Pointer): Integer; virtual; - function Find(ARegExp:shortstring;var P: PpcRegExp):boolean; virtual; - function CheckNew(ARegExp:shortstring):PpcRegExp;virtual; - end; - -Var - PRegExpCache : PRegExpCollection; - - -{$IFDEF VIRTUALPASCAL} {&Cdecl+} {$ENDIF VIRTUALPASCAL} - - { imported original pcre functions } - - function pcre_compile( const pattern : PChar; options : integer; - var errorptr : PChar; var erroroffset : integer; - const tables : PChar ) : pointer {pcre}; external; -{$IFDEF PCRE_7_0} - function pcre_compile2( const pattern : PChar; options : integer; - var errorcodeptr : Integer; - var errorptr : PChar; var erroroffset : integer; - const tables : PChar ) : pointer {pcre}; external; -{$ENDIF} -{$IFDEF PCRE_5_0} - function pcre_config( what : integer; where : pointer) : integer; external; - function pcre_copy_named_substring( const code : pointer {pcre}; - const subject : pchar; - var ovector : integer; - stringcount : integer; - const stringname : pchar; - var buffer : pchar; - size : integer) : integer; external; - function pcre_copy_substring( const subject : pchar; var ovector : integer; - stringcount, stringnumber : integer; - var buffer : pchar; size : integer ) - : integer; external; - function pcre_exec( const argument_re : pointer {pcre}; - const extra_data : pointer {pcre_extra}; -{$ELSE} - function pcre_exec( const external_re : pointer; - const external_extra : pointer; -{$ENDIF} - const subject : PChar; - length, start_offset, options : integer; - offsets : pointer; - offsetcount : integer ) : integer; external; -{$IFDEF PCRE_7_0} - function pcre_dfa_exec( const argument_re : pointer {pcre}; - const extra_data : pointer {pcre_extra}; - const subject : pchar; - length, start_offset, options : integer; - offsets : pointer; - offsetcount : integer; - workspace : pointer; - wscount : integer ) : integer; external; -{$ENDIF} -{$IFDEF PCRE_5_0} - procedure pcre_free_substring( const p : pchar ); external; - procedure pcre_free_substring_list( var p : pchar ); external; - function pcre_fullinfo( const argument_re : pointer {pcre}; - const extra_data : pointer {pcre_extra}; - what : integer; - where : pointer ) : integer; external; - function pcre_get_named_substring( const code : pointer {pcre}; - const subject : pchar; - var ovector : integer; - stringcount : integer; - const stringname : pchar; - var stringptr : pchar ) : integer; external; - function pcre_get_stringnumber( const code : pointer {pcre}; - const stringname : pchar ) : integer; external; - function pcre_get_stringtable_entries( const code : pointer {pcre}; - const stringname : pchar; - var firstptr, - lastptr : pchar ) : integer; external; - function pcre_get_substring( const subject : pchar; var ovector : integer; - stringcount, stringnumber : integer; - var stringptr : pchar ) : integer; external; - function pcre_get_substring_list( const subject : pchar; var ovector : integer; - stringcount : integer; - listptr : pointer {const char ***listptr}) : integer; external; - function pcre_info( const argument_re : pointer {pcre}; - var optptr : integer; - var first_byte : integer ) : integer; external; - function pcre_maketables : pchar; external; -{$ENDIF} -{$IFDEF PCRE_7_0} - function pcre_refcount( const argument_re : pointer {pcre}; - adjust : integer ) : pchar; external; -{$ENDIF} - function pcre_study( const external_re : pointer {pcre}; - options : integer; - var errorptr : PChar ) : pointer {pcre_extra}; external; -{$IFDEF PCRE_5_0} - function pcre_version : pchar; external; -{$ENDIF} - - function pcre_malloc( size : integer ) : pointer; - begin - GetMem( result, size ); - end; - - procedure pcre_free( {var} p : pointer ); - begin - if (p <> nil) then - FreeMem( p, 0 ); - {@p := nil;} - end; - -{$IFDEF PCRE_5_0} -(* Called from PCRE as a result of the (?C) item. We print out where we are in -the match. Yield zero unless more callouts than the fail count, or the callout -data is not zero. *) - - function pcre_callout; - begin - end; -{$ENDIF} - -{$IFDEF VIRTUALPASCAL} {&Cdecl-} {$ENDIF VIRTUALPASCAL} - -// Always include the newest version of the library -{$IFDEF PCRE_7_7} - {$L pcre77.lib} -{$ELSE} - {$IFDEF PCRE_7_0} - {$L pcre70.lib} - {$ELSE} - {$IFDEF PCRE_5_0} - {$L pcre50.lib} - {$ELSE} - {$IFDEF PCRE_3_7} - {$L pcre37.lib} - {$ENDIF PCRE_3_7} - {$ENDIF PCRE_5_0} - {$ENDIF PCRE_7_0} -{$ENDIF PCRE_7_7} - -{TpcRegExp} - - constructor TpcRegExp.Init(const ARegExp:shortstring; AOptions:integer; ALocale : Pointer); - var - pRegExp : PChar; - begin - RegExp:=ARegExp; - RegExpC:=nil; - RegExpExt:=nil; - Matches:=nil; - MatchesCount:=0; - Error:=true; - ErrorMsg:=nil; - ErrorPos:=0; - RunTimeOptions := 0; - if length(RegExp) < 255 then - begin - RegExp[length(RegExp)+1]:=#0; - pRegExp:=@RegExp[1]; - end - else - begin - GetMem(pRegExp,length(RegExp)+1); - pRegExp:=strpcopy(pRegExp,RegExp); - end; - RegExpC := pcre_compile( pRegExp, - AOptions and PCRE_COMPILE_ALLOWED_OPTIONS, - ErrorMsg, ErrorPos, ALocale); - if length(RegExp) = 255 then - StrDispose(pRegExp); - if RegExpC = nil then - exit; - ErrorMsg:=nil; - RegExpExt := pcre_study( RegExpC, 0, ErrorMsg ); - if (RegExpExt = nil) and (ErrorMsg <> nil) then - begin - pcre_free(RegExpC); - exit; - end; - GetMem(Matches,SizeOf(TMatchArray)); - Error:=false; - end; - - destructor TpcRegExp.Done; - begin - if RegExpC <> nil then - pcre_free(RegExpC); - if RegExpExt <> nil then - pcre_free(RegExpExt); - if Matches <> nil then - FreeMem(Matches,SizeOf(TMatchArray)); - end; - - function TpcRegExp.SearchNext( AStr: Pchar; ALen : longint ) : boolean; - var Options: Integer; - begin // must handle PCRE_ERROR_PARTIAL here - Options := (RunTimeOptions or startup.MiscMultiData.cfgRegEx.DefaultOptions) and - PCRE_EXEC_ALLOWED_OPTIONS; - if MatchesCount > 0 then - MatchesCount:=pcre_exec( RegExpC, RegExpExt, AStr, ALen, PMatchArray(Matches)^[1], - Options, Matches, MAX_MATCHES ) else - MatchesCount:=pcre_exec( RegExpC, RegExpExt, AStr, ALen, 0, - Options, Matches, MAX_MATCHES ); -{ if MatchesCount = 0 then - MatchesCount := MatchesCount div 3;} - PartialMatch := MatchesCount = PCRE_ERROR_PARTIAL; - SearchNext := MatchesCount > 0; - end; - - function TpcRegExp.Search( AStr: Pchar; ALen : longint):boolean; - begin - MatchesCount:=0; - Search:=SearchNext(AStr,ALen); - SourceLen:=ALen; - end; - - function TpcRegExp.SearchOfs( AStr: Pchar; ALen, AOfs: longint ) : boolean; - var Options: Integer; - begin - MatchesCount:=0; - Options := (RunTimeOptions or startup.MiscMultiData.cfgRegEx.DefaultOptions) and - PCRE_EXEC_ALLOWED_OPTIONS; - MatchesCount:=pcre_exec( RegExpC, RegExpExt, AStr, ALen, AOfs, - Options, Matches, MAX_MATCHES ); - PartialMatch := MatchesCount = PCRE_ERROR_PARTIAL; - SearchOfs := MatchesCount > 0; - SourceLen := ALen-AOfs; - end; - - function TpcRegExp.MatchSub(ANom:integer; var Pos,Len:longint):boolean; - begin - if (MatchesCount > 0) and (ANom <= (MatchesCount-1)) then - begin - ANom:=ANom*2; - Pos:=PMatchArray(Matches)^[ANom]; - Len:=PMatchArray(Matches)^[ANom+1]-Pos; - MatchSub:=true; - end - else - MatchSub:=false; - end; - - function TpcRegExp.MatchFull(var Pos,Len:longint):boolean; - begin - MatchFull:=MatchSub(0,Pos,Len); - end; - - function TpcRegExp.GetSubStr(ANom: integer; AStr: Pchar):string; - var - s: ansistring; - pos,len: longint; - begin - s:=''; - if MatchSub(ANom, pos, len) then - begin - setlength(s, len); - Move(AStr[pos], s[1], len); - end; - GetSubStr:=s; - end; - - function TpcRegExp.GetPreSubStr(AStr: Pchar):string; - var - s: ansistring; - l: longint; - begin - s:=''; - if (MatchesCount > 0) then - begin - l:=PMatchArray(Matches)^[0]-1; - if l > 0 then - begin - setlength(s,l); - Move(AStr[1],s[1],l); - end; - end; - GetPreSubStr:=s; - end; - - function TpcRegExp.GetPostSubStr(AStr: Pchar):string; - var - s: ansistring; - l: longint; - ANom: integer; - begin - s:=''; - if (MatchesCount > 0) then - begin - ANom:=(MatchesCount-1){*2} shl 1; - l:=SourceLen-PMatchArray(Matches)^[ANom+1]+1; - if l > 0 then - begin - setlength(s,l); - Move(AStr[PMatchArray(Matches)^[ANom+1]],s[1],l); - end; - end; - GetPostSubStr:=s; - end; - - - function TpcRegExp.GetFullStr(AStr: Pchar):string; - var - s: ansistring; - l: longint; - begin - GetFullStr:=GetSubStr(0,AStr); - end; - - function TpcRegExp.GetReplStr(AStr: Pchar; const ARepl: string):string; - var - s: ansistring; - l,i,lasti: longint; - begin - l:=length(ARepl); - i:=1; - lasti:=1; - s:=''; - while i <= l do - begin - case ARepl[i] of - '\' : - begin - if i < l then - begin - s:=s+copy(ARepl,lasti,i-lasti){+ARepl[i+1]}; - {AH 17-10-05 support for POSIX \1-\9 backreferences} - case ARepl[i+1] of - '0' : s:=s+GetFullStr(AStr); - '1'..'9' : s:=s+GetSubStr(ord(ARepl[i+1])-ord('0'),AStr); - else s:=s+ARepl[i+1]; // copy the escaped character - end; - end; - inc(i); - lasti:=i+1; - end; - '$' : - begin - if i < l then - begin - s:=s+copy(ARepl,lasti,i-lasti); - case ARepl[i+1] of - '&' : s:=s+GetFullStr(AStr); - '1'..'9' : s:=s+GetSubStr(ord(ARepl[i+1])-ord('0'),AStr); - '`' : s:=s+GetPreSubStr(AStr); - #39 : s:=s+GetPostSubStr(AStr); - end; - end; - inc(i); - lasti:=i+1; - end; - end; - inc(i); - end; - if lasti <= {AH 25-10-2004 added =, else l==1 won't work} l then - s:=s+copy(ARepl,lasti,l-lasti+1); - GetReplStr:=s; - end; - - function TpcRegExp.ErrorStr:string; - begin - ErrorStr:=StrPas(ErrorMsg); - end; - -{TRegExpCollection} - -constructor TRegExpCollection.Init(AMaxRegExp: integer); -begin - Inherited Init(1,1); - MaxRegExp:=AMaxRegExp; - CompareModeInsert:=true; -end; - -procedure TRegExpCollection.FreeItem(P: Pointer); -begin - if P <> nil then - begin - Dispose(PpcRegExp(P),Done); - end; -end; - -function TRegExpCollection.Compare(P1, P2: Pointer): Integer; -//var -// l,l1,l2,i : byte; -//// wPos: pchar; -begin - if CompareModeInsert then - begin -// l1:=length(PpcRegExp(P1)^.RegExp); -// l2:=length(PpcRegExp(P2)^.RegExp); -// if l1 > l2 then l:=l2 else -// l:=l1; -// for i:=1 to l do -// if PpcRegExp(P1).RegExp[i] <> PpcRegExp(P2).RegExp[i] then break; -// if i <=l then -// Compare:=ord(PpcRegExp(P1).RegExp[i])-ord(PpcRegExp(P2).RegExp[i]) else -// Compare:=l1-l2; - Compare := stringsx.PasStrCmp(PpcRegExp(P1).RegExp, PpcRegExp(P2).RegExp, False); - end - else - begin -// l1:=length(PpcRegExp(P1)^.RegExp); -// l2:=length(SearchRegExp); -// if l1 > l2 then l:=l2 else -// l:=l1; -// for i:=1 to l do -// if PpcRegExp(P1).RegExp[i] <> SearchRegExp[i] then -// begin -// Compare:=ord(PpcRegExp(P1).RegExp[i])-ord(SearchRegExp[i]); -// break; -// end; -// if i > l then Compare:=l1-l2; - Compare := stringsx.PasStrCmp(PpcRegExp(P1).RegExp, SearchRegExp, False); - end; -end; - -function TRegExpCollection.Find(ARegExp:shortstring;var P: PpcRegExp):boolean; -var I : integer; -begin - CompareModeInsert:=false; - SearchRegExp:=ARegExp; - if Search(nil,I) then - begin - P:=PpcRegExp(At(I)); - Find:=true; - end - else - begin - P:=nil; - Find:=false; - end; - CompareModeInsert:=true; -end; - -function TRegExpCollection.CheckNew(ARegExp:shortstring):PpcRegExp; -var - P : PpcRegExp; -begin - if not Find(ARegExp,P) then - begin - if Count = MaxRegExp then - AtFree(0); - P:=New(ppcRegExp,Init(ARegExp,PCRE_CASELESS,nil)); - Insert(P); - end; - CheckNew:=P; -end; - -function pcGrepMatch(WildCard, aStr: string; AOptions:integer; ALocale : Pointer): Boolean; -var - PpcRE:PpcRegExp; -begin - PpcRE:=New(ppcRegExp,Init(WildCard,AOptions,Alocale)); - pcGrepMatch:=PpcRE^.Search(pchar(AStr),Length(AStr)); - Dispose(PpcRE,Done); -end; - -function pcGrepSub(WildCard, aStr, aRepl: string; AOptions:integer; ALocale : Pointer): string; -var - PpcRE:PpcRegExp; -begin - PpcRE:=New(ppcRegExp,Init(WildCard,AOptions,Alocale)); - if PpcRE^.Search(pchar(AStr),Length(AStr)) then - pcGrepSub:=PpcRE^.GetReplStr(pchar(AStr),ARepl) - else - pcGrepSub:=''; - Dispose(PpcRE,Done); -end; - -function pcFastGrepMatch(WildCard, aStr: string): Boolean; -var - PpcRE:PpcRegExp; -begin - PpcRE:=PRegExpCache^.CheckNew(WildCard); - pcFastGrepMatch:=PpcRE^.Search(pchar(AStr),Length(AStr)); -end; - -function pcFastGrepSub(WildCard, aStr, aRepl: string): string; -var - PpcRE:PpcRegExp; -begin - PpcRE:=PRegExpCache^.CheckNew(WildCard); - if PpcRE^.Search(pchar(AStr),Length(AStr)) then - pcFastGrepSub:=PpcRE^.GetReplStr(pchar(AStr),ARepl) - else - pcFastGrepSub:=''; -end; - -{$IFDEF PCRE_5_0} -function pcGetVersion : pchar; assembler; {$FRAME-}{$USES none} -asm - call pcre_version -end; -{$ENDIF PCRE_5_0} - -function pcError; -var P: ppcRegExp absolute pRegExp; -begin - Result := (P = nil) or P^.Error; - If Result and (P <> nil) then - begin -{ if P^.ErrorPos = 0 then - MessageBox(GetString(erRegExpCompile)+'"'+P^.ErrorStr+'"', nil,mfConfirmation+mfOkButton) - else} - MessageBox(GetString(erRegExpCompile)+'"'+P^.ErrorStr+'"'+GetString(erRegExpCompPos), - @P^.ErrorPos,mfConfirmation+mfOkButton); - Dispose(P, Done); - P:=nil; - end; -end; - -function pcInit; -var Options : Integer; -begin - If CaseSens then Options := 0 else Options := PCRE_CASELESS; - Result := New( PpcRegExp, Init( Pattern, - {DefaultOptions} - startup.MiscMultiData.cfgRegEx.DefaultOptions or Options, - DefaultLocaleTable) ); -end; - -Initialization - PRegExpCache:=New(PRegExpCollection,Init(64)); -Finalization - Dispose(PRegExpCache,Done); -End. diff --git a/src/third_party/pcre-8.39/pcregrep.c b/src/third_party/pcre-8.39/pcregrep.c deleted file mode 100644 index cd53c648da2..00000000000 --- a/src/third_party/pcre-8.39/pcregrep.c +++ /dev/null @@ -1,3325 +0,0 @@ -/************************************************* -* pcregrep program * -*************************************************/ - -/* This is a grep program that uses the PCRE regular expression library to do -its pattern matching. On Unix-like, Windows, and native z/OS systems it can -recurse into directories, and in z/OS it can handle PDS files. - -Note that for native z/OS, in addition to defining the NATIVE_ZOS macro, an -additional header is required. That header is not included in the main PCRE -distribution because other apparatus is needed to compile pcregrep for z/OS. -The header can be found in the special z/OS distribution, which is available -from www.zaconsultants.net or from www.cbttape.org. - - Copyright (c) 1997-2014 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include -#include -#include -#include -#include - -#include -#include - -#ifdef HAVE_UNISTD_H -#include -#endif - -#ifdef SUPPORT_LIBZ -#include -#endif - -#ifdef SUPPORT_LIBBZ2 -#include -#endif - -#include "pcre.h" - -#define FALSE 0 -#define TRUE 1 - -typedef int BOOL; - -#define OFFSET_SIZE 99 - -#if BUFSIZ > 8192 -#define MAXPATLEN BUFSIZ -#else -#define MAXPATLEN 8192 -#endif - -#define PATBUFSIZE (MAXPATLEN + 10) /* Allows for prefix+suffix */ - -/* Values for the "filenames" variable, which specifies options for file name -output. The order is important; it is assumed that a file name is wanted for -all values greater than FN_DEFAULT. */ - -enum { FN_NONE, FN_DEFAULT, FN_MATCH_ONLY, FN_NOMATCH_ONLY, FN_FORCE }; - -/* File reading styles */ - -enum { FR_PLAIN, FR_LIBZ, FR_LIBBZ2 }; - -/* Actions for the -d and -D options */ - -enum { dee_READ, dee_SKIP, dee_RECURSE }; -enum { DEE_READ, DEE_SKIP }; - -/* Actions for special processing options (flag bits) */ - -#define PO_WORD_MATCH 0x0001 -#define PO_LINE_MATCH 0x0002 -#define PO_FIXED_STRINGS 0x0004 - -/* Line ending types */ - -enum { EL_LF, EL_CR, EL_CRLF, EL_ANY, EL_ANYCRLF }; - -/* Binary file options */ - -enum { BIN_BINARY, BIN_NOMATCH, BIN_TEXT }; - -/* In newer versions of gcc, with FORTIFY_SOURCE set (the default in some -environments), a warning is issued if the value of fwrite() is ignored. -Unfortunately, casting to (void) does not suppress the warning. To get round -this, we use a macro that compiles a fudge. Oddly, this does not also seem to -apply to fprintf(). */ - -#define FWRITE(a,b,c,d) if (fwrite(a,b,c,d)) {} - - - -/************************************************* -* Global variables * -*************************************************/ - -/* Jeffrey Friedl has some debugging requirements that are not part of the -regular code. */ - -#ifdef JFRIEDL_DEBUG -static int S_arg = -1; -static unsigned int jfriedl_XR = 0; /* repeat regex attempt this many times */ -static unsigned int jfriedl_XT = 0; /* replicate text this many times */ -static const char *jfriedl_prefix = ""; -static const char *jfriedl_postfix = ""; -#endif - -static int endlinetype; - -static char *colour_string = (char *)"1;31"; -static char *colour_option = NULL; -static char *dee_option = NULL; -static char *DEE_option = NULL; -static char *locale = NULL; -static char *main_buffer = NULL; -static char *newline = NULL; -static char *om_separator = (char *)""; -static char *stdin_name = (char *)"(standard input)"; - -static const unsigned char *pcretables = NULL; - -static int after_context = 0; -static int before_context = 0; -static int binary_files = BIN_BINARY; -static int both_context = 0; -static int bufthird = PCREGREP_BUFSIZE; -static int bufsize = 3*PCREGREP_BUFSIZE; - -#if defined HAVE_WINDOWS_H && HAVE_WINDOWS_H -static int dee_action = dee_SKIP; -#else -static int dee_action = dee_READ; -#endif - -static int DEE_action = DEE_READ; -static int error_count = 0; -static int filenames = FN_DEFAULT; -static int pcre_options = 0; -static int process_options = 0; - -#ifdef SUPPORT_PCREGREP_JIT -static int study_options = PCRE_STUDY_JIT_COMPILE; -#else -static int study_options = 0; -#endif - -static unsigned long int match_limit = 0; -static unsigned long int match_limit_recursion = 0; - -static BOOL count_only = FALSE; -static BOOL do_colour = FALSE; -static BOOL file_offsets = FALSE; -static BOOL hyphenpending = FALSE; -static BOOL invert = FALSE; -static BOOL line_buffered = FALSE; -static BOOL line_offsets = FALSE; -static BOOL multiline = FALSE; -static BOOL number = FALSE; -static BOOL omit_zero_count = FALSE; -static BOOL resource_error = FALSE; -static BOOL quiet = FALSE; -static BOOL show_only_matching = FALSE; -static BOOL silent = FALSE; -static BOOL utf8 = FALSE; - -/* Structure for list of --only-matching capturing numbers. */ - -typedef struct omstr { - struct omstr *next; - int groupnum; -} omstr; - -static omstr *only_matching = NULL; -static omstr *only_matching_last = NULL; - -/* Structure for holding the two variables that describe a number chain. */ - -typedef struct omdatastr { - omstr **anchor; - omstr **lastptr; -} omdatastr; - -static omdatastr only_matching_data = { &only_matching, &only_matching_last }; - -/* Structure for list of file names (for -f and --{in,ex}clude-from) */ - -typedef struct fnstr { - struct fnstr *next; - char *name; -} fnstr; - -static fnstr *exclude_from = NULL; -static fnstr *exclude_from_last = NULL; -static fnstr *include_from = NULL; -static fnstr *include_from_last = NULL; - -static fnstr *file_lists = NULL; -static fnstr *file_lists_last = NULL; -static fnstr *pattern_files = NULL; -static fnstr *pattern_files_last = NULL; - -/* Structure for holding the two variables that describe a file name chain. */ - -typedef struct fndatastr { - fnstr **anchor; - fnstr **lastptr; -} fndatastr; - -static fndatastr exclude_from_data = { &exclude_from, &exclude_from_last }; -static fndatastr include_from_data = { &include_from, &include_from_last }; -static fndatastr file_lists_data = { &file_lists, &file_lists_last }; -static fndatastr pattern_files_data = { &pattern_files, &pattern_files_last }; - -/* Structure for pattern and its compiled form; used for matching patterns and -also for include/exclude patterns. */ - -typedef struct patstr { - struct patstr *next; - char *string; - pcre *compiled; - pcre_extra *hint; -} patstr; - -static patstr *patterns = NULL; -static patstr *patterns_last = NULL; -static patstr *include_patterns = NULL; -static patstr *include_patterns_last = NULL; -static patstr *exclude_patterns = NULL; -static patstr *exclude_patterns_last = NULL; -static patstr *include_dir_patterns = NULL; -static patstr *include_dir_patterns_last = NULL; -static patstr *exclude_dir_patterns = NULL; -static patstr *exclude_dir_patterns_last = NULL; - -/* Structure holding the two variables that describe a pattern chain. A pointer -to such structures is used for each appropriate option. */ - -typedef struct patdatastr { - patstr **anchor; - patstr **lastptr; -} patdatastr; - -static patdatastr match_patdata = { &patterns, &patterns_last }; -static patdatastr include_patdata = { &include_patterns, &include_patterns_last }; -static patdatastr exclude_patdata = { &exclude_patterns, &exclude_patterns_last }; -static patdatastr include_dir_patdata = { &include_dir_patterns, &include_dir_patterns_last }; -static patdatastr exclude_dir_patdata = { &exclude_dir_patterns, &exclude_dir_patterns_last }; - -static patstr **incexlist[4] = { &include_patterns, &exclude_patterns, - &include_dir_patterns, &exclude_dir_patterns }; - -static const char *incexname[4] = { "--include", "--exclude", - "--include-dir", "--exclude-dir" }; - -/* Structure for options and list of them */ - -enum { OP_NODATA, OP_STRING, OP_OP_STRING, OP_NUMBER, OP_LONGNUMBER, - OP_OP_NUMBER, OP_OP_NUMBERS, OP_PATLIST, OP_FILELIST, OP_BINFILES }; - -typedef struct option_item { - int type; - int one_char; - void *dataptr; - const char *long_name; - const char *help_text; -} option_item; - -/* Options without a single-letter equivalent get a negative value. This can be -used to identify them. */ - -#define N_COLOUR (-1) -#define N_EXCLUDE (-2) -#define N_EXCLUDE_DIR (-3) -#define N_HELP (-4) -#define N_INCLUDE (-5) -#define N_INCLUDE_DIR (-6) -#define N_LABEL (-7) -#define N_LOCALE (-8) -#define N_NULL (-9) -#define N_LOFFSETS (-10) -#define N_FOFFSETS (-11) -#define N_LBUFFER (-12) -#define N_M_LIMIT (-13) -#define N_M_LIMIT_REC (-14) -#define N_BUFSIZE (-15) -#define N_NOJIT (-16) -#define N_FILE_LIST (-17) -#define N_BINARY_FILES (-18) -#define N_EXCLUDE_FROM (-19) -#define N_INCLUDE_FROM (-20) -#define N_OM_SEPARATOR (-21) - -static option_item optionlist[] = { - { OP_NODATA, N_NULL, NULL, "", "terminate options" }, - { OP_NODATA, N_HELP, NULL, "help", "display this help and exit" }, - { OP_NUMBER, 'A', &after_context, "after-context=number", "set number of following context lines" }, - { OP_NODATA, 'a', NULL, "text", "treat binary files as text" }, - { OP_NUMBER, 'B', &before_context, "before-context=number", "set number of prior context lines" }, - { OP_BINFILES, N_BINARY_FILES, NULL, "binary-files=word", "set treatment of binary files" }, - { OP_NUMBER, N_BUFSIZE,&bufthird, "buffer-size=number", "set processing buffer size parameter" }, - { OP_OP_STRING, N_COLOUR, &colour_option, "color=option", "matched text color option" }, - { OP_OP_STRING, N_COLOUR, &colour_option, "colour=option", "matched text colour option" }, - { OP_NUMBER, 'C', &both_context, "context=number", "set number of context lines, before & after" }, - { OP_NODATA, 'c', NULL, "count", "print only a count of matching lines per FILE" }, - { OP_STRING, 'D', &DEE_option, "devices=action","how to handle devices, FIFOs, and sockets" }, - { OP_STRING, 'd', &dee_option, "directories=action", "how to handle directories" }, - { OP_PATLIST, 'e', &match_patdata, "regex(p)=pattern", "specify pattern (may be used more than once)" }, - { OP_NODATA, 'F', NULL, "fixed-strings", "patterns are sets of newline-separated strings" }, - { OP_FILELIST, 'f', &pattern_files_data, "file=path", "read patterns from file" }, - { OP_FILELIST, N_FILE_LIST, &file_lists_data, "file-list=path","read files to search from file" }, - { OP_NODATA, N_FOFFSETS, NULL, "file-offsets", "output file offsets, not text" }, - { OP_NODATA, 'H', NULL, "with-filename", "force the prefixing filename on output" }, - { OP_NODATA, 'h', NULL, "no-filename", "suppress the prefixing filename on output" }, - { OP_NODATA, 'I', NULL, "", "treat binary files as not matching (ignore)" }, - { OP_NODATA, 'i', NULL, "ignore-case", "ignore case distinctions" }, -#ifdef SUPPORT_PCREGREP_JIT - { OP_NODATA, N_NOJIT, NULL, "no-jit", "do not use just-in-time compiler optimization" }, -#else - { OP_NODATA, N_NOJIT, NULL, "no-jit", "ignored: this pcregrep does not support JIT" }, -#endif - { OP_NODATA, 'l', NULL, "files-with-matches", "print only FILE names containing matches" }, - { OP_NODATA, 'L', NULL, "files-without-match","print only FILE names not containing matches" }, - { OP_STRING, N_LABEL, &stdin_name, "label=name", "set name for standard input" }, - { OP_NODATA, N_LBUFFER, NULL, "line-buffered", "use line buffering" }, - { OP_NODATA, N_LOFFSETS, NULL, "line-offsets", "output line numbers and offsets, not text" }, - { OP_STRING, N_LOCALE, &locale, "locale=locale", "use the named locale" }, - { OP_LONGNUMBER, N_M_LIMIT, &match_limit, "match-limit=number", "set PCRE match limit option" }, - { OP_LONGNUMBER, N_M_LIMIT_REC, &match_limit_recursion, "recursion-limit=number", "set PCRE match recursion limit option" }, - { OP_NODATA, 'M', NULL, "multiline", "run in multiline mode" }, - { OP_STRING, 'N', &newline, "newline=type", "set newline type (CR, LF, CRLF, ANYCRLF or ANY)" }, - { OP_NODATA, 'n', NULL, "line-number", "print line number with output lines" }, - { OP_OP_NUMBERS, 'o', &only_matching_data, "only-matching=n", "show only the part of the line that matched" }, - { OP_STRING, N_OM_SEPARATOR, &om_separator, "om-separator=text", "set separator for multiple -o output" }, - { OP_NODATA, 'q', NULL, "quiet", "suppress output, just set return code" }, - { OP_NODATA, 'r', NULL, "recursive", "recursively scan sub-directories" }, - { OP_PATLIST, N_EXCLUDE,&exclude_patdata, "exclude=pattern","exclude matching files when recursing" }, - { OP_PATLIST, N_INCLUDE,&include_patdata, "include=pattern","include matching files when recursing" }, - { OP_PATLIST, N_EXCLUDE_DIR,&exclude_dir_patdata, "exclude-dir=pattern","exclude matching directories when recursing" }, - { OP_PATLIST, N_INCLUDE_DIR,&include_dir_patdata, "include-dir=pattern","include matching directories when recursing" }, - { OP_FILELIST, N_EXCLUDE_FROM,&exclude_from_data, "exclude-from=path", "read exclude list from file" }, - { OP_FILELIST, N_INCLUDE_FROM,&include_from_data, "include-from=path", "read include list from file" }, - - /* These two were accidentally implemented with underscores instead of - hyphens in the option names. As this was not discovered for several releases, - the incorrect versions are left in the table for compatibility. However, the - --help function misses out any option that has an underscore in its name. */ - - { OP_PATLIST, N_EXCLUDE_DIR,&exclude_dir_patdata, "exclude_dir=pattern","exclude matching directories when recursing" }, - { OP_PATLIST, N_INCLUDE_DIR,&include_dir_patdata, "include_dir=pattern","include matching directories when recursing" }, - -#ifdef JFRIEDL_DEBUG - { OP_OP_NUMBER, 'S', &S_arg, "jeffS", "replace matched (sub)string with X" }, -#endif - { OP_NODATA, 's', NULL, "no-messages", "suppress error messages" }, - { OP_NODATA, 'u', NULL, "utf-8", "use UTF-8 mode" }, - { OP_NODATA, 'V', NULL, "version", "print version information and exit" }, - { OP_NODATA, 'v', NULL, "invert-match", "select non-matching lines" }, - { OP_NODATA, 'w', NULL, "word-regex(p)", "force patterns to match only as words" }, - { OP_NODATA, 'x', NULL, "line-regex(p)", "force patterns to match only whole lines" }, - { OP_NODATA, 0, NULL, NULL, NULL } -}; - -/* Tables for prefixing and suffixing patterns, according to the -w, -x, and -F -options. These set the 1, 2, and 4 bits in process_options, respectively. Note -that the combination of -w and -x has the same effect as -x on its own, so we -can treat them as the same. Note that the MAXPATLEN macro assumes the longest -prefix+suffix is 10 characters; if anything longer is added, it must be -adjusted. */ - -static const char *prefix[] = { - "", "\\b", "^(?:", "^(?:", "\\Q", "\\b\\Q", "^(?:\\Q", "^(?:\\Q" }; - -static const char *suffix[] = { - "", "\\b", ")$", ")$", "\\E", "\\E\\b", "\\E)$", "\\E)$" }; - -/* UTF-8 tables - used only when the newline setting is "any". */ - -const int utf8_table3[] = { 0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01}; - -const char utf8_table4[] = { - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, - 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 }; - - - -/************************************************* -* Exit from the program * -*************************************************/ - -/* If there has been a resource error, give a suitable message. - -Argument: the return code -Returns: does not return -*/ - -static void -pcregrep_exit(int rc) -{ -if (resource_error) - { - fprintf(stderr, "pcregrep: Error %d, %d or %d means that a resource limit " - "was exceeded.\n", PCRE_ERROR_MATCHLIMIT, PCRE_ERROR_RECURSIONLIMIT, - PCRE_ERROR_JIT_STACKLIMIT); - fprintf(stderr, "pcregrep: Check your regex for nested unlimited loops.\n"); - } -exit(rc); -} - - -/************************************************* -* Add item to chain of patterns * -*************************************************/ - -/* Used to add an item onto a chain, or just return an unconnected item if the -"after" argument is NULL. - -Arguments: - s pattern string to add - after if not NULL points to item to insert after - -Returns: new pattern block or NULL on error -*/ - -static patstr * -add_pattern(char *s, patstr *after) -{ -patstr *p = (patstr *)malloc(sizeof(patstr)); -if (p == NULL) - { - fprintf(stderr, "pcregrep: malloc failed\n"); - pcregrep_exit(2); - } -if (strlen(s) > MAXPATLEN) - { - fprintf(stderr, "pcregrep: pattern is too long (limit is %d bytes)\n", - MAXPATLEN); - free(p); - return NULL; - } -p->next = NULL; -p->string = s; -p->compiled = NULL; -p->hint = NULL; - -if (after != NULL) - { - p->next = after->next; - after->next = p; - } -return p; -} - - -/************************************************* -* Free chain of patterns * -*************************************************/ - -/* Used for several chains of patterns. - -Argument: pointer to start of chain -Returns: nothing -*/ - -static void -free_pattern_chain(patstr *pc) -{ -while (pc != NULL) - { - patstr *p = pc; - pc = p->next; - if (p->hint != NULL) pcre_free_study(p->hint); - if (p->compiled != NULL) pcre_free(p->compiled); - free(p); - } -} - - -/************************************************* -* Free chain of file names * -*************************************************/ - -/* -Argument: pointer to start of chain -Returns: nothing -*/ - -static void -free_file_chain(fnstr *fn) -{ -while (fn != NULL) - { - fnstr *f = fn; - fn = f->next; - free(f); - } -} - - -/************************************************* -* OS-specific functions * -*************************************************/ - -/* These functions are defined so that they can be made system specific. -At present there are versions for Unix-style environments, Windows, native -z/OS, and "no support". */ - - -/************* Directory scanning Unix-style and z/OS ***********/ - -#if (defined HAVE_SYS_STAT_H && defined HAVE_DIRENT_H && defined HAVE_SYS_TYPES_H) || defined NATIVE_ZOS -#include -#include -#include - -#if defined NATIVE_ZOS -/************* Directory and PDS/E scanning for z/OS ***********/ -/************* z/OS looks mostly like Unix with USS ************/ -/* However, z/OS needs the #include statements in this header */ -#include "pcrzosfs.h" -/* That header is not included in the main PCRE distribution because - other apparatus is needed to compile pcregrep for z/OS. The header - can be found in the special z/OS distribution, which is available - from www.zaconsultants.net or from www.cbttape.org. */ -#endif - -typedef DIR directory_type; -#define FILESEP '/' - -static int -isdirectory(char *filename) -{ -struct stat statbuf; -if (stat(filename, &statbuf) < 0) - return 0; /* In the expectation that opening as a file will fail */ -return (statbuf.st_mode & S_IFMT) == S_IFDIR; -} - -static directory_type * -opendirectory(char *filename) -{ -return opendir(filename); -} - -static char * -readdirectory(directory_type *dir) -{ -for (;;) - { - struct dirent *dent = readdir(dir); - if (dent == NULL) return NULL; - if (strcmp(dent->d_name, ".") != 0 && strcmp(dent->d_name, "..") != 0) - return dent->d_name; - } -/* Control never reaches here */ -} - -static void -closedirectory(directory_type *dir) -{ -closedir(dir); -} - - -/************* Test for regular file, Unix-style **********/ - -static int -isregfile(char *filename) -{ -struct stat statbuf; -if (stat(filename, &statbuf) < 0) - return 1; /* In the expectation that opening as a file will fail */ -return (statbuf.st_mode & S_IFMT) == S_IFREG; -} - - -#if defined NATIVE_ZOS -/************* Test for a terminal in z/OS **********/ -/* isatty() does not work in a TSO environment, so always give FALSE.*/ - -static BOOL -is_stdout_tty(void) -{ -return FALSE; -} - -static BOOL -is_file_tty(FILE *f) -{ -return FALSE; -} - - -/************* Test for a terminal, Unix-style **********/ - -#else -static BOOL -is_stdout_tty(void) -{ -return isatty(fileno(stdout)); -} - -static BOOL -is_file_tty(FILE *f) -{ -return isatty(fileno(f)); -} -#endif - -/* End of Unix-style or native z/OS environment functions. */ - - -/************* Directory scanning in Windows ***********/ - -/* I (Philip Hazel) have no means of testing this code. It was contributed by -Lionel Fourquaux. David Burgess added a patch to define INVALID_FILE_ATTRIBUTES -when it did not exist. David Byron added a patch that moved the #include of - to before the INVALID_FILE_ATTRIBUTES definition rather than after. -The double test below stops gcc 4.4.4 grumbling that HAVE_WINDOWS_H is -undefined when it is indeed undefined. */ - -#elif defined HAVE_WINDOWS_H && HAVE_WINDOWS_H - -#ifndef STRICT -# define STRICT -#endif -#ifndef WIN32_LEAN_AND_MEAN -# define WIN32_LEAN_AND_MEAN -#endif - -#include - -#ifndef INVALID_FILE_ATTRIBUTES -#define INVALID_FILE_ATTRIBUTES 0xFFFFFFFF -#endif - -typedef struct directory_type -{ -HANDLE handle; -BOOL first; -WIN32_FIND_DATA data; -} directory_type; - -#define FILESEP '/' - -int -isdirectory(char *filename) -{ -DWORD attr = GetFileAttributes(filename); -if (attr == INVALID_FILE_ATTRIBUTES) - return 0; -return (attr & FILE_ATTRIBUTE_DIRECTORY) != 0; -} - -directory_type * -opendirectory(char *filename) -{ -size_t len; -char *pattern; -directory_type *dir; -DWORD err; -len = strlen(filename); -pattern = (char *)malloc(len + 3); -dir = (directory_type *)malloc(sizeof(*dir)); -if ((pattern == NULL) || (dir == NULL)) - { - fprintf(stderr, "pcregrep: malloc failed\n"); - pcregrep_exit(2); - } -memcpy(pattern, filename, len); -memcpy(&(pattern[len]), "\\*", 3); -dir->handle = FindFirstFile(pattern, &(dir->data)); -if (dir->handle != INVALID_HANDLE_VALUE) - { - free(pattern); - dir->first = TRUE; - return dir; - } -err = GetLastError(); -free(pattern); -free(dir); -errno = (err == ERROR_ACCESS_DENIED) ? EACCES : ENOENT; -return NULL; -} - -char * -readdirectory(directory_type *dir) -{ -for (;;) - { - if (!dir->first) - { - if (!FindNextFile(dir->handle, &(dir->data))) - return NULL; - } - else - { - dir->first = FALSE; - } - if (strcmp(dir->data.cFileName, ".") != 0 && strcmp(dir->data.cFileName, "..") != 0) - return dir->data.cFileName; - } -#ifndef _MSC_VER -return NULL; /* Keep compiler happy; never executed */ -#endif -} - -void -closedirectory(directory_type *dir) -{ -FindClose(dir->handle); -free(dir); -} - - -/************* Test for regular file in Windows **********/ - -/* I don't know how to do this, or if it can be done; assume all paths are -regular if they are not directories. */ - -int isregfile(char *filename) -{ -return !isdirectory(filename); -} - - -/************* Test for a terminal in Windows **********/ - -/* I don't know how to do this; assume never */ - -static BOOL -is_stdout_tty(void) -{ -return FALSE; -} - -static BOOL -is_file_tty(FILE *f) -{ -return FALSE; -} - -/* End of Windows functions */ - - -/************* Directory scanning when we can't do it ***********/ - -/* The type is void, and apart from isdirectory(), the functions do nothing. */ - -#else - -#define FILESEP 0 -typedef void directory_type; - -int isdirectory(char *filename) { return 0; } -directory_type * opendirectory(char *filename) { return (directory_type*)0;} -char *readdirectory(directory_type *dir) { return (char*)0;} -void closedirectory(directory_type *dir) {} - - -/************* Test for regular file when we can't do it **********/ - -/* Assume all files are regular. */ - -int isregfile(char *filename) { return 1; } - - -/************* Test for a terminal when we can't do it **********/ - -static BOOL -is_stdout_tty(void) -{ -return FALSE; -} - -static BOOL -is_file_tty(FILE *f) -{ -return FALSE; -} - -#endif /* End of system-specific functions */ - - - -#ifndef HAVE_STRERROR -/************************************************* -* Provide strerror() for non-ANSI libraries * -*************************************************/ - -/* Some old-fashioned systems still around (e.g. SunOS4) don't have strerror() -in their libraries, but can provide the same facility by this simple -alternative function. */ - -extern int sys_nerr; -extern char *sys_errlist[]; - -char * -strerror(int n) -{ -if (n < 0 || n >= sys_nerr) return "unknown error number"; -return sys_errlist[n]; -} -#endif /* HAVE_STRERROR */ - - - -/************************************************* -* Usage function * -*************************************************/ - -static int -usage(int rc) -{ -option_item *op; -fprintf(stderr, "Usage: pcregrep [-"); -for (op = optionlist; op->one_char != 0; op++) - { - if (op->one_char > 0) fprintf(stderr, "%c", op->one_char); - } -fprintf(stderr, "] [long options] [pattern] [files]\n"); -fprintf(stderr, "Type `pcregrep --help' for more information and the long " - "options.\n"); -return rc; -} - - - -/************************************************* -* Help function * -*************************************************/ - -static void -help(void) -{ -option_item *op; - -printf("Usage: pcregrep [OPTION]... [PATTERN] [FILE1 FILE2 ...]\n"); -printf("Search for PATTERN in each FILE or standard input.\n"); -printf("PATTERN must be present if neither -e nor -f is used.\n"); -printf("\"-\" can be used as a file name to mean STDIN.\n"); - -#ifdef SUPPORT_LIBZ -printf("Files whose names end in .gz are read using zlib.\n"); -#endif - -#ifdef SUPPORT_LIBBZ2 -printf("Files whose names end in .bz2 are read using bzlib2.\n"); -#endif - -#if defined SUPPORT_LIBZ || defined SUPPORT_LIBBZ2 -printf("Other files and the standard input are read as plain files.\n\n"); -#else -printf("All files are read as plain files, without any interpretation.\n\n"); -#endif - -printf("Example: pcregrep -i 'hello.*world' menu.h main.c\n\n"); -printf("Options:\n"); - -for (op = optionlist; op->one_char != 0; op++) - { - int n; - char s[4]; - - /* Two options were accidentally implemented and documented with underscores - instead of hyphens in their names, something that was not noticed for quite a - few releases. When fixing this, I left the underscored versions in the list - in case people were using them. However, we don't want to display them in the - help data. There are no other options that contain underscores, and we do not - expect ever to implement such options. Therefore, just omit any option that - contains an underscore. */ - - if (strchr(op->long_name, '_') != NULL) continue; - - if (op->one_char > 0 && (op->long_name)[0] == 0) - n = 31 - printf(" -%c", op->one_char); - else - { - if (op->one_char > 0) sprintf(s, "-%c,", op->one_char); - else strcpy(s, " "); - n = 31 - printf(" %s --%s", s, op->long_name); - } - - if (n < 1) n = 1; - printf("%.*s%s\n", n, " ", op->help_text); - } - -printf("\nNumbers may be followed by K or M, e.g. --buffer-size=100K.\n"); -printf("The default value for --buffer-size is %d.\n", PCREGREP_BUFSIZE); -printf("When reading patterns or file names from a file, trailing white\n"); -printf("space is removed and blank lines are ignored.\n"); -printf("The maximum size of any pattern is %d bytes.\n", MAXPATLEN); - -printf("\nWith no FILEs, read standard input. If fewer than two FILEs given, assume -h.\n"); -printf("Exit status is 0 if any matches, 1 if no matches, and 2 if trouble.\n"); -} - - - -/************************************************* -* Test exclude/includes * -*************************************************/ - -/* If any exclude pattern matches, the path is excluded. Otherwise, unless -there are no includes, the path must match an include pattern. - -Arguments: - path the path to be matched - ip the chain of include patterns - ep the chain of exclude patterns - -Returns: TRUE if the path is not excluded -*/ - -static BOOL -test_incexc(char *path, patstr *ip, patstr *ep) -{ -int plen = strlen(path); - -for (; ep != NULL; ep = ep->next) - { - if (pcre_exec(ep->compiled, NULL, path, plen, 0, 0, NULL, 0) >= 0) - return FALSE; - } - -if (ip == NULL) return TRUE; - -for (; ip != NULL; ip = ip->next) - { - if (pcre_exec(ip->compiled, NULL, path, plen, 0, 0, NULL, 0) >= 0) - return TRUE; - } - -return FALSE; -} - - - -/************************************************* -* Decode integer argument value * -*************************************************/ - -/* Integer arguments can be followed by K or M. Avoid the use of strtoul() -because SunOS4 doesn't have it. This is used only for unpicking arguments, so -just keep it simple. - -Arguments: - option_data the option data string - op the option item (for error messages) - longop TRUE if option given in long form - -Returns: a long integer -*/ - -static long int -decode_number(char *option_data, option_item *op, BOOL longop) -{ -unsigned long int n = 0; -char *endptr = option_data; -while (*endptr != 0 && isspace((unsigned char)(*endptr))) endptr++; -while (isdigit((unsigned char)(*endptr))) - n = n * 10 + (int)(*endptr++ - '0'); -if (toupper(*endptr) == 'K') - { - n *= 1024; - endptr++; - } -else if (toupper(*endptr) == 'M') - { - n *= 1024*1024; - endptr++; - } - -if (*endptr != 0) /* Error */ - { - if (longop) - { - char *equals = strchr(op->long_name, '='); - int nlen = (equals == NULL)? (int)strlen(op->long_name) : - (int)(equals - op->long_name); - fprintf(stderr, "pcregrep: Malformed number \"%s\" after --%.*s\n", - option_data, nlen, op->long_name); - } - else - fprintf(stderr, "pcregrep: Malformed number \"%s\" after -%c\n", - option_data, op->one_char); - pcregrep_exit(usage(2)); - } - -return n; -} - - - -/************************************************* -* Add item to a chain of numbers * -*************************************************/ - -/* Used to add an item onto a chain, or just return an unconnected item if the -"after" argument is NULL. - -Arguments: - n the number to add - after if not NULL points to item to insert after - -Returns: new number block -*/ - -static omstr * -add_number(int n, omstr *after) -{ -omstr *om = (omstr *)malloc(sizeof(omstr)); - -if (om == NULL) - { - fprintf(stderr, "pcregrep: malloc failed\n"); - pcregrep_exit(2); - } -om->next = NULL; -om->groupnum = n; - -if (after != NULL) - { - om->next = after->next; - after->next = om; - } -return om; -} - - - -/************************************************* -* Read one line of input * -*************************************************/ - -/* Normally, input is read using fread() into a large buffer, so many lines may -be read at once. However, doing this for tty input means that no output appears -until a lot of input has been typed. Instead, tty input is handled line by -line. We cannot use fgets() for this, because it does not stop at a binary -zero, and therefore there is no way of telling how many characters it has read, -because there may be binary zeros embedded in the data. - -Arguments: - buffer the buffer to read into - length the maximum number of characters to read - f the file - -Returns: the number of characters read, zero at end of file -*/ - -static unsigned int -read_one_line(char *buffer, int length, FILE *f) -{ -int c; -int yield = 0; -while ((c = fgetc(f)) != EOF) - { - buffer[yield++] = c; - if (c == '\n' || yield >= length) break; - } -return yield; -} - - - -/************************************************* -* Find end of line * -*************************************************/ - -/* The length of the endline sequence that is found is set via lenptr. This may -be zero at the very end of the file if there is no line-ending sequence there. - -Arguments: - p current position in line - endptr end of available data - lenptr where to put the length of the eol sequence - -Returns: pointer after the last byte of the line, - including the newline byte(s) -*/ - -static char * -end_of_line(char *p, char *endptr, int *lenptr) -{ -switch(endlinetype) - { - default: /* Just in case */ - case EL_LF: - while (p < endptr && *p != '\n') p++; - if (p < endptr) - { - *lenptr = 1; - return p + 1; - } - *lenptr = 0; - return endptr; - - case EL_CR: - while (p < endptr && *p != '\r') p++; - if (p < endptr) - { - *lenptr = 1; - return p + 1; - } - *lenptr = 0; - return endptr; - - case EL_CRLF: - for (;;) - { - while (p < endptr && *p != '\r') p++; - if (++p >= endptr) - { - *lenptr = 0; - return endptr; - } - if (*p == '\n') - { - *lenptr = 2; - return p + 1; - } - } - break; - - case EL_ANYCRLF: - while (p < endptr) - { - int extra = 0; - register int c = *((unsigned char *)p); - - if (utf8 && c >= 0xc0) - { - int gcii, gcss; - extra = utf8_table4[c & 0x3f]; /* Number of additional bytes */ - gcss = 6*extra; - c = (c & utf8_table3[extra]) << gcss; - for (gcii = 1; gcii <= extra; gcii++) - { - gcss -= 6; - c |= (p[gcii] & 0x3f) << gcss; - } - } - - p += 1 + extra; - - switch (c) - { - case '\n': - *lenptr = 1; - return p; - - case '\r': - if (p < endptr && *p == '\n') - { - *lenptr = 2; - p++; - } - else *lenptr = 1; - return p; - - default: - break; - } - } /* End of loop for ANYCRLF case */ - - *lenptr = 0; /* Must have hit the end */ - return endptr; - - case EL_ANY: - while (p < endptr) - { - int extra = 0; - register int c = *((unsigned char *)p); - - if (utf8 && c >= 0xc0) - { - int gcii, gcss; - extra = utf8_table4[c & 0x3f]; /* Number of additional bytes */ - gcss = 6*extra; - c = (c & utf8_table3[extra]) << gcss; - for (gcii = 1; gcii <= extra; gcii++) - { - gcss -= 6; - c |= (p[gcii] & 0x3f) << gcss; - } - } - - p += 1 + extra; - - switch (c) - { - case '\n': /* LF */ - case '\v': /* VT */ - case '\f': /* FF */ - *lenptr = 1; - return p; - - case '\r': /* CR */ - if (p < endptr && *p == '\n') - { - *lenptr = 2; - p++; - } - else *lenptr = 1; - return p; - -#ifndef EBCDIC - case 0x85: /* Unicode NEL */ - *lenptr = utf8? 2 : 1; - return p; - - case 0x2028: /* Unicode LS */ - case 0x2029: /* Unicode PS */ - *lenptr = 3; - return p; -#endif /* Not EBCDIC */ - - default: - break; - } - } /* End of loop for ANY case */ - - *lenptr = 0; /* Must have hit the end */ - return endptr; - } /* End of overall switch */ -} - - - -/************************************************* -* Find start of previous line * -*************************************************/ - -/* This is called when looking back for before lines to print. - -Arguments: - p start of the subsequent line - startptr start of available data - -Returns: pointer to the start of the previous line -*/ - -static char * -previous_line(char *p, char *startptr) -{ -switch(endlinetype) - { - default: /* Just in case */ - case EL_LF: - p--; - while (p > startptr && p[-1] != '\n') p--; - return p; - - case EL_CR: - p--; - while (p > startptr && p[-1] != '\n') p--; - return p; - - case EL_CRLF: - for (;;) - { - p -= 2; - while (p > startptr && p[-1] != '\n') p--; - if (p <= startptr + 1 || p[-2] == '\r') return p; - } - /* Control can never get here */ - - case EL_ANY: - case EL_ANYCRLF: - if (*(--p) == '\n' && p > startptr && p[-1] == '\r') p--; - if (utf8) while ((*p & 0xc0) == 0x80) p--; - - while (p > startptr) - { - register unsigned int c; - char *pp = p - 1; - - if (utf8) - { - int extra = 0; - while ((*pp & 0xc0) == 0x80) pp--; - c = *((unsigned char *)pp); - if (c >= 0xc0) - { - int gcii, gcss; - extra = utf8_table4[c & 0x3f]; /* Number of additional bytes */ - gcss = 6*extra; - c = (c & utf8_table3[extra]) << gcss; - for (gcii = 1; gcii <= extra; gcii++) - { - gcss -= 6; - c |= (pp[gcii] & 0x3f) << gcss; - } - } - } - else c = *((unsigned char *)pp); - - if (endlinetype == EL_ANYCRLF) switch (c) - { - case '\n': /* LF */ - case '\r': /* CR */ - return p; - - default: - break; - } - - else switch (c) - { - case '\n': /* LF */ - case '\v': /* VT */ - case '\f': /* FF */ - case '\r': /* CR */ -#ifndef EBCDIE - case 0x85: /* Unicode NEL */ - case 0x2028: /* Unicode LS */ - case 0x2029: /* Unicode PS */ -#endif /* Not EBCDIC */ - return p; - - default: - break; - } - - p = pp; /* Back one character */ - } /* End of loop for ANY case */ - - return startptr; /* Hit start of data */ - } /* End of overall switch */ -} - - - - - -/************************************************* -* Print the previous "after" lines * -*************************************************/ - -/* This is called if we are about to lose said lines because of buffer filling, -and at the end of the file. The data in the line is written using fwrite() so -that a binary zero does not terminate it. - -Arguments: - lastmatchnumber the number of the last matching line, plus one - lastmatchrestart where we restarted after the last match - endptr end of available data - printname filename for printing - -Returns: nothing -*/ - -static void -do_after_lines(int lastmatchnumber, char *lastmatchrestart, char *endptr, - char *printname) -{ -if (after_context > 0 && lastmatchnumber > 0) - { - int count = 0; - while (lastmatchrestart < endptr && count++ < after_context) - { - int ellength; - char *pp = lastmatchrestart; - if (printname != NULL) fprintf(stdout, "%s-", printname); - if (number) fprintf(stdout, "%d-", lastmatchnumber++); - pp = end_of_line(pp, endptr, &ellength); - FWRITE(lastmatchrestart, 1, pp - lastmatchrestart, stdout); - lastmatchrestart = pp; - } - hyphenpending = TRUE; - } -} - - - -/************************************************* -* Apply patterns to subject till one matches * -*************************************************/ - -/* This function is called to run through all patterns, looking for a match. It -is used multiple times for the same subject when colouring is enabled, in order -to find all possible matches. - -Arguments: - matchptr the start of the subject - length the length of the subject to match - options options for pcre_exec - startoffset where to start matching - offsets the offets vector to fill in - mrc address of where to put the result of pcre_exec() - -Returns: TRUE if there was a match - FALSE if there was no match - invert if there was a non-fatal error -*/ - -static BOOL -match_patterns(char *matchptr, size_t length, unsigned int options, - int startoffset, int *offsets, int *mrc) -{ -int i; -size_t slen = length; -patstr *p = patterns; -const char *msg = "this text:\n\n"; - -if (slen > 200) - { - slen = 200; - msg = "text that starts:\n\n"; - } -for (i = 1; p != NULL; p = p->next, i++) - { - *mrc = pcre_exec(p->compiled, p->hint, matchptr, (int)length, - startoffset, options, offsets, OFFSET_SIZE); - if (*mrc >= 0) return TRUE; - if (*mrc == PCRE_ERROR_NOMATCH) continue; - fprintf(stderr, "pcregrep: pcre_exec() gave error %d while matching ", *mrc); - if (patterns->next != NULL) fprintf(stderr, "pattern number %d to ", i); - fprintf(stderr, "%s", msg); - FWRITE(matchptr, 1, slen, stderr); /* In case binary zero included */ - fprintf(stderr, "\n\n"); - if (*mrc == PCRE_ERROR_MATCHLIMIT || *mrc == PCRE_ERROR_RECURSIONLIMIT || - *mrc == PCRE_ERROR_JIT_STACKLIMIT) - resource_error = TRUE; - if (error_count++ > 20) - { - fprintf(stderr, "pcregrep: Too many errors - abandoned.\n"); - pcregrep_exit(2); - } - return invert; /* No more matching; don't show the line again */ - } - -return FALSE; /* No match, no errors */ -} - - - -/************************************************* -* Grep an individual file * -*************************************************/ - -/* This is called from grep_or_recurse() below. It uses a buffer that is three -times the value of bufthird. The matching point is never allowed to stray into -the top third of the buffer, thus keeping more of the file available for -context printing or for multiline scanning. For large files, the pointer will -be in the middle third most of the time, so the bottom third is available for -"before" context printing. - -Arguments: - handle the fopened FILE stream for a normal file - the gzFile pointer when reading is via libz - the BZFILE pointer when reading is via libbz2 - frtype FR_PLAIN, FR_LIBZ, or FR_LIBBZ2 - filename the file name or NULL (for errors) - printname the file name if it is to be printed for each match - or NULL if the file name is not to be printed - it cannot be NULL if filenames[_nomatch]_only is set - -Returns: 0 if there was at least one match - 1 otherwise (no matches) - 2 if an overlong line is encountered - 3 if there is a read error on a .bz2 file -*/ - -static int -pcregrep(void *handle, int frtype, char *filename, char *printname) -{ -int rc = 1; -int linenumber = 1; -int lastmatchnumber = 0; -int count = 0; -int filepos = 0; -int offsets[OFFSET_SIZE]; -char *lastmatchrestart = NULL; -char *ptr = main_buffer; -char *endptr; -size_t bufflength; -BOOL binary = FALSE; -BOOL endhyphenpending = FALSE; -BOOL input_line_buffered = line_buffered; -FILE *in = NULL; /* Ensure initialized */ - -#ifdef SUPPORT_LIBZ -gzFile ingz = NULL; -#endif - -#ifdef SUPPORT_LIBBZ2 -BZFILE *inbz2 = NULL; -#endif - - -/* Do the first read into the start of the buffer and set up the pointer to end -of what we have. In the case of libz, a non-zipped .gz file will be read as a -plain file. However, if a .bz2 file isn't actually bzipped, the first read will -fail. */ - -(void)frtype; - -#ifdef SUPPORT_LIBZ -if (frtype == FR_LIBZ) - { - ingz = (gzFile)handle; - bufflength = gzread (ingz, main_buffer, bufsize); - } -else -#endif - -#ifdef SUPPORT_LIBBZ2 -if (frtype == FR_LIBBZ2) - { - inbz2 = (BZFILE *)handle; - bufflength = BZ2_bzread(inbz2, main_buffer, bufsize); - if ((int)bufflength < 0) return 2; /* Gotcha: bufflength is size_t; */ - } /* without the cast it is unsigned. */ -else -#endif - - { - in = (FILE *)handle; - if (is_file_tty(in)) input_line_buffered = TRUE; - bufflength = input_line_buffered? - read_one_line(main_buffer, bufsize, in) : - fread(main_buffer, 1, bufsize, in); - } - -endptr = main_buffer + bufflength; - -/* Unless binary-files=text, see if we have a binary file. This uses the same -rule as GNU grep, namely, a search for a binary zero byte near the start of the -file. */ - -if (binary_files != BIN_TEXT) - { - binary = - memchr(main_buffer, 0, (bufflength > 1024)? 1024 : bufflength) != NULL; - if (binary && binary_files == BIN_NOMATCH) return 1; - } - -/* Loop while the current pointer is not at the end of the file. For large -files, endptr will be at the end of the buffer when we are in the middle of the -file, but ptr will never get there, because as soon as it gets over 2/3 of the -way, the buffer is shifted left and re-filled. */ - -while (ptr < endptr) - { - int endlinelength; - int mrc = 0; - int startoffset = 0; - int prevoffsets[2]; - unsigned int options = 0; - BOOL match; - char *matchptr = ptr; - char *t = ptr; - size_t length, linelength; - - prevoffsets[0] = prevoffsets[1] = -1; - - /* At this point, ptr is at the start of a line. We need to find the length - of the subject string to pass to pcre_exec(). In multiline mode, it is the - length remainder of the data in the buffer. Otherwise, it is the length of - the next line, excluding the terminating newline. After matching, we always - advance by the length of the next line. In multiline mode the PCRE_FIRSTLINE - option is used for compiling, so that any match is constrained to be in the - first line. */ - - t = end_of_line(t, endptr, &endlinelength); - linelength = t - ptr - endlinelength; - length = multiline? (size_t)(endptr - ptr) : linelength; - - /* Check to see if the line we are looking at extends right to the very end - of the buffer without a line terminator. This means the line is too long to - handle. */ - - if (endlinelength == 0 && t == main_buffer + bufsize) - { - fprintf(stderr, "pcregrep: line %d%s%s is too long for the internal buffer\n" - "pcregrep: check the --buffer-size option\n", - linenumber, - (filename == NULL)? "" : " of file ", - (filename == NULL)? "" : filename); - return 2; - } - - /* Extra processing for Jeffrey Friedl's debugging. */ - -#ifdef JFRIEDL_DEBUG - if (jfriedl_XT || jfriedl_XR) - { -# include -# include - struct timeval start_time, end_time; - struct timezone dummy; - int i; - - if (jfriedl_XT) - { - unsigned long newlen = length * jfriedl_XT + strlen(jfriedl_prefix) + strlen(jfriedl_postfix); - const char *orig = ptr; - ptr = malloc(newlen + 1); - if (!ptr) { - printf("out of memory"); - pcregrep_exit(2); - } - endptr = ptr; - strcpy(endptr, jfriedl_prefix); endptr += strlen(jfriedl_prefix); - for (i = 0; i < jfriedl_XT; i++) { - strncpy(endptr, orig, length); - endptr += length; - } - strcpy(endptr, jfriedl_postfix); endptr += strlen(jfriedl_postfix); - length = newlen; - } - - if (gettimeofday(&start_time, &dummy) != 0) - perror("bad gettimeofday"); - - - for (i = 0; i < jfriedl_XR; i++) - match = (pcre_exec(patterns->compiled, patterns->hint, ptr, length, 0, - PCRE_NOTEMPTY, offsets, OFFSET_SIZE) >= 0); - - if (gettimeofday(&end_time, &dummy) != 0) - perror("bad gettimeofday"); - - double delta = ((end_time.tv_sec + (end_time.tv_usec / 1000000.0)) - - - (start_time.tv_sec + (start_time.tv_usec / 1000000.0))); - - printf("%s TIMER[%.4f]\n", match ? "MATCH" : "FAIL", delta); - return 0; - } -#endif - - /* We come back here after a match when show_only_matching is set, in order - to find any further matches in the same line. This applies to - --only-matching, --file-offsets, and --line-offsets. */ - - ONLY_MATCHING_RESTART: - - /* Run through all the patterns until one matches or there is an error other - than NOMATCH. This code is in a subroutine so that it can be re-used for - finding subsequent matches when colouring matched lines. After finding one - match, set PCRE_NOTEMPTY to disable any further matches of null strings in - this line. */ - - match = match_patterns(matchptr, length, options, startoffset, offsets, &mrc); - options = PCRE_NOTEMPTY; - - /* If it's a match or a not-match (as required), do what's wanted. */ - - if (match != invert) - { - BOOL hyphenprinted = FALSE; - - /* We've failed if we want a file that doesn't have any matches. */ - - if (filenames == FN_NOMATCH_ONLY) return 1; - - /* If all we want is a yes/no answer, stop now. */ - - if (quiet) return 0; - - /* Just count if just counting is wanted. */ - - else if (count_only) count++; - - /* When handling a binary file and binary-files==binary, the "binary" - variable will be set true (it's false in all other cases). In this - situation we just want to output the file name. No need to scan further. */ - - else if (binary) - { - fprintf(stdout, "Binary file %s matches\n", filename); - return 0; - } - - /* If all we want is a file name, there is no need to scan any more lines - in the file. */ - - else if (filenames == FN_MATCH_ONLY) - { - fprintf(stdout, "%s\n", printname); - return 0; - } - - /* The --only-matching option prints just the substring that matched, - and/or one or more captured portions of it, as long as these strings are - not empty. The --file-offsets and --line-offsets options output offsets for - the matching substring (all three set show_only_matching). None of these - mutually exclusive options prints any context. Afterwards, adjust the start - and then jump back to look for further matches in the same line. If we are - in invert mode, however, nothing is printed and we do not restart - this - could still be useful because the return code is set. */ - - else if (show_only_matching) - { - if (!invert) - { - int oldstartoffset = startoffset; - - /* It is possible, when a lookbehind assertion contains \K, for the - same string to be found again. The code below advances startoffset, but - until it is past the "bumpalong" offset that gave the match, the same - substring will be returned. The PCRE1 library does not return the - bumpalong offset, so all we can do is ignore repeated strings. (PCRE2 - does this better.) */ - - if (prevoffsets[0] != offsets[0] || prevoffsets[1] != offsets[1]) - { - prevoffsets[0] = offsets[0]; - prevoffsets[1] = offsets[1]; - - if (printname != NULL) fprintf(stdout, "%s:", printname); - if (number) fprintf(stdout, "%d:", linenumber); - - /* Handle --line-offsets */ - - if (line_offsets) - fprintf(stdout, "%d,%d\n", (int)(matchptr + offsets[0] - ptr), - offsets[1] - offsets[0]); - - /* Handle --file-offsets */ - - else if (file_offsets) - fprintf(stdout, "%d,%d\n", - (int)(filepos + matchptr + offsets[0] - ptr), - offsets[1] - offsets[0]); - - /* Handle --only-matching, which may occur many times */ - - else - { - BOOL printed = FALSE; - omstr *om; - - for (om = only_matching; om != NULL; om = om->next) - { - int n = om->groupnum; - if (n < mrc) - { - int plen = offsets[2*n + 1] - offsets[2*n]; - if (plen > 0) - { - if (printed) fprintf(stdout, "%s", om_separator); - if (do_colour) fprintf(stdout, "%c[%sm", 0x1b, colour_string); - FWRITE(matchptr + offsets[n*2], 1, plen, stdout); - if (do_colour) fprintf(stdout, "%c[00m", 0x1b); - printed = TRUE; - } - } - } - - if (printed || printname != NULL || number) fprintf(stdout, "\n"); - } - } - - /* Prepare to repeat to find the next match. If the patterned contained - a lookbehind tht included \K, it is possible that the end of the match - might be at or before the actual strting offset we have just used. We - need to start one character further on. Unfortunately, for unanchored - patterns, the actual start offset can be greater that the one that was - set as a result of "bumpalong". PCRE1 does not return the actual start - offset, so we have to check against the original start offset. This may - lead to duplicates - we we need the fudge above to avoid printing them. - (PCRE2 does this better.) */ - - match = FALSE; - if (line_buffered) fflush(stdout); - rc = 0; /* Had some success */ - startoffset = offsets[1]; /* Restart after the match */ - if (startoffset <= oldstartoffset) - { - if ((size_t)startoffset >= length) - goto END_ONE_MATCH; /* We were at the end */ - startoffset = oldstartoffset + 1; - if (utf8) - while ((matchptr[startoffset] & 0xc0) == 0x80) startoffset++; - } - goto ONLY_MATCHING_RESTART; - } - } - - /* This is the default case when none of the above options is set. We print - the matching lines(s), possibly preceded and/or followed by other lines of - context. */ - - else - { - /* See if there is a requirement to print some "after" lines from a - previous match. We never print any overlaps. */ - - if (after_context > 0 && lastmatchnumber > 0) - { - int ellength; - int linecount = 0; - char *p = lastmatchrestart; - - while (p < ptr && linecount < after_context) - { - p = end_of_line(p, ptr, &ellength); - linecount++; - } - - /* It is important to advance lastmatchrestart during this printing so - that it interacts correctly with any "before" printing below. Print - each line's data using fwrite() in case there are binary zeroes. */ - - while (lastmatchrestart < p) - { - char *pp = lastmatchrestart; - if (printname != NULL) fprintf(stdout, "%s-", printname); - if (number) fprintf(stdout, "%d-", lastmatchnumber++); - pp = end_of_line(pp, endptr, &ellength); - FWRITE(lastmatchrestart, 1, pp - lastmatchrestart, stdout); - lastmatchrestart = pp; - } - if (lastmatchrestart != ptr) hyphenpending = TRUE; - } - - /* If there were non-contiguous lines printed above, insert hyphens. */ - - if (hyphenpending) - { - fprintf(stdout, "--\n"); - hyphenpending = FALSE; - hyphenprinted = TRUE; - } - - /* See if there is a requirement to print some "before" lines for this - match. Again, don't print overlaps. */ - - if (before_context > 0) - { - int linecount = 0; - char *p = ptr; - - while (p > main_buffer && (lastmatchnumber == 0 || p > lastmatchrestart) && - linecount < before_context) - { - linecount++; - p = previous_line(p, main_buffer); - } - - if (lastmatchnumber > 0 && p > lastmatchrestart && !hyphenprinted) - fprintf(stdout, "--\n"); - - while (p < ptr) - { - int ellength; - char *pp = p; - if (printname != NULL) fprintf(stdout, "%s-", printname); - if (number) fprintf(stdout, "%d-", linenumber - linecount--); - pp = end_of_line(pp, endptr, &ellength); - FWRITE(p, 1, pp - p, stdout); - p = pp; - } - } - - /* Now print the matching line(s); ensure we set hyphenpending at the end - of the file if any context lines are being output. */ - - if (after_context > 0 || before_context > 0) - endhyphenpending = TRUE; - - if (printname != NULL) fprintf(stdout, "%s:", printname); - if (number) fprintf(stdout, "%d:", linenumber); - - /* In multiline mode, we want to print to the end of the line in which - the end of the matched string is found, so we adjust linelength and the - line number appropriately, but only when there actually was a match - (invert not set). Because the PCRE_FIRSTLINE option is set, the start of - the match will always be before the first newline sequence. */ - - if (multiline & !invert) - { - char *endmatch = ptr + offsets[1]; - t = ptr; - while (t <= endmatch) - { - t = end_of_line(t, endptr, &endlinelength); - if (t < endmatch) linenumber++; else break; - } - linelength = t - ptr - endlinelength; - } - - /*** NOTE: Use only fwrite() to output the data line, so that binary - zeroes are treated as just another data character. */ - - /* This extra option, for Jeffrey Friedl's debugging requirements, - replaces the matched string, or a specific captured string if it exists, - with X. When this happens, colouring is ignored. */ - -#ifdef JFRIEDL_DEBUG - if (S_arg >= 0 && S_arg < mrc) - { - int first = S_arg * 2; - int last = first + 1; - FWRITE(ptr, 1, offsets[first], stdout); - fprintf(stdout, "X"); - FWRITE(ptr + offsets[last], 1, linelength - offsets[last], stdout); - } - else -#endif - - /* We have to split the line(s) up if colouring, and search for further - matches, but not of course if the line is a non-match. */ - - if (do_colour && !invert) - { - int plength; - FWRITE(ptr, 1, offsets[0], stdout); - fprintf(stdout, "%c[%sm", 0x1b, colour_string); - FWRITE(ptr + offsets[0], 1, offsets[1] - offsets[0], stdout); - fprintf(stdout, "%c[00m", 0x1b); - for (;;) - { - startoffset = offsets[1]; - if (startoffset >= (int)linelength + endlinelength || - !match_patterns(matchptr, length, options, startoffset, offsets, - &mrc)) - break; - FWRITE(matchptr + startoffset, 1, offsets[0] - startoffset, stdout); - fprintf(stdout, "%c[%sm", 0x1b, colour_string); - FWRITE(matchptr + offsets[0], 1, offsets[1] - offsets[0], stdout); - fprintf(stdout, "%c[00m", 0x1b); - } - - /* In multiline mode, we may have already printed the complete line - and its line-ending characters (if they matched the pattern), so there - may be no more to print. */ - - plength = (int)((linelength + endlinelength) - startoffset); - if (plength > 0) FWRITE(ptr + startoffset, 1, plength, stdout); - } - - /* Not colouring; no need to search for further matches */ - - else FWRITE(ptr, 1, linelength + endlinelength, stdout); - } - - /* End of doing what has to be done for a match. If --line-buffered was - given, flush the output. */ - - if (line_buffered) fflush(stdout); - rc = 0; /* Had some success */ - - /* Remember where the last match happened for after_context. We remember - where we are about to restart, and that line's number. */ - - lastmatchrestart = ptr + linelength + endlinelength; - lastmatchnumber = linenumber + 1; - } - - /* For a match in multiline inverted mode (which of course did not cause - anything to be printed), we have to move on to the end of the match before - proceeding. */ - - if (multiline && invert && match) - { - int ellength; - char *endmatch = ptr + offsets[1]; - t = ptr; - while (t < endmatch) - { - t = end_of_line(t, endptr, &ellength); - if (t <= endmatch) linenumber++; else break; - } - endmatch = end_of_line(endmatch, endptr, &ellength); - linelength = endmatch - ptr - ellength; - } - - /* Advance to after the newline and increment the line number. The file - offset to the current line is maintained in filepos. */ - - END_ONE_MATCH: - ptr += linelength + endlinelength; - filepos += (int)(linelength + endlinelength); - linenumber++; - - /* If input is line buffered, and the buffer is not yet full, read another - line and add it into the buffer. */ - - if (input_line_buffered && bufflength < (size_t)bufsize) - { - int add = read_one_line(ptr, bufsize - (int)(ptr - main_buffer), in); - bufflength += add; - endptr += add; - } - - /* If we haven't yet reached the end of the file (the buffer is full), and - the current point is in the top 1/3 of the buffer, slide the buffer down by - 1/3 and refill it. Before we do this, if some unprinted "after" lines are - about to be lost, print them. */ - - if (bufflength >= (size_t)bufsize && ptr > main_buffer + 2*bufthird) - { - if (after_context > 0 && - lastmatchnumber > 0 && - lastmatchrestart < main_buffer + bufthird) - { - do_after_lines(lastmatchnumber, lastmatchrestart, endptr, printname); - lastmatchnumber = 0; - } - - /* Now do the shuffle */ - - memmove(main_buffer, main_buffer + bufthird, 2*bufthird); - ptr -= bufthird; - -#ifdef SUPPORT_LIBZ - if (frtype == FR_LIBZ) - bufflength = 2*bufthird + - gzread (ingz, main_buffer + 2*bufthird, bufthird); - else -#endif - -#ifdef SUPPORT_LIBBZ2 - if (frtype == FR_LIBBZ2) - bufflength = 2*bufthird + - BZ2_bzread(inbz2, main_buffer + 2*bufthird, bufthird); - else -#endif - - bufflength = 2*bufthird + - (input_line_buffered? - read_one_line(main_buffer + 2*bufthird, bufthird, in) : - fread(main_buffer + 2*bufthird, 1, bufthird, in)); - endptr = main_buffer + bufflength; - - /* Adjust any last match point */ - - if (lastmatchnumber > 0) lastmatchrestart -= bufthird; - } - } /* Loop through the whole file */ - -/* End of file; print final "after" lines if wanted; do_after_lines sets -hyphenpending if it prints something. */ - -if (!show_only_matching && !count_only) - { - do_after_lines(lastmatchnumber, lastmatchrestart, endptr, printname); - hyphenpending |= endhyphenpending; - } - -/* Print the file name if we are looking for those without matches and there -were none. If we found a match, we won't have got this far. */ - -if (filenames == FN_NOMATCH_ONLY) - { - fprintf(stdout, "%s\n", printname); - return 0; - } - -/* Print the match count if wanted */ - -if (count_only && !quiet) - { - if (count > 0 || !omit_zero_count) - { - if (printname != NULL && filenames != FN_NONE) - fprintf(stdout, "%s:", printname); - fprintf(stdout, "%d\n", count); - } - } - -return rc; -} - - - -/************************************************* -* Grep a file or recurse into a directory * -*************************************************/ - -/* Given a path name, if it's a directory, scan all the files if we are -recursing; if it's a file, grep it. - -Arguments: - pathname the path to investigate - dir_recurse TRUE if recursing is wanted (-r or -drecurse) - only_one_at_top TRUE if the path is the only one at toplevel - -Returns: -1 the file/directory was skipped - 0 if there was at least one match - 1 if there were no matches - 2 there was some kind of error - -However, file opening failures are suppressed if "silent" is set. -*/ - -static int -grep_or_recurse(char *pathname, BOOL dir_recurse, BOOL only_one_at_top) -{ -int rc = 1; -int frtype; -void *handle; -char *lastcomp; -FILE *in = NULL; /* Ensure initialized */ - -#ifdef SUPPORT_LIBZ -gzFile ingz = NULL; -#endif - -#ifdef SUPPORT_LIBBZ2 -BZFILE *inbz2 = NULL; -#endif - -#if defined SUPPORT_LIBZ || defined SUPPORT_LIBBZ2 -int pathlen; -#endif - -#if defined NATIVE_ZOS -int zos_type; -FILE *zos_test_file; -#endif - -/* If the file name is "-" we scan stdin */ - -if (strcmp(pathname, "-") == 0) - { - return pcregrep(stdin, FR_PLAIN, stdin_name, - (filenames > FN_DEFAULT || (filenames == FN_DEFAULT && !only_one_at_top))? - stdin_name : NULL); - } - -/* Inclusion and exclusion: --include-dir and --exclude-dir apply only to -directories, whereas --include and --exclude apply to everything else. The test -is against the final component of the path. */ - -lastcomp = strrchr(pathname, FILESEP); -lastcomp = (lastcomp == NULL)? pathname : lastcomp + 1; - -/* If the file is a directory, skip if not recursing or if explicitly excluded. -Otherwise, scan the directory and recurse for each path within it. The scanning -code is localized so it can be made system-specific. */ - - -/* For z/OS, determine the file type. */ - -#if defined NATIVE_ZOS -zos_test_file = fopen(pathname,"rb"); - -if (zos_test_file == NULL) - { - if (!silent) fprintf(stderr, "pcregrep: failed to test next file %s\n", - pathname, strerror(errno)); - return -1; - } -zos_type = identifyzosfiletype (zos_test_file); -fclose (zos_test_file); - -/* Handle a PDS in separate code */ - -if (zos_type == __ZOS_PDS || zos_type == __ZOS_PDSE) - { - return travelonpdsdir (pathname, only_one_at_top); - } - -/* Deal with regular files in the normal way below. These types are: - zos_type == __ZOS_PDS_MEMBER - zos_type == __ZOS_PS - zos_type == __ZOS_VSAM_KSDS - zos_type == __ZOS_VSAM_ESDS - zos_type == __ZOS_VSAM_RRDS -*/ - -/* Handle a z/OS directory using common code. */ - -else if (zos_type == __ZOS_HFS) - { -#endif /* NATIVE_ZOS */ - - -/* Handle directories: common code for all OS */ - -if (isdirectory(pathname)) - { - if (dee_action == dee_SKIP || - !test_incexc(lastcomp, include_dir_patterns, exclude_dir_patterns)) - return -1; - - if (dee_action == dee_RECURSE) - { - char buffer[1024]; - char *nextfile; - directory_type *dir = opendirectory(pathname); - - if (dir == NULL) - { - if (!silent) - fprintf(stderr, "pcregrep: Failed to open directory %s: %s\n", pathname, - strerror(errno)); - return 2; - } - - while ((nextfile = readdirectory(dir)) != NULL) - { - int frc; - sprintf(buffer, "%.512s%c%.128s", pathname, FILESEP, nextfile); - frc = grep_or_recurse(buffer, dir_recurse, FALSE); - if (frc > 1) rc = frc; - else if (frc == 0 && rc == 1) rc = 0; - } - - closedirectory(dir); - return rc; - } - } - -#if defined NATIVE_ZOS - } -#endif - -/* If the file is not a directory, check for a regular file, and if it is not, -skip it if that's been requested. Otherwise, check for an explicit inclusion or -exclusion. */ - -else if ( -#if defined NATIVE_ZOS - (zos_type == __ZOS_NOFILE && DEE_action == DEE_SKIP) || -#else /* all other OS */ - (!isregfile(pathname) && DEE_action == DEE_SKIP) || -#endif - !test_incexc(lastcomp, include_patterns, exclude_patterns)) - return -1; /* File skipped */ - -/* Control reaches here if we have a regular file, or if we have a directory -and recursion or skipping was not requested, or if we have anything else and -skipping was not requested. The scan proceeds. If this is the first and only -argument at top level, we don't show the file name, unless we are only showing -the file name, or the filename was forced (-H). */ - -#if defined SUPPORT_LIBZ || defined SUPPORT_LIBBZ2 -pathlen = (int)(strlen(pathname)); -#endif - -/* Open using zlib if it is supported and the file name ends with .gz. */ - -#ifdef SUPPORT_LIBZ -if (pathlen > 3 && strcmp(pathname + pathlen - 3, ".gz") == 0) - { - ingz = gzopen(pathname, "rb"); - if (ingz == NULL) - { - if (!silent) - fprintf(stderr, "pcregrep: Failed to open %s: %s\n", pathname, - strerror(errno)); - return 2; - } - handle = (void *)ingz; - frtype = FR_LIBZ; - } -else -#endif - -/* Otherwise open with bz2lib if it is supported and the name ends with .bz2. */ - -#ifdef SUPPORT_LIBBZ2 -if (pathlen > 4 && strcmp(pathname + pathlen - 4, ".bz2") == 0) - { - inbz2 = BZ2_bzopen(pathname, "rb"); - handle = (void *)inbz2; - frtype = FR_LIBBZ2; - } -else -#endif - -/* Otherwise use plain fopen(). The label is so that we can come back here if -an attempt to read a .bz2 file indicates that it really is a plain file. */ - -#ifdef SUPPORT_LIBBZ2 -PLAIN_FILE: -#endif - { - in = fopen(pathname, "rb"); - handle = (void *)in; - frtype = FR_PLAIN; - } - -/* All the opening methods return errno when they fail. */ - -if (handle == NULL) - { - if (!silent) - fprintf(stderr, "pcregrep: Failed to open %s: %s\n", pathname, - strerror(errno)); - return 2; - } - -/* Now grep the file */ - -rc = pcregrep(handle, frtype, pathname, (filenames > FN_DEFAULT || - (filenames == FN_DEFAULT && !only_one_at_top))? pathname : NULL); - -/* Close in an appropriate manner. */ - -#ifdef SUPPORT_LIBZ -if (frtype == FR_LIBZ) - gzclose(ingz); -else -#endif - -/* If it is a .bz2 file and the result is 3, it means that the first attempt to -read failed. If the error indicates that the file isn't in fact bzipped, try -again as a normal file. */ - -#ifdef SUPPORT_LIBBZ2 -if (frtype == FR_LIBBZ2) - { - if (rc == 3) - { - int errnum; - const char *err = BZ2_bzerror(inbz2, &errnum); - if (errnum == BZ_DATA_ERROR_MAGIC) - { - BZ2_bzclose(inbz2); - goto PLAIN_FILE; - } - else if (!silent) - fprintf(stderr, "pcregrep: Failed to read %s using bzlib: %s\n", - pathname, err); - rc = 2; /* The normal "something went wrong" code */ - } - BZ2_bzclose(inbz2); - } -else -#endif - -/* Normal file close */ - -fclose(in); - -/* Pass back the yield from pcregrep(). */ - -return rc; -} - - - -/************************************************* -* Handle a single-letter, no data option * -*************************************************/ - -static int -handle_option(int letter, int options) -{ -switch(letter) - { - case N_FOFFSETS: file_offsets = TRUE; break; - case N_HELP: help(); pcregrep_exit(0); - case N_LBUFFER: line_buffered = TRUE; break; - case N_LOFFSETS: line_offsets = number = TRUE; break; - case N_NOJIT: study_options &= ~PCRE_STUDY_JIT_COMPILE; break; - case 'a': binary_files = BIN_TEXT; break; - case 'c': count_only = TRUE; break; - case 'F': process_options |= PO_FIXED_STRINGS; break; - case 'H': filenames = FN_FORCE; break; - case 'I': binary_files = BIN_NOMATCH; break; - case 'h': filenames = FN_NONE; break; - case 'i': options |= PCRE_CASELESS; break; - case 'l': omit_zero_count = TRUE; filenames = FN_MATCH_ONLY; break; - case 'L': filenames = FN_NOMATCH_ONLY; break; - case 'M': multiline = TRUE; options |= PCRE_MULTILINE|PCRE_FIRSTLINE; break; - case 'n': number = TRUE; break; - - case 'o': - only_matching_last = add_number(0, only_matching_last); - if (only_matching == NULL) only_matching = only_matching_last; - break; - - case 'q': quiet = TRUE; break; - case 'r': dee_action = dee_RECURSE; break; - case 's': silent = TRUE; break; - case 'u': options |= PCRE_UTF8; utf8 = TRUE; break; - case 'v': invert = TRUE; break; - case 'w': process_options |= PO_WORD_MATCH; break; - case 'x': process_options |= PO_LINE_MATCH; break; - - case 'V': - fprintf(stdout, "pcregrep version %s\n", pcre_version()); - pcregrep_exit(0); - break; - - default: - fprintf(stderr, "pcregrep: Unknown option -%c\n", letter); - pcregrep_exit(usage(2)); - } - -return options; -} - - - - -/************************************************* -* Construct printed ordinal * -*************************************************/ - -/* This turns a number into "1st", "3rd", etc. */ - -static char * -ordin(int n) -{ -static char buffer[14]; -char *p = buffer; -sprintf(p, "%d", n); -while (*p != 0) p++; -switch (n%10) - { - case 1: strcpy(p, "st"); break; - case 2: strcpy(p, "nd"); break; - case 3: strcpy(p, "rd"); break; - default: strcpy(p, "th"); break; - } -return buffer; -} - - - -/************************************************* -* Compile a single pattern * -*************************************************/ - -/* Do nothing if the pattern has already been compiled. This is the case for -include/exclude patterns read from a file. - -When the -F option has been used, each "pattern" may be a list of strings, -separated by line breaks. They will be matched literally. We split such a -string and compile the first substring, inserting an additional block into the -pattern chain. - -Arguments: - p points to the pattern block - options the PCRE options - popts the processing options - fromfile TRUE if the pattern was read from a file - fromtext file name or identifying text (e.g. "include") - count 0 if this is the only command line pattern, or - number of the command line pattern, or - linenumber for a pattern from a file - -Returns: TRUE on success, FALSE after an error -*/ - -static BOOL -compile_pattern(patstr *p, int options, int popts, int fromfile, - const char *fromtext, int count) -{ -char buffer[PATBUFSIZE]; -const char *error; -char *ps = p->string; -int patlen = strlen(ps); -int errptr; - -if (p->compiled != NULL) return TRUE; - -if ((popts & PO_FIXED_STRINGS) != 0) - { - int ellength; - char *eop = ps + patlen; - char *pe = end_of_line(ps, eop, &ellength); - - if (ellength != 0) - { - if (add_pattern(pe, p) == NULL) return FALSE; - patlen = (int)(pe - ps - ellength); - } - } - -sprintf(buffer, "%s%.*s%s", prefix[popts], patlen, ps, suffix[popts]); -p->compiled = pcre_compile(buffer, options, &error, &errptr, pcretables); -if (p->compiled != NULL) return TRUE; - -/* Handle compile errors */ - -errptr -= (int)strlen(prefix[popts]); -if (errptr > patlen) errptr = patlen; - -if (fromfile) - { - fprintf(stderr, "pcregrep: Error in regex in line %d of %s " - "at offset %d: %s\n", count, fromtext, errptr, error); - } -else - { - if (count == 0) - fprintf(stderr, "pcregrep: Error in %s regex at offset %d: %s\n", - fromtext, errptr, error); - else - fprintf(stderr, "pcregrep: Error in %s %s regex at offset %d: %s\n", - ordin(count), fromtext, errptr, error); - } - -return FALSE; -} - - - -/************************************************* -* Read and compile a file of patterns * -*************************************************/ - -/* This is used for --filelist, --include-from, and --exclude-from. - -Arguments: - name the name of the file; "-" is stdin - patptr pointer to the pattern chain anchor - patlastptr pointer to the last pattern pointer - popts the process options to pass to pattern_compile() - -Returns: TRUE if all went well -*/ - -static BOOL -read_pattern_file(char *name, patstr **patptr, patstr **patlastptr, int popts) -{ -int linenumber = 0; -FILE *f; -char *filename; -char buffer[PATBUFSIZE]; - -if (strcmp(name, "-") == 0) - { - f = stdin; - filename = stdin_name; - } -else - { - f = fopen(name, "r"); - if (f == NULL) - { - fprintf(stderr, "pcregrep: Failed to open %s: %s\n", name, strerror(errno)); - return FALSE; - } - filename = name; - } - -while (fgets(buffer, PATBUFSIZE, f) != NULL) - { - char *s = buffer + (int)strlen(buffer); - while (s > buffer && isspace((unsigned char)(s[-1]))) s--; - *s = 0; - linenumber++; - if (buffer[0] == 0) continue; /* Skip blank lines */ - - /* Note: this call to add_pattern() puts a pointer to the local variable - "buffer" into the pattern chain. However, that pointer is used only when - compiling the pattern, which happens immediately below, so we flatten it - afterwards, as a precaution against any later code trying to use it. */ - - *patlastptr = add_pattern(buffer, *patlastptr); - if (*patlastptr == NULL) - { - if (f != stdin) fclose(f); - return FALSE; - } - if (*patptr == NULL) *patptr = *patlastptr; - - /* This loop is needed because compiling a "pattern" when -F is set may add - on additional literal patterns if the original contains a newline. In the - common case, it never will, because fgets() stops at a newline. However, - the -N option can be used to give pcregrep a different newline setting. */ - - for(;;) - { - if (!compile_pattern(*patlastptr, pcre_options, popts, TRUE, filename, - linenumber)) - { - if (f != stdin) fclose(f); - return FALSE; - } - (*patlastptr)->string = NULL; /* Insurance */ - if ((*patlastptr)->next == NULL) break; - *patlastptr = (*patlastptr)->next; - } - } - -if (f != stdin) fclose(f); -return TRUE; -} - - - -/************************************************* -* Main program * -*************************************************/ - -/* Returns 0 if something matched, 1 if nothing matched, 2 after an error. */ - -int -main(int argc, char **argv) -{ -int i, j; -int rc = 1; -BOOL only_one_at_top; -patstr *cp; -fnstr *fn; -const char *locale_from = "--locale"; -const char *error; - -#ifdef SUPPORT_PCREGREP_JIT -pcre_jit_stack *jit_stack = NULL; -#endif - -/* Set the default line ending value from the default in the PCRE library; -"lf", "cr", "crlf", and "any" are supported. Anything else is treated as "lf". -Note that the return values from pcre_config(), though derived from the ASCII -codes, are the same in EBCDIC environments, so we must use the actual values -rather than escapes such as as '\r'. */ - -(void)pcre_config(PCRE_CONFIG_NEWLINE, &i); -switch(i) - { - default: newline = (char *)"lf"; break; - case 13: newline = (char *)"cr"; break; - case (13 << 8) | 10: newline = (char *)"crlf"; break; - case -1: newline = (char *)"any"; break; - case -2: newline = (char *)"anycrlf"; break; - } - -/* Process the options */ - -for (i = 1; i < argc; i++) - { - option_item *op = NULL; - char *option_data = (char *)""; /* default to keep compiler happy */ - BOOL longop; - BOOL longopwasequals = FALSE; - - if (argv[i][0] != '-') break; - - /* If we hit an argument that is just "-", it may be a reference to STDIN, - but only if we have previously had -e or -f to define the patterns. */ - - if (argv[i][1] == 0) - { - if (pattern_files != NULL || patterns != NULL) break; - else pcregrep_exit(usage(2)); - } - - /* Handle a long name option, or -- to terminate the options */ - - if (argv[i][1] == '-') - { - char *arg = argv[i] + 2; - char *argequals = strchr(arg, '='); - - if (*arg == 0) /* -- terminates options */ - { - i++; - break; /* out of the options-handling loop */ - } - - longop = TRUE; - - /* Some long options have data that follows after =, for example file=name. - Some options have variations in the long name spelling: specifically, we - allow "regexp" because GNU grep allows it, though I personally go along - with Jeffrey Friedl and Larry Wall in preferring "regex" without the "p". - These options are entered in the table as "regex(p)". Options can be in - both these categories. */ - - for (op = optionlist; op->one_char != 0; op++) - { - char *opbra = strchr(op->long_name, '('); - char *equals = strchr(op->long_name, '='); - - /* Handle options with only one spelling of the name */ - - if (opbra == NULL) /* Does not contain '(' */ - { - if (equals == NULL) /* Not thing=data case */ - { - if (strcmp(arg, op->long_name) == 0) break; - } - else /* Special case xxx=data */ - { - int oplen = (int)(equals - op->long_name); - int arglen = (argequals == NULL)? - (int)strlen(arg) : (int)(argequals - arg); - if (oplen == arglen && strncmp(arg, op->long_name, oplen) == 0) - { - option_data = arg + arglen; - if (*option_data == '=') - { - option_data++; - longopwasequals = TRUE; - } - break; - } - } - } - - /* Handle options with an alternate spelling of the name */ - - else - { - char buff1[24]; - char buff2[24]; - - int baselen = (int)(opbra - op->long_name); - int fulllen = (int)(strchr(op->long_name, ')') - op->long_name + 1); - int arglen = (argequals == NULL || equals == NULL)? - (int)strlen(arg) : (int)(argequals - arg); - - sprintf(buff1, "%.*s", baselen, op->long_name); - sprintf(buff2, "%s%.*s", buff1, fulllen - baselen - 2, opbra + 1); - - if (strncmp(arg, buff1, arglen) == 0 || - strncmp(arg, buff2, arglen) == 0) - { - if (equals != NULL && argequals != NULL) - { - option_data = argequals; - if (*option_data == '=') - { - option_data++; - longopwasequals = TRUE; - } - } - break; - } - } - } - - if (op->one_char == 0) - { - fprintf(stderr, "pcregrep: Unknown option %s\n", argv[i]); - pcregrep_exit(usage(2)); - } - } - - /* Jeffrey Friedl's debugging harness uses these additional options which - are not in the right form for putting in the option table because they use - only one hyphen, yet are more than one character long. By putting them - separately here, they will not get displayed as part of the help() output, - but I don't think Jeffrey will care about that. */ - -#ifdef JFRIEDL_DEBUG - else if (strcmp(argv[i], "-pre") == 0) { - jfriedl_prefix = argv[++i]; - continue; - } else if (strcmp(argv[i], "-post") == 0) { - jfriedl_postfix = argv[++i]; - continue; - } else if (strcmp(argv[i], "-XT") == 0) { - sscanf(argv[++i], "%d", &jfriedl_XT); - continue; - } else if (strcmp(argv[i], "-XR") == 0) { - sscanf(argv[++i], "%d", &jfriedl_XR); - continue; - } -#endif - - - /* One-char options; many that have no data may be in a single argument; we - continue till we hit the last one or one that needs data. */ - - else - { - char *s = argv[i] + 1; - longop = FALSE; - - while (*s != 0) - { - for (op = optionlist; op->one_char != 0; op++) - { - if (*s == op->one_char) break; - } - if (op->one_char == 0) - { - fprintf(stderr, "pcregrep: Unknown option letter '%c' in \"%s\"\n", - *s, argv[i]); - pcregrep_exit(usage(2)); - } - - option_data = s+1; - - /* Break out if this is the last character in the string; it's handled - below like a single multi-char option. */ - - if (*option_data == 0) break; - - /* Check for a single-character option that has data: OP_OP_NUMBER(S) - are used for ones that either have a numerical number or defaults, i.e. - the data is optional. If a digit follows, there is data; if not, carry on - with other single-character options in the same string. */ - - if (op->type == OP_OP_NUMBER || op->type == OP_OP_NUMBERS) - { - if (isdigit((unsigned char)s[1])) break; - } - else /* Check for an option with data */ - { - if (op->type != OP_NODATA) break; - } - - /* Handle a single-character option with no data, then loop for the - next character in the string. */ - - pcre_options = handle_option(*s++, pcre_options); - } - } - - /* At this point we should have op pointing to a matched option. If the type - is NO_DATA, it means that there is no data, and the option might set - something in the PCRE options. */ - - if (op->type == OP_NODATA) - { - pcre_options = handle_option(op->one_char, pcre_options); - continue; - } - - /* If the option type is OP_OP_STRING or OP_OP_NUMBER(S), it's an option that - either has a value or defaults to something. It cannot have data in a - separate item. At the moment, the only such options are "colo(u)r", - "only-matching", and Jeffrey Friedl's special -S debugging option. */ - - if (*option_data == 0 && - (op->type == OP_OP_STRING || op->type == OP_OP_NUMBER || - op->type == OP_OP_NUMBERS)) - { - switch (op->one_char) - { - case N_COLOUR: - colour_option = (char *)"auto"; - break; - - case 'o': - only_matching_last = add_number(0, only_matching_last); - if (only_matching == NULL) only_matching = only_matching_last; - break; - -#ifdef JFRIEDL_DEBUG - case 'S': - S_arg = 0; - break; -#endif - } - continue; - } - - /* Otherwise, find the data string for the option. */ - - if (*option_data == 0) - { - if (i >= argc - 1 || longopwasequals) - { - fprintf(stderr, "pcregrep: Data missing after %s\n", argv[i]); - pcregrep_exit(usage(2)); - } - option_data = argv[++i]; - } - - /* If the option type is OP_OP_NUMBERS, the value is a number that is to be - added to a chain of numbers. */ - - if (op->type == OP_OP_NUMBERS) - { - unsigned long int n = decode_number(option_data, op, longop); - omdatastr *omd = (omdatastr *)op->dataptr; - *(omd->lastptr) = add_number((int)n, *(omd->lastptr)); - if (*(omd->anchor) == NULL) *(omd->anchor) = *(omd->lastptr); - } - - /* If the option type is OP_PATLIST, it's the -e option, or one of the - include/exclude options, which can be called multiple times to create lists - of patterns. */ - - else if (op->type == OP_PATLIST) - { - patdatastr *pd = (patdatastr *)op->dataptr; - *(pd->lastptr) = add_pattern(option_data, *(pd->lastptr)); - if (*(pd->lastptr) == NULL) goto EXIT2; - if (*(pd->anchor) == NULL) *(pd->anchor) = *(pd->lastptr); - } - - /* If the option type is OP_FILELIST, it's one of the options that names a - file. */ - - else if (op->type == OP_FILELIST) - { - fndatastr *fd = (fndatastr *)op->dataptr; - fn = (fnstr *)malloc(sizeof(fnstr)); - if (fn == NULL) - { - fprintf(stderr, "pcregrep: malloc failed\n"); - goto EXIT2; - } - fn->next = NULL; - fn->name = option_data; - if (*(fd->anchor) == NULL) - *(fd->anchor) = fn; - else - (*(fd->lastptr))->next = fn; - *(fd->lastptr) = fn; - } - - /* Handle OP_BINARY_FILES */ - - else if (op->type == OP_BINFILES) - { - if (strcmp(option_data, "binary") == 0) - binary_files = BIN_BINARY; - else if (strcmp(option_data, "without-match") == 0) - binary_files = BIN_NOMATCH; - else if (strcmp(option_data, "text") == 0) - binary_files = BIN_TEXT; - else - { - fprintf(stderr, "pcregrep: unknown value \"%s\" for binary-files\n", - option_data); - pcregrep_exit(usage(2)); - } - } - - /* Otherwise, deal with a single string or numeric data value. */ - - else if (op->type != OP_NUMBER && op->type != OP_LONGNUMBER && - op->type != OP_OP_NUMBER) - { - *((char **)op->dataptr) = option_data; - } - else - { - unsigned long int n = decode_number(option_data, op, longop); - if (op->type == OP_LONGNUMBER) *((unsigned long int *)op->dataptr) = n; - else *((int *)op->dataptr) = n; - } - } - -/* Options have been decoded. If -C was used, its value is used as a default -for -A and -B. */ - -if (both_context > 0) - { - if (after_context == 0) after_context = both_context; - if (before_context == 0) before_context = both_context; - } - -/* Only one of --only-matching, --file-offsets, or --line-offsets is permitted. -However, all three set show_only_matching because they display, each in their -own way, only the data that has matched. */ - -if ((only_matching != NULL && (file_offsets || line_offsets)) || - (file_offsets && line_offsets)) - { - fprintf(stderr, "pcregrep: Cannot mix --only-matching, --file-offsets " - "and/or --line-offsets\n"); - pcregrep_exit(usage(2)); - } - -if (only_matching != NULL || file_offsets || line_offsets) - show_only_matching = TRUE; - -/* If a locale has not been provided as an option, see if the LC_CTYPE or -LC_ALL environment variable is set, and if so, use it. */ - -if (locale == NULL) - { - locale = getenv("LC_ALL"); - locale_from = "LCC_ALL"; - } - -if (locale == NULL) - { - locale = getenv("LC_CTYPE"); - locale_from = "LC_CTYPE"; - } - -/* If a locale is set, use it to generate the tables the PCRE needs. Otherwise, -pcretables==NULL, which causes the use of default tables. */ - -if (locale != NULL) - { - if (setlocale(LC_CTYPE, locale) == NULL) - { - fprintf(stderr, "pcregrep: Failed to set locale %s (obtained from %s)\n", - locale, locale_from); - goto EXIT2; - } - pcretables = pcre_maketables(); - } - -/* Sort out colouring */ - -if (colour_option != NULL && strcmp(colour_option, "never") != 0) - { - if (strcmp(colour_option, "always") == 0) do_colour = TRUE; - else if (strcmp(colour_option, "auto") == 0) do_colour = is_stdout_tty(); - else - { - fprintf(stderr, "pcregrep: Unknown colour setting \"%s\"\n", - colour_option); - goto EXIT2; - } - if (do_colour) - { - char *cs = getenv("PCREGREP_COLOUR"); - if (cs == NULL) cs = getenv("PCREGREP_COLOR"); - if (cs != NULL) colour_string = cs; - } - } - -/* Interpret the newline type; the default settings are Unix-like. */ - -if (strcmp(newline, "cr") == 0 || strcmp(newline, "CR") == 0) - { - pcre_options |= PCRE_NEWLINE_CR; - endlinetype = EL_CR; - } -else if (strcmp(newline, "lf") == 0 || strcmp(newline, "LF") == 0) - { - pcre_options |= PCRE_NEWLINE_LF; - endlinetype = EL_LF; - } -else if (strcmp(newline, "crlf") == 0 || strcmp(newline, "CRLF") == 0) - { - pcre_options |= PCRE_NEWLINE_CRLF; - endlinetype = EL_CRLF; - } -else if (strcmp(newline, "any") == 0 || strcmp(newline, "ANY") == 0) - { - pcre_options |= PCRE_NEWLINE_ANY; - endlinetype = EL_ANY; - } -else if (strcmp(newline, "anycrlf") == 0 || strcmp(newline, "ANYCRLF") == 0) - { - pcre_options |= PCRE_NEWLINE_ANYCRLF; - endlinetype = EL_ANYCRLF; - } -else - { - fprintf(stderr, "pcregrep: Invalid newline specifier \"%s\"\n", newline); - goto EXIT2; - } - -/* Interpret the text values for -d and -D */ - -if (dee_option != NULL) - { - if (strcmp(dee_option, "read") == 0) dee_action = dee_READ; - else if (strcmp(dee_option, "recurse") == 0) dee_action = dee_RECURSE; - else if (strcmp(dee_option, "skip") == 0) dee_action = dee_SKIP; - else - { - fprintf(stderr, "pcregrep: Invalid value \"%s\" for -d\n", dee_option); - goto EXIT2; - } - } - -if (DEE_option != NULL) - { - if (strcmp(DEE_option, "read") == 0) DEE_action = DEE_READ; - else if (strcmp(DEE_option, "skip") == 0) DEE_action = DEE_SKIP; - else - { - fprintf(stderr, "pcregrep: Invalid value \"%s\" for -D\n", DEE_option); - goto EXIT2; - } - } - -/* Check the values for Jeffrey Friedl's debugging options. */ - -#ifdef JFRIEDL_DEBUG -if (S_arg > 9) - { - fprintf(stderr, "pcregrep: bad value for -S option\n"); - return 2; - } -if (jfriedl_XT != 0 || jfriedl_XR != 0) - { - if (jfriedl_XT == 0) jfriedl_XT = 1; - if (jfriedl_XR == 0) jfriedl_XR = 1; - } -#endif - -/* Get memory for the main buffer. */ - -bufsize = 3*bufthird; -main_buffer = (char *)malloc(bufsize); - -if (main_buffer == NULL) - { - fprintf(stderr, "pcregrep: malloc failed\n"); - goto EXIT2; - } - -/* If no patterns were provided by -e, and there are no files provided by -f, -the first argument is the one and only pattern, and it must exist. */ - -if (patterns == NULL && pattern_files == NULL) - { - if (i >= argc) return usage(2); - patterns = patterns_last = add_pattern(argv[i++], NULL); - if (patterns == NULL) goto EXIT2; - } - -/* Compile the patterns that were provided on the command line, either by -multiple uses of -e or as a single unkeyed pattern. We cannot do this until -after all the command-line options are read so that we know which PCRE options -to use. When -F is used, compile_pattern() may add another block into the -chain, so we must not access the next pointer till after the compile. */ - -for (j = 1, cp = patterns; cp != NULL; j++, cp = cp->next) - { - if (!compile_pattern(cp, pcre_options, process_options, FALSE, "command-line", - (j == 1 && patterns->next == NULL)? 0 : j)) - goto EXIT2; - } - -/* Read and compile the regular expressions that are provided in files. */ - -for (fn = pattern_files; fn != NULL; fn = fn->next) - { - if (!read_pattern_file(fn->name, &patterns, &patterns_last, process_options)) - goto EXIT2; - } - -/* Study the regular expressions, as we will be running them many times. If an -extra block is needed for a limit, set PCRE_STUDY_EXTRA_NEEDED so that one is -returned, even if studying produces no data. */ - -if (match_limit > 0 || match_limit_recursion > 0) - study_options |= PCRE_STUDY_EXTRA_NEEDED; - -/* Unless JIT has been explicitly disabled, arrange a stack for it to use. */ - -#ifdef SUPPORT_PCREGREP_JIT -if ((study_options & PCRE_STUDY_JIT_COMPILE) != 0) - jit_stack = pcre_jit_stack_alloc(32*1024, 1024*1024); -#endif - -for (j = 1, cp = patterns; cp != NULL; j++, cp = cp->next) - { - cp->hint = pcre_study(cp->compiled, study_options, &error); - if (error != NULL) - { - char s[16]; - if (patterns->next == NULL) s[0] = 0; else sprintf(s, " number %d", j); - fprintf(stderr, "pcregrep: Error while studying regex%s: %s\n", s, error); - goto EXIT2; - } -#ifdef SUPPORT_PCREGREP_JIT - if (jit_stack != NULL && cp->hint != NULL) - pcre_assign_jit_stack(cp->hint, NULL, jit_stack); -#endif - } - -/* If --match-limit or --recursion-limit was set, put the value(s) into the -pcre_extra block for each pattern. There will always be an extra block because -of the use of PCRE_STUDY_EXTRA_NEEDED above. */ - -for (cp = patterns; cp != NULL; cp = cp->next) - { - if (match_limit > 0) - { - cp->hint->flags |= PCRE_EXTRA_MATCH_LIMIT; - cp->hint->match_limit = match_limit; - } - - if (match_limit_recursion > 0) - { - cp->hint->flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION; - cp->hint->match_limit_recursion = match_limit_recursion; - } - } - -/* If there are include or exclude patterns read from the command line, compile -them. -F, -w, and -x do not apply, so the third argument of compile_pattern is -0. */ - -for (j = 0; j < 4; j++) - { - int k; - for (k = 1, cp = *(incexlist[j]); cp != NULL; k++, cp = cp->next) - { - if (!compile_pattern(cp, pcre_options, 0, FALSE, incexname[j], - (k == 1 && cp->next == NULL)? 0 : k)) - goto EXIT2; - } - } - -/* Read and compile include/exclude patterns from files. */ - -for (fn = include_from; fn != NULL; fn = fn->next) - { - if (!read_pattern_file(fn->name, &include_patterns, &include_patterns_last, 0)) - goto EXIT2; - } - -for (fn = exclude_from; fn != NULL; fn = fn->next) - { - if (!read_pattern_file(fn->name, &exclude_patterns, &exclude_patterns_last, 0)) - goto EXIT2; - } - -/* If there are no files that contain lists of files to search, and there are -no file arguments, search stdin, and then exit. */ - -if (file_lists == NULL && i >= argc) - { - rc = pcregrep(stdin, FR_PLAIN, stdin_name, - (filenames > FN_DEFAULT)? stdin_name : NULL); - goto EXIT; - } - -/* If any files that contains a list of files to search have been specified, -read them line by line and search the given files. */ - -for (fn = file_lists; fn != NULL; fn = fn->next) - { - char buffer[PATBUFSIZE]; - FILE *fl; - if (strcmp(fn->name, "-") == 0) fl = stdin; else - { - fl = fopen(fn->name, "rb"); - if (fl == NULL) - { - fprintf(stderr, "pcregrep: Failed to open %s: %s\n", fn->name, - strerror(errno)); - goto EXIT2; - } - } - while (fgets(buffer, PATBUFSIZE, fl) != NULL) - { - int frc; - char *end = buffer + (int)strlen(buffer); - while (end > buffer && isspace(end[-1])) end--; - *end = 0; - if (*buffer != 0) - { - frc = grep_or_recurse(buffer, dee_action == dee_RECURSE, FALSE); - if (frc > 1) rc = frc; - else if (frc == 0 && rc == 1) rc = 0; - } - } - if (fl != stdin) fclose(fl); - } - -/* After handling file-list, work through remaining arguments. Pass in the fact -that there is only one argument at top level - this suppresses the file name if -the argument is not a directory and filenames are not otherwise forced. */ - -only_one_at_top = i == argc - 1 && file_lists == NULL; - -for (; i < argc; i++) - { - int frc = grep_or_recurse(argv[i], dee_action == dee_RECURSE, - only_one_at_top); - if (frc > 1) rc = frc; - else if (frc == 0 && rc == 1) rc = 0; - } - -EXIT: -#ifdef SUPPORT_PCREGREP_JIT -if (jit_stack != NULL) pcre_jit_stack_free(jit_stack); -#endif - -free(main_buffer); -free((void *)pcretables); - -free_pattern_chain(patterns); -free_pattern_chain(include_patterns); -free_pattern_chain(include_dir_patterns); -free_pattern_chain(exclude_patterns); -free_pattern_chain(exclude_dir_patterns); - -free_file_chain(exclude_from); -free_file_chain(include_from); -free_file_chain(pattern_files); -free_file_chain(file_lists); - -while (only_matching != NULL) - { - omstr *this = only_matching; - only_matching = this->next; - free(this); - } - -pcregrep_exit(rc); - -EXIT2: -rc = 2; -goto EXIT; -} - -/* End of pcregrep */ diff --git a/src/third_party/pcre-8.39/pcreposix.c b/src/third_party/pcre-8.39/pcreposix.c deleted file mode 100644 index cf75588c40d..00000000000 --- a/src/third_party/pcre-8.39/pcreposix.c +++ /dev/null @@ -1,432 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -/* PCRE is a library of functions to support regular expressions whose syntax -and semantics are as close as possible to those of the Perl 5 language. - - Written by Philip Hazel - Copyright (c) 1997-2016 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - - -/* This module is a wrapper that provides a POSIX API to the underlying PCRE -functions. */ - - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - - -/* Ensure that the PCREPOSIX_EXP_xxx macros are set appropriately for -compiling these functions. This must come before including pcreposix.h, where -they are set for an application (using these functions) if they have not -previously been set. */ - -#if defined(_WIN32) && !defined(PCRE_STATIC) -# define PCREPOSIX_EXP_DECL extern __declspec(dllexport) -# define PCREPOSIX_EXP_DEFN __declspec(dllexport) -#endif - -/* We include pcre.h before pcre_internal.h so that the PCRE library functions -are declared as "import" for Windows by defining PCRE_EXP_DECL as "import". -This is needed even though pcre_internal.h itself includes pcre.h, because it -does so after it has set PCRE_EXP_DECL to "export" if it is not already set. */ - -#include "pcre.h" -#include "pcre_internal.h" -#include "pcreposix.h" - - -/* Table to translate PCRE compile time error codes into POSIX error codes. */ - -static const int eint[] = { - 0, /* no error */ - REG_EESCAPE, /* \ at end of pattern */ - REG_EESCAPE, /* \c at end of pattern */ - REG_EESCAPE, /* unrecognized character follows \ */ - REG_BADBR, /* numbers out of order in {} quantifier */ - /* 5 */ - REG_BADBR, /* number too big in {} quantifier */ - REG_EBRACK, /* missing terminating ] for character class */ - REG_ECTYPE, /* invalid escape sequence in character class */ - REG_ERANGE, /* range out of order in character class */ - REG_BADRPT, /* nothing to repeat */ - /* 10 */ - REG_BADRPT, /* operand of unlimited repeat could match the empty string */ - REG_ASSERT, /* internal error: unexpected repeat */ - REG_BADPAT, /* unrecognized character after (? */ - REG_BADPAT, /* POSIX named classes are supported only within a class */ - REG_EPAREN, /* missing ) */ - /* 15 */ - REG_ESUBREG, /* reference to non-existent subpattern */ - REG_INVARG, /* erroffset passed as NULL */ - REG_INVARG, /* unknown option bit(s) set */ - REG_EPAREN, /* missing ) after comment */ - REG_ESIZE, /* parentheses nested too deeply */ - /* 20 */ - REG_ESIZE, /* regular expression too large */ - REG_ESPACE, /* failed to get memory */ - REG_EPAREN, /* unmatched parentheses */ - REG_ASSERT, /* internal error: code overflow */ - REG_BADPAT, /* unrecognized character after (?< */ - /* 25 */ - REG_BADPAT, /* lookbehind assertion is not fixed length */ - REG_BADPAT, /* malformed number or name after (?( */ - REG_BADPAT, /* conditional group contains more than two branches */ - REG_BADPAT, /* assertion expected after (?( */ - REG_BADPAT, /* (?R or (?[+-]digits must be followed by ) */ - /* 30 */ - REG_ECTYPE, /* unknown POSIX class name */ - REG_BADPAT, /* POSIX collating elements are not supported */ - REG_INVARG, /* this version of PCRE is not compiled with PCRE_UTF8 support */ - REG_BADPAT, /* spare error */ - REG_BADPAT, /* character value in \x{} or \o{} is too large */ - /* 35 */ - REG_BADPAT, /* invalid condition (?(0) */ - REG_BADPAT, /* \C not allowed in lookbehind assertion */ - REG_EESCAPE, /* PCRE does not support \L, \l, \N, \U, or \u */ - REG_BADPAT, /* number after (?C is > 255 */ - REG_BADPAT, /* closing ) for (?C expected */ - /* 40 */ - REG_BADPAT, /* recursive call could loop indefinitely */ - REG_BADPAT, /* unrecognized character after (?P */ - REG_BADPAT, /* syntax error in subpattern name (missing terminator) */ - REG_BADPAT, /* two named subpatterns have the same name */ - REG_BADPAT, /* invalid UTF-8 string */ - /* 45 */ - REG_BADPAT, /* support for \P, \p, and \X has not been compiled */ - REG_BADPAT, /* malformed \P or \p sequence */ - REG_BADPAT, /* unknown property name after \P or \p */ - REG_BADPAT, /* subpattern name is too long (maximum 32 characters) */ - REG_BADPAT, /* too many named subpatterns (maximum 10,000) */ - /* 50 */ - REG_BADPAT, /* repeated subpattern is too long */ - REG_BADPAT, /* octal value is greater than \377 (not in UTF-8 mode) */ - REG_BADPAT, /* internal error: overran compiling workspace */ - REG_BADPAT, /* internal error: previously-checked referenced subpattern not found */ - REG_BADPAT, /* DEFINE group contains more than one branch */ - /* 55 */ - REG_BADPAT, /* repeating a DEFINE group is not allowed */ - REG_INVARG, /* inconsistent NEWLINE options */ - REG_BADPAT, /* \g is not followed followed by an (optionally braced) non-zero number */ - REG_BADPAT, /* a numbered reference must not be zero */ - REG_BADPAT, /* an argument is not allowed for (*ACCEPT), (*FAIL), or (*COMMIT) */ - /* 60 */ - REG_BADPAT, /* (*VERB) not recognized */ - REG_BADPAT, /* number is too big */ - REG_BADPAT, /* subpattern name expected */ - REG_BADPAT, /* digit expected after (?+ */ - REG_BADPAT, /* ] is an invalid data character in JavaScript compatibility mode */ - /* 65 */ - REG_BADPAT, /* different names for subpatterns of the same number are not allowed */ - REG_BADPAT, /* (*MARK) must have an argument */ - REG_INVARG, /* this version of PCRE is not compiled with PCRE_UCP support */ - REG_BADPAT, /* \c must be followed by an ASCII character */ - REG_BADPAT, /* \k is not followed by a braced, angle-bracketed, or quoted name */ - /* 70 */ - REG_BADPAT, /* internal error: unknown opcode in find_fixedlength() */ - REG_BADPAT, /* \N is not supported in a class */ - REG_BADPAT, /* too many forward references */ - REG_BADPAT, /* disallowed UTF-8/16/32 code point (>= 0xd800 && <= 0xdfff) */ - REG_BADPAT, /* invalid UTF-16 string (should not occur) */ - /* 75 */ - REG_BADPAT, /* overlong MARK name */ - REG_BADPAT, /* character value in \u.... sequence is too large */ - REG_BADPAT, /* invalid UTF-32 string (should not occur) */ - REG_BADPAT, /* setting UTF is disabled by the application */ - REG_BADPAT, /* non-hex character in \\x{} (closing brace missing?) */ - /* 80 */ - REG_BADPAT, /* non-octal character in \o{} (closing brace missing?) */ - REG_BADPAT, /* missing opening brace after \o */ - REG_BADPAT, /* parentheses too deeply nested */ - REG_BADPAT, /* invalid range in character class */ - REG_BADPAT, /* group name must start with a non-digit */ - /* 85 */ - REG_BADPAT, /* parentheses too deeply nested (stack check) */ - REG_BADPAT, /* missing digits in \x{} or \o{} */ - REG_BADPAT /* pattern too complicated */ -}; - -/* Table of texts corresponding to POSIX error codes */ - -static const char *const pstring[] = { - "", /* Dummy for value 0 */ - "internal error", /* REG_ASSERT */ - "invalid repeat counts in {}", /* BADBR */ - "pattern error", /* BADPAT */ - "? * + invalid", /* BADRPT */ - "unbalanced {}", /* EBRACE */ - "unbalanced []", /* EBRACK */ - "collation error - not relevant", /* ECOLLATE */ - "bad class", /* ECTYPE */ - "bad escape sequence", /* EESCAPE */ - "empty expression", /* EMPTY */ - "unbalanced ()", /* EPAREN */ - "bad range inside []", /* ERANGE */ - "expression too big", /* ESIZE */ - "failed to get memory", /* ESPACE */ - "bad back reference", /* ESUBREG */ - "bad argument", /* INVARG */ - "match failed" /* NOMATCH */ -}; - - - - -/************************************************* -* Translate error code to string * -*************************************************/ - -PCREPOSIX_EXP_DEFN size_t PCRE_CALL_CONVENTION -regerror(int errcode, const regex_t *preg, char *errbuf, size_t errbuf_size) -{ -const char *message, *addmessage; -size_t length, addlength; - -message = (errcode >= (int)(sizeof(pstring)/sizeof(char *)))? - "unknown error code" : pstring[errcode]; -length = strlen(message) + 1; - -addmessage = " at offset "; -addlength = (preg != NULL && (int)preg->re_erroffset != -1)? - strlen(addmessage) + 6 : 0; - -if (errbuf_size > 0) - { - if (addlength > 0 && errbuf_size >= length + addlength) - sprintf(errbuf, "%s%s%-6d", message, addmessage, (int)preg->re_erroffset); - else - { - strncpy(errbuf, message, errbuf_size - 1); - errbuf[errbuf_size-1] = 0; - } - } - -return length + addlength; -} - - - - -/************************************************* -* Free store held by a regex * -*************************************************/ - -PCREPOSIX_EXP_DEFN void PCRE_CALL_CONVENTION -regfree(regex_t *preg) -{ -(PUBL(free))(preg->re_pcre); -} - - - - -/************************************************* -* Compile a regular expression * -*************************************************/ - -/* -Arguments: - preg points to a structure for recording the compiled expression - pattern the pattern to compile - cflags compilation flags - -Returns: 0 on success - various non-zero codes on failure -*/ - -PCREPOSIX_EXP_DEFN int PCRE_CALL_CONVENTION -regcomp(regex_t *preg, const char *pattern, int cflags) -{ -const char *errorptr; -int erroffset; -int errorcode; -int options = 0; -int re_nsub = 0; - -if ((cflags & REG_ICASE) != 0) options |= PCRE_CASELESS; -if ((cflags & REG_NEWLINE) != 0) options |= PCRE_MULTILINE; -if ((cflags & REG_DOTALL) != 0) options |= PCRE_DOTALL; -if ((cflags & REG_NOSUB) != 0) options |= PCRE_NO_AUTO_CAPTURE; -if ((cflags & REG_UTF8) != 0) options |= PCRE_UTF8; -if ((cflags & REG_UCP) != 0) options |= PCRE_UCP; -if ((cflags & REG_UNGREEDY) != 0) options |= PCRE_UNGREEDY; - -preg->re_pcre = pcre_compile2(pattern, options, &errorcode, &errorptr, - &erroffset, NULL); -preg->re_erroffset = erroffset; - -/* Safety: if the error code is too big for the translation vector (which -should not happen, but we all make mistakes), return REG_BADPAT. */ - -if (preg->re_pcre == NULL) - { - return (errorcode < (int)(sizeof(eint)/sizeof(const int)))? - eint[errorcode] : REG_BADPAT; - } - -(void)pcre_fullinfo((const pcre *)preg->re_pcre, NULL, PCRE_INFO_CAPTURECOUNT, - &re_nsub); -preg->re_nsub = (size_t)re_nsub; -return 0; -} - - - - -/************************************************* -* Match a regular expression * -*************************************************/ - -/* Unfortunately, PCRE requires 3 ints of working space for each captured -substring, so we have to get and release working store instead of just using -the POSIX structures as was done in earlier releases when PCRE needed only 2 -ints. However, if the number of possible capturing brackets is small, use a -block of store on the stack, to reduce the use of malloc/free. The threshold is -in a macro that can be changed at configure time. - -If REG_NOSUB was specified at compile time, the PCRE_NO_AUTO_CAPTURE flag will -be set. When this is the case, the nmatch and pmatch arguments are ignored, and -the only result is yes/no/error. */ - -PCREPOSIX_EXP_DEFN int PCRE_CALL_CONVENTION -regexec(const regex_t *preg, const char *string, size_t nmatch, - regmatch_t pmatch[], int eflags) -{ -int rc, so, eo; -int options = 0; -int *ovector = NULL; -int small_ovector[POSIX_MALLOC_THRESHOLD * 3]; -BOOL allocated_ovector = FALSE; -BOOL nosub = - (REAL_PCRE_OPTIONS((const pcre *)preg->re_pcre) & PCRE_NO_AUTO_CAPTURE) != 0; - -if ((eflags & REG_NOTBOL) != 0) options |= PCRE_NOTBOL; -if ((eflags & REG_NOTEOL) != 0) options |= PCRE_NOTEOL; -if ((eflags & REG_NOTEMPTY) != 0) options |= PCRE_NOTEMPTY; - -((regex_t *)preg)->re_erroffset = (size_t)(-1); /* Only has meaning after compile */ - -/* When no string data is being returned, or no vector has been passed in which -to put it, ensure that nmatch is zero. Otherwise, ensure the vector for holding -the return data is large enough. */ - -if (nosub || pmatch == NULL) nmatch = 0; - -else if (nmatch > 0) - { - if (nmatch <= POSIX_MALLOC_THRESHOLD) - { - ovector = &(small_ovector[0]); - } - else - { - if (nmatch > INT_MAX/(sizeof(int) * 3)) return REG_ESPACE; - ovector = (int *)malloc(sizeof(int) * nmatch * 3); - if (ovector == NULL) return REG_ESPACE; - allocated_ovector = TRUE; - } - } - -/* REG_STARTEND is a BSD extension, to allow for non-NUL-terminated strings. -The man page from OS X says "REG_STARTEND affects only the location of the -string, not how it is matched". That is why the "so" value is used to bump the -start location rather than being passed as a PCRE "starting offset". */ - -if ((eflags & REG_STARTEND) != 0) - { - if (pmatch == NULL) return REG_INVARG; - so = pmatch[0].rm_so; - eo = pmatch[0].rm_eo; - } -else - { - so = 0; - eo = (int)strlen(string); - } - -rc = pcre_exec((const pcre *)preg->re_pcre, NULL, string + so, (eo - so), - 0, options, ovector, (int)(nmatch * 3)); - -if (rc == 0) rc = (int)nmatch; /* All captured slots were filled in */ - -/* Successful match */ - -if (rc >= 0) - { - size_t i; - if (!nosub) - { - for (i = 0; i < (size_t)rc; i++) - { - pmatch[i].rm_so = ovector[i*2]; - pmatch[i].rm_eo = ovector[i*2+1]; - } - if (allocated_ovector) free(ovector); - for (; i < nmatch; i++) pmatch[i].rm_so = pmatch[i].rm_eo = -1; - } - return 0; - } - -/* Unsuccessful match */ - -if (allocated_ovector) free(ovector); -switch(rc) - { -/* ========================================================================== */ - /* These cases are never obeyed. This is a fudge that causes a compile-time - error if the vector eint, which is indexed by compile-time error number, is - not the correct length. It seems to be the only way to do such a check at - compile time, as the sizeof() operator does not work in the C preprocessor. - As all the PCRE_ERROR_xxx values are negative, we can use 0 and 1. */ - - case 0: - case (sizeof(eint)/sizeof(int) == ERRCOUNT): - return REG_ASSERT; -/* ========================================================================== */ - - case PCRE_ERROR_NOMATCH: return REG_NOMATCH; - case PCRE_ERROR_NULL: return REG_INVARG; - case PCRE_ERROR_BADOPTION: return REG_INVARG; - case PCRE_ERROR_BADMAGIC: return REG_INVARG; - case PCRE_ERROR_UNKNOWN_NODE: return REG_ASSERT; - case PCRE_ERROR_NOMEMORY: return REG_ESPACE; - case PCRE_ERROR_MATCHLIMIT: return REG_ESPACE; - case PCRE_ERROR_BADUTF8: return REG_INVARG; - case PCRE_ERROR_BADUTF8_OFFSET: return REG_INVARG; - case PCRE_ERROR_BADMODE: return REG_INVARG; - default: return REG_ASSERT; - } -} - -/* End of pcreposix.c */ diff --git a/src/third_party/pcre-8.39/pcreposix.h b/src/third_party/pcre-8.39/pcreposix.h deleted file mode 100644 index c77c0b0523c..00000000000 --- a/src/third_party/pcre-8.39/pcreposix.h +++ /dev/null @@ -1,146 +0,0 @@ -/************************************************* -* Perl-Compatible Regular Expressions * -*************************************************/ - -#ifndef _PCREPOSIX_H -#define _PCREPOSIX_H - -/* This is the header for the POSIX wrapper interface to the PCRE Perl- -Compatible Regular Expression library. It defines the things POSIX says should -be there. I hope. - - Copyright (c) 1997-2012 University of Cambridge - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* Have to include stdlib.h in order to ensure that size_t is defined. */ - -#include - -/* Allow for C++ users */ - -#ifdef __cplusplus -extern "C" { -#endif - -/* Options, mostly defined by POSIX, but with some extras. */ - -#define REG_ICASE 0x0001 /* Maps to PCRE_CASELESS */ -#define REG_NEWLINE 0x0002 /* Maps to PCRE_MULTILINE */ -#define REG_NOTBOL 0x0004 /* Maps to PCRE_NOTBOL */ -#define REG_NOTEOL 0x0008 /* Maps to PCRE_NOTEOL */ -#define REG_DOTALL 0x0010 /* NOT defined by POSIX; maps to PCRE_DOTALL */ -#define REG_NOSUB 0x0020 /* Maps to PCRE_NO_AUTO_CAPTURE */ -#define REG_UTF8 0x0040 /* NOT defined by POSIX; maps to PCRE_UTF8 */ -#define REG_STARTEND 0x0080 /* BSD feature: pass subject string by so,eo */ -#define REG_NOTEMPTY 0x0100 /* NOT defined by POSIX; maps to PCRE_NOTEMPTY */ -#define REG_UNGREEDY 0x0200 /* NOT defined by POSIX; maps to PCRE_UNGREEDY */ -#define REG_UCP 0x0400 /* NOT defined by POSIX; maps to PCRE_UCP */ - -/* This is not used by PCRE, but by defining it we make it easier -to slot PCRE into existing programs that make POSIX calls. */ - -#define REG_EXTENDED 0 - -/* Error values. Not all these are relevant or used by the wrapper. */ - -enum { - REG_ASSERT = 1, /* internal error ? */ - REG_BADBR, /* invalid repeat counts in {} */ - REG_BADPAT, /* pattern error */ - REG_BADRPT, /* ? * + invalid */ - REG_EBRACE, /* unbalanced {} */ - REG_EBRACK, /* unbalanced [] */ - REG_ECOLLATE, /* collation error - not relevant */ - REG_ECTYPE, /* bad class */ - REG_EESCAPE, /* bad escape sequence */ - REG_EMPTY, /* empty expression */ - REG_EPAREN, /* unbalanced () */ - REG_ERANGE, /* bad range inside [] */ - REG_ESIZE, /* expression too big */ - REG_ESPACE, /* failed to get memory */ - REG_ESUBREG, /* bad back reference */ - REG_INVARG, /* bad argument */ - REG_NOMATCH /* match failed */ -}; - - -/* The structure representing a compiled regular expression. */ - -typedef struct { - void *re_pcre; - size_t re_nsub; - size_t re_erroffset; -} regex_t; - -/* The structure in which a captured offset is returned. */ - -typedef int regoff_t; - -typedef struct { - regoff_t rm_so; - regoff_t rm_eo; -} regmatch_t; - -/* When an application links to a PCRE DLL in Windows, the symbols that are -imported have to be identified as such. When building PCRE, the appropriate -export settings are needed, and are set in pcreposix.c before including this -file. */ - -#if defined(_WIN32) && !defined(PCRE_STATIC) && !defined(PCREPOSIX_EXP_DECL) -# define PCREPOSIX_EXP_DECL extern __declspec(dllimport) -# define PCREPOSIX_EXP_DEFN __declspec(dllimport) -#endif - -/* By default, we use the standard "extern" declarations. */ - -#ifndef PCREPOSIX_EXP_DECL -# ifdef __cplusplus -# define PCREPOSIX_EXP_DECL extern "C" -# define PCREPOSIX_EXP_DEFN extern "C" -# else -# define PCREPOSIX_EXP_DECL extern -# define PCREPOSIX_EXP_DEFN extern -# endif -#endif - -/* The functions */ - -PCREPOSIX_EXP_DECL int regcomp(regex_t *, const char *, int); -PCREPOSIX_EXP_DECL int regexec(const regex_t *, const char *, size_t, - regmatch_t *, int); -PCREPOSIX_EXP_DECL size_t regerror(int, const regex_t *, char *, size_t); -PCREPOSIX_EXP_DECL void regfree(regex_t *); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* End of pcreposix.h */ diff --git a/src/third_party/pcre-8.39/pcretest.c b/src/third_party/pcre-8.39/pcretest.c deleted file mode 100644 index 78ef5177df7..00000000000 --- a/src/third_party/pcre-8.39/pcretest.c +++ /dev/null @@ -1,5754 +0,0 @@ -/************************************************* -* PCRE testing program * -*************************************************/ - -/* This program was hacked up as a tester for PCRE. I really should have -written it more tidily in the first place. Will I ever learn? It has grown and -been extended and consequently is now rather, er, *very* untidy in places. The -addition of 16-bit support has made it even worse. :-( - ------------------------------------------------------------------------------ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the University of Cambridge nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------ -*/ - -/* This program now supports the testing of all of the 8-bit, 16-bit, and -32-bit PCRE libraries in a single program. This is different from the modules -such as pcre_compile.c in the library itself, which are compiled separately for -each mode. If two modes are enabled, for example, pcre_compile.c is compiled -twice. By contrast, pcretest.c is compiled only once. Therefore, it must not -make use of any of the macros from pcre_internal.h that depend on -COMPILE_PCRE8, COMPILE_PCRE16, or COMPILE_PCRE32. It does, however, make use of -SUPPORT_PCRE8, SUPPORT_PCRE16, and SUPPORT_PCRE32 to ensure that it calls only -supported library functions. */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include -#include -#include -#include -#include -#include -#include - -/* Both libreadline and libedit are optionally supported. The user-supplied -original patch uses readline/readline.h for libedit, but in at least one system -it is installed as editline/readline.h, so the configuration code now looks for -that first, falling back to readline/readline.h. */ - -#if defined(SUPPORT_LIBREADLINE) || defined(SUPPORT_LIBEDIT) -#ifdef HAVE_UNISTD_H -#include -#endif -#if defined(SUPPORT_LIBREADLINE) -#include -#include -#else -#if defined(HAVE_EDITLINE_READLINE_H) -#include -#else -#include -#endif -#endif -#endif - -/* A number of things vary for Windows builds. Originally, pcretest opened its -input and output without "b"; then I was told that "b" was needed in some -environments, so it was added for release 5.0 to both the input and output. (It -makes no difference on Unix-like systems.) Later I was told that it is wrong -for the input on Windows. I've now abstracted the modes into two macros that -are set here, to make it easier to fiddle with them, and removed "b" from the -input mode under Windows. */ - -#if defined(_WIN32) || defined(WIN32) -#include /* For _setmode() */ -#include /* For _O_BINARY */ -#define INPUT_MODE "r" -#define OUTPUT_MODE "wb" - -#ifndef isatty -#define isatty _isatty /* This is what Windows calls them, I'm told, */ -#endif /* though in some environments they seem to */ - /* be already defined, hence the #ifndefs. */ -#ifndef fileno -#define fileno _fileno -#endif - -/* A user sent this fix for Borland Builder 5 under Windows. */ - -#ifdef __BORLANDC__ -#define _setmode(handle, mode) setmode(handle, mode) -#endif - -/* Not Windows */ - -#else -#include /* These two includes are needed */ -#include /* for setrlimit(). */ -#if defined NATIVE_ZOS /* z/OS uses non-binary I/O */ -#define INPUT_MODE "r" -#define OUTPUT_MODE "w" -#else -#define INPUT_MODE "rb" -#define OUTPUT_MODE "wb" -#endif -#endif - -#ifdef __VMS -#include -void vms_setsymbol( char *, char *, int ); -#endif - - -#define PRIV(name) name - -/* We have to include pcre_internal.h because we need the internal info for -displaying the results of pcre_study() and we also need to know about the -internal macros, structures, and other internal data values; pcretest has -"inside information" compared to a program that strictly follows the PCRE API. - -Although pcre_internal.h does itself include pcre.h, we explicitly include it -here before pcre_internal.h so that the PCRE_EXP_xxx macros get set -appropriately for an application, not for building PCRE. */ - -#include "pcre.h" -#include "pcre_internal.h" - -/* The pcre_printint() function, which prints the internal form of a compiled -regex, is held in a separate file so that (a) it can be compiled in either -8-, 16- or 32-bit mode, and (b) it can be #included directly in pcre_compile.c -when that is compiled in debug mode. */ - -#ifdef SUPPORT_PCRE8 -void pcre_printint(pcre *external_re, FILE *f, BOOL print_lengths); -#endif -#ifdef SUPPORT_PCRE16 -void pcre16_printint(pcre *external_re, FILE *f, BOOL print_lengths); -#endif -#ifdef SUPPORT_PCRE32 -void pcre32_printint(pcre *external_re, FILE *f, BOOL print_lengths); -#endif - -/* We need access to some of the data tables that PCRE uses. So as not to have -to keep two copies, we include the source files here, changing the names of the -external symbols to prevent clashes. */ - -#define PCRE_INCLUDED - -#include "pcre_tables.c" -#include "pcre_ucd.c" - -/* The definition of the macro PRINTABLE, which determines whether to print an -output character as-is or as a hex value when showing compiled patterns, is -the same as in the printint.src file. We uses it here in cases when the locale -has not been explicitly changed, so as to get consistent output from systems -that differ in their output from isprint() even in the "C" locale. */ - -#ifdef EBCDIC -#define PRINTABLE(c) ((c) >= 64 && (c) < 255) -#else -#define PRINTABLE(c) ((c) >= 32 && (c) < 127) -#endif - -#define PRINTOK(c) (locale_set? isprint(c) : PRINTABLE(c)) - -/* Posix support is disabled in 16 or 32 bit only mode. */ -#if !defined SUPPORT_PCRE8 && !defined NOPOSIX -#define NOPOSIX -#endif - -/* It is possible to compile this test program without including support for -testing the POSIX interface, though this is not available via the standard -Makefile. */ - -#if !defined NOPOSIX -#include "pcreposix.h" -#endif - -/* It is also possible, originally for the benefit of a version that was -imported into Exim, to build pcretest without support for UTF8 or UTF16 (define -NOUTF), without the interface to the DFA matcher (NODFA). In fact, we -automatically cut out the UTF support if PCRE is built without it. */ - -#ifndef SUPPORT_UTF -#ifndef NOUTF -#define NOUTF -#endif -#endif - -/* To make the code a bit tidier for 8/16/32-bit support, we define macros -for all the pcre[16]_xxx functions (except pcre16_fullinfo, which is called -only from one place and is handled differently). I couldn't dream up any way of -using a single macro to do this in a generic way, because of the many different -argument requirements. We know that at least one of SUPPORT_PCRE8 and -SUPPORT_PCRE16 must be set. First define macros for each individual mode; then -use these in the definitions of generic macros. - -**** Special note about the PCHARSxxx macros: the address of the string to be -printed is always given as two arguments: a base address followed by an offset. -The base address is cast to the correct data size for 8 or 16 bit data; the -offset is in units of this size. If the string were given as base+offset in one -argument, the casting might be incorrectly applied. */ - -#ifdef SUPPORT_PCRE8 - -#define PCHARS8(lv, p, offset, len, f) \ - lv = pchars((pcre_uint8 *)(p) + offset, len, f) - -#define PCHARSV8(p, offset, len, f) \ - (void)pchars((pcre_uint8 *)(p) + offset, len, f) - -#define READ_CAPTURE_NAME8(p, cn8, cn16, cn32, re) \ - p = read_capture_name8(p, cn8, re) - -#define STRLEN8(p) ((int)strlen((char *)p)) - -#define SET_PCRE_CALLOUT8(callout) \ - pcre_callout = callout - -#define SET_PCRE_STACK_GUARD8(stack_guard) \ - pcre_stack_guard = stack_guard - -#define PCRE_ASSIGN_JIT_STACK8(extra, callback, userdata) \ - pcre_assign_jit_stack(extra, callback, userdata) - -#define PCRE_COMPILE8(re, pat, options, error, erroffset, tables) \ - re = pcre_compile((char *)pat, options, error, erroffset, tables) - -#define PCRE_COPY_NAMED_SUBSTRING8(rc, re, bptr, offsets, count, \ - namesptr, cbuffer, size) \ - rc = pcre_copy_named_substring(re, (char *)bptr, offsets, count, \ - (char *)namesptr, cbuffer, size) - -#define PCRE_COPY_SUBSTRING8(rc, bptr, offsets, count, i, cbuffer, size) \ - rc = pcre_copy_substring((char *)bptr, offsets, count, i, cbuffer, size) - -#define PCRE_DFA_EXEC8(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets, workspace, size_workspace) \ - count = pcre_dfa_exec(re, extra, (char *)bptr, len, start_offset, options, \ - offsets, size_offsets, workspace, size_workspace) - -#define PCRE_EXEC8(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets) \ - count = pcre_exec(re, extra, (char *)bptr, len, start_offset, options, \ - offsets, size_offsets) - -#define PCRE_FREE_STUDY8(extra) \ - pcre_free_study(extra) - -#define PCRE_FREE_SUBSTRING8(substring) \ - pcre_free_substring(substring) - -#define PCRE_FREE_SUBSTRING_LIST8(listptr) \ - pcre_free_substring_list(listptr) - -#define PCRE_GET_NAMED_SUBSTRING8(rc, re, bptr, offsets, count, \ - getnamesptr, subsptr) \ - rc = pcre_get_named_substring(re, (char *)bptr, offsets, count, \ - (char *)getnamesptr, subsptr) - -#define PCRE_GET_STRINGNUMBER8(n, rc, ptr) \ - n = pcre_get_stringnumber(re, (char *)ptr) - -#define PCRE_GET_SUBSTRING8(rc, bptr, offsets, count, i, subsptr) \ - rc = pcre_get_substring((char *)bptr, offsets, count, i, subsptr) - -#define PCRE_GET_SUBSTRING_LIST8(rc, bptr, offsets, count, listptr) \ - rc = pcre_get_substring_list((const char *)bptr, offsets, count, listptr) - -#define PCRE_PATTERN_TO_HOST_BYTE_ORDER8(rc, re, extra, tables) \ - rc = pcre_pattern_to_host_byte_order(re, extra, tables) - -#define PCRE_PRINTINT8(re, outfile, debug_lengths) \ - pcre_printint(re, outfile, debug_lengths) - -#define PCRE_STUDY8(extra, re, options, error) \ - extra = pcre_study(re, options, error) - -#define PCRE_JIT_STACK_ALLOC8(startsize, maxsize) \ - pcre_jit_stack_alloc(startsize, maxsize) - -#define PCRE_JIT_STACK_FREE8(stack) \ - pcre_jit_stack_free(stack) - -#define pcre8_maketables pcre_maketables - -#endif /* SUPPORT_PCRE8 */ - -/* -----------------------------------------------------------*/ - -#ifdef SUPPORT_PCRE16 - -#define PCHARS16(lv, p, offset, len, f) \ - lv = pchars16((PCRE_SPTR16)(p) + offset, len, f) - -#define PCHARSV16(p, offset, len, f) \ - (void)pchars16((PCRE_SPTR16)(p) + offset, len, f) - -#define READ_CAPTURE_NAME16(p, cn8, cn16, cn32, re) \ - p = read_capture_name16(p, cn16, re) - -#define STRLEN16(p) ((int)strlen16((PCRE_SPTR16)p)) - -#define SET_PCRE_CALLOUT16(callout) \ - pcre16_callout = (int (*)(pcre16_callout_block *))callout - -#define SET_PCRE_STACK_GUARD16(stack_guard) \ - pcre16_stack_guard = (int (*)(void))stack_guard - -#define PCRE_ASSIGN_JIT_STACK16(extra, callback, userdata) \ - pcre16_assign_jit_stack((pcre16_extra *)extra, \ - (pcre16_jit_callback)callback, userdata) - -#define PCRE_COMPILE16(re, pat, options, error, erroffset, tables) \ - re = (pcre *)pcre16_compile((PCRE_SPTR16)pat, options, error, erroffset, \ - tables) - -#define PCRE_COPY_NAMED_SUBSTRING16(rc, re, bptr, offsets, count, \ - namesptr, cbuffer, size) \ - rc = pcre16_copy_named_substring((pcre16 *)re, (PCRE_SPTR16)bptr, offsets, \ - count, (PCRE_SPTR16)namesptr, (PCRE_UCHAR16 *)cbuffer, size/2) - -#define PCRE_COPY_SUBSTRING16(rc, bptr, offsets, count, i, cbuffer, size) \ - rc = pcre16_copy_substring((PCRE_SPTR16)bptr, offsets, count, i, \ - (PCRE_UCHAR16 *)cbuffer, size/2) - -#define PCRE_DFA_EXEC16(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets, workspace, size_workspace) \ - count = pcre16_dfa_exec((pcre16 *)re, (pcre16_extra *)extra, \ - (PCRE_SPTR16)bptr, len, start_offset, options, offsets, size_offsets, \ - workspace, size_workspace) - -#define PCRE_EXEC16(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets) \ - count = pcre16_exec((pcre16 *)re, (pcre16_extra *)extra, (PCRE_SPTR16)bptr, \ - len, start_offset, options, offsets, size_offsets) - -#define PCRE_FREE_STUDY16(extra) \ - pcre16_free_study((pcre16_extra *)extra) - -#define PCRE_FREE_SUBSTRING16(substring) \ - pcre16_free_substring((PCRE_SPTR16)substring) - -#define PCRE_FREE_SUBSTRING_LIST16(listptr) \ - pcre16_free_substring_list((PCRE_SPTR16 *)listptr) - -#define PCRE_GET_NAMED_SUBSTRING16(rc, re, bptr, offsets, count, \ - getnamesptr, subsptr) \ - rc = pcre16_get_named_substring((pcre16 *)re, (PCRE_SPTR16)bptr, offsets, \ - count, (PCRE_SPTR16)getnamesptr, (PCRE_SPTR16 *)(void*)subsptr) - -#define PCRE_GET_STRINGNUMBER16(n, rc, ptr) \ - n = pcre16_get_stringnumber(re, (PCRE_SPTR16)ptr) - -#define PCRE_GET_SUBSTRING16(rc, bptr, offsets, count, i, subsptr) \ - rc = pcre16_get_substring((PCRE_SPTR16)bptr, offsets, count, i, \ - (PCRE_SPTR16 *)(void*)subsptr) - -#define PCRE_GET_SUBSTRING_LIST16(rc, bptr, offsets, count, listptr) \ - rc = pcre16_get_substring_list((PCRE_SPTR16)bptr, offsets, count, \ - (PCRE_SPTR16 **)(void*)listptr) - -#define PCRE_PATTERN_TO_HOST_BYTE_ORDER16(rc, re, extra, tables) \ - rc = pcre16_pattern_to_host_byte_order((pcre16 *)re, (pcre16_extra *)extra, \ - tables) - -#define PCRE_PRINTINT16(re, outfile, debug_lengths) \ - pcre16_printint(re, outfile, debug_lengths) - -#define PCRE_STUDY16(extra, re, options, error) \ - extra = (pcre_extra *)pcre16_study((pcre16 *)re, options, error) - -#define PCRE_JIT_STACK_ALLOC16(startsize, maxsize) \ - (pcre_jit_stack *)pcre16_jit_stack_alloc(startsize, maxsize) - -#define PCRE_JIT_STACK_FREE16(stack) \ - pcre16_jit_stack_free((pcre16_jit_stack *)stack) - -#endif /* SUPPORT_PCRE16 */ - -/* -----------------------------------------------------------*/ - -#ifdef SUPPORT_PCRE32 - -#define PCHARS32(lv, p, offset, len, f) \ - lv = pchars32((PCRE_SPTR32)(p) + offset, len, use_utf, f) - -#define PCHARSV32(p, offset, len, f) \ - (void)pchars32((PCRE_SPTR32)(p) + offset, len, use_utf, f) - -#define READ_CAPTURE_NAME32(p, cn8, cn16, cn32, re) \ - p = read_capture_name32(p, cn32, re) - -#define STRLEN32(p) ((int)strlen32((PCRE_SPTR32)p)) - -#define SET_PCRE_CALLOUT32(callout) \ - pcre32_callout = (int (*)(pcre32_callout_block *))callout - -#define SET_PCRE_STACK_GUARD32(stack_guard) \ - pcre32_stack_guard = (int (*)(void))stack_guard - -#define PCRE_ASSIGN_JIT_STACK32(extra, callback, userdata) \ - pcre32_assign_jit_stack((pcre32_extra *)extra, \ - (pcre32_jit_callback)callback, userdata) - -#define PCRE_COMPILE32(re, pat, options, error, erroffset, tables) \ - re = (pcre *)pcre32_compile((PCRE_SPTR32)pat, options, error, erroffset, \ - tables) - -#define PCRE_COPY_NAMED_SUBSTRING32(rc, re, bptr, offsets, count, \ - namesptr, cbuffer, size) \ - rc = pcre32_copy_named_substring((pcre32 *)re, (PCRE_SPTR32)bptr, offsets, \ - count, (PCRE_SPTR32)namesptr, (PCRE_UCHAR32 *)cbuffer, size/2) - -#define PCRE_COPY_SUBSTRING32(rc, bptr, offsets, count, i, cbuffer, size) \ - rc = pcre32_copy_substring((PCRE_SPTR32)bptr, offsets, count, i, \ - (PCRE_UCHAR32 *)cbuffer, size/2) - -#define PCRE_DFA_EXEC32(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets, workspace, size_workspace) \ - count = pcre32_dfa_exec((pcre32 *)re, (pcre32_extra *)extra, \ - (PCRE_SPTR32)bptr, len, start_offset, options, offsets, size_offsets, \ - workspace, size_workspace) - -#define PCRE_EXEC32(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets) \ - count = pcre32_exec((pcre32 *)re, (pcre32_extra *)extra, (PCRE_SPTR32)bptr, \ - len, start_offset, options, offsets, size_offsets) - -#define PCRE_FREE_STUDY32(extra) \ - pcre32_free_study((pcre32_extra *)extra) - -#define PCRE_FREE_SUBSTRING32(substring) \ - pcre32_free_substring((PCRE_SPTR32)substring) - -#define PCRE_FREE_SUBSTRING_LIST32(listptr) \ - pcre32_free_substring_list((PCRE_SPTR32 *)listptr) - -#define PCRE_GET_NAMED_SUBSTRING32(rc, re, bptr, offsets, count, \ - getnamesptr, subsptr) \ - rc = pcre32_get_named_substring((pcre32 *)re, (PCRE_SPTR32)bptr, offsets, \ - count, (PCRE_SPTR32)getnamesptr, (PCRE_SPTR32 *)(void*)subsptr) - -#define PCRE_GET_STRINGNUMBER32(n, rc, ptr) \ - n = pcre32_get_stringnumber(re, (PCRE_SPTR32)ptr) - -#define PCRE_GET_SUBSTRING32(rc, bptr, offsets, count, i, subsptr) \ - rc = pcre32_get_substring((PCRE_SPTR32)bptr, offsets, count, i, \ - (PCRE_SPTR32 *)(void*)subsptr) - -#define PCRE_GET_SUBSTRING_LIST32(rc, bptr, offsets, count, listptr) \ - rc = pcre32_get_substring_list((PCRE_SPTR32)bptr, offsets, count, \ - (PCRE_SPTR32 **)(void*)listptr) - -#define PCRE_PATTERN_TO_HOST_BYTE_ORDER32(rc, re, extra, tables) \ - rc = pcre32_pattern_to_host_byte_order((pcre32 *)re, (pcre32_extra *)extra, \ - tables) - -#define PCRE_PRINTINT32(re, outfile, debug_lengths) \ - pcre32_printint(re, outfile, debug_lengths) - -#define PCRE_STUDY32(extra, re, options, error) \ - extra = (pcre_extra *)pcre32_study((pcre32 *)re, options, error) - -#define PCRE_JIT_STACK_ALLOC32(startsize, maxsize) \ - (pcre_jit_stack *)pcre32_jit_stack_alloc(startsize, maxsize) - -#define PCRE_JIT_STACK_FREE32(stack) \ - pcre32_jit_stack_free((pcre32_jit_stack *)stack) - -#endif /* SUPPORT_PCRE32 */ - - -/* ----- More than one mode is supported; a runtime test is needed, except for -pcre_config(), and the JIT stack functions, when it doesn't matter which -available version is called. ----- */ - -enum { - PCRE8_MODE, - PCRE16_MODE, - PCRE32_MODE -}; - -#if (defined (SUPPORT_PCRE8) + defined (SUPPORT_PCRE16) + \ - defined (SUPPORT_PCRE32)) >= 2 - -#define CHAR_SIZE (1 << pcre_mode) - -/* There doesn't seem to be an easy way of writing these macros that can cope -with the 3 pairs of bit sizes plus all three bit sizes. So just handle all the -cases separately. */ - -/* ----- All three modes supported ----- */ - -#if defined(SUPPORT_PCRE8) && defined(SUPPORT_PCRE16) && defined(SUPPORT_PCRE32) - -#define PCHARS(lv, p, offset, len, f) \ - if (pcre_mode == PCRE32_MODE) \ - PCHARS32(lv, p, offset, len, f); \ - else if (pcre_mode == PCRE16_MODE) \ - PCHARS16(lv, p, offset, len, f); \ - else \ - PCHARS8(lv, p, offset, len, f) - -#define PCHARSV(p, offset, len, f) \ - if (pcre_mode == PCRE32_MODE) \ - PCHARSV32(p, offset, len, f); \ - else if (pcre_mode == PCRE16_MODE) \ - PCHARSV16(p, offset, len, f); \ - else \ - PCHARSV8(p, offset, len, f) - -#define READ_CAPTURE_NAME(p, cn8, cn16, cn32, re) \ - if (pcre_mode == PCRE32_MODE) \ - READ_CAPTURE_NAME32(p, cn8, cn16, cn32, re); \ - else if (pcre_mode == PCRE16_MODE) \ - READ_CAPTURE_NAME16(p, cn8, cn16, cn32, re); \ - else \ - READ_CAPTURE_NAME8(p, cn8, cn16, cn32, re) - -#define SET_PCRE_CALLOUT(callout) \ - if (pcre_mode == PCRE32_MODE) \ - SET_PCRE_CALLOUT32(callout); \ - else if (pcre_mode == PCRE16_MODE) \ - SET_PCRE_CALLOUT16(callout); \ - else \ - SET_PCRE_CALLOUT8(callout) - -#define SET_PCRE_STACK_GUARD(stack_guard) \ - if (pcre_mode == PCRE32_MODE) \ - SET_PCRE_STACK_GUARD32(stack_guard); \ - else if (pcre_mode == PCRE16_MODE) \ - SET_PCRE_STACK_GUARD16(stack_guard); \ - else \ - SET_PCRE_STACK_GUARD8(stack_guard) - -#define STRLEN(p) (pcre_mode == PCRE32_MODE ? STRLEN32(p) : pcre_mode == PCRE16_MODE ? STRLEN16(p) : STRLEN8(p)) - -#define PCRE_ASSIGN_JIT_STACK(extra, callback, userdata) \ - if (pcre_mode == PCRE32_MODE) \ - PCRE_ASSIGN_JIT_STACK32(extra, callback, userdata); \ - else if (pcre_mode == PCRE16_MODE) \ - PCRE_ASSIGN_JIT_STACK16(extra, callback, userdata); \ - else \ - PCRE_ASSIGN_JIT_STACK8(extra, callback, userdata) - -#define PCRE_COMPILE(re, pat, options, error, erroffset, tables) \ - if (pcre_mode == PCRE32_MODE) \ - PCRE_COMPILE32(re, pat, options, error, erroffset, tables); \ - else if (pcre_mode == PCRE16_MODE) \ - PCRE_COMPILE16(re, pat, options, error, erroffset, tables); \ - else \ - PCRE_COMPILE8(re, pat, options, error, erroffset, tables) - -#define PCRE_CONFIG pcre_config - -#define PCRE_COPY_NAMED_SUBSTRING(rc, re, bptr, offsets, count, \ - namesptr, cbuffer, size) \ - if (pcre_mode == PCRE32_MODE) \ - PCRE_COPY_NAMED_SUBSTRING32(rc, re, bptr, offsets, count, \ - namesptr, cbuffer, size); \ - else if (pcre_mode == PCRE16_MODE) \ - PCRE_COPY_NAMED_SUBSTRING16(rc, re, bptr, offsets, count, \ - namesptr, cbuffer, size); \ - else \ - PCRE_COPY_NAMED_SUBSTRING8(rc, re, bptr, offsets, count, \ - namesptr, cbuffer, size) - -#define PCRE_COPY_SUBSTRING(rc, bptr, offsets, count, i, cbuffer, size) \ - if (pcre_mode == PCRE32_MODE) \ - PCRE_COPY_SUBSTRING32(rc, bptr, offsets, count, i, cbuffer, size); \ - else if (pcre_mode == PCRE16_MODE) \ - PCRE_COPY_SUBSTRING16(rc, bptr, offsets, count, i, cbuffer, size); \ - else \ - PCRE_COPY_SUBSTRING8(rc, bptr, offsets, count, i, cbuffer, size) - -#define PCRE_DFA_EXEC(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets, workspace, size_workspace) \ - if (pcre_mode == PCRE32_MODE) \ - PCRE_DFA_EXEC32(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets, workspace, size_workspace); \ - else if (pcre_mode == PCRE16_MODE) \ - PCRE_DFA_EXEC16(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets, workspace, size_workspace); \ - else \ - PCRE_DFA_EXEC8(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets, workspace, size_workspace) - -#define PCRE_EXEC(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets) \ - if (pcre_mode == PCRE32_MODE) \ - PCRE_EXEC32(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets); \ - else if (pcre_mode == PCRE16_MODE) \ - PCRE_EXEC16(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets); \ - else \ - PCRE_EXEC8(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets) - -#define PCRE_FREE_STUDY(extra) \ - if (pcre_mode == PCRE32_MODE) \ - PCRE_FREE_STUDY32(extra); \ - else if (pcre_mode == PCRE16_MODE) \ - PCRE_FREE_STUDY16(extra); \ - else \ - PCRE_FREE_STUDY8(extra) - -#define PCRE_FREE_SUBSTRING(substring) \ - if (pcre_mode == PCRE32_MODE) \ - PCRE_FREE_SUBSTRING32(substring); \ - else if (pcre_mode == PCRE16_MODE) \ - PCRE_FREE_SUBSTRING16(substring); \ - else \ - PCRE_FREE_SUBSTRING8(substring) - -#define PCRE_FREE_SUBSTRING_LIST(listptr) \ - if (pcre_mode == PCRE32_MODE) \ - PCRE_FREE_SUBSTRING_LIST32(listptr); \ - else if (pcre_mode == PCRE16_MODE) \ - PCRE_FREE_SUBSTRING_LIST16(listptr); \ - else \ - PCRE_FREE_SUBSTRING_LIST8(listptr) - -#define PCRE_GET_NAMED_SUBSTRING(rc, re, bptr, offsets, count, \ - getnamesptr, subsptr) \ - if (pcre_mode == PCRE32_MODE) \ - PCRE_GET_NAMED_SUBSTRING32(rc, re, bptr, offsets, count, \ - getnamesptr, subsptr); \ - else if (pcre_mode == PCRE16_MODE) \ - PCRE_GET_NAMED_SUBSTRING16(rc, re, bptr, offsets, count, \ - getnamesptr, subsptr); \ - else \ - PCRE_GET_NAMED_SUBSTRING8(rc, re, bptr, offsets, count, \ - getnamesptr, subsptr) - -#define PCRE_GET_STRINGNUMBER(n, rc, ptr) \ - if (pcre_mode == PCRE32_MODE) \ - PCRE_GET_STRINGNUMBER32(n, rc, ptr); \ - else if (pcre_mode == PCRE16_MODE) \ - PCRE_GET_STRINGNUMBER16(n, rc, ptr); \ - else \ - PCRE_GET_STRINGNUMBER8(n, rc, ptr) - -#define PCRE_GET_SUBSTRING(rc, bptr, use_offsets, count, i, subsptr) \ - if (pcre_mode == PCRE32_MODE) \ - PCRE_GET_SUBSTRING32(rc, bptr, use_offsets, count, i, subsptr); \ - else if (pcre_mode == PCRE16_MODE) \ - PCRE_GET_SUBSTRING16(rc, bptr, use_offsets, count, i, subsptr); \ - else \ - PCRE_GET_SUBSTRING8(rc, bptr, use_offsets, count, i, subsptr) - -#define PCRE_GET_SUBSTRING_LIST(rc, bptr, offsets, count, listptr) \ - if (pcre_mode == PCRE32_MODE) \ - PCRE_GET_SUBSTRING_LIST32(rc, bptr, offsets, count, listptr); \ - else if (pcre_mode == PCRE16_MODE) \ - PCRE_GET_SUBSTRING_LIST16(rc, bptr, offsets, count, listptr); \ - else \ - PCRE_GET_SUBSTRING_LIST8(rc, bptr, offsets, count, listptr) - -#define PCRE_JIT_STACK_ALLOC(startsize, maxsize) \ - (pcre_mode == PCRE32_MODE ? \ - PCRE_JIT_STACK_ALLOC32(startsize, maxsize) \ - : pcre_mode == PCRE16_MODE ? \ - PCRE_JIT_STACK_ALLOC16(startsize, maxsize) \ - : PCRE_JIT_STACK_ALLOC8(startsize, maxsize)) - -#define PCRE_JIT_STACK_FREE(stack) \ - if (pcre_mode == PCRE32_MODE) \ - PCRE_JIT_STACK_FREE32(stack); \ - else if (pcre_mode == PCRE16_MODE) \ - PCRE_JIT_STACK_FREE16(stack); \ - else \ - PCRE_JIT_STACK_FREE8(stack) - -#define PCRE_MAKETABLES \ - (pcre_mode == PCRE32_MODE ? pcre32_maketables() : pcre_mode == PCRE16_MODE ? pcre16_maketables() : pcre_maketables()) - -#define PCRE_PATTERN_TO_HOST_BYTE_ORDER(rc, re, extra, tables) \ - if (pcre_mode == PCRE32_MODE) \ - PCRE_PATTERN_TO_HOST_BYTE_ORDER32(rc, re, extra, tables); \ - else if (pcre_mode == PCRE16_MODE) \ - PCRE_PATTERN_TO_HOST_BYTE_ORDER16(rc, re, extra, tables); \ - else \ - PCRE_PATTERN_TO_HOST_BYTE_ORDER8(rc, re, extra, tables) - -#define PCRE_PRINTINT(re, outfile, debug_lengths) \ - if (pcre_mode == PCRE32_MODE) \ - PCRE_PRINTINT32(re, outfile, debug_lengths); \ - else if (pcre_mode == PCRE16_MODE) \ - PCRE_PRINTINT16(re, outfile, debug_lengths); \ - else \ - PCRE_PRINTINT8(re, outfile, debug_lengths) - -#define PCRE_STUDY(extra, re, options, error) \ - if (pcre_mode == PCRE32_MODE) \ - PCRE_STUDY32(extra, re, options, error); \ - else if (pcre_mode == PCRE16_MODE) \ - PCRE_STUDY16(extra, re, options, error); \ - else \ - PCRE_STUDY8(extra, re, options, error) - - -/* ----- Two out of three modes are supported ----- */ - -#else - -/* We can use some macro trickery to make a single set of definitions work in -the three different cases. */ - -/* ----- 32-bit and 16-bit but not 8-bit supported ----- */ - -#if defined(SUPPORT_PCRE32) && defined(SUPPORT_PCRE16) -#define BITONE 32 -#define BITTWO 16 - -/* ----- 32-bit and 8-bit but not 16-bit supported ----- */ - -#elif defined(SUPPORT_PCRE32) && defined(SUPPORT_PCRE8) -#define BITONE 32 -#define BITTWO 8 - -/* ----- 16-bit and 8-bit but not 32-bit supported ----- */ - -#else -#define BITONE 16 -#define BITTWO 8 -#endif - -#define glue(a,b) a##b -#define G(a,b) glue(a,b) - - -/* ----- Common macros for two-mode cases ----- */ - -#define PCHARS(lv, p, offset, len, f) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(PCHARS,BITONE)(lv, p, offset, len, f); \ - else \ - G(PCHARS,BITTWO)(lv, p, offset, len, f) - -#define PCHARSV(p, offset, len, f) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(PCHARSV,BITONE)(p, offset, len, f); \ - else \ - G(PCHARSV,BITTWO)(p, offset, len, f) - -#define READ_CAPTURE_NAME(p, cn8, cn16, cn32, re) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(READ_CAPTURE_NAME,BITONE)(p, cn8, cn16, cn32, re); \ - else \ - G(READ_CAPTURE_NAME,BITTWO)(p, cn8, cn16, cn32, re) - -#define SET_PCRE_CALLOUT(callout) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(SET_PCRE_CALLOUT,BITONE)(callout); \ - else \ - G(SET_PCRE_CALLOUT,BITTWO)(callout) - -#define SET_PCRE_STACK_GUARD(stack_guard) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(SET_PCRE_STACK_GUARD,BITONE)(stack_guard); \ - else \ - G(SET_PCRE_STACK_GUARD,BITTWO)(stack_guard) - -#define STRLEN(p) ((pcre_mode == G(G(PCRE,BITONE),_MODE)) ? \ - G(STRLEN,BITONE)(p) : G(STRLEN,BITTWO)(p)) - -#define PCRE_ASSIGN_JIT_STACK(extra, callback, userdata) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(PCRE_ASSIGN_JIT_STACK,BITONE)(extra, callback, userdata); \ - else \ - G(PCRE_ASSIGN_JIT_STACK,BITTWO)(extra, callback, userdata) - -#define PCRE_COMPILE(re, pat, options, error, erroffset, tables) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(PCRE_COMPILE,BITONE)(re, pat, options, error, erroffset, tables); \ - else \ - G(PCRE_COMPILE,BITTWO)(re, pat, options, error, erroffset, tables) - -#define PCRE_CONFIG G(G(pcre,BITONE),_config) - -#define PCRE_COPY_NAMED_SUBSTRING(rc, re, bptr, offsets, count, \ - namesptr, cbuffer, size) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(PCRE_COPY_NAMED_SUBSTRING,BITONE)(rc, re, bptr, offsets, count, \ - namesptr, cbuffer, size); \ - else \ - G(PCRE_COPY_NAMED_SUBSTRING,BITTWO)(rc, re, bptr, offsets, count, \ - namesptr, cbuffer, size) - -#define PCRE_COPY_SUBSTRING(rc, bptr, offsets, count, i, cbuffer, size) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(PCRE_COPY_SUBSTRING,BITONE)(rc, bptr, offsets, count, i, cbuffer, size); \ - else \ - G(PCRE_COPY_SUBSTRING,BITTWO)(rc, bptr, offsets, count, i, cbuffer, size) - -#define PCRE_DFA_EXEC(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets, workspace, size_workspace) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(PCRE_DFA_EXEC,BITONE)(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets, workspace, size_workspace); \ - else \ - G(PCRE_DFA_EXEC,BITTWO)(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets, workspace, size_workspace) - -#define PCRE_EXEC(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(PCRE_EXEC,BITONE)(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets); \ - else \ - G(PCRE_EXEC,BITTWO)(count, re, extra, bptr, len, start_offset, options, \ - offsets, size_offsets) - -#define PCRE_FREE_STUDY(extra) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(PCRE_FREE_STUDY,BITONE)(extra); \ - else \ - G(PCRE_FREE_STUDY,BITTWO)(extra) - -#define PCRE_FREE_SUBSTRING(substring) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(PCRE_FREE_SUBSTRING,BITONE)(substring); \ - else \ - G(PCRE_FREE_SUBSTRING,BITTWO)(substring) - -#define PCRE_FREE_SUBSTRING_LIST(listptr) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(PCRE_FREE_SUBSTRING_LIST,BITONE)(listptr); \ - else \ - G(PCRE_FREE_SUBSTRING_LIST,BITTWO)(listptr) - -#define PCRE_GET_NAMED_SUBSTRING(rc, re, bptr, offsets, count, \ - getnamesptr, subsptr) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(PCRE_GET_NAMED_SUBSTRING,BITONE)(rc, re, bptr, offsets, count, \ - getnamesptr, subsptr); \ - else \ - G(PCRE_GET_NAMED_SUBSTRING,BITTWO)(rc, re, bptr, offsets, count, \ - getnamesptr, subsptr) - -#define PCRE_GET_STRINGNUMBER(n, rc, ptr) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(PCRE_GET_STRINGNUMBER,BITONE)(n, rc, ptr); \ - else \ - G(PCRE_GET_STRINGNUMBER,BITTWO)(n, rc, ptr) - -#define PCRE_GET_SUBSTRING(rc, bptr, use_offsets, count, i, subsptr) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(PCRE_GET_SUBSTRING,BITONE)(rc, bptr, use_offsets, count, i, subsptr); \ - else \ - G(PCRE_GET_SUBSTRING,BITTWO)(rc, bptr, use_offsets, count, i, subsptr) - -#define PCRE_GET_SUBSTRING_LIST(rc, bptr, offsets, count, listptr) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(PCRE_GET_SUBSTRING_LIST,BITONE)(rc, bptr, offsets, count, listptr); \ - else \ - G(PCRE_GET_SUBSTRING_LIST,BITTWO)(rc, bptr, offsets, count, listptr) - -#define PCRE_JIT_STACK_ALLOC(startsize, maxsize) \ - (pcre_mode == G(G(PCRE,BITONE),_MODE)) ? \ - G(PCRE_JIT_STACK_ALLOC,BITONE)(startsize, maxsize) \ - : G(PCRE_JIT_STACK_ALLOC,BITTWO)(startsize, maxsize) - -#define PCRE_JIT_STACK_FREE(stack) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(PCRE_JIT_STACK_FREE,BITONE)(stack); \ - else \ - G(PCRE_JIT_STACK_FREE,BITTWO)(stack) - -#define PCRE_MAKETABLES \ - (pcre_mode == G(G(PCRE,BITONE),_MODE)) ? \ - G(G(pcre,BITONE),_maketables)() : G(G(pcre,BITTWO),_maketables)() - -#define PCRE_PATTERN_TO_HOST_BYTE_ORDER(rc, re, extra, tables) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(PCRE_PATTERN_TO_HOST_BYTE_ORDER,BITONE)(rc, re, extra, tables); \ - else \ - G(PCRE_PATTERN_TO_HOST_BYTE_ORDER,BITTWO)(rc, re, extra, tables) - -#define PCRE_PRINTINT(re, outfile, debug_lengths) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(PCRE_PRINTINT,BITONE)(re, outfile, debug_lengths); \ - else \ - G(PCRE_PRINTINT,BITTWO)(re, outfile, debug_lengths) - -#define PCRE_STUDY(extra, re, options, error) \ - if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ - G(PCRE_STUDY,BITONE)(extra, re, options, error); \ - else \ - G(PCRE_STUDY,BITTWO)(extra, re, options, error) - -#endif /* Two out of three modes */ - -/* ----- End of cases where more than one mode is supported ----- */ - - -/* ----- Only 8-bit mode is supported ----- */ - -#elif defined SUPPORT_PCRE8 -#define CHAR_SIZE 1 -#define PCHARS PCHARS8 -#define PCHARSV PCHARSV8 -#define READ_CAPTURE_NAME READ_CAPTURE_NAME8 -#define SET_PCRE_CALLOUT SET_PCRE_CALLOUT8 -#define SET_PCRE_STACK_GUARD SET_PCRE_STACK_GUARD8 -#define STRLEN STRLEN8 -#define PCRE_ASSIGN_JIT_STACK PCRE_ASSIGN_JIT_STACK8 -#define PCRE_COMPILE PCRE_COMPILE8 -#define PCRE_CONFIG pcre_config -#define PCRE_COPY_NAMED_SUBSTRING PCRE_COPY_NAMED_SUBSTRING8 -#define PCRE_COPY_SUBSTRING PCRE_COPY_SUBSTRING8 -#define PCRE_DFA_EXEC PCRE_DFA_EXEC8 -#define PCRE_EXEC PCRE_EXEC8 -#define PCRE_FREE_STUDY PCRE_FREE_STUDY8 -#define PCRE_FREE_SUBSTRING PCRE_FREE_SUBSTRING8 -#define PCRE_FREE_SUBSTRING_LIST PCRE_FREE_SUBSTRING_LIST8 -#define PCRE_GET_NAMED_SUBSTRING PCRE_GET_NAMED_SUBSTRING8 -#define PCRE_GET_STRINGNUMBER PCRE_GET_STRINGNUMBER8 -#define PCRE_GET_SUBSTRING PCRE_GET_SUBSTRING8 -#define PCRE_GET_SUBSTRING_LIST PCRE_GET_SUBSTRING_LIST8 -#define PCRE_JIT_STACK_ALLOC PCRE_JIT_STACK_ALLOC8 -#define PCRE_JIT_STACK_FREE PCRE_JIT_STACK_FREE8 -#define PCRE_MAKETABLES pcre_maketables() -#define PCRE_PATTERN_TO_HOST_BYTE_ORDER PCRE_PATTERN_TO_HOST_BYTE_ORDER8 -#define PCRE_PRINTINT PCRE_PRINTINT8 -#define PCRE_STUDY PCRE_STUDY8 - -/* ----- Only 16-bit mode is supported ----- */ - -#elif defined SUPPORT_PCRE16 -#define CHAR_SIZE 2 -#define PCHARS PCHARS16 -#define PCHARSV PCHARSV16 -#define READ_CAPTURE_NAME READ_CAPTURE_NAME16 -#define SET_PCRE_CALLOUT SET_PCRE_CALLOUT16 -#define SET_PCRE_STACK_GUARD SET_PCRE_STACK_GUARD16 -#define STRLEN STRLEN16 -#define PCRE_ASSIGN_JIT_STACK PCRE_ASSIGN_JIT_STACK16 -#define PCRE_COMPILE PCRE_COMPILE16 -#define PCRE_CONFIG pcre16_config -#define PCRE_COPY_NAMED_SUBSTRING PCRE_COPY_NAMED_SUBSTRING16 -#define PCRE_COPY_SUBSTRING PCRE_COPY_SUBSTRING16 -#define PCRE_DFA_EXEC PCRE_DFA_EXEC16 -#define PCRE_EXEC PCRE_EXEC16 -#define PCRE_FREE_STUDY PCRE_FREE_STUDY16 -#define PCRE_FREE_SUBSTRING PCRE_FREE_SUBSTRING16 -#define PCRE_FREE_SUBSTRING_LIST PCRE_FREE_SUBSTRING_LIST16 -#define PCRE_GET_NAMED_SUBSTRING PCRE_GET_NAMED_SUBSTRING16 -#define PCRE_GET_STRINGNUMBER PCRE_GET_STRINGNUMBER16 -#define PCRE_GET_SUBSTRING PCRE_GET_SUBSTRING16 -#define PCRE_GET_SUBSTRING_LIST PCRE_GET_SUBSTRING_LIST16 -#define PCRE_JIT_STACK_ALLOC PCRE_JIT_STACK_ALLOC16 -#define PCRE_JIT_STACK_FREE PCRE_JIT_STACK_FREE16 -#define PCRE_MAKETABLES pcre16_maketables() -#define PCRE_PATTERN_TO_HOST_BYTE_ORDER PCRE_PATTERN_TO_HOST_BYTE_ORDER16 -#define PCRE_PRINTINT PCRE_PRINTINT16 -#define PCRE_STUDY PCRE_STUDY16 - -/* ----- Only 32-bit mode is supported ----- */ - -#elif defined SUPPORT_PCRE32 -#define CHAR_SIZE 4 -#define PCHARS PCHARS32 -#define PCHARSV PCHARSV32 -#define READ_CAPTURE_NAME READ_CAPTURE_NAME32 -#define SET_PCRE_CALLOUT SET_PCRE_CALLOUT32 -#define SET_PCRE_STACK_GUARD SET_PCRE_STACK_GUARD32 -#define STRLEN STRLEN32 -#define PCRE_ASSIGN_JIT_STACK PCRE_ASSIGN_JIT_STACK32 -#define PCRE_COMPILE PCRE_COMPILE32 -#define PCRE_CONFIG pcre32_config -#define PCRE_COPY_NAMED_SUBSTRING PCRE_COPY_NAMED_SUBSTRING32 -#define PCRE_COPY_SUBSTRING PCRE_COPY_SUBSTRING32 -#define PCRE_DFA_EXEC PCRE_DFA_EXEC32 -#define PCRE_EXEC PCRE_EXEC32 -#define PCRE_FREE_STUDY PCRE_FREE_STUDY32 -#define PCRE_FREE_SUBSTRING PCRE_FREE_SUBSTRING32 -#define PCRE_FREE_SUBSTRING_LIST PCRE_FREE_SUBSTRING_LIST32 -#define PCRE_GET_NAMED_SUBSTRING PCRE_GET_NAMED_SUBSTRING32 -#define PCRE_GET_STRINGNUMBER PCRE_GET_STRINGNUMBER32 -#define PCRE_GET_SUBSTRING PCRE_GET_SUBSTRING32 -#define PCRE_GET_SUBSTRING_LIST PCRE_GET_SUBSTRING_LIST32 -#define PCRE_JIT_STACK_ALLOC PCRE_JIT_STACK_ALLOC32 -#define PCRE_JIT_STACK_FREE PCRE_JIT_STACK_FREE32 -#define PCRE_MAKETABLES pcre32_maketables() -#define PCRE_PATTERN_TO_HOST_BYTE_ORDER PCRE_PATTERN_TO_HOST_BYTE_ORDER32 -#define PCRE_PRINTINT PCRE_PRINTINT32 -#define PCRE_STUDY PCRE_STUDY32 - -#endif - -/* ----- End of mode-specific function call macros ----- */ - - -/* Other parameters */ - -#ifndef CLOCKS_PER_SEC -#ifdef CLK_TCK -#define CLOCKS_PER_SEC CLK_TCK -#else -#define CLOCKS_PER_SEC 100 -#endif -#endif - -#if !defined NODFA -#define DFA_WS_DIMENSION 1000 -#endif - -/* This is the default loop count for timing. */ - -#define LOOPREPEAT 500000 - -/* Static variables */ - -static FILE *outfile; -static int log_store = 0; -static int callout_count; -static int callout_extra; -static int callout_fail_count; -static int callout_fail_id; -static int debug_lengths; -static int first_callout; -static int jit_was_used; -static int locale_set = 0; -static int show_malloc; -static int stack_guard_return; -static int use_utf; -static const unsigned char *last_callout_mark = NULL; - -/* The buffers grow automatically if very long input lines are encountered. */ - -static int buffer_size = 50000; -static pcre_uint8 *buffer = NULL; -static pcre_uint8 *pbuffer = NULL; - -/* Just as a safety check, make sure that COMPILE_PCRE[16|32] are *not* set. */ - -#ifdef COMPILE_PCRE16 -#error COMPILE_PCRE16 must not be set when compiling pcretest.c -#endif - -#ifdef COMPILE_PCRE32 -#error COMPILE_PCRE32 must not be set when compiling pcretest.c -#endif - -/* We need buffers for building 16/32-bit strings, and the tables of operator -lengths that are used for 16/32-bit compiling, in order to swap bytes in a -pattern for saving/reloading testing. Luckily, the data for these tables is -defined as a macro. However, we must ensure that LINK_SIZE and IMM2_SIZE (which -are used in the tables) are adjusted appropriately for the 16/32-bit world. -LINK_SIZE is also used later in this program. */ - -#ifdef SUPPORT_PCRE16 -#undef IMM2_SIZE -#define IMM2_SIZE 1 - -#if LINK_SIZE == 2 -#undef LINK_SIZE -#define LINK_SIZE 1 -#elif LINK_SIZE == 3 || LINK_SIZE == 4 -#undef LINK_SIZE -#define LINK_SIZE 2 -#else -#error LINK_SIZE must be either 2, 3, or 4 -#endif - -static int buffer16_size = 0; -static pcre_uint16 *buffer16 = NULL; -static const pcre_uint16 OP_lengths16[] = { OP_LENGTHS }; -#endif /* SUPPORT_PCRE16 */ - -#ifdef SUPPORT_PCRE32 -#undef IMM2_SIZE -#define IMM2_SIZE 1 -#undef LINK_SIZE -#define LINK_SIZE 1 - -static int buffer32_size = 0; -static pcre_uint32 *buffer32 = NULL; -static const pcre_uint32 OP_lengths32[] = { OP_LENGTHS }; -#endif /* SUPPORT_PCRE32 */ - -/* If we have 8-bit support, default to it; if there is also 16-or 32-bit -support, it can be changed by an option. If there is no 8-bit support, there -must be 16-or 32-bit support, so default it to 1. */ - -#if defined SUPPORT_PCRE8 -static int pcre_mode = PCRE8_MODE; -#elif defined SUPPORT_PCRE16 -static int pcre_mode = PCRE16_MODE; -#elif defined SUPPORT_PCRE32 -static int pcre_mode = PCRE32_MODE; -#endif - -/* JIT study options for -s+n and /S+n where '1' <= n <= '7'. */ - -static int jit_study_bits[] = - { - PCRE_STUDY_JIT_COMPILE, - PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE, - PCRE_STUDY_JIT_COMPILE + PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE, - PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE, - PCRE_STUDY_JIT_COMPILE + PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE, - PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE + PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE, - PCRE_STUDY_JIT_COMPILE + PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE + - PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE -}; - -#define PCRE_STUDY_ALLJIT (PCRE_STUDY_JIT_COMPILE | \ - PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE | PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE) - -/* Textual explanations for runtime error codes */ - -static const char *errtexts[] = { - NULL, /* 0 is no error */ - NULL, /* NOMATCH is handled specially */ - "NULL argument passed", - "bad option value", - "magic number missing", - "unknown opcode - pattern overwritten?", - "no more memory", - NULL, /* never returned by pcre_exec() or pcre_dfa_exec() */ - "match limit exceeded", - "callout error code", - NULL, /* BADUTF8/16 is handled specially */ - NULL, /* BADUTF8/16 offset is handled specially */ - NULL, /* PARTIAL is handled specially */ - "not used - internal error", - "internal error - pattern overwritten?", - "bad count value", - "item unsupported for DFA matching", - "backreference condition or recursion test not supported for DFA matching", - "match limit not supported for DFA matching", - "workspace size exceeded in DFA matching", - "too much recursion for DFA matching", - "recursion limit exceeded", - "not used - internal error", - "invalid combination of newline options", - "bad offset value", - NULL, /* SHORTUTF8/16 is handled specially */ - "nested recursion at the same subject position", - "JIT stack limit reached", - "pattern compiled in wrong mode: 8-bit/16-bit error", - "pattern compiled with other endianness", - "invalid data in workspace for DFA restart", - "bad JIT option", - "bad length" -}; - - -/************************************************* -* Alternate character tables * -*************************************************/ - -/* By default, the "tables" pointer when calling PCRE is set to NULL, thereby -using the default tables of the library. However, the T option can be used to -select alternate sets of tables, for different kinds of testing. Note also that -the L (locale) option also adjusts the tables. */ - -/* This is the set of tables distributed as default with PCRE. It recognizes -only ASCII characters. */ - -static const pcre_uint8 tables0[] = { - -/* This table is a lower casing table. */ - - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, - 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 38, 39, - 40, 41, 42, 43, 44, 45, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, - 56, 57, 58, 59, 60, 61, 62, 63, - 64, 97, 98, 99,100,101,102,103, - 104,105,106,107,108,109,110,111, - 112,113,114,115,116,117,118,119, - 120,121,122, 91, 92, 93, 94, 95, - 96, 97, 98, 99,100,101,102,103, - 104,105,106,107,108,109,110,111, - 112,113,114,115,116,117,118,119, - 120,121,122,123,124,125,126,127, - 128,129,130,131,132,133,134,135, - 136,137,138,139,140,141,142,143, - 144,145,146,147,148,149,150,151, - 152,153,154,155,156,157,158,159, - 160,161,162,163,164,165,166,167, - 168,169,170,171,172,173,174,175, - 176,177,178,179,180,181,182,183, - 184,185,186,187,188,189,190,191, - 192,193,194,195,196,197,198,199, - 200,201,202,203,204,205,206,207, - 208,209,210,211,212,213,214,215, - 216,217,218,219,220,221,222,223, - 224,225,226,227,228,229,230,231, - 232,233,234,235,236,237,238,239, - 240,241,242,243,244,245,246,247, - 248,249,250,251,252,253,254,255, - -/* This table is a case flipping table. */ - - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, - 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 38, 39, - 40, 41, 42, 43, 44, 45, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, - 56, 57, 58, 59, 60, 61, 62, 63, - 64, 97, 98, 99,100,101,102,103, - 104,105,106,107,108,109,110,111, - 112,113,114,115,116,117,118,119, - 120,121,122, 91, 92, 93, 94, 95, - 96, 65, 66, 67, 68, 69, 70, 71, - 72, 73, 74, 75, 76, 77, 78, 79, - 80, 81, 82, 83, 84, 85, 86, 87, - 88, 89, 90,123,124,125,126,127, - 128,129,130,131,132,133,134,135, - 136,137,138,139,140,141,142,143, - 144,145,146,147,148,149,150,151, - 152,153,154,155,156,157,158,159, - 160,161,162,163,164,165,166,167, - 168,169,170,171,172,173,174,175, - 176,177,178,179,180,181,182,183, - 184,185,186,187,188,189,190,191, - 192,193,194,195,196,197,198,199, - 200,201,202,203,204,205,206,207, - 208,209,210,211,212,213,214,215, - 216,217,218,219,220,221,222,223, - 224,225,226,227,228,229,230,231, - 232,233,234,235,236,237,238,239, - 240,241,242,243,244,245,246,247, - 248,249,250,251,252,253,254,255, - -/* This table contains bit maps for various character classes. Each map is 32 -bytes long and the bits run from the least significant end of each byte. The -classes that have their own maps are: space, xdigit, digit, upper, lower, word, -graph, print, punct, and cntrl. Other classes are built from combinations. */ - - 0x00,0x3e,0x00,0x00,0x01,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, - 0x7e,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0xfe,0xff,0xff,0x07,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0x07, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, - 0xfe,0xff,0xff,0x87,0xfe,0xff,0xff,0x07, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0xff, - 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0xff,0xff,0xff,0xff, - 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0x00,0x00,0x00,0x00,0xfe,0xff,0x00,0xfc, - 0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0x78, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - - 0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x80, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - -/* This table identifies various classes of character by individual bits: - 0x01 white space character - 0x02 letter - 0x04 decimal digit - 0x08 hexadecimal digit - 0x10 alphanumeric or '_' - 0x80 regular expression metacharacter or binary zero -*/ - - 0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 0- 7 */ - 0x00,0x01,0x01,0x01,0x01,0x01,0x00,0x00, /* 8- 15 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 16- 23 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 24- 31 */ - 0x01,0x00,0x00,0x00,0x80,0x00,0x00,0x00, /* - ' */ - 0x80,0x80,0x80,0x80,0x00,0x00,0x80,0x00, /* ( - / */ - 0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c, /* 0 - 7 */ - 0x1c,0x1c,0x00,0x00,0x00,0x00,0x00,0x80, /* 8 - ? */ - 0x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /* @ - G */ - 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* H - O */ - 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* P - W */ - 0x12,0x12,0x12,0x80,0x80,0x00,0x80,0x10, /* X - _ */ - 0x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /* ` - g */ - 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* h - o */ - 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* p - w */ - 0x12,0x12,0x12,0x80,0x80,0x00,0x00,0x00, /* x -127 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 128-135 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 136-143 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 144-151 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 152-159 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 160-167 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 168-175 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 176-183 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 184-191 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 192-199 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 200-207 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 208-215 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 216-223 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 224-231 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 232-239 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 240-247 */ - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00};/* 248-255 */ - -/* This is a set of tables that came originally from a Windows user. It seems -to be at least an approximation of ISO 8859. In particular, there are -characters greater than 128 that are marked as spaces, letters, etc. */ - -static const pcre_uint8 tables1[] = { -0,1,2,3,4,5,6,7, -8,9,10,11,12,13,14,15, -16,17,18,19,20,21,22,23, -24,25,26,27,28,29,30,31, -32,33,34,35,36,37,38,39, -40,41,42,43,44,45,46,47, -48,49,50,51,52,53,54,55, -56,57,58,59,60,61,62,63, -64,97,98,99,100,101,102,103, -104,105,106,107,108,109,110,111, -112,113,114,115,116,117,118,119, -120,121,122,91,92,93,94,95, -96,97,98,99,100,101,102,103, -104,105,106,107,108,109,110,111, -112,113,114,115,116,117,118,119, -120,121,122,123,124,125,126,127, -128,129,130,131,132,133,134,135, -136,137,138,139,140,141,142,143, -144,145,146,147,148,149,150,151, -152,153,154,155,156,157,158,159, -160,161,162,163,164,165,166,167, -168,169,170,171,172,173,174,175, -176,177,178,179,180,181,182,183, -184,185,186,187,188,189,190,191, -224,225,226,227,228,229,230,231, -232,233,234,235,236,237,238,239, -240,241,242,243,244,245,246,215, -248,249,250,251,252,253,254,223, -224,225,226,227,228,229,230,231, -232,233,234,235,236,237,238,239, -240,241,242,243,244,245,246,247, -248,249,250,251,252,253,254,255, -0,1,2,3,4,5,6,7, -8,9,10,11,12,13,14,15, -16,17,18,19,20,21,22,23, -24,25,26,27,28,29,30,31, -32,33,34,35,36,37,38,39, -40,41,42,43,44,45,46,47, -48,49,50,51,52,53,54,55, -56,57,58,59,60,61,62,63, -64,97,98,99,100,101,102,103, -104,105,106,107,108,109,110,111, -112,113,114,115,116,117,118,119, -120,121,122,91,92,93,94,95, -96,65,66,67,68,69,70,71, -72,73,74,75,76,77,78,79, -80,81,82,83,84,85,86,87, -88,89,90,123,124,125,126,127, -128,129,130,131,132,133,134,135, -136,137,138,139,140,141,142,143, -144,145,146,147,148,149,150,151, -152,153,154,155,156,157,158,159, -160,161,162,163,164,165,166,167, -168,169,170,171,172,173,174,175, -176,177,178,179,180,181,182,183, -184,185,186,187,188,189,190,191, -224,225,226,227,228,229,230,231, -232,233,234,235,236,237,238,239, -240,241,242,243,244,245,246,215, -248,249,250,251,252,253,254,223, -192,193,194,195,196,197,198,199, -200,201,202,203,204,205,206,207, -208,209,210,211,212,213,214,247, -216,217,218,219,220,221,222,255, -0,62,0,0,1,0,0,0, -0,0,0,0,0,0,0,0, -32,0,0,0,1,0,0,0, -0,0,0,0,0,0,0,0, -0,0,0,0,0,0,255,3, -126,0,0,0,126,0,0,0, -0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0, -0,0,0,0,0,0,255,3, -0,0,0,0,0,0,0,0, -0,0,0,0,0,0,12,2, -0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0, -254,255,255,7,0,0,0,0, -0,0,0,0,0,0,0,0, -255,255,127,127,0,0,0,0, -0,0,0,0,0,0,0,0, -0,0,0,0,254,255,255,7, -0,0,0,0,0,4,32,4, -0,0,0,128,255,255,127,255, -0,0,0,0,0,0,255,3, -254,255,255,135,254,255,255,7, -0,0,0,0,0,4,44,6, -255,255,127,255,255,255,127,255, -0,0,0,0,254,255,255,255, -255,255,255,255,255,255,255,127, -0,0,0,0,254,255,255,255, -255,255,255,255,255,255,255,255, -0,2,0,0,255,255,255,255, -255,255,255,255,255,255,255,127, -0,0,0,0,255,255,255,255, -255,255,255,255,255,255,255,255, -0,0,0,0,254,255,0,252, -1,0,0,248,1,0,0,120, -0,0,0,0,254,255,255,255, -0,0,128,0,0,0,128,0, -255,255,255,255,0,0,0,0, -0,0,0,0,0,0,0,128, -255,255,255,255,0,0,0,0, -0,0,0,0,0,0,0,0, -128,0,0,0,0,0,0,0, -0,1,1,0,1,1,0,0, -0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0, -1,0,0,0,128,0,0,0, -128,128,128,128,0,0,128,0, -28,28,28,28,28,28,28,28, -28,28,0,0,0,0,0,128, -0,26,26,26,26,26,26,18, -18,18,18,18,18,18,18,18, -18,18,18,18,18,18,18,18, -18,18,18,128,128,0,128,16, -0,26,26,26,26,26,26,18, -18,18,18,18,18,18,18,18, -18,18,18,18,18,18,18,18, -18,18,18,128,128,0,0,0, -0,0,0,0,0,1,0,0, -0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0, -0,0,0,0,0,0,0,0, -1,0,0,0,0,0,0,0, -0,0,18,0,0,0,0,0, -0,0,20,20,0,18,0,0, -0,20,18,0,0,0,0,0, -18,18,18,18,18,18,18,18, -18,18,18,18,18,18,18,18, -18,18,18,18,18,18,18,0, -18,18,18,18,18,18,18,18, -18,18,18,18,18,18,18,18, -18,18,18,18,18,18,18,18, -18,18,18,18,18,18,18,0, -18,18,18,18,18,18,18,18 -}; - - - - -#ifndef HAVE_STRERROR -/************************************************* -* Provide strerror() for non-ANSI libraries * -*************************************************/ - -/* Some old-fashioned systems still around (e.g. SunOS4) don't have strerror() -in their libraries, but can provide the same facility by this simple -alternative function. */ - -extern int sys_nerr; -extern char *sys_errlist[]; - -char * -strerror(int n) -{ -if (n < 0 || n >= sys_nerr) return "unknown error number"; -return sys_errlist[n]; -} -#endif /* HAVE_STRERROR */ - - - -/************************************************* -* Print newline configuration * -*************************************************/ - -/* -Arguments: - rc the return code from PCRE_CONFIG_NEWLINE - isc TRUE if called from "-C newline" -Returns: nothing -*/ - -static void -print_newline_config(int rc, BOOL isc) -{ -const char *s = NULL; -if (!isc) printf(" Newline sequence is "); -switch(rc) - { - case CHAR_CR: s = "CR"; break; - case CHAR_LF: s = "LF"; break; - case (CHAR_CR<<8 | CHAR_LF): s = "CRLF"; break; - case -1: s = "ANY"; break; - case -2: s = "ANYCRLF"; break; - - default: - printf("a non-standard value: 0x%04x\n", rc); - return; - } - -printf("%s\n", s); -} - - - -/************************************************* -* JIT memory callback * -*************************************************/ - -static pcre_jit_stack* jit_callback(void *arg) -{ -jit_was_used = TRUE; -return (pcre_jit_stack *)arg; -} - - -#if !defined NOUTF || defined SUPPORT_PCRE16 || defined SUPPORT_PCRE32 -/************************************************* -* Convert UTF-8 string to value * -*************************************************/ - -/* This function takes one or more bytes that represents a UTF-8 character, -and returns the value of the character. - -Argument: - utf8bytes a pointer to the byte vector - vptr a pointer to an int to receive the value - -Returns: > 0 => the number of bytes consumed - -6 to 0 => malformed UTF-8 character at offset = (-return) -*/ - -static int -utf82ord(pcre_uint8 *utf8bytes, pcre_uint32 *vptr) -{ -pcre_uint32 c = *utf8bytes++; -pcre_uint32 d = c; -int i, j, s; - -for (i = -1; i < 6; i++) /* i is number of additional bytes */ - { - if ((d & 0x80) == 0) break; - d <<= 1; - } - -if (i == -1) { *vptr = c; return 1; } /* ascii character */ -if (i == 0 || i == 6) return 0; /* invalid UTF-8 */ - -/* i now has a value in the range 1-5 */ - -s = 6*i; -d = (c & utf8_table3[i]) << s; - -for (j = 0; j < i; j++) - { - c = *utf8bytes++; - if ((c & 0xc0) != 0x80) return -(j+1); - s -= 6; - d |= (c & 0x3f) << s; - } - -/* Check that encoding was the correct unique one */ - -for (j = 0; j < utf8_table1_size; j++) - if (d <= (pcre_uint32)utf8_table1[j]) break; -if (j != i) return -(i+1); - -/* Valid value */ - -*vptr = d; -return i+1; -} -#endif /* NOUTF || SUPPORT_PCRE16 */ - - - -#if defined SUPPORT_PCRE8 && !defined NOUTF -/************************************************* -* Convert character value to UTF-8 * -*************************************************/ - -/* This function takes an integer value in the range 0 - 0x7fffffff -and encodes it as a UTF-8 character in 0 to 6 bytes. - -Arguments: - cvalue the character value - utf8bytes pointer to buffer for result - at least 6 bytes long - -Returns: number of characters placed in the buffer -*/ - -static int -ord2utf8(pcre_uint32 cvalue, pcre_uint8 *utf8bytes) -{ -register int i, j; -if (cvalue > 0x7fffffffu) - return -1; -for (i = 0; i < utf8_table1_size; i++) - if (cvalue <= (pcre_uint32)utf8_table1[i]) break; -utf8bytes += i; -for (j = i; j > 0; j--) - { - *utf8bytes-- = 0x80 | (cvalue & 0x3f); - cvalue >>= 6; - } -*utf8bytes = utf8_table2[i] | cvalue; -return i + 1; -} -#endif - - -#ifdef SUPPORT_PCRE16 -/************************************************* -* Convert a string to 16-bit * -*************************************************/ - -/* In non-UTF mode, the space needed for a 16-bit string is exactly double the -8-bit size. For a UTF-8 string, the size needed for UTF-16 is no more than -double, because up to 0xffff uses no more than 3 bytes in UTF-8 but possibly 4 -in UTF-16. Higher values use 4 bytes in UTF-8 and up to 4 bytes in UTF-16. The -result is always left in buffer16. - -Note that this function does not object to surrogate values. This is -deliberate; it makes it possible to construct UTF-16 strings that are invalid, -for the purpose of testing that they are correctly faulted. - -Patterns to be converted are either plain ASCII or UTF-8; data lines are always -in UTF-8 so that values greater than 255 can be handled. - -Arguments: - data TRUE if converting a data line; FALSE for a regex - p points to a byte string - utf true if UTF-8 (to be converted to UTF-16) - len number of bytes in the string (excluding trailing zero) - -Returns: number of 16-bit data items used (excluding trailing zero) - OR -1 if a UTF-8 string is malformed - OR -2 if a value > 0x10ffff is encountered - OR -3 if a value > 0xffff is encountered when not in UTF mode -*/ - -static int -to16(int data, pcre_uint8 *p, int utf, int len) -{ -pcre_uint16 *pp; - -if (buffer16_size < 2*len + 2) - { - if (buffer16 != NULL) free(buffer16); - buffer16_size = 2*len + 2; - buffer16 = (pcre_uint16 *)malloc(buffer16_size); - if (buffer16 == NULL) - { - fprintf(stderr, "pcretest: malloc(%d) failed for buffer16\n", buffer16_size); - exit(1); - } - } - -pp = buffer16; - -if (!utf && !data) - { - while (len-- > 0) *pp++ = *p++; - } - -else - { - pcre_uint32 c = 0; - while (len > 0) - { - int chlen = utf82ord(p, &c); - if (chlen <= 0) return -1; - if (c > 0x10ffff) return -2; - p += chlen; - len -= chlen; - if (c < 0x10000) *pp++ = c; else - { - if (!utf) return -3; - c -= 0x10000; - *pp++ = 0xD800 | (c >> 10); - *pp++ = 0xDC00 | (c & 0x3ff); - } - } - } - -*pp = 0; -return pp - buffer16; -} -#endif - -#ifdef SUPPORT_PCRE32 -/************************************************* -* Convert a string to 32-bit * -*************************************************/ - -/* In non-UTF mode, the space needed for a 32-bit string is exactly four times the -8-bit size. For a UTF-8 string, the size needed for UTF-32 is no more than four -times, because up to 0xffff uses no more than 3 bytes in UTF-8 but possibly 4 -in UTF-32. Higher values use 4 bytes in UTF-8 and up to 4 bytes in UTF-32. The -result is always left in buffer32. - -Note that this function does not object to surrogate values. This is -deliberate; it makes it possible to construct UTF-32 strings that are invalid, -for the purpose of testing that they are correctly faulted. - -Patterns to be converted are either plain ASCII or UTF-8; data lines are always -in UTF-8 so that values greater than 255 can be handled. - -Arguments: - data TRUE if converting a data line; FALSE for a regex - p points to a byte string - utf true if UTF-8 (to be converted to UTF-32) - len number of bytes in the string (excluding trailing zero) - -Returns: number of 32-bit data items used (excluding trailing zero) - OR -1 if a UTF-8 string is malformed - OR -2 if a value > 0x10ffff is encountered - OR -3 if an ill-formed value is encountered (i.e. a surrogate) -*/ - -static int -to32(int data, pcre_uint8 *p, int utf, int len) -{ -pcre_uint32 *pp; - -if (buffer32_size < 4*len + 4) - { - if (buffer32 != NULL) free(buffer32); - buffer32_size = 4*len + 4; - buffer32 = (pcre_uint32 *)malloc(buffer32_size); - if (buffer32 == NULL) - { - fprintf(stderr, "pcretest: malloc(%d) failed for buffer32\n", buffer32_size); - exit(1); - } - } - -pp = buffer32; - -if (!utf && !data) - { - while (len-- > 0) *pp++ = *p++; - } - -else - { - pcre_uint32 c = 0; - while (len > 0) - { - int chlen = utf82ord(p, &c); - if (chlen <= 0) return -1; - if (utf) - { - if (c > 0x10ffff) return -2; - if (!data && (c & 0xfffff800u) == 0xd800u) return -3; - } - - p += chlen; - len -= chlen; - *pp++ = c; - } - } - -*pp = 0; -return pp - buffer32; -} - -/* Check that a 32-bit character string is valid UTF-32. - -Arguments: - string points to the string - length length of string, or -1 if the string is zero-terminated - -Returns: TRUE if the string is a valid UTF-32 string - FALSE otherwise -*/ - -#ifdef NEVER /* Not used */ -#ifdef SUPPORT_UTF -static BOOL -valid_utf32(pcre_uint32 *string, int length) -{ -register pcre_uint32 *p; -register pcre_uint32 c; - -for (p = string; length-- > 0; p++) - { - c = *p; - if (c > 0x10ffffu) return FALSE; /* Too big */ - if ((c & 0xfffff800u) == 0xd800u) return FALSE; /* Surrogate */ - } - -return TRUE; -} -#endif /* SUPPORT_UTF */ -#endif /* NEVER */ -#endif /* SUPPORT_PCRE32 */ - - -/************************************************* -* Read or extend an input line * -*************************************************/ - -/* Input lines are read into buffer, but both patterns and data lines can be -continued over multiple input lines. In addition, if the buffer fills up, we -want to automatically expand it so as to be able to handle extremely large -lines that are needed for certain stress tests. When the input buffer is -expanded, the other two buffers must also be expanded likewise, and the -contents of pbuffer, which are a copy of the input for callouts, must be -preserved (for when expansion happens for a data line). This is not the most -optimal way of handling this, but hey, this is just a test program! - -Arguments: - f the file to read - start where in buffer to start (this *must* be within buffer) - prompt for stdin or readline() - -Returns: pointer to the start of new data - could be a copy of start, or could be moved - NULL if no data read and EOF reached -*/ - -static pcre_uint8 * -extend_inputline(FILE *f, pcre_uint8 *start, const char *prompt) -{ -pcre_uint8 *here = start; - -for (;;) - { - size_t rlen = (size_t)(buffer_size - (here - buffer)); - - if (rlen > 1000) - { - int dlen; - - /* If libreadline or libedit support is required, use readline() to read a - line if the input is a terminal. Note that readline() removes the trailing - newline, so we must put it back again, to be compatible with fgets(). */ - -#if defined(SUPPORT_LIBREADLINE) || defined(SUPPORT_LIBEDIT) - if (isatty(fileno(f))) - { - size_t len; - char *s = readline(prompt); - if (s == NULL) return (here == start)? NULL : start; - len = strlen(s); - if (len > 0) add_history(s); - if (len > rlen - 1) len = rlen - 1; - memcpy(here, s, len); - here[len] = '\n'; - here[len+1] = 0; - free(s); - } - else -#endif - - /* Read the next line by normal means, prompting if the file is stdin. */ - - { - if (f == stdin) printf("%s", prompt); - if (fgets((char *)here, rlen, f) == NULL) - return (here == start)? NULL : start; - } - - dlen = (int)strlen((char *)here); - if (dlen > 0 && here[dlen - 1] == '\n') return start; - here += dlen; - } - - else - { - int new_buffer_size = 2*buffer_size; - pcre_uint8 *new_buffer = (pcre_uint8 *)malloc(new_buffer_size); - pcre_uint8 *new_pbuffer = (pcre_uint8 *)malloc(new_buffer_size); - - if (new_buffer == NULL || new_pbuffer == NULL) - { - fprintf(stderr, "pcretest: malloc(%d) failed\n", new_buffer_size); - exit(1); - } - - memcpy(new_buffer, buffer, buffer_size); - memcpy(new_pbuffer, pbuffer, buffer_size); - - buffer_size = new_buffer_size; - - start = new_buffer + (start - buffer); - here = new_buffer + (here - buffer); - - free(buffer); - free(pbuffer); - - buffer = new_buffer; - pbuffer = new_pbuffer; - } - } - -/* Control never gets here */ -} - - - -/************************************************* -* Read number from string * -*************************************************/ - -/* We don't use strtoul() because SunOS4 doesn't have it. Rather than mess -around with conditional compilation, just do the job by hand. It is only used -for unpicking arguments, so just keep it simple. - -Arguments: - str string to be converted - endptr where to put the end pointer - -Returns: the unsigned long -*/ - -static int -get_value(pcre_uint8 *str, pcre_uint8 **endptr) -{ -int result = 0; -while(*str != 0 && isspace(*str)) str++; -while (isdigit(*str)) result = result * 10 + (int)(*str++ - '0'); -*endptr = str; -return(result); -} - - - -/************************************************* -* Print one character * -*************************************************/ - -/* Print a single character either literally, or as a hex escape. */ - -static int pchar(pcre_uint32 c, FILE *f) -{ -int n = 0; -if (PRINTOK(c)) - { - if (f != NULL) fprintf(f, "%c", c); - return 1; - } - -if (c < 0x100) - { - if (use_utf) - { - if (f != NULL) fprintf(f, "\\x{%02x}", c); - return 6; - } - else - { - if (f != NULL) fprintf(f, "\\x%02x", c); - return 4; - } - } - -if (f != NULL) n = fprintf(f, "\\x{%02x}", c); -return n >= 0 ? n : 0; -} - - - -#ifdef SUPPORT_PCRE8 -/************************************************* -* Print 8-bit character string * -*************************************************/ - -/* Must handle UTF-8 strings in utf8 mode. Yields number of characters printed. -If handed a NULL file, just counts chars without printing. */ - -static int pchars(pcre_uint8 *p, int length, FILE *f) -{ -pcre_uint32 c = 0; -int yield = 0; - -if (length < 0) - length = strlen((char *)p); - -while (length-- > 0) - { -#if !defined NOUTF - if (use_utf) - { - int rc = utf82ord(p, &c); - if (rc > 0 && rc <= length + 1) /* Mustn't run over the end */ - { - length -= rc - 1; - p += rc; - yield += pchar(c, f); - continue; - } - } -#endif - c = *p++; - yield += pchar(c, f); - } - -return yield; -} -#endif - - - -#ifdef SUPPORT_PCRE16 -/************************************************* -* Find length of 0-terminated 16-bit string * -*************************************************/ - -static int strlen16(PCRE_SPTR16 p) -{ -PCRE_SPTR16 pp = p; -while (*pp != 0) pp++; -return (int)(pp - p); -} -#endif /* SUPPORT_PCRE16 */ - - - -#ifdef SUPPORT_PCRE32 -/************************************************* -* Find length of 0-terminated 32-bit string * -*************************************************/ - -static int strlen32(PCRE_SPTR32 p) -{ -PCRE_SPTR32 pp = p; -while (*pp != 0) pp++; -return (int)(pp - p); -} -#endif /* SUPPORT_PCRE32 */ - - - -#ifdef SUPPORT_PCRE16 -/************************************************* -* Print 16-bit character string * -*************************************************/ - -/* Must handle UTF-16 strings in utf mode. Yields number of characters printed. -If handed a NULL file, just counts chars without printing. */ - -static int pchars16(PCRE_SPTR16 p, int length, FILE *f) -{ -int yield = 0; - -if (length < 0) - length = strlen16(p); - -while (length-- > 0) - { - pcre_uint32 c = *p++ & 0xffff; -#if !defined NOUTF - if (use_utf && c >= 0xD800 && c < 0xDC00 && length > 0) - { - int d = *p & 0xffff; - if (d >= 0xDC00 && d <= 0xDFFF) - { - c = ((c & 0x3ff) << 10) + (d & 0x3ff) + 0x10000; - length--; - p++; - } - } -#endif - yield += pchar(c, f); - } - -return yield; -} -#endif /* SUPPORT_PCRE16 */ - - - -#ifdef SUPPORT_PCRE32 -/************************************************* -* Print 32-bit character string * -*************************************************/ - -/* Must handle UTF-32 strings in utf mode. Yields number of characters printed. -If handed a NULL file, just counts chars without printing. */ - -static int pchars32(PCRE_SPTR32 p, int length, BOOL utf, FILE *f) -{ -int yield = 0; - -(void)(utf); /* Avoid compiler warning */ - -if (length < 0) - length = strlen32(p); - -while (length-- > 0) - { - pcre_uint32 c = *p++; - yield += pchar(c, f); - } - -return yield; -} -#endif /* SUPPORT_PCRE32 */ - - - -#ifdef SUPPORT_PCRE8 -/************************************************* -* Read a capture name (8-bit) and check it * -*************************************************/ - -static pcre_uint8 * -read_capture_name8(pcre_uint8 *p, pcre_uint8 **pp, pcre *re) -{ -pcre_uint8 *npp = *pp; -while (isalnum(*p)) *npp++ = *p++; -*npp++ = 0; -*npp = 0; -if (pcre_get_stringnumber(re, (char *)(*pp)) < 0) - { - fprintf(outfile, "no parentheses with name \""); - PCHARSV(*pp, 0, -1, outfile); - fprintf(outfile, "\"\n"); - } - -*pp = npp; -return p; -} -#endif /* SUPPORT_PCRE8 */ - - - -#ifdef SUPPORT_PCRE16 -/************************************************* -* Read a capture name (16-bit) and check it * -*************************************************/ - -/* Note that the text being read is 8-bit. */ - -static pcre_uint8 * -read_capture_name16(pcre_uint8 *p, pcre_uint16 **pp, pcre *re) -{ -pcre_uint16 *npp = *pp; -while (isalnum(*p)) *npp++ = *p++; -*npp++ = 0; -*npp = 0; -if (pcre16_get_stringnumber((pcre16 *)re, (PCRE_SPTR16)(*pp)) < 0) - { - fprintf(outfile, "no parentheses with name \""); - PCHARSV(*pp, 0, -1, outfile); - fprintf(outfile, "\"\n"); - } -*pp = npp; -return p; -} -#endif /* SUPPORT_PCRE16 */ - - - -#ifdef SUPPORT_PCRE32 -/************************************************* -* Read a capture name (32-bit) and check it * -*************************************************/ - -/* Note that the text being read is 8-bit. */ - -static pcre_uint8 * -read_capture_name32(pcre_uint8 *p, pcre_uint32 **pp, pcre *re) -{ -pcre_uint32 *npp = *pp; -while (isalnum(*p)) *npp++ = *p++; -*npp++ = 0; -*npp = 0; -if (pcre32_get_stringnumber((pcre32 *)re, (PCRE_SPTR32)(*pp)) < 0) - { - fprintf(outfile, "no parentheses with name \""); - PCHARSV(*pp, 0, -1, outfile); - fprintf(outfile, "\"\n"); - } -*pp = npp; -return p; -} -#endif /* SUPPORT_PCRE32 */ - - - -/************************************************* -* Stack guard function * -*************************************************/ - -/* Called from PCRE when set in pcre_stack_guard. We give an error (non-zero) -return when a count overflows. */ - -static int stack_guard(void) -{ -return stack_guard_return; -} - -/************************************************* -* Callout function * -*************************************************/ - -/* Called from PCRE as a result of the (?C) item. We print out where we are in -the match. Yield zero unless more callouts than the fail count, or the callout -data is not zero. */ - -static int callout(pcre_callout_block *cb) -{ -FILE *f = (first_callout | callout_extra)? outfile : NULL; -int i, current_position, pre_start, post_start, subject_length; - -if (callout_extra) - { - fprintf(f, "Callout %d: last capture = %d\n", - cb->callout_number, cb->capture_last); - - if (cb->offset_vector != NULL) - { - for (i = 0; i < cb->capture_top * 2; i += 2) - { - if (cb->offset_vector[i] < 0) - fprintf(f, "%2d: \n", i/2); - else - { - fprintf(f, "%2d: ", i/2); - PCHARSV(cb->subject, cb->offset_vector[i], - cb->offset_vector[i+1] - cb->offset_vector[i], f); - fprintf(f, "\n"); - } - } - } - } - -/* Re-print the subject in canonical form, the first time or if giving full -datails. On subsequent calls in the same match, we use pchars just to find the -printed lengths of the substrings. */ - -if (f != NULL) fprintf(f, "--->"); - -/* If a lookbehind is involved, the current position may be earlier than the -match start. If so, use the match start instead. */ - -current_position = (cb->current_position >= cb->start_match)? - cb->current_position : cb->start_match; - -PCHARS(pre_start, cb->subject, 0, cb->start_match, f); -PCHARS(post_start, cb->subject, cb->start_match, - current_position - cb->start_match, f); - -PCHARS(subject_length, cb->subject, 0, cb->subject_length, NULL); - -PCHARSV(cb->subject, current_position, cb->subject_length - current_position, f); - -if (f != NULL) fprintf(f, "\n"); - -/* Always print appropriate indicators, with callout number if not already -shown. For automatic callouts, show the pattern offset. */ - -if (cb->callout_number == 255) - { - fprintf(outfile, "%+3d ", cb->pattern_position); - if (cb->pattern_position > 99) fprintf(outfile, "\n "); - } -else - { - if (callout_extra) fprintf(outfile, " "); - else fprintf(outfile, "%3d ", cb->callout_number); - } - -for (i = 0; i < pre_start; i++) fprintf(outfile, " "); -fprintf(outfile, "^"); - -if (post_start > 0) - { - for (i = 0; i < post_start - 1; i++) fprintf(outfile, " "); - fprintf(outfile, "^"); - } - -for (i = 0; i < subject_length - pre_start - post_start + 4; i++) - fprintf(outfile, " "); - -fprintf(outfile, "%.*s", (cb->next_item_length == 0)? 1 : cb->next_item_length, - pbuffer + cb->pattern_position); - -fprintf(outfile, "\n"); -first_callout = 0; - -if (cb->mark != last_callout_mark) - { - if (cb->mark == NULL) - fprintf(outfile, "Latest Mark: \n"); - else - { - fprintf(outfile, "Latest Mark: "); - PCHARSV(cb->mark, 0, -1, outfile); - putc('\n', outfile); - } - last_callout_mark = cb->mark; - } - -if (cb->callout_data != NULL) - { - int callout_data = *((int *)(cb->callout_data)); - if (callout_data != 0) - { - fprintf(outfile, "Callout data = %d\n", callout_data); - return callout_data; - } - } - -return (cb->callout_number != callout_fail_id)? 0 : - (++callout_count >= callout_fail_count)? 1 : 0; -} - - -/************************************************* -* Local malloc functions * -*************************************************/ - -/* Alternative malloc function, to test functionality and save the size of a -compiled re, which is the first store request that pcre_compile() makes. The -show_malloc variable is set only during matching. */ - -static void *new_malloc(size_t size) -{ -void *block = malloc(size); -if (show_malloc) - fprintf(outfile, "malloc %3d %p\n", (int)size, block); -return block; -} - -static void new_free(void *block) -{ -if (show_malloc) - fprintf(outfile, "free %p\n", block); -free(block); -} - -/* For recursion malloc/free, to test stacking calls */ - -static void *stack_malloc(size_t size) -{ -void *block = malloc(size); -if (show_malloc) - fprintf(outfile, "stack_malloc %3d %p\n", (int)size, block); -return block; -} - -static void stack_free(void *block) -{ -if (show_malloc) - fprintf(outfile, "stack_free %p\n", block); -free(block); -} - - -/************************************************* -* Call pcre_fullinfo() * -*************************************************/ - -/* Get one piece of information from the pcre_fullinfo() function. When only -one of 8-, 16- or 32-bit is supported, pcre_mode should always have the correct -value, but the code is defensive. - -Arguments: - re compiled regex - study study data - option PCRE_INFO_xxx option - ptr where to put the data - -Returns: 0 when OK, < 0 on error -*/ - -static int -new_info(pcre *re, pcre_extra *study, int option, void *ptr) -{ -int rc; - -if (pcre_mode == PCRE32_MODE) -#ifdef SUPPORT_PCRE32 - rc = pcre32_fullinfo((pcre32 *)re, (pcre32_extra *)study, option, ptr); -#else - rc = PCRE_ERROR_BADMODE; -#endif -else if (pcre_mode == PCRE16_MODE) -#ifdef SUPPORT_PCRE16 - rc = pcre16_fullinfo((pcre16 *)re, (pcre16_extra *)study, option, ptr); -#else - rc = PCRE_ERROR_BADMODE; -#endif -else -#ifdef SUPPORT_PCRE8 - rc = pcre_fullinfo(re, study, option, ptr); -#else - rc = PCRE_ERROR_BADMODE; -#endif - -if (rc < 0 && rc != PCRE_ERROR_UNSET) - { - fprintf(outfile, "Error %d from pcre%s_fullinfo(%d)\n", rc, - pcre_mode == PCRE32_MODE ? "32" : pcre_mode == PCRE16_MODE ? "16" : "", option); - if (rc == PCRE_ERROR_BADMODE) - fprintf(outfile, "Running in %d-bit mode but pattern was compiled in " - "%d-bit mode\n", 8 * CHAR_SIZE, - 8 * (REAL_PCRE_FLAGS(re) & PCRE_MODE_MASK)); - } - -return rc; -} - - - -/************************************************* -* Swap byte functions * -*************************************************/ - -/* The following functions swap the bytes of a pcre_uint16 and pcre_uint32 -value, respectively. - -Arguments: - value any number - -Returns: the byte swapped value -*/ - -static pcre_uint32 -swap_uint32(pcre_uint32 value) -{ -return ((value & 0x000000ff) << 24) | - ((value & 0x0000ff00) << 8) | - ((value & 0x00ff0000) >> 8) | - (value >> 24); -} - -static pcre_uint16 -swap_uint16(pcre_uint16 value) -{ -return (value >> 8) | (value << 8); -} - - - -/************************************************* -* Flip bytes in a compiled pattern * -*************************************************/ - -/* This function is called if the 'F' option was present on a pattern that is -to be written to a file. We flip the bytes of all the integer fields in the -regex data block and the study block. In 16-bit mode this also flips relevant -bytes in the pattern itself. This is to make it possible to test PCRE's -ability to reload byte-flipped patterns, e.g. those compiled on a different -architecture. */ - -#if defined SUPPORT_PCRE8 || defined SUPPORT_PCRE16 -static void -regexflip8_or_16(pcre *ere, pcre_extra *extra) -{ -real_pcre8_or_16 *re = (real_pcre8_or_16 *)ere; -#ifdef SUPPORT_PCRE16 -int op; -pcre_uint16 *ptr = (pcre_uint16 *)re + re->name_table_offset; -int length = re->name_count * re->name_entry_size; -#ifdef SUPPORT_UTF -BOOL utf = (re->options & PCRE_UTF16) != 0; -BOOL utf16_char = FALSE; -#endif /* SUPPORT_UTF */ -#endif /* SUPPORT_PCRE16 */ - -/* Always flip the bytes in the main data block and study blocks. */ - -re->magic_number = REVERSED_MAGIC_NUMBER; -re->size = swap_uint32(re->size); -re->options = swap_uint32(re->options); -re->flags = swap_uint32(re->flags); -re->limit_match = swap_uint32(re->limit_match); -re->limit_recursion = swap_uint32(re->limit_recursion); -re->first_char = swap_uint16(re->first_char); -re->req_char = swap_uint16(re->req_char); -re->max_lookbehind = swap_uint16(re->max_lookbehind); -re->top_bracket = swap_uint16(re->top_bracket); -re->top_backref = swap_uint16(re->top_backref); -re->name_table_offset = swap_uint16(re->name_table_offset); -re->name_entry_size = swap_uint16(re->name_entry_size); -re->name_count = swap_uint16(re->name_count); -re->ref_count = swap_uint16(re->ref_count); - -if (extra != NULL && (extra->flags & PCRE_EXTRA_STUDY_DATA) != 0) - { - pcre_study_data *rsd = (pcre_study_data *)(extra->study_data); - rsd->size = swap_uint32(rsd->size); - rsd->flags = swap_uint32(rsd->flags); - rsd->minlength = swap_uint32(rsd->minlength); - } - -/* In 8-bit mode, that is all we need to do. In 16-bit mode we must swap bytes -in the name table, if present, and then in the pattern itself. */ - -#ifdef SUPPORT_PCRE16 -if (pcre_mode != PCRE16_MODE) return; - -while(TRUE) - { - /* Swap previous characters. */ - while (length-- > 0) - { - *ptr = swap_uint16(*ptr); - ptr++; - } -#ifdef SUPPORT_UTF - if (utf16_char) - { - if ((ptr[-1] & 0xfc00) == 0xd800) - { - /* We know that there is only one extra character in UTF-16. */ - *ptr = swap_uint16(*ptr); - ptr++; - } - } - utf16_char = FALSE; -#endif /* SUPPORT_UTF */ - - /* Get next opcode. */ - - length = 0; - op = *ptr; - *ptr++ = swap_uint16(op); - - switch (op) - { - case OP_END: - return; - -#ifdef SUPPORT_UTF - case OP_CHAR: - case OP_CHARI: - case OP_NOT: - case OP_NOTI: - case OP_STAR: - case OP_MINSTAR: - case OP_PLUS: - case OP_MINPLUS: - case OP_QUERY: - case OP_MINQUERY: - case OP_UPTO: - case OP_MINUPTO: - case OP_EXACT: - case OP_POSSTAR: - case OP_POSPLUS: - case OP_POSQUERY: - case OP_POSUPTO: - case OP_STARI: - case OP_MINSTARI: - case OP_PLUSI: - case OP_MINPLUSI: - case OP_QUERYI: - case OP_MINQUERYI: - case OP_UPTOI: - case OP_MINUPTOI: - case OP_EXACTI: - case OP_POSSTARI: - case OP_POSPLUSI: - case OP_POSQUERYI: - case OP_POSUPTOI: - case OP_NOTSTAR: - case OP_NOTMINSTAR: - case OP_NOTPLUS: - case OP_NOTMINPLUS: - case OP_NOTQUERY: - case OP_NOTMINQUERY: - case OP_NOTUPTO: - case OP_NOTMINUPTO: - case OP_NOTEXACT: - case OP_NOTPOSSTAR: - case OP_NOTPOSPLUS: - case OP_NOTPOSQUERY: - case OP_NOTPOSUPTO: - case OP_NOTSTARI: - case OP_NOTMINSTARI: - case OP_NOTPLUSI: - case OP_NOTMINPLUSI: - case OP_NOTQUERYI: - case OP_NOTMINQUERYI: - case OP_NOTUPTOI: - case OP_NOTMINUPTOI: - case OP_NOTEXACTI: - case OP_NOTPOSSTARI: - case OP_NOTPOSPLUSI: - case OP_NOTPOSQUERYI: - case OP_NOTPOSUPTOI: - if (utf) utf16_char = TRUE; -#endif - /* Fall through. */ - - default: - length = OP_lengths16[op] - 1; - break; - - case OP_CLASS: - case OP_NCLASS: - /* Skip the character bit map. */ - ptr += 32/sizeof(pcre_uint16); - length = 0; - break; - - case OP_XCLASS: - /* LINK_SIZE can be 1 or 2 in 16 bit mode. */ - if (LINK_SIZE > 1) - length = (int)((((unsigned int)(ptr[0]) << 16) | (unsigned int)(ptr[1])) - - (1 + LINK_SIZE + 1)); - else - length = (int)((unsigned int)(ptr[0]) - (1 + LINK_SIZE + 1)); - - /* Reverse the size of the XCLASS instance. */ - *ptr = swap_uint16(*ptr); - ptr++; - if (LINK_SIZE > 1) - { - *ptr = swap_uint16(*ptr); - ptr++; - } - - op = *ptr; - *ptr = swap_uint16(op); - ptr++; - if ((op & XCL_MAP) != 0) - { - /* Skip the character bit map. */ - ptr += 32/sizeof(pcre_uint16); - length -= 32/sizeof(pcre_uint16); - } - break; - } - } -/* Control should never reach here in 16 bit mode. */ -#endif /* SUPPORT_PCRE16 */ -} -#endif /* SUPPORT_PCRE[8|16] */ - - - -#if defined SUPPORT_PCRE32 -static void -regexflip_32(pcre *ere, pcre_extra *extra) -{ -real_pcre32 *re = (real_pcre32 *)ere; -int op; -pcre_uint32 *ptr = (pcre_uint32 *)re + re->name_table_offset; -int length = re->name_count * re->name_entry_size; - -/* Always flip the bytes in the main data block and study blocks. */ - -re->magic_number = REVERSED_MAGIC_NUMBER; -re->size = swap_uint32(re->size); -re->options = swap_uint32(re->options); -re->flags = swap_uint32(re->flags); -re->limit_match = swap_uint32(re->limit_match); -re->limit_recursion = swap_uint32(re->limit_recursion); -re->first_char = swap_uint32(re->first_char); -re->req_char = swap_uint32(re->req_char); -re->max_lookbehind = swap_uint16(re->max_lookbehind); -re->top_bracket = swap_uint16(re->top_bracket); -re->top_backref = swap_uint16(re->top_backref); -re->name_table_offset = swap_uint16(re->name_table_offset); -re->name_entry_size = swap_uint16(re->name_entry_size); -re->name_count = swap_uint16(re->name_count); -re->ref_count = swap_uint16(re->ref_count); - -if (extra != NULL && (extra->flags & PCRE_EXTRA_STUDY_DATA) != 0) - { - pcre_study_data *rsd = (pcre_study_data *)(extra->study_data); - rsd->size = swap_uint32(rsd->size); - rsd->flags = swap_uint32(rsd->flags); - rsd->minlength = swap_uint32(rsd->minlength); - } - -/* In 32-bit mode we must swap bytes in the name table, if present, and then in -the pattern itself. */ - -while(TRUE) - { - /* Swap previous characters. */ - while (length-- > 0) - { - *ptr = swap_uint32(*ptr); - ptr++; - } - - /* Get next opcode. */ - - length = 0; - op = *ptr; - *ptr++ = swap_uint32(op); - - switch (op) - { - case OP_END: - return; - - default: - length = OP_lengths32[op] - 1; - break; - - case OP_CLASS: - case OP_NCLASS: - /* Skip the character bit map. */ - ptr += 32/sizeof(pcre_uint32); - length = 0; - break; - - case OP_XCLASS: - /* LINK_SIZE can only be 1 in 32-bit mode. */ - length = (int)((unsigned int)(ptr[0]) - (1 + LINK_SIZE + 1)); - - /* Reverse the size of the XCLASS instance. */ - *ptr = swap_uint32(*ptr); - ptr++; - - op = *ptr; - *ptr = swap_uint32(op); - ptr++; - if ((op & XCL_MAP) != 0) - { - /* Skip the character bit map. */ - ptr += 32/sizeof(pcre_uint32); - length -= 32/sizeof(pcre_uint32); - } - break; - } - } -/* Control should never reach here in 32 bit mode. */ -} - -#endif /* SUPPORT_PCRE32 */ - - - -static void -regexflip(pcre *ere, pcre_extra *extra) -{ -#if defined SUPPORT_PCRE32 - if (REAL_PCRE_FLAGS(ere) & PCRE_MODE32) - regexflip_32(ere, extra); -#endif -#if defined SUPPORT_PCRE8 || defined SUPPORT_PCRE16 - if (REAL_PCRE_FLAGS(ere) & (PCRE_MODE8 | PCRE_MODE16)) - regexflip8_or_16(ere, extra); -#endif -} - - - -/************************************************* -* Check match or recursion limit * -*************************************************/ - -static int -check_match_limit(pcre *re, pcre_extra *extra, pcre_uint8 *bptr, int len, - int start_offset, int options, int *use_offsets, int use_size_offsets, - int flag, unsigned long int *limit, int errnumber, const char *msg) -{ -int count; -int min = 0; -int mid = 64; -int max = -1; - -extra->flags |= flag; - -for (;;) - { - *limit = mid; - - PCRE_EXEC(count, re, extra, bptr, len, start_offset, options, - use_offsets, use_size_offsets); - - if (count == errnumber) - { - /* fprintf(outfile, "Testing %s limit = %d\n", msg, mid); */ - min = mid; - mid = (mid == max - 1)? max : (max > 0)? (min + max)/2 : mid*2; - } - - else if (count >= 0 || count == PCRE_ERROR_NOMATCH || - count == PCRE_ERROR_PARTIAL) - { - if (mid == min + 1) - { - fprintf(outfile, "Minimum %s limit = %d\n", msg, mid); - break; - } - /* fprintf(outfile, "Testing %s limit = %d\n", msg, mid); */ - max = mid; - mid = (min + mid)/2; - } - else break; /* Some other error */ - } - -extra->flags &= ~flag; -return count; -} - - - -/************************************************* -* Case-independent strncmp() function * -*************************************************/ - -/* -Arguments: - s first string - t second string - n number of characters to compare - -Returns: < 0, = 0, or > 0, according to the comparison -*/ - -static int -strncmpic(pcre_uint8 *s, pcre_uint8 *t, int n) -{ -while (n--) - { - int c = tolower(*s++) - tolower(*t++); - if (c) return c; - } -return 0; -} - - - -/************************************************* -* Check multicharacter option * -*************************************************/ - -/* This is used both at compile and run-time to check for escapes. Print -a message and return 0 if there is no match. - -Arguments: - p points after the leading '<' - f file for error message - nl TRUE to check only for newline settings - stype "modifier" or "escape sequence" - -Returns: appropriate PCRE_NEWLINE_xxx flags, or 0 -*/ - -static int -check_mc_option(pcre_uint8 *p, FILE *f, BOOL nl, const char *stype) -{ -if (strncmpic(p, (pcre_uint8 *)"cr>", 3) == 0) return PCRE_NEWLINE_CR; -if (strncmpic(p, (pcre_uint8 *)"lf>", 3) == 0) return PCRE_NEWLINE_LF; -if (strncmpic(p, (pcre_uint8 *)"crlf>", 5) == 0) return PCRE_NEWLINE_CRLF; -if (strncmpic(p, (pcre_uint8 *)"anycrlf>", 8) == 0) return PCRE_NEWLINE_ANYCRLF; -if (strncmpic(p, (pcre_uint8 *)"any>", 4) == 0) return PCRE_NEWLINE_ANY; -if (strncmpic(p, (pcre_uint8 *)"bsr_anycrlf>", 12) == 0) return PCRE_BSR_ANYCRLF; -if (strncmpic(p, (pcre_uint8 *)"bsr_unicode>", 12) == 0) return PCRE_BSR_UNICODE; - -if (!nl) - { - if (strncmpic(p, (pcre_uint8 *)"JS>", 3) == 0) return PCRE_JAVASCRIPT_COMPAT; - } - -fprintf(f, "Unknown %s at: <%s\n", stype, p); -return 0; -} - - - -/************************************************* -* Usage function * -*************************************************/ - -static void -usage(void) -{ -printf("Usage: pcretest [options] [ []]\n\n"); -printf("Input and output default to stdin and stdout.\n"); -#if defined(SUPPORT_LIBREADLINE) || defined(SUPPORT_LIBEDIT) -printf("If input is a terminal, readline() is used to read from it.\n"); -#else -printf("This version of pcretest is not linked with readline().\n"); -#endif -printf("\nOptions:\n"); -#ifdef SUPPORT_PCRE16 -printf(" -16 use the 16-bit library\n"); -#endif -#ifdef SUPPORT_PCRE32 -printf(" -32 use the 32-bit library\n"); -#endif -printf(" -b show compiled code\n"); -printf(" -C show PCRE compile-time options and exit\n"); -printf(" -C arg show a specific compile-time option and exit\n"); -printf(" with its value if numeric (else 0). The arg can be:\n"); -printf(" linksize internal link size [2, 3, 4]\n"); -printf(" pcre8 8 bit library support enabled [0, 1]\n"); -printf(" pcre16 16 bit library support enabled [0, 1]\n"); -printf(" pcre32 32 bit library support enabled [0, 1]\n"); -printf(" utf Unicode Transformation Format supported [0, 1]\n"); -printf(" ucp Unicode Properties supported [0, 1]\n"); -printf(" jit Just-in-time compiler supported [0, 1]\n"); -printf(" newline Newline type [CR, LF, CRLF, ANYCRLF, ANY]\n"); -printf(" bsr \\R type [ANYCRLF, ANY]\n"); -printf(" -d debug: show compiled code and information (-b and -i)\n"); -#if !defined NODFA -printf(" -dfa force DFA matching for all subjects\n"); -#endif -printf(" -help show usage information\n"); -printf(" -i show information about compiled patterns\n" - " -M find MATCH_LIMIT minimum for each subject\n" - " -m output memory used information\n" - " -O set PCRE_NO_AUTO_POSSESS on each pattern\n" - " -o set size of offsets vector to \n"); -#if !defined NOPOSIX -printf(" -p use POSIX interface\n"); -#endif -printf(" -q quiet: do not output PCRE version number at start\n"); -printf(" -S set stack size to megabytes\n"); -printf(" -s force each pattern to be studied at basic level\n" - " -s+ force each pattern to be studied, using JIT if available\n" - " -s++ ditto, verifying when JIT was actually used\n" - " -s+n force each pattern to be studied, using JIT if available,\n" - " where 1 <= n <= 7 selects JIT options\n" - " -s++n ditto, verifying when JIT was actually used\n" - " -t time compilation and execution\n"); -printf(" -t time compilation and execution, repeating times\n"); -printf(" -tm time execution (matching) only\n"); -printf(" -tm time execution (matching) only, repeating times\n"); -printf(" -T same as -t, but show total times at the end\n"); -printf(" -TM same as -tm, but show total time at the end\n"); -} - - - -/************************************************* -* Main Program * -*************************************************/ - -/* Read lines from named file or stdin and write to named file or stdout; lines -consist of a regular expression, in delimiters and optionally followed by -options, followed by a set of test data, terminated by an empty line. */ - -int main(int argc, char **argv) -{ -FILE *infile = stdin; -const char *version; -int options = 0; -int study_options = 0; -int default_find_match_limit = FALSE; -pcre_uint32 default_options = 0; -int op = 1; -int timeit = 0; -int timeitm = 0; -int showtotaltimes = 0; -int showinfo = 0; -int showstore = 0; -int force_study = -1; -int force_study_options = 0; -int quiet = 0; -int size_offsets = 45; -int size_offsets_max; -int *offsets = NULL; -int debug = 0; -int done = 0; -int all_use_dfa = 0; -int verify_jit = 0; -int yield = 0; -int stack_size; -pcre_uint8 *dbuffer = NULL; -pcre_uint8 lockout[24] = { 0 }; -size_t dbuffer_size = 1u << 14; -clock_t total_compile_time = 0; -clock_t total_study_time = 0; -clock_t total_match_time = 0; - -#if !defined NOPOSIX -int posix = 0; -#endif -#if !defined NODFA -int *dfa_workspace = NULL; -#endif - -pcre_jit_stack *jit_stack = NULL; - -/* These vectors store, end-to-end, a list of zero-terminated captured -substring names, each list itself being terminated by an empty name. Assume -that 1024 is plenty long enough for the few names we'll be testing. It is -easiest to keep separate 8-, 16- and 32-bit versions, using the 32-bit version -for the actual memory, to ensure alignment. */ - -pcre_uint32 copynames[1024]; -pcre_uint32 getnames[1024]; - -#ifdef SUPPORT_PCRE32 -pcre_uint32 *cn32ptr; -pcre_uint32 *gn32ptr; -#endif - -#ifdef SUPPORT_PCRE16 -pcre_uint16 *copynames16 = (pcre_uint16 *)copynames; -pcre_uint16 *getnames16 = (pcre_uint16 *)getnames; -pcre_uint16 *cn16ptr; -pcre_uint16 *gn16ptr; -#endif - -#ifdef SUPPORT_PCRE8 -pcre_uint8 *copynames8 = (pcre_uint8 *)copynames; -pcre_uint8 *getnames8 = (pcre_uint8 *)getnames; -pcre_uint8 *cn8ptr; -pcre_uint8 *gn8ptr; -#endif - -/* Get buffers from malloc() so that valgrind will check their misuse when -debugging. They grow automatically when very long lines are read. The 16- -and 32-bit buffers (buffer16, buffer32) are obtained only if needed. */ - -buffer = (pcre_uint8 *)malloc(buffer_size); -pbuffer = (pcre_uint8 *)malloc(buffer_size); - -/* The outfile variable is static so that new_malloc can use it. */ - -outfile = stdout; - -/* The following _setmode() stuff is some Windows magic that tells its runtime -library to translate CRLF into a single LF character. At least, that's what -I've been told: never having used Windows I take this all on trust. Originally -it set 0x8000, but then I was advised that _O_BINARY was better. */ - -#if defined(_WIN32) || defined(WIN32) -_setmode( _fileno( stdout ), _O_BINARY ); -#endif - -/* Get the version number: both pcre_version() and pcre16_version() give the -same answer. We just need to ensure that we call one that is available. */ - -#if defined SUPPORT_PCRE8 -version = pcre_version(); -#elif defined SUPPORT_PCRE16 -version = pcre16_version(); -#elif defined SUPPORT_PCRE32 -version = pcre32_version(); -#endif - -/* Scan options */ - -while (argc > 1 && argv[op][0] == '-') - { - pcre_uint8 *endptr; - char *arg = argv[op]; - - if (strcmp(arg, "-m") == 0) showstore = 1; - else if (strcmp(arg, "-s") == 0) force_study = 0; - - else if (strncmp(arg, "-s+", 3) == 0) - { - arg += 3; - if (*arg == '+') { arg++; verify_jit = TRUE; } - force_study = 1; - if (*arg == 0) - force_study_options = jit_study_bits[6]; - else if (*arg >= '1' && *arg <= '7') - force_study_options = jit_study_bits[*arg - '1']; - else goto BAD_ARG; - } - else if (strcmp(arg, "-8") == 0) - { -#ifdef SUPPORT_PCRE8 - pcre_mode = PCRE8_MODE; -#else - printf("** This version of PCRE was built without 8-bit support\n"); - exit(1); -#endif - } - else if (strcmp(arg, "-16") == 0) - { -#ifdef SUPPORT_PCRE16 - pcre_mode = PCRE16_MODE; -#else - printf("** This version of PCRE was built without 16-bit support\n"); - exit(1); -#endif - } - else if (strcmp(arg, "-32") == 0) - { -#ifdef SUPPORT_PCRE32 - pcre_mode = PCRE32_MODE; -#else - printf("** This version of PCRE was built without 32-bit support\n"); - exit(1); -#endif - } - else if (strcmp(arg, "-q") == 0) quiet = 1; - else if (strcmp(arg, "-b") == 0) debug = 1; - else if (strcmp(arg, "-i") == 0) showinfo = 1; - else if (strcmp(arg, "-d") == 0) showinfo = debug = 1; - else if (strcmp(arg, "-M") == 0) default_find_match_limit = TRUE; - else if (strcmp(arg, "-O") == 0) default_options |= PCRE_NO_AUTO_POSSESS; -#if !defined NODFA - else if (strcmp(arg, "-dfa") == 0) all_use_dfa = 1; -#endif - else if (strcmp(arg, "-o") == 0 && argc > 2 && - ((size_offsets = get_value((pcre_uint8 *)argv[op+1], &endptr)), - *endptr == 0)) - { - op++; - argc--; - } - else if (strcmp(arg, "-t") == 0 || strcmp(arg, "-tm") == 0 || - strcmp(arg, "-T") == 0 || strcmp(arg, "-TM") == 0) - { - int temp; - int both = arg[2] == 0; - showtotaltimes = arg[1] == 'T'; - if (argc > 2 && (temp = get_value((pcre_uint8 *)argv[op+1], &endptr), - *endptr == 0)) - { - timeitm = temp; - op++; - argc--; - } - else timeitm = LOOPREPEAT; - if (both) timeit = timeitm; - } - else if (strcmp(arg, "-S") == 0 && argc > 2 && - ((stack_size = get_value((pcre_uint8 *)argv[op+1], &endptr)), - *endptr == 0)) - { -#if defined(_WIN32) || defined(WIN32) || defined(__minix) || defined(NATIVE_ZOS) || defined(__VMS) - printf("PCRE: -S not supported on this OS\n"); - exit(1); -#else - int rc; - struct rlimit rlim; - getrlimit(RLIMIT_STACK, &rlim); - rlim.rlim_cur = stack_size * 1024 * 1024; - rc = setrlimit(RLIMIT_STACK, &rlim); - if (rc != 0) - { - printf("PCRE: setrlimit() failed with error %d\n", rc); - exit(1); - } - op++; - argc--; -#endif - } -#if !defined NOPOSIX - else if (strcmp(arg, "-p") == 0) posix = 1; -#endif - else if (strcmp(arg, "-C") == 0) - { - int rc; - unsigned long int lrc; - - if (argc > 2) - { - if (strcmp(argv[op + 1], "linksize") == 0) - { - (void)PCRE_CONFIG(PCRE_CONFIG_LINK_SIZE, &rc); - printf("%d\n", rc); - yield = rc; - -#ifdef __VMS - vms_setsymbol("LINKSIZE",0,yield ); -#endif - } - else if (strcmp(argv[op + 1], "pcre8") == 0) - { -#ifdef SUPPORT_PCRE8 - printf("1\n"); - yield = 1; -#else - printf("0\n"); - yield = 0; -#endif -#ifdef __VMS - vms_setsymbol("PCRE8",0,yield ); -#endif - } - else if (strcmp(argv[op + 1], "pcre16") == 0) - { -#ifdef SUPPORT_PCRE16 - printf("1\n"); - yield = 1; -#else - printf("0\n"); - yield = 0; -#endif -#ifdef __VMS - vms_setsymbol("PCRE16",0,yield ); -#endif - } - else if (strcmp(argv[op + 1], "pcre32") == 0) - { -#ifdef SUPPORT_PCRE32 - printf("1\n"); - yield = 1; -#else - printf("0\n"); - yield = 0; -#endif -#ifdef __VMS - vms_setsymbol("PCRE32",0,yield ); -#endif - } - else if (strcmp(argv[op + 1], "utf") == 0) - { -#ifdef SUPPORT_PCRE8 - if (pcre_mode == PCRE8_MODE) - (void)pcre_config(PCRE_CONFIG_UTF8, &rc); -#endif -#ifdef SUPPORT_PCRE16 - if (pcre_mode == PCRE16_MODE) - (void)pcre16_config(PCRE_CONFIG_UTF16, &rc); -#endif -#ifdef SUPPORT_PCRE32 - if (pcre_mode == PCRE32_MODE) - (void)pcre32_config(PCRE_CONFIG_UTF32, &rc); -#endif - printf("%d\n", rc); - yield = rc; -#ifdef __VMS - vms_setsymbol("UTF",0,yield ); -#endif - } - else if (strcmp(argv[op + 1], "ucp") == 0) - { - (void)PCRE_CONFIG(PCRE_CONFIG_UNICODE_PROPERTIES, &rc); - printf("%d\n", rc); - yield = rc; - } - else if (strcmp(argv[op + 1], "jit") == 0) - { - (void)PCRE_CONFIG(PCRE_CONFIG_JIT, &rc); - printf("%d\n", rc); - yield = rc; - } - else if (strcmp(argv[op + 1], "newline") == 0) - { - (void)PCRE_CONFIG(PCRE_CONFIG_NEWLINE, &rc); - print_newline_config(rc, TRUE); - } - else if (strcmp(argv[op + 1], "bsr") == 0) - { - (void)PCRE_CONFIG(PCRE_CONFIG_BSR, &rc); - printf("%s\n", rc? "ANYCRLF" : "ANY"); - } - else if (strcmp(argv[op + 1], "ebcdic") == 0) - { -#ifdef EBCDIC - printf("1\n"); - yield = 1; -#else - printf("0\n"); -#endif - } - else if (strcmp(argv[op + 1], "ebcdic-nl") == 0) - { -#ifdef EBCDIC - printf("0x%02x\n", CHAR_LF); -#else - printf("0\n"); -#endif - } - else - { - printf("Unknown -C option: %s\n", argv[op + 1]); - } - goto EXIT; - } - - /* No argument for -C: output all configuration information. */ - - printf("PCRE version %s\n", version); - printf("Compiled with\n"); - -#ifdef EBCDIC - printf(" EBCDIC code support: LF is 0x%02x\n", CHAR_LF); -#endif - -/* At least one of SUPPORT_PCRE8 and SUPPORT_PCRE16 will be set. If both -are set, either both UTFs are supported or both are not supported. */ - -#ifdef SUPPORT_PCRE8 - printf(" 8-bit support\n"); - (void)pcre_config(PCRE_CONFIG_UTF8, &rc); - printf (" %sUTF-8 support\n", rc ? "" : "No "); -#endif -#ifdef SUPPORT_PCRE16 - printf(" 16-bit support\n"); - (void)pcre16_config(PCRE_CONFIG_UTF16, &rc); - printf (" %sUTF-16 support\n", rc ? "" : "No "); -#endif -#ifdef SUPPORT_PCRE32 - printf(" 32-bit support\n"); - (void)pcre32_config(PCRE_CONFIG_UTF32, &rc); - printf (" %sUTF-32 support\n", rc ? "" : "No "); -#endif - - (void)PCRE_CONFIG(PCRE_CONFIG_UNICODE_PROPERTIES, &rc); - printf(" %sUnicode properties support\n", rc? "" : "No "); - (void)PCRE_CONFIG(PCRE_CONFIG_JIT, &rc); - if (rc) - { - const char *arch; - (void)PCRE_CONFIG(PCRE_CONFIG_JITTARGET, (void *)(&arch)); - printf(" Just-in-time compiler support: %s\n", arch); - } - else - printf(" No just-in-time compiler support\n"); - (void)PCRE_CONFIG(PCRE_CONFIG_NEWLINE, &rc); - print_newline_config(rc, FALSE); - (void)PCRE_CONFIG(PCRE_CONFIG_BSR, &rc); - printf(" \\R matches %s\n", rc? "CR, LF, or CRLF only" : - "all Unicode newlines"); - (void)PCRE_CONFIG(PCRE_CONFIG_LINK_SIZE, &rc); - printf(" Internal link size = %d\n", rc); - (void)PCRE_CONFIG(PCRE_CONFIG_POSIX_MALLOC_THRESHOLD, &rc); - printf(" POSIX malloc threshold = %d\n", rc); - (void)PCRE_CONFIG(PCRE_CONFIG_PARENS_LIMIT, &lrc); - printf(" Parentheses nest limit = %ld\n", lrc); - (void)PCRE_CONFIG(PCRE_CONFIG_MATCH_LIMIT, &lrc); - printf(" Default match limit = %ld\n", lrc); - (void)PCRE_CONFIG(PCRE_CONFIG_MATCH_LIMIT_RECURSION, &lrc); - printf(" Default recursion depth limit = %ld\n", lrc); - (void)PCRE_CONFIG(PCRE_CONFIG_STACKRECURSE, &rc); - printf(" Match recursion uses %s", rc? "stack" : "heap"); - if (showstore) - { - PCRE_EXEC(stack_size, NULL, NULL, NULL, -999, -999, 0, NULL, 0); - printf(": %sframe size = %d bytes", rc? "approximate " : "", -stack_size); - } - printf("\n"); - goto EXIT; - } - else if (strcmp(arg, "-help") == 0 || - strcmp(arg, "--help") == 0) - { - usage(); - goto EXIT; - } - else - { - BAD_ARG: - printf("** Unknown or malformed option %s\n", arg); - usage(); - yield = 1; - goto EXIT; - } - op++; - argc--; - } - -/* Get the store for the offsets vector, and remember what it was */ - -size_offsets_max = size_offsets; -offsets = (int *)malloc(size_offsets_max * sizeof(int)); -if (offsets == NULL) - { - printf("** Failed to get %d bytes of memory for offsets vector\n", - (int)(size_offsets_max * sizeof(int))); - yield = 1; - goto EXIT; - } - -/* Sort out the input and output files */ - -if (argc > 1) - { - infile = fopen(argv[op], INPUT_MODE); - if (infile == NULL) - { - printf("** Failed to open %s\n", argv[op]); - yield = 1; - goto EXIT; - } - } - -if (argc > 2) - { - outfile = fopen(argv[op+1], OUTPUT_MODE); - if (outfile == NULL) - { - printf("** Failed to open %s\n", argv[op+1]); - yield = 1; - goto EXIT; - } - } - -/* Set alternative malloc function */ - -#ifdef SUPPORT_PCRE8 -pcre_malloc = new_malloc; -pcre_free = new_free; -pcre_stack_malloc = stack_malloc; -pcre_stack_free = stack_free; -#endif - -#ifdef SUPPORT_PCRE16 -pcre16_malloc = new_malloc; -pcre16_free = new_free; -pcre16_stack_malloc = stack_malloc; -pcre16_stack_free = stack_free; -#endif - -#ifdef SUPPORT_PCRE32 -pcre32_malloc = new_malloc; -pcre32_free = new_free; -pcre32_stack_malloc = stack_malloc; -pcre32_stack_free = stack_free; -#endif - -/* Heading line unless quiet */ - -if (!quiet) fprintf(outfile, "PCRE version %s\n\n", version); - -/* Main loop */ - -while (!done) - { - pcre *re = NULL; - pcre_extra *extra = NULL; - -#if !defined NOPOSIX /* There are still compilers that require no indent */ - regex_t preg = { NULL, 0, 0} ; - int do_posix = 0; -#endif - - const char *error; - pcre_uint8 *markptr; - pcre_uint8 *p, *pp, *ppp; - pcre_uint8 *to_file = NULL; - const pcre_uint8 *tables = NULL; - unsigned long int get_options; - unsigned long int true_size, true_study_size = 0; - size_t size; - int do_allcaps = 0; - int do_mark = 0; - int do_study = 0; - int no_force_study = 0; - int do_debug = debug; - int do_G = 0; - int do_g = 0; - int do_showinfo = showinfo; - int do_showrest = 0; - int do_showcaprest = 0; - int do_flip = 0; - int erroroffset, len, delimiter, poffset; - -#if !defined NODFA - int dfa_matched = 0; -#endif - - use_utf = 0; - debug_lengths = 1; - SET_PCRE_STACK_GUARD(NULL); - - if (extend_inputline(infile, buffer, " re> ") == NULL) break; - if (infile != stdin) fprintf(outfile, "%s", (char *)buffer); - fflush(outfile); - - p = buffer; - while (isspace(*p)) p++; - if (*p == 0) continue; - - /* Handle option lock-out setting */ - - if (*p == '<' && p[1] == ' ') - { - p += 2; - while (isspace(*p)) p++; - if (strncmp((char *)p, "forbid ", 7) == 0) - { - p += 7; - while (isspace(*p)) p++; - pp = lockout; - while (!isspace(*p) && pp < lockout + sizeof(lockout) - 1) - *pp++ = *p++; - *pp = 0; - } - else - { - printf("** Unrecognized special command '%s'\n", p); - yield = 1; - goto EXIT; - } - continue; - } - - /* See if the pattern is to be loaded pre-compiled from a file. */ - - if (*p == '<' && strchr((char *)(p+1), '<') == NULL) - { - pcre_uint32 magic; - pcre_uint8 sbuf[8]; - FILE *f; - - p++; - if (*p == '!') - { - do_debug = TRUE; - do_showinfo = TRUE; - p++; - } - - pp = p + (int)strlen((char *)p); - while (isspace(pp[-1])) pp--; - *pp = 0; - - f = fopen((char *)p, "rb"); - if (f == NULL) - { - fprintf(outfile, "Failed to open %s: %s\n", p, strerror(errno)); - continue; - } - if (fread(sbuf, 1, 8, f) != 8) goto FAIL_READ; - - true_size = - (sbuf[0] << 24) | (sbuf[1] << 16) | (sbuf[2] << 8) | sbuf[3]; - true_study_size = - (sbuf[4] << 24) | (sbuf[5] << 16) | (sbuf[6] << 8) | sbuf[7]; - - re = (pcre *)new_malloc(true_size); - if (re == NULL) - { - printf("** Failed to get %d bytes of memory for pcre object\n", - (int)true_size); - yield = 1; - goto EXIT; - } - if (fread(re, 1, true_size, f) != true_size) goto FAIL_READ; - - magic = REAL_PCRE_MAGIC(re); - if (magic != MAGIC_NUMBER) - { - if (swap_uint32(magic) == MAGIC_NUMBER) - { - do_flip = 1; - } - else - { - fprintf(outfile, "Data in %s is not a compiled PCRE regex\n", p); - new_free(re); - fclose(f); - continue; - } - } - - /* We hide the byte-invert info for little and big endian tests. */ - fprintf(outfile, "Compiled pattern%s loaded from %s\n", - do_flip && (p[-1] == '<') ? " (byte-inverted)" : "", p); - - /* Now see if there is any following study data. */ - - if (true_study_size != 0) - { - pcre_study_data *psd; - - extra = (pcre_extra *)new_malloc(sizeof(pcre_extra) + true_study_size); - extra->flags = PCRE_EXTRA_STUDY_DATA; - - psd = (pcre_study_data *)(((char *)extra) + sizeof(pcre_extra)); - extra->study_data = psd; - - if (fread(psd, 1, true_study_size, f) != true_study_size) - { - FAIL_READ: - fprintf(outfile, "Failed to read data from %s\n", p); - if (extra != NULL) - { - PCRE_FREE_STUDY(extra); - } - new_free(re); - fclose(f); - continue; - } - fprintf(outfile, "Study data loaded from %s\n", p); - do_study = 1; /* To get the data output if requested */ - } - else fprintf(outfile, "No study data\n"); - - /* Flip the necessary bytes. */ - if (do_flip) - { - int rc; - PCRE_PATTERN_TO_HOST_BYTE_ORDER(rc, re, extra, NULL); - if (rc == PCRE_ERROR_BADMODE) - { - pcre_uint32 flags_in_host_byte_order; - if (REAL_PCRE_MAGIC(re) == MAGIC_NUMBER) - flags_in_host_byte_order = REAL_PCRE_FLAGS(re); - else - flags_in_host_byte_order = swap_uint32(REAL_PCRE_FLAGS(re)); - /* Simulate the result of the function call below. */ - fprintf(outfile, "Error %d from pcre%s_fullinfo(%d)\n", rc, - pcre_mode == PCRE32_MODE ? "32" : pcre_mode == PCRE16_MODE ? "16" : "", - PCRE_INFO_OPTIONS); - fprintf(outfile, "Running in %d-bit mode but pattern was compiled in " - "%d-bit mode\n", 8 * CHAR_SIZE, 8 * (flags_in_host_byte_order & PCRE_MODE_MASK)); - new_free(re); - fclose(f); - continue; - } - } - - /* Need to know if UTF-8 for printing data strings. */ - - if (new_info(re, NULL, PCRE_INFO_OPTIONS, &get_options) < 0) - { - new_free(re); - fclose(f); - continue; - } - use_utf = (get_options & PCRE_UTF8) != 0; - - fclose(f); - goto SHOW_INFO; - } - - /* In-line pattern (the usual case). Get the delimiter and seek the end of - the pattern; if it isn't complete, read more. */ - - delimiter = *p++; - - if (isalnum(delimiter) || delimiter == '\\') - { - fprintf(outfile, "** Delimiter must not be alphanumeric or \\\n"); - goto SKIP_DATA; - } - - pp = p; - poffset = (int)(p - buffer); - - for(;;) - { - while (*pp != 0) - { - if (*pp == '\\' && pp[1] != 0) pp++; - else if (*pp == delimiter) break; - pp++; - } - if (*pp != 0) break; - if ((pp = extend_inputline(infile, pp, " > ")) == NULL) - { - fprintf(outfile, "** Unexpected EOF\n"); - done = 1; - goto CONTINUE; - } - if (infile != stdin) fprintf(outfile, "%s", (char *)pp); - } - - /* The buffer may have moved while being extended; reset the start of data - pointer to the correct relative point in the buffer. */ - - p = buffer + poffset; - - /* If the first character after the delimiter is backslash, make - the pattern end with backslash. This is purely to provide a way - of testing for the error message when a pattern ends with backslash. */ - - if (pp[1] == '\\') *pp++ = '\\'; - - /* Terminate the pattern at the delimiter, and save a copy of the pattern - for callouts. */ - - *pp++ = 0; - strcpy((char *)pbuffer, (char *)p); - - /* Look for modifiers and options after the final delimiter. */ - - options = default_options; - study_options = force_study_options; - log_store = showstore; /* default from command line */ - - while (*pp != 0) - { - /* Check to see whether this modifier has been locked out for this file. - This is complicated for the multi-character options that begin with '<'. - If there is no '>' in the lockout string, all multi-character modifiers are - locked out. */ - - if (strchr((char *)lockout, *pp) != NULL) - { - if (*pp == '<' && strchr((char *)lockout, '>') != NULL) - { - int x = check_mc_option(pp+1, outfile, FALSE, "modifier"); - if (x == 0) goto SKIP_DATA; - - for (ppp = lockout; *ppp != 0; ppp++) - { - if (*ppp == '<') - { - int y = check_mc_option(ppp+1, outfile, FALSE, "modifier"); - if (y == 0) - { - printf("** Error in modifier forbid data - giving up.\n"); - yield = 1; - goto EXIT; - } - if (x == y) - { - ppp = pp; - while (*ppp != '>') ppp++; - printf("** The %.*s modifier is locked out - giving up.\n", - (int)(ppp - pp + 1), pp); - yield = 1; - goto EXIT; - } - } - } - } - - /* The single-character modifiers are straightforward. */ - - else - { - printf("** The /%c modifier is locked out - giving up.\n", *pp); - yield = 1; - goto EXIT; - } - } - - /* The modifier is not locked out; handle it. */ - - switch (*pp++) - { - case 'f': options |= PCRE_FIRSTLINE; break; - case 'g': do_g = 1; break; - case 'i': options |= PCRE_CASELESS; break; - case 'm': options |= PCRE_MULTILINE; break; - case 's': options |= PCRE_DOTALL; break; - case 'x': options |= PCRE_EXTENDED; break; - - case '+': - if (do_showrest) do_showcaprest = 1; else do_showrest = 1; - break; - - case '=': do_allcaps = 1; break; - case 'A': options |= PCRE_ANCHORED; break; - case 'B': do_debug = 1; break; - case 'C': options |= PCRE_AUTO_CALLOUT; break; - case 'D': do_debug = do_showinfo = 1; break; - case 'E': options |= PCRE_DOLLAR_ENDONLY; break; - case 'F': do_flip = 1; break; - case 'G': do_G = 1; break; - case 'I': do_showinfo = 1; break; - case 'J': options |= PCRE_DUPNAMES; break; - case 'K': do_mark = 1; break; - case 'M': log_store = 1; break; - case 'N': options |= PCRE_NO_AUTO_CAPTURE; break; - case 'O': options |= PCRE_NO_AUTO_POSSESS; break; - -#if !defined NOPOSIX - case 'P': do_posix = 1; break; -#endif - - case 'Q': - switch (*pp) - { - case '0': - case '1': - stack_guard_return = *pp++ - '0'; - break; - - default: - fprintf(outfile, "** Missing 0 or 1 after /Q\n"); - goto SKIP_DATA; - } - SET_PCRE_STACK_GUARD(stack_guard); - break; - - case 'S': - do_study = 1; - for (;;) - { - switch (*pp++) - { - case 'S': - do_study = 0; - no_force_study = 1; - break; - - case '!': - study_options |= PCRE_STUDY_EXTRA_NEEDED; - break; - - case '+': - if (*pp == '+') - { - verify_jit = TRUE; - pp++; - } - if (*pp >= '1' && *pp <= '7') - study_options |= jit_study_bits[*pp++ - '1']; - else - study_options |= jit_study_bits[6]; - break; - - case '-': - study_options &= ~PCRE_STUDY_ALLJIT; - break; - - default: - pp--; - goto ENDLOOP; - } - } - ENDLOOP: - break; - - case 'U': options |= PCRE_UNGREEDY; break; - case 'W': options |= PCRE_UCP; break; - case 'X': options |= PCRE_EXTRA; break; - case 'Y': options |= PCRE_NO_START_OPTIMISE; break; - case 'Z': debug_lengths = 0; break; - case '8': options |= PCRE_UTF8; use_utf = 1; break; - case '9': options |= PCRE_NEVER_UTF; break; - case '?': options |= PCRE_NO_UTF8_CHECK; break; - - case 'T': - switch (*pp++) - { - case '0': tables = tables0; break; - case '1': tables = tables1; break; - - case '\r': - case '\n': - case ' ': - case 0: - fprintf(outfile, "** Missing table number after /T\n"); - goto SKIP_DATA; - - default: - fprintf(outfile, "** Bad table number \"%c\" after /T\n", pp[-1]); - goto SKIP_DATA; - } - break; - - case 'L': - ppp = pp; - /* The '\r' test here is so that it works on Windows. */ - /* The '0' test is just in case this is an unterminated line. */ - while (*ppp != 0 && *ppp != '\n' && *ppp != '\r' && *ppp != ' ') ppp++; - *ppp = 0; - if (setlocale(LC_CTYPE, (const char *)pp) == NULL) - { - fprintf(outfile, "** Failed to set locale \"%s\"\n", pp); - goto SKIP_DATA; - } - locale_set = 1; - tables = PCRE_MAKETABLES; - pp = ppp; - break; - - case '>': - to_file = pp; - while (*pp != 0) pp++; - while (isspace(pp[-1])) pp--; - *pp = 0; - break; - - case '<': - { - int x = check_mc_option(pp, outfile, FALSE, "modifier"); - if (x == 0) goto SKIP_DATA; - options |= x; - while (*pp++ != '>'); - } - break; - - case '\r': /* So that it works in Windows */ - case '\n': - case ' ': - break; - - default: - fprintf(outfile, "** Unknown modifier '%c'\n", pp[-1]); - goto SKIP_DATA; - } - } - - /* Handle compiling via the POSIX interface, which doesn't support the - timing, showing, or debugging options, nor the ability to pass over - local character tables. Neither does it have 16-bit support. */ - -#if !defined NOPOSIX - if (posix || do_posix) - { - int rc; - int cflags = 0; - - if ((options & PCRE_CASELESS) != 0) cflags |= REG_ICASE; - if ((options & PCRE_MULTILINE) != 0) cflags |= REG_NEWLINE; - if ((options & PCRE_DOTALL) != 0) cflags |= REG_DOTALL; - if ((options & PCRE_NO_AUTO_CAPTURE) != 0) cflags |= REG_NOSUB; - if ((options & PCRE_UTF8) != 0) cflags |= REG_UTF8; - if ((options & PCRE_UCP) != 0) cflags |= REG_UCP; - if ((options & PCRE_UNGREEDY) != 0) cflags |= REG_UNGREEDY; - - rc = regcomp(&preg, (char *)p, cflags); - - /* Compilation failed; go back for another re, skipping to blank line - if non-interactive. */ - - if (rc != 0) - { - (void)regerror(rc, &preg, (char *)buffer, buffer_size); - fprintf(outfile, "Failed: POSIX code %d: %s\n", rc, buffer); - goto SKIP_DATA; - } - } - - /* Handle compiling via the native interface */ - - else -#endif /* !defined NOPOSIX */ - - { - /* In 16- or 32-bit mode, convert the input. */ - -#ifdef SUPPORT_PCRE16 - if (pcre_mode == PCRE16_MODE) - { - switch(to16(FALSE, p, options & PCRE_UTF8, (int)strlen((char *)p))) - { - case -1: - fprintf(outfile, "**Failed: invalid UTF-8 string cannot be " - "converted to UTF-16\n"); - goto SKIP_DATA; - - case -2: - fprintf(outfile, "**Failed: character value greater than 0x10ffff " - "cannot be converted to UTF-16\n"); - goto SKIP_DATA; - - case -3: /* "Impossible error" when to16 is called arg1 FALSE */ - fprintf(outfile, "**Failed: character value greater than 0xffff " - "cannot be converted to 16-bit in non-UTF mode\n"); - goto SKIP_DATA; - - default: - break; - } - p = (pcre_uint8 *)buffer16; - } -#endif - -#ifdef SUPPORT_PCRE32 - if (pcre_mode == PCRE32_MODE) - { - switch(to32(FALSE, p, options & PCRE_UTF32, (int)strlen((char *)p))) - { - case -1: - fprintf(outfile, "**Failed: invalid UTF-8 string cannot be " - "converted to UTF-32\n"); - goto SKIP_DATA; - - case -2: - fprintf(outfile, "**Failed: character value greater than 0x10ffff " - "cannot be converted to UTF-32\n"); - goto SKIP_DATA; - - case -3: - fprintf(outfile, "**Failed: character value is ill-formed UTF-32\n"); - goto SKIP_DATA; - - default: - break; - } - p = (pcre_uint8 *)buffer32; - } -#endif - - /* Compile many times when timing */ - - if (timeit > 0) - { - register int i; - clock_t time_taken; - clock_t start_time = clock(); - for (i = 0; i < timeit; i++) - { - PCRE_COMPILE(re, p, options, &error, &erroroffset, tables); - if (re != NULL) free(re); - } - total_compile_time += (time_taken = clock() - start_time); - fprintf(outfile, "Compile time %.4f milliseconds\n", - (((double)time_taken * 1000.0) / (double)timeit) / - (double)CLOCKS_PER_SEC); - } - - PCRE_COMPILE(re, p, options, &error, &erroroffset, tables); - - /* Compilation failed; go back for another re, skipping to blank line - if non-interactive. */ - - if (re == NULL) - { - fprintf(outfile, "Failed: %s at offset %d\n", error, erroroffset); - SKIP_DATA: - if (infile != stdin) - { - for (;;) - { - if (extend_inputline(infile, buffer, NULL) == NULL) - { - done = 1; - goto CONTINUE; - } - len = (int)strlen((char *)buffer); - while (len > 0 && isspace(buffer[len-1])) len--; - if (len == 0) break; - } - fprintf(outfile, "\n"); - } - goto CONTINUE; - } - - /* Compilation succeeded. It is now possible to set the UTF-8 option from - within the regex; check for this so that we know how to process the data - lines. */ - - if (new_info(re, NULL, PCRE_INFO_OPTIONS, &get_options) < 0) - goto SKIP_DATA; - if ((get_options & PCRE_UTF8) != 0) use_utf = 1; - - /* Extract the size for possible writing before possibly flipping it, - and remember the store that was got. */ - - true_size = REAL_PCRE_SIZE(re); - - /* Output code size information if requested */ - - if (log_store) - { - int name_count, name_entry_size, real_pcre_size; - - new_info(re, NULL, PCRE_INFO_NAMECOUNT, &name_count); - new_info(re, NULL, PCRE_INFO_NAMEENTRYSIZE, &name_entry_size); - real_pcre_size = 0; -#ifdef SUPPORT_PCRE8 - if (REAL_PCRE_FLAGS(re) & PCRE_MODE8) - real_pcre_size = sizeof(real_pcre); -#endif -#ifdef SUPPORT_PCRE16 - if (REAL_PCRE_FLAGS(re) & PCRE_MODE16) - real_pcre_size = sizeof(real_pcre16); -#endif -#ifdef SUPPORT_PCRE32 - if (REAL_PCRE_FLAGS(re) & PCRE_MODE32) - real_pcre_size = sizeof(real_pcre32); -#endif - new_info(re, NULL, PCRE_INFO_SIZE, &size); - fprintf(outfile, "Memory allocation (code space): %d\n", - (int)(size - real_pcre_size - name_count * name_entry_size)); - } - - /* If -s or /S was present, study the regex to generate additional info to - help with the matching, unless the pattern has the SS option, which - suppresses the effect of /S (used for a few test patterns where studying is - never sensible). */ - - if (do_study || (force_study >= 0 && !no_force_study)) - { - if (timeit > 0) - { - register int i; - clock_t time_taken; - clock_t start_time = clock(); - for (i = 0; i < timeit; i++) - { - PCRE_STUDY(extra, re, study_options, &error); - } - total_study_time = (time_taken = clock() - start_time); - if (extra != NULL) - { - PCRE_FREE_STUDY(extra); - } - fprintf(outfile, " Study time %.4f milliseconds\n", - (((double)time_taken * 1000.0) / (double)timeit) / - (double)CLOCKS_PER_SEC); - } - PCRE_STUDY(extra, re, study_options, &error); - if (error != NULL) - fprintf(outfile, "Failed to study: %s\n", error); - else if (extra != NULL) - { - true_study_size = ((pcre_study_data *)(extra->study_data))->size; - if (log_store) - { - size_t jitsize; - if (new_info(re, extra, PCRE_INFO_JITSIZE, &jitsize) == 0 && - jitsize != 0) - fprintf(outfile, "Memory allocation (JIT code): %d\n", (int)jitsize); - } - } - } - - /* If /K was present, we set up for handling MARK data. */ - - if (do_mark) - { - if (extra == NULL) - { - extra = (pcre_extra *)malloc(sizeof(pcre_extra)); - extra->flags = 0; - } - extra->mark = &markptr; - extra->flags |= PCRE_EXTRA_MARK; - } - - /* Extract and display information from the compiled data if required. */ - - SHOW_INFO: - - if (do_debug) - { - fprintf(outfile, "------------------------------------------------------------------\n"); - PCRE_PRINTINT(re, outfile, debug_lengths); - } - - /* We already have the options in get_options (see above) */ - - if (do_showinfo) - { - unsigned long int all_options; - pcre_uint32 first_char, need_char; - pcre_uint32 match_limit, recursion_limit; - int count, backrefmax, first_char_set, need_char_set, okpartial, jchanged, - hascrorlf, maxlookbehind, match_empty; - int nameentrysize, namecount; - const pcre_uint8 *nametable; - - if (new_info(re, NULL, PCRE_INFO_CAPTURECOUNT, &count) + - new_info(re, NULL, PCRE_INFO_BACKREFMAX, &backrefmax) + - new_info(re, NULL, PCRE_INFO_FIRSTCHARACTER, &first_char) + - new_info(re, NULL, PCRE_INFO_FIRSTCHARACTERFLAGS, &first_char_set) + - new_info(re, NULL, PCRE_INFO_REQUIREDCHAR, &need_char) + - new_info(re, NULL, PCRE_INFO_REQUIREDCHARFLAGS, &need_char_set) + - new_info(re, NULL, PCRE_INFO_NAMEENTRYSIZE, &nameentrysize) + - new_info(re, NULL, PCRE_INFO_NAMECOUNT, &namecount) + - new_info(re, NULL, PCRE_INFO_NAMETABLE, (void *)&nametable) + - new_info(re, NULL, PCRE_INFO_OKPARTIAL, &okpartial) + - new_info(re, NULL, PCRE_INFO_JCHANGED, &jchanged) + - new_info(re, NULL, PCRE_INFO_HASCRORLF, &hascrorlf) + - new_info(re, NULL, PCRE_INFO_MATCH_EMPTY, &match_empty) + - new_info(re, NULL, PCRE_INFO_MAXLOOKBEHIND, &maxlookbehind) - != 0) - goto SKIP_DATA; - - fprintf(outfile, "Capturing subpattern count = %d\n", count); - - if (backrefmax > 0) - fprintf(outfile, "Max back reference = %d\n", backrefmax); - - if (maxlookbehind > 0) - fprintf(outfile, "Max lookbehind = %d\n", maxlookbehind); - - if (new_info(re, NULL, PCRE_INFO_MATCHLIMIT, &match_limit) == 0) - fprintf(outfile, "Match limit = %u\n", match_limit); - - if (new_info(re, NULL, PCRE_INFO_RECURSIONLIMIT, &recursion_limit) == 0) - fprintf(outfile, "Recursion limit = %u\n", recursion_limit); - - if (namecount > 0) - { - fprintf(outfile, "Named capturing subpatterns:\n"); - while (namecount-- > 0) - { - int imm2_size = pcre_mode == PCRE8_MODE ? 2 : 1; - int length = (int)STRLEN(nametable + imm2_size); - fprintf(outfile, " "); - PCHARSV(nametable, imm2_size, length, outfile); - while (length++ < nameentrysize - imm2_size) putc(' ', outfile); -#ifdef SUPPORT_PCRE32 - if (pcre_mode == PCRE32_MODE) - fprintf(outfile, "%3d\n", (int)(((PCRE_SPTR32)nametable)[0])); -#endif -#ifdef SUPPORT_PCRE16 - if (pcre_mode == PCRE16_MODE) - fprintf(outfile, "%3d\n", (int)(((PCRE_SPTR16)nametable)[0])); -#endif -#ifdef SUPPORT_PCRE8 - if (pcre_mode == PCRE8_MODE) - fprintf(outfile, "%3d\n", ((int)nametable[0] << 8) | (int)nametable[1]); -#endif - nametable += nameentrysize * CHAR_SIZE; - } - } - - if (!okpartial) fprintf(outfile, "Partial matching not supported\n"); - if (hascrorlf) fprintf(outfile, "Contains explicit CR or LF match\n"); - if (match_empty) fprintf(outfile, "May match empty string\n"); - - all_options = REAL_PCRE_OPTIONS(re); - if (do_flip) all_options = swap_uint32(all_options); - - if (get_options == 0) fprintf(outfile, "No options\n"); - else fprintf(outfile, "Options:%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n", - ((get_options & PCRE_ANCHORED) != 0)? " anchored" : "", - ((get_options & PCRE_CASELESS) != 0)? " caseless" : "", - ((get_options & PCRE_EXTENDED) != 0)? " extended" : "", - ((get_options & PCRE_MULTILINE) != 0)? " multiline" : "", - ((get_options & PCRE_FIRSTLINE) != 0)? " firstline" : "", - ((get_options & PCRE_DOTALL) != 0)? " dotall" : "", - ((get_options & PCRE_BSR_ANYCRLF) != 0)? " bsr_anycrlf" : "", - ((get_options & PCRE_BSR_UNICODE) != 0)? " bsr_unicode" : "", - ((get_options & PCRE_DOLLAR_ENDONLY) != 0)? " dollar_endonly" : "", - ((get_options & PCRE_EXTRA) != 0)? " extra" : "", - ((get_options & PCRE_UNGREEDY) != 0)? " ungreedy" : "", - ((get_options & PCRE_NO_AUTO_CAPTURE) != 0)? " no_auto_capture" : "", - ((get_options & PCRE_NO_AUTO_POSSESS) != 0)? " no_auto_possessify" : "", - ((get_options & PCRE_UTF8) != 0)? " utf" : "", - ((get_options & PCRE_UCP) != 0)? " ucp" : "", - ((get_options & PCRE_NO_UTF8_CHECK) != 0)? " no_utf_check" : "", - ((get_options & PCRE_NO_START_OPTIMIZE) != 0)? " no_start_optimize" : "", - ((get_options & PCRE_DUPNAMES) != 0)? " dupnames" : "", - ((get_options & PCRE_NEVER_UTF) != 0)? " never_utf" : ""); - - if (jchanged) fprintf(outfile, "Duplicate name status changes\n"); - - switch (get_options & PCRE_NEWLINE_BITS) - { - case PCRE_NEWLINE_CR: - fprintf(outfile, "Forced newline sequence: CR\n"); - break; - - case PCRE_NEWLINE_LF: - fprintf(outfile, "Forced newline sequence: LF\n"); - break; - - case PCRE_NEWLINE_CRLF: - fprintf(outfile, "Forced newline sequence: CRLF\n"); - break; - - case PCRE_NEWLINE_ANYCRLF: - fprintf(outfile, "Forced newline sequence: ANYCRLF\n"); - break; - - case PCRE_NEWLINE_ANY: - fprintf(outfile, "Forced newline sequence: ANY\n"); - break; - - default: - break; - } - - if (first_char_set == 2) - { - fprintf(outfile, "First char at start or follows newline\n"); - } - else if (first_char_set == 1) - { - const char *caseless = - ((REAL_PCRE_FLAGS(re) & PCRE_FCH_CASELESS) == 0)? - "" : " (caseless)"; - - if (PRINTOK(first_char)) - fprintf(outfile, "First char = \'%c\'%s\n", first_char, caseless); - else - { - fprintf(outfile, "First char = "); - pchar(first_char, outfile); - fprintf(outfile, "%s\n", caseless); - } - } - else - { - fprintf(outfile, "No first char\n"); - } - - if (need_char_set == 0) - { - fprintf(outfile, "No need char\n"); - } - else - { - const char *caseless = - ((REAL_PCRE_FLAGS(re) & PCRE_RCH_CASELESS) == 0)? - "" : " (caseless)"; - - if (PRINTOK(need_char)) - fprintf(outfile, "Need char = \'%c\'%s\n", need_char, caseless); - else - { - fprintf(outfile, "Need char = "); - pchar(need_char, outfile); - fprintf(outfile, "%s\n", caseless); - } - } - - /* Don't output study size; at present it is in any case a fixed - value, but it varies, depending on the computer architecture, and - so messes up the test suite. (And with the /F option, it might be - flipped.) If study was forced by an external -s, don't show this - information unless -i or -d was also present. This means that, except - when auto-callouts are involved, the output from runs with and without - -s should be identical. */ - - if (do_study || (force_study >= 0 && showinfo && !no_force_study)) - { - if (extra == NULL) - fprintf(outfile, "Study returned NULL\n"); - else - { - pcre_uint8 *start_bits = NULL; - int minlength; - - if (new_info(re, extra, PCRE_INFO_MINLENGTH, &minlength) == 0) - fprintf(outfile, "Subject length lower bound = %d\n", minlength); - - if (new_info(re, extra, PCRE_INFO_FIRSTTABLE, &start_bits) == 0) - { - if (start_bits == NULL) - fprintf(outfile, "No starting char list\n"); - else - { - int i; - int c = 24; - fprintf(outfile, "Starting chars: "); - for (i = 0; i < 256; i++) - { - if ((start_bits[i/8] & (1<<(i&7))) != 0) - { - if (c > 75) - { - fprintf(outfile, "\n "); - c = 2; - } - if (PRINTOK(i) && i != ' ') - { - fprintf(outfile, "%c ", i); - c += 2; - } - else - { - fprintf(outfile, "\\x%02x ", i); - c += 5; - } - } - } - fprintf(outfile, "\n"); - } - } - } - - /* Show this only if the JIT was set by /S, not by -s. */ - - if ((study_options & PCRE_STUDY_ALLJIT) != 0 && - (force_study_options & PCRE_STUDY_ALLJIT) == 0) - { - int jit; - if (new_info(re, extra, PCRE_INFO_JIT, &jit) == 0) - { - if (jit) - fprintf(outfile, "JIT study was successful\n"); - else -#ifdef SUPPORT_JIT - fprintf(outfile, "JIT study was not successful\n"); -#else - fprintf(outfile, "JIT support is not available in this version of PCRE\n"); -#endif - } - } - } - } - - /* If the '>' option was present, we write out the regex to a file, and - that is all. The first 8 bytes of the file are the regex length and then - the study length, in big-endian order. */ - - if (to_file != NULL) - { - FILE *f = fopen((char *)to_file, "wb"); - if (f == NULL) - { - fprintf(outfile, "Unable to open %s: %s\n", to_file, strerror(errno)); - } - else - { - pcre_uint8 sbuf[8]; - - if (do_flip) regexflip(re, extra); - sbuf[0] = (pcre_uint8)((true_size >> 24) & 255); - sbuf[1] = (pcre_uint8)((true_size >> 16) & 255); - sbuf[2] = (pcre_uint8)((true_size >> 8) & 255); - sbuf[3] = (pcre_uint8)((true_size) & 255); - sbuf[4] = (pcre_uint8)((true_study_size >> 24) & 255); - sbuf[5] = (pcre_uint8)((true_study_size >> 16) & 255); - sbuf[6] = (pcre_uint8)((true_study_size >> 8) & 255); - sbuf[7] = (pcre_uint8)((true_study_size) & 255); - - if (fwrite(sbuf, 1, 8, f) < 8 || - fwrite(re, 1, true_size, f) < true_size) - { - fprintf(outfile, "Write error on %s: %s\n", to_file, strerror(errno)); - } - else - { - fprintf(outfile, "Compiled pattern written to %s\n", to_file); - - /* If there is study data, write it. */ - - if (extra != NULL) - { - if (fwrite(extra->study_data, 1, true_study_size, f) < - true_study_size) - { - fprintf(outfile, "Write error on %s: %s\n", to_file, - strerror(errno)); - } - else fprintf(outfile, "Study data written to %s\n", to_file); - } - } - fclose(f); - } - - new_free(re); - if (extra != NULL) - { - PCRE_FREE_STUDY(extra); - } - if (locale_set) - { - new_free((void *)tables); - setlocale(LC_CTYPE, "C"); - locale_set = 0; - } - continue; /* With next regex */ - } - } /* End of non-POSIX compile */ - - /* Read data lines and test them */ - - for (;;) - { -#ifdef SUPPORT_PCRE8 - pcre_uint8 *q8; -#endif -#ifdef SUPPORT_PCRE16 - pcre_uint16 *q16; -#endif -#ifdef SUPPORT_PCRE32 - pcre_uint32 *q32; -#endif - pcre_uint8 *bptr; - int *use_offsets = offsets; - int use_size_offsets = size_offsets; - int callout_data = 0; - int callout_data_set = 0; - int count; - pcre_uint32 c; - int copystrings = 0; - int find_match_limit = default_find_match_limit; - int getstrings = 0; - int getlist = 0; - int gmatched = 0; - int start_offset = 0; - int start_offset_sign = 1; - int g_notempty = 0; - int use_dfa = 0; - - *copynames = 0; - *getnames = 0; - -#ifdef SUPPORT_PCRE32 - cn32ptr = copynames; - gn32ptr = getnames; -#endif -#ifdef SUPPORT_PCRE16 - cn16ptr = copynames16; - gn16ptr = getnames16; -#endif -#ifdef SUPPORT_PCRE8 - cn8ptr = copynames8; - gn8ptr = getnames8; -#endif - - SET_PCRE_CALLOUT(callout); - first_callout = 1; - last_callout_mark = NULL; - callout_extra = 0; - callout_count = 0; - callout_fail_count = 999999; - callout_fail_id = -1; - show_malloc = 0; - options = 0; - - if (extra != NULL) extra->flags &= - ~(PCRE_EXTRA_MATCH_LIMIT|PCRE_EXTRA_MATCH_LIMIT_RECURSION); - - len = 0; - for (;;) - { - if (extend_inputline(infile, buffer + len, "data> ") == NULL) - { - if (len > 0) /* Reached EOF without hitting a newline */ - { - fprintf(outfile, "\n"); - break; - } - done = 1; - goto CONTINUE; - } - if (infile != stdin) fprintf(outfile, "%s", (char *)buffer); - len = (int)strlen((char *)buffer); - if (buffer[len-1] == '\n') break; - } - - while (len > 0 && isspace(buffer[len-1])) len--; - buffer[len] = 0; - if (len == 0) break; - - p = buffer; - while (isspace(*p)) p++; - -#ifndef NOUTF - /* Check that the data is well-formed UTF-8 if we're in UTF mode. To create - invalid input to pcre_exec, you must use \x?? or \x{} sequences. */ - - if (use_utf) - { - pcre_uint8 *q; - pcre_uint32 cc; - int n = 1; - - for (q = p; n > 0 && *q; q += n) n = utf82ord(q, &cc); - if (n <= 0) - { - fprintf(outfile, "**Failed: invalid UTF-8 string cannot be used as input in UTF mode\n"); - goto NEXT_DATA; - } - } -#endif - -#ifdef SUPPORT_VALGRIND - /* Mark the dbuffer as addressable but undefined again. */ - - if (dbuffer != NULL) - { - VALGRIND_MAKE_MEM_UNDEFINED(dbuffer, dbuffer_size * CHAR_SIZE); - } -#endif - - /* Allocate a buffer to hold the data line; len+1 is an upper bound on - the number of pcre_uchar units that will be needed. */ - - while (dbuffer == NULL || (size_t)len >= dbuffer_size) - { - dbuffer_size *= 2; - dbuffer = (pcre_uint8 *)realloc(dbuffer, dbuffer_size * CHAR_SIZE); - if (dbuffer == NULL) - { - fprintf(stderr, "pcretest: realloc(%d) failed\n", (int)dbuffer_size); - exit(1); - } - } - -#ifdef SUPPORT_PCRE8 - q8 = (pcre_uint8 *) dbuffer; -#endif -#ifdef SUPPORT_PCRE16 - q16 = (pcre_uint16 *) dbuffer; -#endif -#ifdef SUPPORT_PCRE32 - q32 = (pcre_uint32 *) dbuffer; -#endif - - while ((c = *p++) != 0) - { - int i = 0; - int n = 0; - - /* In UTF mode, input can be UTF-8, so just copy all non-backslash bytes. - In non-UTF mode, allow the value of the byte to fall through to later, - where values greater than 127 are turned into UTF-8 when running in - 16-bit or 32-bit mode. */ - - if (c != '\\') - { -#ifndef NOUTF - if (use_utf && HASUTF8EXTRALEN(c)) { GETUTF8INC(c, p); } -#endif - } - - /* Handle backslash escapes */ - - else switch ((c = *p++)) - { - case 'a': c = CHAR_BEL; break; - case 'b': c = '\b'; break; - case 'e': c = CHAR_ESC; break; - case 'f': c = '\f'; break; - case 'n': c = '\n'; break; - case 'r': c = '\r'; break; - case 't': c = '\t'; break; - case 'v': c = '\v'; break; - - case '0': case '1': case '2': case '3': - case '4': case '5': case '6': case '7': - c -= '0'; - while (i++ < 2 && isdigit(*p) && *p != '8' && *p != '9') - c = c * 8 + *p++ - '0'; - break; - - case 'o': - if (*p == '{') - { - pcre_uint8 *pt = p; - c = 0; - for (pt++; isdigit(*pt) && *pt != '8' && *pt != '9'; pt++) - { - if (++i == 12) - fprintf(outfile, "** Too many octal digits in \\o{...} item; " - "using only the first twelve.\n"); - else c = c * 8 + *pt - '0'; - } - if (*pt == '}') p = pt + 1; - else fprintf(outfile, "** Missing } after \\o{ (assumed)\n"); - } - break; - - case 'x': - if (*p == '{') - { - pcre_uint8 *pt = p; - c = 0; - - /* We used to have "while (isxdigit(*(++pt)))" here, but it fails - when isxdigit() is a macro that refers to its argument more than - once. This is banned by the C Standard, but apparently happens in at - least one MacOS environment. */ - - for (pt++; isxdigit(*pt); pt++) - { - if (++i == 9) - fprintf(outfile, "** Too many hex digits in \\x{...} item; " - "using only the first eight.\n"); - else c = c * 16 + tolower(*pt) - ((isdigit(*pt))? '0' : 'a' - 10); - } - if (*pt == '}') - { - p = pt + 1; - break; - } - /* Not correct form for \x{...}; fall through */ - } - - /* \x without {} always defines just one byte in 8-bit mode. This - allows UTF-8 characters to be constructed byte by byte, and also allows - invalid UTF-8 sequences to be made. Just copy the byte in UTF mode. - Otherwise, pass it down to later code so that it can be turned into - UTF-8 when running in 16/32-bit mode. */ - - c = 0; - while (i++ < 2 && isxdigit(*p)) - { - c = c * 16 + tolower(*p) - ((isdigit(*p))? '0' : 'a' - 10); - p++; - } -#if !defined NOUTF && defined SUPPORT_PCRE8 - if (use_utf && (pcre_mode == PCRE8_MODE)) - { - *q8++ = c; - continue; - } -#endif - break; - - case 0: /* \ followed by EOF allows for an empty line */ - p--; - continue; - - case '>': - if (*p == '-') - { - start_offset_sign = -1; - p++; - } - while(isdigit(*p)) start_offset = start_offset * 10 + *p++ - '0'; - start_offset *= start_offset_sign; - continue; - - case 'A': /* Option setting */ - options |= PCRE_ANCHORED; - continue; - - case 'B': - options |= PCRE_NOTBOL; - continue; - - case 'C': - if (isdigit(*p)) /* Set copy string */ - { - while(isdigit(*p)) n = n * 10 + *p++ - '0'; - copystrings |= 1 << n; - } - else if (isalnum(*p)) - { - READ_CAPTURE_NAME(p, &cn8ptr, &cn16ptr, &cn32ptr, re); - } - else if (*p == '+') - { - callout_extra = 1; - p++; - } - else if (*p == '-') - { - SET_PCRE_CALLOUT(NULL); - p++; - } - else if (*p == '!') - { - callout_fail_id = 0; - p++; - while(isdigit(*p)) - callout_fail_id = callout_fail_id * 10 + *p++ - '0'; - callout_fail_count = 0; - if (*p == '!') - { - p++; - while(isdigit(*p)) - callout_fail_count = callout_fail_count * 10 + *p++ - '0'; - } - } - else if (*p == '*') - { - int sign = 1; - callout_data = 0; - if (*(++p) == '-') { sign = -1; p++; } - while(isdigit(*p)) - callout_data = callout_data * 10 + *p++ - '0'; - callout_data *= sign; - callout_data_set = 1; - } - continue; - -#if !defined NODFA - case 'D': -#if !defined NOPOSIX - if (posix || do_posix) - printf("** Can't use dfa matching in POSIX mode: \\D ignored\n"); - else -#endif - use_dfa = 1; - continue; -#endif - -#if !defined NODFA - case 'F': - options |= PCRE_DFA_SHORTEST; - continue; -#endif - - case 'G': - if (isdigit(*p)) - { - while(isdigit(*p)) n = n * 10 + *p++ - '0'; - getstrings |= 1 << n; - } - else if (isalnum(*p)) - { - READ_CAPTURE_NAME(p, &gn8ptr, &gn16ptr, &gn32ptr, re); - } - continue; - - case 'J': - while(isdigit(*p)) n = n * 10 + *p++ - '0'; - if (extra != NULL - && (extra->flags & PCRE_EXTRA_EXECUTABLE_JIT) != 0 - && extra->executable_jit != NULL) - { - if (jit_stack != NULL) { PCRE_JIT_STACK_FREE(jit_stack); } - jit_stack = PCRE_JIT_STACK_ALLOC(1, n * 1024); - PCRE_ASSIGN_JIT_STACK(extra, jit_callback, jit_stack); - } - continue; - - case 'L': - getlist = 1; - continue; - - case 'M': - find_match_limit = 1; - continue; - - case 'N': - if ((options & PCRE_NOTEMPTY) != 0) - options = (options & ~PCRE_NOTEMPTY) | PCRE_NOTEMPTY_ATSTART; - else - options |= PCRE_NOTEMPTY; - continue; - - case 'O': - while(isdigit(*p)) n = n * 10 + *p++ - '0'; - if (n > size_offsets_max) - { - size_offsets_max = n; - free(offsets); - use_offsets = offsets = (int *)malloc(size_offsets_max * sizeof(int)); - if (offsets == NULL) - { - printf("** Failed to get %d bytes of memory for offsets vector\n", - (int)(size_offsets_max * sizeof(int))); - yield = 1; - goto EXIT; - } - } - use_size_offsets = n; - if (n == 0) use_offsets = NULL; /* Ensures it can't write to it */ - else use_offsets = offsets + size_offsets_max - n; /* To catch overruns */ - continue; - - case 'P': - options |= ((options & PCRE_PARTIAL_SOFT) == 0)? - PCRE_PARTIAL_SOFT : PCRE_PARTIAL_HARD; - continue; - - case 'Q': - while(isdigit(*p)) n = n * 10 + *p++ - '0'; - if (extra == NULL) - { - extra = (pcre_extra *)malloc(sizeof(pcre_extra)); - extra->flags = 0; - } - extra->flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION; - extra->match_limit_recursion = n; - continue; - - case 'q': - while(isdigit(*p)) n = n * 10 + *p++ - '0'; - if (extra == NULL) - { - extra = (pcre_extra *)malloc(sizeof(pcre_extra)); - extra->flags = 0; - } - extra->flags |= PCRE_EXTRA_MATCH_LIMIT; - extra->match_limit = n; - continue; - -#if !defined NODFA - case 'R': - options |= PCRE_DFA_RESTART; - continue; -#endif - - case 'S': - show_malloc = 1; - continue; - - case 'Y': - options |= PCRE_NO_START_OPTIMIZE; - continue; - - case 'Z': - options |= PCRE_NOTEOL; - continue; - - case '?': - options |= PCRE_NO_UTF8_CHECK; - continue; - - case '<': - { - int x = check_mc_option(p, outfile, TRUE, "escape sequence"); - if (x == 0) goto NEXT_DATA; - options |= x; - while (*p++ != '>'); - } - continue; - } - - /* We now have a character value in c that may be greater than 255. - In 8-bit mode we convert to UTF-8 if we are in UTF mode. Values greater - than 127 in UTF mode must have come from \x{...} or octal constructs - because values from \x.. get this far only in non-UTF mode. */ - -#ifdef SUPPORT_PCRE8 - if (pcre_mode == PCRE8_MODE) - { -#ifndef NOUTF - if (use_utf) - { - if (c > 0x7fffffff) - { - fprintf(outfile, "** Character \\x{%x} is greater than 0x7fffffff " - "and so cannot be converted to UTF-8\n", c); - goto NEXT_DATA; - } - q8 += ord2utf8(c, q8); - } - else -#endif - { - if (c > 0xffu) - { - fprintf(outfile, "** Character \\x{%x} is greater than 255 " - "and UTF-8 mode is not enabled.\n", c); - fprintf(outfile, "** Truncation will probably give the wrong " - "result.\n"); - } - *q8++ = c; - } - } -#endif -#ifdef SUPPORT_PCRE16 - if (pcre_mode == PCRE16_MODE) - { -#ifndef NOUTF - if (use_utf) - { - if (c > 0x10ffffu) - { - fprintf(outfile, "** Failed: character \\x{%x} is greater than " - "0x10ffff and so cannot be converted to UTF-16\n", c); - goto NEXT_DATA; - } - else if (c >= 0x10000u) - { - c-= 0x10000u; - *q16++ = 0xD800 | (c >> 10); - *q16++ = 0xDC00 | (c & 0x3ff); - } - else - *q16++ = c; - } - else -#endif - { - if (c > 0xffffu) - { - fprintf(outfile, "** Character \\x{%x} is greater than 0xffff " - "and UTF-16 mode is not enabled.\n", c); - fprintf(outfile, "** Truncation will probably give the wrong " - "result.\n"); - } - - *q16++ = c; - } - } -#endif -#ifdef SUPPORT_PCRE32 - if (pcre_mode == PCRE32_MODE) - { - *q32++ = c; - } -#endif - - } - - /* Reached end of subject string */ - -#ifdef SUPPORT_PCRE8 - if (pcre_mode == PCRE8_MODE) - { - *q8 = 0; - len = (int)(q8 - (pcre_uint8 *)dbuffer); - } -#endif -#ifdef SUPPORT_PCRE16 - if (pcre_mode == PCRE16_MODE) - { - *q16 = 0; - len = (int)(q16 - (pcre_uint16 *)dbuffer); - } -#endif -#ifdef SUPPORT_PCRE32 - if (pcre_mode == PCRE32_MODE) - { - *q32 = 0; - len = (int)(q32 - (pcre_uint32 *)dbuffer); - } -#endif - - /* If we're compiling with explicit valgrind support, Mark the data from after - its end to the end of the buffer as unaddressable, so that a read over the end - of the buffer will be seen by valgrind, even if it doesn't cause a crash. - If we're not building with valgrind support, at least move the data to the end - of the buffer so that it might at least cause a crash. - If we are using the POSIX interface, we must include the terminating zero. */ - - bptr = dbuffer; - -#if !defined NOPOSIX - if (posix || do_posix) - { -#ifdef SUPPORT_VALGRIND - VALGRIND_MAKE_MEM_NOACCESS(dbuffer + len + 1, dbuffer_size - (len + 1)); -#else - memmove(bptr + dbuffer_size - len - 1, bptr, len + 1); - bptr += dbuffer_size - len - 1; -#endif - } - else -#endif - { -#ifdef SUPPORT_VALGRIND - VALGRIND_MAKE_MEM_NOACCESS(dbuffer + len * CHAR_SIZE, (dbuffer_size - len) * CHAR_SIZE); -#else - bptr = memmove(bptr + (dbuffer_size - len) * CHAR_SIZE, bptr, len * CHAR_SIZE); -#endif - } - - if ((all_use_dfa || use_dfa) && find_match_limit) - { - printf("**Match limit not relevant for DFA matching: ignored\n"); - find_match_limit = 0; - } - - /* Handle matching via the POSIX interface, which does not - support timing or playing with the match limit or callout data. */ - -#if !defined NOPOSIX - if (posix || do_posix) - { - int rc; - int eflags = 0; - regmatch_t *pmatch = NULL; - if (use_size_offsets > 0) - pmatch = (regmatch_t *)malloc(sizeof(regmatch_t) * use_size_offsets); - if ((options & PCRE_NOTBOL) != 0) eflags |= REG_NOTBOL; - if ((options & PCRE_NOTEOL) != 0) eflags |= REG_NOTEOL; - if ((options & PCRE_NOTEMPTY) != 0) eflags |= REG_NOTEMPTY; - - rc = regexec(&preg, (const char *)bptr, use_size_offsets, pmatch, eflags); - - if (rc != 0) - { - (void)regerror(rc, &preg, (char *)buffer, buffer_size); - fprintf(outfile, "No match: POSIX code %d: %s\n", rc, buffer); - } - else if ((REAL_PCRE_OPTIONS(preg.re_pcre) & PCRE_NO_AUTO_CAPTURE) != 0) - { - fprintf(outfile, "Matched with REG_NOSUB\n"); - } - else - { - size_t i; - for (i = 0; i < (size_t)use_size_offsets; i++) - { - if (pmatch[i].rm_so >= 0) - { - fprintf(outfile, "%2d: ", (int)i); - PCHARSV(dbuffer, pmatch[i].rm_so, - pmatch[i].rm_eo - pmatch[i].rm_so, outfile); - fprintf(outfile, "\n"); - if (do_showcaprest || (i == 0 && do_showrest)) - { - fprintf(outfile, "%2d+ ", (int)i); - PCHARSV(dbuffer, pmatch[i].rm_eo, len - pmatch[i].rm_eo, - outfile); - fprintf(outfile, "\n"); - } - } - } - } - free(pmatch); - goto NEXT_DATA; - } - -#endif /* !defined NOPOSIX */ - - /* Handle matching via the native interface - repeats for /g and /G */ - - /* Ensure that there is a JIT callback if we want to verify that JIT was - actually used. If jit_stack == NULL, no stack has yet been assigned. */ - - if (verify_jit && jit_stack == NULL && extra != NULL) - { PCRE_ASSIGN_JIT_STACK(extra, jit_callback, jit_stack); } - - for (;; gmatched++) /* Loop for /g or /G */ - { - markptr = NULL; - jit_was_used = FALSE; - - if (timeitm > 0) - { - register int i; - clock_t time_taken; - clock_t start_time = clock(); - -#if !defined NODFA - if (all_use_dfa || use_dfa) - { - if ((options & PCRE_DFA_RESTART) != 0) - { - fprintf(outfile, "Timing DFA restarts is not supported\n"); - break; - } - if (dfa_workspace == NULL) - dfa_workspace = (int *)malloc(DFA_WS_DIMENSION*sizeof(int)); - for (i = 0; i < timeitm; i++) - { - PCRE_DFA_EXEC(count, re, extra, bptr, len, start_offset, - (options | g_notempty), use_offsets, use_size_offsets, - dfa_workspace, DFA_WS_DIMENSION); - } - } - else -#endif - - for (i = 0; i < timeitm; i++) - { - PCRE_EXEC(count, re, extra, bptr, len, start_offset, - (options | g_notempty), use_offsets, use_size_offsets); - } - total_match_time += (time_taken = clock() - start_time); - fprintf(outfile, "Execute time %.4f milliseconds\n", - (((double)time_taken * 1000.0) / (double)timeitm) / - (double)CLOCKS_PER_SEC); - } - - /* If find_match_limit is set, we want to do repeated matches with - varying limits in order to find the minimum value for the match limit and - for the recursion limit. The match limits are relevant only to the normal - running of pcre_exec(), so disable the JIT optimization. This makes it - possible to run the same set of tests with and without JIT externally - requested. */ - - if (find_match_limit) - { - if (extra != NULL) { PCRE_FREE_STUDY(extra); } - extra = (pcre_extra *)malloc(sizeof(pcre_extra)); - extra->flags = 0; - - (void)check_match_limit(re, extra, bptr, len, start_offset, - options|g_notempty, use_offsets, use_size_offsets, - PCRE_EXTRA_MATCH_LIMIT, &(extra->match_limit), - PCRE_ERROR_MATCHLIMIT, "match()"); - - count = check_match_limit(re, extra, bptr, len, start_offset, - options|g_notempty, use_offsets, use_size_offsets, - PCRE_EXTRA_MATCH_LIMIT_RECURSION, &(extra->match_limit_recursion), - PCRE_ERROR_RECURSIONLIMIT, "match() recursion"); - } - - /* If callout_data is set, use the interface with additional data */ - - else if (callout_data_set) - { - if (extra == NULL) - { - extra = (pcre_extra *)malloc(sizeof(pcre_extra)); - extra->flags = 0; - } - extra->flags |= PCRE_EXTRA_CALLOUT_DATA; - extra->callout_data = &callout_data; - PCRE_EXEC(count, re, extra, bptr, len, start_offset, - options | g_notempty, use_offsets, use_size_offsets); - extra->flags &= ~PCRE_EXTRA_CALLOUT_DATA; - } - - /* The normal case is just to do the match once, with the default - value of match_limit. */ - -#if !defined NODFA - else if (all_use_dfa || use_dfa) - { - if (dfa_workspace == NULL) - dfa_workspace = (int *)malloc(DFA_WS_DIMENSION*sizeof(int)); - if (dfa_matched++ == 0) - dfa_workspace[0] = -1; /* To catch bad restart */ - PCRE_DFA_EXEC(count, re, extra, bptr, len, start_offset, - (options | g_notempty), use_offsets, use_size_offsets, dfa_workspace, - DFA_WS_DIMENSION); - if (count == 0) - { - fprintf(outfile, "Matched, but offsets vector is too small to show all matches\n"); - count = use_size_offsets/2; - } - } -#endif - - else - { - PCRE_EXEC(count, re, extra, bptr, len, start_offset, - options | g_notempty, use_offsets, use_size_offsets); - if (count == 0) - { - fprintf(outfile, "Matched, but too many substrings\n"); - /* 2 is a special case; match can be returned */ - count = (use_size_offsets == 2)? 1 : use_size_offsets/3; - } - } - - /* Matched */ - - if (count >= 0) - { - int i, maxcount; - void *cnptr, *gnptr; - -#if !defined NODFA - if (all_use_dfa || use_dfa) maxcount = use_size_offsets/2; else -#endif - /* 2 is a special case; match can be returned */ - maxcount = (use_size_offsets == 2)? 1 : use_size_offsets/3; - - /* This is a check against a lunatic return value. */ - - if (count > maxcount) - { - fprintf(outfile, - "** PCRE error: returned count %d is too big for offset size %d\n", - count, use_size_offsets); - count = use_size_offsets/3; - if (do_g || do_G) - { - fprintf(outfile, "** /%c loop abandoned\n", do_g? 'g' : 'G'); - do_g = do_G = FALSE; /* Break g/G loop */ - } - } - - /* do_allcaps requests showing of all captures in the pattern, to check - unset ones at the end. */ - - if (do_allcaps) - { - if (new_info(re, NULL, PCRE_INFO_CAPTURECOUNT, &count) < 0) - goto SKIP_DATA; - count++; /* Allow for full match */ - if (count * 2 > use_size_offsets) count = use_size_offsets/2; - } - - /* Output the captured substrings. Note that, for the matched string, - the use of \K in an assertion can make the start later than the end. */ - - for (i = 0; i < count * 2; i += 2) - { - if (use_offsets[i] < 0) - { - if (use_offsets[i] != -1) - fprintf(outfile, "ERROR: bad negative value %d for offset %d\n", - use_offsets[i], i); - if (use_offsets[i+1] != -1) - fprintf(outfile, "ERROR: bad negative value %d for offset %d\n", - use_offsets[i+1], i+1); - fprintf(outfile, "%2d: \n", i/2); - } - else - { - int start = use_offsets[i]; - int end = use_offsets[i+1]; - - if (start > end) - { - start = use_offsets[i+1]; - end = use_offsets[i]; - fprintf(outfile, "Start of matched string is beyond its end - " - "displaying from end to start.\n"); - } - - fprintf(outfile, "%2d: ", i/2); - PCHARSV(bptr, start, end - start, outfile); - if (verify_jit && jit_was_used) fprintf(outfile, " (JIT)"); - fprintf(outfile, "\n"); - - /* Note: don't use the start/end variables here because we want to - show the text from what is reported as the end. */ - - if (do_showcaprest || (i == 0 && do_showrest)) - { - fprintf(outfile, "%2d+ ", i/2); - PCHARSV(bptr, use_offsets[i+1], len - use_offsets[i+1], - outfile); - fprintf(outfile, "\n"); - } - } - } - - if (markptr != NULL) - { - fprintf(outfile, "MK: "); - PCHARSV(markptr, 0, -1, outfile); - fprintf(outfile, "\n"); - } - - for (i = 0; i < 32; i++) - { - if ((copystrings & (1 << i)) != 0) - { - int rc; - char copybuffer[256]; - PCRE_COPY_SUBSTRING(rc, bptr, use_offsets, count, i, - copybuffer, sizeof(copybuffer)); - if (rc < 0) - fprintf(outfile, "copy substring %d failed %d\n", i, rc); - else - { - fprintf(outfile, "%2dC ", i); - PCHARSV(copybuffer, 0, rc, outfile); - fprintf(outfile, " (%d)\n", rc); - } - } - } - - cnptr = copynames; - for (;;) - { - int rc; - char copybuffer[256]; - -#ifdef SUPPORT_PCRE32 - if (pcre_mode == PCRE32_MODE) - { - if (*(pcre_uint32 *)cnptr == 0) break; - } -#endif -#ifdef SUPPORT_PCRE16 - if (pcre_mode == PCRE16_MODE) - { - if (*(pcre_uint16 *)cnptr == 0) break; - } -#endif -#ifdef SUPPORT_PCRE8 - if (pcre_mode == PCRE8_MODE) - { - if (*(pcre_uint8 *)cnptr == 0) break; - } -#endif - - PCRE_COPY_NAMED_SUBSTRING(rc, re, bptr, use_offsets, count, - cnptr, copybuffer, sizeof(copybuffer)); - - if (rc < 0) - { - fprintf(outfile, "copy substring "); - PCHARSV(cnptr, 0, -1, outfile); - fprintf(outfile, " failed %d\n", rc); - } - else - { - fprintf(outfile, " C "); - PCHARSV(copybuffer, 0, rc, outfile); - fprintf(outfile, " (%d) ", rc); - PCHARSV(cnptr, 0, -1, outfile); - putc('\n', outfile); - } - - cnptr = (char *)cnptr + (STRLEN(cnptr) + 1) * CHAR_SIZE; - } - - for (i = 0; i < 32; i++) - { - if ((getstrings & (1 << i)) != 0) - { - int rc; - const char *substring; - PCRE_GET_SUBSTRING(rc, bptr, use_offsets, count, i, &substring); - if (rc < 0) - fprintf(outfile, "get substring %d failed %d\n", i, rc); - else - { - fprintf(outfile, "%2dG ", i); - PCHARSV(substring, 0, rc, outfile); - fprintf(outfile, " (%d)\n", rc); - PCRE_FREE_SUBSTRING(substring); - } - } - } - - gnptr = getnames; - for (;;) - { - int rc; - const char *substring; - -#ifdef SUPPORT_PCRE32 - if (pcre_mode == PCRE32_MODE) - { - if (*(pcre_uint32 *)gnptr == 0) break; - } -#endif -#ifdef SUPPORT_PCRE16 - if (pcre_mode == PCRE16_MODE) - { - if (*(pcre_uint16 *)gnptr == 0) break; - } -#endif -#ifdef SUPPORT_PCRE8 - if (pcre_mode == PCRE8_MODE) - { - if (*(pcre_uint8 *)gnptr == 0) break; - } -#endif - - PCRE_GET_NAMED_SUBSTRING(rc, re, bptr, use_offsets, count, - gnptr, &substring); - if (rc < 0) - { - fprintf(outfile, "get substring "); - PCHARSV(gnptr, 0, -1, outfile); - fprintf(outfile, " failed %d\n", rc); - } - else - { - fprintf(outfile, " G "); - PCHARSV(substring, 0, rc, outfile); - fprintf(outfile, " (%d) ", rc); - PCHARSV(gnptr, 0, -1, outfile); - PCRE_FREE_SUBSTRING(substring); - putc('\n', outfile); - } - - gnptr = (char *)gnptr + (STRLEN(gnptr) + 1) * CHAR_SIZE; - } - - if (getlist) - { - int rc; - const char **stringlist; - PCRE_GET_SUBSTRING_LIST(rc, bptr, use_offsets, count, &stringlist); - if (rc < 0) - fprintf(outfile, "get substring list failed %d\n", rc); - else - { - for (i = 0; i < count; i++) - { - fprintf(outfile, "%2dL ", i); - PCHARSV(stringlist[i], 0, -1, outfile); - putc('\n', outfile); - } - if (stringlist[i] != NULL) - fprintf(outfile, "string list not terminated by NULL\n"); - PCRE_FREE_SUBSTRING_LIST(stringlist); - } - } - } - - /* There was a partial match. If the bumpalong point is not the same as - the first inspected character, show the offset explicitly. */ - - else if (count == PCRE_ERROR_PARTIAL) - { - fprintf(outfile, "Partial match"); - if (use_size_offsets > 2 && use_offsets[0] != use_offsets[2]) - fprintf(outfile, " at offset %d", use_offsets[2]); - if (markptr != NULL) - { - fprintf(outfile, ", mark="); - PCHARSV(markptr, 0, -1, outfile); - } - if (use_size_offsets > 1) - { - fprintf(outfile, ": "); - PCHARSV(bptr, use_offsets[0], use_offsets[1] - use_offsets[0], - outfile); - } - if (verify_jit && jit_was_used) fprintf(outfile, " (JIT)"); - fprintf(outfile, "\n"); - break; /* Out of the /g loop */ - } - - /* Failed to match. If this is a /g or /G loop and we previously set - g_notempty after a null match, this is not necessarily the end. We want - to advance the start offset, and continue. We won't be at the end of the - string - that was checked before setting g_notempty. - - Complication arises in the case when the newline convention is "any", - "crlf", or "anycrlf". If the previous match was at the end of a line - terminated by CRLF, an advance of one character just passes the \r, - whereas we should prefer the longer newline sequence, as does the code in - pcre_exec(). Fudge the offset value to achieve this. We check for a - newline setting in the pattern; if none was set, use PCRE_CONFIG() to - find the default. - - Otherwise, in the case of UTF-8 matching, the advance must be one - character, not one byte. */ - - else - { - if (g_notempty != 0) - { - int onechar = 1; - unsigned int obits = REAL_PCRE_OPTIONS(re); - use_offsets[0] = start_offset; - if ((obits & PCRE_NEWLINE_BITS) == 0) - { - int d; - (void)PCRE_CONFIG(PCRE_CONFIG_NEWLINE, &d); - /* Note that these values are always the ASCII ones, even in - EBCDIC environments. CR = 13, NL = 10. */ - obits = (d == 13)? PCRE_NEWLINE_CR : - (d == 10)? PCRE_NEWLINE_LF : - (d == (13<<8 | 10))? PCRE_NEWLINE_CRLF : - (d == -2)? PCRE_NEWLINE_ANYCRLF : - (d == -1)? PCRE_NEWLINE_ANY : 0; - } - if (((obits & PCRE_NEWLINE_BITS) == PCRE_NEWLINE_ANY || - (obits & PCRE_NEWLINE_BITS) == PCRE_NEWLINE_CRLF || - (obits & PCRE_NEWLINE_BITS) == PCRE_NEWLINE_ANYCRLF) - && - start_offset < len - 1 && ( -#ifdef SUPPORT_PCRE8 - (pcre_mode == PCRE8_MODE && - bptr[start_offset] == '\r' && - bptr[start_offset + 1] == '\n') || -#endif -#ifdef SUPPORT_PCRE16 - (pcre_mode == PCRE16_MODE && - ((PCRE_SPTR16)bptr)[start_offset] == '\r' && - ((PCRE_SPTR16)bptr)[start_offset + 1] == '\n') || -#endif -#ifdef SUPPORT_PCRE32 - (pcre_mode == PCRE32_MODE && - ((PCRE_SPTR32)bptr)[start_offset] == '\r' && - ((PCRE_SPTR32)bptr)[start_offset + 1] == '\n') || -#endif - 0)) - onechar++; - else if (use_utf) - { - while (start_offset + onechar < len) - { - if ((bptr[start_offset+onechar] & 0xc0) != 0x80) break; - onechar++; - } - } - use_offsets[1] = start_offset + onechar; - } - else - { - switch(count) - { - case PCRE_ERROR_NOMATCH: - if (gmatched == 0) - { - if (markptr == NULL) - { - fprintf(outfile, "No match"); - } - else - { - fprintf(outfile, "No match, mark = "); - PCHARSV(markptr, 0, -1, outfile); - } - if (verify_jit && jit_was_used) fprintf(outfile, " (JIT)"); - putc('\n', outfile); - } - break; - - case PCRE_ERROR_BADUTF8: - case PCRE_ERROR_SHORTUTF8: - fprintf(outfile, "Error %d (%s UTF-%d string)", count, - (count == PCRE_ERROR_BADUTF8)? "bad" : "short", - 8 * CHAR_SIZE); - if (use_size_offsets >= 2) - fprintf(outfile, " offset=%d reason=%d", use_offsets[0], - use_offsets[1]); - fprintf(outfile, "\n"); - break; - - case PCRE_ERROR_BADUTF8_OFFSET: - fprintf(outfile, "Error %d (bad UTF-%d offset)\n", count, - 8 * CHAR_SIZE); - break; - - default: - if (count < 0 && - (-count) < (int)(sizeof(errtexts)/sizeof(const char *))) - fprintf(outfile, "Error %d (%s)\n", count, errtexts[-count]); - else - fprintf(outfile, "Error %d (Unexpected value)\n", count); - break; - } - - break; /* Out of the /g loop */ - } - } - - /* If not /g or /G we are done */ - - if (!do_g && !do_G) break; - - if (use_offsets == NULL) - { - fprintf(outfile, "Cannot do global matching without an ovector\n"); - break; - } - - if (use_size_offsets < 2) - { - fprintf(outfile, "Cannot do global matching with an ovector size < 2\n"); - break; - } - - /* If we have matched an empty string, first check to see if we are at - the end of the subject. If so, the /g loop is over. Otherwise, mimic what - Perl's /g options does. This turns out to be rather cunning. First we set - PCRE_NOTEMPTY_ATSTART and PCRE_ANCHORED and try the match again at the - same point. If this fails (picked up above) we advance to the next - character. */ - - g_notempty = 0; - - if (use_offsets[0] == use_offsets[1]) - { - if (use_offsets[0] == len) break; - g_notempty = PCRE_NOTEMPTY_ATSTART | PCRE_ANCHORED; - } - - /* For /g, update the start offset, leaving the rest alone. There is a - tricky case when \K is used in a positive lookbehind assertion. This can - cause the end of the match to be less than or equal to the start offset. - In this case we restart at one past the start offset. This may return the - same match if the original start offset was bumped along during the - match, but eventually the new start offset will hit the actual start - offset. (In PCRE2 the true start offset is available, and this can be - done better. It is not worth doing more than making sure we do not loop - at this stage in the life of PCRE1.) */ - - if (do_g) - { - if (g_notempty == 0 && use_offsets[1] <= start_offset) - { - if (start_offset >= len) break; /* End of subject */ - start_offset++; - if (use_utf) - { - while (start_offset < len) - { - if ((bptr[start_offset] & 0xc0) != 0x80) break; - start_offset++; - } - } - } - else start_offset = use_offsets[1]; - } - - /* For /G, update the pointer and length */ - - else - { - bptr += use_offsets[1] * CHAR_SIZE; - len -= use_offsets[1]; - } - } /* End of loop for /g and /G */ - - NEXT_DATA: continue; - } /* End of loop for data lines */ - - CONTINUE: - -#if !defined NOPOSIX - if ((posix || do_posix) && preg.re_pcre != 0) regfree(&preg); -#endif - - if (re != NULL) new_free(re); - if (extra != NULL) - { - PCRE_FREE_STUDY(extra); - } - if (locale_set) - { - new_free((void *)tables); - setlocale(LC_CTYPE, "C"); - locale_set = 0; - } - if (jit_stack != NULL) - { - PCRE_JIT_STACK_FREE(jit_stack); - jit_stack = NULL; - } - } - -if (infile == stdin) fprintf(outfile, "\n"); - -if (showtotaltimes) - { - fprintf(outfile, "--------------------------------------\n"); - if (timeit > 0) - { - fprintf(outfile, "Total compile time %.4f milliseconds\n", - (((double)total_compile_time * 1000.0) / (double)timeit) / - (double)CLOCKS_PER_SEC); - fprintf(outfile, "Total study time %.4f milliseconds\n", - (((double)total_study_time * 1000.0) / (double)timeit) / - (double)CLOCKS_PER_SEC); - } - fprintf(outfile, "Total execute time %.4f milliseconds\n", - (((double)total_match_time * 1000.0) / (double)timeitm) / - (double)CLOCKS_PER_SEC); - } - -EXIT: - -if (infile != NULL && infile != stdin) fclose(infile); -if (outfile != NULL && outfile != stdout) fclose(outfile); - -free(buffer); -free(dbuffer); -free(pbuffer); -free(offsets); - -#ifdef SUPPORT_PCRE16 -if (buffer16 != NULL) free(buffer16); -#endif -#ifdef SUPPORT_PCRE32 -if (buffer32 != NULL) free(buffer32); -#endif - -#if !defined NODFA -if (dfa_workspace != NULL) - free(dfa_workspace); -#endif - -#if defined(__VMS) - yield = SS$_NORMAL; /* Return values via DCL symbols */ -#endif - -return yield; -} - -/* End of pcretest.c */ - diff --git a/src/third_party/pcre-8.39/perltest.pl b/src/third_party/pcre-8.39/perltest.pl deleted file mode 100755 index 29b808b5293..00000000000 --- a/src/third_party/pcre-8.39/perltest.pl +++ /dev/null @@ -1,242 +0,0 @@ -#! /usr/bin/env perl - -# Program for testing regular expressions with perl to check that PCRE handles -# them the same. This version needs to have "use utf8" at the start for running -# the UTF-8 tests, but *not* for the other tests. The only way I've found for -# doing this is to cat this line in explicitly in the RunPerlTest script. I've -# also used this method to supply "require Encode" for the UTF-8 tests, so that -# the main test will still run where Encode is not installed. - -#use utf8; -#require Encode; - -# Function for turning a string into a string of printing chars. - -sub pchars { -my($t) = ""; - -if ($utf8) - { - @p = unpack('U*', $_[0]); - foreach $c (@p) - { - if ($c >= 32 && $c < 127) { $t .= chr $c; } - else { $t .= sprintf("\\x{%02x}", $c); - } - } - } -else - { - foreach $c (split(//, $_[0])) - { - if (ord $c >= 32 && ord $c < 127) { $t .= $c; } - else { $t .= sprintf("\\x%02x", ord $c); } - } - } - -$t; -} - - -# Read lines from named file or stdin and write to named file or stdout; lines -# consist of a regular expression, in delimiters and optionally followed by -# options, followed by a set of test data, terminated by an empty line. - -# Sort out the input and output files - -if (@ARGV > 0) - { - open(INFILE, "<$ARGV[0]") || die "Failed to open $ARGV[0]\n"; - $infile = "INFILE"; - } -else { $infile = "STDIN"; } - -if (@ARGV > 1) - { - open(OUTFILE, ">$ARGV[1]") || die "Failed to open $ARGV[1]\n"; - $outfile = "OUTFILE"; - } -else { $outfile = "STDOUT"; } - -printf($outfile "Perl $] Regular Expressions\n\n"); - -# Main loop - -NEXT_RE: -for (;;) - { - printf " re> " if $infile eq "STDIN"; - last if ! ($_ = <$infile>); - printf $outfile "$_" if $infile ne "STDIN"; - next if ($_ =~ /^\s*$/ || $_ =~ /^< forbid/); - - $pattern = $_; - - while ($pattern !~ /^\s*(.).*\1/s) - { - printf " > " if $infile eq "STDIN"; - last if ! ($_ = <$infile>); - printf $outfile "$_" if $infile ne "STDIN"; - $pattern .= $_; - } - - chomp($pattern); - $pattern =~ s/\s+$//; - - # The private /+ modifier means "print $' afterwards". - - $showrest = ($pattern =~ s/\+(?=[a-zA-Z]*$)//); - - # A doubled version is used by pcretest to print remainders after captures - - $pattern =~ s/\+(?=[a-zA-Z]*$)//; - - # Remove /8 from a UTF-8 pattern. - - $utf8 = $pattern =~ s/8(?=[a-zA-Z]*$)//; - - # Remove /J from a pattern with duplicate names. - - $pattern =~ s/J(?=[a-zA-Z]*$)//; - - # Remove /K from a pattern (asks pcretest to check MARK data) */ - - $pattern =~ s/K(?=[a-zA-Z]*$)//; - - # /W asks pcretest to set PCRE_UCP; change this to /u for Perl - - $pattern =~ s/W(?=[a-zA-Z]*$)/u/; - - # Remove /S or /SS from a pattern (asks pcretest to study or not to study) - - $pattern =~ s/S(?=[a-zA-Z]*$)//g; - - # Remove /Y and /O from a pattern (disable PCRE optimizations) - - $pattern =~ s/[YO](?=[a-zA-Z]*$)//; - - # Check that the pattern is valid - - eval "\$_ =~ ${pattern}"; - if ($@) - { - printf $outfile "Error: $@"; - if ($infile != "STDIN") - { - for (;;) - { - last if ! ($_ = <$infile>); - last if $_ =~ /^\s*$/; - } - } - next NEXT_RE; - } - - # If the /g modifier is present, we want to put a loop round the matching; - # otherwise just a single "if". - - $cmd = ($pattern =~ /g[a-z]*$/)? "while" : "if"; - - # If the pattern is actually the null string, Perl uses the most recently - # executed (and successfully compiled) regex is used instead. This is a - # nasty trap for the unwary! The PCRE test suite does contain null strings - # in places - if they are allowed through here all sorts of weird and - # unexpected effects happen. To avoid this, we replace such patterns with - # a non-null pattern that has the same effect. - - $pattern = "/(?#)/$2" if ($pattern =~ /^(.)\1(.*)$/); - - # Read data lines and test them - - for (;;) - { - printf "data> " if $infile eq "STDIN"; - last NEXT_RE if ! ($_ = <$infile>); - chomp; - printf $outfile "$_\n" if $infile ne "STDIN"; - - s/\s+$//; # Remove trailing space - s/^\s+//; # Remove leading space - s/\\Y//g; # Remove \Y (pcretest flag to set PCRE_NO_START_OPTIMIZE) - - last if ($_ eq ""); - $x = eval "\"$_\""; # To get escapes processed - - # Empty array for holding results, ensure $REGERROR and $REGMARK are - # unset, then do the matching. - - @subs = (); - - $pushes = "push \@subs,\$&;" . - "push \@subs,\$1;" . - "push \@subs,\$2;" . - "push \@subs,\$3;" . - "push \@subs,\$4;" . - "push \@subs,\$5;" . - "push \@subs,\$6;" . - "push \@subs,\$7;" . - "push \@subs,\$8;" . - "push \@subs,\$9;" . - "push \@subs,\$10;" . - "push \@subs,\$11;" . - "push \@subs,\$12;" . - "push \@subs,\$13;" . - "push \@subs,\$14;" . - "push \@subs,\$15;" . - "push \@subs,\$16;" . - "push \@subs,\$'; }"; - - undef $REGERROR; - undef $REGMARK; - - eval "${cmd} (\$x =~ ${pattern}) {" . $pushes; - - if ($@) - { - printf $outfile "Error: $@\n"; - next NEXT_RE; - } - elsif (scalar(@subs) == 0) - { - printf $outfile "No match"; - if (defined $REGERROR && $REGERROR != 1) - { printf $outfile (", mark = %s", &pchars($REGERROR)); } - printf $outfile "\n"; - } - else - { - while (scalar(@subs) != 0) - { - printf $outfile (" 0: %s\n", &pchars($subs[0])); - printf $outfile (" 0+ %s\n", &pchars($subs[17])) if $showrest; - $last_printed = 0; - for ($i = 1; $i <= 16; $i++) - { - if (defined $subs[$i]) - { - while ($last_printed++ < $i-1) - { printf $outfile ("%2d: \n", $last_printed); } - printf $outfile ("%2d: %s\n", $i, &pchars($subs[$i])); - $last_printed = $i; - } - } - splice(@subs, 0, 18); - } - - # It seems that $REGMARK is not marked as UTF-8 even when use utf8 is - # set and the input pattern was a UTF-8 string. We can, however, force - # it to be so marked. - - if (defined $REGMARK && $REGMARK != 1) - { - $xx = $REGMARK; - $xx = Encode::decode_utf8($xx) if $utf8; - printf $outfile ("MK: %s\n", &pchars($xx)); - } - } - } - } - -# printf $outfile "\n"; - -# End diff --git a/src/third_party/pcre-8.39/ucp.h b/src/third_party/pcre-8.39/ucp.h deleted file mode 100644 index 2fa00296e42..00000000000 --- a/src/third_party/pcre-8.39/ucp.h +++ /dev/null @@ -1,224 +0,0 @@ -/************************************************* -* Unicode Property Table handler * -*************************************************/ - -#ifndef _UCP_H -#define _UCP_H - -/* This file contains definitions of the property values that are returned by -the UCD access macros. New values that are added for new releases of Unicode -should always be at the end of each enum, for backwards compatibility. - -IMPORTANT: Note also that the specific numeric values of the enums have to be -the same as the values that are generated by the maint/MultiStage2.py script, -where the equivalent property descriptive names are listed in vectors. - -ALSO: The specific values of the first two enums are assumed for the table -called catposstab in pcre_compile.c. */ - -/* These are the general character categories. */ - -enum { - ucp_C, /* Other */ - ucp_L, /* Letter */ - ucp_M, /* Mark */ - ucp_N, /* Number */ - ucp_P, /* Punctuation */ - ucp_S, /* Symbol */ - ucp_Z /* Separator */ -}; - -/* These are the particular character categories. */ - -enum { - ucp_Cc, /* Control */ - ucp_Cf, /* Format */ - ucp_Cn, /* Unassigned */ - ucp_Co, /* Private use */ - ucp_Cs, /* Surrogate */ - ucp_Ll, /* Lower case letter */ - ucp_Lm, /* Modifier letter */ - ucp_Lo, /* Other letter */ - ucp_Lt, /* Title case letter */ - ucp_Lu, /* Upper case letter */ - ucp_Mc, /* Spacing mark */ - ucp_Me, /* Enclosing mark */ - ucp_Mn, /* Non-spacing mark */ - ucp_Nd, /* Decimal number */ - ucp_Nl, /* Letter number */ - ucp_No, /* Other number */ - ucp_Pc, /* Connector punctuation */ - ucp_Pd, /* Dash punctuation */ - ucp_Pe, /* Close punctuation */ - ucp_Pf, /* Final punctuation */ - ucp_Pi, /* Initial punctuation */ - ucp_Po, /* Other punctuation */ - ucp_Ps, /* Open punctuation */ - ucp_Sc, /* Currency symbol */ - ucp_Sk, /* Modifier symbol */ - ucp_Sm, /* Mathematical symbol */ - ucp_So, /* Other symbol */ - ucp_Zl, /* Line separator */ - ucp_Zp, /* Paragraph separator */ - ucp_Zs /* Space separator */ -}; - -/* These are grapheme break properties. Note that the code for processing them -assumes that the values are less than 16. If more values are added that take -the number to 16 or more, the code will have to be rewritten. */ - -enum { - ucp_gbCR, /* 0 */ - ucp_gbLF, /* 1 */ - ucp_gbControl, /* 2 */ - ucp_gbExtend, /* 3 */ - ucp_gbPrepend, /* 4 */ - ucp_gbSpacingMark, /* 5 */ - ucp_gbL, /* 6 Hangul syllable type L */ - ucp_gbV, /* 7 Hangul syllable type V */ - ucp_gbT, /* 8 Hangul syllable type T */ - ucp_gbLV, /* 9 Hangul syllable type LV */ - ucp_gbLVT, /* 10 Hangul syllable type LVT */ - ucp_gbRegionalIndicator, /* 11 */ - ucp_gbOther /* 12 */ -}; - -/* These are the script identifications. */ - -enum { - ucp_Arabic, - ucp_Armenian, - ucp_Bengali, - ucp_Bopomofo, - ucp_Braille, - ucp_Buginese, - ucp_Buhid, - ucp_Canadian_Aboriginal, - ucp_Cherokee, - ucp_Common, - ucp_Coptic, - ucp_Cypriot, - ucp_Cyrillic, - ucp_Deseret, - ucp_Devanagari, - ucp_Ethiopic, - ucp_Georgian, - ucp_Glagolitic, - ucp_Gothic, - ucp_Greek, - ucp_Gujarati, - ucp_Gurmukhi, - ucp_Han, - ucp_Hangul, - ucp_Hanunoo, - ucp_Hebrew, - ucp_Hiragana, - ucp_Inherited, - ucp_Kannada, - ucp_Katakana, - ucp_Kharoshthi, - ucp_Khmer, - ucp_Lao, - ucp_Latin, - ucp_Limbu, - ucp_Linear_B, - ucp_Malayalam, - ucp_Mongolian, - ucp_Myanmar, - ucp_New_Tai_Lue, - ucp_Ogham, - ucp_Old_Italic, - ucp_Old_Persian, - ucp_Oriya, - ucp_Osmanya, - ucp_Runic, - ucp_Shavian, - ucp_Sinhala, - ucp_Syloti_Nagri, - ucp_Syriac, - ucp_Tagalog, - ucp_Tagbanwa, - ucp_Tai_Le, - ucp_Tamil, - ucp_Telugu, - ucp_Thaana, - ucp_Thai, - ucp_Tibetan, - ucp_Tifinagh, - ucp_Ugaritic, - ucp_Yi, - /* New for Unicode 5.0: */ - ucp_Balinese, - ucp_Cuneiform, - ucp_Nko, - ucp_Phags_Pa, - ucp_Phoenician, - /* New for Unicode 5.1: */ - ucp_Carian, - ucp_Cham, - ucp_Kayah_Li, - ucp_Lepcha, - ucp_Lycian, - ucp_Lydian, - ucp_Ol_Chiki, - ucp_Rejang, - ucp_Saurashtra, - ucp_Sundanese, - ucp_Vai, - /* New for Unicode 5.2: */ - ucp_Avestan, - ucp_Bamum, - ucp_Egyptian_Hieroglyphs, - ucp_Imperial_Aramaic, - ucp_Inscriptional_Pahlavi, - ucp_Inscriptional_Parthian, - ucp_Javanese, - ucp_Kaithi, - ucp_Lisu, - ucp_Meetei_Mayek, - ucp_Old_South_Arabian, - ucp_Old_Turkic, - ucp_Samaritan, - ucp_Tai_Tham, - ucp_Tai_Viet, - /* New for Unicode 6.0.0: */ - ucp_Batak, - ucp_Brahmi, - ucp_Mandaic, - /* New for Unicode 6.1.0: */ - ucp_Chakma, - ucp_Meroitic_Cursive, - ucp_Meroitic_Hieroglyphs, - ucp_Miao, - ucp_Sharada, - ucp_Sora_Sompeng, - ucp_Takri, - /* New for Unicode 7.0.0: */ - ucp_Bassa_Vah, - ucp_Caucasian_Albanian, - ucp_Duployan, - ucp_Elbasan, - ucp_Grantha, - ucp_Khojki, - ucp_Khudawadi, - ucp_Linear_A, - ucp_Mahajani, - ucp_Manichaean, - ucp_Mende_Kikakui, - ucp_Modi, - ucp_Mro, - ucp_Nabataean, - ucp_Old_North_Arabian, - ucp_Old_Permic, - ucp_Pahawh_Hmong, - ucp_Palmyrene, - ucp_Psalter_Pahlavi, - ucp_Pau_Cin_Hau, - ucp_Siddham, - ucp_Tirhuta, - ucp_Warang_Citi -}; - -#endif - -/* End of ucp.h */ diff --git a/src/third_party/pcre-8.41/132html b/src/third_party/pcre-8.41/132html new file mode 100755 index 00000000000..e00024970a5 --- /dev/null +++ b/src/third_party/pcre-8.41/132html @@ -0,0 +1,313 @@ +#! /usr/bin/perl -w + +# Script to turn PCRE man pages into HTML + + +# Subroutine to handle font changes and other escapes + +sub do_line { +my($s) = $_[0]; + +$s =~ s/ +$s =~ s/>/>/g; +$s =~ s"\\fI(.*?)\\f[RP]"$1"g; +$s =~ s"\\fB(.*?)\\f[RP]"$1"g; +$s =~ s"\\e"\\"g; +$s =~ s/(?<=Copyright )\(c\)/©/g; +$s; +} + +# Subroutine to ensure not in a paragraph + +sub end_para { +if ($inpara) + { + print TEMP "\n" if ($inpre); + print TEMP "

\n"; + } +$inpara = $inpre = 0; +$wrotetext = 0; +} + +# Subroutine to start a new paragraph + +sub new_para { +&end_para(); +print TEMP "

\n"; +$inpara = 1; +} + + +# Main program + +$innf = 0; +$inpara = 0; +$inpre = 0; +$wrotetext = 0; +$toc = 0; +$ref = 1; + +while ($#ARGV >= 0 && $ARGV[0] =~ /^-/) + { + $toc = 1 if $ARGV[0] eq "-toc"; + shift; + } + +# Initial output to STDOUT + +print < + +$ARGV[0] specification + + +

$ARGV[0] man page

+ +

+This page is part of the PCRE HTML documentation. It was generated automatically +from the original man page. If there is any nonsense in it, please consult the +man page, in case the conversion went wrong. +
+End + +print "

    \n" if ($toc); + +open(TEMP, ">/tmp/$$") || die "Can't open /tmp/$$ for output\n"; + +while () + { + # Handle lines beginning with a dot + + if (/^\./) + { + # Some of the PCRE man pages used to contain instances of .br. However, + # they should have all been removed because they cause trouble in some + # (other) automated systems that translate man pages to HTML. Complain if + # we find .br or .in (another macro that is deprecated). + + if (/^\.br/ || /^\.in/) + { + print STDERR "\n*** Deprecated macro encountered - rewrite needed\n"; + print STDERR "*** $_\n"; + die "*** Processing abandoned\n"; + } + + # Instead of .br, relevent "literal" sections are enclosed in .nf/.fi. + + elsif (/^\.nf/) + { + $innf = 1; + } + + elsif (/^\.fi/) + { + $innf = 0; + } + + # Handling .sp is subtle. If it is inside a literal section, do nothing if + # the next line is a non literal text line; similarly, if not inside a + # literal section, do nothing if a literal follows, unless we are inside + # a .nf/.ne section. The point being that the 
     and
    that delimit + # literal sections will do the spacing. Always skip if no previous output. + + elsif (/^\.sp/) + { + if ($wrotetext) + { + $_ = ; + if ($inpre) + { + print TEMP "\n" if (/^[\s.]/); + } + else + { + print TEMP "
    \n
    \n" if ($innf || !/^[\s.]/); + } + redo; # Now process the lookahead line we just read + } + } + elsif (/^\.TP/ || /^\.PP/ || /^\.P/) + { + &new_para(); + } + elsif (/^\.SH\s*("?)(.*)\1/) + { + # Ignore the NAME section + if ($2 =~ /^NAME\b/) + { + ; + next; + } + + &end_para(); + my($title) = &do_line($2); + if ($toc) + { + printf("
  • $title\n", + $ref, $ref); + printf TEMP ("
    $title
    \n", + $ref); + $ref++; + } + else + { + print TEMP "
    \n$title\n
    \n"; + } + } + elsif (/^\.SS\s*("?)(.*)\1/) + { + &end_para(); + my($title) = &do_line($2); + print TEMP "
    \n$title\n
    \n"; + } + elsif (/^\.B\s*(.*)/) + { + &new_para() if (!$inpara); + $_ = &do_line($1); + s/"(.*?)"/$1/g; + print TEMP "$_\n"; + $wrotetext = 1; + } + elsif (/^\.I\s*(.*)/) + { + &new_para() if (!$inpara); + $_ = &do_line($1); + s/"(.*?)"/$1/g; + print TEMP "$_\n"; + $wrotetext = 1; + } + + # A comment that starts "HREF" takes the next line as a name that + # is turned into a hyperlink, using the text given, which might be + # in a special font. If it ends in () or (digits) or punctuation, they + # aren't part of the link. + + elsif (/^\.\\"\s*HREF/) + { + $_=; + chomp; + $_ = &do_line($_); + $_ =~ s/\s+$//; + $_ =~ /^(?:<.>)?([^<(]+)(?:\(\))?(?:<\/.>)?(?:\(\d+\))?[.,;:]?$/; + print TEMP "$_\n"; + } + + # A comment that starts "HTML" inserts literal HTML + + elsif (/^\.\\"\s*HTML\s*(.*)/) + { + print TEMP $1; + } + + # A comment that starts < inserts that HTML at the end of the + # *next* input line - so as not to get a newline between them. + + elsif (/^\.\\"\s*(<.*>)/) + { + my($markup) = $1; + $_=; + chomp; + $_ = &do_line($_); + $_ =~ s/\s+$//; + print TEMP "$_$markup\n"; + } + + # A comment that starts JOIN joins the next two lines together, with one + # space between them. Then that line is processed. This is used in some + # displays where two lines are needed for the "man" version. JOINSH works + # the same, except that it assumes this is a shell command, so removes + # continuation backslashes. + + elsif (/^\.\\"\s*JOIN(SH)?/) + { + my($one,$two); + $one = ; + $two = ; + $one =~ s/\s*\\e\s*$// if (defined($1)); + chomp($one); + $two =~ s/^\s+//; + $_ = "$one $two"; + redo; # Process the joined lines + } + + # .EX/.EE are used in the pcredemo page to bracket the entire program, + # which is unmodified except for turning backslash into "\e". + + elsif (/^\.EX\s*$/) + { + print TEMP "
    \n";
+      while ()
+        {
+        last if /^\.EE\s*$/;
+        s/\\e/\\/g;
+        s/&/&/g;
+        s//>/g;
+        print TEMP;
+        }
+      }
+
+    # Ignore anything not recognized
+
+    next;
+    }
+
+  # Line does not begin with a dot. Replace blank lines with new paragraphs
+
+  if (/^\s*$/)
+    {
+    &end_para() if ($wrotetext);
+    next;
+    }
+
+  # Convert fonts changes and output an ordinary line. Ensure that indented
+  # lines are marked as literal.
+
+  $_ = &do_line($_);
+  &new_para() if (!$inpara);
+
+  if (/^\s/)
+    {
+    if (!$inpre)
+      {
+      print TEMP "
    \n";
+      $inpre = 1;
+      }
+    }
+  elsif ($inpre)
+    {
+    print TEMP "
    \n";
+    $inpre = 0;
+    }
+
+  # Add 
     to the end of a non-literal line if we are within .nf/.fi
+
+  $_ .= "
    \n" if (!$inpre && $innf);
+
+  print TEMP;
+  $wrotetext = 1;
+  }
+
+# The TOC, if present, will have been written - terminate it
+
+print "
\n" if ($toc); + +# Copy the remainder to the standard output + +close(TEMP); +open(TEMP, "/tmp/$$") || die "Can't open /tmp/$$ for input\n"; + +print while (); + +print < +Return to the PCRE index page. +

+End + +close(TEMP); +unlink("/tmp/$$"); + +# End diff --git a/src/third_party/pcre-8.41/AUTHORS b/src/third_party/pcre-8.41/AUTHORS new file mode 100644 index 00000000000..291657caef1 --- /dev/null +++ b/src/third_party/pcre-8.41/AUTHORS @@ -0,0 +1,45 @@ +THE MAIN PCRE LIBRARY +--------------------- + +Written by: Philip Hazel +Email local part: ph10 +Email domain: cam.ac.uk + +University of Cambridge Computing Service, +Cambridge, England. + +Copyright (c) 1997-2017 University of Cambridge +All rights reserved + + +PCRE JUST-IN-TIME COMPILATION SUPPORT +------------------------------------- + +Written by: Zoltan Herczeg +Email local part: hzmester +Emain domain: freemail.hu + +Copyright(c) 2010-2017 Zoltan Herczeg +All rights reserved. + + +STACK-LESS JUST-IN-TIME COMPILER +-------------------------------- + +Written by: Zoltan Herczeg +Email local part: hzmester +Emain domain: freemail.hu + +Copyright(c) 2009-2017 Zoltan Herczeg +All rights reserved. + + +THE C++ WRAPPER LIBRARY +----------------------- + +Written by: Google Inc. + +Copyright (c) 2007-2012 Google Inc +All rights reserved + +#### diff --git a/src/third_party/pcre-8.41/COPYING b/src/third_party/pcre-8.41/COPYING new file mode 100644 index 00000000000..58eed01b61d --- /dev/null +++ b/src/third_party/pcre-8.41/COPYING @@ -0,0 +1,5 @@ +PCRE LICENCE + +Please see the file LICENCE in the PCRE distribution for licensing details. + +End diff --git a/src/third_party/pcre-8.41/ChangeLog b/src/third_party/pcre-8.41/ChangeLog new file mode 100644 index 00000000000..590a7542885 --- /dev/null +++ b/src/third_party/pcre-8.41/ChangeLog @@ -0,0 +1,6104 @@ +ChangeLog for PCRE +------------------ + +Note that the PCRE 8.xx series (PCRE1) is now in a bugfix-only state. All +development is happening in the PCRE2 10.xx series. + +Version 8.41 05-July-2017 +------------------------- + +1. Fixed typo in CMakeLists.txt (wrong number of arguments for +PCRE_STATIC_RUNTIME (affects MSVC only). + +2. Issue 1 for 8.40 below was not correctly fixed. If pcregrep in multiline +mode with --only-matching matched several lines, it restarted scanning at the +next line instead of moving on to the end of the matched string, which can be +several lines after the start. + +3. Fix a missing else in the JIT compiler reported by 'idaifish'. + +4. A (?# style comment is now ignored between a basic quantifier and a +following '+' or '?' (example: /X+(?#comment)?Y/. + +5. Avoid use of a potentially overflowing buffer in pcregrep (patch by Petr +Pisar). + +6. Fuzzers have reported issues in pcretest. These are NOT serious (it is, +after all, just a test program). However, to stop the reports, some easy ones +are fixed: + + (a) Check for values < 256 when calling isprint() in pcretest. + (b) Give an error for too big a number after \O. + +7. In the 32-bit library in non-UTF mode, an attempt to find a Unicode +property for a character with a code point greater than 0x10ffff (the Unicode +maximum) caused a crash. + +8. The alternative matching function, pcre_dfa_exec() misbehaved if it +encountered a character class with a possessive repeat, for example [a-f]{3}+. + +9. When pcretest called pcre_copy_substring() in 32-bit mode, it set the buffer +length incorrectly, which could result in buffer overflow. + +10. Remove redundant line of code (accidentally left in ages ago). + +11. Applied C++ patch from Irfan Adilovic to guard 'using std::' directives +with namespace pcrecpp (Bugzilla #2084). + +12. Remove a duplication typo in pcre_tables.c. + +13. Fix returned offsets from regexec() when REG_STARTEND is used with a +starting offset greater than zero. + + +Version 8.40 11-January-2017 +---------------------------- + +1. Using -o with -M in pcregrep could cause unnecessary repeated output when + the match extended over a line boundary. + +2. Applied Chris Wilson's second patch (Bugzilla #1681) to CMakeLists.txt for + MSVC static compilation, putting the first patch under a new option. + +3. Fix register overwite in JIT when SSE2 acceleration is enabled. + +4. Ignore "show all captures" (/=) for DFA matching. + +5. Fix JIT unaligned accesses on x86. Patch by Marc Mutz. + +6. In any wide-character mode (8-bit UTF or any 16-bit or 32-bit mode), + without PCRE_UCP set, a negative character type such as \D in a positive + class should cause all characters greater than 255 to match, whatever else + is in the class. There was a bug that caused this not to happen if a + Unicode property item was added to such a class, for example [\D\P{Nd}] or + [\W\pL]. + +7. When pcretest was outputing information from a callout, the caret indicator + for the current position in the subject line was incorrect if it was after + an escape sequence for a character whose code point was greater than + \x{ff}. + +8. A pattern such as (?abc)(?(R)xyz) was incorrectly compiled such that + the conditional was interpreted as a reference to capturing group 1 instead + of a test for recursion. Any group whose name began with R was + misinterpreted in this way. (The reference interpretation should only + happen if the group's name is precisely "R".) + +9. A number of bugs have been mended relating to match start-up optimizations + when the first thing in a pattern is a positive lookahead. These all + applied only when PCRE_NO_START_OPTIMIZE was *not* set: + + (a) A pattern such as (?=.*X)X$ was incorrectly optimized as if it needed + both an initial 'X' and a following 'X'. + (b) Some patterns starting with an assertion that started with .* were + incorrectly optimized as having to match at the start of the subject or + after a newline. There are cases where this is not true, for example, + (?=.*[A-Z])(?=.{8,16})(?!.*[\s]) matches after the start in lines that + start with spaces. Starting .* in an assertion is no longer taken as an + indication of matching at the start (or after a newline). + + +Version 8.39 14-June-2016 +------------------------- + +1. If PCRE_AUTO_CALLOUT was set on a pattern that had a (?# comment between + an item and its qualifier (for example, A(?#comment)?B) pcre_compile() + misbehaved. This bug was found by the LLVM fuzzer. + +2. Similar to the above, if an isolated \E was present between an item and its + qualifier when PCRE_AUTO_CALLOUT was set, pcre_compile() misbehaved. This + bug was found by the LLVM fuzzer. + +3. Further to 8.38/46, negated classes such as [^[:^ascii:]\d] were also not + working correctly in UCP mode. + +4. The POSIX wrapper function regexec() crashed if the option REG_STARTEND + was set when the pmatch argument was NULL. It now returns REG_INVARG. + +5. Allow for up to 32-bit numbers in the ordin() function in pcregrep. + +6. An empty \Q\E sequence between an item and its qualifier caused + pcre_compile() to misbehave when auto callouts were enabled. This bug was + found by the LLVM fuzzer. + +7. If a pattern that was compiled with PCRE_EXTENDED started with white + space or a #-type comment that was followed by (?-x), which turns off + PCRE_EXTENDED, and there was no subsequent (?x) to turn it on again, + pcre_compile() assumed that (?-x) applied to the whole pattern and + consequently mis-compiled it. This bug was found by the LLVM fuzzer. + +8. A call of pcre_copy_named_substring() for a named substring whose number + was greater than the space in the ovector could cause a crash. + +9. Yet another buffer overflow bug involved duplicate named groups with a + group that reset capture numbers (compare 8.38/7 below). Once again, I have + just allowed for more memory, even if not needed. (A proper fix is + implemented in PCRE2, but it involves a lot of refactoring.) + +10. pcre_get_substring_list() crashed if the use of \K in a match caused the + start of the match to be earlier than the end. + +11. Migrating appropriate PCRE2 JIT improvements to PCRE. + +12. A pattern such as /(?<=((?C)0))/, which has a callout inside a lookbehind + assertion, caused pcretest to generate incorrect output, and also to read + uninitialized memory (detected by ASAN or valgrind). + +13. A pattern that included (*ACCEPT) in the middle of a sufficiently deeply + nested set of parentheses of sufficient size caused an overflow of the + compiling workspace (which was diagnosed, but of course is not desirable). + +14. And yet another buffer overflow bug involving duplicate named groups, this + time nested, with a nested back reference. Yet again, I have just allowed + for more memory, because anything more needs all the refactoring that has + been done for PCRE2. An example pattern that provoked this bug is: + /((?J)(?'R'(?'R'(?'R'(?'R'(?'R'(?|(\k'R'))))))))/ and the bug was + registered as CVE-2016-1283. + +15. pcretest went into a loop if global matching was requested with an ovector + size less than 2. It now gives an error message. This bug was found by + afl-fuzz. + +16. An invalid pattern fragment such as (?(?C)0 was not diagnosing an error + ("assertion expected") when (?(?C) was not followed by an opening + parenthesis. + +17. Fixed typo ("&&" for "&") in pcre_study(). Fortunately, this could not + actually affect anything, by sheer luck. + +18. Applied Chris Wilson's patch (Bugzilla #1681) to CMakeLists.txt for MSVC + static compilation. + +19. Modified the RunTest script to incorporate a valgrind suppressions file so + that certain errors, provoked by the SSE2 instruction set when JIT is used, + are ignored. + +20. A racing condition is fixed in JIT reported by Mozilla. + +21. Minor code refactor to avoid "array subscript is below array bounds" + compiler warning. + +22. Minor code refactor to avoid "left shift of negative number" warning. + +23. Fix typo causing compile error when 16- or 32-bit JIT is compiled without + UCP support. + +24. Refactor to avoid compiler warnings in pcrecpp.cc. + +25. Refactor to fix a typo in pcre_jit_test.c + +26. Patch to support compiling pcrecpp.cc with Intel compiler. + + +Version 8.38 23-November-2015 +----------------------------- + +1. If a group that contained a recursive back reference also contained a + forward reference subroutine call followed by a non-forward-reference + subroutine call, for example /.((?2)(?R)\1)()/, pcre_compile() failed to + compile correct code, leading to undefined behaviour or an internally + detected error. This bug was discovered by the LLVM fuzzer. + +2. Quantification of certain items (e.g. atomic back references) could cause + incorrect code to be compiled when recursive forward references were + involved. For example, in this pattern: /(?1)()((((((\1++))\x85)+)|))/. + This bug was discovered by the LLVM fuzzer. + +3. A repeated conditional group whose condition was a reference by name caused + a buffer overflow if there was more than one group with the given name. + This bug was discovered by the LLVM fuzzer. + +4. A recursive back reference by name within a group that had the same name as + another group caused a buffer overflow. For example: + /(?J)(?'d'(?'d'\g{d}))/. This bug was discovered by the LLVM fuzzer. + +5. A forward reference by name to a group whose number is the same as the + current group, for example in this pattern: /(?|(\k'Pm')|(?'Pm'))/, caused + a buffer overflow at compile time. This bug was discovered by the LLVM + fuzzer. + +6. A lookbehind assertion within a set of mutually recursive subpatterns could + provoke a buffer overflow. This bug was discovered by the LLVM fuzzer. + +7. Another buffer overflow bug involved duplicate named groups with a + reference between their definition, with a group that reset capture + numbers, for example: /(?J:(?|(?'R')(\k'R')|((?'R'))))/. This has been + fixed by always allowing for more memory, even if not needed. (A proper fix + is implemented in PCRE2, but it involves more refactoring.) + +8. There was no check for integer overflow in subroutine calls such as (?123). + +9. The table entry for \l in EBCDIC environments was incorrect, leading to its + being treated as a literal 'l' instead of causing an error. + +10. There was a buffer overflow if pcre_exec() was called with an ovector of + size 1. This bug was found by american fuzzy lop. + +11. If a non-capturing group containing a conditional group that could match + an empty string was repeated, it was not identified as matching an empty + string itself. For example: /^(?:(?(1)x|)+)+$()/. + +12. In an EBCDIC environment, pcretest was mishandling the escape sequences + \a and \e in test subject lines. + +13. In an EBCDIC environment, \a in a pattern was converted to the ASCII + instead of the EBCDIC value. + +14. The handling of \c in an EBCDIC environment has been revised so that it is + now compatible with the specification in Perl's perlebcdic page. + +15. The EBCDIC character 0x41 is a non-breaking space, equivalent to 0xa0 in + ASCII/Unicode. This has now been added to the list of characters that are + recognized as white space in EBCDIC. + +16. When PCRE was compiled without UCP support, the use of \p and \P gave an + error (correctly) when used outside a class, but did not give an error + within a class. + +17. \h within a class was incorrectly compiled in EBCDIC environments. + +18. A pattern with an unmatched closing parenthesis that contained a backward + assertion which itself contained a forward reference caused buffer + overflow. And example pattern is: /(?=di(?<=(?1))|(?=(.))))/. + +19. JIT should return with error when the compiled pattern requires more stack + space than the maximum. + +20. A possessively repeated conditional group that could match an empty string, + for example, /(?(R))*+/, was incorrectly compiled. + +21. Fix infinite recursion in the JIT compiler when certain patterns such as + /(?:|a|){100}x/ are analysed. + +22. Some patterns with character classes involving [: and \\ were incorrectly + compiled and could cause reading from uninitialized memory or an incorrect + error diagnosis. + +23. Pathological patterns containing many nested occurrences of [: caused + pcre_compile() to run for a very long time. + +24. A conditional group with only one branch has an implicit empty alternative + branch and must therefore be treated as potentially matching an empty + string. + +25. If (?R was followed by - or + incorrect behaviour happened instead of a + diagnostic. + +26. Arrange to give up on finding the minimum matching length for overly + complex patterns. + +27. Similar to (4) above: in a pattern with duplicated named groups and an + occurrence of (?| it is possible for an apparently non-recursive back + reference to become recursive if a later named group with the relevant + number is encountered. This could lead to a buffer overflow. Wen Guanxing + from Venustech ADLAB discovered this bug. + +28. If pcregrep was given the -q option with -c or -l, or when handling a + binary file, it incorrectly wrote output to stdout. + +29. The JIT compiler did not restore the control verb head in case of *THEN + control verbs. This issue was found by Karl Skomski with a custom LLVM + fuzzer. + +30. Error messages for syntax errors following \g and \k were giving inaccurate + offsets in the pattern. + +31. Added a check for integer overflow in conditions (?() and + (?(R). This omission was discovered by Karl Skomski with the LLVM + fuzzer. + +32. Handling recursive references such as (?2) when the reference is to a group + later in the pattern uses code that is very hacked about and error-prone. + It has been re-written for PCRE2. Here in PCRE1, a check has been added to + give an internal error if it is obvious that compiling has gone wrong. + +33. The JIT compiler should not check repeats after a {0,1} repeat byte code. + This issue was found by Karl Skomski with a custom LLVM fuzzer. + +34. The JIT compiler should restore the control chain for empty possessive + repeats. This issue was found by Karl Skomski with a custom LLVM fuzzer. + +35. Match limit check added to JIT recursion. This issue was found by Karl + Skomski with a custom LLVM fuzzer. + +36. Yet another case similar to 27 above has been circumvented by an + unconditional allocation of extra memory. This issue is fixed "properly" in + PCRE2 by refactoring the way references are handled. Wen Guanxing + from Venustech ADLAB discovered this bug. + +37. Fix two assertion fails in JIT. These issues were found by Karl Skomski + with a custom LLVM fuzzer. + +38. Fixed a corner case of range optimization in JIT. + +39. An incorrect error "overran compiling workspace" was given if there were + exactly enough group forward references such that the last one extended + into the workspace safety margin. The next one would have expanded the + workspace. The test for overflow was not including the safety margin. + +40. A match limit issue is fixed in JIT which was found by Karl Skomski + with a custom LLVM fuzzer. + +41. Remove the use of /dev/null in testdata/testinput2, because it doesn't + work under Windows. (Why has it taken so long for anyone to notice?) + +42. In a character class such as [\W\p{Any}] where both a negative-type escape + ("not a word character") and a property escape were present, the property + escape was being ignored. + +43. Fix crash caused by very long (*MARK) or (*THEN) names. + +44. A sequence such as [[:punct:]b] that is, a POSIX character class followed + by a single ASCII character in a class item, was incorrectly compiled in + UCP mode. The POSIX class got lost, but only if the single character + followed it. + +45. [:punct:] in UCP mode was matching some characters in the range 128-255 + that should not have been matched. + +46. If [:^ascii:] or [:^xdigit:] or [:^cntrl:] are present in a non-negated + class, all characters with code points greater than 255 are in the class. + When a Unicode property was also in the class (if PCRE_UCP is set, escapes + such as \w are turned into Unicode properties), wide characters were not + correctly handled, and could fail to match. + + +Version 8.37 28-April-2015 +-------------------------- + +1. When an (*ACCEPT) is triggered inside capturing parentheses, it arranges + for those parentheses to be closed with whatever has been captured so far. + However, it was failing to mark any other groups between the hightest + capture so far and the currrent group as "unset". Thus, the ovector for + those groups contained whatever was previously there. An example is the + pattern /(x)|((*ACCEPT))/ when matched against "abcd". + +2. If an assertion condition was quantified with a minimum of zero (an odd + thing to do, but it happened), SIGSEGV or other misbehaviour could occur. + +3. If a pattern in pcretest input had the P (POSIX) modifier followed by an + unrecognized modifier, a crash could occur. + +4. An attempt to do global matching in pcretest with a zero-length ovector + caused a crash. + +5. Fixed a memory leak during matching that could occur for a subpattern + subroutine call (recursive or otherwise) if the number of captured groups + that had to be saved was greater than ten. + +6. Catch a bad opcode during auto-possessification after compiling a bad UTF + string with NO_UTF_CHECK. This is a tidyup, not a bug fix, as passing bad + UTF with NO_UTF_CHECK is documented as having an undefined outcome. + +7. A UTF pattern containing a "not" match of a non-ASCII character and a + subroutine reference could loop at compile time. Example: /[^\xff]((?1))/. + +8. When a pattern is compiled, it remembers the highest back reference so that + when matching, if the ovector is too small, extra memory can be obtained to + use instead. A conditional subpattern whose condition is a check on a + capture having happened, such as, for example in the pattern + /^(?:(a)|b)(?(1)A|B)/, is another kind of back reference, but it was not + setting the highest backreference number. This mattered only if pcre_exec() + was called with an ovector that was too small to hold the capture, and there + was no other kind of back reference (a situation which is probably quite + rare). The effect of the bug was that the condition was always treated as + FALSE when the capture could not be consulted, leading to a incorrect + behaviour by pcre_exec(). This bug has been fixed. + +9. A reference to a duplicated named group (either a back reference or a test + for being set in a conditional) that occurred in a part of the pattern where + PCRE_DUPNAMES was not set caused the amount of memory needed for the pattern + to be incorrectly calculated, leading to overwriting. + +10. A mutually recursive set of back references such as (\2)(\1) caused a + segfault at study time (while trying to find the minimum matching length). + The infinite loop is now broken (with the minimum length unset, that is, + zero). + +11. If an assertion that was used as a condition was quantified with a minimum + of zero, matching went wrong. In particular, if the whole group had + unlimited repetition and could match an empty string, a segfault was + likely. The pattern (?(?=0)?)+ is an example that caused this. Perl allows + assertions to be quantified, but not if they are being used as conditions, + so the above pattern is faulted by Perl. PCRE has now been changed so that + it also rejects such patterns. + +12. A possessive capturing group such as (a)*+ with a minimum repeat of zero + failed to allow the zero-repeat case if pcre2_exec() was called with an + ovector too small to capture the group. + +13. Fixed two bugs in pcretest that were discovered by fuzzing and reported by + Red Hat Product Security: + + (a) A crash if /K and /F were both set with the option to save the compiled + pattern. + + (b) Another crash if the option to print captured substrings in a callout + was combined with setting a null ovector, for example \O\C+ as a subject + string. + +14. A pattern such as "((?2){0,1999}())?", which has a group containing a + forward reference repeated a large (but limited) number of times within a + repeated outer group that has a zero minimum quantifier, caused incorrect + code to be compiled, leading to the error "internal error: + previously-checked referenced subpattern not found" when an incorrect + memory address was read. This bug was reported as "heap overflow", + discovered by Kai Lu of Fortinet's FortiGuard Labs and given the CVE number + CVE-2015-2325. + +23. A pattern such as "((?+1)(\1))/" containing a forward reference subroutine + call within a group that also contained a recursive back reference caused + incorrect code to be compiled. This bug was reported as "heap overflow", + discovered by Kai Lu of Fortinet's FortiGuard Labs, and given the CVE + number CVE-2015-2326. + +24. Computing the size of the JIT read-only data in advance has been a source + of various issues, and new ones are still appear unfortunately. To fix + existing and future issues, size computation is eliminated from the code, + and replaced by on-demand memory allocation. + +25. A pattern such as /(?i)[A-`]/, where characters in the other case are + adjacent to the end of the range, and the range contained characters with + more than one other case, caused incorrect behaviour when compiled in UTF + mode. In that example, the range a-j was left out of the class. + +26. Fix JIT compilation of conditional blocks, which assertion + is converted to (*FAIL). E.g: /(?(?!))/. + +27. The pattern /(?(?!)^)/ caused references to random memory. This bug was + discovered by the LLVM fuzzer. + +28. The assertion (?!) is optimized to (*FAIL). This was not handled correctly + when this assertion was used as a condition, for example (?(?!)a|b). In + pcre2_match() it worked by luck; in pcre2_dfa_match() it gave an incorrect + error about an unsupported item. + +29. For some types of pattern, for example /Z*(|d*){216}/, the auto- + possessification code could take exponential time to complete. A recursion + depth limit of 1000 has been imposed to limit the resources used by this + optimization. + +30. A pattern such as /(*UTF)[\S\V\H]/, which contains a negated special class + such as \S in non-UCP mode, explicit wide characters (> 255) can be ignored + because \S ensures they are all in the class. The code for doing this was + interacting badly with the code for computing the amount of space needed to + compile the pattern, leading to a buffer overflow. This bug was discovered + by the LLVM fuzzer. + +31. A pattern such as /((?2)+)((?1))/ which has mutual recursion nested inside + other kinds of group caused stack overflow at compile time. This bug was + discovered by the LLVM fuzzer. + +32. A pattern such as /(?1)(?#?'){8}(a)/ which had a parenthesized comment + between a subroutine call and its quantifier was incorrectly compiled, + leading to buffer overflow or other errors. This bug was discovered by the + LLVM fuzzer. + +33. The illegal pattern /(?(?.*!.*)?)/ was not being diagnosed as missing an + assertion after (?(. The code was failing to check the character after + (?(?< for the ! or = that would indicate a lookbehind assertion. This bug + was discovered by the LLVM fuzzer. + +34. A pattern such as /X((?2)()*+){2}+/ which has a possessive quantifier with + a fixed maximum following a group that contains a subroutine reference was + incorrectly compiled and could trigger buffer overflow. This bug was + discovered by the LLVM fuzzer. + +35. A mutual recursion within a lookbehind assertion such as (?<=((?2))((?1))) + caused a stack overflow instead of the diagnosis of a non-fixed length + lookbehind assertion. This bug was discovered by the LLVM fuzzer. + +36. The use of \K in a positive lookbehind assertion in a non-anchored pattern + (e.g. /(?<=\Ka)/) could make pcregrep loop. + +37. There was a similar problem to 36 in pcretest for global matches. + +38. If a greedy quantified \X was preceded by \C in UTF mode (e.g. \C\X*), + and a subsequent item in the pattern caused a non-match, backtracking over + the repeated \X did not stop, but carried on past the start of the subject, + causing reference to random memory and/or a segfault. There were also some + other cases where backtracking after \C could crash. This set of bugs was + discovered by the LLVM fuzzer. + +39. The function for finding the minimum length of a matching string could take + a very long time if mutual recursion was present many times in a pattern, + for example, /((?2){73}(?2))((?1))/. A better mutual recursion detection + method has been implemented. This infelicity was discovered by the LLVM + fuzzer. + +40. Static linking against the PCRE library using the pkg-config module was + failing on missing pthread symbols. + + +Version 8.36 26-September-2014 +------------------------------ + +1. Got rid of some compiler warnings in the C++ modules that were shown up by + -Wmissing-field-initializers and -Wunused-parameter. + +2. The tests for quantifiers being too big (greater than 65535) were being + applied after reading the number, and stupidly assuming that integer + overflow would give a negative number. The tests are now applied as the + numbers are read. + +3. Tidy code in pcre_exec.c where two branches that used to be different are + now the same. + +4. The JIT compiler did not generate match limit checks for certain + bracketed expressions with quantifiers. This may lead to exponential + backtracking, instead of returning with PCRE_ERROR_MATCHLIMIT. This + issue should be resolved now. + +5. Fixed an issue, which occures when nested alternatives are optimized + with table jumps. + +6. Inserted two casts and changed some ints to size_t in the light of some + reported 64-bit compiler warnings (Bugzilla 1477). + +7. Fixed a bug concerned with zero-minimum possessive groups that could match + an empty string, which sometimes were behaving incorrectly in the + interpreter (though correctly in the JIT matcher). This pcretest input is + an example: + + '\A(?:[^"]++|"(?:[^"]*+|"")*+")++' + NON QUOTED "QUOT""ED" AFTER "NOT MATCHED + + the interpreter was reporting a match of 'NON QUOTED ' only, whereas the + JIT matcher and Perl both matched 'NON QUOTED "QUOT""ED" AFTER '. The test + for an empty string was breaking the inner loop and carrying on at a lower + level, when possessive repeated groups should always return to a higher + level as they have no backtrack points in them. The empty string test now + occurs at the outer level. + +8. Fixed a bug that was incorrectly auto-possessifying \w+ in the pattern + ^\w+(?>\s*)(?<=\w) which caused it not to match "test test". + +9. Give a compile-time error for \o{} (as Perl does) and for \x{} (which Perl + doesn't). + +10. Change 8.34/15 introduced a bug that caused the amount of memory needed + to hold a pattern to be incorrectly computed (too small) when there were + named back references to duplicated names. This could cause "internal + error: code overflow" or "double free or corruption" or other memory + handling errors. + +11. When named subpatterns had the same prefixes, back references could be + confused. For example, in this pattern: + + /(?Pa)?(?Pb)?(?()c|d)*l/ + + the reference to 'Name' was incorrectly treated as a reference to a + duplicate name. + +12. A pattern such as /^s?c/mi8 where the optional character has more than + one "other case" was incorrectly compiled such that it would only try to + match starting at "c". + +13. When a pattern starting with \s was studied, VT was not included in the + list of possible starting characters; this should have been part of the + 8.34/18 patch. + +14. If a character class started [\Qx]... where x is any character, the class + was incorrectly terminated at the ]. + +15. If a pattern that started with a caseless match for a character with more + than one "other case" was studied, PCRE did not set up the starting code + unit bit map for the list of possible characters. Now it does. This is an + optimization improvement, not a bug fix. + +16. The Unicode data tables have been updated to Unicode 7.0.0. + +17. Fixed a number of memory leaks in pcregrep. + +18. Avoid a compiler warning (from some compilers) for a function call with + a cast that removes "const" from an lvalue by using an intermediate + variable (to which the compiler does not object). + +19. Incorrect code was compiled if a group that contained an internal recursive + back reference was optional (had quantifier with a minimum of zero). This + example compiled incorrect code: /(((a\2)|(a*)\g<-1>))*/ and other examples + caused segmentation faults because of stack overflows at compile time. + +20. A pattern such as /((?(R)a|(?1)))+/, which contains a recursion within a + group that is quantified with an indefinite repeat, caused a compile-time + loop which used up all the system stack and provoked a segmentation fault. + This was not the same bug as 19 above. + +21. Add PCRECPP_EXP_DECL declaration to operator<< in pcre_stringpiece.h. + Patch by Mike Frysinger. + + +Version 8.35 04-April-2014 +-------------------------- + +1. A new flag is set, when property checks are present in an XCLASS. + When this flag is not set, PCRE can perform certain optimizations + such as studying these XCLASS-es. + +2. The auto-possessification of character sets were improved: a normal + and an extended character set can be compared now. Furthermore + the JIT compiler optimizes more character set checks. + +3. Got rid of some compiler warnings for potentially uninitialized variables + that show up only when compiled with -O2. + +4. A pattern such as (?=ab\K) that uses \K in an assertion can set the start + of a match later then the end of the match. The pcretest program was not + handling the case sensibly - it was outputting from the start to the next + binary zero. It now reports this situation in a message, and outputs the + text from the end to the start. + +5. Fast forward search is improved in JIT. Instead of the first three + characters, any three characters with fixed position can be searched. + Search order: first, last, middle. + +6. Improve character range checks in JIT. Characters are read by an inprecise + function now, which returns with an unknown value if the character code is + above a certain threshold (e.g: 256). The only limitation is that the value + must be bigger than the threshold as well. This function is useful when + the characters above the threshold are handled in the same way. + +7. The macros whose names start with RAWUCHAR are placeholders for a future + mode in which only the bottom 21 bits of 32-bit data items are used. To + make this more memorable for those maintaining the code, the names have + been changed to start with UCHAR21, and an extensive comment has been added + to their definition. + +8. Add missing (new) files sljitNativeTILEGX.c and sljitNativeTILEGX-encoder.c + to the export list in Makefile.am (they were accidentally omitted from the + 8.34 tarball). + +9. The informational output from pcretest used the phrase "starting byte set" + which is inappropriate for the 16-bit and 32-bit libraries. As the output + for "first char" and "need char" really means "non-UTF-char", I've changed + "byte" to "char", and slightly reworded the output. The documentation about + these values has also been (I hope) clarified. + +10. Another JIT related optimization: use table jumps for selecting the correct + backtracking path, when more than four alternatives are present inside a + bracket. + +11. Empty match is not possible, when the minimum length is greater than zero, + and there is no \K in the pattern. JIT should avoid empty match checks in + such cases. + +12. In a caseless character class with UCP support, when a character with more + than one alternative case was not the first character of a range, not all + the alternative cases were added to the class. For example, s and \x{17f} + are both alternative cases for S: the class [RST] was handled correctly, + but [R-T] was not. + +13. The configure.ac file always checked for pthread support when JIT was + enabled. This is not used in Windows, so I have put this test inside a + check for the presence of windows.h (which was already tested for). + +14. Improve pattern prefix search by a simplified Boyer-Moore algorithm in JIT. + The algorithm provides a way to skip certain starting offsets, and usually + faster than linear prefix searches. + +15. Change 13 for 8.20 updated RunTest to check for the 'fr' locale as well + as for 'fr_FR' and 'french'. For some reason, however, it then used the + Windows-specific input and output files, which have 'french' screwed in. + So this could never have worked. One of the problems with locales is that + they aren't always the same. I have now updated RunTest so that it checks + the output of the locale test (test 3) against three different output + files, and it allows the test to pass if any one of them matches. With luck + this should make the test pass on some versions of Solaris where it was + failing. Because of the uncertainty, the script did not used to stop if + test 3 failed; it now does. If further versions of a French locale ever + come to light, they can now easily be added. + +16. If --with-pcregrep-bufsize was given a non-integer value such as "50K", + there was a message during ./configure, but it did not stop. This now + provokes an error. The invalid example in README has been corrected. + If a value less than the minimum is given, the minimum value has always + been used, but now a warning is given. + +17. If --enable-bsr-anycrlf was set, the special 16/32-bit test failed. This + was a bug in the test system, which is now fixed. Also, the list of various + configurations that are tested for each release did not have one with both + 16/32 bits and --enable-bar-anycrlf. It now does. + +18. pcretest was missing "-C bsr" for displaying the \R default setting. + +19. Little endian PowerPC systems are supported now by the JIT compiler. + +20. The fast forward newline mechanism could enter to an infinite loop on + certain invalid UTF-8 input. Although we don't support these cases + this issue can be fixed by a performance optimization. + +21. Change 33 of 8.34 is not sufficient to ensure stack safety because it does + not take account if existing stack usage. There is now a new global + variable called pcre_stack_guard that can be set to point to an external + function to check stack availability. It is called at the start of + processing every parenthesized group. + +22. A typo in the code meant that in ungreedy mode the max/min qualifier + behaved like a min-possessive qualifier, and, for example, /a{1,3}b/U did + not match "ab". + +23. When UTF was disabled, the JIT program reported some incorrect compile + errors. These messages are silenced now. + +24. Experimental support for ARM-64 and MIPS-64 has been added to the JIT + compiler. + +25. Change all the temporary files used in RunGrepTest to be different to those + used by RunTest so that the tests can be run simultaneously, for example by + "make -j check". + + +Version 8.34 15-December-2013 +----------------------------- + +1. Add pcre[16|32]_jit_free_unused_memory to forcibly free unused JIT + executable memory. Patch inspired by Carsten Klein. + +2. ./configure --enable-coverage defined SUPPORT_GCOV in config.h, although + this macro is never tested and has no effect, because the work to support + coverage involves only compiling and linking options and special targets in + the Makefile. The comment in config.h implied that defining the macro would + enable coverage support, which is totally false. There was also support for + setting this macro in the CMake files (my fault, I just copied it from + configure). SUPPORT_GCOV has now been removed. + +3. Make a small performance improvement in strlen16() and strlen32() in + pcretest. + +4. Change 36 for 8.33 left some unreachable statements in pcre_exec.c, + detected by the Solaris compiler (gcc doesn't seem to be able to diagnose + these cases). There was also one in pcretest.c. + +5. Cleaned up a "may be uninitialized" compiler warning in pcre_exec.c. + +6. In UTF mode, the code for checking whether a group could match an empty + string (which is used for indefinitely repeated groups to allow for + breaking an infinite loop) was broken when the group contained a repeated + negated single-character class with a character that occupied more than one + data item and had a minimum repetition of zero (for example, [^\x{100}]* in + UTF-8 mode). The effect was undefined: the group might or might not be + deemed as matching an empty string, or the program might have crashed. + +7. The code for checking whether a group could match an empty string was not + recognizing that \h, \H, \v, \V, and \R must match a character. + +8. Implemented PCRE_INFO_MATCH_EMPTY, which yields 1 if the pattern can match + an empty string. If it can, pcretest shows this in its information output. + +9. Fixed two related bugs that applied to Unicode extended grapheme clusters + that were repeated with a maximizing qualifier (e.g. \X* or \X{2,5}) when + matched by pcre_exec() without using JIT: + + (a) If the rest of the pattern did not match after a maximal run of + grapheme clusters, the code for backing up to try with fewer of them + did not always back up over a full grapheme when characters that do not + have the modifier quality were involved, e.g. Hangul syllables. + + (b) If the match point in a subject started with modifier character, and + there was no match, the code could incorrectly back up beyond the match + point, and potentially beyond the first character in the subject, + leading to a segfault or an incorrect match result. + +10. A conditional group with an assertion condition could lead to PCRE + recording an incorrect first data item for a match if no other first data + item was recorded. For example, the pattern (?(?=ab)ab) recorded "a" as a + first data item, and therefore matched "ca" after "c" instead of at the + start. + +11. Change 40 for 8.33 (allowing pcregrep to find empty strings) showed up a + bug that caused the command "echo a | ./pcregrep -M '|a'" to loop. + +12. The source of pcregrep now includes z/OS-specific code so that it can be + compiled for z/OS as part of the special z/OS distribution. + +13. Added the -T and -TM options to pcretest. + +14. The code in pcre_compile.c for creating the table of named capturing groups + has been refactored. Instead of creating the table dynamically during the + actual compiling pass, the information is remembered during the pre-compile + pass (on the stack unless there are more than 20 named groups, in which + case malloc() is used) and the whole table is created before the actual + compile happens. This has simplified the code (it is now nearly 150 lines + shorter) and prepared the way for better handling of references to groups + with duplicate names. + +15. A back reference to a named subpattern when there is more than one of the + same name now checks them in the order in which they appear in the pattern. + The first one that is set is used for the reference. Previously only the + first one was inspected. This change makes PCRE more compatible with Perl. + +16. Unicode character properties were updated from Unicode 6.3.0. + +17. The compile-time code for auto-possessification has been refactored, based + on a patch by Zoltan Herczeg. It now happens after instead of during + compilation. The code is cleaner, and more cases are handled. The option + PCRE_NO_AUTO_POSSESS is added for testing purposes, and the -O and /O + options in pcretest are provided to set it. It can also be set by + (*NO_AUTO_POSSESS) at the start of a pattern. + +18. The character VT has been added to the default ("C" locale) set of + characters that match \s and are generally treated as white space, + following this same change in Perl 5.18. There is now no difference between + "Perl space" and "POSIX space". Whether VT is treated as white space in + other locales depends on the locale. + +19. The code for checking named groups as conditions, either for being set or + for being recursed, has been refactored (this is related to 14 and 15 + above). Processing unduplicated named groups should now be as fast at + numerical groups, and processing duplicated groups should be faster than + before. + +20. Two patches to the CMake build system, by Alexander Barkov: + + (1) Replace the "source" command by "." in CMakeLists.txt because + "source" is a bash-ism. + + (2) Add missing HAVE_STDINT_H and HAVE_INTTYPES_H to config-cmake.h.in; + without these the CMake build does not work on Solaris. + +21. Perl has changed its handling of \8 and \9. If there is no previously + encountered capturing group of those numbers, they are treated as the + literal characters 8 and 9 instead of a binary zero followed by the + literals. PCRE now does the same. + +22. Following Perl, added \o{} to specify codepoints in octal, making it + possible to specify values greater than 0777 and also making them + unambiguous. + +23. Perl now gives an error for missing closing braces after \x{... instead of + treating the string as literal. PCRE now does the same. + +24. RunTest used to grumble if an inappropriate test was selected explicitly, + but just skip it when running all tests. This make it awkward to run ranges + of tests when one of them was inappropriate. Now it just skips any + inappropriate tests, as it always did when running all tests. + +25. If PCRE_AUTO_CALLOUT and PCRE_UCP were set for a pattern that contained + character types such as \d or \w, too many callouts were inserted, and the + data that they returned was rubbish. + +26. In UCP mode, \s was not matching two of the characters that Perl matches, + namely NEL (U+0085) and MONGOLIAN VOWEL SEPARATOR (U+180E), though they + were matched by \h. The code has now been refactored so that the lists of + the horizontal and vertical whitespace characters used for \h and \v (which + are defined only in one place) are now also used for \s. + +27. Add JIT support for the 64 bit TileGX architecture. + Patch by Jiong Wang (Tilera Corporation). + +28. Possessive quantifiers for classes (both explicit and automatically + generated) now use special opcodes instead of wrapping in ONCE brackets. + +29. Whereas an item such as A{4}+ ignored the possessivenes of the quantifier + (because it's meaningless), this was not happening when PCRE_CASELESS was + set. Not wrong, but inefficient. + +30. Updated perltest.pl to add /u (force Unicode mode) when /W (use Unicode + properties for \w, \d, etc) is present in a test regex. Otherwise if the + test contains no characters greater than 255, Perl doesn't realise it + should be using Unicode semantics. + +31. Upgraded the handling of the POSIX classes [:graph:], [:print:], and + [:punct:] when PCRE_UCP is set so as to include the same characters as Perl + does in Unicode mode. + +32. Added the "forbid" facility to pcretest so that putting tests into the + wrong test files can sometimes be quickly detected. + +33. There is now a limit (default 250) on the depth of nesting of parentheses. + This limit is imposed to control the amount of system stack used at compile + time. It can be changed at build time by --with-parens-nest-limit=xxx or + the equivalent in CMake. + +34. Character classes such as [A-\d] or [a-[:digit:]] now cause compile-time + errors. Perl warns for these when in warning mode, but PCRE has no facility + for giving warnings. + +35. Change 34 for 8.13 allowed quantifiers on assertions, because Perl does. + However, this was not working for (?!) because it is optimized to (*FAIL), + for which PCRE does not allow quantifiers. The optimization is now disabled + when a quantifier follows (?!). I can't see any use for this, but it makes + things uniform. + +36. Perl no longer allows group names to start with digits, so I have made this + change also in PCRE. It simplifies the code a bit. + +37. In extended mode, Perl ignores spaces before a + that indicates a + possessive quantifier. PCRE allowed a space before the quantifier, but not + before the possessive +. It now does. + +38. The use of \K (reset reported match start) within a repeated possessive + group such as (a\Kb)*+ was not working. + +40. Document that the same character tables must be used at compile time and + run time, and that the facility to pass tables to pcre_exec() and + pcre_dfa_exec() is for use only with saved/restored patterns. + +41. Applied Jeff Trawick's patch CMakeLists.txt, which "provides two new + features for Builds with MSVC: + + 1. Support pcre.rc and/or pcreposix.rc (as is already done for MinGW + builds). The .rc files can be used to set FileDescription and many other + attributes. + + 2. Add an option (-DINSTALL_MSVC_PDB) to enable installation of .pdb files. + This allows higher-level build scripts which want .pdb files to avoid + hard-coding the exact files needed." + +42. Added support for [[:<:]] and [[:>:]] as used in the BSD POSIX library to + mean "start of word" and "end of word", respectively, as a transition aid. + +43. A minimizing repeat of a class containing codepoints greater than 255 in + non-UTF 16-bit or 32-bit modes caused an internal error when PCRE was + compiled to use the heap for recursion. + +44. Got rid of some compiler warnings for unused variables when UTF but not UCP + is configured. + + +Version 8.33 28-May-2013 +------------------------ + +1. Added 'U' to some constants that are compared to unsigned integers, to + avoid compiler signed/unsigned warnings. Added (int) casts to unsigned + variables that are added to signed variables, to ensure the result is + signed and can be negated. + +2. Applied patch by Daniel Richard G for quashing MSVC warnings to the + CMake config files. + +3. Revise the creation of config.h.generic so that all boolean macros are + #undefined, whereas non-boolean macros are #ifndef/#endif-ed. This makes + overriding via -D on the command line possible. + +4. Changing the definition of the variable "op" in pcre_exec.c from pcre_uchar + to unsigned int is reported to make a quite noticeable speed difference in + a specific Windows environment. Testing on Linux did also appear to show + some benefit (and it is clearly not harmful). Also fixed the definition of + Xop which should be unsigned. + +5. Related to (4), changing the definition of the intermediate variable cc + in repeated character loops from pcre_uchar to pcre_uint32 also gave speed + improvements. + +6. Fix forward search in JIT when link size is 3 or greater. Also removed some + unnecessary spaces. + +7. Adjust autogen.sh and configure.ac to lose warnings given by automake 1.12 + and later. + +8. Fix two buffer over read issues in 16 and 32 bit modes. Affects JIT only. + +9. Optimizing fast_forward_start_bits in JIT. + +10. Adding support for callouts in JIT, and fixing some issues revealed + during this work. Namely: + + (a) Unoptimized capturing brackets incorrectly reset on backtrack. + + (b) Minimum length was not checked before the matching is started. + +11. The value of capture_last that is passed to callouts was incorrect in some + cases when there was a capture on one path that was subsequently abandoned + after a backtrack. Also, the capture_last value is now reset after a + recursion, since all captures are also reset in this case. + +12. The interpreter no longer returns the "too many substrings" error in the + case when an overflowing capture is in a branch that is subsequently + abandoned after a backtrack. + +13. In the pathological case when an offset vector of size 2 is used, pcretest + now prints out the matched string after a yield of 0 or 1. + +14. Inlining subpatterns in recursions, when certain conditions are fulfilled. + Only supported by the JIT compiler at the moment. + +15. JIT compiler now supports 32 bit Macs thanks to Lawrence Velazquez. + +16. Partial matches now set offsets[2] to the "bumpalong" value, that is, the + offset of the starting point of the matching process, provided the offsets + vector is large enough. + +17. The \A escape now records a lookbehind value of 1, though its execution + does not actually inspect the previous character. This is to ensure that, + in partial multi-segment matching, at least one character from the old + segment is retained when a new segment is processed. Otherwise, if there + are no lookbehinds in the pattern, \A might match incorrectly at the start + of a new segment. + +18. Added some #ifdef __VMS code into pcretest.c to help VMS implementations. + +19. Redefined some pcre_uchar variables in pcre_exec.c as pcre_uint32; this + gives some modest performance improvement in 8-bit mode. + +20. Added the PCRE-specific property \p{Xuc} for matching characters that can + be expressed in certain programming languages using Universal Character + Names. + +21. Unicode validation has been updated in the light of Unicode Corrigendum #9, + which points out that "non characters" are not "characters that may not + appear in Unicode strings" but rather "characters that are reserved for + internal use and have only local meaning". + +22. When a pattern was compiled with automatic callouts (PCRE_AUTO_CALLOUT) and + there was a conditional group that depended on an assertion, if the + assertion was false, the callout that immediately followed the alternation + in the condition was skipped when pcre_exec() was used for matching. + +23. Allow an explicit callout to be inserted before an assertion that is the + condition for a conditional group, for compatibility with automatic + callouts, which always insert a callout at this point. + +24. In 8.31, (*COMMIT) was confined to within a recursive subpattern. Perl also + confines (*SKIP) and (*PRUNE) in the same way, and this has now been done. + +25. (*PRUNE) is now supported by the JIT compiler. + +26. Fix infinite loop when /(?<=(*SKIP)ac)a/ is matched against aa. + +27. Fix the case where there are two or more SKIPs with arguments that may be + ignored. + +28. (*SKIP) is now supported by the JIT compiler. + +29. (*THEN) is now supported by the JIT compiler. + +30. Update RunTest with additional test selector options. + +31. The way PCRE handles backtracking verbs has been changed in two ways. + + (1) Previously, in something like (*COMMIT)(*SKIP), COMMIT would override + SKIP. Now, PCRE acts on whichever backtracking verb is reached first by + backtracking. In some cases this makes it more Perl-compatible, but Perl's + rather obscure rules do not always do the same thing. + + (2) Previously, backtracking verbs were confined within assertions. This is + no longer the case for positive assertions, except for (*ACCEPT). Again, + this sometimes improves Perl compatibility, and sometimes does not. + +32. A number of tests that were in test 2 because Perl did things differently + have been moved to test 1, because either Perl or PCRE has changed, and + these tests are now compatible. + +32. Backtracking control verbs are now handled in the same way in JIT and + interpreter. + +33. An opening parenthesis in a MARK/PRUNE/SKIP/THEN name in a pattern that + contained a forward subroutine reference caused a compile error. + +34. Auto-detect and optimize limited repetitions in JIT. + +35. Implement PCRE_NEVER_UTF to lock out the use of UTF, in particular, + blocking (*UTF) etc. + +36. In the interpreter, maximizing pattern repetitions for characters and + character types now use tail recursion, which reduces stack usage. + +37. The value of the max lookbehind was not correctly preserved if a compiled + and saved regex was reloaded on a host of different endianness. + +38. Implemented (*LIMIT_MATCH) and (*LIMIT_RECURSION). As part of the extension + of the compiled pattern block, expand the flags field from 16 to 32 bits + because it was almost full. + +39. Try madvise first before posix_madvise. + +40. Change 7 for PCRE 7.9 made it impossible for pcregrep to find empty lines + with a pattern such as ^$. It has taken 4 years for anybody to notice! The + original change locked out all matches of empty strings. This has been + changed so that one match of an empty string per line is recognized. + Subsequent searches on the same line (for colouring or for --only-matching, + for example) do not recognize empty strings. + +41. Applied a user patch to fix a number of spelling mistakes in comments. + +42. Data lines longer than 65536 caused pcretest to crash. + +43. Clarified the data type for length and startoffset arguments for pcre_exec + and pcre_dfa_exec in the function-specific man pages, where they were + explicitly stated to be in bytes, never having been updated. I also added + some clarification to the pcreapi man page. + +44. A call to pcre_dfa_exec() with an output vector size less than 2 caused + a segmentation fault. + + +Version 8.32 30-November-2012 +----------------------------- + +1. Improved JIT compiler optimizations for first character search and single + character iterators. + +2. Supporting IBM XL C compilers for PPC architectures in the JIT compiler. + Patch by Daniel Richard G. + +3. Single character iterator optimizations in the JIT compiler. + +4. Improved JIT compiler optimizations for character ranges. + +5. Rename the "leave" variable names to "quit" to improve WinCE compatibility. + Reported by Giuseppe D'Angelo. + +6. The PCRE_STARTLINE bit, indicating that a match can occur only at the start + of a line, was being set incorrectly in cases where .* appeared inside + atomic brackets at the start of a pattern, or where there was a subsequent + *PRUNE or *SKIP. + +7. Improved instruction cache flush for POWER/PowerPC. + Patch by Daniel Richard G. + +8. Fixed a number of issues in pcregrep, making it more compatible with GNU + grep: + + (a) There is now no limit to the number of patterns to be matched. + + (b) An error is given if a pattern is too long. + + (c) Multiple uses of --exclude, --exclude-dir, --include, and --include-dir + are now supported. + + (d) --exclude-from and --include-from (multiple use) have been added. + + (e) Exclusions and inclusions now apply to all files and directories, not + just to those obtained from scanning a directory recursively. + + (f) Multiple uses of -f and --file-list are now supported. + + (g) In a Windows environment, the default for -d has been changed from + "read" (the GNU grep default) to "skip", because otherwise the presence + of a directory in the file list provokes an error. + + (h) The documentation has been revised and clarified in places. + +9. Improve the matching speed of capturing brackets. + +10. Changed the meaning of \X so that it now matches a Unicode extended + grapheme cluster. + +11. Patch by Daniel Richard G to the autoconf files to add a macro for sorting + out POSIX threads when JIT support is configured. + +12. Added support for PCRE_STUDY_EXTRA_NEEDED. + +13. In the POSIX wrapper regcomp() function, setting re_nsub field in the preg + structure could go wrong in environments where size_t is not the same size + as int. + +14. Applied user-supplied patch to pcrecpp.cc to allow PCRE_NO_UTF8_CHECK to be + set. + +15. The EBCDIC support had decayed; later updates to the code had included + explicit references to (e.g.) \x0a instead of CHAR_LF. There has been a + general tidy up of EBCDIC-related issues, and the documentation was also + not quite right. There is now a test that can be run on ASCII systems to + check some of the EBCDIC-related things (but is it not a full test). + +16. The new PCRE_STUDY_EXTRA_NEEDED option is now used by pcregrep, resulting + in a small tidy to the code. + +17. Fix JIT tests when UTF is disabled and both 8 and 16 bit mode are enabled. + +18. If the --only-matching (-o) option in pcregrep is specified multiple + times, each one causes appropriate output. For example, -o1 -o2 outputs the + substrings matched by the 1st and 2nd capturing parentheses. A separating + string can be specified by --om-separator (default empty). + +19. Improving the first n character searches. + +20. Turn case lists for horizontal and vertical white space into macros so that + they are defined only once. + +21. This set of changes together give more compatible Unicode case-folding + behaviour for characters that have more than one other case when UCP + support is available. + + (a) The Unicode property table now has offsets into a new table of sets of + three or more characters that are case-equivalent. The MultiStage2.py + script that generates these tables (the pcre_ucd.c file) now scans + CaseFolding.txt instead of UnicodeData.txt for character case + information. + + (b) The code for adding characters or ranges of characters to a character + class has been abstracted into a generalized function that also handles + case-independence. In UTF-mode with UCP support, this uses the new data + to handle characters with more than one other case. + + (c) A bug that is fixed as a result of (b) is that codepoints less than 256 + whose other case is greater than 256 are now correctly matched + caselessly. Previously, the high codepoint matched the low one, but not + vice versa. + + (d) The processing of \h, \H, \v, and \ in character classes now makes use + of the new class addition function, using character lists defined as + macros alongside the case definitions of 20 above. + + (e) Caseless back references now work with characters that have more than + one other case. + + (f) General caseless matching of characters with more than one other case + is supported. + +22. Unicode character properties were updated from Unicode 6.2.0 + +23. Improved CMake support under Windows. Patch by Daniel Richard G. + +24. Add support for 32-bit character strings, and UTF-32 + +25. Major JIT compiler update (code refactoring and bugfixing). + Experimental Sparc 32 support is added. + +26. Applied a modified version of Daniel Richard G's patch to create + pcre.h.generic and config.h.generic by "make" instead of in the + PrepareRelease script. + +27. Added a definition for CHAR_NULL (helpful for the z/OS port), and use it in + pcre_compile.c when checking for a zero character. + +28. Introducing a native interface for JIT. Through this interface, the compiled + machine code can be directly executed. The purpose of this interface is to + provide fast pattern matching, so several sanity checks are not performed. + However, feature tests are still performed. The new interface provides + 1.4x speedup compared to the old one. + +29. If pcre_exec() or pcre_dfa_exec() was called with a negative value for + the subject string length, the error given was PCRE_ERROR_BADOFFSET, which + was confusing. There is now a new error PCRE_ERROR_BADLENGTH for this case. + +30. In 8-bit UTF-8 mode, pcretest failed to give an error for data codepoints + greater than 0x7fffffff (which cannot be represented in UTF-8, even under + the "old" RFC 2279). Instead, it ended up passing a negative length to + pcre_exec(). + +31. Add support for GCC's visibility feature to hide internal functions. + +32. Running "pcretest -C pcre8" or "pcretest -C pcre16" gave a spurious error + "unknown -C option" after outputting 0 or 1. + +33. There is now support for generating a code coverage report for the test + suite in environments where gcc is the compiler and lcov is installed. This + is mainly for the benefit of the developers. + +34. If PCRE is built with --enable-valgrind, certain memory regions are marked + unaddressable using valgrind annotations, allowing valgrind to detect + invalid memory accesses. This is mainly for the benefit of the developers. + +25. (*UTF) can now be used to start a pattern in any of the three libraries. + +26. Give configure error if --enable-cpp but no C++ compiler found. + + +Version 8.31 06-July-2012 +------------------------- + +1. Fixing a wrong JIT test case and some compiler warnings. + +2. Removed a bashism from the RunTest script. + +3. Add a cast to pcre_exec.c to fix the warning "unary minus operator applied + to unsigned type, result still unsigned" that was given by an MS compiler + on encountering the code "-sizeof(xxx)". + +4. Partial matching support is added to the JIT compiler. + +5. Fixed several bugs concerned with partial matching of items that consist + of more than one character: + + (a) /^(..)\1/ did not partially match "aba" because checking references was + done on an "all or nothing" basis. This also applied to repeated + references. + + (b) \R did not give a hard partial match if \r was found at the end of the + subject. + + (c) \X did not give a hard partial match after matching one or more + characters at the end of the subject. + + (d) When newline was set to CRLF, a pattern such as /a$/ did not recognize + a partial match for the string "\r". + + (e) When newline was set to CRLF, the metacharacter "." did not recognize + a partial match for a CR character at the end of the subject string. + +6. If JIT is requested using /S++ or -s++ (instead of just /S+ or -s+) when + running pcretest, the text "(JIT)" added to the output whenever JIT is + actually used to run the match. + +7. Individual JIT compile options can be set in pcretest by following -s+[+] + or /S+[+] with a digit between 1 and 7. + +8. OP_NOT now supports any UTF character not just single-byte ones. + +9. (*MARK) control verb is now supported by the JIT compiler. + +10. The command "./RunTest list" lists the available tests without actually + running any of them. (Because I keep forgetting what they all are.) + +11. Add PCRE_INFO_MAXLOOKBEHIND. + +12. Applied a (slightly modified) user-supplied patch that improves performance + when the heap is used for recursion (compiled with --disable-stack-for- + recursion). Instead of malloc and free for each heap frame each time a + logical recursion happens, frames are retained on a chain and re-used where + possible. This sometimes gives as much as 30% improvement. + +13. As documented, (*COMMIT) is now confined to within a recursive subpattern + call. + +14. As documented, (*COMMIT) is now confined to within a positive assertion. + +15. It is now possible to link pcretest with libedit as an alternative to + libreadline. + +16. (*COMMIT) control verb is now supported by the JIT compiler. + +17. The Unicode data tables have been updated to Unicode 6.1.0. + +18. Added --file-list option to pcregrep. + +19. Added binary file support to pcregrep, including the -a, --binary-files, + -I, and --text options. + +20. The madvise function is renamed for posix_madvise for QNX compatibility + reasons. Fixed by Giuseppe D'Angelo. + +21. Fixed a bug for backward assertions with REVERSE 0 in the JIT compiler. + +22. Changed the option for creating symbolic links for 16-bit man pages from + -s to -sf so that re-installing does not cause issues. + +23. Support PCRE_NO_START_OPTIMIZE in JIT as (*MARK) support requires it. + +24. Fixed a very old bug in pcretest that caused errors with restarted DFA + matches in certain environments (the workspace was not being correctly + retained). Also added to pcre_dfa_exec() a simple plausibility check on + some of the workspace data at the beginning of a restart. + +25. \s*\R was auto-possessifying the \s* when it should not, whereas \S*\R + was not doing so when it should - probably a typo introduced by SVN 528 + (change 8.10/14). + +26. When PCRE_UCP was not set, \w+\x{c4} was incorrectly auto-possessifying the + \w+ when the character tables indicated that \x{c4} was a word character. + There were several related cases, all because the tests for doing a table + lookup were testing for characters less than 127 instead of 255. + +27. If a pattern contains capturing parentheses that are not used in a match, + their slots in the ovector are set to -1. For those that are higher than + any matched groups, this happens at the end of processing. In the case when + there were back references that the ovector was too small to contain + (causing temporary malloc'd memory to be used during matching), and the + highest capturing number was not used, memory off the end of the ovector + was incorrectly being set to -1. (It was using the size of the temporary + memory instead of the true size.) + +28. To catch bugs like 27 using valgrind, when pcretest is asked to specify an + ovector size, it uses memory at the end of the block that it has got. + +29. Check for an overlong MARK name and give an error at compile time. The + limit is 255 for the 8-bit library and 65535 for the 16-bit library. + +30. JIT compiler update. + +31. JIT is now supported on jailbroken iOS devices. Thanks for Ruiger + Rill for the patch. + +32. Put spaces around SLJIT_PRINT_D in the JIT compiler. Required by CXX11. + +33. Variable renamings in the PCRE-JIT compiler. No functionality change. + +34. Fixed typos in pcregrep: in two places there was SUPPORT_LIBZ2 instead of + SUPPORT_LIBBZ2. This caused a build problem when bzip2 but not gzip (zlib) + was enabled. + +35. Improve JIT code generation for greedy plus quantifier. + +36. When /((?:a?)*)*c/ or /((?>a?)*)*c/ was matched against "aac", it set group + 1 to "aa" instead of to an empty string. The bug affected repeated groups + that could potentially match an empty string. + +37. Optimizing single character iterators in JIT. + +38. Wide characters specified with \uxxxx in JavaScript mode are now subject to + the same checks as \x{...} characters in non-JavaScript mode. Specifically, + codepoints that are too big for the mode are faulted, and in a UTF mode, + disallowed codepoints are also faulted. + +39. If PCRE was compiled with UTF support, in three places in the DFA + matcher there was code that should only have been obeyed in UTF mode, but + was being obeyed unconditionally. In 8-bit mode this could cause incorrect + processing when bytes with values greater than 127 were present. In 16-bit + mode the bug would be provoked by values in the range 0xfc00 to 0xdc00. In + both cases the values are those that cannot be the first data item in a UTF + character. The three items that might have provoked this were recursions, + possessively repeated groups, and atomic groups. + +40. Ensure that libpcre is explicitly listed in the link commands for pcretest + and pcregrep, because some OS require shared objects to be explicitly + passed to ld, causing the link step to fail if they are not. + +41. There were two incorrect #ifdefs in pcre_study.c, meaning that, in 16-bit + mode, patterns that started with \h* or \R* might be incorrectly matched. + + +Version 8.30 04-February-2012 +----------------------------- + +1. Renamed "isnumber" as "is_a_number" because in some Mac environments this + name is defined in ctype.h. + +2. Fixed a bug in fixed-length calculation for lookbehinds that would show up + only in quite long subpatterns. + +3. Removed the function pcre_info(), which has been obsolete and deprecated + since it was replaced by pcre_fullinfo() in February 2000. + +4. For a non-anchored pattern, if (*SKIP) was given with a name that did not + match a (*MARK), and the match failed at the start of the subject, a + reference to memory before the start of the subject could occur. This bug + was introduced by fix 17 of release 8.21. + +5. A reference to an unset group with zero minimum repetition was giving + totally wrong answers (in non-JavaScript-compatibility mode). For example, + /(another)?(\1?)test/ matched against "hello world test". This bug was + introduced in release 8.13. + +6. Add support for 16-bit character strings (a large amount of work involving + many changes and refactorings). + +7. RunGrepTest failed on msys because \r\n was replaced by whitespace when the + command "pattern=`printf 'xxx\r\njkl'`" was run. The pattern is now taken + from a file. + +8. Ovector size of 2 is also supported by JIT based pcre_exec (the ovector size + rounding is not applied in this particular case). + +9. The invalid Unicode surrogate codepoints U+D800 to U+DFFF are now rejected + if they appear, or are escaped, in patterns. + +10. Get rid of a number of -Wunused-but-set-variable warnings. + +11. The pattern /(?=(*:x))(q|)/ matches an empty string, and returns the mark + "x". The similar pattern /(?=(*:x))((*:y)q|)/ did not return a mark at all. + Oddly, Perl behaves the same way. PCRE has been fixed so that this pattern + also returns the mark "x". This bug applied to capturing parentheses, + non-capturing parentheses, and atomic parentheses. It also applied to some + assertions. + +12. Stephen Kelly's patch to CMakeLists.txt allows it to parse the version + information out of configure.ac instead of relying on pcre.h.generic, which + is not stored in the repository. + +13. Applied Dmitry V. Levin's patch for a more portable method for linking with + -lreadline. + +14. ZH added PCRE_CONFIG_JITTARGET; added its output to pcretest -C. + +15. Applied Graycode's patch to put the top-level frame on the stack rather + than the heap when not using the stack for recursion. This gives a + performance improvement in many cases when recursion is not deep. + +16. Experimental code added to "pcretest -C" to output the stack frame size. + + +Version 8.21 12-Dec-2011 +------------------------ + +1. Updating the JIT compiler. + +2. JIT compiler now supports OP_NCREF, OP_RREF and OP_NRREF. New test cases + are added as well. + +3. Fix cache-flush issue on PowerPC (It is still an experimental JIT port). + PCRE_EXTRA_TABLES is not suported by JIT, and should be checked before + calling _pcre_jit_exec. Some extra comments are added. + +4. (*MARK) settings inside atomic groups that do not contain any capturing + parentheses, for example, (?>a(*:m)), were not being passed out. This bug + was introduced by change 18 for 8.20. + +5. Supporting of \x, \U and \u in JavaScript compatibility mode based on the + ECMA-262 standard. + +6. Lookbehinds such as (?<=a{2}b) that contained a fixed repetition were + erroneously being rejected as "not fixed length" if PCRE_CASELESS was set. + This bug was probably introduced by change 9 of 8.13. + +7. While fixing 6 above, I noticed that a number of other items were being + incorrectly rejected as "not fixed length". This arose partly because newer + opcodes had not been added to the fixed-length checking code. I have (a) + corrected the bug and added tests for these items, and (b) arranged for an + error to occur if an unknown opcode is encountered while checking for fixed + length instead of just assuming "not fixed length". The items that were + rejected were: (*ACCEPT), (*COMMIT), (*FAIL), (*MARK), (*PRUNE), (*SKIP), + (*THEN), \h, \H, \v, \V, and single character negative classes with fixed + repetitions, e.g. [^a]{3}, with and without PCRE_CASELESS. + +8. A possessively repeated conditional subpattern such as (?(?=c)c|d)++ was + being incorrectly compiled and would have given unpredicatble results. + +9. A possessively repeated subpattern with minimum repeat count greater than + one behaved incorrectly. For example, (A){2,}+ behaved as if it was + (A)(A)++ which meant that, after a subsequent mismatch, backtracking into + the first (A) could occur when it should not. + +10. Add a cast and remove a redundant test from the code. + +11. JIT should use pcre_malloc/pcre_free for allocation. + +12. Updated pcre-config so that it no longer shows -L/usr/lib, which seems + best practice nowadays, and helps with cross-compiling. (If the exec_prefix + is anything other than /usr, -L is still shown). + +13. In non-UTF-8 mode, \C is now supported in lookbehinds and DFA matching. + +14. Perl does not support \N without a following name in a [] class; PCRE now + also gives an error. + +15. If a forward reference was repeated with an upper limit of around 2000, + it caused the error "internal error: overran compiling workspace". The + maximum number of forward references (including repeats) was limited by the + internal workspace, and dependent on the LINK_SIZE. The code has been + rewritten so that the workspace expands (via pcre_malloc) if necessary, and + the default depends on LINK_SIZE. There is a new upper limit (for safety) + of around 200,000 forward references. While doing this, I also speeded up + the filling in of repeated forward references. + +16. A repeated forward reference in a pattern such as (a)(?2){2}(.) was + incorrectly expecting the subject to contain another "a" after the start. + +17. When (*SKIP:name) is activated without a corresponding (*MARK:name) earlier + in the match, the SKIP should be ignored. This was not happening; instead + the SKIP was being treated as NOMATCH. For patterns such as + /A(*MARK:A)A+(*SKIP:B)Z|AAC/ this meant that the AAC branch was never + tested. + +18. The behaviour of (*MARK), (*PRUNE), and (*THEN) has been reworked and is + now much more compatible with Perl, in particular in cases where the result + is a non-match for a non-anchored pattern. For example, if + /b(*:m)f|a(*:n)w/ is matched against "abc", the non-match returns the name + "m", where previously it did not return a name. A side effect of this + change is that for partial matches, the last encountered mark name is + returned, as for non matches. A number of tests that were previously not + Perl-compatible have been moved into the Perl-compatible test files. The + refactoring has had the pleasing side effect of removing one argument from + the match() function, thus reducing its stack requirements. + +19. If the /S+ option was used in pcretest to study a pattern using JIT, + subsequent uses of /S (without +) incorrectly behaved like /S+. + +21. Retrieve executable code size support for the JIT compiler and fixing + some warnings. + +22. A caseless match of a UTF-8 character whose other case uses fewer bytes did + not work when the shorter character appeared right at the end of the + subject string. + +23. Added some (int) casts to non-JIT modules to reduce warnings on 64-bit + systems. + +24. Added PCRE_INFO_JITSIZE to pass on the value from (21) above, and also + output it when the /M option is used in pcretest. + +25. The CheckMan script was not being included in the distribution. Also, added + an explicit "perl" to run Perl scripts from the PrepareRelease script + because this is reportedly needed in Windows. + +26. If study data was being save in a file and studying had not found a set of + "starts with" bytes for the pattern, the data written to the file (though + never used) was taken from uninitialized memory and so caused valgrind to + complain. + +27. Updated RunTest.bat as provided by Sheri Pierce. + +28. Fixed a possible uninitialized memory bug in pcre_jit_compile.c. + +29. Computation of memory usage for the table of capturing group names was + giving an unnecessarily large value. + + +Version 8.20 21-Oct-2011 +------------------------ + +1. Change 37 of 8.13 broke patterns like [:a]...[b:] because it thought it had + a POSIX class. After further experiments with Perl, which convinced me that + Perl has bugs and confusions, a closing square bracket is no longer allowed + in a POSIX name. This bug also affected patterns with classes that started + with full stops. + +2. If a pattern such as /(a)b|ac/ is matched against "ac", there is no + captured substring, but while checking the failing first alternative, + substring 1 is temporarily captured. If the output vector supplied to + pcre_exec() was not big enough for this capture, the yield of the function + was still zero ("insufficient space for captured substrings"). This cannot + be totally fixed without adding another stack variable, which seems a lot + of expense for a edge case. However, I have improved the situation in cases + such as /(a)(b)x|abc/ matched against "abc", where the return code + indicates that fewer than the maximum number of slots in the ovector have + been set. + +3. Related to (2) above: when there are more back references in a pattern than + slots in the output vector, pcre_exec() uses temporary memory during + matching, and copies in the captures as far as possible afterwards. It was + using the entire output vector, but this conflicts with the specification + that only 2/3 is used for passing back captured substrings. Now it uses + only the first 2/3, for compatibility. This is, of course, another edge + case. + +4. Zoltan Herczeg's just-in-time compiler support has been integrated into the + main code base, and can be used by building with --enable-jit. When this is + done, pcregrep automatically uses it unless --disable-pcregrep-jit or the + runtime --no-jit option is given. + +5. When the number of matches in a pcre_dfa_exec() run exactly filled the + ovector, the return from the function was zero, implying that there were + other matches that did not fit. The correct "exactly full" value is now + returned. + +6. If a subpattern that was called recursively or as a subroutine contained + (*PRUNE) or any other control that caused it to give a non-standard return, + invalid errors such as "Error -26 (nested recursion at the same subject + position)" or even infinite loops could occur. + +7. If a pattern such as /a(*SKIP)c|b(*ACCEPT)|/ was studied, it stopped + computing the minimum length on reaching *ACCEPT, and so ended up with the + wrong value of 1 rather than 0. Further investigation indicates that + computing a minimum subject length in the presence of *ACCEPT is difficult + (think back references, subroutine calls), and so I have changed the code + so that no minimum is registered for a pattern that contains *ACCEPT. + +8. If (*THEN) was present in the first (true) branch of a conditional group, + it was not handled as intended. [But see 16 below.] + +9. Replaced RunTest.bat and CMakeLists.txt with improved versions provided by + Sheri Pierce. + +10. A pathological pattern such as /(*ACCEPT)a/ was miscompiled, thinking that + the first byte in a match must be "a". + +11. Change 17 for 8.13 increased the recursion depth for patterns like + /a(?:.)*?a/ drastically. I've improved things by remembering whether a + pattern contains any instances of (*THEN). If it does not, the old + optimizations are restored. It would be nice to do this on a per-group + basis, but at the moment that is not feasible. + +12. In some environments, the output of pcretest -C is CRLF terminated. This + broke RunTest's code that checks for the link size. A single white space + character after the value is now allowed for. + +13. RunTest now checks for the "fr" locale as well as for "fr_FR" and "french". + For "fr", it uses the Windows-specific input and output files. + +14. If (*THEN) appeared in a group that was called recursively or as a + subroutine, it did not work as intended. [But see next item.] + +15. Consider the pattern /A (B(*THEN)C) | D/ where A, B, C, and D are complex + pattern fragments (but not containing any | characters). If A and B are + matched, but there is a failure in C so that it backtracks to (*THEN), PCRE + was behaving differently to Perl. PCRE backtracked into A, but Perl goes to + D. In other words, Perl considers parentheses that do not contain any | + characters to be part of a surrounding alternative, whereas PCRE was + treading (B(*THEN)C) the same as (B(*THEN)C|(*FAIL)) -- which Perl handles + differently. PCRE now behaves in the same way as Perl, except in the case + of subroutine/recursion calls such as (?1) which have in any case always + been different (but PCRE had them first :-). + +16. Related to 15 above: Perl does not treat the | in a conditional group as + creating alternatives. Such a group is treated in the same way as an + ordinary group without any | characters when processing (*THEN). PCRE has + been changed to match Perl's behaviour. + +17. If a user had set PCREGREP_COLO(U)R to something other than 1:31, the + RunGrepTest script failed. + +18. Change 22 for version 13 caused atomic groups to use more stack. This is + inevitable for groups that contain captures, but it can lead to a lot of + stack use in large patterns. The old behaviour has been restored for atomic + groups that do not contain any capturing parentheses. + +19. If the PCRE_NO_START_OPTIMIZE option was set for pcre_compile(), it did not + suppress the check for a minimum subject length at run time. (If it was + given to pcre_exec() or pcre_dfa_exec() it did work.) + +20. Fixed an ASCII-dependent infelicity in pcretest that would have made it + fail to work when decoding hex characters in data strings in EBCDIC + environments. + +21. It appears that in at least one Mac OS environment, the isxdigit() function + is implemented as a macro that evaluates to its argument more than once, + contravening the C 90 Standard (I haven't checked a later standard). There + was an instance in pcretest which caused it to go wrong when processing + \x{...} escapes in subject strings. The has been rewritten to avoid using + things like p++ in the argument of isxdigit(). + + +Version 8.13 16-Aug-2011 +------------------------ + +1. The Unicode data tables have been updated to Unicode 6.0.0. + +2. Two minor typos in pcre_internal.h have been fixed. + +3. Added #include to pcre_scanner_unittest.cc, pcrecpp.cc, and + pcrecpp_unittest.cc. They are needed for strcmp(), memset(), and strchr() + in some environments (e.g. Solaris 10/SPARC using Sun Studio 12U2). + +4. There were a number of related bugs in the code for matching backrefences + caselessly in UTF-8 mode when codes for the characters concerned were + different numbers of bytes. For example, U+023A and U+2C65 are an upper + and lower case pair, using 2 and 3 bytes, respectively. The main bugs were: + (a) A reference to 3 copies of a 2-byte code matched only 2 of a 3-byte + code. (b) A reference to 2 copies of a 3-byte code would not match 2 of a + 2-byte code at the end of the subject (it thought there wasn't enough data + left). + +5. Comprehensive information about what went wrong is now returned by + pcre_exec() and pcre_dfa_exec() when the UTF-8 string check fails, as long + as the output vector has at least 2 elements. The offset of the start of + the failing character and a reason code are placed in the vector. + +6. When the UTF-8 string check fails for pcre_compile(), the offset that is + now returned is for the first byte of the failing character, instead of the + last byte inspected. This is an incompatible change, but I hope it is small + enough not to be a problem. It makes the returned offset consistent with + pcre_exec() and pcre_dfa_exec(). + +7. pcretest now gives a text phrase as well as the error number when + pcre_exec() or pcre_dfa_exec() fails; if the error is a UTF-8 check + failure, the offset and reason code are output. + +8. When \R was used with a maximizing quantifier it failed to skip backwards + over a \r\n pair if the subsequent match failed. Instead, it just skipped + back over a single character (\n). This seems wrong (because it treated the + two characters as a single entity when going forwards), conflicts with the + documentation that \R is equivalent to (?>\r\n|\n|...etc), and makes the + behaviour of \R* different to (\R)*, which also seems wrong. The behaviour + has been changed. + +9. Some internal refactoring has changed the processing so that the handling + of the PCRE_CASELESS and PCRE_MULTILINE options is done entirely at compile + time (the PCRE_DOTALL option was changed this way some time ago: version + 7.7 change 16). This has made it possible to abolish the OP_OPT op code, + which was always a bit of a fudge. It also means that there is one less + argument for the match() function, which reduces its stack requirements + slightly. This change also fixes an incompatibility with Perl: the pattern + (?i:([^b]))(?1) should not match "ab", but previously PCRE gave a match. + +10. More internal refactoring has drastically reduced the number of recursive + calls to match() for possessively repeated groups such as (abc)++ when + using pcre_exec(). + +11. While implementing 10, a number of bugs in the handling of groups were + discovered and fixed: + + (?<=(a)+) was not diagnosed as invalid (non-fixed-length lookbehind). + (a|)*(?1) gave a compile-time internal error. + ((a|)+)+ did not notice that the outer group could match an empty string. + (^a|^)+ was not marked as anchored. + (.*a|.*)+ was not marked as matching at start or after a newline. + +12. Yet more internal refactoring has removed another argument from the match() + function. Special calls to this function are now indicated by setting a + value in a variable in the "match data" data block. + +13. Be more explicit in pcre_study() instead of relying on "default" for + opcodes that mean there is no starting character; this means that when new + ones are added and accidentally left out of pcre_study(), testing should + pick them up. + +14. The -s option of pcretest has been documented for ages as being an old + synonym of -m (show memory usage). I have changed it to mean "force study + for every regex", that is, assume /S for every regex. This is similar to -i + and -d etc. It's slightly incompatible, but I'm hoping nobody is still + using it. It makes it easier to run collections of tests with and without + study enabled, and thereby test pcre_study() more easily. All the standard + tests are now run with and without -s (but some patterns can be marked as + "never study" - see 20 below). + +15. When (*ACCEPT) was used in a subpattern that was called recursively, the + restoration of the capturing data to the outer values was not happening + correctly. + +16. If a recursively called subpattern ended with (*ACCEPT) and matched an + empty string, and PCRE_NOTEMPTY was set, pcre_exec() thought the whole + pattern had matched an empty string, and so incorrectly returned a no + match. + +17. There was optimizing code for the last branch of non-capturing parentheses, + and also for the obeyed branch of a conditional subexpression, which used + tail recursion to cut down on stack usage. Unfortunately, now that there is + the possibility of (*THEN) occurring in these branches, tail recursion is + no longer possible because the return has to be checked for (*THEN). These + two optimizations have therefore been removed. [But see 8.20/11 above.] + +18. If a pattern containing \R was studied, it was assumed that \R always + matched two bytes, thus causing the minimum subject length to be + incorrectly computed because \R can also match just one byte. + +19. If a pattern containing (*ACCEPT) was studied, the minimum subject length + was incorrectly computed. + +20. If /S is present twice on a test pattern in pcretest input, it now + *disables* studying, thereby overriding the use of -s on the command line + (see 14 above). This is necessary for one or two tests to keep the output + identical in both cases. + +21. When (*ACCEPT) was used in an assertion that matched an empty string and + PCRE_NOTEMPTY was set, PCRE applied the non-empty test to the assertion. + +22. When an atomic group that contained a capturing parenthesis was + successfully matched, but the branch in which it appeared failed, the + capturing was not being forgotten if a higher numbered group was later + captured. For example, /(?>(a))b|(a)c/ when matching "ac" set capturing + group 1 to "a", when in fact it should be unset. This applied to multi- + branched capturing and non-capturing groups, repeated or not, and also to + positive assertions (capturing in negative assertions does not happen + in PCRE) and also to nested atomic groups. + +23. Add the ++ qualifier feature to pcretest, to show the remainder of the + subject after a captured substring, to make it easier to tell which of a + number of identical substrings has been captured. + +24. The way atomic groups are processed by pcre_exec() has been changed so that + if they are repeated, backtracking one repetition now resets captured + values correctly. For example, if ((?>(a+)b)+aabab) is matched against + "aaaabaaabaabab" the value of captured group 2 is now correctly recorded as + "aaa". Previously, it would have been "a". As part of this code + refactoring, the way recursive calls are handled has also been changed. + +25. If an assertion condition captured any substrings, they were not passed + back unless some other capturing happened later. For example, if + (?(?=(a))a) was matched against "a", no capturing was returned. + +26. When studying a pattern that contained subroutine calls or assertions, + the code for finding the minimum length of a possible match was handling + direct recursions such as (xxx(?1)|yyy) but not mutual recursions (where + group 1 called group 2 while simultaneously a separate group 2 called group + 1). A stack overflow occurred in this case. I have fixed this by limiting + the recursion depth to 10. + +27. Updated RunTest.bat in the distribution to the version supplied by Tom + Fortmann. This supports explicit test numbers on the command line, and has + argument validation and error reporting. + +28. An instance of \X with an unlimited repeat could fail if at any point the + first character it looked at was a mark character. + +29. Some minor code refactoring concerning Unicode properties and scripts + should reduce the stack requirement of match() slightly. + +30. Added the '=' option to pcretest to check the setting of unused capturing + slots at the end of the pattern, which are documented as being -1, but are + not included in the return count. + +31. If \k was not followed by a braced, angle-bracketed, or quoted name, PCRE + compiled something random. Now it gives a compile-time error (as does + Perl). + +32. A *MARK encountered during the processing of a positive assertion is now + recorded and passed back (compatible with Perl). + +33. If --only-matching or --colour was set on a pcregrep call whose pattern + had alternative anchored branches, the search for a second match in a line + was done as if at the line start. Thus, for example, /^01|^02/ incorrectly + matched the line "0102" twice. The same bug affected patterns that started + with a backwards assertion. For example /\b01|\b02/ also matched "0102" + twice. + +34. Previously, PCRE did not allow quantification of assertions. However, Perl + does, and because of capturing effects, quantifying parenthesized + assertions may at times be useful. Quantifiers are now allowed for + parenthesized assertions. + +35. A minor code tidy in pcre_compile() when checking options for \R usage. + +36. \g was being checked for fancy things in a character class, when it should + just be a literal "g". + +37. PCRE was rejecting [:a[:digit:]] whereas Perl was not. It seems that the + appearance of a nested POSIX class supersedes an apparent external class. + For example, [:a[:digit:]b:] matches "a", "b", ":", or a digit. Also, + unescaped square brackets may also appear as part of class names. For + example, [:a[:abc]b:] gives unknown class "[:abc]b:]". PCRE now behaves + more like Perl. (But see 8.20/1 above.) + +38. PCRE was giving an error for \N with a braced quantifier such as {1,} (this + was because it thought it was \N{name}, which is not supported). + +39. Add minix to OS list not supporting the -S option in pcretest. + +40. PCRE tries to detect cases of infinite recursion at compile time, but it + cannot analyze patterns in sufficient detail to catch mutual recursions + such as ((?1))((?2)). There is now a runtime test that gives an error if a + subgroup is called recursively as a subpattern for a second time at the + same position in the subject string. In previous releases this might have + been caught by the recursion limit, or it might have run out of stack. + +41. A pattern such as /(?(R)a+|(?R)b)/ is quite safe, as the recursion can + happen only once. PCRE was, however incorrectly giving a compile time error + "recursive call could loop indefinitely" because it cannot analyze the + pattern in sufficient detail. The compile time test no longer happens when + PCRE is compiling a conditional subpattern, but actual runaway loops are + now caught at runtime (see 40 above). + +42. It seems that Perl allows any characters other than a closing parenthesis + to be part of the NAME in (*MARK:NAME) and other backtracking verbs. PCRE + has been changed to be the same. + +43. Updated configure.ac to put in more quoting round AC_LANG_PROGRAM etc. so + as not to get warnings when autogen.sh is called. Also changed + AC_PROG_LIBTOOL (deprecated) to LT_INIT (the current macro). + +44. To help people who use pcregrep to scan files containing exceedingly long + lines, the following changes have been made: + + (a) The default value of the buffer size parameter has been increased from + 8K to 20K. (The actual buffer used is three times this size.) + + (b) The default can be changed by ./configure --with-pcregrep-bufsize when + PCRE is built. + + (c) A --buffer-size=n option has been added to pcregrep, to allow the size + to be set at run time. + + (d) Numerical values in pcregrep options can be followed by K or M, for + example --buffer-size=50K. + + (e) If a line being scanned overflows pcregrep's buffer, an error is now + given and the return code is set to 2. + +45. Add a pointer to the latest mark to the callout data block. + +46. The pattern /.(*F)/, when applied to "abc" with PCRE_PARTIAL_HARD, gave a + partial match of an empty string instead of no match. This was specific to + the use of ".". + +47. The pattern /f.*/8s, when applied to "for" with PCRE_PARTIAL_HARD, gave a + complete match instead of a partial match. This bug was dependent on both + the PCRE_UTF8 and PCRE_DOTALL options being set. + +48. For a pattern such as /\babc|\bdef/ pcre_study() was failing to set up the + starting byte set, because \b was not being ignored. + + +Version 8.12 15-Jan-2011 +------------------------ + +1. Fixed some typos in the markup of the man pages, and wrote a script that + checks for such things as part of the documentation building process. + +2. On a big-endian 64-bit system, pcregrep did not correctly process the + --match-limit and --recursion-limit options (added for 8.11). In + particular, this made one of the standard tests fail. (The integer value + went into the wrong half of a long int.) + +3. If the --colour option was given to pcregrep with -v (invert match), it + did strange things, either producing crazy output, or crashing. It should, + of course, ignore a request for colour when reporting lines that do not + match. + +4. Another pcregrep bug caused similar problems if --colour was specified with + -M (multiline) and the pattern match finished with a line ending. + +5. In pcregrep, when a pattern that ended with a literal newline sequence was + matched in multiline mode, the following line was shown as part of the + match. This seems wrong, so I have changed it. + +6. Another pcregrep bug in multiline mode, when --colour was specified, caused + the check for further matches in the same line (so they could be coloured) + to overrun the end of the current line. If another match was found, it was + incorrectly shown (and then shown again when found in the next line). + +7. If pcregrep was compiled under Windows, there was a reference to the + function pcregrep_exit() before it was defined. I am assuming this was + the cause of the "error C2371: 'pcregrep_exit' : redefinition;" that was + reported by a user. I've moved the definition above the reference. + + +Version 8.11 10-Dec-2010 +------------------------ + +1. (*THEN) was not working properly if there were untried alternatives prior + to it in the current branch. For example, in ((a|b)(*THEN)(*F)|c..) it + backtracked to try for "b" instead of moving to the next alternative branch + at the same level (in this case, to look for "c"). The Perl documentation + is clear that when (*THEN) is backtracked onto, it goes to the "next + alternative in the innermost enclosing group". + +2. (*COMMIT) was not overriding (*THEN), as it does in Perl. In a pattern + such as (A(*COMMIT)B(*THEN)C|D) any failure after matching A should + result in overall failure. Similarly, (*COMMIT) now overrides (*PRUNE) and + (*SKIP), (*SKIP) overrides (*PRUNE) and (*THEN), and (*PRUNE) overrides + (*THEN). + +3. If \s appeared in a character class, it removed the VT character from + the class, even if it had been included by some previous item, for example + in [\x00-\xff\s]. (This was a bug related to the fact that VT is not part + of \s, but is part of the POSIX "space" class.) + +4. A partial match never returns an empty string (because you can always + match an empty string at the end of the subject); however the checking for + an empty string was starting at the "start of match" point. This has been + changed to the "earliest inspected character" point, because the returned + data for a partial match starts at this character. This means that, for + example, /(?<=abc)def/ gives a partial match for the subject "abc" + (previously it gave "no match"). + +5. Changes have been made to the way PCRE_PARTIAL_HARD affects the matching + of $, \z, \Z, \b, and \B. If the match point is at the end of the string, + previously a full match would be given. However, setting PCRE_PARTIAL_HARD + has an implication that the given string is incomplete (because a partial + match is preferred over a full match). For this reason, these items now + give a partial match in this situation. [Aside: previously, the one case + /t\b/ matched against "cat" with PCRE_PARTIAL_HARD set did return a partial + match rather than a full match, which was wrong by the old rules, but is + now correct.] + +6. There was a bug in the handling of #-introduced comments, recognized when + PCRE_EXTENDED is set, when PCRE_NEWLINE_ANY and PCRE_UTF8 were also set. + If a UTF-8 multi-byte character included the byte 0x85 (e.g. +U0445, whose + UTF-8 encoding is 0xd1,0x85), this was misinterpreted as a newline when + scanning for the end of the comment. (*Character* 0x85 is an "any" newline, + but *byte* 0x85 is not, in UTF-8 mode). This bug was present in several + places in pcre_compile(). + +7. Related to (6) above, when pcre_compile() was skipping #-introduced + comments when looking ahead for named forward references to subpatterns, + the only newline sequence it recognized was NL. It now handles newlines + according to the set newline convention. + +8. SunOS4 doesn't have strerror() or strtoul(); pcregrep dealt with the + former, but used strtoul(), whereas pcretest avoided strtoul() but did not + cater for a lack of strerror(). These oversights have been fixed. + +9. Added --match-limit and --recursion-limit to pcregrep. + +10. Added two casts needed to build with Visual Studio when NO_RECURSE is set. + +11. When the -o option was used, pcregrep was setting a return code of 1, even + when matches were found, and --line-buffered was not being honoured. + +12. Added an optional parentheses number to the -o and --only-matching options + of pcregrep. + +13. Imitating Perl's /g action for multiple matches is tricky when the pattern + can match an empty string. The code to do it in pcretest and pcredemo + needed fixing: + + (a) When the newline convention was "crlf", pcretest got it wrong, skipping + only one byte after an empty string match just before CRLF (this case + just got forgotten; "any" and "anycrlf" were OK). + + (b) The pcretest code also had a bug, causing it to loop forever in UTF-8 + mode when an empty string match preceded an ASCII character followed by + a non-ASCII character. (The code for advancing by one character rather + than one byte was nonsense.) + + (c) The pcredemo.c sample program did not have any code at all to handle + the cases when CRLF is a valid newline sequence. + +14. Neither pcre_exec() nor pcre_dfa_exec() was checking that the value given + as a starting offset was within the subject string. There is now a new + error, PCRE_ERROR_BADOFFSET, which is returned if the starting offset is + negative or greater than the length of the string. In order to test this, + pcretest is extended to allow the setting of negative starting offsets. + +15. In both pcre_exec() and pcre_dfa_exec() the code for checking that the + starting offset points to the beginning of a UTF-8 character was + unnecessarily clumsy. I tidied it up. + +16. Added PCRE_ERROR_SHORTUTF8 to make it possible to distinguish between a + bad UTF-8 sequence and one that is incomplete when using PCRE_PARTIAL_HARD. + +17. Nobody had reported that the --include_dir option, which was added in + release 7.7 should have been called --include-dir (hyphen, not underscore) + for compatibility with GNU grep. I have changed it to --include-dir, but + left --include_dir as an undocumented synonym, and the same for + --exclude-dir, though that is not available in GNU grep, at least as of + release 2.5.4. + +18. At a user's suggestion, the macros GETCHAR and friends (which pick up UTF-8 + characters from a string of bytes) have been redefined so as not to use + loops, in order to improve performance in some environments. At the same + time, I abstracted some of the common code into auxiliary macros to save + repetition (this should not affect the compiled code). + +19. If \c was followed by a multibyte UTF-8 character, bad things happened. A + compile-time error is now given if \c is not followed by an ASCII + character, that is, a byte less than 128. (In EBCDIC mode, the code is + different, and any byte value is allowed.) + +20. Recognize (*NO_START_OPT) at the start of a pattern to set the PCRE_NO_ + START_OPTIMIZE option, which is now allowed at compile time - but just + passed through to pcre_exec() or pcre_dfa_exec(). This makes it available + to pcregrep and other applications that have no direct access to PCRE + options. The new /Y option in pcretest sets this option when calling + pcre_compile(). + +21. Change 18 of release 8.01 broke the use of named subpatterns for recursive + back references. Groups containing recursive back references were forced to + be atomic by that change, but in the case of named groups, the amount of + memory required was incorrectly computed, leading to "Failed: internal + error: code overflow". This has been fixed. + +22. Some patches to pcre_stringpiece.h, pcre_stringpiece_unittest.cc, and + pcretest.c, to avoid build problems in some Borland environments. + + +Version 8.10 25-Jun-2010 +------------------------ + +1. Added support for (*MARK:ARG) and for ARG additions to PRUNE, SKIP, and + THEN. + +2. (*ACCEPT) was not working when inside an atomic group. + +3. Inside a character class, \B is treated as a literal by default, but + faulted if PCRE_EXTRA is set. This mimics Perl's behaviour (the -w option + causes the error). The code is unchanged, but I tidied the documentation. + +4. Inside a character class, PCRE always treated \R and \X as literals, + whereas Perl faults them if its -w option is set. I have changed PCRE so + that it faults them when PCRE_EXTRA is set. + +5. Added support for \N, which always matches any character other than + newline. (It is the same as "." when PCRE_DOTALL is not set.) + +6. When compiling pcregrep with newer versions of gcc which may have + FORTIFY_SOURCE set, several warnings "ignoring return value of 'fwrite', + declared with attribute warn_unused_result" were given. Just casting the + result to (void) does not stop the warnings; a more elaborate fudge is + needed. I've used a macro to implement this. + +7. Minor change to pcretest.c to avoid a compiler warning. + +8. Added four artifical Unicode properties to help with an option to make + \s etc use properties (see next item). The new properties are: Xan + (alphanumeric), Xsp (Perl space), Xps (POSIX space), and Xwd (word). + +9. Added PCRE_UCP to make \b, \d, \s, \w, and certain POSIX character classes + use Unicode properties. (*UCP) at the start of a pattern can be used to set + this option. Modified pcretest to add /W to test this facility. Added + REG_UCP to make it available via the POSIX interface. + +10. Added --line-buffered to pcregrep. + +11. In UTF-8 mode, if a pattern that was compiled with PCRE_CASELESS was + studied, and the match started with a letter with a code point greater than + 127 whose first byte was different to the first byte of the other case of + the letter, the other case of this starting letter was not recognized + (#976). + +12. If a pattern that was studied started with a repeated Unicode property + test, for example, \p{Nd}+, there was the theoretical possibility of + setting up an incorrect bitmap of starting bytes, but fortunately it could + not have actually happened in practice until change 8 above was made (it + added property types that matched character-matching opcodes). + +13. pcre_study() now recognizes \h, \v, and \R when constructing a bit map of + possible starting bytes for non-anchored patterns. + +14. Extended the "auto-possessify" feature of pcre_compile(). It now recognizes + \R, and also a number of cases that involve Unicode properties, both + explicit and implicit when PCRE_UCP is set. + +15. If a repeated Unicode property match (e.g. \p{Lu}*) was used with non-UTF-8 + input, it could crash or give wrong results if characters with values + greater than 0xc0 were present in the subject string. (Detail: it assumed + UTF-8 input when processing these items.) + +16. Added a lot of (int) casts to avoid compiler warnings in systems where + size_t is 64-bit (#991). + +17. Added a check for running out of memory when PCRE is compiled with + --disable-stack-for-recursion (#990). + +18. If the last data line in a file for pcretest does not have a newline on + the end, a newline was missing in the output. + +19. The default pcre_chartables.c file recognizes only ASCII characters (values + less than 128) in its various bitmaps. However, there is a facility for + generating tables according to the current locale when PCRE is compiled. It + turns out that in some environments, 0x85 and 0xa0, which are Unicode space + characters, are recognized by isspace() and therefore were getting set in + these tables, and indeed these tables seem to approximate to ISO 8859. This + caused a problem in UTF-8 mode when pcre_study() was used to create a list + of bytes that can start a match. For \s, it was including 0x85 and 0xa0, + which of course cannot start UTF-8 characters. I have changed the code so + that only real ASCII characters (less than 128) and the correct starting + bytes for UTF-8 encodings are set for characters greater than 127 when in + UTF-8 mode. (When PCRE_UCP is set - see 9 above - the code is different + altogether.) + +20. Added the /T option to pcretest so as to be able to run tests with non- + standard character tables, thus making it possible to include the tests + used for 19 above in the standard set of tests. + +21. A pattern such as (?&t)(?#()(?(DEFINE)(?a)) which has a forward + reference to a subpattern the other side of a comment that contains an + opening parenthesis caused either an internal compiling error, or a + reference to the wrong subpattern. + + +Version 8.02 19-Mar-2010 +------------------------ + +1. The Unicode data tables have been updated to Unicode 5.2.0. + +2. Added the option --libs-cpp to pcre-config, but only when C++ support is + configured. + +3. Updated the licensing terms in the pcregexp.pas file, as agreed with the + original author of that file, following a query about its status. + +4. On systems that do not have stdint.h (e.g. Solaris), check for and include + inttypes.h instead. This fixes a bug that was introduced by change 8.01/8. + +5. A pattern such as (?&t)*+(?(DEFINE)(?.)) which has a possessive + quantifier applied to a forward-referencing subroutine call, could compile + incorrect code or give the error "internal error: previously-checked + referenced subpattern not found". + +6. Both MS Visual Studio and Symbian OS have problems with initializing + variables to point to external functions. For these systems, therefore, + pcre_malloc etc. are now initialized to local functions that call the + relevant global functions. + +7. There were two entries missing in the vectors called coptable and poptable + in pcre_dfa_exec.c. This could lead to memory accesses outsize the vectors. + I've fixed the data, and added a kludgy way of testing at compile time that + the lengths are correct (equal to the number of opcodes). + +8. Following on from 7, I added a similar kludge to check the length of the + eint vector in pcreposix.c. + +9. Error texts for pcre_compile() are held as one long string to avoid too + much relocation at load time. To find a text, the string is searched, + counting zeros. There was no check for running off the end of the string, + which could happen if a new error number was added without updating the + string. + +10. \K gave a compile-time error if it appeared in a lookbehind assersion. + +11. \K was not working if it appeared in an atomic group or in a group that + was called as a "subroutine", or in an assertion. Perl 5.11 documents that + \K is "not well defined" if used in an assertion. PCRE now accepts it if + the assertion is positive, but not if it is negative. + +12. Change 11 fortuitously reduced the size of the stack frame used in the + "match()" function of pcre_exec.c by one pointer. Forthcoming + implementation of support for (*MARK) will need an extra pointer on the + stack; I have reserved it now, so that the stack frame size does not + decrease. + +13. A pattern such as (?P(?P0)|(?P>L2)(?P>L1)) in which the only other + item in branch that calls a recursion is a subroutine call - as in the + second branch in the above example - was incorrectly given the compile- + time error "recursive call could loop indefinitely" because pcre_compile() + was not correctly checking the subroutine for matching a non-empty string. + +14. The checks for overrunning compiling workspace could trigger after an + overrun had occurred. This is a "should never occur" error, but it can be + triggered by pathological patterns such as hundreds of nested parentheses. + The checks now trigger 100 bytes before the end of the workspace. + +15. Fix typo in configure.ac: "srtoq" should be "strtoq". + + +Version 8.01 19-Jan-2010 +------------------------ + +1. If a pattern contained a conditional subpattern with only one branch (in + particular, this includes all (*DEFINE) patterns), a call to pcre_study() + computed the wrong minimum data length (which is of course zero for such + subpatterns). This could cause incorrect "no match" results. + +2. For patterns such as (?i)a(?-i)b|c where an option setting at the start of + the pattern is reset in the first branch, pcre_compile() failed with + "internal error: code overflow at offset...". This happened only when + the reset was to the original external option setting. (An optimization + abstracts leading options settings into an external setting, which was the + cause of this.) + +3. A pattern such as ^(?!a(*SKIP)b) where a negative assertion contained one + of the verbs SKIP, PRUNE, or COMMIT, did not work correctly. When the + assertion pattern did not match (meaning that the assertion was true), it + was incorrectly treated as false if the SKIP had been reached during the + matching. This also applied to assertions used as conditions. + +4. If an item that is not supported by pcre_dfa_exec() was encountered in an + assertion subpattern, including such a pattern used as a condition, + unpredictable results occurred, instead of the error return + PCRE_ERROR_DFA_UITEM. + +5. The C++ GlobalReplace function was not working like Perl for the special + situation when an empty string is matched. It now does the fancy magic + stuff that is necessary. + +6. In pcre_internal.h, obsolete includes to setjmp.h and stdarg.h have been + removed. (These were left over from very, very early versions of PCRE.) + +7. Some cosmetic changes to the code to make life easier when compiling it + as part of something else: + + (a) Change DEBUG to PCRE_DEBUG. + + (b) In pcre_compile(), rename the member of the "branch_chain" structure + called "current" as "current_branch", to prevent a collision with the + Linux macro when compiled as a kernel module. + + (c) In pcre_study(), rename the function set_bit() as set_table_bit(), to + prevent a collision with the Linux macro when compiled as a kernel + module. + +8. In pcre_compile() there are some checks for integer overflows that used to + cast potentially large values to (double). This has been changed to that + when building, a check for int64_t is made, and if it is found, it is used + instead, thus avoiding the use of floating point arithmetic. (There is no + other use of FP in PCRE.) If int64_t is not found, the fallback is to + double. + +9. Added two casts to avoid signed/unsigned warnings from VS Studio Express + 2005 (difference between two addresses compared to an unsigned value). + +10. Change the standard AC_CHECK_LIB test for libbz2 in configure.ac to a + custom one, because of the following reported problem in Windows: + + - libbz2 uses the Pascal calling convention (WINAPI) for the functions + under Win32. + - The standard autoconf AC_CHECK_LIB fails to include "bzlib.h", + therefore missing the function definition. + - The compiler thus generates a "C" signature for the test function. + - The linker fails to find the "C" function. + - PCRE fails to configure if asked to do so against libbz2. + +11. When running libtoolize from libtool-2.2.6b as part of autogen.sh, these + messages were output: + + Consider adding `AC_CONFIG_MACRO_DIR([m4])' to configure.ac and + rerunning libtoolize, to keep the correct libtool macros in-tree. + Consider adding `-I m4' to ACLOCAL_AMFLAGS in Makefile.am. + + I have done both of these things. + +12. Although pcre_dfa_exec() does not use nearly as much stack as pcre_exec() + most of the time, it *can* run out if it is given a pattern that contains a + runaway infinite recursion. I updated the discussion in the pcrestack man + page. + +13. Now that we have gone to the x.xx style of version numbers, the minor + version may start with zero. Using 08 or 09 is a bad idea because users + might check the value of PCRE_MINOR in their code, and 08 or 09 may be + interpreted as invalid octal numbers. I've updated the previous comment in + configure.ac, and also added a check that gives an error if 08 or 09 are + used. + +14. Change 8.00/11 was not quite complete: code had been accidentally omitted, + causing partial matching to fail when the end of the subject matched \W + in a UTF-8 pattern where \W was quantified with a minimum of 3. + +15. There were some discrepancies between the declarations in pcre_internal.h + of _pcre_is_newline(), _pcre_was_newline(), and _pcre_valid_utf8() and + their definitions. The declarations used "const uschar *" and the + definitions used USPTR. Even though USPTR is normally defined as "const + unsigned char *" (and uschar is typedeffed as "unsigned char"), it was + reported that: "This difference in casting confuses some C++ compilers, for + example, SunCC recognizes above declarations as different functions and + generates broken code for hbpcre." I have changed the declarations to use + USPTR. + +16. GNU libtool is named differently on some systems. The autogen.sh script now + tries several variants such as glibtoolize (MacOSX) and libtoolize1x + (FreeBSD). + +17. Applied Craig's patch that fixes an HP aCC compile error in pcre 8.00 + (strtoXX undefined when compiling pcrecpp.cc). The patch contains this + comment: "Figure out how to create a longlong from a string: strtoll and + equivalent. It's not enough to call AC_CHECK_FUNCS: hpux has a strtoll, for + instance, but it only takes 2 args instead of 3!" + +18. A subtle bug concerned with back references has been fixed by a change of + specification, with a corresponding code fix. A pattern such as + ^(xa|=?\1a)+$ which contains a back reference inside the group to which it + refers, was giving matches when it shouldn't. For example, xa=xaaa would + match that pattern. Interestingly, Perl (at least up to 5.11.3) has the + same bug. Such groups have to be quantified to be useful, or contained + inside another quantified group. (If there's no repetition, the reference + can never match.) The problem arises because, having left the group and + moved on to the rest of the pattern, a later failure that backtracks into + the group uses the captured value from the final iteration of the group + rather than the correct earlier one. I have fixed this in PCRE by forcing + any group that contains a reference to itself to be an atomic group; that + is, there cannot be any backtracking into it once it has completed. This is + similar to recursive and subroutine calls. + + +Version 8.00 19-Oct-09 +---------------------- + +1. The table for translating pcre_compile() error codes into POSIX error codes + was out-of-date, and there was no check on the pcre_compile() error code + being within the table. This could lead to an OK return being given in + error. + +2. Changed the call to open a subject file in pcregrep from fopen(pathname, + "r") to fopen(pathname, "rb"), which fixed a problem with some of the tests + in a Windows environment. + +3. The pcregrep --count option prints the count for each file even when it is + zero, as does GNU grep. However, pcregrep was also printing all files when + --files-with-matches was added. Now, when both options are given, it prints + counts only for those files that have at least one match. (GNU grep just + prints the file name in this circumstance, but including the count seems + more useful - otherwise, why use --count?) Also ensured that the + combination -clh just lists non-zero counts, with no names. + +4. The long form of the pcregrep -F option was incorrectly implemented as + --fixed_strings instead of --fixed-strings. This is an incompatible change, + but it seems right to fix it, and I didn't think it was worth preserving + the old behaviour. + +5. The command line items --regex=pattern and --regexp=pattern were not + recognized by pcregrep, which required --regex pattern or --regexp pattern + (with a space rather than an '='). The man page documented the '=' forms, + which are compatible with GNU grep; these now work. + +6. No libpcreposix.pc file was created for pkg-config; there was just + libpcre.pc and libpcrecpp.pc. The omission has been rectified. + +7. Added #ifndef SUPPORT_UCP into the pcre_ucd.c module, to reduce its size + when UCP support is not needed, by modifying the Python script that + generates it from Unicode data files. This should not matter if the module + is correctly used as a library, but I received one complaint about 50K of + unwanted data. My guess is that the person linked everything into his + program rather than using a library. Anyway, it does no harm. + +8. A pattern such as /\x{123}{2,2}+/8 was incorrectly compiled; the trigger + was a minimum greater than 1 for a wide character in a possessive + repetition. The same bug could also affect patterns like /(\x{ff}{0,2})*/8 + which had an unlimited repeat of a nested, fixed maximum repeat of a wide + character. Chaos in the form of incorrect output or a compiling loop could + result. + +9. The restrictions on what a pattern can contain when partial matching is + requested for pcre_exec() have been removed. All patterns can now be + partially matched by this function. In addition, if there are at least two + slots in the offset vector, the offset of the earliest inspected character + for the match and the offset of the end of the subject are set in them when + PCRE_ERROR_PARTIAL is returned. + +10. Partial matching has been split into two forms: PCRE_PARTIAL_SOFT, which is + synonymous with PCRE_PARTIAL, for backwards compatibility, and + PCRE_PARTIAL_HARD, which causes a partial match to supersede a full match, + and may be more useful for multi-segment matching. + +11. Partial matching with pcre_exec() is now more intuitive. A partial match + used to be given if ever the end of the subject was reached; now it is + given only if matching could not proceed because another character was + needed. This makes a difference in some odd cases such as Z(*FAIL) with the + string "Z", which now yields "no match" instead of "partial match". In the + case of pcre_dfa_exec(), "no match" is given if every matching path for the + final character ended with (*FAIL). + +12. Restarting a match using pcre_dfa_exec() after a partial match did not work + if the pattern had a "must contain" character that was already found in the + earlier partial match, unless partial matching was again requested. For + example, with the pattern /dog.(body)?/, the "must contain" character is + "g". If the first part-match was for the string "dog", restarting with + "sbody" failed. This bug has been fixed. + +13. The string returned by pcre_dfa_exec() after a partial match has been + changed so that it starts at the first inspected character rather than the + first character of the match. This makes a difference only if the pattern + starts with a lookbehind assertion or \b or \B (\K is not supported by + pcre_dfa_exec()). It's an incompatible change, but it makes the two + matching functions compatible, and I think it's the right thing to do. + +14. Added a pcredemo man page, created automatically from the pcredemo.c file, + so that the demonstration program is easily available in environments where + PCRE has not been installed from source. + +15. Arranged to add -DPCRE_STATIC to cflags in libpcre.pc, libpcreposix.cp, + libpcrecpp.pc and pcre-config when PCRE is not compiled as a shared + library. + +16. Added REG_UNGREEDY to the pcreposix interface, at the request of a user. + It maps to PCRE_UNGREEDY. It is not, of course, POSIX-compatible, but it + is not the first non-POSIX option to be added. Clearly some people find + these options useful. + +17. If a caller to the POSIX matching function regexec() passes a non-zero + value for nmatch with a NULL value for pmatch, the value of + nmatch is forced to zero. + +18. RunGrepTest did not have a test for the availability of the -u option of + the diff command, as RunTest does. It now checks in the same way as + RunTest, and also checks for the -b option. + +19. If an odd number of negated classes containing just a single character + interposed, within parentheses, between a forward reference to a named + subpattern and the definition of the subpattern, compilation crashed with + an internal error, complaining that it could not find the referenced + subpattern. An example of a crashing pattern is /(?&A)(([^m])(?))/. + [The bug was that it was starting one character too far in when skipping + over the character class, thus treating the ] as data rather than + terminating the class. This meant it could skip too much.] + +20. Added PCRE_NOTEMPTY_ATSTART in order to be able to correctly implement the + /g option in pcretest when the pattern contains \K, which makes it possible + to have an empty string match not at the start, even when the pattern is + anchored. Updated pcretest and pcredemo to use this option. + +21. If the maximum number of capturing subpatterns in a recursion was greater + than the maximum at the outer level, the higher number was returned, but + with unset values at the outer level. The correct (outer level) value is + now given. + +22. If (*ACCEPT) appeared inside capturing parentheses, previous releases of + PCRE did not set those parentheses (unlike Perl). I have now found a way to + make it do so. The string so far is captured, making this feature + compatible with Perl. + +23. The tests have been re-organized, adding tests 11 and 12, to make it + possible to check the Perl 5.10 features against Perl 5.10. + +24. Perl 5.10 allows subroutine calls in lookbehinds, as long as the subroutine + pattern matches a fixed length string. PCRE did not allow this; now it + does. Neither allows recursion. + +25. I finally figured out how to implement a request to provide the minimum + length of subject string that was needed in order to match a given pattern. + (It was back references and recursion that I had previously got hung up + on.) This code has now been added to pcre_study(); it finds a lower bound + to the length of subject needed. It is not necessarily the greatest lower + bound, but using it to avoid searching strings that are too short does give + some useful speed-ups. The value is available to calling programs via + pcre_fullinfo(). + +26. While implementing 25, I discovered to my embarrassment that pcretest had + not been passing the result of pcre_study() to pcre_dfa_exec(), so the + study optimizations had never been tested with that matching function. + Oops. What is worse, even when it was passed study data, there was a bug in + pcre_dfa_exec() that meant it never actually used it. Double oops. There + were also very few tests of studied patterns with pcre_dfa_exec(). + +27. If (?| is used to create subpatterns with duplicate numbers, they are now + allowed to have the same name, even if PCRE_DUPNAMES is not set. However, + on the other side of the coin, they are no longer allowed to have different + names, because these cannot be distinguished in PCRE, and this has caused + confusion. (This is a difference from Perl.) + +28. When duplicate subpattern names are present (necessarily with different + numbers, as required by 27 above), and a test is made by name in a + conditional pattern, either for a subpattern having been matched, or for + recursion in such a pattern, all the associated numbered subpatterns are + tested, and the overall condition is true if the condition is true for any + one of them. This is the way Perl works, and is also more like the way + testing by number works. + + +Version 7.9 11-Apr-09 +--------------------- + +1. When building with support for bzlib/zlib (pcregrep) and/or readline + (pcretest), all targets were linked against these libraries. This included + libpcre, libpcreposix, and libpcrecpp, even though they do not use these + libraries. This caused unwanted dependencies to be created. This problem + has been fixed, and now only pcregrep is linked with bzlib/zlib and only + pcretest is linked with readline. + +2. The "typedef int BOOL" in pcre_internal.h that was included inside the + "#ifndef FALSE" condition by an earlier change (probably 7.8/18) has been + moved outside it again, because FALSE and TRUE are already defined in AIX, + but BOOL is not. + +3. The pcre_config() function was treating the PCRE_MATCH_LIMIT and + PCRE_MATCH_LIMIT_RECURSION values as ints, when they should be long ints. + +4. The pcregrep documentation said spaces were inserted as well as colons (or + hyphens) following file names and line numbers when outputting matching + lines. This is not true; no spaces are inserted. I have also clarified the + wording for the --colour (or --color) option. + +5. In pcregrep, when --colour was used with -o, the list of matching strings + was not coloured; this is different to GNU grep, so I have changed it to be + the same. + +6. When --colo(u)r was used in pcregrep, only the first matching substring in + each matching line was coloured. Now it goes on to look for further matches + of any of the test patterns, which is the same behaviour as GNU grep. + +7. A pattern that could match an empty string could cause pcregrep to loop; it + doesn't make sense to accept an empty string match in pcregrep, so I have + locked it out (using PCRE's PCRE_NOTEMPTY option). By experiment, this + seems to be how GNU grep behaves. [But see later change 40 for release + 8.33.] + +8. The pattern (?(?=.*b)b|^) was incorrectly compiled as "match must be at + start or after a newline", because the conditional assertion was not being + correctly handled. The rule now is that both the assertion and what follows + in the first alternative must satisfy the test. + +9. If auto-callout was enabled in a pattern with a conditional group whose + condition was an assertion, PCRE could crash during matching, both with + pcre_exec() and pcre_dfa_exec(). + +10. The PCRE_DOLLAR_ENDONLY option was not working when pcre_dfa_exec() was + used for matching. + +11. Unicode property support in character classes was not working for + characters (bytes) greater than 127 when not in UTF-8 mode. + +12. Added the -M command line option to pcretest. + +14. Added the non-standard REG_NOTEMPTY option to the POSIX interface. + +15. Added the PCRE_NO_START_OPTIMIZE match-time option. + +16. Added comments and documentation about mis-use of no_arg in the C++ + wrapper. + +17. Implemented support for UTF-8 encoding in EBCDIC environments, a patch + from Martin Jerabek that uses macro names for all relevant character and + string constants. + +18. Added to pcre_internal.h two configuration checks: (a) If both EBCDIC and + SUPPORT_UTF8 are set, give an error; (b) If SUPPORT_UCP is set without + SUPPORT_UTF8, define SUPPORT_UTF8. The "configure" script handles both of + these, but not everybody uses configure. + +19. A conditional group that had only one branch was not being correctly + recognized as an item that could match an empty string. This meant that an + enclosing group might also not be so recognized, causing infinite looping + (and probably a segfault) for patterns such as ^"((?(?=[a])[^"])|b)*"$ + with the subject "ab", where knowledge that the repeated group can match + nothing is needed in order to break the loop. + +20. If a pattern that was compiled with callouts was matched using pcre_dfa_ + exec(), but without supplying a callout function, matching went wrong. + +21. If PCRE_ERROR_MATCHLIMIT occurred during a recursion, there was a memory + leak if the size of the offset vector was greater than 30. When the vector + is smaller, the saved offsets during recursion go onto a local stack + vector, but for larger vectors malloc() is used. It was failing to free + when the recursion yielded PCRE_ERROR_MATCH_LIMIT (or any other "abnormal" + error, in fact). + +22. There was a missing #ifdef SUPPORT_UTF8 round one of the variables in the + heapframe that is used only when UTF-8 support is enabled. This caused no + problem, but was untidy. + +23. Steven Van Ingelgem's patch to CMakeLists.txt to change the name + CMAKE_BINARY_DIR to PROJECT_BINARY_DIR so that it works when PCRE is + included within another project. + +24. Steven Van Ingelgem's patches to add more options to the CMake support, + slightly modified by me: + + (a) PCRE_BUILD_TESTS can be set OFF not to build the tests, including + not building pcregrep. + + (b) PCRE_BUILD_PCREGREP can be see OFF not to build pcregrep, but only + if PCRE_BUILD_TESTS is also set OFF, because the tests use pcregrep. + +25. Forward references, both numeric and by name, in patterns that made use of + duplicate group numbers, could behave incorrectly or give incorrect errors, + because when scanning forward to find the reference group, PCRE was not + taking into account the duplicate group numbers. A pattern such as + ^X(?3)(a)(?|(b)|(q))(Y) is an example. + +26. Changed a few more instances of "const unsigned char *" to USPTR, making + the feature of a custom pointer more persuasive (as requested by a user). + +27. Wrapped the definitions of fileno and isatty for Windows, which appear in + pcretest.c, inside #ifndefs, because it seems they are sometimes already + pre-defined. + +28. Added support for (*UTF8) at the start of a pattern. + +29. Arrange for flags added by the "release type" setting in CMake to be shown + in the configuration summary. + + +Version 7.8 05-Sep-08 +--------------------- + +1. Replaced UCP searching code with optimized version as implemented for Ad + Muncher (http://www.admuncher.com/) by Peter Kankowski. This uses a two- + stage table and inline lookup instead of a function, giving speed ups of 2 + to 5 times on some simple patterns that I tested. Permission was given to + distribute the MultiStage2.py script that generates the tables (it's not in + the tarball, but is in the Subversion repository). + +2. Updated the Unicode datatables to Unicode 5.1.0. This adds yet more + scripts. + +3. Change 12 for 7.7 introduced a bug in pcre_study() when a pattern contained + a group with a zero qualifier. The result of the study could be incorrect, + or the function might crash, depending on the pattern. + +4. Caseless matching was not working for non-ASCII characters in back + references. For example, /(\x{de})\1/8i was not matching \x{de}\x{fe}. + It now works when Unicode Property Support is available. + +5. In pcretest, an escape such as \x{de} in the data was always generating + a UTF-8 string, even in non-UTF-8 mode. Now it generates a single byte in + non-UTF-8 mode. If the value is greater than 255, it gives a warning about + truncation. + +6. Minor bugfix in pcrecpp.cc (change "" == ... to NULL == ...). + +7. Added two (int) casts to pcregrep when printing the difference of two + pointers, in case they are 64-bit values. + +8. Added comments about Mac OS X stack usage to the pcrestack man page and to + test 2 if it fails. + +9. Added PCRE_CALL_CONVENTION just before the names of all exported functions, + and a #define of that name to empty if it is not externally set. This is to + allow users of MSVC to set it if necessary. + +10. The PCRE_EXP_DEFN macro which precedes exported functions was missing from + the convenience functions in the pcre_get.c source file. + +11. An option change at the start of a pattern that had top-level alternatives + could cause overwriting and/or a crash. This command provoked a crash in + some environments: + + printf "/(?i)[\xc3\xa9\xc3\xbd]|[\xc3\xa9\xc3\xbdA]/8\n" | pcretest + + This potential security problem was recorded as CVE-2008-2371. + +12. For a pattern where the match had to start at the beginning or immediately + after a newline (e.g /.*anything/ without the DOTALL flag), pcre_exec() and + pcre_dfa_exec() could read past the end of the passed subject if there was + no match. To help with detecting such bugs (e.g. with valgrind), I modified + pcretest so that it places the subject at the end of its malloc-ed buffer. + +13. The change to pcretest in 12 above threw up a couple more cases when pcre_ + exec() might read past the end of the data buffer in UTF-8 mode. + +14. A similar bug to 7.3/2 existed when the PCRE_FIRSTLINE option was set and + the data contained the byte 0x85 as part of a UTF-8 character within its + first line. This applied both to normal and DFA matching. + +15. Lazy qualifiers were not working in some cases in UTF-8 mode. For example, + /^[^d]*?$/8 failed to match "abc". + +16. Added a missing copyright notice to pcrecpp_internal.h. + +17. Make it more clear in the documentation that values returned from + pcre_exec() in ovector are byte offsets, not character counts. + +18. Tidied a few places to stop certain compilers from issuing warnings. + +19. Updated the Virtual Pascal + BCC files to compile the latest v7.7, as + supplied by Stefan Weber. I made a further small update for 7.8 because + there is a change of source arrangements: the pcre_searchfuncs.c module is + replaced by pcre_ucd.c. + + +Version 7.7 07-May-08 +--------------------- + +1. Applied Craig's patch to sort out a long long problem: "If we can't convert + a string to a long long, pretend we don't even have a long long." This is + done by checking for the strtoq, strtoll, and _strtoi64 functions. + +2. Applied Craig's patch to pcrecpp.cc to restore ABI compatibility with + pre-7.6 versions, which defined a global no_arg variable instead of putting + it in the RE class. (See also #8 below.) + +3. Remove a line of dead code, identified by coverity and reported by Nuno + Lopes. + +4. Fixed two related pcregrep bugs involving -r with --include or --exclude: + + (1) The include/exclude patterns were being applied to the whole pathnames + of files, instead of just to the final components. + + (2) If there was more than one level of directory, the subdirectories were + skipped unless they satisfied the include/exclude conditions. This is + inconsistent with GNU grep (and could even be seen as contrary to the + pcregrep specification - which I improved to make it absolutely clear). + The action now is always to scan all levels of directory, and just + apply the include/exclude patterns to regular files. + +5. Added the --include_dir and --exclude_dir patterns to pcregrep, and used + --exclude_dir in the tests to avoid scanning .svn directories. + +6. Applied Craig's patch to the QuoteMeta function so that it escapes the + NUL character as backslash + 0 rather than backslash + NUL, because PCRE + doesn't support NULs in patterns. + +7. Added some missing "const"s to declarations of static tables in + pcre_compile.c and pcre_dfa_exec.c. + +8. Applied Craig's patch to pcrecpp.cc to fix a problem in OS X that was + caused by fix #2 above. (Subsequently also a second patch to fix the + first patch. And a third patch - this was a messy problem.) + +9. Applied Craig's patch to remove the use of push_back(). + +10. Applied Alan Lehotsky's patch to add REG_STARTEND support to the POSIX + matching function regexec(). + +11. Added support for the Oniguruma syntax \g, \g, \g'name', \g'n', + which, however, unlike Perl's \g{...}, are subroutine calls, not back + references. PCRE supports relative numbers with this syntax (I don't think + Oniguruma does). + +12. Previously, a group with a zero repeat such as (...){0} was completely + omitted from the compiled regex. However, this means that if the group + was called as a subroutine from elsewhere in the pattern, things went wrong + (an internal error was given). Such groups are now left in the compiled + pattern, with a new opcode that causes them to be skipped at execution + time. + +13. Added the PCRE_JAVASCRIPT_COMPAT option. This makes the following changes + to the way PCRE behaves: + + (a) A lone ] character is dis-allowed (Perl treats it as data). + + (b) A back reference to an unmatched subpattern matches an empty string + (Perl fails the current match path). + + (c) A data ] in a character class must be notated as \] because if the + first data character in a class is ], it defines an empty class. (In + Perl it is not possible to have an empty class.) The empty class [] + never matches; it forces failure and is equivalent to (*FAIL) or (?!). + The negative empty class [^] matches any one character, independently + of the DOTALL setting. + +14. A pattern such as /(?2)[]a()b](abc)/ which had a forward reference to a + non-existent subpattern following a character class starting with ']' and + containing () gave an internal compiling error instead of "reference to + non-existent subpattern". Fortunately, when the pattern did exist, the + compiled code was correct. (When scanning forwards to check for the + existence of the subpattern, it was treating the data ']' as terminating + the class, so got the count wrong. When actually compiling, the reference + was subsequently set up correctly.) + +15. The "always fail" assertion (?!) is optimzed to (*FAIL) by pcre_compile; + it was being rejected as not supported by pcre_dfa_exec(), even though + other assertions are supported. I have made pcre_dfa_exec() support + (*FAIL). + +16. The implementation of 13c above involved the invention of a new opcode, + OP_ALLANY, which is like OP_ANY but doesn't check the /s flag. Since /s + cannot be changed at match time, I realized I could make a small + improvement to matching performance by compiling OP_ALLANY instead of + OP_ANY for "." when DOTALL was set, and then removing the runtime tests + on the OP_ANY path. + +17. Compiling pcretest on Windows with readline support failed without the + following two fixes: (1) Make the unistd.h include conditional on + HAVE_UNISTD_H; (2) #define isatty and fileno as _isatty and _fileno. + +18. Changed CMakeLists.txt and cmake/FindReadline.cmake to arrange for the + ncurses library to be included for pcretest when ReadLine support is + requested, but also to allow for it to be overridden. This patch came from + Daniel Bergström. + +19. There was a typo in the file ucpinternal.h where f0_rangeflag was defined + as 0x00f00000 instead of 0x00800000. Luckily, this would not have caused + any errors with the current Unicode tables. Thanks to Peter Kankowski for + spotting this. + + +Version 7.6 28-Jan-08 +--------------------- + +1. A character class containing a very large number of characters with + codepoints greater than 255 (in UTF-8 mode, of course) caused a buffer + overflow. + +2. Patch to cut out the "long long" test in pcrecpp_unittest when + HAVE_LONG_LONG is not defined. + +3. Applied Christian Ehrlicher's patch to update the CMake build files to + bring them up to date and include new features. This patch includes: + + - Fixed PH's badly added libz and libbz2 support. + - Fixed a problem with static linking. + - Added pcredemo. [But later removed - see 7 below.] + - Fixed dftables problem and added an option. + - Added a number of HAVE_XXX tests, including HAVE_WINDOWS_H and + HAVE_LONG_LONG. + - Added readline support for pcretest. + - Added an listing of the option settings after cmake has run. + +4. A user submitted a patch to Makefile that makes it easy to create + "pcre.dll" under mingw when using Configure/Make. I added stuff to + Makefile.am that cause it to include this special target, without + affecting anything else. Note that the same mingw target plus all + the other distribution libraries and programs are now supported + when configuring with CMake (see 6 below) instead of with + Configure/Make. + +5. Applied Craig's patch that moves no_arg into the RE class in the C++ code. + This is an attempt to solve the reported problem "pcrecpp::no_arg is not + exported in the Windows port". It has not yet been confirmed that the patch + solves the problem, but it does no harm. + +6. Applied Sheri's patch to CMakeLists.txt to add NON_STANDARD_LIB_PREFIX and + NON_STANDARD_LIB_SUFFIX for dll names built with mingw when configured + with CMake, and also correct the comment about stack recursion. + +7. Remove the automatic building of pcredemo from the ./configure system and + from CMakeLists.txt. The whole idea of pcredemo.c is that it is an example + of a program that users should build themselves after PCRE is installed, so + building it automatically is not really right. What is more, it gave + trouble in some build environments. + +8. Further tidies to CMakeLists.txt from Sheri and Christian. + + +Version 7.5 10-Jan-08 +--------------------- + +1. Applied a patch from Craig: "This patch makes it possible to 'ignore' + values in parens when parsing an RE using the C++ wrapper." + +2. Negative specials like \S did not work in character classes in UTF-8 mode. + Characters greater than 255 were excluded from the class instead of being + included. + +3. The same bug as (2) above applied to negated POSIX classes such as + [:^space:]. + +4. PCRECPP_STATIC was referenced in pcrecpp_internal.h, but nowhere was it + defined or documented. It seems to have been a typo for PCRE_STATIC, so + I have changed it. + +5. The construct (?&) was not diagnosed as a syntax error (it referenced the + first named subpattern) and a construct such as (?&a) would reference the + first named subpattern whose name started with "a" (in other words, the + length check was missing). Both these problems are fixed. "Subpattern name + expected" is now given for (?&) (a zero-length name), and this patch also + makes it give the same error for \k'' (previously it complained that that + was a reference to a non-existent subpattern). + +6. The erroneous patterns (?+-a) and (?-+a) give different error messages; + this is right because (?- can be followed by option settings as well as by + digits. I have, however, made the messages clearer. + +7. Patterns such as (?(1)a|b) (a pattern that contains fewer subpatterns + than the number used in the conditional) now cause a compile-time error. + This is actually not compatible with Perl, which accepts such patterns, but + treats the conditional as always being FALSE (as PCRE used to), but it + seems to me that giving a diagnostic is better. + +8. Change "alphameric" to the more common word "alphanumeric" in comments + and messages. + +9. Fix two occurrences of "backslash" in comments that should have been + "backspace". + +10. Remove two redundant lines of code that can never be obeyed (their function + was moved elsewhere). + +11. The program that makes PCRE's Unicode character property table had a bug + which caused it to generate incorrect table entries for sequences of + characters that have the same character type, but are in different scripts. + It amalgamated them into a single range, with the script of the first of + them. In other words, some characters were in the wrong script. There were + thirteen such cases, affecting characters in the following ranges: + + U+002b0 - U+002c1 + U+0060c - U+0060d + U+0061e - U+00612 + U+0064b - U+0065e + U+0074d - U+0076d + U+01800 - U+01805 + U+01d00 - U+01d77 + U+01d9b - U+01dbf + U+0200b - U+0200f + U+030fc - U+030fe + U+03260 - U+0327f + U+0fb46 - U+0fbb1 + U+10450 - U+1049d + +12. The -o option (show only the matching part of a line) for pcregrep was not + compatible with GNU grep in that, if there was more than one match in a + line, it showed only the first of them. It now behaves in the same way as + GNU grep. + +13. If the -o and -v options were combined for pcregrep, it printed a blank + line for every non-matching line. GNU grep prints nothing, and pcregrep now + does the same. The return code can be used to tell if there were any + non-matching lines. + +14. Added --file-offsets and --line-offsets to pcregrep. + +15. The pattern (?=something)(?R) was not being diagnosed as a potentially + infinitely looping recursion. The bug was that positive lookaheads were not + being skipped when checking for a possible empty match (negative lookaheads + and both kinds of lookbehind were skipped). + +16. Fixed two typos in the Windows-only code in pcregrep.c, and moved the + inclusion of to before rather than after the definition of + INVALID_FILE_ATTRIBUTES (patch from David Byron). + +17. Specifying a possessive quantifier with a specific limit for a Unicode + character property caused pcre_compile() to compile bad code, which led at + runtime to PCRE_ERROR_INTERNAL (-14). Examples of patterns that caused this + are: /\p{Zl}{2,3}+/8 and /\p{Cc}{2}+/8. It was the possessive "+" that + caused the error; without that there was no problem. + +18. Added --enable-pcregrep-libz and --enable-pcregrep-libbz2. + +19. Added --enable-pcretest-libreadline. + +20. In pcrecpp.cc, the variable 'count' was incremented twice in + RE::GlobalReplace(). As a result, the number of replacements returned was + double what it should be. I removed one of the increments, but Craig sent a + later patch that removed the other one (the right fix) and added unit tests + that check the return values (which was not done before). + +21. Several CMake things: + + (1) Arranged that, when cmake is used on Unix, the libraries end up with + the names libpcre and libpcreposix, not just pcre and pcreposix. + + (2) The above change means that pcretest and pcregrep are now correctly + linked with the newly-built libraries, not previously installed ones. + + (3) Added PCRE_SUPPORT_LIBREADLINE, PCRE_SUPPORT_LIBZ, PCRE_SUPPORT_LIBBZ2. + +22. In UTF-8 mode, with newline set to "any", a pattern such as .*a.*=.b.* + crashed when matching a string such as a\x{2029}b (note that \x{2029} is a + UTF-8 newline character). The key issue is that the pattern starts .*; + this means that the match must be either at the beginning, or after a + newline. The bug was in the code for advancing after a failed match and + checking that the new position followed a newline. It was not taking + account of UTF-8 characters correctly. + +23. PCRE was behaving differently from Perl in the way it recognized POSIX + character classes. PCRE was not treating the sequence [:...:] as a + character class unless the ... were all letters. Perl, however, seems to + allow any characters between [: and :], though of course it rejects as + unknown any "names" that contain non-letters, because all the known class + names consist only of letters. Thus, Perl gives an error for [[:1234:]], + for example, whereas PCRE did not - it did not recognize a POSIX character + class. This seemed a bit dangerous, so the code has been changed to be + closer to Perl. The behaviour is not identical to Perl, because PCRE will + diagnose an unknown class for, for example, [[:l\ower:]] where Perl will + treat it as [[:lower:]]. However, PCRE does now give "unknown" errors where + Perl does, and where it didn't before. + +24. Rewrite so as to remove the single use of %n from pcregrep because in some + Windows environments %n is disabled by default. + + +Version 7.4 21-Sep-07 +--------------------- + +1. Change 7.3/28 was implemented for classes by looking at the bitmap. This + means that a class such as [\s] counted as "explicit reference to CR or + LF". That isn't really right - the whole point of the change was to try to + help when there was an actual mention of one of the two characters. So now + the change happens only if \r or \n (or a literal CR or LF) character is + encountered. + +2. The 32-bit options word was also used for 6 internal flags, but the numbers + of both had grown to the point where there were only 3 bits left. + Fortunately, there was spare space in the data structure, and so I have + moved the internal flags into a new 16-bit field to free up more option + bits. + +3. The appearance of (?J) at the start of a pattern set the DUPNAMES option, + but did not set the internal JCHANGED flag - either of these is enough to + control the way the "get" function works - but the PCRE_INFO_JCHANGED + facility is supposed to tell if (?J) was ever used, so now (?J) at the + start sets both bits. + +4. Added options (at build time, compile time, exec time) to change \R from + matching any Unicode line ending sequence to just matching CR, LF, or CRLF. + +5. doc/pcresyntax.html was missing from the distribution. + +6. Put back the definition of PCRE_ERROR_NULLWSLIMIT, for backward + compatibility, even though it is no longer used. + +7. Added macro for snprintf to pcrecpp_unittest.cc and also for strtoll and + strtoull to pcrecpp.cc to select the available functions in WIN32 when the + windows.h file is present (where different names are used). [This was + reversed later after testing - see 16 below.] + +8. Changed all #include to #include "config.h". There were also + some further cases that I changed to "pcre.h". + +9. When pcregrep was used with the --colour option, it missed the line ending + sequence off the lines that it output. + +10. It was pointed out to me that arrays of string pointers cause lots of + relocations when a shared library is dynamically loaded. A technique of + using a single long string with a table of offsets can drastically reduce + these. I have refactored PCRE in four places to do this. The result is + dramatic: + + Originally: 290 + After changing UCP table: 187 + After changing error message table: 43 + After changing table of "verbs" 36 + After changing table of Posix names 22 + + Thanks to the folks working on Gregex for glib for this insight. + +11. --disable-stack-for-recursion caused compiling to fail unless -enable- + unicode-properties was also set. + +12. Updated the tests so that they work when \R is defaulted to ANYCRLF. + +13. Added checks for ANY and ANYCRLF to pcrecpp.cc where it previously + checked only for CRLF. + +14. Added casts to pcretest.c to avoid compiler warnings. + +15. Added Craig's patch to various pcrecpp modules to avoid compiler warnings. + +16. Added Craig's patch to remove the WINDOWS_H tests, that were not working, + and instead check for _strtoi64 explicitly, and avoid the use of snprintf() + entirely. This removes changes made in 7 above. + +17. The CMake files have been updated, and there is now more information about + building with CMake in the NON-UNIX-USE document. + + +Version 7.3 28-Aug-07 +--------------------- + + 1. In the rejigging of the build system that eventually resulted in 7.1, the + line "#include " was included in pcre_internal.h. The use of angle + brackets there is not right, since it causes compilers to look for an + installed pcre.h, not the version that is in the source that is being + compiled (which of course may be different). I have changed it back to: + + #include "pcre.h" + + I have a vague recollection that the change was concerned with compiling in + different directories, but in the new build system, that is taken care of + by the VPATH setting the Makefile. + + 2. The pattern .*$ when run in not-DOTALL UTF-8 mode with newline=any failed + when the subject happened to end in the byte 0x85 (e.g. if the last + character was \x{1ec5}). *Character* 0x85 is one of the "any" newline + characters but of course it shouldn't be taken as a newline when it is part + of another character. The bug was that, for an unlimited repeat of . in + not-DOTALL UTF-8 mode, PCRE was advancing by bytes rather than by + characters when looking for a newline. + + 3. A small performance improvement in the DOTALL UTF-8 mode .* case. + + 4. Debugging: adjusted the names of opcodes for different kinds of parentheses + in debug output. + + 5. Arrange to use "%I64d" instead of "%lld" and "%I64u" instead of "%llu" for + long printing in the pcrecpp unittest when running under MinGW. + + 6. ESC_K was left out of the EBCDIC table. + + 7. Change 7.0/38 introduced a new limit on the number of nested non-capturing + parentheses; I made it 1000, which seemed large enough. Unfortunately, the + limit also applies to "virtual nesting" when a pattern is recursive, and in + this case 1000 isn't so big. I have been able to remove this limit at the + expense of backing off one optimization in certain circumstances. Normally, + when pcre_exec() would call its internal match() function recursively and + immediately return the result unconditionally, it uses a "tail recursion" + feature to save stack. However, when a subpattern that can match an empty + string has an unlimited repetition quantifier, it no longer makes this + optimization. That gives it a stack frame in which to save the data for + checking that an empty string has been matched. Previously this was taken + from the 1000-entry workspace that had been reserved. So now there is no + explicit limit, but more stack is used. + + 8. Applied Daniel's patches to solve problems with the import/export magic + syntax that is required for Windows, and which was going wrong for the + pcreposix and pcrecpp parts of the library. These were overlooked when this + problem was solved for the main library. + + 9. There were some crude static tests to avoid integer overflow when computing + the size of patterns that contain repeated groups with explicit upper + limits. As the maximum quantifier is 65535, the maximum group length was + set at 30,000 so that the product of these two numbers did not overflow a + 32-bit integer. However, it turns out that people want to use groups that + are longer than 30,000 bytes (though not repeat them that many times). + Change 7.0/17 (the refactoring of the way the pattern size is computed) has + made it possible to implement the integer overflow checks in a much more + dynamic way, which I have now done. The artificial limitation on group + length has been removed - we now have only the limit on the total length of + the compiled pattern, which depends on the LINK_SIZE setting. + +10. Fixed a bug in the documentation for get/copy named substring when + duplicate names are permitted. If none of the named substrings are set, the + functions return PCRE_ERROR_NOSUBSTRING (7); the doc said they returned an + empty string. + +11. Because Perl interprets \Q...\E at a high level, and ignores orphan \E + instances, patterns such as [\Q\E] or [\E] or even [^\E] cause an error, + because the ] is interpreted as the first data character and the + terminating ] is not found. PCRE has been made compatible with Perl in this + regard. Previously, it interpreted [\Q\E] as an empty class, and [\E] could + cause memory overwriting. + +10. Like Perl, PCRE automatically breaks an unlimited repeat after an empty + string has been matched (to stop an infinite loop). It was not recognizing + a conditional subpattern that could match an empty string if that + subpattern was within another subpattern. For example, it looped when + trying to match (((?(1)X|))*) but it was OK with ((?(1)X|)*) where the + condition was not nested. This bug has been fixed. + +12. A pattern like \X?\d or \P{L}?\d in non-UTF-8 mode could cause a backtrack + past the start of the subject in the presence of bytes with the top bit + set, for example "\x8aBCD". + +13. Added Perl 5.10 experimental backtracking controls (*FAIL), (*F), (*PRUNE), + (*SKIP), (*THEN), (*COMMIT), and (*ACCEPT). + +14. Optimized (?!) to (*FAIL). + +15. Updated the test for a valid UTF-8 string to conform to the later RFC 3629. + This restricts code points to be within the range 0 to 0x10FFFF, excluding + the "low surrogate" sequence 0xD800 to 0xDFFF. Previously, PCRE allowed the + full range 0 to 0x7FFFFFFF, as defined by RFC 2279. Internally, it still + does: it's just the validity check that is more restrictive. + +16. Inserted checks for integer overflows during escape sequence (backslash) + processing, and also fixed erroneous offset values for syntax errors during + backslash processing. + +17. Fixed another case of looking too far back in non-UTF-8 mode (cf 12 above) + for patterns like [\PPP\x8a]{1,}\x80 with the subject "A\x80". + +18. An unterminated class in a pattern like (?1)\c[ with a "forward reference" + caused an overrun. + +19. A pattern like (?:[\PPa*]*){8,} which had an "extended class" (one with + something other than just ASCII characters) inside a group that had an + unlimited repeat caused a loop at compile time (while checking to see + whether the group could match an empty string). + +20. Debugging a pattern containing \p or \P could cause a crash. For example, + [\P{Any}] did so. (Error in the code for printing property names.) + +21. An orphan \E inside a character class could cause a crash. + +22. A repeated capturing bracket such as (A)? could cause a wild memory + reference during compilation. + +23. There are several functions in pcre_compile() that scan along a compiled + expression for various reasons (e.g. to see if it's fixed length for look + behind). There were bugs in these functions when a repeated \p or \P was + present in the pattern. These operators have additional parameters compared + with \d, etc, and these were not being taken into account when moving along + the compiled data. Specifically: + + (a) A item such as \p{Yi}{3} in a lookbehind was not treated as fixed + length. + + (b) An item such as \pL+ within a repeated group could cause crashes or + loops. + + (c) A pattern such as \p{Yi}+(\P{Yi}+)(?1) could give an incorrect + "reference to non-existent subpattern" error. + + (d) A pattern like (\P{Yi}{2}\277)? could loop at compile time. + +24. A repeated \S or \W in UTF-8 mode could give wrong answers when multibyte + characters were involved (for example /\S{2}/8g with "A\x{a3}BC"). + +25. Using pcregrep in multiline, inverted mode (-Mv) caused it to loop. + +26. Patterns such as [\P{Yi}A] which include \p or \P and just one other + character were causing crashes (broken optimization). + +27. Patterns such as (\P{Yi}*\277)* (group with possible zero repeat containing + \p or \P) caused a compile-time loop. + +28. More problems have arisen in unanchored patterns when CRLF is a valid line + break. For example, the unstudied pattern [\r\n]A does not match the string + "\r\nA" because change 7.0/46 below moves the current point on by two + characters after failing to match at the start. However, the pattern \nA + *does* match, because it doesn't start till \n, and if [\r\n]A is studied, + the same is true. There doesn't seem any very clean way out of this, but + what I have chosen to do makes the common cases work: PCRE now takes note + of whether there can be an explicit match for \r or \n anywhere in the + pattern, and if so, 7.0/46 no longer applies. As part of this change, + there's a new PCRE_INFO_HASCRORLF option for finding out whether a compiled + pattern has explicit CR or LF references. + +29. Added (*CR) etc for changing newline setting at start of pattern. + + +Version 7.2 19-Jun-07 +--------------------- + + 1. If the fr_FR locale cannot be found for test 3, try the "french" locale, + which is apparently normally available under Windows. + + 2. Re-jig the pcregrep tests with different newline settings in an attempt + to make them independent of the local environment's newline setting. + + 3. Add code to configure.ac to remove -g from the CFLAGS default settings. + + 4. Some of the "internals" tests were previously cut out when the link size + was not 2, because the output contained actual offsets. The recent new + "Z" feature of pcretest means that these can be cut out, making the tests + usable with all link sizes. + + 5. Implemented Stan Switzer's goto replacement for longjmp() when not using + stack recursion. This gives a massive performance boost under BSD, but just + a small improvement under Linux. However, it saves one field in the frame + in all cases. + + 6. Added more features from the forthcoming Perl 5.10: + + (a) (?-n) (where n is a string of digits) is a relative subroutine or + recursion call. It refers to the nth most recently opened parentheses. + + (b) (?+n) is also a relative subroutine call; it refers to the nth next + to be opened parentheses. + + (c) Conditions that refer to capturing parentheses can be specified + relatively, for example, (?(-2)... or (?(+3)... + + (d) \K resets the start of the current match so that everything before + is not part of it. + + (e) \k{name} is synonymous with \k and \k'name' (.NET compatible). + + (f) \g{name} is another synonym - part of Perl 5.10's unification of + reference syntax. + + (g) (?| introduces a group in which the numbering of parentheses in each + alternative starts with the same number. + + (h) \h, \H, \v, and \V match horizontal and vertical whitespace. + + 7. Added two new calls to pcre_fullinfo(): PCRE_INFO_OKPARTIAL and + PCRE_INFO_JCHANGED. + + 8. A pattern such as (.*(.)?)* caused pcre_exec() to fail by either not + terminating or by crashing. Diagnosed by Viktor Griph; it was in the code + for detecting groups that can match an empty string. + + 9. A pattern with a very large number of alternatives (more than several + hundred) was running out of internal workspace during the pre-compile + phase, where pcre_compile() figures out how much memory will be needed. A + bit of new cunning has reduced the workspace needed for groups with + alternatives. The 1000-alternative test pattern now uses 12 bytes of + workspace instead of running out of the 4096 that are available. + +10. Inserted some missing (unsigned int) casts to get rid of compiler warnings. + +11. Applied patch from Google to remove an optimization that didn't quite work. + The report of the bug said: + + pcrecpp::RE("a*").FullMatch("aaa") matches, while + pcrecpp::RE("a*?").FullMatch("aaa") does not, and + pcrecpp::RE("a*?\\z").FullMatch("aaa") does again. + +12. If \p or \P was used in non-UTF-8 mode on a character greater than 127 + it matched the wrong number of bytes. + + +Version 7.1 24-Apr-07 +--------------------- + + 1. Applied Bob Rossi and Daniel G's patches to convert the build system to one + that is more "standard", making use of automake and other Autotools. There + is some re-arrangement of the files and adjustment of comments consequent + on this. + + 2. Part of the patch fixed a problem with the pcregrep tests. The test of -r + for recursive directory scanning broke on some systems because the files + are not scanned in any specific order and on different systems the order + was different. A call to "sort" has been inserted into RunGrepTest for the + approprate test as a short-term fix. In the longer term there may be an + alternative. + + 3. I had an email from Eric Raymond about problems translating some of PCRE's + man pages to HTML (despite the fact that I distribute HTML pages, some + people do their own conversions for various reasons). The problems + concerned the use of low-level troff macros .br and .in. I have therefore + removed all such uses from the man pages (some were redundant, some could + be replaced by .nf/.fi pairs). The 132html script that I use to generate + HTML has been updated to handle .nf/.fi and to complain if it encounters + .br or .in. + + 4. Updated comments in configure.ac that get placed in config.h.in and also + arranged for config.h to be included in the distribution, with the name + config.h.generic, for the benefit of those who have to compile without + Autotools (compare pcre.h, which is now distributed as pcre.h.generic). + + 5. Updated the support (such as it is) for Virtual Pascal, thanks to Stefan + Weber: (1) pcre_internal.h was missing some function renames; (2) updated + makevp.bat for the current PCRE, using the additional files + makevp_c.txt, makevp_l.txt, and pcregexp.pas. + + 6. A Windows user reported a minor discrepancy with test 2, which turned out + to be caused by a trailing space on an input line that had got lost in his + copy. The trailing space was an accident, so I've just removed it. + + 7. Add -Wl,-R... flags in pcre-config.in for *BSD* systems, as I'm told + that is needed. + + 8. Mark ucp_table (in ucptable.h) and ucp_gentype (in pcre_ucp_searchfuncs.c) + as "const" (a) because they are and (b) because it helps the PHP + maintainers who have recently made a script to detect big data structures + in the php code that should be moved to the .rodata section. I remembered + to update Builducptable as well, so it won't revert if ucptable.h is ever + re-created. + + 9. Added some extra #ifdef SUPPORT_UTF8 conditionals into pcretest.c, + pcre_printint.src, pcre_compile.c, pcre_study.c, and pcre_tables.c, in + order to be able to cut out the UTF-8 tables in the latter when UTF-8 + support is not required. This saves 1.5-2K of code, which is important in + some applications. + + Later: more #ifdefs are needed in pcre_ord2utf8.c and pcre_valid_utf8.c + so as not to refer to the tables, even though these functions will never be + called when UTF-8 support is disabled. Otherwise there are problems with a + shared library. + +10. Fixed two bugs in the emulated memmove() function in pcre_internal.h: + + (a) It was defining its arguments as char * instead of void *. + + (b) It was assuming that all moves were upwards in memory; this was true + a long time ago when I wrote it, but is no longer the case. + + The emulated memove() is provided for those environments that have neither + memmove() nor bcopy(). I didn't think anyone used it these days, but that + is clearly not the case, as these two bugs were recently reported. + +11. The script PrepareRelease is now distributed: it calls 132html, CleanTxt, + and Detrail to create the HTML documentation, the .txt form of the man + pages, and it removes trailing spaces from listed files. It also creates + pcre.h.generic and config.h.generic from pcre.h and config.h. In the latter + case, it wraps all the #defines with #ifndefs. This script should be run + before "make dist". + +12. Fixed two fairly obscure bugs concerned with quantified caseless matching + with Unicode property support. + + (a) For a maximizing quantifier, if the two different cases of the + character were of different lengths in their UTF-8 codings (there are + some cases like this - I found 11), and the matching function had to + back up over a mixture of the two cases, it incorrectly assumed they + were both the same length. + + (b) When PCRE was configured to use the heap rather than the stack for + recursion during matching, it was not correctly preserving the data for + the other case of a UTF-8 character when checking ahead for a match + while processing a minimizing repeat. If the check also involved + matching a wide character, but failed, corruption could cause an + erroneous result when trying to check for a repeat of the original + character. + +13. Some tidying changes to the testing mechanism: + + (a) The RunTest script now detects the internal link size and whether there + is UTF-8 and UCP support by running ./pcretest -C instead of relying on + values substituted by "configure". (The RunGrepTest script already did + this for UTF-8.) The configure.ac script no longer substitutes the + relevant variables. + + (b) The debugging options /B and /D in pcretest show the compiled bytecode + with length and offset values. This means that the output is different + for different internal link sizes. Test 2 is skipped for link sizes + other than 2 because of this, bypassing the problem. Unfortunately, + there was also a test in test 3 (the locale tests) that used /B and + failed for link sizes other than 2. Rather than cut the whole test out, + I have added a new /Z option to pcretest that replaces the length and + offset values with spaces. This is now used to make test 3 independent + of link size. (Test 2 will be tidied up later.) + +14. If erroroffset was passed as NULL to pcre_compile, it provoked a + segmentation fault instead of returning the appropriate error message. + +15. In multiline mode when the newline sequence was set to "any", the pattern + ^$ would give a match between the \r and \n of a subject such as "A\r\nB". + This doesn't seem right; it now treats the CRLF combination as the line + ending, and so does not match in that case. It's only a pattern such as ^$ + that would hit this one: something like ^ABC$ would have failed after \r + and then tried again after \r\n. + +16. Changed the comparison command for RunGrepTest from "diff -u" to "diff -ub" + in an attempt to make files that differ only in their line terminators + compare equal. This works on Linux. + +17. Under certain error circumstances pcregrep might try to free random memory + as it exited. This is now fixed, thanks to valgrind. + +19. In pcretest, if the pattern /(?m)^$/g was matched against the string + "abc\r\n\r\n", it found an unwanted second match after the second \r. This + was because its rules for how to advance for /g after matching an empty + string at the end of a line did not allow for this case. They now check for + it specially. + +20. pcretest is supposed to handle patterns and data of any length, by + extending its buffers when necessary. It was getting this wrong when the + buffer for a data line had to be extended. + +21. Added PCRE_NEWLINE_ANYCRLF which is like ANY, but matches only CR, LF, or + CRLF as a newline sequence. + +22. Code for handling Unicode properties in pcre_dfa_exec() wasn't being cut + out by #ifdef SUPPORT_UCP. This did no harm, as it could never be used, but + I have nevertheless tidied it up. + +23. Added some casts to kill warnings from HP-UX ia64 compiler. + +24. Added a man page for pcre-config. + + +Version 7.0 19-Dec-06 +--------------------- + + 1. Fixed a signed/unsigned compiler warning in pcre_compile.c, shown up by + moving to gcc 4.1.1. + + 2. The -S option for pcretest uses setrlimit(); I had omitted to #include + sys/time.h, which is documented as needed for this function. It doesn't + seem to matter on Linux, but it showed up on some releases of OS X. + + 3. It seems that there are systems where bytes whose values are greater than + 127 match isprint() in the "C" locale. The "C" locale should be the + default when a C program starts up. In most systems, only ASCII printing + characters match isprint(). This difference caused the output from pcretest + to vary, making some of the tests fail. I have changed pcretest so that: + + (a) When it is outputting text in the compiled version of a pattern, bytes + other than 32-126 are always shown as hex escapes. + + (b) When it is outputting text that is a matched part of a subject string, + it does the same, unless a different locale has been set for the match + (using the /L modifier). In this case, it uses isprint() to decide. + + 4. Fixed a major bug that caused incorrect computation of the amount of memory + required for a compiled pattern when options that changed within the + pattern affected the logic of the preliminary scan that determines the + length. The relevant options are -x, and -i in UTF-8 mode. The result was + that the computed length was too small. The symptoms of this bug were + either the PCRE error "internal error: code overflow" from pcre_compile(), + or a glibc crash with a message such as "pcretest: free(): invalid next + size (fast)". Examples of patterns that provoked this bug (shown in + pcretest format) are: + + /(?-x: )/x + /(?x)(?-x: \s*#\s*)/ + /((?i)[\x{c0}])/8 + /(?i:[\x{c0}])/8 + + HOWEVER: Change 17 below makes this fix obsolete as the memory computation + is now done differently. + + 5. Applied patches from Google to: (a) add a QuoteMeta function to the C++ + wrapper classes; (b) implement a new function in the C++ scanner that is + more efficient than the old way of doing things because it avoids levels of + recursion in the regex matching; (c) add a paragraph to the documentation + for the FullMatch() function. + + 6. The escape sequence \n was being treated as whatever was defined as + "newline". Not only was this contrary to the documentation, which states + that \n is character 10 (hex 0A), but it also went horribly wrong when + "newline" was defined as CRLF. This has been fixed. + + 7. In pcre_dfa_exec.c the value of an unsigned integer (the variable called c) + was being set to -1 for the "end of line" case (supposedly a value that no + character can have). Though this value is never used (the check for end of + line is "zero bytes in current character"), it caused compiler complaints. + I've changed it to 0xffffffff. + + 8. In pcre_version.c, the version string was being built by a sequence of + C macros that, in the event of PCRE_PRERELEASE being defined as an empty + string (as it is for production releases) called a macro with an empty + argument. The C standard says the result of this is undefined. The gcc + compiler treats it as an empty string (which was what was wanted) but it is + reported that Visual C gives an error. The source has been hacked around to + avoid this problem. + + 9. On the advice of a Windows user, included and in Windows + builds of pcretest, and changed the call to _setmode() to use _O_BINARY + instead of 0x8000. Made all the #ifdefs test both _WIN32 and WIN32 (not all + of them did). + +10. Originally, pcretest opened its input and output without "b"; then I was + told that "b" was needed in some environments, so it was added for release + 5.0 to both the input and output. (It makes no difference on Unix-like + systems.) Later I was told that it is wrong for the input on Windows. I've + now abstracted the modes into two macros, to make it easier to fiddle with + them, and removed "b" from the input mode under Windows. + +11. Added pkgconfig support for the C++ wrapper library, libpcrecpp. + +12. Added -help and --help to pcretest as an official way of being reminded + of the options. + +13. Removed some redundant semicolons after macro calls in pcrecpparg.h.in + and pcrecpp.cc because they annoy compilers at high warning levels. + +14. A bit of tidying/refactoring in pcre_exec.c in the main bumpalong loop. + +15. Fixed an occurrence of == in configure.ac that should have been = (shell + scripts are not C programs :-) and which was not noticed because it works + on Linux. + +16. pcretest is supposed to handle any length of pattern and data line (as one + line or as a continued sequence of lines) by extending its input buffer if + necessary. This feature was broken for very long pattern lines, leading to + a string of junk being passed to pcre_compile() if the pattern was longer + than about 50K. + +17. I have done a major re-factoring of the way pcre_compile() computes the + amount of memory needed for a compiled pattern. Previously, there was code + that made a preliminary scan of the pattern in order to do this. That was + OK when PCRE was new, but as the facilities have expanded, it has become + harder and harder to keep it in step with the real compile phase, and there + have been a number of bugs (see for example, 4 above). I have now found a + cunning way of running the real compile function in a "fake" mode that + enables it to compute how much memory it would need, while actually only + ever using a few hundred bytes of working memory and without too many + tests of the mode. This should make future maintenance and development + easier. A side effect of this work is that the limit of 200 on the nesting + depth of parentheses has been removed (though this was never a serious + limitation, I suspect). However, there is a downside: pcre_compile() now + runs more slowly than before (30% or more, depending on the pattern). I + hope this isn't a big issue. There is no effect on runtime performance. + +18. Fixed a minor bug in pcretest: if a pattern line was not terminated by a + newline (only possible for the last line of a file) and it was a + pattern that set a locale (followed by /Lsomething), pcretest crashed. + +19. Added additional timing features to pcretest. (1) The -tm option now times + matching only, not compiling. (2) Both -t and -tm can be followed, as a + separate command line item, by a number that specifies the number of + repeats to use when timing. The default is 50000; this gives better + precision, but takes uncomfortably long for very large patterns. + +20. Extended pcre_study() to be more clever in cases where a branch of a + subpattern has no definite first character. For example, (a*|b*)[cd] would + previously give no result from pcre_study(). Now it recognizes that the + first character must be a, b, c, or d. + +21. There was an incorrect error "recursive call could loop indefinitely" if + a subpattern (or the entire pattern) that was being tested for matching an + empty string contained only one non-empty item after a nested subpattern. + For example, the pattern (?>\x{100}*)\d(?R) provoked this error + incorrectly, because the \d was being skipped in the check. + +22. The pcretest program now has a new pattern option /B and a command line + option -b, which is equivalent to adding /B to every pattern. This causes + it to show the compiled bytecode, without the additional information that + -d shows. The effect of -d is now the same as -b with -i (and similarly, /D + is the same as /B/I). + +23. A new optimization is now able automatically to treat some sequences such + as a*b as a*+b. More specifically, if something simple (such as a character + or a simple class like \d) has an unlimited quantifier, and is followed by + something that cannot possibly match the quantified thing, the quantifier + is automatically "possessified". + +24. A recursive reference to a subpattern whose number was greater than 39 + went wrong under certain circumstances in UTF-8 mode. This bug could also + have affected the operation of pcre_study(). + +25. Realized that a little bit of performance could be had by replacing + (c & 0xc0) == 0xc0 with c >= 0xc0 when processing UTF-8 characters. + +26. Timing data from pcretest is now shown to 4 decimal places instead of 3. + +27. Possessive quantifiers such as a++ were previously implemented by turning + them into atomic groups such as ($>a+). Now they have their own opcodes, + which improves performance. This includes the automatically created ones + from 23 above. + +28. A pattern such as (?=(\w+))\1: which simulates an atomic group using a + lookahead was broken if it was not anchored. PCRE was mistakenly expecting + the first matched character to be a colon. This applied both to named and + numbered groups. + +29. The ucpinternal.h header file was missing its idempotency #ifdef. + +30. I was sent a "project" file called libpcre.a.dev which I understand makes + building PCRE on Windows easier, so I have included it in the distribution. + +31. There is now a check in pcretest against a ridiculously large number being + returned by pcre_exec() or pcre_dfa_exec(). If this happens in a /g or /G + loop, the loop is abandoned. + +32. Forward references to subpatterns in conditions such as (?(2)...) where + subpattern 2 is defined later cause pcre_compile() to search forwards in + the pattern for the relevant set of parentheses. This search went wrong + when there were unescaped parentheses in a character class, parentheses + escaped with \Q...\E, or parentheses in a #-comment in /x mode. + +33. "Subroutine" calls and backreferences were previously restricted to + referencing subpatterns earlier in the regex. This restriction has now + been removed. + +34. Added a number of extra features that are going to be in Perl 5.10. On the + whole, these are just syntactic alternatives for features that PCRE had + previously implemented using the Python syntax or my own invention. The + other formats are all retained for compatibility. + + (a) Named groups can now be defined as (?...) or (?'name'...) as well + as (?P...). The new forms, as well as being in Perl 5.10, are + also .NET compatible. + + (b) A recursion or subroutine call to a named group can now be defined as + (?&name) as well as (?P>name). + + (c) A backreference to a named group can now be defined as \k or + \k'name' as well as (?P=name). The new forms, as well as being in Perl + 5.10, are also .NET compatible. + + (d) A conditional reference to a named group can now use the syntax + (?() or (?('name') as well as (?(name). + + (e) A "conditional group" of the form (?(DEFINE)...) can be used to define + groups (named and numbered) that are never evaluated inline, but can be + called as "subroutines" from elsewhere. In effect, the DEFINE condition + is always false. There may be only one alternative in such a group. + + (f) A test for recursion can be given as (?(R1).. or (?(R&name)... as well + as the simple (?(R). The condition is true only if the most recent + recursion is that of the given number or name. It does not search out + through the entire recursion stack. + + (g) The escape \gN or \g{N} has been added, where N is a positive or + negative number, specifying an absolute or relative reference. + +35. Tidied to get rid of some further signed/unsigned compiler warnings and + some "unreachable code" warnings. + +36. Updated the Unicode property tables to Unicode version 5.0.0. Amongst other + things, this adds five new scripts. + +37. Perl ignores orphaned \E escapes completely. PCRE now does the same. + There were also incompatibilities regarding the handling of \Q..\E inside + character classes, for example with patterns like [\Qa\E-\Qz\E] where the + hyphen was adjacent to \Q or \E. I hope I've cleared all this up now. + +38. Like Perl, PCRE detects when an indefinitely repeated parenthesized group + matches an empty string, and forcibly breaks the loop. There were bugs in + this code in non-simple cases. For a pattern such as ^(a()*)* matched + against aaaa the result was just "a" rather than "aaaa", for example. Two + separate and independent bugs (that affected different cases) have been + fixed. + +39. Refactored the code to abolish the use of different opcodes for small + capturing bracket numbers. This is a tidy that I avoided doing when I + removed the limit on the number of capturing brackets for 3.5 back in 2001. + The new approach is not only tidier, it makes it possible to reduce the + memory needed to fix the previous bug (38). + +40. Implemented PCRE_NEWLINE_ANY to recognize any of the Unicode newline + sequences (http://unicode.org/unicode/reports/tr18/) as "newline" when + processing dot, circumflex, or dollar metacharacters, or #-comments in /x + mode. + +41. Add \R to match any Unicode newline sequence, as suggested in the Unicode + report. + +42. Applied patch, originally from Ari Pollak, modified by Google, to allow + copy construction and assignment in the C++ wrapper. + +43. Updated pcregrep to support "--newline=any". In the process, I fixed a + couple of bugs that could have given wrong results in the "--newline=crlf" + case. + +44. Added a number of casts and did some reorganization of signed/unsigned int + variables following suggestions from Dair Grant. Also renamed the variable + "this" as "item" because it is a C++ keyword. + +45. Arranged for dftables to add + + #include "pcre_internal.h" + + to pcre_chartables.c because without it, gcc 4.x may remove the array + definition from the final binary if PCRE is built into a static library and + dead code stripping is activated. + +46. For an unanchored pattern, if a match attempt fails at the start of a + newline sequence, and the newline setting is CRLF or ANY, and the next two + characters are CRLF, advance by two characters instead of one. + + +Version 6.7 04-Jul-06 +--------------------- + + 1. In order to handle tests when input lines are enormously long, pcretest has + been re-factored so that it automatically extends its buffers when + necessary. The code is crude, but this _is_ just a test program. The + default size has been increased from 32K to 50K. + + 2. The code in pcre_study() was using the value of the re argument before + testing it for NULL. (Of course, in any sensible call of the function, it + won't be NULL.) + + 3. The memmove() emulation function in pcre_internal.h, which is used on + systems that lack both memmove() and bcopy() - that is, hardly ever - + was missing a "static" storage class specifier. + + 4. When UTF-8 mode was not set, PCRE looped when compiling certain patterns + containing an extended class (one that cannot be represented by a bitmap + because it contains high-valued characters or Unicode property items, e.g. + [\pZ]). Almost always one would set UTF-8 mode when processing such a + pattern, but PCRE should not loop if you do not (it no longer does). + [Detail: two cases were found: (a) a repeated subpattern containing an + extended class; (b) a recursive reference to a subpattern that followed a + previous extended class. It wasn't skipping over the extended class + correctly when UTF-8 mode was not set.] + + 5. A negated single-character class was not being recognized as fixed-length + in lookbehind assertions such as (?<=[^f]), leading to an incorrect + compile error "lookbehind assertion is not fixed length". + + 6. The RunPerlTest auxiliary script was showing an unexpected difference + between PCRE and Perl for UTF-8 tests. It turns out that it is hard to + write a Perl script that can interpret lines of an input file either as + byte characters or as UTF-8, which is what "perltest" was being required to + do for the non-UTF-8 and UTF-8 tests, respectively. Essentially what you + can't do is switch easily at run time between having the "use utf8;" pragma + or not. In the end, I fudged it by using the RunPerlTest script to insert + "use utf8;" explicitly for the UTF-8 tests. + + 7. In multiline (/m) mode, PCRE was matching ^ after a terminating newline at + the end of the subject string, contrary to the documentation and to what + Perl does. This was true of both matching functions. Now it matches only at + the start of the subject and immediately after *internal* newlines. + + 8. A call of pcre_fullinfo() from pcretest to get the option bits was passing + a pointer to an int instead of a pointer to an unsigned long int. This + caused problems on 64-bit systems. + + 9. Applied a patch from the folks at Google to pcrecpp.cc, to fix "another + instance of the 'standard' template library not being so standard". + +10. There was no check on the number of named subpatterns nor the maximum + length of a subpattern name. The product of these values is used to compute + the size of the memory block for a compiled pattern. By supplying a very + long subpattern name and a large number of named subpatterns, the size + computation could be caused to overflow. This is now prevented by limiting + the length of names to 32 characters, and the number of named subpatterns + to 10,000. + +11. Subpatterns that are repeated with specific counts have to be replicated in + the compiled pattern. The size of memory for this was computed from the + length of the subpattern and the repeat count. The latter is limited to + 65535, but there was no limit on the former, meaning that integer overflow + could in principle occur. The compiled length of a repeated subpattern is + now limited to 30,000 bytes in order to prevent this. + +12. Added the optional facility to have named substrings with the same name. + +13. Added the ability to use a named substring as a condition, using the + Python syntax: (?(name)yes|no). This overloads (?(R)... and names that + are numbers (not recommended). Forward references are permitted. + +14. Added forward references in named backreferences (if you see what I mean). + +15. In UTF-8 mode, with the PCRE_DOTALL option set, a quantified dot in the + pattern could run off the end of the subject. For example, the pattern + "(?s)(.{1,5})"8 did this with the subject "ab". + +16. If PCRE_DOTALL or PCRE_MULTILINE were set, pcre_dfa_exec() behaved as if + PCRE_CASELESS was set when matching characters that were quantified with ? + or *. + +17. A character class other than a single negated character that had a minimum + but no maximum quantifier - for example [ab]{6,} - was not handled + correctly by pce_dfa_exec(). It would match only one character. + +18. A valid (though odd) pattern that looked like a POSIX character + class but used an invalid character after [ (for example [[,abc,]]) caused + pcre_compile() to give the error "Failed: internal error: code overflow" or + in some cases to crash with a glibc free() error. This could even happen if + the pattern terminated after [[ but there just happened to be a sequence of + letters, a binary zero, and a closing ] in the memory that followed. + +19. Perl's treatment of octal escapes in the range \400 to \777 has changed + over the years. Originally (before any Unicode support), just the bottom 8 + bits were taken. Thus, for example, \500 really meant \100. Nowadays the + output from "man perlunicode" includes this: + + The regular expression compiler produces polymorphic opcodes. That + is, the pattern adapts to the data and automatically switches to + the Unicode character scheme when presented with Unicode data--or + instead uses a traditional byte scheme when presented with byte + data. + + Sadly, a wide octal escape does not cause a switch, and in a string with + no other multibyte characters, these octal escapes are treated as before. + Thus, in Perl, the pattern /\500/ actually matches \100 but the pattern + /\500|\x{1ff}/ matches \500 or \777 because the whole thing is treated as a + Unicode string. + + I have not perpetrated such confusion in PCRE. Up till now, it took just + the bottom 8 bits, as in old Perl. I have now made octal escapes with + values greater than \377 illegal in non-UTF-8 mode. In UTF-8 mode they + translate to the appropriate multibyte character. + +29. Applied some refactoring to reduce the number of warnings from Microsoft + and Borland compilers. This has included removing the fudge introduced + seven years ago for the OS/2 compiler (see 2.02/2 below) because it caused + a warning about an unused variable. + +21. PCRE has not included VT (character 0x0b) in the set of whitespace + characters since release 4.0, because Perl (from release 5.004) does not. + [Or at least, is documented not to: some releases seem to be in conflict + with the documentation.] However, when a pattern was studied with + pcre_study() and all its branches started with \s, PCRE still included VT + as a possible starting character. Of course, this did no harm; it just + caused an unnecessary match attempt. + +22. Removed a now-redundant internal flag bit that recorded the fact that case + dependency changed within the pattern. This was once needed for "required + byte" processing, but is no longer used. This recovers a now-scarce options + bit. Also moved the least significant internal flag bit to the most- + significant bit of the word, which was not previously used (hangover from + the days when it was an int rather than a uint) to free up another bit for + the future. + +23. Added support for CRLF line endings as well as CR and LF. As well as the + default being selectable at build time, it can now be changed at runtime + via the PCRE_NEWLINE_xxx flags. There are now options for pcregrep to + specify that it is scanning data with non-default line endings. + +24. Changed the definition of CXXLINK to make it agree with the definition of + LINK in the Makefile, by replacing LDFLAGS to CXXFLAGS. + +25. Applied Ian Taylor's patches to avoid using another stack frame for tail + recursions. This makes a big different to stack usage for some patterns. + +26. If a subpattern containing a named recursion or subroutine reference such + as (?P>B) was quantified, for example (xxx(?P>B)){3}, the calculation of + the space required for the compiled pattern went wrong and gave too small a + value. Depending on the environment, this could lead to "Failed: internal + error: code overflow at offset 49" or "glibc detected double free or + corruption" errors. + +27. Applied patches from Google (a) to support the new newline modes and (b) to + advance over multibyte UTF-8 characters in GlobalReplace. + +28. Change free() to pcre_free() in pcredemo.c. Apparently this makes a + difference for some implementation of PCRE in some Windows version. + +29. Added some extra testing facilities to pcretest: + + \q in a data line sets the "match limit" value + \Q in a data line sets the "match recursion limt" value + -S sets the stack size, where is in megabytes + + The -S option isn't available for Windows. + + +Version 6.6 06-Feb-06 +--------------------- + + 1. Change 16(a) for 6.5 broke things, because PCRE_DATA_SCOPE was not defined + in pcreposix.h. I have copied the definition from pcre.h. + + 2. Change 25 for 6.5 broke compilation in a build directory out-of-tree + because pcre.h is no longer a built file. + + 3. Added Jeff Friedl's additional debugging patches to pcregrep. These are + not normally included in the compiled code. + + +Version 6.5 01-Feb-06 +--------------------- + + 1. When using the partial match feature with pcre_dfa_exec(), it was not + anchoring the second and subsequent partial matches at the new starting + point. This could lead to incorrect results. For example, with the pattern + /1234/, partially matching against "123" and then "a4" gave a match. + + 2. Changes to pcregrep: + + (a) All non-match returns from pcre_exec() were being treated as failures + to match the line. Now, unless the error is PCRE_ERROR_NOMATCH, an + error message is output. Some extra information is given for the + PCRE_ERROR_MATCHLIMIT and PCRE_ERROR_RECURSIONLIMIT errors, which are + probably the only errors that are likely to be caused by users (by + specifying a regex that has nested indefinite repeats, for instance). + If there are more than 20 of these errors, pcregrep is abandoned. + + (b) A binary zero was treated as data while matching, but terminated the + output line if it was written out. This has been fixed: binary zeroes + are now no different to any other data bytes. + + (c) Whichever of the LC_ALL or LC_CTYPE environment variables is set is + used to set a locale for matching. The --locale=xxxx long option has + been added (no short equivalent) to specify a locale explicitly on the + pcregrep command, overriding the environment variables. + + (d) When -B was used with -n, some line numbers in the output were one less + than they should have been. + + (e) Added the -o (--only-matching) option. + + (f) If -A or -C was used with -c (count only), some lines of context were + accidentally printed for the final match. + + (g) Added the -H (--with-filename) option. + + (h) The combination of options -rh failed to suppress file names for files + that were found from directory arguments. + + (i) Added the -D (--devices) and -d (--directories) options. + + (j) Added the -F (--fixed-strings) option. + + (k) Allow "-" to be used as a file name for -f as well as for a data file. + + (l) Added the --colo(u)r option. + + (m) Added Jeffrey Friedl's -S testing option, but within #ifdefs so that it + is not present by default. + + 3. A nasty bug was discovered in the handling of recursive patterns, that is, + items such as (?R) or (?1), when the recursion could match a number of + alternatives. If it matched one of the alternatives, but subsequently, + outside the recursion, there was a failure, the code tried to back up into + the recursion. However, because of the way PCRE is implemented, this is not + possible, and the result was an incorrect result from the match. + + In order to prevent this happening, the specification of recursion has + been changed so that all such subpatterns are automatically treated as + atomic groups. Thus, for example, (?R) is treated as if it were (?>(?R)). + + 4. I had overlooked the fact that, in some locales, there are characters for + which isalpha() is true but neither isupper() nor islower() are true. In + the fr_FR locale, for instance, the \xAA and \xBA characters (ordmasculine + and ordfeminine) are like this. This affected the treatment of \w and \W + when they appeared in character classes, but not when they appeared outside + a character class. The bit map for "word" characters is now created + separately from the results of isalnum() instead of just taking it from the + upper, lower, and digit maps. (Plus the underscore character, of course.) + + 5. The above bug also affected the handling of POSIX character classes such as + [[:alpha:]] and [[:alnum:]]. These do not have their own bit maps in PCRE's + permanent tables. Instead, the bit maps for such a class were previously + created as the appropriate unions of the upper, lower, and digit bitmaps. + Now they are created by subtraction from the [[:word:]] class, which has + its own bitmap. + + 6. The [[:blank:]] character class matches horizontal, but not vertical space. + It is created by subtracting the vertical space characters (\x09, \x0a, + \x0b, \x0c) from the [[:space:]] bitmap. Previously, however, the + subtraction was done in the overall bitmap for a character class, meaning + that a class such as [\x0c[:blank:]] was incorrect because \x0c would not + be recognized. This bug has been fixed. + + 7. Patches from the folks at Google: + + (a) pcrecpp.cc: "to handle a corner case that may or may not happen in + real life, but is still worth protecting against". + + (b) pcrecpp.cc: "corrects a bug when negative radixes are used with + regular expressions". + + (c) pcre_scanner.cc: avoid use of std::count() because not all systems + have it. + + (d) Split off pcrecpparg.h from pcrecpp.h and had the former built by + "configure" and the latter not, in order to fix a problem somebody had + with compiling the Arg class on HP-UX. + + (e) Improve the error-handling of the C++ wrapper a little bit. + + (f) New tests for checking recursion limiting. + + 8. The pcre_memmove() function, which is used only if the environment does not + have a standard memmove() function (and is therefore rarely compiled), + contained two bugs: (a) use of int instead of size_t, and (b) it was not + returning a result (though PCRE never actually uses the result). + + 9. In the POSIX regexec() interface, if nmatch is specified as a ridiculously + large number - greater than INT_MAX/(3*sizeof(int)) - REG_ESPACE is + returned instead of calling malloc() with an overflowing number that would + most likely cause subsequent chaos. + +10. The debugging option of pcretest was not showing the NO_AUTO_CAPTURE flag. + +11. The POSIX flag REG_NOSUB is now supported. When a pattern that was compiled + with this option is matched, the nmatch and pmatch options of regexec() are + ignored. + +12. Added REG_UTF8 to the POSIX interface. This is not defined by POSIX, but is + provided in case anyone wants to the the POSIX interface with UTF-8 + strings. + +13. Added CXXLDFLAGS to the Makefile parameters to provide settings only on the + C++ linking (needed for some HP-UX environments). + +14. Avoid compiler warnings in get_ucpname() when compiled without UCP support + (unused parameter) and in the pcre_printint() function (omitted "default" + switch label when the default is to do nothing). + +15. Added some code to make it possible, when PCRE is compiled as a C++ + library, to replace subject pointers for pcre_exec() with a smart pointer + class, thus making it possible to process discontinuous strings. + +16. The two macros PCRE_EXPORT and PCRE_DATA_SCOPE are confusing, and perform + much the same function. They were added by different people who were trying + to make PCRE easy to compile on non-Unix systems. It has been suggested + that PCRE_EXPORT be abolished now that there is more automatic apparatus + for compiling on Windows systems. I have therefore replaced it with + PCRE_DATA_SCOPE. This is set automatically for Windows; if not set it + defaults to "extern" for C or "extern C" for C++, which works fine on + Unix-like systems. It is now possible to override the value of PCRE_DATA_ + SCOPE with something explicit in config.h. In addition: + + (a) pcreposix.h still had just "extern" instead of either of these macros; + I have replaced it with PCRE_DATA_SCOPE. + + (b) Functions such as _pcre_xclass(), which are internal to the library, + but external in the C sense, all had PCRE_EXPORT in their definitions. + This is apparently wrong for the Windows case, so I have removed it. + (It makes no difference on Unix-like systems.) + +17. Added a new limit, MATCH_LIMIT_RECURSION, which limits the depth of nesting + of recursive calls to match(). This is different to MATCH_LIMIT because + that limits the total number of calls to match(), not all of which increase + the depth of recursion. Limiting the recursion depth limits the amount of + stack (or heap if NO_RECURSE is set) that is used. The default can be set + when PCRE is compiled, and changed at run time. A patch from Google adds + this functionality to the C++ interface. + +18. Changes to the handling of Unicode character properties: + + (a) Updated the table to Unicode 4.1.0. + + (b) Recognize characters that are not in the table as "Cn" (undefined). + + (c) I revised the way the table is implemented to a much improved format + which includes recognition of ranges. It now supports the ranges that + are defined in UnicodeData.txt, and it also amalgamates other + characters into ranges. This has reduced the number of entries in the + table from around 16,000 to around 3,000, thus reducing its size + considerably. I realized I did not need to use a tree structure after + all - a binary chop search is just as efficient. Having reduced the + number of entries, I extended their size from 6 bytes to 8 bytes to + allow for more data. + + (d) Added support for Unicode script names via properties such as \p{Han}. + +19. In UTF-8 mode, a backslash followed by a non-Ascii character was not + matching that character. + +20. When matching a repeated Unicode property with a minimum greater than zero, + (for example \pL{2,}), PCRE could look past the end of the subject if it + reached it while seeking the minimum number of characters. This could + happen only if some of the characters were more than one byte long, because + there is a check for at least the minimum number of bytes. + +21. Refactored the implementation of \p and \P so as to be more general, to + allow for more different types of property in future. This has changed the + compiled form incompatibly. Anybody with saved compiled patterns that use + \p or \P will have to recompile them. + +22. Added "Any" and "L&" to the supported property types. + +23. Recognize \x{...} as a code point specifier, even when not in UTF-8 mode, + but give a compile time error if the value is greater than 0xff. + +24. The man pages for pcrepartial, pcreprecompile, and pcre_compile2 were + accidentally not being installed or uninstalled. + +25. The pcre.h file was built from pcre.h.in, but the only changes that were + made were to insert the current release number. This seemed silly, because + it made things harder for people building PCRE on systems that don't run + "configure". I have turned pcre.h into a distributed file, no longer built + by "configure", with the version identification directly included. There is + no longer a pcre.h.in file. + + However, this change necessitated a change to the pcre-config script as + well. It is built from pcre-config.in, and one of the substitutions was the + release number. I have updated configure.ac so that ./configure now finds + the release number by grepping pcre.h. + +26. Added the ability to run the tests under valgrind. + + +Version 6.4 05-Sep-05 +--------------------- + + 1. Change 6.0/10/(l) to pcregrep introduced a bug that caused separator lines + "--" to be printed when multiple files were scanned, even when none of the + -A, -B, or -C options were used. This is not compatible with Gnu grep, so I + consider it to be a bug, and have restored the previous behaviour. + + 2. A couple of code tidies to get rid of compiler warnings. + + 3. The pcretest program used to cheat by referring to symbols in the library + whose names begin with _pcre_. These are internal symbols that are not + really supposed to be visible externally, and in some environments it is + possible to suppress them. The cheating is now confined to including + certain files from the library's source, which is a bit cleaner. + + 4. Renamed pcre.in as pcre.h.in to go with pcrecpp.h.in; it also makes the + file's purpose clearer. + + 5. Reorganized pcre_ucp_findchar(). + + +Version 6.3 15-Aug-05 +--------------------- + + 1. The file libpcre.pc.in did not have general read permission in the tarball. + + 2. There were some problems when building without C++ support: + + (a) If C++ support was not built, "make install" and "make test" still + tried to test it. + + (b) There were problems when the value of CXX was explicitly set. Some + changes have been made to try to fix these, and ... + + (c) --disable-cpp can now be used to explicitly disable C++ support. + + (d) The use of @CPP_OBJ@ directly caused a blank line preceded by a + backslash in a target when C++ was disabled. This confuses some + versions of "make", apparently. Using an intermediate variable solves + this. (Same for CPP_LOBJ.) + + 3. $(LINK_FOR_BUILD) now includes $(CFLAGS_FOR_BUILD) and $(LINK) + (non-Windows) now includes $(CFLAGS) because these flags are sometimes + necessary on certain architectures. + + 4. Added a setting of -export-symbols-regex to the link command to remove + those symbols that are exported in the C sense, but actually are local + within the library, and not documented. Their names all begin with + "_pcre_". This is not a perfect job, because (a) we have to except some + symbols that pcretest ("illegally") uses, and (b) the facility isn't always + available (and never for static libraries). I have made a note to try to + find a way round (a) in the future. + + +Version 6.2 01-Aug-05 +--------------------- + + 1. There was no test for integer overflow of quantifier values. A construction + such as {1111111111111111} would give undefined results. What is worse, if + a minimum quantifier for a parenthesized subpattern overflowed and became + negative, the calculation of the memory size went wrong. This could have + led to memory overwriting. + + 2. Building PCRE using VPATH was broken. Hopefully it is now fixed. + + 3. Added "b" to the 2nd argument of fopen() in dftables.c, for non-Unix-like + operating environments where this matters. + + 4. Applied Giuseppe Maxia's patch to add additional features for controlling + PCRE options from within the C++ wrapper. + + 5. Named capturing subpatterns were not being correctly counted when a pattern + was compiled. This caused two problems: (a) If there were more than 100 + such subpatterns, the calculation of the memory needed for the whole + compiled pattern went wrong, leading to an overflow error. (b) Numerical + back references of the form \12, where the number was greater than 9, were + not recognized as back references, even though there were sufficient + previous subpatterns. + + 6. Two minor patches to pcrecpp.cc in order to allow it to compile on older + versions of gcc, e.g. 2.95.4. + + +Version 6.1 21-Jun-05 +--------------------- + + 1. There was one reference to the variable "posix" in pcretest.c that was not + surrounded by "#if !defined NOPOSIX". + + 2. Make it possible to compile pcretest without DFA support, UTF8 support, or + the cross-check on the old pcre_info() function, for the benefit of the + cut-down version of PCRE that is currently imported into Exim. + + 3. A (silly) pattern starting with (?i)(?-i) caused an internal space + allocation error. I've done the easy fix, which wastes 2 bytes for sensible + patterns that start (?i) but I don't think that matters. The use of (?i) is + just an example; this all applies to the other options as well. + + 4. Since libtool seems to echo the compile commands it is issuing, the output + from "make" can be reduced a bit by putting "@" in front of each libtool + compile command. + + 5. Patch from the folks at Google for configure.in to be a bit more thorough + in checking for a suitable C++ installation before trying to compile the + C++ stuff. This should fix a reported problem when a compiler was present, + but no suitable headers. + + 6. The man pages all had just "PCRE" as their title. I have changed them to + be the relevant file name. I have also arranged that these names are + retained in the file doc/pcre.txt, which is a concatenation in text format + of all the man pages except the little individual ones for each function. + + 7. The NON-UNIX-USE file had not been updated for the different set of source + files that come with release 6. I also added a few comments about the C++ + wrapper. + + +Version 6.0 07-Jun-05 +--------------------- + + 1. Some minor internal re-organization to help with my DFA experiments. + + 2. Some missing #ifdef SUPPORT_UCP conditionals in pcretest and printint that + didn't matter for the library itself when fully configured, but did matter + when compiling without UCP support, or within Exim, where the ucp files are + not imported. + + 3. Refactoring of the library code to split up the various functions into + different source modules. The addition of the new DFA matching code (see + below) to a single monolithic source would have made it really too + unwieldy, quite apart from causing all the code to be include in a + statically linked application, when only some functions are used. This is + relevant even without the DFA addition now that patterns can be compiled in + one application and matched in another. + + The downside of splitting up is that there have to be some external + functions and data tables that are used internally in different modules of + the library but which are not part of the API. These have all had their + names changed to start with "_pcre_" so that they are unlikely to clash + with other external names. + + 4. Added an alternate matching function, pcre_dfa_exec(), which matches using + a different (DFA) algorithm. Although it is slower than the original + function, it does have some advantages for certain types of matching + problem. + + 5. Upgrades to pcretest in order to test the features of pcre_dfa_exec(), + including restarting after a partial match. + + 6. A patch for pcregrep that defines INVALID_FILE_ATTRIBUTES if it is not + defined when compiling for Windows was sent to me. I have put it into the + code, though I have no means of testing or verifying it. + + 7. Added the pcre_refcount() auxiliary function. + + 8. Added the PCRE_FIRSTLINE option. This constrains an unanchored pattern to + match before or at the first newline in the subject string. In pcretest, + the /f option on a pattern can be used to set this. + + 9. A repeated \w when used in UTF-8 mode with characters greater than 256 + would behave wrongly. This has been present in PCRE since release 4.0. + +10. A number of changes to the pcregrep command: + + (a) Refactored how -x works; insert ^(...)$ instead of setting + PCRE_ANCHORED and checking the length, in preparation for adding + something similar for -w. + + (b) Added the -w (match as a word) option. + + (c) Refactored the way lines are read and buffered so as to have more + than one at a time available. + + (d) Implemented a pcregrep test script. + + (e) Added the -M (multiline match) option. This allows patterns to match + over several lines of the subject. The buffering ensures that at least + 8K, or the rest of the document (whichever is the shorter) is available + for matching (and similarly the previous 8K for lookbehind assertions). + + (f) Changed the --help output so that it now says + + -w, --word-regex(p) + + instead of two lines, one with "regex" and the other with "regexp" + because that confused at least one person since the short forms are the + same. (This required a bit of code, as the output is generated + automatically from a table. It wasn't just a text change.) + + (g) -- can be used to terminate pcregrep options if the next thing isn't an + option but starts with a hyphen. Could be a pattern or a path name + starting with a hyphen, for instance. + + (h) "-" can be given as a file name to represent stdin. + + (i) When file names are being printed, "(standard input)" is used for + the standard input, for compatibility with GNU grep. Previously + "" was used. + + (j) The option --label=xxx can be used to supply a name to be used for + stdin when file names are being printed. There is no short form. + + (k) Re-factored the options decoding logic because we are going to add + two more options that take data. Such options can now be given in four + different ways, e.g. "-fname", "-f name", "--file=name", "--file name". + + (l) Added the -A, -B, and -C options for requesting that lines of context + around matches be printed. + + (m) Added the -L option to print the names of files that do not contain + any matching lines, that is, the complement of -l. + + (n) The return code is 2 if any file cannot be opened, but pcregrep does + continue to scan other files. + + (o) The -s option was incorrectly implemented. For compatibility with other + greps, it now suppresses the error message for a non-existent or non- + accessible file (but not the return code). There is a new option called + -q that suppresses the output of matching lines, which was what -s was + previously doing. + + (p) Added --include and --exclude options to specify files for inclusion + and exclusion when recursing. + +11. The Makefile was not using the Autoconf-supported LDFLAGS macro properly. + Hopefully, it now does. + +12. Missing cast in pcre_study(). + +13. Added an "uninstall" target to the makefile. + +14. Replaced "extern" in the function prototypes in Makefile.in with + "PCRE_DATA_SCOPE", which defaults to 'extern' or 'extern "C"' in the Unix + world, but is set differently for Windows. + +15. Added a second compiling function called pcre_compile2(). The only + difference is that it has an extra argument, which is a pointer to an + integer error code. When there is a compile-time failure, this is set + non-zero, in addition to the error test pointer being set to point to an + error message. The new argument may be NULL if no error number is required + (but then you may as well call pcre_compile(), which is now just a + wrapper). This facility is provided because some applications need a + numeric error indication, but it has also enabled me to tidy up the way + compile-time errors are handled in the POSIX wrapper. + +16. Added VPATH=.libs to the makefile; this should help when building with one + prefix path and installing with another. (Or so I'm told by someone who + knows more about this stuff than I do.) + +17. Added a new option, REG_DOTALL, to the POSIX function regcomp(). This + passes PCRE_DOTALL to the pcre_compile() function, making the "." character + match everything, including newlines. This is not POSIX-compatible, but + somebody wanted the feature. From pcretest it can be activated by using + both the P and the s flags. + +18. AC_PROG_LIBTOOL appeared twice in Makefile.in. Removed one. + +19. libpcre.pc was being incorrectly installed as executable. + +20. A couple of places in pcretest check for end-of-line by looking for '\n'; + it now also looks for '\r' so that it will work unmodified on Windows. + +21. Added Google's contributed C++ wrapper to the distribution. + +22. Added some untidy missing memory free() calls in pcretest, to keep + Electric Fence happy when testing. + + + +Version 5.0 13-Sep-04 +--------------------- + + 1. Internal change: literal characters are no longer packed up into items + containing multiple characters in a single byte-string. Each character + is now matched using a separate opcode. However, there may be more than one + byte in the character in UTF-8 mode. + + 2. The pcre_callout_block structure has two new fields: pattern_position and + next_item_length. These contain the offset in the pattern to the next match + item, and its length, respectively. + + 3. The PCRE_AUTO_CALLOUT option for pcre_compile() requests the automatic + insertion of callouts before each pattern item. Added the /C option to + pcretest to make use of this. + + 4. On the advice of a Windows user, the lines + + #if defined(_WIN32) || defined(WIN32) + _setmode( _fileno( stdout ), 0x8000 ); + #endif /* defined(_WIN32) || defined(WIN32) */ + + have been added to the source of pcretest. This apparently does useful + magic in relation to line terminators. + + 5. Changed "r" and "w" in the calls to fopen() in pcretest to "rb" and "wb" + for the benefit of those environments where the "b" makes a difference. + + 6. The icc compiler has the same options as gcc, but "configure" doesn't seem + to know about it. I have put a hack into configure.in that adds in code + to set GCC=yes if CC=icc. This seems to end up at a point in the + generated configure script that is early enough to affect the setting of + compiler options, which is what is needed, but I have no means of testing + whether it really works. (The user who reported this had patched the + generated configure script, which of course I cannot do.) + + LATER: After change 22 below (new libtool files), the configure script + seems to know about icc (and also ecc). Therefore, I have commented out + this hack in configure.in. + + 7. Added support for pkg-config (2 patches were sent in). + + 8. Negated POSIX character classes that used a combination of internal tables + were completely broken. These were [[:^alpha:]], [[:^alnum:]], and + [[:^ascii]]. Typically, they would match almost any characters. The other + POSIX classes were not broken in this way. + + 9. Matching the pattern "\b.*?" against "ab cd", starting at offset 1, failed + to find the match, as PCRE was deluded into thinking that the match had to + start at the start point or following a newline. The same bug applied to + patterns with negative forward assertions or any backward assertions + preceding ".*" at the start, unless the pattern required a fixed first + character. This was a failing pattern: "(?!.bcd).*". The bug is now fixed. + +10. In UTF-8 mode, when moving forwards in the subject after a failed match + starting at the last subject character, bytes beyond the end of the subject + string were read. + +11. Renamed the variable "class" as "classbits" to make life easier for C++ + users. (Previously there was a macro definition, but it apparently wasn't + enough.) + +12. Added the new field "tables" to the extra data so that tables can be passed + in at exec time, or the internal tables can be re-selected. This allows + a compiled regex to be saved and re-used at a later time by a different + program that might have everything at different addresses. + +13. Modified the pcre-config script so that, when run on Solaris, it shows a + -R library as well as a -L library. + +14. The debugging options of pcretest (-d on the command line or D on a + pattern) showed incorrect output for anything following an extended class + that contained multibyte characters and which was followed by a quantifier. + +15. Added optional support for general category Unicode character properties + via the \p, \P, and \X escapes. Unicode property support implies UTF-8 + support. It adds about 90K to the size of the library. The meanings of the + inbuilt class escapes such as \d and \s have NOT been changed. + +16. Updated pcredemo.c to include calls to free() to release the memory for the + compiled pattern. + +17. The generated file chartables.c was being created in the source directory + instead of in the building directory. This caused the build to fail if the + source directory was different from the building directory, and was + read-only. + +18. Added some sample Win commands from Mark Tetrode into the NON-UNIX-USE + file. No doubt somebody will tell me if they don't make sense... Also added + Dan Mooney's comments about building on OpenVMS. + +19. Added support for partial matching via the PCRE_PARTIAL option for + pcre_exec() and the \P data escape in pcretest. + +20. Extended pcretest with 3 new pattern features: + + (i) A pattern option of the form ">rest-of-line" causes pcretest to + write the compiled pattern to the file whose name is "rest-of-line". + This is a straight binary dump of the data, with the saved pointer to + the character tables forced to be NULL. The study data, if any, is + written too. After writing, pcretest reads a new pattern. + + (ii) If, instead of a pattern, ": new target + : new target + : use native compiler + : use native linker + : handle Windows platform correctly + : ditto + : ditto + copy DLL to top builddir before testing + + As part of these changes, -no-undefined was removed again. This was reported + to give trouble on HP-UX 11.0, so getting rid of it seems like a good idea + in any case. + +3. Some tidies to get rid of compiler warnings: + + . In the match_data structure, match_limit was an unsigned long int, whereas + match_call_count was an int. I've made them both unsigned long ints. + + . In pcretest the fact that a const uschar * doesn't automatically cast to + a void * provoked a warning. + + . Turning on some more compiler warnings threw up some "shadow" variables + and a few more missing casts. + +4. If PCRE was complied with UTF-8 support, but called without the PCRE_UTF8 + option, a class that contained a single character with a value between 128 + and 255 (e.g. /[\xFF]/) caused PCRE to crash. + +5. If PCRE was compiled with UTF-8 support, but called without the PCRE_UTF8 + option, a class that contained several characters, but with at least one + whose value was between 128 and 255 caused PCRE to crash. + + +Version 4.1 12-Mar-03 +--------------------- + +1. Compiling with gcc -pedantic found a couple of places where casts were +needed, and a string in dftables.c that was longer than standard compilers are +required to support. + +2. Compiling with Sun's compiler found a few more places where the code could +be tidied up in order to avoid warnings. + +3. The variables for cross-compiling were called HOST_CC and HOST_CFLAGS; the +first of these names is deprecated in the latest Autoconf in favour of the name +CC_FOR_BUILD, because "host" is typically used to mean the system on which the +compiled code will be run. I can't find a reference for HOST_CFLAGS, but by +analogy I have changed it to CFLAGS_FOR_BUILD. + +4. Added -no-undefined to the linking command in the Makefile, because this is +apparently helpful for Windows. To make it work, also added "-L. -lpcre" to the +linking step for the pcreposix library. + +5. PCRE was failing to diagnose the case of two named groups with the same +name. + +6. A problem with one of PCRE's optimizations was discovered. PCRE remembers a +literal character that is needed in the subject for a match, and scans along to +ensure that it is present before embarking on the full matching process. This +saves time in cases of nested unlimited repeats that are never going to match. +Problem: the scan can take a lot of time if the subject is very long (e.g. +megabytes), thus penalizing straightforward matches. It is now done only if the +amount of subject to be scanned is less than 1000 bytes. + +7. A lesser problem with the same optimization is that it was recording the +first character of an anchored pattern as "needed", thus provoking a search +right along the subject, even when the first match of the pattern was going to +fail. The "needed" character is now not set for anchored patterns, unless it +follows something in the pattern that is of non-fixed length. Thus, it still +fulfils its original purpose of finding quick non-matches in cases of nested +unlimited repeats, but isn't used for simple anchored patterns such as /^abc/. + + +Version 4.0 17-Feb-03 +--------------------- + +1. If a comment in an extended regex that started immediately after a meta-item +extended to the end of string, PCRE compiled incorrect data. This could lead to +all kinds of weird effects. Example: /#/ was bad; /()#/ was bad; /a#/ was not. + +2. Moved to autoconf 2.53 and libtool 1.4.2. + +3. Perl 5.8 no longer needs "use utf8" for doing UTF-8 things. Consequently, +the special perltest8 script is no longer needed - all the tests can be run +from a single perltest script. + +4. From 5.004, Perl has not included the VT character (0x0b) in the set defined +by \s. It has now been removed in PCRE. This means it isn't recognized as +whitespace in /x regexes too, which is the same as Perl. Note that the POSIX +class [:space:] *does* include VT, thereby creating a mess. + +5. Added the class [:blank:] (a GNU extension from Perl 5.8) to match only +space and tab. + +6. Perl 5.005 was a long time ago. It's time to amalgamate the tests that use +its new features into the main test script, reducing the number of scripts. + +7. Perl 5.8 has changed the meaning of patterns like /a(?i)b/. Earlier versions +were backward compatible, and made the (?i) apply to the whole pattern, as if +/i were given. Now it behaves more logically, and applies the option setting +only to what follows. PCRE has been changed to follow suit. However, if it +finds options settings right at the start of the pattern, it extracts them into +the global options, as before. Thus, they show up in the info data. + +8. Added support for the \Q...\E escape sequence. Characters in between are +treated as literals. This is slightly different from Perl in that $ and @ are +also handled as literals inside the quotes. In Perl, they will cause variable +interpolation. Note the following examples: + + Pattern PCRE matches Perl matches + + \Qabc$xyz\E abc$xyz abc followed by the contents of $xyz + \Qabc\$xyz\E abc\$xyz abc\$xyz + \Qabc\E\$\Qxyz\E abc$xyz abc$xyz + +For compatibility with Perl, \Q...\E sequences are recognized inside character +classes as well as outside them. + +9. Re-organized 3 code statements in pcretest to avoid "overflow in +floating-point constant arithmetic" warnings from a Microsoft compiler. Added a +(size_t) cast to one statement in pcretest and one in pcreposix to avoid +signed/unsigned warnings. + +10. SunOS4 doesn't have strtoul(). This was used only for unpicking the -o +option for pcretest, so I've replaced it by a simple function that does just +that job. + +11. pcregrep was ending with code 0 instead of 2 for the commands "pcregrep" or +"pcregrep -". + +12. Added "possessive quantifiers" ?+, *+, ++, and {,}+ which come from Sun's +Java package. This provides some syntactic sugar for simple cases of what my +documentation calls "once-only subpatterns". A pattern such as x*+ is the same +as (?>x*). In other words, if what is inside (?>...) is just a single repeated +item, you can use this simplified notation. Note that only makes sense with +greedy quantifiers. Consequently, the use of the possessive quantifier forces +greediness, whatever the setting of the PCRE_UNGREEDY option. + +13. A change of greediness default within a pattern was not taking effect at +the current level for patterns like /(b+(?U)a+)/. It did apply to parenthesized +subpatterns that followed. Patterns like /b+(?U)a+/ worked because the option +was abstracted outside. + +14. PCRE now supports the \G assertion. It is true when the current matching +position is at the start point of the match. This differs from \A when the +starting offset is non-zero. Used with the /g option of pcretest (or similar +code), it works in the same way as it does for Perl's /g option. If all +alternatives of a regex begin with \G, the expression is anchored to the start +match position, and the "anchored" flag is set in the compiled expression. + +15. Some bugs concerning the handling of certain option changes within patterns +have been fixed. These applied to options other than (?ims). For example, +"a(?x: b c )d" did not match "XabcdY" but did match "Xa b c dY". It should have +been the other way round. Some of this was related to change 7 above. + +16. PCRE now gives errors for /[.x.]/ and /[=x=]/ as unsupported POSIX +features, as Perl does. Previously, PCRE gave the warnings only for /[[.x.]]/ +and /[[=x=]]/. PCRE now also gives an error for /[:name:]/ because it supports +POSIX classes only within a class (e.g. /[[:alpha:]]/). + +17. Added support for Perl's \C escape. This matches one byte, even in UTF8 +mode. Unlike ".", it always matches newline, whatever the setting of +PCRE_DOTALL. However, PCRE does not permit \C to appear in lookbehind +assertions. Perl allows it, but it doesn't (in general) work because it can't +calculate the length of the lookbehind. At least, that's the case for Perl +5.8.0 - I've been told they are going to document that it doesn't work in +future. + +18. Added an error diagnosis for escapes that PCRE does not support: these are +\L, \l, \N, \P, \p, \U, \u, and \X. + +19. Although correctly diagnosing a missing ']' in a character class, PCRE was +reading past the end of the pattern in cases such as /[abcd/. + +20. PCRE was getting more memory than necessary for patterns with classes that +contained both POSIX named classes and other characters, e.g. /[[:space:]abc/. + +21. Added some code, conditional on #ifdef VPCOMPAT, to make life easier for +compiling PCRE for use with Virtual Pascal. + +22. Small fix to the Makefile to make it work properly if the build is done +outside the source tree. + +23. Added a new extension: a condition to go with recursion. If a conditional +subpattern starts with (?(R) the "true" branch is used if recursion has +happened, whereas the "false" branch is used only at the top level. + +24. When there was a very long string of literal characters (over 255 bytes +without UTF support, over 250 bytes with UTF support), the computation of how +much memory was required could be incorrect, leading to segfaults or other +strange effects. + +25. PCRE was incorrectly assuming anchoring (either to start of subject or to +start of line for a non-DOTALL pattern) when a pattern started with (.*) and +there was a subsequent back reference to those brackets. This meant that, for +example, /(.*)\d+\1/ failed to match "abc123bc". Unfortunately, it isn't +possible to check for precisely this case. All we can do is abandon the +optimization if .* occurs inside capturing brackets when there are any back +references whatsoever. (See below for a better fix that came later.) + +26. The handling of the optimization for finding the first character of a +non-anchored pattern, and for finding a character that is required later in the +match were failing in some cases. This didn't break the matching; it just +failed to optimize when it could. The way this is done has been re-implemented. + +27. Fixed typo in error message for invalid (?R item (it said "(?p"). + +28. Added a new feature that provides some of the functionality that Perl +provides with (?{...}). The facility is termed a "callout". The way it is done +in PCRE is for the caller to provide an optional function, by setting +pcre_callout to its entry point. Like pcre_malloc and pcre_free, this is a +global variable. By default it is unset, which disables all calling out. To get +the function called, the regex must include (?C) at appropriate points. This +is, in fact, equivalent to (?C0), and any number <= 255 may be given with (?C). +This provides a means of identifying different callout points. When PCRE +reaches such a point in the regex, if pcre_callout has been set, the external +function is called. It is provided with data in a structure called +pcre_callout_block, which is defined in pcre.h. If the function returns 0, +matching continues; if it returns a non-zero value, the match at the current +point fails. However, backtracking will occur if possible. [This was changed +later and other features added - see item 49 below.] + +29. pcretest is upgraded to test the callout functionality. It provides a +callout function that displays information. By default, it shows the start of +the match and the current position in the text. There are some new data escapes +to vary what happens: + + \C+ in addition, show current contents of captured substrings + \C- do not supply a callout function + \C!n return 1 when callout number n is reached + \C!n!m return 1 when callout number n is reached for the mth time + +30. If pcregrep was called with the -l option and just a single file name, it +output "" if a match was found, instead of the file name. + +31. Improve the efficiency of the POSIX API to PCRE. If the number of capturing +slots is less than POSIX_MALLOC_THRESHOLD, use a block on the stack to pass to +pcre_exec(). This saves a malloc/free per call. The default value of +POSIX_MALLOC_THRESHOLD is 10; it can be changed by --with-posix-malloc-threshold +when configuring. + +32. The default maximum size of a compiled pattern is 64K. There have been a +few cases of people hitting this limit. The code now uses macros to handle the +storing of links as offsets within the compiled pattern. It defaults to 2-byte +links, but this can be changed to 3 or 4 bytes by --with-link-size when +configuring. Tests 2 and 5 work only with 2-byte links because they output +debugging information about compiled patterns. + +33. Internal code re-arrangements: + +(a) Moved the debugging function for printing out a compiled regex into + its own source file (printint.c) and used #include to pull it into + pcretest.c and, when DEBUG is defined, into pcre.c, instead of having two + separate copies. + +(b) Defined the list of op-code names for debugging as a macro in + internal.h so that it is next to the definition of the opcodes. + +(c) Defined a table of op-code lengths for simpler skipping along compiled + code. This is again a macro in internal.h so that it is next to the + definition of the opcodes. + +34. Added support for recursive calls to individual subpatterns, along the +lines of Robin Houston's patch (but implemented somewhat differently). + +35. Further mods to the Makefile to help Win32. Also, added code to pcregrep to +allow it to read and process whole directories in Win32. This code was +contributed by Lionel Fourquaux; it has not been tested by me. + +36. Added support for named subpatterns. The Python syntax (?P...) is +used to name a group. Names consist of alphanumerics and underscores, and must +be unique. Back references use the syntax (?P=name) and recursive calls use +(?P>name) which is a PCRE extension to the Python extension. Groups still have +numbers. The function pcre_fullinfo() can be used after compilation to extract +a name/number map. There are three relevant calls: + + PCRE_INFO_NAMEENTRYSIZE yields the size of each entry in the map + PCRE_INFO_NAMECOUNT yields the number of entries + PCRE_INFO_NAMETABLE yields a pointer to the map. + +The map is a vector of fixed-size entries. The size of each entry depends on +the length of the longest name used. The first two bytes of each entry are the +group number, most significant byte first. There follows the corresponding +name, zero terminated. The names are in alphabetical order. + +37. Make the maximum literal string in the compiled code 250 for the non-UTF-8 +case instead of 255. Making it the same both with and without UTF-8 support +means that the same test output works with both. + +38. There was a case of malloc(0) in the POSIX testing code in pcretest. Avoid +calling malloc() with a zero argument. + +39. Change 25 above had to resort to a heavy-handed test for the .* anchoring +optimization. I've improved things by keeping a bitmap of backreferences with +numbers 1-31 so that if .* occurs inside capturing brackets that are not in +fact referenced, the optimization can be applied. It is unlikely that a +relevant occurrence of .* (i.e. one which might indicate anchoring or forcing +the match to follow \n) will appear inside brackets with a number greater than +31, but if it does, any back reference > 31 suppresses the optimization. + +40. Added a new compile-time option PCRE_NO_AUTO_CAPTURE. This has the effect +of disabling numbered capturing parentheses. Any opening parenthesis that is +not followed by ? behaves as if it were followed by ?: but named parentheses +can still be used for capturing (and they will acquire numbers in the usual +way). + +41. Redesigned the return codes from the match() function into yes/no/error so +that errors can be passed back from deep inside the nested calls. A malloc +failure while inside a recursive subpattern call now causes the +PCRE_ERROR_NOMEMORY return instead of quietly going wrong. + +42. It is now possible to set a limit on the number of times the match() +function is called in a call to pcre_exec(). This facility makes it possible to +limit the amount of recursion and backtracking, though not in a directly +obvious way, because the match() function is used in a number of different +circumstances. The count starts from zero for each position in the subject +string (for non-anchored patterns). The default limit is, for compatibility, a +large number, namely 10 000 000. You can change this in two ways: + +(a) When configuring PCRE before making, you can use --with-match-limit=n + to set a default value for the compiled library. + +(b) For each call to pcre_exec(), you can pass a pcre_extra block in which + a different value is set. See 45 below. + +If the limit is exceeded, pcre_exec() returns PCRE_ERROR_MATCHLIMIT. + +43. Added a new function pcre_config(int, void *) to enable run-time extraction +of things that can be changed at compile time. The first argument specifies +what is wanted and the second points to where the information is to be placed. +The current list of available information is: + + PCRE_CONFIG_UTF8 + +The output is an integer that is set to one if UTF-8 support is available; +otherwise it is set to zero. + + PCRE_CONFIG_NEWLINE + +The output is an integer that it set to the value of the code that is used for +newline. It is either LF (10) or CR (13). + + PCRE_CONFIG_LINK_SIZE + +The output is an integer that contains the number of bytes used for internal +linkage in compiled expressions. The value is 2, 3, or 4. See item 32 above. + + PCRE_CONFIG_POSIX_MALLOC_THRESHOLD + +The output is an integer that contains the threshold above which the POSIX +interface uses malloc() for output vectors. See item 31 above. + + PCRE_CONFIG_MATCH_LIMIT + +The output is an unsigned integer that contains the default limit of the number +of match() calls in a pcre_exec() execution. See 42 above. + +44. pcretest has been upgraded by the addition of the -C option. This causes it +to extract all the available output from the new pcre_config() function, and to +output it. The program then exits immediately. + +45. A need has arisen to pass over additional data with calls to pcre_exec() in +order to support additional features. One way would have been to define +pcre_exec2() (for example) with extra arguments, but this would not have been +extensible, and would also have required all calls to the original function to +be mapped to the new one. Instead, I have chosen to extend the mechanism that +is used for passing in "extra" data from pcre_study(). + +The pcre_extra structure is now exposed and defined in pcre.h. It currently +contains the following fields: + + flags a bitmap indicating which of the following fields are set + study_data opaque data from pcre_study() + match_limit a way of specifying a limit on match() calls for a specific + call to pcre_exec() + callout_data data for callouts (see 49 below) + +The flag bits are also defined in pcre.h, and are + + PCRE_EXTRA_STUDY_DATA + PCRE_EXTRA_MATCH_LIMIT + PCRE_EXTRA_CALLOUT_DATA + +The pcre_study() function now returns one of these new pcre_extra blocks, with +the actual study data pointed to by the study_data field, and the +PCRE_EXTRA_STUDY_DATA flag set. This can be passed directly to pcre_exec() as +before. That is, this change is entirely upwards-compatible and requires no +change to existing code. + +If you want to pass in additional data to pcre_exec(), you can either place it +in a pcre_extra block provided by pcre_study(), or create your own pcre_extra +block. + +46. pcretest has been extended to test the PCRE_EXTRA_MATCH_LIMIT feature. If a +data string contains the escape sequence \M, pcretest calls pcre_exec() several +times with different match limits, until it finds the minimum value needed for +pcre_exec() to complete. The value is then output. This can be instructive; for +most simple matches the number is quite small, but for pathological cases it +gets very large very quickly. + +47. There's a new option for pcre_fullinfo() called PCRE_INFO_STUDYSIZE. It +returns the size of the data block pointed to by the study_data field in a +pcre_extra block, that is, the value that was passed as the argument to +pcre_malloc() when PCRE was getting memory in which to place the information +created by pcre_study(). The fourth argument should point to a size_t variable. +pcretest has been extended so that this information is shown after a successful +pcre_study() call when information about the compiled regex is being displayed. + +48. Cosmetic change to Makefile: there's no need to have / after $(DESTDIR) +because what follows is always an absolute path. (Later: it turns out that this +is more than cosmetic for MinGW, because it doesn't like empty path +components.) + +49. Some changes have been made to the callout feature (see 28 above): + +(i) A callout function now has three choices for what it returns: + + 0 => success, carry on matching + > 0 => failure at this point, but backtrack if possible + < 0 => serious error, return this value from pcre_exec() + + Negative values should normally be chosen from the set of PCRE_ERROR_xxx + values. In particular, returning PCRE_ERROR_NOMATCH forces a standard + "match failed" error. The error number PCRE_ERROR_CALLOUT is reserved for + use by callout functions. It will never be used by PCRE itself. + +(ii) The pcre_extra structure (see 45 above) has a void * field called + callout_data, with corresponding flag bit PCRE_EXTRA_CALLOUT_DATA. The + pcre_callout_block structure has a field of the same name. The contents of + the field passed in the pcre_extra structure are passed to the callout + function in the corresponding field in the callout block. This makes it + easier to use the same callout-containing regex from multiple threads. For + testing, the pcretest program has a new data escape + + \C*n pass the number n (may be negative) as callout_data + + If the callout function in pcretest receives a non-zero value as + callout_data, it returns that value. + +50. Makefile wasn't handling CFLAGS properly when compiling dftables. Also, +there were some redundant $(CFLAGS) in commands that are now specified as +$(LINK), which already includes $(CFLAGS). + +51. Extensions to UTF-8 support are listed below. These all apply when (a) PCRE +has been compiled with UTF-8 support *and* pcre_compile() has been compiled +with the PCRE_UTF8 flag. Patterns that are compiled without that flag assume +one-byte characters throughout. Note that case-insensitive matching applies +only to characters whose values are less than 256. PCRE doesn't support the +notion of cases for higher-valued characters. + +(i) A character class whose characters are all within 0-255 is handled as + a bit map, and the map is inverted for negative classes. Previously, a + character > 255 always failed to match such a class; however it should + match if the class was a negative one (e.g. [^ab]). This has been fixed. + +(ii) A negated character class with a single character < 255 is coded as + "not this character" (OP_NOT). This wasn't working properly when the test + character was multibyte, either singly or repeated. + +(iii) Repeats of multibyte characters are now handled correctly in UTF-8 + mode, for example: \x{100}{2,3}. + +(iv) The character escapes \b, \B, \d, \D, \s, \S, \w, and \W (either + singly or repeated) now correctly test multibyte characters. However, + PCRE doesn't recognize any characters with values greater than 255 as + digits, spaces, or word characters. Such characters always match \D, \S, + and \W, and never match \d, \s, or \w. + +(v) Classes may now contain characters and character ranges with values + greater than 255. For example: [ab\x{100}-\x{400}]. + +(vi) pcregrep now has a --utf-8 option (synonym -u) which makes it call + PCRE in UTF-8 mode. + +52. The info request value PCRE_INFO_FIRSTCHAR has been renamed +PCRE_INFO_FIRSTBYTE because it is a byte value. However, the old name is +retained for backwards compatibility. (Note that LASTLITERAL is also a byte +value.) + +53. The single man page has become too large. I have therefore split it up into +a number of separate man pages. These also give rise to individual HTML pages; +these are now put in a separate directory, and there is an index.html page that +lists them all. Some hyperlinking between the pages has been installed. + +54. Added convenience functions for handling named capturing parentheses. + +55. Unknown escapes inside character classes (e.g. [\M]) and escapes that +aren't interpreted therein (e.g. [\C]) are literals in Perl. This is now also +true in PCRE, except when the PCRE_EXTENDED option is set, in which case they +are faulted. + +56. Introduced HOST_CC and HOST_CFLAGS which can be set in the environment when +calling configure. These values are used when compiling the dftables.c program +which is run to generate the source of the default character tables. They +default to the values of CC and CFLAGS. If you are cross-compiling PCRE, +you will need to set these values. + +57. Updated the building process for Windows DLL, as provided by Fred Cox. + + +Version 3.9 02-Jan-02 +--------------------- + +1. A bit of extraneous text had somehow crept into the pcregrep documentation. + +2. If --disable-static was given, the building process failed when trying to +build pcretest and pcregrep. (For some reason it was using libtool to compile +them, which is not right, as they aren't part of the library.) + + +Version 3.8 18-Dec-01 +--------------------- + +1. The experimental UTF-8 code was completely screwed up. It was packing the +bytes in the wrong order. How dumb can you get? + + +Version 3.7 29-Oct-01 +--------------------- + +1. In updating pcretest to check change 1 of version 3.6, I screwed up. +This caused pcretest, when used on the test data, to segfault. Unfortunately, +this didn't happen under Solaris 8, where I normally test things. + +2. The Makefile had to be changed to make it work on BSD systems, where 'make' +doesn't seem to recognize that ./xxx and xxx are the same file. (This entry +isn't in ChangeLog distributed with 3.7 because I forgot when I hastily made +this fix an hour or so after the initial 3.7 release.) + + +Version 3.6 23-Oct-01 +--------------------- + +1. Crashed with /(sens|respons)e and \1ibility/ and "sense and sensibility" if +offsets passed as NULL with zero offset count. + +2. The config.guess and config.sub files had not been updated when I moved to +the latest autoconf. + + +Version 3.5 15-Aug-01 +--------------------- + +1. Added some missing #if !defined NOPOSIX conditionals in pcretest.c that +had been forgotten. + +2. By using declared but undefined structures, we can avoid using "void" +definitions in pcre.h while keeping the internal definitions of the structures +private. + +3. The distribution is now built using autoconf 2.50 and libtool 1.4. From a +user point of view, this means that both static and shared libraries are built +by default, but this can be individually controlled. More of the work of +handling this static/shared cases is now inside libtool instead of PCRE's make +file. + +4. The pcretest utility is now installed along with pcregrep because it is +useful for users (to test regexs) and by doing this, it automatically gets +relinked by libtool. The documentation has been turned into a man page, so +there are now .1, .txt, and .html versions in /doc. + +5. Upgrades to pcregrep: + (i) Added long-form option names like gnu grep. + (ii) Added --help to list all options with an explanatory phrase. + (iii) Added -r, --recursive to recurse into sub-directories. + (iv) Added -f, --file to read patterns from a file. + +6. pcre_exec() was referring to its "code" argument before testing that +argument for NULL (and giving an error if it was NULL). + +7. Upgraded Makefile.in to allow for compiling in a different directory from +the source directory. + +8. Tiny buglet in pcretest: when pcre_fullinfo() was called to retrieve the +options bits, the pointer it was passed was to an int instead of to an unsigned +long int. This mattered only on 64-bit systems. + +9. Fixed typo (3.4/1) in pcre.h again. Sigh. I had changed pcre.h (which is +generated) instead of pcre.in, which it its source. Also made the same change +in several of the .c files. + +10. A new release of gcc defines printf() as a macro, which broke pcretest +because it had an ifdef in the middle of a string argument for printf(). Fixed +by using separate calls to printf(). + +11. Added --enable-newline-is-cr and --enable-newline-is-lf to the configure +script, to force use of CR or LF instead of \n in the source. On non-Unix +systems, the value can be set in config.h. + +12. The limit of 200 on non-capturing parentheses is a _nesting_ limit, not an +absolute limit. Changed the text of the error message to make this clear, and +likewise updated the man page. + +13. The limit of 99 on the number of capturing subpatterns has been removed. +The new limit is 65535, which I hope will not be a "real" limit. + + +Version 3.4 22-Aug-00 +--------------------- + +1. Fixed typo in pcre.h: unsigned const char * changed to const unsigned char *. + +2. Diagnose condition (?(0) as an error instead of crashing on matching. + + +Version 3.3 01-Aug-00 +--------------------- + +1. If an octal character was given, but the value was greater than \377, it +was not getting masked to the least significant bits, as documented. This could +lead to crashes in some systems. + +2. Perl 5.6 (if not earlier versions) accepts classes like [a-\d] and treats +the hyphen as a literal. PCRE used to give an error; it now behaves like Perl. + +3. Added the functions pcre_free_substring() and pcre_free_substring_list(). +These just pass their arguments on to (pcre_free)(), but they are provided +because some uses of PCRE bind it to non-C systems that can call its functions, +but cannot call free() or pcre_free() directly. + +4. Add "make test" as a synonym for "make check". Corrected some comments in +the Makefile. + +5. Add $(DESTDIR)/ in front of all the paths in the "install" target in the +Makefile. + +6. Changed the name of pgrep to pcregrep, because Solaris has introduced a +command called pgrep for grepping around the active processes. + +7. Added the beginnings of support for UTF-8 character strings. + +8. Arranged for the Makefile to pass over the settings of CC, CFLAGS, and +RANLIB to ./ltconfig so that they are used by libtool. I think these are all +the relevant ones. (AR is not passed because ./ltconfig does its own figuring +out for the ar command.) + + +Version 3.2 12-May-00 +--------------------- + +This is purely a bug fixing release. + +1. If the pattern /((Z)+|A)*/ was matched agained ZABCDEFG it matched Z instead +of ZA. This was just one example of several cases that could provoke this bug, +which was introduced by change 9 of version 2.00. The code for breaking +infinite loops after an iteration that matches an empty string was't working +correctly. + +2. The pcretest program was not imitating Perl correctly for the pattern /a*/g +when matched against abbab (for example). After matching an empty string, it +wasn't forcing anchoring when setting PCRE_NOTEMPTY for the next attempt; this +caused it to match further down the string than it should. + +3. The code contained an inclusion of sys/types.h. It isn't clear why this +was there because it doesn't seem to be needed, and it causes trouble on some +systems, as it is not a Standard C header. It has been removed. + +4. Made 4 silly changes to the source to avoid stupid compiler warnings that +were reported on the Macintosh. The changes were from + + while ((c = *(++ptr)) != 0 && c != '\n'); +to + while ((c = *(++ptr)) != 0 && c != '\n') ; + +Totally extraordinary, but if that's what it takes... + +5. PCRE is being used in one environment where neither memmove() nor bcopy() is +available. Added HAVE_BCOPY and an autoconf test for it; if neither +HAVE_MEMMOVE nor HAVE_BCOPY is set, use a built-in emulation function which +assumes the way PCRE uses memmove() (always moving upwards). + +6. PCRE is being used in one environment where strchr() is not available. There +was only one use in pcre.c, and writing it out to avoid strchr() probably gives +faster code anyway. + + +Version 3.1 09-Feb-00 +--------------------- + +The only change in this release is the fixing of some bugs in Makefile.in for +the "install" target: + +(1) It was failing to install pcreposix.h. + +(2) It was overwriting the pcre.3 man page with the pcreposix.3 man page. + + +Version 3.0 01-Feb-00 +--------------------- + +1. Add support for the /+ modifier to perltest (to output $` like it does in +pcretest). + +2. Add support for the /g modifier to perltest. + +3. Fix pcretest so that it behaves even more like Perl for /g when the pattern +matches null strings. + +4. Fix perltest so that it doesn't do unwanted things when fed an empty +pattern. Perl treats empty patterns specially - it reuses the most recent +pattern, which is not what we want. Replace // by /(?#)/ in order to avoid this +effect. + +5. The POSIX interface was broken in that it was just handing over the POSIX +captured string vector to pcre_exec(), but (since release 2.00) PCRE has +required a bigger vector, with some working space on the end. This means that +the POSIX wrapper now has to get and free some memory, and copy the results. + +6. Added some simple autoconf support, placing the test data and the +documentation in separate directories, re-organizing some of the +information files, and making it build pcre-config (a GNU standard). Also added +libtool support for building PCRE as a shared library, which is now the +default. + +7. Got rid of the leading zero in the definition of PCRE_MINOR because 08 and +09 are not valid octal constants. Single digits will be used for minor values +less than 10. + +8. Defined REG_EXTENDED and REG_NOSUB as zero in the POSIX header, so that +existing programs that set these in the POSIX interface can use PCRE without +modification. + +9. Added a new function, pcre_fullinfo() with an extensible interface. It can +return all that pcre_info() returns, plus additional data. The pcre_info() +function is retained for compatibility, but is considered to be obsolete. + +10. Added experimental recursion feature (?R) to handle one common case that +Perl 5.6 will be able to do with (?p{...}). + +11. Added support for POSIX character classes like [:alpha:], which Perl is +adopting. + + +Version 2.08 31-Aug-99 +---------------------- + +1. When startoffset was not zero and the pattern began with ".*", PCRE was not +trying to match at the startoffset position, but instead was moving forward to +the next newline as if a previous match had failed. + +2. pcretest was not making use of PCRE_NOTEMPTY when repeating for /g and /G, +and could get into a loop if a null string was matched other than at the start +of the subject. + +3. Added definitions of PCRE_MAJOR and PCRE_MINOR to pcre.h so the version can +be distinguished at compile time, and for completeness also added PCRE_DATE. + +5. Added Paul Sokolovsky's minor changes to make it easy to compile a Win32 DLL +in GnuWin32 environments. + + +Version 2.07 29-Jul-99 +---------------------- + +1. The documentation is now supplied in plain text form and HTML as well as in +the form of man page sources. + +2. C++ compilers don't like assigning (void *) values to other pointer types. +In particular this affects malloc(). Although there is no problem in Standard +C, I've put in casts to keep C++ compilers happy. + +3. Typo on pcretest.c; a cast of (unsigned char *) in the POSIX regexec() call +should be (const char *). + +4. If NOPOSIX is defined, pcretest.c compiles without POSIX support. This may +be useful for non-Unix systems who don't want to bother with the POSIX stuff. +However, I haven't made this a standard facility. The documentation doesn't +mention it, and the Makefile doesn't support it. + +5. The Makefile now contains an "install" target, with editable destinations at +the top of the file. The pcretest program is not installed. + +6. pgrep -V now gives the PCRE version number and date. + +7. Fixed bug: a zero repetition after a literal string (e.g. /abcde{0}/) was +causing the entire string to be ignored, instead of just the last character. + +8. If a pattern like /"([^\\"]+|\\.)*"/ is applied in the normal way to a +non-matching string, it can take a very, very long time, even for strings of +quite modest length, because of the nested recursion. PCRE now does better in +some of these cases. It does this by remembering the last required literal +character in the pattern, and pre-searching the subject to ensure it is present +before running the real match. In other words, it applies a heuristic to detect +some types of certain failure quickly, and in the above example, if presented +with a string that has no trailing " it gives "no match" very quickly. + +9. A new runtime option PCRE_NOTEMPTY causes null string matches to be ignored; +other alternatives are tried instead. + + +Version 2.06 09-Jun-99 +---------------------- + +1. Change pcretest's output for amount of store used to show just the code +space, because the remainder (the data block) varies in size between 32-bit and +64-bit systems. + +2. Added an extra argument to pcre_exec() to supply an offset in the subject to +start matching at. This allows lookbehinds to work when searching for multiple +occurrences in a string. + +3. Added additional options to pcretest for testing multiple occurrences: + + /+ outputs the rest of the string that follows a match + /g loops for multiple occurrences, using the new startoffset argument + /G loops for multiple occurrences by passing an incremented pointer + +4. PCRE wasn't doing the "first character" optimization for patterns starting +with \b or \B, though it was doing it for other lookbehind assertions. That is, +it wasn't noticing that a match for a pattern such as /\bxyz/ has to start with +the letter 'x'. On long subject strings, this gives a significant speed-up. + + +Version 2.05 21-Apr-99 +---------------------- + +1. Changed the type of magic_number from int to long int so that it works +properly on 16-bit systems. + +2. Fixed a bug which caused patterns starting with .* not to work correctly +when the subject string contained newline characters. PCRE was assuming +anchoring for such patterns in all cases, which is not correct because .* will +not pass a newline unless PCRE_DOTALL is set. It now assumes anchoring only if +DOTALL is set at top level; otherwise it knows that patterns starting with .* +must be retried after every newline in the subject. + + +Version 2.04 18-Feb-99 +---------------------- + +1. For parenthesized subpatterns with repeats whose minimum was zero, the +computation of the store needed to hold the pattern was incorrect (too large). +If such patterns were nested a few deep, this could multiply and become a real +problem. + +2. Added /M option to pcretest to show the memory requirement of a specific +pattern. Made -m a synonym of -s (which does this globally) for compatibility. + +3. Subpatterns of the form (regex){n,m} (i.e. limited maximum) were being +compiled in such a way that the backtracking after subsequent failure was +pessimal. Something like (a){0,3} was compiled as (a)?(a)?(a)? instead of +((a)((a)(a)?)?)? with disastrous performance if the maximum was of any size. + + +Version 2.03 02-Feb-99 +---------------------- + +1. Fixed typo and small mistake in man page. + +2. Added 4th condition (GPL supersedes if conflict) and created separate +LICENCE file containing the conditions. + +3. Updated pcretest so that patterns such as /abc\/def/ work like they do in +Perl, that is the internal \ allows the delimiter to be included in the +pattern. Locked out the use of \ as a delimiter. If \ immediately follows +the final delimiter, add \ to the end of the pattern (to test the error). + +4. Added the convenience functions for extracting substrings after a successful +match. Updated pcretest to make it able to test these functions. + + +Version 2.02 14-Jan-99 +---------------------- + +1. Initialized the working variables associated with each extraction so that +their saving and restoring doesn't refer to uninitialized store. + +2. Put dummy code into study.c in order to trick the optimizer of the IBM C +compiler for OS/2 into generating correct code. Apparently IBM isn't going to +fix the problem. + +3. Pcretest: the timing code wasn't using LOOPREPEAT for timing execution +calls, and wasn't printing the correct value for compiling calls. Increased the +default value of LOOPREPEAT, and the number of significant figures in the +times. + +4. Changed "/bin/rm" in the Makefile to "-rm" so it works on Windows NT. + +5. Renamed "deftables" as "dftables" to get it down to 8 characters, to avoid +a building problem on Windows NT with a FAT file system. + + +Version 2.01 21-Oct-98 +---------------------- + +1. Changed the API for pcre_compile() to allow for the provision of a pointer +to character tables built by pcre_maketables() in the current locale. If NULL +is passed, the default tables are used. + + +Version 2.00 24-Sep-98 +---------------------- + +1. Since the (>?) facility is in Perl 5.005, don't require PCRE_EXTRA to enable +it any more. + +2. Allow quantification of (?>) groups, and make it work correctly. + +3. The first character computation wasn't working for (?>) groups. + +4. Correct the implementation of \Z (it is permitted to match on the \n at the +end of the subject) and add 5.005's \z, which really does match only at the +very end of the subject. + +5. Remove the \X "cut" facility; Perl doesn't have it, and (?> is neater. + +6. Remove the ability to specify CASELESS, MULTILINE, DOTALL, and +DOLLAR_END_ONLY at runtime, to make it possible to implement the Perl 5.005 +localized options. All options to pcre_study() were also removed. + +7. Add other new features from 5.005: + + $(?<= positive lookbehind + $(?a*))*/ (a PCRE_EXTRA facility). + + +Version 1.00 18-Nov-97 +---------------------- + +1. Added compile-time macros to support systems such as SunOS4 which don't have +memmove() or strerror() but have other things that can be used instead. + +2. Arranged that "make clean" removes the executables. + + +Version 0.99 27-Oct-97 +---------------------- + +1. Fixed bug in code for optimizing classes with only one character. It was +initializing a 32-byte map regardless, which could cause it to run off the end +of the memory it had got. + +2. Added, conditional on PCRE_EXTRA, the proposed (?>REGEX) construction. + + +Version 0.98 22-Oct-97 +---------------------- + +1. Fixed bug in code for handling temporary memory usage when there are more +back references than supplied space in the ovector. This could cause segfaults. + + +Version 0.97 21-Oct-97 +---------------------- + +1. Added the \X "cut" facility, conditional on PCRE_EXTRA. + +2. Optimized negated single characters not to use a bit map. + +3. Brought error texts together as macro definitions; clarified some of them; +fixed one that was wrong - it said "range out of order" when it meant "invalid +escape sequence". + +4. Changed some char * arguments to const char *. + +5. Added PCRE_NOTBOL and PCRE_NOTEOL (from POSIX). + +6. Added the POSIX-style API wrapper in pcreposix.a and testing facilities in +pcretest. + + +Version 0.96 16-Oct-97 +---------------------- + +1. Added a simple "pgrep" utility to the distribution. + +2. Fixed an incompatibility with Perl: "{" is now treated as a normal character +unless it appears in one of the precise forms "{ddd}", "{ddd,}", or "{ddd,ddd}" +where "ddd" means "one or more decimal digits". + +3. Fixed serious bug. If a pattern had a back reference, but the call to +pcre_exec() didn't supply a large enough ovector to record the related +identifying subpattern, the match always failed. PCRE now remembers the number +of the largest back reference, and gets some temporary memory in which to save +the offsets during matching if necessary, in order to ensure that +backreferences always work. + +4. Increased the compatibility with Perl in a number of ways: + + (a) . no longer matches \n by default; an option PCRE_DOTALL is provided + to request this handling. The option can be set at compile or exec time. + + (b) $ matches before a terminating newline by default; an option + PCRE_DOLLAR_ENDONLY is provided to override this (but not in multiline + mode). The option can be set at compile or exec time. + + (c) The handling of \ followed by a digit other than 0 is now supposed to be + the same as Perl's. If the decimal number it represents is less than 10 + or there aren't that many previous left capturing parentheses, an octal + escape is read. Inside a character class, it's always an octal escape, + even if it is a single digit. + + (d) An escaped but undefined alphabetic character is taken as a literal, + unless PCRE_EXTRA is set. Currently this just reserves the remaining + escapes. + + (e) {0} is now permitted. (The previous item is removed from the compiled + pattern). + +5. Changed all the names of code files so that the basic parts are no longer +than 10 characters, and abolished the teeny "globals.c" file. + +6. Changed the handling of character classes; they are now done with a 32-byte +bit map always. + +7. Added the -d and /D options to pcretest to make it possible to look at the +internals of compilation without having to recompile pcre. + + +Version 0.95 23-Sep-97 +---------------------- + +1. Fixed bug in pre-pass concerning escaped "normal" characters such as \x5c or +\x20 at the start of a run of normal characters. These were being treated as +real characters, instead of the source characters being re-checked. + + +Version 0.94 18-Sep-97 +---------------------- + +1. The functions are now thread-safe, with the caveat that the global variables +containing pointers to malloc() and free() or alternative functions are the +same for all threads. + +2. Get pcre_study() to generate a bitmap of initial characters for non- +anchored patterns when this is possible, and use it if passed to pcre_exec(). + + +Version 0.93 15-Sep-97 +---------------------- + +1. /(b)|(:+)/ was computing an incorrect first character. + +2. Add pcre_study() to the API and the passing of pcre_extra to pcre_exec(), +but not actually doing anything yet. + +3. Treat "-" characters in classes that cannot be part of ranges as literals, +as Perl does (e.g. [-az] or [az-]). + +4. Set the anchored flag if a branch starts with .* or .*? because that tests +all possible positions. + +5. Split up into different modules to avoid including unneeded functions in a +compiled binary. However, compile and exec are still in one module. The "study" +function is split off. + +6. The character tables are now in a separate module whose source is generated +by an auxiliary program - but can then be edited by hand if required. There are +now no calls to isalnum(), isspace(), isdigit(), isxdigit(), tolower() or +toupper() in the code. + +7. Turn the malloc/free funtions variables into pcre_malloc and pcre_free and +make them global. Abolish the function for setting them, as the caller can now +set them directly. + + +Version 0.92 11-Sep-97 +---------------------- + +1. A repeat with a fixed maximum and a minimum of 1 for an ordinary character +(e.g. /a{1,3}/) was broken (I mis-optimized it). + +2. Caseless matching was not working in character classes if the characters in +the pattern were in upper case. + +3. Make ranges like [W-c] work in the same way as Perl for caseless matching. + +4. Make PCRE_ANCHORED public and accept as a compile option. + +5. Add an options word to pcre_exec() and accept PCRE_ANCHORED and +PCRE_CASELESS at run time. Add escapes \A and \I to pcretest to cause it to +pass them. + +6. Give an error if bad option bits passed at compile or run time. + +7. Add PCRE_MULTILINE at compile and exec time, and (?m) as well. Add \M to +pcretest to cause it to pass that flag. + +8. Add pcre_info(), to get the number of identifying subpatterns, the stored +options, and the first character, if set. + +9. Recognize C+ or C{n,m} where n >= 1 as providing a fixed starting character. + + +Version 0.91 10-Sep-97 +---------------------- + +1. PCRE was failing to diagnose unlimited repeats of subpatterns that could +match the empty string as in /(a*)*/. It was looping and ultimately crashing. + +2. PCRE was looping on encountering an indefinitely repeated back reference to +a subpattern that had matched an empty string, e.g. /(a|)\1*/. It now does what +Perl does - treats the match as successful. + +**** diff --git a/src/third_party/pcre-8.41/CheckMan b/src/third_party/pcre-8.41/CheckMan new file mode 100755 index 00000000000..480d735481e --- /dev/null +++ b/src/third_party/pcre-8.41/CheckMan @@ -0,0 +1,67 @@ +#! /usr/bin/perl + +# A script to scan PCRE's man pages to check for typos in the control +# sequences. I use only a small set of the available repertoire, so it is +# straightforward to check that nothing else has slipped in by mistake. This +# script should be called in the doc directory. + +$yield = 0; + +while (scalar(@ARGV) > 0) + { + $line = 0; + $file = shift @ARGV; + + open (IN, $file) || die "Failed to open $file\n"; + + while () + { + $line++; + if (/^\s*$/) + { + printf "Empty line $line of $file\n"; + $yield = 1; + } + elsif (/^\./) + { + if (!/^\.\s*$| + ^\.B\s+\S| + ^\.TH\s\S| + ^\.SH\s\S| + ^\.SS\s\S| + ^\.TP(?:\s?\d+)?\s*$| + ^\.SM\s*$| + ^\.br\s*$| + ^\.rs\s*$| + ^\.sp\s*$| + ^\.nf\s*$| + ^\.fi\s*$| + ^\.P\s*$| + ^\.PP\s*$| + ^\.\\"(?:\ HREF)?\s*$| + ^\.\\"\sHTML\s\s*$| + ^\.\\"\sHTML\s<\/a>\s*$| + ^\.\\"\s<\/a>\s*$| + ^\.\\"\sJOINSH\s*$| + ^\.\\"\sJOIN\s*$/x + ) + { + printf "Bad control line $line of $file\n"; + $yield = 1; + } + } + else + { + if (/\\[^ef]|\\f[^IBP]/) + { + printf "Bad backslash in line $line of $file\n"; + $yield = 1; + } + } + } + + close(IN); + } + +exit $yield; +# End diff --git a/src/third_party/pcre-8.41/CleanTxt b/src/third_party/pcre-8.41/CleanTxt new file mode 100755 index 00000000000..1f42519c8d7 --- /dev/null +++ b/src/third_party/pcre-8.41/CleanTxt @@ -0,0 +1,113 @@ +#! /usr/bin/perl -w + +# Script to take the output of nroff -man and remove all the backspacing and +# the page footers and the screen commands etc so that it is more usefully +# readable online. In fact, in the latest nroff, intermediate footers don't +# seem to be generated any more. + +$blankcount = 0; +$lastwascut = 0; +$firstheader = 1; + +# Input on STDIN; output to STDOUT. + +while () + { + s/\x1b\[\d+m//g; # Remove screen controls "ESC [ number m" + s/.\x8//g; # Remove "char, backspace" + + # Handle header lines. Retain only the first one we encounter, but remove + # the blank line that follows. Any others (e.g. at end of document) and the + # following blank line are dropped. + + if (/^PCRE(\w*)\(([13])\)\s+PCRE\1\(\2\)$/) + { + if ($firstheader) + { + $firstheader = 0; + print; + $lastprinted = $_; + $lastwascut = 0; + } + $_=; # Remove a blank that follows + next; + } + + # Count runs of empty lines + + if (/^\s*$/) + { + $blankcount++; + $lastwascut = 0; + next; + } + + # If a chunk of lines has been cut out (page footer) and the next line + # has a different indentation, put back one blank line. + + if ($lastwascut && $blankcount < 1 && defined($lastprinted)) + { + ($a) = $lastprinted =~ /^(\s*)/; + ($b) = $_ =~ /^(\s*)/; + $blankcount++ if ($a ne $b); + } + + # We get here only when we have a non-blank line in hand. If it was preceded + # by 3 or more blank lines, read the next 3 lines and see if they are blank. + # If so, remove all 7 lines, and remember that we have just done a cut. + + if ($blankcount >= 3) + { + for ($i = 0; $i < 3; $i++) + { + $next[$i] = ; + $next[$i] = "" if !defined $next[$i]; + $next[$i] =~ s/\x1b\[\d+m//g; # Remove screen controls "ESC [ number m" + $next[$i] =~ s/.\x8//g; # Remove "char, backspace" + } + + # Cut out chunks of the form <3 blanks><3 blanks> + + if ($next[0] =~ /^\s*$/ && + $next[1] =~ /^\s*$/ && + $next[2] =~ /^\s*$/) + { + $blankcount -= 3; + $lastwascut = 1; + } + + # Otherwise output the saved blanks, the current, and the next three + # lines. Remember the last printed line. + + else + { + for ($i = 0; $i < $blankcount; $i++) { print "\n"; } + print; + for ($i = 0; $i < 3; $i++) + { + $next[$i] =~ s/.\x8//g; + print $next[$i]; + $lastprinted = $_; + } + $lastwascut = 0; + $blankcount = 0; + } + } + + # This non-blank line is not preceded by 3 or more blank lines. Output + # any blanks there are, and the line. Remember it. Force two blank lines + # before headings. + + else + { + $blankcount = 2 if /^\S/ && !/^Last updated/ && !/^Copyright/ && + defined($lastprinted); + for ($i = 0; $i < $blankcount; $i++) { print "\n"; } + print; + $lastprinted = $_; + $lastwascut = 0; + $blankcount = 0; + } + } + +# End diff --git a/src/third_party/pcre-8.41/Detrail b/src/third_party/pcre-8.41/Detrail new file mode 100755 index 00000000000..1c5c7e9cae1 --- /dev/null +++ b/src/third_party/pcre-8.41/Detrail @@ -0,0 +1,35 @@ +#!/usr/bin/perl + +# This is a script for removing trailing whitespace from lines in files that +# are listed on the command line. + +# This subroutine does the work for one file. + +sub detrail { +my($file) = $_[0]; +my($changed) = 0; +open(IN, "$file") || die "Can't open $file for input"; +@lines = ; +close(IN); +foreach (@lines) + { + if (/\s+\n$/) + { + s/\s+\n$/\n/; + $changed = 1; + } + } +if ($changed) + { + open(OUT, ">$file") || die "Can't open $file for output"; + print OUT @lines; + close(OUT); + } +} + +# This is the main program + +$, = ""; # Output field separator +for ($i = 0; $i < @ARGV; $i++) { &detrail($ARGV[$i]); } + +# End diff --git a/src/third_party/pcre-8.41/HACKING b/src/third_party/pcre-8.41/HACKING new file mode 100644 index 00000000000..691b7a14e50 --- /dev/null +++ b/src/third_party/pcre-8.41/HACKING @@ -0,0 +1,528 @@ +Technical Notes about PCRE +-------------------------- + +These are very rough technical notes that record potentially useful information +about PCRE internals. For information about testing PCRE, see the pcretest +documentation and the comment at the head of the RunTest file. + + +Historical note 1 +----------------- + +Many years ago I implemented some regular expression functions to an algorithm +suggested by Martin Richards. These were not Unix-like in form, and were quite +restricted in what they could do by comparison with Perl. The interesting part +about the algorithm was that the amount of space required to hold the compiled +form of an expression was known in advance. The code to apply an expression did +not operate by backtracking, as the original Henry Spencer code and current +Perl code does, but instead checked all possibilities simultaneously by keeping +a list of current states and checking all of them as it advanced through the +subject string. In the terminology of Jeffrey Friedl's book, it was a "DFA +algorithm", though it was not a traditional Finite State Machine (FSM). When +the pattern was all used up, all remaining states were possible matches, and +the one matching the longest subset of the subject string was chosen. This did +not necessarily maximize the individual wild portions of the pattern, as is +expected in Unix and Perl-style regular expressions. + + +Historical note 2 +----------------- + +By contrast, the code originally written by Henry Spencer (which was +subsequently heavily modified for Perl) compiles the expression twice: once in +a dummy mode in order to find out how much store will be needed, and then for +real. (The Perl version probably doesn't do this any more; I'm talking about +the original library.) The execution function operates by backtracking and +maximizing (or, optionally, minimizing in Perl) the amount of the subject that +matches individual wild portions of the pattern. This is an "NFA algorithm" in +Friedl's terminology. + + +OK, here's the real stuff +------------------------- + +For the set of functions that form the "basic" PCRE library (which are +unrelated to those mentioned above), I tried at first to invent an algorithm +that used an amount of store bounded by a multiple of the number of characters +in the pattern, to save on compiling time. However, because of the greater +complexity in Perl regular expressions, I couldn't do this. In any case, a +first pass through the pattern is helpful for other reasons. + + +Support for 16-bit and 32-bit data strings +------------------------------------------- + +From release 8.30, PCRE supports 16-bit as well as 8-bit data strings; and from +release 8.32, PCRE supports 32-bit data strings. The library can be compiled +in any combination of 8-bit, 16-bit or 32-bit modes, creating up to three +different libraries. In the description that follows, the word "short" is used +for a 16-bit data quantity, and the word "unit" is used for a quantity that is +a byte in 8-bit mode, a short in 16-bit mode and a 32-bit word in 32-bit mode. +However, so as not to over-complicate the text, the names of PCRE functions are +given in 8-bit form only. + + +Computing the memory requirement: how it was +-------------------------------------------- + +Up to and including release 6.7, PCRE worked by running a very degenerate first +pass to calculate a maximum store size, and then a second pass to do the real +compile - which might use a bit less than the predicted amount of memory. The +idea was that this would turn out faster than the Henry Spencer code because +the first pass is degenerate and the second pass can just store stuff straight +into the vector, which it knows is big enough. + + +Computing the memory requirement: how it is +------------------------------------------- + +By the time I was working on a potential 6.8 release, the degenerate first pass +had become very complicated and hard to maintain. Indeed one of the early +things I did for 6.8 was to fix Yet Another Bug in the memory computation. Then +I had a flash of inspiration as to how I could run the real compile function in +a "fake" mode that enables it to compute how much memory it would need, while +actually only ever using a few hundred bytes of working memory, and without too +many tests of the mode that might slow it down. So I refactored the compiling +functions to work this way. This got rid of about 600 lines of source. It +should make future maintenance and development easier. As this was such a major +change, I never released 6.8, instead upping the number to 7.0 (other quite +major changes were also present in the 7.0 release). + +A side effect of this work was that the previous limit of 200 on the nesting +depth of parentheses was removed. However, there is a downside: pcre_compile() +runs more slowly than before (30% or more, depending on the pattern) because it +is doing a full analysis of the pattern. My hope was that this would not be a +big issue, and in the event, nobody has commented on it. + +At release 8.34, a limit on the nesting depth of parentheses was re-introduced +(default 250, settable at build time) so as to put a limit on the amount of +system stack used by pcre_compile(). This is a safety feature for environments +with small stacks where the patterns are provided by users. + + +Traditional matching function +----------------------------- + +The "traditional", and original, matching function is called pcre_exec(), and +it implements an NFA algorithm, similar to the original Henry Spencer algorithm +and the way that Perl works. This is not surprising, since it is intended to be +as compatible with Perl as possible. This is the function most users of PCRE +will use most of the time. From release 8.20, if PCRE is compiled with +just-in-time (JIT) support, and studying a compiled pattern with JIT is +successful, the JIT code is run instead of the normal pcre_exec() code, but the +result is the same. + + +Supplementary matching function +------------------------------- + +From PCRE 6.0, there is also a supplementary matching function called +pcre_dfa_exec(). This implements a DFA matching algorithm that searches +simultaneously for all possible matches that start at one point in the subject +string. (Going back to my roots: see Historical Note 1 above.) This function +intreprets the same compiled pattern data as pcre_exec(); however, not all the +facilities are available, and those that are do not always work in quite the +same way. See the user documentation for details. + +The algorithm that is used for pcre_dfa_exec() is not a traditional FSM, +because it may have a number of states active at one time. More work would be +needed at compile time to produce a traditional FSM where only one state is +ever active at once. I believe some other regex matchers work this way. JIT +support is not available for this kind of matching. + + +Changeable options +------------------ + +The /i, /m, or /s options (PCRE_CASELESS, PCRE_MULTILINE, PCRE_DOTALL, and some +others) may change in the middle of patterns. From PCRE 8.13, their processing +is handled entirely at compile time by generating different opcodes for the +different settings. The runtime functions do not need to keep track of an +options state any more. + + +Format of compiled patterns +--------------------------- + +The compiled form of a pattern is a vector of unsigned units (bytes in 8-bit +mode, shorts in 16-bit mode, 32-bit words in 32-bit mode), containing items of +variable length. The first unit in an item contains an opcode, and the length +of the item is either implicit in the opcode or contained in the data that +follows it. + +In many cases listed below, LINK_SIZE data values are specified for offsets +within the compiled pattern. LINK_SIZE always specifies a number of bytes. The +default value for LINK_SIZE is 2, but PCRE can be compiled to use 3-byte or +4-byte values for these offsets, although this impairs the performance. (3-byte +LINK_SIZE values are available only in 8-bit mode.) Specifing a LINK_SIZE +larger than 2 is necessary only when patterns whose compiled length is greater +than 64K are going to be processed. In this description, we assume the "normal" +compilation options. Data values that are counts (e.g. quantifiers) are two +bytes long in 8-bit mode (most significant byte first), or one unit in 16-bit +and 32-bit modes. + + +Opcodes with no following data +------------------------------ + +These items are all just one unit long + + OP_END end of pattern + OP_ANY match any one character other than newline + OP_ALLANY match any one character, including newline + OP_ANYBYTE match any single unit, even in UTF-8/16 mode + OP_SOD match start of data: \A + OP_SOM, start of match (subject + offset): \G + OP_SET_SOM, set start of match (\K) + OP_CIRC ^ (start of data) + OP_CIRCM ^ multiline mode (start of data or after newline) + OP_NOT_WORD_BOUNDARY \W + OP_WORD_BOUNDARY \w + OP_NOT_DIGIT \D + OP_DIGIT \d + OP_NOT_HSPACE \H + OP_HSPACE \h + OP_NOT_WHITESPACE \S + OP_WHITESPACE \s + OP_NOT_VSPACE \V + OP_VSPACE \v + OP_NOT_WORDCHAR \W + OP_WORDCHAR \w + OP_EODN match end of data or newline at end: \Z + OP_EOD match end of data: \z + OP_DOLL $ (end of data, or before final newline) + OP_DOLLM $ multiline mode (end of data or before newline) + OP_EXTUNI match an extended Unicode grapheme cluster + OP_ANYNL match any Unicode newline sequence + + OP_ASSERT_ACCEPT ) + OP_ACCEPT ) These are Perl 5.10's "backtracking control + OP_COMMIT ) verbs". If OP_ACCEPT is inside capturing + OP_FAIL ) parentheses, it may be preceded by one or more + OP_PRUNE ) OP_CLOSE, each followed by a count that + OP_SKIP ) indicates which parentheses must be closed. + OP_THEN ) + +OP_ASSERT_ACCEPT is used when (*ACCEPT) is encountered within an assertion. +This ends the assertion, not the entire pattern match. + + +Backtracking control verbs with optional data +--------------------------------------------- + +(*THEN) without an argument generates the opcode OP_THEN and no following data. +OP_MARK is followed by the mark name, preceded by a one-unit length, and +followed by a binary zero. For (*PRUNE), (*SKIP), and (*THEN) with arguments, +the opcodes OP_PRUNE_ARG, OP_SKIP_ARG, and OP_THEN_ARG are used, with the name +following in the same format as OP_MARK. + + +Matching literal characters +--------------------------- + +The OP_CHAR opcode is followed by a single character that is to be matched +casefully. For caseless matching, OP_CHARI is used. In UTF-8 or UTF-16 modes, +the character may be more than one unit long. In UTF-32 mode, characters +are always exactly one unit long. + +If there is only one character in a character class, OP_CHAR or OP_CHARI is +used for a positive class, and OP_NOT or OP_NOTI for a negative one (that is, +for something like [^a]). + + +Repeating single characters +--------------------------- + +The common repeats (*, +, ?), when applied to a single character, use the +following opcodes, which come in caseful and caseless versions: + + Caseful Caseless + OP_STAR OP_STARI + OP_MINSTAR OP_MINSTARI + OP_POSSTAR OP_POSSTARI + OP_PLUS OP_PLUSI + OP_MINPLUS OP_MINPLUSI + OP_POSPLUS OP_POSPLUSI + OP_QUERY OP_QUERYI + OP_MINQUERY OP_MINQUERYI + OP_POSQUERY OP_POSQUERYI + +Each opcode is followed by the character that is to be repeated. In ASCII mode, +these are two-unit items; in UTF-8 or UTF-16 modes, the length is variable; in +UTF-32 mode these are one-unit items. Those with "MIN" in their names are the +minimizing versions. Those with "POS" in their names are possessive versions. +Other repeats make use of these opcodes: + + Caseful Caseless + OP_UPTO OP_UPTOI + OP_MINUPTO OP_MINUPTOI + OP_POSUPTO OP_POSUPTOI + OP_EXACT OP_EXACTI + +Each of these is followed by a count and then the repeated character. OP_UPTO +matches from 0 to the given number. A repeat with a non-zero minimum and a +fixed maximum is coded as an OP_EXACT followed by an OP_UPTO (or OP_MINUPTO or +OPT_POSUPTO). + +Another set of matching repeating opcodes (called OP_NOTSTAR, OP_NOTSTARI, +etc.) are used for repeated, negated, single-character classes such as [^a]*. +The normal single-character opcodes (OP_STAR, etc.) are used for repeated +positive single-character classes. + + +Repeating character types +------------------------- + +Repeats of things like \d are done exactly as for single characters, except +that instead of a character, the opcode for the type is stored in the data +unit. The opcodes are: + + OP_TYPESTAR + OP_TYPEMINSTAR + OP_TYPEPOSSTAR + OP_TYPEPLUS + OP_TYPEMINPLUS + OP_TYPEPOSPLUS + OP_TYPEQUERY + OP_TYPEMINQUERY + OP_TYPEPOSQUERY + OP_TYPEUPTO + OP_TYPEMINUPTO + OP_TYPEPOSUPTO + OP_TYPEEXACT + + +Match by Unicode property +------------------------- + +OP_PROP and OP_NOTPROP are used for positive and negative matches of a +character by testing its Unicode property (the \p and \P escape sequences). +Each is followed by two units that encode the desired property as a type and a +value. The types are a set of #defines of the form PT_xxx, and the values are +enumerations of the form ucp_xx, defined in the ucp.h source file. The value is +relevant only for PT_GC (General Category), PT_PC (Particular Category), and +PT_SC (Script). + +Repeats of these items use the OP_TYPESTAR etc. set of opcodes, followed by +three units: OP_PROP or OP_NOTPROP, and then the desired property type and +value. + + +Character classes +----------------- + +If there is only one character in a class, OP_CHAR or OP_CHARI is used for a +positive class, and OP_NOT or OP_NOTI for a negative one (that is, for +something like [^a]). + +A set of repeating opcodes (called OP_NOTSTAR etc.) are used for repeated, +negated, single-character classes. The normal single-character opcodes +(OP_STAR, etc.) are used for repeated positive single-character classes. + +When there is more than one character in a class, and all the code points are +less than 256, OP_CLASS is used for a positive class, and OP_NCLASS for a +negative one. In either case, the opcode is followed by a 32-byte (16-short, +8-word) bit map containing a 1 bit for every character that is acceptable. The +bits are counted from the least significant end of each unit. In caseless mode, +bits for both cases are set. + +The reason for having both OP_CLASS and OP_NCLASS is so that, in UTF-8/16/32 +mode, subject characters with values greater than 255 can be handled correctly. +For OP_CLASS they do not match, whereas for OP_NCLASS they do. + +For classes containing characters with values greater than 255 or that contain +\p or \P, OP_XCLASS is used. It optionally uses a bit map if any code points +are less than 256, followed by a list of pairs (for a range) and single +characters. In caseless mode, both cases are explicitly listed. + +OP_XCLASS is followed by a unit containing flag bits: XCL_NOT indicates that +this is a negative class, and XCL_MAP indicates that a bit map is present. +There follows the bit map, if XCL_MAP is set, and then a sequence of items +coded as follows: + + XCL_END marks the end of the list + XCL_SINGLE one character follows + XCL_RANGE two characters follow + XCL_PROP a Unicode property (type, value) follows + XCL_NOTPROP a Unicode property (type, value) follows + +If a range starts with a code point less than 256 and ends with one greater +than 256, an XCL_RANGE item is used, without setting any bits in the bit map. +This means that if no other items in the class set bits in the map, a map is +not needed. + + +Back references +--------------- + +OP_REF (caseful) or OP_REFI (caseless) is followed by a count containing the +reference number if the reference is to a unique capturing group (either by +number or by name). When named groups are used, there may be more than one +group with the same name. In this case, a reference by name generates OP_DNREF +or OP_DNREFI. These are followed by two counts: the index (not the byte offset) +in the group name table of the first entry for the requred name, followed by +the number of groups with the same name. + + +Repeating character classes and back references +----------------------------------------------- + +Single-character classes are handled specially (see above). This section +applies to other classes and also to back references. In both cases, the repeat +information follows the base item. The matching code looks at the following +opcode to see if it is one of + + OP_CRSTAR + OP_CRMINSTAR + OP_CRPOSSTAR + OP_CRPLUS + OP_CRMINPLUS + OP_CRPOSPLUS + OP_CRQUERY + OP_CRMINQUERY + OP_CRPOSQUERY + OP_CRRANGE + OP_CRMINRANGE + OP_CRPOSRANGE + +All but the last three are single-unit items, with no data. The others are +followed by the minimum and maximum repeat counts. + + +Brackets and alternation +------------------------ + +A pair of non-capturing round brackets is wrapped round each expression at +compile time, so alternation always happens in the context of brackets. + +[Note for North Americans: "bracket" to some English speakers, including +myself, can be round, square, curly, or pointy. Hence this usage rather than +"parentheses".] + +Non-capturing brackets use the opcode OP_BRA. Originally PCRE was limited to 99 +capturing brackets and it used a different opcode for each one. From release +3.5, the limit was removed by putting the bracket number into the data for +higher-numbered brackets. From release 7.0 all capturing brackets are handled +this way, using the single opcode OP_CBRA. + +A bracket opcode is followed by LINK_SIZE bytes which give the offset to the +next alternative OP_ALT or, if there aren't any branches, to the matching +OP_KET opcode. Each OP_ALT is followed by LINK_SIZE bytes giving the offset to +the next one, or to the OP_KET opcode. For capturing brackets, the bracket +number is a count that immediately follows the offset. + +OP_KET is used for subpatterns that do not repeat indefinitely, and OP_KETRMIN +and OP_KETRMAX are used for indefinite repetitions, minimally or maximally +respectively (see below for possessive repetitions). All three are followed by +LINK_SIZE bytes giving (as a positive number) the offset back to the matching +bracket opcode. + +If a subpattern is quantified such that it is permitted to match zero times, it +is preceded by one of OP_BRAZERO, OP_BRAMINZERO, or OP_SKIPZERO. These are +single-unit opcodes that tell the matcher that skipping the following +subpattern entirely is a valid branch. In the case of the first two, not +skipping the pattern is also valid (greedy and non-greedy). The third is used +when a pattern has the quantifier {0,0}. It cannot be entirely discarded, +because it may be called as a subroutine from elsewhere in the regex. + +A subpattern with an indefinite maximum repetition is replicated in the +compiled data its minimum number of times (or once with OP_BRAZERO if the +minimum is zero), with the final copy terminating with OP_KETRMIN or OP_KETRMAX +as appropriate. + +A subpattern with a bounded maximum repetition is replicated in a nested +fashion up to the maximum number of times, with OP_BRAZERO or OP_BRAMINZERO +before each replication after the minimum, so that, for example, (abc){2,5} is +compiled as (abc)(abc)((abc)((abc)(abc)?)?)?, except that each bracketed group +has the same number. + +When a repeated subpattern has an unbounded upper limit, it is checked to see +whether it could match an empty string. If this is the case, the opcode in the +final replication is changed to OP_SBRA or OP_SCBRA. This tells the matcher +that it needs to check for matching an empty string when it hits OP_KETRMIN or +OP_KETRMAX, and if so, to break the loop. + + +Possessive brackets +------------------- + +When a repeated group (capturing or non-capturing) is marked as possessive by +the "+" notation, e.g. (abc)++, different opcodes are used. Their names all +have POS on the end, e.g. OP_BRAPOS instead of OP_BRA and OP_SCPBRPOS instead +of OP_SCBRA. The end of such a group is marked by OP_KETRPOS. If the minimum +repetition is zero, the group is preceded by OP_BRAPOSZERO. + + +Once-only (atomic) groups +------------------------- + +These are just like other subpatterns, but they start with the opcode +OP_ONCE or OP_ONCE_NC. The former is used when there are no capturing brackets +within the atomic group; the latter when there are. The distinction is needed +for when there is a backtrack to before the group - any captures within the +group must be reset, so it is necessary to retain backtracking points inside +the group even after it is complete in order to do this. When there are no +captures in an atomic group, all the backtracking can be discarded when it is +complete. This is more efficient, and also uses less stack. + +The check for matching an empty string in an unbounded repeat is handled +entirely at runtime, so there are just these two opcodes for atomic groups. + + +Assertions +---------- + +Forward assertions are also just like other subpatterns, but starting with one +of the opcodes OP_ASSERT or OP_ASSERT_NOT. Backward assertions use the opcodes +OP_ASSERTBACK and OP_ASSERTBACK_NOT, and the first opcode inside the assertion +is OP_REVERSE, followed by a count of the number of characters to move back the +pointer in the subject string. In ASCII mode, the count is a number of units, +but in UTF-8/16 mode each character may occupy more than one unit; in UTF-32 +mode each character occupies exactly one unit. A separate count is present in +each alternative of a lookbehind assertion, allowing them to have different +fixed lengths. + + +Conditional subpatterns +----------------------- + +These are like other subpatterns, but they start with the opcode OP_COND, or +OP_SCOND for one that might match an empty string in an unbounded repeat. If +the condition is a back reference, this is stored at the start of the +subpattern using the opcode OP_CREF followed by a count containing the +reference number, provided that the reference is to a unique capturing group. +If the reference was by name and there is more than one group with that name, +OP_DNCREF is used instead. It is followed by two counts: the index in the group +names table, and the number of groups with the same name. + +If the condition is "in recursion" (coded as "(?(R)"), or "in recursion of +group x" (coded as "(?(Rx)"), the group number is stored at the start of the +subpattern using the opcode OP_RREF (with a value of zero for "the whole +pattern") or OP_DNRREF (with data as for OP_DNCREF). For a DEFINE condition, +just the single unit OP_DEF is used (it has no associated data). Otherwise, a +conditional subpattern always starts with one of the assertions. + + +Recursion +--------- + +Recursion either matches the current regex, or some subexpression. The opcode +OP_RECURSE is followed by aLINK_SIZE value that is the offset to the starting +bracket from the start of the whole pattern. From release 6.5, OP_RECURSE is +automatically wrapped inside OP_ONCE brackets, because otherwise some patterns +broke it. OP_RECURSE is also used for "subroutine" calls, even though they are +not strictly a recursion. + + +Callout +------- + +OP_CALLOUT is followed by one unit of data that holds a callout number in the +range 0 to 254 for manual callouts, or 255 for an automatic callout. In both +cases there follows a count giving the offset in the pattern string to the +start of the following item, and another count giving the length of this item. +These values make is possible for pcretest to output useful tracing information +using automatic callouts. + +Philip Hazel +November 2013 diff --git a/src/third_party/pcre-8.41/INSTALL b/src/third_party/pcre-8.41/INSTALL new file mode 100644 index 00000000000..2099840756e --- /dev/null +++ b/src/third_party/pcre-8.41/INSTALL @@ -0,0 +1,370 @@ +Installation Instructions +************************* + +Copyright (C) 1994-1996, 1999-2002, 2004-2013 Free Software Foundation, +Inc. + + Copying and distribution of this file, with or without modification, +are permitted in any medium without royalty provided the copyright +notice and this notice are preserved. This file is offered as-is, +without warranty of any kind. + +Basic Installation +================== + + Briefly, the shell command `./configure && make && make install' +should configure, build, and install this package. The following +more-detailed instructions are generic; see the `README' file for +instructions specific to this package. Some packages provide this +`INSTALL' file but do not implement all of the features documented +below. The lack of an optional feature in a given package is not +necessarily a bug. More recommendations for GNU packages can be found +in *note Makefile Conventions: (standards)Makefile Conventions. + + The `configure' shell script attempts to guess correct values for +various system-dependent variables used during compilation. It uses +those values to create a `Makefile' in each directory of the package. +It may also create one or more `.h' files containing system-dependent +definitions. Finally, it creates a shell script `config.status' that +you can run in the future to recreate the current configuration, and a +file `config.log' containing compiler output (useful mainly for +debugging `configure'). + + It can also use an optional file (typically called `config.cache' +and enabled with `--cache-file=config.cache' or simply `-C') that saves +the results of its tests to speed up reconfiguring. Caching is +disabled by default to prevent problems with accidental use of stale +cache files. + + If you need to do unusual things to compile the package, please try +to figure out how `configure' could check whether to do them, and mail +diffs or instructions to the address given in the `README' so they can +be considered for the next release. If you are using the cache, and at +some point `config.cache' contains results you don't want to keep, you +may remove or edit it. + + The file `configure.ac' (or `configure.in') is used to create +`configure' by a program called `autoconf'. You need `configure.ac' if +you want to change it or regenerate `configure' using a newer version +of `autoconf'. + + The simplest way to compile this package is: + + 1. `cd' to the directory containing the package's source code and type + `./configure' to configure the package for your system. + + Running `configure' might take a while. While running, it prints + some messages telling which features it is checking for. + + 2. Type `make' to compile the package. + + 3. Optionally, type `make check' to run any self-tests that come with + the package, generally using the just-built uninstalled binaries. + + 4. Type `make install' to install the programs and any data files and + documentation. When installing into a prefix owned by root, it is + recommended that the package be configured and built as a regular + user, and only the `make install' phase executed with root + privileges. + + 5. Optionally, type `make installcheck' to repeat any self-tests, but + this time using the binaries in their final installed location. + This target does not install anything. Running this target as a + regular user, particularly if the prior `make install' required + root privileges, verifies that the installation completed + correctly. + + 6. You can remove the program binaries and object files from the + source code directory by typing `make clean'. To also remove the + files that `configure' created (so you can compile the package for + a different kind of computer), type `make distclean'. There is + also a `make maintainer-clean' target, but that is intended mainly + for the package's developers. If you use it, you may have to get + all sorts of other programs in order to regenerate files that came + with the distribution. + + 7. Often, you can also type `make uninstall' to remove the installed + files again. In practice, not all packages have tested that + uninstallation works correctly, even though it is required by the + GNU Coding Standards. + + 8. Some packages, particularly those that use Automake, provide `make + distcheck', which can by used by developers to test that all other + targets like `make install' and `make uninstall' work correctly. + This target is generally not run by end users. + +Compilers and Options +===================== + + Some systems require unusual options for compilation or linking that +the `configure' script does not know about. Run `./configure --help' +for details on some of the pertinent environment variables. + + You can give `configure' initial values for configuration parameters +by setting variables in the command line or in the environment. Here +is an example: + + ./configure CC=c99 CFLAGS=-g LIBS=-lposix + + *Note Defining Variables::, for more details. + +Compiling For Multiple Architectures +==================================== + + You can compile the package for more than one kind of computer at the +same time, by placing the object files for each architecture in their +own directory. To do this, you can use GNU `make'. `cd' to the +directory where you want the object files and executables to go and run +the `configure' script. `configure' automatically checks for the +source code in the directory that `configure' is in and in `..'. This +is known as a "VPATH" build. + + With a non-GNU `make', it is safer to compile the package for one +architecture at a time in the source code directory. After you have +installed the package for one architecture, use `make distclean' before +reconfiguring for another architecture. + + On MacOS X 10.5 and later systems, you can create libraries and +executables that work on multiple system types--known as "fat" or +"universal" binaries--by specifying multiple `-arch' options to the +compiler but only a single `-arch' option to the preprocessor. Like +this: + + ./configure CC="gcc -arch i386 -arch x86_64 -arch ppc -arch ppc64" \ + CXX="g++ -arch i386 -arch x86_64 -arch ppc -arch ppc64" \ + CPP="gcc -E" CXXCPP="g++ -E" + + This is not guaranteed to produce working output in all cases, you +may have to build one architecture at a time and combine the results +using the `lipo' tool if you have problems. + +Installation Names +================== + + By default, `make install' installs the package's commands under +`/usr/local/bin', include files under `/usr/local/include', etc. You +can specify an installation prefix other than `/usr/local' by giving +`configure' the option `--prefix=PREFIX', where PREFIX must be an +absolute file name. + + You can specify separate installation prefixes for +architecture-specific files and architecture-independent files. If you +pass the option `--exec-prefix=PREFIX' to `configure', the package uses +PREFIX as the prefix for installing programs and libraries. +Documentation and other data files still use the regular prefix. + + In addition, if you use an unusual directory layout you can give +options like `--bindir=DIR' to specify different values for particular +kinds of files. Run `configure --help' for a list of the directories +you can set and what kinds of files go in them. In general, the +default for these options is expressed in terms of `${prefix}', so that +specifying just `--prefix' will affect all of the other directory +specifications that were not explicitly provided. + + The most portable way to affect installation locations is to pass the +correct locations to `configure'; however, many packages provide one or +both of the following shortcuts of passing variable assignments to the +`make install' command line to change installation locations without +having to reconfigure or recompile. + + The first method involves providing an override variable for each +affected directory. For example, `make install +prefix=/alternate/directory' will choose an alternate location for all +directory configuration variables that were expressed in terms of +`${prefix}'. Any directories that were specified during `configure', +but not in terms of `${prefix}', must each be overridden at install +time for the entire installation to be relocated. The approach of +makefile variable overrides for each directory variable is required by +the GNU Coding Standards, and ideally causes no recompilation. +However, some platforms have known limitations with the semantics of +shared libraries that end up requiring recompilation when using this +method, particularly noticeable in packages that use GNU Libtool. + + The second method involves providing the `DESTDIR' variable. For +example, `make install DESTDIR=/alternate/directory' will prepend +`/alternate/directory' before all installation names. The approach of +`DESTDIR' overrides is not required by the GNU Coding Standards, and +does not work on platforms that have drive letters. On the other hand, +it does better at avoiding recompilation issues, and works well even +when some directory options were not specified in terms of `${prefix}' +at `configure' time. + +Optional Features +================= + + If the package supports it, you can cause programs to be installed +with an extra prefix or suffix on their names by giving `configure' the +option `--program-prefix=PREFIX' or `--program-suffix=SUFFIX'. + + Some packages pay attention to `--enable-FEATURE' options to +`configure', where FEATURE indicates an optional part of the package. +They may also pay attention to `--with-PACKAGE' options, where PACKAGE +is something like `gnu-as' or `x' (for the X Window System). The +`README' should mention any `--enable-' and `--with-' options that the +package recognizes. + + For packages that use the X Window System, `configure' can usually +find the X include and library files automatically, but if it doesn't, +you can use the `configure' options `--x-includes=DIR' and +`--x-libraries=DIR' to specify their locations. + + Some packages offer the ability to configure how verbose the +execution of `make' will be. For these packages, running `./configure +--enable-silent-rules' sets the default to minimal output, which can be +overridden with `make V=1'; while running `./configure +--disable-silent-rules' sets the default to verbose, which can be +overridden with `make V=0'. + +Particular systems +================== + + On HP-UX, the default C compiler is not ANSI C compatible. If GNU +CC is not installed, it is recommended to use the following options in +order to use an ANSI C compiler: + + ./configure CC="cc -Ae -D_XOPEN_SOURCE=500" + +and if that doesn't work, install pre-built binaries of GCC for HP-UX. + + HP-UX `make' updates targets which have the same time stamps as +their prerequisites, which makes it generally unusable when shipped +generated files such as `configure' are involved. Use GNU `make' +instead. + + On OSF/1 a.k.a. Tru64, some versions of the default C compiler cannot +parse its `' header file. The option `-nodtk' can be used as +a workaround. If GNU CC is not installed, it is therefore recommended +to try + + ./configure CC="cc" + +and if that doesn't work, try + + ./configure CC="cc -nodtk" + + On Solaris, don't put `/usr/ucb' early in your `PATH'. This +directory contains several dysfunctional programs; working variants of +these programs are available in `/usr/bin'. So, if you need `/usr/ucb' +in your `PATH', put it _after_ `/usr/bin'. + + On Haiku, software installed for all users goes in `/boot/common', +not `/usr/local'. It is recommended to use the following options: + + ./configure --prefix=/boot/common + +Specifying the System Type +========================== + + There may be some features `configure' cannot figure out +automatically, but needs to determine by the type of machine the package +will run on. Usually, assuming the package is built to be run on the +_same_ architectures, `configure' can figure that out, but if it prints +a message saying it cannot guess the machine type, give it the +`--build=TYPE' option. TYPE can either be a short name for the system +type, such as `sun4', or a canonical name which has the form: + + CPU-COMPANY-SYSTEM + +where SYSTEM can have one of these forms: + + OS + KERNEL-OS + + See the file `config.sub' for the possible values of each field. If +`config.sub' isn't included in this package, then this package doesn't +need to know the machine type. + + If you are _building_ compiler tools for cross-compiling, you should +use the option `--target=TYPE' to select the type of system they will +produce code for. + + If you want to _use_ a cross compiler, that generates code for a +platform different from the build platform, you should specify the +"host" platform (i.e., that on which the generated programs will +eventually be run) with `--host=TYPE'. + +Sharing Defaults +================ + + If you want to set default values for `configure' scripts to share, +you can create a site shell script called `config.site' that gives +default values for variables like `CC', `cache_file', and `prefix'. +`configure' looks for `PREFIX/share/config.site' if it exists, then +`PREFIX/etc/config.site' if it exists. Or, you can set the +`CONFIG_SITE' environment variable to the location of the site script. +A warning: not all `configure' scripts look for a site script. + +Defining Variables +================== + + Variables not defined in a site shell script can be set in the +environment passed to `configure'. However, some packages may run +configure again during the build, and the customized values of these +variables may be lost. In order to avoid this problem, you should set +them in the `configure' command line, using `VAR=value'. For example: + + ./configure CC=/usr/local2/bin/gcc + +causes the specified `gcc' to be used as the C compiler (unless it is +overridden in the site shell script). + +Unfortunately, this technique does not work for `CONFIG_SHELL' due to +an Autoconf limitation. Until the limitation is lifted, you can use +this workaround: + + CONFIG_SHELL=/bin/bash ./configure CONFIG_SHELL=/bin/bash + +`configure' Invocation +====================== + + `configure' recognizes the following options to control how it +operates. + +`--help' +`-h' + Print a summary of all of the options to `configure', and exit. + +`--help=short' +`--help=recursive' + Print a summary of the options unique to this package's + `configure', and exit. The `short' variant lists options used + only in the top level, while the `recursive' variant lists options + also present in any nested packages. + +`--version' +`-V' + Print the version of Autoconf used to generate the `configure' + script, and exit. + +`--cache-file=FILE' + Enable the cache: use and save the results of the tests in FILE, + traditionally `config.cache'. FILE defaults to `/dev/null' to + disable caching. + +`--config-cache' +`-C' + Alias for `--cache-file=config.cache'. + +`--quiet' +`--silent' +`-q' + Do not print messages saying which checks are being made. To + suppress all normal output, redirect it to `/dev/null' (any error + messages will still be shown). + +`--srcdir=DIR' + Look for the package's source code in directory DIR. Usually + `configure' can determine that directory automatically. + +`--prefix=DIR' + Use DIR as the installation prefix. *note Installation Names:: + for more details, including other options available for fine-tuning + the installation locations. + +`--no-create' +`-n' + Run the configure checks, but stop before creating any output + files. + +`configure' also accepts some other, not widely useful, options. Run +`configure --help' for more details. diff --git a/src/third_party/pcre-8.41/LICENCE b/src/third_party/pcre-8.41/LICENCE new file mode 100644 index 00000000000..dd9071a8dd8 --- /dev/null +++ b/src/third_party/pcre-8.41/LICENCE @@ -0,0 +1,93 @@ +PCRE LICENCE +------------ + +PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + +Release 8 of PCRE is distributed under the terms of the "BSD" licence, as +specified below. The documentation for PCRE, supplied in the "doc" +directory, is distributed under the same terms as the software itself. The data +in the testdata directory is not copyrighted and is in the public domain. + +The basic library functions are written in C and are freestanding. Also +included in the distribution is a set of C++ wrapper functions, and a +just-in-time compiler that can be used to optimize pattern matching. These +are both optional features that can be omitted when the library is built. + + +THE BASIC LIBRARY FUNCTIONS +--------------------------- + +Written by: Philip Hazel +Email local part: ph10 +Email domain: cam.ac.uk + +University of Cambridge Computing Service, +Cambridge, England. + +Copyright (c) 1997-2017 University of Cambridge +All rights reserved. + + +PCRE JUST-IN-TIME COMPILATION SUPPORT +------------------------------------- + +Written by: Zoltan Herczeg +Email local part: hzmester +Emain domain: freemail.hu + +Copyright(c) 2010-2017 Zoltan Herczeg +All rights reserved. + + +STACK-LESS JUST-IN-TIME COMPILER +-------------------------------- + +Written by: Zoltan Herczeg +Email local part: hzmester +Emain domain: freemail.hu + +Copyright(c) 2009-2017 Zoltan Herczeg +All rights reserved. + + +THE C++ WRAPPER FUNCTIONS +------------------------- + +Contributed by: Google Inc. + +Copyright (c) 2007-2012, Google Inc. +All rights reserved. + + +THE "BSD" LICENCE +----------------- + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the name of Google + Inc. nor the names of their contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +End diff --git a/src/third_party/pcre-8.41/NEWS b/src/third_party/pcre-8.41/NEWS new file mode 100644 index 00000000000..36be07cb880 --- /dev/null +++ b/src/third_party/pcre-8.41/NEWS @@ -0,0 +1,737 @@ +News about PCRE releases +------------------------ + +Release 8.41 13-June-2017 +------------------------- + +This is a bug-fix release. + + +Release 8.40 11-January-2017 +---------------------------- + +This is a bug-fix release. + + +Release 8.39 14-June-2016 +------------------------- + +Some appropriate PCRE2 JIT improvements have been retro-fitted to PCRE1. Apart +from that, this is another bug-fix release. Note that this library (now called +PCRE1) is now being maintained for bug fixes only. New projects are advised to +use the new PCRE2 libraries. + + +Release 8.38 23-November-2015 +----------------------------- + +This is bug-fix release. Note that this library (now called PCRE1) is now being +maintained for bug fixes only. New projects are advised to use the new PCRE2 +libraries. + + +Release 8.37 28-April-2015 +-------------------------- + +This is bug-fix release. Note that this library (now called PCRE1) is now being +maintained for bug fixes only. New projects are advised to use the new PCRE2 +libraries. + + +Release 8.36 26-September-2014 +------------------------------ + +This is primarily a bug-fix release. However, in addition, the Unicode data +tables have been updated to Unicode 7.0.0. + + +Release 8.35 04-April-2014 +-------------------------- + +There have been performance improvements for classes containing non-ASCII +characters and the "auto-possessification" feature has been extended. Other +minor improvements have been implemented and bugs fixed. There is a new callout +feature to enable applications to do detailed stack checks at compile time, to +avoid running out of stack for deeply nested parentheses. The JIT compiler has +been extended with experimental support for ARM-64, MIPS-64, and PPC-LE. + + +Release 8.34 15-December-2013 +----------------------------- + +As well as fixing the inevitable bugs, performance has been improved by +refactoring and extending the amount of "auto-possessification" that PCRE does. +Other notable changes: + +. Implemented PCRE_INFO_MATCH_EMPTY, which yields 1 if the pattern can match + an empty string. If it can, pcretest shows this in its information output. + +. A back reference to a named subpattern when there is more than one of the + same name now checks them in the order in which they appear in the pattern. + The first one that is set is used for the reference. Previously only the + first one was inspected. This change makes PCRE more compatible with Perl. + +. Unicode character properties were updated from Unicode 6.3.0. + +. The character VT has been added to the set of characters that match \s and + are generally treated as white space, following this same change in Perl + 5.18. There is now no difference between "Perl space" and "POSIX space". + +. Perl has changed its handling of \8 and \9. If there is no previously + encountered capturing group of those numbers, they are treated as the + literal characters 8 and 9 instead of a binary zero followed by the + literals. PCRE now does the same. + +. Following Perl, added \o{} to specify codepoints in octal, making it + possible to specify values greater than 0777 and also making them + unambiguous. + +. In UCP mode, \s was not matching two of the characters that Perl matches, + namely NEL (U+0085) and MONGOLIAN VOWEL SEPARATOR (U+180E), though they + were matched by \h. + +. Add JIT support for the 64 bit TileGX architecture. + +. Upgraded the handling of the POSIX classes [:graph:], [:print:], and + [:punct:] when PCRE_UCP is set so as to include the same characters as Perl + does in Unicode mode. + +. Perl no longer allows group names to start with digits, so I have made this + change also in PCRE. + +. Added support for [[:<:]] and [[:>:]] as used in the BSD POSIX library to + mean "start of word" and "end of word", respectively, as a transition aid. + + +Release 8.33 28-May-2013 +-------------------------- + +A number of bugs are fixed, and some performance improvements have been made. +There are also some new features, of which these are the most important: + +. The behaviour of the backtracking verbs has been rationalized and + documented in more detail. + +. JIT now supports callouts and all of the backtracking verbs. + +. Unicode validation has been updated in the light of Unicode Corrigendum #9, + which points out that "non characters" are not "characters that may not + appear in Unicode strings" but rather "characters that are reserved for + internal use and have only local meaning". + +. (*LIMIT_MATCH=d) and (*LIMIT_RECURSION=d) have been added so that the + creator of a pattern can specify lower (but not higher) limits for the + matching process. + +. The PCRE_NEVER_UTF option is available to prevent pattern-writers from using + the (*UTF) feature, as this could be a security issue. + + +Release 8.32 30-November-2012 +----------------------------- + +This release fixes a number of bugs, but also has some new features. These are +the highlights: + +. There is now support for 32-bit character strings and UTF-32. Like the + 16-bit support, this is done by compiling a separate 32-bit library. + +. \X now matches a Unicode extended grapheme cluster. + +. Case-independent matching of Unicode characters that have more than one + "other case" now makes all three (or more) characters equivalent. This + applies, for example, to Greek Sigma, which has two lowercase versions. + +. Unicode character properties are updated to Unicode 6.2.0. + +. The EBCDIC support, which had decayed, has had a spring clean. + +. A number of JIT optimizations have been added, which give faster JIT + execution speed. In addition, a new direct interface to JIT execution is + available. This bypasses some of the sanity checks of pcre_exec() to give a + noticeable speed-up. + +. A number of issues in pcregrep have been fixed, making it more compatible + with GNU grep. In particular, --exclude and --include (and variants) apply + to all files now, not just those obtained from scanning a directory + recursively. In Windows environments, the default action for directories is + now "skip" instead of "read" (which provokes an error). + +. If the --only-matching (-o) option in pcregrep is specified multiple + times, each one causes appropriate output. For example, -o1 -o2 outputs the + substrings matched by the 1st and 2nd capturing parentheses. A separating + string can be specified by --om-separator (default empty). + +. When PCRE is built via Autotools using a version of gcc that has the + "visibility" feature, it is used to hide internal library functions that are + not part of the public API. + + +Release 8.31 06-July-2012 +------------------------- + +This is mainly a bug-fixing release, with a small number of developments: + +. The JIT compiler now supports partial matching and the (*MARK) and + (*COMMIT) verbs. + +. PCRE_INFO_MAXLOOKBEHIND can be used to find the longest lookbehind in a + pattern. + +. There should be a performance improvement when using the heap instead of the + stack for recursion. + +. pcregrep can now be linked with libedit as an alternative to libreadline. + +. pcregrep now has a --file-list option where the list of files to scan is + given as a file. + +. pcregrep now recognizes binary files and there are related options. + +. The Unicode tables have been updated to 6.1.0. + +As always, the full list of changes is in the ChangeLog file. + + +Release 8.30 04-February-2012 +----------------------------- + +Release 8.30 introduces a major new feature: support for 16-bit character +strings, compiled as a separate library. There are a few changes to the +8-bit library, in addition to some bug fixes. + +. The pcre_info() function, which has been obsolete for over 10 years, has + been removed. + +. When a compiled pattern was saved to a file and later reloaded on a host + with different endianness, PCRE used automatically to swap the bytes in some + of the data fields. With the advent of the 16-bit library, where more of this + swapping is needed, it is no longer done automatically. Instead, the bad + endianness is detected and a specific error is given. The user can then call + a new function called pcre_pattern_to_host_byte_order() (or an equivalent + 16-bit function) to do the swap. + +. In UTF-8 mode, the values 0xd800 to 0xdfff are not legal Unicode + code points and are now faulted. (They are the so-called "surrogates" + that are reserved for coding high values in UTF-16.) + + +Release 8.21 12-Dec-2011 +------------------------ + +This is almost entirely a bug-fix release. The only new feature is the ability +to obtain the size of the memory used by the JIT compiler. + + +Release 8.20 21-Oct-2011 +------------------------ + +The main change in this release is the inclusion of Zoltan Herczeg's +just-in-time compiler support, which can be accessed by building PCRE with +--enable-jit. Large performance benefits can be had in many situations. 8.20 +also fixes an unfortunate bug that was introduced in 8.13 as well as tidying up +a number of infelicities and differences from Perl. + + +Release 8.13 16-Aug-2011 +------------------------ + +This is mainly a bug-fix release. There has been a lot of internal refactoring. +The Unicode tables have been updated. The only new feature in the library is +the passing of *MARK information to callouts. Some additions have been made to +pcretest to make testing easier and more comprehensive. There is a new option +for pcregrep to adjust its internal buffer size. + + +Release 8.12 15-Jan-2011 +------------------------ + +This release fixes some bugs in pcregrep, one of which caused the tests to fail +on 64-bit big-endian systems. There are no changes to the code of the library. + + +Release 8.11 10-Dec-2010 +------------------------ + +A number of bugs in the library and in pcregrep have been fixed. As always, see +ChangeLog for details. The following are the non-bug-fix changes: + +. Added --match-limit and --recursion-limit to pcregrep. + +. Added an optional parentheses number to the -o and --only-matching options + of pcregrep. + +. Changed the way PCRE_PARTIAL_HARD affects the matching of $, \z, \Z, \b, and + \B. + +. Added PCRE_ERROR_SHORTUTF8 to make it possible to distinguish between a + bad UTF-8 sequence and one that is incomplete when using PCRE_PARTIAL_HARD. + +. Recognize (*NO_START_OPT) at the start of a pattern to set the PCRE_NO_ + START_OPTIMIZE option, which is now allowed at compile time + + +Release 8.10 25-Jun-2010 +------------------------ + +There are two major additions: support for (*MARK) and friends, and the option +PCRE_UCP, which changes the behaviour of \b, \d, \s, and \w (and their +opposites) so that they make use of Unicode properties. There are also a number +of lesser new features, and several bugs have been fixed. A new option, +--line-buffered, has been added to pcregrep, for use when it is connected to +pipes. + + +Release 8.02 19-Mar-2010 +------------------------ + +Another bug-fix release. + + +Release 8.01 19-Jan-2010 +------------------------ + +This is a bug-fix release. Several bugs in the code itself and some bugs and +infelicities in the build system have been fixed. + + +Release 8.00 19-Oct-09 +---------------------- + +Bugs have been fixed in the library and in pcregrep. There are also some +enhancements. Restrictions on patterns used for partial matching have been +removed, extra information is given for partial matches, the partial matching +process has been improved, and an option to make a partial match override a +full match is available. The "study" process has been enhanced by finding a +lower bound matching length. Groups with duplicate numbers may now have +duplicated names without the use of PCRE_DUPNAMES. However, they may not have +different names. The documentation has been revised to reflect these changes. +The version number has been expanded to 3 digits as it is clear that the rate +of change is not slowing down. + + +Release 7.9 11-Apr-09 +--------------------- + +Mostly bugfixes and tidies with just a couple of minor functional additions. + + +Release 7.8 05-Sep-08 +--------------------- + +More bug fixes, plus a performance improvement in Unicode character property +lookup. + + +Release 7.7 07-May-08 +--------------------- + +This is once again mainly a bug-fix release, but there are a couple of new +features. + + +Release 7.6 28-Jan-08 +--------------------- + +The main reason for having this release so soon after 7.5 is because it fixes a +potential buffer overflow problem in pcre_compile() when run in UTF-8 mode. In +addition, the CMake configuration files have been brought up to date. + + +Release 7.5 10-Jan-08 +--------------------- + +This is mainly a bug-fix release. However the ability to link pcregrep with +libz or libbz2 and the ability to link pcretest with libreadline have been +added. Also the --line-offsets and --file-offsets options were added to +pcregrep. + + +Release 7.4 21-Sep-07 +--------------------- + +The only change of specification is the addition of options to control whether +\R matches any Unicode line ending (the default) or just CR, LF, and CRLF. +Otherwise, the changes are bug fixes and a refactoring to reduce the number of +relocations needed in a shared library. There have also been some documentation +updates, in particular, some more information about using CMake to build PCRE +has been added to the NON-UNIX-USE file. + + +Release 7.3 28-Aug-07 +--------------------- + +Most changes are bug fixes. Some that are not: + +1. There is some support for Perl 5.10's experimental "backtracking control + verbs" such as (*PRUNE). + +2. UTF-8 checking is now as per RFC 3629 instead of RFC 2279; this is more + restrictive in the strings it accepts. + +3. Checking for potential integer overflow has been made more dynamic, and as a + consequence there is no longer a hard limit on the size of a subpattern that + has a limited repeat count. + +4. When CRLF is a valid line-ending sequence, pcre_exec() and pcre_dfa_exec() + no longer advance by two characters instead of one when an unanchored match + fails at CRLF if there are explicit CR or LF matches within the pattern. + This gets rid of some anomalous effects that previously occurred. + +5. Some PCRE-specific settings for varying the newline options at the start of + a pattern have been added. + + +Release 7.2 19-Jun-07 +--------------------- + +WARNING: saved patterns that were compiled by earlier versions of PCRE must be +recompiled for use with 7.2 (necessitated by the addition of \K, \h, \H, \v, +and \V). + +Correction to the notes for 7.1: the note about shared libraries for Windows is +wrong. Previously, three libraries were built, but each could function +independently. For example, the pcreposix library also included all the +functions from the basic pcre library. The change is that the three libraries +are no longer independent. They are like the Unix libraries. To use the +pcreposix functions, for example, you need to link with both the pcreposix and +the basic pcre library. + +Some more features from Perl 5.10 have been added: + + (?-n) and (?+n) relative references for recursion and subroutines. + + (?(-n) and (?(+n) relative references as conditions. + + \k{name} and \g{name} are synonyms for \k. + + \K to reset the start of the matched string; for example, (foo)\Kbar + matches bar preceded by foo, but only sets bar as the matched string. + + (?| introduces a group where the capturing parentheses in each alternative + start from the same number; for example, (?|(abc)|(xyz)) sets capturing + parentheses number 1 in both cases. + + \h, \H, \v, \V match horizontal and vertical whitespace, respectively. + + +Release 7.1 24-Apr-07 +--------------------- + +There is only one new feature in this release: a linebreak setting of +PCRE_NEWLINE_ANYCRLF. It is a cut-down version of PCRE_NEWLINE_ANY, which +recognizes only CRLF, CR, and LF as linebreaks. + +A few bugs are fixed (see ChangeLog for details), but the major change is a +complete re-implementation of the build system. This now has full Autotools +support and so is now "standard" in some sense. It should help with compiling +PCRE in a wide variety of environments. + +NOTE: when building shared libraries for Windows, three dlls are now built, +called libpcre, libpcreposix, and libpcrecpp. Previously, everything was +included in a single dll. + +Another important change is that the dftables auxiliary program is no longer +compiled and run at "make" time by default. Instead, a default set of character +tables (assuming ASCII coding) is used. If you want to use dftables to generate +the character tables as previously, add --enable-rebuild-chartables to the +"configure" command. You must do this if you are compiling PCRE to run on a +system that uses EBCDIC code. + +There is a discussion about character tables in the README file. The default is +not to use dftables so that that there is no problem when cross-compiling. + + +Release 7.0 19-Dec-06 +--------------------- + +This release has a new major number because there have been some internal +upheavals to facilitate the addition of new optimizations and other facilities, +and to make subsequent maintenance and extension easier. Compilation is likely +to be a bit slower, but there should be no major effect on runtime performance. +Previously compiled patterns are NOT upwards compatible with this release. If +you have saved compiled patterns from a previous release, you will have to +re-compile them. Important changes that are visible to users are: + +1. The Unicode property tables have been updated to Unicode 5.0.0, which adds + some more scripts. + +2. The option PCRE_NEWLINE_ANY causes PCRE to recognize any Unicode newline + sequence as a newline. + +3. The \R escape matches a single Unicode newline sequence as a single unit. + +4. New features that will appear in Perl 5.10 are now in PCRE. These include + alternative Perl syntax for named parentheses, and Perl syntax for + recursion. + +5. The C++ wrapper interface has been extended by the addition of a + QuoteMeta function and the ability to allow copy construction and + assignment. + +For a complete list of changes, see the ChangeLog file. + + +Release 6.7 04-Jul-06 +--------------------- + +The main additions to this release are the ability to use the same name for +multiple sets of parentheses, and support for CRLF line endings in both the +library and pcregrep (and in pcretest for testing). + +Thanks to Ian Taylor, the stack usage for many kinds of pattern has been +significantly reduced for certain subject strings. + + +Release 6.5 01-Feb-06 +--------------------- + +Important changes in this release: + +1. A number of new features have been added to pcregrep. + +2. The Unicode property tables have been updated to Unicode 4.1.0, and the + supported properties have been extended with script names such as "Arabic", + and the derived properties "Any" and "L&". This has necessitated a change to + the interal format of compiled patterns. Any saved compiled patterns that + use \p or \P must be recompiled. + +3. The specification of recursion in patterns has been changed so that all + recursive subpatterns are automatically treated as atomic groups. Thus, for + example, (?R) is treated as if it were (?>(?R)). This is necessary because + otherwise there are situations where recursion does not work. + +See the ChangeLog for a complete list of changes, which include a number of bug +fixes and tidies. + + +Release 6.0 07-Jun-05 +--------------------- + +The release number has been increased to 6.0 because of the addition of several +major new pieces of functionality. + +A new function, pcre_dfa_exec(), which implements pattern matching using a DFA +algorithm, has been added. This has a number of advantages for certain cases, +though it does run more slowly, and lacks the ability to capture substrings. On +the other hand, it does find all matches, not just the first, and it works +better for partial matching. The pcrematching man page discusses the +differences. + +The pcretest program has been enhanced so that it can make use of the new +pcre_dfa_exec() matching function and the extra features it provides. + +The distribution now includes a C++ wrapper library. This is built +automatically if a C++ compiler is found. The pcrecpp man page discusses this +interface. + +The code itself has been re-organized into many more files, one for each +function, so it no longer requires everything to be linked in when static +linkage is used. As a consequence, some internal functions have had to have +their names exposed. These functions all have names starting with _pcre_. They +are undocumented, and are not intended for use by outside callers. + +The pcregrep program has been enhanced with new functionality such as +multiline-matching and options for output more matching context. See the +ChangeLog for a complete list of changes to the library and the utility +programs. + + +Release 5.0 13-Sep-04 +--------------------- + +The licence under which PCRE is released has been changed to the more +conventional "BSD" licence. + +In the code, some bugs have been fixed, and there are also some major changes +in this release (which is why I've increased the number to 5.0). Some changes +are internal rearrangements, and some provide a number of new facilities. The +new features are: + +1. There's an "automatic callout" feature that inserts callouts before every + item in the regex, and there's a new callout field that gives the position + in the pattern - useful for debugging and tracing. + +2. The extra_data structure can now be used to pass in a set of character + tables at exec time. This is useful if compiled regex are saved and re-used + at a later time when the tables may not be at the same address. If the + default internal tables are used, the pointer saved with the compiled + pattern is now set to NULL, which means that you don't need to do anything + special unless you are using custom tables. + +3. It is possible, with some restrictions on the content of the regex, to + request "partial" matching. A special return code is given if all of the + subject string matched part of the regex. This could be useful for testing + an input field as it is being typed. + +4. There is now some optional support for Unicode character properties, which + means that the patterns items such as \p{Lu} and \X can now be used. Only + the general category properties are supported. If PCRE is compiled with this + support, an additional 90K data structure is include, which increases the + size of the library dramatically. + +5. There is support for saving compiled patterns and re-using them later. + +6. There is support for running regular expressions that were compiled on a + different host with the opposite endianness. + +7. The pcretest program has been extended to accommodate the new features. + +The main internal rearrangement is that sequences of literal characters are no +longer handled as strings. Instead, each character is handled on its own. This +makes some UTF-8 handling easier, and makes the support of partial matching +possible. Compiled patterns containing long literal strings will be larger as a +result of this change; I hope that performance will not be much affected. + + +Release 4.5 01-Dec-03 +--------------------- + +Again mainly a bug-fix and tidying release, with only a couple of new features: + +1. It's possible now to compile PCRE so that it does not use recursive +function calls when matching. Instead it gets memory from the heap. This slows +things down, but may be necessary on systems with limited stacks. + +2. UTF-8 string checking has been tightened to reject overlong sequences and to +check that a starting offset points to the start of a character. Failure of the +latter returns a new error code: PCRE_ERROR_BADUTF8_OFFSET. + +3. PCRE can now be compiled for systems that use EBCDIC code. + + +Release 4.4 21-Aug-03 +--------------------- + +This is mainly a bug-fix and tidying release. The only new feature is that PCRE +checks UTF-8 strings for validity by default. There is an option to suppress +this, just in case anybody wants that teeny extra bit of performance. + + +Releases 4.1 - 4.3 +------------------ + +Sorry, I forgot about updating the NEWS file for these releases. Please take a +look at ChangeLog. + + +Release 4.0 17-Feb-03 +--------------------- + +There have been a lot of changes for the 4.0 release, adding additional +functionality and mending bugs. Below is a list of the highlights of the new +functionality. For full details of these features, please consult the +documentation. For a complete list of changes, see the ChangeLog file. + +1. Support for Perl's \Q...\E escapes. + +2. "Possessive quantifiers" ?+, *+, ++, and {,}+ which come from Sun's Java +package. They provide some syntactic sugar for simple cases of "atomic +grouping". + +3. Support for the \G assertion. It is true when the current matching position +is at the start point of the match. + +4. A new feature that provides some of the functionality that Perl provides +with (?{...}). The facility is termed a "callout". The way it is done in PCRE +is for the caller to provide an optional function, by setting pcre_callout to +its entry point. To get the function called, the regex must include (?C) at +appropriate points. + +5. Support for recursive calls to individual subpatterns. This makes it really +easy to get totally confused. + +6. Support for named subpatterns. The Python syntax (?P...) is used to +name a group. + +7. Several extensions to UTF-8 support; it is now fairly complete. There is an +option for pcregrep to make it operate in UTF-8 mode. + +8. The single man page has been split into a number of separate man pages. +These also give rise to individual HTML pages which are put in a separate +directory. There is an index.html page that lists them all. Some hyperlinking +between the pages has been installed. + + +Release 3.5 15-Aug-01 +--------------------- + +1. The configuring system has been upgraded to use later versions of autoconf +and libtool. By default it builds both a shared and a static library if the OS +supports it. You can use --disable-shared or --disable-static on the configure +command if you want only one of them. + +2. The pcretest utility is now installed along with pcregrep because it is +useful for users (to test regexs) and by doing this, it automatically gets +relinked by libtool. The documentation has been turned into a man page, so +there are now .1, .txt, and .html versions in /doc. + +3. Upgrades to pcregrep: + (i) Added long-form option names like gnu grep. + (ii) Added --help to list all options with an explanatory phrase. + (iii) Added -r, --recursive to recurse into sub-directories. + (iv) Added -f, --file to read patterns from a file. + +4. Added --enable-newline-is-cr and --enable-newline-is-lf to the configure +script, to force use of CR or LF instead of \n in the source. On non-Unix +systems, the value can be set in config.h. + +5. The limit of 200 on non-capturing parentheses is a _nesting_ limit, not an +absolute limit. Changed the text of the error message to make this clear, and +likewise updated the man page. + +6. The limit of 99 on the number of capturing subpatterns has been removed. +The new limit is 65535, which I hope will not be a "real" limit. + + +Release 3.3 01-Aug-00 +--------------------- + +There is some support for UTF-8 character strings. This is incomplete and +experimental. The documentation describes what is and what is not implemented. +Otherwise, this is just a bug-fixing release. + + +Release 3.0 01-Feb-00 +--------------------- + +1. A "configure" script is now used to configure PCRE for Unix systems. It +builds a Makefile, a config.h file, and the pcre-config script. + +2. PCRE is built as a shared library by default. + +3. There is support for POSIX classes such as [:alpha:]. + +5. There is an experimental recursion feature. + +---------------------------------------------------------------------------- + IMPORTANT FOR THOSE UPGRADING FROM VERSIONS BEFORE 2.00 + +Please note that there has been a change in the API such that a larger +ovector is required at matching time, to provide some additional workspace. +The new man page has details. This change was necessary in order to support +some of the new functionality in Perl 5.005. + + IMPORTANT FOR THOSE UPGRADING FROM VERSION 2.00 + +Another (I hope this is the last!) change has been made to the API for the +pcre_compile() function. An additional argument has been added to make it +possible to pass over a pointer to character tables built in the current +locale by pcre_maketables(). To use the default tables, this new argument +should be passed as NULL. + + IMPORTANT FOR THOSE UPGRADING FROM VERSION 2.05 + +Yet another (and again I hope this really is the last) change has been made +to the API for the pcre_exec() function. An additional argument has been +added to make it possible to start the match other than at the start of the +subject string. This is important if there are lookbehinds. The new man +page has the details, but you just want to convert existing programs, all +you need to do is to stick in a new fifth argument to pcre_exec(), with a +value of zero. For example, change + + pcre_exec(pattern, extra, subject, length, options, ovec, ovecsize) +to + pcre_exec(pattern, extra, subject, length, 0, options, ovec, ovecsize) + +**** diff --git a/src/third_party/pcre-8.41/NON-AUTOTOOLS-BUILD b/src/third_party/pcre-8.41/NON-AUTOTOOLS-BUILD new file mode 100644 index 00000000000..3910059106b --- /dev/null +++ b/src/third_party/pcre-8.41/NON-AUTOTOOLS-BUILD @@ -0,0 +1,772 @@ +Building PCRE without using autotools +------------------------------------- + +NOTE: This document relates to PCRE releases that use the original API, with +library names libpcre, libpcre16, and libpcre32. January 2015 saw the first +release of a new API, known as PCRE2, with release numbers starting at 10.00 +and library names libpcre2-8, libpcre2-16, and libpcre2-32. The old libraries +(now called PCRE1) are still being maintained for bug fixes, but there will be +no new development. New projects are advised to use the new PCRE2 libraries. + + +This document contains the following sections: + + General + Generic instructions for the PCRE C library + The C++ wrapper functions + Building for virtual Pascal + Stack size in Windows environments + Linking programs in Windows environments + Calling conventions in Windows environments + Comments about Win32 builds + Building PCRE on Windows with CMake + Use of relative paths with CMake on Windows + Testing with RunTest.bat + Building under Windows CE with Visual Studio 200x + Building under Windows with BCC5.5 + Building using Borland C++ Builder 2007 (CB2007) and higher + Building PCRE on OpenVMS + Building PCRE on Stratus OpenVOS + Building PCRE on native z/OS and z/VM + + +GENERAL + +I (Philip Hazel) have no experience of Windows or VMS sytems and how their +libraries work. The items in the PCRE distribution and Makefile that relate to +anything other than Linux systems are untested by me. + +There are some other comments and files (including some documentation in CHM +format) in the Contrib directory on the FTP site: + + ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/Contrib + +The basic PCRE library consists entirely of code written in Standard C, and so +should compile successfully on any system that has a Standard C compiler and +library. The C++ wrapper functions are a separate issue (see below). + +The PCRE distribution includes a "configure" file for use by the configure/make +(autotools) build system, as found in many Unix-like environments. The README +file contains information about the options for "configure". + +There is also support for CMake, which some users prefer, especially in Windows +environments, though it can also be run in Unix-like environments. See the +section entitled "Building PCRE on Windows with CMake" below. + +Versions of config.h and pcre.h are distributed in the PCRE tarballs under the +names config.h.generic and pcre.h.generic. These are provided for those who +build PCRE without using "configure" or CMake. If you use "configure" or CMake, +the .generic versions are not used. + + +GENERIC INSTRUCTIONS FOR THE PCRE C LIBRARY + +The following are generic instructions for building the PCRE C library "by +hand". If you are going to use CMake, this section does not apply to you; you +can skip ahead to the CMake section. + + (1) Copy or rename the file config.h.generic as config.h, and edit the macro + settings that it contains to whatever is appropriate for your environment. + + In particular, you can alter the definition of the NEWLINE macro to + specify what character(s) you want to be interpreted as line terminators. + In an EBCDIC environment, you MUST change NEWLINE, because its default + value is 10, an ASCII LF. The usual EBCDIC newline character is 21 (0x15, + NL), though in some cases it may be 37 (0x25). + + When you compile any of the PCRE modules, you must specify -DHAVE_CONFIG_H + to your compiler so that config.h is included in the sources. + + An alternative approach is not to edit config.h, but to use -D on the + compiler command line to make any changes that you need to the + configuration options. In this case -DHAVE_CONFIG_H must not be set. + + NOTE: There have been occasions when the way in which certain parameters + in config.h are used has changed between releases. (In the configure/make + world, this is handled automatically.) When upgrading to a new release, + you are strongly advised to review config.h.generic before re-using what + you had previously. + + (2) Copy or rename the file pcre.h.generic as pcre.h. + + (3) EITHER: + Copy or rename file pcre_chartables.c.dist as pcre_chartables.c. + + OR: + Compile dftables.c as a stand-alone program (using -DHAVE_CONFIG_H if + you have set up config.h), and then run it with the single argument + "pcre_chartables.c". This generates a set of standard character tables + and writes them to that file. The tables are generated using the default + C locale for your system. If you want to use a locale that is specified + by LC_xxx environment variables, add the -L option to the dftables + command. You must use this method if you are building on a system that + uses EBCDIC code. + + The tables in pcre_chartables.c are defaults. The caller of PCRE can + specify alternative tables at run time. + + (4) Ensure that you have the following header files: + + pcre_internal.h + ucp.h + + (5) For an 8-bit library, compile the following source files, setting + -DHAVE_CONFIG_H as a compiler option if you have set up config.h with your + configuration, or else use other -D settings to change the configuration + as required. + + pcre_byte_order.c + pcre_chartables.c + pcre_compile.c + pcre_config.c + pcre_dfa_exec.c + pcre_exec.c + pcre_fullinfo.c + pcre_get.c + pcre_globals.c + pcre_jit_compile.c + pcre_maketables.c + pcre_newline.c + pcre_ord2utf8.c + pcre_refcount.c + pcre_string_utils.c + pcre_study.c + pcre_tables.c + pcre_ucd.c + pcre_valid_utf8.c + pcre_version.c + pcre_xclass.c + + Make sure that you include -I. in the compiler command (or equivalent for + an unusual compiler) so that all included PCRE header files are first + sought in the current directory. Otherwise you run the risk of picking up + a previously-installed file from somewhere else. + + Note that you must still compile pcre_jit_compile.c, even if you have not + defined SUPPORT_JIT in config.h, because when JIT support is not + configured, dummy functions are compiled. When JIT support IS configured, + pcre_jit_compile.c #includes sources from the sljit subdirectory, where + there should be 16 files, all of whose names begin with "sljit". + + (6) Now link all the compiled code into an object library in whichever form + your system keeps such libraries. This is the basic PCRE C 8-bit library. + If your system has static and shared libraries, you may have to do this + once for each type. + + (7) If you want to build a 16-bit library (as well as, or instead of the 8-bit + or 32-bit libraries) repeat steps 5-6 with the following files: + + pcre16_byte_order.c + pcre16_chartables.c + pcre16_compile.c + pcre16_config.c + pcre16_dfa_exec.c + pcre16_exec.c + pcre16_fullinfo.c + pcre16_get.c + pcre16_globals.c + pcre16_jit_compile.c + pcre16_maketables.c + pcre16_newline.c + pcre16_ord2utf16.c + pcre16_refcount.c + pcre16_string_utils.c + pcre16_study.c + pcre16_tables.c + pcre16_ucd.c + pcre16_utf16_utils.c + pcre16_valid_utf16.c + pcre16_version.c + pcre16_xclass.c + + (8) If you want to build a 32-bit library (as well as, or instead of the 8-bit + or 16-bit libraries) repeat steps 5-6 with the following files: + + pcre32_byte_order.c + pcre32_chartables.c + pcre32_compile.c + pcre32_config.c + pcre32_dfa_exec.c + pcre32_exec.c + pcre32_fullinfo.c + pcre32_get.c + pcre32_globals.c + pcre32_jit_compile.c + pcre32_maketables.c + pcre32_newline.c + pcre32_ord2utf32.c + pcre32_refcount.c + pcre32_string_utils.c + pcre32_study.c + pcre32_tables.c + pcre32_ucd.c + pcre32_utf32_utils.c + pcre32_valid_utf32.c + pcre32_version.c + pcre32_xclass.c + + (9) If you want to build the POSIX wrapper functions (which apply only to the + 8-bit library), ensure that you have the pcreposix.h file and then compile + pcreposix.c (remembering -DHAVE_CONFIG_H if necessary). Link the result + (on its own) as the pcreposix library. + +(10) The pcretest program can be linked with any combination of the 8-bit, + 16-bit and 32-bit libraries (depending on what you selected in config.h). + Compile pcretest.c and pcre_printint.c (again, don't forget + -DHAVE_CONFIG_H) and link them together with the appropriate library/ies. + If you compiled an 8-bit library, pcretest also needs the pcreposix + wrapper library unless you compiled it with -DNOPOSIX. + +(11) Run pcretest on the testinput files in the testdata directory, and check + that the output matches the corresponding testoutput files. There are + comments about what each test does in the section entitled "Testing PCRE" + in the README file. If you compiled more than one of the 8-bit, 16-bit and + 32-bit libraries, you need to run pcretest with the -16 option to do + 16-bit tests and with the -32 option to do 32-bit tests. + + Some tests are relevant only when certain build-time options are selected. + For example, test 4 is for UTF-8/UTF-16/UTF-32 support, and will not run + if you have built PCRE without it. See the comments at the start of each + testinput file. If you have a suitable Unix-like shell, the RunTest script + will run the appropriate tests for you. The command "RunTest list" will + output a list of all the tests. + + Note that the supplied files are in Unix format, with just LF characters + as line terminators. You may need to edit them to change this if your + system uses a different convention. If you are using Windows, you probably + should use the wintestinput3 file instead of testinput3 (and the + corresponding output file). This is a locale test; wintestinput3 sets the + locale to "french" rather than "fr_FR", and there some minor output + differences. + +(12) If you have built PCRE with SUPPORT_JIT, the JIT features will be tested + by the testdata files. However, you might also like to build and run + the freestanding JIT test program, pcre_jit_test.c. + +(13) If you want to use the pcregrep command, compile and link pcregrep.c; it + uses only the basic 8-bit PCRE library (it does not need the pcreposix + library). + + +THE C++ WRAPPER FUNCTIONS + +The PCRE distribution also contains some C++ wrapper functions and tests, +applicable to the 8-bit library, which were contributed by Google Inc. On a +system that can use "configure" and "make", the functions are automatically +built into a library called pcrecpp. It should be straightforward to compile +the .cc files manually on other systems. The files called xxx_unittest.cc are +test programs for each of the corresponding xxx.cc files. + + +BUILDING FOR VIRTUAL PASCAL + +A script for building PCRE using Borland's C++ compiler for use with VPASCAL +was contributed by Alexander Tokarev. Stefan Weber updated the script and added +additional files. The following files in the distribution are for building PCRE +for use with VP/Borland: makevp_c.txt, makevp_l.txt, makevp.bat, pcregexp.pas. + + +STACK SIZE IN WINDOWS ENVIRONMENTS + +The default processor stack size of 1Mb in some Windows environments is too +small for matching patterns that need much recursion. In particular, test 2 may +fail because of this. Normally, running out of stack causes a crash, but there +have been cases where the test program has just died silently. See your linker +documentation for how to increase stack size if you experience problems. The +Linux default of 8Mb is a reasonable choice for the stack, though even that can +be too small for some pattern/subject combinations. + +PCRE has a compile configuration option to disable the use of stack for +recursion so that heap is used instead. However, pattern matching is +significantly slower when this is done. There is more about stack usage in the +"pcrestack" documentation. + + +LINKING PROGRAMS IN WINDOWS ENVIRONMENTS + +If you want to statically link a program against a PCRE library in the form of +a non-dll .a file, you must define PCRE_STATIC before including pcre.h or +pcrecpp.h, otherwise the pcre_malloc() and pcre_free() exported functions will +be declared __declspec(dllimport), with unwanted results. + + +CALLING CONVENTIONS IN WINDOWS ENVIRONMENTS + +It is possible to compile programs to use different calling conventions using +MSVC. Search the web for "calling conventions" for more information. To make it +easier to change the calling convention for the exported functions in the +PCRE library, the macro PCRE_CALL_CONVENTION is present in all the external +definitions. It can be set externally when compiling (e.g. in CFLAGS). If it is +not set, it defaults to empty; the default calling convention is then used +(which is what is wanted most of the time). + + +COMMENTS ABOUT WIN32 BUILDS (see also "BUILDING PCRE ON WINDOWS WITH CMAKE") + +There are two ways of building PCRE using the "configure, make, make install" +paradigm on Windows systems: using MinGW or using Cygwin. These are not at all +the same thing; they are completely different from each other. There is also +support for building using CMake, which some users find a more straightforward +way of building PCRE under Windows. + +The MinGW home page (http://www.mingw.org/) says this: + + MinGW: A collection of freely available and freely distributable Windows + specific header files and import libraries combined with GNU toolsets that + allow one to produce native Windows programs that do not rely on any + 3rd-party C runtime DLLs. + +The Cygwin home page (http://www.cygwin.com/) says this: + + Cygwin is a Linux-like environment for Windows. It consists of two parts: + + . A DLL (cygwin1.dll) which acts as a Linux API emulation layer providing + substantial Linux API functionality + + . A collection of tools which provide Linux look and feel. + + The Cygwin DLL currently works with all recent, commercially released x86 32 + bit and 64 bit versions of Windows, with the exception of Windows CE. + +On both MinGW and Cygwin, PCRE should build correctly using: + + ./configure && make && make install + +This should create two libraries called libpcre and libpcreposix, and, if you +have enabled building the C++ wrapper, a third one called libpcrecpp. These are +independent libraries: when you link with libpcreposix or libpcrecpp you must +also link with libpcre, which contains the basic functions. (Some earlier +releases of PCRE included the basic libpcre functions in libpcreposix. This no +longer happens.) + +A user submitted a special-purpose patch that makes it easy to create +"pcre.dll" under mingw32 using the "msys" environment. It provides "pcre.dll" +as a special target. If you use this target, no other files are built, and in +particular, the pcretest and pcregrep programs are not built. An example of how +this might be used is: + + ./configure --enable-utf --disable-cpp CFLAGS="-03 -s"; make pcre.dll + +Using Cygwin's compiler generates libraries and executables that depend on +cygwin1.dll. If a library that is generated this way is distributed, +cygwin1.dll has to be distributed as well. Since cygwin1.dll is under the GPL +licence, this forces not only PCRE to be under the GPL, but also the entire +application. A distributor who wants to keep their own code proprietary must +purchase an appropriate Cygwin licence. + +MinGW has no such restrictions. The MinGW compiler generates a library or +executable that can run standalone on Windows without any third party dll or +licensing issues. + +But there is more complication: + +If a Cygwin user uses the -mno-cygwin Cygwin gcc flag, what that really does is +to tell Cygwin's gcc to use the MinGW gcc. Cygwin's gcc is only acting as a +front end to MinGW's gcc (if you install Cygwin's gcc, you get both Cygwin's +gcc and MinGW's gcc). So, a user can: + +. Build native binaries by using MinGW or by getting Cygwin and using + -mno-cygwin. + +. Build binaries that depend on cygwin1.dll by using Cygwin with the normal + compiler flags. + +The test files that are supplied with PCRE are in UNIX format, with LF +characters as line terminators. Unless your PCRE library uses a default newline +option that includes LF as a valid newline, it may be necessary to change the +line terminators in the test files to get some of the tests to work. + + +BUILDING PCRE ON WINDOWS WITH CMAKE + +CMake is an alternative configuration facility that can be used instead of +"configure". CMake creates project files (make files, solution files, etc.) +tailored to numerous development environments, including Visual Studio, +Borland, Msys, MinGW, NMake, and Unix. If possible, use short paths with no +spaces in the names for your CMake installation and your PCRE source and build +directories. + +The following instructions were contributed by a PCRE user. If they are not +followed exactly, errors may occur. In the event that errors do occur, it is +recommended that you delete the CMake cache before attempting to repeat the +CMake build process. In the CMake GUI, the cache can be deleted by selecting +"File > Delete Cache". + +1. Install the latest CMake version available from http://www.cmake.org/, and + ensure that cmake\bin is on your path. + +2. Unzip (retaining folder structure) the PCRE source tree into a source + directory such as C:\pcre. You should ensure your local date and time + is not earlier than the file dates in your source dir if the release is + very new. + +3. Create a new, empty build directory, preferably a subdirectory of the + source dir. For example, C:\pcre\pcre-xx\build. + +4. Run cmake-gui from the Shell envirornment of your build tool, for example, + Msys for Msys/MinGW or Visual Studio Command Prompt for VC/VC++. Do not try + to start Cmake from the Windows Start menu, as this can lead to errors. + +5. Enter C:\pcre\pcre-xx and C:\pcre\pcre-xx\build for the source and build + directories, respectively. + +6. Hit the "Configure" button. + +7. Select the particular IDE / build tool that you are using (Visual + Studio, MSYS makefiles, MinGW makefiles, etc.) + +8. The GUI will then list several configuration options. This is where + you can enable UTF-8 support or other PCRE optional features. + +9. Hit "Configure" again. The adjacent "Generate" button should now be + active. + +10. Hit "Generate". + +11. The build directory should now contain a usable build system, be it a + solution file for Visual Studio, makefiles for MinGW, etc. Exit from + cmake-gui and use the generated build system with your compiler or IDE. + E.g., for MinGW you can run "make", or for Visual Studio, open the PCRE + solution, select the desired configuration (Debug, or Release, etc.) and + build the ALL_BUILD project. + +12. If during configuration with cmake-gui you've elected to build the test + programs, you can execute them by building the test project. E.g., for + MinGW: "make test"; for Visual Studio build the RUN_TESTS project. The + most recent build configuration is targeted by the tests. A summary of + test results is presented. Complete test output is subsequently + available for review in Testing\Temporary under your build dir. + + +USE OF RELATIVE PATHS WITH CMAKE ON WINDOWS + +A PCRE user comments as follows: I thought that others may want to know the +current state of CMAKE_USE_RELATIVE_PATHS support on Windows. Here it is: + +-- AdditionalIncludeDirectories is only partially modified (only the + first path - see below) +-- Only some of the contained file paths are modified - shown below for + pcre.vcproj +-- It properly modifies + +I am sure CMake people can fix that if they want to. Until then one will +need to replace existing absolute paths in project files with relative +paths manually (e.g. from VS) - relative to project file location. I did +just that before being told to try CMAKE_USE_RELATIVE_PATHS. Not a big +deal. + +AdditionalIncludeDirectories="E:\builds\pcre\build;E:\builds\pcre\pcre-7.5;" +AdditionalIncludeDirectories=".;E:\builds\pcre\pcre-7.5;" + +RelativePath="pcre.h" +RelativePath="pcre_chartables.c" +RelativePath="pcre_chartables.c.rule" + + +TESTING WITH RUNTEST.BAT + +If configured with CMake, building the test project ("make test" or building +ALL_TESTS in Visual Studio) creates (and runs) pcre_test.bat (and depending +on your configuration options, possibly other test programs) in the build +directory. Pcre_test.bat runs RunTest.Bat with correct source and exe paths. + +For manual testing with RunTest.bat, provided the build dir is a subdirectory +of the source directory: Open command shell window. Chdir to the location +of your pcretest.exe and pcregrep.exe programs. Call RunTest.bat with +"..\RunTest.Bat" or "..\..\RunTest.bat" as appropriate. + +To run only a particular test with RunTest.Bat provide a test number argument. + +Otherwise: + +1. Copy RunTest.bat into the directory where pcretest.exe and pcregrep.exe + have been created. + +2. Edit RunTest.bat to indentify the full or relative location of + the pcre source (wherein which the testdata folder resides), e.g.: + + set srcdir=C:\pcre\pcre-8.20 + +3. In a Windows command environment, chdir to the location of your bat and + exe programs. + +4. Run RunTest.bat. Test outputs will automatically be compared to expected + results, and discrepancies will be identified in the console output. + +To independently test the just-in-time compiler, run pcre_jit_test.exe. +To test pcrecpp, run pcrecpp_unittest.exe, pcre_stringpiece_unittest.exe and +pcre_scanner_unittest.exe. + + +BUILDING UNDER WINDOWS CE WITH VISUAL STUDIO 200x + +Vincent Richomme sent a zip archive of files to help with this process. They +can be found in the file "pcre-vsbuild.zip" in the Contrib directory of the FTP +site. + + +BUILDING UNDER WINDOWS WITH BCC5.5 + +Michael Roy sent these comments about building PCRE under Windows with BCC5.5: + +Some of the core BCC libraries have a version of PCRE from 1998 built in, which +can lead to pcre_exec() giving an erroneous PCRE_ERROR_NULL from a version +mismatch. I'm including an easy workaround below, if you'd like to include it +in the non-unix instructions: + +When linking a project with BCC5.5, pcre.lib must be included before any of the +libraries cw32.lib, cw32i.lib, cw32mt.lib, and cw32mti.lib on the command line. + + +BUILDING USING BORLAND C++ BUILDER 2007 (CB2007) AND HIGHER + +A PCRE user sent these comments about this environment (see also the comment +from another user that follows them): + +The XE versions of C++ Builder come with a RegularExpressionsCore class which +contain a version of TPerlRegEx. However, direct use of the C PCRE library may +be desirable. + +The default makevp.bat, however, supplied with PCRE builds a version of PCRE +that is not usable with any version of C++ Builder because the compiler ships +with an embedded version of PCRE, version 2.01 from 1998! [See also the note +about BCC5.5 above.] If you want to use PCRE you'll need to rename the +functions (pcre_compile to pcre_compile_bcc, etc) or do as I have done and just +use the 16 bit versions. I'm using std::wstring everywhere anyway. Since the +embedded version of PCRE does not have the 16 bit function names, there is no +conflict. + +Building PCRE using a C++ Builder static library project file (recommended): + +1. Rename or remove pcre.h, pcreposi.h, and pcreposix.h from your C++ Builder +original include path. + +2. Download PCRE from pcre.org and extract to a directory. + +3. Rename pcre_chartables.c.dist to pcre_chartables.c, pcre.h.generic to +pcre.h, and config.h.generic to config.h. + +4. Edit pcre.h and pcre_config.c so that they include config.h. + +5. Edit config.h like so: + +Comment out the following lines: +#define PACKAGE "pcre" +#define PACKAGE_BUGREPORT "" +#define PACKAGE_NAME "PCRE" +#define PACKAGE_STRING "PCRE 8.32" +#define PACKAGE_TARNAME "pcre" +#define PACKAGE_URL "" +#define PACKAGE_VERSION "8.32" + +Add the following lines: +#ifndef SUPPORT_UTF +#define SUPPORT_UTF 100 // any value is fine +#endif + +#ifndef SUPPORT_UCP +#define SUPPORT_UCP 101 // any value is fine +#endif + +#ifndef SUPPORT_UCP +#define SUPPORT_PCRE16 102 // any value is fine +#endif + +#ifndef SUPPORT_UTF8 +#define SUPPORT_UTF8 103 // any value is fine +#endif + +6. Build a C++ Builder project using the IDE. Go to File / New / Other and +choose Static Library. You can name it pcre.cbproj or whatever. Now set your +paths by going to Project / Options. Set the Include path. Do this from the +"Base" option to apply to both Release and Debug builds. Now add the following +files to the project: + +pcre.h +pcre16_byte_order.c +pcre16_chartables.c +pcre16_compile.c +pcre16_config.c +pcre16_dfa_exec.c +pcre16_exec.c +pcre16_fullinfo.c +pcre16_get.c +pcre16_globals.c +pcre16_maketables.c +pcre16_newline.c +pcre16_ord2utf16.c +pcre16_printint.c +pcre16_refcount.c +pcre16_string_utils.c +pcre16_study.c +pcre16_tables.c +pcre16_ucd.c +pcre16_utf16_utils.c +pcre16_valid_utf16.c +pcre16_version.c +pcre16_xclass.c + +//Optional +pcre_version.c + +7. After compiling the .lib file, copy the .lib and header files to a project +you want to use PCRE with. Enjoy. + +Optional ... Building PCRE using the makevp.bat file: + +1. Edit makevp_c.txt and makevp_l.txt and change all the names to the 16 bit +versions. + +2. Edit makevp.bat and set the path to C++ Builder. Run makevp.bat. + +Another PCRE user added this comment: + +Another approach I successfully used for some years with BCB 5 and 6 was to +make sure that include and library paths of PCRE are configured before the +default paths of the IDE in the dialogs where one can manage those paths. +Afterwards one can open the project files using a text editor and manually add +the self created library for pcre itself, pcrecpp doesn't ship with the IDE, in +the library nodes where the IDE manages its own libraries to link against in +front of the IDE-own libraries. This way one can use the default PCRE function +names without getting access violations on runtime. + + + + +BUILDING PCRE ON OPENVMS + +Stephen Hoffman sent the following, in December 2012: + +"Here is a very short write-up on the +OpenVMS port and here + + + +is a zip with the OpenVMS files, and with one modified testing-related PCRE +file." This is a port of PCRE 8.32. + +Earlier, Dan Mooney sent the following comments about building PCRE on OpenVMS. +They relate to an older version of PCRE that used fewer source files, so the +exact commands will need changing. See the current list of source files above. + +"It was quite easy to compile and link the library. I don't have a formal +make file but the attached file [reproduced below] contains the OpenVMS DCL +commands I used to build the library. I had to add #define +POSIX_MALLOC_THRESHOLD 10 to pcre.h since it was not defined anywhere. + +The library was built on: +O/S: HP OpenVMS v7.3-1 +Compiler: Compaq C v6.5-001-48BCD +Linker: vA13-01 + +The test results did not match 100% due to the issues you mention in your +documentation regarding isprint(), iscntrl(), isgraph() and ispunct(). I +modified some of the character tables temporarily and was able to get the +results to match. Tests using the fr locale did not match since I don't have +that locale loaded. The study size was always reported to be 3 less than the +value in the standard test output files." + +========================= +$! This DCL procedure builds PCRE on OpenVMS +$! +$! I followed the instructions in the non-unix-use file in the distribution. +$! +$ COMPILE == "CC/LIST/NOMEMBER_ALIGNMENT/PREFIX_LIBRARY_ENTRIES=ALL_ENTRIES +$ COMPILE DFTABLES.C +$ LINK/EXE=DFTABLES.EXE DFTABLES.OBJ +$ RUN DFTABLES.EXE/OUTPUT=CHARTABLES.C +$ COMPILE MAKETABLES.C +$ COMPILE GET.C +$ COMPILE STUDY.C +$! I had to set POSIX_MALLOC_THRESHOLD to 10 in PCRE.H since the symbol +$! did not seem to be defined anywhere. +$! I edited pcre.h and added #DEFINE SUPPORT_UTF8 to enable UTF8 support. +$ COMPILE PCRE.C +$ LIB/CREATE PCRE MAKETABLES.OBJ, GET.OBJ, STUDY.OBJ, PCRE.OBJ +$! I had to set POSIX_MALLOC_THRESHOLD to 10 in PCRE.H since the symbol +$! did not seem to be defined anywhere. +$ COMPILE PCREPOSIX.C +$ LIB/CREATE PCREPOSIX PCREPOSIX.OBJ +$ COMPILE PCRETEST.C +$ LINK/EXE=PCRETEST.EXE PCRETEST.OBJ, PCRE/LIB, PCREPOSIX/LIB +$! C programs that want access to command line arguments must be +$! defined as a symbol +$ PCRETEST :== "$ SYS$ROADSUSERS:[DMOONEY.REGEXP]PCRETEST.EXE" +$! Arguments must be enclosed in quotes. +$ PCRETEST "-C" +$! Test results: +$! +$! The test results did not match 100%. The functions isprint(), iscntrl(), +$! isgraph() and ispunct() on OpenVMS must not produce the same results +$! as the system that built the test output files provided with the +$! distribution. +$! +$! The study size did not match and was always 3 less on OpenVMS. +$! +$! Locale could not be set to fr +$! +========================= + + +BUILDING PCRE ON STRATUS OPENVOS + +These notes on the port of PCRE to VOS (lightly edited) were supplied by +Ashutosh Warikoo, whose email address has the local part awarikoo and the +domain nse.co.in. The port was for version 7.9 in August 2009. + +1. Building PCRE + +I built pcre on OpenVOS Release 17.0.1at using GNU Tools 3.4a without any +problems. I used the following packages to build PCRE: + + ftp://ftp.stratus.com/pub/vos/posix/ga/posix.save.evf.gz + +Please read and follow the instructions that come with these packages. To start +the build of pcre, from the root of the package type: + + ./build.sh + +2. Installing PCRE + +Once you have successfully built PCRE, login to the SysAdmin group, switch to +the root user, and type + + [ !create_dir (master_disk)>usr --if needed ] + [ !create_dir (master_disk)>usr>local --if needed ] + !gmake install + +This installs PCRE and its man pages into /usr/local. You can add +(master_disk)>usr>local>bin to your command search paths, or if you are in +BASH, add /usr/local/bin to the PATH environment variable. + +4. Restrictions + +This port requires readline library optionally. However during the build I +faced some yet unexplored errors while linking with readline. As it was an +optional component I chose to disable it. + +5. Known Problems + +I ran the test suite, but you will have to be your own judge of whether this +command, and this port, suits your purposes. If you find any problems that +appear to be related to the port itself, please let me know. Please see the +build.log file in the root of the package also. + + +BUILDING PCRE ON NATIVE Z/OS AND Z/VM + +z/OS and z/VM are operating systems for mainframe computers, produced by IBM. +The character code used is EBCDIC, not ASCII or Unicode. In z/OS, UNIX APIs and +applications can be supported through UNIX System Services, and in such an +environment PCRE can be built in the same way as in other systems. However, in +native z/OS (without UNIX System Services) and in z/VM, special ports are +required. For details, please see this web site: + + http://www.zaconsultants.net + +You may download PCRE from WWW.CBTTAPE.ORG, file 882.  Everything, source and +executable, is in EBCDIC and native z/OS file formats and this is the +recommended download site. + +========================== +Last Updated: 25 June 2015 diff --git a/src/third_party/pcre-8.41/NON-UNIX-USE b/src/third_party/pcre-8.41/NON-UNIX-USE new file mode 100644 index 00000000000..a25546b6ff5 --- /dev/null +++ b/src/third_party/pcre-8.41/NON-UNIX-USE @@ -0,0 +1,7 @@ +Compiling PCRE on non-Unix systems +---------------------------------- + +This has been renamed to better reflect its contents. Please see the file +NON-AUTOTOOLS-BUILD for details of how to build PCRE without using autotools. + +#### diff --git a/src/third_party/pcre-8.41/PrepareRelease b/src/third_party/pcre-8.41/PrepareRelease new file mode 100755 index 00000000000..9891e08d67f --- /dev/null +++ b/src/third_party/pcre-8.41/PrepareRelease @@ -0,0 +1,258 @@ +#/bin/sh + +# Script to prepare the files for building a PCRE release. It does some +# processing of the documentation, detrails files, and creates pcre.h.generic +# and config.h.generic (for use by builders who can't run ./configure). + +# You must run this script before runnning "make dist". If its first argument +# is "doc", it stops after preparing the documentation. There are no other +# arguments. The script makes use of the following files: + +# 132html A Perl script that converts a .1 or .3 man page into HTML. It +# "knows" the relevant troff constructs that are used in the PCRE +# man pages. + +# CheckMan A Perl script that checks man pages for typos in the mark up. + +# CleanTxt A Perl script that cleans up the output of "nroff -man" by +# removing backspaces and other redundant text so as to produce +# a readable .txt file. + +# Detrail A Perl script that removes trailing spaces from files. + +# doc/index.html.src +# A file that is copied as index.html into the doc/html directory +# when the HTML documentation is built. It works like this so that +# doc/html can be deleted and re-created from scratch. + +# README & NON-AUTOTOOLS-BUILD +# These files are copied into the doc/html directory, with .txt +# extensions so that they can by hyperlinked from the HTML +# documentation, because some people just go to the HTML without +# looking for text files. + + +# First, sort out the documentation. Remove pcredemo.3 first because it won't +# pass the markup check (it is created below, using markup that none of the +# other pages use). + +cd doc +echo Processing documentation + +/bin/rm -f pcredemo.3 + +# Check the remaining man pages + +perl ../CheckMan *.1 *.3 +if [ $? != 0 ] ; then exit 1; fi + +# Make Text form of the documentation. It needs some mangling to make it +# tidy for online reading. Concatenate all the .3 stuff, but omit the +# individual function pages. + +cat <pcre.txt +----------------------------------------------------------------------------- +This file contains a concatenation of the PCRE man pages, converted to plain +text format for ease of searching with a text editor, or for use on systems +that do not have a man page processor. The small individual files that give +synopses of each function in the library have not been included. Neither has +the pcredemo program. There are separate text files for the pcregrep and +pcretest commands. +----------------------------------------------------------------------------- + + +End + +echo "Making pcre.txt" +for file in pcre pcre16 pcre32 pcrebuild pcrematching pcreapi pcrecallout \ + pcrecompat pcrepattern pcresyntax pcreunicode pcrejit pcrepartial \ + pcreprecompile pcreperform pcreposix pcrecpp pcresample \ + pcrelimits pcrestack ; do + echo " Processing $file.3" + nroff -c -man $file.3 >$file.rawtxt + perl ../CleanTxt <$file.rawtxt >>pcre.txt + /bin/rm $file.rawtxt + echo "------------------------------------------------------------------------------" >>pcre.txt + if [ "$file" != "pcresample" ] ; then + echo " " >>pcre.txt + echo " " >>pcre.txt + fi +done + +# The three commands +for file in pcretest pcregrep pcre-config ; do + echo Making $file.txt + nroff -c -man $file.1 >$file.rawtxt + perl ../CleanTxt <$file.rawtxt >$file.txt + /bin/rm $file.rawtxt +done + + +# Make pcredemo.3 from the pcredemo.c source file + +echo "Making pcredemo.3" +perl <<"END" >pcredemo.3 + open(IN, "../pcredemo.c") || die "Failed to open pcredemo.c\n"; + open(OUT, ">pcredemo.3") || die "Failed to open pcredemo.3\n"; + print OUT ".\\\" Start example.\n" . + ".de EX\n" . + ". nr mE \\\\n(.f\n" . + ". nf\n" . + ". nh\n" . + ". ft CW\n" . + "..\n" . + ".\n" . + ".\n" . + ".\\\" End example.\n" . + ".de EE\n" . + ". ft \\\\n(mE\n" . + ". fi\n" . + ". hy \\\\n(HY\n" . + "..\n" . + ".\n" . + ".EX\n" ; + while () + { + s/\\/\\e/g; + print OUT; + } + print OUT ".EE\n"; + close(IN); + close(OUT); +END +if [ $? != 0 ] ; then exit 1; fi + + +# Make HTML form of the documentation. + +echo "Making HTML documentation" +/bin/rm html/* +cp index.html.src html/index.html +cp ../README html/README.txt +cp ../NON-AUTOTOOLS-BUILD html/NON-AUTOTOOLS-BUILD.txt + +for file in *.1 ; do + base=`basename $file .1` + echo " Making $base.html" + perl ../132html -toc $base <$file >html/$base.html +done + +# Exclude table of contents for function summaries. It seems that expr +# forces an anchored regex. Also exclude them for small pages that have +# only one section. + +for file in *.3 ; do + base=`basename $file .3` + toc=-toc + if [ `expr $base : '.*_'` -ne 0 ] ; then toc="" ; fi + if [ "$base" = "pcresample" ] || \ + [ "$base" = "pcrestack" ] || \ + [ "$base" = "pcrecompat" ] || \ + [ "$base" = "pcrelimits" ] || \ + [ "$base" = "pcreperform" ] || \ + [ "$base" = "pcreunicode" ] ; then + toc="" + fi + echo " Making $base.html" + perl ../132html $toc $base <$file >html/$base.html + if [ $? != 0 ] ; then exit 1; fi +done + +# End of documentation processing; stop if only documentation required. + +cd .. +echo Documentation done +if [ "$1" = "doc" ] ; then exit; fi + +# These files are detrailed; do not detrail the test data because there may be +# significant trailing spaces. Do not detrail RunTest.bat, because it has CRLF +# line endings and the detrail script removes all trailing white space. The +# configure files are also omitted from the detrailing. We don't bother with +# those pcre[16|32]_xx files that just define COMPILE_PCRE16 and then #include the +# common file, because they aren't going to change. + +files="\ + Makefile.am \ + Makefile.in \ + configure.ac \ + README \ + LICENCE \ + COPYING \ + AUTHORS \ + NEWS \ + NON-UNIX-USE \ + NON-AUTOTOOLS-BUILD \ + INSTALL \ + 132html \ + CleanTxt \ + Detrail \ + ChangeLog \ + CMakeLists.txt \ + RunGrepTest \ + RunTest \ + pcre-config.in \ + libpcre.pc.in \ + libpcre16.pc.in \ + libpcre32.pc.in \ + libpcreposix.pc.in \ + libpcrecpp.pc.in \ + config.h.in \ + pcre_chartables.c.dist \ + pcredemo.c \ + pcregrep.c \ + pcretest.c \ + dftables.c \ + pcreposix.c \ + pcreposix.h \ + pcre.h.in \ + pcre_internal.h \ + pcre_byte_order.c \ + pcre_compile.c \ + pcre_config.c \ + pcre_dfa_exec.c \ + pcre_exec.c \ + pcre_fullinfo.c \ + pcre_get.c \ + pcre_globals.c \ + pcre_jit_compile.c \ + pcre_jit_test.c \ + pcre_maketables.c \ + pcre_newline.c \ + pcre_ord2utf8.c \ + pcre16_ord2utf16.c \ + pcre32_ord2utf32.c \ + pcre_printint.c \ + pcre_refcount.c \ + pcre_string_utils.c \ + pcre_study.c \ + pcre_tables.c \ + pcre_valid_utf8.c \ + pcre_version.c \ + pcre_xclass.c \ + pcre16_utf16_utils.c \ + pcre32_utf32_utils.c \ + pcre16_valid_utf16.c \ + pcre32_valid_utf32.c \ + pcre_scanner.cc \ + pcre_scanner.h \ + pcre_scanner_unittest.cc \ + pcrecpp.cc \ + pcrecpp.h \ + pcrecpparg.h.in \ + pcrecpp_unittest.cc \ + pcre_stringpiece.cc \ + pcre_stringpiece.h.in \ + pcre_stringpiece_unittest.cc \ + perltest.pl \ + ucp.h \ + makevp.bat \ + pcre.def \ + libpcre.def \ + libpcreposix.def" + +echo Detrailing +perl ./Detrail $files doc/p* doc/html/* + +echo Done + +#End diff --git a/src/third_party/pcre-8.41/README b/src/third_party/pcre-8.41/README new file mode 100644 index 00000000000..4887ebf350e --- /dev/null +++ b/src/third_party/pcre-8.41/README @@ -0,0 +1,1002 @@ +README file for PCRE (Perl-compatible regular expression library) +----------------------------------------------------------------- + +NOTE: This set of files relates to PCRE releases that use the original API, +with library names libpcre, libpcre16, and libpcre32. January 2015 saw the +first release of a new API, known as PCRE2, with release numbers starting at +10.00 and library names libpcre2-8, libpcre2-16, and libpcre2-32. The old +libraries (now called PCRE1) are still being maintained for bug fixes, but +there will be no new development. New projects are advised to use the new PCRE2 +libraries. + + +The latest release of PCRE1 is always available in three alternative formats +from: + + ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-xxx.tar.gz + ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-xxx.tar.bz2 + ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-xxx.zip + +There is a mailing list for discussion about the development of PCRE at +pcre-dev@exim.org. You can access the archives and subscribe or manage your +subscription here: + + https://lists.exim.org/mailman/listinfo/pcre-dev + +Please read the NEWS file if you are upgrading from a previous release. +The contents of this README file are: + + The PCRE APIs + Documentation for PCRE + Contributions by users of PCRE + Building PCRE on non-Unix-like systems + Building PCRE without using autotools + Building PCRE using autotools + Retrieving configuration information + Shared libraries + Cross-compiling using autotools + Using HP's ANSI C++ compiler (aCC) + Compiling in Tru64 using native compilers + Using Sun's compilers for Solaris + Using PCRE from MySQL + Making new tarballs + Testing PCRE + Character tables + File manifest + + +The PCRE APIs +------------- + +PCRE is written in C, and it has its own API. There are three sets of +functions, one for the 8-bit library, which processes strings of bytes, one for +the 16-bit library, which processes strings of 16-bit values, and one for the +32-bit library, which processes strings of 32-bit values. The distribution also +includes a set of C++ wrapper functions (see the pcrecpp man page for details), +courtesy of Google Inc., which can be used to call the 8-bit PCRE library from +C++. Other C++ wrappers have been created from time to time. See, for example: +https://github.com/YasserAsmi/regexp, which aims to be simple and similar in +style to the C API. + +The distribution also contains a set of C wrapper functions (again, just for +the 8-bit library) that are based on the POSIX regular expression API (see the +pcreposix man page). These end up in the library called libpcreposix. Note that +this just provides a POSIX calling interface to PCRE; the regular expressions +themselves still follow Perl syntax and semantics. The POSIX API is restricted, +and does not give full access to all of PCRE's facilities. + +The header file for the POSIX-style functions is called pcreposix.h. The +official POSIX name is regex.h, but I did not want to risk possible problems +with existing files of that name by distributing it that way. To use PCRE with +an existing program that uses the POSIX API, pcreposix.h will have to be +renamed or pointed at by a link. + +If you are using the POSIX interface to PCRE and there is already a POSIX regex +library installed on your system, as well as worrying about the regex.h header +file (as mentioned above), you must also take care when linking programs to +ensure that they link with PCRE's libpcreposix library. Otherwise they may pick +up the POSIX functions of the same name from the other library. + +One way of avoiding this confusion is to compile PCRE with the addition of +-Dregcomp=PCREregcomp (and similarly for the other POSIX functions) to the +compiler flags (CFLAGS if you are using "configure" -- see below). This has the +effect of renaming the functions so that the names no longer clash. Of course, +you have to do the same thing for your applications, or write them using the +new names. + + +Documentation for PCRE +---------------------- + +If you install PCRE in the normal way on a Unix-like system, you will end up +with a set of man pages whose names all start with "pcre". The one that is just +called "pcre" lists all the others. In addition to these man pages, the PCRE +documentation is supplied in two other forms: + + 1. There are files called doc/pcre.txt, doc/pcregrep.txt, and + doc/pcretest.txt in the source distribution. The first of these is a + concatenation of the text forms of all the section 3 man pages except + the listing of pcredemo.c and those that summarize individual functions. + The other two are the text forms of the section 1 man pages for the + pcregrep and pcretest commands. These text forms are provided for ease of + scanning with text editors or similar tools. They are installed in + /share/doc/pcre, where is the installation prefix + (defaulting to /usr/local). + + 2. A set of files containing all the documentation in HTML form, hyperlinked + in various ways, and rooted in a file called index.html, is distributed in + doc/html and installed in /share/doc/pcre/html. + +Users of PCRE have contributed files containing the documentation for various +releases in CHM format. These can be found in the Contrib directory of the FTP +site (see next section). + + +Contributions by users of PCRE +------------------------------ + +You can find contributions from PCRE users in the directory + + ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/Contrib + +There is a README file giving brief descriptions of what they are. Some are +complete in themselves; others are pointers to URLs containing relevant files. +Some of this material is likely to be well out-of-date. Several of the earlier +contributions provided support for compiling PCRE on various flavours of +Windows (I myself do not use Windows). Nowadays there is more Windows support +in the standard distribution, so these contibutions have been archived. + +A PCRE user maintains downloadable Windows binaries of the pcregrep and +pcretest programs here: + + http://www.rexegg.com/pcregrep-pcretest.html + + +Building PCRE on non-Unix-like systems +-------------------------------------- + +For a non-Unix-like system, please read the comments in the file +NON-AUTOTOOLS-BUILD, though if your system supports the use of "configure" and +"make" you may be able to build PCRE using autotools in the same way as for +many Unix-like systems. + +PCRE can also be configured using the GUI facility provided by CMake's +cmake-gui command. This creates Makefiles, solution files, etc. The file +NON-AUTOTOOLS-BUILD has information about CMake. + +PCRE has been compiled on many different operating systems. It should be +straightforward to build PCRE on any system that has a Standard C compiler and +library, because it uses only Standard C functions. + + +Building PCRE without using autotools +------------------------------------- + +The use of autotools (in particular, libtool) is problematic in some +environments, even some that are Unix or Unix-like. See the NON-AUTOTOOLS-BUILD +file for ways of building PCRE without using autotools. + + +Building PCRE using autotools +----------------------------- + +If you are using HP's ANSI C++ compiler (aCC), please see the special note +in the section entitled "Using HP's ANSI C++ compiler (aCC)" below. + +The following instructions assume the use of the widely used "configure; make; +make install" (autotools) process. + +To build PCRE on system that supports autotools, first run the "configure" +command from the PCRE distribution directory, with your current directory set +to the directory where you want the files to be created. This command is a +standard GNU "autoconf" configuration script, for which generic instructions +are supplied in the file INSTALL. + +Most commonly, people build PCRE within its own distribution directory, and in +this case, on many systems, just running "./configure" is sufficient. However, +the usual methods of changing standard defaults are available. For example: + +CFLAGS='-O2 -Wall' ./configure --prefix=/opt/local + +This command specifies that the C compiler should be run with the flags '-O2 +-Wall' instead of the default, and that "make install" should install PCRE +under /opt/local instead of the default /usr/local. + +If you want to build in a different directory, just run "configure" with that +directory as current. For example, suppose you have unpacked the PCRE source +into /source/pcre/pcre-xxx, but you want to build it in /build/pcre/pcre-xxx: + +cd /build/pcre/pcre-xxx +/source/pcre/pcre-xxx/configure + +PCRE is written in C and is normally compiled as a C library. However, it is +possible to build it as a C++ library, though the provided building apparatus +does not have any features to support this. + +There are some optional features that can be included or omitted from the PCRE +library. They are also documented in the pcrebuild man page. + +. By default, both shared and static libraries are built. You can change this + by adding one of these options to the "configure" command: + + --disable-shared + --disable-static + + (See also "Shared libraries on Unix-like systems" below.) + +. By default, only the 8-bit library is built. If you add --enable-pcre16 to + the "configure" command, the 16-bit library is also built. If you add + --enable-pcre32 to the "configure" command, the 32-bit library is also built. + If you want only the 16-bit or 32-bit library, use --disable-pcre8 to disable + building the 8-bit library. + +. If you are building the 8-bit library and want to suppress the building of + the C++ wrapper library, you can add --disable-cpp to the "configure" + command. Otherwise, when "configure" is run without --disable-pcre8, it will + try to find a C++ compiler and C++ header files, and if it succeeds, it will + try to build the C++ wrapper. + +. If you want to include support for just-in-time compiling, which can give + large performance improvements on certain platforms, add --enable-jit to the + "configure" command. This support is available only for certain hardware + architectures. If you try to enable it on an unsupported architecture, there + will be a compile time error. + +. When JIT support is enabled, pcregrep automatically makes use of it, unless + you add --disable-pcregrep-jit to the "configure" command. + +. If you want to make use of the support for UTF-8 Unicode character strings in + the 8-bit library, or UTF-16 Unicode character strings in the 16-bit library, + or UTF-32 Unicode character strings in the 32-bit library, you must add + --enable-utf to the "configure" command. Without it, the code for handling + UTF-8, UTF-16 and UTF-8 is not included in the relevant library. Even + when --enable-utf is included, the use of a UTF encoding still has to be + enabled by an option at run time. When PCRE is compiled with this option, its + input can only either be ASCII or UTF-8/16/32, even when running on EBCDIC + platforms. It is not possible to use both --enable-utf and --enable-ebcdic at + the same time. + +. There are no separate options for enabling UTF-8, UTF-16 and UTF-32 + independently because that would allow ridiculous settings such as requesting + UTF-16 support while building only the 8-bit library. However, the option + --enable-utf8 is retained for backwards compatibility with earlier releases + that did not support 16-bit or 32-bit character strings. It is synonymous with + --enable-utf. It is not possible to configure one library with UTF support + and the other without in the same configuration. + +. If, in addition to support for UTF-8/16/32 character strings, you want to + include support for the \P, \p, and \X sequences that recognize Unicode + character properties, you must add --enable-unicode-properties to the + "configure" command. This adds about 30K to the size of the library (in the + form of a property table); only the basic two-letter properties such as Lu + are supported. + +. You can build PCRE to recognize either CR or LF or the sequence CRLF or any + of the preceding, or any of the Unicode newline sequences as indicating the + end of a line. Whatever you specify at build time is the default; the caller + of PCRE can change the selection at run time. The default newline indicator + is a single LF character (the Unix standard). You can specify the default + newline indicator by adding --enable-newline-is-cr or --enable-newline-is-lf + or --enable-newline-is-crlf or --enable-newline-is-anycrlf or + --enable-newline-is-any to the "configure" command, respectively. + + If you specify --enable-newline-is-cr or --enable-newline-is-crlf, some of + the standard tests will fail, because the lines in the test files end with + LF. Even if the files are edited to change the line endings, there are likely + to be some failures. With --enable-newline-is-anycrlf or + --enable-newline-is-any, many tests should succeed, but there may be some + failures. + +. By default, the sequence \R in a pattern matches any Unicode line ending + sequence. This is independent of the option specifying what PCRE considers to + be the end of a line (see above). However, the caller of PCRE can restrict \R + to match only CR, LF, or CRLF. You can make this the default by adding + --enable-bsr-anycrlf to the "configure" command (bsr = "backslash R"). + +. When called via the POSIX interface, PCRE uses malloc() to get additional + storage for processing capturing parentheses if there are more than 10 of + them in a pattern. You can increase this threshold by setting, for example, + + --with-posix-malloc-threshold=20 + + on the "configure" command. + +. PCRE has a counter that limits the depth of nesting of parentheses in a + pattern. This limits the amount of system stack that a pattern uses when it + is compiled. The default is 250, but you can change it by setting, for + example, + + --with-parens-nest-limit=500 + +. PCRE has a counter that can be set to limit the amount of resources it uses + when matching a pattern. If the limit is exceeded during a match, the match + fails. The default is ten million. You can change the default by setting, for + example, + + --with-match-limit=500000 + + on the "configure" command. This is just the default; individual calls to + pcre_exec() can supply their own value. There is more discussion on the + pcreapi man page. + +. There is a separate counter that limits the depth of recursive function calls + during a matching process. This also has a default of ten million, which is + essentially "unlimited". You can change the default by setting, for example, + + --with-match-limit-recursion=500000 + + Recursive function calls use up the runtime stack; running out of stack can + cause programs to crash in strange ways. There is a discussion about stack + sizes in the pcrestack man page. + +. The default maximum compiled pattern size is around 64K. You can increase + this by adding --with-link-size=3 to the "configure" command. In the 8-bit + library, PCRE then uses three bytes instead of two for offsets to different + parts of the compiled pattern. In the 16-bit library, --with-link-size=3 is + the same as --with-link-size=4, which (in both libraries) uses four-byte + offsets. Increasing the internal link size reduces performance. In the 32-bit + library, the only supported link size is 4. + +. You can build PCRE so that its internal match() function that is called from + pcre_exec() does not call itself recursively. Instead, it uses memory blocks + obtained from the heap via the special functions pcre_stack_malloc() and + pcre_stack_free() to save data that would otherwise be saved on the stack. To + build PCRE like this, use + + --disable-stack-for-recursion + + on the "configure" command. PCRE runs more slowly in this mode, but it may be + necessary in environments with limited stack sizes. This applies only to the + normal execution of the pcre_exec() function; if JIT support is being + successfully used, it is not relevant. Equally, it does not apply to + pcre_dfa_exec(), which does not use deeply nested recursion. There is a + discussion about stack sizes in the pcrestack man page. + +. For speed, PCRE uses four tables for manipulating and identifying characters + whose code point values are less than 256. By default, it uses a set of + tables for ASCII encoding that is part of the distribution. If you specify + + --enable-rebuild-chartables + + a program called dftables is compiled and run in the default C locale when + you obey "make". It builds a source file called pcre_chartables.c. If you do + not specify this option, pcre_chartables.c is created as a copy of + pcre_chartables.c.dist. See "Character tables" below for further information. + +. It is possible to compile PCRE for use on systems that use EBCDIC as their + character code (as opposed to ASCII/Unicode) by specifying + + --enable-ebcdic + + This automatically implies --enable-rebuild-chartables (see above). However, + when PCRE is built this way, it always operates in EBCDIC. It cannot support + both EBCDIC and UTF-8/16/32. There is a second option, --enable-ebcdic-nl25, + which specifies that the code value for the EBCDIC NL character is 0x25 + instead of the default 0x15. + +. In environments where valgrind is installed, if you specify + + --enable-valgrind + + PCRE will use valgrind annotations to mark certain memory regions as + unaddressable. This allows it to detect invalid memory accesses, and is + mostly useful for debugging PCRE itself. + +. In environments where the gcc compiler is used and lcov version 1.6 or above + is installed, if you specify + + --enable-coverage + + the build process implements a code coverage report for the test suite. The + report is generated by running "make coverage". If ccache is installed on + your system, it must be disabled when building PCRE for coverage reporting. + You can do this by setting the environment variable CCACHE_DISABLE=1 before + running "make" to build PCRE. There is more information about coverage + reporting in the "pcrebuild" documentation. + +. The pcregrep program currently supports only 8-bit data files, and so + requires the 8-bit PCRE library. It is possible to compile pcregrep to use + libz and/or libbz2, in order to read .gz and .bz2 files (respectively), by + specifying one or both of + + --enable-pcregrep-libz + --enable-pcregrep-libbz2 + + Of course, the relevant libraries must be installed on your system. + +. The default size (in bytes) of the internal buffer used by pcregrep can be + set by, for example: + + --with-pcregrep-bufsize=51200 + + The value must be a plain integer. The default is 20480. + +. It is possible to compile pcretest so that it links with the libreadline + or libedit libraries, by specifying, respectively, + + --enable-pcretest-libreadline or --enable-pcretest-libedit + + If this is done, when pcretest's input is from a terminal, it reads it using + the readline() function. This provides line-editing and history facilities. + Note that libreadline is GPL-licenced, so if you distribute a binary of + pcretest linked in this way, there may be licensing issues. These can be + avoided by linking with libedit (which has a BSD licence) instead. + + Enabling libreadline causes the -lreadline option to be added to the pcretest + build. In many operating environments with a sytem-installed readline + library this is sufficient. However, in some environments (e.g. if an + unmodified distribution version of readline is in use), it may be necessary + to specify something like LIBS="-lncurses" as well. This is because, to quote + the readline INSTALL, "Readline uses the termcap functions, but does not link + with the termcap or curses library itself, allowing applications which link + with readline the to choose an appropriate library." If you get error + messages about missing functions tgetstr, tgetent, tputs, tgetflag, or tgoto, + this is the problem, and linking with the ncurses library should fix it. + +The "configure" script builds the following files for the basic C library: + +. Makefile the makefile that builds the library +. config.h build-time configuration options for the library +. pcre.h the public PCRE header file +. pcre-config script that shows the building settings such as CFLAGS + that were set for "configure" +. libpcre.pc ) data for the pkg-config command +. libpcre16.pc ) +. libpcre32.pc ) +. libpcreposix.pc ) +. libtool script that builds shared and/or static libraries + +Versions of config.h and pcre.h are distributed in the PCRE tarballs under the +names config.h.generic and pcre.h.generic. These are provided for those who +have to built PCRE without using "configure" or CMake. If you use "configure" +or CMake, the .generic versions are not used. + +When building the 8-bit library, if a C++ compiler is found, the following +files are also built: + +. libpcrecpp.pc data for the pkg-config command +. pcrecpparg.h header file for calling PCRE via the C++ wrapper +. pcre_stringpiece.h header for the C++ "stringpiece" functions + +The "configure" script also creates config.status, which is an executable +script that can be run to recreate the configuration, and config.log, which +contains compiler output from tests that "configure" runs. + +Once "configure" has run, you can run "make". This builds the the libraries +libpcre, libpcre16 and/or libpcre32, and a test program called pcretest. If you +enabled JIT support with --enable-jit, a test program called pcre_jit_test is +built as well. + +If the 8-bit library is built, libpcreposix and the pcregrep command are also +built, and if a C++ compiler was found on your system, and you did not disable +it with --disable-cpp, "make" builds the C++ wrapper library, which is called +libpcrecpp, as well as some test programs called pcrecpp_unittest, +pcre_scanner_unittest, and pcre_stringpiece_unittest. + +The command "make check" runs all the appropriate tests. Details of the PCRE +tests are given below in a separate section of this document. + +You can use "make install" to install PCRE into live directories on your +system. The following are installed (file names are all relative to the + that is set when "configure" is run): + + Commands (bin): + pcretest + pcregrep (if 8-bit support is enabled) + pcre-config + + Libraries (lib): + libpcre16 (if 16-bit support is enabled) + libpcre32 (if 32-bit support is enabled) + libpcre (if 8-bit support is enabled) + libpcreposix (if 8-bit support is enabled) + libpcrecpp (if 8-bit and C++ support is enabled) + + Configuration information (lib/pkgconfig): + libpcre16.pc + libpcre32.pc + libpcre.pc + libpcreposix.pc + libpcrecpp.pc (if C++ support is enabled) + + Header files (include): + pcre.h + pcreposix.h + pcre_scanner.h ) + pcre_stringpiece.h ) if C++ support is enabled + pcrecpp.h ) + pcrecpparg.h ) + + Man pages (share/man/man{1,3}): + pcregrep.1 + pcretest.1 + pcre-config.1 + pcre.3 + pcre*.3 (lots more pages, all starting "pcre") + + HTML documentation (share/doc/pcre/html): + index.html + *.html (lots more pages, hyperlinked from index.html) + + Text file documentation (share/doc/pcre): + AUTHORS + COPYING + ChangeLog + LICENCE + NEWS + README + pcre.txt (a concatenation of the man(3) pages) + pcretest.txt the pcretest man page + pcregrep.txt the pcregrep man page + pcre-config.txt the pcre-config man page + +If you want to remove PCRE from your system, you can run "make uninstall". +This removes all the files that "make install" installed. However, it does not +remove any directories, because these are often shared with other programs. + + +Retrieving configuration information +------------------------------------ + +Running "make install" installs the command pcre-config, which can be used to +recall information about the PCRE configuration and installation. For example: + + pcre-config --version + +prints the version number, and + + pcre-config --libs + +outputs information about where the library is installed. This command can be +included in makefiles for programs that use PCRE, saving the programmer from +having to remember too many details. + +The pkg-config command is another system for saving and retrieving information +about installed libraries. Instead of separate commands for each library, a +single command is used. For example: + + pkg-config --cflags pcre + +The data is held in *.pc files that are installed in a directory called +/lib/pkgconfig. + + +Shared libraries +---------------- + +The default distribution builds PCRE as shared libraries and static libraries, +as long as the operating system supports shared libraries. Shared library +support relies on the "libtool" script which is built as part of the +"configure" process. + +The libtool script is used to compile and link both shared and static +libraries. They are placed in a subdirectory called .libs when they are newly +built. The programs pcretest and pcregrep are built to use these uninstalled +libraries (by means of wrapper scripts in the case of shared libraries). When +you use "make install" to install shared libraries, pcregrep and pcretest are +automatically re-built to use the newly installed shared libraries before being +installed themselves. However, the versions left in the build directory still +use the uninstalled libraries. + +To build PCRE using static libraries only you must use --disable-shared when +configuring it. For example: + +./configure --prefix=/usr/gnu --disable-shared + +Then run "make" in the usual way. Similarly, you can use --disable-static to +build only shared libraries. + + +Cross-compiling using autotools +------------------------------- + +You can specify CC and CFLAGS in the normal way to the "configure" command, in +order to cross-compile PCRE for some other host. However, you should NOT +specify --enable-rebuild-chartables, because if you do, the dftables.c source +file is compiled and run on the local host, in order to generate the inbuilt +character tables (the pcre_chartables.c file). This will probably not work, +because dftables.c needs to be compiled with the local compiler, not the cross +compiler. + +When --enable-rebuild-chartables is not specified, pcre_chartables.c is created +by making a copy of pcre_chartables.c.dist, which is a default set of tables +that assumes ASCII code. Cross-compiling with the default tables should not be +a problem. + +If you need to modify the character tables when cross-compiling, you should +move pcre_chartables.c.dist out of the way, then compile dftables.c by hand and +run it on the local host to make a new version of pcre_chartables.c.dist. +Then when you cross-compile PCRE this new version of the tables will be used. + + +Using HP's ANSI C++ compiler (aCC) +---------------------------------- + +Unless C++ support is disabled by specifying the "--disable-cpp" option of the +"configure" script, you must include the "-AA" option in the CXXFLAGS +environment variable in order for the C++ components to compile correctly. + +Also, note that the aCC compiler on PA-RISC platforms may have a defect whereby +needed libraries fail to get included when specifying the "-AA" compiler +option. If you experience unresolved symbols when linking the C++ programs, +use the workaround of specifying the following environment variable prior to +running the "configure" script: + + CXXLDFLAGS="-lstd_v2 -lCsup_v2" + + +Compiling in Tru64 using native compilers +----------------------------------------- + +The following error may occur when compiling with native compilers in the Tru64 +operating system: + + CXX libpcrecpp_la-pcrecpp.lo +cxx: Error: /usr/lib/cmplrs/cxx/V7.1-006/include/cxx/iosfwd, line 58: #error + directive: "cannot include iosfwd -- define __USE_STD_IOSTREAM to + override default - see section 7.1.2 of the C++ Using Guide" +#error "cannot include iosfwd -- define __USE_STD_IOSTREAM to override default +- see section 7.1.2 of the C++ Using Guide" + +This may be followed by other errors, complaining that 'namespace "std" has no +member'. The solution to this is to add the line + +#define __USE_STD_IOSTREAM 1 + +to the config.h file. + + +Using Sun's compilers for Solaris +--------------------------------- + +A user reports that the following configurations work on Solaris 9 sparcv9 and +Solaris 9 x86 (32-bit): + + Solaris 9 sparcv9: ./configure --disable-cpp CC=/bin/cc CFLAGS="-m64 -g" + Solaris 9 x86: ./configure --disable-cpp CC=/bin/cc CFLAGS="-g" + + +Using PCRE from MySQL +--------------------- + +On systems where both PCRE and MySQL are installed, it is possible to make use +of PCRE from within MySQL, as an alternative to the built-in pattern matching. +There is a web page that tells you how to do this: + + http://www.mysqludf.org/lib_mysqludf_preg/index.php + + +Making new tarballs +------------------- + +The command "make dist" creates three PCRE tarballs, in tar.gz, tar.bz2, and +zip formats. The command "make distcheck" does the same, but then does a trial +build of the new distribution to ensure that it works. + +If you have modified any of the man page sources in the doc directory, you +should first run the PrepareRelease script before making a distribution. This +script creates the .txt and HTML forms of the documentation from the man pages. + + +Testing PCRE +------------ + +To test the basic PCRE library on a Unix-like system, run the RunTest script. +There is another script called RunGrepTest that tests the options of the +pcregrep command. If the C++ wrapper library is built, three test programs +called pcrecpp_unittest, pcre_scanner_unittest, and pcre_stringpiece_unittest +are also built. When JIT support is enabled, another test program called +pcre_jit_test is built. + +Both the scripts and all the program tests are run if you obey "make check" or +"make test". For other environments, see the instructions in +NON-AUTOTOOLS-BUILD. + +The RunTest script runs the pcretest test program (which is documented in its +own man page) on each of the relevant testinput files in the testdata +directory, and compares the output with the contents of the corresponding +testoutput files. RunTest uses a file called testtry to hold the main output +from pcretest. Other files whose names begin with "test" are used as working +files in some tests. + +Some tests are relevant only when certain build-time options were selected. For +example, the tests for UTF-8/16/32 support are run only if --enable-utf was +used. RunTest outputs a comment when it skips a test. + +Many of the tests that are not skipped are run up to three times. The second +run forces pcre_study() to be called for all patterns except for a few in some +tests that are marked "never study" (see the pcretest program for how this is +done). If JIT support is available, the non-DFA tests are run a third time, +this time with a forced pcre_study() with the PCRE_STUDY_JIT_COMPILE option. +This testing can be suppressed by putting "nojit" on the RunTest command line. + +The entire set of tests is run once for each of the 8-bit, 16-bit and 32-bit +libraries that are enabled. If you want to run just one set of tests, call +RunTest with either the -8, -16 or -32 option. + +If valgrind is installed, you can run the tests under it by putting "valgrind" +on the RunTest command line. To run pcretest on just one or more specific test +files, give their numbers as arguments to RunTest, for example: + + RunTest 2 7 11 + +You can also specify ranges of tests such as 3-6 or 3- (meaning 3 to the +end), or a number preceded by ~ to exclude a test. For example: + + Runtest 3-15 ~10 + +This runs tests 3 to 15, excluding test 10, and just ~13 runs all the tests +except test 13. Whatever order the arguments are in, the tests are always run +in numerical order. + +You can also call RunTest with the single argument "list" to cause it to output +a list of tests. + +The first test file can be fed directly into the perltest.pl script to check +that Perl gives the same results. The only difference you should see is in the +first few lines, where the Perl version is given instead of the PCRE version. + +The second set of tests check pcre_fullinfo(), pcre_study(), +pcre_copy_substring(), pcre_get_substring(), pcre_get_substring_list(), error +detection, and run-time flags that are specific to PCRE, as well as the POSIX +wrapper API. It also uses the debugging flags to check some of the internals of +pcre_compile(). + +If you build PCRE with a locale setting that is not the standard C locale, the +character tables may be different (see next paragraph). In some cases, this may +cause failures in the second set of tests. For example, in a locale where the +isprint() function yields TRUE for characters in the range 128-255, the use of +[:isascii:] inside a character class defines a different set of characters, and +this shows up in this test as a difference in the compiled code, which is being +listed for checking. Where the comparison test output contains [\x00-\x7f] the +test will contain [\x00-\xff], and similarly in some other cases. This is not a +bug in PCRE. + +The third set of tests checks pcre_maketables(), the facility for building a +set of character tables for a specific locale and using them instead of the +default tables. The tests make use of the "fr_FR" (French) locale. Before +running the test, the script checks for the presence of this locale by running +the "locale" command. If that command fails, or if it doesn't include "fr_FR" +in the list of available locales, the third test cannot be run, and a comment +is output to say why. If running this test produces instances of the error + + ** Failed to set locale "fr_FR" + +in the comparison output, it means that locale is not available on your system, +despite being listed by "locale". This does not mean that PCRE is broken. + +[If you are trying to run this test on Windows, you may be able to get it to +work by changing "fr_FR" to "french" everywhere it occurs. Alternatively, use +RunTest.bat. The version of RunTest.bat included with PCRE 7.4 and above uses +Windows versions of test 2. More info on using RunTest.bat is included in the +document entitled NON-UNIX-USE.] + +The fourth and fifth tests check the UTF-8/16/32 support and error handling and +internal UTF features of PCRE that are not relevant to Perl, respectively. The +sixth and seventh tests do the same for Unicode character properties support. + +The eighth, ninth, and tenth tests check the pcre_dfa_exec() alternative +matching function, in non-UTF-8/16/32 mode, UTF-8/16/32 mode, and UTF-8/16/32 +mode with Unicode property support, respectively. + +The eleventh test checks some internal offsets and code size features; it is +run only when the default "link size" of 2 is set (in other cases the sizes +change) and when Unicode property support is enabled. + +The twelfth test is run only when JIT support is available, and the thirteenth +test is run only when JIT support is not available. They test some JIT-specific +features such as information output from pcretest about JIT compilation. + +The fourteenth, fifteenth, and sixteenth tests are run only in 8-bit mode, and +the seventeenth, eighteenth, and nineteenth tests are run only in 16/32-bit +mode. These are tests that generate different output in the two modes. They are +for general cases, UTF-8/16/32 support, and Unicode property support, +respectively. + +The twentieth test is run only in 16/32-bit mode. It tests some specific +16/32-bit features of the DFA matching engine. + +The twenty-first and twenty-second tests are run only in 16/32-bit mode, when +the link size is set to 2 for the 16-bit library. They test reloading +pre-compiled patterns. + +The twenty-third and twenty-fourth tests are run only in 16-bit mode. They are +for general cases, and UTF-16 support, respectively. + +The twenty-fifth and twenty-sixth tests are run only in 32-bit mode. They are +for general cases, and UTF-32 support, respectively. + + +Character tables +---------------- + +For speed, PCRE uses four tables for manipulating and identifying characters +whose code point values are less than 256. The final argument of the +pcre_compile() function is a pointer to a block of memory containing the +concatenated tables. A call to pcre_maketables() can be used to generate a set +of tables in the current locale. If the final argument for pcre_compile() is +passed as NULL, a set of default tables that is built into the binary is used. + +The source file called pcre_chartables.c contains the default set of tables. By +default, this is created as a copy of pcre_chartables.c.dist, which contains +tables for ASCII coding. However, if --enable-rebuild-chartables is specified +for ./configure, a different version of pcre_chartables.c is built by the +program dftables (compiled from dftables.c), which uses the ANSI C character +handling functions such as isalnum(), isalpha(), isupper(), islower(), etc. to +build the table sources. This means that the default C locale which is set for +your system will control the contents of these default tables. You can change +the default tables by editing pcre_chartables.c and then re-building PCRE. If +you do this, you should take care to ensure that the file does not get +automatically re-generated. The best way to do this is to move +pcre_chartables.c.dist out of the way and replace it with your customized +tables. + +When the dftables program is run as a result of --enable-rebuild-chartables, +it uses the default C locale that is set on your system. It does not pay +attention to the LC_xxx environment variables. In other words, it uses the +system's default locale rather than whatever the compiling user happens to have +set. If you really do want to build a source set of character tables in a +locale that is specified by the LC_xxx variables, you can run the dftables +program by hand with the -L option. For example: + + ./dftables -L pcre_chartables.c.special + +The first two 256-byte tables provide lower casing and case flipping functions, +respectively. The next table consists of three 32-byte bit maps which identify +digits, "word" characters, and white space, respectively. These are used when +building 32-byte bit maps that represent character classes for code points less +than 256. + +The final 256-byte table has bits indicating various character types, as +follows: + + 1 white space character + 2 letter + 4 decimal digit + 8 hexadecimal digit + 16 alphanumeric or '_' + 128 regular expression metacharacter or binary zero + +You should not alter the set of characters that contain the 128 bit, as that +will cause PCRE to malfunction. + + +File manifest +------------- + +The distribution should contain the files listed below. Where a file name is +given as pcre[16|32]_xxx it means that there are three files, one with the name +pcre_xxx, one with the name pcre16_xx, and a third with the name pcre32_xxx. + +(A) Source files of the PCRE library functions and their headers: + + dftables.c auxiliary program for building pcre_chartables.c + when --enable-rebuild-chartables is specified + + pcre_chartables.c.dist a default set of character tables that assume ASCII + coding; used, unless --enable-rebuild-chartables is + specified, by copying to pcre[16]_chartables.c + + pcreposix.c ) + pcre[16|32]_byte_order.c ) + pcre[16|32]_compile.c ) + pcre[16|32]_config.c ) + pcre[16|32]_dfa_exec.c ) + pcre[16|32]_exec.c ) + pcre[16|32]_fullinfo.c ) + pcre[16|32]_get.c ) sources for the functions in the library, + pcre[16|32]_globals.c ) and some internal functions that they use + pcre[16|32]_jit_compile.c ) + pcre[16|32]_maketables.c ) + pcre[16|32]_newline.c ) + pcre[16|32]_refcount.c ) + pcre[16|32]_string_utils.c ) + pcre[16|32]_study.c ) + pcre[16|32]_tables.c ) + pcre[16|32]_ucd.c ) + pcre[16|32]_version.c ) + pcre[16|32]_xclass.c ) + pcre_ord2utf8.c ) + pcre_valid_utf8.c ) + pcre16_ord2utf16.c ) + pcre16_utf16_utils.c ) + pcre16_valid_utf16.c ) + pcre32_utf32_utils.c ) + pcre32_valid_utf32.c ) + + pcre[16|32]_printint.c ) debugging function that is used by pcretest, + ) and can also be #included in pcre_compile() + + pcre.h.in template for pcre.h when built by "configure" + pcreposix.h header for the external POSIX wrapper API + pcre_internal.h header for internal use + sljit/* 16 files that make up the JIT compiler + ucp.h header for Unicode property handling + + config.h.in template for config.h, which is built by "configure" + + pcrecpp.h public header file for the C++ wrapper + pcrecpparg.h.in template for another C++ header file + pcre_scanner.h public header file for C++ scanner functions + pcrecpp.cc ) + pcre_scanner.cc ) source for the C++ wrapper library + + pcre_stringpiece.h.in template for pcre_stringpiece.h, the header for the + C++ stringpiece functions + pcre_stringpiece.cc source for the C++ stringpiece functions + +(B) Source files for programs that use PCRE: + + pcredemo.c simple demonstration of coding calls to PCRE + pcregrep.c source of a grep utility that uses PCRE + pcretest.c comprehensive test program + +(C) Auxiliary files: + + 132html script to turn "man" pages into HTML + AUTHORS information about the author of PCRE + ChangeLog log of changes to the code + CleanTxt script to clean nroff output for txt man pages + Detrail script to remove trailing spaces + HACKING some notes about the internals of PCRE + INSTALL generic installation instructions + LICENCE conditions for the use of PCRE + COPYING the same, using GNU's standard name + Makefile.in ) template for Unix Makefile, which is built by + ) "configure" + Makefile.am ) the automake input that was used to create + ) Makefile.in + NEWS important changes in this release + NON-UNIX-USE the previous name for NON-AUTOTOOLS-BUILD + NON-AUTOTOOLS-BUILD notes on building PCRE without using autotools + PrepareRelease script to make preparations for "make dist" + README this file + RunTest a Unix shell script for running tests + RunGrepTest a Unix shell script for pcregrep tests + aclocal.m4 m4 macros (generated by "aclocal") + config.guess ) files used by libtool, + config.sub ) used only when building a shared library + configure a configuring shell script (built by autoconf) + configure.ac ) the autoconf input that was used to build + ) "configure" and config.h + depcomp ) script to find program dependencies, generated by + ) automake + doc/*.3 man page sources for PCRE + doc/*.1 man page sources for pcregrep and pcretest + doc/index.html.src the base HTML page + doc/html/* HTML documentation + doc/pcre.txt plain text version of the man pages + doc/pcretest.txt plain text documentation of test program + doc/perltest.txt plain text documentation of Perl test program + install-sh a shell script for installing files + libpcre16.pc.in template for libpcre16.pc for pkg-config + libpcre32.pc.in template for libpcre32.pc for pkg-config + libpcre.pc.in template for libpcre.pc for pkg-config + libpcreposix.pc.in template for libpcreposix.pc for pkg-config + libpcrecpp.pc.in template for libpcrecpp.pc for pkg-config + ltmain.sh file used to build a libtool script + missing ) common stub for a few missing GNU programs while + ) installing, generated by automake + mkinstalldirs script for making install directories + perltest.pl Perl test program + pcre-config.in source of script which retains PCRE information + pcre_jit_test.c test program for the JIT compiler + pcrecpp_unittest.cc ) + pcre_scanner_unittest.cc ) test programs for the C++ wrapper + pcre_stringpiece_unittest.cc ) + testdata/testinput* test data for main library tests + testdata/testoutput* expected test results + testdata/grep* input and output for pcregrep tests + testdata/* other supporting test files + +(D) Auxiliary files for cmake support + + cmake/COPYING-CMAKE-SCRIPTS + cmake/FindPackageHandleStandardArgs.cmake + cmake/FindEditline.cmake + cmake/FindReadline.cmake + CMakeLists.txt + config-cmake.h.in + +(E) Auxiliary files for VPASCAL + + makevp.bat + makevp_c.txt + makevp_l.txt + pcregexp.pas + +(F) Auxiliary files for building PCRE "by hand" + + pcre.h.generic ) a version of the public PCRE header file + ) for use in non-"configure" environments + config.h.generic ) a version of config.h for use in non-"configure" + ) environments + +(F) Miscellaneous + + RunTest.bat a script for running tests under Windows + +Philip Hazel +Email local part: ph10 +Email domain: cam.ac.uk +Last updated: 10 February 2015 diff --git a/src/third_party/pcre-8.41/RunGrepTest b/src/third_party/pcre-8.41/RunGrepTest new file mode 100755 index 00000000000..721ec5184ab --- /dev/null +++ b/src/third_party/pcre-8.41/RunGrepTest @@ -0,0 +1,593 @@ +#! /bin/sh + +# Run pcregrep tests. The assumption is that the PCRE tests check the library +# itself. What we are checking here is the file handling and options that are +# supported by pcregrep. This script must be run in the build directory. + +# Set the C locale, so that sort(1) behaves predictably. + +LC_ALL=C +export LC_ALL + +# Remove any non-default colouring and aliases that the caller may have set. + +unset PCREGREP_COLOUR PCREGREP_COLOR +unset cp ls mv rm + +# Remember the current (build) directory, set the program to be tested, and +# valgrind settings when requested. + +builddir=`pwd` +pcregrep=$builddir/pcregrep + +valgrind= +while [ $# -gt 0 ] ; do + case $1 in + valgrind) valgrind="valgrind -q --leak-check=no --smc-check=all";; + *) echo "RunGrepTest: Unknown argument $1"; exit 1;; + esac + shift +done + +echo " " +pcregrep_version=`$pcregrep -V` +if [ "$valgrind" = "" ] ; then + echo "Testing $pcregrep_version" +else + echo "Testing $pcregrep_version using valgrind" +fi + +# Set up a suitable "diff" command for comparison. Some systems have a diff +# that lacks a -u option. Try to deal with this; better do the test for the -b +# option as well. + +cf="diff" +diff -b /dev/null /dev/null 2>/dev/null && cf="diff -b" +diff -u /dev/null /dev/null 2>/dev/null && cf="diff -u" +diff -ub /dev/null /dev/null 2>/dev/null && cf="diff -ub" + +# If this test is being run from "make check", $srcdir will be set. If not, set +# it to the current or parent directory, whichever one contains the test data. +# Subsequently, we run most of the pcregrep tests in the source directory so +# that the file names in the output are always the same. + +if [ -z "$srcdir" -o ! -d "$srcdir/testdata" ] ; then + if [ -d "./testdata" ] ; then + srcdir=. + elif [ -d "../testdata" ] ; then + srcdir=.. + else + echo "Cannot find the testdata directory" + exit 1 + fi +fi + +# Check for the availability of UTF-8 support + +./pcretest -C utf >/dev/null +utf8=$? + +# We need valgrind suppressions when JIT is in use. (This isn't perfect because +# some tests are run with -no-jit, but as PCRE1 is in maintenance only, I have +# not bothered about that.) + +./pcretest -C jit >/dev/null +if [ $? -eq 1 -a "$valgrind" != "" ] ; then + valgrind="$valgrind --suppressions=./testdata/valgrind-jit.supp" +fi + +echo "Testing pcregrep main features" + +echo "---------------------------- Test 1 ------------------------------" >testtrygrep +(cd $srcdir; $valgrind $pcregrep PATTERN ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 2 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep '^PATTERN' ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 3 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -in PATTERN ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 4 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -ic PATTERN ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 5 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -in PATTERN ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 6 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -inh PATTERN ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 7 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -il PATTERN ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 8 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -l PATTERN ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 9 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -q PATTERN ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 10 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -q NEVER-PATTERN ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 11 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -vn pattern ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 12 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -ix pattern ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 13 -----------------------------" >>testtrygrep +echo seventeen >testtemp1grep +(cd $srcdir; $valgrind $pcregrep -f./testdata/greplist -f $builddir/testtemp1grep ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 14 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -w pat ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 15 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep 'abc^*' ./testdata/grepinput) 2>>testtrygrep >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 16 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep abc ./testdata/grepinput ./testdata/nonexistfile) 2>>testtrygrep >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 17 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -M 'the\noutput' ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 18 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -Mn '(the\noutput|dog\.\n--)' ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 19 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -Mix 'Pattern' ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 20 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -Mixn 'complete pair\nof lines' ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 21 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -nA3 'four' ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 22 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -nB3 'four' ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 23 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -C3 'four' ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 24 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -A9 'four' ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 25 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -nB9 'four' ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 26 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -A9 -B9 'four' ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 27 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -A10 'four' ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 28 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -nB10 'four' ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 29 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -C12 -B10 'four' ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 30 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -inB3 'pattern' ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 31 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -inA3 'pattern' ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 32 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -L 'fox' ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 33 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep 'fox' ./testdata/grepnonexist) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 34 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -s 'fox' ./testdata/grepnonexist) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 35 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -L -r --include=grepinputx --include grepinput8 --exclude-dir='^\.' 'fox' ./testdata | sort) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 36 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -L -r --include=grepinput --exclude 'grepinput$' --exclude=grepinput8 --exclude-dir='^\.' 'fox' ./testdata | sort) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 37 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep '^(a+)*\d' ./testdata/grepinput) >>testtrygrep 2>teststderrgrep +echo "RC=$?" >>testtrygrep +echo "======== STDERR ========" >>testtrygrep +cat teststderrgrep >>testtrygrep + +echo "---------------------------- Test 38 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep '>\x00<' ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 39 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -A1 'before the binary zero' ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 40 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -B1 'after the binary zero' ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 41 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -B1 -o '\w+ the binary zero' ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 42 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -B1 -onH '\w+ the binary zero' ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 43 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -on 'before|zero|after' ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 44 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -on -e before -ezero -e after ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 45 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -on -f ./testdata/greplist -e binary ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 46 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -eabc -e '(unclosed' ./testdata/grepinput) 2>>testtrygrep >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 47 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -Fx "AB.VE +elephant" ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 48 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -F "AB.VE +elephant" ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 49 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -F -e DATA -e "AB.VE +elephant" ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 50 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep "^(abc|def|ghi|jkl)" ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 51 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -Mv "brown\sfox" ./testdata/grepinputv) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 52 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --colour=always jumps ./testdata/grepinputv) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 53 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --file-offsets 'before|zero|after' ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 54 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --line-offsets 'before|zero|after' ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 55 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -f./testdata/greplist --color=always ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 56 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -c lazy ./testdata/grepinput*) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 57 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -c -l lazy ./testdata/grepinput*) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 58 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --regex=PATTERN ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 59 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --regexp=PATTERN ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 60 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --regex PATTERN ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 61 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --regexp PATTERN ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 62 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --match-limit=1000 --no-jit -M 'This is a file(.|\R)*file.' ./testdata/grepinput) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 63 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --recursion-limit=1000 --no-jit -M 'This is a file(.|\R)*file.' ./testdata/grepinput) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 64 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -o1 '(?<=PAT)TERN (ap(pear)s)' ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 65 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -o2 '(?<=PAT)TERN (ap(pear)s)' ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 66 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -o3 '(?<=PAT)TERN (ap(pear)s)' ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 67 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -o12 '(?<=PAT)TERN (ap(pear)s)' ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 68 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --only-matching=2 '(?<=PAT)TERN (ap(pear)s)' ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 69 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -vn --colour=always pattern ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 70 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --color=always -M "triple:\t.*\n\n" ./testdata/grepinput3) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 71 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -o "^01|^02|^03" ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 72 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --color=always "^01|^02|^03" ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 73 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -o --colour=always "^01|^02|^03" ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 74 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -o "^01|02|^03" ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 75 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --color=always "^01|02|^03" ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 76 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -o --colour=always "^01|02|^03" ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 77 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -o "^01|^02|03" ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 78 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --color=always "^01|^02|03" ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 79 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -o --colour=always "^01|^02|03" ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 80 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -o "\b01|\b02" ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 81 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --color=always "\\b01|\\b02" ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 82 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -o --colour=always "\\b01|\\b02" ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 83 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --buffer-size=100 "^a" ./testdata/grepinput3) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 84 -----------------------------" >>testtrygrep +echo testdata/grepinput3 >testtemp1grep +(cd $srcdir; $valgrind $pcregrep --file-list ./testdata/grepfilelist --file-list $builddir/testtemp1grep "fox|complete|t7") >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 85 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --file-list=./testdata/grepfilelist "dolor" ./testdata/grepinput3) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 86 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep "dog" ./testdata/grepbinary) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 87 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep "cat" ./testdata/grepbinary) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 88 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -v "cat" ./testdata/grepbinary) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 89 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -I "dog" ./testdata/grepbinary) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 90 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --binary-files=without-match "dog" ./testdata/grepbinary) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 91 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -a "dog" ./testdata/grepbinary) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 92 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --binary-files=text "dog" ./testdata/grepbinary) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 93 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --text "dog" ./testdata/grepbinary) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 94 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -L -r --include=grepinputx --include grepinput8 'fox' ./testdata/grepinput* | sort) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 95 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --file-list ./testdata/grepfilelist --exclude grepinputv "fox|complete") >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 96 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -L -r --include-dir=testdata --exclude '^(?!grepinput)' 'fox' ./test* | sort) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 97 -----------------------------" >>testtrygrep +echo "grepinput$" >testtemp1grep +echo "grepinput8" >>testtemp1grep +(cd $srcdir; $valgrind $pcregrep -L -r --include=grepinput --exclude-from $builddir/testtemp1grep --exclude-dir='^\.' 'fox' ./testdata | sort) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 98 -----------------------------" >>testtrygrep +echo "grepinput$" >testtemp1grep +echo "grepinput8" >>testtemp1grep +(cd $srcdir; $valgrind $pcregrep -L -r --exclude=grepinput3 --include=grepinput --exclude-from $builddir/testtemp1grep --exclude-dir='^\.' 'fox' ./testdata | sort) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 99 -----------------------------" >>testtrygrep +echo "grepinput$" >testtemp1grep +echo "grepinput8" >testtemp2grep +(cd $srcdir; $valgrind $pcregrep -L -r --include grepinput --exclude-from $builddir/testtemp1grep --exclude-from=$builddir/testtemp2grep --exclude-dir='^\.' 'fox' ./testdata | sort) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 100 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -Ho2 --only-matching=1 -o3 '(\w+) binary (\w+)(\.)?' ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 101 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -o3 -Ho2 -o12 --only-matching=1 -o3 --colour=always --om-separator='|' '(\w+) binary (\w+)(\.)?' ./testdata/grepinput) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 102 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -n "^$" ./testdata/grepinput3) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 103 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --only-matching "^$" ./testdata/grepinput3) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 104 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -n --only-matching "^$" ./testdata/grepinput3) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 105 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep --colour=always "ipsum|" ./testdata/grepinput3) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 106 -----------------------------" >>testtrygrep +(cd $srcdir; echo "a" | $valgrind $pcregrep -M "|a" ) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 107 -----------------------------" >>testtrygrep +echo "a" >testtemp1grep +echo "aaaaa" >>testtemp1grep +(cd $srcdir; $valgrind $pcregrep --line-offsets '(?<=\Ka)' $builddir/testtemp1grep) >>testtrygrep 2>&1 +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 108 ------------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -lq PATTERN ./testdata/grepinput ./testdata/grepinputx) >>testtrygrep +echo "RC=$?" >>testtrygrep + +echo "---------------------------- Test 109 -----------------------------" >>testtrygrep +(cd $srcdir; $valgrind $pcregrep -cq lazy ./testdata/grepinput*) >>testtrygrep +echo "RC=$?" >>testtrygrep + +# Now compare the results. + +$cf $srcdir/testdata/grepoutput testtrygrep +if [ $? != 0 ] ; then exit 1; fi + + +# These tests require UTF-8 support + +if [ $utf8 -ne 0 ] ; then + echo "Testing pcregrep UTF-8 features" + + echo "---------------------------- Test U1 ------------------------------" >testtrygrep + (cd $srcdir; $valgrind $pcregrep -n -u --newline=any "^X" ./testdata/grepinput8) >>testtrygrep + echo "RC=$?" >>testtrygrep + + echo "---------------------------- Test U2 ------------------------------" >>testtrygrep + (cd $srcdir; $valgrind $pcregrep -n -u -C 3 --newline=any "Match" ./testdata/grepinput8) >>testtrygrep + echo "RC=$?" >>testtrygrep + + $cf $srcdir/testdata/grepoutput8 testtrygrep + if [ $? != 0 ] ; then exit 1; fi + +else + echo "Skipping pcregrep UTF-8 tests: no UTF-8 support in PCRE library" +fi + + +# We go to some contortions to try to ensure that the tests for the various +# newline settings will work in environments where the normal newline sequence +# is not \n. Do not use exported files, whose line endings might be changed. +# Instead, create an input file using printf so that its contents are exactly +# what we want. Note the messy fudge to get printf to write a string that +# starts with a hyphen. These tests are run in the build directory. + +echo "Testing pcregrep newline settings" +printf "abc\rdef\r\nghi\njkl" >testNinputgrep + +printf "%c--------------------------- Test N1 ------------------------------\r\n" - >testtrygrep +$valgrind $pcregrep -n -N CR "^(abc|def|ghi|jkl)" testNinputgrep >>testtrygrep + +printf "%c--------------------------- Test N2 ------------------------------\r\n" - >>testtrygrep +$valgrind $pcregrep -n --newline=crlf "^(abc|def|ghi|jkl)" testNinputgrep >>testtrygrep + +printf "%c--------------------------- Test N3 ------------------------------\r\n" - >>testtrygrep +pattern=`printf 'def\rjkl'` +$valgrind $pcregrep -n --newline=cr -F "$pattern" testNinputgrep >>testtrygrep + +printf "%c--------------------------- Test N4 ------------------------------\r\n" - >>testtrygrep +$valgrind $pcregrep -n --newline=crlf -F -f $srcdir/testdata/greppatN4 testNinputgrep >>testtrygrep + +printf "%c--------------------------- Test N5 ------------------------------\r\n" - >>testtrygrep +$valgrind $pcregrep -n --newline=any "^(abc|def|ghi|jkl)" testNinputgrep >>testtrygrep + +printf "%c--------------------------- Test N6 ------------------------------\r\n" - >>testtrygrep +$valgrind $pcregrep -n --newline=anycrlf "^(abc|def|ghi|jkl)" testNinputgrep >>testtrygrep + +$cf $srcdir/testdata/grepoutputN testtrygrep +if [ $? != 0 ] ; then exit 1; fi + +exit 0 + +# End diff --git a/src/third_party/pcre-8.41/RunTest b/src/third_party/pcre-8.41/RunTest new file mode 100755 index 00000000000..357a739b75f --- /dev/null +++ b/src/third_party/pcre-8.41/RunTest @@ -0,0 +1,1014 @@ +#! /bin/sh + +############################################################################### +# Run the PCRE tests using the pcretest program. The appropriate tests are +# selected, depending on which build-time options were used. +# +# All tests are now run both with and without -s, to ensure that everything is +# tested with and without studying. However, there are some tests that produce +# different output after studying, typically when we are tracing the actual +# matching process (for example, using auto-callouts). In these few cases, the +# tests are duplicated in the files, one with /S to force studying always, and +# one with /SS to force *not* studying always. The use of -s doesn't then make +# any difference to their output. There is also one test which compiles invalid +# UTF-8 with the UTF-8 check turned off; for this, studying must also be +# disabled with /SS. +# +# When JIT support is available, all appropriate tests are also run with -s+ to +# test (again, almost) everything with studying and the JIT option, unless +# "nojit" is given on the command line. There are also two tests for +# JIT-specific features, one to be run when JIT support is available (unless +# "nojit" is specified), and one when it is not. +# +# Whichever of the 8-, 16- and 32-bit libraries exist are tested. It is also +# possible to select which to test by giving "-8", "-16" or "-32" on the +# command line. +# +# As well as "nojit", "-8", "-16", and "-32", arguments for this script are +# individual test numbers, ranges of tests such as 3-6 or 3- (meaning 3 to the +# end), or a number preceded by ~ to exclude a test. For example, "3-15 ~10" +# runs tests 3 to 15, excluding test 10, and just "~10" runs all the tests +# except test 10. Whatever order the arguments are in, the tests are always run +# in numerical order. +# +# The special argument "3S" runs test 3, stopping if it fails. Test 3 is the +# locale test, and failure usually means there's an issue with the locale +# rather than a bug in PCRE, so normally subsequent tests are run. "3S" is +# useful when you want to debug or update the test. +# +# Inappropriate tests are automatically skipped (with a comment to say so): for +# example, if JIT support is not compiled, test 12 is skipped, whereas if JIT +# support is compiled, test 13 is skipped. +# +# Other arguments can be one of the words "valgrind", "valgrind-log", or "sim" +# followed by an argument to run cross-compiled executables under a simulator, +# for example: +# +# RunTest 3 sim "qemu-arm -s 8388608" +# +# There are two special cases where only one argument is allowed: +# +# If the first and only argument is "ebcdic", the script runs the special +# EBCDIC test that can be useful for checking certain EBCDIC features, even +# when run in an ASCII environment. +# +# If the script is obeyed as "RunTest list", a list of available tests is +# output, but none of them are run. +############################################################################### + +# Define test titles in variables so that they can be output as a list. Some +# of them are modified (e.g. with -8 or -16) when used in the actual tests. + +title1="Test 1: Main functionality (Compatible with Perl >= 5.10)" +title2="Test 2: API, errors, internals, and non-Perl stuff" +title3="Test 3: Locale-specific features" +title4A="Test 4: UTF" +title4B=" support (Compatible with Perl >= 5.10)" +title5="Test 5: API, internals, and non-Perl stuff for UTF" +title6="Test 6: Unicode property support (Compatible with Perl >= 5.10)" +title7="Test 7: API, internals, and non-Perl stuff for Unicode property support" +title8="Test 8: DFA matching main functionality" +title9="Test 9: DFA matching with UTF" +title10="Test 10: DFA matching with Unicode properties" +title11="Test 11: Internal offsets and code size tests" +title12="Test 12: JIT-specific features (when JIT is available)" +title13="Test 13: JIT-specific features (when JIT is not available)" +title14="Test 14: Specials for the basic 8-bit library" +title15="Test 15: Specials for the 8-bit library with UTF-8 support" +title16="Test 16: Specials for the 8-bit library with Unicode propery support" +title17="Test 17: Specials for the basic 16/32-bit library" +title18="Test 18: Specials for the 16/32-bit library with UTF-16/32 support" +title19="Test 19: Specials for the 16/32-bit library with Unicode property support" +title20="Test 20: DFA specials for the basic 16/32-bit library" +title21="Test 21: Reloads for the basic 16/32-bit library" +title22="Test 22: Reloads for the 16/32-bit library with UTF-16/32 support" +title23="Test 23: Specials for the 16-bit library" +title24="Test 24: Specials for the 16-bit library with UTF-16 support" +title25="Test 25: Specials for the 32-bit library" +title26="Test 26: Specials for the 32-bit library with UTF-32 support" + +maxtest=26 + +if [ $# -eq 1 -a "$1" = "list" ]; then + echo $title1 + echo $title2 "(not UTF)" + echo $title3 + echo $title4A $title4B + echo $title5 support + echo $title6 + echo $title7 + echo $title8 + echo $title9 + echo $title10 + echo $title11 + echo $title12 + echo $title13 + echo $title14 + echo $title15 + echo $title16 + echo $title17 + echo $title18 + echo $title19 + echo $title20 + echo $title21 + echo $title22 + echo $title23 + echo $title24 + echo $title25 + echo $title26 + exit 0 +fi + +# Set up a suitable "diff" command for comparison. Some systems +# have a diff that lacks a -u option. Try to deal with this. + +cf="diff" +diff -u /dev/null /dev/null 2>/dev/null && cf="diff -u" + +# Find the test data + +if [ -n "$srcdir" -a -d "$srcdir" ] ; then + testdata="$srcdir/testdata" +elif [ -d "./testdata" ] ; then + testdata=./testdata +elif [ -d "../testdata" ] ; then + testdata=../testdata +else + echo "Cannot find the testdata directory" + exit 1 +fi + + +# ------ Special EBCDIC Test ------- + +if [ $# -eq 1 -a "$1" = "ebcdic" ]; then + ./pcretest -C ebcdic >/dev/null + ebcdic=$? + if [ $ebcdic -ne 1 ] ; then + echo "Cannot run EBCDIC tests: EBCDIC support not compiled" + exit 1 + fi + + for opt in "" "-s" "-dfa" "-s -dfa"; do + ./pcretest -q $opt $testdata/testinputEBC >testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutputEBC testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + if [ "$opt" = "-s" ] ; then echo " OK with study" + elif [ "$opt" = "-dfa" ] ; then echo " OK using DFA" + elif [ "$opt" = "-s -dfa" ] ; then echo " OK using DFA with study" + else echo " OK" + fi + done + +exit 0 +fi + + +# ------ Normal Tests ------ + +# Default values + +arg8= +arg16= +arg32= +nojit= +sim= +skip= +valgrind= +vjs= + +# This is in case the caller has set aliases (as I do - PH) +unset cp ls mv rm + +# Process options and select which tests to run; for those that are explicitly +# requested, check that the necessary optional facilities are available. + +do1=no +do2=no +do3=no +do4=no +do5=no +do6=no +do7=no +do8=no +do9=no +do10=no +do11=no +do12=no +do13=no +do14=no +do15=no +do16=no +do17=no +do18=no +do19=no +do20=no +do21=no +do22=no +do23=no +do24=no +do25=no +do26=no + +while [ $# -gt 0 ] ; do + case $1 in + 1) do1=yes;; + 2) do2=yes;; + 3) do3=yes;; + 4) do4=yes;; + 5) do5=yes;; + 6) do6=yes;; + 7) do7=yes;; + 8) do8=yes;; + 9) do9=yes;; + 10) do10=yes;; + 11) do11=yes;; + 12) do12=yes;; + 13) do13=yes;; + 14) do14=yes;; + 15) do15=yes;; + 16) do16=yes;; + 17) do17=yes;; + 18) do18=yes;; + 19) do19=yes;; + 20) do20=yes;; + 21) do21=yes;; + 22) do22=yes;; + 23) do23=yes;; + 24) do24=yes;; + 25) do25=yes;; + 26) do26=yes;; + -8) arg8=yes;; + -16) arg16=yes;; + -32) arg32=yes;; + nojit) nojit=yes;; + sim) shift; sim=$1;; + valgrind) valgrind="valgrind --tool=memcheck -q --smc-check=all";; + valgrind-log) valgrind="valgrind --tool=memcheck --num-callers=30 --leak-check=no --error-limit=no --smc-check=all --log-file=report.%p ";; + ~*) + if expr "$1" : '~[0-9][0-9]*$' >/dev/null; then + skip="$skip `expr "$1" : '~\([0-9]*\)*$'`" + else + echo "Unknown option or test selector '$1'"; exit 1 + fi + ;; + *-*) + if expr "$1" : '[0-9][0-9]*-[0-9]*$' >/dev/null; then + tf=`expr "$1" : '\([0-9]*\)'` + tt=`expr "$1" : '.*-\([0-9]*\)'` + if [ "$tt" = "" ] ; then tt=$maxtest; fi + if expr \( "$tf" "<" 1 \) \| \( "$tt" ">" "$maxtest" \) >/dev/null; then + echo "Invalid test range '$1'"; exit 1 + fi + while expr "$tf" "<=" "$tt" >/dev/null; do + eval do${tf}=yes + tf=`expr $tf + 1` + done + else + echo "Invalid test range '$1'"; exit 1 + fi + ;; + *) echo "Unknown option or test selector '$1'"; exit 1;; + esac + shift +done + +# Find which optional facilities are available. + +$sim ./pcretest -C linksize >/dev/null +link_size=$? +if [ $link_size -lt 2 ] ; then + echo "Failed to find internal link size" + exit 1 +fi +if [ $link_size -gt 4 ] ; then + echo "Failed to find internal link size" + exit 1 +fi + +# All of 8-bit, 16-bit, and 32-bit character strings may be supported, but only +# one need be. + +$sim ./pcretest -C pcre8 >/dev/null +support8=$? +$sim ./pcretest -C pcre16 >/dev/null +support16=$? +$sim ./pcretest -C pcre32 >/dev/null +support32=$? + +# Initialize all bitsizes skipped + +test8=skip +test16=skip +test32=skip + +# If no bitsize arguments, select all that are available + +if [ "$arg8$arg16$arg32" = "" ] ; then + if [ $support8 -ne 0 ] ; then + test8= + fi + if [ $support16 -ne 0 ] ; then + test16=-16 + fi + if [ $support32 -ne 0 ] ; then + test32=-32 + fi + +# Select requested bit sizes + +else + if [ "$arg8" = yes ] ; then + if [ $support8 -eq 0 ] ; then + echo "Cannot run 8-bit library tests: 8-bit library not compiled" + exit 1 + fi + test8= + fi + if [ "$arg16" = yes ] ; then + if [ $support16 -eq 0 ] ; then + echo "Cannot run 16-bit library tests: 16-bit library not compiled" + exit 1 + fi + test16=-16 + fi + if [ "$arg32" = yes ] ; then + if [ $support32 -eq 0 ] ; then + echo "Cannot run 32-bit library tests: 32-bit library not compiled" + exit 1 + fi + test32=-32 + fi +fi + +# UTF support always applies to all bit sizes if both are supported; we can't +# have UTF-8 support without UTF-16 support (for example). + +$sim ./pcretest -C utf >/dev/null +utf=$? + +$sim ./pcretest -C ucp >/dev/null +ucp=$? + +jitopt= +$sim ./pcretest -C jit >/dev/null +jit=$? +if [ $jit -ne 0 -a "$nojit" != "yes" ] ; then + jitopt=-s+ + if [ "$valgrind" != "" ] ; then + vjs="--suppressions=$testdata/valgrind-jit.supp" + fi +fi + +# If no specific tests were requested, select all. Those that are not +# relevant will be automatically skipped. + +if [ $do1 = no -a $do2 = no -a $do3 = no -a $do4 = no -a \ + $do5 = no -a $do6 = no -a $do7 = no -a $do8 = no -a \ + $do9 = no -a $do10 = no -a $do11 = no -a $do12 = no -a \ + $do13 = no -a $do14 = no -a $do15 = no -a $do16 = no -a \ + $do17 = no -a $do18 = no -a $do19 = no -a $do20 = no -a \ + $do21 = no -a $do22 = no -a $do23 = no -a $do24 = no -a \ + $do25 = no -a $do26 = no ] ; then + do1=yes + do2=yes + do3=yes + do4=yes + do5=yes + do6=yes + do7=yes + do8=yes + do9=yes + do10=yes + do11=yes + do12=yes + do13=yes + do14=yes + do15=yes + do16=yes + do17=yes + do18=yes + do19=yes + do20=yes + do21=yes + do22=yes + do23=yes + do24=yes + do25=yes + do26=yes +fi + +# Handle any explicit skips at this stage, so that an argument list may consist +# only of explicit skips. + +for i in $skip; do eval do$i=no; done + +# Show which release and which test data + +echo "" +echo PCRE C library tests using test data from $testdata +$sim ./pcretest /dev/null + +for bmode in "$test8" "$test16" "$test32"; do + case "$bmode" in + skip) continue;; + -16) if [ "$test8$test32" != "skipskip" ] ; then echo ""; fi + bits=16; echo "---- Testing 16-bit library ----"; echo "";; + -32) if [ "$test8$test16" != "skipskip" ] ; then echo ""; fi + bits=32; echo "---- Testing 32-bit library ----"; echo "";; + *) bits=8; echo "---- Testing 8-bit library ----"; echo "";; + esac + +# Primary test, compatible with JIT and all versions of Perl >= 5.8 + +if [ $do1 = yes ] ; then + echo $title1 + for opt in "" "-s" $jitopt; do + $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput1 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput1 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + if [ "$opt" = "-s" ] ; then echo " OK with study" + elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" + else echo " OK" + fi + done +fi + +# PCRE tests that are not JIT or Perl-compatible: API, errors, internals + +if [ $do2 = yes ] ; then + echo $title2 "(not UTF-$bits)" + for opt in "" "-s" $jitopt; do + $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput2 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput2 testtry + if [ $? != 0 ] ; then exit 1; fi + else + echo " " + echo "** Test 2 requires a lot of stack. If it has crashed with a" + echo "** segmentation fault, it may be that you do not have enough" + echo "** stack available by default. Please see the 'pcrestack' man" + echo "** page for a discussion of PCRE's stack usage." + echo " " + exit 1 + fi + if [ "$opt" = "-s" ] ; then echo " OK with study" + elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" + else echo " OK" + fi + done +fi + +# Locale-specific tests, provided that either the "fr_FR" or the "french" +# locale is available. The former is the Unix-like standard; the latter is +# for Windows. Another possibility is "fr". Unfortunately, different versions +# of the French locale give different outputs for some items. This test passes +# if the output matches any one of the alternative output files. + +if [ $do3 = yes ] ; then + locale -a | grep '^fr_FR$' >/dev/null + if [ $? -eq 0 ] ; then + locale=fr_FR + infile=$testdata/testinput3 + outfile=$testdata/testoutput3 + outfile2=$testdata/testoutput3A + outfile3=$testdata/testoutput3B + else + infile=test3input + outfile=test3output + outfile2=test3outputA + outfile3=test3outputB + locale -a | grep '^french$' >/dev/null + if [ $? -eq 0 ] ; then + locale=french + sed 's/fr_FR/french/' $testdata/testinput3 >test3input + sed 's/fr_FR/french/' $testdata/testoutput3 >test3output + sed 's/fr_FR/french/' $testdata/testoutput3A >test3outputA + sed 's/fr_FR/french/' $testdata/testoutput3B >test3outputB + else + locale -a | grep '^fr$' >/dev/null + if [ $? -eq 0 ] ; then + locale=fr + sed 's/fr_FR/fr/' $testdata/intestinput3 >test3input + sed 's/fr_FR/fr/' $testdata/intestoutput3 >test3output + sed 's/fr_FR/fr/' $testdata/intestoutput3A >test3outputA + sed 's/fr_FR/fr/' $testdata/intestoutput3B >test3outputB + else + locale= + fi + fi + fi + + if [ "$locale" != "" ] ; then + echo $title3 "(using '$locale' locale)" + for opt in "" "-s" $jitopt; do + $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $infile testtry + if [ $? = 0 ] ; then + if $cf $outfile testtry >teststdout || \ + $cf $outfile2 testtry >teststdout || \ + $cf $outfile3 testtry >teststdout + then + if [ "$opt" = "-s" ] ; then echo " OK with study" + elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" + else echo " OK" + fi + else + echo "** Locale test did not run successfully. The output did not match" + echo " $outfile, $outfile2 or $outfile3." + echo " This may mean that there is a problem with the locale settings rather" + echo " than a bug in PCRE." + exit 1 + fi + else exit 1 + fi + done + else + echo "Cannot test locale-specific features - none of the 'fr_FR', 'fr' or" + echo "'french' locales exist, or the \"locale\" command is not available" + echo "to check for them." + echo " " + fi +fi + +# Additional tests for UTF support + +if [ $do4 = yes ] ; then + echo ${title4A}-${bits}${title4B} + if [ $utf -eq 0 ] ; then + echo " Skipped because UTF-$bits support is not available" + else + for opt in "" "-s" $jitopt; do + $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput4 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput4 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + if [ "$opt" = "-s" ] ; then echo " OK with study" + elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" + else echo " OK" + fi + done + fi +fi + +if [ $do5 = yes ] ; then + echo ${title5}-${bits} support + if [ $utf -eq 0 ] ; then + echo " Skipped because UTF-$bits support is not available" + else + for opt in "" "-s" $jitopt; do + $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput5 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput5 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + if [ "$opt" = "-s" ] ; then echo " OK with study" + elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" + else echo " OK" + fi + done + fi +fi + +if [ $do6 = yes ] ; then + echo $title6 + if [ $utf -eq 0 -o $ucp -eq 0 ] ; then + echo " Skipped because Unicode property support is not available" + else + for opt in "" "-s" $jitopt; do + $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput6 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput6 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + if [ "$opt" = "-s" ] ; then echo " OK with study" + elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" + else echo " OK" + fi + done + fi +fi + +# Test non-Perl-compatible Unicode property support + +if [ $do7 = yes ] ; then + echo $title7 + if [ $utf -eq 0 -o $ucp -eq 0 ] ; then + echo " Skipped because Unicode property support is not available" + else + for opt in "" "-s" $jitopt; do + $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput7 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput7 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + if [ "$opt" = "-s" ] ; then echo " OK with study" + elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" + else echo " OK" + fi + done + fi +fi + +# Tests for DFA matching support + +if [ $do8 = yes ] ; then + echo $title8 + for opt in "" "-s"; do + $sim $valgrind ./pcretest -q $bmode $opt -dfa $testdata/testinput8 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput8 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + if [ "$opt" = "-s" ] ; then echo " OK with study" ; else echo " OK"; fi + done +fi + +if [ $do9 = yes ] ; then + echo ${title9}-${bits} + if [ $utf -eq 0 ] ; then + echo " Skipped because UTF-$bits support is not available" + else + for opt in "" "-s"; do + $sim $valgrind ./pcretest -q $bmode $opt -dfa $testdata/testinput9 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput9 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + if [ "$opt" = "-s" ] ; then echo " OK with study" ; else echo " OK"; fi + done + fi +fi + +if [ $do10 = yes ] ; then + echo $title10 + if [ $utf -eq 0 -o $ucp -eq 0 ] ; then + echo " Skipped because Unicode property support is not available" + else + for opt in "" "-s"; do + $sim $valgrind ./pcretest -q $bmode $opt -dfa $testdata/testinput10 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput10 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + if [ "$opt" = "-s" ] ; then echo " OK with study" ; else echo " OK"; fi + done + fi +fi + +# Test of internal offsets and code sizes. This test is run only when there +# is Unicode property support and the link size is 2. The actual tests are +# mostly the same as in some of the above, but in this test we inspect some +# offsets and sizes that require a known link size. This is a doublecheck for +# the maintainer, just in case something changes unexpectely. The output from +# this test is not the same in 8-bit and 16-bit modes. + +if [ $do11 = yes ] ; then + echo $title11 + if [ $link_size -ne 2 ] ; then + echo " Skipped because link size is not 2" + elif [ $ucp -eq 0 ] ; then + echo " Skipped because Unicode property support is not available" + else + for opt in "" "-s"; do + $sim $valgrind ./pcretest -q $bmode $opt $testdata/testinput11 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput11-$bits testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + if [ "$opt" = "-s" ] ; then echo " OK with study" ; else echo " OK"; fi + done + fi +fi + +# Test JIT-specific features when JIT is available + +if [ $do12 = yes ] ; then + echo $title12 + if [ $jit -eq 0 -o "$nojit" = "yes" ] ; then + echo " Skipped because JIT is not available or not usable" + else + $sim $valgrind $vjs ./pcretest -q $bmode $testdata/testinput12 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput12 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + echo " OK" + fi +fi + +# Test JIT-specific features when JIT is not available + +if [ $do13 = yes ] ; then + echo $title13 + if [ $jit -ne 0 ] ; then + echo " Skipped because JIT is available" + else + $sim $valgrind ./pcretest -q $bmode $testdata/testinput13 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput13 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + echo " OK" + fi +fi + +# Tests for 8-bit-specific features + +if [ "$do14" = yes ] ; then + echo $title14 + if [ "$bits" = "16" -o "$bits" = "32" ] ; then + echo " Skipped when running 16/32-bit tests" + else + cp -f $testdata/saved16 testsaved16 + cp -f $testdata/saved32 testsaved32 + for opt in "" "-s" $jitopt; do + $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput14 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput14 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + if [ "$opt" = "-s" ] ; then echo " OK with study" + elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" + else echo " OK" + fi + done + fi +fi + +# Tests for 8-bit-specific features (needs UTF-8 support) + +if [ "$do15" = yes ] ; then + echo $title15 + if [ "$bits" = "16" -o "$bits" = "32" ] ; then + echo " Skipped when running 16/32-bit tests" + elif [ $utf -eq 0 ] ; then + echo " Skipped because UTF-$bits support is not available" + else + for opt in "" "-s" $jitopt; do + $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput15 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput15 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + if [ "$opt" = "-s" ] ; then echo " OK with study" + elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" + else echo " OK" + fi + done + fi +fi + +# Tests for 8-bit-specific features (Unicode property support) + +if [ $do16 = yes ] ; then + echo $title16 + if [ "$bits" = "16" -o "$bits" = "32" ] ; then + echo " Skipped when running 16/32-bit tests" + elif [ $ucp -eq 0 ] ; then + echo " Skipped because Unicode property support is not available" + else + for opt in "" "-s" $jitopt; do + $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput16 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput16 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + if [ "$opt" = "-s" ] ; then echo " OK with study" + elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" + else echo " OK" + fi + done + fi +fi + +# Tests for 16/32-bit-specific features + +if [ $do17 = yes ] ; then + echo $title17 + if [ "$bits" = "8" ] ; then + echo " Skipped when running 8-bit tests" + else + for opt in "" "-s" $jitopt; do + $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput17 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput17 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + if [ "$opt" = "-s" ] ; then echo " OK with study" + elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" + else echo " OK" + fi + done + fi +fi + +# Tests for 16/32-bit-specific features (UTF-16/32 support) + +if [ $do18 = yes ] ; then + echo $title18 + if [ "$bits" = "8" ] ; then + echo " Skipped when running 8-bit tests" + elif [ $utf -eq 0 ] ; then + echo " Skipped because UTF-$bits support is not available" + else + for opt in "" "-s" $jitopt; do + $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput18 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput18-$bits testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + if [ "$opt" = "-s" ] ; then echo " OK with study" + elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" + else echo " OK" + fi + done + fi +fi + +# Tests for 16/32-bit-specific features (Unicode property support) + +if [ $do19 = yes ] ; then + echo $title19 + if [ "$bits" = "8" ] ; then + echo " Skipped when running 8-bit tests" + elif [ $ucp -eq 0 ] ; then + echo " Skipped because Unicode property support is not available" + else + for opt in "" "-s" $jitopt; do + $sim $valgrind ${opt:+$vjs} ./pcretest -q $bmode $opt $testdata/testinput19 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput19 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + if [ "$opt" = "-s" ] ; then echo " OK with study" + elif [ "$opt" = "-s+" ] ; then echo " OK with JIT study" + else echo " OK" + fi + done + fi +fi + +# Tests for 16/32-bit-specific features in DFA non-UTF-16/32 mode + +if [ $do20 = yes ] ; then + echo $title20 + if [ "$bits" = "8" ] ; then + echo " Skipped when running 8-bit tests" + else + for opt in "" "-s"; do + $sim $valgrind ./pcretest -q $bmode $opt -dfa $testdata/testinput20 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput20 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + if [ "$opt" = "-s" ] ; then echo " OK with study" + else echo " OK" + fi + done + fi +fi + +# Tests for reloads with 16/32-bit library + +if [ $do21 = yes ] ; then + echo $title21 + if [ "$bits" = "8" ] ; then + echo " Skipped when running 8-bit tests" + elif [ $link_size -ne 2 ] ; then + echo " Skipped because link size is not 2" + else + cp -f $testdata/saved8 testsaved8 + cp -f $testdata/saved16LE-1 testsaved16LE-1 + cp -f $testdata/saved16BE-1 testsaved16BE-1 + cp -f $testdata/saved32LE-1 testsaved32LE-1 + cp -f $testdata/saved32BE-1 testsaved32BE-1 + $sim $valgrind ./pcretest -q $bmode $testdata/testinput21 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput21-$bits testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + echo " OK" + fi +fi + +# Tests for reloads with 16/32-bit library (UTF-16 support) + +if [ $do22 = yes ] ; then + echo $title22 + if [ "$bits" = "8" ] ; then + echo " Skipped when running 8-bit tests" + elif [ $utf -eq 0 ] ; then + echo " Skipped because UTF-$bits support is not available" + elif [ $link_size -ne 2 ] ; then + echo " Skipped because link size is not 2" + else + cp -f $testdata/saved16LE-2 testsaved16LE-2 + cp -f $testdata/saved16BE-2 testsaved16BE-2 + cp -f $testdata/saved32LE-2 testsaved32LE-2 + cp -f $testdata/saved32BE-2 testsaved32BE-2 + $sim $valgrind ./pcretest -q $bmode $testdata/testinput22 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput22-$bits testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + echo " OK" + fi +fi + +if [ $do23 = yes ] ; then + echo $title23 + if [ "$bits" = "8" -o "$bits" = "32" ] ; then + echo " Skipped when running 8/32-bit tests" + else + $sim $valgrind ./pcretest -q $bmode $testdata/testinput23 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput23 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + echo " OK" + fi +fi + +if [ $do24 = yes ] ; then + echo $title24 + if [ "$bits" = "8" -o "$bits" = "32" ] ; then + echo " Skipped when running 8/32-bit tests" + elif [ $utf -eq 0 ] ; then + echo " Skipped because UTF-$bits support is not available" + else + $sim $valgrind ./pcretest -q $bmode $testdata/testinput24 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput24 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + echo " OK" + fi +fi + +if [ $do25 = yes ] ; then + echo $title25 + if [ "$bits" = "8" -o "$bits" = "16" ] ; then + echo " Skipped when running 8/16-bit tests" + else + $sim $valgrind ./pcretest -q $bmode $testdata/testinput25 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput25 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + echo " OK" + fi +fi + +if [ $do26 = yes ] ; then + echo $title26 + if [ "$bits" = "8" -o "$bits" = "16" ] ; then + echo " Skipped when running 8/16-bit tests" + elif [ $utf -eq 0 ] ; then + echo " Skipped because UTF-$bits support is not available" + else + $sim $valgrind ./pcretest -q $bmode $testdata/testinput26 testtry + if [ $? = 0 ] ; then + $cf $testdata/testoutput26 testtry + if [ $? != 0 ] ; then exit 1; fi + else exit 1 + fi + echo " OK" + fi +fi + +# End of loop for 8/16/32-bit tests +done + +# Clean up local working files +rm -f test3input test3output test3outputA testNinput testsaved* teststderr teststdout testtry + +# End diff --git a/src/third_party/pcre-8.41/RunTest.bat b/src/third_party/pcre-8.41/RunTest.bat new file mode 100644 index 00000000000..35d7f71f9e8 --- /dev/null +++ b/src/third_party/pcre-8.41/RunTest.bat @@ -0,0 +1,616 @@ +@echo off +@rem This file must use CRLF linebreaks to function properly +@rem and requires both pcretest and pcregrep +@rem This file was originally contributed by Ralf Junker, and touched up by +@rem Daniel Richard G. Tests 10-12 added by Philip H. +@rem Philip H also changed test 3 to use "wintest" files. +@rem +@rem Updated by Tom Fortmann to support explicit test numbers on the command line. +@rem Added argument validation and added error reporting. +@rem +@rem MS Windows batch file to run pcretest on testfiles with the correct +@rem options. +@rem +@rem Sheri Pierce added logic to skip feature dependent tests +@rem tests 4 5 9 15 and 18 require utf support +@rem tests 6 7 10 16 and 19 require ucp support +@rem 11 requires ucp and link size 2 +@rem 12 requires presence of jit support +@rem 13 requires absence of jit support +@rem Sheri P also added override tests for study and jit testing +@rem Zoltan Herczeg added libpcre16 support +@rem Zoltan Herczeg added libpcre32 support + +setlocal enabledelayedexpansion +if [%srcdir%]==[] ( +if exist testdata\ set srcdir=.) +if [%srcdir%]==[] ( +if exist ..\testdata\ set srcdir=..) +if [%srcdir%]==[] ( +if exist ..\..\testdata\ set srcdir=..\..) +if NOT exist %srcdir%\testdata\ ( +Error: echo distribution testdata folder not found! +call :conferror +exit /b 1 +goto :eof +) + +if [%pcretest%]==[] set pcretest=.\pcretest.exe + +echo source dir is %srcdir% +echo pcretest=%pcretest% + +if NOT exist %pcretest% ( +echo Error: %pcretest% not found! +echo. +call :conferror +exit /b 1 +) + +%pcretest% -C linksize >NUL +set link_size=%ERRORLEVEL% +%pcretest% -C pcre8 >NUL +set support8=%ERRORLEVEL% +%pcretest% -C pcre16 >NUL +set support16=%ERRORLEVEL% +%pcretest% -C pcre32 >NUL +set support32=%ERRORLEVEL% +%pcretest% -C utf >NUL +set utf=%ERRORLEVEL% +%pcretest% -C ucp >NUL +set ucp=%ERRORLEVEL% +%pcretest% -C jit >NUL +set jit=%ERRORLEVEL% + +if %support8% EQU 1 ( +if not exist testout8 md testout8 +if not exist testoutstudy8 md testoutstudy8 +if not exist testoutjit8 md testoutjit8 +) + +if %support16% EQU 1 ( +if not exist testout16 md testout16 +if not exist testoutstudy16 md testoutstudy16 +if not exist testoutjit16 md testoutjit16 +) + +if %support16% EQU 1 ( +if not exist testout32 md testout32 +if not exist testoutstudy32 md testoutstudy32 +if not exist testoutjit32 md testoutjit32 +) + +set do1=no +set do2=no +set do3=no +set do4=no +set do5=no +set do6=no +set do7=no +set do8=no +set do9=no +set do10=no +set do11=no +set do12=no +set do13=no +set do14=no +set do15=no +set do16=no +set do17=no +set do18=no +set do19=no +set do20=no +set do21=no +set do22=no +set do23=no +set do24=no +set do25=no +set do26=no +set all=yes + +for %%a in (%*) do ( + set valid=no + for %%v in (1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26) do if %%v == %%a set valid=yes + if "!valid!" == "yes" ( + set do%%a=yes + set all=no +) else ( + echo Invalid test number - %%a! + echo Usage %0 [ test_number ] ... + echo Where test_number is one or more optional test numbers 1 through 26, default is all tests. + exit /b 1 +) +) +set failed="no" + +if "%all%" == "yes" ( + set do1=yes + set do2=yes + set do3=yes + set do4=yes + set do5=yes + set do6=yes + set do7=yes + set do8=yes + set do9=yes + set do10=yes + set do11=yes + set do12=yes + set do13=yes + set do14=yes + set do15=yes + set do16=yes + set do17=yes + set do18=yes + set do19=yes + set do20=yes + set do21=yes + set do22=yes + set do23=yes + set do24=yes + set do25=yes + set do26=yes +) + +@echo RunTest.bat's pcretest output is written to newly created subfolders named +@echo testout, testoutstudy and testoutjit. +@echo. + +set mode= +set bits=8 + +:nextMode +if "%mode%" == "" ( + if %support8% EQU 0 goto modeSkip + echo. + echo ---- Testing 8-bit library ---- + echo. +) +if "%mode%" == "-16" ( + if %support16% EQU 0 goto modeSkip + echo. + echo ---- Testing 16-bit library ---- + echo. +) +if "%mode%" == "-32" ( + if %support32% EQU 0 goto modeSkip + echo. + echo ---- Testing 32-bit library ---- + echo. +) +if "%do1%" == "yes" call :do1 +if "%do2%" == "yes" call :do2 +if "%do3%" == "yes" call :do3 +if "%do4%" == "yes" call :do4 +if "%do5%" == "yes" call :do5 +if "%do6%" == "yes" call :do6 +if "%do7%" == "yes" call :do7 +if "%do8%" == "yes" call :do8 +if "%do9%" == "yes" call :do9 +if "%do10%" == "yes" call :do10 +if "%do11%" == "yes" call :do11 +if "%do12%" == "yes" call :do12 +if "%do13%" == "yes" call :do13 +if "%do14%" == "yes" call :do14 +if "%do15%" == "yes" call :do15 +if "%do16%" == "yes" call :do16 +if "%do17%" == "yes" call :do17 +if "%do18%" == "yes" call :do18 +if "%do19%" == "yes" call :do19 +if "%do20%" == "yes" call :do20 +if "%do21%" == "yes" call :do21 +if "%do22%" == "yes" call :do22 +if "%do23%" == "yes" call :do23 +if "%do24%" == "yes" call :do24 +if "%do25%" == "yes" call :do25 +if "%do26%" == "yes" call :do26 +:modeSkip +if "%mode%" == "" ( + set mode=-16 + set bits=16 + goto nextMode +) +if "%mode%" == "-16" ( + set mode=-32 + set bits=32 + goto nextMode +) + +@rem If mode is -32, testing is finished +if %failed% == "yes" ( +echo In above output, one or more of the various tests failed! +exit /b 1 +) +echo All OK +goto :eof + +:runsub +@rem Function to execute pcretest and compare the output +@rem Arguments are as follows: +@rem +@rem 1 = test number +@rem 2 = outputdir +@rem 3 = test name use double quotes +@rem 4 - 9 = pcretest options + +if [%1] == [] ( + echo Missing test number argument! + exit /b 1 +) + +if [%2] == [] ( + echo Missing outputdir! + exit /b 1 +) + +if [%3] == [] ( + echo Missing test name argument! + exit /b 1 +) + +set testinput=testinput%1 +set testoutput=testoutput%1 +if exist %srcdir%\testdata\win%testinput% ( + set testinput=wintestinput%1 + set testoutput=wintestoutput%1 +) + +echo Test %1: %3 +%pcretest% %mode% %4 %5 %6 %7 %8 %9 %srcdir%\testdata\%testinput% >%2%bits%\%testoutput% +if errorlevel 1 ( + echo. failed executing command-line: + echo. %pcretest% %mode% %4 %5 %6 %7 %8 %9 %srcdir%\testdata\%testinput% ^>%2%bits%\%testoutput% + set failed="yes" + goto :eof +) + +set type= +if [%1]==[11] ( + set type=-%bits% +) +if [%1]==[18] ( + set type=-%bits% +) +if [%1]==[21] ( + set type=-%bits% +) +if [%1]==[22] ( + set type=-%bits% +) + +fc /n %srcdir%\testdata\%testoutput%%type% %2%bits%\%testoutput% >NUL + +if errorlevel 1 ( + echo. failed comparison: fc /n %srcdir%\testdata\%testoutput% %2%bits%\%testoutput% + if [%1]==[2] ( + echo. + echo ** Test 2 requires a lot of stack. PCRE can be configured to + echo ** use heap for recursion. Otherwise, to pass Test 2 + echo ** you generally need to allocate 8 mb stack to PCRE. + echo ** See the 'pcrestack' page for a discussion of PCRE's + echo ** stack usage. + echo. +) + if [%1]==[3] ( + echo. + echo ** Test 3 failure usually means french locale is not + echo ** available on the system, rather than a bug or problem with PCRE. + echo. + goto :eof +) + + set failed="yes" + goto :eof +) + +echo. Passed. +goto :eof + +:do1 +call :runsub 1 testout "Main functionality (Compatible with Perl >= 5.10)" -q +call :runsub 1 testoutstudy "Test with Study Override" -q -s +if %jit% EQU 1 call :runsub 1 testoutjit "Test with JIT Override" -q -s+ +goto :eof + +:do2 + call :runsub 2 testout "API, errors, internals, and non-Perl stuff" -q + call :runsub 2 testoutstudy "Test with Study Override" -q -s + if %jit% EQU 1 call :runsub 2 testoutjit "Test with JIT Override" -q -s+ +goto :eof + +:do3 + call :runsub 3 testout "Locale-specific features" -q + call :runsub 3 testoutstudy "Test with Study Override" -q -s + if %jit% EQU 1 call :runsub 3 testoutjit "Test with JIT Override" -q -s+ +goto :eof + +:do4 +if %utf% EQU 0 ( + echo Test 4 Skipped due to absence of UTF-%bits% support. + goto :eof +) + call :runsub 4 testout "UTF-%bits% support - (Compatible with Perl >= 5.10)" -q + call :runsub 4 testoutstudy "Test with Study Override" -q -s + if %jit% EQU 1 call :runsub 4 testoutjit "Test with JIT Override" -q -s+ +goto :eof + +:do5 +if %utf% EQU 0 ( + echo Test 5 Skipped due to absence of UTF-%bits% support. + goto :eof +) + call :runsub 5 testout "API, internals, and non-Perl stuff for UTF-%bits%" -q + call :runsub 5 testoutstudy "Test with Study Override" -q -s + if %jit% EQU 1 call :runsub 5 testoutjit "Test with JIT Override" -q -s+ +goto :eof + +:do6 +if %ucp% EQU 0 ( + echo Test 6 Skipped due to absence of Unicode property support. + goto :eof +) + call :runsub 6 testout "Unicode property support (Compatible with Perl >= 5.10)" -q + call :runsub 6 testoutstudy "Test with Study Override" -q -s + if %jit% EQU 1 call :runsub 6 testoutjit "Test with JIT Override" -q -s+ +goto :eof + +:do7 +if %ucp% EQU 0 ( + echo Test 7 Skipped due to absence of Unicode property support. + goto :eof +) + call :runsub 7 testout "API, internals, and non-Perl stuff for Unicode property support" -q + call :runsub 7 testoutstudy "Test with Study Override" -q -s + if %jit% EQU 1 call :runsub 7 testoutjit "Test with JIT Override" -q -s+ +goto :eof + +:do8 + call :runsub 8 testout "DFA matching main functionality" -q -dfa + call :runsub 8 testoutstudy "Test with Study Override" -q -dfa -s +goto :eof + +:do9 +if %utf% EQU 0 ( + echo Test 9 Skipped due to absence of UTF-%bits% support. + goto :eof +) + call :runsub 9 testout "DFA matching with UTF-%bits%" -q -dfa + call :runsub 9 testoutstudy "Test with Study Override" -q -dfa -s + goto :eof + +:do10 +if %ucp% EQU 0 ( + echo Test 10 Skipped due to absence of Unicode property support. + goto :eof +) + call :runsub 10 testout "DFA matching with Unicode properties" -q -dfa + call :runsub 10 testoutstudy "Test with Study Override" -q -dfa -s +goto :eof + +:do11 +if NOT %link_size% EQU 2 ( + echo Test 11 Skipped because link size is not 2. + goto :eof +) +if %ucp% EQU 0 ( + echo Test 11 Skipped due to absence of Unicode property support. + goto :eof +) + call :runsub 11 testout "Internal offsets and code size tests" -q + call :runsub 11 testoutstudy "Test with Study Override" -q -s +goto :eof + +:do12 +if %jit% EQU 0 ( + echo Test 12 Skipped due to absence of JIT support. + goto :eof +) + call :runsub 12 testout "JIT-specific features (JIT available)" -q +goto :eof + +:do13 +if %jit% EQU 1 ( + echo Test 13 Skipped due to presence of JIT support. + goto :eof +) + call :runsub 13 testout "JIT-specific features (JIT not available)" -q +goto :eof + +:do14 +if NOT %bits% EQU 8 ( + echo Test 14 Skipped when running 16/32-bit tests. + goto :eof +) + copy /Y %srcdir%\testdata\saved16 testsaved16 + copy /Y %srcdir%\testdata\saved32 testsaved32 + call :runsub 14 testout "Specials for the basic 8-bit library" -q + call :runsub 14 testoutstudy "Test with Study Override" -q -s + if %jit% EQU 1 call :runsub 14 testoutjit "Test with JIT Override" -q -s+ +goto :eof + +:do15 +if NOT %bits% EQU 8 ( + echo Test 15 Skipped when running 16/32-bit tests. + goto :eof +) +if %utf% EQU 0 ( + echo Test 15 Skipped due to absence of UTF-%bits% support. + goto :eof +) + call :runsub 15 testout "Specials for the 8-bit library with UTF-%bits% support" -q + call :runsub 15 testoutstudy "Test with Study Override" -q -s + if %jit% EQU 1 call :runsub 15 testoutjit "Test with JIT Override" -q -s+ +goto :eof + +:do16 +if NOT %bits% EQU 8 ( + echo Test 16 Skipped when running 16/32-bit tests. + goto :eof +) +if %ucp% EQU 0 ( + echo Test 16 Skipped due to absence of Unicode property support. + goto :eof +) + call :runsub 16 testout "Specials for the 8-bit library with Unicode propery support" -q + call :runsub 16 testoutstudy "Test with Study Override" -q -s + if %jit% EQU 1 call :runsub 16 testoutjit "Test with JIT Override" -q -s+ +goto :eof + +:do17 +if %bits% EQU 8 ( + echo Test 17 Skipped when running 8-bit tests. + goto :eof +) + call :runsub 17 testout "Specials for the basic 16/32-bit library" -q + call :runsub 17 testoutstudy "Test with Study Override" -q -s + if %jit% EQU 1 call :runsub 17 testoutjit "Test with JIT Override" -q -s+ +goto :eof + +:do18 +if %bits% EQU 8 ( + echo Test 18 Skipped when running 8-bit tests. + goto :eof +) +if %utf% EQU 0 ( + echo Test 18 Skipped due to absence of UTF-%bits% support. + goto :eof +) + call :runsub 18 testout "Specials for the 16/32-bit library with UTF-%bits% support" -q + call :runsub 18 testoutstudy "Test with Study Override" -q -s + if %jit% EQU 1 call :runsub 18 testoutjit "Test with JIT Override" -q -s+ +goto :eof + +:do19 +if %bits% EQU 8 ( + echo Test 19 Skipped when running 8-bit tests. + goto :eof +) +if %ucp% EQU 0 ( + echo Test 19 Skipped due to absence of Unicode property support. + goto :eof +) + call :runsub 19 testout "Specials for the 16/32-bit library with Unicode property support" -q + call :runsub 19 testoutstudy "Test with Study Override" -q -s + if %jit% EQU 1 call :runsub 19 testoutjit "Test with JIT Override" -q -s+ +goto :eof + +:do20 +if %bits% EQU 8 ( + echo Test 20 Skipped when running 8-bit tests. + goto :eof +) + call :runsub 20 testout "DFA specials for the basic 16/32-bit library" -q -dfa + call :runsub 20 testoutstudy "Test with Study Override" -q -dfa -s +goto :eof + +:do21 +if %bits% EQU 8 ( + echo Test 21 Skipped when running 8-bit tests. + goto :eof +) +if NOT %link_size% EQU 2 ( + echo Test 21 Skipped because link size is not 2. + goto :eof +) +copy /Y %srcdir%\testdata\saved8 testsaved8 +copy /Y %srcdir%\testdata\saved16LE-1 testsaved16LE-1 +copy /Y %srcdir%\testdata\saved16BE-1 testsaved16BE-1 +copy /Y %srcdir%\testdata\saved32LE-1 testsaved32LE-1 +copy /Y %srcdir%\testdata\saved32BE-1 testsaved32BE-1 +call :runsub 21 testout "Reloads for the basic 16/32-bit library" -q +call :runsub 21 testoutstudy "Test with Study Override" -q -s +if %jit% EQU 1 call :runsub 21 testoutjit "Test with JIT Override" -q -s+ +goto :eof + +:do22 +if %bits% EQU 8 ( + echo Test 22 Skipped when running 8-bit tests. + goto :eof +) +if %utf% EQU 0 ( + echo Test 22 Skipped due to absence of UTF-%bits% support. + goto :eof +) +if NOT %link_size% EQU 2 ( + echo Test 22 Skipped because link size is not 2. + goto :eof +) +copy /Y %srcdir%\testdata\saved16LE-2 testsaved16LE-2 +copy /Y %srcdir%\testdata\saved16BE-2 testsaved16BE-2 +copy /Y %srcdir%\testdata\saved32LE-2 testsaved32LE-2 +copy /Y %srcdir%\testdata\saved32BE-2 testsaved32BE-2 +call :runsub 22 testout "Reloads for the 16/32-bit library with UTF-16/32 support" -q +call :runsub 22 testoutstudy "Test with Study Override" -q -s +if %jit% EQU 1 call :runsub 22 testoutjit "Test with JIT Override" -q -s+ +goto :eof + +:do23 +if NOT %bits% EQU 16 ( + echo Test 23 Skipped when running 8/32-bit tests. + goto :eof +) +call :runsub 23 testout "Specials for the 16-bit library" -q +call :runsub 23 testoutstudy "Test with Study Override" -q -s +if %jit% EQU 1 call :runsub 23 testoutjit "Test with JIT Override" -q -s+ +goto :eof + +:do24 +if NOT %bits% EQU 16 ( + echo Test 24 Skipped when running 8/32-bit tests. + goto :eof +) +if %utf% EQU 0 ( + echo Test 24 Skipped due to absence of UTF-%bits% support. + goto :eof +) +call :runsub 24 testout "Specials for the 16-bit library with UTF-16 support" -q +call :runsub 24 testoutstudy "Test with Study Override" -q -s +if %jit% EQU 1 call :runsub 24 testoutjit "Test with JIT Override" -q -s+ +goto :eof + +:do25 +if NOT %bits% EQU 32 ( + echo Test 25 Skipped when running 8/16-bit tests. + goto :eof +) +call :runsub 25 testout "Specials for the 32-bit library" -q +call :runsub 25 testoutstudy "Test with Study Override" -q -s +if %jit% EQU 1 call :runsub 25 testoutjit "Test with JIT Override" -q -s+ +goto :eof + +:do26 +if NOT %bits% EQU 32 ( + echo Test 26 Skipped when running 8/16-bit tests. + goto :eof +) +if %utf% EQU 0 ( + echo Test 26 Skipped due to absence of UTF-%bits% support. + goto :eof +) +call :runsub 26 testout "Specials for the 32-bit library with UTF-32 support" -q +call :runsub 26 testoutstudy "Test with Study Override" -q -s +if %jit% EQU 1 call :runsub 26 testoutjit "Test with JIT Override" -q -s+ +goto :eof + +:conferror +@echo. +@echo Either your build is incomplete or you have a configuration error. +@echo. +@echo If configured with cmake and executed via "make test" or the MSVC "RUN_TESTS" +@echo project, pcre_test.bat defines variables and automatically calls RunTest.bat. +@echo For manual testing of all available features, after configuring with cmake +@echo and building, you can run the built pcre_test.bat. For best results with +@echo cmake builds and tests avoid directories with full path names that include +@echo spaces for source or build. +@echo. +@echo Otherwise, if the build dir is in a subdir of the source dir, testdata needed +@echo for input and verification should be found automatically when (from the +@echo location of the the built exes) you call RunTest.bat. By default RunTest.bat +@echo runs all tests compatible with the linked pcre library but it can be given +@echo a test number as an argument. +@echo. +@echo If the build dir is not under the source dir you can either copy your exes +@echo to the source folder or copy RunTest.bat and the testdata folder to the +@echo location of your built exes and then run RunTest.bat. +@echo. +goto :eof diff --git a/src/third_party/pcre-8.41/SConscript b/src/third_party/pcre-8.41/SConscript new file mode 100644 index 00000000000..e7b9a59a550 --- /dev/null +++ b/src/third_party/pcre-8.41/SConscript @@ -0,0 +1,68 @@ +# -*- mode: python -*- + +Import("env") + +env = env.Clone() +env.Append( CPPDEFINES=[ "HAVE_CONFIG_H", ] ) + +def removeIfPresent(lst, item): + try: + lst.remove(item) + except ValueError: + pass + +for to_remove in ['-Werror', '-Wall', '-W']: + removeIfPresent(env['CCFLAGS'], to_remove) + +# Directories that include generated config.h for various platforms +# +# Generated via +# AutoTools (non-Windows) +# ./configure --disable-stack-for-recursion --enable-utf --enable-unicode-properties +# --with-match-limit=200000 --with-match-limit-recursion=4000 --enable-shared=no +# CMake (Windows) +# -DPCRE_SUPPORT_PCREGREP_JIT:BOOL="0" -DPCRE_BUILD_TESTS:BOOL="0" +# -DPCRE_POSIX_MALLOC_THRESHOLD:STRING="10" -DPCRE_MATCH_LIMIT_RECURSION:STRING="4000" +# -DPCRE_NO_RECURSE:BOOL="1" -DPCRE_LINK_SIZE:STRING="2" -DPCRE_NEWLINE:STRING="LF" +# -DPCRE_SUPPORT_UNICODE_PROPERTIES:BOOL="1" -DPCREGREP_BUFSIZE:STRING="20480" +# -DPCRE_MATCH_LIMIT:STRING="200000" -DPCRE_PARENS_NEST_LIMIT:STRING="250" +# -DPCRE_SUPPORT_UTF:BOOL="1" +# +if env.TargetOSIs('windows'): + env.Append(CPPPATH=["build_windows"]) +elif env.TargetOSIs('solaris'): + env.Append(CPPPATH=["build_solaris"]) +else: + env.Append(CPPPATH=["build_posix"]) + +env.Library( "pcrecpp", [ + # pcre + "pcre_byte_order.c", + "pcre_compile.c", + "pcre_config.c", + "pcre_dfa_exec.c", + "pcre_exec.c", + "pcre_fullinfo.c", + "pcre_get.c", + "pcre_globals.c", + "pcre_maketables.c", + "pcre_newline.c", + "pcre_ord2utf8.c", + "pcre_refcount.c", + "pcre_string_utils.c", + "pcre_study.c", + "pcre_tables.c", + "pcre_ucd.c", + "pcre_valid_utf8.c", + "pcre_version.c", + "pcre_xclass.c", + # pcre nodist + "pcre_chartables.c", + # pcre cpp + "pcrecpp.cc", + "pcre_scanner.cc", + "pcre_stringpiece.cc", + # pcre posix + "pcreposix.c", + ] ) + diff --git a/src/third_party/pcre-8.41/ar-lib b/src/third_party/pcre-8.41/ar-lib new file mode 100755 index 00000000000..463b9ec0206 --- /dev/null +++ b/src/third_party/pcre-8.41/ar-lib @@ -0,0 +1,270 @@ +#! /bin/sh +# Wrapper for Microsoft lib.exe + +me=ar-lib +scriptversion=2012-03-01.08; # UTC + +# Copyright (C) 2010-2014 Free Software Foundation, Inc. +# Written by Peter Rosin . +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# This file is maintained in Automake, please report +# bugs to or send patches to +# . + + +# func_error message +func_error () +{ + echo "$me: $1" 1>&2 + exit 1 +} + +file_conv= + +# func_file_conv build_file +# Convert a $build file to $host form and store it in $file +# Currently only supports Windows hosts. +func_file_conv () +{ + file=$1 + case $file in + / | /[!/]*) # absolute file, and not a UNC file + if test -z "$file_conv"; then + # lazily determine how to convert abs files + case `uname -s` in + MINGW*) + file_conv=mingw + ;; + CYGWIN*) + file_conv=cygwin + ;; + *) + file_conv=wine + ;; + esac + fi + case $file_conv in + mingw) + file=`cmd //C echo "$file " | sed -e 's/"\(.*\) " *$/\1/'` + ;; + cygwin) + file=`cygpath -m "$file" || echo "$file"` + ;; + wine) + file=`winepath -w "$file" || echo "$file"` + ;; + esac + ;; + esac +} + +# func_at_file at_file operation archive +# Iterate over all members in AT_FILE performing OPERATION on ARCHIVE +# for each of them. +# When interpreting the content of the @FILE, do NOT use func_file_conv, +# since the user would need to supply preconverted file names to +# binutils ar, at least for MinGW. +func_at_file () +{ + operation=$2 + archive=$3 + at_file_contents=`cat "$1"` + eval set x "$at_file_contents" + shift + + for member + do + $AR -NOLOGO $operation:"$member" "$archive" || exit $? + done +} + +case $1 in + '') + func_error "no command. Try '$0 --help' for more information." + ;; + -h | --h*) + cat < header file. */ +/* #undef HAVE_BITS_TYPE_TRAITS_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_BZLIB_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DIRENT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_EDITLINE_READLINE_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_EDIT_READLINE_READLINE_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* Define to 1 if the system has the type `long long'. */ +#define HAVE_LONG_LONG 1 + +/* Define to 1 if you have the `memmove' function. */ +#define HAVE_MEMMOVE 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define if you have POSIX threads libraries and header files. */ +/* #undef HAVE_PTHREAD */ + +/* Have PTHREAD_PRIO_INHERIT. */ +/* #undef HAVE_PTHREAD_PRIO_INHERIT */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_READLINE_HISTORY_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_READLINE_READLINE_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the `strerror' function. */ +#define HAVE_STRERROR 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have `strtoimax'. */ +/* #undef HAVE_STRTOIMAX */ + +/* Define to 1 if you have `strtoll'. */ +/* #undef HAVE_STRTOLL */ + +/* Define to 1 if you have `strtoq'. */ +#define HAVE_STRTOQ 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_TYPE_TRAITS_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if the system has the type `unsigned long long'. */ +#define HAVE_UNSIGNED_LONG_LONG 1 + +/* Define to 1 if the compiler supports simple visibility declarations. */ +#define HAVE_VISIBILITY 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WINDOWS_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_ZLIB_H 1 + +/* Define to 1 if you have `_strtoi64'. */ +/* #undef HAVE__STRTOI64 */ + +/* The value of LINK_SIZE determines the number of bytes used to store links + as offsets within the compiled regex. The default is 2, which allows for + compiled patterns up to 64K long. This covers the vast majority of cases. + However, PCRE can also be compiled to use 3 or 4 bytes instead. This allows + for longer patterns in extreme cases. */ +#define LINK_SIZE 2 + +/* Define to the sub-directory where libtool stores uninstalled libraries. */ +#define LT_OBJDIR ".libs/" + +/* The value of MATCH_LIMIT determines the default number of times the + internal match() function can be called during a single execution of + pcre_exec(). There is a runtime interface for setting a different limit. + The limit exists in order to catch runaway regular expressions that take + for ever to determine that they do not match. The default is set very large + so that it does not accidentally catch legitimate cases. */ +#define MATCH_LIMIT 200000 + +/* The above limit applies to all calls of match(), whether or not they + increase the recursion depth. In some environments it is desirable to limit + the depth of recursive calls of match() more strictly, in order to restrict + the maximum amount of stack (or heap, if NO_RECURSE is defined) that is + used. The value of MATCH_LIMIT_RECURSION applies only to recursive calls of + match(). To have any useful effect, it must be less than the value of + MATCH_LIMIT. The default is to use the same value as MATCH_LIMIT. There is + a runtime method for setting a different limit. */ +#define MATCH_LIMIT_RECURSION 4000 + +/* This limit is parameterized just in case anybody ever wants to change it. + Care must be taken if it is increased, because it guards against integer + overflow caused by enormously large patterns. */ +#define MAX_NAME_COUNT 10000 + +/* This limit is parameterized just in case anybody ever wants to change it. + Care must be taken if it is increased, because it guards against integer + overflow caused by enormously large patterns. */ +#define MAX_NAME_SIZE 32 + +/* The value of NEWLINE determines the default newline character sequence. + PCRE client programs can override this by selecting other values at run + time. In ASCII environments, the value can be 10 (LF), 13 (CR), or 3338 + (CRLF); in EBCDIC environments the value can be 21 or 37 (LF), 13 (CR), or + 3349 or 3365 (CRLF) because there are two alternative codepoints (0x15 and + 0x25) that are used as the NL line terminator that is equivalent to ASCII + LF. In both ASCII and EBCDIC environments the value can also be -1 (ANY), + or -2 (ANYCRLF). */ +#define NEWLINE 10 + +/* PCRE uses recursive function calls to handle backtracking while matching. + This can sometimes be a problem on systems that have stacks of limited + size. Define NO_RECURSE to any value to get a version that doesn't use + recursion in the match() function; instead it creates its own stack by + steam using pcre_recurse_malloc() to obtain memory from the heap. For more + detail, see the comments and other stuff just above the match() function. + */ +#define NO_RECURSE /**/ + +/* Name of package */ +#define PACKAGE "pcre" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "PCRE" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "PCRE 8.41" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "pcre" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "8.41" + +/* The value of PARENS_NEST_LIMIT specifies the maximum depth of nested + parentheses (of any kind) in a pattern. This limits the amount of system + stack that is used while compiling a pattern. */ +#define PARENS_NEST_LIMIT 250 + +/* to make a symbol visible */ +#define PCRECPP_EXP_DECL extern __attribute__ ((visibility ("default"))) + +/* to make a symbol visible */ +#define PCRECPP_EXP_DEFN __attribute__ ((visibility ("default"))) + +/* The value of PCREGREP_BUFSIZE determines the size of buffer used by + pcregrep to hold parts of the file it is searching. This is also the + minimum value. The actual amount of memory used by pcregrep is three times + this number, because it allows for the buffering of "before" and "after" + lines. */ +#define PCREGREP_BUFSIZE 20480 + +/* to make a symbol visible */ +#define PCREPOSIX_EXP_DECL extern __attribute__ ((visibility ("default"))) + +/* to make a symbol visible */ +#define PCREPOSIX_EXP_DEFN extern __attribute__ ((visibility ("default"))) + +/* to make a symbol visible */ +#define PCRE_EXP_DATA_DEFN __attribute__ ((visibility ("default"))) + +/* to make a symbol visible */ +#define PCRE_EXP_DECL extern __attribute__ ((visibility ("default"))) + + +/* If you are compiling for a system other than a Unix-like system or + Win32, and it needs some magic to be inserted before the definition + of a function that is exported by the library, define this macro to + contain the relevant magic. If you do not define this macro, a suitable + __declspec value is used for Windows systems; in other environments + "extern" is used for a C compiler and "extern C" for a C++ compiler. + This macro apears at the start of every exported function that is part + of the external API. It does not appear on functions that are "external" + in the C sense, but which are internal to the library. */ +#define PCRE_EXP_DEFN __attribute__ ((visibility ("default"))) + +/* Define to any value if linking statically (TODO: make nice with Libtool) */ +#define PCRE_STATIC 1 + +/* When calling PCRE via the POSIX interface, additional working storage is + required for holding the pointers to capturing substrings because PCRE + requires three integers per substring, whereas the POSIX interface provides + only two. If the number of expected substrings is small, the wrapper + function uses space on the stack, because this is faster than using + malloc() for each call. The threshold above which the stack is no longer + used is defined by POSIX_MALLOC_THRESHOLD. */ +#define POSIX_MALLOC_THRESHOLD 10 + +/* Define to necessary symbol if this constant uses a non-standard name on + your system. */ +/* #undef PTHREAD_CREATE_JOINABLE */ + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define to any value to enable support for Just-In-Time compiling. */ +/* #undef SUPPORT_JIT */ + +/* Define to any value to allow pcregrep to be linked with libbz2, so that it + is able to handle .bz2 files. */ +/* #undef SUPPORT_LIBBZ2 */ + +/* Define to any value to allow pcretest to be linked with libedit. */ +/* #undef SUPPORT_LIBEDIT */ + +/* Define to any value to allow pcretest to be linked with libreadline. */ +/* #undef SUPPORT_LIBREADLINE */ + +/* Define to any value to allow pcregrep to be linked with libz, so that it is + able to handle .gz files. */ +/* #undef SUPPORT_LIBZ */ + +/* Define to any value to enable the 16 bit PCRE library. */ +/* #undef SUPPORT_PCRE16 */ + +/* Define to any value to enable the 32 bit PCRE library. */ +/* #undef SUPPORT_PCRE32 */ + +/* Define to any value to enable the 8 bit PCRE library. */ +#define SUPPORT_PCRE8 /**/ + +/* Define to any value to enable JIT support in pcregrep. */ +/* #undef SUPPORT_PCREGREP_JIT */ + +/* Define to any value to enable support for Unicode properties. */ +#define SUPPORT_UCP /**/ + +/* Define to any value to enable support for the UTF-8/16/32 Unicode encoding. + This will work even in an EBCDIC environment, but it is incompatible with + the EBCDIC macro. That is, PCRE can support *either* EBCDIC code *or* + ASCII/UTF-8/16/32, but not both at once. */ +#define SUPPORT_UTF /**/ + +/* Define to any value for valgrind support to find invalid memory reads. */ +/* #undef SUPPORT_VALGRIND */ + +/* Version number of package */ +#define VERSION "8.41" + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Define to the type of a signed integer type of width exactly 64 bits if + such a type exists and the standard includes do not define it. */ +/* #undef int64_t */ + +/* Define to `unsigned int' if does not define. */ +/* #undef size_t */ diff --git a/src/third_party/pcre-8.41/build_solaris/config.h b/src/third_party/pcre-8.41/build_solaris/config.h new file mode 100644 index 00000000000..164bd8a2e39 --- /dev/null +++ b/src/third_party/pcre-8.41/build_solaris/config.h @@ -0,0 +1,348 @@ +/* config.h. Generated from config.h.in by configure. */ +/* config.h.in. Generated from configure.ac by autoheader. */ + + +/* PCRE is written in Standard C, but there are a few non-standard things it +can cope with, allowing it to run on SunOS4 and other "close to standard" +systems. + +In environments that support the GNU autotools, config.h.in is converted into +config.h by the "configure" script. In environments that use CMake, +config-cmake.in is converted into config.h. If you are going to build PCRE "by +hand" without using "configure" or CMake, you should copy the distributed +config.h.generic to config.h, and edit the macro definitions to be the way you +need them. You must then add -DHAVE_CONFIG_H to all of your compile commands, +so that config.h is included at the start of every source. + +Alternatively, you can avoid editing by using -D on the compiler command line +to set the macro values. In this case, you do not have to set -DHAVE_CONFIG_H, +but if you do, default values will be taken from config.h for non-boolean +macros that are not defined on the command line. + +Boolean macros such as HAVE_STDLIB_H and SUPPORT_PCRE8 should either be defined +(conventionally to 1) for TRUE, and not defined at all for FALSE. All such +macros are listed as a commented #undef in config.h.generic. Macros such as +MATCH_LIMIT, whose actual value is relevant, have defaults defined, but are +surrounded by #ifndef/#endif lines so that the value can be overridden by -D. + +PCRE uses memmove() if HAVE_MEMMOVE is defined; otherwise it uses bcopy() if +HAVE_BCOPY is defined. If your system has neither bcopy() nor memmove(), make +sure both macros are undefined; an emulation function will then be used. */ + +/* By default, the \R escape sequence matches any Unicode line ending + character or sequence of characters. If BSR_ANYCRLF is defined (to any + value), this is changed so that backslash-R matches only CR, LF, or CRLF. + The build-time default can be overridden by the user of PCRE at runtime. */ +/* #undef BSR_ANYCRLF */ + +/* If you are compiling for a system that uses EBCDIC instead of ASCII + character codes, define this macro to any value. You must also edit the + NEWLINE macro below to set a suitable EBCDIC newline, commonly 21 (0x15). + On systems that can use "configure" or CMake to set EBCDIC, NEWLINE is + automatically adjusted. When EBCDIC is set, PCRE assumes that all input + strings are in EBCDIC. If you do not define this macro, PCRE will assume + input strings are ASCII or UTF-8/16/32 Unicode. It is not possible to build + a version of PCRE that supports both EBCDIC and UTF-8/16/32. */ +/* #undef EBCDIC */ + +/* In an EBCDIC environment, define this macro to any value to arrange for the + NL character to be 0x25 instead of the default 0x15. NL plays the role that + LF does in an ASCII/Unicode environment. The value must also be set in the + NEWLINE macro below. On systems that can use "configure" or CMake to set + EBCDIC_NL25, the adjustment of NEWLINE is automatic. */ +/* #undef EBCDIC_NL25 */ + +/* Define to 1 if you have the `bcopy' function. */ +#define HAVE_BCOPY 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_BITS_TYPE_TRAITS_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_BZLIB_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DIRENT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_EDITLINE_READLINE_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_EDIT_READLINE_READLINE_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* Define to 1 if the system has the type `long long'. */ +#define HAVE_LONG_LONG 1 + +/* Define to 1 if you have the `memmove' function. */ +#define HAVE_MEMMOVE 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define if you have POSIX threads libraries and header files. */ +/* #undef HAVE_PTHREAD */ + +/* Have PTHREAD_PRIO_INHERIT. */ +/* #undef HAVE_PTHREAD_PRIO_INHERIT */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_READLINE_HISTORY_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_READLINE_READLINE_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the `strerror' function. */ +#define HAVE_STRERROR 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have `strtoimax'. */ +/* #undef HAVE_STRTOIMAX */ + +/* Define to 1 if you have `strtoll'. */ +#define HAVE_STRTOLL 1 + +/* Define to 1 if you have `strtoq'. */ +/* #undef HAVE_STRTOQ */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_TYPE_TRAITS_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if the system has the type `unsigned long long'. */ +#define HAVE_UNSIGNED_LONG_LONG 1 + +/* Define to 1 if the compiler supports simple visibility declarations. */ +#define HAVE_VISIBILITY 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_WINDOWS_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_ZLIB_H 1 + +/* Define to 1 if you have `_strtoi64'. */ +/* #undef HAVE__STRTOI64 */ + +/* The value of LINK_SIZE determines the number of bytes used to store links + as offsets within the compiled regex. The default is 2, which allows for + compiled patterns up to 64K long. This covers the vast majority of cases. + However, PCRE can also be compiled to use 3 or 4 bytes instead. This allows + for longer patterns in extreme cases. */ +#define LINK_SIZE 2 + +/* Define to the sub-directory where libtool stores uninstalled libraries. */ +#define LT_OBJDIR ".libs/" + +/* The value of MATCH_LIMIT determines the default number of times the + internal match() function can be called during a single execution of + pcre_exec(). There is a runtime interface for setting a different limit. + The limit exists in order to catch runaway regular expressions that take + for ever to determine that they do not match. The default is set very large + so that it does not accidentally catch legitimate cases. */ +#define MATCH_LIMIT 200000 + +/* The above limit applies to all calls of match(), whether or not they + increase the recursion depth. In some environments it is desirable to limit + the depth of recursive calls of match() more strictly, in order to restrict + the maximum amount of stack (or heap, if NO_RECURSE is defined) that is + used. The value of MATCH_LIMIT_RECURSION applies only to recursive calls of + match(). To have any useful effect, it must be less than the value of + MATCH_LIMIT. The default is to use the same value as MATCH_LIMIT. There is + a runtime method for setting a different limit. */ +#define MATCH_LIMIT_RECURSION 4000 + +/* This limit is parameterized just in case anybody ever wants to change it. + Care must be taken if it is increased, because it guards against integer + overflow caused by enormously large patterns. */ +#define MAX_NAME_COUNT 10000 + +/* This limit is parameterized just in case anybody ever wants to change it. + Care must be taken if it is increased, because it guards against integer + overflow caused by enormously large patterns. */ +#define MAX_NAME_SIZE 32 + +/* The value of NEWLINE determines the default newline character sequence. + PCRE client programs can override this by selecting other values at run + time. In ASCII environments, the value can be 10 (LF), 13 (CR), or 3338 + (CRLF); in EBCDIC environments the value can be 21 or 37 (LF), 13 (CR), or + 3349 or 3365 (CRLF) because there are two alternative codepoints (0x15 and + 0x25) that are used as the NL line terminator that is equivalent to ASCII + LF. In both ASCII and EBCDIC environments the value can also be -1 (ANY), + or -2 (ANYCRLF). */ +#define NEWLINE 10 + +/* PCRE uses recursive function calls to handle backtracking while matching. + This can sometimes be a problem on systems that have stacks of limited + size. Define NO_RECURSE to any value to get a version that doesn't use + recursion in the match() function; instead it creates its own stack by + steam using pcre_recurse_malloc() to obtain memory from the heap. For more + detail, see the comments and other stuff just above the match() function. + */ +#define NO_RECURSE /**/ + +/* Name of package */ +#define PACKAGE "pcre" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "PCRE" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "PCRE 8.41" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "pcre" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "8.41" + +/* The value of PARENS_NEST_LIMIT specifies the maximum depth of nested + parentheses (of any kind) in a pattern. This limits the amount of system + stack that is used while compiling a pattern. */ +#define PARENS_NEST_LIMIT 250 + +/* to make a symbol visible */ +#define PCRECPP_EXP_DECL extern __attribute__ ((visibility ("default"))) + +/* to make a symbol visible */ +#define PCRECPP_EXP_DEFN __attribute__ ((visibility ("default"))) + +/* The value of PCREGREP_BUFSIZE determines the size of buffer used by + pcregrep to hold parts of the file it is searching. This is also the + minimum value. The actual amount of memory used by pcregrep is three times + this number, because it allows for the buffering of "before" and "after" + lines. */ +#define PCREGREP_BUFSIZE 20480 + +/* to make a symbol visible */ +#define PCREPOSIX_EXP_DECL extern __attribute__ ((visibility ("default"))) + +/* to make a symbol visible */ +#define PCREPOSIX_EXP_DEFN extern __attribute__ ((visibility ("default"))) + +/* to make a symbol visible */ +#define PCRE_EXP_DATA_DEFN __attribute__ ((visibility ("default"))) + +/* to make a symbol visible */ +#define PCRE_EXP_DECL extern __attribute__ ((visibility ("default"))) + + +/* If you are compiling for a system other than a Unix-like system or + Win32, and it needs some magic to be inserted before the definition + of a function that is exported by the library, define this macro to + contain the relevant magic. If you do not define this macro, a suitable + __declspec value is used for Windows systems; in other environments + "extern" is used for a C compiler and "extern C" for a C++ compiler. + This macro apears at the start of every exported function that is part + of the external API. It does not appear on functions that are "external" + in the C sense, but which are internal to the library. */ +#define PCRE_EXP_DEFN __attribute__ ((visibility ("default"))) + +/* Define to any value if linking statically (TODO: make nice with Libtool) */ +#define PCRE_STATIC 1 + +/* When calling PCRE via the POSIX interface, additional working storage is + required for holding the pointers to capturing substrings because PCRE + requires three integers per substring, whereas the POSIX interface provides + only two. If the number of expected substrings is small, the wrapper + function uses space on the stack, because this is faster than using + malloc() for each call. The threshold above which the stack is no longer + used is defined by POSIX_MALLOC_THRESHOLD. */ +#define POSIX_MALLOC_THRESHOLD 10 + +/* Define to necessary symbol if this constant uses a non-standard name on + your system. */ +/* #undef PTHREAD_CREATE_JOINABLE */ + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define to any value to enable support for Just-In-Time compiling. */ +/* #undef SUPPORT_JIT */ + +/* Define to any value to allow pcregrep to be linked with libbz2, so that it + is able to handle .bz2 files. */ +/* #undef SUPPORT_LIBBZ2 */ + +/* Define to any value to allow pcretest to be linked with libedit. */ +/* #undef SUPPORT_LIBEDIT */ + +/* Define to any value to allow pcretest to be linked with libreadline. */ +/* #undef SUPPORT_LIBREADLINE */ + +/* Define to any value to allow pcregrep to be linked with libz, so that it is + able to handle .gz files. */ +/* #undef SUPPORT_LIBZ */ + +/* Define to any value to enable the 16 bit PCRE library. */ +/* #undef SUPPORT_PCRE16 */ + +/* Define to any value to enable the 32 bit PCRE library. */ +/* #undef SUPPORT_PCRE32 */ + +/* Define to any value to enable the 8 bit PCRE library. */ +#define SUPPORT_PCRE8 /**/ + +/* Define to any value to enable JIT support in pcregrep. */ +/* #undef SUPPORT_PCREGREP_JIT */ + +/* Define to any value to enable support for Unicode properties. */ +#define SUPPORT_UCP /**/ + +/* Define to any value to enable support for the UTF-8/16/32 Unicode encoding. + This will work even in an EBCDIC environment, but it is incompatible with + the EBCDIC macro. That is, PCRE can support *either* EBCDIC code *or* + ASCII/UTF-8/16/32, but not both at once. */ +#define SUPPORT_UTF /**/ + +/* Define to any value for valgrind support to find invalid memory reads. */ +/* #undef SUPPORT_VALGRIND */ + +/* Version number of package */ +#define VERSION "8.41" + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Define to the type of a signed integer type of width exactly 64 bits if + such a type exists and the standard includes do not define it. */ +/* #undef int64_t */ + +/* Define to `unsigned int' if does not define. */ +/* #undef size_t */ diff --git a/src/third_party/pcre-8.41/build_windows/config.h b/src/third_party/pcre-8.41/build_windows/config.h new file mode 100755 index 00000000000..2e8d4ec7b6a --- /dev/null +++ b/src/third_party/pcre-8.41/build_windows/config.h @@ -0,0 +1,57 @@ +/* config.h for CMake builds */ + +/* #undef HAVE_DIRENT_H */ +#define HAVE_SYS_STAT_H 1 +#define HAVE_SYS_TYPES_H 1 +/* #undef HAVE_UNISTD_H */ +#define HAVE_WINDOWS_H 1 +#define HAVE_STDINT_H 1 +#define HAVE_INTTYPES_H 1 + +/* #undef HAVE_TYPE_TRAITS_H */ +/* #undef HAVE_BITS_TYPE_TRAITS_H */ + +/* #undef HAVE_BCOPY */ +#define HAVE_MEMMOVE 1 +#define HAVE_STRERROR 1 +#define HAVE_STRTOLL 1 +/* #undef HAVE_STRTOQ */ +#define HAVE__STRTOI64 1 + +#define PCRE_STATIC 1 + +#define SUPPORT_PCRE8 1 +/* #undef SUPPORT_PCRE16 */ +/* #undef SUPPORT_PCRE32 */ +/* #undef SUPPORT_JIT */ +/* #undef SUPPORT_PCREGREP_JIT */ +#define SUPPORT_UTF 1 +#define SUPPORT_UCP 1 +/* #undef EBCDIC */ +/* #undef EBCDIC_NL25 */ +/* #undef BSR_ANYCRLF */ +#define NO_RECURSE 1 + +#define HAVE_LONG_LONG 1 +#define HAVE_UNSIGNED_LONG_LONG 1 + +/* #undef SUPPORT_LIBBZ2 */ +/* #undef SUPPORT_LIBZ */ +/* #undef SUPPORT_LIBEDIT */ +/* #undef SUPPORT_LIBREADLINE */ + +/* #undef SUPPORT_VALGRIND */ +/* #undef SUPPORT_GCOV */ + +#define NEWLINE 10 +#define POSIX_MALLOC_THRESHOLD 10 +#define LINK_SIZE 2 +#define PARENS_NEST_LIMIT 250 +#define MATCH_LIMIT 200000 +#define MATCH_LIMIT_RECURSION 4000 +#define PCREGREP_BUFSIZE 20480 + +#define MAX_NAME_SIZE 32 +#define MAX_NAME_COUNT 10000 + +/* end config.h for CMake builds */ diff --git a/src/third_party/pcre-8.41/dftables.c b/src/third_party/pcre-8.41/dftables.c new file mode 100644 index 00000000000..1fdc8e0f231 --- /dev/null +++ b/src/third_party/pcre-8.41/dftables.c @@ -0,0 +1,212 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This is a freestanding support program to generate a file containing +character tables for PCRE. The tables are built according to the current +locale. Now that pcre_maketables is a function visible to the outside world, we +make use of its code from here in order to be consistent. */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include "pcre_internal.h" + +#define DFTABLES /* pcre_maketables.c notices this */ +#include "pcre_maketables.c" + + +int main(int argc, char **argv) +{ +FILE *f; +int i = 1; +const unsigned char *tables; +const unsigned char *base_of_tables; + +/* By default, the default C locale is used rather than what the building user +happens to have set. However, if the -L option is given, set the locale from +the LC_xxx environment variables. */ + +if (argc > 1 && strcmp(argv[1], "-L") == 0) + { + setlocale(LC_ALL, ""); /* Set from environment variables */ + i++; + } + +if (argc < i + 1) + { + fprintf(stderr, "dftables: one filename argument is required\n"); + return 1; + } + +tables = pcre_maketables(); +base_of_tables = tables; + +f = fopen(argv[i], "wb"); +if (f == NULL) + { + fprintf(stderr, "dftables: failed to open %s for writing\n", argv[1]); + return 1; + } + +/* There are several fprintf() calls here, because gcc in pedantic mode +complains about the very long string otherwise. */ + +fprintf(f, + "/*************************************************\n" + "* Perl-Compatible Regular Expressions *\n" + "*************************************************/\n\n" + "/* This file was automatically written by the dftables auxiliary\n" + "program. It contains character tables that are used when no external\n" + "tables are passed to PCRE by the application that calls it. The tables\n" + "are used only for characters whose code values are less than 256.\n\n"); +fprintf(f, + "The following #includes are present because without them gcc 4.x may remove\n" + "the array definition from the final binary if PCRE is built into a static\n" + "library and dead code stripping is activated. This leads to link errors.\n" + "Pulling in the header ensures that the array gets flagged as \"someone\n" + "outside this compilation unit might reference this\" and so it will always\n" + "be supplied to the linker. */\n\n"); + +/* Force config.h in z/OS */ + +#if defined NATIVE_ZOS +fprintf(f, + "/* For z/OS, config.h is forced */\n" + "#ifndef HAVE_CONFIG_H\n" + "#define HAVE_CONFIG_H 1\n" + "#endif\n\n"); +#endif + +fprintf(f, + "#ifdef HAVE_CONFIG_H\n" + "#include \"config.h\"\n" + "#endif\n\n" + "#include \"pcre_internal.h\"\n\n"); + +fprintf(f, + "const pcre_uint8 PRIV(default_tables)[] = {\n\n" + "/* This table is a lower casing table. */\n\n"); + +fprintf(f, " "); +for (i = 0; i < 256; i++) + { + if ((i & 7) == 0 && i != 0) fprintf(f, "\n "); + fprintf(f, "%3d", *tables++); + if (i != 255) fprintf(f, ","); + } +fprintf(f, ",\n\n"); + +fprintf(f, "/* This table is a case flipping table. */\n\n"); + +fprintf(f, " "); +for (i = 0; i < 256; i++) + { + if ((i & 7) == 0 && i != 0) fprintf(f, "\n "); + fprintf(f, "%3d", *tables++); + if (i != 255) fprintf(f, ","); + } +fprintf(f, ",\n\n"); + +fprintf(f, + "/* This table contains bit maps for various character classes.\n" + "Each map is 32 bytes long and the bits run from the least\n" + "significant end of each byte. The classes that have their own\n" + "maps are: space, xdigit, digit, upper, lower, word, graph\n" + "print, punct, and cntrl. Other classes are built from combinations. */\n\n"); + +fprintf(f, " "); +for (i = 0; i < cbit_length; i++) + { + if ((i & 7) == 0 && i != 0) + { + if ((i & 31) == 0) fprintf(f, "\n"); + fprintf(f, "\n "); + } + fprintf(f, "0x%02x", *tables++); + if (i != cbit_length - 1) fprintf(f, ","); + } +fprintf(f, ",\n\n"); + +fprintf(f, + "/* This table identifies various classes of character by individual bits:\n" + " 0x%02x white space character\n" + " 0x%02x letter\n" + " 0x%02x decimal digit\n" + " 0x%02x hexadecimal digit\n" + " 0x%02x alphanumeric or '_'\n" + " 0x%02x regular expression metacharacter or binary zero\n*/\n\n", + ctype_space, ctype_letter, ctype_digit, ctype_xdigit, ctype_word, + ctype_meta); + +fprintf(f, " "); +for (i = 0; i < 256; i++) + { + if ((i & 7) == 0 && i != 0) + { + fprintf(f, " /* "); + if (isprint(i-8)) fprintf(f, " %c -", i-8); + else fprintf(f, "%3d-", i-8); + if (isprint(i-1)) fprintf(f, " %c ", i-1); + else fprintf(f, "%3d", i-1); + fprintf(f, " */\n "); + } + fprintf(f, "0x%02x", *tables++); + if (i != 255) fprintf(f, ","); + } + +fprintf(f, "};/* "); +if (isprint(i-8)) fprintf(f, " %c -", i-8); + else fprintf(f, "%3d-", i-8); +if (isprint(i-1)) fprintf(f, " %c ", i-1); + else fprintf(f, "%3d", i-1); +fprintf(f, " */\n\n/* End of pcre_chartables.c */\n"); + +fclose(f); +free((void *)base_of_tables); +return 0; +} + +/* End of dftables.c */ diff --git a/src/third_party/pcre-8.41/libpcre.pc.in b/src/third_party/pcre-8.41/libpcre.pc.in new file mode 100644 index 00000000000..0a35da87f85 --- /dev/null +++ b/src/third_party/pcre-8.41/libpcre.pc.in @@ -0,0 +1,13 @@ +# Package Information for pkg-config + +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: libpcre +Description: PCRE - Perl compatible regular expressions C library with 8 bit character support +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lpcre +Libs.private: @PTHREAD_CFLAGS@ @PTHREAD_LIBS@ +Cflags: -I${includedir} @PCRE_STATIC_CFLAG@ diff --git a/src/third_party/pcre-8.41/libpcre16.pc.in b/src/third_party/pcre-8.41/libpcre16.pc.in new file mode 100644 index 00000000000..080c9dcfe8a --- /dev/null +++ b/src/third_party/pcre-8.41/libpcre16.pc.in @@ -0,0 +1,13 @@ +# Package Information for pkg-config + +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: libpcre16 +Description: PCRE - Perl compatible regular expressions C library with 16 bit character support +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lpcre16 +Libs.private: @PTHREAD_CFLAGS@ @PTHREAD_LIBS@ +Cflags: -I${includedir} @PCRE_STATIC_CFLAG@ diff --git a/src/third_party/pcre-8.41/libpcre32.pc.in b/src/third_party/pcre-8.41/libpcre32.pc.in new file mode 100644 index 00000000000..a3ae0e11fcf --- /dev/null +++ b/src/third_party/pcre-8.41/libpcre32.pc.in @@ -0,0 +1,13 @@ +# Package Information for pkg-config + +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: libpcre32 +Description: PCRE - Perl compatible regular expressions C library with 32 bit character support +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lpcre32 +Libs.private: @PTHREAD_CFLAGS@ @PTHREAD_LIBS@ +Cflags: -I${includedir} @PCRE_STATIC_CFLAG@ diff --git a/src/third_party/pcre-8.41/libpcrecpp.pc.in b/src/third_party/pcre-8.41/libpcrecpp.pc.in new file mode 100644 index 00000000000..ef006fe47aa --- /dev/null +++ b/src/third_party/pcre-8.41/libpcrecpp.pc.in @@ -0,0 +1,12 @@ +# Package Information for pkg-config + +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: libpcrecpp +Description: PCRECPP - C++ wrapper for PCRE +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lpcre -lpcrecpp +Cflags: -I${includedir} @PCRE_STATIC_CFLAG@ diff --git a/src/third_party/pcre-8.41/libpcreposix.pc.in b/src/third_party/pcre-8.41/libpcreposix.pc.in new file mode 100644 index 00000000000..c6c0b0c6c66 --- /dev/null +++ b/src/third_party/pcre-8.41/libpcreposix.pc.in @@ -0,0 +1,13 @@ +# Package Information for pkg-config + +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: libpcreposix +Description: PCREPosix - Posix compatible interface to libpcre +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lpcreposix +Cflags: -I${includedir} @PCRE_STATIC_CFLAG@ +Requires.private: libpcre diff --git a/src/third_party/pcre-8.41/makevp.bat b/src/third_party/pcre-8.41/makevp.bat new file mode 100644 index 00000000000..5f795487eb9 --- /dev/null +++ b/src/third_party/pcre-8.41/makevp.bat @@ -0,0 +1,66 @@ +:: AH 20-12-06 modified for new PCRE-7.0 and VP/BCC +:: PH 19-03-07 renamed !compile.txt and !linklib.txt as makevp-compile.txt and +:: makevp-linklib.txt +:: PH 26-03-07 re-renamed !compile.txt and !linklib.txt as makevp-c.txt and +:: makevp-l.txt +:: PH 29-03-07 hopefully the final rename to makevp_c and makevp_l +:: AH 27.08.08 updated for new PCRE-7.7 +:: required PCRE.H and CONFIG.H will be generated if not existing + +@echo off +echo. +echo Compiling PCRE with BORLAND C++ for VIRTUAL PASCAL +echo. + +REM This file was contributed by Alexander Tokarev for building PCRE for use +REM with Virtual Pascal. It has not been tested with the latest PCRE release. + +REM This file has been modified and extended to compile with newer PCRE releases +REM by Stefan Weber (Angels Holocaust). + +REM CHANGE THIS FOR YOUR BORLAND C++ COMPILER PATH +SET BORLAND=f:\bcc +REM location of the TASM binaries, if compiling with the -B BCC switch +SET TASM=f:\tasm + +SET PATH=%PATH%;%BORLAND%\bin;%TASM%\bin +SET PCRE_VER=77 +SET COMPILE_DEFAULTS=-DHAVE_CONFIG_H -DPCRE_STATIC -I%BORLAND%\include + +del pcre%PCRE_VER%.lib >nul 2>nul + +:: sh configure + +:: check for needed header files +if not exist pcre.h copy pcre.h.generic pcre.h +if not exist config.h copy config.h.generic config.h + +bcc32 -DDFTABLES %COMPILE_DEFAULTS% -L%BORLAND%\lib dftables.c +IF ERRORLEVEL 1 GOTO ERROR + +:: dftables > chartables.c +dftables pcre_chartables.c + +REM compile and link the PCRE library into lib: option -B for ASM compile works too +bcc32 -a4 -c -RT- -y- -v- -u- -R- -Q- -X -d -fp -ff -P- -O2 -Oc -Ov -3 -w-8004 -w-8064 -w-8065 -w-8012 -UDFTABLES -DVPCOMPAT %COMPILE_DEFAULTS% @makevp_c.txt +IF ERRORLEVEL 1 GOTO ERROR + +tlib %BORLAND%\lib\cw32.lib *calloc *del *strncmp *memcpy *memmove *memset *memcmp *strlen +IF ERRORLEVEL 1 GOTO ERROR +tlib pcre%PCRE_VER%.lib @makevp_l.txt +calloc.obj +del.obj +strncmp.obj +memcpy.obj +memmove.obj +memset.obj +memcmp.obj +strlen.obj +IF ERRORLEVEL 1 GOTO ERROR + +del *.obj *.tds *.bak >nul 2>nul + +echo --- +echo Now the library should be complete. Please check all messages above. +echo Don't care for warnings, it's OK. +goto END + +:ERROR +echo --- +echo Error while compiling PCRE. Aborting... +pause +goto END + +:END diff --git a/src/third_party/pcre-8.41/makevp_c.txt b/src/third_party/pcre-8.41/makevp_c.txt new file mode 100644 index 00000000000..5648115405d --- /dev/null +++ b/src/third_party/pcre-8.41/makevp_c.txt @@ -0,0 +1,21 @@ +pcre_byte_order.c +pcre_chartables.c +pcre_compile.c +pcre_config.c +pcre_dfa_exec.c +pcre_exec.c +pcre_fullinfo.c +pcre_get.c +pcre_globals.c +pcre_jit_compile.c +pcre_maketables.c +pcre_newline.c +pcre_ord2utf8.c +pcre_refcount.c +pcre_string_utils.c +pcre_study.c +pcre_tables.c +pcre_ucd.c +pcre_valid_utf8.c +pcre_version.c +pcre_xclass.c diff --git a/src/third_party/pcre-8.41/makevp_l.txt b/src/third_party/pcre-8.41/makevp_l.txt new file mode 100644 index 00000000000..9b071e0f90a --- /dev/null +++ b/src/third_party/pcre-8.41/makevp_l.txt @@ -0,0 +1,21 @@ ++pcre_byte_order.obj & ++pcre_chartables.obj & ++pcre_compile.obj & ++pcre_config.obj & ++pcre_dfa_exec.obj & ++pcre_exec.obj & ++pcre_fullinfo.obj & ++pcre_get.obj & ++pcre_globals.obj & ++pcre_jit_compile.obj & ++pcre_maketables.obj & ++pcre_newline.obj & ++pcre_ord2utf8.obj & ++pcre_refcount.obj & ++pcre_string_utils.obj & ++pcre_study.obj & ++pcre_tables.obj & ++pcre_ucd.obj & ++pcre_valid_utf8.obj & ++pcre_version.obj & ++pcre_xclass.obj diff --git a/src/third_party/pcre-8.41/pcre-config.in b/src/third_party/pcre-8.41/pcre-config.in new file mode 100644 index 00000000000..ac06a3325bc --- /dev/null +++ b/src/third_party/pcre-8.41/pcre-config.in @@ -0,0 +1,133 @@ +#!/bin/sh + +prefix=@prefix@ +exec_prefix=@exec_prefix@ +exec_prefix_set=no + +cflags="[--cflags]" + +if test @enable_cpp@ = yes ; then + libs="[--libs-cpp]" +else + libs= +fi + +if test @enable_pcre16@ = yes ; then + libs="[--libs16] $libs" +fi + +if test @enable_pcre32@ = yes ; then + libs="[--libs32] $libs" +fi + +if test @enable_pcre8@ = yes ; then + libs="[--libs] [--libs-posix] $libs" + cflags="$cflags [--cflags-posix]" +fi + +usage="Usage: pcre-config [--prefix] [--exec-prefix] [--version] $libs $cflags" + +if test $# -eq 0; then + echo "${usage}" 1>&2 + exit 1 +fi + +libR= +case `uname -s` in + *SunOS*) + libR=" -R@libdir@" + ;; + *BSD*) + libR=" -Wl,-R@libdir@" + ;; +esac + +libS= +if test @libdir@ != /usr/lib ; then + libS=-L@libdir@ +fi + +while test $# -gt 0; do + case "$1" in + -*=*) optarg=`echo "$1" | sed 's/[-_a-zA-Z0-9]*=//'` ;; + *) optarg= ;; + esac + + case $1 in + --prefix=*) + prefix=$optarg + if test $exec_prefix_set = no ; then + exec_prefix=$optarg + fi + ;; + --prefix) + echo $prefix + ;; + --exec-prefix=*) + exec_prefix=$optarg + exec_prefix_set=yes + ;; + --exec-prefix) + echo $exec_prefix + ;; + --version) + echo @PACKAGE_VERSION@ + ;; + --cflags) + if test @includedir@ != /usr/include ; then + includes=-I@includedir@ + fi + echo $includes @PCRE_STATIC_CFLAG@ + ;; + --cflags-posix) + if test @enable_pcre8@ = yes ; then + if test @includedir@ != /usr/include ; then + includes=-I@includedir@ + fi + echo $includes @PCRE_STATIC_CFLAG@ + else + echo "${usage}" 1>&2 + fi + ;; + --libs-posix) + if test @enable_pcre8@ = yes ; then + echo $libS$libR -lpcreposix -lpcre + else + echo "${usage}" 1>&2 + fi + ;; + --libs) + if test @enable_pcre8@ = yes ; then + echo $libS$libR -lpcre + else + echo "${usage}" 1>&2 + fi + ;; + --libs16) + if test @enable_pcre16@ = yes ; then + echo $libS$libR -lpcre16 + else + echo "${usage}" 1>&2 + fi + ;; + --libs32) + if test @enable_pcre32@ = yes ; then + echo $libS$libR -lpcre32 + else + echo "${usage}" 1>&2 + fi + ;; + --libs-cpp) + if test @enable_cpp@ = yes ; then + echo $libS$libR -lpcrecpp -lpcre + else + echo "${usage}" 1>&2 + fi + ;; + *) + echo "${usage}" 1>&2 + exit 1 + ;; + esac + shift +done diff --git a/src/third_party/pcre-8.41/pcre.h b/src/third_party/pcre-8.41/pcre.h new file mode 100644 index 00000000000..442c6bdb5fe --- /dev/null +++ b/src/third_party/pcre-8.41/pcre.h @@ -0,0 +1,677 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* This is the public header file for the PCRE library, to be #included by +applications that call the PCRE functions. + + Copyright (c) 1997-2014 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +#ifndef _PCRE_H +#define _PCRE_H + +/* The current PCRE version information. */ + +#define PCRE_MAJOR 8 +#define PCRE_MINOR 41 +#define PCRE_PRERELEASE +#define PCRE_DATE 2017-07-05 + +/* When an application links to a PCRE DLL in Windows, the symbols that are +imported have to be identified as such. When building PCRE, the appropriate +export setting is defined in pcre_internal.h, which includes this file. So we +don't change existing definitions of PCRE_EXP_DECL and PCRECPP_EXP_DECL. */ + +#if defined(_WIN32) && !defined(PCRE_STATIC) +# ifndef PCRE_EXP_DECL +# define PCRE_EXP_DECL extern __declspec(dllimport) +# endif +# ifdef __cplusplus +# ifndef PCRECPP_EXP_DECL +# define PCRECPP_EXP_DECL extern __declspec(dllimport) +# endif +# ifndef PCRECPP_EXP_DEFN +# define PCRECPP_EXP_DEFN __declspec(dllimport) +# endif +# endif +#endif + +/* By default, we use the standard "extern" declarations. */ + +#ifndef PCRE_EXP_DECL +# ifdef __cplusplus +# define PCRE_EXP_DECL extern "C" +# else +# define PCRE_EXP_DECL extern +# endif +#endif + +#ifdef __cplusplus +# ifndef PCRECPP_EXP_DECL +# define PCRECPP_EXP_DECL extern +# endif +# ifndef PCRECPP_EXP_DEFN +# define PCRECPP_EXP_DEFN +# endif +#endif + +/* Have to include stdlib.h in order to ensure that size_t is defined; +it is needed here for malloc. */ + +#include + +/* Allow for C++ users */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Public options. Some are compile-time only, some are run-time only, and some +are both. Most of the compile-time options are saved with the compiled regex so +that they can be inspected during studying (and therefore JIT compiling). Note +that pcre_study() has its own set of options. Originally, all the options +defined here used distinct bits. However, almost all the bits in a 32-bit word +are now used, so in order to conserve them, option bits that were previously +only recognized at matching time (i.e. by pcre_exec() or pcre_dfa_exec()) may +also be used for compile-time options that affect only compiling and are not +relevant for studying or JIT compiling. + +Some options for pcre_compile() change its behaviour but do not affect the +behaviour of the execution functions. Other options are passed through to the +execution functions and affect their behaviour, with or without affecting the +behaviour of pcre_compile(). + +Options that can be passed to pcre_compile() are tagged Cx below, with these +variants: + +C1 Affects compile only +C2 Does not affect compile; affects exec, dfa_exec +C3 Affects compile, exec, dfa_exec +C4 Affects compile, exec, dfa_exec, study +C5 Affects compile, exec, study + +Options that can be set for pcre_exec() and/or pcre_dfa_exec() are flagged with +E and D, respectively. They take precedence over C3, C4, and C5 settings passed +from pcre_compile(). Those that are compatible with JIT execution are flagged +with J. */ + +#define PCRE_CASELESS 0x00000001 /* C1 */ +#define PCRE_MULTILINE 0x00000002 /* C1 */ +#define PCRE_DOTALL 0x00000004 /* C1 */ +#define PCRE_EXTENDED 0x00000008 /* C1 */ +#define PCRE_ANCHORED 0x00000010 /* C4 E D */ +#define PCRE_DOLLAR_ENDONLY 0x00000020 /* C2 */ +#define PCRE_EXTRA 0x00000040 /* C1 */ +#define PCRE_NOTBOL 0x00000080 /* E D J */ +#define PCRE_NOTEOL 0x00000100 /* E D J */ +#define PCRE_UNGREEDY 0x00000200 /* C1 */ +#define PCRE_NOTEMPTY 0x00000400 /* E D J */ +#define PCRE_UTF8 0x00000800 /* C4 ) */ +#define PCRE_UTF16 0x00000800 /* C4 ) Synonyms */ +#define PCRE_UTF32 0x00000800 /* C4 ) */ +#define PCRE_NO_AUTO_CAPTURE 0x00001000 /* C1 */ +#define PCRE_NO_UTF8_CHECK 0x00002000 /* C1 E D J ) */ +#define PCRE_NO_UTF16_CHECK 0x00002000 /* C1 E D J ) Synonyms */ +#define PCRE_NO_UTF32_CHECK 0x00002000 /* C1 E D J ) */ +#define PCRE_AUTO_CALLOUT 0x00004000 /* C1 */ +#define PCRE_PARTIAL_SOFT 0x00008000 /* E D J ) Synonyms */ +#define PCRE_PARTIAL 0x00008000 /* E D J ) */ + +/* This pair use the same bit. */ +#define PCRE_NEVER_UTF 0x00010000 /* C1 ) Overlaid */ +#define PCRE_DFA_SHORTEST 0x00010000 /* D ) Overlaid */ + +/* This pair use the same bit. */ +#define PCRE_NO_AUTO_POSSESS 0x00020000 /* C1 ) Overlaid */ +#define PCRE_DFA_RESTART 0x00020000 /* D ) Overlaid */ + +#define PCRE_FIRSTLINE 0x00040000 /* C3 */ +#define PCRE_DUPNAMES 0x00080000 /* C1 */ +#define PCRE_NEWLINE_CR 0x00100000 /* C3 E D */ +#define PCRE_NEWLINE_LF 0x00200000 /* C3 E D */ +#define PCRE_NEWLINE_CRLF 0x00300000 /* C3 E D */ +#define PCRE_NEWLINE_ANY 0x00400000 /* C3 E D */ +#define PCRE_NEWLINE_ANYCRLF 0x00500000 /* C3 E D */ +#define PCRE_BSR_ANYCRLF 0x00800000 /* C3 E D */ +#define PCRE_BSR_UNICODE 0x01000000 /* C3 E D */ +#define PCRE_JAVASCRIPT_COMPAT 0x02000000 /* C5 */ +#define PCRE_NO_START_OPTIMIZE 0x04000000 /* C2 E D ) Synonyms */ +#define PCRE_NO_START_OPTIMISE 0x04000000 /* C2 E D ) */ +#define PCRE_PARTIAL_HARD 0x08000000 /* E D J */ +#define PCRE_NOTEMPTY_ATSTART 0x10000000 /* E D J */ +#define PCRE_UCP 0x20000000 /* C3 */ + +/* Exec-time and get/set-time error codes */ + +#define PCRE_ERROR_NOMATCH (-1) +#define PCRE_ERROR_NULL (-2) +#define PCRE_ERROR_BADOPTION (-3) +#define PCRE_ERROR_BADMAGIC (-4) +#define PCRE_ERROR_UNKNOWN_OPCODE (-5) +#define PCRE_ERROR_UNKNOWN_NODE (-5) /* For backward compatibility */ +#define PCRE_ERROR_NOMEMORY (-6) +#define PCRE_ERROR_NOSUBSTRING (-7) +#define PCRE_ERROR_MATCHLIMIT (-8) +#define PCRE_ERROR_CALLOUT (-9) /* Never used by PCRE itself */ +#define PCRE_ERROR_BADUTF8 (-10) /* Same for 8/16/32 */ +#define PCRE_ERROR_BADUTF16 (-10) /* Same for 8/16/32 */ +#define PCRE_ERROR_BADUTF32 (-10) /* Same for 8/16/32 */ +#define PCRE_ERROR_BADUTF8_OFFSET (-11) /* Same for 8/16 */ +#define PCRE_ERROR_BADUTF16_OFFSET (-11) /* Same for 8/16 */ +#define PCRE_ERROR_PARTIAL (-12) +#define PCRE_ERROR_BADPARTIAL (-13) +#define PCRE_ERROR_INTERNAL (-14) +#define PCRE_ERROR_BADCOUNT (-15) +#define PCRE_ERROR_DFA_UITEM (-16) +#define PCRE_ERROR_DFA_UCOND (-17) +#define PCRE_ERROR_DFA_UMLIMIT (-18) +#define PCRE_ERROR_DFA_WSSIZE (-19) +#define PCRE_ERROR_DFA_RECURSE (-20) +#define PCRE_ERROR_RECURSIONLIMIT (-21) +#define PCRE_ERROR_NULLWSLIMIT (-22) /* No longer actually used */ +#define PCRE_ERROR_BADNEWLINE (-23) +#define PCRE_ERROR_BADOFFSET (-24) +#define PCRE_ERROR_SHORTUTF8 (-25) +#define PCRE_ERROR_SHORTUTF16 (-25) /* Same for 8/16 */ +#define PCRE_ERROR_RECURSELOOP (-26) +#define PCRE_ERROR_JIT_STACKLIMIT (-27) +#define PCRE_ERROR_BADMODE (-28) +#define PCRE_ERROR_BADENDIANNESS (-29) +#define PCRE_ERROR_DFA_BADRESTART (-30) +#define PCRE_ERROR_JIT_BADOPTION (-31) +#define PCRE_ERROR_BADLENGTH (-32) +#define PCRE_ERROR_UNSET (-33) + +/* Specific error codes for UTF-8 validity checks */ + +#define PCRE_UTF8_ERR0 0 +#define PCRE_UTF8_ERR1 1 +#define PCRE_UTF8_ERR2 2 +#define PCRE_UTF8_ERR3 3 +#define PCRE_UTF8_ERR4 4 +#define PCRE_UTF8_ERR5 5 +#define PCRE_UTF8_ERR6 6 +#define PCRE_UTF8_ERR7 7 +#define PCRE_UTF8_ERR8 8 +#define PCRE_UTF8_ERR9 9 +#define PCRE_UTF8_ERR10 10 +#define PCRE_UTF8_ERR11 11 +#define PCRE_UTF8_ERR12 12 +#define PCRE_UTF8_ERR13 13 +#define PCRE_UTF8_ERR14 14 +#define PCRE_UTF8_ERR15 15 +#define PCRE_UTF8_ERR16 16 +#define PCRE_UTF8_ERR17 17 +#define PCRE_UTF8_ERR18 18 +#define PCRE_UTF8_ERR19 19 +#define PCRE_UTF8_ERR20 20 +#define PCRE_UTF8_ERR21 21 +#define PCRE_UTF8_ERR22 22 /* Unused (was non-character) */ + +/* Specific error codes for UTF-16 validity checks */ + +#define PCRE_UTF16_ERR0 0 +#define PCRE_UTF16_ERR1 1 +#define PCRE_UTF16_ERR2 2 +#define PCRE_UTF16_ERR3 3 +#define PCRE_UTF16_ERR4 4 /* Unused (was non-character) */ + +/* Specific error codes for UTF-32 validity checks */ + +#define PCRE_UTF32_ERR0 0 +#define PCRE_UTF32_ERR1 1 +#define PCRE_UTF32_ERR2 2 /* Unused (was non-character) */ +#define PCRE_UTF32_ERR3 3 + +/* Request types for pcre_fullinfo() */ + +#define PCRE_INFO_OPTIONS 0 +#define PCRE_INFO_SIZE 1 +#define PCRE_INFO_CAPTURECOUNT 2 +#define PCRE_INFO_BACKREFMAX 3 +#define PCRE_INFO_FIRSTBYTE 4 +#define PCRE_INFO_FIRSTCHAR 4 /* For backwards compatibility */ +#define PCRE_INFO_FIRSTTABLE 5 +#define PCRE_INFO_LASTLITERAL 6 +#define PCRE_INFO_NAMEENTRYSIZE 7 +#define PCRE_INFO_NAMECOUNT 8 +#define PCRE_INFO_NAMETABLE 9 +#define PCRE_INFO_STUDYSIZE 10 +#define PCRE_INFO_DEFAULT_TABLES 11 +#define PCRE_INFO_OKPARTIAL 12 +#define PCRE_INFO_JCHANGED 13 +#define PCRE_INFO_HASCRORLF 14 +#define PCRE_INFO_MINLENGTH 15 +#define PCRE_INFO_JIT 16 +#define PCRE_INFO_JITSIZE 17 +#define PCRE_INFO_MAXLOOKBEHIND 18 +#define PCRE_INFO_FIRSTCHARACTER 19 +#define PCRE_INFO_FIRSTCHARACTERFLAGS 20 +#define PCRE_INFO_REQUIREDCHAR 21 +#define PCRE_INFO_REQUIREDCHARFLAGS 22 +#define PCRE_INFO_MATCHLIMIT 23 +#define PCRE_INFO_RECURSIONLIMIT 24 +#define PCRE_INFO_MATCH_EMPTY 25 + +/* Request types for pcre_config(). Do not re-arrange, in order to remain +compatible. */ + +#define PCRE_CONFIG_UTF8 0 +#define PCRE_CONFIG_NEWLINE 1 +#define PCRE_CONFIG_LINK_SIZE 2 +#define PCRE_CONFIG_POSIX_MALLOC_THRESHOLD 3 +#define PCRE_CONFIG_MATCH_LIMIT 4 +#define PCRE_CONFIG_STACKRECURSE 5 +#define PCRE_CONFIG_UNICODE_PROPERTIES 6 +#define PCRE_CONFIG_MATCH_LIMIT_RECURSION 7 +#define PCRE_CONFIG_BSR 8 +#define PCRE_CONFIG_JIT 9 +#define PCRE_CONFIG_UTF16 10 +#define PCRE_CONFIG_JITTARGET 11 +#define PCRE_CONFIG_UTF32 12 +#define PCRE_CONFIG_PARENS_LIMIT 13 + +/* Request types for pcre_study(). Do not re-arrange, in order to remain +compatible. */ + +#define PCRE_STUDY_JIT_COMPILE 0x0001 +#define PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE 0x0002 +#define PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE 0x0004 +#define PCRE_STUDY_EXTRA_NEEDED 0x0008 + +/* Bit flags for the pcre[16|32]_extra structure. Do not re-arrange or redefine +these bits, just add new ones on the end, in order to remain compatible. */ + +#define PCRE_EXTRA_STUDY_DATA 0x0001 +#define PCRE_EXTRA_MATCH_LIMIT 0x0002 +#define PCRE_EXTRA_CALLOUT_DATA 0x0004 +#define PCRE_EXTRA_TABLES 0x0008 +#define PCRE_EXTRA_MATCH_LIMIT_RECURSION 0x0010 +#define PCRE_EXTRA_MARK 0x0020 +#define PCRE_EXTRA_EXECUTABLE_JIT 0x0040 + +/* Types */ + +struct real_pcre; /* declaration; the definition is private */ +typedef struct real_pcre pcre; + +struct real_pcre16; /* declaration; the definition is private */ +typedef struct real_pcre16 pcre16; + +struct real_pcre32; /* declaration; the definition is private */ +typedef struct real_pcre32 pcre32; + +struct real_pcre_jit_stack; /* declaration; the definition is private */ +typedef struct real_pcre_jit_stack pcre_jit_stack; + +struct real_pcre16_jit_stack; /* declaration; the definition is private */ +typedef struct real_pcre16_jit_stack pcre16_jit_stack; + +struct real_pcre32_jit_stack; /* declaration; the definition is private */ +typedef struct real_pcre32_jit_stack pcre32_jit_stack; + +/* If PCRE is compiled with 16 bit character support, PCRE_UCHAR16 must contain +a 16 bit wide signed data type. Otherwise it can be a dummy data type since +pcre16 functions are not implemented. There is a check for this in pcre_internal.h. */ +#ifndef PCRE_UCHAR16 +#define PCRE_UCHAR16 unsigned short +#endif + +#ifndef PCRE_SPTR16 +#define PCRE_SPTR16 const PCRE_UCHAR16 * +#endif + +/* If PCRE is compiled with 32 bit character support, PCRE_UCHAR32 must contain +a 32 bit wide signed data type. Otherwise it can be a dummy data type since +pcre32 functions are not implemented. There is a check for this in pcre_internal.h. */ +#ifndef PCRE_UCHAR32 +#define PCRE_UCHAR32 unsigned int +#endif + +#ifndef PCRE_SPTR32 +#define PCRE_SPTR32 const PCRE_UCHAR32 * +#endif + +/* When PCRE is compiled as a C++ library, the subject pointer type can be +replaced with a custom type. For conventional use, the public interface is a +const char *. */ + +#ifndef PCRE_SPTR +#define PCRE_SPTR const char * +#endif + +/* The structure for passing additional data to pcre_exec(). This is defined in +such as way as to be extensible. Always add new fields at the end, in order to +remain compatible. */ + +typedef struct pcre_extra { + unsigned long int flags; /* Bits for which fields are set */ + void *study_data; /* Opaque data from pcre_study() */ + unsigned long int match_limit; /* Maximum number of calls to match() */ + void *callout_data; /* Data passed back in callouts */ + const unsigned char *tables; /* Pointer to character tables */ + unsigned long int match_limit_recursion; /* Max recursive calls to match() */ + unsigned char **mark; /* For passing back a mark pointer */ + void *executable_jit; /* Contains a pointer to a compiled jit code */ +} pcre_extra; + +/* Same structure as above, but with 16 bit char pointers. */ + +typedef struct pcre16_extra { + unsigned long int flags; /* Bits for which fields are set */ + void *study_data; /* Opaque data from pcre_study() */ + unsigned long int match_limit; /* Maximum number of calls to match() */ + void *callout_data; /* Data passed back in callouts */ + const unsigned char *tables; /* Pointer to character tables */ + unsigned long int match_limit_recursion; /* Max recursive calls to match() */ + PCRE_UCHAR16 **mark; /* For passing back a mark pointer */ + void *executable_jit; /* Contains a pointer to a compiled jit code */ +} pcre16_extra; + +/* Same structure as above, but with 32 bit char pointers. */ + +typedef struct pcre32_extra { + unsigned long int flags; /* Bits for which fields are set */ + void *study_data; /* Opaque data from pcre_study() */ + unsigned long int match_limit; /* Maximum number of calls to match() */ + void *callout_data; /* Data passed back in callouts */ + const unsigned char *tables; /* Pointer to character tables */ + unsigned long int match_limit_recursion; /* Max recursive calls to match() */ + PCRE_UCHAR32 **mark; /* For passing back a mark pointer */ + void *executable_jit; /* Contains a pointer to a compiled jit code */ +} pcre32_extra; + +/* The structure for passing out data via the pcre_callout_function. We use a +structure so that new fields can be added on the end in future versions, +without changing the API of the function, thereby allowing old clients to work +without modification. */ + +typedef struct pcre_callout_block { + int version; /* Identifies version of block */ + /* ------------------------ Version 0 ------------------------------- */ + int callout_number; /* Number compiled into pattern */ + int *offset_vector; /* The offset vector */ + PCRE_SPTR subject; /* The subject being matched */ + int subject_length; /* The length of the subject */ + int start_match; /* Offset to start of this match attempt */ + int current_position; /* Where we currently are in the subject */ + int capture_top; /* Max current capture */ + int capture_last; /* Most recently closed capture */ + void *callout_data; /* Data passed in with the call */ + /* ------------------- Added for Version 1 -------------------------- */ + int pattern_position; /* Offset to next item in the pattern */ + int next_item_length; /* Length of next item in the pattern */ + /* ------------------- Added for Version 2 -------------------------- */ + const unsigned char *mark; /* Pointer to current mark or NULL */ + /* ------------------------------------------------------------------ */ +} pcre_callout_block; + +/* Same structure as above, but with 16 bit char pointers. */ + +typedef struct pcre16_callout_block { + int version; /* Identifies version of block */ + /* ------------------------ Version 0 ------------------------------- */ + int callout_number; /* Number compiled into pattern */ + int *offset_vector; /* The offset vector */ + PCRE_SPTR16 subject; /* The subject being matched */ + int subject_length; /* The length of the subject */ + int start_match; /* Offset to start of this match attempt */ + int current_position; /* Where we currently are in the subject */ + int capture_top; /* Max current capture */ + int capture_last; /* Most recently closed capture */ + void *callout_data; /* Data passed in with the call */ + /* ------------------- Added for Version 1 -------------------------- */ + int pattern_position; /* Offset to next item in the pattern */ + int next_item_length; /* Length of next item in the pattern */ + /* ------------------- Added for Version 2 -------------------------- */ + const PCRE_UCHAR16 *mark; /* Pointer to current mark or NULL */ + /* ------------------------------------------------------------------ */ +} pcre16_callout_block; + +/* Same structure as above, but with 32 bit char pointers. */ + +typedef struct pcre32_callout_block { + int version; /* Identifies version of block */ + /* ------------------------ Version 0 ------------------------------- */ + int callout_number; /* Number compiled into pattern */ + int *offset_vector; /* The offset vector */ + PCRE_SPTR32 subject; /* The subject being matched */ + int subject_length; /* The length of the subject */ + int start_match; /* Offset to start of this match attempt */ + int current_position; /* Where we currently are in the subject */ + int capture_top; /* Max current capture */ + int capture_last; /* Most recently closed capture */ + void *callout_data; /* Data passed in with the call */ + /* ------------------- Added for Version 1 -------------------------- */ + int pattern_position; /* Offset to next item in the pattern */ + int next_item_length; /* Length of next item in the pattern */ + /* ------------------- Added for Version 2 -------------------------- */ + const PCRE_UCHAR32 *mark; /* Pointer to current mark or NULL */ + /* ------------------------------------------------------------------ */ +} pcre32_callout_block; + +/* Indirection for store get and free functions. These can be set to +alternative malloc/free functions if required. Special ones are used in the +non-recursive case for "frames". There is also an optional callout function +that is triggered by the (?) regex item. For Virtual Pascal, these definitions +have to take another form. */ + +#ifndef VPCOMPAT +PCRE_EXP_DECL void *(*pcre_malloc)(size_t); +PCRE_EXP_DECL void (*pcre_free)(void *); +PCRE_EXP_DECL void *(*pcre_stack_malloc)(size_t); +PCRE_EXP_DECL void (*pcre_stack_free)(void *); +PCRE_EXP_DECL int (*pcre_callout)(pcre_callout_block *); +PCRE_EXP_DECL int (*pcre_stack_guard)(void); + +PCRE_EXP_DECL void *(*pcre16_malloc)(size_t); +PCRE_EXP_DECL void (*pcre16_free)(void *); +PCRE_EXP_DECL void *(*pcre16_stack_malloc)(size_t); +PCRE_EXP_DECL void (*pcre16_stack_free)(void *); +PCRE_EXP_DECL int (*pcre16_callout)(pcre16_callout_block *); +PCRE_EXP_DECL int (*pcre16_stack_guard)(void); + +PCRE_EXP_DECL void *(*pcre32_malloc)(size_t); +PCRE_EXP_DECL void (*pcre32_free)(void *); +PCRE_EXP_DECL void *(*pcre32_stack_malloc)(size_t); +PCRE_EXP_DECL void (*pcre32_stack_free)(void *); +PCRE_EXP_DECL int (*pcre32_callout)(pcre32_callout_block *); +PCRE_EXP_DECL int (*pcre32_stack_guard)(void); +#else /* VPCOMPAT */ +PCRE_EXP_DECL void *pcre_malloc(size_t); +PCRE_EXP_DECL void pcre_free(void *); +PCRE_EXP_DECL void *pcre_stack_malloc(size_t); +PCRE_EXP_DECL void pcre_stack_free(void *); +PCRE_EXP_DECL int pcre_callout(pcre_callout_block *); +PCRE_EXP_DECL int pcre_stack_guard(void); + +PCRE_EXP_DECL void *pcre16_malloc(size_t); +PCRE_EXP_DECL void pcre16_free(void *); +PCRE_EXP_DECL void *pcre16_stack_malloc(size_t); +PCRE_EXP_DECL void pcre16_stack_free(void *); +PCRE_EXP_DECL int pcre16_callout(pcre16_callout_block *); +PCRE_EXP_DECL int pcre16_stack_guard(void); + +PCRE_EXP_DECL void *pcre32_malloc(size_t); +PCRE_EXP_DECL void pcre32_free(void *); +PCRE_EXP_DECL void *pcre32_stack_malloc(size_t); +PCRE_EXP_DECL void pcre32_stack_free(void *); +PCRE_EXP_DECL int pcre32_callout(pcre32_callout_block *); +PCRE_EXP_DECL int pcre32_stack_guard(void); +#endif /* VPCOMPAT */ + +/* User defined callback which provides a stack just before the match starts. */ + +typedef pcre_jit_stack *(*pcre_jit_callback)(void *); +typedef pcre16_jit_stack *(*pcre16_jit_callback)(void *); +typedef pcre32_jit_stack *(*pcre32_jit_callback)(void *); + +/* Exported PCRE functions */ + +PCRE_EXP_DECL pcre *pcre_compile(const char *, int, const char **, int *, + const unsigned char *); +PCRE_EXP_DECL pcre16 *pcre16_compile(PCRE_SPTR16, int, const char **, int *, + const unsigned char *); +PCRE_EXP_DECL pcre32 *pcre32_compile(PCRE_SPTR32, int, const char **, int *, + const unsigned char *); +PCRE_EXP_DECL pcre *pcre_compile2(const char *, int, int *, const char **, + int *, const unsigned char *); +PCRE_EXP_DECL pcre16 *pcre16_compile2(PCRE_SPTR16, int, int *, const char **, + int *, const unsigned char *); +PCRE_EXP_DECL pcre32 *pcre32_compile2(PCRE_SPTR32, int, int *, const char **, + int *, const unsigned char *); +PCRE_EXP_DECL int pcre_config(int, void *); +PCRE_EXP_DECL int pcre16_config(int, void *); +PCRE_EXP_DECL int pcre32_config(int, void *); +PCRE_EXP_DECL int pcre_copy_named_substring(const pcre *, const char *, + int *, int, const char *, char *, int); +PCRE_EXP_DECL int pcre16_copy_named_substring(const pcre16 *, PCRE_SPTR16, + int *, int, PCRE_SPTR16, PCRE_UCHAR16 *, int); +PCRE_EXP_DECL int pcre32_copy_named_substring(const pcre32 *, PCRE_SPTR32, + int *, int, PCRE_SPTR32, PCRE_UCHAR32 *, int); +PCRE_EXP_DECL int pcre_copy_substring(const char *, int *, int, int, + char *, int); +PCRE_EXP_DECL int pcre16_copy_substring(PCRE_SPTR16, int *, int, int, + PCRE_UCHAR16 *, int); +PCRE_EXP_DECL int pcre32_copy_substring(PCRE_SPTR32, int *, int, int, + PCRE_UCHAR32 *, int); +PCRE_EXP_DECL int pcre_dfa_exec(const pcre *, const pcre_extra *, + const char *, int, int, int, int *, int , int *, int); +PCRE_EXP_DECL int pcre16_dfa_exec(const pcre16 *, const pcre16_extra *, + PCRE_SPTR16, int, int, int, int *, int , int *, int); +PCRE_EXP_DECL int pcre32_dfa_exec(const pcre32 *, const pcre32_extra *, + PCRE_SPTR32, int, int, int, int *, int , int *, int); +PCRE_EXP_DECL int pcre_exec(const pcre *, const pcre_extra *, PCRE_SPTR, + int, int, int, int *, int); +PCRE_EXP_DECL int pcre16_exec(const pcre16 *, const pcre16_extra *, + PCRE_SPTR16, int, int, int, int *, int); +PCRE_EXP_DECL int pcre32_exec(const pcre32 *, const pcre32_extra *, + PCRE_SPTR32, int, int, int, int *, int); +PCRE_EXP_DECL int pcre_jit_exec(const pcre *, const pcre_extra *, + PCRE_SPTR, int, int, int, int *, int, + pcre_jit_stack *); +PCRE_EXP_DECL int pcre16_jit_exec(const pcre16 *, const pcre16_extra *, + PCRE_SPTR16, int, int, int, int *, int, + pcre16_jit_stack *); +PCRE_EXP_DECL int pcre32_jit_exec(const pcre32 *, const pcre32_extra *, + PCRE_SPTR32, int, int, int, int *, int, + pcre32_jit_stack *); +PCRE_EXP_DECL void pcre_free_substring(const char *); +PCRE_EXP_DECL void pcre16_free_substring(PCRE_SPTR16); +PCRE_EXP_DECL void pcre32_free_substring(PCRE_SPTR32); +PCRE_EXP_DECL void pcre_free_substring_list(const char **); +PCRE_EXP_DECL void pcre16_free_substring_list(PCRE_SPTR16 *); +PCRE_EXP_DECL void pcre32_free_substring_list(PCRE_SPTR32 *); +PCRE_EXP_DECL int pcre_fullinfo(const pcre *, const pcre_extra *, int, + void *); +PCRE_EXP_DECL int pcre16_fullinfo(const pcre16 *, const pcre16_extra *, int, + void *); +PCRE_EXP_DECL int pcre32_fullinfo(const pcre32 *, const pcre32_extra *, int, + void *); +PCRE_EXP_DECL int pcre_get_named_substring(const pcre *, const char *, + int *, int, const char *, const char **); +PCRE_EXP_DECL int pcre16_get_named_substring(const pcre16 *, PCRE_SPTR16, + int *, int, PCRE_SPTR16, PCRE_SPTR16 *); +PCRE_EXP_DECL int pcre32_get_named_substring(const pcre32 *, PCRE_SPTR32, + int *, int, PCRE_SPTR32, PCRE_SPTR32 *); +PCRE_EXP_DECL int pcre_get_stringnumber(const pcre *, const char *); +PCRE_EXP_DECL int pcre16_get_stringnumber(const pcre16 *, PCRE_SPTR16); +PCRE_EXP_DECL int pcre32_get_stringnumber(const pcre32 *, PCRE_SPTR32); +PCRE_EXP_DECL int pcre_get_stringtable_entries(const pcre *, const char *, + char **, char **); +PCRE_EXP_DECL int pcre16_get_stringtable_entries(const pcre16 *, PCRE_SPTR16, + PCRE_UCHAR16 **, PCRE_UCHAR16 **); +PCRE_EXP_DECL int pcre32_get_stringtable_entries(const pcre32 *, PCRE_SPTR32, + PCRE_UCHAR32 **, PCRE_UCHAR32 **); +PCRE_EXP_DECL int pcre_get_substring(const char *, int *, int, int, + const char **); +PCRE_EXP_DECL int pcre16_get_substring(PCRE_SPTR16, int *, int, int, + PCRE_SPTR16 *); +PCRE_EXP_DECL int pcre32_get_substring(PCRE_SPTR32, int *, int, int, + PCRE_SPTR32 *); +PCRE_EXP_DECL int pcre_get_substring_list(const char *, int *, int, + const char ***); +PCRE_EXP_DECL int pcre16_get_substring_list(PCRE_SPTR16, int *, int, + PCRE_SPTR16 **); +PCRE_EXP_DECL int pcre32_get_substring_list(PCRE_SPTR32, int *, int, + PCRE_SPTR32 **); +PCRE_EXP_DECL const unsigned char *pcre_maketables(void); +PCRE_EXP_DECL const unsigned char *pcre16_maketables(void); +PCRE_EXP_DECL const unsigned char *pcre32_maketables(void); +PCRE_EXP_DECL int pcre_refcount(pcre *, int); +PCRE_EXP_DECL int pcre16_refcount(pcre16 *, int); +PCRE_EXP_DECL int pcre32_refcount(pcre32 *, int); +PCRE_EXP_DECL pcre_extra *pcre_study(const pcre *, int, const char **); +PCRE_EXP_DECL pcre16_extra *pcre16_study(const pcre16 *, int, const char **); +PCRE_EXP_DECL pcre32_extra *pcre32_study(const pcre32 *, int, const char **); +PCRE_EXP_DECL void pcre_free_study(pcre_extra *); +PCRE_EXP_DECL void pcre16_free_study(pcre16_extra *); +PCRE_EXP_DECL void pcre32_free_study(pcre32_extra *); +PCRE_EXP_DECL const char *pcre_version(void); +PCRE_EXP_DECL const char *pcre16_version(void); +PCRE_EXP_DECL const char *pcre32_version(void); + +/* Utility functions for byte order swaps. */ +PCRE_EXP_DECL int pcre_pattern_to_host_byte_order(pcre *, pcre_extra *, + const unsigned char *); +PCRE_EXP_DECL int pcre16_pattern_to_host_byte_order(pcre16 *, pcre16_extra *, + const unsigned char *); +PCRE_EXP_DECL int pcre32_pattern_to_host_byte_order(pcre32 *, pcre32_extra *, + const unsigned char *); +PCRE_EXP_DECL int pcre16_utf16_to_host_byte_order(PCRE_UCHAR16 *, + PCRE_SPTR16, int, int *, int); +PCRE_EXP_DECL int pcre32_utf32_to_host_byte_order(PCRE_UCHAR32 *, + PCRE_SPTR32, int, int *, int); + +/* JIT compiler related functions. */ + +PCRE_EXP_DECL pcre_jit_stack *pcre_jit_stack_alloc(int, int); +PCRE_EXP_DECL pcre16_jit_stack *pcre16_jit_stack_alloc(int, int); +PCRE_EXP_DECL pcre32_jit_stack *pcre32_jit_stack_alloc(int, int); +PCRE_EXP_DECL void pcre_jit_stack_free(pcre_jit_stack *); +PCRE_EXP_DECL void pcre16_jit_stack_free(pcre16_jit_stack *); +PCRE_EXP_DECL void pcre32_jit_stack_free(pcre32_jit_stack *); +PCRE_EXP_DECL void pcre_assign_jit_stack(pcre_extra *, + pcre_jit_callback, void *); +PCRE_EXP_DECL void pcre16_assign_jit_stack(pcre16_extra *, + pcre16_jit_callback, void *); +PCRE_EXP_DECL void pcre32_assign_jit_stack(pcre32_extra *, + pcre32_jit_callback, void *); +PCRE_EXP_DECL void pcre_jit_free_unused_memory(void); +PCRE_EXP_DECL void pcre16_jit_free_unused_memory(void); +PCRE_EXP_DECL void pcre32_jit_free_unused_memory(void); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* End of pcre.h */ diff --git a/src/third_party/pcre-8.41/pcre.h.generic b/src/third_party/pcre-8.41/pcre.h.generic new file mode 100644 index 00000000000..442c6bdb5fe --- /dev/null +++ b/src/third_party/pcre-8.41/pcre.h.generic @@ -0,0 +1,677 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* This is the public header file for the PCRE library, to be #included by +applications that call the PCRE functions. + + Copyright (c) 1997-2014 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +#ifndef _PCRE_H +#define _PCRE_H + +/* The current PCRE version information. */ + +#define PCRE_MAJOR 8 +#define PCRE_MINOR 41 +#define PCRE_PRERELEASE +#define PCRE_DATE 2017-07-05 + +/* When an application links to a PCRE DLL in Windows, the symbols that are +imported have to be identified as such. When building PCRE, the appropriate +export setting is defined in pcre_internal.h, which includes this file. So we +don't change existing definitions of PCRE_EXP_DECL and PCRECPP_EXP_DECL. */ + +#if defined(_WIN32) && !defined(PCRE_STATIC) +# ifndef PCRE_EXP_DECL +# define PCRE_EXP_DECL extern __declspec(dllimport) +# endif +# ifdef __cplusplus +# ifndef PCRECPP_EXP_DECL +# define PCRECPP_EXP_DECL extern __declspec(dllimport) +# endif +# ifndef PCRECPP_EXP_DEFN +# define PCRECPP_EXP_DEFN __declspec(dllimport) +# endif +# endif +#endif + +/* By default, we use the standard "extern" declarations. */ + +#ifndef PCRE_EXP_DECL +# ifdef __cplusplus +# define PCRE_EXP_DECL extern "C" +# else +# define PCRE_EXP_DECL extern +# endif +#endif + +#ifdef __cplusplus +# ifndef PCRECPP_EXP_DECL +# define PCRECPP_EXP_DECL extern +# endif +# ifndef PCRECPP_EXP_DEFN +# define PCRECPP_EXP_DEFN +# endif +#endif + +/* Have to include stdlib.h in order to ensure that size_t is defined; +it is needed here for malloc. */ + +#include + +/* Allow for C++ users */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Public options. Some are compile-time only, some are run-time only, and some +are both. Most of the compile-time options are saved with the compiled regex so +that they can be inspected during studying (and therefore JIT compiling). Note +that pcre_study() has its own set of options. Originally, all the options +defined here used distinct bits. However, almost all the bits in a 32-bit word +are now used, so in order to conserve them, option bits that were previously +only recognized at matching time (i.e. by pcre_exec() or pcre_dfa_exec()) may +also be used for compile-time options that affect only compiling and are not +relevant for studying or JIT compiling. + +Some options for pcre_compile() change its behaviour but do not affect the +behaviour of the execution functions. Other options are passed through to the +execution functions and affect their behaviour, with or without affecting the +behaviour of pcre_compile(). + +Options that can be passed to pcre_compile() are tagged Cx below, with these +variants: + +C1 Affects compile only +C2 Does not affect compile; affects exec, dfa_exec +C3 Affects compile, exec, dfa_exec +C4 Affects compile, exec, dfa_exec, study +C5 Affects compile, exec, study + +Options that can be set for pcre_exec() and/or pcre_dfa_exec() are flagged with +E and D, respectively. They take precedence over C3, C4, and C5 settings passed +from pcre_compile(). Those that are compatible with JIT execution are flagged +with J. */ + +#define PCRE_CASELESS 0x00000001 /* C1 */ +#define PCRE_MULTILINE 0x00000002 /* C1 */ +#define PCRE_DOTALL 0x00000004 /* C1 */ +#define PCRE_EXTENDED 0x00000008 /* C1 */ +#define PCRE_ANCHORED 0x00000010 /* C4 E D */ +#define PCRE_DOLLAR_ENDONLY 0x00000020 /* C2 */ +#define PCRE_EXTRA 0x00000040 /* C1 */ +#define PCRE_NOTBOL 0x00000080 /* E D J */ +#define PCRE_NOTEOL 0x00000100 /* E D J */ +#define PCRE_UNGREEDY 0x00000200 /* C1 */ +#define PCRE_NOTEMPTY 0x00000400 /* E D J */ +#define PCRE_UTF8 0x00000800 /* C4 ) */ +#define PCRE_UTF16 0x00000800 /* C4 ) Synonyms */ +#define PCRE_UTF32 0x00000800 /* C4 ) */ +#define PCRE_NO_AUTO_CAPTURE 0x00001000 /* C1 */ +#define PCRE_NO_UTF8_CHECK 0x00002000 /* C1 E D J ) */ +#define PCRE_NO_UTF16_CHECK 0x00002000 /* C1 E D J ) Synonyms */ +#define PCRE_NO_UTF32_CHECK 0x00002000 /* C1 E D J ) */ +#define PCRE_AUTO_CALLOUT 0x00004000 /* C1 */ +#define PCRE_PARTIAL_SOFT 0x00008000 /* E D J ) Synonyms */ +#define PCRE_PARTIAL 0x00008000 /* E D J ) */ + +/* This pair use the same bit. */ +#define PCRE_NEVER_UTF 0x00010000 /* C1 ) Overlaid */ +#define PCRE_DFA_SHORTEST 0x00010000 /* D ) Overlaid */ + +/* This pair use the same bit. */ +#define PCRE_NO_AUTO_POSSESS 0x00020000 /* C1 ) Overlaid */ +#define PCRE_DFA_RESTART 0x00020000 /* D ) Overlaid */ + +#define PCRE_FIRSTLINE 0x00040000 /* C3 */ +#define PCRE_DUPNAMES 0x00080000 /* C1 */ +#define PCRE_NEWLINE_CR 0x00100000 /* C3 E D */ +#define PCRE_NEWLINE_LF 0x00200000 /* C3 E D */ +#define PCRE_NEWLINE_CRLF 0x00300000 /* C3 E D */ +#define PCRE_NEWLINE_ANY 0x00400000 /* C3 E D */ +#define PCRE_NEWLINE_ANYCRLF 0x00500000 /* C3 E D */ +#define PCRE_BSR_ANYCRLF 0x00800000 /* C3 E D */ +#define PCRE_BSR_UNICODE 0x01000000 /* C3 E D */ +#define PCRE_JAVASCRIPT_COMPAT 0x02000000 /* C5 */ +#define PCRE_NO_START_OPTIMIZE 0x04000000 /* C2 E D ) Synonyms */ +#define PCRE_NO_START_OPTIMISE 0x04000000 /* C2 E D ) */ +#define PCRE_PARTIAL_HARD 0x08000000 /* E D J */ +#define PCRE_NOTEMPTY_ATSTART 0x10000000 /* E D J */ +#define PCRE_UCP 0x20000000 /* C3 */ + +/* Exec-time and get/set-time error codes */ + +#define PCRE_ERROR_NOMATCH (-1) +#define PCRE_ERROR_NULL (-2) +#define PCRE_ERROR_BADOPTION (-3) +#define PCRE_ERROR_BADMAGIC (-4) +#define PCRE_ERROR_UNKNOWN_OPCODE (-5) +#define PCRE_ERROR_UNKNOWN_NODE (-5) /* For backward compatibility */ +#define PCRE_ERROR_NOMEMORY (-6) +#define PCRE_ERROR_NOSUBSTRING (-7) +#define PCRE_ERROR_MATCHLIMIT (-8) +#define PCRE_ERROR_CALLOUT (-9) /* Never used by PCRE itself */ +#define PCRE_ERROR_BADUTF8 (-10) /* Same for 8/16/32 */ +#define PCRE_ERROR_BADUTF16 (-10) /* Same for 8/16/32 */ +#define PCRE_ERROR_BADUTF32 (-10) /* Same for 8/16/32 */ +#define PCRE_ERROR_BADUTF8_OFFSET (-11) /* Same for 8/16 */ +#define PCRE_ERROR_BADUTF16_OFFSET (-11) /* Same for 8/16 */ +#define PCRE_ERROR_PARTIAL (-12) +#define PCRE_ERROR_BADPARTIAL (-13) +#define PCRE_ERROR_INTERNAL (-14) +#define PCRE_ERROR_BADCOUNT (-15) +#define PCRE_ERROR_DFA_UITEM (-16) +#define PCRE_ERROR_DFA_UCOND (-17) +#define PCRE_ERROR_DFA_UMLIMIT (-18) +#define PCRE_ERROR_DFA_WSSIZE (-19) +#define PCRE_ERROR_DFA_RECURSE (-20) +#define PCRE_ERROR_RECURSIONLIMIT (-21) +#define PCRE_ERROR_NULLWSLIMIT (-22) /* No longer actually used */ +#define PCRE_ERROR_BADNEWLINE (-23) +#define PCRE_ERROR_BADOFFSET (-24) +#define PCRE_ERROR_SHORTUTF8 (-25) +#define PCRE_ERROR_SHORTUTF16 (-25) /* Same for 8/16 */ +#define PCRE_ERROR_RECURSELOOP (-26) +#define PCRE_ERROR_JIT_STACKLIMIT (-27) +#define PCRE_ERROR_BADMODE (-28) +#define PCRE_ERROR_BADENDIANNESS (-29) +#define PCRE_ERROR_DFA_BADRESTART (-30) +#define PCRE_ERROR_JIT_BADOPTION (-31) +#define PCRE_ERROR_BADLENGTH (-32) +#define PCRE_ERROR_UNSET (-33) + +/* Specific error codes for UTF-8 validity checks */ + +#define PCRE_UTF8_ERR0 0 +#define PCRE_UTF8_ERR1 1 +#define PCRE_UTF8_ERR2 2 +#define PCRE_UTF8_ERR3 3 +#define PCRE_UTF8_ERR4 4 +#define PCRE_UTF8_ERR5 5 +#define PCRE_UTF8_ERR6 6 +#define PCRE_UTF8_ERR7 7 +#define PCRE_UTF8_ERR8 8 +#define PCRE_UTF8_ERR9 9 +#define PCRE_UTF8_ERR10 10 +#define PCRE_UTF8_ERR11 11 +#define PCRE_UTF8_ERR12 12 +#define PCRE_UTF8_ERR13 13 +#define PCRE_UTF8_ERR14 14 +#define PCRE_UTF8_ERR15 15 +#define PCRE_UTF8_ERR16 16 +#define PCRE_UTF8_ERR17 17 +#define PCRE_UTF8_ERR18 18 +#define PCRE_UTF8_ERR19 19 +#define PCRE_UTF8_ERR20 20 +#define PCRE_UTF8_ERR21 21 +#define PCRE_UTF8_ERR22 22 /* Unused (was non-character) */ + +/* Specific error codes for UTF-16 validity checks */ + +#define PCRE_UTF16_ERR0 0 +#define PCRE_UTF16_ERR1 1 +#define PCRE_UTF16_ERR2 2 +#define PCRE_UTF16_ERR3 3 +#define PCRE_UTF16_ERR4 4 /* Unused (was non-character) */ + +/* Specific error codes for UTF-32 validity checks */ + +#define PCRE_UTF32_ERR0 0 +#define PCRE_UTF32_ERR1 1 +#define PCRE_UTF32_ERR2 2 /* Unused (was non-character) */ +#define PCRE_UTF32_ERR3 3 + +/* Request types for pcre_fullinfo() */ + +#define PCRE_INFO_OPTIONS 0 +#define PCRE_INFO_SIZE 1 +#define PCRE_INFO_CAPTURECOUNT 2 +#define PCRE_INFO_BACKREFMAX 3 +#define PCRE_INFO_FIRSTBYTE 4 +#define PCRE_INFO_FIRSTCHAR 4 /* For backwards compatibility */ +#define PCRE_INFO_FIRSTTABLE 5 +#define PCRE_INFO_LASTLITERAL 6 +#define PCRE_INFO_NAMEENTRYSIZE 7 +#define PCRE_INFO_NAMECOUNT 8 +#define PCRE_INFO_NAMETABLE 9 +#define PCRE_INFO_STUDYSIZE 10 +#define PCRE_INFO_DEFAULT_TABLES 11 +#define PCRE_INFO_OKPARTIAL 12 +#define PCRE_INFO_JCHANGED 13 +#define PCRE_INFO_HASCRORLF 14 +#define PCRE_INFO_MINLENGTH 15 +#define PCRE_INFO_JIT 16 +#define PCRE_INFO_JITSIZE 17 +#define PCRE_INFO_MAXLOOKBEHIND 18 +#define PCRE_INFO_FIRSTCHARACTER 19 +#define PCRE_INFO_FIRSTCHARACTERFLAGS 20 +#define PCRE_INFO_REQUIREDCHAR 21 +#define PCRE_INFO_REQUIREDCHARFLAGS 22 +#define PCRE_INFO_MATCHLIMIT 23 +#define PCRE_INFO_RECURSIONLIMIT 24 +#define PCRE_INFO_MATCH_EMPTY 25 + +/* Request types for pcre_config(). Do not re-arrange, in order to remain +compatible. */ + +#define PCRE_CONFIG_UTF8 0 +#define PCRE_CONFIG_NEWLINE 1 +#define PCRE_CONFIG_LINK_SIZE 2 +#define PCRE_CONFIG_POSIX_MALLOC_THRESHOLD 3 +#define PCRE_CONFIG_MATCH_LIMIT 4 +#define PCRE_CONFIG_STACKRECURSE 5 +#define PCRE_CONFIG_UNICODE_PROPERTIES 6 +#define PCRE_CONFIG_MATCH_LIMIT_RECURSION 7 +#define PCRE_CONFIG_BSR 8 +#define PCRE_CONFIG_JIT 9 +#define PCRE_CONFIG_UTF16 10 +#define PCRE_CONFIG_JITTARGET 11 +#define PCRE_CONFIG_UTF32 12 +#define PCRE_CONFIG_PARENS_LIMIT 13 + +/* Request types for pcre_study(). Do not re-arrange, in order to remain +compatible. */ + +#define PCRE_STUDY_JIT_COMPILE 0x0001 +#define PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE 0x0002 +#define PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE 0x0004 +#define PCRE_STUDY_EXTRA_NEEDED 0x0008 + +/* Bit flags for the pcre[16|32]_extra structure. Do not re-arrange or redefine +these bits, just add new ones on the end, in order to remain compatible. */ + +#define PCRE_EXTRA_STUDY_DATA 0x0001 +#define PCRE_EXTRA_MATCH_LIMIT 0x0002 +#define PCRE_EXTRA_CALLOUT_DATA 0x0004 +#define PCRE_EXTRA_TABLES 0x0008 +#define PCRE_EXTRA_MATCH_LIMIT_RECURSION 0x0010 +#define PCRE_EXTRA_MARK 0x0020 +#define PCRE_EXTRA_EXECUTABLE_JIT 0x0040 + +/* Types */ + +struct real_pcre; /* declaration; the definition is private */ +typedef struct real_pcre pcre; + +struct real_pcre16; /* declaration; the definition is private */ +typedef struct real_pcre16 pcre16; + +struct real_pcre32; /* declaration; the definition is private */ +typedef struct real_pcre32 pcre32; + +struct real_pcre_jit_stack; /* declaration; the definition is private */ +typedef struct real_pcre_jit_stack pcre_jit_stack; + +struct real_pcre16_jit_stack; /* declaration; the definition is private */ +typedef struct real_pcre16_jit_stack pcre16_jit_stack; + +struct real_pcre32_jit_stack; /* declaration; the definition is private */ +typedef struct real_pcre32_jit_stack pcre32_jit_stack; + +/* If PCRE is compiled with 16 bit character support, PCRE_UCHAR16 must contain +a 16 bit wide signed data type. Otherwise it can be a dummy data type since +pcre16 functions are not implemented. There is a check for this in pcre_internal.h. */ +#ifndef PCRE_UCHAR16 +#define PCRE_UCHAR16 unsigned short +#endif + +#ifndef PCRE_SPTR16 +#define PCRE_SPTR16 const PCRE_UCHAR16 * +#endif + +/* If PCRE is compiled with 32 bit character support, PCRE_UCHAR32 must contain +a 32 bit wide signed data type. Otherwise it can be a dummy data type since +pcre32 functions are not implemented. There is a check for this in pcre_internal.h. */ +#ifndef PCRE_UCHAR32 +#define PCRE_UCHAR32 unsigned int +#endif + +#ifndef PCRE_SPTR32 +#define PCRE_SPTR32 const PCRE_UCHAR32 * +#endif + +/* When PCRE is compiled as a C++ library, the subject pointer type can be +replaced with a custom type. For conventional use, the public interface is a +const char *. */ + +#ifndef PCRE_SPTR +#define PCRE_SPTR const char * +#endif + +/* The structure for passing additional data to pcre_exec(). This is defined in +such as way as to be extensible. Always add new fields at the end, in order to +remain compatible. */ + +typedef struct pcre_extra { + unsigned long int flags; /* Bits for which fields are set */ + void *study_data; /* Opaque data from pcre_study() */ + unsigned long int match_limit; /* Maximum number of calls to match() */ + void *callout_data; /* Data passed back in callouts */ + const unsigned char *tables; /* Pointer to character tables */ + unsigned long int match_limit_recursion; /* Max recursive calls to match() */ + unsigned char **mark; /* For passing back a mark pointer */ + void *executable_jit; /* Contains a pointer to a compiled jit code */ +} pcre_extra; + +/* Same structure as above, but with 16 bit char pointers. */ + +typedef struct pcre16_extra { + unsigned long int flags; /* Bits for which fields are set */ + void *study_data; /* Opaque data from pcre_study() */ + unsigned long int match_limit; /* Maximum number of calls to match() */ + void *callout_data; /* Data passed back in callouts */ + const unsigned char *tables; /* Pointer to character tables */ + unsigned long int match_limit_recursion; /* Max recursive calls to match() */ + PCRE_UCHAR16 **mark; /* For passing back a mark pointer */ + void *executable_jit; /* Contains a pointer to a compiled jit code */ +} pcre16_extra; + +/* Same structure as above, but with 32 bit char pointers. */ + +typedef struct pcre32_extra { + unsigned long int flags; /* Bits for which fields are set */ + void *study_data; /* Opaque data from pcre_study() */ + unsigned long int match_limit; /* Maximum number of calls to match() */ + void *callout_data; /* Data passed back in callouts */ + const unsigned char *tables; /* Pointer to character tables */ + unsigned long int match_limit_recursion; /* Max recursive calls to match() */ + PCRE_UCHAR32 **mark; /* For passing back a mark pointer */ + void *executable_jit; /* Contains a pointer to a compiled jit code */ +} pcre32_extra; + +/* The structure for passing out data via the pcre_callout_function. We use a +structure so that new fields can be added on the end in future versions, +without changing the API of the function, thereby allowing old clients to work +without modification. */ + +typedef struct pcre_callout_block { + int version; /* Identifies version of block */ + /* ------------------------ Version 0 ------------------------------- */ + int callout_number; /* Number compiled into pattern */ + int *offset_vector; /* The offset vector */ + PCRE_SPTR subject; /* The subject being matched */ + int subject_length; /* The length of the subject */ + int start_match; /* Offset to start of this match attempt */ + int current_position; /* Where we currently are in the subject */ + int capture_top; /* Max current capture */ + int capture_last; /* Most recently closed capture */ + void *callout_data; /* Data passed in with the call */ + /* ------------------- Added for Version 1 -------------------------- */ + int pattern_position; /* Offset to next item in the pattern */ + int next_item_length; /* Length of next item in the pattern */ + /* ------------------- Added for Version 2 -------------------------- */ + const unsigned char *mark; /* Pointer to current mark or NULL */ + /* ------------------------------------------------------------------ */ +} pcre_callout_block; + +/* Same structure as above, but with 16 bit char pointers. */ + +typedef struct pcre16_callout_block { + int version; /* Identifies version of block */ + /* ------------------------ Version 0 ------------------------------- */ + int callout_number; /* Number compiled into pattern */ + int *offset_vector; /* The offset vector */ + PCRE_SPTR16 subject; /* The subject being matched */ + int subject_length; /* The length of the subject */ + int start_match; /* Offset to start of this match attempt */ + int current_position; /* Where we currently are in the subject */ + int capture_top; /* Max current capture */ + int capture_last; /* Most recently closed capture */ + void *callout_data; /* Data passed in with the call */ + /* ------------------- Added for Version 1 -------------------------- */ + int pattern_position; /* Offset to next item in the pattern */ + int next_item_length; /* Length of next item in the pattern */ + /* ------------------- Added for Version 2 -------------------------- */ + const PCRE_UCHAR16 *mark; /* Pointer to current mark or NULL */ + /* ------------------------------------------------------------------ */ +} pcre16_callout_block; + +/* Same structure as above, but with 32 bit char pointers. */ + +typedef struct pcre32_callout_block { + int version; /* Identifies version of block */ + /* ------------------------ Version 0 ------------------------------- */ + int callout_number; /* Number compiled into pattern */ + int *offset_vector; /* The offset vector */ + PCRE_SPTR32 subject; /* The subject being matched */ + int subject_length; /* The length of the subject */ + int start_match; /* Offset to start of this match attempt */ + int current_position; /* Where we currently are in the subject */ + int capture_top; /* Max current capture */ + int capture_last; /* Most recently closed capture */ + void *callout_data; /* Data passed in with the call */ + /* ------------------- Added for Version 1 -------------------------- */ + int pattern_position; /* Offset to next item in the pattern */ + int next_item_length; /* Length of next item in the pattern */ + /* ------------------- Added for Version 2 -------------------------- */ + const PCRE_UCHAR32 *mark; /* Pointer to current mark or NULL */ + /* ------------------------------------------------------------------ */ +} pcre32_callout_block; + +/* Indirection for store get and free functions. These can be set to +alternative malloc/free functions if required. Special ones are used in the +non-recursive case for "frames". There is also an optional callout function +that is triggered by the (?) regex item. For Virtual Pascal, these definitions +have to take another form. */ + +#ifndef VPCOMPAT +PCRE_EXP_DECL void *(*pcre_malloc)(size_t); +PCRE_EXP_DECL void (*pcre_free)(void *); +PCRE_EXP_DECL void *(*pcre_stack_malloc)(size_t); +PCRE_EXP_DECL void (*pcre_stack_free)(void *); +PCRE_EXP_DECL int (*pcre_callout)(pcre_callout_block *); +PCRE_EXP_DECL int (*pcre_stack_guard)(void); + +PCRE_EXP_DECL void *(*pcre16_malloc)(size_t); +PCRE_EXP_DECL void (*pcre16_free)(void *); +PCRE_EXP_DECL void *(*pcre16_stack_malloc)(size_t); +PCRE_EXP_DECL void (*pcre16_stack_free)(void *); +PCRE_EXP_DECL int (*pcre16_callout)(pcre16_callout_block *); +PCRE_EXP_DECL int (*pcre16_stack_guard)(void); + +PCRE_EXP_DECL void *(*pcre32_malloc)(size_t); +PCRE_EXP_DECL void (*pcre32_free)(void *); +PCRE_EXP_DECL void *(*pcre32_stack_malloc)(size_t); +PCRE_EXP_DECL void (*pcre32_stack_free)(void *); +PCRE_EXP_DECL int (*pcre32_callout)(pcre32_callout_block *); +PCRE_EXP_DECL int (*pcre32_stack_guard)(void); +#else /* VPCOMPAT */ +PCRE_EXP_DECL void *pcre_malloc(size_t); +PCRE_EXP_DECL void pcre_free(void *); +PCRE_EXP_DECL void *pcre_stack_malloc(size_t); +PCRE_EXP_DECL void pcre_stack_free(void *); +PCRE_EXP_DECL int pcre_callout(pcre_callout_block *); +PCRE_EXP_DECL int pcre_stack_guard(void); + +PCRE_EXP_DECL void *pcre16_malloc(size_t); +PCRE_EXP_DECL void pcre16_free(void *); +PCRE_EXP_DECL void *pcre16_stack_malloc(size_t); +PCRE_EXP_DECL void pcre16_stack_free(void *); +PCRE_EXP_DECL int pcre16_callout(pcre16_callout_block *); +PCRE_EXP_DECL int pcre16_stack_guard(void); + +PCRE_EXP_DECL void *pcre32_malloc(size_t); +PCRE_EXP_DECL void pcre32_free(void *); +PCRE_EXP_DECL void *pcre32_stack_malloc(size_t); +PCRE_EXP_DECL void pcre32_stack_free(void *); +PCRE_EXP_DECL int pcre32_callout(pcre32_callout_block *); +PCRE_EXP_DECL int pcre32_stack_guard(void); +#endif /* VPCOMPAT */ + +/* User defined callback which provides a stack just before the match starts. */ + +typedef pcre_jit_stack *(*pcre_jit_callback)(void *); +typedef pcre16_jit_stack *(*pcre16_jit_callback)(void *); +typedef pcre32_jit_stack *(*pcre32_jit_callback)(void *); + +/* Exported PCRE functions */ + +PCRE_EXP_DECL pcre *pcre_compile(const char *, int, const char **, int *, + const unsigned char *); +PCRE_EXP_DECL pcre16 *pcre16_compile(PCRE_SPTR16, int, const char **, int *, + const unsigned char *); +PCRE_EXP_DECL pcre32 *pcre32_compile(PCRE_SPTR32, int, const char **, int *, + const unsigned char *); +PCRE_EXP_DECL pcre *pcre_compile2(const char *, int, int *, const char **, + int *, const unsigned char *); +PCRE_EXP_DECL pcre16 *pcre16_compile2(PCRE_SPTR16, int, int *, const char **, + int *, const unsigned char *); +PCRE_EXP_DECL pcre32 *pcre32_compile2(PCRE_SPTR32, int, int *, const char **, + int *, const unsigned char *); +PCRE_EXP_DECL int pcre_config(int, void *); +PCRE_EXP_DECL int pcre16_config(int, void *); +PCRE_EXP_DECL int pcre32_config(int, void *); +PCRE_EXP_DECL int pcre_copy_named_substring(const pcre *, const char *, + int *, int, const char *, char *, int); +PCRE_EXP_DECL int pcre16_copy_named_substring(const pcre16 *, PCRE_SPTR16, + int *, int, PCRE_SPTR16, PCRE_UCHAR16 *, int); +PCRE_EXP_DECL int pcre32_copy_named_substring(const pcre32 *, PCRE_SPTR32, + int *, int, PCRE_SPTR32, PCRE_UCHAR32 *, int); +PCRE_EXP_DECL int pcre_copy_substring(const char *, int *, int, int, + char *, int); +PCRE_EXP_DECL int pcre16_copy_substring(PCRE_SPTR16, int *, int, int, + PCRE_UCHAR16 *, int); +PCRE_EXP_DECL int pcre32_copy_substring(PCRE_SPTR32, int *, int, int, + PCRE_UCHAR32 *, int); +PCRE_EXP_DECL int pcre_dfa_exec(const pcre *, const pcre_extra *, + const char *, int, int, int, int *, int , int *, int); +PCRE_EXP_DECL int pcre16_dfa_exec(const pcre16 *, const pcre16_extra *, + PCRE_SPTR16, int, int, int, int *, int , int *, int); +PCRE_EXP_DECL int pcre32_dfa_exec(const pcre32 *, const pcre32_extra *, + PCRE_SPTR32, int, int, int, int *, int , int *, int); +PCRE_EXP_DECL int pcre_exec(const pcre *, const pcre_extra *, PCRE_SPTR, + int, int, int, int *, int); +PCRE_EXP_DECL int pcre16_exec(const pcre16 *, const pcre16_extra *, + PCRE_SPTR16, int, int, int, int *, int); +PCRE_EXP_DECL int pcre32_exec(const pcre32 *, const pcre32_extra *, + PCRE_SPTR32, int, int, int, int *, int); +PCRE_EXP_DECL int pcre_jit_exec(const pcre *, const pcre_extra *, + PCRE_SPTR, int, int, int, int *, int, + pcre_jit_stack *); +PCRE_EXP_DECL int pcre16_jit_exec(const pcre16 *, const pcre16_extra *, + PCRE_SPTR16, int, int, int, int *, int, + pcre16_jit_stack *); +PCRE_EXP_DECL int pcre32_jit_exec(const pcre32 *, const pcre32_extra *, + PCRE_SPTR32, int, int, int, int *, int, + pcre32_jit_stack *); +PCRE_EXP_DECL void pcre_free_substring(const char *); +PCRE_EXP_DECL void pcre16_free_substring(PCRE_SPTR16); +PCRE_EXP_DECL void pcre32_free_substring(PCRE_SPTR32); +PCRE_EXP_DECL void pcre_free_substring_list(const char **); +PCRE_EXP_DECL void pcre16_free_substring_list(PCRE_SPTR16 *); +PCRE_EXP_DECL void pcre32_free_substring_list(PCRE_SPTR32 *); +PCRE_EXP_DECL int pcre_fullinfo(const pcre *, const pcre_extra *, int, + void *); +PCRE_EXP_DECL int pcre16_fullinfo(const pcre16 *, const pcre16_extra *, int, + void *); +PCRE_EXP_DECL int pcre32_fullinfo(const pcre32 *, const pcre32_extra *, int, + void *); +PCRE_EXP_DECL int pcre_get_named_substring(const pcre *, const char *, + int *, int, const char *, const char **); +PCRE_EXP_DECL int pcre16_get_named_substring(const pcre16 *, PCRE_SPTR16, + int *, int, PCRE_SPTR16, PCRE_SPTR16 *); +PCRE_EXP_DECL int pcre32_get_named_substring(const pcre32 *, PCRE_SPTR32, + int *, int, PCRE_SPTR32, PCRE_SPTR32 *); +PCRE_EXP_DECL int pcre_get_stringnumber(const pcre *, const char *); +PCRE_EXP_DECL int pcre16_get_stringnumber(const pcre16 *, PCRE_SPTR16); +PCRE_EXP_DECL int pcre32_get_stringnumber(const pcre32 *, PCRE_SPTR32); +PCRE_EXP_DECL int pcre_get_stringtable_entries(const pcre *, const char *, + char **, char **); +PCRE_EXP_DECL int pcre16_get_stringtable_entries(const pcre16 *, PCRE_SPTR16, + PCRE_UCHAR16 **, PCRE_UCHAR16 **); +PCRE_EXP_DECL int pcre32_get_stringtable_entries(const pcre32 *, PCRE_SPTR32, + PCRE_UCHAR32 **, PCRE_UCHAR32 **); +PCRE_EXP_DECL int pcre_get_substring(const char *, int *, int, int, + const char **); +PCRE_EXP_DECL int pcre16_get_substring(PCRE_SPTR16, int *, int, int, + PCRE_SPTR16 *); +PCRE_EXP_DECL int pcre32_get_substring(PCRE_SPTR32, int *, int, int, + PCRE_SPTR32 *); +PCRE_EXP_DECL int pcre_get_substring_list(const char *, int *, int, + const char ***); +PCRE_EXP_DECL int pcre16_get_substring_list(PCRE_SPTR16, int *, int, + PCRE_SPTR16 **); +PCRE_EXP_DECL int pcre32_get_substring_list(PCRE_SPTR32, int *, int, + PCRE_SPTR32 **); +PCRE_EXP_DECL const unsigned char *pcre_maketables(void); +PCRE_EXP_DECL const unsigned char *pcre16_maketables(void); +PCRE_EXP_DECL const unsigned char *pcre32_maketables(void); +PCRE_EXP_DECL int pcre_refcount(pcre *, int); +PCRE_EXP_DECL int pcre16_refcount(pcre16 *, int); +PCRE_EXP_DECL int pcre32_refcount(pcre32 *, int); +PCRE_EXP_DECL pcre_extra *pcre_study(const pcre *, int, const char **); +PCRE_EXP_DECL pcre16_extra *pcre16_study(const pcre16 *, int, const char **); +PCRE_EXP_DECL pcre32_extra *pcre32_study(const pcre32 *, int, const char **); +PCRE_EXP_DECL void pcre_free_study(pcre_extra *); +PCRE_EXP_DECL void pcre16_free_study(pcre16_extra *); +PCRE_EXP_DECL void pcre32_free_study(pcre32_extra *); +PCRE_EXP_DECL const char *pcre_version(void); +PCRE_EXP_DECL const char *pcre16_version(void); +PCRE_EXP_DECL const char *pcre32_version(void); + +/* Utility functions for byte order swaps. */ +PCRE_EXP_DECL int pcre_pattern_to_host_byte_order(pcre *, pcre_extra *, + const unsigned char *); +PCRE_EXP_DECL int pcre16_pattern_to_host_byte_order(pcre16 *, pcre16_extra *, + const unsigned char *); +PCRE_EXP_DECL int pcre32_pattern_to_host_byte_order(pcre32 *, pcre32_extra *, + const unsigned char *); +PCRE_EXP_DECL int pcre16_utf16_to_host_byte_order(PCRE_UCHAR16 *, + PCRE_SPTR16, int, int *, int); +PCRE_EXP_DECL int pcre32_utf32_to_host_byte_order(PCRE_UCHAR32 *, + PCRE_SPTR32, int, int *, int); + +/* JIT compiler related functions. */ + +PCRE_EXP_DECL pcre_jit_stack *pcre_jit_stack_alloc(int, int); +PCRE_EXP_DECL pcre16_jit_stack *pcre16_jit_stack_alloc(int, int); +PCRE_EXP_DECL pcre32_jit_stack *pcre32_jit_stack_alloc(int, int); +PCRE_EXP_DECL void pcre_jit_stack_free(pcre_jit_stack *); +PCRE_EXP_DECL void pcre16_jit_stack_free(pcre16_jit_stack *); +PCRE_EXP_DECL void pcre32_jit_stack_free(pcre32_jit_stack *); +PCRE_EXP_DECL void pcre_assign_jit_stack(pcre_extra *, + pcre_jit_callback, void *); +PCRE_EXP_DECL void pcre16_assign_jit_stack(pcre16_extra *, + pcre16_jit_callback, void *); +PCRE_EXP_DECL void pcre32_assign_jit_stack(pcre32_extra *, + pcre32_jit_callback, void *); +PCRE_EXP_DECL void pcre_jit_free_unused_memory(void); +PCRE_EXP_DECL void pcre16_jit_free_unused_memory(void); +PCRE_EXP_DECL void pcre32_jit_free_unused_memory(void); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* End of pcre.h */ diff --git a/src/third_party/pcre-8.41/pcre.h.in b/src/third_party/pcre-8.41/pcre.h.in new file mode 100644 index 00000000000..667a45ed575 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre.h.in @@ -0,0 +1,677 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* This is the public header file for the PCRE library, to be #included by +applications that call the PCRE functions. + + Copyright (c) 1997-2014 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +#ifndef _PCRE_H +#define _PCRE_H + +/* The current PCRE version information. */ + +#define PCRE_MAJOR @PCRE_MAJOR@ +#define PCRE_MINOR @PCRE_MINOR@ +#define PCRE_PRERELEASE @PCRE_PRERELEASE@ +#define PCRE_DATE @PCRE_DATE@ + +/* When an application links to a PCRE DLL in Windows, the symbols that are +imported have to be identified as such. When building PCRE, the appropriate +export setting is defined in pcre_internal.h, which includes this file. So we +don't change existing definitions of PCRE_EXP_DECL and PCRECPP_EXP_DECL. */ + +#if defined(_WIN32) && !defined(PCRE_STATIC) +# ifndef PCRE_EXP_DECL +# define PCRE_EXP_DECL extern __declspec(dllimport) +# endif +# ifdef __cplusplus +# ifndef PCRECPP_EXP_DECL +# define PCRECPP_EXP_DECL extern __declspec(dllimport) +# endif +# ifndef PCRECPP_EXP_DEFN +# define PCRECPP_EXP_DEFN __declspec(dllimport) +# endif +# endif +#endif + +/* By default, we use the standard "extern" declarations. */ + +#ifndef PCRE_EXP_DECL +# ifdef __cplusplus +# define PCRE_EXP_DECL extern "C" +# else +# define PCRE_EXP_DECL extern +# endif +#endif + +#ifdef __cplusplus +# ifndef PCRECPP_EXP_DECL +# define PCRECPP_EXP_DECL extern +# endif +# ifndef PCRECPP_EXP_DEFN +# define PCRECPP_EXP_DEFN +# endif +#endif + +/* Have to include stdlib.h in order to ensure that size_t is defined; +it is needed here for malloc. */ + +#include + +/* Allow for C++ users */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Public options. Some are compile-time only, some are run-time only, and some +are both. Most of the compile-time options are saved with the compiled regex so +that they can be inspected during studying (and therefore JIT compiling). Note +that pcre_study() has its own set of options. Originally, all the options +defined here used distinct bits. However, almost all the bits in a 32-bit word +are now used, so in order to conserve them, option bits that were previously +only recognized at matching time (i.e. by pcre_exec() or pcre_dfa_exec()) may +also be used for compile-time options that affect only compiling and are not +relevant for studying or JIT compiling. + +Some options for pcre_compile() change its behaviour but do not affect the +behaviour of the execution functions. Other options are passed through to the +execution functions and affect their behaviour, with or without affecting the +behaviour of pcre_compile(). + +Options that can be passed to pcre_compile() are tagged Cx below, with these +variants: + +C1 Affects compile only +C2 Does not affect compile; affects exec, dfa_exec +C3 Affects compile, exec, dfa_exec +C4 Affects compile, exec, dfa_exec, study +C5 Affects compile, exec, study + +Options that can be set for pcre_exec() and/or pcre_dfa_exec() are flagged with +E and D, respectively. They take precedence over C3, C4, and C5 settings passed +from pcre_compile(). Those that are compatible with JIT execution are flagged +with J. */ + +#define PCRE_CASELESS 0x00000001 /* C1 */ +#define PCRE_MULTILINE 0x00000002 /* C1 */ +#define PCRE_DOTALL 0x00000004 /* C1 */ +#define PCRE_EXTENDED 0x00000008 /* C1 */ +#define PCRE_ANCHORED 0x00000010 /* C4 E D */ +#define PCRE_DOLLAR_ENDONLY 0x00000020 /* C2 */ +#define PCRE_EXTRA 0x00000040 /* C1 */ +#define PCRE_NOTBOL 0x00000080 /* E D J */ +#define PCRE_NOTEOL 0x00000100 /* E D J */ +#define PCRE_UNGREEDY 0x00000200 /* C1 */ +#define PCRE_NOTEMPTY 0x00000400 /* E D J */ +#define PCRE_UTF8 0x00000800 /* C4 ) */ +#define PCRE_UTF16 0x00000800 /* C4 ) Synonyms */ +#define PCRE_UTF32 0x00000800 /* C4 ) */ +#define PCRE_NO_AUTO_CAPTURE 0x00001000 /* C1 */ +#define PCRE_NO_UTF8_CHECK 0x00002000 /* C1 E D J ) */ +#define PCRE_NO_UTF16_CHECK 0x00002000 /* C1 E D J ) Synonyms */ +#define PCRE_NO_UTF32_CHECK 0x00002000 /* C1 E D J ) */ +#define PCRE_AUTO_CALLOUT 0x00004000 /* C1 */ +#define PCRE_PARTIAL_SOFT 0x00008000 /* E D J ) Synonyms */ +#define PCRE_PARTIAL 0x00008000 /* E D J ) */ + +/* This pair use the same bit. */ +#define PCRE_NEVER_UTF 0x00010000 /* C1 ) Overlaid */ +#define PCRE_DFA_SHORTEST 0x00010000 /* D ) Overlaid */ + +/* This pair use the same bit. */ +#define PCRE_NO_AUTO_POSSESS 0x00020000 /* C1 ) Overlaid */ +#define PCRE_DFA_RESTART 0x00020000 /* D ) Overlaid */ + +#define PCRE_FIRSTLINE 0x00040000 /* C3 */ +#define PCRE_DUPNAMES 0x00080000 /* C1 */ +#define PCRE_NEWLINE_CR 0x00100000 /* C3 E D */ +#define PCRE_NEWLINE_LF 0x00200000 /* C3 E D */ +#define PCRE_NEWLINE_CRLF 0x00300000 /* C3 E D */ +#define PCRE_NEWLINE_ANY 0x00400000 /* C3 E D */ +#define PCRE_NEWLINE_ANYCRLF 0x00500000 /* C3 E D */ +#define PCRE_BSR_ANYCRLF 0x00800000 /* C3 E D */ +#define PCRE_BSR_UNICODE 0x01000000 /* C3 E D */ +#define PCRE_JAVASCRIPT_COMPAT 0x02000000 /* C5 */ +#define PCRE_NO_START_OPTIMIZE 0x04000000 /* C2 E D ) Synonyms */ +#define PCRE_NO_START_OPTIMISE 0x04000000 /* C2 E D ) */ +#define PCRE_PARTIAL_HARD 0x08000000 /* E D J */ +#define PCRE_NOTEMPTY_ATSTART 0x10000000 /* E D J */ +#define PCRE_UCP 0x20000000 /* C3 */ + +/* Exec-time and get/set-time error codes */ + +#define PCRE_ERROR_NOMATCH (-1) +#define PCRE_ERROR_NULL (-2) +#define PCRE_ERROR_BADOPTION (-3) +#define PCRE_ERROR_BADMAGIC (-4) +#define PCRE_ERROR_UNKNOWN_OPCODE (-5) +#define PCRE_ERROR_UNKNOWN_NODE (-5) /* For backward compatibility */ +#define PCRE_ERROR_NOMEMORY (-6) +#define PCRE_ERROR_NOSUBSTRING (-7) +#define PCRE_ERROR_MATCHLIMIT (-8) +#define PCRE_ERROR_CALLOUT (-9) /* Never used by PCRE itself */ +#define PCRE_ERROR_BADUTF8 (-10) /* Same for 8/16/32 */ +#define PCRE_ERROR_BADUTF16 (-10) /* Same for 8/16/32 */ +#define PCRE_ERROR_BADUTF32 (-10) /* Same for 8/16/32 */ +#define PCRE_ERROR_BADUTF8_OFFSET (-11) /* Same for 8/16 */ +#define PCRE_ERROR_BADUTF16_OFFSET (-11) /* Same for 8/16 */ +#define PCRE_ERROR_PARTIAL (-12) +#define PCRE_ERROR_BADPARTIAL (-13) +#define PCRE_ERROR_INTERNAL (-14) +#define PCRE_ERROR_BADCOUNT (-15) +#define PCRE_ERROR_DFA_UITEM (-16) +#define PCRE_ERROR_DFA_UCOND (-17) +#define PCRE_ERROR_DFA_UMLIMIT (-18) +#define PCRE_ERROR_DFA_WSSIZE (-19) +#define PCRE_ERROR_DFA_RECURSE (-20) +#define PCRE_ERROR_RECURSIONLIMIT (-21) +#define PCRE_ERROR_NULLWSLIMIT (-22) /* No longer actually used */ +#define PCRE_ERROR_BADNEWLINE (-23) +#define PCRE_ERROR_BADOFFSET (-24) +#define PCRE_ERROR_SHORTUTF8 (-25) +#define PCRE_ERROR_SHORTUTF16 (-25) /* Same for 8/16 */ +#define PCRE_ERROR_RECURSELOOP (-26) +#define PCRE_ERROR_JIT_STACKLIMIT (-27) +#define PCRE_ERROR_BADMODE (-28) +#define PCRE_ERROR_BADENDIANNESS (-29) +#define PCRE_ERROR_DFA_BADRESTART (-30) +#define PCRE_ERROR_JIT_BADOPTION (-31) +#define PCRE_ERROR_BADLENGTH (-32) +#define PCRE_ERROR_UNSET (-33) + +/* Specific error codes for UTF-8 validity checks */ + +#define PCRE_UTF8_ERR0 0 +#define PCRE_UTF8_ERR1 1 +#define PCRE_UTF8_ERR2 2 +#define PCRE_UTF8_ERR3 3 +#define PCRE_UTF8_ERR4 4 +#define PCRE_UTF8_ERR5 5 +#define PCRE_UTF8_ERR6 6 +#define PCRE_UTF8_ERR7 7 +#define PCRE_UTF8_ERR8 8 +#define PCRE_UTF8_ERR9 9 +#define PCRE_UTF8_ERR10 10 +#define PCRE_UTF8_ERR11 11 +#define PCRE_UTF8_ERR12 12 +#define PCRE_UTF8_ERR13 13 +#define PCRE_UTF8_ERR14 14 +#define PCRE_UTF8_ERR15 15 +#define PCRE_UTF8_ERR16 16 +#define PCRE_UTF8_ERR17 17 +#define PCRE_UTF8_ERR18 18 +#define PCRE_UTF8_ERR19 19 +#define PCRE_UTF8_ERR20 20 +#define PCRE_UTF8_ERR21 21 +#define PCRE_UTF8_ERR22 22 /* Unused (was non-character) */ + +/* Specific error codes for UTF-16 validity checks */ + +#define PCRE_UTF16_ERR0 0 +#define PCRE_UTF16_ERR1 1 +#define PCRE_UTF16_ERR2 2 +#define PCRE_UTF16_ERR3 3 +#define PCRE_UTF16_ERR4 4 /* Unused (was non-character) */ + +/* Specific error codes for UTF-32 validity checks */ + +#define PCRE_UTF32_ERR0 0 +#define PCRE_UTF32_ERR1 1 +#define PCRE_UTF32_ERR2 2 /* Unused (was non-character) */ +#define PCRE_UTF32_ERR3 3 + +/* Request types for pcre_fullinfo() */ + +#define PCRE_INFO_OPTIONS 0 +#define PCRE_INFO_SIZE 1 +#define PCRE_INFO_CAPTURECOUNT 2 +#define PCRE_INFO_BACKREFMAX 3 +#define PCRE_INFO_FIRSTBYTE 4 +#define PCRE_INFO_FIRSTCHAR 4 /* For backwards compatibility */ +#define PCRE_INFO_FIRSTTABLE 5 +#define PCRE_INFO_LASTLITERAL 6 +#define PCRE_INFO_NAMEENTRYSIZE 7 +#define PCRE_INFO_NAMECOUNT 8 +#define PCRE_INFO_NAMETABLE 9 +#define PCRE_INFO_STUDYSIZE 10 +#define PCRE_INFO_DEFAULT_TABLES 11 +#define PCRE_INFO_OKPARTIAL 12 +#define PCRE_INFO_JCHANGED 13 +#define PCRE_INFO_HASCRORLF 14 +#define PCRE_INFO_MINLENGTH 15 +#define PCRE_INFO_JIT 16 +#define PCRE_INFO_JITSIZE 17 +#define PCRE_INFO_MAXLOOKBEHIND 18 +#define PCRE_INFO_FIRSTCHARACTER 19 +#define PCRE_INFO_FIRSTCHARACTERFLAGS 20 +#define PCRE_INFO_REQUIREDCHAR 21 +#define PCRE_INFO_REQUIREDCHARFLAGS 22 +#define PCRE_INFO_MATCHLIMIT 23 +#define PCRE_INFO_RECURSIONLIMIT 24 +#define PCRE_INFO_MATCH_EMPTY 25 + +/* Request types for pcre_config(). Do not re-arrange, in order to remain +compatible. */ + +#define PCRE_CONFIG_UTF8 0 +#define PCRE_CONFIG_NEWLINE 1 +#define PCRE_CONFIG_LINK_SIZE 2 +#define PCRE_CONFIG_POSIX_MALLOC_THRESHOLD 3 +#define PCRE_CONFIG_MATCH_LIMIT 4 +#define PCRE_CONFIG_STACKRECURSE 5 +#define PCRE_CONFIG_UNICODE_PROPERTIES 6 +#define PCRE_CONFIG_MATCH_LIMIT_RECURSION 7 +#define PCRE_CONFIG_BSR 8 +#define PCRE_CONFIG_JIT 9 +#define PCRE_CONFIG_UTF16 10 +#define PCRE_CONFIG_JITTARGET 11 +#define PCRE_CONFIG_UTF32 12 +#define PCRE_CONFIG_PARENS_LIMIT 13 + +/* Request types for pcre_study(). Do not re-arrange, in order to remain +compatible. */ + +#define PCRE_STUDY_JIT_COMPILE 0x0001 +#define PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE 0x0002 +#define PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE 0x0004 +#define PCRE_STUDY_EXTRA_NEEDED 0x0008 + +/* Bit flags for the pcre[16|32]_extra structure. Do not re-arrange or redefine +these bits, just add new ones on the end, in order to remain compatible. */ + +#define PCRE_EXTRA_STUDY_DATA 0x0001 +#define PCRE_EXTRA_MATCH_LIMIT 0x0002 +#define PCRE_EXTRA_CALLOUT_DATA 0x0004 +#define PCRE_EXTRA_TABLES 0x0008 +#define PCRE_EXTRA_MATCH_LIMIT_RECURSION 0x0010 +#define PCRE_EXTRA_MARK 0x0020 +#define PCRE_EXTRA_EXECUTABLE_JIT 0x0040 + +/* Types */ + +struct real_pcre; /* declaration; the definition is private */ +typedef struct real_pcre pcre; + +struct real_pcre16; /* declaration; the definition is private */ +typedef struct real_pcre16 pcre16; + +struct real_pcre32; /* declaration; the definition is private */ +typedef struct real_pcre32 pcre32; + +struct real_pcre_jit_stack; /* declaration; the definition is private */ +typedef struct real_pcre_jit_stack pcre_jit_stack; + +struct real_pcre16_jit_stack; /* declaration; the definition is private */ +typedef struct real_pcre16_jit_stack pcre16_jit_stack; + +struct real_pcre32_jit_stack; /* declaration; the definition is private */ +typedef struct real_pcre32_jit_stack pcre32_jit_stack; + +/* If PCRE is compiled with 16 bit character support, PCRE_UCHAR16 must contain +a 16 bit wide signed data type. Otherwise it can be a dummy data type since +pcre16 functions are not implemented. There is a check for this in pcre_internal.h. */ +#ifndef PCRE_UCHAR16 +#define PCRE_UCHAR16 unsigned short +#endif + +#ifndef PCRE_SPTR16 +#define PCRE_SPTR16 const PCRE_UCHAR16 * +#endif + +/* If PCRE is compiled with 32 bit character support, PCRE_UCHAR32 must contain +a 32 bit wide signed data type. Otherwise it can be a dummy data type since +pcre32 functions are not implemented. There is a check for this in pcre_internal.h. */ +#ifndef PCRE_UCHAR32 +#define PCRE_UCHAR32 unsigned int +#endif + +#ifndef PCRE_SPTR32 +#define PCRE_SPTR32 const PCRE_UCHAR32 * +#endif + +/* When PCRE is compiled as a C++ library, the subject pointer type can be +replaced with a custom type. For conventional use, the public interface is a +const char *. */ + +#ifndef PCRE_SPTR +#define PCRE_SPTR const char * +#endif + +/* The structure for passing additional data to pcre_exec(). This is defined in +such as way as to be extensible. Always add new fields at the end, in order to +remain compatible. */ + +typedef struct pcre_extra { + unsigned long int flags; /* Bits for which fields are set */ + void *study_data; /* Opaque data from pcre_study() */ + unsigned long int match_limit; /* Maximum number of calls to match() */ + void *callout_data; /* Data passed back in callouts */ + const unsigned char *tables; /* Pointer to character tables */ + unsigned long int match_limit_recursion; /* Max recursive calls to match() */ + unsigned char **mark; /* For passing back a mark pointer */ + void *executable_jit; /* Contains a pointer to a compiled jit code */ +} pcre_extra; + +/* Same structure as above, but with 16 bit char pointers. */ + +typedef struct pcre16_extra { + unsigned long int flags; /* Bits for which fields are set */ + void *study_data; /* Opaque data from pcre_study() */ + unsigned long int match_limit; /* Maximum number of calls to match() */ + void *callout_data; /* Data passed back in callouts */ + const unsigned char *tables; /* Pointer to character tables */ + unsigned long int match_limit_recursion; /* Max recursive calls to match() */ + PCRE_UCHAR16 **mark; /* For passing back a mark pointer */ + void *executable_jit; /* Contains a pointer to a compiled jit code */ +} pcre16_extra; + +/* Same structure as above, but with 32 bit char pointers. */ + +typedef struct pcre32_extra { + unsigned long int flags; /* Bits for which fields are set */ + void *study_data; /* Opaque data from pcre_study() */ + unsigned long int match_limit; /* Maximum number of calls to match() */ + void *callout_data; /* Data passed back in callouts */ + const unsigned char *tables; /* Pointer to character tables */ + unsigned long int match_limit_recursion; /* Max recursive calls to match() */ + PCRE_UCHAR32 **mark; /* For passing back a mark pointer */ + void *executable_jit; /* Contains a pointer to a compiled jit code */ +} pcre32_extra; + +/* The structure for passing out data via the pcre_callout_function. We use a +structure so that new fields can be added on the end in future versions, +without changing the API of the function, thereby allowing old clients to work +without modification. */ + +typedef struct pcre_callout_block { + int version; /* Identifies version of block */ + /* ------------------------ Version 0 ------------------------------- */ + int callout_number; /* Number compiled into pattern */ + int *offset_vector; /* The offset vector */ + PCRE_SPTR subject; /* The subject being matched */ + int subject_length; /* The length of the subject */ + int start_match; /* Offset to start of this match attempt */ + int current_position; /* Where we currently are in the subject */ + int capture_top; /* Max current capture */ + int capture_last; /* Most recently closed capture */ + void *callout_data; /* Data passed in with the call */ + /* ------------------- Added for Version 1 -------------------------- */ + int pattern_position; /* Offset to next item in the pattern */ + int next_item_length; /* Length of next item in the pattern */ + /* ------------------- Added for Version 2 -------------------------- */ + const unsigned char *mark; /* Pointer to current mark or NULL */ + /* ------------------------------------------------------------------ */ +} pcre_callout_block; + +/* Same structure as above, but with 16 bit char pointers. */ + +typedef struct pcre16_callout_block { + int version; /* Identifies version of block */ + /* ------------------------ Version 0 ------------------------------- */ + int callout_number; /* Number compiled into pattern */ + int *offset_vector; /* The offset vector */ + PCRE_SPTR16 subject; /* The subject being matched */ + int subject_length; /* The length of the subject */ + int start_match; /* Offset to start of this match attempt */ + int current_position; /* Where we currently are in the subject */ + int capture_top; /* Max current capture */ + int capture_last; /* Most recently closed capture */ + void *callout_data; /* Data passed in with the call */ + /* ------------------- Added for Version 1 -------------------------- */ + int pattern_position; /* Offset to next item in the pattern */ + int next_item_length; /* Length of next item in the pattern */ + /* ------------------- Added for Version 2 -------------------------- */ + const PCRE_UCHAR16 *mark; /* Pointer to current mark or NULL */ + /* ------------------------------------------------------------------ */ +} pcre16_callout_block; + +/* Same structure as above, but with 32 bit char pointers. */ + +typedef struct pcre32_callout_block { + int version; /* Identifies version of block */ + /* ------------------------ Version 0 ------------------------------- */ + int callout_number; /* Number compiled into pattern */ + int *offset_vector; /* The offset vector */ + PCRE_SPTR32 subject; /* The subject being matched */ + int subject_length; /* The length of the subject */ + int start_match; /* Offset to start of this match attempt */ + int current_position; /* Where we currently are in the subject */ + int capture_top; /* Max current capture */ + int capture_last; /* Most recently closed capture */ + void *callout_data; /* Data passed in with the call */ + /* ------------------- Added for Version 1 -------------------------- */ + int pattern_position; /* Offset to next item in the pattern */ + int next_item_length; /* Length of next item in the pattern */ + /* ------------------- Added for Version 2 -------------------------- */ + const PCRE_UCHAR32 *mark; /* Pointer to current mark or NULL */ + /* ------------------------------------------------------------------ */ +} pcre32_callout_block; + +/* Indirection for store get and free functions. These can be set to +alternative malloc/free functions if required. Special ones are used in the +non-recursive case for "frames". There is also an optional callout function +that is triggered by the (?) regex item. For Virtual Pascal, these definitions +have to take another form. */ + +#ifndef VPCOMPAT +PCRE_EXP_DECL void *(*pcre_malloc)(size_t); +PCRE_EXP_DECL void (*pcre_free)(void *); +PCRE_EXP_DECL void *(*pcre_stack_malloc)(size_t); +PCRE_EXP_DECL void (*pcre_stack_free)(void *); +PCRE_EXP_DECL int (*pcre_callout)(pcre_callout_block *); +PCRE_EXP_DECL int (*pcre_stack_guard)(void); + +PCRE_EXP_DECL void *(*pcre16_malloc)(size_t); +PCRE_EXP_DECL void (*pcre16_free)(void *); +PCRE_EXP_DECL void *(*pcre16_stack_malloc)(size_t); +PCRE_EXP_DECL void (*pcre16_stack_free)(void *); +PCRE_EXP_DECL int (*pcre16_callout)(pcre16_callout_block *); +PCRE_EXP_DECL int (*pcre16_stack_guard)(void); + +PCRE_EXP_DECL void *(*pcre32_malloc)(size_t); +PCRE_EXP_DECL void (*pcre32_free)(void *); +PCRE_EXP_DECL void *(*pcre32_stack_malloc)(size_t); +PCRE_EXP_DECL void (*pcre32_stack_free)(void *); +PCRE_EXP_DECL int (*pcre32_callout)(pcre32_callout_block *); +PCRE_EXP_DECL int (*pcre32_stack_guard)(void); +#else /* VPCOMPAT */ +PCRE_EXP_DECL void *pcre_malloc(size_t); +PCRE_EXP_DECL void pcre_free(void *); +PCRE_EXP_DECL void *pcre_stack_malloc(size_t); +PCRE_EXP_DECL void pcre_stack_free(void *); +PCRE_EXP_DECL int pcre_callout(pcre_callout_block *); +PCRE_EXP_DECL int pcre_stack_guard(void); + +PCRE_EXP_DECL void *pcre16_malloc(size_t); +PCRE_EXP_DECL void pcre16_free(void *); +PCRE_EXP_DECL void *pcre16_stack_malloc(size_t); +PCRE_EXP_DECL void pcre16_stack_free(void *); +PCRE_EXP_DECL int pcre16_callout(pcre16_callout_block *); +PCRE_EXP_DECL int pcre16_stack_guard(void); + +PCRE_EXP_DECL void *pcre32_malloc(size_t); +PCRE_EXP_DECL void pcre32_free(void *); +PCRE_EXP_DECL void *pcre32_stack_malloc(size_t); +PCRE_EXP_DECL void pcre32_stack_free(void *); +PCRE_EXP_DECL int pcre32_callout(pcre32_callout_block *); +PCRE_EXP_DECL int pcre32_stack_guard(void); +#endif /* VPCOMPAT */ + +/* User defined callback which provides a stack just before the match starts. */ + +typedef pcre_jit_stack *(*pcre_jit_callback)(void *); +typedef pcre16_jit_stack *(*pcre16_jit_callback)(void *); +typedef pcre32_jit_stack *(*pcre32_jit_callback)(void *); + +/* Exported PCRE functions */ + +PCRE_EXP_DECL pcre *pcre_compile(const char *, int, const char **, int *, + const unsigned char *); +PCRE_EXP_DECL pcre16 *pcre16_compile(PCRE_SPTR16, int, const char **, int *, + const unsigned char *); +PCRE_EXP_DECL pcre32 *pcre32_compile(PCRE_SPTR32, int, const char **, int *, + const unsigned char *); +PCRE_EXP_DECL pcre *pcre_compile2(const char *, int, int *, const char **, + int *, const unsigned char *); +PCRE_EXP_DECL pcre16 *pcre16_compile2(PCRE_SPTR16, int, int *, const char **, + int *, const unsigned char *); +PCRE_EXP_DECL pcre32 *pcre32_compile2(PCRE_SPTR32, int, int *, const char **, + int *, const unsigned char *); +PCRE_EXP_DECL int pcre_config(int, void *); +PCRE_EXP_DECL int pcre16_config(int, void *); +PCRE_EXP_DECL int pcre32_config(int, void *); +PCRE_EXP_DECL int pcre_copy_named_substring(const pcre *, const char *, + int *, int, const char *, char *, int); +PCRE_EXP_DECL int pcre16_copy_named_substring(const pcre16 *, PCRE_SPTR16, + int *, int, PCRE_SPTR16, PCRE_UCHAR16 *, int); +PCRE_EXP_DECL int pcre32_copy_named_substring(const pcre32 *, PCRE_SPTR32, + int *, int, PCRE_SPTR32, PCRE_UCHAR32 *, int); +PCRE_EXP_DECL int pcre_copy_substring(const char *, int *, int, int, + char *, int); +PCRE_EXP_DECL int pcre16_copy_substring(PCRE_SPTR16, int *, int, int, + PCRE_UCHAR16 *, int); +PCRE_EXP_DECL int pcre32_copy_substring(PCRE_SPTR32, int *, int, int, + PCRE_UCHAR32 *, int); +PCRE_EXP_DECL int pcre_dfa_exec(const pcre *, const pcre_extra *, + const char *, int, int, int, int *, int , int *, int); +PCRE_EXP_DECL int pcre16_dfa_exec(const pcre16 *, const pcre16_extra *, + PCRE_SPTR16, int, int, int, int *, int , int *, int); +PCRE_EXP_DECL int pcre32_dfa_exec(const pcre32 *, const pcre32_extra *, + PCRE_SPTR32, int, int, int, int *, int , int *, int); +PCRE_EXP_DECL int pcre_exec(const pcre *, const pcre_extra *, PCRE_SPTR, + int, int, int, int *, int); +PCRE_EXP_DECL int pcre16_exec(const pcre16 *, const pcre16_extra *, + PCRE_SPTR16, int, int, int, int *, int); +PCRE_EXP_DECL int pcre32_exec(const pcre32 *, const pcre32_extra *, + PCRE_SPTR32, int, int, int, int *, int); +PCRE_EXP_DECL int pcre_jit_exec(const pcre *, const pcre_extra *, + PCRE_SPTR, int, int, int, int *, int, + pcre_jit_stack *); +PCRE_EXP_DECL int pcre16_jit_exec(const pcre16 *, const pcre16_extra *, + PCRE_SPTR16, int, int, int, int *, int, + pcre16_jit_stack *); +PCRE_EXP_DECL int pcre32_jit_exec(const pcre32 *, const pcre32_extra *, + PCRE_SPTR32, int, int, int, int *, int, + pcre32_jit_stack *); +PCRE_EXP_DECL void pcre_free_substring(const char *); +PCRE_EXP_DECL void pcre16_free_substring(PCRE_SPTR16); +PCRE_EXP_DECL void pcre32_free_substring(PCRE_SPTR32); +PCRE_EXP_DECL void pcre_free_substring_list(const char **); +PCRE_EXP_DECL void pcre16_free_substring_list(PCRE_SPTR16 *); +PCRE_EXP_DECL void pcre32_free_substring_list(PCRE_SPTR32 *); +PCRE_EXP_DECL int pcre_fullinfo(const pcre *, const pcre_extra *, int, + void *); +PCRE_EXP_DECL int pcre16_fullinfo(const pcre16 *, const pcre16_extra *, int, + void *); +PCRE_EXP_DECL int pcre32_fullinfo(const pcre32 *, const pcre32_extra *, int, + void *); +PCRE_EXP_DECL int pcre_get_named_substring(const pcre *, const char *, + int *, int, const char *, const char **); +PCRE_EXP_DECL int pcre16_get_named_substring(const pcre16 *, PCRE_SPTR16, + int *, int, PCRE_SPTR16, PCRE_SPTR16 *); +PCRE_EXP_DECL int pcre32_get_named_substring(const pcre32 *, PCRE_SPTR32, + int *, int, PCRE_SPTR32, PCRE_SPTR32 *); +PCRE_EXP_DECL int pcre_get_stringnumber(const pcre *, const char *); +PCRE_EXP_DECL int pcre16_get_stringnumber(const pcre16 *, PCRE_SPTR16); +PCRE_EXP_DECL int pcre32_get_stringnumber(const pcre32 *, PCRE_SPTR32); +PCRE_EXP_DECL int pcre_get_stringtable_entries(const pcre *, const char *, + char **, char **); +PCRE_EXP_DECL int pcre16_get_stringtable_entries(const pcre16 *, PCRE_SPTR16, + PCRE_UCHAR16 **, PCRE_UCHAR16 **); +PCRE_EXP_DECL int pcre32_get_stringtable_entries(const pcre32 *, PCRE_SPTR32, + PCRE_UCHAR32 **, PCRE_UCHAR32 **); +PCRE_EXP_DECL int pcre_get_substring(const char *, int *, int, int, + const char **); +PCRE_EXP_DECL int pcre16_get_substring(PCRE_SPTR16, int *, int, int, + PCRE_SPTR16 *); +PCRE_EXP_DECL int pcre32_get_substring(PCRE_SPTR32, int *, int, int, + PCRE_SPTR32 *); +PCRE_EXP_DECL int pcre_get_substring_list(const char *, int *, int, + const char ***); +PCRE_EXP_DECL int pcre16_get_substring_list(PCRE_SPTR16, int *, int, + PCRE_SPTR16 **); +PCRE_EXP_DECL int pcre32_get_substring_list(PCRE_SPTR32, int *, int, + PCRE_SPTR32 **); +PCRE_EXP_DECL const unsigned char *pcre_maketables(void); +PCRE_EXP_DECL const unsigned char *pcre16_maketables(void); +PCRE_EXP_DECL const unsigned char *pcre32_maketables(void); +PCRE_EXP_DECL int pcre_refcount(pcre *, int); +PCRE_EXP_DECL int pcre16_refcount(pcre16 *, int); +PCRE_EXP_DECL int pcre32_refcount(pcre32 *, int); +PCRE_EXP_DECL pcre_extra *pcre_study(const pcre *, int, const char **); +PCRE_EXP_DECL pcre16_extra *pcre16_study(const pcre16 *, int, const char **); +PCRE_EXP_DECL pcre32_extra *pcre32_study(const pcre32 *, int, const char **); +PCRE_EXP_DECL void pcre_free_study(pcre_extra *); +PCRE_EXP_DECL void pcre16_free_study(pcre16_extra *); +PCRE_EXP_DECL void pcre32_free_study(pcre32_extra *); +PCRE_EXP_DECL const char *pcre_version(void); +PCRE_EXP_DECL const char *pcre16_version(void); +PCRE_EXP_DECL const char *pcre32_version(void); + +/* Utility functions for byte order swaps. */ +PCRE_EXP_DECL int pcre_pattern_to_host_byte_order(pcre *, pcre_extra *, + const unsigned char *); +PCRE_EXP_DECL int pcre16_pattern_to_host_byte_order(pcre16 *, pcre16_extra *, + const unsigned char *); +PCRE_EXP_DECL int pcre32_pattern_to_host_byte_order(pcre32 *, pcre32_extra *, + const unsigned char *); +PCRE_EXP_DECL int pcre16_utf16_to_host_byte_order(PCRE_UCHAR16 *, + PCRE_SPTR16, int, int *, int); +PCRE_EXP_DECL int pcre32_utf32_to_host_byte_order(PCRE_UCHAR32 *, + PCRE_SPTR32, int, int *, int); + +/* JIT compiler related functions. */ + +PCRE_EXP_DECL pcre_jit_stack *pcre_jit_stack_alloc(int, int); +PCRE_EXP_DECL pcre16_jit_stack *pcre16_jit_stack_alloc(int, int); +PCRE_EXP_DECL pcre32_jit_stack *pcre32_jit_stack_alloc(int, int); +PCRE_EXP_DECL void pcre_jit_stack_free(pcre_jit_stack *); +PCRE_EXP_DECL void pcre16_jit_stack_free(pcre16_jit_stack *); +PCRE_EXP_DECL void pcre32_jit_stack_free(pcre32_jit_stack *); +PCRE_EXP_DECL void pcre_assign_jit_stack(pcre_extra *, + pcre_jit_callback, void *); +PCRE_EXP_DECL void pcre16_assign_jit_stack(pcre16_extra *, + pcre16_jit_callback, void *); +PCRE_EXP_DECL void pcre32_assign_jit_stack(pcre32_extra *, + pcre32_jit_callback, void *); +PCRE_EXP_DECL void pcre_jit_free_unused_memory(void); +PCRE_EXP_DECL void pcre16_jit_free_unused_memory(void); +PCRE_EXP_DECL void pcre32_jit_free_unused_memory(void); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* End of pcre.h */ diff --git a/src/third_party/pcre-8.41/pcre16_byte_order.c b/src/third_party/pcre-8.41/pcre16_byte_order.c new file mode 100644 index 00000000000..11d2973a3db --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_byte_order.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_byte_order.c" + +/* End of pcre16_byte_order.c */ diff --git a/src/third_party/pcre-8.41/pcre16_chartables.c b/src/third_party/pcre-8.41/pcre16_chartables.c new file mode 100644 index 00000000000..7c0ff35f5e0 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_chartables.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_chartables.c" + +/* End of pcre16_chartables.c */ diff --git a/src/third_party/pcre-8.41/pcre16_compile.c b/src/third_party/pcre-8.41/pcre16_compile.c new file mode 100644 index 00000000000..e499b670877 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_compile.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_compile.c" + +/* End of pcre16_compile.c */ diff --git a/src/third_party/pcre-8.41/pcre16_config.c b/src/third_party/pcre-8.41/pcre16_config.c new file mode 100644 index 00000000000..b52138764f6 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_config.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_config.c" + +/* End of pcre16_config.c */ diff --git a/src/third_party/pcre-8.41/pcre16_dfa_exec.c b/src/third_party/pcre-8.41/pcre16_dfa_exec.c new file mode 100644 index 00000000000..2ba740e972b --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_dfa_exec.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_dfa_exec.c" + +/* End of pcre16_dfa_exec.c */ diff --git a/src/third_party/pcre-8.41/pcre16_exec.c b/src/third_party/pcre-8.41/pcre16_exec.c new file mode 100644 index 00000000000..7417b1770c6 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_exec.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_exec.c" + +/* End of pcre16_exec.c */ diff --git a/src/third_party/pcre-8.41/pcre16_fullinfo.c b/src/third_party/pcre-8.41/pcre16_fullinfo.c new file mode 100644 index 00000000000..544dca6ed5c --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_fullinfo.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_fullinfo.c" + +/* End of pcre16_fullinfo.c */ diff --git a/src/third_party/pcre-8.41/pcre16_get.c b/src/third_party/pcre-8.41/pcre16_get.c new file mode 100644 index 00000000000..3ded08c622c --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_get.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_get.c" + +/* End of pcre16_get.c */ diff --git a/src/third_party/pcre-8.41/pcre16_globals.c b/src/third_party/pcre-8.41/pcre16_globals.c new file mode 100644 index 00000000000..a136b3d8c22 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_globals.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_globals.c" + +/* End of pcre16_globals.c */ diff --git a/src/third_party/pcre-8.41/pcre16_jit_compile.c b/src/third_party/pcre-8.41/pcre16_jit_compile.c new file mode 100644 index 00000000000..ab0cacd7646 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_jit_compile.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_jit_compile.c" + +/* End of pcre16_jit_compile.c */ diff --git a/src/third_party/pcre-8.41/pcre16_maketables.c b/src/third_party/pcre-8.41/pcre16_maketables.c new file mode 100644 index 00000000000..b1cd1c579d6 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_maketables.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_maketables.c" + +/* End of pcre16_maketables.c */ diff --git a/src/third_party/pcre-8.41/pcre16_newline.c b/src/third_party/pcre-8.41/pcre16_newline.c new file mode 100644 index 00000000000..7fe201400f5 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_newline.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_newline.c" + +/* End of pcre16_newline.c */ diff --git a/src/third_party/pcre-8.41/pcre16_ord2utf16.c b/src/third_party/pcre-8.41/pcre16_ord2utf16.c new file mode 100644 index 00000000000..8e2ce5ea6c5 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_ord2utf16.c @@ -0,0 +1,90 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This file contains a private PCRE function that converts an ordinal +character value into a UTF16 string. */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_internal.h" + +/************************************************* +* Convert character value to UTF-16 * +*************************************************/ + +/* This function takes an integer value in the range 0 - 0x10ffff +and encodes it as a UTF-16 character in 1 to 2 pcre_uchars. + +Arguments: + cvalue the character value + buffer pointer to buffer for result - at least 2 pcre_uchars long + +Returns: number of characters placed in the buffer +*/ + +unsigned int +PRIV(ord2utf)(pcre_uint32 cvalue, pcre_uchar *buffer) +{ +#ifdef SUPPORT_UTF + +if (cvalue <= 0xffff) + { + *buffer = (pcre_uchar)cvalue; + return 1; + } + +cvalue -= 0x10000; +*buffer++ = 0xd800 | (cvalue >> 10); +*buffer = 0xdc00 | (cvalue & 0x3ff); +return 2; + +#else /* SUPPORT_UTF */ +(void)(cvalue); /* Keep compiler happy; this function won't ever be */ +(void)(buffer); /* called when SUPPORT_UTF is not defined. */ +return 0; +#endif /* SUPPORT_UTF */ +} + +/* End of pcre16_ord2utf16.c */ diff --git a/src/third_party/pcre-8.41/pcre16_printint.c b/src/third_party/pcre-8.41/pcre16_printint.c new file mode 100644 index 00000000000..33d8c340200 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_printint.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_printint.c" + +/* End of pcre16_printint.c */ diff --git a/src/third_party/pcre-8.41/pcre16_refcount.c b/src/third_party/pcre-8.41/pcre16_refcount.c new file mode 100644 index 00000000000..d3d15439737 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_refcount.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_refcount.c" + +/* End of pcre16_refcount.c */ diff --git a/src/third_party/pcre-8.41/pcre16_string_utils.c b/src/third_party/pcre-8.41/pcre16_string_utils.c new file mode 100644 index 00000000000..382c40799fb --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_string_utils.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_string_utils.c" + +/* End of pcre16_string_utils.c */ diff --git a/src/third_party/pcre-8.41/pcre16_study.c b/src/third_party/pcre-8.41/pcre16_study.c new file mode 100644 index 00000000000..f87de081fc4 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_study.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_study.c" + +/* End of pcre16_study.c */ diff --git a/src/third_party/pcre-8.41/pcre16_tables.c b/src/third_party/pcre-8.41/pcre16_tables.c new file mode 100644 index 00000000000..d84297093a4 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_tables.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_tables.c" + +/* End of pcre16_tables.c */ diff --git a/src/third_party/pcre-8.41/pcre16_ucd.c b/src/third_party/pcre-8.41/pcre16_ucd.c new file mode 100644 index 00000000000..ee23439a013 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_ucd.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_ucd.c" + +/* End of pcre16_ucd.c */ diff --git a/src/third_party/pcre-8.41/pcre16_utf16_utils.c b/src/third_party/pcre-8.41/pcre16_utf16_utils.c new file mode 100644 index 00000000000..49ced0c0b1c --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_utf16_utils.c @@ -0,0 +1,130 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This module contains a function for converting any UTF-16 character +strings to host byte order. */ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_internal.h" + +/************************************************* +* Convert any UTF-16 string to host byte order * +*************************************************/ + +/* This function takes an UTF-16 string and converts +it to host byte order. The length can be explicitly set, +or automatically detected for zero terminated strings. +BOMs can be kept or discarded during the conversion. +Conversion can be done in place (output == input). + +Arguments: + output the output buffer, its size must be greater + or equal than the input string + input any UTF-16 string + length the number of 16-bit units in the input string + can be less than zero for zero terminated strings + host_byte_order + A non-zero value means the input is in host byte + order, which can be dynamically changed by BOMs later. + Initially it contains the starting byte order and returns + with the last byte order so it can be used for stream + processing. It can be NULL, which set the host byte + order mode by default. + keep_boms for a non-zero value, the BOM (0xfeff) characters + are copied as well + +Returns: the number of 16-bit units placed into the output buffer, + including the zero-terminator +*/ + +int +pcre16_utf16_to_host_byte_order(PCRE_UCHAR16 *output, PCRE_SPTR16 input, + int length, int *host_byte_order, int keep_boms) +{ +#ifdef SUPPORT_UTF +/* This function converts any UTF-16 string to host byte order and optionally +removes any Byte Order Marks (BOMS). Returns with the remainig length. */ +int host_bo = host_byte_order != NULL ? *host_byte_order : 1; +pcre_uchar *optr = (pcre_uchar *)output; +const pcre_uchar *iptr = (const pcre_uchar *)input; +const pcre_uchar *end; +/* The c variable must be unsigned. */ +register pcre_uchar c; + +if (length < 0) + length = STRLEN_UC(iptr) + 1; +end = iptr + length; + +while (iptr < end) + { + c = *iptr++; + if (c == 0xfeff || c == 0xfffe) + { + /* Detecting the byte order of the machine is unnecessary, it is + enough to know that the UTF-16 string has the same byte order or not. */ + host_bo = c == 0xfeff; + if (keep_boms != 0) + *optr++ = 0xfeff; + else + length--; + } + else + *optr++ = host_bo ? c : ((c >> 8) | (c << 8)); /* Flip bytes if needed. */ + } +if (host_byte_order != NULL) + *host_byte_order = host_bo; + +#else /* Not SUPPORT_UTF */ +(void)(output); /* Keep picky compilers happy */ +(void)(input); +(void)(keep_boms); +(void)(host_byte_order); +#endif /* SUPPORT_UTF */ +return length; +} + +/* End of pcre16_utf16_utils.c */ diff --git a/src/third_party/pcre-8.41/pcre16_valid_utf16.c b/src/third_party/pcre-8.41/pcre16_valid_utf16.c new file mode 100644 index 00000000000..09076539d09 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_valid_utf16.c @@ -0,0 +1,137 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2013 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This module contains an internal function for validating UTF-16 character +strings. */ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_internal.h" + + +/************************************************* +* Validate a UTF-16 string * +*************************************************/ + +/* This function is called (optionally) at the start of compile or match, to +check that a supposed UTF-16 string is actually valid. The early check means +that subsequent code can assume it is dealing with a valid string. The check +can be turned off for maximum performance, but the consequences of supplying an +invalid string are then undefined. + +From release 8.21 more information about the details of the error are passed +back in the returned value: + +PCRE_UTF16_ERR0 No error +PCRE_UTF16_ERR1 Missing low surrogate at the end of the string +PCRE_UTF16_ERR2 Invalid low surrogate +PCRE_UTF16_ERR3 Isolated low surrogate +PCRE_UTF16_ERR4 Unused (was non-character) + +Arguments: + string points to the string + length length of string, or -1 if the string is zero-terminated + errp pointer to an error position offset variable + +Returns: = 0 if the string is a valid UTF-16 string + > 0 otherwise, setting the offset of the bad character +*/ + +int +PRIV(valid_utf)(PCRE_PUCHAR string, int length, int *erroroffset) +{ +#ifdef SUPPORT_UTF +register PCRE_PUCHAR p; +register pcre_uint32 c; + +if (length < 0) + { + for (p = string; *p != 0; p++); + length = p - string; + } + +for (p = string; length-- > 0; p++) + { + c = *p; + + if ((c & 0xf800) != 0xd800) + { + /* Normal UTF-16 code point. Neither high nor low surrogate. */ + } + else if ((c & 0x0400) == 0) + { + /* High surrogate. Must be a followed by a low surrogate. */ + if (length == 0) + { + *erroroffset = p - string; + return PCRE_UTF16_ERR1; + } + p++; + length--; + if ((*p & 0xfc00) != 0xdc00) + { + *erroroffset = p - string; + return PCRE_UTF16_ERR2; + } + } + else + { + /* Isolated low surrogate. Always an error. */ + *erroroffset = p - string; + return PCRE_UTF16_ERR3; + } + } + +#else /* SUPPORT_UTF */ +(void)(string); /* Keep picky compilers happy */ +(void)(length); +(void)(erroroffset); +#endif /* SUPPORT_UTF */ + +return PCRE_UTF16_ERR0; /* This indicates success */ +} + +/* End of pcre16_valid_utf16.c */ diff --git a/src/third_party/pcre-8.41/pcre16_version.c b/src/third_party/pcre-8.41/pcre16_version.c new file mode 100644 index 00000000000..e991b1a8cfd --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_version.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_version.c" + +/* End of pcre16_version.c */ diff --git a/src/third_party/pcre-8.41/pcre16_xclass.c b/src/third_party/pcre-8.41/pcre16_xclass.c new file mode 100644 index 00000000000..5aac2a36c68 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre16_xclass.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 16 bit character support. */ +#define COMPILE_PCRE16 + +#include "pcre_xclass.c" + +/* End of pcre16_xclass.c */ diff --git a/src/third_party/pcre-8.41/pcre32_byte_order.c b/src/third_party/pcre-8.41/pcre32_byte_order.c new file mode 100644 index 00000000000..9cf5362730a --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_byte_order.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_byte_order.c" + +/* End of pcre32_byte_order.c */ diff --git a/src/third_party/pcre-8.41/pcre32_chartables.c b/src/third_party/pcre-8.41/pcre32_chartables.c new file mode 100644 index 00000000000..b5d8c23dbf1 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_chartables.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_chartables.c" + +/* End of pcre32_chartables.c */ diff --git a/src/third_party/pcre-8.41/pcre32_compile.c b/src/third_party/pcre-8.41/pcre32_compile.c new file mode 100644 index 00000000000..d781eb377e0 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_compile.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_compile.c" + +/* End of pcre32_compile.c */ diff --git a/src/third_party/pcre-8.41/pcre32_config.c b/src/third_party/pcre-8.41/pcre32_config.c new file mode 100644 index 00000000000..d63f3e9ea23 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_config.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_config.c" + +/* End of pcre32_config.c */ diff --git a/src/third_party/pcre-8.41/pcre32_dfa_exec.c b/src/third_party/pcre-8.41/pcre32_dfa_exec.c new file mode 100644 index 00000000000..b0bfd34f04d --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_dfa_exec.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_dfa_exec.c" + +/* End of pcre32_dfa_exec.c */ diff --git a/src/third_party/pcre-8.41/pcre32_exec.c b/src/third_party/pcre-8.41/pcre32_exec.c new file mode 100644 index 00000000000..8170ed77d35 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_exec.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_exec.c" + +/* End of pcre32_exec.c */ diff --git a/src/third_party/pcre-8.41/pcre32_fullinfo.c b/src/third_party/pcre-8.41/pcre32_fullinfo.c new file mode 100644 index 00000000000..6ecc5209a08 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_fullinfo.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_fullinfo.c" + +/* End of pcre32_fullinfo.c */ diff --git a/src/third_party/pcre-8.41/pcre32_get.c b/src/third_party/pcre-8.41/pcre32_get.c new file mode 100644 index 00000000000..d35deee0cd4 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_get.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_get.c" + +/* End of pcre32_get.c */ diff --git a/src/third_party/pcre-8.41/pcre32_globals.c b/src/third_party/pcre-8.41/pcre32_globals.c new file mode 100644 index 00000000000..32e0914ca6d --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_globals.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_globals.c" + +/* End of pcre32_globals.c */ diff --git a/src/third_party/pcre-8.41/pcre32_jit_compile.c b/src/third_party/pcre-8.41/pcre32_jit_compile.c new file mode 100644 index 00000000000..2e7c6f97c96 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_jit_compile.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_jit_compile.c" + +/* End of pcre32_jit_compile.c */ diff --git a/src/third_party/pcre-8.41/pcre32_maketables.c b/src/third_party/pcre-8.41/pcre32_maketables.c new file mode 100644 index 00000000000..5d1b1c64c96 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_maketables.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_maketables.c" + +/* End of pcre32_maketables.c */ diff --git a/src/third_party/pcre-8.41/pcre32_newline.c b/src/third_party/pcre-8.41/pcre32_newline.c new file mode 100644 index 00000000000..7f8d5360cdc --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_newline.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_newline.c" + +/* End of pcre32_newline.c */ diff --git a/src/third_party/pcre-8.41/pcre32_ord2utf32.c b/src/third_party/pcre-8.41/pcre32_ord2utf32.c new file mode 100644 index 00000000000..606bcb3d7ba --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_ord2utf32.c @@ -0,0 +1,82 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This file contains a private PCRE function that converts an ordinal +character value into a UTF32 string. */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_internal.h" + +/************************************************* +* Convert character value to UTF-32 * +*************************************************/ + +/* This function takes an integer value in the range 0 - 0x10ffff +and encodes it as a UTF-32 character in 1 pcre_uchars. + +Arguments: + cvalue the character value + buffer pointer to buffer for result - at least 1 pcre_uchars long + +Returns: number of characters placed in the buffer +*/ + +unsigned int +PRIV(ord2utf)(pcre_uint32 cvalue, pcre_uchar *buffer) +{ +#ifdef SUPPORT_UTF + +*buffer = (pcre_uchar)cvalue; +return 1; + +#else /* SUPPORT_UTF */ +(void)(cvalue); /* Keep compiler happy; this function won't ever be */ +(void)(buffer); /* called when SUPPORT_UTF is not defined. */ +return 0; +#endif /* SUPPORT_UTF */ +} + +/* End of pcre32_ord2utf32.c */ diff --git a/src/third_party/pcre-8.41/pcre32_printint.c b/src/third_party/pcre-8.41/pcre32_printint.c new file mode 100644 index 00000000000..f3fd7b25e2c --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_printint.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_printint.c" + +/* End of pcre32_printint.c */ diff --git a/src/third_party/pcre-8.41/pcre32_refcount.c b/src/third_party/pcre-8.41/pcre32_refcount.c new file mode 100644 index 00000000000..dbdf432d82a --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_refcount.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_refcount.c" + +/* End of pcre32_refcount.c */ diff --git a/src/third_party/pcre-8.41/pcre32_string_utils.c b/src/third_party/pcre-8.41/pcre32_string_utils.c new file mode 100644 index 00000000000..e37b3d4805f --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_string_utils.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_string_utils.c" + +/* End of pcre32_string_utils.c */ diff --git a/src/third_party/pcre-8.41/pcre32_study.c b/src/third_party/pcre-8.41/pcre32_study.c new file mode 100644 index 00000000000..d3a3afed791 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_study.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_study.c" + +/* End of pcre32_study.c */ diff --git a/src/third_party/pcre-8.41/pcre32_tables.c b/src/third_party/pcre-8.41/pcre32_tables.c new file mode 100644 index 00000000000..3d94cca33a1 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_tables.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_tables.c" + +/* End of pcre32_tables.c */ diff --git a/src/third_party/pcre-8.41/pcre32_ucd.c b/src/third_party/pcre-8.41/pcre32_ucd.c new file mode 100644 index 00000000000..befe22d3435 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_ucd.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_ucd.c" + +/* End of pcre32_ucd.c */ diff --git a/src/third_party/pcre-8.41/pcre32_utf32_utils.c b/src/third_party/pcre-8.41/pcre32_utf32_utils.c new file mode 100644 index 00000000000..f844e237165 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_utf32_utils.c @@ -0,0 +1,141 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This module contains a function for converting any UTF-32 character +strings to host byte order. */ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_internal.h" + +#ifdef SUPPORT_UTF +static pcre_uint32 +swap_uint32(pcre_uint32 value) +{ +return ((value & 0x000000ff) << 24) | + ((value & 0x0000ff00) << 8) | + ((value & 0x00ff0000) >> 8) | + (value >> 24); +} +#endif + + +/************************************************* +* Convert any UTF-32 string to host byte order * +*************************************************/ + +/* This function takes an UTF-32 string and converts +it to host byte order. The length can be explicitly set, +or automatically detected for zero terminated strings. +BOMs can be kept or discarded during the conversion. +Conversion can be done in place (output == input). + +Arguments: + output the output buffer, its size must be greater + or equal than the input string + input any UTF-32 string + length the number of 32-bit units in the input string + can be less than zero for zero terminated strings + host_byte_order + A non-zero value means the input is in host byte + order, which can be dynamically changed by BOMs later. + Initially it contains the starting byte order and returns + with the last byte order so it can be used for stream + processing. It can be NULL, which set the host byte + order mode by default. + keep_boms for a non-zero value, the BOM (0xfeff) characters + are copied as well + +Returns: the number of 32-bit units placed into the output buffer, + including the zero-terminator +*/ + +int +pcre32_utf32_to_host_byte_order(PCRE_UCHAR32 *output, PCRE_SPTR32 input, + int length, int *host_byte_order, int keep_boms) +{ +#ifdef SUPPORT_UTF +/* This function converts any UTF-32 string to host byte order and optionally +removes any Byte Order Marks (BOMS). Returns with the remainig length. */ +int host_bo = host_byte_order != NULL ? *host_byte_order : 1; +pcre_uchar *optr = (pcre_uchar *)output; +const pcre_uchar *iptr = (const pcre_uchar *)input; +const pcre_uchar *end; +/* The c variable must be unsigned. */ +register pcre_uchar c; + +if (length < 0) + end = iptr + STRLEN_UC(iptr) + 1; +else + end = iptr + length; + +while (iptr < end) + { + c = *iptr++; + if (c == 0x0000feffu || c == 0xfffe0000u) + { + /* Detecting the byte order of the machine is unnecessary, it is + enough to know that the UTF-32 string has the same byte order or not. */ + host_bo = c == 0x0000feffu; + if (keep_boms != 0) + *optr++ = 0x0000feffu; + } + else + *optr++ = host_bo ? c : swap_uint32(c); + } +if (host_byte_order != NULL) + *host_byte_order = host_bo; + +#else /* SUPPORT_UTF */ +(void)(output); /* Keep picky compilers happy */ +(void)(input); +(void)(keep_boms); +(void)(host_byte_order); +#endif /* SUPPORT_UTF */ +return length; +} + +/* End of pcre32_utf32_utils.c */ diff --git a/src/third_party/pcre-8.41/pcre32_valid_utf32.c b/src/third_party/pcre-8.41/pcre32_valid_utf32.c new file mode 100644 index 00000000000..94cda1a2c4c --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_valid_utf32.c @@ -0,0 +1,124 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2013 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This module contains an internal function for validating UTF-32 character +strings. */ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_internal.h" + +/************************************************* +* Validate a UTF-32 string * +*************************************************/ + +/* This function is called (optionally) at the start of compile or match, to +check that a supposed UTF-32 string is actually valid. The early check means +that subsequent code can assume it is dealing with a valid string. The check +can be turned off for maximum performance, but the consequences of supplying an +invalid string are then undefined. + +More information about the details of the error are passed +back in the returned value: + +PCRE_UTF32_ERR0 No error +PCRE_UTF32_ERR1 Surrogate character +PCRE_UTF32_ERR2 Unused (was non-character) +PCRE_UTF32_ERR3 Character > 0x10ffff + +Arguments: + string points to the string + length length of string, or -1 if the string is zero-terminated + errp pointer to an error position offset variable + +Returns: = 0 if the string is a valid UTF-32 string + > 0 otherwise, setting the offset of the bad character +*/ + +int +PRIV(valid_utf)(PCRE_PUCHAR string, int length, int *erroroffset) +{ +#ifdef SUPPORT_UTF +register PCRE_PUCHAR p; +register pcre_uchar c; + +if (length < 0) + { + for (p = string; *p != 0; p++); + length = p - string; + } + +for (p = string; length-- > 0; p++) + { + c = *p; + + if ((c & 0xfffff800u) != 0xd800u) + { + /* Normal UTF-32 code point. Neither high nor low surrogate. */ + if (c > 0x10ffffu) + { + *erroroffset = p - string; + return PCRE_UTF32_ERR3; + } + } + else + { + /* A surrogate */ + *erroroffset = p - string; + return PCRE_UTF32_ERR1; + } + } + +#else /* SUPPORT_UTF */ +(void)(string); /* Keep picky compilers happy */ +(void)(length); +(void)(erroroffset); +#endif /* SUPPORT_UTF */ + +return PCRE_UTF32_ERR0; /* This indicates success */ +} + +/* End of pcre32_valid_utf32.c */ diff --git a/src/third_party/pcre-8.41/pcre32_version.c b/src/third_party/pcre-8.41/pcre32_version.c new file mode 100644 index 00000000000..fdaad9b0859 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_version.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_version.c" + +/* End of pcre32_version.c */ diff --git a/src/third_party/pcre-8.41/pcre32_xclass.c b/src/third_party/pcre-8.41/pcre32_xclass.c new file mode 100644 index 00000000000..5662408ad5f --- /dev/null +++ b/src/third_party/pcre-8.41/pcre32_xclass.c @@ -0,0 +1,45 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Generate code with 32 bit character support. */ +#define COMPILE_PCRE32 + +#include "pcre_xclass.c" + +/* End of pcre32_xclass.c */ diff --git a/src/third_party/pcre-8.41/pcre_byte_order.c b/src/third_party/pcre-8.41/pcre_byte_order.c new file mode 100644 index 00000000000..cf5f12b04ea --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_byte_order.c @@ -0,0 +1,319 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2014 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This module contains an internal function that tests a compiled pattern to +see if it was compiled with the opposite endianness. If so, it uses an +auxiliary local function to flip the appropriate bytes. */ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "pcre_internal.h" + + +/************************************************* +* Swap byte functions * +*************************************************/ + +/* The following functions swap the bytes of a pcre_uint16 +and pcre_uint32 value. + +Arguments: + value any number + +Returns: the byte swapped value +*/ + +static pcre_uint32 +swap_uint32(pcre_uint32 value) +{ +return ((value & 0x000000ff) << 24) | + ((value & 0x0000ff00) << 8) | + ((value & 0x00ff0000) >> 8) | + (value >> 24); +} + +static pcre_uint16 +swap_uint16(pcre_uint16 value) +{ +return (value >> 8) | (value << 8); +} + + +/************************************************* +* Test for a byte-flipped compiled regex * +*************************************************/ + +/* This function swaps the bytes of a compiled pattern usually +loaded form the disk. It also sets the tables pointer, which +is likely an invalid pointer after reload. + +Arguments: + argument_re points to the compiled expression + extra_data points to extra data or is NULL + tables points to the character tables or NULL + +Returns: 0 if the swap is successful, negative on error +*/ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DECL int pcre_pattern_to_host_byte_order(pcre *argument_re, + pcre_extra *extra_data, const unsigned char *tables) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DECL int pcre16_pattern_to_host_byte_order(pcre16 *argument_re, + pcre16_extra *extra_data, const unsigned char *tables) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DECL int pcre32_pattern_to_host_byte_order(pcre32 *argument_re, + pcre32_extra *extra_data, const unsigned char *tables) +#endif +{ +REAL_PCRE *re = (REAL_PCRE *)argument_re; +pcre_study_data *study; +#ifndef COMPILE_PCRE8 +pcre_uchar *ptr; +int length; +#if defined SUPPORT_UTF && defined COMPILE_PCRE16 +BOOL utf; +BOOL utf16_char; +#endif /* SUPPORT_UTF && COMPILE_PCRE16 */ +#endif /* !COMPILE_PCRE8 */ + +if (re == NULL) return PCRE_ERROR_NULL; +if (re->magic_number == MAGIC_NUMBER) + { + if ((re->flags & PCRE_MODE) == 0) return PCRE_ERROR_BADMODE; + re->tables = tables; + return 0; + } + +if (re->magic_number != REVERSED_MAGIC_NUMBER) return PCRE_ERROR_BADMAGIC; +if ((swap_uint32(re->flags) & PCRE_MODE) == 0) return PCRE_ERROR_BADMODE; + +re->magic_number = MAGIC_NUMBER; +re->size = swap_uint32(re->size); +re->options = swap_uint32(re->options); +re->flags = swap_uint32(re->flags); +re->limit_match = swap_uint32(re->limit_match); +re->limit_recursion = swap_uint32(re->limit_recursion); + +#if defined COMPILE_PCRE8 || defined COMPILE_PCRE16 +re->first_char = swap_uint16(re->first_char); +re->req_char = swap_uint16(re->req_char); +#elif defined COMPILE_PCRE32 +re->first_char = swap_uint32(re->first_char); +re->req_char = swap_uint32(re->req_char); +#endif + +re->max_lookbehind = swap_uint16(re->max_lookbehind); +re->top_bracket = swap_uint16(re->top_bracket); +re->top_backref = swap_uint16(re->top_backref); +re->name_table_offset = swap_uint16(re->name_table_offset); +re->name_entry_size = swap_uint16(re->name_entry_size); +re->name_count = swap_uint16(re->name_count); +re->ref_count = swap_uint16(re->ref_count); +re->tables = tables; + +if (extra_data != NULL && (extra_data->flags & PCRE_EXTRA_STUDY_DATA) != 0) + { + study = (pcre_study_data *)extra_data->study_data; + study->size = swap_uint32(study->size); + study->flags = swap_uint32(study->flags); + study->minlength = swap_uint32(study->minlength); + } + +#ifndef COMPILE_PCRE8 +ptr = (pcre_uchar *)re + re->name_table_offset; +length = re->name_count * re->name_entry_size; +#if defined SUPPORT_UTF && defined COMPILE_PCRE16 +utf = (re->options & PCRE_UTF16) != 0; +utf16_char = FALSE; +#endif /* SUPPORT_UTF && COMPILE_PCRE16 */ + +while(TRUE) + { + /* Swap previous characters. */ + while (length-- > 0) + { +#if defined COMPILE_PCRE16 + *ptr = swap_uint16(*ptr); +#elif defined COMPILE_PCRE32 + *ptr = swap_uint32(*ptr); +#endif + ptr++; + } +#if defined SUPPORT_UTF && defined COMPILE_PCRE16 + if (utf16_char) + { + if (HAS_EXTRALEN(ptr[-1])) + { + /* We know that there is only one extra character in UTF-16. */ + *ptr = swap_uint16(*ptr); + ptr++; + } + } + utf16_char = FALSE; +#endif /* SUPPORT_UTF */ + + /* Get next opcode. */ + length = 0; +#if defined COMPILE_PCRE16 + *ptr = swap_uint16(*ptr); +#elif defined COMPILE_PCRE32 + *ptr = swap_uint32(*ptr); +#endif + switch (*ptr) + { + case OP_END: + return 0; + +#if defined SUPPORT_UTF && defined COMPILE_PCRE16 + case OP_CHAR: + case OP_CHARI: + case OP_NOT: + case OP_NOTI: + case OP_STAR: + case OP_MINSTAR: + case OP_PLUS: + case OP_MINPLUS: + case OP_QUERY: + case OP_MINQUERY: + case OP_UPTO: + case OP_MINUPTO: + case OP_EXACT: + case OP_POSSTAR: + case OP_POSPLUS: + case OP_POSQUERY: + case OP_POSUPTO: + case OP_STARI: + case OP_MINSTARI: + case OP_PLUSI: + case OP_MINPLUSI: + case OP_QUERYI: + case OP_MINQUERYI: + case OP_UPTOI: + case OP_MINUPTOI: + case OP_EXACTI: + case OP_POSSTARI: + case OP_POSPLUSI: + case OP_POSQUERYI: + case OP_POSUPTOI: + case OP_NOTSTAR: + case OP_NOTMINSTAR: + case OP_NOTPLUS: + case OP_NOTMINPLUS: + case OP_NOTQUERY: + case OP_NOTMINQUERY: + case OP_NOTUPTO: + case OP_NOTMINUPTO: + case OP_NOTEXACT: + case OP_NOTPOSSTAR: + case OP_NOTPOSPLUS: + case OP_NOTPOSQUERY: + case OP_NOTPOSUPTO: + case OP_NOTSTARI: + case OP_NOTMINSTARI: + case OP_NOTPLUSI: + case OP_NOTMINPLUSI: + case OP_NOTQUERYI: + case OP_NOTMINQUERYI: + case OP_NOTUPTOI: + case OP_NOTMINUPTOI: + case OP_NOTEXACTI: + case OP_NOTPOSSTARI: + case OP_NOTPOSPLUSI: + case OP_NOTPOSQUERYI: + case OP_NOTPOSUPTOI: + if (utf) utf16_char = TRUE; +#endif + /* Fall through. */ + + default: + length = PRIV(OP_lengths)[*ptr] - 1; + break; + + case OP_CLASS: + case OP_NCLASS: + /* Skip the character bit map. */ + ptr += 32/sizeof(pcre_uchar); + length = 0; + break; + + case OP_XCLASS: + /* Reverse the size of the XCLASS instance. */ + ptr++; +#if defined COMPILE_PCRE16 + *ptr = swap_uint16(*ptr); +#elif defined COMPILE_PCRE32 + *ptr = swap_uint32(*ptr); +#endif +#ifndef COMPILE_PCRE32 + if (LINK_SIZE > 1) + { + /* LINK_SIZE can be 1 or 2 in 16 bit mode. */ + ptr++; + *ptr = swap_uint16(*ptr); + } +#endif + ptr++; + length = (GET(ptr, -LINK_SIZE)) - (1 + LINK_SIZE + 1); +#if defined COMPILE_PCRE16 + *ptr = swap_uint16(*ptr); +#elif defined COMPILE_PCRE32 + *ptr = swap_uint32(*ptr); +#endif + if ((*ptr & XCL_MAP) != 0) + { + /* Skip the character bit map. */ + ptr += 32/sizeof(pcre_uchar); + length -= 32/sizeof(pcre_uchar); + } + break; + } + ptr++; + } +/* Control should never reach here in 16/32 bit mode. */ +#else /* In 8-bit mode, the pattern does not need to be processed. */ +return 0; +#endif /* !COMPILE_PCRE8 */ +} + +/* End of pcre_byte_order.c */ diff --git a/src/third_party/pcre-8.41/pcre_chartables.c b/src/third_party/pcre-8.41/pcre_chartables.c new file mode 100644 index 00000000000..1e20ec29d05 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_chartables.c @@ -0,0 +1,198 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* This file contains character tables that are used when no external tables +are passed to PCRE by the application that calls it. The tables are used only +for characters whose code values are less than 256. + +This is a default version of the tables that assumes ASCII encoding. A program +called dftables (which is distributed with PCRE) can be used to build +alternative versions of this file. This is necessary if you are running in an +EBCDIC environment, or if you want to default to a different encoding, for +example ISO-8859-1. When dftables is run, it creates these tables in the +current locale. If PCRE is configured with --enable-rebuild-chartables, this +happens automatically. + +The following #includes are present because without them gcc 4.x may remove the +array definition from the final binary if PCRE is built into a static library +and dead code stripping is activated. This leads to link errors. Pulling in the +header ensures that the array gets flagged as "someone outside this compilation +unit might reference this" and so it will always be supplied to the linker. */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "pcre_internal.h" + +const pcre_uint8 PRIV(default_tables)[] = { + +/* This table is a lower casing table. */ + + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, + 64, 97, 98, 99,100,101,102,103, + 104,105,106,107,108,109,110,111, + 112,113,114,115,116,117,118,119, + 120,121,122, 91, 92, 93, 94, 95, + 96, 97, 98, 99,100,101,102,103, + 104,105,106,107,108,109,110,111, + 112,113,114,115,116,117,118,119, + 120,121,122,123,124,125,126,127, + 128,129,130,131,132,133,134,135, + 136,137,138,139,140,141,142,143, + 144,145,146,147,148,149,150,151, + 152,153,154,155,156,157,158,159, + 160,161,162,163,164,165,166,167, + 168,169,170,171,172,173,174,175, + 176,177,178,179,180,181,182,183, + 184,185,186,187,188,189,190,191, + 192,193,194,195,196,197,198,199, + 200,201,202,203,204,205,206,207, + 208,209,210,211,212,213,214,215, + 216,217,218,219,220,221,222,223, + 224,225,226,227,228,229,230,231, + 232,233,234,235,236,237,238,239, + 240,241,242,243,244,245,246,247, + 248,249,250,251,252,253,254,255, + +/* This table is a case flipping table. */ + + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, + 64, 97, 98, 99,100,101,102,103, + 104,105,106,107,108,109,110,111, + 112,113,114,115,116,117,118,119, + 120,121,122, 91, 92, 93, 94, 95, + 96, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 74, 75, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 87, + 88, 89, 90,123,124,125,126,127, + 128,129,130,131,132,133,134,135, + 136,137,138,139,140,141,142,143, + 144,145,146,147,148,149,150,151, + 152,153,154,155,156,157,158,159, + 160,161,162,163,164,165,166,167, + 168,169,170,171,172,173,174,175, + 176,177,178,179,180,181,182,183, + 184,185,186,187,188,189,190,191, + 192,193,194,195,196,197,198,199, + 200,201,202,203,204,205,206,207, + 208,209,210,211,212,213,214,215, + 216,217,218,219,220,221,222,223, + 224,225,226,227,228,229,230,231, + 232,233,234,235,236,237,238,239, + 240,241,242,243,244,245,246,247, + 248,249,250,251,252,253,254,255, + +/* This table contains bit maps for various character classes. Each map is 32 +bytes long and the bits run from the least significant end of each byte. The +classes that have their own maps are: space, xdigit, digit, upper, lower, word, +graph, print, punct, and cntrl. Other classes are built from combinations. */ + + 0x00,0x3e,0x00,0x00,0x01,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, + 0x7e,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xfe,0xff,0xff,0x07,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0x07, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, + 0xfe,0xff,0xff,0x87,0xfe,0xff,0xff,0x07, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0xfe,0xff,0x00,0xfc, + 0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0x78, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x80, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + +/* This table identifies various classes of character by individual bits: + 0x01 white space character + 0x02 letter + 0x04 decimal digit + 0x08 hexadecimal digit + 0x10 alphanumeric or '_' + 0x80 regular expression metacharacter or binary zero +*/ + + 0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 0- 7 */ + 0x00,0x01,0x01,0x01,0x01,0x01,0x00,0x00, /* 8- 15 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 16- 23 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 24- 31 */ + 0x01,0x00,0x00,0x00,0x80,0x00,0x00,0x00, /* - ' */ + 0x80,0x80,0x80,0x80,0x00,0x00,0x80,0x00, /* ( - / */ + 0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c, /* 0 - 7 */ + 0x1c,0x1c,0x00,0x00,0x00,0x00,0x00,0x80, /* 8 - ? */ + 0x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /* @ - G */ + 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* H - O */ + 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* P - W */ + 0x12,0x12,0x12,0x80,0x80,0x00,0x80,0x10, /* X - _ */ + 0x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /* ` - g */ + 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* h - o */ + 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* p - w */ + 0x12,0x12,0x12,0x80,0x80,0x00,0x00,0x00, /* x -127 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 128-135 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 136-143 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 144-151 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 152-159 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 160-167 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 168-175 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 176-183 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 184-191 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 192-199 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 200-207 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 208-215 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 216-223 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 224-231 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 232-239 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 240-247 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00};/* 248-255 */ + +/* End of pcre_chartables.c */ diff --git a/src/third_party/pcre-8.41/pcre_chartables.c.dist b/src/third_party/pcre-8.41/pcre_chartables.c.dist new file mode 100644 index 00000000000..1e20ec29d05 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_chartables.c.dist @@ -0,0 +1,198 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* This file contains character tables that are used when no external tables +are passed to PCRE by the application that calls it. The tables are used only +for characters whose code values are less than 256. + +This is a default version of the tables that assumes ASCII encoding. A program +called dftables (which is distributed with PCRE) can be used to build +alternative versions of this file. This is necessary if you are running in an +EBCDIC environment, or if you want to default to a different encoding, for +example ISO-8859-1. When dftables is run, it creates these tables in the +current locale. If PCRE is configured with --enable-rebuild-chartables, this +happens automatically. + +The following #includes are present because without them gcc 4.x may remove the +array definition from the final binary if PCRE is built into a static library +and dead code stripping is activated. This leads to link errors. Pulling in the +header ensures that the array gets flagged as "someone outside this compilation +unit might reference this" and so it will always be supplied to the linker. */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "pcre_internal.h" + +const pcre_uint8 PRIV(default_tables)[] = { + +/* This table is a lower casing table. */ + + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, + 64, 97, 98, 99,100,101,102,103, + 104,105,106,107,108,109,110,111, + 112,113,114,115,116,117,118,119, + 120,121,122, 91, 92, 93, 94, 95, + 96, 97, 98, 99,100,101,102,103, + 104,105,106,107,108,109,110,111, + 112,113,114,115,116,117,118,119, + 120,121,122,123,124,125,126,127, + 128,129,130,131,132,133,134,135, + 136,137,138,139,140,141,142,143, + 144,145,146,147,148,149,150,151, + 152,153,154,155,156,157,158,159, + 160,161,162,163,164,165,166,167, + 168,169,170,171,172,173,174,175, + 176,177,178,179,180,181,182,183, + 184,185,186,187,188,189,190,191, + 192,193,194,195,196,197,198,199, + 200,201,202,203,204,205,206,207, + 208,209,210,211,212,213,214,215, + 216,217,218,219,220,221,222,223, + 224,225,226,227,228,229,230,231, + 232,233,234,235,236,237,238,239, + 240,241,242,243,244,245,246,247, + 248,249,250,251,252,253,254,255, + +/* This table is a case flipping table. */ + + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, + 64, 97, 98, 99,100,101,102,103, + 104,105,106,107,108,109,110,111, + 112,113,114,115,116,117,118,119, + 120,121,122, 91, 92, 93, 94, 95, + 96, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 74, 75, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 87, + 88, 89, 90,123,124,125,126,127, + 128,129,130,131,132,133,134,135, + 136,137,138,139,140,141,142,143, + 144,145,146,147,148,149,150,151, + 152,153,154,155,156,157,158,159, + 160,161,162,163,164,165,166,167, + 168,169,170,171,172,173,174,175, + 176,177,178,179,180,181,182,183, + 184,185,186,187,188,189,190,191, + 192,193,194,195,196,197,198,199, + 200,201,202,203,204,205,206,207, + 208,209,210,211,212,213,214,215, + 216,217,218,219,220,221,222,223, + 224,225,226,227,228,229,230,231, + 232,233,234,235,236,237,238,239, + 240,241,242,243,244,245,246,247, + 248,249,250,251,252,253,254,255, + +/* This table contains bit maps for various character classes. Each map is 32 +bytes long and the bits run from the least significant end of each byte. The +classes that have their own maps are: space, xdigit, digit, upper, lower, word, +graph, print, punct, and cntrl. Other classes are built from combinations. */ + + 0x00,0x3e,0x00,0x00,0x01,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, + 0x7e,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xfe,0xff,0xff,0x07,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0x07, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, + 0xfe,0xff,0xff,0x87,0xfe,0xff,0xff,0x07, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0xfe,0xff,0x00,0xfc, + 0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0x78, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x80, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + +/* This table identifies various classes of character by individual bits: + 0x01 white space character + 0x02 letter + 0x04 decimal digit + 0x08 hexadecimal digit + 0x10 alphanumeric or '_' + 0x80 regular expression metacharacter or binary zero +*/ + + 0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 0- 7 */ + 0x00,0x01,0x01,0x01,0x01,0x01,0x00,0x00, /* 8- 15 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 16- 23 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 24- 31 */ + 0x01,0x00,0x00,0x00,0x80,0x00,0x00,0x00, /* - ' */ + 0x80,0x80,0x80,0x80,0x00,0x00,0x80,0x00, /* ( - / */ + 0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c, /* 0 - 7 */ + 0x1c,0x1c,0x00,0x00,0x00,0x00,0x00,0x80, /* 8 - ? */ + 0x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /* @ - G */ + 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* H - O */ + 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* P - W */ + 0x12,0x12,0x12,0x80,0x80,0x00,0x80,0x10, /* X - _ */ + 0x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /* ` - g */ + 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* h - o */ + 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* p - w */ + 0x12,0x12,0x12,0x80,0x80,0x00,0x00,0x00, /* x -127 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 128-135 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 136-143 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 144-151 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 152-159 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 160-167 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 168-175 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 176-183 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 184-191 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 192-199 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 200-207 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 208-215 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 216-223 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 224-231 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 232-239 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 240-247 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00};/* 248-255 */ + +/* End of pcre_chartables.c */ diff --git a/src/third_party/pcre-8.41/pcre_compile.c b/src/third_party/pcre-8.41/pcre_compile.c new file mode 100644 index 00000000000..42f204cdfff --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_compile.c @@ -0,0 +1,9795 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2016 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This module contains the external function pcre_compile(), along with +supporting internal functions that are not used by other modules. */ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define NLBLOCK cd /* Block containing newline information */ +#define PSSTART start_pattern /* Field containing pattern start */ +#define PSEND end_pattern /* Field containing pattern end */ + +#include "pcre_internal.h" + + +/* When PCRE_DEBUG is defined, we need the pcre(16|32)_printint() function, which +is also used by pcretest. PCRE_DEBUG is not defined when building a production +library. We do not need to select pcre16_printint.c specially, because the +COMPILE_PCREx macro will already be appropriately set. */ + +#ifdef PCRE_DEBUG +/* pcre_printint.c should not include any headers */ +#define PCRE_INCLUDED +#include "pcre_printint.c" +#undef PCRE_INCLUDED +#endif + + +/* Macro for setting individual bits in class bitmaps. */ + +#define SETBIT(a,b) a[(b)/8] |= (1 << ((b)&7)) + +/* Maximum length value to check against when making sure that the integer that +holds the compiled pattern length does not overflow. We make it a bit less than +INT_MAX to allow for adding in group terminating bytes, so that we don't have +to check them every time. */ + +#define OFLOW_MAX (INT_MAX - 20) + +/* Definitions to allow mutual recursion */ + +static int + add_list_to_class(pcre_uint8 *, pcre_uchar **, int, compile_data *, + const pcre_uint32 *, unsigned int); + +static BOOL + compile_regex(int, pcre_uchar **, const pcre_uchar **, int *, BOOL, BOOL, int, int, + pcre_uint32 *, pcre_int32 *, pcre_uint32 *, pcre_int32 *, branch_chain *, + compile_data *, int *); + + + +/************************************************* +* Code parameters and static tables * +*************************************************/ + +/* This value specifies the size of stack workspace that is used during the +first pre-compile phase that determines how much memory is required. The regex +is partly compiled into this space, but the compiled parts are discarded as +soon as they can be, so that hopefully there will never be an overrun. The code +does, however, check for an overrun. The largest amount I've seen used is 218, +so this number is very generous. + +The same workspace is used during the second, actual compile phase for +remembering forward references to groups so that they can be filled in at the +end. Each entry in this list occupies LINK_SIZE bytes, so even when LINK_SIZE +is 4 there is plenty of room for most patterns. However, the memory can get +filled up by repetitions of forward references, for example patterns like +/(?1){0,1999}(b)/, and one user did hit the limit. The code has been changed so +that the workspace is expanded using malloc() in this situation. The value +below is therefore a minimum, and we put a maximum on it for safety. The +minimum is now also defined in terms of LINK_SIZE so that the use of malloc() +kicks in at the same number of forward references in all cases. */ + +#define COMPILE_WORK_SIZE (2048*LINK_SIZE) +#define COMPILE_WORK_SIZE_MAX (100*COMPILE_WORK_SIZE) + +/* This value determines the size of the initial vector that is used for +remembering named groups during the pre-compile. It is allocated on the stack, +but if it is too small, it is expanded using malloc(), in a similar way to the +workspace. The value is the number of slots in the list. */ + +#define NAMED_GROUP_LIST_SIZE 20 + +/* The overrun tests check for a slightly smaller size so that they detect the +overrun before it actually does run off the end of the data block. */ + +#define WORK_SIZE_SAFETY_MARGIN (100) + +/* Private flags added to firstchar and reqchar. */ + +#define REQ_CASELESS (1 << 0) /* Indicates caselessness */ +#define REQ_VARY (1 << 1) /* Reqchar followed non-literal item */ +/* Negative values for the firstchar and reqchar flags */ +#define REQ_UNSET (-2) +#define REQ_NONE (-1) + +/* Repeated character flags. */ + +#define UTF_LENGTH 0x10000000l /* The char contains its length. */ + +/* Table for handling escaped characters in the range '0'-'z'. Positive returns +are simple data values; negative values are for special things like \d and so +on. Zero means further processing is needed (for things like \x), or the escape +is invalid. */ + +#ifndef EBCDIC + +/* This is the "normal" table for ASCII systems or for EBCDIC systems running +in UTF-8 mode. */ + +static const short int escapes[] = { + 0, 0, + 0, 0, + 0, 0, + 0, 0, + 0, 0, + CHAR_COLON, CHAR_SEMICOLON, + CHAR_LESS_THAN_SIGN, CHAR_EQUALS_SIGN, + CHAR_GREATER_THAN_SIGN, CHAR_QUESTION_MARK, + CHAR_COMMERCIAL_AT, -ESC_A, + -ESC_B, -ESC_C, + -ESC_D, -ESC_E, + 0, -ESC_G, + -ESC_H, 0, + 0, -ESC_K, + 0, 0, + -ESC_N, 0, + -ESC_P, -ESC_Q, + -ESC_R, -ESC_S, + 0, 0, + -ESC_V, -ESC_W, + -ESC_X, 0, + -ESC_Z, CHAR_LEFT_SQUARE_BRACKET, + CHAR_BACKSLASH, CHAR_RIGHT_SQUARE_BRACKET, + CHAR_CIRCUMFLEX_ACCENT, CHAR_UNDERSCORE, + CHAR_GRAVE_ACCENT, ESC_a, + -ESC_b, 0, + -ESC_d, ESC_e, + ESC_f, 0, + -ESC_h, 0, + 0, -ESC_k, + 0, 0, + ESC_n, 0, + -ESC_p, 0, + ESC_r, -ESC_s, + ESC_tee, 0, + -ESC_v, -ESC_w, + 0, 0, + -ESC_z +}; + +#else + +/* This is the "abnormal" table for EBCDIC systems without UTF-8 support. */ + +static const short int escapes[] = { +/* 48 */ 0, 0, 0, '.', '<', '(', '+', '|', +/* 50 */ '&', 0, 0, 0, 0, 0, 0, 0, +/* 58 */ 0, 0, '!', '$', '*', ')', ';', '~', +/* 60 */ '-', '/', 0, 0, 0, 0, 0, 0, +/* 68 */ 0, 0, '|', ',', '%', '_', '>', '?', +/* 70 */ 0, 0, 0, 0, 0, 0, 0, 0, +/* 78 */ 0, '`', ':', '#', '@', '\'', '=', '"', +/* 80 */ 0, ESC_a, -ESC_b, 0, -ESC_d, ESC_e, ESC_f, 0, +/* 88 */-ESC_h, 0, 0, '{', 0, 0, 0, 0, +/* 90 */ 0, 0, -ESC_k, 0, 0, ESC_n, 0, -ESC_p, +/* 98 */ 0, ESC_r, 0, '}', 0, 0, 0, 0, +/* A0 */ 0, '~', -ESC_s, ESC_tee, 0,-ESC_v, -ESC_w, 0, +/* A8 */ 0,-ESC_z, 0, 0, 0, '[', 0, 0, +/* B0 */ 0, 0, 0, 0, 0, 0, 0, 0, +/* B8 */ 0, 0, 0, 0, 0, ']', '=', '-', +/* C0 */ '{',-ESC_A, -ESC_B, -ESC_C, -ESC_D,-ESC_E, 0, -ESC_G, +/* C8 */-ESC_H, 0, 0, 0, 0, 0, 0, 0, +/* D0 */ '}', 0, -ESC_K, 0, 0,-ESC_N, 0, -ESC_P, +/* D8 */-ESC_Q,-ESC_R, 0, 0, 0, 0, 0, 0, +/* E0 */ '\\', 0, -ESC_S, 0, 0,-ESC_V, -ESC_W, -ESC_X, +/* E8 */ 0,-ESC_Z, 0, 0, 0, 0, 0, 0, +/* F0 */ 0, 0, 0, 0, 0, 0, 0, 0, +/* F8 */ 0, 0, 0, 0, 0, 0, 0, 0 +}; + +/* We also need a table of characters that may follow \c in an EBCDIC +environment for characters 0-31. */ + +static unsigned char ebcdic_escape_c[] = "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_"; + +#endif + + +/* Table of special "verbs" like (*PRUNE). This is a short table, so it is +searched linearly. Put all the names into a single string, in order to reduce +the number of relocations when a shared library is dynamically linked. The +string is built from string macros so that it works in UTF-8 mode on EBCDIC +platforms. */ + +typedef struct verbitem { + int len; /* Length of verb name */ + int op; /* Op when no arg, or -1 if arg mandatory */ + int op_arg; /* Op when arg present, or -1 if not allowed */ +} verbitem; + +static const char verbnames[] = + "\0" /* Empty name is a shorthand for MARK */ + STRING_MARK0 + STRING_ACCEPT0 + STRING_COMMIT0 + STRING_F0 + STRING_FAIL0 + STRING_PRUNE0 + STRING_SKIP0 + STRING_THEN; + +static const verbitem verbs[] = { + { 0, -1, OP_MARK }, + { 4, -1, OP_MARK }, + { 6, OP_ACCEPT, -1 }, + { 6, OP_COMMIT, -1 }, + { 1, OP_FAIL, -1 }, + { 4, OP_FAIL, -1 }, + { 5, OP_PRUNE, OP_PRUNE_ARG }, + { 4, OP_SKIP, OP_SKIP_ARG }, + { 4, OP_THEN, OP_THEN_ARG } +}; + +static const int verbcount = sizeof(verbs)/sizeof(verbitem); + + +/* Substitutes for [[:<:]] and [[:>:]], which mean start and end of word in +another regex library. */ + +static const pcre_uchar sub_start_of_word[] = { + CHAR_BACKSLASH, CHAR_b, CHAR_LEFT_PARENTHESIS, CHAR_QUESTION_MARK, + CHAR_EQUALS_SIGN, CHAR_BACKSLASH, CHAR_w, CHAR_RIGHT_PARENTHESIS, '\0' }; + +static const pcre_uchar sub_end_of_word[] = { + CHAR_BACKSLASH, CHAR_b, CHAR_LEFT_PARENTHESIS, CHAR_QUESTION_MARK, + CHAR_LESS_THAN_SIGN, CHAR_EQUALS_SIGN, CHAR_BACKSLASH, CHAR_w, + CHAR_RIGHT_PARENTHESIS, '\0' }; + + +/* Tables of names of POSIX character classes and their lengths. The names are +now all in a single string, to reduce the number of relocations when a shared +library is dynamically loaded. The list of lengths is terminated by a zero +length entry. The first three must be alpha, lower, upper, as this is assumed +for handling case independence. The indices for graph, print, and punct are +needed, so identify them. */ + +static const char posix_names[] = + STRING_alpha0 STRING_lower0 STRING_upper0 STRING_alnum0 + STRING_ascii0 STRING_blank0 STRING_cntrl0 STRING_digit0 + STRING_graph0 STRING_print0 STRING_punct0 STRING_space0 + STRING_word0 STRING_xdigit; + +static const pcre_uint8 posix_name_lengths[] = { + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 6, 0 }; + +#define PC_GRAPH 8 +#define PC_PRINT 9 +#define PC_PUNCT 10 + + +/* Table of class bit maps for each POSIX class. Each class is formed from a +base map, with an optional addition or removal of another map. Then, for some +classes, there is some additional tweaking: for [:blank:] the vertical space +characters are removed, and for [:alpha:] and [:alnum:] the underscore +character is removed. The triples in the table consist of the base map offset, +second map offset or -1 if no second map, and a non-negative value for map +addition or a negative value for map subtraction (if there are two maps). The +absolute value of the third field has these meanings: 0 => no tweaking, 1 => +remove vertical space characters, 2 => remove underscore. */ + +static const int posix_class_maps[] = { + cbit_word, cbit_digit, -2, /* alpha */ + cbit_lower, -1, 0, /* lower */ + cbit_upper, -1, 0, /* upper */ + cbit_word, -1, 2, /* alnum - word without underscore */ + cbit_print, cbit_cntrl, 0, /* ascii */ + cbit_space, -1, 1, /* blank - a GNU extension */ + cbit_cntrl, -1, 0, /* cntrl */ + cbit_digit, -1, 0, /* digit */ + cbit_graph, -1, 0, /* graph */ + cbit_print, -1, 0, /* print */ + cbit_punct, -1, 0, /* punct */ + cbit_space, -1, 0, /* space */ + cbit_word, -1, 0, /* word - a Perl extension */ + cbit_xdigit,-1, 0 /* xdigit */ +}; + +/* Table of substitutes for \d etc when PCRE_UCP is set. They are replaced by +Unicode property escapes. */ + +#ifdef SUPPORT_UCP +static const pcre_uchar string_PNd[] = { + CHAR_BACKSLASH, CHAR_P, CHAR_LEFT_CURLY_BRACKET, + CHAR_N, CHAR_d, CHAR_RIGHT_CURLY_BRACKET, '\0' }; +static const pcre_uchar string_pNd[] = { + CHAR_BACKSLASH, CHAR_p, CHAR_LEFT_CURLY_BRACKET, + CHAR_N, CHAR_d, CHAR_RIGHT_CURLY_BRACKET, '\0' }; +static const pcre_uchar string_PXsp[] = { + CHAR_BACKSLASH, CHAR_P, CHAR_LEFT_CURLY_BRACKET, + CHAR_X, CHAR_s, CHAR_p, CHAR_RIGHT_CURLY_BRACKET, '\0' }; +static const pcre_uchar string_pXsp[] = { + CHAR_BACKSLASH, CHAR_p, CHAR_LEFT_CURLY_BRACKET, + CHAR_X, CHAR_s, CHAR_p, CHAR_RIGHT_CURLY_BRACKET, '\0' }; +static const pcre_uchar string_PXwd[] = { + CHAR_BACKSLASH, CHAR_P, CHAR_LEFT_CURLY_BRACKET, + CHAR_X, CHAR_w, CHAR_d, CHAR_RIGHT_CURLY_BRACKET, '\0' }; +static const pcre_uchar string_pXwd[] = { + CHAR_BACKSLASH, CHAR_p, CHAR_LEFT_CURLY_BRACKET, + CHAR_X, CHAR_w, CHAR_d, CHAR_RIGHT_CURLY_BRACKET, '\0' }; + +static const pcre_uchar *substitutes[] = { + string_PNd, /* \D */ + string_pNd, /* \d */ + string_PXsp, /* \S */ /* Xsp is Perl space, but from 8.34, Perl */ + string_pXsp, /* \s */ /* space and POSIX space are the same. */ + string_PXwd, /* \W */ + string_pXwd /* \w */ +}; + +/* The POSIX class substitutes must be in the order of the POSIX class names, +defined above, and there are both positive and negative cases. NULL means no +general substitute of a Unicode property escape (\p or \P). However, for some +POSIX classes (e.g. graph, print, punct) a special property code is compiled +directly. */ + +static const pcre_uchar string_pL[] = { + CHAR_BACKSLASH, CHAR_p, CHAR_LEFT_CURLY_BRACKET, + CHAR_L, CHAR_RIGHT_CURLY_BRACKET, '\0' }; +static const pcre_uchar string_pLl[] = { + CHAR_BACKSLASH, CHAR_p, CHAR_LEFT_CURLY_BRACKET, + CHAR_L, CHAR_l, CHAR_RIGHT_CURLY_BRACKET, '\0' }; +static const pcre_uchar string_pLu[] = { + CHAR_BACKSLASH, CHAR_p, CHAR_LEFT_CURLY_BRACKET, + CHAR_L, CHAR_u, CHAR_RIGHT_CURLY_BRACKET, '\0' }; +static const pcre_uchar string_pXan[] = { + CHAR_BACKSLASH, CHAR_p, CHAR_LEFT_CURLY_BRACKET, + CHAR_X, CHAR_a, CHAR_n, CHAR_RIGHT_CURLY_BRACKET, '\0' }; +static const pcre_uchar string_h[] = { + CHAR_BACKSLASH, CHAR_h, '\0' }; +static const pcre_uchar string_pXps[] = { + CHAR_BACKSLASH, CHAR_p, CHAR_LEFT_CURLY_BRACKET, + CHAR_X, CHAR_p, CHAR_s, CHAR_RIGHT_CURLY_BRACKET, '\0' }; +static const pcre_uchar string_PL[] = { + CHAR_BACKSLASH, CHAR_P, CHAR_LEFT_CURLY_BRACKET, + CHAR_L, CHAR_RIGHT_CURLY_BRACKET, '\0' }; +static const pcre_uchar string_PLl[] = { + CHAR_BACKSLASH, CHAR_P, CHAR_LEFT_CURLY_BRACKET, + CHAR_L, CHAR_l, CHAR_RIGHT_CURLY_BRACKET, '\0' }; +static const pcre_uchar string_PLu[] = { + CHAR_BACKSLASH, CHAR_P, CHAR_LEFT_CURLY_BRACKET, + CHAR_L, CHAR_u, CHAR_RIGHT_CURLY_BRACKET, '\0' }; +static const pcre_uchar string_PXan[] = { + CHAR_BACKSLASH, CHAR_P, CHAR_LEFT_CURLY_BRACKET, + CHAR_X, CHAR_a, CHAR_n, CHAR_RIGHT_CURLY_BRACKET, '\0' }; +static const pcre_uchar string_H[] = { + CHAR_BACKSLASH, CHAR_H, '\0' }; +static const pcre_uchar string_PXps[] = { + CHAR_BACKSLASH, CHAR_P, CHAR_LEFT_CURLY_BRACKET, + CHAR_X, CHAR_p, CHAR_s, CHAR_RIGHT_CURLY_BRACKET, '\0' }; + +static const pcre_uchar *posix_substitutes[] = { + string_pL, /* alpha */ + string_pLl, /* lower */ + string_pLu, /* upper */ + string_pXan, /* alnum */ + NULL, /* ascii */ + string_h, /* blank */ + NULL, /* cntrl */ + string_pNd, /* digit */ + NULL, /* graph */ + NULL, /* print */ + NULL, /* punct */ + string_pXps, /* space */ /* Xps is POSIX space, but from 8.34 */ + string_pXwd, /* word */ /* Perl and POSIX space are the same */ + NULL, /* xdigit */ + /* Negated cases */ + string_PL, /* ^alpha */ + string_PLl, /* ^lower */ + string_PLu, /* ^upper */ + string_PXan, /* ^alnum */ + NULL, /* ^ascii */ + string_H, /* ^blank */ + NULL, /* ^cntrl */ + string_PNd, /* ^digit */ + NULL, /* ^graph */ + NULL, /* ^print */ + NULL, /* ^punct */ + string_PXps, /* ^space */ /* Xps is POSIX space, but from 8.34 */ + string_PXwd, /* ^word */ /* Perl and POSIX space are the same */ + NULL /* ^xdigit */ +}; +#define POSIX_SUBSIZE (sizeof(posix_substitutes) / sizeof(pcre_uchar *)) +#endif + +#define STRING(a) # a +#define XSTRING(s) STRING(s) + +/* The texts of compile-time error messages. These are "char *" because they +are passed to the outside world. Do not ever re-use any error number, because +they are documented. Always add a new error instead. Messages marked DEAD below +are no longer used. This used to be a table of strings, but in order to reduce +the number of relocations needed when a shared library is loaded dynamically, +it is now one long string. We cannot use a table of offsets, because the +lengths of inserts such as XSTRING(MAX_NAME_SIZE) are not known. Instead, we +simply count through to the one we want - this isn't a performance issue +because these strings are used only when there is a compilation error. + +Each substring ends with \0 to insert a null character. This includes the final +substring, so that the whole string ends with \0\0, which can be detected when +counting through. */ + +static const char error_texts[] = + "no error\0" + "\\ at end of pattern\0" + "\\c at end of pattern\0" + "unrecognized character follows \\\0" + "numbers out of order in {} quantifier\0" + /* 5 */ + "number too big in {} quantifier\0" + "missing terminating ] for character class\0" + "invalid escape sequence in character class\0" + "range out of order in character class\0" + "nothing to repeat\0" + /* 10 */ + "internal error: invalid forward reference offset\0" + "internal error: unexpected repeat\0" + "unrecognized character after (? or (?-\0" + "POSIX named classes are supported only within a class\0" + "missing )\0" + /* 15 */ + "reference to non-existent subpattern\0" + "erroffset passed as NULL\0" + "unknown option bit(s) set\0" + "missing ) after comment\0" + "parentheses nested too deeply\0" /** DEAD **/ + /* 20 */ + "regular expression is too large\0" + "failed to get memory\0" + "unmatched parentheses\0" + "internal error: code overflow\0" + "unrecognized character after (?<\0" + /* 25 */ + "lookbehind assertion is not fixed length\0" + "malformed number or name after (?(\0" + "conditional group contains more than two branches\0" + "assertion expected after (?( or (?(?C)\0" + "(?R or (?[+-]digits must be followed by )\0" + /* 30 */ + "unknown POSIX class name\0" + "POSIX collating elements are not supported\0" + "this version of PCRE is compiled without UTF support\0" + "spare error\0" /** DEAD **/ + "character value in \\x{} or \\o{} is too large\0" + /* 35 */ + "invalid condition (?(0)\0" + "\\C not allowed in lookbehind assertion\0" + "PCRE does not support \\L, \\l, \\N{name}, \\U, or \\u\0" + "number after (?C is > 255\0" + "closing ) for (?C expected\0" + /* 40 */ + "recursive call could loop indefinitely\0" + "unrecognized character after (?P\0" + "syntax error in subpattern name (missing terminator)\0" + "two named subpatterns have the same name\0" + "invalid UTF-8 string\0" + /* 45 */ + "support for \\P, \\p, and \\X has not been compiled\0" + "malformed \\P or \\p sequence\0" + "unknown property name after \\P or \\p\0" + "subpattern name is too long (maximum " XSTRING(MAX_NAME_SIZE) " characters)\0" + "too many named subpatterns (maximum " XSTRING(MAX_NAME_COUNT) ")\0" + /* 50 */ + "repeated subpattern is too long\0" /** DEAD **/ + "octal value is greater than \\377 in 8-bit non-UTF-8 mode\0" + "internal error: overran compiling workspace\0" + "internal error: previously-checked referenced subpattern not found\0" + "DEFINE group contains more than one branch\0" + /* 55 */ + "repeating a DEFINE group is not allowed\0" /** DEAD **/ + "inconsistent NEWLINE options\0" + "\\g is not followed by a braced, angle-bracketed, or quoted name/number or by a plain number\0" + "a numbered reference must not be zero\0" + "an argument is not allowed for (*ACCEPT), (*FAIL), or (*COMMIT)\0" + /* 60 */ + "(*VERB) not recognized or malformed\0" + "number is too big\0" + "subpattern name expected\0" + "digit expected after (?+\0" + "] is an invalid data character in JavaScript compatibility mode\0" + /* 65 */ + "different names for subpatterns of the same number are not allowed\0" + "(*MARK) must have an argument\0" + "this version of PCRE is not compiled with Unicode property support\0" +#ifndef EBCDIC + "\\c must be followed by an ASCII character\0" +#else + "\\c must be followed by a letter or one of [\\]^_?\0" +#endif + "\\k is not followed by a braced, angle-bracketed, or quoted name\0" + /* 70 */ + "internal error: unknown opcode in find_fixedlength()\0" + "\\N is not supported in a class\0" + "too many forward references\0" + "disallowed Unicode code point (>= 0xd800 && <= 0xdfff)\0" + "invalid UTF-16 string\0" + /* 75 */ + "name is too long in (*MARK), (*PRUNE), (*SKIP), or (*THEN)\0" + "character value in \\u.... sequence is too large\0" + "invalid UTF-32 string\0" + "setting UTF is disabled by the application\0" + "non-hex character in \\x{} (closing brace missing?)\0" + /* 80 */ + "non-octal character in \\o{} (closing brace missing?)\0" + "missing opening brace after \\o\0" + "parentheses are too deeply nested\0" + "invalid range in character class\0" + "group name must start with a non-digit\0" + /* 85 */ + "parentheses are too deeply nested (stack check)\0" + "digits missing in \\x{} or \\o{}\0" + "regular expression is too complicated\0" + ; + +/* Table to identify digits and hex digits. This is used when compiling +patterns. Note that the tables in chartables are dependent on the locale, and +may mark arbitrary characters as digits - but the PCRE compiling code expects +to handle only 0-9, a-z, and A-Z as digits when compiling. That is why we have +a private table here. It costs 256 bytes, but it is a lot faster than doing +character value tests (at least in some simple cases I timed), and in some +applications one wants PCRE to compile efficiently as well as match +efficiently. + +For convenience, we use the same bit definitions as in chartables: + + 0x04 decimal digit + 0x08 hexadecimal digit + +Then we can use ctype_digit and ctype_xdigit in the code. */ + +/* Using a simple comparison for decimal numbers rather than a memory read +is much faster, and the resulting code is simpler (the compiler turns it +into a subtraction and unsigned comparison). */ + +#define IS_DIGIT(x) ((x) >= CHAR_0 && (x) <= CHAR_9) + +#ifndef EBCDIC + +/* This is the "normal" case, for ASCII systems, and EBCDIC systems running in +UTF-8 mode. */ + +static const pcre_uint8 digitab[] = + { + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 0- 7 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 8- 15 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 16- 23 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 24- 31 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* - ' */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* ( - / */ + 0x0c,0x0c,0x0c,0x0c,0x0c,0x0c,0x0c,0x0c, /* 0 - 7 */ + 0x0c,0x0c,0x00,0x00,0x00,0x00,0x00,0x00, /* 8 - ? */ + 0x00,0x08,0x08,0x08,0x08,0x08,0x08,0x00, /* @ - G */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* H - O */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* P - W */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* X - _ */ + 0x00,0x08,0x08,0x08,0x08,0x08,0x08,0x00, /* ` - g */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* h - o */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* p - w */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* x -127 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 128-135 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 136-143 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 144-151 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 152-159 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 160-167 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 168-175 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 176-183 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 184-191 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 192-199 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 200-207 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 208-215 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 216-223 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 224-231 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 232-239 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 240-247 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00};/* 248-255 */ + +#else + +/* This is the "abnormal" case, for EBCDIC systems not running in UTF-8 mode. */ + +static const pcre_uint8 digitab[] = + { + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 0- 7 0 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 8- 15 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 16- 23 10 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 24- 31 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 32- 39 20 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 40- 47 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 48- 55 30 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 56- 63 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* - 71 40 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 72- | */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* & - 87 50 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 88- 95 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* - -103 60 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 104- ? */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 112-119 70 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 120- " */ + 0x00,0x08,0x08,0x08,0x08,0x08,0x08,0x00, /* 128- g 80 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* h -143 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 144- p 90 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* q -159 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 160- x A0 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* y -175 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* ^ -183 B0 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 184-191 */ + 0x00,0x08,0x08,0x08,0x08,0x08,0x08,0x00, /* { - G C0 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* H -207 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* } - P D0 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* Q -223 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* \ - X E0 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* Y -239 */ + 0x0c,0x0c,0x0c,0x0c,0x0c,0x0c,0x0c,0x0c, /* 0 - 7 F0 */ + 0x0c,0x0c,0x00,0x00,0x00,0x00,0x00,0x00};/* 8 -255 */ + +static const pcre_uint8 ebcdic_chartab[] = { /* chartable partial dup */ + 0x80,0x00,0x00,0x00,0x00,0x01,0x00,0x00, /* 0- 7 */ + 0x00,0x00,0x00,0x00,0x01,0x01,0x00,0x00, /* 8- 15 */ + 0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00, /* 16- 23 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 24- 31 */ + 0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00, /* 32- 39 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 40- 47 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 48- 55 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 56- 63 */ + 0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* - 71 */ + 0x00,0x00,0x00,0x80,0x00,0x80,0x80,0x80, /* 72- | */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* & - 87 */ + 0x00,0x00,0x00,0x80,0x80,0x80,0x00,0x00, /* 88- 95 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* - -103 */ + 0x00,0x00,0x00,0x00,0x00,0x10,0x00,0x80, /* 104- ? */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 112-119 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 120- " */ + 0x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /* 128- g */ + 0x12,0x12,0x00,0x00,0x00,0x00,0x00,0x00, /* h -143 */ + 0x00,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* 144- p */ + 0x12,0x12,0x00,0x00,0x00,0x00,0x00,0x00, /* q -159 */ + 0x00,0x00,0x12,0x12,0x12,0x12,0x12,0x12, /* 160- x */ + 0x12,0x12,0x00,0x00,0x00,0x00,0x00,0x00, /* y -175 */ + 0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* ^ -183 */ + 0x00,0x00,0x80,0x00,0x00,0x00,0x00,0x00, /* 184-191 */ + 0x80,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /* { - G */ + 0x12,0x12,0x00,0x00,0x00,0x00,0x00,0x00, /* H -207 */ + 0x00,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* } - P */ + 0x12,0x12,0x00,0x00,0x00,0x00,0x00,0x00, /* Q -223 */ + 0x00,0x00,0x12,0x12,0x12,0x12,0x12,0x12, /* \ - X */ + 0x12,0x12,0x00,0x00,0x00,0x00,0x00,0x00, /* Y -239 */ + 0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c, /* 0 - 7 */ + 0x1c,0x1c,0x00,0x00,0x00,0x00,0x00,0x00};/* 8 -255 */ +#endif + + +/* This table is used to check whether auto-possessification is possible +between adjacent character-type opcodes. The left-hand (repeated) opcode is +used to select the row, and the right-hand opcode is use to select the column. +A value of 1 means that auto-possessification is OK. For example, the second +value in the first row means that \D+\d can be turned into \D++\d. + +The Unicode property types (\P and \p) have to be present to fill out the table +because of what their opcode values are, but the table values should always be +zero because property types are handled separately in the code. The last four +columns apply to items that cannot be repeated, so there is no need to have +rows for them. Note that OP_DIGIT etc. are generated only when PCRE_UCP is +*not* set. When it is set, \d etc. are converted into OP_(NOT_)PROP codes. */ + +#define APTROWS (LAST_AUTOTAB_LEFT_OP - FIRST_AUTOTAB_OP + 1) +#define APTCOLS (LAST_AUTOTAB_RIGHT_OP - FIRST_AUTOTAB_OP + 1) + +static const pcre_uint8 autoposstab[APTROWS][APTCOLS] = { +/* \D \d \S \s \W \w . .+ \C \P \p \R \H \h \V \v \X \Z \z $ $M */ + { 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 }, /* \D */ + { 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1 }, /* \d */ + { 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1 }, /* \S */ + { 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 }, /* \s */ + { 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 }, /* \W */ + { 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1 }, /* \w */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0 }, /* . */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 }, /* .+ */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 }, /* \C */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, /* \P */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, /* \p */ + { 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0 }, /* \R */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0 }, /* \H */ + { 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0 }, /* \h */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0 }, /* \V */ + { 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0 }, /* \v */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0 } /* \X */ +}; + + +/* This table is used to check whether auto-possessification is possible +between adjacent Unicode property opcodes (OP_PROP and OP_NOTPROP). The +left-hand (repeated) opcode is used to select the row, and the right-hand +opcode is used to select the column. The values are as follows: + + 0 Always return FALSE (never auto-possessify) + 1 Character groups are distinct (possessify if both are OP_PROP) + 2 Check character categories in the same group (general or particular) + 3 TRUE if the two opcodes are not the same (PROP vs NOTPROP) + + 4 Check left general category vs right particular category + 5 Check right general category vs left particular category + + 6 Left alphanum vs right general category + 7 Left space vs right general category + 8 Left word vs right general category + + 9 Right alphanum vs left general category + 10 Right space vs left general category + 11 Right word vs left general category + + 12 Left alphanum vs right particular category + 13 Left space vs right particular category + 14 Left word vs right particular category + + 15 Right alphanum vs left particular category + 16 Right space vs left particular category + 17 Right word vs left particular category +*/ + +static const pcre_uint8 propposstab[PT_TABSIZE][PT_TABSIZE] = { +/* ANY LAMP GC PC SC ALNUM SPACE PXSPACE WORD CLIST UCNC */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, /* PT_ANY */ + { 0, 3, 0, 0, 0, 3, 1, 1, 0, 0, 0 }, /* PT_LAMP */ + { 0, 0, 2, 4, 0, 9, 10, 10, 11, 0, 0 }, /* PT_GC */ + { 0, 0, 5, 2, 0, 15, 16, 16, 17, 0, 0 }, /* PT_PC */ + { 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0 }, /* PT_SC */ + { 0, 3, 6, 12, 0, 3, 1, 1, 0, 0, 0 }, /* PT_ALNUM */ + { 0, 1, 7, 13, 0, 1, 3, 3, 1, 0, 0 }, /* PT_SPACE */ + { 0, 1, 7, 13, 0, 1, 3, 3, 1, 0, 0 }, /* PT_PXSPACE */ + { 0, 0, 8, 14, 0, 0, 1, 1, 3, 0, 0 }, /* PT_WORD */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, /* PT_CLIST */ + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3 } /* PT_UCNC */ +}; + +/* This table is used to check whether auto-possessification is possible +between adjacent Unicode property opcodes (OP_PROP and OP_NOTPROP) when one +specifies a general category and the other specifies a particular category. The +row is selected by the general category and the column by the particular +category. The value is 1 if the particular category is not part of the general +category. */ + +static const pcre_uint8 catposstab[7][30] = { +/* Cc Cf Cn Co Cs Ll Lm Lo Lt Lu Mc Me Mn Nd Nl No Pc Pd Pe Pf Pi Po Ps Sc Sk Sm So Zl Zp Zs */ + { 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, /* C */ + { 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, /* L */ + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, /* M */ + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, /* N */ + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1 }, /* P */ + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1 }, /* S */ + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0 } /* Z */ +}; + +/* This table is used when checking ALNUM, (PX)SPACE, SPACE, and WORD against +a general or particular category. The properties in each row are those +that apply to the character set in question. Duplication means that a little +unnecessary work is done when checking, but this keeps things much simpler +because they can all use the same code. For more details see the comment where +this table is used. + +Note: SPACE and PXSPACE used to be different because Perl excluded VT from +"space", but from Perl 5.18 it's included, so both categories are treated the +same here. */ + +static const pcre_uint8 posspropstab[3][4] = { + { ucp_L, ucp_N, ucp_N, ucp_Nl }, /* ALNUM, 3rd and 4th values redundant */ + { ucp_Z, ucp_Z, ucp_C, ucp_Cc }, /* SPACE and PXSPACE, 2nd value redundant */ + { ucp_L, ucp_N, ucp_P, ucp_Po } /* WORD */ +}; + +/* This table is used when converting repeating opcodes into possessified +versions as a result of an explicit possessive quantifier such as ++. A zero +value means there is no possessified version - in those cases the item in +question must be wrapped in ONCE brackets. The table is truncated at OP_CALLOUT +because all relevant opcodes are less than that. */ + +static const pcre_uint8 opcode_possessify[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0 - 15 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 16 - 31 */ + + 0, /* NOTI */ + OP_POSSTAR, 0, /* STAR, MINSTAR */ + OP_POSPLUS, 0, /* PLUS, MINPLUS */ + OP_POSQUERY, 0, /* QUERY, MINQUERY */ + OP_POSUPTO, 0, /* UPTO, MINUPTO */ + 0, /* EXACT */ + 0, 0, 0, 0, /* POS{STAR,PLUS,QUERY,UPTO} */ + + OP_POSSTARI, 0, /* STARI, MINSTARI */ + OP_POSPLUSI, 0, /* PLUSI, MINPLUSI */ + OP_POSQUERYI, 0, /* QUERYI, MINQUERYI */ + OP_POSUPTOI, 0, /* UPTOI, MINUPTOI */ + 0, /* EXACTI */ + 0, 0, 0, 0, /* POS{STARI,PLUSI,QUERYI,UPTOI} */ + + OP_NOTPOSSTAR, 0, /* NOTSTAR, NOTMINSTAR */ + OP_NOTPOSPLUS, 0, /* NOTPLUS, NOTMINPLUS */ + OP_NOTPOSQUERY, 0, /* NOTQUERY, NOTMINQUERY */ + OP_NOTPOSUPTO, 0, /* NOTUPTO, NOTMINUPTO */ + 0, /* NOTEXACT */ + 0, 0, 0, 0, /* NOTPOS{STAR,PLUS,QUERY,UPTO} */ + + OP_NOTPOSSTARI, 0, /* NOTSTARI, NOTMINSTARI */ + OP_NOTPOSPLUSI, 0, /* NOTPLUSI, NOTMINPLUSI */ + OP_NOTPOSQUERYI, 0, /* NOTQUERYI, NOTMINQUERYI */ + OP_NOTPOSUPTOI, 0, /* NOTUPTOI, NOTMINUPTOI */ + 0, /* NOTEXACTI */ + 0, 0, 0, 0, /* NOTPOS{STARI,PLUSI,QUERYI,UPTOI} */ + + OP_TYPEPOSSTAR, 0, /* TYPESTAR, TYPEMINSTAR */ + OP_TYPEPOSPLUS, 0, /* TYPEPLUS, TYPEMINPLUS */ + OP_TYPEPOSQUERY, 0, /* TYPEQUERY, TYPEMINQUERY */ + OP_TYPEPOSUPTO, 0, /* TYPEUPTO, TYPEMINUPTO */ + 0, /* TYPEEXACT */ + 0, 0, 0, 0, /* TYPEPOS{STAR,PLUS,QUERY,UPTO} */ + + OP_CRPOSSTAR, 0, /* CRSTAR, CRMINSTAR */ + OP_CRPOSPLUS, 0, /* CRPLUS, CRMINPLUS */ + OP_CRPOSQUERY, 0, /* CRQUERY, CRMINQUERY */ + OP_CRPOSRANGE, 0, /* CRRANGE, CRMINRANGE */ + 0, 0, 0, 0, /* CRPOS{STAR,PLUS,QUERY,RANGE} */ + + 0, 0, 0, /* CLASS, NCLASS, XCLASS */ + 0, 0, /* REF, REFI */ + 0, 0, /* DNREF, DNREFI */ + 0, 0 /* RECURSE, CALLOUT */ +}; + + + +/************************************************* +* Find an error text * +*************************************************/ + +/* The error texts are now all in one long string, to save on relocations. As +some of the text is of unknown length, we can't use a table of offsets. +Instead, just count through the strings. This is not a performance issue +because it happens only when there has been a compilation error. + +Argument: the error number +Returns: pointer to the error string +*/ + +static const char * +find_error_text(int n) +{ +const char *s = error_texts; +for (; n > 0; n--) + { + while (*s++ != CHAR_NULL) {}; + if (*s == CHAR_NULL) return "Error text not found (please report)"; + } +return s; +} + + + +/************************************************* +* Expand the workspace * +*************************************************/ + +/* This function is called during the second compiling phase, if the number of +forward references fills the existing workspace, which is originally a block on +the stack. A larger block is obtained from malloc() unless the ultimate limit +has been reached or the increase will be rather small. + +Argument: pointer to the compile data block +Returns: 0 if all went well, else an error number +*/ + +static int +expand_workspace(compile_data *cd) +{ +pcre_uchar *newspace; +int newsize = cd->workspace_size * 2; + +if (newsize > COMPILE_WORK_SIZE_MAX) newsize = COMPILE_WORK_SIZE_MAX; +if (cd->workspace_size >= COMPILE_WORK_SIZE_MAX || + newsize - cd->workspace_size < WORK_SIZE_SAFETY_MARGIN) + return ERR72; + +newspace = (PUBL(malloc))(IN_UCHARS(newsize)); +if (newspace == NULL) return ERR21; +memcpy(newspace, cd->start_workspace, cd->workspace_size * sizeof(pcre_uchar)); +cd->hwm = (pcre_uchar *)newspace + (cd->hwm - cd->start_workspace); +if (cd->workspace_size > COMPILE_WORK_SIZE) + (PUBL(free))((void *)cd->start_workspace); +cd->start_workspace = newspace; +cd->workspace_size = newsize; +return 0; +} + + + +/************************************************* +* Check for counted repeat * +*************************************************/ + +/* This function is called when a '{' is encountered in a place where it might +start a quantifier. It looks ahead to see if it really is a quantifier or not. +It is only a quantifier if it is one of the forms {ddd} {ddd,} or {ddd,ddd} +where the ddds are digits. + +Arguments: + p pointer to the first char after '{' + +Returns: TRUE or FALSE +*/ + +static BOOL +is_counted_repeat(const pcre_uchar *p) +{ +if (!IS_DIGIT(*p)) return FALSE; +p++; +while (IS_DIGIT(*p)) p++; +if (*p == CHAR_RIGHT_CURLY_BRACKET) return TRUE; + +if (*p++ != CHAR_COMMA) return FALSE; +if (*p == CHAR_RIGHT_CURLY_BRACKET) return TRUE; + +if (!IS_DIGIT(*p)) return FALSE; +p++; +while (IS_DIGIT(*p)) p++; + +return (*p == CHAR_RIGHT_CURLY_BRACKET); +} + + + +/************************************************* +* Handle escapes * +*************************************************/ + +/* This function is called when a \ has been encountered. It either returns a +positive value for a simple escape such as \n, or 0 for a data character which +will be placed in chptr. A backreference to group n is returned as negative n. +When UTF-8 is enabled, a positive value greater than 255 may be returned in +chptr. On entry, ptr is pointing at the \. On exit, it is on the final +character of the escape sequence. + +Arguments: + ptrptr points to the pattern position pointer + chptr points to a returned data character + errorcodeptr points to the errorcode variable + bracount number of previous extracting brackets + options the options bits + isclass TRUE if inside a character class + +Returns: zero => a data character + positive => a special escape sequence + negative => a back reference + on error, errorcodeptr is set +*/ + +static int +check_escape(const pcre_uchar **ptrptr, pcre_uint32 *chptr, int *errorcodeptr, + int bracount, int options, BOOL isclass) +{ +/* PCRE_UTF16 has the same value as PCRE_UTF8. */ +BOOL utf = (options & PCRE_UTF8) != 0; +const pcre_uchar *ptr = *ptrptr + 1; +pcre_uint32 c; +int escape = 0; +int i; + +GETCHARINCTEST(c, ptr); /* Get character value, increment pointer */ +ptr--; /* Set pointer back to the last byte */ + +/* If backslash is at the end of the pattern, it's an error. */ + +if (c == CHAR_NULL) *errorcodeptr = ERR1; + +/* Non-alphanumerics are literals. For digits or letters, do an initial lookup +in a table. A non-zero result is something that can be returned immediately. +Otherwise further processing may be required. */ + +#ifndef EBCDIC /* ASCII/UTF-8 coding */ +/* Not alphanumeric */ +else if (c < CHAR_0 || c > CHAR_z) {} +else if ((i = escapes[c - CHAR_0]) != 0) + { if (i > 0) c = (pcre_uint32)i; else escape = -i; } + +#else /* EBCDIC coding */ +/* Not alphanumeric */ +else if (c < CHAR_a || (!MAX_255(c) || (ebcdic_chartab[c] & 0x0E) == 0)) {} +else if ((i = escapes[c - 0x48]) != 0) { if (i > 0) c = (pcre_uint32)i; else escape = -i; } +#endif + +/* Escapes that need further processing, or are illegal. */ + +else + { + const pcre_uchar *oldptr; + BOOL braced, negated, overflow; + int s; + + switch (c) + { + /* A number of Perl escapes are not handled by PCRE. We give an explicit + error. */ + + case CHAR_l: + case CHAR_L: + *errorcodeptr = ERR37; + break; + + case CHAR_u: + if ((options & PCRE_JAVASCRIPT_COMPAT) != 0) + { + /* In JavaScript, \u must be followed by four hexadecimal numbers. + Otherwise it is a lowercase u letter. */ + if (MAX_255(ptr[1]) && (digitab[ptr[1]] & ctype_xdigit) != 0 + && MAX_255(ptr[2]) && (digitab[ptr[2]] & ctype_xdigit) != 0 + && MAX_255(ptr[3]) && (digitab[ptr[3]] & ctype_xdigit) != 0 + && MAX_255(ptr[4]) && (digitab[ptr[4]] & ctype_xdigit) != 0) + { + c = 0; + for (i = 0; i < 4; ++i) + { + register pcre_uint32 cc = *(++ptr); +#ifndef EBCDIC /* ASCII/UTF-8 coding */ + if (cc >= CHAR_a) cc -= 32; /* Convert to upper case */ + c = (c << 4) + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10)); +#else /* EBCDIC coding */ + if (cc >= CHAR_a && cc <= CHAR_z) cc += 64; /* Convert to upper case */ + c = (c << 4) + cc - ((cc >= CHAR_0)? CHAR_0 : (CHAR_A - 10)); +#endif + } + +#if defined COMPILE_PCRE8 + if (c > (utf ? 0x10ffffU : 0xffU)) +#elif defined COMPILE_PCRE16 + if (c > (utf ? 0x10ffffU : 0xffffU)) +#elif defined COMPILE_PCRE32 + if (utf && c > 0x10ffffU) +#endif + { + *errorcodeptr = ERR76; + } + else if (utf && c >= 0xd800 && c <= 0xdfff) *errorcodeptr = ERR73; + } + } + else + *errorcodeptr = ERR37; + break; + + case CHAR_U: + /* In JavaScript, \U is an uppercase U letter. */ + if ((options & PCRE_JAVASCRIPT_COMPAT) == 0) *errorcodeptr = ERR37; + break; + + /* In a character class, \g is just a literal "g". Outside a character + class, \g must be followed by one of a number of specific things: + + (1) A number, either plain or braced. If positive, it is an absolute + backreference. If negative, it is a relative backreference. This is a Perl + 5.10 feature. + + (2) Perl 5.10 also supports \g{name} as a reference to a named group. This + is part of Perl's movement towards a unified syntax for back references. As + this is synonymous with \k{name}, we fudge it up by pretending it really + was \k. + + (3) For Oniguruma compatibility we also support \g followed by a name or a + number either in angle brackets or in single quotes. However, these are + (possibly recursive) subroutine calls, _not_ backreferences. Just return + the ESC_g code (cf \k). */ + + case CHAR_g: + if (isclass) break; + if (ptr[1] == CHAR_LESS_THAN_SIGN || ptr[1] == CHAR_APOSTROPHE) + { + escape = ESC_g; + break; + } + + /* Handle the Perl-compatible cases */ + + if (ptr[1] == CHAR_LEFT_CURLY_BRACKET) + { + const pcre_uchar *p; + for (p = ptr+2; *p != CHAR_NULL && *p != CHAR_RIGHT_CURLY_BRACKET; p++) + if (*p != CHAR_MINUS && !IS_DIGIT(*p)) break; + if (*p != CHAR_NULL && *p != CHAR_RIGHT_CURLY_BRACKET) + { + escape = ESC_k; + break; + } + braced = TRUE; + ptr++; + } + else braced = FALSE; + + if (ptr[1] == CHAR_MINUS) + { + negated = TRUE; + ptr++; + } + else negated = FALSE; + + /* The integer range is limited by the machine's int representation. */ + s = 0; + overflow = FALSE; + while (IS_DIGIT(ptr[1])) + { + if (s > INT_MAX / 10 - 1) /* Integer overflow */ + { + overflow = TRUE; + break; + } + s = s * 10 + (int)(*(++ptr) - CHAR_0); + } + if (overflow) /* Integer overflow */ + { + while (IS_DIGIT(ptr[1])) + ptr++; + *errorcodeptr = ERR61; + break; + } + + if (braced && *(++ptr) != CHAR_RIGHT_CURLY_BRACKET) + { + *errorcodeptr = ERR57; + break; + } + + if (s == 0) + { + *errorcodeptr = ERR58; + break; + } + + if (negated) + { + if (s > bracount) + { + *errorcodeptr = ERR15; + break; + } + s = bracount - (s - 1); + } + + escape = -s; + break; + + /* The handling of escape sequences consisting of a string of digits + starting with one that is not zero is not straightforward. Perl has changed + over the years. Nowadays \g{} for backreferences and \o{} for octal are + recommended to avoid the ambiguities in the old syntax. + + Outside a character class, the digits are read as a decimal number. If the + number is less than 8 (used to be 10), or if there are that many previous + extracting left brackets, then it is a back reference. Otherwise, up to + three octal digits are read to form an escaped byte. Thus \123 is likely to + be octal 123 (cf \0123, which is octal 012 followed by the literal 3). If + the octal value is greater than 377, the least significant 8 bits are + taken. \8 and \9 are treated as the literal characters 8 and 9. + + Inside a character class, \ followed by a digit is always either a literal + 8 or 9 or an octal number. */ + + case CHAR_1: case CHAR_2: case CHAR_3: case CHAR_4: case CHAR_5: + case CHAR_6: case CHAR_7: case CHAR_8: case CHAR_9: + + if (!isclass) + { + oldptr = ptr; + /* The integer range is limited by the machine's int representation. */ + s = (int)(c -CHAR_0); + overflow = FALSE; + while (IS_DIGIT(ptr[1])) + { + if (s > INT_MAX / 10 - 1) /* Integer overflow */ + { + overflow = TRUE; + break; + } + s = s * 10 + (int)(*(++ptr) - CHAR_0); + } + if (overflow) /* Integer overflow */ + { + while (IS_DIGIT(ptr[1])) + ptr++; + *errorcodeptr = ERR61; + break; + } + if (s < 8 || s <= bracount) /* Check for back reference */ + { + escape = -s; + break; + } + ptr = oldptr; /* Put the pointer back and fall through */ + } + + /* Handle a digit following \ when the number is not a back reference. If + the first digit is 8 or 9, Perl used to generate a binary zero byte and + then treat the digit as a following literal. At least by Perl 5.18 this + changed so as not to insert the binary zero. */ + + if ((c = *ptr) >= CHAR_8) break; + + /* Fall through with a digit less than 8 */ + + /* \0 always starts an octal number, but we may drop through to here with a + larger first octal digit. The original code used just to take the least + significant 8 bits of octal numbers (I think this is what early Perls used + to do). Nowadays we allow for larger numbers in UTF-8 mode and 16-bit mode, + but no more than 3 octal digits. */ + + case CHAR_0: + c -= CHAR_0; + while(i++ < 2 && ptr[1] >= CHAR_0 && ptr[1] <= CHAR_7) + c = c * 8 + *(++ptr) - CHAR_0; +#ifdef COMPILE_PCRE8 + if (!utf && c > 0xff) *errorcodeptr = ERR51; +#endif + break; + + /* \o is a relatively new Perl feature, supporting a more general way of + specifying character codes in octal. The only supported form is \o{ddd}. */ + + case CHAR_o: + if (ptr[1] != CHAR_LEFT_CURLY_BRACKET) *errorcodeptr = ERR81; else + if (ptr[2] == CHAR_RIGHT_CURLY_BRACKET) *errorcodeptr = ERR86; else + { + ptr += 2; + c = 0; + overflow = FALSE; + while (*ptr >= CHAR_0 && *ptr <= CHAR_7) + { + register pcre_uint32 cc = *ptr++; + if (c == 0 && cc == CHAR_0) continue; /* Leading zeroes */ +#ifdef COMPILE_PCRE32 + if (c >= 0x20000000l) { overflow = TRUE; break; } +#endif + c = (c << 3) + cc - CHAR_0 ; +#if defined COMPILE_PCRE8 + if (c > (utf ? 0x10ffffU : 0xffU)) { overflow = TRUE; break; } +#elif defined COMPILE_PCRE16 + if (c > (utf ? 0x10ffffU : 0xffffU)) { overflow = TRUE; break; } +#elif defined COMPILE_PCRE32 + if (utf && c > 0x10ffffU) { overflow = TRUE; break; } +#endif + } + if (overflow) + { + while (*ptr >= CHAR_0 && *ptr <= CHAR_7) ptr++; + *errorcodeptr = ERR34; + } + else if (*ptr == CHAR_RIGHT_CURLY_BRACKET) + { + if (utf && c >= 0xd800 && c <= 0xdfff) *errorcodeptr = ERR73; + } + else *errorcodeptr = ERR80; + } + break; + + /* \x is complicated. In JavaScript, \x must be followed by two hexadecimal + numbers. Otherwise it is a lowercase x letter. */ + + case CHAR_x: + if ((options & PCRE_JAVASCRIPT_COMPAT) != 0) + { + if (MAX_255(ptr[1]) && (digitab[ptr[1]] & ctype_xdigit) != 0 + && MAX_255(ptr[2]) && (digitab[ptr[2]] & ctype_xdigit) != 0) + { + c = 0; + for (i = 0; i < 2; ++i) + { + register pcre_uint32 cc = *(++ptr); +#ifndef EBCDIC /* ASCII/UTF-8 coding */ + if (cc >= CHAR_a) cc -= 32; /* Convert to upper case */ + c = (c << 4) + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10)); +#else /* EBCDIC coding */ + if (cc >= CHAR_a && cc <= CHAR_z) cc += 64; /* Convert to upper case */ + c = (c << 4) + cc - ((cc >= CHAR_0)? CHAR_0 : (CHAR_A - 10)); +#endif + } + } + } /* End JavaScript handling */ + + /* Handle \x in Perl's style. \x{ddd} is a character number which can be + greater than 0xff in utf or non-8bit mode, but only if the ddd are hex + digits. If not, { used to be treated as a data character. However, Perl + seems to read hex digits up to the first non-such, and ignore the rest, so + that, for example \x{zz} matches a binary zero. This seems crazy, so PCRE + now gives an error. */ + + else + { + if (ptr[1] == CHAR_LEFT_CURLY_BRACKET) + { + ptr += 2; + if (*ptr == CHAR_RIGHT_CURLY_BRACKET) + { + *errorcodeptr = ERR86; + break; + } + c = 0; + overflow = FALSE; + while (MAX_255(*ptr) && (digitab[*ptr] & ctype_xdigit) != 0) + { + register pcre_uint32 cc = *ptr++; + if (c == 0 && cc == CHAR_0) continue; /* Leading zeroes */ + +#ifdef COMPILE_PCRE32 + if (c >= 0x10000000l) { overflow = TRUE; break; } +#endif + +#ifndef EBCDIC /* ASCII/UTF-8 coding */ + if (cc >= CHAR_a) cc -= 32; /* Convert to upper case */ + c = (c << 4) + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10)); +#else /* EBCDIC coding */ + if (cc >= CHAR_a && cc <= CHAR_z) cc += 64; /* Convert to upper case */ + c = (c << 4) + cc - ((cc >= CHAR_0)? CHAR_0 : (CHAR_A - 10)); +#endif + +#if defined COMPILE_PCRE8 + if (c > (utf ? 0x10ffffU : 0xffU)) { overflow = TRUE; break; } +#elif defined COMPILE_PCRE16 + if (c > (utf ? 0x10ffffU : 0xffffU)) { overflow = TRUE; break; } +#elif defined COMPILE_PCRE32 + if (utf && c > 0x10ffffU) { overflow = TRUE; break; } +#endif + } + + if (overflow) + { + while (MAX_255(*ptr) && (digitab[*ptr] & ctype_xdigit) != 0) ptr++; + *errorcodeptr = ERR34; + } + + else if (*ptr == CHAR_RIGHT_CURLY_BRACKET) + { + if (utf && c >= 0xd800 && c <= 0xdfff) *errorcodeptr = ERR73; + } + + /* If the sequence of hex digits does not end with '}', give an error. + We used just to recognize this construct and fall through to the normal + \x handling, but nowadays Perl gives an error, which seems much more + sensible, so we do too. */ + + else *errorcodeptr = ERR79; + } /* End of \x{} processing */ + + /* Read a single-byte hex-defined char (up to two hex digits after \x) */ + + else + { + c = 0; + while (i++ < 2 && MAX_255(ptr[1]) && (digitab[ptr[1]] & ctype_xdigit) != 0) + { + pcre_uint32 cc; /* Some compilers don't like */ + cc = *(++ptr); /* ++ in initializers */ +#ifndef EBCDIC /* ASCII/UTF-8 coding */ + if (cc >= CHAR_a) cc -= 32; /* Convert to upper case */ + c = c * 16 + cc - ((cc < CHAR_A)? CHAR_0 : (CHAR_A - 10)); +#else /* EBCDIC coding */ + if (cc <= CHAR_z) cc += 64; /* Convert to upper case */ + c = c * 16 + cc - ((cc >= CHAR_0)? CHAR_0 : (CHAR_A - 10)); +#endif + } + } /* End of \xdd handling */ + } /* End of Perl-style \x handling */ + break; + + /* For \c, a following letter is upper-cased; then the 0x40 bit is flipped. + An error is given if the byte following \c is not an ASCII character. This + coding is ASCII-specific, but then the whole concept of \cx is + ASCII-specific. (However, an EBCDIC equivalent has now been added.) */ + + case CHAR_c: + c = *(++ptr); + if (c == CHAR_NULL) + { + *errorcodeptr = ERR2; + break; + } +#ifndef EBCDIC /* ASCII/UTF-8 coding */ + if (c > 127) /* Excludes all non-ASCII in either mode */ + { + *errorcodeptr = ERR68; + break; + } + if (c >= CHAR_a && c <= CHAR_z) c -= 32; + c ^= 0x40; +#else /* EBCDIC coding */ + if (c >= CHAR_a && c <= CHAR_z) c += 64; + if (c == CHAR_QUESTION_MARK) + c = ('\\' == 188 && '`' == 74)? 0x5f : 0xff; + else + { + for (i = 0; i < 32; i++) + { + if (c == ebcdic_escape_c[i]) break; + } + if (i < 32) c = i; else *errorcodeptr = ERR68; + } +#endif + break; + + /* PCRE_EXTRA enables extensions to Perl in the matter of escapes. Any + other alphanumeric following \ is an error if PCRE_EXTRA was set; + otherwise, for Perl compatibility, it is a literal. This code looks a bit + odd, but there used to be some cases other than the default, and there may + be again in future, so I haven't "optimized" it. */ + + default: + if ((options & PCRE_EXTRA) != 0) switch(c) + { + default: + *errorcodeptr = ERR3; + break; + } + break; + } + } + +/* Perl supports \N{name} for character names, as well as plain \N for "not +newline". PCRE does not support \N{name}. However, it does support +quantification such as \N{2,3}. */ + +if (escape == ESC_N && ptr[1] == CHAR_LEFT_CURLY_BRACKET && + !is_counted_repeat(ptr+2)) + *errorcodeptr = ERR37; + +/* If PCRE_UCP is set, we change the values for \d etc. */ + +if ((options & PCRE_UCP) != 0 && escape >= ESC_D && escape <= ESC_w) + escape += (ESC_DU - ESC_D); + +/* Set the pointer to the final character before returning. */ + +*ptrptr = ptr; +*chptr = c; +return escape; +} + + + +#ifdef SUPPORT_UCP +/************************************************* +* Handle \P and \p * +*************************************************/ + +/* This function is called after \P or \p has been encountered, provided that +PCRE is compiled with support for Unicode properties. On entry, ptrptr is +pointing at the P or p. On exit, it is pointing at the final character of the +escape sequence. + +Argument: + ptrptr points to the pattern position pointer + negptr points to a boolean that is set TRUE for negation else FALSE + ptypeptr points to an unsigned int that is set to the type value + pdataptr points to an unsigned int that is set to the detailed property value + errorcodeptr points to the error code variable + +Returns: TRUE if the type value was found, or FALSE for an invalid type +*/ + +static BOOL +get_ucp(const pcre_uchar **ptrptr, BOOL *negptr, unsigned int *ptypeptr, + unsigned int *pdataptr, int *errorcodeptr) +{ +pcre_uchar c; +int i, bot, top; +const pcre_uchar *ptr = *ptrptr; +pcre_uchar name[32]; + +c = *(++ptr); +if (c == CHAR_NULL) goto ERROR_RETURN; + +*negptr = FALSE; + +/* \P or \p can be followed by a name in {}, optionally preceded by ^ for +negation. */ + +if (c == CHAR_LEFT_CURLY_BRACKET) + { + if (ptr[1] == CHAR_CIRCUMFLEX_ACCENT) + { + *negptr = TRUE; + ptr++; + } + for (i = 0; i < (int)(sizeof(name) / sizeof(pcre_uchar)) - 1; i++) + { + c = *(++ptr); + if (c == CHAR_NULL) goto ERROR_RETURN; + if (c == CHAR_RIGHT_CURLY_BRACKET) break; + name[i] = c; + } + if (c != CHAR_RIGHT_CURLY_BRACKET) goto ERROR_RETURN; + name[i] = 0; + } + +/* Otherwise there is just one following character */ + +else + { + name[0] = c; + name[1] = 0; + } + +*ptrptr = ptr; + +/* Search for a recognized property name using binary chop */ + +bot = 0; +top = PRIV(utt_size); + +while (bot < top) + { + int r; + i = (bot + top) >> 1; + r = STRCMP_UC_C8(name, PRIV(utt_names) + PRIV(utt)[i].name_offset); + if (r == 0) + { + *ptypeptr = PRIV(utt)[i].type; + *pdataptr = PRIV(utt)[i].value; + return TRUE; + } + if (r > 0) bot = i + 1; else top = i; + } + +*errorcodeptr = ERR47; +*ptrptr = ptr; +return FALSE; + +ERROR_RETURN: +*errorcodeptr = ERR46; +*ptrptr = ptr; +return FALSE; +} +#endif + + + +/************************************************* +* Read repeat counts * +*************************************************/ + +/* Read an item of the form {n,m} and return the values. This is called only +after is_counted_repeat() has confirmed that a repeat-count quantifier exists, +so the syntax is guaranteed to be correct, but we need to check the values. + +Arguments: + p pointer to first char after '{' + minp pointer to int for min + maxp pointer to int for max + returned as -1 if no max + errorcodeptr points to error code variable + +Returns: pointer to '}' on success; + current ptr on error, with errorcodeptr set non-zero +*/ + +static const pcre_uchar * +read_repeat_counts(const pcre_uchar *p, int *minp, int *maxp, int *errorcodeptr) +{ +int min = 0; +int max = -1; + +while (IS_DIGIT(*p)) + { + min = min * 10 + (int)(*p++ - CHAR_0); + if (min > 65535) + { + *errorcodeptr = ERR5; + return p; + } + } + +if (*p == CHAR_RIGHT_CURLY_BRACKET) max = min; else + { + if (*(++p) != CHAR_RIGHT_CURLY_BRACKET) + { + max = 0; + while(IS_DIGIT(*p)) + { + max = max * 10 + (int)(*p++ - CHAR_0); + if (max > 65535) + { + *errorcodeptr = ERR5; + return p; + } + } + if (max < min) + { + *errorcodeptr = ERR4; + return p; + } + } + } + +*minp = min; +*maxp = max; +return p; +} + + + +/************************************************* +* Find first significant op code * +*************************************************/ + +/* This is called by several functions that scan a compiled expression looking +for a fixed first character, or an anchoring op code etc. It skips over things +that do not influence this. For some calls, it makes sense to skip negative +forward and all backward assertions, and also the \b assertion; for others it +does not. + +Arguments: + code pointer to the start of the group + skipassert TRUE if certain assertions are to be skipped + +Returns: pointer to the first significant opcode +*/ + +static const pcre_uchar* +first_significant_code(const pcre_uchar *code, BOOL skipassert) +{ +for (;;) + { + switch ((int)*code) + { + case OP_ASSERT_NOT: + case OP_ASSERTBACK: + case OP_ASSERTBACK_NOT: + if (!skipassert) return code; + do code += GET(code, 1); while (*code == OP_ALT); + code += PRIV(OP_lengths)[*code]; + break; + + case OP_WORD_BOUNDARY: + case OP_NOT_WORD_BOUNDARY: + if (!skipassert) return code; + /* Fall through */ + + case OP_CALLOUT: + case OP_CREF: + case OP_DNCREF: + case OP_RREF: + case OP_DNRREF: + case OP_DEF: + code += PRIV(OP_lengths)[*code]; + break; + + default: + return code; + } + } +/* Control never reaches here */ +} + + + +/************************************************* +* Find the fixed length of a branch * +*************************************************/ + +/* Scan a branch and compute the fixed length of subject that will match it, +if the length is fixed. This is needed for dealing with backward assertions. +In UTF8 mode, the result is in characters rather than bytes. The branch is +temporarily terminated with OP_END when this function is called. + +This function is called when a backward assertion is encountered, so that if it +fails, the error message can point to the correct place in the pattern. +However, we cannot do this when the assertion contains subroutine calls, +because they can be forward references. We solve this by remembering this case +and doing the check at the end; a flag specifies which mode we are running in. + +Arguments: + code points to the start of the pattern (the bracket) + utf TRUE in UTF-8 / UTF-16 / UTF-32 mode + atend TRUE if called when the pattern is complete + cd the "compile data" structure + recurses chain of recurse_check to catch mutual recursion + +Returns: the fixed length, + or -1 if there is no fixed length, + or -2 if \C was encountered (in UTF-8 mode only) + or -3 if an OP_RECURSE item was encountered and atend is FALSE + or -4 if an unknown opcode was encountered (internal error) +*/ + +static int +find_fixedlength(pcre_uchar *code, BOOL utf, BOOL atend, compile_data *cd, + recurse_check *recurses) +{ +int length = -1; +recurse_check this_recurse; +register int branchlength = 0; +register pcre_uchar *cc = code + 1 + LINK_SIZE; + +/* Scan along the opcodes for this branch. If we get to the end of the +branch, check the length against that of the other branches. */ + +for (;;) + { + int d; + pcre_uchar *ce, *cs; + register pcre_uchar op = *cc; + + switch (op) + { + /* We only need to continue for OP_CBRA (normal capturing bracket) and + OP_BRA (normal non-capturing bracket) because the other variants of these + opcodes are all concerned with unlimited repeated groups, which of course + are not of fixed length. */ + + case OP_CBRA: + case OP_BRA: + case OP_ONCE: + case OP_ONCE_NC: + case OP_COND: + d = find_fixedlength(cc + ((op == OP_CBRA)? IMM2_SIZE : 0), utf, atend, cd, + recurses); + if (d < 0) return d; + branchlength += d; + do cc += GET(cc, 1); while (*cc == OP_ALT); + cc += 1 + LINK_SIZE; + break; + + /* Reached end of a branch; if it's a ket it is the end of a nested call. + If it's ALT it is an alternation in a nested call. An ACCEPT is effectively + an ALT. If it is END it's the end of the outer call. All can be handled by + the same code. Note that we must not include the OP_KETRxxx opcodes here, + because they all imply an unlimited repeat. */ + + case OP_ALT: + case OP_KET: + case OP_END: + case OP_ACCEPT: + case OP_ASSERT_ACCEPT: + if (length < 0) length = branchlength; + else if (length != branchlength) return -1; + if (*cc != OP_ALT) return length; + cc += 1 + LINK_SIZE; + branchlength = 0; + break; + + /* A true recursion implies not fixed length, but a subroutine call may + be OK. If the subroutine is a forward reference, we can't deal with + it until the end of the pattern, so return -3. */ + + case OP_RECURSE: + if (!atend) return -3; + cs = ce = (pcre_uchar *)cd->start_code + GET(cc, 1); /* Start subpattern */ + do ce += GET(ce, 1); while (*ce == OP_ALT); /* End subpattern */ + if (cc > cs && cc < ce) return -1; /* Recursion */ + else /* Check for mutual recursion */ + { + recurse_check *r = recurses; + for (r = recurses; r != NULL; r = r->prev) if (r->group == cs) break; + if (r != NULL) return -1; /* Mutual recursion */ + } + this_recurse.prev = recurses; + this_recurse.group = cs; + d = find_fixedlength(cs + IMM2_SIZE, utf, atend, cd, &this_recurse); + if (d < 0) return d; + branchlength += d; + cc += 1 + LINK_SIZE; + break; + + /* Skip over assertive subpatterns */ + + case OP_ASSERT: + case OP_ASSERT_NOT: + case OP_ASSERTBACK: + case OP_ASSERTBACK_NOT: + do cc += GET(cc, 1); while (*cc == OP_ALT); + cc += 1 + LINK_SIZE; + break; + + /* Skip over things that don't match chars */ + + case OP_MARK: + case OP_PRUNE_ARG: + case OP_SKIP_ARG: + case OP_THEN_ARG: + cc += cc[1] + PRIV(OP_lengths)[*cc]; + break; + + case OP_CALLOUT: + case OP_CIRC: + case OP_CIRCM: + case OP_CLOSE: + case OP_COMMIT: + case OP_CREF: + case OP_DEF: + case OP_DNCREF: + case OP_DNRREF: + case OP_DOLL: + case OP_DOLLM: + case OP_EOD: + case OP_EODN: + case OP_FAIL: + case OP_NOT_WORD_BOUNDARY: + case OP_PRUNE: + case OP_REVERSE: + case OP_RREF: + case OP_SET_SOM: + case OP_SKIP: + case OP_SOD: + case OP_SOM: + case OP_THEN: + case OP_WORD_BOUNDARY: + cc += PRIV(OP_lengths)[*cc]; + break; + + /* Handle literal characters */ + + case OP_CHAR: + case OP_CHARI: + case OP_NOT: + case OP_NOTI: + branchlength++; + cc += 2; +#ifdef SUPPORT_UTF + if (utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); +#endif + break; + + /* Handle exact repetitions. The count is already in characters, but we + need to skip over a multibyte character in UTF8 mode. */ + + case OP_EXACT: + case OP_EXACTI: + case OP_NOTEXACT: + case OP_NOTEXACTI: + branchlength += (int)GET2(cc,1); + cc += 2 + IMM2_SIZE; +#ifdef SUPPORT_UTF + if (utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); +#endif + break; + + case OP_TYPEEXACT: + branchlength += GET2(cc,1); + if (cc[1 + IMM2_SIZE] == OP_PROP || cc[1 + IMM2_SIZE] == OP_NOTPROP) + cc += 2; + cc += 1 + IMM2_SIZE + 1; + break; + + /* Handle single-char matchers */ + + case OP_PROP: + case OP_NOTPROP: + cc += 2; + /* Fall through */ + + case OP_HSPACE: + case OP_VSPACE: + case OP_NOT_HSPACE: + case OP_NOT_VSPACE: + case OP_NOT_DIGIT: + case OP_DIGIT: + case OP_NOT_WHITESPACE: + case OP_WHITESPACE: + case OP_NOT_WORDCHAR: + case OP_WORDCHAR: + case OP_ANY: + case OP_ALLANY: + branchlength++; + cc++; + break; + + /* The single-byte matcher isn't allowed. This only happens in UTF-8 mode; + otherwise \C is coded as OP_ALLANY. */ + + case OP_ANYBYTE: + return -2; + + /* Check a class for variable quantification */ + + case OP_CLASS: + case OP_NCLASS: +#if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + case OP_XCLASS: + /* The original code caused an unsigned overflow in 64 bit systems, + so now we use a conditional statement. */ + if (op == OP_XCLASS) + cc += GET(cc, 1); + else + cc += PRIV(OP_lengths)[OP_CLASS]; +#else + cc += PRIV(OP_lengths)[OP_CLASS]; +#endif + + switch (*cc) + { + case OP_CRSTAR: + case OP_CRMINSTAR: + case OP_CRPLUS: + case OP_CRMINPLUS: + case OP_CRQUERY: + case OP_CRMINQUERY: + case OP_CRPOSSTAR: + case OP_CRPOSPLUS: + case OP_CRPOSQUERY: + return -1; + + case OP_CRRANGE: + case OP_CRMINRANGE: + case OP_CRPOSRANGE: + if (GET2(cc,1) != GET2(cc,1+IMM2_SIZE)) return -1; + branchlength += (int)GET2(cc,1); + cc += 1 + 2 * IMM2_SIZE; + break; + + default: + branchlength++; + } + break; + + /* Anything else is variable length */ + + case OP_ANYNL: + case OP_BRAMINZERO: + case OP_BRAPOS: + case OP_BRAPOSZERO: + case OP_BRAZERO: + case OP_CBRAPOS: + case OP_EXTUNI: + case OP_KETRMAX: + case OP_KETRMIN: + case OP_KETRPOS: + case OP_MINPLUS: + case OP_MINPLUSI: + case OP_MINQUERY: + case OP_MINQUERYI: + case OP_MINSTAR: + case OP_MINSTARI: + case OP_MINUPTO: + case OP_MINUPTOI: + case OP_NOTMINPLUS: + case OP_NOTMINPLUSI: + case OP_NOTMINQUERY: + case OP_NOTMINQUERYI: + case OP_NOTMINSTAR: + case OP_NOTMINSTARI: + case OP_NOTMINUPTO: + case OP_NOTMINUPTOI: + case OP_NOTPLUS: + case OP_NOTPLUSI: + case OP_NOTPOSPLUS: + case OP_NOTPOSPLUSI: + case OP_NOTPOSQUERY: + case OP_NOTPOSQUERYI: + case OP_NOTPOSSTAR: + case OP_NOTPOSSTARI: + case OP_NOTPOSUPTO: + case OP_NOTPOSUPTOI: + case OP_NOTQUERY: + case OP_NOTQUERYI: + case OP_NOTSTAR: + case OP_NOTSTARI: + case OP_NOTUPTO: + case OP_NOTUPTOI: + case OP_PLUS: + case OP_PLUSI: + case OP_POSPLUS: + case OP_POSPLUSI: + case OP_POSQUERY: + case OP_POSQUERYI: + case OP_POSSTAR: + case OP_POSSTARI: + case OP_POSUPTO: + case OP_POSUPTOI: + case OP_QUERY: + case OP_QUERYI: + case OP_REF: + case OP_REFI: + case OP_DNREF: + case OP_DNREFI: + case OP_SBRA: + case OP_SBRAPOS: + case OP_SCBRA: + case OP_SCBRAPOS: + case OP_SCOND: + case OP_SKIPZERO: + case OP_STAR: + case OP_STARI: + case OP_TYPEMINPLUS: + case OP_TYPEMINQUERY: + case OP_TYPEMINSTAR: + case OP_TYPEMINUPTO: + case OP_TYPEPLUS: + case OP_TYPEPOSPLUS: + case OP_TYPEPOSQUERY: + case OP_TYPEPOSSTAR: + case OP_TYPEPOSUPTO: + case OP_TYPEQUERY: + case OP_TYPESTAR: + case OP_TYPEUPTO: + case OP_UPTO: + case OP_UPTOI: + return -1; + + /* Catch unrecognized opcodes so that when new ones are added they + are not forgotten, as has happened in the past. */ + + default: + return -4; + } + } +/* Control never gets here */ +} + + + +/************************************************* +* Scan compiled regex for specific bracket * +*************************************************/ + +/* This little function scans through a compiled pattern until it finds a +capturing bracket with the given number, or, if the number is negative, an +instance of OP_REVERSE for a lookbehind. The function is global in the C sense +so that it can be called from pcre_study() when finding the minimum matching +length. + +Arguments: + code points to start of expression + utf TRUE in UTF-8 / UTF-16 / UTF-32 mode + number the required bracket number or negative to find a lookbehind + +Returns: pointer to the opcode for the bracket, or NULL if not found +*/ + +const pcre_uchar * +PRIV(find_bracket)(const pcre_uchar *code, BOOL utf, int number) +{ +for (;;) + { + register pcre_uchar c = *code; + + if (c == OP_END) return NULL; + + /* XCLASS is used for classes that cannot be represented just by a bit + map. This includes negated single high-valued characters. The length in + the table is zero; the actual length is stored in the compiled code. */ + + if (c == OP_XCLASS) code += GET(code, 1); + + /* Handle recursion */ + + else if (c == OP_REVERSE) + { + if (number < 0) return (pcre_uchar *)code; + code += PRIV(OP_lengths)[c]; + } + + /* Handle capturing bracket */ + + else if (c == OP_CBRA || c == OP_SCBRA || + c == OP_CBRAPOS || c == OP_SCBRAPOS) + { + int n = (int)GET2(code, 1+LINK_SIZE); + if (n == number) return (pcre_uchar *)code; + code += PRIV(OP_lengths)[c]; + } + + /* Otherwise, we can get the item's length from the table, except that for + repeated character types, we have to test for \p and \P, which have an extra + two bytes of parameters, and for MARK/PRUNE/SKIP/THEN with an argument, we + must add in its length. */ + + else + { + switch(c) + { + case OP_TYPESTAR: + case OP_TYPEMINSTAR: + case OP_TYPEPLUS: + case OP_TYPEMINPLUS: + case OP_TYPEQUERY: + case OP_TYPEMINQUERY: + case OP_TYPEPOSSTAR: + case OP_TYPEPOSPLUS: + case OP_TYPEPOSQUERY: + if (code[1] == OP_PROP || code[1] == OP_NOTPROP) code += 2; + break; + + case OP_TYPEUPTO: + case OP_TYPEMINUPTO: + case OP_TYPEEXACT: + case OP_TYPEPOSUPTO: + if (code[1 + IMM2_SIZE] == OP_PROP || code[1 + IMM2_SIZE] == OP_NOTPROP) + code += 2; + break; + + case OP_MARK: + case OP_PRUNE_ARG: + case OP_SKIP_ARG: + case OP_THEN_ARG: + code += code[1]; + break; + } + + /* Add in the fixed length from the table */ + + code += PRIV(OP_lengths)[c]; + + /* In UTF-8 mode, opcodes that are followed by a character may be followed by + a multi-byte character. The length in the table is a minimum, so we have to + arrange to skip the extra bytes. */ + +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (utf) switch(c) + { + case OP_CHAR: + case OP_CHARI: + case OP_NOT: + case OP_NOTI: + case OP_EXACT: + case OP_EXACTI: + case OP_NOTEXACT: + case OP_NOTEXACTI: + case OP_UPTO: + case OP_UPTOI: + case OP_NOTUPTO: + case OP_NOTUPTOI: + case OP_MINUPTO: + case OP_MINUPTOI: + case OP_NOTMINUPTO: + case OP_NOTMINUPTOI: + case OP_POSUPTO: + case OP_POSUPTOI: + case OP_NOTPOSUPTO: + case OP_NOTPOSUPTOI: + case OP_STAR: + case OP_STARI: + case OP_NOTSTAR: + case OP_NOTSTARI: + case OP_MINSTAR: + case OP_MINSTARI: + case OP_NOTMINSTAR: + case OP_NOTMINSTARI: + case OP_POSSTAR: + case OP_POSSTARI: + case OP_NOTPOSSTAR: + case OP_NOTPOSSTARI: + case OP_PLUS: + case OP_PLUSI: + case OP_NOTPLUS: + case OP_NOTPLUSI: + case OP_MINPLUS: + case OP_MINPLUSI: + case OP_NOTMINPLUS: + case OP_NOTMINPLUSI: + case OP_POSPLUS: + case OP_POSPLUSI: + case OP_NOTPOSPLUS: + case OP_NOTPOSPLUSI: + case OP_QUERY: + case OP_QUERYI: + case OP_NOTQUERY: + case OP_NOTQUERYI: + case OP_MINQUERY: + case OP_MINQUERYI: + case OP_NOTMINQUERY: + case OP_NOTMINQUERYI: + case OP_POSQUERY: + case OP_POSQUERYI: + case OP_NOTPOSQUERY: + case OP_NOTPOSQUERYI: + if (HAS_EXTRALEN(code[-1])) code += GET_EXTRALEN(code[-1]); + break; + } +#else + (void)(utf); /* Keep compiler happy by referencing function argument */ +#endif + } + } +} + + + +/************************************************* +* Scan compiled regex for recursion reference * +*************************************************/ + +/* This little function scans through a compiled pattern until it finds an +instance of OP_RECURSE. + +Arguments: + code points to start of expression + utf TRUE in UTF-8 / UTF-16 / UTF-32 mode + +Returns: pointer to the opcode for OP_RECURSE, or NULL if not found +*/ + +static const pcre_uchar * +find_recurse(const pcre_uchar *code, BOOL utf) +{ +for (;;) + { + register pcre_uchar c = *code; + if (c == OP_END) return NULL; + if (c == OP_RECURSE) return code; + + /* XCLASS is used for classes that cannot be represented just by a bit + map. This includes negated single high-valued characters. The length in + the table is zero; the actual length is stored in the compiled code. */ + + if (c == OP_XCLASS) code += GET(code, 1); + + /* Otherwise, we can get the item's length from the table, except that for + repeated character types, we have to test for \p and \P, which have an extra + two bytes of parameters, and for MARK/PRUNE/SKIP/THEN with an argument, we + must add in its length. */ + + else + { + switch(c) + { + case OP_TYPESTAR: + case OP_TYPEMINSTAR: + case OP_TYPEPLUS: + case OP_TYPEMINPLUS: + case OP_TYPEQUERY: + case OP_TYPEMINQUERY: + case OP_TYPEPOSSTAR: + case OP_TYPEPOSPLUS: + case OP_TYPEPOSQUERY: + if (code[1] == OP_PROP || code[1] == OP_NOTPROP) code += 2; + break; + + case OP_TYPEPOSUPTO: + case OP_TYPEUPTO: + case OP_TYPEMINUPTO: + case OP_TYPEEXACT: + if (code[1 + IMM2_SIZE] == OP_PROP || code[1 + IMM2_SIZE] == OP_NOTPROP) + code += 2; + break; + + case OP_MARK: + case OP_PRUNE_ARG: + case OP_SKIP_ARG: + case OP_THEN_ARG: + code += code[1]; + break; + } + + /* Add in the fixed length from the table */ + + code += PRIV(OP_lengths)[c]; + + /* In UTF-8 mode, opcodes that are followed by a character may be followed + by a multi-byte character. The length in the table is a minimum, so we have + to arrange to skip the extra bytes. */ + +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (utf) switch(c) + { + case OP_CHAR: + case OP_CHARI: + case OP_NOT: + case OP_NOTI: + case OP_EXACT: + case OP_EXACTI: + case OP_NOTEXACT: + case OP_NOTEXACTI: + case OP_UPTO: + case OP_UPTOI: + case OP_NOTUPTO: + case OP_NOTUPTOI: + case OP_MINUPTO: + case OP_MINUPTOI: + case OP_NOTMINUPTO: + case OP_NOTMINUPTOI: + case OP_POSUPTO: + case OP_POSUPTOI: + case OP_NOTPOSUPTO: + case OP_NOTPOSUPTOI: + case OP_STAR: + case OP_STARI: + case OP_NOTSTAR: + case OP_NOTSTARI: + case OP_MINSTAR: + case OP_MINSTARI: + case OP_NOTMINSTAR: + case OP_NOTMINSTARI: + case OP_POSSTAR: + case OP_POSSTARI: + case OP_NOTPOSSTAR: + case OP_NOTPOSSTARI: + case OP_PLUS: + case OP_PLUSI: + case OP_NOTPLUS: + case OP_NOTPLUSI: + case OP_MINPLUS: + case OP_MINPLUSI: + case OP_NOTMINPLUS: + case OP_NOTMINPLUSI: + case OP_POSPLUS: + case OP_POSPLUSI: + case OP_NOTPOSPLUS: + case OP_NOTPOSPLUSI: + case OP_QUERY: + case OP_QUERYI: + case OP_NOTQUERY: + case OP_NOTQUERYI: + case OP_MINQUERY: + case OP_MINQUERYI: + case OP_NOTMINQUERY: + case OP_NOTMINQUERYI: + case OP_POSQUERY: + case OP_POSQUERYI: + case OP_NOTPOSQUERY: + case OP_NOTPOSQUERYI: + if (HAS_EXTRALEN(code[-1])) code += GET_EXTRALEN(code[-1]); + break; + } +#else + (void)(utf); /* Keep compiler happy by referencing function argument */ +#endif + } + } +} + + + +/************************************************* +* Scan compiled branch for non-emptiness * +*************************************************/ + +/* This function scans through a branch of a compiled pattern to see whether it +can match the empty string or not. It is called from could_be_empty() +below and from compile_branch() when checking for an unlimited repeat of a +group that can match nothing. Note that first_significant_code() skips over +backward and negative forward assertions when its final argument is TRUE. If we +hit an unclosed bracket, we return "empty" - this means we've struck an inner +bracket whose current branch will already have been scanned. + +Arguments: + code points to start of search + endcode points to where to stop + utf TRUE if in UTF-8 / UTF-16 / UTF-32 mode + cd contains pointers to tables etc. + recurses chain of recurse_check to catch mutual recursion + +Returns: TRUE if what is matched could be empty +*/ + +static BOOL +could_be_empty_branch(const pcre_uchar *code, const pcre_uchar *endcode, + BOOL utf, compile_data *cd, recurse_check *recurses) +{ +register pcre_uchar c; +recurse_check this_recurse; + +for (code = first_significant_code(code + PRIV(OP_lengths)[*code], TRUE); + code < endcode; + code = first_significant_code(code + PRIV(OP_lengths)[c], TRUE)) + { + const pcre_uchar *ccode; + + c = *code; + + /* Skip over forward assertions; the other assertions are skipped by + first_significant_code() with a TRUE final argument. */ + + if (c == OP_ASSERT) + { + do code += GET(code, 1); while (*code == OP_ALT); + c = *code; + continue; + } + + /* For a recursion/subroutine call, if its end has been reached, which + implies a backward reference subroutine call, we can scan it. If it's a + forward reference subroutine call, we can't. To detect forward reference + we have to scan up the list that is kept in the workspace. This function is + called only when doing the real compile, not during the pre-compile that + measures the size of the compiled pattern. */ + + if (c == OP_RECURSE) + { + const pcre_uchar *scode = cd->start_code + GET(code, 1); + const pcre_uchar *endgroup = scode; + BOOL empty_branch; + + /* Test for forward reference or uncompleted reference. This is disabled + when called to scan a completed pattern by setting cd->start_workspace to + NULL. */ + + if (cd->start_workspace != NULL) + { + const pcre_uchar *tcode; + for (tcode = cd->start_workspace; tcode < cd->hwm; tcode += LINK_SIZE) + if ((int)GET(tcode, 0) == (int)(code + 1 - cd->start_code)) return TRUE; + if (GET(scode, 1) == 0) return TRUE; /* Unclosed */ + } + + /* If the reference is to a completed group, we need to detect whether this + is a recursive call, as otherwise there will be an infinite loop. If it is + a recursion, just skip over it. Simple recursions are easily detected. For + mutual recursions we keep a chain on the stack. */ + + do endgroup += GET(endgroup, 1); while (*endgroup == OP_ALT); + if (code >= scode && code <= endgroup) continue; /* Simple recursion */ + else + { + recurse_check *r = recurses; + for (r = recurses; r != NULL; r = r->prev) + if (r->group == scode) break; + if (r != NULL) continue; /* Mutual recursion */ + } + + /* Completed reference; scan the referenced group, remembering it on the + stack chain to detect mutual recursions. */ + + empty_branch = FALSE; + this_recurse.prev = recurses; + this_recurse.group = scode; + + do + { + if (could_be_empty_branch(scode, endcode, utf, cd, &this_recurse)) + { + empty_branch = TRUE; + break; + } + scode += GET(scode, 1); + } + while (*scode == OP_ALT); + + if (!empty_branch) return FALSE; /* All branches are non-empty */ + continue; + } + + /* Groups with zero repeats can of course be empty; skip them. */ + + if (c == OP_BRAZERO || c == OP_BRAMINZERO || c == OP_SKIPZERO || + c == OP_BRAPOSZERO) + { + code += PRIV(OP_lengths)[c]; + do code += GET(code, 1); while (*code == OP_ALT); + c = *code; + continue; + } + + /* A nested group that is already marked as "could be empty" can just be + skipped. */ + + if (c == OP_SBRA || c == OP_SBRAPOS || + c == OP_SCBRA || c == OP_SCBRAPOS) + { + do code += GET(code, 1); while (*code == OP_ALT); + c = *code; + continue; + } + + /* For other groups, scan the branches. */ + + if (c == OP_BRA || c == OP_BRAPOS || + c == OP_CBRA || c == OP_CBRAPOS || + c == OP_ONCE || c == OP_ONCE_NC || + c == OP_COND || c == OP_SCOND) + { + BOOL empty_branch; + if (GET(code, 1) == 0) return TRUE; /* Hit unclosed bracket */ + + /* If a conditional group has only one branch, there is a second, implied, + empty branch, so just skip over the conditional, because it could be empty. + Otherwise, scan the individual branches of the group. */ + + if (c == OP_COND && code[GET(code, 1)] != OP_ALT) + code += GET(code, 1); + else + { + empty_branch = FALSE; + do + { + if (!empty_branch && could_be_empty_branch(code, endcode, utf, cd, + recurses)) empty_branch = TRUE; + code += GET(code, 1); + } + while (*code == OP_ALT); + if (!empty_branch) return FALSE; /* All branches are non-empty */ + } + + c = *code; + continue; + } + + /* Handle the other opcodes */ + + switch (c) + { + /* Check for quantifiers after a class. XCLASS is used for classes that + cannot be represented just by a bit map. This includes negated single + high-valued characters. The length in PRIV(OP_lengths)[] is zero; the + actual length is stored in the compiled code, so we must update "code" + here. */ + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: + ccode = code += GET(code, 1); + goto CHECK_CLASS_REPEAT; +#endif + + case OP_CLASS: + case OP_NCLASS: + ccode = code + PRIV(OP_lengths)[OP_CLASS]; + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + CHECK_CLASS_REPEAT: +#endif + + switch (*ccode) + { + case OP_CRSTAR: /* These could be empty; continue */ + case OP_CRMINSTAR: + case OP_CRQUERY: + case OP_CRMINQUERY: + case OP_CRPOSSTAR: + case OP_CRPOSQUERY: + break; + + default: /* Non-repeat => class must match */ + case OP_CRPLUS: /* These repeats aren't empty */ + case OP_CRMINPLUS: + case OP_CRPOSPLUS: + return FALSE; + + case OP_CRRANGE: + case OP_CRMINRANGE: + case OP_CRPOSRANGE: + if (GET2(ccode, 1) > 0) return FALSE; /* Minimum > 0 */ + break; + } + break; + + /* Opcodes that must match a character */ + + case OP_ANY: + case OP_ALLANY: + case OP_ANYBYTE: + + case OP_PROP: + case OP_NOTPROP: + case OP_ANYNL: + + case OP_NOT_HSPACE: + case OP_HSPACE: + case OP_NOT_VSPACE: + case OP_VSPACE: + case OP_EXTUNI: + + case OP_NOT_DIGIT: + case OP_DIGIT: + case OP_NOT_WHITESPACE: + case OP_WHITESPACE: + case OP_NOT_WORDCHAR: + case OP_WORDCHAR: + + case OP_CHAR: + case OP_CHARI: + case OP_NOT: + case OP_NOTI: + + case OP_PLUS: + case OP_PLUSI: + case OP_MINPLUS: + case OP_MINPLUSI: + + case OP_NOTPLUS: + case OP_NOTPLUSI: + case OP_NOTMINPLUS: + case OP_NOTMINPLUSI: + + case OP_POSPLUS: + case OP_POSPLUSI: + case OP_NOTPOSPLUS: + case OP_NOTPOSPLUSI: + + case OP_EXACT: + case OP_EXACTI: + case OP_NOTEXACT: + case OP_NOTEXACTI: + + case OP_TYPEPLUS: + case OP_TYPEMINPLUS: + case OP_TYPEPOSPLUS: + case OP_TYPEEXACT: + + return FALSE; + + /* These are going to continue, as they may be empty, but we have to + fudge the length for the \p and \P cases. */ + + case OP_TYPESTAR: + case OP_TYPEMINSTAR: + case OP_TYPEPOSSTAR: + case OP_TYPEQUERY: + case OP_TYPEMINQUERY: + case OP_TYPEPOSQUERY: + if (code[1] == OP_PROP || code[1] == OP_NOTPROP) code += 2; + break; + + /* Same for these */ + + case OP_TYPEUPTO: + case OP_TYPEMINUPTO: + case OP_TYPEPOSUPTO: + if (code[1 + IMM2_SIZE] == OP_PROP || code[1 + IMM2_SIZE] == OP_NOTPROP) + code += 2; + break; + + /* End of branch */ + + case OP_KET: + case OP_KETRMAX: + case OP_KETRMIN: + case OP_KETRPOS: + case OP_ALT: + return TRUE; + + /* In UTF-8 mode, STAR, MINSTAR, POSSTAR, QUERY, MINQUERY, POSQUERY, UPTO, + MINUPTO, and POSUPTO and their caseless and negative versions may be + followed by a multibyte character. */ + +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + case OP_STAR: + case OP_STARI: + case OP_NOTSTAR: + case OP_NOTSTARI: + + case OP_MINSTAR: + case OP_MINSTARI: + case OP_NOTMINSTAR: + case OP_NOTMINSTARI: + + case OP_POSSTAR: + case OP_POSSTARI: + case OP_NOTPOSSTAR: + case OP_NOTPOSSTARI: + + case OP_QUERY: + case OP_QUERYI: + case OP_NOTQUERY: + case OP_NOTQUERYI: + + case OP_MINQUERY: + case OP_MINQUERYI: + case OP_NOTMINQUERY: + case OP_NOTMINQUERYI: + + case OP_POSQUERY: + case OP_POSQUERYI: + case OP_NOTPOSQUERY: + case OP_NOTPOSQUERYI: + + if (utf && HAS_EXTRALEN(code[1])) code += GET_EXTRALEN(code[1]); + break; + + case OP_UPTO: + case OP_UPTOI: + case OP_NOTUPTO: + case OP_NOTUPTOI: + + case OP_MINUPTO: + case OP_MINUPTOI: + case OP_NOTMINUPTO: + case OP_NOTMINUPTOI: + + case OP_POSUPTO: + case OP_POSUPTOI: + case OP_NOTPOSUPTO: + case OP_NOTPOSUPTOI: + + if (utf && HAS_EXTRALEN(code[1 + IMM2_SIZE])) code += GET_EXTRALEN(code[1 + IMM2_SIZE]); + break; +#endif + + /* MARK, and PRUNE/SKIP/THEN with an argument must skip over the argument + string. */ + + case OP_MARK: + case OP_PRUNE_ARG: + case OP_SKIP_ARG: + case OP_THEN_ARG: + code += code[1]; + break; + + /* None of the remaining opcodes are required to match a character. */ + + default: + break; + } + } + +return TRUE; +} + + + +/************************************************* +* Scan compiled regex for non-emptiness * +*************************************************/ + +/* This function is called to check for left recursive calls. We want to check +the current branch of the current pattern to see if it could match the empty +string. If it could, we must look outwards for branches at other levels, +stopping when we pass beyond the bracket which is the subject of the recursion. +This function is called only during the real compile, not during the +pre-compile. + +Arguments: + code points to start of the recursion + endcode points to where to stop (current RECURSE item) + bcptr points to the chain of current (unclosed) branch starts + utf TRUE if in UTF-8 / UTF-16 / UTF-32 mode + cd pointers to tables etc + +Returns: TRUE if what is matched could be empty +*/ + +static BOOL +could_be_empty(const pcre_uchar *code, const pcre_uchar *endcode, + branch_chain *bcptr, BOOL utf, compile_data *cd) +{ +while (bcptr != NULL && bcptr->current_branch >= code) + { + if (!could_be_empty_branch(bcptr->current_branch, endcode, utf, cd, NULL)) + return FALSE; + bcptr = bcptr->outer; + } +return TRUE; +} + + + +/************************************************* +* Base opcode of repeated opcodes * +*************************************************/ + +/* Returns the base opcode for repeated single character type opcodes. If the +opcode is not a repeated character type, it returns with the original value. + +Arguments: c opcode +Returns: base opcode for the type +*/ + +static pcre_uchar +get_repeat_base(pcre_uchar c) +{ +return (c > OP_TYPEPOSUPTO)? c : + (c >= OP_TYPESTAR)? OP_TYPESTAR : + (c >= OP_NOTSTARI)? OP_NOTSTARI : + (c >= OP_NOTSTAR)? OP_NOTSTAR : + (c >= OP_STARI)? OP_STARI : + OP_STAR; +} + + + +#ifdef SUPPORT_UCP +/************************************************* +* Check a character and a property * +*************************************************/ + +/* This function is called by check_auto_possessive() when a property item +is adjacent to a fixed character. + +Arguments: + c the character + ptype the property type + pdata the data for the type + negated TRUE if it's a negated property (\P or \p{^) + +Returns: TRUE if auto-possessifying is OK +*/ + +static BOOL +check_char_prop(pcre_uint32 c, unsigned int ptype, unsigned int pdata, + BOOL negated) +{ +const pcre_uint32 *p; +const ucd_record *prop = GET_UCD(c); + +switch(ptype) + { + case PT_LAMP: + return (prop->chartype == ucp_Lu || + prop->chartype == ucp_Ll || + prop->chartype == ucp_Lt) == negated; + + case PT_GC: + return (pdata == PRIV(ucp_gentype)[prop->chartype]) == negated; + + case PT_PC: + return (pdata == prop->chartype) == negated; + + case PT_SC: + return (pdata == prop->script) == negated; + + /* These are specials */ + + case PT_ALNUM: + return (PRIV(ucp_gentype)[prop->chartype] == ucp_L || + PRIV(ucp_gentype)[prop->chartype] == ucp_N) == negated; + + /* Perl space used to exclude VT, but from Perl 5.18 it is included, which + means that Perl space and POSIX space are now identical. PCRE was changed + at release 8.34. */ + + case PT_SPACE: /* Perl space */ + case PT_PXSPACE: /* POSIX space */ + switch(c) + { + HSPACE_CASES: + VSPACE_CASES: + return negated; + + default: + return (PRIV(ucp_gentype)[prop->chartype] == ucp_Z) == negated; + } + break; /* Control never reaches here */ + + case PT_WORD: + return (PRIV(ucp_gentype)[prop->chartype] == ucp_L || + PRIV(ucp_gentype)[prop->chartype] == ucp_N || + c == CHAR_UNDERSCORE) == negated; + + case PT_CLIST: + p = PRIV(ucd_caseless_sets) + prop->caseset; + for (;;) + { + if (c < *p) return !negated; + if (c == *p++) return negated; + } + break; /* Control never reaches here */ + } + +return FALSE; +} +#endif /* SUPPORT_UCP */ + + + +/************************************************* +* Fill the character property list * +*************************************************/ + +/* Checks whether the code points to an opcode that can take part in auto- +possessification, and if so, fills a list with its properties. + +Arguments: + code points to start of expression + utf TRUE if in UTF-8 / UTF-16 / UTF-32 mode + fcc points to case-flipping table + list points to output list + list[0] will be filled with the opcode + list[1] will be non-zero if this opcode + can match an empty character string + list[2..7] depends on the opcode + +Returns: points to the start of the next opcode if *code is accepted + NULL if *code is not accepted +*/ + +static const pcre_uchar * +get_chr_property_list(const pcre_uchar *code, BOOL utf, + const pcre_uint8 *fcc, pcre_uint32 *list) +{ +pcre_uchar c = *code; +pcre_uchar base; +const pcre_uchar *end; +pcre_uint32 chr; + +#ifdef SUPPORT_UCP +pcre_uint32 *clist_dest; +const pcre_uint32 *clist_src; +#else +utf = utf; /* Suppress "unused parameter" compiler warning */ +#endif + +list[0] = c; +list[1] = FALSE; +code++; + +if (c >= OP_STAR && c <= OP_TYPEPOSUPTO) + { + base = get_repeat_base(c); + c -= (base - OP_STAR); + + if (c == OP_UPTO || c == OP_MINUPTO || c == OP_EXACT || c == OP_POSUPTO) + code += IMM2_SIZE; + + list[1] = (c != OP_PLUS && c != OP_MINPLUS && c != OP_EXACT && c != OP_POSPLUS); + + switch(base) + { + case OP_STAR: + list[0] = OP_CHAR; + break; + + case OP_STARI: + list[0] = OP_CHARI; + break; + + case OP_NOTSTAR: + list[0] = OP_NOT; + break; + + case OP_NOTSTARI: + list[0] = OP_NOTI; + break; + + case OP_TYPESTAR: + list[0] = *code; + code++; + break; + } + c = list[0]; + } + +switch(c) + { + case OP_NOT_DIGIT: + case OP_DIGIT: + case OP_NOT_WHITESPACE: + case OP_WHITESPACE: + case OP_NOT_WORDCHAR: + case OP_WORDCHAR: + case OP_ANY: + case OP_ALLANY: + case OP_ANYNL: + case OP_NOT_HSPACE: + case OP_HSPACE: + case OP_NOT_VSPACE: + case OP_VSPACE: + case OP_EXTUNI: + case OP_EODN: + case OP_EOD: + case OP_DOLL: + case OP_DOLLM: + return code; + + case OP_CHAR: + case OP_NOT: + GETCHARINCTEST(chr, code); + list[2] = chr; + list[3] = NOTACHAR; + return code; + + case OP_CHARI: + case OP_NOTI: + list[0] = (c == OP_CHARI) ? OP_CHAR : OP_NOT; + GETCHARINCTEST(chr, code); + list[2] = chr; + +#ifdef SUPPORT_UCP + if (chr < 128 || (chr < 256 && !utf)) + list[3] = fcc[chr]; + else + list[3] = UCD_OTHERCASE(chr); +#elif defined SUPPORT_UTF || !defined COMPILE_PCRE8 + list[3] = (chr < 256) ? fcc[chr] : chr; +#else + list[3] = fcc[chr]; +#endif + + /* The othercase might be the same value. */ + + if (chr == list[3]) + list[3] = NOTACHAR; + else + list[4] = NOTACHAR; + return code; + +#ifdef SUPPORT_UCP + case OP_PROP: + case OP_NOTPROP: + if (code[0] != PT_CLIST) + { + list[2] = code[0]; + list[3] = code[1]; + return code + 2; + } + + /* Convert only if we have enough space. */ + + clist_src = PRIV(ucd_caseless_sets) + code[1]; + clist_dest = list + 2; + code += 2; + + do { + if (clist_dest >= list + 8) + { + /* Early return if there is not enough space. This should never + happen, since all clists are shorter than 5 character now. */ + list[2] = code[0]; + list[3] = code[1]; + return code; + } + *clist_dest++ = *clist_src; + } + while(*clist_src++ != NOTACHAR); + + /* All characters are stored. The terminating NOTACHAR + is copied form the clist itself. */ + + list[0] = (c == OP_PROP) ? OP_CHAR : OP_NOT; + return code; +#endif + + case OP_NCLASS: + case OP_CLASS: +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: + if (c == OP_XCLASS) + end = code + GET(code, 0) - 1; + else +#endif + end = code + 32 / sizeof(pcre_uchar); + + switch(*end) + { + case OP_CRSTAR: + case OP_CRMINSTAR: + case OP_CRQUERY: + case OP_CRMINQUERY: + case OP_CRPOSSTAR: + case OP_CRPOSQUERY: + list[1] = TRUE; + end++; + break; + + case OP_CRPLUS: + case OP_CRMINPLUS: + case OP_CRPOSPLUS: + end++; + break; + + case OP_CRRANGE: + case OP_CRMINRANGE: + case OP_CRPOSRANGE: + list[1] = (GET2(end, 1) == 0); + end += 1 + 2 * IMM2_SIZE; + break; + } + list[2] = (pcre_uint32)(end - code); + return end; + } +return NULL; /* Opcode not accepted */ +} + + + +/************************************************* +* Scan further character sets for match * +*************************************************/ + +/* Checks whether the base and the current opcode have a common character, in +which case the base cannot be possessified. + +Arguments: + code points to the byte code + utf TRUE in UTF-8 / UTF-16 / UTF-32 mode + cd static compile data + base_list the data list of the base opcode + +Returns: TRUE if the auto-possessification is possible +*/ + +static BOOL +compare_opcodes(const pcre_uchar *code, BOOL utf, const compile_data *cd, + const pcre_uint32 *base_list, const pcre_uchar *base_end, int *rec_limit) +{ +pcre_uchar c; +pcre_uint32 list[8]; +const pcre_uint32 *chr_ptr; +const pcre_uint32 *ochr_ptr; +const pcre_uint32 *list_ptr; +const pcre_uchar *next_code; +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 +const pcre_uchar *xclass_flags; +#endif +const pcre_uint8 *class_bitset; +const pcre_uint8 *set1, *set2, *set_end; +pcre_uint32 chr; +BOOL accepted, invert_bits; +BOOL entered_a_group = FALSE; + +if (*rec_limit == 0) return FALSE; +--(*rec_limit); + +/* Note: the base_list[1] contains whether the current opcode has greedy +(represented by a non-zero value) quantifier. This is a different from +other character type lists, which stores here that the character iterator +matches to an empty string (also represented by a non-zero value). */ + +for(;;) + { + /* All operations move the code pointer forward. + Therefore infinite recursions are not possible. */ + + c = *code; + + /* Skip over callouts */ + + if (c == OP_CALLOUT) + { + code += PRIV(OP_lengths)[c]; + continue; + } + + if (c == OP_ALT) + { + do code += GET(code, 1); while (*code == OP_ALT); + c = *code; + } + + switch(c) + { + case OP_END: + case OP_KETRPOS: + /* TRUE only in greedy case. The non-greedy case could be replaced by + an OP_EXACT, but it is probably not worth it. (And note that OP_EXACT + uses more memory, which we cannot get at this stage.) */ + + return base_list[1] != 0; + + case OP_KET: + /* If the bracket is capturing, and referenced by an OP_RECURSE, or + it is an atomic sub-pattern (assert, once, etc.) the non-greedy case + cannot be converted to a possessive form. */ + + if (base_list[1] == 0) return FALSE; + + switch(*(code - GET(code, 1))) + { + case OP_ASSERT: + case OP_ASSERT_NOT: + case OP_ASSERTBACK: + case OP_ASSERTBACK_NOT: + case OP_ONCE: + case OP_ONCE_NC: + /* Atomic sub-patterns and assertions can always auto-possessify their + last iterator. However, if the group was entered as a result of checking + a previous iterator, this is not possible. */ + + return !entered_a_group; + } + + code += PRIV(OP_lengths)[c]; + continue; + + case OP_ONCE: + case OP_ONCE_NC: + case OP_BRA: + case OP_CBRA: + next_code = code + GET(code, 1); + code += PRIV(OP_lengths)[c]; + + while (*next_code == OP_ALT) + { + if (!compare_opcodes(code, utf, cd, base_list, base_end, rec_limit)) + return FALSE; + code = next_code + 1 + LINK_SIZE; + next_code += GET(next_code, 1); + } + + entered_a_group = TRUE; + continue; + + case OP_BRAZERO: + case OP_BRAMINZERO: + + next_code = code + 1; + if (*next_code != OP_BRA && *next_code != OP_CBRA + && *next_code != OP_ONCE && *next_code != OP_ONCE_NC) return FALSE; + + do next_code += GET(next_code, 1); while (*next_code == OP_ALT); + + /* The bracket content will be checked by the + OP_BRA/OP_CBRA case above. */ + next_code += 1 + LINK_SIZE; + if (!compare_opcodes(next_code, utf, cd, base_list, base_end, rec_limit)) + return FALSE; + + code += PRIV(OP_lengths)[c]; + continue; + + default: + break; + } + + /* Check for a supported opcode, and load its properties. */ + + code = get_chr_property_list(code, utf, cd->fcc, list); + if (code == NULL) return FALSE; /* Unsupported */ + + /* If either opcode is a small character list, set pointers for comparing + characters from that list with another list, or with a property. */ + + if (base_list[0] == OP_CHAR) + { + chr_ptr = base_list + 2; + list_ptr = list; + } + else if (list[0] == OP_CHAR) + { + chr_ptr = list + 2; + list_ptr = base_list; + } + + /* Character bitsets can also be compared to certain opcodes. */ + + else if (base_list[0] == OP_CLASS || list[0] == OP_CLASS +#ifdef COMPILE_PCRE8 + /* In 8 bit, non-UTF mode, OP_CLASS and OP_NCLASS are the same. */ + || (!utf && (base_list[0] == OP_NCLASS || list[0] == OP_NCLASS)) +#endif + ) + { +#ifdef COMPILE_PCRE8 + if (base_list[0] == OP_CLASS || (!utf && base_list[0] == OP_NCLASS)) +#else + if (base_list[0] == OP_CLASS) +#endif + { + set1 = (pcre_uint8 *)(base_end - base_list[2]); + list_ptr = list; + } + else + { + set1 = (pcre_uint8 *)(code - list[2]); + list_ptr = base_list; + } + + invert_bits = FALSE; + switch(list_ptr[0]) + { + case OP_CLASS: + case OP_NCLASS: + set2 = (pcre_uint8 *) + ((list_ptr == list ? code : base_end) - list_ptr[2]); + break; + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: + xclass_flags = (list_ptr == list ? code : base_end) - list_ptr[2] + LINK_SIZE; + if ((*xclass_flags & XCL_HASPROP) != 0) return FALSE; + if ((*xclass_flags & XCL_MAP) == 0) + { + /* No bits are set for characters < 256. */ + if (list[1] == 0) return TRUE; + /* Might be an empty repeat. */ + continue; + } + set2 = (pcre_uint8 *)(xclass_flags + 1); + break; +#endif + + case OP_NOT_DIGIT: + invert_bits = TRUE; + /* Fall through */ + case OP_DIGIT: + set2 = (pcre_uint8 *)(cd->cbits + cbit_digit); + break; + + case OP_NOT_WHITESPACE: + invert_bits = TRUE; + /* Fall through */ + case OP_WHITESPACE: + set2 = (pcre_uint8 *)(cd->cbits + cbit_space); + break; + + case OP_NOT_WORDCHAR: + invert_bits = TRUE; + /* Fall through */ + case OP_WORDCHAR: + set2 = (pcre_uint8 *)(cd->cbits + cbit_word); + break; + + default: + return FALSE; + } + + /* Because the sets are unaligned, we need + to perform byte comparison here. */ + set_end = set1 + 32; + if (invert_bits) + { + do + { + if ((*set1++ & ~(*set2++)) != 0) return FALSE; + } + while (set1 < set_end); + } + else + { + do + { + if ((*set1++ & *set2++) != 0) return FALSE; + } + while (set1 < set_end); + } + + if (list[1] == 0) return TRUE; + /* Might be an empty repeat. */ + continue; + } + + /* Some property combinations also acceptable. Unicode property opcodes are + processed specially; the rest can be handled with a lookup table. */ + + else + { + pcre_uint32 leftop, rightop; + + leftop = base_list[0]; + rightop = list[0]; + +#ifdef SUPPORT_UCP + accepted = FALSE; /* Always set in non-unicode case. */ + if (leftop == OP_PROP || leftop == OP_NOTPROP) + { + if (rightop == OP_EOD) + accepted = TRUE; + else if (rightop == OP_PROP || rightop == OP_NOTPROP) + { + int n; + const pcre_uint8 *p; + BOOL same = leftop == rightop; + BOOL lisprop = leftop == OP_PROP; + BOOL risprop = rightop == OP_PROP; + BOOL bothprop = lisprop && risprop; + + /* There's a table that specifies how each combination is to be + processed: + 0 Always return FALSE (never auto-possessify) + 1 Character groups are distinct (possessify if both are OP_PROP) + 2 Check character categories in the same group (general or particular) + 3 Return TRUE if the two opcodes are not the same + ... see comments below + */ + + n = propposstab[base_list[2]][list[2]]; + switch(n) + { + case 0: break; + case 1: accepted = bothprop; break; + case 2: accepted = (base_list[3] == list[3]) != same; break; + case 3: accepted = !same; break; + + case 4: /* Left general category, right particular category */ + accepted = risprop && catposstab[base_list[3]][list[3]] == same; + break; + + case 5: /* Right general category, left particular category */ + accepted = lisprop && catposstab[list[3]][base_list[3]] == same; + break; + + /* This code is logically tricky. Think hard before fiddling with it. + The posspropstab table has four entries per row. Each row relates to + one of PCRE's special properties such as ALNUM or SPACE or WORD. + Only WORD actually needs all four entries, but using repeats for the + others means they can all use the same code below. + + The first two entries in each row are Unicode general categories, and + apply always, because all the characters they include are part of the + PCRE character set. The third and fourth entries are a general and a + particular category, respectively, that include one or more relevant + characters. One or the other is used, depending on whether the check + is for a general or a particular category. However, in both cases the + category contains more characters than the specials that are defined + for the property being tested against. Therefore, it cannot be used + in a NOTPROP case. + + Example: the row for WORD contains ucp_L, ucp_N, ucp_P, ucp_Po. + Underscore is covered by ucp_P or ucp_Po. */ + + case 6: /* Left alphanum vs right general category */ + case 7: /* Left space vs right general category */ + case 8: /* Left word vs right general category */ + p = posspropstab[n-6]; + accepted = risprop && lisprop == + (list[3] != p[0] && + list[3] != p[1] && + (list[3] != p[2] || !lisprop)); + break; + + case 9: /* Right alphanum vs left general category */ + case 10: /* Right space vs left general category */ + case 11: /* Right word vs left general category */ + p = posspropstab[n-9]; + accepted = lisprop && risprop == + (base_list[3] != p[0] && + base_list[3] != p[1] && + (base_list[3] != p[2] || !risprop)); + break; + + case 12: /* Left alphanum vs right particular category */ + case 13: /* Left space vs right particular category */ + case 14: /* Left word vs right particular category */ + p = posspropstab[n-12]; + accepted = risprop && lisprop == + (catposstab[p[0]][list[3]] && + catposstab[p[1]][list[3]] && + (list[3] != p[3] || !lisprop)); + break; + + case 15: /* Right alphanum vs left particular category */ + case 16: /* Right space vs left particular category */ + case 17: /* Right word vs left particular category */ + p = posspropstab[n-15]; + accepted = lisprop && risprop == + (catposstab[p[0]][base_list[3]] && + catposstab[p[1]][base_list[3]] && + (base_list[3] != p[3] || !risprop)); + break; + } + } + } + + else +#endif /* SUPPORT_UCP */ + + accepted = leftop >= FIRST_AUTOTAB_OP && leftop <= LAST_AUTOTAB_LEFT_OP && + rightop >= FIRST_AUTOTAB_OP && rightop <= LAST_AUTOTAB_RIGHT_OP && + autoposstab[leftop - FIRST_AUTOTAB_OP][rightop - FIRST_AUTOTAB_OP]; + + if (!accepted) return FALSE; + + if (list[1] == 0) return TRUE; + /* Might be an empty repeat. */ + continue; + } + + /* Control reaches here only if one of the items is a small character list. + All characters are checked against the other side. */ + + do + { + chr = *chr_ptr; + + switch(list_ptr[0]) + { + case OP_CHAR: + ochr_ptr = list_ptr + 2; + do + { + if (chr == *ochr_ptr) return FALSE; + ochr_ptr++; + } + while(*ochr_ptr != NOTACHAR); + break; + + case OP_NOT: + ochr_ptr = list_ptr + 2; + do + { + if (chr == *ochr_ptr) + break; + ochr_ptr++; + } + while(*ochr_ptr != NOTACHAR); + if (*ochr_ptr == NOTACHAR) return FALSE; /* Not found */ + break; + + /* Note that OP_DIGIT etc. are generated only when PCRE_UCP is *not* + set. When it is set, \d etc. are converted into OP_(NOT_)PROP codes. */ + + case OP_DIGIT: + if (chr < 256 && (cd->ctypes[chr] & ctype_digit) != 0) return FALSE; + break; + + case OP_NOT_DIGIT: + if (chr > 255 || (cd->ctypes[chr] & ctype_digit) == 0) return FALSE; + break; + + case OP_WHITESPACE: + if (chr < 256 && (cd->ctypes[chr] & ctype_space) != 0) return FALSE; + break; + + case OP_NOT_WHITESPACE: + if (chr > 255 || (cd->ctypes[chr] & ctype_space) == 0) return FALSE; + break; + + case OP_WORDCHAR: + if (chr < 255 && (cd->ctypes[chr] & ctype_word) != 0) return FALSE; + break; + + case OP_NOT_WORDCHAR: + if (chr > 255 || (cd->ctypes[chr] & ctype_word) == 0) return FALSE; + break; + + case OP_HSPACE: + switch(chr) + { + HSPACE_CASES: return FALSE; + default: break; + } + break; + + case OP_NOT_HSPACE: + switch(chr) + { + HSPACE_CASES: break; + default: return FALSE; + } + break; + + case OP_ANYNL: + case OP_VSPACE: + switch(chr) + { + VSPACE_CASES: return FALSE; + default: break; + } + break; + + case OP_NOT_VSPACE: + switch(chr) + { + VSPACE_CASES: break; + default: return FALSE; + } + break; + + case OP_DOLL: + case OP_EODN: + switch (chr) + { + case CHAR_CR: + case CHAR_LF: + case CHAR_VT: + case CHAR_FF: + case CHAR_NEL: +#ifndef EBCDIC + case 0x2028: + case 0x2029: +#endif /* Not EBCDIC */ + return FALSE; + } + break; + + case OP_EOD: /* Can always possessify before \z */ + break; + +#ifdef SUPPORT_UCP + case OP_PROP: + case OP_NOTPROP: + if (!check_char_prop(chr, list_ptr[2], list_ptr[3], + list_ptr[0] == OP_NOTPROP)) + return FALSE; + break; +#endif + + case OP_NCLASS: + if (chr > 255) return FALSE; + /* Fall through */ + + case OP_CLASS: + if (chr > 255) break; + class_bitset = (pcre_uint8 *) + ((list_ptr == list ? code : base_end) - list_ptr[2]); + if ((class_bitset[chr >> 3] & (1 << (chr & 7))) != 0) return FALSE; + break; + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: + if (PRIV(xclass)(chr, (list_ptr == list ? code : base_end) - + list_ptr[2] + LINK_SIZE, utf)) return FALSE; + break; +#endif + + default: + return FALSE; + } + + chr_ptr++; + } + while(*chr_ptr != NOTACHAR); + + /* At least one character must be matched from this opcode. */ + + if (list[1] == 0) return TRUE; + } + +/* Control never reaches here. There used to be a fail-save return FALSE; here, +but some compilers complain about an unreachable statement. */ + +} + + + +/************************************************* +* Scan compiled regex for auto-possession * +*************************************************/ + +/* Replaces single character iterations with their possessive alternatives +if appropriate. This function modifies the compiled opcode! + +Arguments: + code points to start of the byte code + utf TRUE in UTF-8 / UTF-16 / UTF-32 mode + cd static compile data + +Returns: nothing +*/ + +static void +auto_possessify(pcre_uchar *code, BOOL utf, const compile_data *cd) +{ +register pcre_uchar c; +const pcre_uchar *end; +pcre_uchar *repeat_opcode; +pcre_uint32 list[8]; +int rec_limit; + +for (;;) + { + c = *code; + + /* When a pattern with bad UTF-8 encoding is compiled with NO_UTF_CHECK, + it may compile without complaining, but may get into a loop here if the code + pointer points to a bad value. This is, of course a documentated possibility, + when NO_UTF_CHECK is set, so it isn't a bug, but we can detect this case and + just give up on this optimization. */ + + if (c >= OP_TABLE_LENGTH) return; + + if (c >= OP_STAR && c <= OP_TYPEPOSUPTO) + { + c -= get_repeat_base(c) - OP_STAR; + end = (c <= OP_MINUPTO) ? + get_chr_property_list(code, utf, cd->fcc, list) : NULL; + list[1] = c == OP_STAR || c == OP_PLUS || c == OP_QUERY || c == OP_UPTO; + + rec_limit = 1000; + if (end != NULL && compare_opcodes(end, utf, cd, list, end, &rec_limit)) + { + switch(c) + { + case OP_STAR: + *code += OP_POSSTAR - OP_STAR; + break; + + case OP_MINSTAR: + *code += OP_POSSTAR - OP_MINSTAR; + break; + + case OP_PLUS: + *code += OP_POSPLUS - OP_PLUS; + break; + + case OP_MINPLUS: + *code += OP_POSPLUS - OP_MINPLUS; + break; + + case OP_QUERY: + *code += OP_POSQUERY - OP_QUERY; + break; + + case OP_MINQUERY: + *code += OP_POSQUERY - OP_MINQUERY; + break; + + case OP_UPTO: + *code += OP_POSUPTO - OP_UPTO; + break; + + case OP_MINUPTO: + *code += OP_POSUPTO - OP_MINUPTO; + break; + } + } + c = *code; + } + else if (c == OP_CLASS || c == OP_NCLASS || c == OP_XCLASS) + { +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + if (c == OP_XCLASS) + repeat_opcode = code + GET(code, 1); + else +#endif + repeat_opcode = code + 1 + (32 / sizeof(pcre_uchar)); + + c = *repeat_opcode; + if (c >= OP_CRSTAR && c <= OP_CRMINRANGE) + { + /* end must not be NULL. */ + end = get_chr_property_list(code, utf, cd->fcc, list); + + list[1] = (c & 1) == 0; + + rec_limit = 1000; + if (compare_opcodes(end, utf, cd, list, end, &rec_limit)) + { + switch (c) + { + case OP_CRSTAR: + case OP_CRMINSTAR: + *repeat_opcode = OP_CRPOSSTAR; + break; + + case OP_CRPLUS: + case OP_CRMINPLUS: + *repeat_opcode = OP_CRPOSPLUS; + break; + + case OP_CRQUERY: + case OP_CRMINQUERY: + *repeat_opcode = OP_CRPOSQUERY; + break; + + case OP_CRRANGE: + case OP_CRMINRANGE: + *repeat_opcode = OP_CRPOSRANGE; + break; + } + } + } + c = *code; + } + + switch(c) + { + case OP_END: + return; + + case OP_TYPESTAR: + case OP_TYPEMINSTAR: + case OP_TYPEPLUS: + case OP_TYPEMINPLUS: + case OP_TYPEQUERY: + case OP_TYPEMINQUERY: + case OP_TYPEPOSSTAR: + case OP_TYPEPOSPLUS: + case OP_TYPEPOSQUERY: + if (code[1] == OP_PROP || code[1] == OP_NOTPROP) code += 2; + break; + + case OP_TYPEUPTO: + case OP_TYPEMINUPTO: + case OP_TYPEEXACT: + case OP_TYPEPOSUPTO: + if (code[1 + IMM2_SIZE] == OP_PROP || code[1 + IMM2_SIZE] == OP_NOTPROP) + code += 2; + break; + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: + code += GET(code, 1); + break; +#endif + + case OP_MARK: + case OP_PRUNE_ARG: + case OP_SKIP_ARG: + case OP_THEN_ARG: + code += code[1]; + break; + } + + /* Add in the fixed length from the table */ + + code += PRIV(OP_lengths)[c]; + + /* In UTF-8 mode, opcodes that are followed by a character may be followed by + a multi-byte character. The length in the table is a minimum, so we have to + arrange to skip the extra bytes. */ + +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (utf) switch(c) + { + case OP_CHAR: + case OP_CHARI: + case OP_NOT: + case OP_NOTI: + case OP_STAR: + case OP_MINSTAR: + case OP_PLUS: + case OP_MINPLUS: + case OP_QUERY: + case OP_MINQUERY: + case OP_UPTO: + case OP_MINUPTO: + case OP_EXACT: + case OP_POSSTAR: + case OP_POSPLUS: + case OP_POSQUERY: + case OP_POSUPTO: + case OP_STARI: + case OP_MINSTARI: + case OP_PLUSI: + case OP_MINPLUSI: + case OP_QUERYI: + case OP_MINQUERYI: + case OP_UPTOI: + case OP_MINUPTOI: + case OP_EXACTI: + case OP_POSSTARI: + case OP_POSPLUSI: + case OP_POSQUERYI: + case OP_POSUPTOI: + case OP_NOTSTAR: + case OP_NOTMINSTAR: + case OP_NOTPLUS: + case OP_NOTMINPLUS: + case OP_NOTQUERY: + case OP_NOTMINQUERY: + case OP_NOTUPTO: + case OP_NOTMINUPTO: + case OP_NOTEXACT: + case OP_NOTPOSSTAR: + case OP_NOTPOSPLUS: + case OP_NOTPOSQUERY: + case OP_NOTPOSUPTO: + case OP_NOTSTARI: + case OP_NOTMINSTARI: + case OP_NOTPLUSI: + case OP_NOTMINPLUSI: + case OP_NOTQUERYI: + case OP_NOTMINQUERYI: + case OP_NOTUPTOI: + case OP_NOTMINUPTOI: + case OP_NOTEXACTI: + case OP_NOTPOSSTARI: + case OP_NOTPOSPLUSI: + case OP_NOTPOSQUERYI: + case OP_NOTPOSUPTOI: + if (HAS_EXTRALEN(code[-1])) code += GET_EXTRALEN(code[-1]); + break; + } +#else + (void)(utf); /* Keep compiler happy by referencing function argument */ +#endif + } +} + + + +/************************************************* +* Check for POSIX class syntax * +*************************************************/ + +/* This function is called when the sequence "[:" or "[." or "[=" is +encountered in a character class. It checks whether this is followed by a +sequence of characters terminated by a matching ":]" or ".]" or "=]". If we +reach an unescaped ']' without the special preceding character, return FALSE. + +Originally, this function only recognized a sequence of letters between the +terminators, but it seems that Perl recognizes any sequence of characters, +though of course unknown POSIX names are subsequently rejected. Perl gives an +"Unknown POSIX class" error for [:f\oo:] for example, where previously PCRE +didn't consider this to be a POSIX class. Likewise for [:1234:]. + +The problem in trying to be exactly like Perl is in the handling of escapes. We +have to be sure that [abc[:x\]pqr] is *not* treated as containing a POSIX +class, but [abc[:x\]pqr:]] is (so that an error can be generated). The code +below handles the special cases \\ and \], but does not try to do any other +escape processing. This makes it different from Perl for cases such as +[:l\ower:] where Perl recognizes it as the POSIX class "lower" but PCRE does +not recognize "l\ower". This is a lesser evil than not diagnosing bad classes +when Perl does, I think. + +A user pointed out that PCRE was rejecting [:a[:digit:]] whereas Perl was not. +It seems that the appearance of a nested POSIX class supersedes an apparent +external class. For example, [:a[:digit:]b:] matches "a", "b", ":", or +a digit. + +In Perl, unescaped square brackets may also appear as part of class names. For +example, [:a[:abc]b:] gives unknown POSIX class "[:abc]b:]". However, for +[:a[:abc]b][b:] it gives unknown POSIX class "[:abc]b][b:]", which does not +seem right at all. PCRE does not allow closing square brackets in POSIX class +names. + +Arguments: + ptr pointer to the initial [ + endptr where to return the end pointer + +Returns: TRUE or FALSE +*/ + +static BOOL +check_posix_syntax(const pcre_uchar *ptr, const pcre_uchar **endptr) +{ +pcre_uchar terminator; /* Don't combine these lines; the Solaris cc */ +terminator = *(++ptr); /* compiler warns about "non-constant" initializer. */ +for (++ptr; *ptr != CHAR_NULL; ptr++) + { + if (*ptr == CHAR_BACKSLASH && + (ptr[1] == CHAR_RIGHT_SQUARE_BRACKET || + ptr[1] == CHAR_BACKSLASH)) + ptr++; + else if ((*ptr == CHAR_LEFT_SQUARE_BRACKET && ptr[1] == terminator) || + *ptr == CHAR_RIGHT_SQUARE_BRACKET) return FALSE; + else if (*ptr == terminator && ptr[1] == CHAR_RIGHT_SQUARE_BRACKET) + { + *endptr = ptr; + return TRUE; + } + } +return FALSE; +} + + + + +/************************************************* +* Check POSIX class name * +*************************************************/ + +/* This function is called to check the name given in a POSIX-style class entry +such as [:alnum:]. + +Arguments: + ptr points to the first letter + len the length of the name + +Returns: a value representing the name, or -1 if unknown +*/ + +static int +check_posix_name(const pcre_uchar *ptr, int len) +{ +const char *pn = posix_names; +register int yield = 0; +while (posix_name_lengths[yield] != 0) + { + if (len == posix_name_lengths[yield] && + STRNCMP_UC_C8(ptr, pn, (unsigned int)len) == 0) return yield; + pn += posix_name_lengths[yield] + 1; + yield++; + } +return -1; +} + + +/************************************************* +* Adjust OP_RECURSE items in repeated group * +*************************************************/ + +/* OP_RECURSE items contain an offset from the start of the regex to the group +that is referenced. This means that groups can be replicated for fixed +repetition simply by copying (because the recursion is allowed to refer to +earlier groups that are outside the current group). However, when a group is +optional (i.e. the minimum quantifier is zero), OP_BRAZERO or OP_SKIPZERO is +inserted before it, after it has been compiled. This means that any OP_RECURSE +items within it that refer to the group itself or any contained groups have to +have their offsets adjusted. That one of the jobs of this function. Before it +is called, the partially compiled regex must be temporarily terminated with +OP_END. + +This function has been extended to cope with forward references for recursions +and subroutine calls. It must check the list of such references for the +group we are dealing with. If it finds that one of the recursions in the +current group is on this list, it does not adjust the value in the reference +(which is a group number). After the group has been scanned, all the offsets in +the forward reference list for the group are adjusted. + +Arguments: + group points to the start of the group + adjust the amount by which the group is to be moved + utf TRUE in UTF-8 / UTF-16 / UTF-32 mode + cd contains pointers to tables etc. + save_hwm_offset the hwm forward reference offset at the start of the group + +Returns: nothing +*/ + +static void +adjust_recurse(pcre_uchar *group, int adjust, BOOL utf, compile_data *cd, + size_t save_hwm_offset) +{ +int offset; +pcre_uchar *hc; +pcre_uchar *ptr = group; + +while ((ptr = (pcre_uchar *)find_recurse(ptr, utf)) != NULL) + { + for (hc = (pcre_uchar *)cd->start_workspace + save_hwm_offset; hc < cd->hwm; + hc += LINK_SIZE) + { + offset = (int)GET(hc, 0); + if (cd->start_code + offset == ptr + 1) break; + } + + /* If we have not found this recursion on the forward reference list, adjust + the recursion's offset if it's after the start of this group. */ + + if (hc >= cd->hwm) + { + offset = (int)GET(ptr, 1); + if (cd->start_code + offset >= group) PUT(ptr, 1, offset + adjust); + } + + ptr += 1 + LINK_SIZE; + } + +/* Now adjust all forward reference offsets for the group. */ + +for (hc = (pcre_uchar *)cd->start_workspace + save_hwm_offset; hc < cd->hwm; + hc += LINK_SIZE) + { + offset = (int)GET(hc, 0); + PUT(hc, 0, offset + adjust); + } +} + + + +/************************************************* +* Insert an automatic callout point * +*************************************************/ + +/* This function is called when the PCRE_AUTO_CALLOUT option is set, to insert +callout points before each pattern item. + +Arguments: + code current code pointer + ptr current pattern pointer + cd pointers to tables etc + +Returns: new code pointer +*/ + +static pcre_uchar * +auto_callout(pcre_uchar *code, const pcre_uchar *ptr, compile_data *cd) +{ +*code++ = OP_CALLOUT; +*code++ = 255; +PUT(code, 0, (int)(ptr - cd->start_pattern)); /* Pattern offset */ +PUT(code, LINK_SIZE, 0); /* Default length */ +return code + 2 * LINK_SIZE; +} + + + +/************************************************* +* Complete a callout item * +*************************************************/ + +/* A callout item contains the length of the next item in the pattern, which +we can't fill in till after we have reached the relevant point. This is used +for both automatic and manual callouts. + +Arguments: + previous_callout points to previous callout item + ptr current pattern pointer + cd pointers to tables etc + +Returns: nothing +*/ + +static void +complete_callout(pcre_uchar *previous_callout, const pcre_uchar *ptr, compile_data *cd) +{ +int length = (int)(ptr - cd->start_pattern - GET(previous_callout, 2)); +PUT(previous_callout, 2 + LINK_SIZE, length); +} + + + +#ifdef SUPPORT_UCP +/************************************************* +* Get othercase range * +*************************************************/ + +/* This function is passed the start and end of a class range, in UTF-8 mode +with UCP support. It searches up the characters, looking for ranges of +characters in the "other" case. Each call returns the next one, updating the +start address. A character with multiple other cases is returned on its own +with a special return value. + +Arguments: + cptr points to starting character value; updated + d end value + ocptr where to put start of othercase range + odptr where to put end of othercase range + +Yield: -1 when no more + 0 when a range is returned + >0 the CASESET offset for char with multiple other cases + in this case, ocptr contains the original +*/ + +static int +get_othercase_range(pcre_uint32 *cptr, pcre_uint32 d, pcre_uint32 *ocptr, + pcre_uint32 *odptr) +{ +pcre_uint32 c, othercase, next; +unsigned int co; + +/* Find the first character that has an other case. If it has multiple other +cases, return its case offset value. */ + +for (c = *cptr; c <= d; c++) + { + if ((co = UCD_CASESET(c)) != 0) + { + *ocptr = c++; /* Character that has the set */ + *cptr = c; /* Rest of input range */ + return (int)co; + } + if ((othercase = UCD_OTHERCASE(c)) != c) break; + } + +if (c > d) return -1; /* Reached end of range */ + +/* Found a character that has a single other case. Search for the end of the +range, which is either the end of the input range, or a character that has zero +or more than one other cases. */ + +*ocptr = othercase; +next = othercase + 1; + +for (++c; c <= d; c++) + { + if ((co = UCD_CASESET(c)) != 0 || UCD_OTHERCASE(c) != next) break; + next++; + } + +*odptr = next - 1; /* End of othercase range */ +*cptr = c; /* Rest of input range */ +return 0; +} +#endif /* SUPPORT_UCP */ + + + +/************************************************* +* Add a character or range to a class * +*************************************************/ + +/* This function packages up the logic of adding a character or range of +characters to a class. The character values in the arguments will be within the +valid values for the current mode (8-bit, 16-bit, UTF, etc). This function is +mutually recursive with the function immediately below. + +Arguments: + classbits the bit map for characters < 256 + uchardptr points to the pointer for extra data + options the options word + cd contains pointers to tables etc. + start start of range character + end end of range character + +Returns: the number of < 256 characters added + the pointer to extra data is updated +*/ + +static int +add_to_class(pcre_uint8 *classbits, pcre_uchar **uchardptr, int options, + compile_data *cd, pcre_uint32 start, pcre_uint32 end) +{ +pcre_uint32 c; +pcre_uint32 classbits_end = (end <= 0xff ? end : 0xff); +int n8 = 0; + +/* If caseless matching is required, scan the range and process alternate +cases. In Unicode, there are 8-bit characters that have alternate cases that +are greater than 255 and vice-versa. Sometimes we can just extend the original +range. */ + +if ((options & PCRE_CASELESS) != 0) + { +#ifdef SUPPORT_UCP + if ((options & PCRE_UTF8) != 0) + { + int rc; + pcre_uint32 oc, od; + + options &= ~PCRE_CASELESS; /* Remove for recursive calls */ + c = start; + + while ((rc = get_othercase_range(&c, end, &oc, &od)) >= 0) + { + /* Handle a single character that has more than one other case. */ + + if (rc > 0) n8 += add_list_to_class(classbits, uchardptr, options, cd, + PRIV(ucd_caseless_sets) + rc, oc); + + /* Do nothing if the other case range is within the original range. */ + + else if (oc >= start && od <= end) continue; + + /* Extend the original range if there is overlap, noting that if oc < c, we + can't have od > end because a subrange is always shorter than the basic + range. Otherwise, use a recursive call to add the additional range. */ + + else if (oc < start && od >= start - 1) start = oc; /* Extend downwards */ + else if (od > end && oc <= end + 1) + { + end = od; /* Extend upwards */ + if (end > classbits_end) classbits_end = (end <= 0xff ? end : 0xff); + } + else n8 += add_to_class(classbits, uchardptr, options, cd, oc, od); + } + } + else +#endif /* SUPPORT_UCP */ + + /* Not UTF-mode, or no UCP */ + + for (c = start; c <= classbits_end; c++) + { + SETBIT(classbits, cd->fcc[c]); + n8++; + } + } + +/* Now handle the original range. Adjust the final value according to the bit +length - this means that the same lists of (e.g.) horizontal spaces can be used +in all cases. */ + +#if defined COMPILE_PCRE8 +#ifdef SUPPORT_UTF + if ((options & PCRE_UTF8) == 0) +#endif + if (end > 0xff) end = 0xff; + +#elif defined COMPILE_PCRE16 +#ifdef SUPPORT_UTF + if ((options & PCRE_UTF16) == 0) +#endif + if (end > 0xffff) end = 0xffff; + +#endif /* COMPILE_PCRE[8|16] */ + +/* Use the bitmap for characters < 256. Otherwise use extra data.*/ + +for (c = start; c <= classbits_end; c++) + { + /* Regardless of start, c will always be <= 255. */ + SETBIT(classbits, c); + n8++; + } + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 +if (start <= 0xff) start = 0xff + 1; + +if (end >= start) + { + pcre_uchar *uchardata = *uchardptr; +#ifdef SUPPORT_UTF + if ((options & PCRE_UTF8) != 0) /* All UTFs use the same flag bit */ + { + if (start < end) + { + *uchardata++ = XCL_RANGE; + uchardata += PRIV(ord2utf)(start, uchardata); + uchardata += PRIV(ord2utf)(end, uchardata); + } + else if (start == end) + { + *uchardata++ = XCL_SINGLE; + uchardata += PRIV(ord2utf)(start, uchardata); + } + } + else +#endif /* SUPPORT_UTF */ + + /* Without UTF support, character values are constrained by the bit length, + and can only be > 256 for 16-bit and 32-bit libraries. */ + +#ifdef COMPILE_PCRE8 + {} +#else + if (start < end) + { + *uchardata++ = XCL_RANGE; + *uchardata++ = start; + *uchardata++ = end; + } + else if (start == end) + { + *uchardata++ = XCL_SINGLE; + *uchardata++ = start; + } +#endif + + *uchardptr = uchardata; /* Updata extra data pointer */ + } +#endif /* SUPPORT_UTF || !COMPILE_PCRE8 */ + +return n8; /* Number of 8-bit characters */ +} + + + + +/************************************************* +* Add a list of characters to a class * +*************************************************/ + +/* This function is used for adding a list of case-equivalent characters to a +class, and also for adding a list of horizontal or vertical whitespace. If the +list is in order (which it should be), ranges of characters are detected and +handled appropriately. This function is mutually recursive with the function +above. + +Arguments: + classbits the bit map for characters < 256 + uchardptr points to the pointer for extra data + options the options word + cd contains pointers to tables etc. + p points to row of 32-bit values, terminated by NOTACHAR + except character to omit; this is used when adding lists of + case-equivalent characters to avoid including the one we + already know about + +Returns: the number of < 256 characters added + the pointer to extra data is updated +*/ + +static int +add_list_to_class(pcre_uint8 *classbits, pcre_uchar **uchardptr, int options, + compile_data *cd, const pcre_uint32 *p, unsigned int except) +{ +int n8 = 0; +while (p[0] < NOTACHAR) + { + int n = 0; + if (p[0] != except) + { + while(p[n+1] == p[0] + n + 1) n++; + n8 += add_to_class(classbits, uchardptr, options, cd, p[0], p[n]); + } + p += n + 1; + } +return n8; +} + + + +/************************************************* +* Add characters not in a list to a class * +*************************************************/ + +/* This function is used for adding the complement of a list of horizontal or +vertical whitespace to a class. The list must be in order. + +Arguments: + classbits the bit map for characters < 256 + uchardptr points to the pointer for extra data + options the options word + cd contains pointers to tables etc. + p points to row of 32-bit values, terminated by NOTACHAR + +Returns: the number of < 256 characters added + the pointer to extra data is updated +*/ + +static int +add_not_list_to_class(pcre_uint8 *classbits, pcre_uchar **uchardptr, + int options, compile_data *cd, const pcre_uint32 *p) +{ +BOOL utf = (options & PCRE_UTF8) != 0; +int n8 = 0; +if (p[0] > 0) + n8 += add_to_class(classbits, uchardptr, options, cd, 0, p[0] - 1); +while (p[0] < NOTACHAR) + { + while (p[1] == p[0] + 1) p++; + n8 += add_to_class(classbits, uchardptr, options, cd, p[0] + 1, + (p[1] == NOTACHAR) ? (utf ? 0x10ffffu : 0xffffffffu) : p[1] - 1); + p++; + } +return n8; +} + + + +/************************************************* +* Compile one branch * +*************************************************/ + +/* Scan the pattern, compiling it into the a vector. If the options are +changed during the branch, the pointer is used to change the external options +bits. This function is used during the pre-compile phase when we are trying +to find out the amount of memory needed, as well as during the real compile +phase. The value of lengthptr distinguishes the two phases. + +Arguments: + optionsptr pointer to the option bits + codeptr points to the pointer to the current code point + ptrptr points to the current pattern pointer + errorcodeptr points to error code variable + firstcharptr place to put the first required character + firstcharflagsptr place to put the first character flags, or a negative number + reqcharptr place to put the last required character + reqcharflagsptr place to put the last required character flags, or a negative number + bcptr points to current branch chain + cond_depth conditional nesting depth + cd contains pointers to tables etc. + lengthptr NULL during the real compile phase + points to length accumulator during pre-compile phase + +Returns: TRUE on success + FALSE, with *errorcodeptr set non-zero on error +*/ + +static BOOL +compile_branch(int *optionsptr, pcre_uchar **codeptr, + const pcre_uchar **ptrptr, int *errorcodeptr, + pcre_uint32 *firstcharptr, pcre_int32 *firstcharflagsptr, + pcre_uint32 *reqcharptr, pcre_int32 *reqcharflagsptr, + branch_chain *bcptr, int cond_depth, + compile_data *cd, int *lengthptr) +{ +int repeat_type, op_type; +int repeat_min = 0, repeat_max = 0; /* To please picky compilers */ +int bravalue = 0; +int greedy_default, greedy_non_default; +pcre_uint32 firstchar, reqchar; +pcre_int32 firstcharflags, reqcharflags; +pcre_uint32 zeroreqchar, zerofirstchar; +pcre_int32 zeroreqcharflags, zerofirstcharflags; +pcre_int32 req_caseopt, reqvary, tempreqvary; +int options = *optionsptr; /* May change dynamically */ +int after_manual_callout = 0; +int length_prevgroup = 0; +register pcre_uint32 c; +int escape; +register pcre_uchar *code = *codeptr; +pcre_uchar *last_code = code; +pcre_uchar *orig_code = code; +pcre_uchar *tempcode; +BOOL inescq = FALSE; +BOOL groupsetfirstchar = FALSE; +const pcre_uchar *ptr = *ptrptr; +const pcre_uchar *tempptr; +const pcre_uchar *nestptr = NULL; +pcre_uchar *previous = NULL; +pcre_uchar *previous_callout = NULL; +size_t item_hwm_offset = 0; +pcre_uint8 classbits[32]; + +/* We can fish out the UTF-8 setting once and for all into a BOOL, but we +must not do this for other options (e.g. PCRE_EXTENDED) because they may change +dynamically as we process the pattern. */ + +#ifdef SUPPORT_UTF +/* PCRE_UTF[16|32] have the same value as PCRE_UTF8. */ +BOOL utf = (options & PCRE_UTF8) != 0; +#ifndef COMPILE_PCRE32 +pcre_uchar utf_chars[6]; +#endif +#else +BOOL utf = FALSE; +#endif + +/* Helper variables for OP_XCLASS opcode (for characters > 255). We define +class_uchardata always so that it can be passed to add_to_class() always, +though it will not be used in non-UTF 8-bit cases. This avoids having to supply +alternative calls for the different cases. */ + +pcre_uchar *class_uchardata; +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 +BOOL xclass; +pcre_uchar *class_uchardata_base; +#endif + +#ifdef PCRE_DEBUG +if (lengthptr != NULL) DPRINTF((">> start branch\n")); +#endif + +/* Set up the default and non-default settings for greediness */ + +greedy_default = ((options & PCRE_UNGREEDY) != 0); +greedy_non_default = greedy_default ^ 1; + +/* Initialize no first byte, no required byte. REQ_UNSET means "no char +matching encountered yet". It gets changed to REQ_NONE if we hit something that +matches a non-fixed char first char; reqchar just remains unset if we never +find one. + +When we hit a repeat whose minimum is zero, we may have to adjust these values +to take the zero repeat into account. This is implemented by setting them to +zerofirstbyte and zeroreqchar when such a repeat is encountered. The individual +item types that can be repeated set these backoff variables appropriately. */ + +firstchar = reqchar = zerofirstchar = zeroreqchar = 0; +firstcharflags = reqcharflags = zerofirstcharflags = zeroreqcharflags = REQ_UNSET; + +/* The variable req_caseopt contains either the REQ_CASELESS value +or zero, according to the current setting of the caseless flag. The +REQ_CASELESS leaves the lower 28 bit empty. It is added into the +firstchar or reqchar variables to record the case status of the +value. This is used only for ASCII characters. */ + +req_caseopt = ((options & PCRE_CASELESS) != 0)? REQ_CASELESS:0; + +/* Switch on next character until the end of the branch */ + +for (;; ptr++) + { + BOOL negate_class; + BOOL should_flip_negation; + BOOL possessive_quantifier; + BOOL is_quantifier; + BOOL is_recurse; + BOOL reset_bracount; + int class_has_8bitchar; + int class_one_char; +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + BOOL xclass_has_prop; +#endif + int newoptions; + int recno; + int refsign; + int skipbytes; + pcre_uint32 subreqchar, subfirstchar; + pcre_int32 subreqcharflags, subfirstcharflags; + int terminator; + unsigned int mclength; + unsigned int tempbracount; + pcre_uint32 ec; + pcre_uchar mcbuffer[8]; + + /* Come here to restart the loop without advancing the pointer. */ + + REDO_LOOP: + + /* Get next character in the pattern */ + + c = *ptr; + + /* If we are at the end of a nested substitution, revert to the outer level + string. Nesting only happens one level deep. */ + + if (c == CHAR_NULL && nestptr != NULL) + { + ptr = nestptr; + nestptr = NULL; + c = *ptr; + } + + /* If we are in the pre-compile phase, accumulate the length used for the + previous cycle of this loop. */ + + if (lengthptr != NULL) + { +#ifdef PCRE_DEBUG + if (code > cd->hwm) cd->hwm = code; /* High water info */ +#endif + if (code > cd->start_workspace + cd->workspace_size - + WORK_SIZE_SAFETY_MARGIN) /* Check for overrun */ + { + *errorcodeptr = (code >= cd->start_workspace + cd->workspace_size)? + ERR52 : ERR87; + goto FAILED; + } + + /* There is at least one situation where code goes backwards: this is the + case of a zero quantifier after a class (e.g. [ab]{0}). At compile time, + the class is simply eliminated. However, it is created first, so we have to + allow memory for it. Therefore, don't ever reduce the length at this point. + */ + + if (code < last_code) code = last_code; + + /* Paranoid check for integer overflow */ + + if (OFLOW_MAX - *lengthptr < code - last_code) + { + *errorcodeptr = ERR20; + goto FAILED; + } + + *lengthptr += (int)(code - last_code); + DPRINTF(("length=%d added %d c=%c (0x%x)\n", *lengthptr, + (int)(code - last_code), c, c)); + + /* If "previous" is set and it is not at the start of the work space, move + it back to there, in order to avoid filling up the work space. Otherwise, + if "previous" is NULL, reset the current code pointer to the start. */ + + if (previous != NULL) + { + if (previous > orig_code) + { + memmove(orig_code, previous, IN_UCHARS(code - previous)); + code -= previous - orig_code; + previous = orig_code; + } + } + else code = orig_code; + + /* Remember where this code item starts so we can pick up the length + next time round. */ + + last_code = code; + } + + /* In the real compile phase, just check the workspace used by the forward + reference list. */ + + else if (cd->hwm > cd->start_workspace + cd->workspace_size) + { + *errorcodeptr = ERR52; + goto FAILED; + } + + /* If in \Q...\E, check for the end; if not, we have a literal. Otherwise an + isolated \E is ignored. */ + + if (c != CHAR_NULL) + { + if (c == CHAR_BACKSLASH && ptr[1] == CHAR_E) + { + inescq = FALSE; + ptr++; + continue; + } + else if (inescq) + { + if (previous_callout != NULL) + { + if (lengthptr == NULL) /* Don't attempt in pre-compile phase */ + complete_callout(previous_callout, ptr, cd); + previous_callout = NULL; + } + if ((options & PCRE_AUTO_CALLOUT) != 0) + { + previous_callout = code; + code = auto_callout(code, ptr, cd); + } + goto NORMAL_CHAR; + } + + /* Check for the start of a \Q...\E sequence. We must do this here rather + than later in case it is immediately followed by \E, which turns it into a + "do nothing" sequence. */ + + if (c == CHAR_BACKSLASH && ptr[1] == CHAR_Q) + { + inescq = TRUE; + ptr++; + continue; + } + } + + /* In extended mode, skip white space and comments. */ + + if ((options & PCRE_EXTENDED) != 0) + { + const pcre_uchar *wscptr = ptr; + while (MAX_255(c) && (cd->ctypes[c] & ctype_space) != 0) c = *(++ptr); + if (c == CHAR_NUMBER_SIGN) + { + ptr++; + while (*ptr != CHAR_NULL) + { + if (IS_NEWLINE(ptr)) /* For non-fixed-length newline cases, */ + { /* IS_NEWLINE sets cd->nllen. */ + ptr += cd->nllen; + break; + } + ptr++; +#ifdef SUPPORT_UTF + if (utf) FORWARDCHAR(ptr); +#endif + } + } + + /* If we skipped any characters, restart the loop. Otherwise, we didn't see + a comment. */ + + if (ptr > wscptr) goto REDO_LOOP; + } + + /* Skip over (?# comments. We need to do this here because we want to know if + the next thing is a quantifier, and these comments may come between an item + and its quantifier. */ + + if (c == CHAR_LEFT_PARENTHESIS && ptr[1] == CHAR_QUESTION_MARK && + ptr[2] == CHAR_NUMBER_SIGN) + { + ptr += 3; + while (*ptr != CHAR_NULL && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++; + if (*ptr == CHAR_NULL) + { + *errorcodeptr = ERR18; + goto FAILED; + } + continue; + } + + /* See if the next thing is a quantifier. */ + + is_quantifier = + c == CHAR_ASTERISK || c == CHAR_PLUS || c == CHAR_QUESTION_MARK || + (c == CHAR_LEFT_CURLY_BRACKET && is_counted_repeat(ptr+1)); + + /* Fill in length of a previous callout, except when the next thing is a + quantifier or when processing a property substitution string in UCP mode. */ + + if (!is_quantifier && previous_callout != NULL && nestptr == NULL && + after_manual_callout-- <= 0) + { + if (lengthptr == NULL) /* Don't attempt in pre-compile phase */ + complete_callout(previous_callout, ptr, cd); + previous_callout = NULL; + } + + /* Create auto callout, except for quantifiers, or while processing property + strings that are substituted for \w etc in UCP mode. */ + + if ((options & PCRE_AUTO_CALLOUT) != 0 && !is_quantifier && nestptr == NULL) + { + previous_callout = code; + code = auto_callout(code, ptr, cd); + } + + /* Process the next pattern item. */ + + switch(c) + { + /* ===================================================================*/ + case CHAR_NULL: /* The branch terminates at string end */ + case CHAR_VERTICAL_LINE: /* or | or ) */ + case CHAR_RIGHT_PARENTHESIS: + *firstcharptr = firstchar; + *firstcharflagsptr = firstcharflags; + *reqcharptr = reqchar; + *reqcharflagsptr = reqcharflags; + *codeptr = code; + *ptrptr = ptr; + if (lengthptr != NULL) + { + if (OFLOW_MAX - *lengthptr < code - last_code) + { + *errorcodeptr = ERR20; + goto FAILED; + } + *lengthptr += (int)(code - last_code); /* To include callout length */ + DPRINTF((">> end branch\n")); + } + return TRUE; + + + /* ===================================================================*/ + /* Handle single-character metacharacters. In multiline mode, ^ disables + the setting of any following char as a first character. */ + + case CHAR_CIRCUMFLEX_ACCENT: + previous = NULL; + if ((options & PCRE_MULTILINE) != 0) + { + if (firstcharflags == REQ_UNSET) + zerofirstcharflags = firstcharflags = REQ_NONE; + *code++ = OP_CIRCM; + } + else *code++ = OP_CIRC; + break; + + case CHAR_DOLLAR_SIGN: + previous = NULL; + *code++ = ((options & PCRE_MULTILINE) != 0)? OP_DOLLM : OP_DOLL; + break; + + /* There can never be a first char if '.' is first, whatever happens about + repeats. The value of reqchar doesn't change either. */ + + case CHAR_DOT: + if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; + zerofirstchar = firstchar; + zerofirstcharflags = firstcharflags; + zeroreqchar = reqchar; + zeroreqcharflags = reqcharflags; + previous = code; + item_hwm_offset = cd->hwm - cd->start_workspace; + *code++ = ((options & PCRE_DOTALL) != 0)? OP_ALLANY: OP_ANY; + break; + + + /* ===================================================================*/ + /* Character classes. If the included characters are all < 256, we build a + 32-byte bitmap of the permitted characters, except in the special case + where there is only one such character. For negated classes, we build the + map as usual, then invert it at the end. However, we use a different opcode + so that data characters > 255 can be handled correctly. + + If the class contains characters outside the 0-255 range, a different + opcode is compiled. It may optionally have a bit map for characters < 256, + but those above are are explicitly listed afterwards. A flag byte tells + whether the bitmap is present, and whether this is a negated class or not. + + In JavaScript compatibility mode, an isolated ']' causes an error. In + default (Perl) mode, it is treated as a data character. */ + + case CHAR_RIGHT_SQUARE_BRACKET: + if ((cd->external_options & PCRE_JAVASCRIPT_COMPAT) != 0) + { + *errorcodeptr = ERR64; + goto FAILED; + } + goto NORMAL_CHAR; + + /* In another (POSIX) regex library, the ugly syntax [[:<:]] and [[:>:]] is + used for "start of word" and "end of word". As these are otherwise illegal + sequences, we don't break anything by recognizing them. They are replaced + by \b(?=\w) and \b(?<=\w) respectively. Sequences like [a[:<:]] are + erroneous and are handled by the normal code below. */ + + case CHAR_LEFT_SQUARE_BRACKET: + if (STRNCMP_UC_C8(ptr+1, STRING_WEIRD_STARTWORD, 6) == 0) + { + nestptr = ptr + 7; + ptr = sub_start_of_word; + goto REDO_LOOP; + } + + if (STRNCMP_UC_C8(ptr+1, STRING_WEIRD_ENDWORD, 6) == 0) + { + nestptr = ptr + 7; + ptr = sub_end_of_word; + goto REDO_LOOP; + } + + /* Handle a real character class. */ + + previous = code; + item_hwm_offset = cd->hwm - cd->start_workspace; + + /* PCRE supports POSIX class stuff inside a class. Perl gives an error if + they are encountered at the top level, so we'll do that too. */ + + if ((ptr[1] == CHAR_COLON || ptr[1] == CHAR_DOT || + ptr[1] == CHAR_EQUALS_SIGN) && + check_posix_syntax(ptr, &tempptr)) + { + *errorcodeptr = (ptr[1] == CHAR_COLON)? ERR13 : ERR31; + goto FAILED; + } + + /* If the first character is '^', set the negation flag and skip it. Also, + if the first few characters (either before or after ^) are \Q\E or \E we + skip them too. This makes for compatibility with Perl. */ + + negate_class = FALSE; + for (;;) + { + c = *(++ptr); + if (c == CHAR_BACKSLASH) + { + if (ptr[1] == CHAR_E) + ptr++; + else if (STRNCMP_UC_C8(ptr + 1, STR_Q STR_BACKSLASH STR_E, 3) == 0) + ptr += 3; + else + break; + } + else if (!negate_class && c == CHAR_CIRCUMFLEX_ACCENT) + negate_class = TRUE; + else break; + } + + /* Empty classes are allowed in JavaScript compatibility mode. Otherwise, + an initial ']' is taken as a data character -- the code below handles + that. In JS mode, [] must always fail, so generate OP_FAIL, whereas + [^] must match any character, so generate OP_ALLANY. */ + + if (c == CHAR_RIGHT_SQUARE_BRACKET && + (cd->external_options & PCRE_JAVASCRIPT_COMPAT) != 0) + { + *code++ = negate_class? OP_ALLANY : OP_FAIL; + if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; + zerofirstchar = firstchar; + zerofirstcharflags = firstcharflags; + break; + } + + /* If a class contains a negative special such as \S, we need to flip the + negation flag at the end, so that support for characters > 255 works + correctly (they are all included in the class). */ + + should_flip_negation = FALSE; + + /* Extended class (xclass) will be used when characters > 255 + might match. */ + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + xclass = FALSE; + class_uchardata = code + LINK_SIZE + 2; /* For XCLASS items */ + class_uchardata_base = class_uchardata; /* Save the start */ +#endif + + /* For optimization purposes, we track some properties of the class: + class_has_8bitchar will be non-zero if the class contains at least one < + 256 character; class_one_char will be 1 if the class contains just one + character; xclass_has_prop will be TRUE if unicode property checks + are present in the class. */ + + class_has_8bitchar = 0; + class_one_char = 0; +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + xclass_has_prop = FALSE; +#endif + + /* Initialize the 32-char bit map to all zeros. We build the map in a + temporary bit of memory, in case the class contains fewer than two + 8-bit characters because in that case the compiled code doesn't use the bit + map. */ + + memset(classbits, 0, 32 * sizeof(pcre_uint8)); + + /* Process characters until ] is reached. By writing this as a "do" it + means that an initial ] is taken as a data character. At the start of the + loop, c contains the first byte of the character. */ + + if (c != CHAR_NULL) do + { + const pcre_uchar *oldptr; + +#ifdef SUPPORT_UTF + if (utf && HAS_EXTRALEN(c)) + { /* Braces are required because the */ + GETCHARLEN(c, ptr, ptr); /* macro generates multiple statements */ + } +#endif + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + /* In the pre-compile phase, accumulate the length of any extra + data and reset the pointer. This is so that very large classes that + contain a zillion > 255 characters no longer overwrite the work space + (which is on the stack). We have to remember that there was XCLASS data, + however. */ + + if (class_uchardata > class_uchardata_base) xclass = TRUE; + + if (lengthptr != NULL && class_uchardata > class_uchardata_base) + { + *lengthptr += (int)(class_uchardata - class_uchardata_base); + class_uchardata = class_uchardata_base; + } +#endif + + /* Inside \Q...\E everything is literal except \E */ + + if (inescq) + { + if (c == CHAR_BACKSLASH && ptr[1] == CHAR_E) /* If we are at \E */ + { + inescq = FALSE; /* Reset literal state */ + ptr++; /* Skip the 'E' */ + continue; /* Carry on with next */ + } + goto CHECK_RANGE; /* Could be range if \E follows */ + } + + /* Handle POSIX class names. Perl allows a negation extension of the + form [:^name:]. A square bracket that doesn't match the syntax is + treated as a literal. We also recognize the POSIX constructions + [.ch.] and [=ch=] ("collating elements") and fault them, as Perl + 5.6 and 5.8 do. */ + + if (c == CHAR_LEFT_SQUARE_BRACKET && + (ptr[1] == CHAR_COLON || ptr[1] == CHAR_DOT || + ptr[1] == CHAR_EQUALS_SIGN) && check_posix_syntax(ptr, &tempptr)) + { + BOOL local_negate = FALSE; + int posix_class, taboffset, tabopt; + register const pcre_uint8 *cbits = cd->cbits; + pcre_uint8 pbits[32]; + + if (ptr[1] != CHAR_COLON) + { + *errorcodeptr = ERR31; + goto FAILED; + } + + ptr += 2; + if (*ptr == CHAR_CIRCUMFLEX_ACCENT) + { + local_negate = TRUE; + should_flip_negation = TRUE; /* Note negative special */ + ptr++; + } + + posix_class = check_posix_name(ptr, (int)(tempptr - ptr)); + if (posix_class < 0) + { + *errorcodeptr = ERR30; + goto FAILED; + } + + /* If matching is caseless, upper and lower are converted to + alpha. This relies on the fact that the class table starts with + alpha, lower, upper as the first 3 entries. */ + + if ((options & PCRE_CASELESS) != 0 && posix_class <= 2) + posix_class = 0; + + /* When PCRE_UCP is set, some of the POSIX classes are converted to + different escape sequences that use Unicode properties \p or \P. Others + that are not available via \p or \P generate XCL_PROP/XCL_NOTPROP + directly. */ + +#ifdef SUPPORT_UCP + if ((options & PCRE_UCP) != 0) + { + unsigned int ptype = 0; + int pc = posix_class + ((local_negate)? POSIX_SUBSIZE/2 : 0); + + /* The posix_substitutes table specifies which POSIX classes can be + converted to \p or \P items. */ + + if (posix_substitutes[pc] != NULL) + { + nestptr = tempptr + 1; + ptr = posix_substitutes[pc] - 1; + continue; + } + + /* There are three other classes that generate special property calls + that are recognized only in an XCLASS. */ + + else switch(posix_class) + { + case PC_GRAPH: + ptype = PT_PXGRAPH; + /* Fall through */ + case PC_PRINT: + if (ptype == 0) ptype = PT_PXPRINT; + /* Fall through */ + case PC_PUNCT: + if (ptype == 0) ptype = PT_PXPUNCT; + *class_uchardata++ = local_negate? XCL_NOTPROP : XCL_PROP; + *class_uchardata++ = ptype; + *class_uchardata++ = 0; + xclass_has_prop = TRUE; + ptr = tempptr + 1; + continue; + + /* For the other POSIX classes (ascii, cntrl, xdigit) we are going + to fall through to the non-UCP case and build a bit map for + characters with code points less than 256. If we are in a negated + POSIX class, characters with code points greater than 255 must + either all match or all not match. In the special case where we + have not yet generated any xclass data, and this is the final item + in the overall class, we need do nothing: later on, the opcode + OP_NCLASS will be used to indicate that characters greater than 255 + are acceptable. If we have already seen an xclass item or one may + follow (we have to assume that it might if this is not the end of + the class), explicitly list all wide codepoints, which will then + either not match or match, depending on whether the class is or is + not negated. */ + + default: + if (local_negate && + (xclass || tempptr[2] != CHAR_RIGHT_SQUARE_BRACKET)) + { + *class_uchardata++ = XCL_RANGE; + class_uchardata += PRIV(ord2utf)(0x100, class_uchardata); + class_uchardata += PRIV(ord2utf)(0x10ffff, class_uchardata); + } + break; + } + } +#endif + /* In the non-UCP case, or when UCP makes no difference, we build the + bit map for the POSIX class in a chunk of local store because we may be + adding and subtracting from it, and we don't want to subtract bits that + may be in the main map already. At the end we or the result into the + bit map that is being built. */ + + posix_class *= 3; + + /* Copy in the first table (always present) */ + + memcpy(pbits, cbits + posix_class_maps[posix_class], + 32 * sizeof(pcre_uint8)); + + /* If there is a second table, add or remove it as required. */ + + taboffset = posix_class_maps[posix_class + 1]; + tabopt = posix_class_maps[posix_class + 2]; + + if (taboffset >= 0) + { + if (tabopt >= 0) + for (c = 0; c < 32; c++) pbits[c] |= cbits[c + taboffset]; + else + for (c = 0; c < 32; c++) pbits[c] &= ~cbits[c + taboffset]; + } + + /* Now see if we need to remove any special characters. An option + value of 1 removes vertical space and 2 removes underscore. */ + + if (tabopt < 0) tabopt = -tabopt; + if (tabopt == 1) pbits[1] &= ~0x3c; + else if (tabopt == 2) pbits[11] &= 0x7f; + + /* Add the POSIX table or its complement into the main table that is + being built and we are done. */ + + if (local_negate) + for (c = 0; c < 32; c++) classbits[c] |= ~pbits[c]; + else + for (c = 0; c < 32; c++) classbits[c] |= pbits[c]; + + ptr = tempptr + 1; + /* Every class contains at least one < 256 character. */ + class_has_8bitchar = 1; + /* Every class contains at least two characters. */ + class_one_char = 2; + continue; /* End of POSIX syntax handling */ + } + + /* Backslash may introduce a single character, or it may introduce one + of the specials, which just set a flag. The sequence \b is a special + case. Inside a class (and only there) it is treated as backspace. We + assume that other escapes have more than one character in them, so + speculatively set both class_has_8bitchar and class_one_char bigger + than one. Unrecognized escapes fall through and are either treated + as literal characters (by default), or are faulted if + PCRE_EXTRA is set. */ + + if (c == CHAR_BACKSLASH) + { + escape = check_escape(&ptr, &ec, errorcodeptr, cd->bracount, options, + TRUE); + if (*errorcodeptr != 0) goto FAILED; + if (escape == 0) c = ec; + else if (escape == ESC_b) c = CHAR_BS; /* \b is backspace in a class */ + else if (escape == ESC_N) /* \N is not supported in a class */ + { + *errorcodeptr = ERR71; + goto FAILED; + } + else if (escape == ESC_Q) /* Handle start of quoted string */ + { + if (ptr[1] == CHAR_BACKSLASH && ptr[2] == CHAR_E) + { + ptr += 2; /* avoid empty string */ + } + else inescq = TRUE; + continue; + } + else if (escape == ESC_E) continue; /* Ignore orphan \E */ + + else + { + register const pcre_uint8 *cbits = cd->cbits; + /* Every class contains at least two < 256 characters. */ + class_has_8bitchar++; + /* Every class contains at least two characters. */ + class_one_char += 2; + + switch (escape) + { +#ifdef SUPPORT_UCP + case ESC_du: /* These are the values given for \d etc */ + case ESC_DU: /* when PCRE_UCP is set. We replace the */ + case ESC_wu: /* escape sequence with an appropriate \p */ + case ESC_WU: /* or \P to test Unicode properties instead */ + case ESC_su: /* of the default ASCII testing. */ + case ESC_SU: + nestptr = ptr; + ptr = substitutes[escape - ESC_DU] - 1; /* Just before substitute */ + class_has_8bitchar--; /* Undo! */ + continue; +#endif + case ESC_d: + for (c = 0; c < 32; c++) classbits[c] |= cbits[c+cbit_digit]; + continue; + + case ESC_D: + should_flip_negation = TRUE; + for (c = 0; c < 32; c++) classbits[c] |= ~cbits[c+cbit_digit]; + continue; + + case ESC_w: + for (c = 0; c < 32; c++) classbits[c] |= cbits[c+cbit_word]; + continue; + + case ESC_W: + should_flip_negation = TRUE; + for (c = 0; c < 32; c++) classbits[c] |= ~cbits[c+cbit_word]; + continue; + + /* Perl 5.004 onwards omitted VT from \s, but restored it at Perl + 5.18. Before PCRE 8.34, we had to preserve the VT bit if it was + previously set by something earlier in the character class. + Luckily, the value of CHAR_VT is 0x0b in both ASCII and EBCDIC, so + we could just adjust the appropriate bit. From PCRE 8.34 we no + longer treat \s and \S specially. */ + + case ESC_s: + for (c = 0; c < 32; c++) classbits[c] |= cbits[c+cbit_space]; + continue; + + case ESC_S: + should_flip_negation = TRUE; + for (c = 0; c < 32; c++) classbits[c] |= ~cbits[c+cbit_space]; + continue; + + /* The rest apply in both UCP and non-UCP cases. */ + + case ESC_h: + (void)add_list_to_class(classbits, &class_uchardata, options, cd, + PRIV(hspace_list), NOTACHAR); + continue; + + case ESC_H: + (void)add_not_list_to_class(classbits, &class_uchardata, options, + cd, PRIV(hspace_list)); + continue; + + case ESC_v: + (void)add_list_to_class(classbits, &class_uchardata, options, cd, + PRIV(vspace_list), NOTACHAR); + continue; + + case ESC_V: + (void)add_not_list_to_class(classbits, &class_uchardata, options, + cd, PRIV(vspace_list)); + continue; + + case ESC_p: + case ESC_P: +#ifdef SUPPORT_UCP + { + BOOL negated; + unsigned int ptype = 0, pdata = 0; + if (!get_ucp(&ptr, &negated, &ptype, &pdata, errorcodeptr)) + goto FAILED; + *class_uchardata++ = ((escape == ESC_p) != negated)? + XCL_PROP : XCL_NOTPROP; + *class_uchardata++ = ptype; + *class_uchardata++ = pdata; + xclass_has_prop = TRUE; + class_has_8bitchar--; /* Undo! */ + continue; + } +#else + *errorcodeptr = ERR45; + goto FAILED; +#endif + /* Unrecognized escapes are faulted if PCRE is running in its + strict mode. By default, for compatibility with Perl, they are + treated as literals. */ + + default: + if ((options & PCRE_EXTRA) != 0) + { + *errorcodeptr = ERR7; + goto FAILED; + } + class_has_8bitchar--; /* Undo the speculative increase. */ + class_one_char -= 2; /* Undo the speculative increase. */ + c = *ptr; /* Get the final character and fall through */ + break; + } + } + + /* Fall through if the escape just defined a single character (c >= 0). + This may be greater than 256. */ + + escape = 0; + + } /* End of backslash handling */ + + /* A character may be followed by '-' to form a range. However, Perl does + not permit ']' to be the end of the range. A '-' character at the end is + treated as a literal. Perl ignores orphaned \E sequences entirely. The + code for handling \Q and \E is messy. */ + + CHECK_RANGE: + while (ptr[1] == CHAR_BACKSLASH && ptr[2] == CHAR_E) + { + inescq = FALSE; + ptr += 2; + } + oldptr = ptr; + + /* Remember if \r or \n were explicitly used */ + + if (c == CHAR_CR || c == CHAR_NL) cd->external_flags |= PCRE_HASCRORLF; + + /* Check for range */ + + if (!inescq && ptr[1] == CHAR_MINUS) + { + pcre_uint32 d; + ptr += 2; + while (*ptr == CHAR_BACKSLASH && ptr[1] == CHAR_E) ptr += 2; + + /* If we hit \Q (not followed by \E) at this point, go into escaped + mode. */ + + while (*ptr == CHAR_BACKSLASH && ptr[1] == CHAR_Q) + { + ptr += 2; + if (*ptr == CHAR_BACKSLASH && ptr[1] == CHAR_E) + { ptr += 2; continue; } + inescq = TRUE; + break; + } + + /* Minus (hyphen) at the end of a class is treated as a literal, so put + back the pointer and jump to handle the character that preceded it. */ + + if (*ptr == CHAR_NULL || (!inescq && *ptr == CHAR_RIGHT_SQUARE_BRACKET)) + { + ptr = oldptr; + goto CLASS_SINGLE_CHARACTER; + } + + /* Otherwise, we have a potential range; pick up the next character */ + +#ifdef SUPPORT_UTF + if (utf) + { /* Braces are required because the */ + GETCHARLEN(d, ptr, ptr); /* macro generates multiple statements */ + } + else +#endif + d = *ptr; /* Not UTF-8 mode */ + + /* The second part of a range can be a single-character escape + sequence, but not any of the other escapes. Perl treats a hyphen as a + literal in such circumstances. However, in Perl's warning mode, a + warning is given, so PCRE now faults it as it is almost certainly a + mistake on the user's part. */ + + if (!inescq) + { + if (d == CHAR_BACKSLASH) + { + int descape; + descape = check_escape(&ptr, &d, errorcodeptr, cd->bracount, options, TRUE); + if (*errorcodeptr != 0) goto FAILED; + + /* 0 means a character was put into d; \b is backspace; any other + special causes an error. */ + + if (descape != 0) + { + if (descape == ESC_b) d = CHAR_BS; else + { + *errorcodeptr = ERR83; + goto FAILED; + } + } + } + + /* A hyphen followed by a POSIX class is treated in the same way. */ + + else if (d == CHAR_LEFT_SQUARE_BRACKET && + (ptr[1] == CHAR_COLON || ptr[1] == CHAR_DOT || + ptr[1] == CHAR_EQUALS_SIGN) && + check_posix_syntax(ptr, &tempptr)) + { + *errorcodeptr = ERR83; + goto FAILED; + } + } + + /* Check that the two values are in the correct order. Optimize + one-character ranges. */ + + if (d < c) + { + *errorcodeptr = ERR8; + goto FAILED; + } + if (d == c) goto CLASS_SINGLE_CHARACTER; /* A few lines below */ + + /* We have found a character range, so single character optimizations + cannot be done anymore. Any value greater than 1 indicates that there + is more than one character. */ + + class_one_char = 2; + + /* Remember an explicit \r or \n, and add the range to the class. */ + + if (d == CHAR_CR || d == CHAR_NL) cd->external_flags |= PCRE_HASCRORLF; + + class_has_8bitchar += + add_to_class(classbits, &class_uchardata, options, cd, c, d); + + continue; /* Go get the next char in the class */ + } + + /* Handle a single character - we can get here for a normal non-escape + char, or after \ that introduces a single character or for an apparent + range that isn't. Only the value 1 matters for class_one_char, so don't + increase it if it is already 2 or more ... just in case there's a class + with a zillion characters in it. */ + + CLASS_SINGLE_CHARACTER: + if (class_one_char < 2) class_one_char++; + + /* If xclass_has_prop is false and class_one_char is 1, we have the first + single character in the class, and there have been no prior ranges, or + XCLASS items generated by escapes. If this is the final character in the + class, we can optimize by turning the item into a 1-character OP_CHAR[I] + if it's positive, or OP_NOT[I] if it's negative. In the positive case, it + can cause firstchar to be set. Otherwise, there can be no first char if + this item is first, whatever repeat count may follow. In the case of + reqchar, save the previous value for reinstating. */ + + if (!inescq && +#ifdef SUPPORT_UCP + !xclass_has_prop && +#endif + class_one_char == 1 && ptr[1] == CHAR_RIGHT_SQUARE_BRACKET) + { + ptr++; + zeroreqchar = reqchar; + zeroreqcharflags = reqcharflags; + + if (negate_class) + { +#ifdef SUPPORT_UCP + int d; +#endif + if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; + zerofirstchar = firstchar; + zerofirstcharflags = firstcharflags; + + /* For caseless UTF-8 mode when UCP support is available, check + whether this character has more than one other case. If so, generate + a special OP_NOTPROP item instead of OP_NOTI. */ + +#ifdef SUPPORT_UCP + if (utf && (options & PCRE_CASELESS) != 0 && + (d = UCD_CASESET(c)) != 0) + { + *code++ = OP_NOTPROP; + *code++ = PT_CLIST; + *code++ = d; + } + else +#endif + /* Char has only one other case, or UCP not available */ + + { + *code++ = ((options & PCRE_CASELESS) != 0)? OP_NOTI: OP_NOT; +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (utf && c > MAX_VALUE_FOR_SINGLE_CHAR) + code += PRIV(ord2utf)(c, code); + else +#endif + *code++ = c; + } + + /* We are finished with this character class */ + + goto END_CLASS; + } + + /* For a single, positive character, get the value into mcbuffer, and + then we can handle this with the normal one-character code. */ + +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (utf && c > MAX_VALUE_FOR_SINGLE_CHAR) + mclength = PRIV(ord2utf)(c, mcbuffer); + else +#endif + { + mcbuffer[0] = c; + mclength = 1; + } + goto ONE_CHAR; + } /* End of 1-char optimization */ + + /* There is more than one character in the class, or an XCLASS item + has been generated. Add this character to the class. */ + + class_has_8bitchar += + add_to_class(classbits, &class_uchardata, options, cd, c, c); + } + + /* Loop until ']' reached. This "while" is the end of the "do" far above. + If we are at the end of an internal nested string, revert to the outer + string. */ + + while (((c = *(++ptr)) != CHAR_NULL || + (nestptr != NULL && + (ptr = nestptr, nestptr = NULL, c = *(++ptr)) != CHAR_NULL)) && + (c != CHAR_RIGHT_SQUARE_BRACKET || inescq)); + + /* Check for missing terminating ']' */ + + if (c == CHAR_NULL) + { + *errorcodeptr = ERR6; + goto FAILED; + } + + /* We will need an XCLASS if data has been placed in class_uchardata. In + the second phase this is a sufficient test. However, in the pre-compile + phase, class_uchardata gets emptied to prevent workspace overflow, so it + only if the very last character in the class needs XCLASS will it contain + anything at this point. For this reason, xclass gets set TRUE above when + uchar_classdata is emptied, and that's why this code is the way it is here + instead of just doing a test on class_uchardata below. */ + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + if (class_uchardata > class_uchardata_base) xclass = TRUE; +#endif + + /* If this is the first thing in the branch, there can be no first char + setting, whatever the repeat count. Any reqchar setting must remain + unchanged after any kind of repeat. */ + + if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; + zerofirstchar = firstchar; + zerofirstcharflags = firstcharflags; + zeroreqchar = reqchar; + zeroreqcharflags = reqcharflags; + + /* If there are characters with values > 255, we have to compile an + extended class, with its own opcode, unless there was a negated special + such as \S in the class, and PCRE_UCP is not set, because in that case all + characters > 255 are in the class, so any that were explicitly given as + well can be ignored. If (when there are explicit characters > 255 that must + be listed) there are no characters < 256, we can omit the bitmap in the + actual compiled code. */ + +#ifdef SUPPORT_UTF + if (xclass && (xclass_has_prop || !should_flip_negation || + (options & PCRE_UCP) != 0)) +#elif !defined COMPILE_PCRE8 + if (xclass && (xclass_has_prop || !should_flip_negation)) +#endif +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + { + /* For non-UCP wide characters, in a non-negative class containing \S or + similar (should_flip_negation is set), all characters greater than 255 + must be in the class. */ + + if ( +#if defined COMPILE_PCRE8 + utf && +#endif + should_flip_negation && !negate_class && (options & PCRE_UCP) == 0) + { + *class_uchardata++ = XCL_RANGE; + if (utf) /* Will always be utf in the 8-bit library */ + { + class_uchardata += PRIV(ord2utf)(0x100, class_uchardata); + class_uchardata += PRIV(ord2utf)(0x10ffff, class_uchardata); + } + else /* Can only happen for the 16-bit & 32-bit libraries */ + { +#if defined COMPILE_PCRE16 + *class_uchardata++ = 0x100; + *class_uchardata++ = 0xffffu; +#elif defined COMPILE_PCRE32 + *class_uchardata++ = 0x100; + *class_uchardata++ = 0xffffffffu; +#endif + } + } + + *class_uchardata++ = XCL_END; /* Marks the end of extra data */ + *code++ = OP_XCLASS; + code += LINK_SIZE; + *code = negate_class? XCL_NOT:0; + if (xclass_has_prop) *code |= XCL_HASPROP; + + /* If the map is required, move up the extra data to make room for it; + otherwise just move the code pointer to the end of the extra data. */ + + if (class_has_8bitchar > 0) + { + *code++ |= XCL_MAP; + memmove(code + (32 / sizeof(pcre_uchar)), code, + IN_UCHARS(class_uchardata - code)); + if (negate_class && !xclass_has_prop) + for (c = 0; c < 32; c++) classbits[c] = ~classbits[c]; + memcpy(code, classbits, 32); + code = class_uchardata + (32 / sizeof(pcre_uchar)); + } + else code = class_uchardata; + + /* Now fill in the complete length of the item */ + + PUT(previous, 1, (int)(code - previous)); + break; /* End of class handling */ + } + + /* Even though any XCLASS list is now discarded, we must allow for + its memory. */ + + if (lengthptr != NULL) + *lengthptr += (int)(class_uchardata - class_uchardata_base); +#endif + + /* If there are no characters > 255, or they are all to be included or + excluded, set the opcode to OP_CLASS or OP_NCLASS, depending on whether the + whole class was negated and whether there were negative specials such as \S + (non-UCP) in the class. Then copy the 32-byte map into the code vector, + negating it if necessary. */ + + *code++ = (negate_class == should_flip_negation) ? OP_CLASS : OP_NCLASS; + if (lengthptr == NULL) /* Save time in the pre-compile phase */ + { + if (negate_class) + for (c = 0; c < 32; c++) classbits[c] = ~classbits[c]; + memcpy(code, classbits, 32); + } + code += 32 / sizeof(pcre_uchar); + + END_CLASS: + break; + + + /* ===================================================================*/ + /* Various kinds of repeat; '{' is not necessarily a quantifier, but this + has been tested above. */ + + case CHAR_LEFT_CURLY_BRACKET: + if (!is_quantifier) goto NORMAL_CHAR; + ptr = read_repeat_counts(ptr+1, &repeat_min, &repeat_max, errorcodeptr); + if (*errorcodeptr != 0) goto FAILED; + goto REPEAT; + + case CHAR_ASTERISK: + repeat_min = 0; + repeat_max = -1; + goto REPEAT; + + case CHAR_PLUS: + repeat_min = 1; + repeat_max = -1; + goto REPEAT; + + case CHAR_QUESTION_MARK: + repeat_min = 0; + repeat_max = 1; + + REPEAT: + if (previous == NULL) + { + *errorcodeptr = ERR9; + goto FAILED; + } + + if (repeat_min == 0) + { + firstchar = zerofirstchar; /* Adjust for zero repeat */ + firstcharflags = zerofirstcharflags; + reqchar = zeroreqchar; /* Ditto */ + reqcharflags = zeroreqcharflags; + } + + /* Remember whether this is a variable length repeat */ + + reqvary = (repeat_min == repeat_max)? 0 : REQ_VARY; + + op_type = 0; /* Default single-char op codes */ + possessive_quantifier = FALSE; /* Default not possessive quantifier */ + + /* Save start of previous item, in case we have to move it up in order to + insert something before it. */ + + tempcode = previous; + + /* Before checking for a possessive quantifier, we must skip over + whitespace and comments in extended mode because Perl allows white space at + this point. */ + + if ((options & PCRE_EXTENDED) != 0) + { + const pcre_uchar *p = ptr + 1; + for (;;) + { + while (MAX_255(*p) && (cd->ctypes[*p] & ctype_space) != 0) p++; + if (*p != CHAR_NUMBER_SIGN) break; + p++; + while (*p != CHAR_NULL) + { + if (IS_NEWLINE(p)) /* For non-fixed-length newline cases, */ + { /* IS_NEWLINE sets cd->nllen. */ + p += cd->nllen; + break; + } + p++; +#ifdef SUPPORT_UTF + if (utf) FORWARDCHAR(p); +#endif + } /* Loop for comment characters */ + } /* Loop for multiple comments */ + ptr = p - 1; /* Character before the next significant one. */ + } + + /* We also need to skip over (?# comments, which are not dependent on + extended mode. */ + + if (ptr[1] == CHAR_LEFT_PARENTHESIS && ptr[2] == CHAR_QUESTION_MARK && + ptr[3] == CHAR_NUMBER_SIGN) + { + ptr += 4; + while (*ptr != CHAR_NULL && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++; + if (*ptr == CHAR_NULL) + { + *errorcodeptr = ERR18; + goto FAILED; + } + } + + /* If the next character is '+', we have a possessive quantifier. This + implies greediness, whatever the setting of the PCRE_UNGREEDY option. + If the next character is '?' this is a minimizing repeat, by default, + but if PCRE_UNGREEDY is set, it works the other way round. We change the + repeat type to the non-default. */ + + if (ptr[1] == CHAR_PLUS) + { + repeat_type = 0; /* Force greedy */ + possessive_quantifier = TRUE; + ptr++; + } + else if (ptr[1] == CHAR_QUESTION_MARK) + { + repeat_type = greedy_non_default; + ptr++; + } + else repeat_type = greedy_default; + + /* If previous was a recursion call, wrap it in atomic brackets so that + previous becomes the atomic group. All recursions were so wrapped in the + past, but it no longer happens for non-repeated recursions. In fact, the + repeated ones could be re-implemented independently so as not to need this, + but for the moment we rely on the code for repeating groups. */ + + if (*previous == OP_RECURSE) + { + memmove(previous + 1 + LINK_SIZE, previous, IN_UCHARS(1 + LINK_SIZE)); + *previous = OP_ONCE; + PUT(previous, 1, 2 + 2*LINK_SIZE); + previous[2 + 2*LINK_SIZE] = OP_KET; + PUT(previous, 3 + 2*LINK_SIZE, 2 + 2*LINK_SIZE); + code += 2 + 2 * LINK_SIZE; + length_prevgroup = 3 + 3*LINK_SIZE; + + /* When actually compiling, we need to check whether this was a forward + reference, and if so, adjust the offset. */ + + if (lengthptr == NULL && cd->hwm >= cd->start_workspace + LINK_SIZE) + { + int offset = GET(cd->hwm, -LINK_SIZE); + if (offset == previous + 1 - cd->start_code) + PUT(cd->hwm, -LINK_SIZE, offset + 1 + LINK_SIZE); + } + } + + /* Now handle repetition for the different types of item. */ + + /* If previous was a character or negated character match, abolish the item + and generate a repeat item instead. If a char item has a minimum of more + than one, ensure that it is set in reqchar - it might not be if a sequence + such as x{3} is the first thing in a branch because the x will have gone + into firstchar instead. */ + + if (*previous == OP_CHAR || *previous == OP_CHARI + || *previous == OP_NOT || *previous == OP_NOTI) + { + switch (*previous) + { + default: /* Make compiler happy. */ + case OP_CHAR: op_type = OP_STAR - OP_STAR; break; + case OP_CHARI: op_type = OP_STARI - OP_STAR; break; + case OP_NOT: op_type = OP_NOTSTAR - OP_STAR; break; + case OP_NOTI: op_type = OP_NOTSTARI - OP_STAR; break; + } + + /* Deal with UTF characters that take up more than one character. It's + easier to write this out separately than try to macrify it. Use c to + hold the length of the character in bytes, plus UTF_LENGTH to flag that + it's a length rather than a small character. */ + +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (utf && NOT_FIRSTCHAR(code[-1])) + { + pcre_uchar *lastchar = code - 1; + BACKCHAR(lastchar); + c = (int)(code - lastchar); /* Length of UTF-8 character */ + memcpy(utf_chars, lastchar, IN_UCHARS(c)); /* Save the char */ + c |= UTF_LENGTH; /* Flag c as a length */ + } + else +#endif /* SUPPORT_UTF */ + + /* Handle the case of a single charater - either with no UTF support, or + with UTF disabled, or for a single character UTF character. */ + { + c = code[-1]; + if (*previous <= OP_CHARI && repeat_min > 1) + { + reqchar = c; + reqcharflags = req_caseopt | cd->req_varyopt; + } + } + + goto OUTPUT_SINGLE_REPEAT; /* Code shared with single character types */ + } + + /* If previous was a character type match (\d or similar), abolish it and + create a suitable repeat item. The code is shared with single-character + repeats by setting op_type to add a suitable offset into repeat_type. Note + the the Unicode property types will be present only when SUPPORT_UCP is + defined, but we don't wrap the little bits of code here because it just + makes it horribly messy. */ + + else if (*previous < OP_EODN) + { + pcre_uchar *oldcode; + int prop_type, prop_value; + op_type = OP_TYPESTAR - OP_STAR; /* Use type opcodes */ + c = *previous; + + OUTPUT_SINGLE_REPEAT: + if (*previous == OP_PROP || *previous == OP_NOTPROP) + { + prop_type = previous[1]; + prop_value = previous[2]; + } + else prop_type = prop_value = -1; + + oldcode = code; + code = previous; /* Usually overwrite previous item */ + + /* If the maximum is zero then the minimum must also be zero; Perl allows + this case, so we do too - by simply omitting the item altogether. */ + + if (repeat_max == 0) goto END_REPEAT; + + /* Combine the op_type with the repeat_type */ + + repeat_type += op_type; + + /* A minimum of zero is handled either as the special case * or ?, or as + an UPTO, with the maximum given. */ + + if (repeat_min == 0) + { + if (repeat_max == -1) *code++ = OP_STAR + repeat_type; + else if (repeat_max == 1) *code++ = OP_QUERY + repeat_type; + else + { + *code++ = OP_UPTO + repeat_type; + PUT2INC(code, 0, repeat_max); + } + } + + /* A repeat minimum of 1 is optimized into some special cases. If the + maximum is unlimited, we use OP_PLUS. Otherwise, the original item is + left in place and, if the maximum is greater than 1, we use OP_UPTO with + one less than the maximum. */ + + else if (repeat_min == 1) + { + if (repeat_max == -1) + *code++ = OP_PLUS + repeat_type; + else + { + code = oldcode; /* leave previous item in place */ + if (repeat_max == 1) goto END_REPEAT; + *code++ = OP_UPTO + repeat_type; + PUT2INC(code, 0, repeat_max - 1); + } + } + + /* The case {n,n} is just an EXACT, while the general case {n,m} is + handled as an EXACT followed by an UPTO. */ + + else + { + *code++ = OP_EXACT + op_type; /* NB EXACT doesn't have repeat_type */ + PUT2INC(code, 0, repeat_min); + + /* If the maximum is unlimited, insert an OP_STAR. Before doing so, + we have to insert the character for the previous code. For a repeated + Unicode property match, there are two extra bytes that define the + required property. In UTF-8 mode, long characters have their length in + c, with the UTF_LENGTH bit as a flag. */ + + if (repeat_max < 0) + { +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (utf && (c & UTF_LENGTH) != 0) + { + memcpy(code, utf_chars, IN_UCHARS(c & 7)); + code += c & 7; + } + else +#endif + { + *code++ = c; + if (prop_type >= 0) + { + *code++ = prop_type; + *code++ = prop_value; + } + } + *code++ = OP_STAR + repeat_type; + } + + /* Else insert an UPTO if the max is greater than the min, again + preceded by the character, for the previously inserted code. If the + UPTO is just for 1 instance, we can use QUERY instead. */ + + else if (repeat_max != repeat_min) + { +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (utf && (c & UTF_LENGTH) != 0) + { + memcpy(code, utf_chars, IN_UCHARS(c & 7)); + code += c & 7; + } + else +#endif + *code++ = c; + if (prop_type >= 0) + { + *code++ = prop_type; + *code++ = prop_value; + } + repeat_max -= repeat_min; + + if (repeat_max == 1) + { + *code++ = OP_QUERY + repeat_type; + } + else + { + *code++ = OP_UPTO + repeat_type; + PUT2INC(code, 0, repeat_max); + } + } + } + + /* The character or character type itself comes last in all cases. */ + +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (utf && (c & UTF_LENGTH) != 0) + { + memcpy(code, utf_chars, IN_UCHARS(c & 7)); + code += c & 7; + } + else +#endif + *code++ = c; + + /* For a repeated Unicode property match, there are two extra bytes that + define the required property. */ + +#ifdef SUPPORT_UCP + if (prop_type >= 0) + { + *code++ = prop_type; + *code++ = prop_value; + } +#endif + } + + /* If previous was a character class or a back reference, we put the repeat + stuff after it, but just skip the item if the repeat was {0,0}. */ + + else if (*previous == OP_CLASS || *previous == OP_NCLASS || +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + *previous == OP_XCLASS || +#endif + *previous == OP_REF || *previous == OP_REFI || + *previous == OP_DNREF || *previous == OP_DNREFI) + { + if (repeat_max == 0) + { + code = previous; + goto END_REPEAT; + } + + if (repeat_min == 0 && repeat_max == -1) + *code++ = OP_CRSTAR + repeat_type; + else if (repeat_min == 1 && repeat_max == -1) + *code++ = OP_CRPLUS + repeat_type; + else if (repeat_min == 0 && repeat_max == 1) + *code++ = OP_CRQUERY + repeat_type; + else + { + *code++ = OP_CRRANGE + repeat_type; + PUT2INC(code, 0, repeat_min); + if (repeat_max == -1) repeat_max = 0; /* 2-byte encoding for max */ + PUT2INC(code, 0, repeat_max); + } + } + + /* If previous was a bracket group, we may have to replicate it in certain + cases. Note that at this point we can encounter only the "basic" bracket + opcodes such as BRA and CBRA, as this is the place where they get converted + into the more special varieties such as BRAPOS and SBRA. A test for >= + OP_ASSERT and <= OP_COND includes ASSERT, ASSERT_NOT, ASSERTBACK, + ASSERTBACK_NOT, ONCE, ONCE_NC, BRA, BRAPOS, CBRA, CBRAPOS, and COND. + Originally, PCRE did not allow repetition of assertions, but now it does, + for Perl compatibility. */ + + else if (*previous >= OP_ASSERT && *previous <= OP_COND) + { + register int i; + int len = (int)(code - previous); + size_t base_hwm_offset = item_hwm_offset; + pcre_uchar *bralink = NULL; + pcre_uchar *brazeroptr = NULL; + + /* Repeating a DEFINE group is pointless, but Perl allows the syntax, so + we just ignore the repeat. */ + + if (*previous == OP_COND && previous[LINK_SIZE+1] == OP_DEF) + goto END_REPEAT; + + /* There is no sense in actually repeating assertions. The only potential + use of repetition is in cases when the assertion is optional. Therefore, + if the minimum is greater than zero, just ignore the repeat. If the + maximum is not zero or one, set it to 1. */ + + if (*previous < OP_ONCE) /* Assertion */ + { + if (repeat_min > 0) goto END_REPEAT; + if (repeat_max < 0 || repeat_max > 1) repeat_max = 1; + } + + /* The case of a zero minimum is special because of the need to stick + OP_BRAZERO in front of it, and because the group appears once in the + data, whereas in other cases it appears the minimum number of times. For + this reason, it is simplest to treat this case separately, as otherwise + the code gets far too messy. There are several special subcases when the + minimum is zero. */ + + if (repeat_min == 0) + { + /* If the maximum is also zero, we used to just omit the group from the + output altogether, like this: + + ** if (repeat_max == 0) + ** { + ** code = previous; + ** goto END_REPEAT; + ** } + + However, that fails when a group or a subgroup within it is referenced + as a subroutine from elsewhere in the pattern, so now we stick in + OP_SKIPZERO in front of it so that it is skipped on execution. As we + don't have a list of which groups are referenced, we cannot do this + selectively. + + If the maximum is 1 or unlimited, we just have to stick in the BRAZERO + and do no more at this point. However, we do need to adjust any + OP_RECURSE calls inside the group that refer to the group itself or any + internal or forward referenced group, because the offset is from the + start of the whole regex. Temporarily terminate the pattern while doing + this. */ + + if (repeat_max <= 1) /* Covers 0, 1, and unlimited */ + { + *code = OP_END; + adjust_recurse(previous, 1, utf, cd, item_hwm_offset); + memmove(previous + 1, previous, IN_UCHARS(len)); + code++; + if (repeat_max == 0) + { + *previous++ = OP_SKIPZERO; + goto END_REPEAT; + } + brazeroptr = previous; /* Save for possessive optimizing */ + *previous++ = OP_BRAZERO + repeat_type; + } + + /* If the maximum is greater than 1 and limited, we have to replicate + in a nested fashion, sticking OP_BRAZERO before each set of brackets. + The first one has to be handled carefully because it's the original + copy, which has to be moved up. The remainder can be handled by code + that is common with the non-zero minimum case below. We have to + adjust the value or repeat_max, since one less copy is required. Once + again, we may have to adjust any OP_RECURSE calls inside the group. */ + + else + { + int offset; + *code = OP_END; + adjust_recurse(previous, 2 + LINK_SIZE, utf, cd, item_hwm_offset); + memmove(previous + 2 + LINK_SIZE, previous, IN_UCHARS(len)); + code += 2 + LINK_SIZE; + *previous++ = OP_BRAZERO + repeat_type; + *previous++ = OP_BRA; + + /* We chain together the bracket offset fields that have to be + filled in later when the ends of the brackets are reached. */ + + offset = (bralink == NULL)? 0 : (int)(previous - bralink); + bralink = previous; + PUTINC(previous, 0, offset); + } + + repeat_max--; + } + + /* If the minimum is greater than zero, replicate the group as many + times as necessary, and adjust the maximum to the number of subsequent + copies that we need. If we set a first char from the group, and didn't + set a required char, copy the latter from the former. If there are any + forward reference subroutine calls in the group, there will be entries on + the workspace list; replicate these with an appropriate increment. */ + + else + { + if (repeat_min > 1) + { + /* In the pre-compile phase, we don't actually do the replication. We + just adjust the length as if we had. Do some paranoid checks for + potential integer overflow. The INT64_OR_DOUBLE type is a 64-bit + integer type when available, otherwise double. */ + + if (lengthptr != NULL) + { + int delta = (repeat_min - 1)*length_prevgroup; + if ((INT64_OR_DOUBLE)(repeat_min - 1)* + (INT64_OR_DOUBLE)length_prevgroup > + (INT64_OR_DOUBLE)INT_MAX || + OFLOW_MAX - *lengthptr < delta) + { + *errorcodeptr = ERR20; + goto FAILED; + } + *lengthptr += delta; + } + + /* This is compiling for real. If there is a set first byte for + the group, and we have not yet set a "required byte", set it. Make + sure there is enough workspace for copying forward references before + doing the copy. */ + + else + { + if (groupsetfirstchar && reqcharflags < 0) + { + reqchar = firstchar; + reqcharflags = firstcharflags; + } + + for (i = 1; i < repeat_min; i++) + { + pcre_uchar *hc; + size_t this_hwm_offset = cd->hwm - cd->start_workspace; + memcpy(code, previous, IN_UCHARS(len)); + + while (cd->hwm > cd->start_workspace + cd->workspace_size - + WORK_SIZE_SAFETY_MARGIN - + (this_hwm_offset - base_hwm_offset)) + { + *errorcodeptr = expand_workspace(cd); + if (*errorcodeptr != 0) goto FAILED; + } + + for (hc = (pcre_uchar *)cd->start_workspace + base_hwm_offset; + hc < (pcre_uchar *)cd->start_workspace + this_hwm_offset; + hc += LINK_SIZE) + { + PUT(cd->hwm, 0, GET(hc, 0) + len); + cd->hwm += LINK_SIZE; + } + base_hwm_offset = this_hwm_offset; + code += len; + } + } + } + + if (repeat_max > 0) repeat_max -= repeat_min; + } + + /* This code is common to both the zero and non-zero minimum cases. If + the maximum is limited, it replicates the group in a nested fashion, + remembering the bracket starts on a stack. In the case of a zero minimum, + the first one was set up above. In all cases the repeat_max now specifies + the number of additional copies needed. Again, we must remember to + replicate entries on the forward reference list. */ + + if (repeat_max >= 0) + { + /* In the pre-compile phase, we don't actually do the replication. We + just adjust the length as if we had. For each repetition we must add 1 + to the length for BRAZERO and for all but the last repetition we must + add 2 + 2*LINKSIZE to allow for the nesting that occurs. Do some + paranoid checks to avoid integer overflow. The INT64_OR_DOUBLE type is + a 64-bit integer type when available, otherwise double. */ + + if (lengthptr != NULL && repeat_max > 0) + { + int delta = repeat_max * (length_prevgroup + 1 + 2 + 2*LINK_SIZE) - + 2 - 2*LINK_SIZE; /* Last one doesn't nest */ + if ((INT64_OR_DOUBLE)repeat_max * + (INT64_OR_DOUBLE)(length_prevgroup + 1 + 2 + 2*LINK_SIZE) + > (INT64_OR_DOUBLE)INT_MAX || + OFLOW_MAX - *lengthptr < delta) + { + *errorcodeptr = ERR20; + goto FAILED; + } + *lengthptr += delta; + } + + /* This is compiling for real */ + + else for (i = repeat_max - 1; i >= 0; i--) + { + pcre_uchar *hc; + size_t this_hwm_offset = cd->hwm - cd->start_workspace; + + *code++ = OP_BRAZERO + repeat_type; + + /* All but the final copy start a new nesting, maintaining the + chain of brackets outstanding. */ + + if (i != 0) + { + int offset; + *code++ = OP_BRA; + offset = (bralink == NULL)? 0 : (int)(code - bralink); + bralink = code; + PUTINC(code, 0, offset); + } + + memcpy(code, previous, IN_UCHARS(len)); + + /* Ensure there is enough workspace for forward references before + copying them. */ + + while (cd->hwm > cd->start_workspace + cd->workspace_size - + WORK_SIZE_SAFETY_MARGIN - + (this_hwm_offset - base_hwm_offset)) + { + *errorcodeptr = expand_workspace(cd); + if (*errorcodeptr != 0) goto FAILED; + } + + for (hc = (pcre_uchar *)cd->start_workspace + base_hwm_offset; + hc < (pcre_uchar *)cd->start_workspace + this_hwm_offset; + hc += LINK_SIZE) + { + PUT(cd->hwm, 0, GET(hc, 0) + len + ((i != 0)? 2+LINK_SIZE : 1)); + cd->hwm += LINK_SIZE; + } + base_hwm_offset = this_hwm_offset; + code += len; + } + + /* Now chain through the pending brackets, and fill in their length + fields (which are holding the chain links pro tem). */ + + while (bralink != NULL) + { + int oldlinkoffset; + int offset = (int)(code - bralink + 1); + pcre_uchar *bra = code - offset; + oldlinkoffset = GET(bra, 1); + bralink = (oldlinkoffset == 0)? NULL : bralink - oldlinkoffset; + *code++ = OP_KET; + PUTINC(code, 0, offset); + PUT(bra, 1, offset); + } + } + + /* If the maximum is unlimited, set a repeater in the final copy. For + ONCE brackets, that's all we need to do. However, possessively repeated + ONCE brackets can be converted into non-capturing brackets, as the + behaviour of (?:xx)++ is the same as (?>xx)++ and this saves having to + deal with possessive ONCEs specially. + + Otherwise, when we are doing the actual compile phase, check to see + whether this group is one that could match an empty string. If so, + convert the initial operator to the S form (e.g. OP_BRA -> OP_SBRA) so + that runtime checking can be done. [This check is also applied to ONCE + groups at runtime, but in a different way.] + + Then, if the quantifier was possessive and the bracket is not a + conditional, we convert the BRA code to the POS form, and the KET code to + KETRPOS. (It turns out to be convenient at runtime to detect this kind of + subpattern at both the start and at the end.) The use of special opcodes + makes it possible to reduce greatly the stack usage in pcre_exec(). If + the group is preceded by OP_BRAZERO, convert this to OP_BRAPOSZERO. + + Then, if the minimum number of matches is 1 or 0, cancel the possessive + flag so that the default action below, of wrapping everything inside + atomic brackets, does not happen. When the minimum is greater than 1, + there will be earlier copies of the group, and so we still have to wrap + the whole thing. */ + + else + { + pcre_uchar *ketcode = code - 1 - LINK_SIZE; + pcre_uchar *bracode = ketcode - GET(ketcode, 1); + + /* Convert possessive ONCE brackets to non-capturing */ + + if ((*bracode == OP_ONCE || *bracode == OP_ONCE_NC) && + possessive_quantifier) *bracode = OP_BRA; + + /* For non-possessive ONCE brackets, all we need to do is to + set the KET. */ + + if (*bracode == OP_ONCE || *bracode == OP_ONCE_NC) + *ketcode = OP_KETRMAX + repeat_type; + + /* Handle non-ONCE brackets and possessive ONCEs (which have been + converted to non-capturing above). */ + + else + { + /* In the compile phase, check for empty string matching. */ + + if (lengthptr == NULL) + { + pcre_uchar *scode = bracode; + do + { + if (could_be_empty_branch(scode, ketcode, utf, cd, NULL)) + { + *bracode += OP_SBRA - OP_BRA; + break; + } + scode += GET(scode, 1); + } + while (*scode == OP_ALT); + } + + /* A conditional group with only one branch has an implicit empty + alternative branch. */ + + if (*bracode == OP_COND && bracode[GET(bracode,1)] != OP_ALT) + *bracode = OP_SCOND; + + /* Handle possessive quantifiers. */ + + if (possessive_quantifier) + { + /* For COND brackets, we wrap the whole thing in a possessively + repeated non-capturing bracket, because we have not invented POS + versions of the COND opcodes. Because we are moving code along, we + must ensure that any pending recursive references are updated. */ + + if (*bracode == OP_COND || *bracode == OP_SCOND) + { + int nlen = (int)(code - bracode); + *code = OP_END; + adjust_recurse(bracode, 1 + LINK_SIZE, utf, cd, item_hwm_offset); + memmove(bracode + 1 + LINK_SIZE, bracode, IN_UCHARS(nlen)); + code += 1 + LINK_SIZE; + nlen += 1 + LINK_SIZE; + *bracode = (*bracode == OP_COND)? OP_BRAPOS : OP_SBRAPOS; + *code++ = OP_KETRPOS; + PUTINC(code, 0, nlen); + PUT(bracode, 1, nlen); + } + + /* For non-COND brackets, we modify the BRA code and use KETRPOS. */ + + else + { + *bracode += 1; /* Switch to xxxPOS opcodes */ + *ketcode = OP_KETRPOS; + } + + /* If the minimum is zero, mark it as possessive, then unset the + possessive flag when the minimum is 0 or 1. */ + + if (brazeroptr != NULL) *brazeroptr = OP_BRAPOSZERO; + if (repeat_min < 2) possessive_quantifier = FALSE; + } + + /* Non-possessive quantifier */ + + else *ketcode = OP_KETRMAX + repeat_type; + } + } + } + + /* If previous is OP_FAIL, it was generated by an empty class [] in + JavaScript mode. The other ways in which OP_FAIL can be generated, that is + by (*FAIL) or (?!) set previous to NULL, which gives a "nothing to repeat" + error above. We can just ignore the repeat in JS case. */ + + else if (*previous == OP_FAIL) goto END_REPEAT; + + /* Else there's some kind of shambles */ + + else + { + *errorcodeptr = ERR11; + goto FAILED; + } + + /* If the character following a repeat is '+', possessive_quantifier is + TRUE. For some opcodes, there are special alternative opcodes for this + case. For anything else, we wrap the entire repeated item inside OP_ONCE + brackets. Logically, the '+' notation is just syntactic sugar, taken from + Sun's Java package, but the special opcodes can optimize it. + + Some (but not all) possessively repeated subpatterns have already been + completely handled in the code just above. For them, possessive_quantifier + is always FALSE at this stage. Note that the repeated item starts at + tempcode, not at previous, which might be the first part of a string whose + (former) last char we repeated. */ + + if (possessive_quantifier) + { + int len; + + /* Possessifying an EXACT quantifier has no effect, so we can ignore it. + However, QUERY, STAR, or UPTO may follow (for quantifiers such as {5,6}, + {5,}, or {5,10}). We skip over an EXACT item; if the length of what + remains is greater than zero, there's a further opcode that can be + handled. If not, do nothing, leaving the EXACT alone. */ + + switch(*tempcode) + { + case OP_TYPEEXACT: + tempcode += PRIV(OP_lengths)[*tempcode] + + ((tempcode[1 + IMM2_SIZE] == OP_PROP + || tempcode[1 + IMM2_SIZE] == OP_NOTPROP)? 2 : 0); + break; + + /* CHAR opcodes are used for exacts whose count is 1. */ + + case OP_CHAR: + case OP_CHARI: + case OP_NOT: + case OP_NOTI: + case OP_EXACT: + case OP_EXACTI: + case OP_NOTEXACT: + case OP_NOTEXACTI: + tempcode += PRIV(OP_lengths)[*tempcode]; +#ifdef SUPPORT_UTF + if (utf && HAS_EXTRALEN(tempcode[-1])) + tempcode += GET_EXTRALEN(tempcode[-1]); +#endif + break; + + /* For the class opcodes, the repeat operator appears at the end; + adjust tempcode to point to it. */ + + case OP_CLASS: + case OP_NCLASS: + tempcode += 1 + 32/sizeof(pcre_uchar); + break; + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: + tempcode += GET(tempcode, 1); + break; +#endif + } + + /* If tempcode is equal to code (which points to the end of the repeated + item), it means we have skipped an EXACT item but there is no following + QUERY, STAR, or UPTO; the value of len will be 0, and we do nothing. In + all other cases, tempcode will be pointing to the repeat opcode, and will + be less than code, so the value of len will be greater than 0. */ + + len = (int)(code - tempcode); + if (len > 0) + { + unsigned int repcode = *tempcode; + + /* There is a table for possessifying opcodes, all of which are less + than OP_CALLOUT. A zero entry means there is no possessified version. + */ + + if (repcode < OP_CALLOUT && opcode_possessify[repcode] > 0) + *tempcode = opcode_possessify[repcode]; + + /* For opcode without a special possessified version, wrap the item in + ONCE brackets. Because we are moving code along, we must ensure that any + pending recursive references are updated. */ + + else + { + *code = OP_END; + adjust_recurse(tempcode, 1 + LINK_SIZE, utf, cd, item_hwm_offset); + memmove(tempcode + 1 + LINK_SIZE, tempcode, IN_UCHARS(len)); + code += 1 + LINK_SIZE; + len += 1 + LINK_SIZE; + tempcode[0] = OP_ONCE; + *code++ = OP_KET; + PUTINC(code, 0, len); + PUT(tempcode, 1, len); + } + } + +#ifdef NEVER + if (len > 0) switch (*tempcode) + { + case OP_STAR: *tempcode = OP_POSSTAR; break; + case OP_PLUS: *tempcode = OP_POSPLUS; break; + case OP_QUERY: *tempcode = OP_POSQUERY; break; + case OP_UPTO: *tempcode = OP_POSUPTO; break; + + case OP_STARI: *tempcode = OP_POSSTARI; break; + case OP_PLUSI: *tempcode = OP_POSPLUSI; break; + case OP_QUERYI: *tempcode = OP_POSQUERYI; break; + case OP_UPTOI: *tempcode = OP_POSUPTOI; break; + + case OP_NOTSTAR: *tempcode = OP_NOTPOSSTAR; break; + case OP_NOTPLUS: *tempcode = OP_NOTPOSPLUS; break; + case OP_NOTQUERY: *tempcode = OP_NOTPOSQUERY; break; + case OP_NOTUPTO: *tempcode = OP_NOTPOSUPTO; break; + + case OP_NOTSTARI: *tempcode = OP_NOTPOSSTARI; break; + case OP_NOTPLUSI: *tempcode = OP_NOTPOSPLUSI; break; + case OP_NOTQUERYI: *tempcode = OP_NOTPOSQUERYI; break; + case OP_NOTUPTOI: *tempcode = OP_NOTPOSUPTOI; break; + + case OP_TYPESTAR: *tempcode = OP_TYPEPOSSTAR; break; + case OP_TYPEPLUS: *tempcode = OP_TYPEPOSPLUS; break; + case OP_TYPEQUERY: *tempcode = OP_TYPEPOSQUERY; break; + case OP_TYPEUPTO: *tempcode = OP_TYPEPOSUPTO; break; + + case OP_CRSTAR: *tempcode = OP_CRPOSSTAR; break; + case OP_CRPLUS: *tempcode = OP_CRPOSPLUS; break; + case OP_CRQUERY: *tempcode = OP_CRPOSQUERY; break; + case OP_CRRANGE: *tempcode = OP_CRPOSRANGE; break; + + /* Because we are moving code along, we must ensure that any + pending recursive references are updated. */ + + default: + *code = OP_END; + adjust_recurse(tempcode, 1 + LINK_SIZE, utf, cd, item_hwm_offset); + memmove(tempcode + 1 + LINK_SIZE, tempcode, IN_UCHARS(len)); + code += 1 + LINK_SIZE; + len += 1 + LINK_SIZE; + tempcode[0] = OP_ONCE; + *code++ = OP_KET; + PUTINC(code, 0, len); + PUT(tempcode, 1, len); + break; + } +#endif + } + + /* In all case we no longer have a previous item. We also set the + "follows varying string" flag for subsequently encountered reqchars if + it isn't already set and we have just passed a varying length item. */ + + END_REPEAT: + previous = NULL; + cd->req_varyopt |= reqvary; + break; + + + /* ===================================================================*/ + /* Start of nested parenthesized sub-expression, or comment or lookahead or + lookbehind or option setting or condition or all the other extended + parenthesis forms. */ + + case CHAR_LEFT_PARENTHESIS: + ptr++; + + /* Now deal with various "verbs" that can be introduced by '*'. */ + + if (ptr[0] == CHAR_ASTERISK && (ptr[1] == ':' + || (MAX_255(ptr[1]) && ((cd->ctypes[ptr[1]] & ctype_letter) != 0)))) + { + int i, namelen; + int arglen = 0; + const char *vn = verbnames; + const pcre_uchar *name = ptr + 1; + const pcre_uchar *arg = NULL; + previous = NULL; + ptr++; + while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_letter) != 0) ptr++; + namelen = (int)(ptr - name); + + /* It appears that Perl allows any characters whatsoever, other than + a closing parenthesis, to appear in arguments, so we no longer insist on + letters, digits, and underscores. */ + + if (*ptr == CHAR_COLON) + { + arg = ++ptr; + while (*ptr != CHAR_NULL && *ptr != CHAR_RIGHT_PARENTHESIS) ptr++; + arglen = (int)(ptr - arg); + if ((unsigned int)arglen > MAX_MARK) + { + *errorcodeptr = ERR75; + goto FAILED; + } + } + + if (*ptr != CHAR_RIGHT_PARENTHESIS) + { + *errorcodeptr = ERR60; + goto FAILED; + } + + /* Scan the table of verb names */ + + for (i = 0; i < verbcount; i++) + { + if (namelen == verbs[i].len && + STRNCMP_UC_C8(name, vn, namelen) == 0) + { + int setverb; + + /* Check for open captures before ACCEPT and convert it to + ASSERT_ACCEPT if in an assertion. */ + + if (verbs[i].op == OP_ACCEPT) + { + open_capitem *oc; + if (arglen != 0) + { + *errorcodeptr = ERR59; + goto FAILED; + } + cd->had_accept = TRUE; + for (oc = cd->open_caps; oc != NULL; oc = oc->next) + { + if (lengthptr != NULL) + { +#ifdef COMPILE_PCRE8 + *lengthptr += 1 + IMM2_SIZE; +#elif defined COMPILE_PCRE16 + *lengthptr += 2 + IMM2_SIZE; +#elif defined COMPILE_PCRE32 + *lengthptr += 4 + IMM2_SIZE; +#endif + } + else + { + *code++ = OP_CLOSE; + PUT2INC(code, 0, oc->number); + } + } + setverb = *code++ = + (cd->assert_depth > 0)? OP_ASSERT_ACCEPT : OP_ACCEPT; + + /* Do not set firstchar after *ACCEPT */ + if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; + } + + /* Handle other cases with/without an argument */ + + else if (arglen == 0) + { + if (verbs[i].op < 0) /* Argument is mandatory */ + { + *errorcodeptr = ERR66; + goto FAILED; + } + setverb = *code++ = verbs[i].op; + } + + else + { + if (verbs[i].op_arg < 0) /* Argument is forbidden */ + { + *errorcodeptr = ERR59; + goto FAILED; + } + setverb = *code++ = verbs[i].op_arg; + if (lengthptr != NULL) /* In pass 1 just add in the length */ + { /* to avoid potential workspace */ + *lengthptr += arglen; /* overflow. */ + *code++ = 0; + } + else + { + *code++ = arglen; + memcpy(code, arg, IN_UCHARS(arglen)); + code += arglen; + } + *code++ = 0; + } + + switch (setverb) + { + case OP_THEN: + case OP_THEN_ARG: + cd->external_flags |= PCRE_HASTHEN; + break; + + case OP_PRUNE: + case OP_PRUNE_ARG: + case OP_SKIP: + case OP_SKIP_ARG: + cd->had_pruneorskip = TRUE; + break; + } + + break; /* Found verb, exit loop */ + } + + vn += verbs[i].len + 1; + } + + if (i < verbcount) continue; /* Successfully handled a verb */ + *errorcodeptr = ERR60; /* Verb not recognized */ + goto FAILED; + } + + /* Initialize for "real" parentheses */ + + newoptions = options; + skipbytes = 0; + bravalue = OP_CBRA; + item_hwm_offset = cd->hwm - cd->start_workspace; + reset_bracount = FALSE; + + /* Deal with the extended parentheses; all are introduced by '?', and the + appearance of any of them means that this is not a capturing group. */ + + if (*ptr == CHAR_QUESTION_MARK) + { + int i, set, unset, namelen; + int *optset; + const pcre_uchar *name; + pcre_uchar *slot; + + switch (*(++ptr)) + { + /* ------------------------------------------------------------ */ + case CHAR_VERTICAL_LINE: /* Reset capture count for each branch */ + reset_bracount = TRUE; + cd->dupgroups = TRUE; /* Record (?| encountered */ + /* Fall through */ + + /* ------------------------------------------------------------ */ + case CHAR_COLON: /* Non-capturing bracket */ + bravalue = OP_BRA; + ptr++; + break; + + + /* ------------------------------------------------------------ */ + case CHAR_LEFT_PARENTHESIS: + bravalue = OP_COND; /* Conditional group */ + tempptr = ptr; + + /* A condition can be an assertion, a number (referring to a numbered + group's having been set), a name (referring to a named group), or 'R', + referring to recursion. R and R&name are also permitted for + recursion tests. + + There are ways of testing a named group: (?(name)) is used by Python; + Perl 5.10 onwards uses (?() or (?('name')). + + There is one unfortunate ambiguity, caused by history. 'R' can be the + recursive thing or the name 'R' (and similarly for 'R' followed by + digits). We look for a name first; if not found, we try the other case. + + For compatibility with auto-callouts, we allow a callout to be + specified before a condition that is an assertion. First, check for the + syntax of a callout; if found, adjust the temporary pointer that is + used to check for an assertion condition. That's all that is needed! */ + + if (ptr[1] == CHAR_QUESTION_MARK && ptr[2] == CHAR_C) + { + for (i = 3;; i++) if (!IS_DIGIT(ptr[i])) break; + if (ptr[i] == CHAR_RIGHT_PARENTHESIS) + tempptr += i + 1; + + /* tempptr should now be pointing to the opening parenthesis of the + assertion condition. */ + + if (*tempptr != CHAR_LEFT_PARENTHESIS) + { + *errorcodeptr = ERR28; + goto FAILED; + } + } + + /* For conditions that are assertions, check the syntax, and then exit + the switch. This will take control down to where bracketed groups, + including assertions, are processed. */ + + if (tempptr[1] == CHAR_QUESTION_MARK && + (tempptr[2] == CHAR_EQUALS_SIGN || + tempptr[2] == CHAR_EXCLAMATION_MARK || + (tempptr[2] == CHAR_LESS_THAN_SIGN && + (tempptr[3] == CHAR_EQUALS_SIGN || + tempptr[3] == CHAR_EXCLAMATION_MARK)))) + { + cd->iscondassert = TRUE; + break; + } + + /* Other conditions use OP_CREF/OP_DNCREF/OP_RREF/OP_DNRREF, and all + need to skip at least 1+IMM2_SIZE bytes at the start of the group. */ + + code[1+LINK_SIZE] = OP_CREF; + skipbytes = 1+IMM2_SIZE; + refsign = -1; /* => not a number */ + namelen = -1; /* => not a name; must set to avoid warning */ + name = NULL; /* Always set to avoid warning */ + recno = 0; /* Always set to avoid warning */ + + /* Check for a test for recursion in a named group. */ + + ptr++; + if (*ptr == CHAR_R && ptr[1] == CHAR_AMPERSAND) + { + terminator = -1; + ptr += 2; + code[1+LINK_SIZE] = OP_RREF; /* Change the type of test */ + } + + /* Check for a test for a named group's having been set, using the Perl + syntax (?() or (?('name'), and also allow for the original PCRE + syntax of (?(name) or for (?(+n), (?(-n), and just (?(n). */ + + else if (*ptr == CHAR_LESS_THAN_SIGN) + { + terminator = CHAR_GREATER_THAN_SIGN; + ptr++; + } + else if (*ptr == CHAR_APOSTROPHE) + { + terminator = CHAR_APOSTROPHE; + ptr++; + } + else + { + terminator = CHAR_NULL; + if (*ptr == CHAR_MINUS || *ptr == CHAR_PLUS) refsign = *ptr++; + else if (IS_DIGIT(*ptr)) refsign = 0; + } + + /* Handle a number */ + + if (refsign >= 0) + { + while (IS_DIGIT(*ptr)) + { + if (recno > INT_MAX / 10 - 1) /* Integer overflow */ + { + while (IS_DIGIT(*ptr)) ptr++; + *errorcodeptr = ERR61; + goto FAILED; + } + recno = recno * 10 + (int)(*ptr - CHAR_0); + ptr++; + } + } + + /* Otherwise we expect to read a name; anything else is an error. When + a name is one of a number of duplicates, a different opcode is used and + it needs more memory. Unfortunately we cannot tell whether a name is a + duplicate in the first pass, so we have to allow for more memory. */ + + else + { + if (IS_DIGIT(*ptr)) + { + *errorcodeptr = ERR84; + goto FAILED; + } + if (!MAX_255(*ptr) || (cd->ctypes[*ptr] & ctype_word) == 0) + { + *errorcodeptr = ERR28; /* Assertion expected */ + goto FAILED; + } + name = ptr++; + while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_word) != 0) + { + ptr++; + } + namelen = (int)(ptr - name); + if (lengthptr != NULL) skipbytes += IMM2_SIZE; + } + + /* Check the terminator */ + + if ((terminator > 0 && *ptr++ != (pcre_uchar)terminator) || + *ptr++ != CHAR_RIGHT_PARENTHESIS) + { + ptr--; /* Error offset */ + *errorcodeptr = ERR26; /* Malformed number or name */ + goto FAILED; + } + + /* Do no further checking in the pre-compile phase. */ + + if (lengthptr != NULL) break; + + /* In the real compile we do the work of looking for the actual + reference. If refsign is not negative, it means we have a number in + recno. */ + + if (refsign >= 0) + { + if (recno <= 0) + { + *errorcodeptr = ERR35; + goto FAILED; + } + if (refsign != 0) recno = (refsign == CHAR_MINUS)? + cd->bracount - recno + 1 : recno + cd->bracount; + if (recno <= 0 || recno > cd->final_bracount) + { + *errorcodeptr = ERR15; + goto FAILED; + } + PUT2(code, 2+LINK_SIZE, recno); + if (recno > cd->top_backref) cd->top_backref = recno; + break; + } + + /* Otherwise look for the name. */ + + slot = cd->name_table; + for (i = 0; i < cd->names_found; i++) + { + if (STRNCMP_UC_UC(name, slot+IMM2_SIZE, namelen) == 0 && + slot[IMM2_SIZE+namelen] == 0) break; + slot += cd->name_entry_size; + } + + /* Found the named subpattern. If the name is duplicated, add one to + the opcode to change CREF/RREF into DNCREF/DNRREF and insert + appropriate data values. Otherwise, just insert the unique subpattern + number. */ + + if (i < cd->names_found) + { + int offset = i++; + int count = 1; + recno = GET2(slot, 0); /* Number from first found */ + if (recno > cd->top_backref) cd->top_backref = recno; + for (; i < cd->names_found; i++) + { + slot += cd->name_entry_size; + if (STRNCMP_UC_UC(name, slot+IMM2_SIZE, namelen) != 0 || + (slot+IMM2_SIZE)[namelen] != 0) break; + count++; + } + + if (count > 1) + { + PUT2(code, 2+LINK_SIZE, offset); + PUT2(code, 2+LINK_SIZE+IMM2_SIZE, count); + skipbytes += IMM2_SIZE; + code[1+LINK_SIZE]++; + } + else /* Not a duplicated name */ + { + PUT2(code, 2+LINK_SIZE, recno); + } + } + + /* If terminator == CHAR_NULL it means that the name followed directly + after the opening parenthesis [e.g. (?(abc)...] and in this case there + are some further alternatives to try. For the cases where terminator != + CHAR_NULL [things like (?(... or (?('name')... or (?(R&name)... ] + we have now checked all the possibilities, so give an error. */ + + else if (terminator != CHAR_NULL) + { + *errorcodeptr = ERR15; + goto FAILED; + } + + /* Check for (?(R) for recursion. Allow digits after R to specify a + specific group number. */ + + else if (*name == CHAR_R) + { + recno = 0; + for (i = 1; i < namelen; i++) + { + if (!IS_DIGIT(name[i])) + { + *errorcodeptr = ERR15; + goto FAILED; + } + if (recno > INT_MAX / 10 - 1) /* Integer overflow */ + { + *errorcodeptr = ERR61; + goto FAILED; + } + recno = recno * 10 + name[i] - CHAR_0; + } + if (recno == 0) recno = RREF_ANY; + code[1+LINK_SIZE] = OP_RREF; /* Change test type */ + PUT2(code, 2+LINK_SIZE, recno); + } + + /* Similarly, check for the (?(DEFINE) "condition", which is always + false. */ + + else if (namelen == 6 && STRNCMP_UC_C8(name, STRING_DEFINE, 6) == 0) + { + code[1+LINK_SIZE] = OP_DEF; + skipbytes = 1; + } + + /* Reference to an unidentified subpattern. */ + + else + { + *errorcodeptr = ERR15; + goto FAILED; + } + break; + + + /* ------------------------------------------------------------ */ + case CHAR_EQUALS_SIGN: /* Positive lookahead */ + bravalue = OP_ASSERT; + cd->assert_depth += 1; + ptr++; + break; + + /* Optimize (?!) to (*FAIL) unless it is quantified - which is a weird + thing to do, but Perl allows all assertions to be quantified, and when + they contain capturing parentheses there may be a potential use for + this feature. Not that that applies to a quantified (?!) but we allow + it for uniformity. */ + + /* ------------------------------------------------------------ */ + case CHAR_EXCLAMATION_MARK: /* Negative lookahead */ + ptr++; + if (*ptr == CHAR_RIGHT_PARENTHESIS && ptr[1] != CHAR_ASTERISK && + ptr[1] != CHAR_PLUS && ptr[1] != CHAR_QUESTION_MARK && + (ptr[1] != CHAR_LEFT_CURLY_BRACKET || !is_counted_repeat(ptr+2))) + { + *code++ = OP_FAIL; + previous = NULL; + continue; + } + bravalue = OP_ASSERT_NOT; + cd->assert_depth += 1; + break; + + + /* ------------------------------------------------------------ */ + case CHAR_LESS_THAN_SIGN: /* Lookbehind or named define */ + switch (ptr[1]) + { + case CHAR_EQUALS_SIGN: /* Positive lookbehind */ + bravalue = OP_ASSERTBACK; + cd->assert_depth += 1; + ptr += 2; + break; + + case CHAR_EXCLAMATION_MARK: /* Negative lookbehind */ + bravalue = OP_ASSERTBACK_NOT; + cd->assert_depth += 1; + ptr += 2; + break; + + default: /* Could be name define, else bad */ + if (MAX_255(ptr[1]) && (cd->ctypes[ptr[1]] & ctype_word) != 0) + goto DEFINE_NAME; + ptr++; /* Correct offset for error */ + *errorcodeptr = ERR24; + goto FAILED; + } + break; + + + /* ------------------------------------------------------------ */ + case CHAR_GREATER_THAN_SIGN: /* One-time brackets */ + bravalue = OP_ONCE; + ptr++; + break; + + + /* ------------------------------------------------------------ */ + case CHAR_C: /* Callout - may be followed by digits; */ + previous_callout = code; /* Save for later completion */ + after_manual_callout = 1; /* Skip one item before completing */ + *code++ = OP_CALLOUT; + { + int n = 0; + ptr++; + while(IS_DIGIT(*ptr)) + n = n * 10 + *ptr++ - CHAR_0; + if (*ptr != CHAR_RIGHT_PARENTHESIS) + { + *errorcodeptr = ERR39; + goto FAILED; + } + if (n > 255) + { + *errorcodeptr = ERR38; + goto FAILED; + } + *code++ = n; + PUT(code, 0, (int)(ptr - cd->start_pattern + 1)); /* Pattern offset */ + PUT(code, LINK_SIZE, 0); /* Default length */ + code += 2 * LINK_SIZE; + } + previous = NULL; + continue; + + + /* ------------------------------------------------------------ */ + case CHAR_P: /* Python-style named subpattern handling */ + if (*(++ptr) == CHAR_EQUALS_SIGN || + *ptr == CHAR_GREATER_THAN_SIGN) /* Reference or recursion */ + { + is_recurse = *ptr == CHAR_GREATER_THAN_SIGN; + terminator = CHAR_RIGHT_PARENTHESIS; + goto NAMED_REF_OR_RECURSE; + } + else if (*ptr != CHAR_LESS_THAN_SIGN) /* Test for Python-style defn */ + { + *errorcodeptr = ERR41; + goto FAILED; + } + /* Fall through to handle (?P< as (?< is handled */ + + + /* ------------------------------------------------------------ */ + DEFINE_NAME: /* Come here from (?< handling */ + case CHAR_APOSTROPHE: + terminator = (*ptr == CHAR_LESS_THAN_SIGN)? + CHAR_GREATER_THAN_SIGN : CHAR_APOSTROPHE; + name = ++ptr; + if (IS_DIGIT(*ptr)) + { + *errorcodeptr = ERR84; /* Group name must start with non-digit */ + goto FAILED; + } + while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_word) != 0) ptr++; + namelen = (int)(ptr - name); + + /* In the pre-compile phase, do a syntax check, remember the longest + name, and then remember the group in a vector, expanding it if + necessary. Duplicates for the same number are skipped; other duplicates + are checked for validity. In the actual compile, there is nothing to + do. */ + + if (lengthptr != NULL) + { + named_group *ng; + pcre_uint32 number = cd->bracount + 1; + + if (*ptr != (pcre_uchar)terminator) + { + *errorcodeptr = ERR42; + goto FAILED; + } + + if (cd->names_found >= MAX_NAME_COUNT) + { + *errorcodeptr = ERR49; + goto FAILED; + } + + if (namelen + IMM2_SIZE + 1 > cd->name_entry_size) + { + cd->name_entry_size = namelen + IMM2_SIZE + 1; + if (namelen > MAX_NAME_SIZE) + { + *errorcodeptr = ERR48; + goto FAILED; + } + } + + /* Scan the list to check for duplicates. For duplicate names, if the + number is the same, break the loop, which causes the name to be + discarded; otherwise, if DUPNAMES is not set, give an error. + If it is set, allow the name with a different number, but continue + scanning in case this is a duplicate with the same number. For + non-duplicate names, give an error if the number is duplicated. */ + + ng = cd->named_groups; + for (i = 0; i < cd->names_found; i++, ng++) + { + if (namelen == ng->length && + STRNCMP_UC_UC(name, ng->name, namelen) == 0) + { + if (ng->number == number) break; + if ((options & PCRE_DUPNAMES) == 0) + { + *errorcodeptr = ERR43; + goto FAILED; + } + cd->dupnames = TRUE; /* Duplicate names exist */ + } + else if (ng->number == number) + { + *errorcodeptr = ERR65; + goto FAILED; + } + } + + if (i >= cd->names_found) /* Not a duplicate with same number */ + { + /* Increase the list size if necessary */ + + if (cd->names_found >= cd->named_group_list_size) + { + int newsize = cd->named_group_list_size * 2; + named_group *newspace = (PUBL(malloc)) + (newsize * sizeof(named_group)); + + if (newspace == NULL) + { + *errorcodeptr = ERR21; + goto FAILED; + } + + memcpy(newspace, cd->named_groups, + cd->named_group_list_size * sizeof(named_group)); + if (cd->named_group_list_size > NAMED_GROUP_LIST_SIZE) + (PUBL(free))((void *)cd->named_groups); + cd->named_groups = newspace; + cd->named_group_list_size = newsize; + } + + cd->named_groups[cd->names_found].name = name; + cd->named_groups[cd->names_found].length = namelen; + cd->named_groups[cd->names_found].number = number; + cd->names_found++; + } + } + + ptr++; /* Move past > or ' in both passes. */ + goto NUMBERED_GROUP; + + + /* ------------------------------------------------------------ */ + case CHAR_AMPERSAND: /* Perl recursion/subroutine syntax */ + terminator = CHAR_RIGHT_PARENTHESIS; + is_recurse = TRUE; + /* Fall through */ + + /* We come here from the Python syntax above that handles both + references (?P=name) and recursion (?P>name), as well as falling + through from the Perl recursion syntax (?&name). We also come here from + the Perl \k or \k'name' back reference syntax and the \k{name} + .NET syntax, and the Oniguruma \g<...> and \g'...' subroutine syntax. */ + + NAMED_REF_OR_RECURSE: + name = ++ptr; + if (IS_DIGIT(*ptr)) + { + *errorcodeptr = ERR84; /* Group name must start with non-digit */ + goto FAILED; + } + while (MAX_255(*ptr) && (cd->ctypes[*ptr] & ctype_word) != 0) ptr++; + namelen = (int)(ptr - name); + + /* In the pre-compile phase, do a syntax check. We used to just set + a dummy reference number, because it was not used in the first pass. + However, with the change of recursive back references to be atomic, + we have to look for the number so that this state can be identified, as + otherwise the incorrect length is computed. If it's not a backwards + reference, the dummy number will do. */ + + if (lengthptr != NULL) + { + named_group *ng; + recno = 0; + + if (namelen == 0) + { + *errorcodeptr = ERR62; + goto FAILED; + } + if (*ptr != (pcre_uchar)terminator) + { + *errorcodeptr = ERR42; + goto FAILED; + } + if (namelen > MAX_NAME_SIZE) + { + *errorcodeptr = ERR48; + goto FAILED; + } + + /* Count named back references. */ + + if (!is_recurse) cd->namedrefcount++; + + /* We have to allow for a named reference to a duplicated name (this + cannot be determined until the second pass). This needs an extra + 16-bit data item. */ + + *lengthptr += IMM2_SIZE; + + /* If this is a forward reference and we are within a (?|...) group, + the reference may end up as the number of a group which we are + currently inside, that is, it could be a recursive reference. In the + real compile this will be picked up and the reference wrapped with + OP_ONCE to make it atomic, so we must space in case this occurs. */ + + /* In fact, this can happen for a non-forward reference because + another group with the same number might be created later. This + issue is fixed "properly" in PCRE2. As PCRE1 is now in maintenance + only mode, we finesse the bug by allowing more memory always. */ + + *lengthptr += 4 + 4*LINK_SIZE; + + /* It is even worse than that. The current reference may be to an + existing named group with a different number (so apparently not + recursive) but which later on is also attached to a group with the + current number. This can only happen if $(| has been previous + encountered. In that case, we allow yet more memory, just in case. + (Again, this is fixed "properly" in PCRE2. */ + + if (cd->dupgroups) *lengthptr += 4 + 4*LINK_SIZE; + + /* Otherwise, check for recursion here. The name table does not exist + in the first pass; instead we must scan the list of names encountered + so far in order to get the number. If the name is not found, leave + the value of recno as 0 for a forward reference. */ + + /* This patch (removing "else") fixes a problem when a reference is + to multiple identically named nested groups from within the nest. + Once again, it is not the "proper" fix, and it results in an + over-allocation of memory. */ + + /* else */ + { + ng = cd->named_groups; + for (i = 0; i < cd->names_found; i++, ng++) + { + if (namelen == ng->length && + STRNCMP_UC_UC(name, ng->name, namelen) == 0) + { + open_capitem *oc; + recno = ng->number; + if (is_recurse) break; + for (oc = cd->open_caps; oc != NULL; oc = oc->next) + { + if (oc->number == recno) + { + oc->flag = TRUE; + break; + } + } + } + } + } + } + + /* In the real compile, search the name table. We check the name + first, and then check that we have reached the end of the name in the + table. That way, if the name is longer than any in the table, the + comparison will fail without reading beyond the table entry. */ + + else + { + slot = cd->name_table; + for (i = 0; i < cd->names_found; i++) + { + if (STRNCMP_UC_UC(name, slot+IMM2_SIZE, namelen) == 0 && + slot[IMM2_SIZE+namelen] == 0) + break; + slot += cd->name_entry_size; + } + + if (i < cd->names_found) + { + recno = GET2(slot, 0); + } + else + { + *errorcodeptr = ERR15; + goto FAILED; + } + } + + /* In both phases, for recursions, we can now go to the code than + handles numerical recursion. */ + + if (is_recurse) goto HANDLE_RECURSION; + + /* In the second pass we must see if the name is duplicated. If so, we + generate a different opcode. */ + + if (lengthptr == NULL && cd->dupnames) + { + int count = 1; + unsigned int index = i; + pcre_uchar *cslot = slot + cd->name_entry_size; + + for (i++; i < cd->names_found; i++) + { + if (STRCMP_UC_UC(slot + IMM2_SIZE, cslot + IMM2_SIZE) != 0) break; + count++; + cslot += cd->name_entry_size; + } + + if (count > 1) + { + if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; + previous = code; + item_hwm_offset = cd->hwm - cd->start_workspace; + *code++ = ((options & PCRE_CASELESS) != 0)? OP_DNREFI : OP_DNREF; + PUT2INC(code, 0, index); + PUT2INC(code, 0, count); + + /* Process each potentially referenced group. */ + + for (; slot < cslot; slot += cd->name_entry_size) + { + open_capitem *oc; + recno = GET2(slot, 0); + cd->backref_map |= (recno < 32)? (1 << recno) : 1; + if (recno > cd->top_backref) cd->top_backref = recno; + + /* Check to see if this back reference is recursive, that it, it + is inside the group that it references. A flag is set so that the + group can be made atomic. */ + + for (oc = cd->open_caps; oc != NULL; oc = oc->next) + { + if (oc->number == recno) + { + oc->flag = TRUE; + break; + } + } + } + + continue; /* End of back ref handling */ + } + } + + /* First pass, or a non-duplicated name. */ + + goto HANDLE_REFERENCE; + + + /* ------------------------------------------------------------ */ + case CHAR_R: /* Recursion, same as (?0) */ + recno = 0; + if (*(++ptr) != CHAR_RIGHT_PARENTHESIS) + { + *errorcodeptr = ERR29; + goto FAILED; + } + goto HANDLE_RECURSION; + + + /* ------------------------------------------------------------ */ + case CHAR_MINUS: case CHAR_PLUS: /* Recursion or subroutine */ + case CHAR_0: case CHAR_1: case CHAR_2: case CHAR_3: case CHAR_4: + case CHAR_5: case CHAR_6: case CHAR_7: case CHAR_8: case CHAR_9: + { + const pcre_uchar *called; + terminator = CHAR_RIGHT_PARENTHESIS; + + /* Come here from the \g<...> and \g'...' code (Oniguruma + compatibility). However, the syntax has been checked to ensure that + the ... are a (signed) number, so that neither ERR63 nor ERR29 will + be called on this path, nor with the jump to OTHER_CHAR_AFTER_QUERY + ever be taken. */ + + HANDLE_NUMERICAL_RECURSION: + + if ((refsign = *ptr) == CHAR_PLUS) + { + ptr++; + if (!IS_DIGIT(*ptr)) + { + *errorcodeptr = ERR63; + goto FAILED; + } + } + else if (refsign == CHAR_MINUS) + { + if (!IS_DIGIT(ptr[1])) + goto OTHER_CHAR_AFTER_QUERY; + ptr++; + } + + recno = 0; + while(IS_DIGIT(*ptr)) + { + if (recno > INT_MAX / 10 - 1) /* Integer overflow */ + { + while (IS_DIGIT(*ptr)) ptr++; + *errorcodeptr = ERR61; + goto FAILED; + } + recno = recno * 10 + *ptr++ - CHAR_0; + } + + if (*ptr != (pcre_uchar)terminator) + { + *errorcodeptr = ERR29; + goto FAILED; + } + + if (refsign == CHAR_MINUS) + { + if (recno == 0) + { + *errorcodeptr = ERR58; + goto FAILED; + } + recno = cd->bracount - recno + 1; + if (recno <= 0) + { + *errorcodeptr = ERR15; + goto FAILED; + } + } + else if (refsign == CHAR_PLUS) + { + if (recno == 0) + { + *errorcodeptr = ERR58; + goto FAILED; + } + recno += cd->bracount; + } + + /* Come here from code above that handles a named recursion */ + + HANDLE_RECURSION: + + previous = code; + item_hwm_offset = cd->hwm - cd->start_workspace; + called = cd->start_code; + + /* When we are actually compiling, find the bracket that is being + referenced. Temporarily end the regex in case it doesn't exist before + this point. If we end up with a forward reference, first check that + the bracket does occur later so we can give the error (and position) + now. Then remember this forward reference in the workspace so it can + be filled in at the end. */ + + if (lengthptr == NULL) + { + *code = OP_END; + if (recno != 0) + called = PRIV(find_bracket)(cd->start_code, utf, recno); + + /* Forward reference */ + + if (called == NULL) + { + if (recno > cd->final_bracount) + { + *errorcodeptr = ERR15; + goto FAILED; + } + + /* Fudge the value of "called" so that when it is inserted as an + offset below, what it actually inserted is the reference number + of the group. Then remember the forward reference. */ + + called = cd->start_code + recno; + if (cd->hwm >= cd->start_workspace + cd->workspace_size - + WORK_SIZE_SAFETY_MARGIN) + { + *errorcodeptr = expand_workspace(cd); + if (*errorcodeptr != 0) goto FAILED; + } + PUTINC(cd->hwm, 0, (int)(code + 1 - cd->start_code)); + } + + /* If not a forward reference, and the subpattern is still open, + this is a recursive call. We check to see if this is a left + recursion that could loop for ever, and diagnose that case. We + must not, however, do this check if we are in a conditional + subpattern because the condition might be testing for recursion in + a pattern such as /(?(R)a+|(?R)b)/, which is perfectly valid. + Forever loops are also detected at runtime, so those that occur in + conditional subpatterns will be picked up then. */ + + else if (GET(called, 1) == 0 && cond_depth <= 0 && + could_be_empty(called, code, bcptr, utf, cd)) + { + *errorcodeptr = ERR40; + goto FAILED; + } + } + + /* Insert the recursion/subroutine item. It does not have a set first + character (relevant if it is repeated, because it will then be + wrapped with ONCE brackets). */ + + *code = OP_RECURSE; + PUT(code, 1, (int)(called - cd->start_code)); + code += 1 + LINK_SIZE; + groupsetfirstchar = FALSE; + } + + /* Can't determine a first byte now */ + + if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; + continue; + + + /* ------------------------------------------------------------ */ + default: /* Other characters: check option setting */ + OTHER_CHAR_AFTER_QUERY: + set = unset = 0; + optset = &set; + + while (*ptr != CHAR_RIGHT_PARENTHESIS && *ptr != CHAR_COLON) + { + switch (*ptr++) + { + case CHAR_MINUS: optset = &unset; break; + + case CHAR_J: /* Record that it changed in the external options */ + *optset |= PCRE_DUPNAMES; + cd->external_flags |= PCRE_JCHANGED; + break; + + case CHAR_i: *optset |= PCRE_CASELESS; break; + case CHAR_m: *optset |= PCRE_MULTILINE; break; + case CHAR_s: *optset |= PCRE_DOTALL; break; + case CHAR_x: *optset |= PCRE_EXTENDED; break; + case CHAR_U: *optset |= PCRE_UNGREEDY; break; + case CHAR_X: *optset |= PCRE_EXTRA; break; + + default: *errorcodeptr = ERR12; + ptr--; /* Correct the offset */ + goto FAILED; + } + } + + /* Set up the changed option bits, but don't change anything yet. */ + + newoptions = (options | set) & (~unset); + + /* If the options ended with ')' this is not the start of a nested + group with option changes, so the options change at this level. + If we are not at the pattern start, reset the greedy defaults and the + case value for firstchar and reqchar. */ + + if (*ptr == CHAR_RIGHT_PARENTHESIS) + { + greedy_default = ((newoptions & PCRE_UNGREEDY) != 0); + greedy_non_default = greedy_default ^ 1; + req_caseopt = ((newoptions & PCRE_CASELESS) != 0)? REQ_CASELESS:0; + + /* Change options at this level, and pass them back for use + in subsequent branches. */ + + *optionsptr = options = newoptions; + previous = NULL; /* This item can't be repeated */ + continue; /* It is complete */ + } + + /* If the options ended with ':' we are heading into a nested group + with possible change of options. Such groups are non-capturing and are + not assertions of any kind. All we need to do is skip over the ':'; + the newoptions value is handled below. */ + + bravalue = OP_BRA; + ptr++; + } /* End of switch for character following (? */ + } /* End of (? handling */ + + /* Opening parenthesis not followed by '*' or '?'. If PCRE_NO_AUTO_CAPTURE + is set, all unadorned brackets become non-capturing and behave like (?:...) + brackets. */ + + else if ((options & PCRE_NO_AUTO_CAPTURE) != 0) + { + bravalue = OP_BRA; + } + + /* Else we have a capturing group. */ + + else + { + NUMBERED_GROUP: + cd->bracount += 1; + PUT2(code, 1+LINK_SIZE, cd->bracount); + skipbytes = IMM2_SIZE; + } + + /* Process nested bracketed regex. First check for parentheses nested too + deeply. */ + + if ((cd->parens_depth += 1) > PARENS_NEST_LIMIT) + { + *errorcodeptr = ERR82; + goto FAILED; + } + + /* All assertions used not to be repeatable, but this was changed for Perl + compatibility. All kinds can now be repeated except for assertions that are + conditions (Perl also forbids these to be repeated). We copy code into a + non-register variable (tempcode) in order to be able to pass its address + because some compilers complain otherwise. At the start of a conditional + group whose condition is an assertion, cd->iscondassert is set. We unset it + here so as to allow assertions later in the group to be quantified. */ + + if (bravalue >= OP_ASSERT && bravalue <= OP_ASSERTBACK_NOT && + cd->iscondassert) + { + previous = NULL; + cd->iscondassert = FALSE; + } + else + { + previous = code; + item_hwm_offset = cd->hwm - cd->start_workspace; + } + + *code = bravalue; + tempcode = code; + tempreqvary = cd->req_varyopt; /* Save value before bracket */ + tempbracount = cd->bracount; /* Save value before bracket */ + length_prevgroup = 0; /* Initialize for pre-compile phase */ + + if (!compile_regex( + newoptions, /* The complete new option state */ + &tempcode, /* Where to put code (updated) */ + &ptr, /* Input pointer (updated) */ + errorcodeptr, /* Where to put an error message */ + (bravalue == OP_ASSERTBACK || + bravalue == OP_ASSERTBACK_NOT), /* TRUE if back assert */ + reset_bracount, /* True if (?| group */ + skipbytes, /* Skip over bracket number */ + cond_depth + + ((bravalue == OP_COND)?1:0), /* Depth of condition subpatterns */ + &subfirstchar, /* For possible first char */ + &subfirstcharflags, + &subreqchar, /* For possible last char */ + &subreqcharflags, + bcptr, /* Current branch chain */ + cd, /* Tables block */ + (lengthptr == NULL)? NULL : /* Actual compile phase */ + &length_prevgroup /* Pre-compile phase */ + )) + goto FAILED; + + cd->parens_depth -= 1; + + /* If this was an atomic group and there are no capturing groups within it, + generate OP_ONCE_NC instead of OP_ONCE. */ + + if (bravalue == OP_ONCE && cd->bracount <= tempbracount) + *code = OP_ONCE_NC; + + if (bravalue >= OP_ASSERT && bravalue <= OP_ASSERTBACK_NOT) + cd->assert_depth -= 1; + + /* At the end of compiling, code is still pointing to the start of the + group, while tempcode has been updated to point past the end of the group. + The pattern pointer (ptr) is on the bracket. + + If this is a conditional bracket, check that there are no more than + two branches in the group, or just one if it's a DEFINE group. We do this + in the real compile phase, not in the pre-pass, where the whole group may + not be available. */ + + if (bravalue == OP_COND && lengthptr == NULL) + { + pcre_uchar *tc = code; + int condcount = 0; + + do { + condcount++; + tc += GET(tc,1); + } + while (*tc != OP_KET); + + /* A DEFINE group is never obeyed inline (the "condition" is always + false). It must have only one branch. */ + + if (code[LINK_SIZE+1] == OP_DEF) + { + if (condcount > 1) + { + *errorcodeptr = ERR54; + goto FAILED; + } + bravalue = OP_DEF; /* Just a flag to suppress char handling below */ + } + + /* A "normal" conditional group. If there is just one branch, we must not + make use of its firstchar or reqchar, because this is equivalent to an + empty second branch. */ + + else + { + if (condcount > 2) + { + *errorcodeptr = ERR27; + goto FAILED; + } + if (condcount == 1) subfirstcharflags = subreqcharflags = REQ_NONE; + } + } + + /* Error if hit end of pattern */ + + if (*ptr != CHAR_RIGHT_PARENTHESIS) + { + *errorcodeptr = ERR14; + goto FAILED; + } + + /* In the pre-compile phase, update the length by the length of the group, + less the brackets at either end. Then reduce the compiled code to just a + set of non-capturing brackets so that it doesn't use much memory if it is + duplicated by a quantifier.*/ + + if (lengthptr != NULL) + { + if (OFLOW_MAX - *lengthptr < length_prevgroup - 2 - 2*LINK_SIZE) + { + *errorcodeptr = ERR20; + goto FAILED; + } + *lengthptr += length_prevgroup - 2 - 2*LINK_SIZE; + code++; /* This already contains bravalue */ + PUTINC(code, 0, 1 + LINK_SIZE); + *code++ = OP_KET; + PUTINC(code, 0, 1 + LINK_SIZE); + break; /* No need to waste time with special character handling */ + } + + /* Otherwise update the main code pointer to the end of the group. */ + + code = tempcode; + + /* For a DEFINE group, required and first character settings are not + relevant. */ + + if (bravalue == OP_DEF) break; + + /* Handle updating of the required and first characters for other types of + group. Update for normal brackets of all kinds, and conditions with two + branches (see code above). If the bracket is followed by a quantifier with + zero repeat, we have to back off. Hence the definition of zeroreqchar and + zerofirstchar outside the main loop so that they can be accessed for the + back off. */ + + zeroreqchar = reqchar; + zeroreqcharflags = reqcharflags; + zerofirstchar = firstchar; + zerofirstcharflags = firstcharflags; + groupsetfirstchar = FALSE; + + if (bravalue >= OP_ONCE) + { + /* If we have not yet set a firstchar in this branch, take it from the + subpattern, remembering that it was set here so that a repeat of more + than one can replicate it as reqchar if necessary. If the subpattern has + no firstchar, set "none" for the whole branch. In both cases, a zero + repeat forces firstchar to "none". */ + + if (firstcharflags == REQ_UNSET) + { + if (subfirstcharflags >= 0) + { + firstchar = subfirstchar; + firstcharflags = subfirstcharflags; + groupsetfirstchar = TRUE; + } + else firstcharflags = REQ_NONE; + zerofirstcharflags = REQ_NONE; + } + + /* If firstchar was previously set, convert the subpattern's firstchar + into reqchar if there wasn't one, using the vary flag that was in + existence beforehand. */ + + else if (subfirstcharflags >= 0 && subreqcharflags < 0) + { + subreqchar = subfirstchar; + subreqcharflags = subfirstcharflags | tempreqvary; + } + + /* If the subpattern set a required byte (or set a first byte that isn't + really the first byte - see above), set it. */ + + if (subreqcharflags >= 0) + { + reqchar = subreqchar; + reqcharflags = subreqcharflags; + } + } + + /* For a forward assertion, we take the reqchar, if set, provided that the + group has also set a first char. This can be helpful if the pattern that + follows the assertion doesn't set a different char. For example, it's + useful for /(?=abcde).+/. We can't set firstchar for an assertion, however + because it leads to incorrect effect for patterns such as /(?=a)a.+/ when + the "real" "a" would then become a reqchar instead of a firstchar. This is + overcome by a scan at the end if there's no firstchar, looking for an + asserted first char. */ + + else if (bravalue == OP_ASSERT && subreqcharflags >= 0 && + subfirstcharflags >= 0) + { + reqchar = subreqchar; + reqcharflags = subreqcharflags; + } + break; /* End of processing '(' */ + + + /* ===================================================================*/ + /* Handle metasequences introduced by \. For ones like \d, the ESC_ values + are arranged to be the negation of the corresponding OP_values in the + default case when PCRE_UCP is not set. For the back references, the values + are negative the reference number. Only back references and those types + that consume a character may be repeated. We can test for values between + ESC_b and ESC_Z for the latter; this may have to change if any new ones are + ever created. */ + + case CHAR_BACKSLASH: + tempptr = ptr; + escape = check_escape(&ptr, &ec, errorcodeptr, cd->bracount, options, FALSE); + if (*errorcodeptr != 0) goto FAILED; + + if (escape == 0) /* The escape coded a single character */ + c = ec; + else + { + /* For metasequences that actually match a character, we disable the + setting of a first character if it hasn't already been set. */ + + if (firstcharflags == REQ_UNSET && escape > ESC_b && escape < ESC_Z) + firstcharflags = REQ_NONE; + + /* Set values to reset to if this is followed by a zero repeat. */ + + zerofirstchar = firstchar; + zerofirstcharflags = firstcharflags; + zeroreqchar = reqchar; + zeroreqcharflags = reqcharflags; + + /* \g or \g'name' is a subroutine call by name and \g or \g'n' + is a subroutine call by number (Oniguruma syntax). In fact, the value + ESC_g is returned only for these cases. So we don't need to check for < + or ' if the value is ESC_g. For the Perl syntax \g{n} the value is + -n, and for the Perl syntax \g{name} the result is ESC_k (as + that is a synonym for a named back reference). */ + + if (escape == ESC_g) + { + const pcre_uchar *p; + pcre_uint32 cf; + + item_hwm_offset = cd->hwm - cd->start_workspace; /* Normally this is set when '(' is read */ + terminator = (*(++ptr) == CHAR_LESS_THAN_SIGN)? + CHAR_GREATER_THAN_SIGN : CHAR_APOSTROPHE; + + /* These two statements stop the compiler for warning about possibly + unset variables caused by the jump to HANDLE_NUMERICAL_RECURSION. In + fact, because we do the check for a number below, the paths that + would actually be in error are never taken. */ + + skipbytes = 0; + reset_bracount = FALSE; + + /* If it's not a signed or unsigned number, treat it as a name. */ + + cf = ptr[1]; + if (cf != CHAR_PLUS && cf != CHAR_MINUS && !IS_DIGIT(cf)) + { + is_recurse = TRUE; + goto NAMED_REF_OR_RECURSE; + } + + /* Signed or unsigned number (cf = ptr[1]) is known to be plus or minus + or a digit. */ + + p = ptr + 2; + while (IS_DIGIT(*p)) p++; + if (*p != (pcre_uchar)terminator) + { + *errorcodeptr = ERR57; + goto FAILED; + } + ptr++; + goto HANDLE_NUMERICAL_RECURSION; + } + + /* \k or \k'name' is a back reference by name (Perl syntax). + We also support \k{name} (.NET syntax). */ + + if (escape == ESC_k) + { + if ((ptr[1] != CHAR_LESS_THAN_SIGN && + ptr[1] != CHAR_APOSTROPHE && ptr[1] != CHAR_LEFT_CURLY_BRACKET)) + { + *errorcodeptr = ERR69; + goto FAILED; + } + is_recurse = FALSE; + terminator = (*(++ptr) == CHAR_LESS_THAN_SIGN)? + CHAR_GREATER_THAN_SIGN : (*ptr == CHAR_APOSTROPHE)? + CHAR_APOSTROPHE : CHAR_RIGHT_CURLY_BRACKET; + goto NAMED_REF_OR_RECURSE; + } + + /* Back references are handled specially; must disable firstchar if + not set to cope with cases like (?=(\w+))\1: which would otherwise set + ':' later. */ + + if (escape < 0) + { + open_capitem *oc; + recno = -escape; + + /* Come here from named backref handling when the reference is to a + single group (i.e. not to a duplicated name. */ + + HANDLE_REFERENCE: + if (firstcharflags == REQ_UNSET) firstcharflags = REQ_NONE; + previous = code; + item_hwm_offset = cd->hwm - cd->start_workspace; + *code++ = ((options & PCRE_CASELESS) != 0)? OP_REFI : OP_REF; + PUT2INC(code, 0, recno); + cd->backref_map |= (recno < 32)? (1 << recno) : 1; + if (recno > cd->top_backref) cd->top_backref = recno; + + /* Check to see if this back reference is recursive, that it, it + is inside the group that it references. A flag is set so that the + group can be made atomic. */ + + for (oc = cd->open_caps; oc != NULL; oc = oc->next) + { + if (oc->number == recno) + { + oc->flag = TRUE; + break; + } + } + } + + /* So are Unicode property matches, if supported. */ + +#ifdef SUPPORT_UCP + else if (escape == ESC_P || escape == ESC_p) + { + BOOL negated; + unsigned int ptype = 0, pdata = 0; + if (!get_ucp(&ptr, &negated, &ptype, &pdata, errorcodeptr)) + goto FAILED; + previous = code; + item_hwm_offset = cd->hwm - cd->start_workspace; + *code++ = ((escape == ESC_p) != negated)? OP_PROP : OP_NOTPROP; + *code++ = ptype; + *code++ = pdata; + } +#else + + /* If Unicode properties are not supported, \X, \P, and \p are not + allowed. */ + + else if (escape == ESC_X || escape == ESC_P || escape == ESC_p) + { + *errorcodeptr = ERR45; + goto FAILED; + } +#endif + + /* For the rest (including \X when Unicode properties are supported), we + can obtain the OP value by negating the escape value in the default + situation when PCRE_UCP is not set. When it *is* set, we substitute + Unicode property tests. Note that \b and \B do a one-character + lookbehind, and \A also behaves as if it does. */ + + else + { + if ((escape == ESC_b || escape == ESC_B || escape == ESC_A) && + cd->max_lookbehind == 0) + cd->max_lookbehind = 1; +#ifdef SUPPORT_UCP + if (escape >= ESC_DU && escape <= ESC_wu) + { + nestptr = ptr + 1; /* Where to resume */ + ptr = substitutes[escape - ESC_DU] - 1; /* Just before substitute */ + } + else +#endif + /* In non-UTF-8 mode, we turn \C into OP_ALLANY instead of OP_ANYBYTE + so that it works in DFA mode and in lookbehinds. */ + + { + previous = (escape > ESC_b && escape < ESC_Z)? code : NULL; + item_hwm_offset = cd->hwm - cd->start_workspace; + *code++ = (!utf && escape == ESC_C)? OP_ALLANY : escape; + } + } + continue; + } + + /* We have a data character whose value is in c. In UTF-8 mode it may have + a value > 127. We set its representation in the length/buffer, and then + handle it as a data character. */ + +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (utf && c > MAX_VALUE_FOR_SINGLE_CHAR) + mclength = PRIV(ord2utf)(c, mcbuffer); + else +#endif + + { + mcbuffer[0] = c; + mclength = 1; + } + goto ONE_CHAR; + + + /* ===================================================================*/ + /* Handle a literal character. It is guaranteed not to be whitespace or # + when the extended flag is set. If we are in a UTF mode, it may be a + multi-unit literal character. */ + + default: + NORMAL_CHAR: + mclength = 1; + mcbuffer[0] = c; + +#ifdef SUPPORT_UTF + if (utf && HAS_EXTRALEN(c)) + ACROSSCHAR(TRUE, ptr[1], mcbuffer[mclength++] = *(++ptr)); +#endif + + /* At this point we have the character's bytes in mcbuffer, and the length + in mclength. When not in UTF-8 mode, the length is always 1. */ + + ONE_CHAR: + previous = code; + item_hwm_offset = cd->hwm - cd->start_workspace; + + /* For caseless UTF-8 mode when UCP support is available, check whether + this character has more than one other case. If so, generate a special + OP_PROP item instead of OP_CHARI. */ + +#ifdef SUPPORT_UCP + if (utf && (options & PCRE_CASELESS) != 0) + { + GETCHAR(c, mcbuffer); + if ((c = UCD_CASESET(c)) != 0) + { + *code++ = OP_PROP; + *code++ = PT_CLIST; + *code++ = c; + if (firstcharflags == REQ_UNSET) + firstcharflags = zerofirstcharflags = REQ_NONE; + break; + } + } +#endif + + /* Caseful matches, or not one of the multicase characters. */ + + *code++ = ((options & PCRE_CASELESS) != 0)? OP_CHARI : OP_CHAR; + for (c = 0; c < mclength; c++) *code++ = mcbuffer[c]; + + /* Remember if \r or \n were seen */ + + if (mcbuffer[0] == CHAR_CR || mcbuffer[0] == CHAR_NL) + cd->external_flags |= PCRE_HASCRORLF; + + /* Set the first and required bytes appropriately. If no previous first + byte, set it from this character, but revert to none on a zero repeat. + Otherwise, leave the firstchar value alone, and don't change it on a zero + repeat. */ + + if (firstcharflags == REQ_UNSET) + { + zerofirstcharflags = REQ_NONE; + zeroreqchar = reqchar; + zeroreqcharflags = reqcharflags; + + /* If the character is more than one byte long, we can set firstchar + only if it is not to be matched caselessly. */ + + if (mclength == 1 || req_caseopt == 0) + { + firstchar = mcbuffer[0]; + firstcharflags = req_caseopt; + + if (mclength != 1) + { + reqchar = code[-1]; + reqcharflags = cd->req_varyopt; + } + } + else firstcharflags = reqcharflags = REQ_NONE; + } + + /* firstchar was previously set; we can set reqchar only if the length is + 1 or the matching is caseful. */ + + else + { + zerofirstchar = firstchar; + zerofirstcharflags = firstcharflags; + zeroreqchar = reqchar; + zeroreqcharflags = reqcharflags; + if (mclength == 1 || req_caseopt == 0) + { + reqchar = code[-1]; + reqcharflags = req_caseopt | cd->req_varyopt; + } + } + + break; /* End of literal character handling */ + } + } /* end of big loop */ + + +/* Control never reaches here by falling through, only by a goto for all the +error states. Pass back the position in the pattern so that it can be displayed +to the user for diagnosing the error. */ + +FAILED: +*ptrptr = ptr; +return FALSE; +} + + + +/************************************************* +* Compile sequence of alternatives * +*************************************************/ + +/* On entry, ptr is pointing past the bracket character, but on return it +points to the closing bracket, or vertical bar, or end of string. The code +variable is pointing at the byte into which the BRA operator has been stored. +This function is used during the pre-compile phase when we are trying to find +out the amount of memory needed, as well as during the real compile phase. The +value of lengthptr distinguishes the two phases. + +Arguments: + options option bits, including any changes for this subpattern + codeptr -> the address of the current code pointer + ptrptr -> the address of the current pattern pointer + errorcodeptr -> pointer to error code variable + lookbehind TRUE if this is a lookbehind assertion + reset_bracount TRUE to reset the count for each branch + skipbytes skip this many bytes at start (for brackets and OP_COND) + cond_depth depth of nesting for conditional subpatterns + firstcharptr place to put the first required character + firstcharflagsptr place to put the first character flags, or a negative number + reqcharptr place to put the last required character + reqcharflagsptr place to put the last required character flags, or a negative number + bcptr pointer to the chain of currently open branches + cd points to the data block with tables pointers etc. + lengthptr NULL during the real compile phase + points to length accumulator during pre-compile phase + +Returns: TRUE on success +*/ + +static BOOL +compile_regex(int options, pcre_uchar **codeptr, const pcre_uchar **ptrptr, + int *errorcodeptr, BOOL lookbehind, BOOL reset_bracount, int skipbytes, + int cond_depth, + pcre_uint32 *firstcharptr, pcre_int32 *firstcharflagsptr, + pcre_uint32 *reqcharptr, pcre_int32 *reqcharflagsptr, + branch_chain *bcptr, compile_data *cd, int *lengthptr) +{ +const pcre_uchar *ptr = *ptrptr; +pcre_uchar *code = *codeptr; +pcre_uchar *last_branch = code; +pcre_uchar *start_bracket = code; +pcre_uchar *reverse_count = NULL; +open_capitem capitem; +int capnumber = 0; +pcre_uint32 firstchar, reqchar; +pcre_int32 firstcharflags, reqcharflags; +pcre_uint32 branchfirstchar, branchreqchar; +pcre_int32 branchfirstcharflags, branchreqcharflags; +int length; +unsigned int orig_bracount; +unsigned int max_bracount; +branch_chain bc; +size_t save_hwm_offset; + +/* If set, call the external function that checks for stack availability. */ + +if (PUBL(stack_guard) != NULL && PUBL(stack_guard)()) + { + *errorcodeptr= ERR85; + return FALSE; + } + +/* Miscellaneous initialization */ + +bc.outer = bcptr; +bc.current_branch = code; + +firstchar = reqchar = 0; +firstcharflags = reqcharflags = REQ_UNSET; + +save_hwm_offset = cd->hwm - cd->start_workspace; + +/* Accumulate the length for use in the pre-compile phase. Start with the +length of the BRA and KET and any extra bytes that are required at the +beginning. We accumulate in a local variable to save frequent testing of +lenthptr for NULL. We cannot do this by looking at the value of code at the +start and end of each alternative, because compiled items are discarded during +the pre-compile phase so that the work space is not exceeded. */ + +length = 2 + 2*LINK_SIZE + skipbytes; + +/* WARNING: If the above line is changed for any reason, you must also change +the code that abstracts option settings at the start of the pattern and makes +them global. It tests the value of length for (2 + 2*LINK_SIZE) in the +pre-compile phase to find out whether anything has yet been compiled or not. */ + +/* If this is a capturing subpattern, add to the chain of open capturing items +so that we can detect them if (*ACCEPT) is encountered. This is also used to +detect groups that contain recursive back references to themselves. Note that +only OP_CBRA need be tested here; changing this opcode to one of its variants, +e.g. OP_SCBRAPOS, happens later, after the group has been compiled. */ + +if (*code == OP_CBRA) + { + capnumber = GET2(code, 1 + LINK_SIZE); + capitem.number = capnumber; + capitem.next = cd->open_caps; + capitem.flag = FALSE; + cd->open_caps = &capitem; + } + +/* Offset is set zero to mark that this bracket is still open */ + +PUT(code, 1, 0); +code += 1 + LINK_SIZE + skipbytes; + +/* Loop for each alternative branch */ + +orig_bracount = max_bracount = cd->bracount; +for (;;) + { + /* For a (?| group, reset the capturing bracket count so that each branch + uses the same numbers. */ + + if (reset_bracount) cd->bracount = orig_bracount; + + /* Set up dummy OP_REVERSE if lookbehind assertion */ + + if (lookbehind) + { + *code++ = OP_REVERSE; + reverse_count = code; + PUTINC(code, 0, 0); + length += 1 + LINK_SIZE; + } + + /* Now compile the branch; in the pre-compile phase its length gets added + into the length. */ + + if (!compile_branch(&options, &code, &ptr, errorcodeptr, &branchfirstchar, + &branchfirstcharflags, &branchreqchar, &branchreqcharflags, &bc, + cond_depth, cd, (lengthptr == NULL)? NULL : &length)) + { + *ptrptr = ptr; + return FALSE; + } + + /* Keep the highest bracket count in case (?| was used and some branch + has fewer than the rest. */ + + if (cd->bracount > max_bracount) max_bracount = cd->bracount; + + /* In the real compile phase, there is some post-processing to be done. */ + + if (lengthptr == NULL) + { + /* If this is the first branch, the firstchar and reqchar values for the + branch become the values for the regex. */ + + if (*last_branch != OP_ALT) + { + firstchar = branchfirstchar; + firstcharflags = branchfirstcharflags; + reqchar = branchreqchar; + reqcharflags = branchreqcharflags; + } + + /* If this is not the first branch, the first char and reqchar have to + match the values from all the previous branches, except that if the + previous value for reqchar didn't have REQ_VARY set, it can still match, + and we set REQ_VARY for the regex. */ + + else + { + /* If we previously had a firstchar, but it doesn't match the new branch, + we have to abandon the firstchar for the regex, but if there was + previously no reqchar, it takes on the value of the old firstchar. */ + + if (firstcharflags >= 0 && + (firstcharflags != branchfirstcharflags || firstchar != branchfirstchar)) + { + if (reqcharflags < 0) + { + reqchar = firstchar; + reqcharflags = firstcharflags; + } + firstcharflags = REQ_NONE; + } + + /* If we (now or from before) have no firstchar, a firstchar from the + branch becomes a reqchar if there isn't a branch reqchar. */ + + if (firstcharflags < 0 && branchfirstcharflags >= 0 && branchreqcharflags < 0) + { + branchreqchar = branchfirstchar; + branchreqcharflags = branchfirstcharflags; + } + + /* Now ensure that the reqchars match */ + + if (((reqcharflags & ~REQ_VARY) != (branchreqcharflags & ~REQ_VARY)) || + reqchar != branchreqchar) + reqcharflags = REQ_NONE; + else + { + reqchar = branchreqchar; + reqcharflags |= branchreqcharflags; /* To "or" REQ_VARY */ + } + } + + /* If lookbehind, check that this branch matches a fixed-length string, and + put the length into the OP_REVERSE item. Temporarily mark the end of the + branch with OP_END. If the branch contains OP_RECURSE, the result is -3 + because there may be forward references that we can't check here. Set a + flag to cause another lookbehind check at the end. Why not do it all at the + end? Because common, erroneous checks are picked up here and the offset of + the problem can be shown. */ + + if (lookbehind) + { + int fixed_length; + *code = OP_END; + fixed_length = find_fixedlength(last_branch, (options & PCRE_UTF8) != 0, + FALSE, cd, NULL); + DPRINTF(("fixed length = %d\n", fixed_length)); + if (fixed_length == -3) + { + cd->check_lookbehind = TRUE; + } + else if (fixed_length < 0) + { + *errorcodeptr = (fixed_length == -2)? ERR36 : + (fixed_length == -4)? ERR70: ERR25; + *ptrptr = ptr; + return FALSE; + } + else + { + if (fixed_length > cd->max_lookbehind) + cd->max_lookbehind = fixed_length; + PUT(reverse_count, 0, fixed_length); + } + } + } + + /* Reached end of expression, either ')' or end of pattern. In the real + compile phase, go back through the alternative branches and reverse the chain + of offsets, with the field in the BRA item now becoming an offset to the + first alternative. If there are no alternatives, it points to the end of the + group. The length in the terminating ket is always the length of the whole + bracketed item. Return leaving the pointer at the terminating char. */ + + if (*ptr != CHAR_VERTICAL_LINE) + { + if (lengthptr == NULL) + { + int branch_length = (int)(code - last_branch); + do + { + int prev_length = GET(last_branch, 1); + PUT(last_branch, 1, branch_length); + branch_length = prev_length; + last_branch -= branch_length; + } + while (branch_length > 0); + } + + /* Fill in the ket */ + + *code = OP_KET; + PUT(code, 1, (int)(code - start_bracket)); + code += 1 + LINK_SIZE; + + /* If it was a capturing subpattern, check to see if it contained any + recursive back references. If so, we must wrap it in atomic brackets. + Because we are moving code along, we must ensure that any pending recursive + references are updated. In any event, remove the block from the chain. */ + + if (capnumber > 0) + { + if (cd->open_caps->flag) + { + *code = OP_END; + adjust_recurse(start_bracket, 1 + LINK_SIZE, + (options & PCRE_UTF8) != 0, cd, save_hwm_offset); + memmove(start_bracket + 1 + LINK_SIZE, start_bracket, + IN_UCHARS(code - start_bracket)); + *start_bracket = OP_ONCE; + code += 1 + LINK_SIZE; + PUT(start_bracket, 1, (int)(code - start_bracket)); + *code = OP_KET; + PUT(code, 1, (int)(code - start_bracket)); + code += 1 + LINK_SIZE; + length += 2 + 2*LINK_SIZE; + } + cd->open_caps = cd->open_caps->next; + } + + /* Retain the highest bracket number, in case resetting was used. */ + + cd->bracount = max_bracount; + + /* Set values to pass back */ + + *codeptr = code; + *ptrptr = ptr; + *firstcharptr = firstchar; + *firstcharflagsptr = firstcharflags; + *reqcharptr = reqchar; + *reqcharflagsptr = reqcharflags; + if (lengthptr != NULL) + { + if (OFLOW_MAX - *lengthptr < length) + { + *errorcodeptr = ERR20; + return FALSE; + } + *lengthptr += length; + } + return TRUE; + } + + /* Another branch follows. In the pre-compile phase, we can move the code + pointer back to where it was for the start of the first branch. (That is, + pretend that each branch is the only one.) + + In the real compile phase, insert an ALT node. Its length field points back + to the previous branch while the bracket remains open. At the end the chain + is reversed. It's done like this so that the start of the bracket has a + zero offset until it is closed, making it possible to detect recursion. */ + + if (lengthptr != NULL) + { + code = *codeptr + 1 + LINK_SIZE + skipbytes; + length += 1 + LINK_SIZE; + } + else + { + *code = OP_ALT; + PUT(code, 1, (int)(code - last_branch)); + bc.current_branch = last_branch = code; + code += 1 + LINK_SIZE; + } + + ptr++; + } +/* Control never reaches here */ +} + + + + +/************************************************* +* Check for anchored expression * +*************************************************/ + +/* Try to find out if this is an anchored regular expression. Consider each +alternative branch. If they all start with OP_SOD or OP_CIRC, or with a bracket +all of whose alternatives start with OP_SOD or OP_CIRC (recurse ad lib), then +it's anchored. However, if this is a multiline pattern, then only OP_SOD will +be found, because ^ generates OP_CIRCM in that mode. + +We can also consider a regex to be anchored if OP_SOM starts all its branches. +This is the code for \G, which means "match at start of match position, taking +into account the match offset". + +A branch is also implicitly anchored if it starts with .* and DOTALL is set, +because that will try the rest of the pattern at all possible matching points, +so there is no point trying again.... er .... + +.... except when the .* appears inside capturing parentheses, and there is a +subsequent back reference to those parentheses. We haven't enough information +to catch that case precisely. + +At first, the best we could do was to detect when .* was in capturing brackets +and the highest back reference was greater than or equal to that level. +However, by keeping a bitmap of the first 31 back references, we can catch some +of the more common cases more precisely. + +... A second exception is when the .* appears inside an atomic group, because +this prevents the number of characters it matches from being adjusted. + +Arguments: + code points to start of expression (the bracket) + bracket_map a bitmap of which brackets we are inside while testing; this + handles up to substring 31; after that we just have to take + the less precise approach + cd points to the compile data block + atomcount atomic group level + +Returns: TRUE or FALSE +*/ + +static BOOL +is_anchored(register const pcre_uchar *code, unsigned int bracket_map, + compile_data *cd, int atomcount) +{ +do { + const pcre_uchar *scode = first_significant_code( + code + PRIV(OP_lengths)[*code], FALSE); + register int op = *scode; + + /* Non-capturing brackets */ + + if (op == OP_BRA || op == OP_BRAPOS || + op == OP_SBRA || op == OP_SBRAPOS) + { + if (!is_anchored(scode, bracket_map, cd, atomcount)) return FALSE; + } + + /* Capturing brackets */ + + else if (op == OP_CBRA || op == OP_CBRAPOS || + op == OP_SCBRA || op == OP_SCBRAPOS) + { + int n = GET2(scode, 1+LINK_SIZE); + int new_map = bracket_map | ((n < 32)? (1 << n) : 1); + if (!is_anchored(scode, new_map, cd, atomcount)) return FALSE; + } + + /* Positive forward assertions and conditions */ + + else if (op == OP_ASSERT || op == OP_COND) + { + if (!is_anchored(scode, bracket_map, cd, atomcount)) return FALSE; + } + + /* Atomic groups */ + + else if (op == OP_ONCE || op == OP_ONCE_NC) + { + if (!is_anchored(scode, bracket_map, cd, atomcount + 1)) + return FALSE; + } + + /* .* is not anchored unless DOTALL is set (which generates OP_ALLANY) and + it isn't in brackets that are or may be referenced or inside an atomic + group. */ + + else if ((op == OP_TYPESTAR || op == OP_TYPEMINSTAR || + op == OP_TYPEPOSSTAR)) + { + if (scode[1] != OP_ALLANY || (bracket_map & cd->backref_map) != 0 || + atomcount > 0 || cd->had_pruneorskip) + return FALSE; + } + + /* Check for explicit anchoring */ + + else if (op != OP_SOD && op != OP_SOM && op != OP_CIRC) return FALSE; + + code += GET(code, 1); + } +while (*code == OP_ALT); /* Loop for each alternative */ +return TRUE; +} + + + +/************************************************* +* Check for starting with ^ or .* * +*************************************************/ + +/* This is called to find out if every branch starts with ^ or .* so that +"first char" processing can be done to speed things up in multiline +matching and for non-DOTALL patterns that start with .* (which must start at +the beginning or after \n). As in the case of is_anchored() (see above), we +have to take account of back references to capturing brackets that contain .* +because in that case we can't make the assumption. Also, the appearance of .* +inside atomic brackets or in an assertion, or in a pattern that contains *PRUNE +or *SKIP does not count, because once again the assumption no longer holds. + +Arguments: + code points to start of expression (the bracket) + bracket_map a bitmap of which brackets we are inside while testing; this + handles up to substring 31; after that we just have to take + the less precise approach + cd points to the compile data + atomcount atomic group level + inassert TRUE if in an assertion + +Returns: TRUE or FALSE +*/ + +static BOOL +is_startline(const pcre_uchar *code, unsigned int bracket_map, + compile_data *cd, int atomcount, BOOL inassert) +{ +do { + const pcre_uchar *scode = first_significant_code( + code + PRIV(OP_lengths)[*code], FALSE); + register int op = *scode; + + /* If we are at the start of a conditional assertion group, *both* the + conditional assertion *and* what follows the condition must satisfy the test + for start of line. Other kinds of condition fail. Note that there may be an + auto-callout at the start of a condition. */ + + if (op == OP_COND) + { + scode += 1 + LINK_SIZE; + if (*scode == OP_CALLOUT) scode += PRIV(OP_lengths)[OP_CALLOUT]; + switch (*scode) + { + case OP_CREF: + case OP_DNCREF: + case OP_RREF: + case OP_DNRREF: + case OP_DEF: + case OP_FAIL: + return FALSE; + + default: /* Assertion */ + if (!is_startline(scode, bracket_map, cd, atomcount, TRUE)) return FALSE; + do scode += GET(scode, 1); while (*scode == OP_ALT); + scode += 1 + LINK_SIZE; + break; + } + scode = first_significant_code(scode, FALSE); + op = *scode; + } + + /* Non-capturing brackets */ + + if (op == OP_BRA || op == OP_BRAPOS || + op == OP_SBRA || op == OP_SBRAPOS) + { + if (!is_startline(scode, bracket_map, cd, atomcount, inassert)) return FALSE; + } + + /* Capturing brackets */ + + else if (op == OP_CBRA || op == OP_CBRAPOS || + op == OP_SCBRA || op == OP_SCBRAPOS) + { + int n = GET2(scode, 1+LINK_SIZE); + int new_map = bracket_map | ((n < 32)? (1 << n) : 1); + if (!is_startline(scode, new_map, cd, atomcount, inassert)) return FALSE; + } + + /* Positive forward assertions */ + + else if (op == OP_ASSERT) + { + if (!is_startline(scode, bracket_map, cd, atomcount, TRUE)) return FALSE; + } + + /* Atomic brackets */ + + else if (op == OP_ONCE || op == OP_ONCE_NC) + { + if (!is_startline(scode, bracket_map, cd, atomcount + 1, inassert)) return FALSE; + } + + /* .* means "start at start or after \n" if it isn't in atomic brackets or + brackets that may be referenced or an assertion, as long as the pattern does + not contain *PRUNE or *SKIP, because these break the feature. Consider, for + example, /.*?a(*PRUNE)b/ with the subject "aab", which matches "ab", i.e. + not at the start of a line. */ + + else if (op == OP_TYPESTAR || op == OP_TYPEMINSTAR || op == OP_TYPEPOSSTAR) + { + if (scode[1] != OP_ANY || (bracket_map & cd->backref_map) != 0 || + atomcount > 0 || cd->had_pruneorskip || inassert) + return FALSE; + } + + /* Check for explicit circumflex; anything else gives a FALSE result. Note + in particular that this includes atomic brackets OP_ONCE and OP_ONCE_NC + because the number of characters matched by .* cannot be adjusted inside + them. */ + + else if (op != OP_CIRC && op != OP_CIRCM) return FALSE; + + /* Move on to the next alternative */ + + code += GET(code, 1); + } +while (*code == OP_ALT); /* Loop for each alternative */ +return TRUE; +} + + + +/************************************************* +* Check for asserted fixed first char * +*************************************************/ + +/* During compilation, the "first char" settings from forward assertions are +discarded, because they can cause conflicts with actual literals that follow. +However, if we end up without a first char setting for an unanchored pattern, +it is worth scanning the regex to see if there is an initial asserted first +char. If all branches start with the same asserted char, or with a +non-conditional bracket all of whose alternatives start with the same asserted +char (recurse ad lib), then we return that char, with the flags set to zero or +REQ_CASELESS; otherwise return zero with REQ_NONE in the flags. + +Arguments: + code points to start of expression (the bracket) + flags points to the first char flags, or to REQ_NONE + inassert TRUE if in an assertion + +Returns: the fixed first char, or 0 with REQ_NONE in flags +*/ + +static pcre_uint32 +find_firstassertedchar(const pcre_uchar *code, pcre_int32 *flags, + BOOL inassert) +{ +register pcre_uint32 c = 0; +int cflags = REQ_NONE; + +*flags = REQ_NONE; +do { + pcre_uint32 d; + int dflags; + int xl = (*code == OP_CBRA || *code == OP_SCBRA || + *code == OP_CBRAPOS || *code == OP_SCBRAPOS)? IMM2_SIZE:0; + const pcre_uchar *scode = first_significant_code(code + 1+LINK_SIZE + xl, + TRUE); + register pcre_uchar op = *scode; + + switch(op) + { + default: + return 0; + + case OP_BRA: + case OP_BRAPOS: + case OP_CBRA: + case OP_SCBRA: + case OP_CBRAPOS: + case OP_SCBRAPOS: + case OP_ASSERT: + case OP_ONCE: + case OP_ONCE_NC: + d = find_firstassertedchar(scode, &dflags, op == OP_ASSERT); + if (dflags < 0) + return 0; + if (cflags < 0) { c = d; cflags = dflags; } else if (c != d || cflags != dflags) return 0; + break; + + case OP_EXACT: + scode += IMM2_SIZE; + /* Fall through */ + + case OP_CHAR: + case OP_PLUS: + case OP_MINPLUS: + case OP_POSPLUS: + if (!inassert) return 0; + if (cflags < 0) { c = scode[1]; cflags = 0; } + else if (c != scode[1]) return 0; + break; + + case OP_EXACTI: + scode += IMM2_SIZE; + /* Fall through */ + + case OP_CHARI: + case OP_PLUSI: + case OP_MINPLUSI: + case OP_POSPLUSI: + if (!inassert) return 0; + if (cflags < 0) { c = scode[1]; cflags = REQ_CASELESS; } + else if (c != scode[1]) return 0; + break; + } + + code += GET(code, 1); + } +while (*code == OP_ALT); + +*flags = cflags; +return c; +} + + + +/************************************************* +* Add an entry to the name/number table * +*************************************************/ + +/* This function is called between compiling passes to add an entry to the +name/number table, maintaining alphabetical order. Checking for permitted +and forbidden duplicates has already been done. + +Arguments: + cd the compile data block + name the name to add + length the length of the name + groupno the group number + +Returns: nothing +*/ + +static void +add_name(compile_data *cd, const pcre_uchar *name, int length, + unsigned int groupno) +{ +int i; +pcre_uchar *slot = cd->name_table; + +for (i = 0; i < cd->names_found; i++) + { + int crc = memcmp(name, slot+IMM2_SIZE, IN_UCHARS(length)); + if (crc == 0 && slot[IMM2_SIZE+length] != 0) + crc = -1; /* Current name is a substring */ + + /* Make space in the table and break the loop for an earlier name. For a + duplicate or later name, carry on. We do this for duplicates so that in the + simple case (when ?(| is not used) they are in order of their numbers. In all + cases they are in the order in which they appear in the pattern. */ + + if (crc < 0) + { + memmove(slot + cd->name_entry_size, slot, + IN_UCHARS((cd->names_found - i) * cd->name_entry_size)); + break; + } + + /* Continue the loop for a later or duplicate name */ + + slot += cd->name_entry_size; + } + +PUT2(slot, 0, groupno); +memcpy(slot + IMM2_SIZE, name, IN_UCHARS(length)); +slot[IMM2_SIZE + length] = 0; +cd->names_found++; +} + + + +/************************************************* +* Compile a Regular Expression * +*************************************************/ + +/* This function takes a string and returns a pointer to a block of store +holding a compiled version of the expression. The original API for this +function had no error code return variable; it is retained for backwards +compatibility. The new function is given a new name. + +Arguments: + pattern the regular expression + options various option bits + errorcodeptr pointer to error code variable (pcre_compile2() only) + can be NULL if you don't want a code value + errorptr pointer to pointer to error text + erroroffset ptr offset in pattern where error was detected + tables pointer to character tables or NULL + +Returns: pointer to compiled data block, or NULL on error, + with errorptr and erroroffset set +*/ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN pcre * PCRE_CALL_CONVENTION +pcre_compile(const char *pattern, int options, const char **errorptr, + int *erroroffset, const unsigned char *tables) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN pcre16 * PCRE_CALL_CONVENTION +pcre16_compile(PCRE_SPTR16 pattern, int options, const char **errorptr, + int *erroroffset, const unsigned char *tables) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN pcre32 * PCRE_CALL_CONVENTION +pcre32_compile(PCRE_SPTR32 pattern, int options, const char **errorptr, + int *erroroffset, const unsigned char *tables) +#endif +{ +#if defined COMPILE_PCRE8 +return pcre_compile2(pattern, options, NULL, errorptr, erroroffset, tables); +#elif defined COMPILE_PCRE16 +return pcre16_compile2(pattern, options, NULL, errorptr, erroroffset, tables); +#elif defined COMPILE_PCRE32 +return pcre32_compile2(pattern, options, NULL, errorptr, erroroffset, tables); +#endif +} + + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN pcre * PCRE_CALL_CONVENTION +pcre_compile2(const char *pattern, int options, int *errorcodeptr, + const char **errorptr, int *erroroffset, const unsigned char *tables) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN pcre16 * PCRE_CALL_CONVENTION +pcre16_compile2(PCRE_SPTR16 pattern, int options, int *errorcodeptr, + const char **errorptr, int *erroroffset, const unsigned char *tables) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN pcre32 * PCRE_CALL_CONVENTION +pcre32_compile2(PCRE_SPTR32 pattern, int options, int *errorcodeptr, + const char **errorptr, int *erroroffset, const unsigned char *tables) +#endif +{ +REAL_PCRE *re; +int length = 1; /* For final END opcode */ +pcre_int32 firstcharflags, reqcharflags; +pcre_uint32 firstchar, reqchar; +pcre_uint32 limit_match = PCRE_UINT32_MAX; +pcre_uint32 limit_recursion = PCRE_UINT32_MAX; +int newline; +int errorcode = 0; +int skipatstart = 0; +BOOL utf; +BOOL never_utf = FALSE; +size_t size; +pcre_uchar *code; +const pcre_uchar *codestart; +const pcre_uchar *ptr; +compile_data compile_block; +compile_data *cd = &compile_block; + +/* This space is used for "compiling" into during the first phase, when we are +computing the amount of memory that is needed. Compiled items are thrown away +as soon as possible, so that a fairly large buffer should be sufficient for +this purpose. The same space is used in the second phase for remembering where +to fill in forward references to subpatterns. That may overflow, in which case +new memory is obtained from malloc(). */ + +pcre_uchar cworkspace[COMPILE_WORK_SIZE]; + +/* This vector is used for remembering name groups during the pre-compile. In a +similar way to cworkspace, it can be expanded using malloc() if necessary. */ + +named_group named_groups[NAMED_GROUP_LIST_SIZE]; + +/* Set this early so that early errors get offset 0. */ + +ptr = (const pcre_uchar *)pattern; + +/* We can't pass back an error message if errorptr is NULL; I guess the best we +can do is just return NULL, but we can set a code value if there is a code +pointer. */ + +if (errorptr == NULL) + { + if (errorcodeptr != NULL) *errorcodeptr = 99; + return NULL; + } + +*errorptr = NULL; +if (errorcodeptr != NULL) *errorcodeptr = ERR0; + +/* However, we can give a message for this error */ + +if (erroroffset == NULL) + { + errorcode = ERR16; + goto PCRE_EARLY_ERROR_RETURN2; + } + +*erroroffset = 0; + +/* Set up pointers to the individual character tables */ + +if (tables == NULL) tables = PRIV(default_tables); +cd->lcc = tables + lcc_offset; +cd->fcc = tables + fcc_offset; +cd->cbits = tables + cbits_offset; +cd->ctypes = tables + ctypes_offset; + +/* Check that all undefined public option bits are zero */ + +if ((options & ~PUBLIC_COMPILE_OPTIONS) != 0) + { + errorcode = ERR17; + goto PCRE_EARLY_ERROR_RETURN; + } + +/* If PCRE_NEVER_UTF is set, remember it. */ + +if ((options & PCRE_NEVER_UTF) != 0) never_utf = TRUE; + +/* Check for global one-time settings at the start of the pattern, and remember +the offset for later. */ + +cd->external_flags = 0; /* Initialize here for LIMIT_MATCH/RECURSION */ + +while (ptr[skipatstart] == CHAR_LEFT_PARENTHESIS && + ptr[skipatstart+1] == CHAR_ASTERISK) + { + int newnl = 0; + int newbsr = 0; + +/* For completeness and backward compatibility, (*UTFn) is supported in the +relevant libraries, but (*UTF) is generic and always supported. Note that +PCRE_UTF8 == PCRE_UTF16 == PCRE_UTF32. */ + +#ifdef COMPILE_PCRE8 + if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_UTF8_RIGHTPAR, 5) == 0) + { skipatstart += 7; options |= PCRE_UTF8; continue; } +#endif +#ifdef COMPILE_PCRE16 + if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_UTF16_RIGHTPAR, 6) == 0) + { skipatstart += 8; options |= PCRE_UTF16; continue; } +#endif +#ifdef COMPILE_PCRE32 + if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_UTF32_RIGHTPAR, 6) == 0) + { skipatstart += 8; options |= PCRE_UTF32; continue; } +#endif + + else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_UTF_RIGHTPAR, 4) == 0) + { skipatstart += 6; options |= PCRE_UTF8; continue; } + else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_UCP_RIGHTPAR, 4) == 0) + { skipatstart += 6; options |= PCRE_UCP; continue; } + else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_NO_AUTO_POSSESS_RIGHTPAR, 16) == 0) + { skipatstart += 18; options |= PCRE_NO_AUTO_POSSESS; continue; } + else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_NO_START_OPT_RIGHTPAR, 13) == 0) + { skipatstart += 15; options |= PCRE_NO_START_OPTIMIZE; continue; } + + else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_LIMIT_MATCH_EQ, 12) == 0) + { + pcre_uint32 c = 0; + int p = skipatstart + 14; + while (isdigit(ptr[p])) + { + if (c > PCRE_UINT32_MAX / 10 - 1) break; /* Integer overflow */ + c = c*10 + ptr[p++] - CHAR_0; + } + if (ptr[p++] != CHAR_RIGHT_PARENTHESIS) break; + if (c < limit_match) + { + limit_match = c; + cd->external_flags |= PCRE_MLSET; + } + skipatstart = p; + continue; + } + + else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_LIMIT_RECURSION_EQ, 16) == 0) + { + pcre_uint32 c = 0; + int p = skipatstart + 18; + while (isdigit(ptr[p])) + { + if (c > PCRE_UINT32_MAX / 10 - 1) break; /* Integer overflow check */ + c = c*10 + ptr[p++] - CHAR_0; + } + if (ptr[p++] != CHAR_RIGHT_PARENTHESIS) break; + if (c < limit_recursion) + { + limit_recursion = c; + cd->external_flags |= PCRE_RLSET; + } + skipatstart = p; + continue; + } + + if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_CR_RIGHTPAR, 3) == 0) + { skipatstart += 5; newnl = PCRE_NEWLINE_CR; } + else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_LF_RIGHTPAR, 3) == 0) + { skipatstart += 5; newnl = PCRE_NEWLINE_LF; } + else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_CRLF_RIGHTPAR, 5) == 0) + { skipatstart += 7; newnl = PCRE_NEWLINE_CR + PCRE_NEWLINE_LF; } + else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_ANY_RIGHTPAR, 4) == 0) + { skipatstart += 6; newnl = PCRE_NEWLINE_ANY; } + else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_ANYCRLF_RIGHTPAR, 8) == 0) + { skipatstart += 10; newnl = PCRE_NEWLINE_ANYCRLF; } + + else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_BSR_ANYCRLF_RIGHTPAR, 12) == 0) + { skipatstart += 14; newbsr = PCRE_BSR_ANYCRLF; } + else if (STRNCMP_UC_C8(ptr+skipatstart+2, STRING_BSR_UNICODE_RIGHTPAR, 12) == 0) + { skipatstart += 14; newbsr = PCRE_BSR_UNICODE; } + + if (newnl != 0) + options = (options & ~PCRE_NEWLINE_BITS) | newnl; + else if (newbsr != 0) + options = (options & ~(PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE)) | newbsr; + else break; + } + +/* PCRE_UTF(16|32) have the same value as PCRE_UTF8. */ +utf = (options & PCRE_UTF8) != 0; +if (utf && never_utf) + { + errorcode = ERR78; + goto PCRE_EARLY_ERROR_RETURN2; + } + +/* Can't support UTF unless PCRE has been compiled to include the code. The +return of an error code from PRIV(valid_utf)() is a new feature, introduced in +release 8.13. It is passed back from pcre_[dfa_]exec(), but at the moment is +not used here. */ + +#ifdef SUPPORT_UTF +if (utf && (options & PCRE_NO_UTF8_CHECK) == 0 && + (errorcode = PRIV(valid_utf)((PCRE_PUCHAR)pattern, -1, erroroffset)) != 0) + { +#if defined COMPILE_PCRE8 + errorcode = ERR44; +#elif defined COMPILE_PCRE16 + errorcode = ERR74; +#elif defined COMPILE_PCRE32 + errorcode = ERR77; +#endif + goto PCRE_EARLY_ERROR_RETURN2; + } +#else +if (utf) + { + errorcode = ERR32; + goto PCRE_EARLY_ERROR_RETURN; + } +#endif + +/* Can't support UCP unless PCRE has been compiled to include the code. */ + +#ifndef SUPPORT_UCP +if ((options & PCRE_UCP) != 0) + { + errorcode = ERR67; + goto PCRE_EARLY_ERROR_RETURN; + } +#endif + +/* Check validity of \R options. */ + +if ((options & (PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE)) == + (PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE)) + { + errorcode = ERR56; + goto PCRE_EARLY_ERROR_RETURN; + } + +/* Handle different types of newline. The three bits give seven cases. The +current code allows for fixed one- or two-byte sequences, plus "any" and +"anycrlf". */ + +switch (options & PCRE_NEWLINE_BITS) + { + case 0: newline = NEWLINE; break; /* Build-time default */ + case PCRE_NEWLINE_CR: newline = CHAR_CR; break; + case PCRE_NEWLINE_LF: newline = CHAR_NL; break; + case PCRE_NEWLINE_CR+ + PCRE_NEWLINE_LF: newline = (CHAR_CR << 8) | CHAR_NL; break; + case PCRE_NEWLINE_ANY: newline = -1; break; + case PCRE_NEWLINE_ANYCRLF: newline = -2; break; + default: errorcode = ERR56; goto PCRE_EARLY_ERROR_RETURN; + } + +if (newline == -2) + { + cd->nltype = NLTYPE_ANYCRLF; + } +else if (newline < 0) + { + cd->nltype = NLTYPE_ANY; + } +else + { + cd->nltype = NLTYPE_FIXED; + if (newline > 255) + { + cd->nllen = 2; + cd->nl[0] = (newline >> 8) & 255; + cd->nl[1] = newline & 255; + } + else + { + cd->nllen = 1; + cd->nl[0] = newline; + } + } + +/* Maximum back reference and backref bitmap. The bitmap records up to 31 back +references to help in deciding whether (.*) can be treated as anchored or not. +*/ + +cd->top_backref = 0; +cd->backref_map = 0; + +/* Reflect pattern for debugging output */ + +DPRINTF(("------------------------------------------------------------------\n")); +#ifdef PCRE_DEBUG +print_puchar(stdout, (PCRE_PUCHAR)pattern); +#endif +DPRINTF(("\n")); + +/* Pretend to compile the pattern while actually just accumulating the length +of memory required. This behaviour is triggered by passing a non-NULL final +argument to compile_regex(). We pass a block of workspace (cworkspace) for it +to compile parts of the pattern into; the compiled code is discarded when it is +no longer needed, so hopefully this workspace will never overflow, though there +is a test for its doing so. */ + +cd->bracount = cd->final_bracount = 0; +cd->names_found = 0; +cd->name_entry_size = 0; +cd->name_table = NULL; +cd->dupnames = FALSE; +cd->dupgroups = FALSE; +cd->namedrefcount = 0; +cd->start_code = cworkspace; +cd->hwm = cworkspace; +cd->iscondassert = FALSE; +cd->start_workspace = cworkspace; +cd->workspace_size = COMPILE_WORK_SIZE; +cd->named_groups = named_groups; +cd->named_group_list_size = NAMED_GROUP_LIST_SIZE; +cd->start_pattern = (const pcre_uchar *)pattern; +cd->end_pattern = (const pcre_uchar *)(pattern + STRLEN_UC((const pcre_uchar *)pattern)); +cd->req_varyopt = 0; +cd->parens_depth = 0; +cd->assert_depth = 0; +cd->max_lookbehind = 0; +cd->external_options = options; +cd->open_caps = NULL; + +/* Now do the pre-compile. On error, errorcode will be set non-zero, so we +don't need to look at the result of the function here. The initial options have +been put into the cd block so that they can be changed if an option setting is +found within the regex right at the beginning. Bringing initial option settings +outside can help speed up starting point checks. */ + +ptr += skipatstart; +code = cworkspace; +*code = OP_BRA; + +(void)compile_regex(cd->external_options, &code, &ptr, &errorcode, FALSE, + FALSE, 0, 0, &firstchar, &firstcharflags, &reqchar, &reqcharflags, NULL, + cd, &length); +if (errorcode != 0) goto PCRE_EARLY_ERROR_RETURN; + +DPRINTF(("end pre-compile: length=%d workspace=%d\n", length, + (int)(cd->hwm - cworkspace))); + +if (length > MAX_PATTERN_SIZE) + { + errorcode = ERR20; + goto PCRE_EARLY_ERROR_RETURN; + } + +/* Compute the size of the data block for storing the compiled pattern. Integer +overflow should no longer be possible because nowadays we limit the maximum +value of cd->names_found and cd->name_entry_size. */ + +size = sizeof(REAL_PCRE) + + (length + cd->names_found * cd->name_entry_size) * sizeof(pcre_uchar); + +/* Get the memory. */ + +re = (REAL_PCRE *)(PUBL(malloc))(size); +if (re == NULL) + { + errorcode = ERR21; + goto PCRE_EARLY_ERROR_RETURN; + } + +/* Put in the magic number, and save the sizes, initial options, internal +flags, and character table pointer. NULL is used for the default character +tables. The nullpad field is at the end; it's there to help in the case when a +regex compiled on a system with 4-byte pointers is run on another with 8-byte +pointers. */ + +re->magic_number = MAGIC_NUMBER; +re->size = (int)size; +re->options = cd->external_options; +re->flags = cd->external_flags; +re->limit_match = limit_match; +re->limit_recursion = limit_recursion; +re->first_char = 0; +re->req_char = 0; +re->name_table_offset = sizeof(REAL_PCRE) / sizeof(pcre_uchar); +re->name_entry_size = cd->name_entry_size; +re->name_count = cd->names_found; +re->ref_count = 0; +re->tables = (tables == PRIV(default_tables))? NULL : tables; +re->nullpad = NULL; +#ifdef COMPILE_PCRE32 +re->dummy = 0; +#else +re->dummy1 = re->dummy2 = re->dummy3 = 0; +#endif + +/* The starting points of the name/number translation table and of the code are +passed around in the compile data block. The start/end pattern and initial +options are already set from the pre-compile phase, as is the name_entry_size +field. Reset the bracket count and the names_found field. Also reset the hwm +field; this time it's used for remembering forward references to subpatterns. +*/ + +cd->final_bracount = cd->bracount; /* Save for checking forward references */ +cd->parens_depth = 0; +cd->assert_depth = 0; +cd->bracount = 0; +cd->max_lookbehind = 0; +cd->name_table = (pcre_uchar *)re + re->name_table_offset; +codestart = cd->name_table + re->name_entry_size * re->name_count; +cd->start_code = codestart; +cd->hwm = (pcre_uchar *)(cd->start_workspace); +cd->iscondassert = FALSE; +cd->req_varyopt = 0; +cd->had_accept = FALSE; +cd->had_pruneorskip = FALSE; +cd->check_lookbehind = FALSE; +cd->open_caps = NULL; + +/* If any named groups were found, create the name/number table from the list +created in the first pass. */ + +if (cd->names_found > 0) + { + int i = cd->names_found; + named_group *ng = cd->named_groups; + cd->names_found = 0; + for (; i > 0; i--, ng++) + add_name(cd, ng->name, ng->length, ng->number); + if (cd->named_group_list_size > NAMED_GROUP_LIST_SIZE) + (PUBL(free))((void *)cd->named_groups); + } + +/* Set up a starting, non-extracting bracket, then compile the expression. On +error, errorcode will be set non-zero, so we don't need to look at the result +of the function here. */ + +ptr = (const pcre_uchar *)pattern + skipatstart; +code = (pcre_uchar *)codestart; +*code = OP_BRA; +(void)compile_regex(re->options, &code, &ptr, &errorcode, FALSE, FALSE, 0, 0, + &firstchar, &firstcharflags, &reqchar, &reqcharflags, NULL, cd, NULL); +re->top_bracket = cd->bracount; +re->top_backref = cd->top_backref; +re->max_lookbehind = cd->max_lookbehind; +re->flags = cd->external_flags | PCRE_MODE; + +if (cd->had_accept) + { + reqchar = 0; /* Must disable after (*ACCEPT) */ + reqcharflags = REQ_NONE; + } + +/* If not reached end of pattern on success, there's an excess bracket. */ + +if (errorcode == 0 && *ptr != CHAR_NULL) errorcode = ERR22; + +/* Fill in the terminating state and check for disastrous overflow, but +if debugging, leave the test till after things are printed out. */ + +*code++ = OP_END; + +#ifndef PCRE_DEBUG +if (code - codestart > length) errorcode = ERR23; +#endif + +#ifdef SUPPORT_VALGRIND +/* If the estimated length exceeds the really used length, mark the extra +allocated memory as unaddressable, so that any out-of-bound reads can be +detected. */ +VALGRIND_MAKE_MEM_NOACCESS(code, (length - (code - codestart)) * sizeof(pcre_uchar)); +#endif + +/* Fill in any forward references that are required. There may be repeated +references; optimize for them, as searching a large regex takes time. */ + +if (cd->hwm > cd->start_workspace) + { + int prev_recno = -1; + const pcre_uchar *groupptr = NULL; + while (errorcode == 0 && cd->hwm > cd->start_workspace) + { + int offset, recno; + cd->hwm -= LINK_SIZE; + offset = GET(cd->hwm, 0); + + /* Check that the hwm handling hasn't gone wrong. This whole area is + rewritten in PCRE2 because there are some obscure cases. */ + + if (offset == 0 || codestart[offset-1] != OP_RECURSE) + { + errorcode = ERR10; + break; + } + + recno = GET(codestart, offset); + if (recno != prev_recno) + { + groupptr = PRIV(find_bracket)(codestart, utf, recno); + prev_recno = recno; + } + if (groupptr == NULL) errorcode = ERR53; + else PUT(((pcre_uchar *)codestart), offset, (int)(groupptr - codestart)); + } + } + +/* If the workspace had to be expanded, free the new memory. Set the pointer to +NULL to indicate that forward references have been filled in. */ + +if (cd->workspace_size > COMPILE_WORK_SIZE) + (PUBL(free))((void *)cd->start_workspace); +cd->start_workspace = NULL; + +/* Give an error if there's back reference to a non-existent capturing +subpattern. */ + +if (errorcode == 0 && re->top_backref > re->top_bracket) errorcode = ERR15; + +/* Unless disabled, check whether any single character iterators can be +auto-possessified. The function overwrites the appropriate opcode values, so +the type of the pointer must be cast. NOTE: the intermediate variable "temp" is +used in this code because at least one compiler gives a warning about loss of +"const" attribute if the cast (pcre_uchar *)codestart is used directly in the +function call. */ + +if (errorcode == 0 && (options & PCRE_NO_AUTO_POSSESS) == 0) + { + pcre_uchar *temp = (pcre_uchar *)codestart; + auto_possessify(temp, utf, cd); + } + +/* If there were any lookbehind assertions that contained OP_RECURSE +(recursions or subroutine calls), a flag is set for them to be checked here, +because they may contain forward references. Actual recursions cannot be fixed +length, but subroutine calls can. It is done like this so that those without +OP_RECURSE that are not fixed length get a diagnosic with a useful offset. The +exceptional ones forgo this. We scan the pattern to check that they are fixed +length, and set their lengths. */ + +if (errorcode == 0 && cd->check_lookbehind) + { + pcre_uchar *cc = (pcre_uchar *)codestart; + + /* Loop, searching for OP_REVERSE items, and process those that do not have + their length set. (Actually, it will also re-process any that have a length + of zero, but that is a pathological case, and it does no harm.) When we find + one, we temporarily terminate the branch it is in while we scan it. */ + + for (cc = (pcre_uchar *)PRIV(find_bracket)(codestart, utf, -1); + cc != NULL; + cc = (pcre_uchar *)PRIV(find_bracket)(cc, utf, -1)) + { + if (GET(cc, 1) == 0) + { + int fixed_length; + pcre_uchar *be = cc - 1 - LINK_SIZE + GET(cc, -LINK_SIZE); + int end_op = *be; + *be = OP_END; + fixed_length = find_fixedlength(cc, (re->options & PCRE_UTF8) != 0, TRUE, + cd, NULL); + *be = end_op; + DPRINTF(("fixed length = %d\n", fixed_length)); + if (fixed_length < 0) + { + errorcode = (fixed_length == -2)? ERR36 : + (fixed_length == -4)? ERR70 : ERR25; + break; + } + if (fixed_length > cd->max_lookbehind) cd->max_lookbehind = fixed_length; + PUT(cc, 1, fixed_length); + } + cc += 1 + LINK_SIZE; + } + } + +/* Failed to compile, or error while post-processing */ + +if (errorcode != 0) + { + (PUBL(free))(re); + PCRE_EARLY_ERROR_RETURN: + *erroroffset = (int)(ptr - (const pcre_uchar *)pattern); + PCRE_EARLY_ERROR_RETURN2: + *errorptr = find_error_text(errorcode); + if (errorcodeptr != NULL) *errorcodeptr = errorcode; + return NULL; + } + +/* If the anchored option was not passed, set the flag if we can determine that +the pattern is anchored by virtue of ^ characters or \A or anything else, such +as starting with non-atomic .* when DOTALL is set and there are no occurrences +of *PRUNE or *SKIP. + +Otherwise, if we know what the first byte has to be, save it, because that +speeds up unanchored matches no end. If not, see if we can set the +PCRE_STARTLINE flag. This is helpful for multiline matches when all branches +start with ^. and also when all branches start with non-atomic .* for +non-DOTALL matches when *PRUNE and SKIP are not present. */ + +if ((re->options & PCRE_ANCHORED) == 0) + { + if (is_anchored(codestart, 0, cd, 0)) re->options |= PCRE_ANCHORED; + else + { + if (firstcharflags < 0) + firstchar = find_firstassertedchar(codestart, &firstcharflags, FALSE); + if (firstcharflags >= 0) /* Remove caseless flag for non-caseable chars */ + { +#if defined COMPILE_PCRE8 + re->first_char = firstchar & 0xff; +#elif defined COMPILE_PCRE16 + re->first_char = firstchar & 0xffff; +#elif defined COMPILE_PCRE32 + re->first_char = firstchar; +#endif + if ((firstcharflags & REQ_CASELESS) != 0) + { +#if defined SUPPORT_UCP && !(defined COMPILE_PCRE8) + /* We ignore non-ASCII first chars in 8 bit mode. */ + if (utf) + { + if (re->first_char < 128) + { + if (cd->fcc[re->first_char] != re->first_char) + re->flags |= PCRE_FCH_CASELESS; + } + else if (UCD_OTHERCASE(re->first_char) != re->first_char) + re->flags |= PCRE_FCH_CASELESS; + } + else +#endif + if (MAX_255(re->first_char) + && cd->fcc[re->first_char] != re->first_char) + re->flags |= PCRE_FCH_CASELESS; + } + + re->flags |= PCRE_FIRSTSET; + } + + else if (is_startline(codestart, 0, cd, 0, FALSE)) re->flags |= PCRE_STARTLINE; + } + } + +/* For an anchored pattern, we use the "required byte" only if it follows a +variable length item in the regex. Remove the caseless flag for non-caseable +bytes. */ + +if (reqcharflags >= 0 && + ((re->options & PCRE_ANCHORED) == 0 || (reqcharflags & REQ_VARY) != 0)) + { +#if defined COMPILE_PCRE8 + re->req_char = reqchar & 0xff; +#elif defined COMPILE_PCRE16 + re->req_char = reqchar & 0xffff; +#elif defined COMPILE_PCRE32 + re->req_char = reqchar; +#endif + if ((reqcharflags & REQ_CASELESS) != 0) + { +#if defined SUPPORT_UCP && !(defined COMPILE_PCRE8) + /* We ignore non-ASCII first chars in 8 bit mode. */ + if (utf) + { + if (re->req_char < 128) + { + if (cd->fcc[re->req_char] != re->req_char) + re->flags |= PCRE_RCH_CASELESS; + } + else if (UCD_OTHERCASE(re->req_char) != re->req_char) + re->flags |= PCRE_RCH_CASELESS; + } + else +#endif + if (MAX_255(re->req_char) && cd->fcc[re->req_char] != re->req_char) + re->flags |= PCRE_RCH_CASELESS; + } + + re->flags |= PCRE_REQCHSET; + } + +/* Print out the compiled data if debugging is enabled. This is never the +case when building a production library. */ + +#ifdef PCRE_DEBUG +printf("Length = %d top_bracket = %d top_backref = %d\n", + length, re->top_bracket, re->top_backref); + +printf("Options=%08x\n", re->options); + +if ((re->flags & PCRE_FIRSTSET) != 0) + { + pcre_uchar ch = re->first_char; + const char *caseless = + ((re->flags & PCRE_FCH_CASELESS) == 0)? "" : " (caseless)"; + if (PRINTABLE(ch)) printf("First char = %c%s\n", ch, caseless); + else printf("First char = \\x%02x%s\n", ch, caseless); + } + +if ((re->flags & PCRE_REQCHSET) != 0) + { + pcre_uchar ch = re->req_char; + const char *caseless = + ((re->flags & PCRE_RCH_CASELESS) == 0)? "" : " (caseless)"; + if (PRINTABLE(ch)) printf("Req char = %c%s\n", ch, caseless); + else printf("Req char = \\x%02x%s\n", ch, caseless); + } + +#if defined COMPILE_PCRE8 +pcre_printint((pcre *)re, stdout, TRUE); +#elif defined COMPILE_PCRE16 +pcre16_printint((pcre *)re, stdout, TRUE); +#elif defined COMPILE_PCRE32 +pcre32_printint((pcre *)re, stdout, TRUE); +#endif + +/* This check is done here in the debugging case so that the code that +was compiled can be seen. */ + +if (code - codestart > length) + { + (PUBL(free))(re); + *errorptr = find_error_text(ERR23); + *erroroffset = ptr - (pcre_uchar *)pattern; + if (errorcodeptr != NULL) *errorcodeptr = ERR23; + return NULL; + } +#endif /* PCRE_DEBUG */ + +/* Check for a pattern than can match an empty string, so that this information +can be provided to applications. */ + +do + { + if (could_be_empty_branch(codestart, code, utf, cd, NULL)) + { + re->flags |= PCRE_MATCH_EMPTY; + break; + } + codestart += GET(codestart, 1); + } +while (*codestart == OP_ALT); + +#if defined COMPILE_PCRE8 +return (pcre *)re; +#elif defined COMPILE_PCRE16 +return (pcre16 *)re; +#elif defined COMPILE_PCRE32 +return (pcre32 *)re; +#endif +} + +/* End of pcre_compile.c */ diff --git a/src/third_party/pcre-8.41/pcre_config.c b/src/third_party/pcre-8.41/pcre_config.c new file mode 100644 index 00000000000..1cbdd9c960c --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_config.c @@ -0,0 +1,190 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This module contains the external function pcre_config(). */ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* Keep the original link size. */ +static int real_link_size = LINK_SIZE; + +#include "pcre_internal.h" + + +/************************************************* +* Return info about what features are configured * +*************************************************/ + +/* This function has an extensible interface so that additional items can be +added compatibly. + +Arguments: + what what information is required + where where to put the information + +Returns: 0 if data returned, negative on error +*/ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre_config(int what, void *where) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre16_config(int what, void *where) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre32_config(int what, void *where) +#endif +{ +switch (what) + { + case PCRE_CONFIG_UTF8: +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + *((int *)where) = 0; + return PCRE_ERROR_BADOPTION; +#else +#if defined SUPPORT_UTF + *((int *)where) = 1; +#else + *((int *)where) = 0; +#endif + break; +#endif + + case PCRE_CONFIG_UTF16: +#if defined COMPILE_PCRE8 || defined COMPILE_PCRE32 + *((int *)where) = 0; + return PCRE_ERROR_BADOPTION; +#else +#if defined SUPPORT_UTF + *((int *)where) = 1; +#else + *((int *)where) = 0; +#endif + break; +#endif + + case PCRE_CONFIG_UTF32: +#if defined COMPILE_PCRE8 || defined COMPILE_PCRE16 + *((int *)where) = 0; + return PCRE_ERROR_BADOPTION; +#else +#if defined SUPPORT_UTF + *((int *)where) = 1; +#else + *((int *)where) = 0; +#endif + break; +#endif + + case PCRE_CONFIG_UNICODE_PROPERTIES: +#ifdef SUPPORT_UCP + *((int *)where) = 1; +#else + *((int *)where) = 0; +#endif + break; + + case PCRE_CONFIG_JIT: +#ifdef SUPPORT_JIT + *((int *)where) = 1; +#else + *((int *)where) = 0; +#endif + break; + + case PCRE_CONFIG_JITTARGET: +#ifdef SUPPORT_JIT + *((const char **)where) = PRIV(jit_get_target)(); +#else + *((const char **)where) = NULL; +#endif + break; + + case PCRE_CONFIG_NEWLINE: + *((int *)where) = NEWLINE; + break; + + case PCRE_CONFIG_BSR: +#ifdef BSR_ANYCRLF + *((int *)where) = 1; +#else + *((int *)where) = 0; +#endif + break; + + case PCRE_CONFIG_LINK_SIZE: + *((int *)where) = real_link_size; + break; + + case PCRE_CONFIG_POSIX_MALLOC_THRESHOLD: + *((int *)where) = POSIX_MALLOC_THRESHOLD; + break; + + case PCRE_CONFIG_PARENS_LIMIT: + *((unsigned long int *)where) = PARENS_NEST_LIMIT; + break; + + case PCRE_CONFIG_MATCH_LIMIT: + *((unsigned long int *)where) = MATCH_LIMIT; + break; + + case PCRE_CONFIG_MATCH_LIMIT_RECURSION: + *((unsigned long int *)where) = MATCH_LIMIT_RECURSION; + break; + + case PCRE_CONFIG_STACKRECURSE: +#ifdef NO_RECURSE + *((int *)where) = 0; +#else + *((int *)where) = 1; +#endif + break; + + default: return PCRE_ERROR_BADOPTION; + } + +return 0; +} + +/* End of pcre_config.c */ diff --git a/src/third_party/pcre-8.41/pcre_dfa_exec.c b/src/third_party/pcre-8.41/pcre_dfa_exec.c new file mode 100644 index 00000000000..bc09ced3a7c --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_dfa_exec.c @@ -0,0 +1,3674 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language (but see +below for why this module is different). + + Written by Philip Hazel + Copyright (c) 1997-2017 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* This module contains the external function pcre_dfa_exec(), which is an +alternative matching function that uses a sort of DFA algorithm (not a true +FSM). This is NOT Perl-compatible, but it has advantages in certain +applications. */ + + +/* NOTE ABOUT PERFORMANCE: A user of this function sent some code that improved +the performance of his patterns greatly. I could not use it as it stood, as it +was not thread safe, and made assumptions about pattern sizes. Also, it caused +test 7 to loop, and test 9 to crash with a segfault. + +The issue is the check for duplicate states, which is done by a simple linear +search up the state list. (Grep for "duplicate" below to find the code.) For +many patterns, there will never be many states active at one time, so a simple +linear search is fine. In patterns that have many active states, it might be a +bottleneck. The suggested code used an indexing scheme to remember which states +had previously been used for each character, and avoided the linear search when +it knew there was no chance of a duplicate. This was implemented when adding +states to the state lists. + +I wrote some thread-safe, not-limited code to try something similar at the time +of checking for duplicates (instead of when adding states), using index vectors +on the stack. It did give a 13% improvement with one specially constructed +pattern for certain subject strings, but on other strings and on many of the +simpler patterns in the test suite it did worse. The major problem, I think, +was the extra time to initialize the index. This had to be done for each call +of internal_dfa_exec(). (The supplied patch used a static vector, initialized +only once - I suspect this was the cause of the problems with the tests.) + +Overall, I concluded that the gains in some cases did not outweigh the losses +in others, so I abandoned this code. */ + + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define NLBLOCK md /* Block containing newline information */ +#define PSSTART start_subject /* Field containing processed string start */ +#define PSEND end_subject /* Field containing processed string end */ + +#include "pcre_internal.h" + + +/* For use to indent debugging output */ + +#define SP " " + + +/************************************************* +* Code parameters and static tables * +*************************************************/ + +/* These are offsets that are used to turn the OP_TYPESTAR and friends opcodes +into others, under special conditions. A gap of 20 between the blocks should be +enough. The resulting opcodes don't have to be less than 256 because they are +never stored, so we push them well clear of the normal opcodes. */ + +#define OP_PROP_EXTRA 300 +#define OP_EXTUNI_EXTRA 320 +#define OP_ANYNL_EXTRA 340 +#define OP_HSPACE_EXTRA 360 +#define OP_VSPACE_EXTRA 380 + + +/* This table identifies those opcodes that are followed immediately by a +character that is to be tested in some way. This makes it possible to +centralize the loading of these characters. In the case of Type * etc, the +"character" is the opcode for \D, \d, \S, \s, \W, or \w, which will always be a +small value. Non-zero values in the table are the offsets from the opcode where +the character is to be found. ***NOTE*** If the start of this table is +modified, the three tables that follow must also be modified. */ + +static const pcre_uint8 coptable[] = { + 0, /* End */ + 0, 0, 0, 0, 0, /* \A, \G, \K, \B, \b */ + 0, 0, 0, 0, 0, 0, /* \D, \d, \S, \s, \W, \w */ + 0, 0, 0, /* Any, AllAny, Anybyte */ + 0, 0, /* \P, \p */ + 0, 0, 0, 0, 0, /* \R, \H, \h, \V, \v */ + 0, /* \X */ + 0, 0, 0, 0, 0, 0, /* \Z, \z, $, $M, ^, ^M */ + 1, /* Char */ + 1, /* Chari */ + 1, /* not */ + 1, /* noti */ + /* Positive single-char repeats */ + 1, 1, 1, 1, 1, 1, /* *, *?, +, +?, ?, ?? */ + 1+IMM2_SIZE, 1+IMM2_SIZE, /* upto, minupto */ + 1+IMM2_SIZE, /* exact */ + 1, 1, 1, 1+IMM2_SIZE, /* *+, ++, ?+, upto+ */ + 1, 1, 1, 1, 1, 1, /* *I, *?I, +I, +?I, ?I, ??I */ + 1+IMM2_SIZE, 1+IMM2_SIZE, /* upto I, minupto I */ + 1+IMM2_SIZE, /* exact I */ + 1, 1, 1, 1+IMM2_SIZE, /* *+I, ++I, ?+I, upto+I */ + /* Negative single-char repeats - only for chars < 256 */ + 1, 1, 1, 1, 1, 1, /* NOT *, *?, +, +?, ?, ?? */ + 1+IMM2_SIZE, 1+IMM2_SIZE, /* NOT upto, minupto */ + 1+IMM2_SIZE, /* NOT exact */ + 1, 1, 1, 1+IMM2_SIZE, /* NOT *+, ++, ?+, upto+ */ + 1, 1, 1, 1, 1, 1, /* NOT *I, *?I, +I, +?I, ?I, ??I */ + 1+IMM2_SIZE, 1+IMM2_SIZE, /* NOT upto I, minupto I */ + 1+IMM2_SIZE, /* NOT exact I */ + 1, 1, 1, 1+IMM2_SIZE, /* NOT *+I, ++I, ?+I, upto+I */ + /* Positive type repeats */ + 1, 1, 1, 1, 1, 1, /* Type *, *?, +, +?, ?, ?? */ + 1+IMM2_SIZE, 1+IMM2_SIZE, /* Type upto, minupto */ + 1+IMM2_SIZE, /* Type exact */ + 1, 1, 1, 1+IMM2_SIZE, /* Type *+, ++, ?+, upto+ */ + /* Character class & ref repeats */ + 0, 0, 0, 0, 0, 0, /* *, *?, +, +?, ?, ?? */ + 0, 0, /* CRRANGE, CRMINRANGE */ + 0, 0, 0, 0, /* Possessive *+, ++, ?+, CRPOSRANGE */ + 0, /* CLASS */ + 0, /* NCLASS */ + 0, /* XCLASS - variable length */ + 0, /* REF */ + 0, /* REFI */ + 0, /* DNREF */ + 0, /* DNREFI */ + 0, /* RECURSE */ + 0, /* CALLOUT */ + 0, /* Alt */ + 0, /* Ket */ + 0, /* KetRmax */ + 0, /* KetRmin */ + 0, /* KetRpos */ + 0, /* Reverse */ + 0, /* Assert */ + 0, /* Assert not */ + 0, /* Assert behind */ + 0, /* Assert behind not */ + 0, 0, /* ONCE, ONCE_NC */ + 0, 0, 0, 0, 0, /* BRA, BRAPOS, CBRA, CBRAPOS, COND */ + 0, 0, 0, 0, 0, /* SBRA, SBRAPOS, SCBRA, SCBRAPOS, SCOND */ + 0, 0, /* CREF, DNCREF */ + 0, 0, /* RREF, DNRREF */ + 0, /* DEF */ + 0, 0, 0, /* BRAZERO, BRAMINZERO, BRAPOSZERO */ + 0, 0, 0, /* MARK, PRUNE, PRUNE_ARG */ + 0, 0, 0, 0, /* SKIP, SKIP_ARG, THEN, THEN_ARG */ + 0, 0, 0, 0, /* COMMIT, FAIL, ACCEPT, ASSERT_ACCEPT */ + 0, 0 /* CLOSE, SKIPZERO */ +}; + +/* This table identifies those opcodes that inspect a character. It is used to +remember the fact that a character could have been inspected when the end of +the subject is reached. ***NOTE*** If the start of this table is modified, the +two tables that follow must also be modified. */ + +static const pcre_uint8 poptable[] = { + 0, /* End */ + 0, 0, 0, 1, 1, /* \A, \G, \K, \B, \b */ + 1, 1, 1, 1, 1, 1, /* \D, \d, \S, \s, \W, \w */ + 1, 1, 1, /* Any, AllAny, Anybyte */ + 1, 1, /* \P, \p */ + 1, 1, 1, 1, 1, /* \R, \H, \h, \V, \v */ + 1, /* \X */ + 0, 0, 0, 0, 0, 0, /* \Z, \z, $, $M, ^, ^M */ + 1, /* Char */ + 1, /* Chari */ + 1, /* not */ + 1, /* noti */ + /* Positive single-char repeats */ + 1, 1, 1, 1, 1, 1, /* *, *?, +, +?, ?, ?? */ + 1, 1, 1, /* upto, minupto, exact */ + 1, 1, 1, 1, /* *+, ++, ?+, upto+ */ + 1, 1, 1, 1, 1, 1, /* *I, *?I, +I, +?I, ?I, ??I */ + 1, 1, 1, /* upto I, minupto I, exact I */ + 1, 1, 1, 1, /* *+I, ++I, ?+I, upto+I */ + /* Negative single-char repeats - only for chars < 256 */ + 1, 1, 1, 1, 1, 1, /* NOT *, *?, +, +?, ?, ?? */ + 1, 1, 1, /* NOT upto, minupto, exact */ + 1, 1, 1, 1, /* NOT *+, ++, ?+, upto+ */ + 1, 1, 1, 1, 1, 1, /* NOT *I, *?I, +I, +?I, ?I, ??I */ + 1, 1, 1, /* NOT upto I, minupto I, exact I */ + 1, 1, 1, 1, /* NOT *+I, ++I, ?+I, upto+I */ + /* Positive type repeats */ + 1, 1, 1, 1, 1, 1, /* Type *, *?, +, +?, ?, ?? */ + 1, 1, 1, /* Type upto, minupto, exact */ + 1, 1, 1, 1, /* Type *+, ++, ?+, upto+ */ + /* Character class & ref repeats */ + 1, 1, 1, 1, 1, 1, /* *, *?, +, +?, ?, ?? */ + 1, 1, /* CRRANGE, CRMINRANGE */ + 1, 1, 1, 1, /* Possessive *+, ++, ?+, CRPOSRANGE */ + 1, /* CLASS */ + 1, /* NCLASS */ + 1, /* XCLASS - variable length */ + 0, /* REF */ + 0, /* REFI */ + 0, /* DNREF */ + 0, /* DNREFI */ + 0, /* RECURSE */ + 0, /* CALLOUT */ + 0, /* Alt */ + 0, /* Ket */ + 0, /* KetRmax */ + 0, /* KetRmin */ + 0, /* KetRpos */ + 0, /* Reverse */ + 0, /* Assert */ + 0, /* Assert not */ + 0, /* Assert behind */ + 0, /* Assert behind not */ + 0, 0, /* ONCE, ONCE_NC */ + 0, 0, 0, 0, 0, /* BRA, BRAPOS, CBRA, CBRAPOS, COND */ + 0, 0, 0, 0, 0, /* SBRA, SBRAPOS, SCBRA, SCBRAPOS, SCOND */ + 0, 0, /* CREF, DNCREF */ + 0, 0, /* RREF, DNRREF */ + 0, /* DEF */ + 0, 0, 0, /* BRAZERO, BRAMINZERO, BRAPOSZERO */ + 0, 0, 0, /* MARK, PRUNE, PRUNE_ARG */ + 0, 0, 0, 0, /* SKIP, SKIP_ARG, THEN, THEN_ARG */ + 0, 0, 0, 0, /* COMMIT, FAIL, ACCEPT, ASSERT_ACCEPT */ + 0, 0 /* CLOSE, SKIPZERO */ +}; + +/* These 2 tables allow for compact code for testing for \D, \d, \S, \s, \W, +and \w */ + +static const pcre_uint8 toptable1[] = { + 0, 0, 0, 0, 0, 0, + ctype_digit, ctype_digit, + ctype_space, ctype_space, + ctype_word, ctype_word, + 0, 0 /* OP_ANY, OP_ALLANY */ +}; + +static const pcre_uint8 toptable2[] = { + 0, 0, 0, 0, 0, 0, + ctype_digit, 0, + ctype_space, 0, + ctype_word, 0, + 1, 1 /* OP_ANY, OP_ALLANY */ +}; + + +/* Structure for holding data about a particular state, which is in effect the +current data for an active path through the match tree. It must consist +entirely of ints because the working vector we are passed, and which we put +these structures in, is a vector of ints. */ + +typedef struct stateblock { + int offset; /* Offset to opcode */ + int count; /* Count for repeats */ + int data; /* Some use extra data */ +} stateblock; + +#define INTS_PER_STATEBLOCK (int)(sizeof(stateblock)/sizeof(int)) + + +#ifdef PCRE_DEBUG +/************************************************* +* Print character string * +*************************************************/ + +/* Character string printing function for debugging. + +Arguments: + p points to string + length number of bytes + f where to print + +Returns: nothing +*/ + +static void +pchars(const pcre_uchar *p, int length, FILE *f) +{ +pcre_uint32 c; +while (length-- > 0) + { + if (isprint(c = *(p++))) + fprintf(f, "%c", c); + else + fprintf(f, "\\x{%02x}", c); + } +} +#endif + + + +/************************************************* +* Execute a Regular Expression - DFA engine * +*************************************************/ + +/* This internal function applies a compiled pattern to a subject string, +starting at a given point, using a DFA engine. This function is called from the +external one, possibly multiple times if the pattern is not anchored. The +function calls itself recursively for some kinds of subpattern. + +Arguments: + md the match_data block with fixed information + this_start_code the opening bracket of this subexpression's code + current_subject where we currently are in the subject string + start_offset start offset in the subject string + offsets vector to contain the matching string offsets + offsetcount size of same + workspace vector of workspace + wscount size of same + rlevel function call recursion level + +Returns: > 0 => number of match offset pairs placed in offsets + = 0 => offsets overflowed; longest matches are present + -1 => failed to match + < -1 => some kind of unexpected problem + +The following macros are used for adding states to the two state vectors (one +for the current character, one for the following character). */ + +#define ADD_ACTIVE(x,y) \ + if (active_count++ < wscount) \ + { \ + next_active_state->offset = (x); \ + next_active_state->count = (y); \ + next_active_state++; \ + DPRINTF(("%.*sADD_ACTIVE(%d,%d)\n", rlevel*2-2, SP, (x), (y))); \ + } \ + else return PCRE_ERROR_DFA_WSSIZE + +#define ADD_ACTIVE_DATA(x,y,z) \ + if (active_count++ < wscount) \ + { \ + next_active_state->offset = (x); \ + next_active_state->count = (y); \ + next_active_state->data = (z); \ + next_active_state++; \ + DPRINTF(("%.*sADD_ACTIVE_DATA(%d,%d,%d)\n", rlevel*2-2, SP, (x), (y), (z))); \ + } \ + else return PCRE_ERROR_DFA_WSSIZE + +#define ADD_NEW(x,y) \ + if (new_count++ < wscount) \ + { \ + next_new_state->offset = (x); \ + next_new_state->count = (y); \ + next_new_state++; \ + DPRINTF(("%.*sADD_NEW(%d,%d)\n", rlevel*2-2, SP, (x), (y))); \ + } \ + else return PCRE_ERROR_DFA_WSSIZE + +#define ADD_NEW_DATA(x,y,z) \ + if (new_count++ < wscount) \ + { \ + next_new_state->offset = (x); \ + next_new_state->count = (y); \ + next_new_state->data = (z); \ + next_new_state++; \ + DPRINTF(("%.*sADD_NEW_DATA(%d,%d,%d) line %d\n", rlevel*2-2, SP, \ + (x), (y), (z), __LINE__)); \ + } \ + else return PCRE_ERROR_DFA_WSSIZE + +/* And now, here is the code */ + +static int +internal_dfa_exec( + dfa_match_data *md, + const pcre_uchar *this_start_code, + const pcre_uchar *current_subject, + int start_offset, + int *offsets, + int offsetcount, + int *workspace, + int wscount, + int rlevel) +{ +stateblock *active_states, *new_states, *temp_states; +stateblock *next_active_state, *next_new_state; + +const pcre_uint8 *ctypes, *lcc, *fcc; +const pcre_uchar *ptr; +const pcre_uchar *end_code, *first_op; + +dfa_recursion_info new_recursive; + +int active_count, new_count, match_count; + +/* Some fields in the md block are frequently referenced, so we load them into +independent variables in the hope that this will perform better. */ + +const pcre_uchar *start_subject = md->start_subject; +const pcre_uchar *end_subject = md->end_subject; +const pcre_uchar *start_code = md->start_code; + +#ifdef SUPPORT_UTF +BOOL utf = (md->poptions & PCRE_UTF8) != 0; +#else +BOOL utf = FALSE; +#endif + +BOOL reset_could_continue = FALSE; + +rlevel++; +offsetcount &= (-2); + +wscount -= 2; +wscount = (wscount - (wscount % (INTS_PER_STATEBLOCK * 2))) / + (2 * INTS_PER_STATEBLOCK); + +DPRINTF(("\n%.*s---------------------\n" + "%.*sCall to internal_dfa_exec f=%d\n", + rlevel*2-2, SP, rlevel*2-2, SP, rlevel)); + +ctypes = md->tables + ctypes_offset; +lcc = md->tables + lcc_offset; +fcc = md->tables + fcc_offset; + +match_count = PCRE_ERROR_NOMATCH; /* A negative number */ + +active_states = (stateblock *)(workspace + 2); +next_new_state = new_states = active_states + wscount; +new_count = 0; + +first_op = this_start_code + 1 + LINK_SIZE + + ((*this_start_code == OP_CBRA || *this_start_code == OP_SCBRA || + *this_start_code == OP_CBRAPOS || *this_start_code == OP_SCBRAPOS) + ? IMM2_SIZE:0); + +/* The first thing in any (sub) pattern is a bracket of some sort. Push all +the alternative states onto the list, and find out where the end is. This +makes is possible to use this function recursively, when we want to stop at a +matching internal ket rather than at the end. + +If the first opcode in the first alternative is OP_REVERSE, we are dealing with +a backward assertion. In that case, we have to find out the maximum amount to +move back, and set up each alternative appropriately. */ + +if (*first_op == OP_REVERSE) + { + int max_back = 0; + int gone_back; + + end_code = this_start_code; + do + { + int back = GET(end_code, 2+LINK_SIZE); + if (back > max_back) max_back = back; + end_code += GET(end_code, 1); + } + while (*end_code == OP_ALT); + + /* If we can't go back the amount required for the longest lookbehind + pattern, go back as far as we can; some alternatives may still be viable. */ + +#ifdef SUPPORT_UTF + /* In character mode we have to step back character by character */ + + if (utf) + { + for (gone_back = 0; gone_back < max_back; gone_back++) + { + if (current_subject <= start_subject) break; + current_subject--; + ACROSSCHAR(current_subject > start_subject, *current_subject, current_subject--); + } + } + else +#endif + + /* In byte-mode we can do this quickly. */ + + { + gone_back = (current_subject - max_back < start_subject)? + (int)(current_subject - start_subject) : max_back; + current_subject -= gone_back; + } + + /* Save the earliest consulted character */ + + if (current_subject < md->start_used_ptr) + md->start_used_ptr = current_subject; + + /* Now we can process the individual branches. */ + + end_code = this_start_code; + do + { + int back = GET(end_code, 2+LINK_SIZE); + if (back <= gone_back) + { + int bstate = (int)(end_code - start_code + 2 + 2*LINK_SIZE); + ADD_NEW_DATA(-bstate, 0, gone_back - back); + } + end_code += GET(end_code, 1); + } + while (*end_code == OP_ALT); + } + +/* This is the code for a "normal" subpattern (not a backward assertion). The +start of a whole pattern is always one of these. If we are at the top level, +we may be asked to restart matching from the same point that we reached for a +previous partial match. We still have to scan through the top-level branches to +find the end state. */ + +else + { + end_code = this_start_code; + + /* Restarting */ + + if (rlevel == 1 && (md->moptions & PCRE_DFA_RESTART) != 0) + { + do { end_code += GET(end_code, 1); } while (*end_code == OP_ALT); + new_count = workspace[1]; + if (!workspace[0]) + memcpy(new_states, active_states, new_count * sizeof(stateblock)); + } + + /* Not restarting */ + + else + { + int length = 1 + LINK_SIZE + + ((*this_start_code == OP_CBRA || *this_start_code == OP_SCBRA || + *this_start_code == OP_CBRAPOS || *this_start_code == OP_SCBRAPOS) + ? IMM2_SIZE:0); + do + { + ADD_NEW((int)(end_code - start_code + length), 0); + end_code += GET(end_code, 1); + length = 1 + LINK_SIZE; + } + while (*end_code == OP_ALT); + } + } + +workspace[0] = 0; /* Bit indicating which vector is current */ + +DPRINTF(("%.*sEnd state = %d\n", rlevel*2-2, SP, (int)(end_code - start_code))); + +/* Loop for scanning the subject */ + +ptr = current_subject; +for (;;) + { + int i, j; + int clen, dlen; + pcre_uint32 c, d; + int forced_fail = 0; + BOOL partial_newline = FALSE; + BOOL could_continue = reset_could_continue; + reset_could_continue = FALSE; + + /* Make the new state list into the active state list and empty the + new state list. */ + + temp_states = active_states; + active_states = new_states; + new_states = temp_states; + active_count = new_count; + new_count = 0; + + workspace[0] ^= 1; /* Remember for the restarting feature */ + workspace[1] = active_count; + +#ifdef PCRE_DEBUG + printf("%.*sNext character: rest of subject = \"", rlevel*2-2, SP); + pchars(ptr, STRLEN_UC(ptr), stdout); + printf("\"\n"); + + printf("%.*sActive states: ", rlevel*2-2, SP); + for (i = 0; i < active_count; i++) + printf("%d/%d ", active_states[i].offset, active_states[i].count); + printf("\n"); +#endif + + /* Set the pointers for adding new states */ + + next_active_state = active_states + active_count; + next_new_state = new_states; + + /* Load the current character from the subject outside the loop, as many + different states may want to look at it, and we assume that at least one + will. */ + + if (ptr < end_subject) + { + clen = 1; /* Number of data items in the character */ +#ifdef SUPPORT_UTF + GETCHARLENTEST(c, ptr, clen); +#else + c = *ptr; +#endif /* SUPPORT_UTF */ + } + else + { + clen = 0; /* This indicates the end of the subject */ + c = NOTACHAR; /* This value should never actually be used */ + } + + /* Scan up the active states and act on each one. The result of an action + may be to add more states to the currently active list (e.g. on hitting a + parenthesis) or it may be to put states on the new list, for considering + when we move the character pointer on. */ + + for (i = 0; i < active_count; i++) + { + stateblock *current_state = active_states + i; + BOOL caseless = FALSE; + const pcre_uchar *code; + int state_offset = current_state->offset; + int codevalue, rrc; + int count; + +#ifdef PCRE_DEBUG + printf ("%.*sProcessing state %d c=", rlevel*2-2, SP, state_offset); + if (clen == 0) printf("EOL\n"); + else if (c > 32 && c < 127) printf("'%c'\n", c); + else printf("0x%02x\n", c); +#endif + + /* A negative offset is a special case meaning "hold off going to this + (negated) state until the number of characters in the data field have + been skipped". If the could_continue flag was passed over from a previous + state, arrange for it to passed on. */ + + if (state_offset < 0) + { + if (current_state->data > 0) + { + DPRINTF(("%.*sSkipping this character\n", rlevel*2-2, SP)); + ADD_NEW_DATA(state_offset, current_state->count, + current_state->data - 1); + if (could_continue) reset_could_continue = TRUE; + continue; + } + else + { + current_state->offset = state_offset = -state_offset; + } + } + + /* Check for a duplicate state with the same count, and skip if found. + See the note at the head of this module about the possibility of improving + performance here. */ + + for (j = 0; j < i; j++) + { + if (active_states[j].offset == state_offset && + active_states[j].count == current_state->count) + { + DPRINTF(("%.*sDuplicate state: skipped\n", rlevel*2-2, SP)); + goto NEXT_ACTIVE_STATE; + } + } + + /* The state offset is the offset to the opcode */ + + code = start_code + state_offset; + codevalue = *code; + + /* If this opcode inspects a character, but we are at the end of the + subject, remember the fact for use when testing for a partial match. */ + + if (clen == 0 && poptable[codevalue] != 0) + could_continue = TRUE; + + /* If this opcode is followed by an inline character, load it. It is + tempting to test for the presence of a subject character here, but that + is wrong, because sometimes zero repetitions of the subject are + permitted. + + We also use this mechanism for opcodes such as OP_TYPEPLUS that take an + argument that is not a data character - but is always one byte long because + the values are small. We have to take special action to deal with \P, \p, + \H, \h, \V, \v and \X in this case. To keep the other cases fast, convert + these ones to new opcodes. */ + + if (coptable[codevalue] > 0) + { + dlen = 1; +#ifdef SUPPORT_UTF + if (utf) { GETCHARLEN(d, (code + coptable[codevalue]), dlen); } else +#endif /* SUPPORT_UTF */ + d = code[coptable[codevalue]]; + if (codevalue >= OP_TYPESTAR) + { + switch(d) + { + case OP_ANYBYTE: return PCRE_ERROR_DFA_UITEM; + case OP_NOTPROP: + case OP_PROP: codevalue += OP_PROP_EXTRA; break; + case OP_ANYNL: codevalue += OP_ANYNL_EXTRA; break; + case OP_EXTUNI: codevalue += OP_EXTUNI_EXTRA; break; + case OP_NOT_HSPACE: + case OP_HSPACE: codevalue += OP_HSPACE_EXTRA; break; + case OP_NOT_VSPACE: + case OP_VSPACE: codevalue += OP_VSPACE_EXTRA; break; + default: break; + } + } + } + else + { + dlen = 0; /* Not strictly necessary, but compilers moan */ + d = NOTACHAR; /* if these variables are not set. */ + } + + + /* Now process the individual opcodes */ + + switch (codevalue) + { +/* ========================================================================== */ + /* These cases are never obeyed. This is a fudge that causes a compile- + time error if the vectors coptable or poptable, which are indexed by + opcode, are not the correct length. It seems to be the only way to do + such a check at compile time, as the sizeof() operator does not work + in the C preprocessor. */ + + case OP_TABLE_LENGTH: + case OP_TABLE_LENGTH + + ((sizeof(coptable) == OP_TABLE_LENGTH) && + (sizeof(poptable) == OP_TABLE_LENGTH)): + break; + +/* ========================================================================== */ + /* Reached a closing bracket. If not at the end of the pattern, carry + on with the next opcode. For repeating opcodes, also add the repeat + state. Note that KETRPOS will always be encountered at the end of the + subpattern, because the possessive subpattern repeats are always handled + using recursive calls. Thus, it never adds any new states. + + At the end of the (sub)pattern, unless we have an empty string and + PCRE_NOTEMPTY is set, or PCRE_NOTEMPTY_ATSTART is set and we are at the + start of the subject, save the match data, shifting up all previous + matches so we always have the longest first. */ + + case OP_KET: + case OP_KETRMIN: + case OP_KETRMAX: + case OP_KETRPOS: + if (code != end_code) + { + ADD_ACTIVE(state_offset + 1 + LINK_SIZE, 0); + if (codevalue != OP_KET) + { + ADD_ACTIVE(state_offset - GET(code, 1), 0); + } + } + else + { + if (ptr > current_subject || + ((md->moptions & PCRE_NOTEMPTY) == 0 && + ((md->moptions & PCRE_NOTEMPTY_ATSTART) == 0 || + current_subject > start_subject + md->start_offset))) + { + if (match_count < 0) match_count = (offsetcount >= 2)? 1 : 0; + else if (match_count > 0 && ++match_count * 2 > offsetcount) + match_count = 0; + count = ((match_count == 0)? offsetcount : match_count * 2) - 2; + if (count > 0) memmove(offsets + 2, offsets, count * sizeof(int)); + if (offsetcount >= 2) + { + offsets[0] = (int)(current_subject - start_subject); + offsets[1] = (int)(ptr - start_subject); + DPRINTF(("%.*sSet matched string = \"%.*s\"\n", rlevel*2-2, SP, + offsets[1] - offsets[0], (char *)current_subject)); + } + if ((md->moptions & PCRE_DFA_SHORTEST) != 0) + { + DPRINTF(("%.*sEnd of internal_dfa_exec %d: returning %d\n" + "%.*s---------------------\n\n", rlevel*2-2, SP, rlevel, + match_count, rlevel*2-2, SP)); + return match_count; + } + } + } + break; + +/* ========================================================================== */ + /* These opcodes add to the current list of states without looking + at the current character. */ + + /*-----------------------------------------------------------------*/ + case OP_ALT: + do { code += GET(code, 1); } while (*code == OP_ALT); + ADD_ACTIVE((int)(code - start_code), 0); + break; + + /*-----------------------------------------------------------------*/ + case OP_BRA: + case OP_SBRA: + do + { + ADD_ACTIVE((int)(code - start_code + 1 + LINK_SIZE), 0); + code += GET(code, 1); + } + while (*code == OP_ALT); + break; + + /*-----------------------------------------------------------------*/ + case OP_CBRA: + case OP_SCBRA: + ADD_ACTIVE((int)(code - start_code + 1 + LINK_SIZE + IMM2_SIZE), 0); + code += GET(code, 1); + while (*code == OP_ALT) + { + ADD_ACTIVE((int)(code - start_code + 1 + LINK_SIZE), 0); + code += GET(code, 1); + } + break; + + /*-----------------------------------------------------------------*/ + case OP_BRAZERO: + case OP_BRAMINZERO: + ADD_ACTIVE(state_offset + 1, 0); + code += 1 + GET(code, 2); + while (*code == OP_ALT) code += GET(code, 1); + ADD_ACTIVE((int)(code - start_code + 1 + LINK_SIZE), 0); + break; + + /*-----------------------------------------------------------------*/ + case OP_SKIPZERO: + code += 1 + GET(code, 2); + while (*code == OP_ALT) code += GET(code, 1); + ADD_ACTIVE((int)(code - start_code + 1 + LINK_SIZE), 0); + break; + + /*-----------------------------------------------------------------*/ + case OP_CIRC: + if (ptr == start_subject && (md->moptions & PCRE_NOTBOL) == 0) + { ADD_ACTIVE(state_offset + 1, 0); } + break; + + /*-----------------------------------------------------------------*/ + case OP_CIRCM: + if ((ptr == start_subject && (md->moptions & PCRE_NOTBOL) == 0) || + (ptr != end_subject && WAS_NEWLINE(ptr))) + { ADD_ACTIVE(state_offset + 1, 0); } + break; + + /*-----------------------------------------------------------------*/ + case OP_EOD: + if (ptr >= end_subject) + { + if ((md->moptions & PCRE_PARTIAL_HARD) != 0) + could_continue = TRUE; + else { ADD_ACTIVE(state_offset + 1, 0); } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_SOD: + if (ptr == start_subject) { ADD_ACTIVE(state_offset + 1, 0); } + break; + + /*-----------------------------------------------------------------*/ + case OP_SOM: + if (ptr == start_subject + start_offset) { ADD_ACTIVE(state_offset + 1, 0); } + break; + + +/* ========================================================================== */ + /* These opcodes inspect the next subject character, and sometimes + the previous one as well, but do not have an argument. The variable + clen contains the length of the current character and is zero if we are + at the end of the subject. */ + + /*-----------------------------------------------------------------*/ + case OP_ANY: + if (clen > 0 && !IS_NEWLINE(ptr)) + { + if (ptr + 1 >= md->end_subject && + (md->moptions & (PCRE_PARTIAL_HARD)) != 0 && + NLBLOCK->nltype == NLTYPE_FIXED && + NLBLOCK->nllen == 2 && + c == NLBLOCK->nl[0]) + { + could_continue = partial_newline = TRUE; + } + else + { + ADD_NEW(state_offset + 1, 0); + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_ALLANY: + if (clen > 0) + { ADD_NEW(state_offset + 1, 0); } + break; + + /*-----------------------------------------------------------------*/ + case OP_EODN: + if (clen == 0 && (md->moptions & PCRE_PARTIAL_HARD) != 0) + could_continue = TRUE; + else if (clen == 0 || (IS_NEWLINE(ptr) && ptr == end_subject - md->nllen)) + { ADD_ACTIVE(state_offset + 1, 0); } + break; + + /*-----------------------------------------------------------------*/ + case OP_DOLL: + if ((md->moptions & PCRE_NOTEOL) == 0) + { + if (clen == 0 && (md->moptions & PCRE_PARTIAL_HARD) != 0) + could_continue = TRUE; + else if (clen == 0 || + ((md->poptions & PCRE_DOLLAR_ENDONLY) == 0 && IS_NEWLINE(ptr) && + (ptr == end_subject - md->nllen) + )) + { ADD_ACTIVE(state_offset + 1, 0); } + else if (ptr + 1 >= md->end_subject && + (md->moptions & (PCRE_PARTIAL_HARD|PCRE_PARTIAL_SOFT)) != 0 && + NLBLOCK->nltype == NLTYPE_FIXED && + NLBLOCK->nllen == 2 && + c == NLBLOCK->nl[0]) + { + if ((md->moptions & PCRE_PARTIAL_HARD) != 0) + { + reset_could_continue = TRUE; + ADD_NEW_DATA(-(state_offset + 1), 0, 1); + } + else could_continue = partial_newline = TRUE; + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_DOLLM: + if ((md->moptions & PCRE_NOTEOL) == 0) + { + if (clen == 0 && (md->moptions & PCRE_PARTIAL_HARD) != 0) + could_continue = TRUE; + else if (clen == 0 || + ((md->poptions & PCRE_DOLLAR_ENDONLY) == 0 && IS_NEWLINE(ptr))) + { ADD_ACTIVE(state_offset + 1, 0); } + else if (ptr + 1 >= md->end_subject && + (md->moptions & (PCRE_PARTIAL_HARD|PCRE_PARTIAL_SOFT)) != 0 && + NLBLOCK->nltype == NLTYPE_FIXED && + NLBLOCK->nllen == 2 && + c == NLBLOCK->nl[0]) + { + if ((md->moptions & PCRE_PARTIAL_HARD) != 0) + { + reset_could_continue = TRUE; + ADD_NEW_DATA(-(state_offset + 1), 0, 1); + } + else could_continue = partial_newline = TRUE; + } + } + else if (IS_NEWLINE(ptr)) + { ADD_ACTIVE(state_offset + 1, 0); } + break; + + /*-----------------------------------------------------------------*/ + + case OP_DIGIT: + case OP_WHITESPACE: + case OP_WORDCHAR: + if (clen > 0 && c < 256 && + ((ctypes[c] & toptable1[codevalue]) ^ toptable2[codevalue]) != 0) + { ADD_NEW(state_offset + 1, 0); } + break; + + /*-----------------------------------------------------------------*/ + case OP_NOT_DIGIT: + case OP_NOT_WHITESPACE: + case OP_NOT_WORDCHAR: + if (clen > 0 && (c >= 256 || + ((ctypes[c] & toptable1[codevalue]) ^ toptable2[codevalue]) != 0)) + { ADD_NEW(state_offset + 1, 0); } + break; + + /*-----------------------------------------------------------------*/ + case OP_WORD_BOUNDARY: + case OP_NOT_WORD_BOUNDARY: + { + int left_word, right_word; + + if (ptr > start_subject) + { + const pcre_uchar *temp = ptr - 1; + if (temp < md->start_used_ptr) md->start_used_ptr = temp; +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (utf) { BACKCHAR(temp); } +#endif + GETCHARTEST(d, temp); +#ifdef SUPPORT_UCP + if ((md->poptions & PCRE_UCP) != 0) + { + if (d == '_') left_word = TRUE; else + { + int cat = UCD_CATEGORY(d); + left_word = (cat == ucp_L || cat == ucp_N); + } + } + else +#endif + left_word = d < 256 && (ctypes[d] & ctype_word) != 0; + } + else left_word = FALSE; + + if (clen > 0) + { +#ifdef SUPPORT_UCP + if ((md->poptions & PCRE_UCP) != 0) + { + if (c == '_') right_word = TRUE; else + { + int cat = UCD_CATEGORY(c); + right_word = (cat == ucp_L || cat == ucp_N); + } + } + else +#endif + right_word = c < 256 && (ctypes[c] & ctype_word) != 0; + } + else right_word = FALSE; + + if ((left_word == right_word) == (codevalue == OP_NOT_WORD_BOUNDARY)) + { ADD_ACTIVE(state_offset + 1, 0); } + } + break; + + + /*-----------------------------------------------------------------*/ + /* Check the next character by Unicode property. We will get here only + if the support is in the binary; otherwise a compile-time error occurs. + */ + +#ifdef SUPPORT_UCP + case OP_PROP: + case OP_NOTPROP: + if (clen > 0) + { + BOOL OK; + const pcre_uint32 *cp; + const ucd_record * prop = GET_UCD(c); + switch(code[1]) + { + case PT_ANY: + OK = TRUE; + break; + + case PT_LAMP: + OK = prop->chartype == ucp_Lu || prop->chartype == ucp_Ll || + prop->chartype == ucp_Lt; + break; + + case PT_GC: + OK = PRIV(ucp_gentype)[prop->chartype] == code[2]; + break; + + case PT_PC: + OK = prop->chartype == code[2]; + break; + + case PT_SC: + OK = prop->script == code[2]; + break; + + /* These are specials for combination cases. */ + + case PT_ALNUM: + OK = PRIV(ucp_gentype)[prop->chartype] == ucp_L || + PRIV(ucp_gentype)[prop->chartype] == ucp_N; + break; + + /* Perl space used to exclude VT, but from Perl 5.18 it is included, + which means that Perl space and POSIX space are now identical. PCRE + was changed at release 8.34. */ + + case PT_SPACE: /* Perl space */ + case PT_PXSPACE: /* POSIX space */ + switch(c) + { + HSPACE_CASES: + VSPACE_CASES: + OK = TRUE; + break; + + default: + OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z; + break; + } + break; + + case PT_WORD: + OK = PRIV(ucp_gentype)[prop->chartype] == ucp_L || + PRIV(ucp_gentype)[prop->chartype] == ucp_N || + c == CHAR_UNDERSCORE; + break; + + case PT_CLIST: + cp = PRIV(ucd_caseless_sets) + code[2]; + for (;;) + { + if (c < *cp) { OK = FALSE; break; } + if (c == *cp++) { OK = TRUE; break; } + } + break; + + case PT_UCNC: + OK = c == CHAR_DOLLAR_SIGN || c == CHAR_COMMERCIAL_AT || + c == CHAR_GRAVE_ACCENT || (c >= 0xa0 && c <= 0xd7ff) || + c >= 0xe000; + break; + + /* Should never occur, but keep compilers from grumbling. */ + + default: + OK = codevalue != OP_PROP; + break; + } + + if (OK == (codevalue == OP_PROP)) { ADD_NEW(state_offset + 3, 0); } + } + break; +#endif + + + +/* ========================================================================== */ + /* These opcodes likewise inspect the subject character, but have an + argument that is not a data character. It is one of these opcodes: + OP_ANY, OP_ALLANY, OP_DIGIT, OP_NOT_DIGIT, OP_WHITESPACE, OP_NOT_SPACE, + OP_WORDCHAR, OP_NOT_WORDCHAR. The value is loaded into d. */ + + case OP_TYPEPLUS: + case OP_TYPEMINPLUS: + case OP_TYPEPOSPLUS: + count = current_state->count; /* Already matched */ + if (count > 0) { ADD_ACTIVE(state_offset + 2, 0); } + if (clen > 0) + { + if (d == OP_ANY && ptr + 1 >= md->end_subject && + (md->moptions & (PCRE_PARTIAL_HARD)) != 0 && + NLBLOCK->nltype == NLTYPE_FIXED && + NLBLOCK->nllen == 2 && + c == NLBLOCK->nl[0]) + { + could_continue = partial_newline = TRUE; + } + else if ((c >= 256 && d != OP_DIGIT && d != OP_WHITESPACE && d != OP_WORDCHAR) || + (c < 256 && + (d != OP_ANY || !IS_NEWLINE(ptr)) && + ((ctypes[c] & toptable1[d]) ^ toptable2[d]) != 0)) + { + if (count > 0 && codevalue == OP_TYPEPOSPLUS) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + count++; + ADD_NEW(state_offset, count); + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_TYPEQUERY: + case OP_TYPEMINQUERY: + case OP_TYPEPOSQUERY: + ADD_ACTIVE(state_offset + 2, 0); + if (clen > 0) + { + if (d == OP_ANY && ptr + 1 >= md->end_subject && + (md->moptions & (PCRE_PARTIAL_HARD)) != 0 && + NLBLOCK->nltype == NLTYPE_FIXED && + NLBLOCK->nllen == 2 && + c == NLBLOCK->nl[0]) + { + could_continue = partial_newline = TRUE; + } + else if ((c >= 256 && d != OP_DIGIT && d != OP_WHITESPACE && d != OP_WORDCHAR) || + (c < 256 && + (d != OP_ANY || !IS_NEWLINE(ptr)) && + ((ctypes[c] & toptable1[d]) ^ toptable2[d]) != 0)) + { + if (codevalue == OP_TYPEPOSQUERY) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + ADD_NEW(state_offset + 2, 0); + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_TYPESTAR: + case OP_TYPEMINSTAR: + case OP_TYPEPOSSTAR: + ADD_ACTIVE(state_offset + 2, 0); + if (clen > 0) + { + if (d == OP_ANY && ptr + 1 >= md->end_subject && + (md->moptions & (PCRE_PARTIAL_HARD)) != 0 && + NLBLOCK->nltype == NLTYPE_FIXED && + NLBLOCK->nllen == 2 && + c == NLBLOCK->nl[0]) + { + could_continue = partial_newline = TRUE; + } + else if ((c >= 256 && d != OP_DIGIT && d != OP_WHITESPACE && d != OP_WORDCHAR) || + (c < 256 && + (d != OP_ANY || !IS_NEWLINE(ptr)) && + ((ctypes[c] & toptable1[d]) ^ toptable2[d]) != 0)) + { + if (codevalue == OP_TYPEPOSSTAR) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + ADD_NEW(state_offset, 0); + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_TYPEEXACT: + count = current_state->count; /* Number already matched */ + if (clen > 0) + { + if (d == OP_ANY && ptr + 1 >= md->end_subject && + (md->moptions & (PCRE_PARTIAL_HARD)) != 0 && + NLBLOCK->nltype == NLTYPE_FIXED && + NLBLOCK->nllen == 2 && + c == NLBLOCK->nl[0]) + { + could_continue = partial_newline = TRUE; + } + else if ((c >= 256 && d != OP_DIGIT && d != OP_WHITESPACE && d != OP_WORDCHAR) || + (c < 256 && + (d != OP_ANY || !IS_NEWLINE(ptr)) && + ((ctypes[c] & toptable1[d]) ^ toptable2[d]) != 0)) + { + if (++count >= (int)GET2(code, 1)) + { ADD_NEW(state_offset + 1 + IMM2_SIZE + 1, 0); } + else + { ADD_NEW(state_offset, count); } + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_TYPEUPTO: + case OP_TYPEMINUPTO: + case OP_TYPEPOSUPTO: + ADD_ACTIVE(state_offset + 2 + IMM2_SIZE, 0); + count = current_state->count; /* Number already matched */ + if (clen > 0) + { + if (d == OP_ANY && ptr + 1 >= md->end_subject && + (md->moptions & (PCRE_PARTIAL_HARD)) != 0 && + NLBLOCK->nltype == NLTYPE_FIXED && + NLBLOCK->nllen == 2 && + c == NLBLOCK->nl[0]) + { + could_continue = partial_newline = TRUE; + } + else if ((c >= 256 && d != OP_DIGIT && d != OP_WHITESPACE && d != OP_WORDCHAR) || + (c < 256 && + (d != OP_ANY || !IS_NEWLINE(ptr)) && + ((ctypes[c] & toptable1[d]) ^ toptable2[d]) != 0)) + { + if (codevalue == OP_TYPEPOSUPTO) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + if (++count >= (int)GET2(code, 1)) + { ADD_NEW(state_offset + 2 + IMM2_SIZE, 0); } + else + { ADD_NEW(state_offset, count); } + } + } + break; + +/* ========================================================================== */ + /* These are virtual opcodes that are used when something like + OP_TYPEPLUS has OP_PROP, OP_NOTPROP, OP_ANYNL, or OP_EXTUNI as its + argument. It keeps the code above fast for the other cases. The argument + is in the d variable. */ + +#ifdef SUPPORT_UCP + case OP_PROP_EXTRA + OP_TYPEPLUS: + case OP_PROP_EXTRA + OP_TYPEMINPLUS: + case OP_PROP_EXTRA + OP_TYPEPOSPLUS: + count = current_state->count; /* Already matched */ + if (count > 0) { ADD_ACTIVE(state_offset + 4, 0); } + if (clen > 0) + { + BOOL OK; + const pcre_uint32 *cp; + const ucd_record * prop = GET_UCD(c); + switch(code[2]) + { + case PT_ANY: + OK = TRUE; + break; + + case PT_LAMP: + OK = prop->chartype == ucp_Lu || prop->chartype == ucp_Ll || + prop->chartype == ucp_Lt; + break; + + case PT_GC: + OK = PRIV(ucp_gentype)[prop->chartype] == code[3]; + break; + + case PT_PC: + OK = prop->chartype == code[3]; + break; + + case PT_SC: + OK = prop->script == code[3]; + break; + + /* These are specials for combination cases. */ + + case PT_ALNUM: + OK = PRIV(ucp_gentype)[prop->chartype] == ucp_L || + PRIV(ucp_gentype)[prop->chartype] == ucp_N; + break; + + /* Perl space used to exclude VT, but from Perl 5.18 it is included, + which means that Perl space and POSIX space are now identical. PCRE + was changed at release 8.34. */ + + case PT_SPACE: /* Perl space */ + case PT_PXSPACE: /* POSIX space */ + switch(c) + { + HSPACE_CASES: + VSPACE_CASES: + OK = TRUE; + break; + + default: + OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z; + break; + } + break; + + case PT_WORD: + OK = PRIV(ucp_gentype)[prop->chartype] == ucp_L || + PRIV(ucp_gentype)[prop->chartype] == ucp_N || + c == CHAR_UNDERSCORE; + break; + + case PT_CLIST: + cp = PRIV(ucd_caseless_sets) + code[3]; + for (;;) + { + if (c < *cp) { OK = FALSE; break; } + if (c == *cp++) { OK = TRUE; break; } + } + break; + + case PT_UCNC: + OK = c == CHAR_DOLLAR_SIGN || c == CHAR_COMMERCIAL_AT || + c == CHAR_GRAVE_ACCENT || (c >= 0xa0 && c <= 0xd7ff) || + c >= 0xe000; + break; + + /* Should never occur, but keep compilers from grumbling. */ + + default: + OK = codevalue != OP_PROP; + break; + } + + if (OK == (d == OP_PROP)) + { + if (count > 0 && codevalue == OP_PROP_EXTRA + OP_TYPEPOSPLUS) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + count++; + ADD_NEW(state_offset, count); + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_EXTUNI_EXTRA + OP_TYPEPLUS: + case OP_EXTUNI_EXTRA + OP_TYPEMINPLUS: + case OP_EXTUNI_EXTRA + OP_TYPEPOSPLUS: + count = current_state->count; /* Already matched */ + if (count > 0) { ADD_ACTIVE(state_offset + 2, 0); } + if (clen > 0) + { + int lgb, rgb; + const pcre_uchar *nptr = ptr + clen; + int ncount = 0; + if (count > 0 && codevalue == OP_EXTUNI_EXTRA + OP_TYPEPOSPLUS) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + lgb = UCD_GRAPHBREAK(c); + while (nptr < end_subject) + { + dlen = 1; + if (!utf) d = *nptr; else { GETCHARLEN(d, nptr, dlen); } + rgb = UCD_GRAPHBREAK(d); + if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; + ncount++; + lgb = rgb; + nptr += dlen; + } + count++; + ADD_NEW_DATA(-state_offset, count, ncount); + } + break; +#endif + + /*-----------------------------------------------------------------*/ + case OP_ANYNL_EXTRA + OP_TYPEPLUS: + case OP_ANYNL_EXTRA + OP_TYPEMINPLUS: + case OP_ANYNL_EXTRA + OP_TYPEPOSPLUS: + count = current_state->count; /* Already matched */ + if (count > 0) { ADD_ACTIVE(state_offset + 2, 0); } + if (clen > 0) + { + int ncount = 0; + switch (c) + { + case CHAR_VT: + case CHAR_FF: + case CHAR_NEL: +#ifndef EBCDIC + case 0x2028: + case 0x2029: +#endif /* Not EBCDIC */ + if ((md->moptions & PCRE_BSR_ANYCRLF) != 0) break; + goto ANYNL01; + + case CHAR_CR: + if (ptr + 1 < end_subject && UCHAR21TEST(ptr + 1) == CHAR_LF) ncount = 1; + /* Fall through */ + + ANYNL01: + case CHAR_LF: + if (count > 0 && codevalue == OP_ANYNL_EXTRA + OP_TYPEPOSPLUS) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + count++; + ADD_NEW_DATA(-state_offset, count, ncount); + break; + + default: + break; + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_VSPACE_EXTRA + OP_TYPEPLUS: + case OP_VSPACE_EXTRA + OP_TYPEMINPLUS: + case OP_VSPACE_EXTRA + OP_TYPEPOSPLUS: + count = current_state->count; /* Already matched */ + if (count > 0) { ADD_ACTIVE(state_offset + 2, 0); } + if (clen > 0) + { + BOOL OK; + switch (c) + { + VSPACE_CASES: + OK = TRUE; + break; + + default: + OK = FALSE; + break; + } + + if (OK == (d == OP_VSPACE)) + { + if (count > 0 && codevalue == OP_VSPACE_EXTRA + OP_TYPEPOSPLUS) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + count++; + ADD_NEW_DATA(-state_offset, count, 0); + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_HSPACE_EXTRA + OP_TYPEPLUS: + case OP_HSPACE_EXTRA + OP_TYPEMINPLUS: + case OP_HSPACE_EXTRA + OP_TYPEPOSPLUS: + count = current_state->count; /* Already matched */ + if (count > 0) { ADD_ACTIVE(state_offset + 2, 0); } + if (clen > 0) + { + BOOL OK; + switch (c) + { + HSPACE_CASES: + OK = TRUE; + break; + + default: + OK = FALSE; + break; + } + + if (OK == (d == OP_HSPACE)) + { + if (count > 0 && codevalue == OP_HSPACE_EXTRA + OP_TYPEPOSPLUS) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + count++; + ADD_NEW_DATA(-state_offset, count, 0); + } + } + break; + + /*-----------------------------------------------------------------*/ +#ifdef SUPPORT_UCP + case OP_PROP_EXTRA + OP_TYPEQUERY: + case OP_PROP_EXTRA + OP_TYPEMINQUERY: + case OP_PROP_EXTRA + OP_TYPEPOSQUERY: + count = 4; + goto QS1; + + case OP_PROP_EXTRA + OP_TYPESTAR: + case OP_PROP_EXTRA + OP_TYPEMINSTAR: + case OP_PROP_EXTRA + OP_TYPEPOSSTAR: + count = 0; + + QS1: + + ADD_ACTIVE(state_offset + 4, 0); + if (clen > 0) + { + BOOL OK; + const pcre_uint32 *cp; + const ucd_record * prop = GET_UCD(c); + switch(code[2]) + { + case PT_ANY: + OK = TRUE; + break; + + case PT_LAMP: + OK = prop->chartype == ucp_Lu || prop->chartype == ucp_Ll || + prop->chartype == ucp_Lt; + break; + + case PT_GC: + OK = PRIV(ucp_gentype)[prop->chartype] == code[3]; + break; + + case PT_PC: + OK = prop->chartype == code[3]; + break; + + case PT_SC: + OK = prop->script == code[3]; + break; + + /* These are specials for combination cases. */ + + case PT_ALNUM: + OK = PRIV(ucp_gentype)[prop->chartype] == ucp_L || + PRIV(ucp_gentype)[prop->chartype] == ucp_N; + break; + + /* Perl space used to exclude VT, but from Perl 5.18 it is included, + which means that Perl space and POSIX space are now identical. PCRE + was changed at release 8.34. */ + + case PT_SPACE: /* Perl space */ + case PT_PXSPACE: /* POSIX space */ + switch(c) + { + HSPACE_CASES: + VSPACE_CASES: + OK = TRUE; + break; + + default: + OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z; + break; + } + break; + + case PT_WORD: + OK = PRIV(ucp_gentype)[prop->chartype] == ucp_L || + PRIV(ucp_gentype)[prop->chartype] == ucp_N || + c == CHAR_UNDERSCORE; + break; + + case PT_CLIST: + cp = PRIV(ucd_caseless_sets) + code[3]; + for (;;) + { + if (c < *cp) { OK = FALSE; break; } + if (c == *cp++) { OK = TRUE; break; } + } + break; + + case PT_UCNC: + OK = c == CHAR_DOLLAR_SIGN || c == CHAR_COMMERCIAL_AT || + c == CHAR_GRAVE_ACCENT || (c >= 0xa0 && c <= 0xd7ff) || + c >= 0xe000; + break; + + /* Should never occur, but keep compilers from grumbling. */ + + default: + OK = codevalue != OP_PROP; + break; + } + + if (OK == (d == OP_PROP)) + { + if (codevalue == OP_PROP_EXTRA + OP_TYPEPOSSTAR || + codevalue == OP_PROP_EXTRA + OP_TYPEPOSQUERY) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + ADD_NEW(state_offset + count, 0); + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_EXTUNI_EXTRA + OP_TYPEQUERY: + case OP_EXTUNI_EXTRA + OP_TYPEMINQUERY: + case OP_EXTUNI_EXTRA + OP_TYPEPOSQUERY: + count = 2; + goto QS2; + + case OP_EXTUNI_EXTRA + OP_TYPESTAR: + case OP_EXTUNI_EXTRA + OP_TYPEMINSTAR: + case OP_EXTUNI_EXTRA + OP_TYPEPOSSTAR: + count = 0; + + QS2: + + ADD_ACTIVE(state_offset + 2, 0); + if (clen > 0) + { + int lgb, rgb; + const pcre_uchar *nptr = ptr + clen; + int ncount = 0; + if (codevalue == OP_EXTUNI_EXTRA + OP_TYPEPOSSTAR || + codevalue == OP_EXTUNI_EXTRA + OP_TYPEPOSQUERY) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + lgb = UCD_GRAPHBREAK(c); + while (nptr < end_subject) + { + dlen = 1; + if (!utf) d = *nptr; else { GETCHARLEN(d, nptr, dlen); } + rgb = UCD_GRAPHBREAK(d); + if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; + ncount++; + lgb = rgb; + nptr += dlen; + } + ADD_NEW_DATA(-(state_offset + count), 0, ncount); + } + break; +#endif + + /*-----------------------------------------------------------------*/ + case OP_ANYNL_EXTRA + OP_TYPEQUERY: + case OP_ANYNL_EXTRA + OP_TYPEMINQUERY: + case OP_ANYNL_EXTRA + OP_TYPEPOSQUERY: + count = 2; + goto QS3; + + case OP_ANYNL_EXTRA + OP_TYPESTAR: + case OP_ANYNL_EXTRA + OP_TYPEMINSTAR: + case OP_ANYNL_EXTRA + OP_TYPEPOSSTAR: + count = 0; + + QS3: + ADD_ACTIVE(state_offset + 2, 0); + if (clen > 0) + { + int ncount = 0; + switch (c) + { + case CHAR_VT: + case CHAR_FF: + case CHAR_NEL: +#ifndef EBCDIC + case 0x2028: + case 0x2029: +#endif /* Not EBCDIC */ + if ((md->moptions & PCRE_BSR_ANYCRLF) != 0) break; + goto ANYNL02; + + case CHAR_CR: + if (ptr + 1 < end_subject && UCHAR21TEST(ptr + 1) == CHAR_LF) ncount = 1; + /* Fall through */ + + ANYNL02: + case CHAR_LF: + if (codevalue == OP_ANYNL_EXTRA + OP_TYPEPOSSTAR || + codevalue == OP_ANYNL_EXTRA + OP_TYPEPOSQUERY) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + ADD_NEW_DATA(-(state_offset + (int)count), 0, ncount); + break; + + default: + break; + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_VSPACE_EXTRA + OP_TYPEQUERY: + case OP_VSPACE_EXTRA + OP_TYPEMINQUERY: + case OP_VSPACE_EXTRA + OP_TYPEPOSQUERY: + count = 2; + goto QS4; + + case OP_VSPACE_EXTRA + OP_TYPESTAR: + case OP_VSPACE_EXTRA + OP_TYPEMINSTAR: + case OP_VSPACE_EXTRA + OP_TYPEPOSSTAR: + count = 0; + + QS4: + ADD_ACTIVE(state_offset + 2, 0); + if (clen > 0) + { + BOOL OK; + switch (c) + { + VSPACE_CASES: + OK = TRUE; + break; + + default: + OK = FALSE; + break; + } + if (OK == (d == OP_VSPACE)) + { + if (codevalue == OP_VSPACE_EXTRA + OP_TYPEPOSSTAR || + codevalue == OP_VSPACE_EXTRA + OP_TYPEPOSQUERY) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + ADD_NEW_DATA(-(state_offset + (int)count), 0, 0); + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_HSPACE_EXTRA + OP_TYPEQUERY: + case OP_HSPACE_EXTRA + OP_TYPEMINQUERY: + case OP_HSPACE_EXTRA + OP_TYPEPOSQUERY: + count = 2; + goto QS5; + + case OP_HSPACE_EXTRA + OP_TYPESTAR: + case OP_HSPACE_EXTRA + OP_TYPEMINSTAR: + case OP_HSPACE_EXTRA + OP_TYPEPOSSTAR: + count = 0; + + QS5: + ADD_ACTIVE(state_offset + 2, 0); + if (clen > 0) + { + BOOL OK; + switch (c) + { + HSPACE_CASES: + OK = TRUE; + break; + + default: + OK = FALSE; + break; + } + + if (OK == (d == OP_HSPACE)) + { + if (codevalue == OP_HSPACE_EXTRA + OP_TYPEPOSSTAR || + codevalue == OP_HSPACE_EXTRA + OP_TYPEPOSQUERY) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + ADD_NEW_DATA(-(state_offset + (int)count), 0, 0); + } + } + break; + + /*-----------------------------------------------------------------*/ +#ifdef SUPPORT_UCP + case OP_PROP_EXTRA + OP_TYPEEXACT: + case OP_PROP_EXTRA + OP_TYPEUPTO: + case OP_PROP_EXTRA + OP_TYPEMINUPTO: + case OP_PROP_EXTRA + OP_TYPEPOSUPTO: + if (codevalue != OP_PROP_EXTRA + OP_TYPEEXACT) + { ADD_ACTIVE(state_offset + 1 + IMM2_SIZE + 3, 0); } + count = current_state->count; /* Number already matched */ + if (clen > 0) + { + BOOL OK; + const pcre_uint32 *cp; + const ucd_record * prop = GET_UCD(c); + switch(code[1 + IMM2_SIZE + 1]) + { + case PT_ANY: + OK = TRUE; + break; + + case PT_LAMP: + OK = prop->chartype == ucp_Lu || prop->chartype == ucp_Ll || + prop->chartype == ucp_Lt; + break; + + case PT_GC: + OK = PRIV(ucp_gentype)[prop->chartype] == code[1 + IMM2_SIZE + 2]; + break; + + case PT_PC: + OK = prop->chartype == code[1 + IMM2_SIZE + 2]; + break; + + case PT_SC: + OK = prop->script == code[1 + IMM2_SIZE + 2]; + break; + + /* These are specials for combination cases. */ + + case PT_ALNUM: + OK = PRIV(ucp_gentype)[prop->chartype] == ucp_L || + PRIV(ucp_gentype)[prop->chartype] == ucp_N; + break; + + /* Perl space used to exclude VT, but from Perl 5.18 it is included, + which means that Perl space and POSIX space are now identical. PCRE + was changed at release 8.34. */ + + case PT_SPACE: /* Perl space */ + case PT_PXSPACE: /* POSIX space */ + switch(c) + { + HSPACE_CASES: + VSPACE_CASES: + OK = TRUE; + break; + + default: + OK = PRIV(ucp_gentype)[prop->chartype] == ucp_Z; + break; + } + break; + + case PT_WORD: + OK = PRIV(ucp_gentype)[prop->chartype] == ucp_L || + PRIV(ucp_gentype)[prop->chartype] == ucp_N || + c == CHAR_UNDERSCORE; + break; + + case PT_CLIST: + cp = PRIV(ucd_caseless_sets) + code[1 + IMM2_SIZE + 2]; + for (;;) + { + if (c < *cp) { OK = FALSE; break; } + if (c == *cp++) { OK = TRUE; break; } + } + break; + + case PT_UCNC: + OK = c == CHAR_DOLLAR_SIGN || c == CHAR_COMMERCIAL_AT || + c == CHAR_GRAVE_ACCENT || (c >= 0xa0 && c <= 0xd7ff) || + c >= 0xe000; + break; + + /* Should never occur, but keep compilers from grumbling. */ + + default: + OK = codevalue != OP_PROP; + break; + } + + if (OK == (d == OP_PROP)) + { + if (codevalue == OP_PROP_EXTRA + OP_TYPEPOSUPTO) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + if (++count >= (int)GET2(code, 1)) + { ADD_NEW(state_offset + 1 + IMM2_SIZE + 3, 0); } + else + { ADD_NEW(state_offset, count); } + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_EXTUNI_EXTRA + OP_TYPEEXACT: + case OP_EXTUNI_EXTRA + OP_TYPEUPTO: + case OP_EXTUNI_EXTRA + OP_TYPEMINUPTO: + case OP_EXTUNI_EXTRA + OP_TYPEPOSUPTO: + if (codevalue != OP_EXTUNI_EXTRA + OP_TYPEEXACT) + { ADD_ACTIVE(state_offset + 2 + IMM2_SIZE, 0); } + count = current_state->count; /* Number already matched */ + if (clen > 0) + { + int lgb, rgb; + const pcre_uchar *nptr = ptr + clen; + int ncount = 0; + if (codevalue == OP_EXTUNI_EXTRA + OP_TYPEPOSUPTO) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + lgb = UCD_GRAPHBREAK(c); + while (nptr < end_subject) + { + dlen = 1; + if (!utf) d = *nptr; else { GETCHARLEN(d, nptr, dlen); } + rgb = UCD_GRAPHBREAK(d); + if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; + ncount++; + lgb = rgb; + nptr += dlen; + } + if (nptr >= end_subject && (md->moptions & PCRE_PARTIAL_HARD) != 0) + reset_could_continue = TRUE; + if (++count >= (int)GET2(code, 1)) + { ADD_NEW_DATA(-(state_offset + 2 + IMM2_SIZE), 0, ncount); } + else + { ADD_NEW_DATA(-state_offset, count, ncount); } + } + break; +#endif + + /*-----------------------------------------------------------------*/ + case OP_ANYNL_EXTRA + OP_TYPEEXACT: + case OP_ANYNL_EXTRA + OP_TYPEUPTO: + case OP_ANYNL_EXTRA + OP_TYPEMINUPTO: + case OP_ANYNL_EXTRA + OP_TYPEPOSUPTO: + if (codevalue != OP_ANYNL_EXTRA + OP_TYPEEXACT) + { ADD_ACTIVE(state_offset + 2 + IMM2_SIZE, 0); } + count = current_state->count; /* Number already matched */ + if (clen > 0) + { + int ncount = 0; + switch (c) + { + case CHAR_VT: + case CHAR_FF: + case CHAR_NEL: +#ifndef EBCDIC + case 0x2028: + case 0x2029: +#endif /* Not EBCDIC */ + if ((md->moptions & PCRE_BSR_ANYCRLF) != 0) break; + goto ANYNL03; + + case CHAR_CR: + if (ptr + 1 < end_subject && UCHAR21TEST(ptr + 1) == CHAR_LF) ncount = 1; + /* Fall through */ + + ANYNL03: + case CHAR_LF: + if (codevalue == OP_ANYNL_EXTRA + OP_TYPEPOSUPTO) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + if (++count >= (int)GET2(code, 1)) + { ADD_NEW_DATA(-(state_offset + 2 + IMM2_SIZE), 0, ncount); } + else + { ADD_NEW_DATA(-state_offset, count, ncount); } + break; + + default: + break; + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_VSPACE_EXTRA + OP_TYPEEXACT: + case OP_VSPACE_EXTRA + OP_TYPEUPTO: + case OP_VSPACE_EXTRA + OP_TYPEMINUPTO: + case OP_VSPACE_EXTRA + OP_TYPEPOSUPTO: + if (codevalue != OP_VSPACE_EXTRA + OP_TYPEEXACT) + { ADD_ACTIVE(state_offset + 2 + IMM2_SIZE, 0); } + count = current_state->count; /* Number already matched */ + if (clen > 0) + { + BOOL OK; + switch (c) + { + VSPACE_CASES: + OK = TRUE; + break; + + default: + OK = FALSE; + } + + if (OK == (d == OP_VSPACE)) + { + if (codevalue == OP_VSPACE_EXTRA + OP_TYPEPOSUPTO) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + if (++count >= (int)GET2(code, 1)) + { ADD_NEW_DATA(-(state_offset + 2 + IMM2_SIZE), 0, 0); } + else + { ADD_NEW_DATA(-state_offset, count, 0); } + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_HSPACE_EXTRA + OP_TYPEEXACT: + case OP_HSPACE_EXTRA + OP_TYPEUPTO: + case OP_HSPACE_EXTRA + OP_TYPEMINUPTO: + case OP_HSPACE_EXTRA + OP_TYPEPOSUPTO: + if (codevalue != OP_HSPACE_EXTRA + OP_TYPEEXACT) + { ADD_ACTIVE(state_offset + 2 + IMM2_SIZE, 0); } + count = current_state->count; /* Number already matched */ + if (clen > 0) + { + BOOL OK; + switch (c) + { + HSPACE_CASES: + OK = TRUE; + break; + + default: + OK = FALSE; + break; + } + + if (OK == (d == OP_HSPACE)) + { + if (codevalue == OP_HSPACE_EXTRA + OP_TYPEPOSUPTO) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + if (++count >= (int)GET2(code, 1)) + { ADD_NEW_DATA(-(state_offset + 2 + IMM2_SIZE), 0, 0); } + else + { ADD_NEW_DATA(-state_offset, count, 0); } + } + } + break; + +/* ========================================================================== */ + /* These opcodes are followed by a character that is usually compared + to the current subject character; it is loaded into d. We still get + here even if there is no subject character, because in some cases zero + repetitions are permitted. */ + + /*-----------------------------------------------------------------*/ + case OP_CHAR: + if (clen > 0 && c == d) { ADD_NEW(state_offset + dlen + 1, 0); } + break; + + /*-----------------------------------------------------------------*/ + case OP_CHARI: + if (clen == 0) break; + +#ifdef SUPPORT_UTF + if (utf) + { + if (c == d) { ADD_NEW(state_offset + dlen + 1, 0); } else + { + unsigned int othercase; + if (c < 128) + othercase = fcc[c]; + else + /* If we have Unicode property support, we can use it to test the + other case of the character. */ +#ifdef SUPPORT_UCP + othercase = UCD_OTHERCASE(c); +#else + othercase = NOTACHAR; +#endif + + if (d == othercase) { ADD_NEW(state_offset + dlen + 1, 0); } + } + } + else +#endif /* SUPPORT_UTF */ + /* Not UTF mode */ + { + if (TABLE_GET(c, lcc, c) == TABLE_GET(d, lcc, d)) + { ADD_NEW(state_offset + 2, 0); } + } + break; + + +#ifdef SUPPORT_UCP + /*-----------------------------------------------------------------*/ + /* This is a tricky one because it can match more than one character. + Find out how many characters to skip, and then set up a negative state + to wait for them to pass before continuing. */ + + case OP_EXTUNI: + if (clen > 0) + { + int lgb, rgb; + const pcre_uchar *nptr = ptr + clen; + int ncount = 0; + lgb = UCD_GRAPHBREAK(c); + while (nptr < end_subject) + { + dlen = 1; + if (!utf) d = *nptr; else { GETCHARLEN(d, nptr, dlen); } + rgb = UCD_GRAPHBREAK(d); + if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; + ncount++; + lgb = rgb; + nptr += dlen; + } + if (nptr >= end_subject && (md->moptions & PCRE_PARTIAL_HARD) != 0) + reset_could_continue = TRUE; + ADD_NEW_DATA(-(state_offset + 1), 0, ncount); + } + break; +#endif + + /*-----------------------------------------------------------------*/ + /* This is a tricky like EXTUNI because it too can match more than one + character (when CR is followed by LF). In this case, set up a negative + state to wait for one character to pass before continuing. */ + + case OP_ANYNL: + if (clen > 0) switch(c) + { + case CHAR_VT: + case CHAR_FF: + case CHAR_NEL: +#ifndef EBCDIC + case 0x2028: + case 0x2029: +#endif /* Not EBCDIC */ + if ((md->moptions & PCRE_BSR_ANYCRLF) != 0) break; + + case CHAR_LF: + ADD_NEW(state_offset + 1, 0); + break; + + case CHAR_CR: + if (ptr + 1 >= end_subject) + { + ADD_NEW(state_offset + 1, 0); + if ((md->moptions & PCRE_PARTIAL_HARD) != 0) + reset_could_continue = TRUE; + } + else if (UCHAR21TEST(ptr + 1) == CHAR_LF) + { + ADD_NEW_DATA(-(state_offset + 1), 0, 1); + } + else + { + ADD_NEW(state_offset + 1, 0); + } + break; + } + break; + + /*-----------------------------------------------------------------*/ + case OP_NOT_VSPACE: + if (clen > 0) switch(c) + { + VSPACE_CASES: + break; + + default: + ADD_NEW(state_offset + 1, 0); + break; + } + break; + + /*-----------------------------------------------------------------*/ + case OP_VSPACE: + if (clen > 0) switch(c) + { + VSPACE_CASES: + ADD_NEW(state_offset + 1, 0); + break; + + default: + break; + } + break; + + /*-----------------------------------------------------------------*/ + case OP_NOT_HSPACE: + if (clen > 0) switch(c) + { + HSPACE_CASES: + break; + + default: + ADD_NEW(state_offset + 1, 0); + break; + } + break; + + /*-----------------------------------------------------------------*/ + case OP_HSPACE: + if (clen > 0) switch(c) + { + HSPACE_CASES: + ADD_NEW(state_offset + 1, 0); + break; + + default: + break; + } + break; + + /*-----------------------------------------------------------------*/ + /* Match a negated single character casefully. */ + + case OP_NOT: + if (clen > 0 && c != d) { ADD_NEW(state_offset + dlen + 1, 0); } + break; + + /*-----------------------------------------------------------------*/ + /* Match a negated single character caselessly. */ + + case OP_NOTI: + if (clen > 0) + { + unsigned int otherd; +#ifdef SUPPORT_UTF + if (utf && d >= 128) + { +#ifdef SUPPORT_UCP + otherd = UCD_OTHERCASE(d); +#endif /* SUPPORT_UCP */ + } + else +#endif /* SUPPORT_UTF */ + otherd = TABLE_GET(d, fcc, d); + if (c != d && c != otherd) + { ADD_NEW(state_offset + dlen + 1, 0); } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_PLUSI: + case OP_MINPLUSI: + case OP_POSPLUSI: + case OP_NOTPLUSI: + case OP_NOTMINPLUSI: + case OP_NOTPOSPLUSI: + caseless = TRUE; + codevalue -= OP_STARI - OP_STAR; + + /* Fall through */ + case OP_PLUS: + case OP_MINPLUS: + case OP_POSPLUS: + case OP_NOTPLUS: + case OP_NOTMINPLUS: + case OP_NOTPOSPLUS: + count = current_state->count; /* Already matched */ + if (count > 0) { ADD_ACTIVE(state_offset + dlen + 1, 0); } + if (clen > 0) + { + pcre_uint32 otherd = NOTACHAR; + if (caseless) + { +#ifdef SUPPORT_UTF + if (utf && d >= 128) + { +#ifdef SUPPORT_UCP + otherd = UCD_OTHERCASE(d); +#endif /* SUPPORT_UCP */ + } + else +#endif /* SUPPORT_UTF */ + otherd = TABLE_GET(d, fcc, d); + } + if ((c == d || c == otherd) == (codevalue < OP_NOTSTAR)) + { + if (count > 0 && + (codevalue == OP_POSPLUS || codevalue == OP_NOTPOSPLUS)) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + count++; + ADD_NEW(state_offset, count); + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_QUERYI: + case OP_MINQUERYI: + case OP_POSQUERYI: + case OP_NOTQUERYI: + case OP_NOTMINQUERYI: + case OP_NOTPOSQUERYI: + caseless = TRUE; + codevalue -= OP_STARI - OP_STAR; + /* Fall through */ + case OP_QUERY: + case OP_MINQUERY: + case OP_POSQUERY: + case OP_NOTQUERY: + case OP_NOTMINQUERY: + case OP_NOTPOSQUERY: + ADD_ACTIVE(state_offset + dlen + 1, 0); + if (clen > 0) + { + pcre_uint32 otherd = NOTACHAR; + if (caseless) + { +#ifdef SUPPORT_UTF + if (utf && d >= 128) + { +#ifdef SUPPORT_UCP + otherd = UCD_OTHERCASE(d); +#endif /* SUPPORT_UCP */ + } + else +#endif /* SUPPORT_UTF */ + otherd = TABLE_GET(d, fcc, d); + } + if ((c == d || c == otherd) == (codevalue < OP_NOTSTAR)) + { + if (codevalue == OP_POSQUERY || codevalue == OP_NOTPOSQUERY) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + ADD_NEW(state_offset + dlen + 1, 0); + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_STARI: + case OP_MINSTARI: + case OP_POSSTARI: + case OP_NOTSTARI: + case OP_NOTMINSTARI: + case OP_NOTPOSSTARI: + caseless = TRUE; + codevalue -= OP_STARI - OP_STAR; + /* Fall through */ + case OP_STAR: + case OP_MINSTAR: + case OP_POSSTAR: + case OP_NOTSTAR: + case OP_NOTMINSTAR: + case OP_NOTPOSSTAR: + ADD_ACTIVE(state_offset + dlen + 1, 0); + if (clen > 0) + { + pcre_uint32 otherd = NOTACHAR; + if (caseless) + { +#ifdef SUPPORT_UTF + if (utf && d >= 128) + { +#ifdef SUPPORT_UCP + otherd = UCD_OTHERCASE(d); +#endif /* SUPPORT_UCP */ + } + else +#endif /* SUPPORT_UTF */ + otherd = TABLE_GET(d, fcc, d); + } + if ((c == d || c == otherd) == (codevalue < OP_NOTSTAR)) + { + if (codevalue == OP_POSSTAR || codevalue == OP_NOTPOSSTAR) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + ADD_NEW(state_offset, 0); + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_EXACTI: + case OP_NOTEXACTI: + caseless = TRUE; + codevalue -= OP_STARI - OP_STAR; + /* Fall through */ + case OP_EXACT: + case OP_NOTEXACT: + count = current_state->count; /* Number already matched */ + if (clen > 0) + { + pcre_uint32 otherd = NOTACHAR; + if (caseless) + { +#ifdef SUPPORT_UTF + if (utf && d >= 128) + { +#ifdef SUPPORT_UCP + otherd = UCD_OTHERCASE(d); +#endif /* SUPPORT_UCP */ + } + else +#endif /* SUPPORT_UTF */ + otherd = TABLE_GET(d, fcc, d); + } + if ((c == d || c == otherd) == (codevalue < OP_NOTSTAR)) + { + if (++count >= (int)GET2(code, 1)) + { ADD_NEW(state_offset + dlen + 1 + IMM2_SIZE, 0); } + else + { ADD_NEW(state_offset, count); } + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_UPTOI: + case OP_MINUPTOI: + case OP_POSUPTOI: + case OP_NOTUPTOI: + case OP_NOTMINUPTOI: + case OP_NOTPOSUPTOI: + caseless = TRUE; + codevalue -= OP_STARI - OP_STAR; + /* Fall through */ + case OP_UPTO: + case OP_MINUPTO: + case OP_POSUPTO: + case OP_NOTUPTO: + case OP_NOTMINUPTO: + case OP_NOTPOSUPTO: + ADD_ACTIVE(state_offset + dlen + 1 + IMM2_SIZE, 0); + count = current_state->count; /* Number already matched */ + if (clen > 0) + { + pcre_uint32 otherd = NOTACHAR; + if (caseless) + { +#ifdef SUPPORT_UTF + if (utf && d >= 128) + { +#ifdef SUPPORT_UCP + otherd = UCD_OTHERCASE(d); +#endif /* SUPPORT_UCP */ + } + else +#endif /* SUPPORT_UTF */ + otherd = TABLE_GET(d, fcc, d); + } + if ((c == d || c == otherd) == (codevalue < OP_NOTSTAR)) + { + if (codevalue == OP_POSUPTO || codevalue == OP_NOTPOSUPTO) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + if (++count >= (int)GET2(code, 1)) + { ADD_NEW(state_offset + dlen + 1 + IMM2_SIZE, 0); } + else + { ADD_NEW(state_offset, count); } + } + } + break; + + +/* ========================================================================== */ + /* These are the class-handling opcodes */ + + case OP_CLASS: + case OP_NCLASS: + case OP_XCLASS: + { + BOOL isinclass = FALSE; + int next_state_offset; + const pcre_uchar *ecode; + + /* For a simple class, there is always just a 32-byte table, and we + can set isinclass from it. */ + + if (codevalue != OP_XCLASS) + { + ecode = code + 1 + (32 / sizeof(pcre_uchar)); + if (clen > 0) + { + isinclass = (c > 255)? (codevalue == OP_NCLASS) : + ((((pcre_uint8 *)(code + 1))[c/8] & (1 << (c&7))) != 0); + } + } + + /* An extended class may have a table or a list of single characters, + ranges, or both, and it may be positive or negative. There's a + function that sorts all this out. */ + + else + { + ecode = code + GET(code, 1); + if (clen > 0) isinclass = PRIV(xclass)(c, code + 1 + LINK_SIZE, utf); + } + + /* At this point, isinclass is set for all kinds of class, and ecode + points to the byte after the end of the class. If there is a + quantifier, this is where it will be. */ + + next_state_offset = (int)(ecode - start_code); + + switch (*ecode) + { + case OP_CRSTAR: + case OP_CRMINSTAR: + case OP_CRPOSSTAR: + ADD_ACTIVE(next_state_offset + 1, 0); + if (isinclass) + { + if (*ecode == OP_CRPOSSTAR) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + ADD_NEW(state_offset, 0); + } + break; + + case OP_CRPLUS: + case OP_CRMINPLUS: + case OP_CRPOSPLUS: + count = current_state->count; /* Already matched */ + if (count > 0) { ADD_ACTIVE(next_state_offset + 1, 0); } + if (isinclass) + { + if (count > 0 && *ecode == OP_CRPOSPLUS) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + count++; + ADD_NEW(state_offset, count); + } + break; + + case OP_CRQUERY: + case OP_CRMINQUERY: + case OP_CRPOSQUERY: + ADD_ACTIVE(next_state_offset + 1, 0); + if (isinclass) + { + if (*ecode == OP_CRPOSQUERY) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + ADD_NEW(next_state_offset + 1, 0); + } + break; + + case OP_CRRANGE: + case OP_CRMINRANGE: + case OP_CRPOSRANGE: + count = current_state->count; /* Already matched */ + if (count >= (int)GET2(ecode, 1)) + { ADD_ACTIVE(next_state_offset + 1 + 2 * IMM2_SIZE, 0); } + if (isinclass) + { + int max = (int)GET2(ecode, 1 + IMM2_SIZE); + if (*ecode == OP_CRPOSRANGE && count >= (int)GET2(ecode, 1)) + { + active_count--; /* Remove non-match possibility */ + next_active_state--; + } + if (++count >= max && max != 0) /* Max 0 => no limit */ + { ADD_NEW(next_state_offset + 1 + 2 * IMM2_SIZE, 0); } + else + { ADD_NEW(state_offset, count); } + } + break; + + default: + if (isinclass) { ADD_NEW(next_state_offset, 0); } + break; + } + } + break; + +/* ========================================================================== */ + /* These are the opcodes for fancy brackets of various kinds. We have + to use recursion in order to handle them. The "always failing" assertion + (?!) is optimised to OP_FAIL when compiling, so we have to support that, + though the other "backtracking verbs" are not supported. */ + + case OP_FAIL: + forced_fail++; /* Count FAILs for multiple states */ + break; + + case OP_ASSERT: + case OP_ASSERT_NOT: + case OP_ASSERTBACK: + case OP_ASSERTBACK_NOT: + { + int rc; + int local_offsets[2]; + int local_workspace[1000]; + const pcre_uchar *endasscode = code + GET(code, 1); + + while (*endasscode == OP_ALT) endasscode += GET(endasscode, 1); + + rc = internal_dfa_exec( + md, /* static match data */ + code, /* this subexpression's code */ + ptr, /* where we currently are */ + (int)(ptr - start_subject), /* start offset */ + local_offsets, /* offset vector */ + sizeof(local_offsets)/sizeof(int), /* size of same */ + local_workspace, /* workspace vector */ + sizeof(local_workspace)/sizeof(int), /* size of same */ + rlevel); /* function recursion level */ + + if (rc == PCRE_ERROR_DFA_UITEM) return rc; + if ((rc >= 0) == (codevalue == OP_ASSERT || codevalue == OP_ASSERTBACK)) + { ADD_ACTIVE((int)(endasscode + LINK_SIZE + 1 - start_code), 0); } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_COND: + case OP_SCOND: + { + int local_offsets[1000]; + int local_workspace[1000]; + int codelink = GET(code, 1); + int condcode; + + /* Because of the way auto-callout works during compile, a callout item + is inserted between OP_COND and an assertion condition. This does not + happen for the other conditions. */ + + if (code[LINK_SIZE+1] == OP_CALLOUT) + { + rrc = 0; + if (PUBL(callout) != NULL) + { + PUBL(callout_block) cb; + cb.version = 1; /* Version 1 of the callout block */ + cb.callout_number = code[LINK_SIZE+2]; + cb.offset_vector = offsets; +#if defined COMPILE_PCRE8 + cb.subject = (PCRE_SPTR)start_subject; +#elif defined COMPILE_PCRE16 + cb.subject = (PCRE_SPTR16)start_subject; +#elif defined COMPILE_PCRE32 + cb.subject = (PCRE_SPTR32)start_subject; +#endif + cb.subject_length = (int)(end_subject - start_subject); + cb.start_match = (int)(current_subject - start_subject); + cb.current_position = (int)(ptr - start_subject); + cb.pattern_position = GET(code, LINK_SIZE + 3); + cb.next_item_length = GET(code, 3 + 2*LINK_SIZE); + cb.capture_top = 1; + cb.capture_last = -1; + cb.callout_data = md->callout_data; + cb.mark = NULL; /* No (*MARK) support */ + if ((rrc = (*PUBL(callout))(&cb)) < 0) return rrc; /* Abandon */ + } + if (rrc > 0) break; /* Fail this thread */ + code += PRIV(OP_lengths)[OP_CALLOUT]; /* Skip callout data */ + } + + condcode = code[LINK_SIZE+1]; + + /* Back reference conditions and duplicate named recursion conditions + are not supported */ + + if (condcode == OP_CREF || condcode == OP_DNCREF || + condcode == OP_DNRREF) + return PCRE_ERROR_DFA_UCOND; + + /* The DEFINE condition is always false, and the assertion (?!) is + converted to OP_FAIL. */ + + if (condcode == OP_DEF || condcode == OP_FAIL) + { ADD_ACTIVE(state_offset + codelink + LINK_SIZE + 1, 0); } + + /* The only supported version of OP_RREF is for the value RREF_ANY, + which means "test if in any recursion". We can't test for specifically + recursed groups. */ + + else if (condcode == OP_RREF) + { + int value = GET2(code, LINK_SIZE + 2); + if (value != RREF_ANY) return PCRE_ERROR_DFA_UCOND; + if (md->recursive != NULL) + { ADD_ACTIVE(state_offset + LINK_SIZE + 2 + IMM2_SIZE, 0); } + else { ADD_ACTIVE(state_offset + codelink + LINK_SIZE + 1, 0); } + } + + /* Otherwise, the condition is an assertion */ + + else + { + int rc; + const pcre_uchar *asscode = code + LINK_SIZE + 1; + const pcre_uchar *endasscode = asscode + GET(asscode, 1); + + while (*endasscode == OP_ALT) endasscode += GET(endasscode, 1); + + rc = internal_dfa_exec( + md, /* fixed match data */ + asscode, /* this subexpression's code */ + ptr, /* where we currently are */ + (int)(ptr - start_subject), /* start offset */ + local_offsets, /* offset vector */ + sizeof(local_offsets)/sizeof(int), /* size of same */ + local_workspace, /* workspace vector */ + sizeof(local_workspace)/sizeof(int), /* size of same */ + rlevel); /* function recursion level */ + + if (rc == PCRE_ERROR_DFA_UITEM) return rc; + if ((rc >= 0) == + (condcode == OP_ASSERT || condcode == OP_ASSERTBACK)) + { ADD_ACTIVE((int)(endasscode + LINK_SIZE + 1 - start_code), 0); } + else + { ADD_ACTIVE(state_offset + codelink + LINK_SIZE + 1, 0); } + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_RECURSE: + { + dfa_recursion_info *ri; + int local_offsets[1000]; + int local_workspace[1000]; + const pcre_uchar *callpat = start_code + GET(code, 1); + int recno = (callpat == md->start_code)? 0 : + GET2(callpat, 1 + LINK_SIZE); + int rc; + + DPRINTF(("%.*sStarting regex recursion\n", rlevel*2-2, SP)); + + /* Check for repeating a recursion without advancing the subject + pointer. This should catch convoluted mutual recursions. (Some simple + cases are caught at compile time.) */ + + for (ri = md->recursive; ri != NULL; ri = ri->prevrec) + if (recno == ri->group_num && ptr == ri->subject_position) + return PCRE_ERROR_RECURSELOOP; + + /* Remember this recursion and where we started it so as to + catch infinite loops. */ + + new_recursive.group_num = recno; + new_recursive.subject_position = ptr; + new_recursive.prevrec = md->recursive; + md->recursive = &new_recursive; + + rc = internal_dfa_exec( + md, /* fixed match data */ + callpat, /* this subexpression's code */ + ptr, /* where we currently are */ + (int)(ptr - start_subject), /* start offset */ + local_offsets, /* offset vector */ + sizeof(local_offsets)/sizeof(int), /* size of same */ + local_workspace, /* workspace vector */ + sizeof(local_workspace)/sizeof(int), /* size of same */ + rlevel); /* function recursion level */ + + md->recursive = new_recursive.prevrec; /* Done this recursion */ + + DPRINTF(("%.*sReturn from regex recursion: rc=%d\n", rlevel*2-2, SP, + rc)); + + /* Ran out of internal offsets */ + + if (rc == 0) return PCRE_ERROR_DFA_RECURSE; + + /* For each successful matched substring, set up the next state with a + count of characters to skip before trying it. Note that the count is in + characters, not bytes. */ + + if (rc > 0) + { + for (rc = rc*2 - 2; rc >= 0; rc -= 2) + { + int charcount = local_offsets[rc+1] - local_offsets[rc]; +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (utf) + { + const pcre_uchar *p = start_subject + local_offsets[rc]; + const pcre_uchar *pp = start_subject + local_offsets[rc+1]; + while (p < pp) if (NOT_FIRSTCHAR(*p++)) charcount--; + } +#endif + if (charcount > 0) + { + ADD_NEW_DATA(-(state_offset + LINK_SIZE + 1), 0, (charcount - 1)); + } + else + { + ADD_ACTIVE(state_offset + LINK_SIZE + 1, 0); + } + } + } + else if (rc != PCRE_ERROR_NOMATCH) return rc; + } + break; + + /*-----------------------------------------------------------------*/ + case OP_BRAPOS: + case OP_SBRAPOS: + case OP_CBRAPOS: + case OP_SCBRAPOS: + case OP_BRAPOSZERO: + { + int charcount, matched_count; + const pcre_uchar *local_ptr = ptr; + BOOL allow_zero; + + if (codevalue == OP_BRAPOSZERO) + { + allow_zero = TRUE; + codevalue = *(++code); /* Codevalue will be one of above BRAs */ + } + else allow_zero = FALSE; + + /* Loop to match the subpattern as many times as possible as if it were + a complete pattern. */ + + for (matched_count = 0;; matched_count++) + { + int local_offsets[2]; + int local_workspace[1000]; + + int rc = internal_dfa_exec( + md, /* fixed match data */ + code, /* this subexpression's code */ + local_ptr, /* where we currently are */ + (int)(ptr - start_subject), /* start offset */ + local_offsets, /* offset vector */ + sizeof(local_offsets)/sizeof(int), /* size of same */ + local_workspace, /* workspace vector */ + sizeof(local_workspace)/sizeof(int), /* size of same */ + rlevel); /* function recursion level */ + + /* Failed to match */ + + if (rc < 0) + { + if (rc != PCRE_ERROR_NOMATCH) return rc; + break; + } + + /* Matched: break the loop if zero characters matched. */ + + charcount = local_offsets[1] - local_offsets[0]; + if (charcount == 0) break; + local_ptr += charcount; /* Advance temporary position ptr */ + } + + /* At this point we have matched the subpattern matched_count + times, and local_ptr is pointing to the character after the end of the + last match. */ + + if (matched_count > 0 || allow_zero) + { + const pcre_uchar *end_subpattern = code; + int next_state_offset; + + do { end_subpattern += GET(end_subpattern, 1); } + while (*end_subpattern == OP_ALT); + next_state_offset = + (int)(end_subpattern - start_code + LINK_SIZE + 1); + + /* Optimization: if there are no more active states, and there + are no new states yet set up, then skip over the subject string + right here, to save looping. Otherwise, set up the new state to swing + into action when the end of the matched substring is reached. */ + + if (i + 1 >= active_count && new_count == 0) + { + ptr = local_ptr; + clen = 0; + ADD_NEW(next_state_offset, 0); + } + else + { + const pcre_uchar *p = ptr; + const pcre_uchar *pp = local_ptr; + charcount = (int)(pp - p); +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (utf) while (p < pp) if (NOT_FIRSTCHAR(*p++)) charcount--; +#endif + ADD_NEW_DATA(-next_state_offset, 0, (charcount - 1)); + } + } + } + break; + + /*-----------------------------------------------------------------*/ + case OP_ONCE: + case OP_ONCE_NC: + { + int local_offsets[2]; + int local_workspace[1000]; + + int rc = internal_dfa_exec( + md, /* fixed match data */ + code, /* this subexpression's code */ + ptr, /* where we currently are */ + (int)(ptr - start_subject), /* start offset */ + local_offsets, /* offset vector */ + sizeof(local_offsets)/sizeof(int), /* size of same */ + local_workspace, /* workspace vector */ + sizeof(local_workspace)/sizeof(int), /* size of same */ + rlevel); /* function recursion level */ + + if (rc >= 0) + { + const pcre_uchar *end_subpattern = code; + int charcount = local_offsets[1] - local_offsets[0]; + int next_state_offset, repeat_state_offset; + + do { end_subpattern += GET(end_subpattern, 1); } + while (*end_subpattern == OP_ALT); + next_state_offset = + (int)(end_subpattern - start_code + LINK_SIZE + 1); + + /* If the end of this subpattern is KETRMAX or KETRMIN, we must + arrange for the repeat state also to be added to the relevant list. + Calculate the offset, or set -1 for no repeat. */ + + repeat_state_offset = (*end_subpattern == OP_KETRMAX || + *end_subpattern == OP_KETRMIN)? + (int)(end_subpattern - start_code - GET(end_subpattern, 1)) : -1; + + /* If we have matched an empty string, add the next state at the + current character pointer. This is important so that the duplicate + checking kicks in, which is what breaks infinite loops that match an + empty string. */ + + if (charcount == 0) + { + ADD_ACTIVE(next_state_offset, 0); + } + + /* Optimization: if there are no more active states, and there + are no new states yet set up, then skip over the subject string + right here, to save looping. Otherwise, set up the new state to swing + into action when the end of the matched substring is reached. */ + + else if (i + 1 >= active_count && new_count == 0) + { + ptr += charcount; + clen = 0; + ADD_NEW(next_state_offset, 0); + + /* If we are adding a repeat state at the new character position, + we must fudge things so that it is the only current state. + Otherwise, it might be a duplicate of one we processed before, and + that would cause it to be skipped. */ + + if (repeat_state_offset >= 0) + { + next_active_state = active_states; + active_count = 0; + i = -1; + ADD_ACTIVE(repeat_state_offset, 0); + } + } + else + { +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (utf) + { + const pcre_uchar *p = start_subject + local_offsets[0]; + const pcre_uchar *pp = start_subject + local_offsets[1]; + while (p < pp) if (NOT_FIRSTCHAR(*p++)) charcount--; + } +#endif + ADD_NEW_DATA(-next_state_offset, 0, (charcount - 1)); + if (repeat_state_offset >= 0) + { ADD_NEW_DATA(-repeat_state_offset, 0, (charcount - 1)); } + } + } + else if (rc != PCRE_ERROR_NOMATCH) return rc; + } + break; + + +/* ========================================================================== */ + /* Handle callouts */ + + case OP_CALLOUT: + rrc = 0; + if (PUBL(callout) != NULL) + { + PUBL(callout_block) cb; + cb.version = 1; /* Version 1 of the callout block */ + cb.callout_number = code[1]; + cb.offset_vector = offsets; +#if defined COMPILE_PCRE8 + cb.subject = (PCRE_SPTR)start_subject; +#elif defined COMPILE_PCRE16 + cb.subject = (PCRE_SPTR16)start_subject; +#elif defined COMPILE_PCRE32 + cb.subject = (PCRE_SPTR32)start_subject; +#endif + cb.subject_length = (int)(end_subject - start_subject); + cb.start_match = (int)(current_subject - start_subject); + cb.current_position = (int)(ptr - start_subject); + cb.pattern_position = GET(code, 2); + cb.next_item_length = GET(code, 2 + LINK_SIZE); + cb.capture_top = 1; + cb.capture_last = -1; + cb.callout_data = md->callout_data; + cb.mark = NULL; /* No (*MARK) support */ + if ((rrc = (*PUBL(callout))(&cb)) < 0) return rrc; /* Abandon */ + } + if (rrc == 0) + { ADD_ACTIVE(state_offset + PRIV(OP_lengths)[OP_CALLOUT], 0); } + break; + + +/* ========================================================================== */ + default: /* Unsupported opcode */ + return PCRE_ERROR_DFA_UITEM; + } + + NEXT_ACTIVE_STATE: continue; + + } /* End of loop scanning active states */ + + /* We have finished the processing at the current subject character. If no + new states have been set for the next character, we have found all the + matches that we are going to find. If we are at the top level and partial + matching has been requested, check for appropriate conditions. + + The "forced_ fail" variable counts the number of (*F) encountered for the + character. If it is equal to the original active_count (saved in + workspace[1]) it means that (*F) was found on every active state. In this + case we don't want to give a partial match. + + The "could_continue" variable is true if a state could have continued but + for the fact that the end of the subject was reached. */ + + if (new_count <= 0) + { + if (rlevel == 1 && /* Top level, and */ + could_continue && /* Some could go on, and */ + forced_fail != workspace[1] && /* Not all forced fail & */ + ( /* either... */ + (md->moptions & PCRE_PARTIAL_HARD) != 0 /* Hard partial */ + || /* or... */ + ((md->moptions & PCRE_PARTIAL_SOFT) != 0 && /* Soft partial and */ + match_count < 0) /* no matches */ + ) && /* And... */ + ( + partial_newline || /* Either partial NL */ + ( /* or ... */ + ptr >= end_subject && /* End of subject and */ + ptr > md->start_used_ptr) /* Inspected non-empty string */ + ) + ) + match_count = PCRE_ERROR_PARTIAL; + DPRINTF(("%.*sEnd of internal_dfa_exec %d: returning %d\n" + "%.*s---------------------\n\n", rlevel*2-2, SP, rlevel, match_count, + rlevel*2-2, SP)); + break; /* In effect, "return", but see the comment below */ + } + + /* One or more states are active for the next character. */ + + ptr += clen; /* Advance to next subject character */ + } /* Loop to move along the subject string */ + +/* Control gets here from "break" a few lines above. We do it this way because +if we use "return" above, we have compiler trouble. Some compilers warn if +there's nothing here because they think the function doesn't return a value. On +the other hand, if we put a dummy statement here, some more clever compilers +complain that it can't be reached. Sigh. */ + +return match_count; +} + + + + +/************************************************* +* Execute a Regular Expression - DFA engine * +*************************************************/ + +/* This external function applies a compiled re to a subject string using a DFA +engine. This function calls the internal function multiple times if the pattern +is not anchored. + +Arguments: + argument_re points to the compiled expression + extra_data points to extra data or is NULL + subject points to the subject string + length length of subject string (may contain binary zeros) + start_offset where to start in the subject string + options option bits + offsets vector of match offsets + offsetcount size of same + workspace workspace vector + wscount size of same + +Returns: > 0 => number of match offset pairs placed in offsets + = 0 => offsets overflowed; longest matches are present + -1 => failed to match + < -1 => some kind of unexpected problem +*/ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre_dfa_exec(const pcre *argument_re, const pcre_extra *extra_data, + const char *subject, int length, int start_offset, int options, int *offsets, + int offsetcount, int *workspace, int wscount) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre16_dfa_exec(const pcre16 *argument_re, const pcre16_extra *extra_data, + PCRE_SPTR16 subject, int length, int start_offset, int options, int *offsets, + int offsetcount, int *workspace, int wscount) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre32_dfa_exec(const pcre32 *argument_re, const pcre32_extra *extra_data, + PCRE_SPTR32 subject, int length, int start_offset, int options, int *offsets, + int offsetcount, int *workspace, int wscount) +#endif +{ +REAL_PCRE *re = (REAL_PCRE *)argument_re; +dfa_match_data match_block; +dfa_match_data *md = &match_block; +BOOL utf, anchored, startline, firstline; +const pcre_uchar *current_subject, *end_subject; +const pcre_study_data *study = NULL; + +const pcre_uchar *req_char_ptr; +const pcre_uint8 *start_bits = NULL; +BOOL has_first_char = FALSE; +BOOL has_req_char = FALSE; +pcre_uchar first_char = 0; +pcre_uchar first_char2 = 0; +pcre_uchar req_char = 0; +pcre_uchar req_char2 = 0; +int newline; + +/* Plausibility checks */ + +if ((options & ~PUBLIC_DFA_EXEC_OPTIONS) != 0) return PCRE_ERROR_BADOPTION; +if (re == NULL || subject == NULL || workspace == NULL || + (offsets == NULL && offsetcount > 0)) return PCRE_ERROR_NULL; +if (offsetcount < 0) return PCRE_ERROR_BADCOUNT; +if (wscount < 20) return PCRE_ERROR_DFA_WSSIZE; +if (length < 0) return PCRE_ERROR_BADLENGTH; +if (start_offset < 0 || start_offset > length) return PCRE_ERROR_BADOFFSET; + +/* Check that the first field in the block is the magic number. If it is not, +return with PCRE_ERROR_BADMAGIC. However, if the magic number is equal to +REVERSED_MAGIC_NUMBER we return with PCRE_ERROR_BADENDIANNESS, which +means that the pattern is likely compiled with different endianness. */ + +if (re->magic_number != MAGIC_NUMBER) + return re->magic_number == REVERSED_MAGIC_NUMBER? + PCRE_ERROR_BADENDIANNESS:PCRE_ERROR_BADMAGIC; +if ((re->flags & PCRE_MODE) == 0) return PCRE_ERROR_BADMODE; + +/* If restarting after a partial match, do some sanity checks on the contents +of the workspace. */ + +if ((options & PCRE_DFA_RESTART) != 0) + { + if ((workspace[0] & (-2)) != 0 || workspace[1] < 1 || + workspace[1] > (wscount - 2)/INTS_PER_STATEBLOCK) + return PCRE_ERROR_DFA_BADRESTART; + } + +/* Set up study, callout, and table data */ + +md->tables = re->tables; +md->callout_data = NULL; + +if (extra_data != NULL) + { + unsigned long int flags = extra_data->flags; + if ((flags & PCRE_EXTRA_STUDY_DATA) != 0) + study = (const pcre_study_data *)extra_data->study_data; + if ((flags & PCRE_EXTRA_MATCH_LIMIT) != 0) return PCRE_ERROR_DFA_UMLIMIT; + if ((flags & PCRE_EXTRA_MATCH_LIMIT_RECURSION) != 0) + return PCRE_ERROR_DFA_UMLIMIT; + if ((flags & PCRE_EXTRA_CALLOUT_DATA) != 0) + md->callout_data = extra_data->callout_data; + if ((flags & PCRE_EXTRA_TABLES) != 0) + md->tables = extra_data->tables; + } + +/* Set some local values */ + +current_subject = (const pcre_uchar *)subject + start_offset; +end_subject = (const pcre_uchar *)subject + length; +req_char_ptr = current_subject - 1; + +#ifdef SUPPORT_UTF +/* PCRE_UTF(16|32) have the same value as PCRE_UTF8. */ +utf = (re->options & PCRE_UTF8) != 0; +#else +utf = FALSE; +#endif + +anchored = (options & (PCRE_ANCHORED|PCRE_DFA_RESTART)) != 0 || + (re->options & PCRE_ANCHORED) != 0; + +/* The remaining fixed data for passing around. */ + +md->start_code = (const pcre_uchar *)argument_re + + re->name_table_offset + re->name_count * re->name_entry_size; +md->start_subject = (const pcre_uchar *)subject; +md->end_subject = end_subject; +md->start_offset = start_offset; +md->moptions = options; +md->poptions = re->options; + +/* If the BSR option is not set at match time, copy what was set +at compile time. */ + +if ((md->moptions & (PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE)) == 0) + { + if ((re->options & (PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE)) != 0) + md->moptions |= re->options & (PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE); +#ifdef BSR_ANYCRLF + else md->moptions |= PCRE_BSR_ANYCRLF; +#endif + } + +/* Handle different types of newline. The three bits give eight cases. If +nothing is set at run time, whatever was used at compile time applies. */ + +switch ((((options & PCRE_NEWLINE_BITS) == 0)? re->options : (pcre_uint32)options) & + PCRE_NEWLINE_BITS) + { + case 0: newline = NEWLINE; break; /* Compile-time default */ + case PCRE_NEWLINE_CR: newline = CHAR_CR; break; + case PCRE_NEWLINE_LF: newline = CHAR_NL; break; + case PCRE_NEWLINE_CR+ + PCRE_NEWLINE_LF: newline = (CHAR_CR << 8) | CHAR_NL; break; + case PCRE_NEWLINE_ANY: newline = -1; break; + case PCRE_NEWLINE_ANYCRLF: newline = -2; break; + default: return PCRE_ERROR_BADNEWLINE; + } + +if (newline == -2) + { + md->nltype = NLTYPE_ANYCRLF; + } +else if (newline < 0) + { + md->nltype = NLTYPE_ANY; + } +else + { + md->nltype = NLTYPE_FIXED; + if (newline > 255) + { + md->nllen = 2; + md->nl[0] = (newline >> 8) & 255; + md->nl[1] = newline & 255; + } + else + { + md->nllen = 1; + md->nl[0] = newline; + } + } + +/* Check a UTF-8 string if required. Unfortunately there's no way of passing +back the character offset. */ + +#ifdef SUPPORT_UTF +if (utf && (options & PCRE_NO_UTF8_CHECK) == 0) + { + int erroroffset; + int errorcode = PRIV(valid_utf)((pcre_uchar *)subject, length, &erroroffset); + if (errorcode != 0) + { + if (offsetcount >= 2) + { + offsets[0] = erroroffset; + offsets[1] = errorcode; + } +#if defined COMPILE_PCRE8 + return (errorcode <= PCRE_UTF8_ERR5 && (options & PCRE_PARTIAL_HARD) != 0) ? + PCRE_ERROR_SHORTUTF8 : PCRE_ERROR_BADUTF8; +#elif defined COMPILE_PCRE16 + return (errorcode <= PCRE_UTF16_ERR1 && (options & PCRE_PARTIAL_HARD) != 0) ? + PCRE_ERROR_SHORTUTF16 : PCRE_ERROR_BADUTF16; +#elif defined COMPILE_PCRE32 + return PCRE_ERROR_BADUTF32; +#endif + } +#if defined COMPILE_PCRE8 || defined COMPILE_PCRE16 + if (start_offset > 0 && start_offset < length && + NOT_FIRSTCHAR(((PCRE_PUCHAR)subject)[start_offset])) + return PCRE_ERROR_BADUTF8_OFFSET; +#endif + } +#endif + +/* If the exec call supplied NULL for tables, use the inbuilt ones. This +is a feature that makes it possible to save compiled regex and re-use them +in other programs later. */ + +if (md->tables == NULL) md->tables = PRIV(default_tables); + +/* The "must be at the start of a line" flags are used in a loop when finding +where to start. */ + +startline = (re->flags & PCRE_STARTLINE) != 0; +firstline = (re->options & PCRE_FIRSTLINE) != 0; + +/* Set up the first character to match, if available. The first_byte value is +never set for an anchored regular expression, but the anchoring may be forced +at run time, so we have to test for anchoring. The first char may be unset for +an unanchored pattern, of course. If there's no first char and the pattern was +studied, there may be a bitmap of possible first characters. */ + +if (!anchored) + { + if ((re->flags & PCRE_FIRSTSET) != 0) + { + has_first_char = TRUE; + first_char = first_char2 = (pcre_uchar)(re->first_char); + if ((re->flags & PCRE_FCH_CASELESS) != 0) + { + first_char2 = TABLE_GET(first_char, md->tables + fcc_offset, first_char); +#if defined SUPPORT_UCP && !(defined COMPILE_PCRE8) + if (utf && first_char > 127) + first_char2 = UCD_OTHERCASE(first_char); +#endif + } + } + else + { + if (!startline && study != NULL && + (study->flags & PCRE_STUDY_MAPPED) != 0) + start_bits = study->start_bits; + } + } + +/* For anchored or unanchored matches, there may be a "last known required +character" set. */ + +if ((re->flags & PCRE_REQCHSET) != 0) + { + has_req_char = TRUE; + req_char = req_char2 = (pcre_uchar)(re->req_char); + if ((re->flags & PCRE_RCH_CASELESS) != 0) + { + req_char2 = TABLE_GET(req_char, md->tables + fcc_offset, req_char); +#if defined SUPPORT_UCP && !(defined COMPILE_PCRE8) + if (utf && req_char > 127) + req_char2 = UCD_OTHERCASE(req_char); +#endif + } + } + +/* Call the main matching function, looping for a non-anchored regex after a +failed match. If not restarting, perform certain optimizations at the start of +a match. */ + +for (;;) + { + int rc; + + if ((options & PCRE_DFA_RESTART) == 0) + { + const pcre_uchar *save_end_subject = end_subject; + + /* If firstline is TRUE, the start of the match is constrained to the first + line of a multiline string. Implement this by temporarily adjusting + end_subject so that we stop scanning at a newline. If the match fails at + the newline, later code breaks this loop. */ + + if (firstline) + { + PCRE_PUCHAR t = current_subject; +#ifdef SUPPORT_UTF + if (utf) + { + while (t < md->end_subject && !IS_NEWLINE(t)) + { + t++; + ACROSSCHAR(t < end_subject, *t, t++); + } + } + else +#endif + while (t < md->end_subject && !IS_NEWLINE(t)) t++; + end_subject = t; + } + + /* There are some optimizations that avoid running the match if a known + starting point is not found. However, there is an option that disables + these, for testing and for ensuring that all callouts do actually occur. + The option can be set in the regex by (*NO_START_OPT) or passed in + match-time options. */ + + if (((options | re->options) & PCRE_NO_START_OPTIMIZE) == 0) + { + /* Advance to a known first pcre_uchar (i.e. data item) */ + + if (has_first_char) + { + if (first_char != first_char2) + { + pcre_uchar csc; + while (current_subject < end_subject && + (csc = UCHAR21TEST(current_subject)) != first_char && csc != first_char2) + current_subject++; + } + else + while (current_subject < end_subject && + UCHAR21TEST(current_subject) != first_char) + current_subject++; + } + + /* Or to just after a linebreak for a multiline match if possible */ + + else if (startline) + { + if (current_subject > md->start_subject + start_offset) + { +#ifdef SUPPORT_UTF + if (utf) + { + while (current_subject < end_subject && + !WAS_NEWLINE(current_subject)) + { + current_subject++; + ACROSSCHAR(current_subject < end_subject, *current_subject, + current_subject++); + } + } + else +#endif + while (current_subject < end_subject && !WAS_NEWLINE(current_subject)) + current_subject++; + + /* If we have just passed a CR and the newline option is ANY or + ANYCRLF, and we are now at a LF, advance the match position by one + more character. */ + + if (UCHAR21TEST(current_subject - 1) == CHAR_CR && + (md->nltype == NLTYPE_ANY || md->nltype == NLTYPE_ANYCRLF) && + current_subject < end_subject && + UCHAR21TEST(current_subject) == CHAR_NL) + current_subject++; + } + } + + /* Advance to a non-unique first pcre_uchar after study */ + + else if (start_bits != NULL) + { + while (current_subject < end_subject) + { + register pcre_uint32 c = UCHAR21TEST(current_subject); +#ifndef COMPILE_PCRE8 + if (c > 255) c = 255; +#endif + if ((start_bits[c/8] & (1 << (c&7))) != 0) break; + current_subject++; + } + } + } + + /* Restore fudged end_subject */ + + end_subject = save_end_subject; + + /* The following two optimizations are disabled for partial matching or if + disabling is explicitly requested (and of course, by the test above, this + code is not obeyed when restarting after a partial match). */ + + if (((options | re->options) & PCRE_NO_START_OPTIMIZE) == 0 && + (options & (PCRE_PARTIAL_HARD|PCRE_PARTIAL_SOFT)) == 0) + { + /* If the pattern was studied, a minimum subject length may be set. This + is a lower bound; no actual string of that length may actually match the + pattern. Although the value is, strictly, in characters, we treat it as + in pcre_uchar units to avoid spending too much time in this optimization. + */ + + if (study != NULL && (study->flags & PCRE_STUDY_MINLEN) != 0 && + (pcre_uint32)(end_subject - current_subject) < study->minlength) + return PCRE_ERROR_NOMATCH; + + /* If req_char is set, we know that that pcre_uchar must appear in the + subject for the match to succeed. If the first pcre_uchar is set, + req_char must be later in the subject; otherwise the test starts at the + match point. This optimization can save a huge amount of work in patterns + with nested unlimited repeats that aren't going to match. Writing + separate code for cased/caseless versions makes it go faster, as does + using an autoincrement and backing off on a match. + + HOWEVER: when the subject string is very, very long, searching to its end + can take a long time, and give bad performance on quite ordinary + patterns. This showed up when somebody was matching /^C/ on a 32-megabyte + string... so we don't do this when the string is sufficiently long. */ + + if (has_req_char && end_subject - current_subject < REQ_BYTE_MAX) + { + register PCRE_PUCHAR p = current_subject + (has_first_char? 1:0); + + /* We don't need to repeat the search if we haven't yet reached the + place we found it at last time. */ + + if (p > req_char_ptr) + { + if (req_char != req_char2) + { + while (p < end_subject) + { + register pcre_uint32 pp = UCHAR21INCTEST(p); + if (pp == req_char || pp == req_char2) { p--; break; } + } + } + else + { + while (p < end_subject) + { + if (UCHAR21INCTEST(p) == req_char) { p--; break; } + } + } + + /* If we can't find the required pcre_uchar, break the matching loop, + which will cause a return or PCRE_ERROR_NOMATCH. */ + + if (p >= end_subject) break; + + /* If we have found the required pcre_uchar, save the point where we + found it, so that we don't search again next time round the loop if + the start hasn't passed this point yet. */ + + req_char_ptr = p; + } + } + } + } /* End of optimizations that are done when not restarting */ + + /* OK, now we can do the business */ + + md->start_used_ptr = current_subject; + md->recursive = NULL; + + rc = internal_dfa_exec( + md, /* fixed match data */ + md->start_code, /* this subexpression's code */ + current_subject, /* where we currently are */ + start_offset, /* start offset in subject */ + offsets, /* offset vector */ + offsetcount, /* size of same */ + workspace, /* workspace vector */ + wscount, /* size of same */ + 0); /* function recurse level */ + + /* Anything other than "no match" means we are done, always; otherwise, carry + on only if not anchored. */ + + if (rc != PCRE_ERROR_NOMATCH || anchored) + { + if (rc == PCRE_ERROR_PARTIAL && offsetcount >= 2) + { + offsets[0] = (int)(md->start_used_ptr - (PCRE_PUCHAR)subject); + offsets[1] = (int)(end_subject - (PCRE_PUCHAR)subject); + if (offsetcount > 2) + offsets[2] = (int)(current_subject - (PCRE_PUCHAR)subject); + } + return rc; + } + + /* Advance to the next subject character unless we are at the end of a line + and firstline is set. */ + + if (firstline && IS_NEWLINE(current_subject)) break; + current_subject++; +#ifdef SUPPORT_UTF + if (utf) + { + ACROSSCHAR(current_subject < end_subject, *current_subject, + current_subject++); + } +#endif + if (current_subject > end_subject) break; + + /* If we have just passed a CR and we are now at a LF, and the pattern does + not contain any explicit matches for \r or \n, and the newline option is CRLF + or ANY or ANYCRLF, advance the match position by one more character. */ + + if (UCHAR21TEST(current_subject - 1) == CHAR_CR && + current_subject < end_subject && + UCHAR21TEST(current_subject) == CHAR_NL && + (re->flags & PCRE_HASCRORLF) == 0 && + (md->nltype == NLTYPE_ANY || + md->nltype == NLTYPE_ANYCRLF || + md->nllen == 2)) + current_subject++; + + } /* "Bumpalong" loop */ + +return PCRE_ERROR_NOMATCH; +} + +/* End of pcre_dfa_exec.c */ diff --git a/src/third_party/pcre-8.41/pcre_exec.c b/src/third_party/pcre-8.41/pcre_exec.c new file mode 100644 index 00000000000..1a9bdd546ee --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_exec.c @@ -0,0 +1,7173 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2014 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* This module contains pcre_exec(), the externally visible function that does +pattern matching using an NFA algorithm, trying to mimic Perl as closely as +possible. There are also some static supporting functions. */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define NLBLOCK md /* Block containing newline information */ +#define PSSTART start_subject /* Field containing processed string start */ +#define PSEND end_subject /* Field containing processed string end */ + +#include "pcre_internal.h" + +/* Undefine some potentially clashing cpp symbols */ + +#undef min +#undef max + +/* The md->capture_last field uses the lower 16 bits for the last captured +substring (which can never be greater than 65535) and a bit in the top half +to mean "capture vector overflowed". This odd way of doing things was +implemented when it was realized that preserving and restoring the overflow bit +whenever the last capture number was saved/restored made for a neater +interface, and doing it this way saved on (a) another variable, which would +have increased the stack frame size (a big NO-NO in PCRE) and (b) another +separate set of save/restore instructions. The following defines are used in +implementing this. */ + +#define CAPLMASK 0x0000ffff /* The bits used for last_capture */ +#define OVFLMASK 0xffff0000 /* The bits used for the overflow flag */ +#define OVFLBIT 0x00010000 /* The bit that is set for overflow */ + +/* Values for setting in md->match_function_type to indicate two special types +of call to match(). We do it this way to save on using another stack variable, +as stack usage is to be discouraged. */ + +#define MATCH_CONDASSERT 1 /* Called to check a condition assertion */ +#define MATCH_CBEGROUP 2 /* Could-be-empty unlimited repeat group */ + +/* Non-error returns from the match() function. Error returns are externally +defined PCRE_ERROR_xxx codes, which are all negative. */ + +#define MATCH_MATCH 1 +#define MATCH_NOMATCH 0 + +/* Special internal returns from the match() function. Make them sufficiently +negative to avoid the external error codes. */ + +#define MATCH_ACCEPT (-999) +#define MATCH_KETRPOS (-998) +#define MATCH_ONCE (-997) +/* The next 5 must be kept together and in sequence so that a test that checks +for any one of them can use a range. */ +#define MATCH_COMMIT (-996) +#define MATCH_PRUNE (-995) +#define MATCH_SKIP (-994) +#define MATCH_SKIP_ARG (-993) +#define MATCH_THEN (-992) +#define MATCH_BACKTRACK_MAX MATCH_THEN +#define MATCH_BACKTRACK_MIN MATCH_COMMIT + +/* Maximum number of ints of offset to save on the stack for recursive calls. +If the offset vector is bigger, malloc is used. This should be a multiple of 3, +because the offset vector is always a multiple of 3 long. */ + +#define REC_STACK_SAVE_MAX 30 + +/* Min and max values for the common repeats; for the maxima, 0 => infinity */ + +static const char rep_min[] = { 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, }; +static const char rep_max[] = { 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, }; + +#ifdef PCRE_DEBUG +/************************************************* +* Debugging function to print chars * +*************************************************/ + +/* Print a sequence of chars in printable format, stopping at the end of the +subject if the requested. + +Arguments: + p points to characters + length number to print + is_subject TRUE if printing from within md->start_subject + md pointer to matching data block, if is_subject is TRUE + +Returns: nothing +*/ + +static void +pchars(const pcre_uchar *p, int length, BOOL is_subject, match_data *md) +{ +pcre_uint32 c; +BOOL utf = md->utf; +if (is_subject && length > md->end_subject - p) length = md->end_subject - p; +while (length-- > 0) + if (isprint(c = UCHAR21INCTEST(p))) printf("%c", (char)c); else printf("\\x{%02x}", c); +} +#endif + + + +/************************************************* +* Match a back-reference * +*************************************************/ + +/* Normally, if a back reference hasn't been set, the length that is passed is +negative, so the match always fails. However, in JavaScript compatibility mode, +the length passed is zero. Note that in caseless UTF-8 mode, the number of +subject bytes matched may be different to the number of reference bytes. + +Arguments: + offset index into the offset vector + eptr pointer into the subject + length length of reference to be matched (number of bytes) + md points to match data block + caseless TRUE if caseless + +Returns: >= 0 the number of subject bytes matched + -1 no match + -2 partial match; always given if at end subject +*/ + +static int +match_ref(int offset, register PCRE_PUCHAR eptr, int length, match_data *md, + BOOL caseless) +{ +PCRE_PUCHAR eptr_start = eptr; +register PCRE_PUCHAR p = md->start_subject + md->offset_vector[offset]; +#if defined SUPPORT_UTF && defined SUPPORT_UCP +BOOL utf = md->utf; +#endif + +#ifdef PCRE_DEBUG +if (eptr >= md->end_subject) + printf("matching subject "); +else + { + printf("matching subject "); + pchars(eptr, length, TRUE, md); + } +printf(" against backref "); +pchars(p, length, FALSE, md); +printf("\n"); +#endif + +/* Always fail if reference not set (and not JavaScript compatible - in that +case the length is passed as zero). */ + +if (length < 0) return -1; + +/* Separate the caseless case for speed. In UTF-8 mode we can only do this +properly if Unicode properties are supported. Otherwise, we can check only +ASCII characters. */ + +if (caseless) + { +#if defined SUPPORT_UTF && defined SUPPORT_UCP + if (utf) + { + /* Match characters up to the end of the reference. NOTE: the number of + data units matched may differ, because in UTF-8 there are some characters + whose upper and lower case versions code have different numbers of bytes. + For example, U+023A (2 bytes in UTF-8) is the upper case version of U+2C65 + (3 bytes in UTF-8); a sequence of 3 of the former uses 6 bytes, as does a + sequence of two of the latter. It is important, therefore, to check the + length along the reference, not along the subject (earlier code did this + wrong). */ + + PCRE_PUCHAR endptr = p + length; + while (p < endptr) + { + pcre_uint32 c, d; + const ucd_record *ur; + if (eptr >= md->end_subject) return -2; /* Partial match */ + GETCHARINC(c, eptr); + GETCHARINC(d, p); + ur = GET_UCD(d); + if (c != d && c != d + ur->other_case) + { + const pcre_uint32 *pp = PRIV(ucd_caseless_sets) + ur->caseset; + for (;;) + { + if (c < *pp) return -1; + if (c == *pp++) break; + } + } + } + } + else +#endif + + /* The same code works when not in UTF-8 mode and in UTF-8 mode when there + is no UCP support. */ + { + while (length-- > 0) + { + pcre_uint32 cc, cp; + if (eptr >= md->end_subject) return -2; /* Partial match */ + cc = UCHAR21TEST(eptr); + cp = UCHAR21TEST(p); + if (TABLE_GET(cp, md->lcc, cp) != TABLE_GET(cc, md->lcc, cc)) return -1; + p++; + eptr++; + } + } + } + +/* In the caseful case, we can just compare the bytes, whether or not we +are in UTF-8 mode. */ + +else + { + while (length-- > 0) + { + if (eptr >= md->end_subject) return -2; /* Partial match */ + if (UCHAR21INCTEST(p) != UCHAR21INCTEST(eptr)) return -1; + } + } + +return (int)(eptr - eptr_start); +} + + + +/*************************************************************************** +**************************************************************************** + RECURSION IN THE match() FUNCTION + +The match() function is highly recursive, though not every recursive call +increases the recursive depth. Nevertheless, some regular expressions can cause +it to recurse to a great depth. I was writing for Unix, so I just let it call +itself recursively. This uses the stack for saving everything that has to be +saved for a recursive call. On Unix, the stack can be large, and this works +fine. + +It turns out that on some non-Unix-like systems there are problems with +programs that use a lot of stack. (This despite the fact that every last chip +has oodles of memory these days, and techniques for extending the stack have +been known for decades.) So.... + +There is a fudge, triggered by defining NO_RECURSE, which avoids recursive +calls by keeping local variables that need to be preserved in blocks of memory +obtained from malloc() instead instead of on the stack. Macros are used to +achieve this so that the actual code doesn't look very different to what it +always used to. + +The original heap-recursive code used longjmp(). However, it seems that this +can be very slow on some operating systems. Following a suggestion from Stan +Switzer, the use of longjmp() has been abolished, at the cost of having to +provide a unique number for each call to RMATCH. There is no way of generating +a sequence of numbers at compile time in C. I have given them names, to make +them stand out more clearly. + +Crude tests on x86 Linux show a small speedup of around 5-8%. However, on +FreeBSD, avoiding longjmp() more than halves the time taken to run the standard +tests. Furthermore, not using longjmp() means that local dynamic variables +don't have indeterminate values; this has meant that the frame size can be +reduced because the result can be "passed back" by straight setting of the +variable instead of being passed in the frame. +**************************************************************************** +***************************************************************************/ + +/* Numbers for RMATCH calls. When this list is changed, the code at HEAP_RETURN +below must be updated in sync. */ + +enum { RM1=1, RM2, RM3, RM4, RM5, RM6, RM7, RM8, RM9, RM10, + RM11, RM12, RM13, RM14, RM15, RM16, RM17, RM18, RM19, RM20, + RM21, RM22, RM23, RM24, RM25, RM26, RM27, RM28, RM29, RM30, + RM31, RM32, RM33, RM34, RM35, RM36, RM37, RM38, RM39, RM40, + RM41, RM42, RM43, RM44, RM45, RM46, RM47, RM48, RM49, RM50, + RM51, RM52, RM53, RM54, RM55, RM56, RM57, RM58, RM59, RM60, + RM61, RM62, RM63, RM64, RM65, RM66, RM67 }; + +/* These versions of the macros use the stack, as normal. There are debugging +versions and production versions. Note that the "rw" argument of RMATCH isn't +actually used in this definition. */ + +#ifndef NO_RECURSE +#define REGISTER register + +#ifdef PCRE_DEBUG +#define RMATCH(ra,rb,rc,rd,re,rw) \ + { \ + printf("match() called in line %d\n", __LINE__); \ + rrc = match(ra,rb,mstart,rc,rd,re,rdepth+1); \ + printf("to line %d\n", __LINE__); \ + } +#define RRETURN(ra) \ + { \ + printf("match() returned %d from line %d\n", ra, __LINE__); \ + return ra; \ + } +#else +#define RMATCH(ra,rb,rc,rd,re,rw) \ + rrc = match(ra,rb,mstart,rc,rd,re,rdepth+1) +#define RRETURN(ra) return ra +#endif + +#else + + +/* These versions of the macros manage a private stack on the heap. Note that +the "rd" argument of RMATCH isn't actually used in this definition. It's the md +argument of match(), which never changes. */ + +#define REGISTER + +#define RMATCH(ra,rb,rc,rd,re,rw)\ + {\ + heapframe *newframe = frame->Xnextframe;\ + if (newframe == NULL)\ + {\ + newframe = (heapframe *)(PUBL(stack_malloc))(sizeof(heapframe));\ + if (newframe == NULL) RRETURN(PCRE_ERROR_NOMEMORY);\ + newframe->Xnextframe = NULL;\ + frame->Xnextframe = newframe;\ + }\ + frame->Xwhere = rw;\ + newframe->Xeptr = ra;\ + newframe->Xecode = rb;\ + newframe->Xmstart = mstart;\ + newframe->Xoffset_top = rc;\ + newframe->Xeptrb = re;\ + newframe->Xrdepth = frame->Xrdepth + 1;\ + newframe->Xprevframe = frame;\ + frame = newframe;\ + DPRINTF(("restarting from line %d\n", __LINE__));\ + goto HEAP_RECURSE;\ + L_##rw:\ + DPRINTF(("jumped back to line %d\n", __LINE__));\ + } + +#define RRETURN(ra)\ + {\ + heapframe *oldframe = frame;\ + frame = oldframe->Xprevframe;\ + if (frame != NULL)\ + {\ + rrc = ra;\ + goto HEAP_RETURN;\ + }\ + return ra;\ + } + + +/* Structure for remembering the local variables in a private frame */ + +typedef struct heapframe { + struct heapframe *Xprevframe; + struct heapframe *Xnextframe; + + /* Function arguments that may change */ + + PCRE_PUCHAR Xeptr; + const pcre_uchar *Xecode; + PCRE_PUCHAR Xmstart; + int Xoffset_top; + eptrblock *Xeptrb; + unsigned int Xrdepth; + + /* Function local variables */ + + PCRE_PUCHAR Xcallpat; +#ifdef SUPPORT_UTF + PCRE_PUCHAR Xcharptr; +#endif + PCRE_PUCHAR Xdata; + PCRE_PUCHAR Xnext; + PCRE_PUCHAR Xpp; + PCRE_PUCHAR Xprev; + PCRE_PUCHAR Xsaved_eptr; + + recursion_info Xnew_recursive; + + BOOL Xcur_is_word; + BOOL Xcondition; + BOOL Xprev_is_word; + +#ifdef SUPPORT_UCP + int Xprop_type; + unsigned int Xprop_value; + int Xprop_fail_result; + int Xoclength; + pcre_uchar Xocchars[6]; +#endif + + int Xcodelink; + int Xctype; + unsigned int Xfc; + int Xfi; + int Xlength; + int Xmax; + int Xmin; + unsigned int Xnumber; + int Xoffset; + unsigned int Xop; + pcre_int32 Xsave_capture_last; + int Xsave_offset1, Xsave_offset2, Xsave_offset3; + int Xstacksave[REC_STACK_SAVE_MAX]; + + eptrblock Xnewptrb; + + /* Where to jump back to */ + + int Xwhere; + +} heapframe; + +#endif + + +/*************************************************************************** +***************************************************************************/ + + + +/************************************************* +* Match from current position * +*************************************************/ + +/* This function is called recursively in many circumstances. Whenever it +returns a negative (error) response, the outer incarnation must also return the +same response. */ + +/* These macros pack up tests that are used for partial matching, and which +appear several times in the code. We set the "hit end" flag if the pointer is +at the end of the subject and also past the start of the subject (i.e. +something has been matched). For hard partial matching, we then return +immediately. The second one is used when we already know we are past the end of +the subject. */ + +#define CHECK_PARTIAL()\ + if (md->partial != 0 && eptr >= md->end_subject && \ + eptr > md->start_used_ptr) \ + { \ + md->hitend = TRUE; \ + if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); \ + } + +#define SCHECK_PARTIAL()\ + if (md->partial != 0 && eptr > md->start_used_ptr) \ + { \ + md->hitend = TRUE; \ + if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); \ + } + + +/* Performance note: It might be tempting to extract commonly used fields from +the md structure (e.g. utf, end_subject) into individual variables to improve +performance. Tests using gcc on a SPARC disproved this; in the first case, it +made performance worse. + +Arguments: + eptr pointer to current character in subject + ecode pointer to current position in compiled code + mstart pointer to the current match start position (can be modified + by encountering \K) + offset_top current top pointer + md pointer to "static" info for the match + eptrb pointer to chain of blocks containing eptr at start of + brackets - for testing for empty matches + rdepth the recursion depth + +Returns: MATCH_MATCH if matched ) these values are >= 0 + MATCH_NOMATCH if failed to match ) + a negative MATCH_xxx value for PRUNE, SKIP, etc + a negative PCRE_ERROR_xxx value if aborted by an error condition + (e.g. stopped by repeated call or recursion limit) +*/ + +static int +match(REGISTER PCRE_PUCHAR eptr, REGISTER const pcre_uchar *ecode, + PCRE_PUCHAR mstart, int offset_top, match_data *md, eptrblock *eptrb, + unsigned int rdepth) +{ +/* These variables do not need to be preserved over recursion in this function, +so they can be ordinary variables in all cases. Mark some of them with +"register" because they are used a lot in loops. */ + +register int rrc; /* Returns from recursive calls */ +register int i; /* Used for loops not involving calls to RMATCH() */ +register pcre_uint32 c; /* Character values not kept over RMATCH() calls */ +register BOOL utf; /* Local copy of UTF flag for speed */ + +BOOL minimize, possessive; /* Quantifier options */ +BOOL caseless; +int condcode; + +/* When recursion is not being used, all "local" variables that have to be +preserved over calls to RMATCH() are part of a "frame". We set up the top-level +frame on the stack here; subsequent instantiations are obtained from the heap +whenever RMATCH() does a "recursion". See the macro definitions above. Putting +the top-level on the stack rather than malloc-ing them all gives a performance +boost in many cases where there is not much "recursion". */ + +#ifdef NO_RECURSE +heapframe *frame = (heapframe *)md->match_frames_base; + +/* Copy in the original argument variables */ + +frame->Xeptr = eptr; +frame->Xecode = ecode; +frame->Xmstart = mstart; +frame->Xoffset_top = offset_top; +frame->Xeptrb = eptrb; +frame->Xrdepth = rdepth; + +/* This is where control jumps back to to effect "recursion" */ + +HEAP_RECURSE: + +/* Macros make the argument variables come from the current frame */ + +#define eptr frame->Xeptr +#define ecode frame->Xecode +#define mstart frame->Xmstart +#define offset_top frame->Xoffset_top +#define eptrb frame->Xeptrb +#define rdepth frame->Xrdepth + +/* Ditto for the local variables */ + +#ifdef SUPPORT_UTF +#define charptr frame->Xcharptr +#endif +#define callpat frame->Xcallpat +#define codelink frame->Xcodelink +#define data frame->Xdata +#define next frame->Xnext +#define pp frame->Xpp +#define prev frame->Xprev +#define saved_eptr frame->Xsaved_eptr + +#define new_recursive frame->Xnew_recursive + +#define cur_is_word frame->Xcur_is_word +#define condition frame->Xcondition +#define prev_is_word frame->Xprev_is_word + +#ifdef SUPPORT_UCP +#define prop_type frame->Xprop_type +#define prop_value frame->Xprop_value +#define prop_fail_result frame->Xprop_fail_result +#define oclength frame->Xoclength +#define occhars frame->Xocchars +#endif + +#define ctype frame->Xctype +#define fc frame->Xfc +#define fi frame->Xfi +#define length frame->Xlength +#define max frame->Xmax +#define min frame->Xmin +#define number frame->Xnumber +#define offset frame->Xoffset +#define op frame->Xop +#define save_capture_last frame->Xsave_capture_last +#define save_offset1 frame->Xsave_offset1 +#define save_offset2 frame->Xsave_offset2 +#define save_offset3 frame->Xsave_offset3 +#define stacksave frame->Xstacksave + +#define newptrb frame->Xnewptrb + +/* When recursion is being used, local variables are allocated on the stack and +get preserved during recursion in the normal way. In this environment, fi and +i, and fc and c, can be the same variables. */ + +#else /* NO_RECURSE not defined */ +#define fi i +#define fc c + +/* Many of the following variables are used only in small blocks of the code. +My normal style of coding would have declared them within each of those blocks. +However, in order to accommodate the version of this code that uses an external +"stack" implemented on the heap, it is easier to declare them all here, so the +declarations can be cut out in a block. The only declarations within blocks +below are for variables that do not have to be preserved over a recursive call +to RMATCH(). */ + +#ifdef SUPPORT_UTF +const pcre_uchar *charptr; +#endif +const pcre_uchar *callpat; +const pcre_uchar *data; +const pcre_uchar *next; +PCRE_PUCHAR pp; +const pcre_uchar *prev; +PCRE_PUCHAR saved_eptr; + +recursion_info new_recursive; + +BOOL cur_is_word; +BOOL condition; +BOOL prev_is_word; + +#ifdef SUPPORT_UCP +int prop_type; +unsigned int prop_value; +int prop_fail_result; +int oclength; +pcre_uchar occhars[6]; +#endif + +int codelink; +int ctype; +int length; +int max; +int min; +unsigned int number; +int offset; +unsigned int op; +pcre_int32 save_capture_last; +int save_offset1, save_offset2, save_offset3; +int stacksave[REC_STACK_SAVE_MAX]; + +eptrblock newptrb; + +/* There is a special fudge for calling match() in a way that causes it to +measure the size of its basic stack frame when the stack is being used for +recursion. The second argument (ecode) being NULL triggers this behaviour. It +cannot normally ever be NULL. The return is the negated value of the frame +size. */ + +if (ecode == NULL) + { + if (rdepth == 0) + return match((PCRE_PUCHAR)&rdepth, NULL, NULL, 0, NULL, NULL, 1); + else + { + int len = (int)((char *)&rdepth - (char *)eptr); + return (len > 0)? -len : len; + } + } +#endif /* NO_RECURSE */ + +/* To save space on the stack and in the heap frame, I have doubled up on some +of the local variables that are used only in localised parts of the code, but +still need to be preserved over recursive calls of match(). These macros define +the alternative names that are used. */ + +#define allow_zero cur_is_word +#define cbegroup condition +#define code_offset codelink +#define condassert condition +#define matched_once prev_is_word +#define foc number +#define save_mark data + +/* These statements are here to stop the compiler complaining about unitialized +variables. */ + +#ifdef SUPPORT_UCP +prop_value = 0; +prop_fail_result = 0; +#endif + + +/* This label is used for tail recursion, which is used in a few cases even +when NO_RECURSE is not defined, in order to reduce the amount of stack that is +used. Thanks to Ian Taylor for noticing this possibility and sending the +original patch. */ + +TAIL_RECURSE: + +/* OK, now we can get on with the real code of the function. Recursive calls +are specified by the macro RMATCH and RRETURN is used to return. When +NO_RECURSE is *not* defined, these just turn into a recursive call to match() +and a "return", respectively (possibly with some debugging if PCRE_DEBUG is +defined). However, RMATCH isn't like a function call because it's quite a +complicated macro. It has to be used in one particular way. This shouldn't, +however, impact performance when true recursion is being used. */ + +#ifdef SUPPORT_UTF +utf = md->utf; /* Local copy of the flag */ +#else +utf = FALSE; +#endif + +/* First check that we haven't called match() too many times, or that we +haven't exceeded the recursive call limit. */ + +if (md->match_call_count++ >= md->match_limit) RRETURN(PCRE_ERROR_MATCHLIMIT); +if (rdepth >= md->match_limit_recursion) RRETURN(PCRE_ERROR_RECURSIONLIMIT); + +/* At the start of a group with an unlimited repeat that may match an empty +string, the variable md->match_function_type is set to MATCH_CBEGROUP. It is +done this way to save having to use another function argument, which would take +up space on the stack. See also MATCH_CONDASSERT below. + +When MATCH_CBEGROUP is set, add the current subject pointer to the chain of +such remembered pointers, to be checked when we hit the closing ket, in order +to break infinite loops that match no characters. When match() is called in +other circumstances, don't add to the chain. The MATCH_CBEGROUP feature must +NOT be used with tail recursion, because the memory block that is used is on +the stack, so a new one may be required for each match(). */ + +if (md->match_function_type == MATCH_CBEGROUP) + { + newptrb.epb_saved_eptr = eptr; + newptrb.epb_prev = eptrb; + eptrb = &newptrb; + md->match_function_type = 0; + } + +/* Now start processing the opcodes. */ + +for (;;) + { + minimize = possessive = FALSE; + op = *ecode; + + switch(op) + { + case OP_MARK: + md->nomatch_mark = ecode + 2; + md->mark = NULL; /* In case previously set by assertion */ + RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode] + ecode[1], offset_top, md, + eptrb, RM55); + if ((rrc == MATCH_MATCH || rrc == MATCH_ACCEPT) && + md->mark == NULL) md->mark = ecode + 2; + + /* A return of MATCH_SKIP_ARG means that matching failed at SKIP with an + argument, and we must check whether that argument matches this MARK's + argument. It is passed back in md->start_match_ptr (an overloading of that + variable). If it does match, we reset that variable to the current subject + position and return MATCH_SKIP. Otherwise, pass back the return code + unaltered. */ + + else if (rrc == MATCH_SKIP_ARG && + STRCMP_UC_UC_TEST(ecode + 2, md->start_match_ptr) == 0) + { + md->start_match_ptr = eptr; + RRETURN(MATCH_SKIP); + } + RRETURN(rrc); + + case OP_FAIL: + RRETURN(MATCH_NOMATCH); + + case OP_COMMIT: + RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode], offset_top, md, + eptrb, RM52); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + RRETURN(MATCH_COMMIT); + + case OP_PRUNE: + RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode], offset_top, md, + eptrb, RM51); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + RRETURN(MATCH_PRUNE); + + case OP_PRUNE_ARG: + md->nomatch_mark = ecode + 2; + md->mark = NULL; /* In case previously set by assertion */ + RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode] + ecode[1], offset_top, md, + eptrb, RM56); + if ((rrc == MATCH_MATCH || rrc == MATCH_ACCEPT) && + md->mark == NULL) md->mark = ecode + 2; + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + RRETURN(MATCH_PRUNE); + + case OP_SKIP: + RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode], offset_top, md, + eptrb, RM53); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + md->start_match_ptr = eptr; /* Pass back current position */ + RRETURN(MATCH_SKIP); + + /* Note that, for Perl compatibility, SKIP with an argument does NOT set + nomatch_mark. When a pattern match ends with a SKIP_ARG for which there was + not a matching mark, we have to re-run the match, ignoring the SKIP_ARG + that failed and any that precede it (either they also failed, or were not + triggered). To do this, we maintain a count of executed SKIP_ARGs. If a + SKIP_ARG gets to top level, the match is re-run with md->ignore_skip_arg + set to the count of the one that failed. */ + + case OP_SKIP_ARG: + md->skip_arg_count++; + if (md->skip_arg_count <= md->ignore_skip_arg) + { + ecode += PRIV(OP_lengths)[*ecode] + ecode[1]; + break; + } + RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode] + ecode[1], offset_top, md, + eptrb, RM57); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + + /* Pass back the current skip name by overloading md->start_match_ptr and + returning the special MATCH_SKIP_ARG return code. This will either be + caught by a matching MARK, or get to the top, where it causes a rematch + with md->ignore_skip_arg set to the value of md->skip_arg_count. */ + + md->start_match_ptr = ecode + 2; + RRETURN(MATCH_SKIP_ARG); + + /* For THEN (and THEN_ARG) we pass back the address of the opcode, so that + the branch in which it occurs can be determined. Overload the start of + match pointer to do this. */ + + case OP_THEN: + RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode], offset_top, md, + eptrb, RM54); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + md->start_match_ptr = ecode; + RRETURN(MATCH_THEN); + + case OP_THEN_ARG: + md->nomatch_mark = ecode + 2; + md->mark = NULL; /* In case previously set by assertion */ + RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode] + ecode[1], offset_top, + md, eptrb, RM58); + if ((rrc == MATCH_MATCH || rrc == MATCH_ACCEPT) && + md->mark == NULL) md->mark = ecode + 2; + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + md->start_match_ptr = ecode; + RRETURN(MATCH_THEN); + + /* Handle an atomic group that does not contain any capturing parentheses. + This can be handled like an assertion. Prior to 8.13, all atomic groups + were handled this way. In 8.13, the code was changed as below for ONCE, so + that backups pass through the group and thereby reset captured values. + However, this uses a lot more stack, so in 8.20, atomic groups that do not + contain any captures generate OP_ONCE_NC, which can be handled in the old, + less stack intensive way. + + Check the alternative branches in turn - the matching won't pass the KET + for this kind of subpattern. If any one branch matches, we carry on as at + the end of a normal bracket, leaving the subject pointer, but resetting + the start-of-match value in case it was changed by \K. */ + + case OP_ONCE_NC: + prev = ecode; + saved_eptr = eptr; + save_mark = md->mark; + do + { + RMATCH(eptr, ecode + 1 + LINK_SIZE, offset_top, md, eptrb, RM64); + if (rrc == MATCH_MATCH) /* Note: _not_ MATCH_ACCEPT */ + { + mstart = md->start_match_ptr; + break; + } + if (rrc == MATCH_THEN) + { + next = ecode + GET(ecode,1); + if (md->start_match_ptr < next && + (*ecode == OP_ALT || *next == OP_ALT)) + rrc = MATCH_NOMATCH; + } + + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + ecode += GET(ecode,1); + md->mark = save_mark; + } + while (*ecode == OP_ALT); + + /* If hit the end of the group (which could be repeated), fail */ + + if (*ecode != OP_ONCE_NC && *ecode != OP_ALT) RRETURN(MATCH_NOMATCH); + + /* Continue as from after the group, updating the offsets high water + mark, since extracts may have been taken. */ + + do ecode += GET(ecode, 1); while (*ecode == OP_ALT); + + offset_top = md->end_offset_top; + eptr = md->end_match_ptr; + + /* For a non-repeating ket, just continue at this level. This also + happens for a repeating ket if no characters were matched in the group. + This is the forcible breaking of infinite loops as implemented in Perl + 5.005. */ + + if (*ecode == OP_KET || eptr == saved_eptr) + { + ecode += 1+LINK_SIZE; + break; + } + + /* The repeating kets try the rest of the pattern or restart from the + preceding bracket, in the appropriate order. The second "call" of match() + uses tail recursion, to avoid using another stack frame. */ + + if (*ecode == OP_KETRMIN) + { + RMATCH(eptr, ecode + 1 + LINK_SIZE, offset_top, md, eptrb, RM65); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + ecode = prev; + goto TAIL_RECURSE; + } + else /* OP_KETRMAX */ + { + RMATCH(eptr, prev, offset_top, md, eptrb, RM66); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + ecode += 1 + LINK_SIZE; + goto TAIL_RECURSE; + } + /* Control never gets here */ + + /* Handle a capturing bracket, other than those that are possessive with an + unlimited repeat. If there is space in the offset vector, save the current + subject position in the working slot at the top of the vector. We mustn't + change the current values of the data slot, because they may be set from a + previous iteration of this group, and be referred to by a reference inside + the group. A failure to match might occur after the group has succeeded, + if something later on doesn't match. For this reason, we need to restore + the working value and also the values of the final offsets, in case they + were set by a previous iteration of the same bracket. + + If there isn't enough space in the offset vector, treat this as if it were + a non-capturing bracket. Don't worry about setting the flag for the error + case here; that is handled in the code for KET. */ + + case OP_CBRA: + case OP_SCBRA: + number = GET2(ecode, 1+LINK_SIZE); + offset = number << 1; + +#ifdef PCRE_DEBUG + printf("start bracket %d\n", number); + printf("subject="); + pchars(eptr, 16, TRUE, md); + printf("\n"); +#endif + + if (offset < md->offset_max) + { + save_offset1 = md->offset_vector[offset]; + save_offset2 = md->offset_vector[offset+1]; + save_offset3 = md->offset_vector[md->offset_end - number]; + save_capture_last = md->capture_last; + save_mark = md->mark; + + DPRINTF(("saving %d %d %d\n", save_offset1, save_offset2, save_offset3)); + md->offset_vector[md->offset_end - number] = + (int)(eptr - md->start_subject); + + for (;;) + { + if (op >= OP_SBRA) md->match_function_type = MATCH_CBEGROUP; + RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode], offset_top, md, + eptrb, RM1); + if (rrc == MATCH_ONCE) break; /* Backing up through an atomic group */ + + /* If we backed up to a THEN, check whether it is within the current + branch by comparing the address of the THEN that is passed back with + the end of the branch. If it is within the current branch, and the + branch is one of two or more alternatives (it either starts or ends + with OP_ALT), we have reached the limit of THEN's action, so convert + the return code to NOMATCH, which will cause normal backtracking to + happen from now on. Otherwise, THEN is passed back to an outer + alternative. This implements Perl's treatment of parenthesized groups, + where a group not containing | does not affect the current alternative, + that is, (X) is NOT the same as (X|(*F)). */ + + if (rrc == MATCH_THEN) + { + next = ecode + GET(ecode,1); + if (md->start_match_ptr < next && + (*ecode == OP_ALT || *next == OP_ALT)) + rrc = MATCH_NOMATCH; + } + + /* Anything other than NOMATCH is passed back. */ + + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + md->capture_last = save_capture_last; + ecode += GET(ecode, 1); + md->mark = save_mark; + if (*ecode != OP_ALT) break; + } + + DPRINTF(("bracket %d failed\n", number)); + md->offset_vector[offset] = save_offset1; + md->offset_vector[offset+1] = save_offset2; + md->offset_vector[md->offset_end - number] = save_offset3; + + /* At this point, rrc will be one of MATCH_ONCE or MATCH_NOMATCH. */ + + RRETURN(rrc); + } + + /* FALL THROUGH ... Insufficient room for saving captured contents. Treat + as a non-capturing bracket. */ + + /* VVVVVVVVVVVVVVVVVVVVVVVVV */ + /* VVVVVVVVVVVVVVVVVVVVVVVVV */ + + DPRINTF(("insufficient capture room: treat as non-capturing\n")); + + /* VVVVVVVVVVVVVVVVVVVVVVVVV */ + /* VVVVVVVVVVVVVVVVVVVVVVVVV */ + + /* Non-capturing or atomic group, except for possessive with unlimited + repeat and ONCE group with no captures. Loop for all the alternatives. + + When we get to the final alternative within the brackets, we used to return + the result of a recursive call to match() whatever happened so it was + possible to reduce stack usage by turning this into a tail recursion, + except in the case of a possibly empty group. However, now that there is + the possiblity of (*THEN) occurring in the final alternative, this + optimization is no longer always possible. + + We can optimize if we know there are no (*THEN)s in the pattern; at present + this is the best that can be done. + + MATCH_ONCE is returned when the end of an atomic group is successfully + reached, but subsequent matching fails. It passes back up the tree (causing + captured values to be reset) until the original atomic group level is + reached. This is tested by comparing md->once_target with the start of the + group. At this point, the return is converted into MATCH_NOMATCH so that + previous backup points can be taken. */ + + case OP_ONCE: + case OP_BRA: + case OP_SBRA: + DPRINTF(("start non-capturing bracket\n")); + + for (;;) + { + if (op >= OP_SBRA || op == OP_ONCE) + md->match_function_type = MATCH_CBEGROUP; + + /* If this is not a possibly empty group, and there are no (*THEN)s in + the pattern, and this is the final alternative, optimize as described + above. */ + + else if (!md->hasthen && ecode[GET(ecode, 1)] != OP_ALT) + { + ecode += PRIV(OP_lengths)[*ecode]; + goto TAIL_RECURSE; + } + + /* In all other cases, we have to make another call to match(). */ + + save_mark = md->mark; + save_capture_last = md->capture_last; + RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode], offset_top, md, eptrb, + RM2); + + /* See comment in the code for capturing groups above about handling + THEN. */ + + if (rrc == MATCH_THEN) + { + next = ecode + GET(ecode,1); + if (md->start_match_ptr < next && + (*ecode == OP_ALT || *next == OP_ALT)) + rrc = MATCH_NOMATCH; + } + + if (rrc != MATCH_NOMATCH) + { + if (rrc == MATCH_ONCE) + { + const pcre_uchar *scode = ecode; + if (*scode != OP_ONCE) /* If not at start, find it */ + { + while (*scode == OP_ALT) scode += GET(scode, 1); + scode -= GET(scode, 1); + } + if (md->once_target == scode) rrc = MATCH_NOMATCH; + } + RRETURN(rrc); + } + ecode += GET(ecode, 1); + md->mark = save_mark; + if (*ecode != OP_ALT) break; + md->capture_last = save_capture_last; + } + + RRETURN(MATCH_NOMATCH); + + /* Handle possessive capturing brackets with an unlimited repeat. We come + here from BRAZERO with allow_zero set TRUE. The offset_vector values are + handled similarly to the normal case above. However, the matching is + different. The end of these brackets will always be OP_KETRPOS, which + returns MATCH_KETRPOS without going further in the pattern. By this means + we can handle the group by iteration rather than recursion, thereby + reducing the amount of stack needed. */ + + case OP_CBRAPOS: + case OP_SCBRAPOS: + allow_zero = FALSE; + + POSSESSIVE_CAPTURE: + number = GET2(ecode, 1+LINK_SIZE); + offset = number << 1; + +#ifdef PCRE_DEBUG + printf("start possessive bracket %d\n", number); + printf("subject="); + pchars(eptr, 16, TRUE, md); + printf("\n"); +#endif + + if (offset >= md->offset_max) goto POSSESSIVE_NON_CAPTURE; + + matched_once = FALSE; + code_offset = (int)(ecode - md->start_code); + + save_offset1 = md->offset_vector[offset]; + save_offset2 = md->offset_vector[offset+1]; + save_offset3 = md->offset_vector[md->offset_end - number]; + save_capture_last = md->capture_last; + + DPRINTF(("saving %d %d %d\n", save_offset1, save_offset2, save_offset3)); + + /* Each time round the loop, save the current subject position for use + when the group matches. For MATCH_MATCH, the group has matched, so we + restart it with a new subject starting position, remembering that we had + at least one match. For MATCH_NOMATCH, carry on with the alternatives, as + usual. If we haven't matched any alternatives in any iteration, check to + see if a previous iteration matched. If so, the group has matched; + continue from afterwards. Otherwise it has failed; restore the previous + capture values before returning NOMATCH. */ + + for (;;) + { + md->offset_vector[md->offset_end - number] = + (int)(eptr - md->start_subject); + if (op >= OP_SBRA) md->match_function_type = MATCH_CBEGROUP; + RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode], offset_top, md, + eptrb, RM63); + if (rrc == MATCH_KETRPOS) + { + offset_top = md->end_offset_top; + ecode = md->start_code + code_offset; + save_capture_last = md->capture_last; + matched_once = TRUE; + mstart = md->start_match_ptr; /* In case \K changed it */ + if (eptr == md->end_match_ptr) /* Matched an empty string */ + { + do ecode += GET(ecode, 1); while (*ecode == OP_ALT); + break; + } + eptr = md->end_match_ptr; + continue; + } + + /* See comment in the code for capturing groups above about handling + THEN. */ + + if (rrc == MATCH_THEN) + { + next = ecode + GET(ecode,1); + if (md->start_match_ptr < next && + (*ecode == OP_ALT || *next == OP_ALT)) + rrc = MATCH_NOMATCH; + } + + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + md->capture_last = save_capture_last; + ecode += GET(ecode, 1); + if (*ecode != OP_ALT) break; + } + + if (!matched_once) + { + md->offset_vector[offset] = save_offset1; + md->offset_vector[offset+1] = save_offset2; + md->offset_vector[md->offset_end - number] = save_offset3; + } + + if (allow_zero || matched_once) + { + ecode += 1 + LINK_SIZE; + break; + } + + RRETURN(MATCH_NOMATCH); + + /* Non-capturing possessive bracket with unlimited repeat. We come here + from BRAZERO with allow_zero = TRUE. The code is similar to the above, + without the capturing complication. It is written out separately for speed + and cleanliness. */ + + case OP_BRAPOS: + case OP_SBRAPOS: + allow_zero = FALSE; + + POSSESSIVE_NON_CAPTURE: + matched_once = FALSE; + code_offset = (int)(ecode - md->start_code); + save_capture_last = md->capture_last; + + for (;;) + { + if (op >= OP_SBRA) md->match_function_type = MATCH_CBEGROUP; + RMATCH(eptr, ecode + PRIV(OP_lengths)[*ecode], offset_top, md, + eptrb, RM48); + if (rrc == MATCH_KETRPOS) + { + offset_top = md->end_offset_top; + ecode = md->start_code + code_offset; + matched_once = TRUE; + mstart = md->start_match_ptr; /* In case \K reset it */ + if (eptr == md->end_match_ptr) /* Matched an empty string */ + { + do ecode += GET(ecode, 1); while (*ecode == OP_ALT); + break; + } + eptr = md->end_match_ptr; + continue; + } + + /* See comment in the code for capturing groups above about handling + THEN. */ + + if (rrc == MATCH_THEN) + { + next = ecode + GET(ecode,1); + if (md->start_match_ptr < next && + (*ecode == OP_ALT || *next == OP_ALT)) + rrc = MATCH_NOMATCH; + } + + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + ecode += GET(ecode, 1); + if (*ecode != OP_ALT) break; + md->capture_last = save_capture_last; + } + + if (matched_once || allow_zero) + { + ecode += 1 + LINK_SIZE; + break; + } + RRETURN(MATCH_NOMATCH); + + /* Control never reaches here. */ + + /* Conditional group: compilation checked that there are no more than two + branches. If the condition is false, skipping the first branch takes us + past the end of the item if there is only one branch, but that's exactly + what we want. */ + + case OP_COND: + case OP_SCOND: + + /* The variable codelink will be added to ecode when the condition is + false, to get to the second branch. Setting it to the offset to the ALT + or KET, then incrementing ecode achieves this effect. We now have ecode + pointing to the condition or callout. */ + + codelink = GET(ecode, 1); /* Offset to the second branch */ + ecode += 1 + LINK_SIZE; /* From this opcode */ + + /* Because of the way auto-callout works during compile, a callout item is + inserted between OP_COND and an assertion condition. */ + + if (*ecode == OP_CALLOUT) + { + if (PUBL(callout) != NULL) + { + PUBL(callout_block) cb; + cb.version = 2; /* Version 1 of the callout block */ + cb.callout_number = ecode[1]; + cb.offset_vector = md->offset_vector; +#if defined COMPILE_PCRE8 + cb.subject = (PCRE_SPTR)md->start_subject; +#elif defined COMPILE_PCRE16 + cb.subject = (PCRE_SPTR16)md->start_subject; +#elif defined COMPILE_PCRE32 + cb.subject = (PCRE_SPTR32)md->start_subject; +#endif + cb.subject_length = (int)(md->end_subject - md->start_subject); + cb.start_match = (int)(mstart - md->start_subject); + cb.current_position = (int)(eptr - md->start_subject); + cb.pattern_position = GET(ecode, 2); + cb.next_item_length = GET(ecode, 2 + LINK_SIZE); + cb.capture_top = offset_top/2; + cb.capture_last = md->capture_last & CAPLMASK; + /* Internal change requires this for API compatibility. */ + if (cb.capture_last == 0) cb.capture_last = -1; + cb.callout_data = md->callout_data; + cb.mark = md->nomatch_mark; + if ((rrc = (*PUBL(callout))(&cb)) > 0) RRETURN(MATCH_NOMATCH); + if (rrc < 0) RRETURN(rrc); + } + + /* Advance ecode past the callout, so it now points to the condition. We + must adjust codelink so that the value of ecode+codelink is unchanged. */ + + ecode += PRIV(OP_lengths)[OP_CALLOUT]; + codelink -= PRIV(OP_lengths)[OP_CALLOUT]; + } + + /* Test the various possible conditions */ + + condition = FALSE; + switch(condcode = *ecode) + { + case OP_RREF: /* Numbered group recursion test */ + if (md->recursive != NULL) /* Not recursing => FALSE */ + { + unsigned int recno = GET2(ecode, 1); /* Recursion group number*/ + condition = (recno == RREF_ANY || recno == md->recursive->group_num); + } + break; + + case OP_DNRREF: /* Duplicate named group recursion test */ + if (md->recursive != NULL) + { + int count = GET2(ecode, 1 + IMM2_SIZE); + pcre_uchar *slot = md->name_table + GET2(ecode, 1) * md->name_entry_size; + while (count-- > 0) + { + unsigned int recno = GET2(slot, 0); + condition = recno == md->recursive->group_num; + if (condition) break; + slot += md->name_entry_size; + } + } + break; + + case OP_CREF: /* Numbered group used test */ + offset = GET2(ecode, 1) << 1; /* Doubled ref number */ + condition = offset < offset_top && md->offset_vector[offset] >= 0; + break; + + case OP_DNCREF: /* Duplicate named group used test */ + { + int count = GET2(ecode, 1 + IMM2_SIZE); + pcre_uchar *slot = md->name_table + GET2(ecode, 1) * md->name_entry_size; + while (count-- > 0) + { + offset = GET2(slot, 0) << 1; + condition = offset < offset_top && md->offset_vector[offset] >= 0; + if (condition) break; + slot += md->name_entry_size; + } + } + break; + + case OP_DEF: /* DEFINE - always false */ + case OP_FAIL: /* From optimized (?!) condition */ + break; + + /* The condition is an assertion. Call match() to evaluate it - setting + md->match_function_type to MATCH_CONDASSERT causes it to stop at the end + of an assertion. */ + + default: + md->match_function_type = MATCH_CONDASSERT; + RMATCH(eptr, ecode, offset_top, md, NULL, RM3); + if (rrc == MATCH_MATCH) + { + if (md->end_offset_top > offset_top) + offset_top = md->end_offset_top; /* Captures may have happened */ + condition = TRUE; + + /* Advance ecode past the assertion to the start of the first branch, + but adjust it so that the general choosing code below works. If the + assertion has a quantifier that allows zero repeats we must skip over + the BRAZERO. This is a lunatic thing to do, but somebody did! */ + + if (*ecode == OP_BRAZERO) ecode++; + ecode += GET(ecode, 1); + while (*ecode == OP_ALT) ecode += GET(ecode, 1); + ecode += 1 + LINK_SIZE - PRIV(OP_lengths)[condcode]; + } + + /* PCRE doesn't allow the effect of (*THEN) to escape beyond an + assertion; it is therefore treated as NOMATCH. Any other return is an + error. */ + + else if (rrc != MATCH_NOMATCH && rrc != MATCH_THEN) + { + RRETURN(rrc); /* Need braces because of following else */ + } + break; + } + + /* Choose branch according to the condition */ + + ecode += condition? PRIV(OP_lengths)[condcode] : codelink; + + /* We are now at the branch that is to be obeyed. As there is only one, we + can use tail recursion to avoid using another stack frame, except when + there is unlimited repeat of a possibly empty group. In the latter case, a + recursive call to match() is always required, unless the second alternative + doesn't exist, in which case we can just plough on. Note that, for + compatibility with Perl, the | in a conditional group is NOT treated as + creating two alternatives. If a THEN is encountered in the branch, it + propagates out to the enclosing alternative (unless nested in a deeper set + of alternatives, of course). */ + + if (condition || ecode[-(1+LINK_SIZE)] == OP_ALT) + { + if (op != OP_SCOND) + { + goto TAIL_RECURSE; + } + + md->match_function_type = MATCH_CBEGROUP; + RMATCH(eptr, ecode, offset_top, md, eptrb, RM49); + RRETURN(rrc); + } + + /* Condition false & no alternative; continue after the group. */ + + else + { + } + break; + + + /* Before OP_ACCEPT there may be any number of OP_CLOSE opcodes, + to close any currently open capturing brackets. */ + + case OP_CLOSE: + number = GET2(ecode, 1); /* Must be less than 65536 */ + offset = number << 1; + +#ifdef PCRE_DEBUG + printf("end bracket %d at *ACCEPT", number); + printf("\n"); +#endif + + md->capture_last = (md->capture_last & OVFLMASK) | number; + if (offset >= md->offset_max) md->capture_last |= OVFLBIT; else + { + md->offset_vector[offset] = + md->offset_vector[md->offset_end - number]; + md->offset_vector[offset+1] = (int)(eptr - md->start_subject); + + /* If this group is at or above the current highwater mark, ensure that + any groups between the current high water mark and this group are marked + unset and then update the high water mark. */ + + if (offset >= offset_top) + { + register int *iptr = md->offset_vector + offset_top; + register int *iend = md->offset_vector + offset; + while (iptr < iend) *iptr++ = -1; + offset_top = offset + 2; + } + } + ecode += 1 + IMM2_SIZE; + break; + + + /* End of the pattern, either real or forced. */ + + case OP_END: + case OP_ACCEPT: + case OP_ASSERT_ACCEPT: + + /* If we have matched an empty string, fail if not in an assertion and not + in a recursion if either PCRE_NOTEMPTY is set, or if PCRE_NOTEMPTY_ATSTART + is set and we have matched at the start of the subject. In both cases, + backtracking will then try other alternatives, if any. */ + + if (eptr == mstart && op != OP_ASSERT_ACCEPT && + md->recursive == NULL && + (md->notempty || + (md->notempty_atstart && + mstart == md->start_subject + md->start_offset))) + RRETURN(MATCH_NOMATCH); + + /* Otherwise, we have a match. */ + + md->end_match_ptr = eptr; /* Record where we ended */ + md->end_offset_top = offset_top; /* and how many extracts were taken */ + md->start_match_ptr = mstart; /* and the start (\K can modify) */ + + /* For some reason, the macros don't work properly if an expression is + given as the argument to RRETURN when the heap is in use. */ + + rrc = (op == OP_END)? MATCH_MATCH : MATCH_ACCEPT; + RRETURN(rrc); + + /* Assertion brackets. Check the alternative branches in turn - the + matching won't pass the KET for an assertion. If any one branch matches, + the assertion is true. Lookbehind assertions have an OP_REVERSE item at the + start of each branch to move the current point backwards, so the code at + this level is identical to the lookahead case. When the assertion is part + of a condition, we want to return immediately afterwards. The caller of + this incarnation of the match() function will have set MATCH_CONDASSERT in + md->match_function type, and one of these opcodes will be the first opcode + that is processed. We use a local variable that is preserved over calls to + match() to remember this case. */ + + case OP_ASSERT: + case OP_ASSERTBACK: + save_mark = md->mark; + if (md->match_function_type == MATCH_CONDASSERT) + { + condassert = TRUE; + md->match_function_type = 0; + } + else condassert = FALSE; + + /* Loop for each branch */ + + do + { + RMATCH(eptr, ecode + 1 + LINK_SIZE, offset_top, md, NULL, RM4); + + /* A match means that the assertion is true; break out of the loop + that matches its alternatives. */ + + if (rrc == MATCH_MATCH || rrc == MATCH_ACCEPT) + { + mstart = md->start_match_ptr; /* In case \K reset it */ + break; + } + + /* If not matched, restore the previous mark setting. */ + + md->mark = save_mark; + + /* See comment in the code for capturing groups above about handling + THEN. */ + + if (rrc == MATCH_THEN) + { + next = ecode + GET(ecode,1); + if (md->start_match_ptr < next && + (*ecode == OP_ALT || *next == OP_ALT)) + rrc = MATCH_NOMATCH; + } + + /* Anything other than NOMATCH causes the entire assertion to fail, + passing back the return code. This includes COMMIT, SKIP, PRUNE and an + uncaptured THEN, which means they take their normal effect. This + consistent approach does not always have exactly the same effect as in + Perl. */ + + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + ecode += GET(ecode, 1); + } + while (*ecode == OP_ALT); /* Continue for next alternative */ + + /* If we have tried all the alternative branches, the assertion has + failed. If not, we broke out after a match. */ + + if (*ecode == OP_KET) RRETURN(MATCH_NOMATCH); + + /* If checking an assertion for a condition, return MATCH_MATCH. */ + + if (condassert) RRETURN(MATCH_MATCH); + + /* Continue from after a successful assertion, updating the offsets high + water mark, since extracts may have been taken during the assertion. */ + + do ecode += GET(ecode,1); while (*ecode == OP_ALT); + ecode += 1 + LINK_SIZE; + offset_top = md->end_offset_top; + continue; + + /* Negative assertion: all branches must fail to match for the assertion to + succeed. */ + + case OP_ASSERT_NOT: + case OP_ASSERTBACK_NOT: + save_mark = md->mark; + if (md->match_function_type == MATCH_CONDASSERT) + { + condassert = TRUE; + md->match_function_type = 0; + } + else condassert = FALSE; + + /* Loop for each alternative branch. */ + + do + { + RMATCH(eptr, ecode + 1 + LINK_SIZE, offset_top, md, NULL, RM5); + md->mark = save_mark; /* Always restore the mark setting */ + + switch(rrc) + { + case MATCH_MATCH: /* A successful match means */ + case MATCH_ACCEPT: /* the assertion has failed. */ + RRETURN(MATCH_NOMATCH); + + case MATCH_NOMATCH: /* Carry on with next branch */ + break; + + /* See comment in the code for capturing groups above about handling + THEN. */ + + case MATCH_THEN: + next = ecode + GET(ecode,1); + if (md->start_match_ptr < next && + (*ecode == OP_ALT || *next == OP_ALT)) + { + rrc = MATCH_NOMATCH; + break; + } + /* Otherwise fall through. */ + + /* COMMIT, SKIP, PRUNE, and an uncaptured THEN cause the whole + assertion to fail to match, without considering any more alternatives. + Failing to match means the assertion is true. This is a consistent + approach, but does not always have the same effect as in Perl. */ + + case MATCH_COMMIT: + case MATCH_SKIP: + case MATCH_SKIP_ARG: + case MATCH_PRUNE: + do ecode += GET(ecode,1); while (*ecode == OP_ALT); + goto NEG_ASSERT_TRUE; /* Break out of alternation loop */ + + /* Anything else is an error */ + + default: + RRETURN(rrc); + } + + /* Continue with next branch */ + + ecode += GET(ecode,1); + } + while (*ecode == OP_ALT); + + /* All branches in the assertion failed to match. */ + + NEG_ASSERT_TRUE: + if (condassert) RRETURN(MATCH_MATCH); /* Condition assertion */ + ecode += 1 + LINK_SIZE; /* Continue with current branch */ + continue; + + /* Move the subject pointer back. This occurs only at the start of + each branch of a lookbehind assertion. If we are too close to the start to + move back, this match function fails. When working with UTF-8 we move + back a number of characters, not bytes. */ + + case OP_REVERSE: +#ifdef SUPPORT_UTF + if (utf) + { + i = GET(ecode, 1); + while (i-- > 0) + { + eptr--; + if (eptr < md->start_subject) RRETURN(MATCH_NOMATCH); + BACKCHAR(eptr); + } + } + else +#endif + + /* No UTF-8 support, or not in UTF-8 mode: count is byte count */ + + { + eptr -= GET(ecode, 1); + if (eptr < md->start_subject) RRETURN(MATCH_NOMATCH); + } + + /* Save the earliest consulted character, then skip to next op code */ + + if (eptr < md->start_used_ptr) md->start_used_ptr = eptr; + ecode += 1 + LINK_SIZE; + break; + + /* The callout item calls an external function, if one is provided, passing + details of the match so far. This is mainly for debugging, though the + function is able to force a failure. */ + + case OP_CALLOUT: + if (PUBL(callout) != NULL) + { + PUBL(callout_block) cb; + cb.version = 2; /* Version 1 of the callout block */ + cb.callout_number = ecode[1]; + cb.offset_vector = md->offset_vector; +#if defined COMPILE_PCRE8 + cb.subject = (PCRE_SPTR)md->start_subject; +#elif defined COMPILE_PCRE16 + cb.subject = (PCRE_SPTR16)md->start_subject; +#elif defined COMPILE_PCRE32 + cb.subject = (PCRE_SPTR32)md->start_subject; +#endif + cb.subject_length = (int)(md->end_subject - md->start_subject); + cb.start_match = (int)(mstart - md->start_subject); + cb.current_position = (int)(eptr - md->start_subject); + cb.pattern_position = GET(ecode, 2); + cb.next_item_length = GET(ecode, 2 + LINK_SIZE); + cb.capture_top = offset_top/2; + cb.capture_last = md->capture_last & CAPLMASK; + /* Internal change requires this for API compatibility. */ + if (cb.capture_last == 0) cb.capture_last = -1; + cb.callout_data = md->callout_data; + cb.mark = md->nomatch_mark; + if ((rrc = (*PUBL(callout))(&cb)) > 0) RRETURN(MATCH_NOMATCH); + if (rrc < 0) RRETURN(rrc); + } + ecode += 2 + 2*LINK_SIZE; + break; + + /* Recursion either matches the current regex, or some subexpression. The + offset data is the offset to the starting bracket from the start of the + whole pattern. (This is so that it works from duplicated subpatterns.) + + The state of the capturing groups is preserved over recursion, and + re-instated afterwards. We don't know how many are started and not yet + finished (offset_top records the completed total) so we just have to save + all the potential data. There may be up to 65535 such values, which is too + large to put on the stack, but using malloc for small numbers seems + expensive. As a compromise, the stack is used when there are no more than + REC_STACK_SAVE_MAX values to store; otherwise malloc is used. + + There are also other values that have to be saved. We use a chained + sequence of blocks that actually live on the stack. Thanks to Robin Houston + for the original version of this logic. It has, however, been hacked around + a lot, so he is not to blame for the current way it works. */ + + case OP_RECURSE: + { + recursion_info *ri; + unsigned int recno; + + callpat = md->start_code + GET(ecode, 1); + recno = (callpat == md->start_code)? 0 : + GET2(callpat, 1 + LINK_SIZE); + + /* Check for repeating a recursion without advancing the subject pointer. + This should catch convoluted mutual recursions. (Some simple cases are + caught at compile time.) */ + + for (ri = md->recursive; ri != NULL; ri = ri->prevrec) + if (recno == ri->group_num && eptr == ri->subject_position) + RRETURN(PCRE_ERROR_RECURSELOOP); + + /* Add to "recursing stack" */ + + new_recursive.group_num = recno; + new_recursive.saved_capture_last = md->capture_last; + new_recursive.subject_position = eptr; + new_recursive.prevrec = md->recursive; + md->recursive = &new_recursive; + + /* Where to continue from afterwards */ + + ecode += 1 + LINK_SIZE; + + /* Now save the offset data */ + + new_recursive.saved_max = md->offset_end; + if (new_recursive.saved_max <= REC_STACK_SAVE_MAX) + new_recursive.offset_save = stacksave; + else + { + new_recursive.offset_save = + (int *)(PUBL(malloc))(new_recursive.saved_max * sizeof(int)); + if (new_recursive.offset_save == NULL) RRETURN(PCRE_ERROR_NOMEMORY); + } + memcpy(new_recursive.offset_save, md->offset_vector, + new_recursive.saved_max * sizeof(int)); + + /* OK, now we can do the recursion. After processing each alternative, + restore the offset data and the last captured value. If there were nested + recursions, md->recursive might be changed, so reset it before looping. + */ + + DPRINTF(("Recursing into group %d\n", new_recursive.group_num)); + cbegroup = (*callpat >= OP_SBRA); + do + { + if (cbegroup) md->match_function_type = MATCH_CBEGROUP; + RMATCH(eptr, callpat + PRIV(OP_lengths)[*callpat], offset_top, + md, eptrb, RM6); + memcpy(md->offset_vector, new_recursive.offset_save, + new_recursive.saved_max * sizeof(int)); + md->capture_last = new_recursive.saved_capture_last; + md->recursive = new_recursive.prevrec; + if (rrc == MATCH_MATCH || rrc == MATCH_ACCEPT) + { + DPRINTF(("Recursion matched\n")); + if (new_recursive.offset_save != stacksave) + (PUBL(free))(new_recursive.offset_save); + + /* Set where we got to in the subject, and reset the start in case + it was changed by \K. This *is* propagated back out of a recursion, + for Perl compatibility. */ + + eptr = md->end_match_ptr; + mstart = md->start_match_ptr; + goto RECURSION_MATCHED; /* Exit loop; end processing */ + } + + /* PCRE does not allow THEN, SKIP, PRUNE or COMMIT to escape beyond a + recursion; they cause a NOMATCH for the entire recursion. These codes + are defined in a range that can be tested for. */ + + if (rrc >= MATCH_BACKTRACK_MIN && rrc <= MATCH_BACKTRACK_MAX) + { + if (new_recursive.offset_save != stacksave) + (PUBL(free))(new_recursive.offset_save); + RRETURN(MATCH_NOMATCH); + } + + /* Any return code other than NOMATCH is an error. */ + + if (rrc != MATCH_NOMATCH) + { + DPRINTF(("Recursion gave error %d\n", rrc)); + if (new_recursive.offset_save != stacksave) + (PUBL(free))(new_recursive.offset_save); + RRETURN(rrc); + } + + md->recursive = &new_recursive; + callpat += GET(callpat, 1); + } + while (*callpat == OP_ALT); + + DPRINTF(("Recursion didn't match\n")); + md->recursive = new_recursive.prevrec; + if (new_recursive.offset_save != stacksave) + (PUBL(free))(new_recursive.offset_save); + RRETURN(MATCH_NOMATCH); + } + + RECURSION_MATCHED: + break; + + /* An alternation is the end of a branch; scan along to find the end of the + bracketed group and go to there. */ + + case OP_ALT: + do ecode += GET(ecode,1); while (*ecode == OP_ALT); + break; + + /* BRAZERO, BRAMINZERO and SKIPZERO occur just before a bracket group, + indicating that it may occur zero times. It may repeat infinitely, or not + at all - i.e. it could be ()* or ()? or even (){0} in the pattern. Brackets + with fixed upper repeat limits are compiled as a number of copies, with the + optional ones preceded by BRAZERO or BRAMINZERO. */ + + case OP_BRAZERO: + next = ecode + 1; + RMATCH(eptr, next, offset_top, md, eptrb, RM10); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + do next += GET(next, 1); while (*next == OP_ALT); + ecode = next + 1 + LINK_SIZE; + break; + + case OP_BRAMINZERO: + next = ecode + 1; + do next += GET(next, 1); while (*next == OP_ALT); + RMATCH(eptr, next + 1+LINK_SIZE, offset_top, md, eptrb, RM11); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + ecode++; + break; + + case OP_SKIPZERO: + next = ecode+1; + do next += GET(next,1); while (*next == OP_ALT); + ecode = next + 1 + LINK_SIZE; + break; + + /* BRAPOSZERO occurs before a possessive bracket group. Don't do anything + here; just jump to the group, with allow_zero set TRUE. */ + + case OP_BRAPOSZERO: + op = *(++ecode); + allow_zero = TRUE; + if (op == OP_CBRAPOS || op == OP_SCBRAPOS) goto POSSESSIVE_CAPTURE; + goto POSSESSIVE_NON_CAPTURE; + + /* End of a group, repeated or non-repeating. */ + + case OP_KET: + case OP_KETRMIN: + case OP_KETRMAX: + case OP_KETRPOS: + prev = ecode - GET(ecode, 1); + + /* If this was a group that remembered the subject start, in order to break + infinite repeats of empty string matches, retrieve the subject start from + the chain. Otherwise, set it NULL. */ + + if (*prev >= OP_SBRA || *prev == OP_ONCE) + { + saved_eptr = eptrb->epb_saved_eptr; /* Value at start of group */ + eptrb = eptrb->epb_prev; /* Backup to previous group */ + } + else saved_eptr = NULL; + + /* If we are at the end of an assertion group or a non-capturing atomic + group, stop matching and return MATCH_MATCH, but record the current high + water mark for use by positive assertions. We also need to record the match + start in case it was changed by \K. */ + + if ((*prev >= OP_ASSERT && *prev <= OP_ASSERTBACK_NOT) || + *prev == OP_ONCE_NC) + { + md->end_match_ptr = eptr; /* For ONCE_NC */ + md->end_offset_top = offset_top; + md->start_match_ptr = mstart; + RRETURN(MATCH_MATCH); /* Sets md->mark */ + } + + /* For capturing groups we have to check the group number back at the start + and if necessary complete handling an extraction by setting the offsets and + bumping the high water mark. Whole-pattern recursion is coded as a recurse + into group 0, so it won't be picked up here. Instead, we catch it when the + OP_END is reached. Other recursion is handled here. We just have to record + the current subject position and start match pointer and give a MATCH + return. */ + + if (*prev == OP_CBRA || *prev == OP_SCBRA || + *prev == OP_CBRAPOS || *prev == OP_SCBRAPOS) + { + number = GET2(prev, 1+LINK_SIZE); + offset = number << 1; + +#ifdef PCRE_DEBUG + printf("end bracket %d", number); + printf("\n"); +#endif + + /* Handle a recursively called group. */ + + if (md->recursive != NULL && md->recursive->group_num == number) + { + md->end_match_ptr = eptr; + md->start_match_ptr = mstart; + RRETURN(MATCH_MATCH); + } + + /* Deal with capturing */ + + md->capture_last = (md->capture_last & OVFLMASK) | number; + if (offset >= md->offset_max) md->capture_last |= OVFLBIT; else + { + /* If offset is greater than offset_top, it means that we are + "skipping" a capturing group, and that group's offsets must be marked + unset. In earlier versions of PCRE, all the offsets were unset at the + start of matching, but this doesn't work because atomic groups and + assertions can cause a value to be set that should later be unset. + Example: matching /(?>(a))b|(a)c/ against "ac". This sets group 1 as + part of the atomic group, but this is not on the final matching path, + so must be unset when 2 is set. (If there is no group 2, there is no + problem, because offset_top will then be 2, indicating no capture.) */ + + if (offset > offset_top) + { + register int *iptr = md->offset_vector + offset_top; + register int *iend = md->offset_vector + offset; + while (iptr < iend) *iptr++ = -1; + } + + /* Now make the extraction */ + + md->offset_vector[offset] = + md->offset_vector[md->offset_end - number]; + md->offset_vector[offset+1] = (int)(eptr - md->start_subject); + if (offset_top <= offset) offset_top = offset + 2; + } + } + + /* OP_KETRPOS is a possessive repeating ket. Remember the current position, + and return the MATCH_KETRPOS. This makes it possible to do the repeats one + at a time from the outer level, thus saving stack. This must precede the + empty string test - in this case that test is done at the outer level. */ + + if (*ecode == OP_KETRPOS) + { + md->start_match_ptr = mstart; /* In case \K reset it */ + md->end_match_ptr = eptr; + md->end_offset_top = offset_top; + RRETURN(MATCH_KETRPOS); + } + + /* For an ordinary non-repeating ket, just continue at this level. This + also happens for a repeating ket if no characters were matched in the + group. This is the forcible breaking of infinite loops as implemented in + Perl 5.005. For a non-repeating atomic group that includes captures, + establish a backup point by processing the rest of the pattern at a lower + level. If this results in a NOMATCH return, pass MATCH_ONCE back to the + original OP_ONCE level, thereby bypassing intermediate backup points, but + resetting any captures that happened along the way. */ + + if (*ecode == OP_KET || eptr == saved_eptr) + { + if (*prev == OP_ONCE) + { + RMATCH(eptr, ecode + 1 + LINK_SIZE, offset_top, md, eptrb, RM12); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + md->once_target = prev; /* Level at which to change to MATCH_NOMATCH */ + RRETURN(MATCH_ONCE); + } + ecode += 1 + LINK_SIZE; /* Carry on at this level */ + break; + } + + /* The normal repeating kets try the rest of the pattern or restart from + the preceding bracket, in the appropriate order. In the second case, we can + use tail recursion to avoid using another stack frame, unless we have an + an atomic group or an unlimited repeat of a group that can match an empty + string. */ + + if (*ecode == OP_KETRMIN) + { + RMATCH(eptr, ecode + 1 + LINK_SIZE, offset_top, md, eptrb, RM7); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (*prev == OP_ONCE) + { + RMATCH(eptr, prev, offset_top, md, eptrb, RM8); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + md->once_target = prev; /* Level at which to change to MATCH_NOMATCH */ + RRETURN(MATCH_ONCE); + } + if (*prev >= OP_SBRA) /* Could match an empty string */ + { + RMATCH(eptr, prev, offset_top, md, eptrb, RM50); + RRETURN(rrc); + } + ecode = prev; + goto TAIL_RECURSE; + } + else /* OP_KETRMAX */ + { + RMATCH(eptr, prev, offset_top, md, eptrb, RM13); + if (rrc == MATCH_ONCE && md->once_target == prev) rrc = MATCH_NOMATCH; + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (*prev == OP_ONCE) + { + RMATCH(eptr, ecode + 1 + LINK_SIZE, offset_top, md, eptrb, RM9); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + md->once_target = prev; + RRETURN(MATCH_ONCE); + } + ecode += 1 + LINK_SIZE; + goto TAIL_RECURSE; + } + /* Control never gets here */ + + /* Not multiline mode: start of subject assertion, unless notbol. */ + + case OP_CIRC: + if (md->notbol && eptr == md->start_subject) RRETURN(MATCH_NOMATCH); + + /* Start of subject assertion */ + + case OP_SOD: + if (eptr != md->start_subject) RRETURN(MATCH_NOMATCH); + ecode++; + break; + + /* Multiline mode: start of subject unless notbol, or after any newline. */ + + case OP_CIRCM: + if (md->notbol && eptr == md->start_subject) RRETURN(MATCH_NOMATCH); + if (eptr != md->start_subject && + (eptr == md->end_subject || !WAS_NEWLINE(eptr))) + RRETURN(MATCH_NOMATCH); + ecode++; + break; + + /* Start of match assertion */ + + case OP_SOM: + if (eptr != md->start_subject + md->start_offset) RRETURN(MATCH_NOMATCH); + ecode++; + break; + + /* Reset the start of match point */ + + case OP_SET_SOM: + mstart = eptr; + ecode++; + break; + + /* Multiline mode: assert before any newline, or before end of subject + unless noteol is set. */ + + case OP_DOLLM: + if (eptr < md->end_subject) + { + if (!IS_NEWLINE(eptr)) + { + if (md->partial != 0 && + eptr + 1 >= md->end_subject && + NLBLOCK->nltype == NLTYPE_FIXED && + NLBLOCK->nllen == 2 && + UCHAR21TEST(eptr) == NLBLOCK->nl[0]) + { + md->hitend = TRUE; + if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); + } + RRETURN(MATCH_NOMATCH); + } + } + else + { + if (md->noteol) RRETURN(MATCH_NOMATCH); + SCHECK_PARTIAL(); + } + ecode++; + break; + + /* Not multiline mode: assert before a terminating newline or before end of + subject unless noteol is set. */ + + case OP_DOLL: + if (md->noteol) RRETURN(MATCH_NOMATCH); + if (!md->endonly) goto ASSERT_NL_OR_EOS; + + /* ... else fall through for endonly */ + + /* End of subject assertion (\z) */ + + case OP_EOD: + if (eptr < md->end_subject) RRETURN(MATCH_NOMATCH); + SCHECK_PARTIAL(); + ecode++; + break; + + /* End of subject or ending \n assertion (\Z) */ + + case OP_EODN: + ASSERT_NL_OR_EOS: + if (eptr < md->end_subject && + (!IS_NEWLINE(eptr) || eptr != md->end_subject - md->nllen)) + { + if (md->partial != 0 && + eptr + 1 >= md->end_subject && + NLBLOCK->nltype == NLTYPE_FIXED && + NLBLOCK->nllen == 2 && + UCHAR21TEST(eptr) == NLBLOCK->nl[0]) + { + md->hitend = TRUE; + if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); + } + RRETURN(MATCH_NOMATCH); + } + + /* Either at end of string or \n before end. */ + + SCHECK_PARTIAL(); + ecode++; + break; + + /* Word boundary assertions */ + + case OP_NOT_WORD_BOUNDARY: + case OP_WORD_BOUNDARY: + { + + /* Find out if the previous and current characters are "word" characters. + It takes a bit more work in UTF-8 mode. Characters > 255 are assumed to + be "non-word" characters. Remember the earliest consulted character for + partial matching. */ + +#ifdef SUPPORT_UTF + if (utf) + { + /* Get status of previous character */ + + if (eptr == md->start_subject) prev_is_word = FALSE; else + { + PCRE_PUCHAR lastptr = eptr - 1; + BACKCHAR(lastptr); + if (lastptr < md->start_used_ptr) md->start_used_ptr = lastptr; + GETCHAR(c, lastptr); +#ifdef SUPPORT_UCP + if (md->use_ucp) + { + if (c == '_') prev_is_word = TRUE; else + { + int cat = UCD_CATEGORY(c); + prev_is_word = (cat == ucp_L || cat == ucp_N); + } + } + else +#endif + prev_is_word = c < 256 && (md->ctypes[c] & ctype_word) != 0; + } + + /* Get status of next character */ + + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + cur_is_word = FALSE; + } + else + { + GETCHAR(c, eptr); +#ifdef SUPPORT_UCP + if (md->use_ucp) + { + if (c == '_') cur_is_word = TRUE; else + { + int cat = UCD_CATEGORY(c); + cur_is_word = (cat == ucp_L || cat == ucp_N); + } + } + else +#endif + cur_is_word = c < 256 && (md->ctypes[c] & ctype_word) != 0; + } + } + else +#endif + + /* Not in UTF-8 mode, but we may still have PCRE_UCP set, and for + consistency with the behaviour of \w we do use it in this case. */ + + { + /* Get status of previous character */ + + if (eptr == md->start_subject) prev_is_word = FALSE; else + { + if (eptr <= md->start_used_ptr) md->start_used_ptr = eptr - 1; +#ifdef SUPPORT_UCP + if (md->use_ucp) + { + c = eptr[-1]; + if (c == '_') prev_is_word = TRUE; else + { + int cat = UCD_CATEGORY(c); + prev_is_word = (cat == ucp_L || cat == ucp_N); + } + } + else +#endif + prev_is_word = MAX_255(eptr[-1]) + && ((md->ctypes[eptr[-1]] & ctype_word) != 0); + } + + /* Get status of next character */ + + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + cur_is_word = FALSE; + } + else +#ifdef SUPPORT_UCP + if (md->use_ucp) + { + c = *eptr; + if (c == '_') cur_is_word = TRUE; else + { + int cat = UCD_CATEGORY(c); + cur_is_word = (cat == ucp_L || cat == ucp_N); + } + } + else +#endif + cur_is_word = MAX_255(*eptr) + && ((md->ctypes[*eptr] & ctype_word) != 0); + } + + /* Now see if the situation is what we want */ + + if ((*ecode++ == OP_WORD_BOUNDARY)? + cur_is_word == prev_is_word : cur_is_word != prev_is_word) + RRETURN(MATCH_NOMATCH); + } + break; + + /* Match any single character type except newline; have to take care with + CRLF newlines and partial matching. */ + + case OP_ANY: + if (IS_NEWLINE(eptr)) RRETURN(MATCH_NOMATCH); + if (md->partial != 0 && + eptr + 1 >= md->end_subject && + NLBLOCK->nltype == NLTYPE_FIXED && + NLBLOCK->nllen == 2 && + UCHAR21TEST(eptr) == NLBLOCK->nl[0]) + { + md->hitend = TRUE; + if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); + } + + /* Fall through */ + + /* Match any single character whatsoever. */ + + case OP_ALLANY: + if (eptr >= md->end_subject) /* DO NOT merge the eptr++ here; it must */ + { /* not be updated before SCHECK_PARTIAL. */ + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + eptr++; +#ifdef SUPPORT_UTF + if (utf) ACROSSCHAR(eptr < md->end_subject, *eptr, eptr++); +#endif + ecode++; + break; + + /* Match a single byte, even in UTF-8 mode. This opcode really does match + any byte, even newline, independent of the setting of PCRE_DOTALL. */ + + case OP_ANYBYTE: + if (eptr >= md->end_subject) /* DO NOT merge the eptr++ here; it must */ + { /* not be updated before SCHECK_PARTIAL. */ + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + eptr++; + ecode++; + break; + + case OP_NOT_DIGIT: + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + if ( +#if defined SUPPORT_UTF || !(defined COMPILE_PCRE8) + c < 256 && +#endif + (md->ctypes[c] & ctype_digit) != 0 + ) + RRETURN(MATCH_NOMATCH); + ecode++; + break; + + case OP_DIGIT: + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + if ( +#if defined SUPPORT_UTF || !(defined COMPILE_PCRE8) + c > 255 || +#endif + (md->ctypes[c] & ctype_digit) == 0 + ) + RRETURN(MATCH_NOMATCH); + ecode++; + break; + + case OP_NOT_WHITESPACE: + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + if ( +#if defined SUPPORT_UTF || !(defined COMPILE_PCRE8) + c < 256 && +#endif + (md->ctypes[c] & ctype_space) != 0 + ) + RRETURN(MATCH_NOMATCH); + ecode++; + break; + + case OP_WHITESPACE: + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + if ( +#if defined SUPPORT_UTF || !(defined COMPILE_PCRE8) + c > 255 || +#endif + (md->ctypes[c] & ctype_space) == 0 + ) + RRETURN(MATCH_NOMATCH); + ecode++; + break; + + case OP_NOT_WORDCHAR: + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + if ( +#if defined SUPPORT_UTF || !(defined COMPILE_PCRE8) + c < 256 && +#endif + (md->ctypes[c] & ctype_word) != 0 + ) + RRETURN(MATCH_NOMATCH); + ecode++; + break; + + case OP_WORDCHAR: + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + if ( +#if defined SUPPORT_UTF || !(defined COMPILE_PCRE8) + c > 255 || +#endif + (md->ctypes[c] & ctype_word) == 0 + ) + RRETURN(MATCH_NOMATCH); + ecode++; + break; + + case OP_ANYNL: + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + switch(c) + { + default: RRETURN(MATCH_NOMATCH); + + case CHAR_CR: + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + } + else if (UCHAR21TEST(eptr) == CHAR_LF) eptr++; + break; + + case CHAR_LF: + break; + + case CHAR_VT: + case CHAR_FF: + case CHAR_NEL: +#ifndef EBCDIC + case 0x2028: + case 0x2029: +#endif /* Not EBCDIC */ + if (md->bsr_anycrlf) RRETURN(MATCH_NOMATCH); + break; + } + ecode++; + break; + + case OP_NOT_HSPACE: + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + switch(c) + { + HSPACE_CASES: RRETURN(MATCH_NOMATCH); /* Byte and multibyte cases */ + default: break; + } + ecode++; + break; + + case OP_HSPACE: + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + switch(c) + { + HSPACE_CASES: break; /* Byte and multibyte cases */ + default: RRETURN(MATCH_NOMATCH); + } + ecode++; + break; + + case OP_NOT_VSPACE: + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + switch(c) + { + VSPACE_CASES: RRETURN(MATCH_NOMATCH); + default: break; + } + ecode++; + break; + + case OP_VSPACE: + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + switch(c) + { + VSPACE_CASES: break; + default: RRETURN(MATCH_NOMATCH); + } + ecode++; + break; + +#ifdef SUPPORT_UCP + /* Check the next character by Unicode property. We will get here only + if the support is in the binary; otherwise a compile-time error occurs. */ + + case OP_PROP: + case OP_NOTPROP: + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + { + const pcre_uint32 *cp; + const ucd_record *prop = GET_UCD(c); + + switch(ecode[1]) + { + case PT_ANY: + if (op == OP_NOTPROP) RRETURN(MATCH_NOMATCH); + break; + + case PT_LAMP: + if ((prop->chartype == ucp_Lu || + prop->chartype == ucp_Ll || + prop->chartype == ucp_Lt) == (op == OP_NOTPROP)) + RRETURN(MATCH_NOMATCH); + break; + + case PT_GC: + if ((ecode[2] != PRIV(ucp_gentype)[prop->chartype]) == (op == OP_PROP)) + RRETURN(MATCH_NOMATCH); + break; + + case PT_PC: + if ((ecode[2] != prop->chartype) == (op == OP_PROP)) + RRETURN(MATCH_NOMATCH); + break; + + case PT_SC: + if ((ecode[2] != prop->script) == (op == OP_PROP)) + RRETURN(MATCH_NOMATCH); + break; + + /* These are specials */ + + case PT_ALNUM: + if ((PRIV(ucp_gentype)[prop->chartype] == ucp_L || + PRIV(ucp_gentype)[prop->chartype] == ucp_N) == (op == OP_NOTPROP)) + RRETURN(MATCH_NOMATCH); + break; + + /* Perl space used to exclude VT, but from Perl 5.18 it is included, + which means that Perl space and POSIX space are now identical. PCRE + was changed at release 8.34. */ + + case PT_SPACE: /* Perl space */ + case PT_PXSPACE: /* POSIX space */ + switch(c) + { + HSPACE_CASES: + VSPACE_CASES: + if (op == OP_NOTPROP) RRETURN(MATCH_NOMATCH); + break; + + default: + if ((PRIV(ucp_gentype)[prop->chartype] == ucp_Z) == + (op == OP_NOTPROP)) RRETURN(MATCH_NOMATCH); + break; + } + break; + + case PT_WORD: + if ((PRIV(ucp_gentype)[prop->chartype] == ucp_L || + PRIV(ucp_gentype)[prop->chartype] == ucp_N || + c == CHAR_UNDERSCORE) == (op == OP_NOTPROP)) + RRETURN(MATCH_NOMATCH); + break; + + case PT_CLIST: + cp = PRIV(ucd_caseless_sets) + ecode[2]; + for (;;) + { + if (c < *cp) + { if (op == OP_PROP) { RRETURN(MATCH_NOMATCH); } else break; } + if (c == *cp++) + { if (op == OP_PROP) break; else { RRETURN(MATCH_NOMATCH); } } + } + break; + + case PT_UCNC: + if ((c == CHAR_DOLLAR_SIGN || c == CHAR_COMMERCIAL_AT || + c == CHAR_GRAVE_ACCENT || (c >= 0xa0 && c <= 0xd7ff) || + c >= 0xe000) == (op == OP_NOTPROP)) + RRETURN(MATCH_NOMATCH); + break; + + /* This should never occur */ + + default: + RRETURN(PCRE_ERROR_INTERNAL); + } + + ecode += 3; + } + break; + + /* Match an extended Unicode sequence. We will get here only if the support + is in the binary; otherwise a compile-time error occurs. */ + + case OP_EXTUNI: + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + else + { + int lgb, rgb; + GETCHARINCTEST(c, eptr); + lgb = UCD_GRAPHBREAK(c); + while (eptr < md->end_subject) + { + int len = 1; + if (!utf) c = *eptr; else { GETCHARLEN(c, eptr, len); } + rgb = UCD_GRAPHBREAK(c); + if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; + lgb = rgb; + eptr += len; + } + } + CHECK_PARTIAL(); + ecode++; + break; +#endif /* SUPPORT_UCP */ + + + /* Match a back reference, possibly repeatedly. Look past the end of the + item to see if there is repeat information following. The code is similar + to that for character classes, but repeated for efficiency. Then obey + similar code to character type repeats - written out again for speed. + However, if the referenced string is the empty string, always treat + it as matched, any number of times (otherwise there could be infinite + loops). If the reference is unset, there are two possibilities: + + (a) In the default, Perl-compatible state, set the length negative; + this ensures that every attempt at a match fails. We can't just fail + here, because of the possibility of quantifiers with zero minima. + + (b) If the JavaScript compatibility flag is set, set the length to zero + so that the back reference matches an empty string. + + Otherwise, set the length to the length of what was matched by the + referenced subpattern. + + The OP_REF and OP_REFI opcodes are used for a reference to a numbered group + or to a non-duplicated named group. For a duplicated named group, OP_DNREF + and OP_DNREFI are used. In this case we must scan the list of groups to + which the name refers, and use the first one that is set. */ + + case OP_DNREF: + case OP_DNREFI: + caseless = op == OP_DNREFI; + { + int count = GET2(ecode, 1+IMM2_SIZE); + pcre_uchar *slot = md->name_table + GET2(ecode, 1) * md->name_entry_size; + ecode += 1 + 2*IMM2_SIZE; + + /* Setting the default length first and initializing 'offset' avoids + compiler warnings in the REF_REPEAT code. */ + + length = (md->jscript_compat)? 0 : -1; + offset = 0; + + while (count-- > 0) + { + offset = GET2(slot, 0) << 1; + if (offset < offset_top && md->offset_vector[offset] >= 0) + { + length = md->offset_vector[offset+1] - md->offset_vector[offset]; + break; + } + slot += md->name_entry_size; + } + } + goto REF_REPEAT; + + case OP_REF: + case OP_REFI: + caseless = op == OP_REFI; + offset = GET2(ecode, 1) << 1; /* Doubled ref number */ + ecode += 1 + IMM2_SIZE; + if (offset >= offset_top || md->offset_vector[offset] < 0) + length = (md->jscript_compat)? 0 : -1; + else + length = md->offset_vector[offset+1] - md->offset_vector[offset]; + + /* Set up for repetition, or handle the non-repeated case */ + + REF_REPEAT: + switch (*ecode) + { + case OP_CRSTAR: + case OP_CRMINSTAR: + case OP_CRPLUS: + case OP_CRMINPLUS: + case OP_CRQUERY: + case OP_CRMINQUERY: + c = *ecode++ - OP_CRSTAR; + minimize = (c & 1) != 0; + min = rep_min[c]; /* Pick up values from tables; */ + max = rep_max[c]; /* zero for max => infinity */ + if (max == 0) max = INT_MAX; + break; + + case OP_CRRANGE: + case OP_CRMINRANGE: + minimize = (*ecode == OP_CRMINRANGE); + min = GET2(ecode, 1); + max = GET2(ecode, 1 + IMM2_SIZE); + if (max == 0) max = INT_MAX; + ecode += 1 + 2 * IMM2_SIZE; + break; + + default: /* No repeat follows */ + if ((length = match_ref(offset, eptr, length, md, caseless)) < 0) + { + if (length == -2) eptr = md->end_subject; /* Partial match */ + CHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + eptr += length; + continue; /* With the main loop */ + } + + /* Handle repeated back references. If the length of the reference is + zero, just continue with the main loop. If the length is negative, it + means the reference is unset in non-Java-compatible mode. If the minimum is + zero, we can continue at the same level without recursion. For any other + minimum, carrying on will result in NOMATCH. */ + + if (length == 0) continue; + if (length < 0 && min == 0) continue; + + /* First, ensure the minimum number of matches are present. We get back + the length of the reference string explicitly rather than passing the + address of eptr, so that eptr can be a register variable. */ + + for (i = 1; i <= min; i++) + { + int slength; + if ((slength = match_ref(offset, eptr, length, md, caseless)) < 0) + { + if (slength == -2) eptr = md->end_subject; /* Partial match */ + CHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + eptr += slength; + } + + /* If min = max, continue at the same level without recursion. + They are not both allowed to be zero. */ + + if (min == max) continue; + + /* If minimizing, keep trying and advancing the pointer */ + + if (minimize) + { + for (fi = min;; fi++) + { + int slength; + RMATCH(eptr, ecode, offset_top, md, eptrb, RM14); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if ((slength = match_ref(offset, eptr, length, md, caseless)) < 0) + { + if (slength == -2) eptr = md->end_subject; /* Partial match */ + CHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + eptr += slength; + } + /* Control never gets here */ + } + + /* If maximizing, find the longest string and work backwards */ + + else + { + pp = eptr; + for (i = min; i < max; i++) + { + int slength; + if ((slength = match_ref(offset, eptr, length, md, caseless)) < 0) + { + /* Can't use CHECK_PARTIAL because we don't want to update eptr in + the soft partial matching case. */ + + if (slength == -2 && md->partial != 0 && + md->end_subject > md->start_used_ptr) + { + md->hitend = TRUE; + if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); + } + break; + } + eptr += slength; + } + + while (eptr >= pp) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM15); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + eptr -= length; + } + RRETURN(MATCH_NOMATCH); + } + /* Control never gets here */ + + /* Match a bit-mapped character class, possibly repeatedly. This op code is + used when all the characters in the class have values in the range 0-255, + and either the matching is caseful, or the characters are in the range + 0-127 when UTF-8 processing is enabled. The only difference between + OP_CLASS and OP_NCLASS occurs when a data character outside the range is + encountered. + + First, look past the end of the item to see if there is repeat information + following. Then obey similar code to character type repeats - written out + again for speed. */ + + case OP_NCLASS: + case OP_CLASS: + { + /* The data variable is saved across frames, so the byte map needs to + be stored there. */ +#define BYTE_MAP ((pcre_uint8 *)data) + data = ecode + 1; /* Save for matching */ + ecode += 1 + (32 / sizeof(pcre_uchar)); /* Advance past the item */ + + switch (*ecode) + { + case OP_CRSTAR: + case OP_CRMINSTAR: + case OP_CRPLUS: + case OP_CRMINPLUS: + case OP_CRQUERY: + case OP_CRMINQUERY: + case OP_CRPOSSTAR: + case OP_CRPOSPLUS: + case OP_CRPOSQUERY: + c = *ecode++ - OP_CRSTAR; + if (c < OP_CRPOSSTAR - OP_CRSTAR) minimize = (c & 1) != 0; + else possessive = TRUE; + min = rep_min[c]; /* Pick up values from tables; */ + max = rep_max[c]; /* zero for max => infinity */ + if (max == 0) max = INT_MAX; + break; + + case OP_CRRANGE: + case OP_CRMINRANGE: + case OP_CRPOSRANGE: + minimize = (*ecode == OP_CRMINRANGE); + possessive = (*ecode == OP_CRPOSRANGE); + min = GET2(ecode, 1); + max = GET2(ecode, 1 + IMM2_SIZE); + if (max == 0) max = INT_MAX; + ecode += 1 + 2 * IMM2_SIZE; + break; + + default: /* No repeat follows */ + min = max = 1; + break; + } + + /* First, ensure the minimum number of matches are present. */ + +#ifdef SUPPORT_UTF + if (utf) + { + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINC(c, eptr); + if (c > 255) + { + if (op == OP_CLASS) RRETURN(MATCH_NOMATCH); + } + else + if ((BYTE_MAP[c/8] & (1 << (c&7))) == 0) RRETURN(MATCH_NOMATCH); + } + } + else +#endif + /* Not UTF mode */ + { + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + c = *eptr++; +#ifndef COMPILE_PCRE8 + if (c > 255) + { + if (op == OP_CLASS) RRETURN(MATCH_NOMATCH); + } + else +#endif + if ((BYTE_MAP[c/8] & (1 << (c&7))) == 0) RRETURN(MATCH_NOMATCH); + } + } + + /* If max == min we can continue with the main loop without the + need to recurse. */ + + if (min == max) continue; + + /* If minimizing, keep testing the rest of the expression and advancing + the pointer while it matches the class. */ + + if (minimize) + { +#ifdef SUPPORT_UTF + if (utf) + { + for (fi = min;; fi++) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM16); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINC(c, eptr); + if (c > 255) + { + if (op == OP_CLASS) RRETURN(MATCH_NOMATCH); + } + else + if ((BYTE_MAP[c/8] & (1 << (c&7))) == 0) RRETURN(MATCH_NOMATCH); + } + } + else +#endif + /* Not UTF mode */ + { + for (fi = min;; fi++) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM17); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + c = *eptr++; +#ifndef COMPILE_PCRE8 + if (c > 255) + { + if (op == OP_CLASS) RRETURN(MATCH_NOMATCH); + } + else +#endif + if ((BYTE_MAP[c/8] & (1 << (c&7))) == 0) RRETURN(MATCH_NOMATCH); + } + } + /* Control never gets here */ + } + + /* If maximizing, find the longest possible run, then work backwards. */ + + else + { + pp = eptr; + +#ifdef SUPPORT_UTF + if (utf) + { + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLEN(c, eptr, len); + if (c > 255) + { + if (op == OP_CLASS) break; + } + else + if ((BYTE_MAP[c/8] & (1 << (c&7))) == 0) break; + eptr += len; + } + + if (possessive) continue; /* No backtracking */ + + for (;;) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM18); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (eptr-- == pp) break; /* Stop if tried at original pos */ + BACKCHAR(eptr); + } + } + else +#endif + /* Not UTF mode */ + { + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + c = *eptr; +#ifndef COMPILE_PCRE8 + if (c > 255) + { + if (op == OP_CLASS) break; + } + else +#endif + if ((BYTE_MAP[c/8] & (1 << (c&7))) == 0) break; + eptr++; + } + + if (possessive) continue; /* No backtracking */ + + while (eptr >= pp) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM19); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + eptr--; + } + } + + RRETURN(MATCH_NOMATCH); + } +#undef BYTE_MAP + } + /* Control never gets here */ + + + /* Match an extended character class. In the 8-bit library, this opcode is + encountered only when UTF-8 mode mode is supported. In the 16-bit and + 32-bit libraries, codepoints greater than 255 may be encountered even when + UTF is not supported. */ + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: + { + data = ecode + 1 + LINK_SIZE; /* Save for matching */ + ecode += GET(ecode, 1); /* Advance past the item */ + + switch (*ecode) + { + case OP_CRSTAR: + case OP_CRMINSTAR: + case OP_CRPLUS: + case OP_CRMINPLUS: + case OP_CRQUERY: + case OP_CRMINQUERY: + case OP_CRPOSSTAR: + case OP_CRPOSPLUS: + case OP_CRPOSQUERY: + c = *ecode++ - OP_CRSTAR; + if (c < OP_CRPOSSTAR - OP_CRSTAR) minimize = (c & 1) != 0; + else possessive = TRUE; + min = rep_min[c]; /* Pick up values from tables; */ + max = rep_max[c]; /* zero for max => infinity */ + if (max == 0) max = INT_MAX; + break; + + case OP_CRRANGE: + case OP_CRMINRANGE: + case OP_CRPOSRANGE: + minimize = (*ecode == OP_CRMINRANGE); + possessive = (*ecode == OP_CRPOSRANGE); + min = GET2(ecode, 1); + max = GET2(ecode, 1 + IMM2_SIZE); + if (max == 0) max = INT_MAX; + ecode += 1 + 2 * IMM2_SIZE; + break; + + default: /* No repeat follows */ + min = max = 1; + break; + } + + /* First, ensure the minimum number of matches are present. */ + + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + if (!PRIV(xclass)(c, data, utf)) RRETURN(MATCH_NOMATCH); + } + + /* If max == min we can continue with the main loop without the + need to recurse. */ + + if (min == max) continue; + + /* If minimizing, keep testing the rest of the expression and advancing + the pointer while it matches the class. */ + + if (minimize) + { + for (fi = min;; fi++) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM20); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + if (!PRIV(xclass)(c, data, utf)) RRETURN(MATCH_NOMATCH); + } + /* Control never gets here */ + } + + /* If maximizing, find the longest possible run, then work backwards. */ + + else + { + pp = eptr; + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } +#ifdef SUPPORT_UTF + GETCHARLENTEST(c, eptr, len); +#else + c = *eptr; +#endif + if (!PRIV(xclass)(c, data, utf)) break; + eptr += len; + } + + if (possessive) continue; /* No backtracking */ + + for(;;) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM21); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (eptr-- == pp) break; /* Stop if tried at original pos */ +#ifdef SUPPORT_UTF + if (utf) BACKCHAR(eptr); +#endif + } + RRETURN(MATCH_NOMATCH); + } + + /* Control never gets here */ + } +#endif /* End of XCLASS */ + + /* Match a single character, casefully */ + + case OP_CHAR: +#ifdef SUPPORT_UTF + if (utf) + { + length = 1; + ecode++; + GETCHARLEN(fc, ecode, length); + if (length > md->end_subject - eptr) + { + CHECK_PARTIAL(); /* Not SCHECK_PARTIAL() */ + RRETURN(MATCH_NOMATCH); + } + while (length-- > 0) if (*ecode++ != UCHAR21INC(eptr)) RRETURN(MATCH_NOMATCH); + } + else +#endif + /* Not UTF mode */ + { + if (md->end_subject - eptr < 1) + { + SCHECK_PARTIAL(); /* This one can use SCHECK_PARTIAL() */ + RRETURN(MATCH_NOMATCH); + } + if (ecode[1] != *eptr++) RRETURN(MATCH_NOMATCH); + ecode += 2; + } + break; + + /* Match a single character, caselessly. If we are at the end of the + subject, give up immediately. */ + + case OP_CHARI: + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + +#ifdef SUPPORT_UTF + if (utf) + { + length = 1; + ecode++; + GETCHARLEN(fc, ecode, length); + + /* If the pattern character's value is < 128, we have only one byte, and + we know that its other case must also be one byte long, so we can use the + fast lookup table. We know that there is at least one byte left in the + subject. */ + + if (fc < 128) + { + pcre_uint32 cc = UCHAR21(eptr); + if (md->lcc[fc] != TABLE_GET(cc, md->lcc, cc)) RRETURN(MATCH_NOMATCH); + ecode++; + eptr++; + } + + /* Otherwise we must pick up the subject character. Note that we cannot + use the value of "length" to check for sufficient bytes left, because the + other case of the character may have more or fewer bytes. */ + + else + { + pcre_uint32 dc; + GETCHARINC(dc, eptr); + ecode += length; + + /* If we have Unicode property support, we can use it to test the other + case of the character, if there is one. */ + + if (fc != dc) + { +#ifdef SUPPORT_UCP + if (dc != UCD_OTHERCASE(fc)) +#endif + RRETURN(MATCH_NOMATCH); + } + } + } + else +#endif /* SUPPORT_UTF */ + + /* Not UTF mode */ + { + if (TABLE_GET(ecode[1], md->lcc, ecode[1]) + != TABLE_GET(*eptr, md->lcc, *eptr)) RRETURN(MATCH_NOMATCH); + eptr++; + ecode += 2; + } + break; + + /* Match a single character repeatedly. */ + + case OP_EXACT: + case OP_EXACTI: + min = max = GET2(ecode, 1); + ecode += 1 + IMM2_SIZE; + goto REPEATCHAR; + + case OP_POSUPTO: + case OP_POSUPTOI: + possessive = TRUE; + /* Fall through */ + + case OP_UPTO: + case OP_UPTOI: + case OP_MINUPTO: + case OP_MINUPTOI: + min = 0; + max = GET2(ecode, 1); + minimize = *ecode == OP_MINUPTO || *ecode == OP_MINUPTOI; + ecode += 1 + IMM2_SIZE; + goto REPEATCHAR; + + case OP_POSSTAR: + case OP_POSSTARI: + possessive = TRUE; + min = 0; + max = INT_MAX; + ecode++; + goto REPEATCHAR; + + case OP_POSPLUS: + case OP_POSPLUSI: + possessive = TRUE; + min = 1; + max = INT_MAX; + ecode++; + goto REPEATCHAR; + + case OP_POSQUERY: + case OP_POSQUERYI: + possessive = TRUE; + min = 0; + max = 1; + ecode++; + goto REPEATCHAR; + + case OP_STAR: + case OP_STARI: + case OP_MINSTAR: + case OP_MINSTARI: + case OP_PLUS: + case OP_PLUSI: + case OP_MINPLUS: + case OP_MINPLUSI: + case OP_QUERY: + case OP_QUERYI: + case OP_MINQUERY: + case OP_MINQUERYI: + c = *ecode++ - ((op < OP_STARI)? OP_STAR : OP_STARI); + minimize = (c & 1) != 0; + min = rep_min[c]; /* Pick up values from tables; */ + max = rep_max[c]; /* zero for max => infinity */ + if (max == 0) max = INT_MAX; + + /* Common code for all repeated single-character matches. We first check + for the minimum number of characters. If the minimum equals the maximum, we + are done. Otherwise, if minimizing, check the rest of the pattern for a + match; if there isn't one, advance up to the maximum, one character at a + time. + + If maximizing, advance up to the maximum number of matching characters, + until eptr is past the end of the maximum run. If possessive, we are + then done (no backing up). Otherwise, match at this position; anything + other than no match is immediately returned. For nomatch, back up one + character, unless we are matching \R and the last thing matched was + \r\n, in which case, back up two bytes. When we reach the first optional + character position, we can save stack by doing a tail recurse. + + The various UTF/non-UTF and caseful/caseless cases are handled separately, + for speed. */ + + REPEATCHAR: +#ifdef SUPPORT_UTF + if (utf) + { + length = 1; + charptr = ecode; + GETCHARLEN(fc, ecode, length); + ecode += length; + + /* Handle multibyte character matching specially here. There is + support for caseless matching if UCP support is present. */ + + if (length > 1) + { +#ifdef SUPPORT_UCP + pcre_uint32 othercase; + if (op >= OP_STARI && /* Caseless */ + (othercase = UCD_OTHERCASE(fc)) != fc) + oclength = PRIV(ord2utf)(othercase, occhars); + else oclength = 0; +#endif /* SUPPORT_UCP */ + + for (i = 1; i <= min; i++) + { + if (eptr <= md->end_subject - length && + memcmp(eptr, charptr, IN_UCHARS(length)) == 0) eptr += length; +#ifdef SUPPORT_UCP + else if (oclength > 0 && + eptr <= md->end_subject - oclength && + memcmp(eptr, occhars, IN_UCHARS(oclength)) == 0) eptr += oclength; +#endif /* SUPPORT_UCP */ + else + { + CHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + } + + if (min == max) continue; + + if (minimize) + { + for (fi = min;; fi++) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM22); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr <= md->end_subject - length && + memcmp(eptr, charptr, IN_UCHARS(length)) == 0) eptr += length; +#ifdef SUPPORT_UCP + else if (oclength > 0 && + eptr <= md->end_subject - oclength && + memcmp(eptr, occhars, IN_UCHARS(oclength)) == 0) eptr += oclength; +#endif /* SUPPORT_UCP */ + else + { + CHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + } + /* Control never gets here */ + } + + else /* Maximize */ + { + pp = eptr; + for (i = min; i < max; i++) + { + if (eptr <= md->end_subject - length && + memcmp(eptr, charptr, IN_UCHARS(length)) == 0) eptr += length; +#ifdef SUPPORT_UCP + else if (oclength > 0 && + eptr <= md->end_subject - oclength && + memcmp(eptr, occhars, IN_UCHARS(oclength)) == 0) eptr += oclength; +#endif /* SUPPORT_UCP */ + else + { + CHECK_PARTIAL(); + break; + } + } + + if (possessive) continue; /* No backtracking */ + for(;;) + { + if (eptr <= pp) goto TAIL_RECURSE; + RMATCH(eptr, ecode, offset_top, md, eptrb, RM23); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); +#ifdef SUPPORT_UCP + eptr--; + BACKCHAR(eptr); +#else /* without SUPPORT_UCP */ + eptr -= length; +#endif /* SUPPORT_UCP */ + } + } + /* Control never gets here */ + } + + /* If the length of a UTF-8 character is 1, we fall through here, and + obey the code as for non-UTF-8 characters below, though in this case the + value of fc will always be < 128. */ + } + else +#endif /* SUPPORT_UTF */ + /* When not in UTF-8 mode, load a single-byte character. */ + fc = *ecode++; + + /* The value of fc at this point is always one character, though we may + or may not be in UTF mode. The code is duplicated for the caseless and + caseful cases, for speed, since matching characters is likely to be quite + common. First, ensure the minimum number of matches are present. If min = + max, continue at the same level without recursing. Otherwise, if + minimizing, keep trying the rest of the expression and advancing one + matching character if failing, up to the maximum. Alternatively, if + maximizing, find the maximum number of characters and work backwards. */ + + DPRINTF(("matching %c{%d,%d} against subject %.*s\n", fc, min, max, + max, (char *)eptr)); + + if (op >= OP_STARI) /* Caseless */ + { +#ifdef COMPILE_PCRE8 + /* fc must be < 128 if UTF is enabled. */ + foc = md->fcc[fc]; +#else +#ifdef SUPPORT_UTF +#ifdef SUPPORT_UCP + if (utf && fc > 127) + foc = UCD_OTHERCASE(fc); +#else + if (utf && fc > 127) + foc = fc; +#endif /* SUPPORT_UCP */ + else +#endif /* SUPPORT_UTF */ + foc = TABLE_GET(fc, md->fcc, fc); +#endif /* COMPILE_PCRE8 */ + + for (i = 1; i <= min; i++) + { + pcre_uint32 cc; /* Faster than pcre_uchar */ + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + cc = UCHAR21TEST(eptr); + if (fc != cc && foc != cc) RRETURN(MATCH_NOMATCH); + eptr++; + } + if (min == max) continue; + if (minimize) + { + for (fi = min;; fi++) + { + pcre_uint32 cc; /* Faster than pcre_uchar */ + RMATCH(eptr, ecode, offset_top, md, eptrb, RM24); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + cc = UCHAR21TEST(eptr); + if (fc != cc && foc != cc) RRETURN(MATCH_NOMATCH); + eptr++; + } + /* Control never gets here */ + } + else /* Maximize */ + { + pp = eptr; + for (i = min; i < max; i++) + { + pcre_uint32 cc; /* Faster than pcre_uchar */ + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + cc = UCHAR21TEST(eptr); + if (fc != cc && foc != cc) break; + eptr++; + } + if (possessive) continue; /* No backtracking */ + for (;;) + { + if (eptr == pp) goto TAIL_RECURSE; + RMATCH(eptr, ecode, offset_top, md, eptrb, RM25); + eptr--; + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + } + /* Control never gets here */ + } + } + + /* Caseful comparisons (includes all multi-byte characters) */ + + else + { + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + if (fc != UCHAR21INCTEST(eptr)) RRETURN(MATCH_NOMATCH); + } + + if (min == max) continue; + + if (minimize) + { + for (fi = min;; fi++) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM26); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + if (fc != UCHAR21INCTEST(eptr)) RRETURN(MATCH_NOMATCH); + } + /* Control never gets here */ + } + else /* Maximize */ + { + pp = eptr; + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + if (fc != UCHAR21TEST(eptr)) break; + eptr++; + } + if (possessive) continue; /* No backtracking */ + for (;;) + { + if (eptr == pp) goto TAIL_RECURSE; + RMATCH(eptr, ecode, offset_top, md, eptrb, RM27); + eptr--; + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + } + /* Control never gets here */ + } + } + /* Control never gets here */ + + /* Match a negated single one-byte character. The character we are + checking can be multibyte. */ + + case OP_NOT: + case OP_NOTI: + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } +#ifdef SUPPORT_UTF + if (utf) + { + register pcre_uint32 ch, och; + + ecode++; + GETCHARINC(ch, ecode); + GETCHARINC(c, eptr); + + if (op == OP_NOT) + { + if (ch == c) RRETURN(MATCH_NOMATCH); + } + else + { +#ifdef SUPPORT_UCP + if (ch > 127) + och = UCD_OTHERCASE(ch); +#else + if (ch > 127) + och = ch; +#endif /* SUPPORT_UCP */ + else + och = TABLE_GET(ch, md->fcc, ch); + if (ch == c || och == c) RRETURN(MATCH_NOMATCH); + } + } + else +#endif + { + register pcre_uint32 ch = ecode[1]; + c = *eptr++; + if (ch == c || (op == OP_NOTI && TABLE_GET(ch, md->fcc, ch) == c)) + RRETURN(MATCH_NOMATCH); + ecode += 2; + } + break; + + /* Match a negated single one-byte character repeatedly. This is almost a + repeat of the code for a repeated single character, but I haven't found a + nice way of commoning these up that doesn't require a test of the + positive/negative option for each character match. Maybe that wouldn't add + very much to the time taken, but character matching *is* what this is all + about... */ + + case OP_NOTEXACT: + case OP_NOTEXACTI: + min = max = GET2(ecode, 1); + ecode += 1 + IMM2_SIZE; + goto REPEATNOTCHAR; + + case OP_NOTUPTO: + case OP_NOTUPTOI: + case OP_NOTMINUPTO: + case OP_NOTMINUPTOI: + min = 0; + max = GET2(ecode, 1); + minimize = *ecode == OP_NOTMINUPTO || *ecode == OP_NOTMINUPTOI; + ecode += 1 + IMM2_SIZE; + goto REPEATNOTCHAR; + + case OP_NOTPOSSTAR: + case OP_NOTPOSSTARI: + possessive = TRUE; + min = 0; + max = INT_MAX; + ecode++; + goto REPEATNOTCHAR; + + case OP_NOTPOSPLUS: + case OP_NOTPOSPLUSI: + possessive = TRUE; + min = 1; + max = INT_MAX; + ecode++; + goto REPEATNOTCHAR; + + case OP_NOTPOSQUERY: + case OP_NOTPOSQUERYI: + possessive = TRUE; + min = 0; + max = 1; + ecode++; + goto REPEATNOTCHAR; + + case OP_NOTPOSUPTO: + case OP_NOTPOSUPTOI: + possessive = TRUE; + min = 0; + max = GET2(ecode, 1); + ecode += 1 + IMM2_SIZE; + goto REPEATNOTCHAR; + + case OP_NOTSTAR: + case OP_NOTSTARI: + case OP_NOTMINSTAR: + case OP_NOTMINSTARI: + case OP_NOTPLUS: + case OP_NOTPLUSI: + case OP_NOTMINPLUS: + case OP_NOTMINPLUSI: + case OP_NOTQUERY: + case OP_NOTQUERYI: + case OP_NOTMINQUERY: + case OP_NOTMINQUERYI: + c = *ecode++ - ((op >= OP_NOTSTARI)? OP_NOTSTARI: OP_NOTSTAR); + minimize = (c & 1) != 0; + min = rep_min[c]; /* Pick up values from tables; */ + max = rep_max[c]; /* zero for max => infinity */ + if (max == 0) max = INT_MAX; + + /* Common code for all repeated single-byte matches. */ + + REPEATNOTCHAR: + GETCHARINCTEST(fc, ecode); + + /* The code is duplicated for the caseless and caseful cases, for speed, + since matching characters is likely to be quite common. First, ensure the + minimum number of matches are present. If min = max, continue at the same + level without recursing. Otherwise, if minimizing, keep trying the rest of + the expression and advancing one matching character if failing, up to the + maximum. Alternatively, if maximizing, find the maximum number of + characters and work backwards. */ + + DPRINTF(("negative matching %c{%d,%d} against subject %.*s\n", fc, min, max, + max, (char *)eptr)); + + if (op >= OP_NOTSTARI) /* Caseless */ + { +#ifdef SUPPORT_UTF +#ifdef SUPPORT_UCP + if (utf && fc > 127) + foc = UCD_OTHERCASE(fc); +#else + if (utf && fc > 127) + foc = fc; +#endif /* SUPPORT_UCP */ + else +#endif /* SUPPORT_UTF */ + foc = TABLE_GET(fc, md->fcc, fc); + +#ifdef SUPPORT_UTF + if (utf) + { + register pcre_uint32 d; + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINC(d, eptr); + if (fc == d || (unsigned int)foc == d) RRETURN(MATCH_NOMATCH); + } + } + else +#endif /* SUPPORT_UTF */ + /* Not UTF mode */ + { + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + if (fc == *eptr || foc == *eptr) RRETURN(MATCH_NOMATCH); + eptr++; + } + } + + if (min == max) continue; + + if (minimize) + { +#ifdef SUPPORT_UTF + if (utf) + { + register pcre_uint32 d; + for (fi = min;; fi++) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM28); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINC(d, eptr); + if (fc == d || (unsigned int)foc == d) RRETURN(MATCH_NOMATCH); + } + } + else +#endif /*SUPPORT_UTF */ + /* Not UTF mode */ + { + for (fi = min;; fi++) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM29); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + if (fc == *eptr || foc == *eptr) RRETURN(MATCH_NOMATCH); + eptr++; + } + } + /* Control never gets here */ + } + + /* Maximize case */ + + else + { + pp = eptr; + +#ifdef SUPPORT_UTF + if (utf) + { + register pcre_uint32 d; + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLEN(d, eptr, len); + if (fc == d || (unsigned int)foc == d) break; + eptr += len; + } + if (possessive) continue; /* No backtracking */ + for(;;) + { + if (eptr <= pp) goto TAIL_RECURSE; + RMATCH(eptr, ecode, offset_top, md, eptrb, RM30); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + eptr--; + BACKCHAR(eptr); + } + } + else +#endif /* SUPPORT_UTF */ + /* Not UTF mode */ + { + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + if (fc == *eptr || foc == *eptr) break; + eptr++; + } + if (possessive) continue; /* No backtracking */ + for (;;) + { + if (eptr == pp) goto TAIL_RECURSE; + RMATCH(eptr, ecode, offset_top, md, eptrb, RM31); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + eptr--; + } + } + /* Control never gets here */ + } + } + + /* Caseful comparisons */ + + else + { +#ifdef SUPPORT_UTF + if (utf) + { + register pcre_uint32 d; + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINC(d, eptr); + if (fc == d) RRETURN(MATCH_NOMATCH); + } + } + else +#endif + /* Not UTF mode */ + { + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + if (fc == *eptr++) RRETURN(MATCH_NOMATCH); + } + } + + if (min == max) continue; + + if (minimize) + { +#ifdef SUPPORT_UTF + if (utf) + { + register pcre_uint32 d; + for (fi = min;; fi++) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM32); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINC(d, eptr); + if (fc == d) RRETURN(MATCH_NOMATCH); + } + } + else +#endif + /* Not UTF mode */ + { + for (fi = min;; fi++) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM33); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + if (fc == *eptr++) RRETURN(MATCH_NOMATCH); + } + } + /* Control never gets here */ + } + + /* Maximize case */ + + else + { + pp = eptr; + +#ifdef SUPPORT_UTF + if (utf) + { + register pcre_uint32 d; + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLEN(d, eptr, len); + if (fc == d) break; + eptr += len; + } + if (possessive) continue; /* No backtracking */ + for(;;) + { + if (eptr <= pp) goto TAIL_RECURSE; + RMATCH(eptr, ecode, offset_top, md, eptrb, RM34); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + eptr--; + BACKCHAR(eptr); + } + } + else +#endif + /* Not UTF mode */ + { + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + if (fc == *eptr) break; + eptr++; + } + if (possessive) continue; /* No backtracking */ + for (;;) + { + if (eptr == pp) goto TAIL_RECURSE; + RMATCH(eptr, ecode, offset_top, md, eptrb, RM35); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + eptr--; + } + } + /* Control never gets here */ + } + } + /* Control never gets here */ + + /* Match a single character type repeatedly; several different opcodes + share code. This is very similar to the code for single characters, but we + repeat it in the interests of efficiency. */ + + case OP_TYPEEXACT: + min = max = GET2(ecode, 1); + minimize = TRUE; + ecode += 1 + IMM2_SIZE; + goto REPEATTYPE; + + case OP_TYPEUPTO: + case OP_TYPEMINUPTO: + min = 0; + max = GET2(ecode, 1); + minimize = *ecode == OP_TYPEMINUPTO; + ecode += 1 + IMM2_SIZE; + goto REPEATTYPE; + + case OP_TYPEPOSSTAR: + possessive = TRUE; + min = 0; + max = INT_MAX; + ecode++; + goto REPEATTYPE; + + case OP_TYPEPOSPLUS: + possessive = TRUE; + min = 1; + max = INT_MAX; + ecode++; + goto REPEATTYPE; + + case OP_TYPEPOSQUERY: + possessive = TRUE; + min = 0; + max = 1; + ecode++; + goto REPEATTYPE; + + case OP_TYPEPOSUPTO: + possessive = TRUE; + min = 0; + max = GET2(ecode, 1); + ecode += 1 + IMM2_SIZE; + goto REPEATTYPE; + + case OP_TYPESTAR: + case OP_TYPEMINSTAR: + case OP_TYPEPLUS: + case OP_TYPEMINPLUS: + case OP_TYPEQUERY: + case OP_TYPEMINQUERY: + c = *ecode++ - OP_TYPESTAR; + minimize = (c & 1) != 0; + min = rep_min[c]; /* Pick up values from tables; */ + max = rep_max[c]; /* zero for max => infinity */ + if (max == 0) max = INT_MAX; + + /* Common code for all repeated single character type matches. Note that + in UTF-8 mode, '.' matches a character of any length, but for the other + character types, the valid characters are all one-byte long. */ + + REPEATTYPE: + ctype = *ecode++; /* Code for the character type */ + +#ifdef SUPPORT_UCP + if (ctype == OP_PROP || ctype == OP_NOTPROP) + { + prop_fail_result = ctype == OP_NOTPROP; + prop_type = *ecode++; + prop_value = *ecode++; + } + else prop_type = -1; +#endif + + /* First, ensure the minimum number of matches are present. Use inline + code for maximizing the speed, and do the type test once at the start + (i.e. keep it out of the loop). Separate the UTF-8 code completely as that + is tidier. Also separate the UCP code, which can be the same for both UTF-8 + and single-bytes. */ + + if (min > 0) + { +#ifdef SUPPORT_UCP + if (prop_type >= 0) + { + switch(prop_type) + { + case PT_ANY: + if (prop_fail_result) RRETURN(MATCH_NOMATCH); + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + } + break; + + case PT_LAMP: + for (i = 1; i <= min; i++) + { + int chartype; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + chartype = UCD_CHARTYPE(c); + if ((chartype == ucp_Lu || + chartype == ucp_Ll || + chartype == ucp_Lt) == prop_fail_result) + RRETURN(MATCH_NOMATCH); + } + break; + + case PT_GC: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + if ((UCD_CATEGORY(c) == prop_value) == prop_fail_result) + RRETURN(MATCH_NOMATCH); + } + break; + + case PT_PC: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + if ((UCD_CHARTYPE(c) == prop_value) == prop_fail_result) + RRETURN(MATCH_NOMATCH); + } + break; + + case PT_SC: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + if ((UCD_SCRIPT(c) == prop_value) == prop_fail_result) + RRETURN(MATCH_NOMATCH); + } + break; + + case PT_ALNUM: + for (i = 1; i <= min; i++) + { + int category; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + category = UCD_CATEGORY(c); + if ((category == ucp_L || category == ucp_N) == prop_fail_result) + RRETURN(MATCH_NOMATCH); + } + break; + + /* Perl space used to exclude VT, but from Perl 5.18 it is included, + which means that Perl space and POSIX space are now identical. PCRE + was changed at release 8.34. */ + + case PT_SPACE: /* Perl space */ + case PT_PXSPACE: /* POSIX space */ + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + switch(c) + { + HSPACE_CASES: + VSPACE_CASES: + if (prop_fail_result) RRETURN(MATCH_NOMATCH); + break; + + default: + if ((UCD_CATEGORY(c) == ucp_Z) == prop_fail_result) + RRETURN(MATCH_NOMATCH); + break; + } + } + break; + + case PT_WORD: + for (i = 1; i <= min; i++) + { + int category; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + category = UCD_CATEGORY(c); + if ((category == ucp_L || category == ucp_N || c == CHAR_UNDERSCORE) + == prop_fail_result) + RRETURN(MATCH_NOMATCH); + } + break; + + case PT_CLIST: + for (i = 1; i <= min; i++) + { + const pcre_uint32 *cp; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + cp = PRIV(ucd_caseless_sets) + prop_value; + for (;;) + { + if (c < *cp) + { if (prop_fail_result) break; else { RRETURN(MATCH_NOMATCH); } } + if (c == *cp++) + { if (prop_fail_result) { RRETURN(MATCH_NOMATCH); } else break; } + } + } + break; + + case PT_UCNC: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + if ((c == CHAR_DOLLAR_SIGN || c == CHAR_COMMERCIAL_AT || + c == CHAR_GRAVE_ACCENT || (c >= 0xa0 && c <= 0xd7ff) || + c >= 0xe000) == prop_fail_result) + RRETURN(MATCH_NOMATCH); + } + break; + + /* This should not occur */ + + default: + RRETURN(PCRE_ERROR_INTERNAL); + } + } + + /* Match extended Unicode sequences. We will get here only if the + support is in the binary; otherwise a compile-time error occurs. */ + + else if (ctype == OP_EXTUNI) + { + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + else + { + int lgb, rgb; + GETCHARINCTEST(c, eptr); + lgb = UCD_GRAPHBREAK(c); + while (eptr < md->end_subject) + { + int len = 1; + if (!utf) c = *eptr; else { GETCHARLEN(c, eptr, len); } + rgb = UCD_GRAPHBREAK(c); + if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; + lgb = rgb; + eptr += len; + } + } + CHECK_PARTIAL(); + } + } + + else +#endif /* SUPPORT_UCP */ + +/* Handle all other cases when the coding is UTF-8 */ + +#ifdef SUPPORT_UTF + if (utf) switch(ctype) + { + case OP_ANY: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + if (IS_NEWLINE(eptr)) RRETURN(MATCH_NOMATCH); + if (md->partial != 0 && + eptr + 1 >= md->end_subject && + NLBLOCK->nltype == NLTYPE_FIXED && + NLBLOCK->nllen == 2 && + UCHAR21(eptr) == NLBLOCK->nl[0]) + { + md->hitend = TRUE; + if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); + } + eptr++; + ACROSSCHAR(eptr < md->end_subject, *eptr, eptr++); + } + break; + + case OP_ALLANY: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + eptr++; + ACROSSCHAR(eptr < md->end_subject, *eptr, eptr++); + } + break; + + case OP_ANYBYTE: + if (eptr > md->end_subject - min) RRETURN(MATCH_NOMATCH); + eptr += min; + break; + + case OP_ANYNL: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINC(c, eptr); + switch(c) + { + default: RRETURN(MATCH_NOMATCH); + + case CHAR_CR: + if (eptr < md->end_subject && UCHAR21(eptr) == CHAR_LF) eptr++; + break; + + case CHAR_LF: + break; + + case CHAR_VT: + case CHAR_FF: + case CHAR_NEL: +#ifndef EBCDIC + case 0x2028: + case 0x2029: +#endif /* Not EBCDIC */ + if (md->bsr_anycrlf) RRETURN(MATCH_NOMATCH); + break; + } + } + break; + + case OP_NOT_HSPACE: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINC(c, eptr); + switch(c) + { + HSPACE_CASES: RRETURN(MATCH_NOMATCH); /* Byte and multibyte cases */ + default: break; + } + } + break; + + case OP_HSPACE: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINC(c, eptr); + switch(c) + { + HSPACE_CASES: break; /* Byte and multibyte cases */ + default: RRETURN(MATCH_NOMATCH); + } + } + break; + + case OP_NOT_VSPACE: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINC(c, eptr); + switch(c) + { + VSPACE_CASES: RRETURN(MATCH_NOMATCH); + default: break; + } + } + break; + + case OP_VSPACE: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINC(c, eptr); + switch(c) + { + VSPACE_CASES: break; + default: RRETURN(MATCH_NOMATCH); + } + } + break; + + case OP_NOT_DIGIT: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINC(c, eptr); + if (c < 128 && (md->ctypes[c] & ctype_digit) != 0) + RRETURN(MATCH_NOMATCH); + } + break; + + case OP_DIGIT: + for (i = 1; i <= min; i++) + { + pcre_uint32 cc; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + cc = UCHAR21(eptr); + if (cc >= 128 || (md->ctypes[cc] & ctype_digit) == 0) + RRETURN(MATCH_NOMATCH); + eptr++; + /* No need to skip more bytes - we know it's a 1-byte character */ + } + break; + + case OP_NOT_WHITESPACE: + for (i = 1; i <= min; i++) + { + pcre_uint32 cc; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + cc = UCHAR21(eptr); + if (cc < 128 && (md->ctypes[cc] & ctype_space) != 0) + RRETURN(MATCH_NOMATCH); + eptr++; + ACROSSCHAR(eptr < md->end_subject, *eptr, eptr++); + } + break; + + case OP_WHITESPACE: + for (i = 1; i <= min; i++) + { + pcre_uint32 cc; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + cc = UCHAR21(eptr); + if (cc >= 128 || (md->ctypes[cc] & ctype_space) == 0) + RRETURN(MATCH_NOMATCH); + eptr++; + /* No need to skip more bytes - we know it's a 1-byte character */ + } + break; + + case OP_NOT_WORDCHAR: + for (i = 1; i <= min; i++) + { + pcre_uint32 cc; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + cc = UCHAR21(eptr); + if (cc < 128 && (md->ctypes[cc] & ctype_word) != 0) + RRETURN(MATCH_NOMATCH); + eptr++; + ACROSSCHAR(eptr < md->end_subject, *eptr, eptr++); + } + break; + + case OP_WORDCHAR: + for (i = 1; i <= min; i++) + { + pcre_uint32 cc; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + cc = UCHAR21(eptr); + if (cc >= 128 || (md->ctypes[cc] & ctype_word) == 0) + RRETURN(MATCH_NOMATCH); + eptr++; + /* No need to skip more bytes - we know it's a 1-byte character */ + } + break; + + default: + RRETURN(PCRE_ERROR_INTERNAL); + } /* End switch(ctype) */ + + else +#endif /* SUPPORT_UTF */ + + /* Code for the non-UTF-8 case for minimum matching of operators other + than OP_PROP and OP_NOTPROP. */ + + switch(ctype) + { + case OP_ANY: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + if (IS_NEWLINE(eptr)) RRETURN(MATCH_NOMATCH); + if (md->partial != 0 && + eptr + 1 >= md->end_subject && + NLBLOCK->nltype == NLTYPE_FIXED && + NLBLOCK->nllen == 2 && + *eptr == NLBLOCK->nl[0]) + { + md->hitend = TRUE; + if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); + } + eptr++; + } + break; + + case OP_ALLANY: + if (eptr > md->end_subject - min) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + eptr += min; + break; + + case OP_ANYBYTE: + if (eptr > md->end_subject - min) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + eptr += min; + break; + + case OP_ANYNL: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + switch(*eptr++) + { + default: RRETURN(MATCH_NOMATCH); + + case CHAR_CR: + if (eptr < md->end_subject && *eptr == CHAR_LF) eptr++; + break; + + case CHAR_LF: + break; + + case CHAR_VT: + case CHAR_FF: + case CHAR_NEL: +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + case 0x2028: + case 0x2029: +#endif + if (md->bsr_anycrlf) RRETURN(MATCH_NOMATCH); + break; + } + } + break; + + case OP_NOT_HSPACE: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + switch(*eptr++) + { + default: break; + HSPACE_BYTE_CASES: +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + HSPACE_MULTIBYTE_CASES: +#endif + RRETURN(MATCH_NOMATCH); + } + } + break; + + case OP_HSPACE: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + switch(*eptr++) + { + default: RRETURN(MATCH_NOMATCH); + HSPACE_BYTE_CASES: +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + HSPACE_MULTIBYTE_CASES: +#endif + break; + } + } + break; + + case OP_NOT_VSPACE: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + switch(*eptr++) + { + VSPACE_BYTE_CASES: +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + VSPACE_MULTIBYTE_CASES: +#endif + RRETURN(MATCH_NOMATCH); + default: break; + } + } + break; + + case OP_VSPACE: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + switch(*eptr++) + { + default: RRETURN(MATCH_NOMATCH); + VSPACE_BYTE_CASES: +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + VSPACE_MULTIBYTE_CASES: +#endif + break; + } + } + break; + + case OP_NOT_DIGIT: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + if (MAX_255(*eptr) && (md->ctypes[*eptr] & ctype_digit) != 0) + RRETURN(MATCH_NOMATCH); + eptr++; + } + break; + + case OP_DIGIT: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + if (!MAX_255(*eptr) || (md->ctypes[*eptr] & ctype_digit) == 0) + RRETURN(MATCH_NOMATCH); + eptr++; + } + break; + + case OP_NOT_WHITESPACE: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + if (MAX_255(*eptr) && (md->ctypes[*eptr] & ctype_space) != 0) + RRETURN(MATCH_NOMATCH); + eptr++; + } + break; + + case OP_WHITESPACE: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + if (!MAX_255(*eptr) || (md->ctypes[*eptr] & ctype_space) == 0) + RRETURN(MATCH_NOMATCH); + eptr++; + } + break; + + case OP_NOT_WORDCHAR: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + if (MAX_255(*eptr) && (md->ctypes[*eptr] & ctype_word) != 0) + RRETURN(MATCH_NOMATCH); + eptr++; + } + break; + + case OP_WORDCHAR: + for (i = 1; i <= min; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + if (!MAX_255(*eptr) || (md->ctypes[*eptr] & ctype_word) == 0) + RRETURN(MATCH_NOMATCH); + eptr++; + } + break; + + default: + RRETURN(PCRE_ERROR_INTERNAL); + } + } + + /* If min = max, continue at the same level without recursing */ + + if (min == max) continue; + + /* If minimizing, we have to test the rest of the pattern before each + subsequent match. Again, separate the UTF-8 case for speed, and also + separate the UCP cases. */ + + if (minimize) + { +#ifdef SUPPORT_UCP + if (prop_type >= 0) + { + switch(prop_type) + { + case PT_ANY: + for (fi = min;; fi++) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM36); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + if (prop_fail_result) RRETURN(MATCH_NOMATCH); + } + /* Control never gets here */ + + case PT_LAMP: + for (fi = min;; fi++) + { + int chartype; + RMATCH(eptr, ecode, offset_top, md, eptrb, RM37); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + chartype = UCD_CHARTYPE(c); + if ((chartype == ucp_Lu || + chartype == ucp_Ll || + chartype == ucp_Lt) == prop_fail_result) + RRETURN(MATCH_NOMATCH); + } + /* Control never gets here */ + + case PT_GC: + for (fi = min;; fi++) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM38); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + if ((UCD_CATEGORY(c) == prop_value) == prop_fail_result) + RRETURN(MATCH_NOMATCH); + } + /* Control never gets here */ + + case PT_PC: + for (fi = min;; fi++) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM39); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + if ((UCD_CHARTYPE(c) == prop_value) == prop_fail_result) + RRETURN(MATCH_NOMATCH); + } + /* Control never gets here */ + + case PT_SC: + for (fi = min;; fi++) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM40); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + if ((UCD_SCRIPT(c) == prop_value) == prop_fail_result) + RRETURN(MATCH_NOMATCH); + } + /* Control never gets here */ + + case PT_ALNUM: + for (fi = min;; fi++) + { + int category; + RMATCH(eptr, ecode, offset_top, md, eptrb, RM59); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + category = UCD_CATEGORY(c); + if ((category == ucp_L || category == ucp_N) == prop_fail_result) + RRETURN(MATCH_NOMATCH); + } + /* Control never gets here */ + + /* Perl space used to exclude VT, but from Perl 5.18 it is included, + which means that Perl space and POSIX space are now identical. PCRE + was changed at release 8.34. */ + + case PT_SPACE: /* Perl space */ + case PT_PXSPACE: /* POSIX space */ + for (fi = min;; fi++) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM61); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + switch(c) + { + HSPACE_CASES: + VSPACE_CASES: + if (prop_fail_result) RRETURN(MATCH_NOMATCH); + break; + + default: + if ((UCD_CATEGORY(c) == ucp_Z) == prop_fail_result) + RRETURN(MATCH_NOMATCH); + break; + } + } + /* Control never gets here */ + + case PT_WORD: + for (fi = min;; fi++) + { + int category; + RMATCH(eptr, ecode, offset_top, md, eptrb, RM62); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + category = UCD_CATEGORY(c); + if ((category == ucp_L || + category == ucp_N || + c == CHAR_UNDERSCORE) + == prop_fail_result) + RRETURN(MATCH_NOMATCH); + } + /* Control never gets here */ + + case PT_CLIST: + for (fi = min;; fi++) + { + const pcre_uint32 *cp; + RMATCH(eptr, ecode, offset_top, md, eptrb, RM67); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + cp = PRIV(ucd_caseless_sets) + prop_value; + for (;;) + { + if (c < *cp) + { if (prop_fail_result) break; else { RRETURN(MATCH_NOMATCH); } } + if (c == *cp++) + { if (prop_fail_result) { RRETURN(MATCH_NOMATCH); } else break; } + } + } + /* Control never gets here */ + + case PT_UCNC: + for (fi = min;; fi++) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM60); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + GETCHARINCTEST(c, eptr); + if ((c == CHAR_DOLLAR_SIGN || c == CHAR_COMMERCIAL_AT || + c == CHAR_GRAVE_ACCENT || (c >= 0xa0 && c <= 0xd7ff) || + c >= 0xe000) == prop_fail_result) + RRETURN(MATCH_NOMATCH); + } + /* Control never gets here */ + + /* This should never occur */ + default: + RRETURN(PCRE_ERROR_INTERNAL); + } + } + + /* Match extended Unicode sequences. We will get here only if the + support is in the binary; otherwise a compile-time error occurs. */ + + else if (ctype == OP_EXTUNI) + { + for (fi = min;; fi++) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM41); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + else + { + int lgb, rgb; + GETCHARINCTEST(c, eptr); + lgb = UCD_GRAPHBREAK(c); + while (eptr < md->end_subject) + { + int len = 1; + if (!utf) c = *eptr; else { GETCHARLEN(c, eptr, len); } + rgb = UCD_GRAPHBREAK(c); + if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; + lgb = rgb; + eptr += len; + } + } + CHECK_PARTIAL(); + } + } + else +#endif /* SUPPORT_UCP */ + +#ifdef SUPPORT_UTF + if (utf) + { + for (fi = min;; fi++) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM42); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + if (ctype == OP_ANY && IS_NEWLINE(eptr)) + RRETURN(MATCH_NOMATCH); + GETCHARINC(c, eptr); + switch(ctype) + { + case OP_ANY: /* This is the non-NL case */ + if (md->partial != 0 && /* Take care with CRLF partial */ + eptr >= md->end_subject && + NLBLOCK->nltype == NLTYPE_FIXED && + NLBLOCK->nllen == 2 && + c == NLBLOCK->nl[0]) + { + md->hitend = TRUE; + if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); + } + break; + + case OP_ALLANY: + case OP_ANYBYTE: + break; + + case OP_ANYNL: + switch(c) + { + default: RRETURN(MATCH_NOMATCH); + case CHAR_CR: + if (eptr < md->end_subject && UCHAR21(eptr) == CHAR_LF) eptr++; + break; + + case CHAR_LF: + break; + + case CHAR_VT: + case CHAR_FF: + case CHAR_NEL: +#ifndef EBCDIC + case 0x2028: + case 0x2029: +#endif /* Not EBCDIC */ + if (md->bsr_anycrlf) RRETURN(MATCH_NOMATCH); + break; + } + break; + + case OP_NOT_HSPACE: + switch(c) + { + HSPACE_CASES: RRETURN(MATCH_NOMATCH); + default: break; + } + break; + + case OP_HSPACE: + switch(c) + { + HSPACE_CASES: break; + default: RRETURN(MATCH_NOMATCH); + } + break; + + case OP_NOT_VSPACE: + switch(c) + { + VSPACE_CASES: RRETURN(MATCH_NOMATCH); + default: break; + } + break; + + case OP_VSPACE: + switch(c) + { + VSPACE_CASES: break; + default: RRETURN(MATCH_NOMATCH); + } + break; + + case OP_NOT_DIGIT: + if (c < 256 && (md->ctypes[c] & ctype_digit) != 0) + RRETURN(MATCH_NOMATCH); + break; + + case OP_DIGIT: + if (c >= 256 || (md->ctypes[c] & ctype_digit) == 0) + RRETURN(MATCH_NOMATCH); + break; + + case OP_NOT_WHITESPACE: + if (c < 256 && (md->ctypes[c] & ctype_space) != 0) + RRETURN(MATCH_NOMATCH); + break; + + case OP_WHITESPACE: + if (c >= 256 || (md->ctypes[c] & ctype_space) == 0) + RRETURN(MATCH_NOMATCH); + break; + + case OP_NOT_WORDCHAR: + if (c < 256 && (md->ctypes[c] & ctype_word) != 0) + RRETURN(MATCH_NOMATCH); + break; + + case OP_WORDCHAR: + if (c >= 256 || (md->ctypes[c] & ctype_word) == 0) + RRETURN(MATCH_NOMATCH); + break; + + default: + RRETURN(PCRE_ERROR_INTERNAL); + } + } + } + else +#endif + /* Not UTF mode */ + { + for (fi = min;; fi++) + { + RMATCH(eptr, ecode, offset_top, md, eptrb, RM43); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + if (fi >= max) RRETURN(MATCH_NOMATCH); + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + RRETURN(MATCH_NOMATCH); + } + if (ctype == OP_ANY && IS_NEWLINE(eptr)) + RRETURN(MATCH_NOMATCH); + c = *eptr++; + switch(ctype) + { + case OP_ANY: /* This is the non-NL case */ + if (md->partial != 0 && /* Take care with CRLF partial */ + eptr >= md->end_subject && + NLBLOCK->nltype == NLTYPE_FIXED && + NLBLOCK->nllen == 2 && + c == NLBLOCK->nl[0]) + { + md->hitend = TRUE; + if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); + } + break; + + case OP_ALLANY: + case OP_ANYBYTE: + break; + + case OP_ANYNL: + switch(c) + { + default: RRETURN(MATCH_NOMATCH); + case CHAR_CR: + if (eptr < md->end_subject && *eptr == CHAR_LF) eptr++; + break; + + case CHAR_LF: + break; + + case CHAR_VT: + case CHAR_FF: + case CHAR_NEL: +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + case 0x2028: + case 0x2029: +#endif + if (md->bsr_anycrlf) RRETURN(MATCH_NOMATCH); + break; + } + break; + + case OP_NOT_HSPACE: + switch(c) + { + default: break; + HSPACE_BYTE_CASES: +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + HSPACE_MULTIBYTE_CASES: +#endif + RRETURN(MATCH_NOMATCH); + } + break; + + case OP_HSPACE: + switch(c) + { + default: RRETURN(MATCH_NOMATCH); + HSPACE_BYTE_CASES: +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + HSPACE_MULTIBYTE_CASES: +#endif + break; + } + break; + + case OP_NOT_VSPACE: + switch(c) + { + default: break; + VSPACE_BYTE_CASES: +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + VSPACE_MULTIBYTE_CASES: +#endif + RRETURN(MATCH_NOMATCH); + } + break; + + case OP_VSPACE: + switch(c) + { + default: RRETURN(MATCH_NOMATCH); + VSPACE_BYTE_CASES: +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + VSPACE_MULTIBYTE_CASES: +#endif + break; + } + break; + + case OP_NOT_DIGIT: + if (MAX_255(c) && (md->ctypes[c] & ctype_digit) != 0) RRETURN(MATCH_NOMATCH); + break; + + case OP_DIGIT: + if (!MAX_255(c) || (md->ctypes[c] & ctype_digit) == 0) RRETURN(MATCH_NOMATCH); + break; + + case OP_NOT_WHITESPACE: + if (MAX_255(c) && (md->ctypes[c] & ctype_space) != 0) RRETURN(MATCH_NOMATCH); + break; + + case OP_WHITESPACE: + if (!MAX_255(c) || (md->ctypes[c] & ctype_space) == 0) RRETURN(MATCH_NOMATCH); + break; + + case OP_NOT_WORDCHAR: + if (MAX_255(c) && (md->ctypes[c] & ctype_word) != 0) RRETURN(MATCH_NOMATCH); + break; + + case OP_WORDCHAR: + if (!MAX_255(c) || (md->ctypes[c] & ctype_word) == 0) RRETURN(MATCH_NOMATCH); + break; + + default: + RRETURN(PCRE_ERROR_INTERNAL); + } + } + } + /* Control never gets here */ + } + + /* If maximizing, it is worth using inline code for speed, doing the type + test once at the start (i.e. keep it out of the loop). Again, keep the + UTF-8 and UCP stuff separate. */ + + else + { + pp = eptr; /* Remember where we started */ + +#ifdef SUPPORT_UCP + if (prop_type >= 0) + { + switch(prop_type) + { + case PT_ANY: + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLENTEST(c, eptr, len); + if (prop_fail_result) break; + eptr+= len; + } + break; + + case PT_LAMP: + for (i = min; i < max; i++) + { + int chartype; + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLENTEST(c, eptr, len); + chartype = UCD_CHARTYPE(c); + if ((chartype == ucp_Lu || + chartype == ucp_Ll || + chartype == ucp_Lt) == prop_fail_result) + break; + eptr+= len; + } + break; + + case PT_GC: + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLENTEST(c, eptr, len); + if ((UCD_CATEGORY(c) == prop_value) == prop_fail_result) break; + eptr+= len; + } + break; + + case PT_PC: + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLENTEST(c, eptr, len); + if ((UCD_CHARTYPE(c) == prop_value) == prop_fail_result) break; + eptr+= len; + } + break; + + case PT_SC: + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLENTEST(c, eptr, len); + if ((UCD_SCRIPT(c) == prop_value) == prop_fail_result) break; + eptr+= len; + } + break; + + case PT_ALNUM: + for (i = min; i < max; i++) + { + int category; + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLENTEST(c, eptr, len); + category = UCD_CATEGORY(c); + if ((category == ucp_L || category == ucp_N) == prop_fail_result) + break; + eptr+= len; + } + break; + + /* Perl space used to exclude VT, but from Perl 5.18 it is included, + which means that Perl space and POSIX space are now identical. PCRE + was changed at release 8.34. */ + + case PT_SPACE: /* Perl space */ + case PT_PXSPACE: /* POSIX space */ + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLENTEST(c, eptr, len); + switch(c) + { + HSPACE_CASES: + VSPACE_CASES: + if (prop_fail_result) goto ENDLOOP99; /* Break the loop */ + break; + + default: + if ((UCD_CATEGORY(c) == ucp_Z) == prop_fail_result) + goto ENDLOOP99; /* Break the loop */ + break; + } + eptr+= len; + } + ENDLOOP99: + break; + + case PT_WORD: + for (i = min; i < max; i++) + { + int category; + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLENTEST(c, eptr, len); + category = UCD_CATEGORY(c); + if ((category == ucp_L || category == ucp_N || + c == CHAR_UNDERSCORE) == prop_fail_result) + break; + eptr+= len; + } + break; + + case PT_CLIST: + for (i = min; i < max; i++) + { + const pcre_uint32 *cp; + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLENTEST(c, eptr, len); + cp = PRIV(ucd_caseless_sets) + prop_value; + for (;;) + { + if (c < *cp) + { if (prop_fail_result) break; else goto GOT_MAX; } + if (c == *cp++) + { if (prop_fail_result) goto GOT_MAX; else break; } + } + eptr += len; + } + GOT_MAX: + break; + + case PT_UCNC: + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLENTEST(c, eptr, len); + if ((c == CHAR_DOLLAR_SIGN || c == CHAR_COMMERCIAL_AT || + c == CHAR_GRAVE_ACCENT || (c >= 0xa0 && c <= 0xd7ff) || + c >= 0xe000) == prop_fail_result) + break; + eptr += len; + } + break; + + default: + RRETURN(PCRE_ERROR_INTERNAL); + } + + /* eptr is now past the end of the maximum run */ + + if (possessive) continue; /* No backtracking */ + for(;;) + { + if (eptr <= pp) goto TAIL_RECURSE; + RMATCH(eptr, ecode, offset_top, md, eptrb, RM44); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + eptr--; + if (utf) BACKCHAR(eptr); + } + } + + /* Match extended Unicode grapheme clusters. We will get here only if the + support is in the binary; otherwise a compile-time error occurs. */ + + else if (ctype == OP_EXTUNI) + { + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + else + { + int lgb, rgb; + GETCHARINCTEST(c, eptr); + lgb = UCD_GRAPHBREAK(c); + while (eptr < md->end_subject) + { + int len = 1; + if (!utf) c = *eptr; else { GETCHARLEN(c, eptr, len); } + rgb = UCD_GRAPHBREAK(c); + if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; + lgb = rgb; + eptr += len; + } + } + CHECK_PARTIAL(); + } + + /* eptr is now past the end of the maximum run */ + + if (possessive) continue; /* No backtracking */ + + /* We use <= pp rather than == pp to detect the start of the run while + backtracking because the use of \C in UTF mode can cause BACKCHAR to + move back past pp. This is just palliative; the use of \C in UTF mode + is fraught with danger. */ + + for(;;) + { + int lgb, rgb; + PCRE_PUCHAR fptr; + + if (eptr <= pp) goto TAIL_RECURSE; /* At start of char run */ + RMATCH(eptr, ecode, offset_top, md, eptrb, RM45); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + + /* Backtracking over an extended grapheme cluster involves inspecting + the previous two characters (if present) to see if a break is + permitted between them. */ + + eptr--; + if (!utf) c = *eptr; else + { + BACKCHAR(eptr); + GETCHAR(c, eptr); + } + rgb = UCD_GRAPHBREAK(c); + + for (;;) + { + if (eptr <= pp) goto TAIL_RECURSE; /* At start of char run */ + fptr = eptr - 1; + if (!utf) c = *fptr; else + { + BACKCHAR(fptr); + GETCHAR(c, fptr); + } + lgb = UCD_GRAPHBREAK(c); + if ((PRIV(ucp_gbtable)[lgb] & (1 << rgb)) == 0) break; + eptr = fptr; + rgb = lgb; + } + } + } + + else +#endif /* SUPPORT_UCP */ + +#ifdef SUPPORT_UTF + if (utf) + { + switch(ctype) + { + case OP_ANY: + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + if (IS_NEWLINE(eptr)) break; + if (md->partial != 0 && /* Take care with CRLF partial */ + eptr + 1 >= md->end_subject && + NLBLOCK->nltype == NLTYPE_FIXED && + NLBLOCK->nllen == 2 && + UCHAR21(eptr) == NLBLOCK->nl[0]) + { + md->hitend = TRUE; + if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); + } + eptr++; + ACROSSCHAR(eptr < md->end_subject, *eptr, eptr++); + } + break; + + case OP_ALLANY: + if (max < INT_MAX) + { + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + eptr++; + ACROSSCHAR(eptr < md->end_subject, *eptr, eptr++); + } + } + else + { + eptr = md->end_subject; /* Unlimited UTF-8 repeat */ + SCHECK_PARTIAL(); + } + break; + + /* The byte case is the same as non-UTF8 */ + + case OP_ANYBYTE: + c = max - min; + if (c > (unsigned int)(md->end_subject - eptr)) + { + eptr = md->end_subject; + SCHECK_PARTIAL(); + } + else eptr += c; + break; + + case OP_ANYNL: + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLEN(c, eptr, len); + if (c == CHAR_CR) + { + if (++eptr >= md->end_subject) break; + if (UCHAR21(eptr) == CHAR_LF) eptr++; + } + else + { + if (c != CHAR_LF && + (md->bsr_anycrlf || + (c != CHAR_VT && c != CHAR_FF && c != CHAR_NEL +#ifndef EBCDIC + && c != 0x2028 && c != 0x2029 +#endif /* Not EBCDIC */ + ))) + break; + eptr += len; + } + } + break; + + case OP_NOT_HSPACE: + case OP_HSPACE: + for (i = min; i < max; i++) + { + BOOL gotspace; + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLEN(c, eptr, len); + switch(c) + { + HSPACE_CASES: gotspace = TRUE; break; + default: gotspace = FALSE; break; + } + if (gotspace == (ctype == OP_NOT_HSPACE)) break; + eptr += len; + } + break; + + case OP_NOT_VSPACE: + case OP_VSPACE: + for (i = min; i < max; i++) + { + BOOL gotspace; + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLEN(c, eptr, len); + switch(c) + { + VSPACE_CASES: gotspace = TRUE; break; + default: gotspace = FALSE; break; + } + if (gotspace == (ctype == OP_NOT_VSPACE)) break; + eptr += len; + } + break; + + case OP_NOT_DIGIT: + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLEN(c, eptr, len); + if (c < 256 && (md->ctypes[c] & ctype_digit) != 0) break; + eptr+= len; + } + break; + + case OP_DIGIT: + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLEN(c, eptr, len); + if (c >= 256 ||(md->ctypes[c] & ctype_digit) == 0) break; + eptr+= len; + } + break; + + case OP_NOT_WHITESPACE: + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLEN(c, eptr, len); + if (c < 256 && (md->ctypes[c] & ctype_space) != 0) break; + eptr+= len; + } + break; + + case OP_WHITESPACE: + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLEN(c, eptr, len); + if (c >= 256 ||(md->ctypes[c] & ctype_space) == 0) break; + eptr+= len; + } + break; + + case OP_NOT_WORDCHAR: + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLEN(c, eptr, len); + if (c < 256 && (md->ctypes[c] & ctype_word) != 0) break; + eptr+= len; + } + break; + + case OP_WORDCHAR: + for (i = min; i < max; i++) + { + int len = 1; + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + GETCHARLEN(c, eptr, len); + if (c >= 256 || (md->ctypes[c] & ctype_word) == 0) break; + eptr+= len; + } + break; + + default: + RRETURN(PCRE_ERROR_INTERNAL); + } + + if (possessive) continue; /* No backtracking */ + for(;;) + { + if (eptr <= pp) goto TAIL_RECURSE; + RMATCH(eptr, ecode, offset_top, md, eptrb, RM46); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + eptr--; + BACKCHAR(eptr); + if (ctype == OP_ANYNL && eptr > pp && UCHAR21(eptr) == CHAR_NL && + UCHAR21(eptr - 1) == CHAR_CR) eptr--; + } + } + else +#endif /* SUPPORT_UTF */ + /* Not UTF mode */ + { + switch(ctype) + { + case OP_ANY: + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + if (IS_NEWLINE(eptr)) break; + if (md->partial != 0 && /* Take care with CRLF partial */ + eptr + 1 >= md->end_subject && + NLBLOCK->nltype == NLTYPE_FIXED && + NLBLOCK->nllen == 2 && + *eptr == NLBLOCK->nl[0]) + { + md->hitend = TRUE; + if (md->partial > 1) RRETURN(PCRE_ERROR_PARTIAL); + } + eptr++; + } + break; + + case OP_ALLANY: + case OP_ANYBYTE: + c = max - min; + if (c > (unsigned int)(md->end_subject - eptr)) + { + eptr = md->end_subject; + SCHECK_PARTIAL(); + } + else eptr += c; + break; + + case OP_ANYNL: + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + c = *eptr; + if (c == CHAR_CR) + { + if (++eptr >= md->end_subject) break; + if (*eptr == CHAR_LF) eptr++; + } + else + { + if (c != CHAR_LF && (md->bsr_anycrlf || + (c != CHAR_VT && c != CHAR_FF && c != CHAR_NEL +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + && c != 0x2028 && c != 0x2029 +#endif + ))) break; + eptr++; + } + } + break; + + case OP_NOT_HSPACE: + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + switch(*eptr) + { + default: eptr++; break; + HSPACE_BYTE_CASES: +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + HSPACE_MULTIBYTE_CASES: +#endif + goto ENDLOOP00; + } + } + ENDLOOP00: + break; + + case OP_HSPACE: + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + switch(*eptr) + { + default: goto ENDLOOP01; + HSPACE_BYTE_CASES: +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + HSPACE_MULTIBYTE_CASES: +#endif + eptr++; break; + } + } + ENDLOOP01: + break; + + case OP_NOT_VSPACE: + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + switch(*eptr) + { + default: eptr++; break; + VSPACE_BYTE_CASES: +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + VSPACE_MULTIBYTE_CASES: +#endif + goto ENDLOOP02; + } + } + ENDLOOP02: + break; + + case OP_VSPACE: + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + switch(*eptr) + { + default: goto ENDLOOP03; + VSPACE_BYTE_CASES: +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + VSPACE_MULTIBYTE_CASES: +#endif + eptr++; break; + } + } + ENDLOOP03: + break; + + case OP_NOT_DIGIT: + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + if (MAX_255(*eptr) && (md->ctypes[*eptr] & ctype_digit) != 0) break; + eptr++; + } + break; + + case OP_DIGIT: + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + if (!MAX_255(*eptr) || (md->ctypes[*eptr] & ctype_digit) == 0) break; + eptr++; + } + break; + + case OP_NOT_WHITESPACE: + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + if (MAX_255(*eptr) && (md->ctypes[*eptr] & ctype_space) != 0) break; + eptr++; + } + break; + + case OP_WHITESPACE: + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + if (!MAX_255(*eptr) || (md->ctypes[*eptr] & ctype_space) == 0) break; + eptr++; + } + break; + + case OP_NOT_WORDCHAR: + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + if (MAX_255(*eptr) && (md->ctypes[*eptr] & ctype_word) != 0) break; + eptr++; + } + break; + + case OP_WORDCHAR: + for (i = min; i < max; i++) + { + if (eptr >= md->end_subject) + { + SCHECK_PARTIAL(); + break; + } + if (!MAX_255(*eptr) || (md->ctypes[*eptr] & ctype_word) == 0) break; + eptr++; + } + break; + + default: + RRETURN(PCRE_ERROR_INTERNAL); + } + + if (possessive) continue; /* No backtracking */ + for (;;) + { + if (eptr == pp) goto TAIL_RECURSE; + RMATCH(eptr, ecode, offset_top, md, eptrb, RM47); + if (rrc != MATCH_NOMATCH) RRETURN(rrc); + eptr--; + if (ctype == OP_ANYNL && eptr > pp && *eptr == CHAR_LF && + eptr[-1] == CHAR_CR) eptr--; + } + } + + /* Control never gets here */ + } + + /* There's been some horrible disaster. Arrival here can only mean there is + something seriously wrong in the code above or the OP_xxx definitions. */ + + default: + DPRINTF(("Unknown opcode %d\n", *ecode)); + RRETURN(PCRE_ERROR_UNKNOWN_OPCODE); + } + + /* Do not stick any code in here without much thought; it is assumed + that "continue" in the code above comes out to here to repeat the main + loop. */ + + } /* End of main loop */ +/* Control never reaches here */ + + +/* When compiling to use the heap rather than the stack for recursive calls to +match(), the RRETURN() macro jumps here. The number that is saved in +frame->Xwhere indicates which label we actually want to return to. */ + +#ifdef NO_RECURSE +#define LBL(val) case val: goto L_RM##val; +HEAP_RETURN: +switch (frame->Xwhere) + { + LBL( 1) LBL( 2) LBL( 3) LBL( 4) LBL( 5) LBL( 6) LBL( 7) LBL( 8) + LBL( 9) LBL(10) LBL(11) LBL(12) LBL(13) LBL(14) LBL(15) LBL(17) + LBL(19) LBL(24) LBL(25) LBL(26) LBL(27) LBL(29) LBL(31) LBL(33) + LBL(35) LBL(43) LBL(47) LBL(48) LBL(49) LBL(50) LBL(51) LBL(52) + LBL(53) LBL(54) LBL(55) LBL(56) LBL(57) LBL(58) LBL(63) LBL(64) + LBL(65) LBL(66) +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + LBL(20) LBL(21) +#endif +#ifdef SUPPORT_UTF + LBL(16) LBL(18) + LBL(22) LBL(23) LBL(28) LBL(30) + LBL(32) LBL(34) LBL(42) LBL(46) +#ifdef SUPPORT_UCP + LBL(36) LBL(37) LBL(38) LBL(39) LBL(40) LBL(41) LBL(44) LBL(45) + LBL(59) LBL(60) LBL(61) LBL(62) LBL(67) +#endif /* SUPPORT_UCP */ +#endif /* SUPPORT_UTF */ + default: + DPRINTF(("jump error in pcre match: label %d non-existent\n", frame->Xwhere)); + return PCRE_ERROR_INTERNAL; + } +#undef LBL +#endif /* NO_RECURSE */ +} + + +/*************************************************************************** +**************************************************************************** + RECURSION IN THE match() FUNCTION + +Undefine all the macros that were defined above to handle this. */ + +#ifdef NO_RECURSE +#undef eptr +#undef ecode +#undef mstart +#undef offset_top +#undef eptrb +#undef flags + +#undef callpat +#undef charptr +#undef data +#undef next +#undef pp +#undef prev +#undef saved_eptr + +#undef new_recursive + +#undef cur_is_word +#undef condition +#undef prev_is_word + +#undef ctype +#undef length +#undef max +#undef min +#undef number +#undef offset +#undef op +#undef save_capture_last +#undef save_offset1 +#undef save_offset2 +#undef save_offset3 +#undef stacksave + +#undef newptrb + +#endif + +/* These two are defined as macros in both cases */ + +#undef fc +#undef fi + +/*************************************************************************** +***************************************************************************/ + + +#ifdef NO_RECURSE +/************************************************* +* Release allocated heap frames * +*************************************************/ + +/* This function releases all the allocated frames. The base frame is on the +machine stack, and so must not be freed. + +Argument: the address of the base frame +Returns: nothing +*/ + +static void +release_match_heapframes (heapframe *frame_base) +{ +heapframe *nextframe = frame_base->Xnextframe; +while (nextframe != NULL) + { + heapframe *oldframe = nextframe; + nextframe = nextframe->Xnextframe; + (PUBL(stack_free))(oldframe); + } +} +#endif + + +/************************************************* +* Execute a Regular Expression * +*************************************************/ + +/* This function applies a compiled re to a subject string and picks out +portions of the string if it matches. Two elements in the vector are set for +each substring: the offsets to the start and end of the substring. + +Arguments: + argument_re points to the compiled expression + extra_data points to extra data or is NULL + subject points to the subject string + length length of subject string (may contain binary zeros) + start_offset where to start in the subject string + options option bits + offsets points to a vector of ints to be filled in with offsets + offsetcount the number of elements in the vector + +Returns: > 0 => success; value is the number of elements filled in + = 0 => success, but offsets is not big enough + -1 => failed to match + < -1 => some kind of unexpected problem +*/ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre_exec(const pcre *argument_re, const pcre_extra *extra_data, + PCRE_SPTR subject, int length, int start_offset, int options, int *offsets, + int offsetcount) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre16_exec(const pcre16 *argument_re, const pcre16_extra *extra_data, + PCRE_SPTR16 subject, int length, int start_offset, int options, int *offsets, + int offsetcount) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre32_exec(const pcre32 *argument_re, const pcre32_extra *extra_data, + PCRE_SPTR32 subject, int length, int start_offset, int options, int *offsets, + int offsetcount) +#endif +{ +int rc, ocount, arg_offset_max; +int newline; +BOOL using_temporary_offsets = FALSE; +BOOL anchored; +BOOL startline; +BOOL firstline; +BOOL utf; +BOOL has_first_char = FALSE; +BOOL has_req_char = FALSE; +pcre_uchar first_char = 0; +pcre_uchar first_char2 = 0; +pcre_uchar req_char = 0; +pcre_uchar req_char2 = 0; +match_data match_block; +match_data *md = &match_block; +const pcre_uint8 *tables; +const pcre_uint8 *start_bits = NULL; +PCRE_PUCHAR start_match = (PCRE_PUCHAR)subject + start_offset; +PCRE_PUCHAR end_subject; +PCRE_PUCHAR start_partial = NULL; +PCRE_PUCHAR match_partial = NULL; +PCRE_PUCHAR req_char_ptr = start_match - 1; + +const pcre_study_data *study; +const REAL_PCRE *re = (const REAL_PCRE *)argument_re; + +#ifdef NO_RECURSE +heapframe frame_zero; +frame_zero.Xprevframe = NULL; /* Marks the top level */ +frame_zero.Xnextframe = NULL; /* None are allocated yet */ +md->match_frames_base = &frame_zero; +#endif + +/* Check for the special magic call that measures the size of the stack used +per recursive call of match(). Without the funny casting for sizeof, a Windows +compiler gave this error: "unary minus operator applied to unsigned type, +result still unsigned". Hopefully the cast fixes that. */ + +if (re == NULL && extra_data == NULL && subject == NULL && length == -999 && + start_offset == -999) +#ifdef NO_RECURSE + return -((int)sizeof(heapframe)); +#else + return match(NULL, NULL, NULL, 0, NULL, NULL, 0); +#endif + +/* Plausibility checks */ + +if ((options & ~PUBLIC_EXEC_OPTIONS) != 0) return PCRE_ERROR_BADOPTION; +if (re == NULL || subject == NULL || (offsets == NULL && offsetcount > 0)) + return PCRE_ERROR_NULL; +if (offsetcount < 0) return PCRE_ERROR_BADCOUNT; +if (length < 0) return PCRE_ERROR_BADLENGTH; +if (start_offset < 0 || start_offset > length) return PCRE_ERROR_BADOFFSET; + +/* Check that the first field in the block is the magic number. If it is not, +return with PCRE_ERROR_BADMAGIC. However, if the magic number is equal to +REVERSED_MAGIC_NUMBER we return with PCRE_ERROR_BADENDIANNESS, which +means that the pattern is likely compiled with different endianness. */ + +if (re->magic_number != MAGIC_NUMBER) + return re->magic_number == REVERSED_MAGIC_NUMBER? + PCRE_ERROR_BADENDIANNESS:PCRE_ERROR_BADMAGIC; +if ((re->flags & PCRE_MODE) == 0) return PCRE_ERROR_BADMODE; + +/* These two settings are used in the code for checking a UTF-8 string that +follows immediately afterwards. Other values in the md block are used only +during "normal" pcre_exec() processing, not when the JIT support is in use, +so they are set up later. */ + +/* PCRE_UTF16 has the same value as PCRE_UTF8. */ +utf = md->utf = (re->options & PCRE_UTF8) != 0; +md->partial = ((options & PCRE_PARTIAL_HARD) != 0)? 2 : + ((options & PCRE_PARTIAL_SOFT) != 0)? 1 : 0; + +/* Check a UTF-8 string if required. Pass back the character offset and error +code for an invalid string if a results vector is available. */ + +#ifdef SUPPORT_UTF +if (utf && (options & PCRE_NO_UTF8_CHECK) == 0) + { + int erroroffset; + int errorcode = PRIV(valid_utf)((PCRE_PUCHAR)subject, length, &erroroffset); + if (errorcode != 0) + { + if (offsetcount >= 2) + { + offsets[0] = erroroffset; + offsets[1] = errorcode; + } +#if defined COMPILE_PCRE8 + return (errorcode <= PCRE_UTF8_ERR5 && md->partial > 1)? + PCRE_ERROR_SHORTUTF8 : PCRE_ERROR_BADUTF8; +#elif defined COMPILE_PCRE16 + return (errorcode <= PCRE_UTF16_ERR1 && md->partial > 1)? + PCRE_ERROR_SHORTUTF16 : PCRE_ERROR_BADUTF16; +#elif defined COMPILE_PCRE32 + return PCRE_ERROR_BADUTF32; +#endif + } +#if defined COMPILE_PCRE8 || defined COMPILE_PCRE16 + /* Check that a start_offset points to the start of a UTF character. */ + if (start_offset > 0 && start_offset < length && + NOT_FIRSTCHAR(((PCRE_PUCHAR)subject)[start_offset])) + return PCRE_ERROR_BADUTF8_OFFSET; +#endif + } +#endif + +/* If the pattern was successfully studied with JIT support, run the JIT +executable instead of the rest of this function. Most options must be set at +compile time for the JIT code to be usable. Fallback to the normal code path if +an unsupported flag is set. */ + +#ifdef SUPPORT_JIT +if (extra_data != NULL + && (extra_data->flags & (PCRE_EXTRA_EXECUTABLE_JIT | + PCRE_EXTRA_TABLES)) == PCRE_EXTRA_EXECUTABLE_JIT + && extra_data->executable_jit != NULL + && (options & ~PUBLIC_JIT_EXEC_OPTIONS) == 0) + { + rc = PRIV(jit_exec)(extra_data, (const pcre_uchar *)subject, length, + start_offset, options, offsets, offsetcount); + + /* PCRE_ERROR_NULL means that the selected normal or partial matching + mode is not compiled. In this case we simply fallback to interpreter. */ + + if (rc != PCRE_ERROR_JIT_BADOPTION) return rc; + } +#endif + +/* Carry on with non-JIT matching. This information is for finding all the +numbers associated with a given name, for condition testing. */ + +md->name_table = (pcre_uchar *)re + re->name_table_offset; +md->name_count = re->name_count; +md->name_entry_size = re->name_entry_size; + +/* Fish out the optional data from the extra_data structure, first setting +the default values. */ + +study = NULL; +md->match_limit = MATCH_LIMIT; +md->match_limit_recursion = MATCH_LIMIT_RECURSION; +md->callout_data = NULL; + +/* The table pointer is always in native byte order. */ + +tables = re->tables; + +/* The two limit values override the defaults, whatever their value. */ + +if (extra_data != NULL) + { + unsigned long int flags = extra_data->flags; + if ((flags & PCRE_EXTRA_STUDY_DATA) != 0) + study = (const pcre_study_data *)extra_data->study_data; + if ((flags & PCRE_EXTRA_MATCH_LIMIT) != 0) + md->match_limit = extra_data->match_limit; + if ((flags & PCRE_EXTRA_MATCH_LIMIT_RECURSION) != 0) + md->match_limit_recursion = extra_data->match_limit_recursion; + if ((flags & PCRE_EXTRA_CALLOUT_DATA) != 0) + md->callout_data = extra_data->callout_data; + if ((flags & PCRE_EXTRA_TABLES) != 0) tables = extra_data->tables; + } + +/* Limits in the regex override only if they are smaller. */ + +if ((re->flags & PCRE_MLSET) != 0 && re->limit_match < md->match_limit) + md->match_limit = re->limit_match; + +if ((re->flags & PCRE_RLSET) != 0 && + re->limit_recursion < md->match_limit_recursion) + md->match_limit_recursion = re->limit_recursion; + +/* If the exec call supplied NULL for tables, use the inbuilt ones. This +is a feature that makes it possible to save compiled regex and re-use them +in other programs later. */ + +if (tables == NULL) tables = PRIV(default_tables); + +/* Set up other data */ + +anchored = ((re->options | options) & PCRE_ANCHORED) != 0; +startline = (re->flags & PCRE_STARTLINE) != 0; +firstline = (re->options & PCRE_FIRSTLINE) != 0; + +/* The code starts after the real_pcre block and the capture name table. */ + +md->start_code = (const pcre_uchar *)re + re->name_table_offset + + re->name_count * re->name_entry_size; + +md->start_subject = (PCRE_PUCHAR)subject; +md->start_offset = start_offset; +md->end_subject = md->start_subject + length; +end_subject = md->end_subject; + +md->endonly = (re->options & PCRE_DOLLAR_ENDONLY) != 0; +md->use_ucp = (re->options & PCRE_UCP) != 0; +md->jscript_compat = (re->options & PCRE_JAVASCRIPT_COMPAT) != 0; +md->ignore_skip_arg = 0; + +/* Some options are unpacked into BOOL variables in the hope that testing +them will be faster than individual option bits. */ + +md->notbol = (options & PCRE_NOTBOL) != 0; +md->noteol = (options & PCRE_NOTEOL) != 0; +md->notempty = (options & PCRE_NOTEMPTY) != 0; +md->notempty_atstart = (options & PCRE_NOTEMPTY_ATSTART) != 0; + +md->hitend = FALSE; +md->mark = md->nomatch_mark = NULL; /* In case never set */ + +md->recursive = NULL; /* No recursion at top level */ +md->hasthen = (re->flags & PCRE_HASTHEN) != 0; + +md->lcc = tables + lcc_offset; +md->fcc = tables + fcc_offset; +md->ctypes = tables + ctypes_offset; + +/* Handle different \R options. */ + +switch (options & (PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE)) + { + case 0: + if ((re->options & (PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE)) != 0) + md->bsr_anycrlf = (re->options & PCRE_BSR_ANYCRLF) != 0; + else +#ifdef BSR_ANYCRLF + md->bsr_anycrlf = TRUE; +#else + md->bsr_anycrlf = FALSE; +#endif + break; + + case PCRE_BSR_ANYCRLF: + md->bsr_anycrlf = TRUE; + break; + + case PCRE_BSR_UNICODE: + md->bsr_anycrlf = FALSE; + break; + + default: return PCRE_ERROR_BADNEWLINE; + } + +/* Handle different types of newline. The three bits give eight cases. If +nothing is set at run time, whatever was used at compile time applies. */ + +switch ((((options & PCRE_NEWLINE_BITS) == 0)? re->options : + (pcre_uint32)options) & PCRE_NEWLINE_BITS) + { + case 0: newline = NEWLINE; break; /* Compile-time default */ + case PCRE_NEWLINE_CR: newline = CHAR_CR; break; + case PCRE_NEWLINE_LF: newline = CHAR_NL; break; + case PCRE_NEWLINE_CR+ + PCRE_NEWLINE_LF: newline = (CHAR_CR << 8) | CHAR_NL; break; + case PCRE_NEWLINE_ANY: newline = -1; break; + case PCRE_NEWLINE_ANYCRLF: newline = -2; break; + default: return PCRE_ERROR_BADNEWLINE; + } + +if (newline == -2) + { + md->nltype = NLTYPE_ANYCRLF; + } +else if (newline < 0) + { + md->nltype = NLTYPE_ANY; + } +else + { + md->nltype = NLTYPE_FIXED; + if (newline > 255) + { + md->nllen = 2; + md->nl[0] = (newline >> 8) & 255; + md->nl[1] = newline & 255; + } + else + { + md->nllen = 1; + md->nl[0] = newline; + } + } + +/* Partial matching was originally supported only for a restricted set of +regexes; from release 8.00 there are no restrictions, but the bits are still +defined (though never set). So there's no harm in leaving this code. */ + +if (md->partial && (re->flags & PCRE_NOPARTIAL) != 0) + return PCRE_ERROR_BADPARTIAL; + +/* If the expression has got more back references than the offsets supplied can +hold, we get a temporary chunk of working store to use during the matching. +Otherwise, we can use the vector supplied, rounding down its size to a multiple +of 3. */ + +ocount = offsetcount - (offsetcount % 3); +arg_offset_max = (2*ocount)/3; + +if (re->top_backref > 0 && re->top_backref >= ocount/3) + { + ocount = re->top_backref * 3 + 3; + md->offset_vector = (int *)(PUBL(malloc))(ocount * sizeof(int)); + if (md->offset_vector == NULL) return PCRE_ERROR_NOMEMORY; + using_temporary_offsets = TRUE; + DPRINTF(("Got memory to hold back references\n")); + } +else md->offset_vector = offsets; +md->offset_end = ocount; +md->offset_max = (2*ocount)/3; +md->capture_last = 0; + +/* Reset the working variable associated with each extraction. These should +never be used unless previously set, but they get saved and restored, and so we +initialize them to avoid reading uninitialized locations. Also, unset the +offsets for the matched string. This is really just for tidiness with callouts, +in case they inspect these fields. */ + +if (md->offset_vector != NULL) + { + register int *iptr = md->offset_vector + ocount; + register int *iend = iptr - re->top_bracket; + if (iend < md->offset_vector + 2) iend = md->offset_vector + 2; + while (--iptr >= iend) *iptr = -1; + if (offsetcount > 0) md->offset_vector[0] = -1; + if (offsetcount > 1) md->offset_vector[1] = -1; + } + +/* Set up the first character to match, if available. The first_char value is +never set for an anchored regular expression, but the anchoring may be forced +at run time, so we have to test for anchoring. The first char may be unset for +an unanchored pattern, of course. If there's no first char and the pattern was +studied, there may be a bitmap of possible first characters. */ + +if (!anchored) + { + if ((re->flags & PCRE_FIRSTSET) != 0) + { + has_first_char = TRUE; + first_char = first_char2 = (pcre_uchar)(re->first_char); + if ((re->flags & PCRE_FCH_CASELESS) != 0) + { + first_char2 = TABLE_GET(first_char, md->fcc, first_char); +#if defined SUPPORT_UCP && !(defined COMPILE_PCRE8) + if (utf && first_char > 127) + first_char2 = UCD_OTHERCASE(first_char); +#endif + } + } + else + if (!startline && study != NULL && + (study->flags & PCRE_STUDY_MAPPED) != 0) + start_bits = study->start_bits; + } + +/* For anchored or unanchored matches, there may be a "last known required +character" set. */ + +if ((re->flags & PCRE_REQCHSET) != 0) + { + has_req_char = TRUE; + req_char = req_char2 = (pcre_uchar)(re->req_char); + if ((re->flags & PCRE_RCH_CASELESS) != 0) + { + req_char2 = TABLE_GET(req_char, md->fcc, req_char); +#if defined SUPPORT_UCP && !(defined COMPILE_PCRE8) + if (utf && req_char > 127) + req_char2 = UCD_OTHERCASE(req_char); +#endif + } + } + + +/* ==========================================================================*/ + +/* Loop for handling unanchored repeated matching attempts; for anchored regexs +the loop runs just once. */ + +for(;;) + { + PCRE_PUCHAR save_end_subject = end_subject; + PCRE_PUCHAR new_start_match; + + /* If firstline is TRUE, the start of the match is constrained to the first + line of a multiline string. That is, the match must be before or at the first + newline. Implement this by temporarily adjusting end_subject so that we stop + scanning at a newline. If the match fails at the newline, later code breaks + this loop. */ + + if (firstline) + { + PCRE_PUCHAR t = start_match; +#ifdef SUPPORT_UTF + if (utf) + { + while (t < md->end_subject && !IS_NEWLINE(t)) + { + t++; + ACROSSCHAR(t < end_subject, *t, t++); + } + } + else +#endif + while (t < md->end_subject && !IS_NEWLINE(t)) t++; + end_subject = t; + } + + /* There are some optimizations that avoid running the match if a known + starting point is not found, or if a known later character is not present. + However, there is an option that disables these, for testing and for ensuring + that all callouts do actually occur. The option can be set in the regex by + (*NO_START_OPT) or passed in match-time options. */ + + if (((options | re->options) & PCRE_NO_START_OPTIMIZE) == 0) + { + /* Advance to a unique first char if there is one. */ + + if (has_first_char) + { + pcre_uchar smc; + + if (first_char != first_char2) + while (start_match < end_subject && + (smc = UCHAR21TEST(start_match)) != first_char && smc != first_char2) + start_match++; + else + while (start_match < end_subject && UCHAR21TEST(start_match) != first_char) + start_match++; + } + + /* Or to just after a linebreak for a multiline match */ + + else if (startline) + { + if (start_match > md->start_subject + start_offset) + { +#ifdef SUPPORT_UTF + if (utf) + { + while (start_match < end_subject && !WAS_NEWLINE(start_match)) + { + start_match++; + ACROSSCHAR(start_match < end_subject, *start_match, + start_match++); + } + } + else +#endif + while (start_match < end_subject && !WAS_NEWLINE(start_match)) + start_match++; + + /* If we have just passed a CR and the newline option is ANY or ANYCRLF, + and we are now at a LF, advance the match position by one more character. + */ + + if (start_match[-1] == CHAR_CR && + (md->nltype == NLTYPE_ANY || md->nltype == NLTYPE_ANYCRLF) && + start_match < end_subject && + UCHAR21TEST(start_match) == CHAR_NL) + start_match++; + } + } + + /* Or to a non-unique first byte after study */ + + else if (start_bits != NULL) + { + while (start_match < end_subject) + { + register pcre_uint32 c = UCHAR21TEST(start_match); +#ifndef COMPILE_PCRE8 + if (c > 255) c = 255; +#endif + if ((start_bits[c/8] & (1 << (c&7))) != 0) break; + start_match++; + } + } + } /* Starting optimizations */ + + /* Restore fudged end_subject */ + + end_subject = save_end_subject; + + /* The following two optimizations are disabled for partial matching or if + disabling is explicitly requested. */ + + if (((options | re->options) & PCRE_NO_START_OPTIMIZE) == 0 && !md->partial) + { + /* If the pattern was studied, a minimum subject length may be set. This is + a lower bound; no actual string of that length may actually match the + pattern. Although the value is, strictly, in characters, we treat it as + bytes to avoid spending too much time in this optimization. */ + + if (study != NULL && (study->flags & PCRE_STUDY_MINLEN) != 0 && + (pcre_uint32)(end_subject - start_match) < study->minlength) + { + rc = MATCH_NOMATCH; + break; + } + + /* If req_char is set, we know that that character must appear in the + subject for the match to succeed. If the first character is set, req_char + must be later in the subject; otherwise the test starts at the match point. + This optimization can save a huge amount of backtracking in patterns with + nested unlimited repeats that aren't going to match. Writing separate code + for cased/caseless versions makes it go faster, as does using an + autoincrement and backing off on a match. + + HOWEVER: when the subject string is very, very long, searching to its end + can take a long time, and give bad performance on quite ordinary patterns. + This showed up when somebody was matching something like /^\d+C/ on a + 32-megabyte string... so we don't do this when the string is sufficiently + long. */ + + if (has_req_char && end_subject - start_match < REQ_BYTE_MAX) + { + register PCRE_PUCHAR p = start_match + (has_first_char? 1:0); + + /* We don't need to repeat the search if we haven't yet reached the + place we found it at last time. */ + + if (p > req_char_ptr) + { + if (req_char != req_char2) + { + while (p < end_subject) + { + register pcre_uint32 pp = UCHAR21INCTEST(p); + if (pp == req_char || pp == req_char2) { p--; break; } + } + } + else + { + while (p < end_subject) + { + if (UCHAR21INCTEST(p) == req_char) { p--; break; } + } + } + + /* If we can't find the required character, break the matching loop, + forcing a match failure. */ + + if (p >= end_subject) + { + rc = MATCH_NOMATCH; + break; + } + + /* If we have found the required character, save the point where we + found it, so that we don't search again next time round the loop if + the start hasn't passed this character yet. */ + + req_char_ptr = p; + } + } + } + +#ifdef PCRE_DEBUG /* Sigh. Some compilers never learn. */ + printf(">>>> Match against: "); + pchars(start_match, end_subject - start_match, TRUE, md); + printf("\n"); +#endif + + /* OK, we can now run the match. If "hitend" is set afterwards, remember the + first starting point for which a partial match was found. */ + + md->start_match_ptr = start_match; + md->start_used_ptr = start_match; + md->match_call_count = 0; + md->match_function_type = 0; + md->end_offset_top = 0; + md->skip_arg_count = 0; + rc = match(start_match, md->start_code, start_match, 2, md, NULL, 0); + if (md->hitend && start_partial == NULL) + { + start_partial = md->start_used_ptr; + match_partial = start_match; + } + + switch(rc) + { + /* If MATCH_SKIP_ARG reaches this level it means that a MARK that matched + the SKIP's arg was not found. In this circumstance, Perl ignores the SKIP + entirely. The only way we can do that is to re-do the match at the same + point, with a flag to force SKIP with an argument to be ignored. Just + treating this case as NOMATCH does not work because it does not check other + alternatives in patterns such as A(*SKIP:A)B|AC when the subject is AC. */ + + case MATCH_SKIP_ARG: + new_start_match = start_match; + md->ignore_skip_arg = md->skip_arg_count; + break; + + /* SKIP passes back the next starting point explicitly, but if it is no + greater than the match we have just done, treat it as NOMATCH. */ + + case MATCH_SKIP: + if (md->start_match_ptr > start_match) + { + new_start_match = md->start_match_ptr; + break; + } + /* Fall through */ + + /* NOMATCH and PRUNE advance by one character. THEN at this level acts + exactly like PRUNE. Unset ignore SKIP-with-argument. */ + + case MATCH_NOMATCH: + case MATCH_PRUNE: + case MATCH_THEN: + md->ignore_skip_arg = 0; + new_start_match = start_match + 1; +#ifdef SUPPORT_UTF + if (utf) + ACROSSCHAR(new_start_match < end_subject, *new_start_match, + new_start_match++); +#endif + break; + + /* COMMIT disables the bumpalong, but otherwise behaves as NOMATCH. */ + + case MATCH_COMMIT: + rc = MATCH_NOMATCH; + goto ENDLOOP; + + /* Any other return is either a match, or some kind of error. */ + + default: + goto ENDLOOP; + } + + /* Control reaches here for the various types of "no match at this point" + result. Reset the code to MATCH_NOMATCH for subsequent checking. */ + + rc = MATCH_NOMATCH; + + /* If PCRE_FIRSTLINE is set, the match must happen before or at the first + newline in the subject (though it may continue over the newline). Therefore, + if we have just failed to match, starting at a newline, do not continue. */ + + if (firstline && IS_NEWLINE(start_match)) break; + + /* Advance to new matching position */ + + start_match = new_start_match; + + /* Break the loop if the pattern is anchored or if we have passed the end of + the subject. */ + + if (anchored || start_match > end_subject) break; + + /* If we have just passed a CR and we are now at a LF, and the pattern does + not contain any explicit matches for \r or \n, and the newline option is CRLF + or ANY or ANYCRLF, advance the match position by one more character. In + normal matching start_match will aways be greater than the first position at + this stage, but a failed *SKIP can cause a return at the same point, which is + why the first test exists. */ + + if (start_match > (PCRE_PUCHAR)subject + start_offset && + start_match[-1] == CHAR_CR && + start_match < end_subject && + *start_match == CHAR_NL && + (re->flags & PCRE_HASCRORLF) == 0 && + (md->nltype == NLTYPE_ANY || + md->nltype == NLTYPE_ANYCRLF || + md->nllen == 2)) + start_match++; + + md->mark = NULL; /* Reset for start of next match attempt */ + } /* End of for(;;) "bumpalong" loop */ + +/* ==========================================================================*/ + +/* We reach here when rc is not MATCH_NOMATCH, or if one of the stopping +conditions is true: + +(1) The pattern is anchored or the match was failed by (*COMMIT); + +(2) We are past the end of the subject; + +(3) PCRE_FIRSTLINE is set and we have failed to match at a newline, because + this option requests that a match occur at or before the first newline in + the subject. + +When we have a match and the offset vector is big enough to deal with any +backreferences, captured substring offsets will already be set up. In the case +where we had to get some local store to hold offsets for backreference +processing, copy those that we can. In this case there need not be overflow if +certain parts of the pattern were not used, even though there are more +capturing parentheses than vector slots. */ + +ENDLOOP: + +if (rc == MATCH_MATCH || rc == MATCH_ACCEPT) + { + if (using_temporary_offsets) + { + if (arg_offset_max >= 4) + { + memcpy(offsets + 2, md->offset_vector + 2, + (arg_offset_max - 2) * sizeof(int)); + DPRINTF(("Copied offsets from temporary memory\n")); + } + if (md->end_offset_top > arg_offset_max) md->capture_last |= OVFLBIT; + DPRINTF(("Freeing temporary memory\n")); + (PUBL(free))(md->offset_vector); + } + + /* Set the return code to the number of captured strings, or 0 if there were + too many to fit into the vector. */ + + rc = ((md->capture_last & OVFLBIT) != 0 && + md->end_offset_top >= arg_offset_max)? + 0 : md->end_offset_top/2; + + /* If there is space in the offset vector, set any unused pairs at the end of + the pattern to -1 for backwards compatibility. It is documented that this + happens. In earlier versions, the whole set of potential capturing offsets + was set to -1 each time round the loop, but this is handled differently now. + "Gaps" are set to -1 dynamically instead (this fixes a bug). Thus, it is only + those at the end that need unsetting here. We can't just unset them all at + the start of the whole thing because they may get set in one branch that is + not the final matching branch. */ + + if (md->end_offset_top/2 <= re->top_bracket && offsets != NULL) + { + register int *iptr, *iend; + int resetcount = 2 + re->top_bracket * 2; + if (resetcount > offsetcount) resetcount = offsetcount; + iptr = offsets + md->end_offset_top; + iend = offsets + resetcount; + while (iptr < iend) *iptr++ = -1; + } + + /* If there is space, set up the whole thing as substring 0. The value of + md->start_match_ptr might be modified if \K was encountered on the success + matching path. */ + + if (offsetcount < 2) rc = 0; else + { + offsets[0] = (int)(md->start_match_ptr - md->start_subject); + offsets[1] = (int)(md->end_match_ptr - md->start_subject); + } + + /* Return MARK data if requested */ + + if (extra_data != NULL && (extra_data->flags & PCRE_EXTRA_MARK) != 0) + *(extra_data->mark) = (pcre_uchar *)md->mark; + DPRINTF((">>>> returning %d\n", rc)); +#ifdef NO_RECURSE + release_match_heapframes(&frame_zero); +#endif + return rc; + } + +/* Control gets here if there has been an error, or if the overall match +attempt has failed at all permitted starting positions. */ + +if (using_temporary_offsets) + { + DPRINTF(("Freeing temporary memory\n")); + (PUBL(free))(md->offset_vector); + } + +/* For anything other than nomatch or partial match, just return the code. */ + +if (rc != MATCH_NOMATCH && rc != PCRE_ERROR_PARTIAL) + { + DPRINTF((">>>> error: returning %d\n", rc)); +#ifdef NO_RECURSE + release_match_heapframes(&frame_zero); +#endif + return rc; + } + +/* Handle partial matches - disable any mark data */ + +if (match_partial != NULL) + { + DPRINTF((">>>> returning PCRE_ERROR_PARTIAL\n")); + md->mark = NULL; + if (offsetcount > 1) + { + offsets[0] = (int)(start_partial - (PCRE_PUCHAR)subject); + offsets[1] = (int)(end_subject - (PCRE_PUCHAR)subject); + if (offsetcount > 2) + offsets[2] = (int)(match_partial - (PCRE_PUCHAR)subject); + } + rc = PCRE_ERROR_PARTIAL; + } + +/* This is the classic nomatch case */ + +else + { + DPRINTF((">>>> returning PCRE_ERROR_NOMATCH\n")); + rc = PCRE_ERROR_NOMATCH; + } + +/* Return the MARK data if it has been requested. */ + +if (extra_data != NULL && (extra_data->flags & PCRE_EXTRA_MARK) != 0) + *(extra_data->mark) = (pcre_uchar *)md->nomatch_mark; +#ifdef NO_RECURSE + release_match_heapframes(&frame_zero); +#endif +return rc; +} + +/* End of pcre_exec.c */ diff --git a/src/third_party/pcre-8.41/pcre_fullinfo.c b/src/third_party/pcre-8.41/pcre_fullinfo.c new file mode 100644 index 00000000000..a6c2ece6ca5 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_fullinfo.c @@ -0,0 +1,245 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2013 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This module contains the external function pcre_fullinfo(), which returns +information about a compiled pattern. */ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "pcre_internal.h" + + +/************************************************* +* Return info about compiled pattern * +*************************************************/ + +/* This is a newer "info" function which has an extensible interface so +that additional items can be added compatibly. + +Arguments: + argument_re points to compiled code + extra_data points extra data, or NULL + what what information is required + where where to put the information + +Returns: 0 if data returned, negative on error +*/ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre_fullinfo(const pcre *argument_re, const pcre_extra *extra_data, + int what, void *where) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre16_fullinfo(const pcre16 *argument_re, const pcre16_extra *extra_data, + int what, void *where) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre32_fullinfo(const pcre32 *argument_re, const pcre32_extra *extra_data, + int what, void *where) +#endif +{ +const REAL_PCRE *re = (const REAL_PCRE *)argument_re; +const pcre_study_data *study = NULL; + +if (re == NULL || where == NULL) return PCRE_ERROR_NULL; + +if (extra_data != NULL && (extra_data->flags & PCRE_EXTRA_STUDY_DATA) != 0) + study = (const pcre_study_data *)extra_data->study_data; + +/* Check that the first field in the block is the magic number. If it is not, +return with PCRE_ERROR_BADMAGIC. However, if the magic number is equal to +REVERSED_MAGIC_NUMBER we return with PCRE_ERROR_BADENDIANNESS, which +means that the pattern is likely compiled with different endianness. */ + +if (re->magic_number != MAGIC_NUMBER) + return re->magic_number == REVERSED_MAGIC_NUMBER? + PCRE_ERROR_BADENDIANNESS:PCRE_ERROR_BADMAGIC; + +/* Check that this pattern was compiled in the correct bit mode */ + +if ((re->flags & PCRE_MODE) == 0) return PCRE_ERROR_BADMODE; + +switch (what) + { + case PCRE_INFO_OPTIONS: + *((unsigned long int *)where) = re->options & PUBLIC_COMPILE_OPTIONS; + break; + + case PCRE_INFO_SIZE: + *((size_t *)where) = re->size; + break; + + case PCRE_INFO_STUDYSIZE: + *((size_t *)where) = (study == NULL)? 0 : study->size; + break; + + case PCRE_INFO_JITSIZE: +#ifdef SUPPORT_JIT + *((size_t *)where) = + (extra_data != NULL && + (extra_data->flags & PCRE_EXTRA_EXECUTABLE_JIT) != 0 && + extra_data->executable_jit != NULL)? + PRIV(jit_get_size)(extra_data->executable_jit) : 0; +#else + *((size_t *)where) = 0; +#endif + break; + + case PCRE_INFO_CAPTURECOUNT: + *((int *)where) = re->top_bracket; + break; + + case PCRE_INFO_BACKREFMAX: + *((int *)where) = re->top_backref; + break; + + case PCRE_INFO_FIRSTBYTE: + *((int *)where) = + ((re->flags & PCRE_FIRSTSET) != 0)? (int)re->first_char : + ((re->flags & PCRE_STARTLINE) != 0)? -1 : -2; + break; + + case PCRE_INFO_FIRSTCHARACTER: + *((pcre_uint32 *)where) = + (re->flags & PCRE_FIRSTSET) != 0 ? re->first_char : 0; + break; + + case PCRE_INFO_FIRSTCHARACTERFLAGS: + *((int *)where) = + ((re->flags & PCRE_FIRSTSET) != 0) ? 1 : + ((re->flags & PCRE_STARTLINE) != 0) ? 2 : 0; + break; + + /* Make sure we pass back the pointer to the bit vector in the external + block, not the internal copy (with flipped integer fields). */ + + case PCRE_INFO_FIRSTTABLE: + *((const pcre_uint8 **)where) = + (study != NULL && (study->flags & PCRE_STUDY_MAPPED) != 0)? + ((const pcre_study_data *)extra_data->study_data)->start_bits : NULL; + break; + + case PCRE_INFO_MINLENGTH: + *((int *)where) = + (study != NULL && (study->flags & PCRE_STUDY_MINLEN) != 0)? + (int)(study->minlength) : -1; + break; + + case PCRE_INFO_JIT: + *((int *)where) = extra_data != NULL && + (extra_data->flags & PCRE_EXTRA_EXECUTABLE_JIT) != 0 && + extra_data->executable_jit != NULL; + break; + + case PCRE_INFO_LASTLITERAL: + *((int *)where) = + ((re->flags & PCRE_REQCHSET) != 0)? (int)re->req_char : -1; + break; + + case PCRE_INFO_REQUIREDCHAR: + *((pcre_uint32 *)where) = + ((re->flags & PCRE_REQCHSET) != 0) ? re->req_char : 0; + break; + + case PCRE_INFO_REQUIREDCHARFLAGS: + *((int *)where) = + ((re->flags & PCRE_REQCHSET) != 0); + break; + + case PCRE_INFO_NAMEENTRYSIZE: + *((int *)where) = re->name_entry_size; + break; + + case PCRE_INFO_NAMECOUNT: + *((int *)where) = re->name_count; + break; + + case PCRE_INFO_NAMETABLE: + *((const pcre_uchar **)where) = (const pcre_uchar *)re + re->name_table_offset; + break; + + case PCRE_INFO_DEFAULT_TABLES: + *((const pcre_uint8 **)where) = (const pcre_uint8 *)(PRIV(default_tables)); + break; + + /* From release 8.00 this will always return TRUE because NOPARTIAL is + no longer ever set (the restrictions have been removed). */ + + case PCRE_INFO_OKPARTIAL: + *((int *)where) = (re->flags & PCRE_NOPARTIAL) == 0; + break; + + case PCRE_INFO_JCHANGED: + *((int *)where) = (re->flags & PCRE_JCHANGED) != 0; + break; + + case PCRE_INFO_HASCRORLF: + *((int *)where) = (re->flags & PCRE_HASCRORLF) != 0; + break; + + case PCRE_INFO_MAXLOOKBEHIND: + *((int *)where) = re->max_lookbehind; + break; + + case PCRE_INFO_MATCHLIMIT: + if ((re->flags & PCRE_MLSET) == 0) return PCRE_ERROR_UNSET; + *((pcre_uint32 *)where) = re->limit_match; + break; + + case PCRE_INFO_RECURSIONLIMIT: + if ((re->flags & PCRE_RLSET) == 0) return PCRE_ERROR_UNSET; + *((pcre_uint32 *)where) = re->limit_recursion; + break; + + case PCRE_INFO_MATCH_EMPTY: + *((int *)where) = (re->flags & PCRE_MATCH_EMPTY) != 0; + break; + + default: return PCRE_ERROR_BADOPTION; + } + +return 0; +} + +/* End of pcre_fullinfo.c */ diff --git a/src/third_party/pcre-8.41/pcre_get.c b/src/third_party/pcre-8.41/pcre_get.c new file mode 100644 index 00000000000..9475d5e88cd --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_get.c @@ -0,0 +1,669 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This module contains some convenience functions for extracting substrings +from the subject string after a regex match has succeeded. The original idea +for these functions came from Scott Wimer. */ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "pcre_internal.h" + + +/************************************************* +* Find number for named string * +*************************************************/ + +/* This function is used by the get_first_set() function below, as well +as being generally available. It assumes that names are unique. + +Arguments: + code the compiled regex + stringname the name whose number is required + +Returns: the number of the named parentheses, or a negative number + (PCRE_ERROR_NOSUBSTRING) if not found +*/ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre_get_stringnumber(const pcre *code, const char *stringname) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre16_get_stringnumber(const pcre16 *code, PCRE_SPTR16 stringname) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre32_get_stringnumber(const pcre32 *code, PCRE_SPTR32 stringname) +#endif +{ +int rc; +int entrysize; +int top, bot; +pcre_uchar *nametable; + +#ifdef COMPILE_PCRE8 +if ((rc = pcre_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top)) != 0) + return rc; +if (top <= 0) return PCRE_ERROR_NOSUBSTRING; + +if ((rc = pcre_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize)) != 0) + return rc; +if ((rc = pcre_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, &nametable)) != 0) + return rc; +#endif +#ifdef COMPILE_PCRE16 +if ((rc = pcre16_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top)) != 0) + return rc; +if (top <= 0) return PCRE_ERROR_NOSUBSTRING; + +if ((rc = pcre16_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize)) != 0) + return rc; +if ((rc = pcre16_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, &nametable)) != 0) + return rc; +#endif +#ifdef COMPILE_PCRE32 +if ((rc = pcre32_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top)) != 0) + return rc; +if (top <= 0) return PCRE_ERROR_NOSUBSTRING; + +if ((rc = pcre32_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize)) != 0) + return rc; +if ((rc = pcre32_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, &nametable)) != 0) + return rc; +#endif + +bot = 0; +while (top > bot) + { + int mid = (top + bot) / 2; + pcre_uchar *entry = nametable + entrysize*mid; + int c = STRCMP_UC_UC((pcre_uchar *)stringname, + (pcre_uchar *)(entry + IMM2_SIZE)); + if (c == 0) return GET2(entry, 0); + if (c > 0) bot = mid + 1; else top = mid; + } + +return PCRE_ERROR_NOSUBSTRING; +} + + + +/************************************************* +* Find (multiple) entries for named string * +*************************************************/ + +/* This is used by the get_first_set() function below, as well as being +generally available. It is used when duplicated names are permitted. + +Arguments: + code the compiled regex + stringname the name whose entries required + firstptr where to put the pointer to the first entry + lastptr where to put the pointer to the last entry + +Returns: the length of each entry, or a negative number + (PCRE_ERROR_NOSUBSTRING) if not found +*/ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre_get_stringtable_entries(const pcre *code, const char *stringname, + char **firstptr, char **lastptr) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre16_get_stringtable_entries(const pcre16 *code, PCRE_SPTR16 stringname, + PCRE_UCHAR16 **firstptr, PCRE_UCHAR16 **lastptr) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre32_get_stringtable_entries(const pcre32 *code, PCRE_SPTR32 stringname, + PCRE_UCHAR32 **firstptr, PCRE_UCHAR32 **lastptr) +#endif +{ +int rc; +int entrysize; +int top, bot; +pcre_uchar *nametable, *lastentry; + +#ifdef COMPILE_PCRE8 +if ((rc = pcre_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top)) != 0) + return rc; +if (top <= 0) return PCRE_ERROR_NOSUBSTRING; + +if ((rc = pcre_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize)) != 0) + return rc; +if ((rc = pcre_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, &nametable)) != 0) + return rc; +#endif +#ifdef COMPILE_PCRE16 +if ((rc = pcre16_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top)) != 0) + return rc; +if (top <= 0) return PCRE_ERROR_NOSUBSTRING; + +if ((rc = pcre16_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize)) != 0) + return rc; +if ((rc = pcre16_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, &nametable)) != 0) + return rc; +#endif +#ifdef COMPILE_PCRE32 +if ((rc = pcre32_fullinfo(code, NULL, PCRE_INFO_NAMECOUNT, &top)) != 0) + return rc; +if (top <= 0) return PCRE_ERROR_NOSUBSTRING; + +if ((rc = pcre32_fullinfo(code, NULL, PCRE_INFO_NAMEENTRYSIZE, &entrysize)) != 0) + return rc; +if ((rc = pcre32_fullinfo(code, NULL, PCRE_INFO_NAMETABLE, &nametable)) != 0) + return rc; +#endif + +lastentry = nametable + entrysize * (top - 1); +bot = 0; +while (top > bot) + { + int mid = (top + bot) / 2; + pcre_uchar *entry = nametable + entrysize*mid; + int c = STRCMP_UC_UC((pcre_uchar *)stringname, + (pcre_uchar *)(entry + IMM2_SIZE)); + if (c == 0) + { + pcre_uchar *first = entry; + pcre_uchar *last = entry; + while (first > nametable) + { + if (STRCMP_UC_UC((pcre_uchar *)stringname, + (pcre_uchar *)(first - entrysize + IMM2_SIZE)) != 0) break; + first -= entrysize; + } + while (last < lastentry) + { + if (STRCMP_UC_UC((pcre_uchar *)stringname, + (pcre_uchar *)(last + entrysize + IMM2_SIZE)) != 0) break; + last += entrysize; + } +#if defined COMPILE_PCRE8 + *firstptr = (char *)first; + *lastptr = (char *)last; +#elif defined COMPILE_PCRE16 + *firstptr = (PCRE_UCHAR16 *)first; + *lastptr = (PCRE_UCHAR16 *)last; +#elif defined COMPILE_PCRE32 + *firstptr = (PCRE_UCHAR32 *)first; + *lastptr = (PCRE_UCHAR32 *)last; +#endif + return entrysize; + } + if (c > 0) bot = mid + 1; else top = mid; + } + +return PCRE_ERROR_NOSUBSTRING; +} + + + +/************************************************* +* Find first set of multiple named strings * +*************************************************/ + +/* This function allows for duplicate names in the table of named substrings. +It returns the number of the first one that was set in a pattern match. + +Arguments: + code the compiled regex + stringname the name of the capturing substring + ovector the vector of matched substrings + stringcount number of captured substrings + +Returns: the number of the first that is set, + or the number of the last one if none are set, + or a negative number on error +*/ + +#if defined COMPILE_PCRE8 +static int +get_first_set(const pcre *code, const char *stringname, int *ovector, + int stringcount) +#elif defined COMPILE_PCRE16 +static int +get_first_set(const pcre16 *code, PCRE_SPTR16 stringname, int *ovector, + int stringcount) +#elif defined COMPILE_PCRE32 +static int +get_first_set(const pcre32 *code, PCRE_SPTR32 stringname, int *ovector, + int stringcount) +#endif +{ +const REAL_PCRE *re = (const REAL_PCRE *)code; +int entrysize; +pcre_uchar *entry; +#if defined COMPILE_PCRE8 +char *first, *last; +#elif defined COMPILE_PCRE16 +PCRE_UCHAR16 *first, *last; +#elif defined COMPILE_PCRE32 +PCRE_UCHAR32 *first, *last; +#endif + +#if defined COMPILE_PCRE8 +if ((re->options & PCRE_DUPNAMES) == 0 && (re->flags & PCRE_JCHANGED) == 0) + return pcre_get_stringnumber(code, stringname); +entrysize = pcre_get_stringtable_entries(code, stringname, &first, &last); +#elif defined COMPILE_PCRE16 +if ((re->options & PCRE_DUPNAMES) == 0 && (re->flags & PCRE_JCHANGED) == 0) + return pcre16_get_stringnumber(code, stringname); +entrysize = pcre16_get_stringtable_entries(code, stringname, &first, &last); +#elif defined COMPILE_PCRE32 +if ((re->options & PCRE_DUPNAMES) == 0 && (re->flags & PCRE_JCHANGED) == 0) + return pcre32_get_stringnumber(code, stringname); +entrysize = pcre32_get_stringtable_entries(code, stringname, &first, &last); +#endif +if (entrysize <= 0) return entrysize; +for (entry = (pcre_uchar *)first; entry <= (pcre_uchar *)last; entry += entrysize) + { + int n = GET2(entry, 0); + if (n < stringcount && ovector[n*2] >= 0) return n; + } +return GET2(entry, 0); +} + + + + +/************************************************* +* Copy captured string to given buffer * +*************************************************/ + +/* This function copies a single captured substring into a given buffer. +Note that we use memcpy() rather than strncpy() in case there are binary zeros +in the string. + +Arguments: + subject the subject string that was matched + ovector pointer to the offsets table + stringcount the number of substrings that were captured + (i.e. the yield of the pcre_exec call, unless + that was zero, in which case it should be 1/3 + of the offset table size) + stringnumber the number of the required substring + buffer where to put the substring + size the size of the buffer + +Returns: if successful: + the length of the copied string, not including the zero + that is put on the end; can be zero + if not successful: + PCRE_ERROR_NOMEMORY (-6) buffer too small + PCRE_ERROR_NOSUBSTRING (-7) no such captured substring +*/ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre_copy_substring(const char *subject, int *ovector, int stringcount, + int stringnumber, char *buffer, int size) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre16_copy_substring(PCRE_SPTR16 subject, int *ovector, int stringcount, + int stringnumber, PCRE_UCHAR16 *buffer, int size) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre32_copy_substring(PCRE_SPTR32 subject, int *ovector, int stringcount, + int stringnumber, PCRE_UCHAR32 *buffer, int size) +#endif +{ +int yield; +if (stringnumber < 0 || stringnumber >= stringcount) + return PCRE_ERROR_NOSUBSTRING; +stringnumber *= 2; +yield = ovector[stringnumber+1] - ovector[stringnumber]; +if (size < yield + 1) return PCRE_ERROR_NOMEMORY; +memcpy(buffer, subject + ovector[stringnumber], IN_UCHARS(yield)); +buffer[yield] = 0; +return yield; +} + + + +/************************************************* +* Copy named captured string to given buffer * +*************************************************/ + +/* This function copies a single captured substring into a given buffer, +identifying it by name. If the regex permits duplicate names, the first +substring that is set is chosen. + +Arguments: + code the compiled regex + subject the subject string that was matched + ovector pointer to the offsets table + stringcount the number of substrings that were captured + (i.e. the yield of the pcre_exec call, unless + that was zero, in which case it should be 1/3 + of the offset table size) + stringname the name of the required substring + buffer where to put the substring + size the size of the buffer + +Returns: if successful: + the length of the copied string, not including the zero + that is put on the end; can be zero + if not successful: + PCRE_ERROR_NOMEMORY (-6) buffer too small + PCRE_ERROR_NOSUBSTRING (-7) no such captured substring +*/ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre_copy_named_substring(const pcre *code, const char *subject, + int *ovector, int stringcount, const char *stringname, + char *buffer, int size) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre16_copy_named_substring(const pcre16 *code, PCRE_SPTR16 subject, + int *ovector, int stringcount, PCRE_SPTR16 stringname, + PCRE_UCHAR16 *buffer, int size) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre32_copy_named_substring(const pcre32 *code, PCRE_SPTR32 subject, + int *ovector, int stringcount, PCRE_SPTR32 stringname, + PCRE_UCHAR32 *buffer, int size) +#endif +{ +int n = get_first_set(code, stringname, ovector, stringcount); +if (n <= 0) return n; +#if defined COMPILE_PCRE8 +return pcre_copy_substring(subject, ovector, stringcount, n, buffer, size); +#elif defined COMPILE_PCRE16 +return pcre16_copy_substring(subject, ovector, stringcount, n, buffer, size); +#elif defined COMPILE_PCRE32 +return pcre32_copy_substring(subject, ovector, stringcount, n, buffer, size); +#endif +} + + + +/************************************************* +* Copy all captured strings to new store * +*************************************************/ + +/* This function gets one chunk of store and builds a list of pointers and all +of the captured substrings in it. A NULL pointer is put on the end of the list. + +Arguments: + subject the subject string that was matched + ovector pointer to the offsets table + stringcount the number of substrings that were captured + (i.e. the yield of the pcre_exec call, unless + that was zero, in which case it should be 1/3 + of the offset table size) + listptr set to point to the list of pointers + +Returns: if successful: 0 + if not successful: + PCRE_ERROR_NOMEMORY (-6) failed to get store +*/ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre_get_substring_list(const char *subject, int *ovector, int stringcount, + const char ***listptr) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre16_get_substring_list(PCRE_SPTR16 subject, int *ovector, int stringcount, + PCRE_SPTR16 **listptr) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre32_get_substring_list(PCRE_SPTR32 subject, int *ovector, int stringcount, + PCRE_SPTR32 **listptr) +#endif +{ +int i; +int size = sizeof(pcre_uchar *); +int double_count = stringcount * 2; +pcre_uchar **stringlist; +pcre_uchar *p; + +for (i = 0; i < double_count; i += 2) + { + size += sizeof(pcre_uchar *) + IN_UCHARS(1); + if (ovector[i+1] > ovector[i]) size += IN_UCHARS(ovector[i+1] - ovector[i]); + } + +stringlist = (pcre_uchar **)(PUBL(malloc))(size); +if (stringlist == NULL) return PCRE_ERROR_NOMEMORY; + +#if defined COMPILE_PCRE8 +*listptr = (const char **)stringlist; +#elif defined COMPILE_PCRE16 +*listptr = (PCRE_SPTR16 *)stringlist; +#elif defined COMPILE_PCRE32 +*listptr = (PCRE_SPTR32 *)stringlist; +#endif +p = (pcre_uchar *)(stringlist + stringcount + 1); + +for (i = 0; i < double_count; i += 2) + { + int len = (ovector[i+1] > ovector[i])? (ovector[i+1] - ovector[i]) : 0; + memcpy(p, subject + ovector[i], IN_UCHARS(len)); + *stringlist++ = p; + p += len; + *p++ = 0; + } + +*stringlist = NULL; +return 0; +} + + + +/************************************************* +* Free store obtained by get_substring_list * +*************************************************/ + +/* This function exists for the benefit of people calling PCRE from non-C +programs that can call its functions, but not free() or (PUBL(free))() +directly. + +Argument: the result of a previous pcre_get_substring_list() +Returns: nothing +*/ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN void PCRE_CALL_CONVENTION +pcre_free_substring_list(const char **pointer) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN void PCRE_CALL_CONVENTION +pcre16_free_substring_list(PCRE_SPTR16 *pointer) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN void PCRE_CALL_CONVENTION +pcre32_free_substring_list(PCRE_SPTR32 *pointer) +#endif +{ +(PUBL(free))((void *)pointer); +} + + + +/************************************************* +* Copy captured string to new store * +*************************************************/ + +/* This function copies a single captured substring into a piece of new +store + +Arguments: + subject the subject string that was matched + ovector pointer to the offsets table + stringcount the number of substrings that were captured + (i.e. the yield of the pcre_exec call, unless + that was zero, in which case it should be 1/3 + of the offset table size) + stringnumber the number of the required substring + stringptr where to put a pointer to the substring + +Returns: if successful: + the length of the string, not including the zero that + is put on the end; can be zero + if not successful: + PCRE_ERROR_NOMEMORY (-6) failed to get store + PCRE_ERROR_NOSUBSTRING (-7) substring not present +*/ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre_get_substring(const char *subject, int *ovector, int stringcount, + int stringnumber, const char **stringptr) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre16_get_substring(PCRE_SPTR16 subject, int *ovector, int stringcount, + int stringnumber, PCRE_SPTR16 *stringptr) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre32_get_substring(PCRE_SPTR32 subject, int *ovector, int stringcount, + int stringnumber, PCRE_SPTR32 *stringptr) +#endif +{ +int yield; +pcre_uchar *substring; +if (stringnumber < 0 || stringnumber >= stringcount) + return PCRE_ERROR_NOSUBSTRING; +stringnumber *= 2; +yield = ovector[stringnumber+1] - ovector[stringnumber]; +substring = (pcre_uchar *)(PUBL(malloc))(IN_UCHARS(yield + 1)); +if (substring == NULL) return PCRE_ERROR_NOMEMORY; +memcpy(substring, subject + ovector[stringnumber], IN_UCHARS(yield)); +substring[yield] = 0; +#if defined COMPILE_PCRE8 +*stringptr = (const char *)substring; +#elif defined COMPILE_PCRE16 +*stringptr = (PCRE_SPTR16)substring; +#elif defined COMPILE_PCRE32 +*stringptr = (PCRE_SPTR32)substring; +#endif +return yield; +} + + + +/************************************************* +* Copy named captured string to new store * +*************************************************/ + +/* This function copies a single captured substring, identified by name, into +new store. If the regex permits duplicate names, the first substring that is +set is chosen. + +Arguments: + code the compiled regex + subject the subject string that was matched + ovector pointer to the offsets table + stringcount the number of substrings that were captured + (i.e. the yield of the pcre_exec call, unless + that was zero, in which case it should be 1/3 + of the offset table size) + stringname the name of the required substring + stringptr where to put the pointer + +Returns: if successful: + the length of the copied string, not including the zero + that is put on the end; can be zero + if not successful: + PCRE_ERROR_NOMEMORY (-6) couldn't get memory + PCRE_ERROR_NOSUBSTRING (-7) no such captured substring +*/ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre_get_named_substring(const pcre *code, const char *subject, + int *ovector, int stringcount, const char *stringname, + const char **stringptr) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre16_get_named_substring(const pcre16 *code, PCRE_SPTR16 subject, + int *ovector, int stringcount, PCRE_SPTR16 stringname, + PCRE_SPTR16 *stringptr) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre32_get_named_substring(const pcre32 *code, PCRE_SPTR32 subject, + int *ovector, int stringcount, PCRE_SPTR32 stringname, + PCRE_SPTR32 *stringptr) +#endif +{ +int n = get_first_set(code, stringname, ovector, stringcount); +if (n <= 0) return n; +#if defined COMPILE_PCRE8 +return pcre_get_substring(subject, ovector, stringcount, n, stringptr); +#elif defined COMPILE_PCRE16 +return pcre16_get_substring(subject, ovector, stringcount, n, stringptr); +#elif defined COMPILE_PCRE32 +return pcre32_get_substring(subject, ovector, stringcount, n, stringptr); +#endif +} + + + + +/************************************************* +* Free store obtained by get_substring * +*************************************************/ + +/* This function exists for the benefit of people calling PCRE from non-C +programs that can call its functions, but not free() or (PUBL(free))() +directly. + +Argument: the result of a previous pcre_get_substring() +Returns: nothing +*/ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN void PCRE_CALL_CONVENTION +pcre_free_substring(const char *pointer) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN void PCRE_CALL_CONVENTION +pcre16_free_substring(PCRE_SPTR16 pointer) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN void PCRE_CALL_CONVENTION +pcre32_free_substring(PCRE_SPTR32 pointer) +#endif +{ +(PUBL(free))((void *)pointer); +} + +/* End of pcre_get.c */ diff --git a/src/third_party/pcre-8.41/pcre_globals.c b/src/third_party/pcre-8.41/pcre_globals.c new file mode 100644 index 00000000000..0f106aa9013 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_globals.c @@ -0,0 +1,86 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2014 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This module contains global variables that are exported by the PCRE library. +PCRE is thread-clean and doesn't use any global variables in the normal sense. +However, it calls memory allocation and freeing functions via the four +indirections below, and it can optionally do callouts, using the fifth +indirection. These values can be changed by the caller, but are shared between +all threads. + +For MS Visual Studio and Symbian OS, there are problems in initializing these +variables to non-local functions. In these cases, therefore, an indirection via +a local function is used. + +Also, when compiling for Virtual Pascal, things are done differently, and +global variables are not used. */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "pcre_internal.h" + +#if defined _MSC_VER || defined __SYMBIAN32__ +static void* LocalPcreMalloc(size_t aSize) + { + return malloc(aSize); + } +static void LocalPcreFree(void* aPtr) + { + free(aPtr); + } +PCRE_EXP_DATA_DEFN void *(*PUBL(malloc))(size_t) = LocalPcreMalloc; +PCRE_EXP_DATA_DEFN void (*PUBL(free))(void *) = LocalPcreFree; +PCRE_EXP_DATA_DEFN void *(*PUBL(stack_malloc))(size_t) = LocalPcreMalloc; +PCRE_EXP_DATA_DEFN void (*PUBL(stack_free))(void *) = LocalPcreFree; +PCRE_EXP_DATA_DEFN int (*PUBL(callout))(PUBL(callout_block) *) = NULL; +PCRE_EXP_DATA_DEFN int (*PUBL(stack_guard))(void) = NULL; + +#elif !defined VPCOMPAT +PCRE_EXP_DATA_DEFN void *(*PUBL(malloc))(size_t) = malloc; +PCRE_EXP_DATA_DEFN void (*PUBL(free))(void *) = free; +PCRE_EXP_DATA_DEFN void *(*PUBL(stack_malloc))(size_t) = malloc; +PCRE_EXP_DATA_DEFN void (*PUBL(stack_free))(void *) = free; +PCRE_EXP_DATA_DEFN int (*PUBL(callout))(PUBL(callout_block) *) = NULL; +PCRE_EXP_DATA_DEFN int (*PUBL(stack_guard))(void) = NULL; +#endif + +/* End of pcre_globals.c */ diff --git a/src/third_party/pcre-8.41/pcre_internal.h b/src/third_party/pcre-8.41/pcre_internal.h new file mode 100644 index 00000000000..97ff55d03b3 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_internal.h @@ -0,0 +1,2807 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2016 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* This header contains definitions that are shared between the different +modules, but which are not relevant to the exported API. This includes some +functions whose names all begin with "_pcre_", "_pcre16_" or "_pcre32_" +depending on the PRIV macro. */ + +#ifndef PCRE_INTERNAL_H +#define PCRE_INTERNAL_H + +/* Define PCRE_DEBUG to get debugging output on stdout. */ + +#if 0 +#define PCRE_DEBUG +#endif + +/* PCRE is compiled as an 8 bit library if it is not requested otherwise. */ + +#if !defined COMPILE_PCRE16 && !defined COMPILE_PCRE32 +#define COMPILE_PCRE8 +#endif + +/* If SUPPORT_UCP is defined, SUPPORT_UTF must also be defined. The +"configure" script ensures this, but not everybody uses "configure". */ + +#if defined SUPPORT_UCP && !(defined SUPPORT_UTF) +#define SUPPORT_UTF 1 +#endif + +/* We define SUPPORT_UTF if SUPPORT_UTF8 is enabled for compatibility +reasons with existing code. */ + +#if defined SUPPORT_UTF8 && !(defined SUPPORT_UTF) +#define SUPPORT_UTF 1 +#endif + +/* Fixme: SUPPORT_UTF8 should be eventually disappear from the code. +Until then we define it if SUPPORT_UTF is defined. */ + +#if defined SUPPORT_UTF && !(defined SUPPORT_UTF8) +#define SUPPORT_UTF8 1 +#endif + +/* We do not support both EBCDIC and UTF-8/16/32 at the same time. The "configure" +script prevents both being selected, but not everybody uses "configure". */ + +#if defined EBCDIC && defined SUPPORT_UTF +#error The use of both EBCDIC and SUPPORT_UTF is not supported. +#endif + +/* Use a macro for debugging printing, 'cause that eliminates the use of #ifdef +inline, and there are *still* stupid compilers about that don't like indented +pre-processor statements, or at least there were when I first wrote this. After +all, it had only been about 10 years then... + +It turns out that the Mac Debugging.h header also defines the macro DPRINTF, so +be absolutely sure we get our version. */ + +#undef DPRINTF +#ifdef PCRE_DEBUG +#define DPRINTF(p) printf p +#else +#define DPRINTF(p) /* Nothing */ +#endif + + +/* Standard C headers plus the external interface definition. The only time +setjmp and stdarg are used is when NO_RECURSE is set. */ + +#include +#include +#include +#include +#include +#include + +/* Valgrind (memcheck) support */ + +#ifdef SUPPORT_VALGRIND +#include +#endif + +/* When compiling a DLL for Windows, the exported symbols have to be declared +using some MS magic. I found some useful information on this web page: +http://msdn2.microsoft.com/en-us/library/y4h7bcy6(VS.80).aspx. According to the +information there, using __declspec(dllexport) without "extern" we have a +definition; with "extern" we have a declaration. The settings here override the +setting in pcre.h (which is included below); it defines only PCRE_EXP_DECL, +which is all that is needed for applications (they just import the symbols). We +use: + + PCRE_EXP_DECL for declarations + PCRE_EXP_DEFN for definitions of exported functions + PCRE_EXP_DATA_DEFN for definitions of exported variables + +The reason for the two DEFN macros is that in non-Windows environments, one +does not want to have "extern" before variable definitions because it leads to +compiler warnings. So we distinguish between functions and variables. In +Windows, the two should always be the same. + +The reason for wrapping this in #ifndef PCRE_EXP_DECL is so that pcretest, +which is an application, but needs to import this file in order to "peek" at +internals, can #include pcre.h first to get an application's-eye view. + +In principle, people compiling for non-Windows, non-Unix-like (i.e. uncommon, +special-purpose environments) might want to stick other stuff in front of +exported symbols. That's why, in the non-Windows case, we set PCRE_EXP_DEFN and +PCRE_EXP_DATA_DEFN only if they are not already set. */ + +#ifndef PCRE_EXP_DECL +# ifdef _WIN32 +# ifndef PCRE_STATIC +# define PCRE_EXP_DECL extern __declspec(dllexport) +# define PCRE_EXP_DEFN __declspec(dllexport) +# define PCRE_EXP_DATA_DEFN __declspec(dllexport) +# else +# define PCRE_EXP_DECL extern +# define PCRE_EXP_DEFN +# define PCRE_EXP_DATA_DEFN +# endif +# else +# ifdef __cplusplus +# define PCRE_EXP_DECL extern "C" +# else +# define PCRE_EXP_DECL extern +# endif +# ifndef PCRE_EXP_DEFN +# define PCRE_EXP_DEFN PCRE_EXP_DECL +# endif +# ifndef PCRE_EXP_DATA_DEFN +# define PCRE_EXP_DATA_DEFN +# endif +# endif +#endif + +/* When compiling with the MSVC compiler, it is sometimes necessary to include +a "calling convention" before exported function names. (This is secondhand +information; I know nothing about MSVC myself). For example, something like + + void __cdecl function(....) + +might be needed. In order so make this easy, all the exported functions have +PCRE_CALL_CONVENTION just before their names. It is rarely needed; if not +set, we ensure here that it has no effect. */ + +#ifndef PCRE_CALL_CONVENTION +#define PCRE_CALL_CONVENTION +#endif + +/* We need to have types that specify unsigned 8, 16 and 32-bit integers. We +cannot determine these outside the compilation (e.g. by running a program as +part of "configure") because PCRE is often cross-compiled for use on other +systems. Instead we make use of the maximum sizes that are available at +preprocessor time in standard C environments. */ + +typedef unsigned char pcre_uint8; + +#if USHRT_MAX == 65535 +typedef unsigned short pcre_uint16; +typedef short pcre_int16; +#define PCRE_UINT16_MAX USHRT_MAX +#define PCRE_INT16_MAX SHRT_MAX +#elif UINT_MAX == 65535 +typedef unsigned int pcre_uint16; +typedef int pcre_int16; +#define PCRE_UINT16_MAX UINT_MAX +#define PCRE_INT16_MAX INT_MAX +#else +#error Cannot determine a type for 16-bit integers +#endif + +#if UINT_MAX == 4294967295U +typedef unsigned int pcre_uint32; +typedef int pcre_int32; +#define PCRE_UINT32_MAX UINT_MAX +#define PCRE_INT32_MAX INT_MAX +#elif ULONG_MAX == 4294967295UL +typedef unsigned long int pcre_uint32; +typedef long int pcre_int32; +#define PCRE_UINT32_MAX ULONG_MAX +#define PCRE_INT32_MAX LONG_MAX +#else +#error Cannot determine a type for 32-bit integers +#endif + +/* When checking for integer overflow in pcre_compile(), we need to handle +large integers. If a 64-bit integer type is available, we can use that. +Otherwise we have to cast to double, which of course requires floating point +arithmetic. Handle this by defining a macro for the appropriate type. If +stdint.h is available, include it; it may define INT64_MAX. Systems that do not +have stdint.h (e.g. Solaris) may have inttypes.h. The macro int64_t may be set +by "configure". */ + +#if defined HAVE_STDINT_H +#include +#elif defined HAVE_INTTYPES_H +#include +#endif + +#if defined INT64_MAX || defined int64_t +#define INT64_OR_DOUBLE int64_t +#else +#define INT64_OR_DOUBLE double +#endif + +/* All character handling must be done as unsigned characters. Otherwise there +are problems with top-bit-set characters and functions such as isspace(). +However, we leave the interface to the outside world as char * or short *, +because that should make things easier for callers. This character type is +called pcre_uchar. + +The IN_UCHARS macro multiply its argument with the byte size of the current +pcre_uchar type. Useful for memcpy and such operations, whose require the +byte size of their input/output buffers. + +The MAX_255 macro checks whether its pcre_uchar input is less than 256. + +The TABLE_GET macro is designed for accessing elements of tables whose contain +exactly 256 items. When the character is able to contain more than 256 +items, some check is needed before accessing these tables. +*/ + +#if defined COMPILE_PCRE8 + +typedef unsigned char pcre_uchar; +#define IN_UCHARS(x) (x) +#define MAX_255(c) 1 +#define TABLE_GET(c, table, default) ((table)[c]) + +#elif defined COMPILE_PCRE16 + +#if USHRT_MAX != 65535 +/* This is a warning message. Change PCRE_UCHAR16 to a 16 bit data type in +pcre.h(.in) and disable (comment out) this message. */ +#error Warning: PCRE_UCHAR16 is not a 16 bit data type. +#endif + +typedef pcre_uint16 pcre_uchar; +#define UCHAR_SHIFT (1) +#define IN_UCHARS(x) ((x) * 2) +#define MAX_255(c) ((c) <= 255u) +#define TABLE_GET(c, table, default) (MAX_255(c)? ((table)[c]):(default)) + +#elif defined COMPILE_PCRE32 + +typedef pcre_uint32 pcre_uchar; +#define UCHAR_SHIFT (2) +#define IN_UCHARS(x) ((x) * 4) +#define MAX_255(c) ((c) <= 255u) +#define TABLE_GET(c, table, default) (MAX_255(c)? ((table)[c]):(default)) + +#else +#error Unsupported compiling mode +#endif /* COMPILE_PCRE[8|16|32] */ + +/* This is an unsigned int value that no character can ever have. UTF-8 +characters only go up to 0x7fffffff (though Unicode doesn't go beyond +0x0010ffff). */ + +#define NOTACHAR 0xffffffff + +/* PCRE is able to support several different kinds of newline (CR, LF, CRLF, +"any" and "anycrlf" at present). The following macros are used to package up +testing for newlines. NLBLOCK, PSSTART, and PSEND are defined in the various +modules to indicate in which datablock the parameters exist, and what the +start/end of string field names are. */ + +#define NLTYPE_FIXED 0 /* Newline is a fixed length string */ +#define NLTYPE_ANY 1 /* Newline is any Unicode line ending */ +#define NLTYPE_ANYCRLF 2 /* Newline is CR, LF, or CRLF */ + +/* This macro checks for a newline at the given position */ + +#define IS_NEWLINE(p) \ + ((NLBLOCK->nltype != NLTYPE_FIXED)? \ + ((p) < NLBLOCK->PSEND && \ + PRIV(is_newline)((p), NLBLOCK->nltype, NLBLOCK->PSEND, \ + &(NLBLOCK->nllen), utf)) \ + : \ + ((p) <= NLBLOCK->PSEND - NLBLOCK->nllen && \ + UCHAR21TEST(p) == NLBLOCK->nl[0] && \ + (NLBLOCK->nllen == 1 || UCHAR21TEST(p+1) == NLBLOCK->nl[1]) \ + ) \ + ) + +/* This macro checks for a newline immediately preceding the given position */ + +#define WAS_NEWLINE(p) \ + ((NLBLOCK->nltype != NLTYPE_FIXED)? \ + ((p) > NLBLOCK->PSSTART && \ + PRIV(was_newline)((p), NLBLOCK->nltype, NLBLOCK->PSSTART, \ + &(NLBLOCK->nllen), utf)) \ + : \ + ((p) >= NLBLOCK->PSSTART + NLBLOCK->nllen && \ + UCHAR21TEST(p - NLBLOCK->nllen) == NLBLOCK->nl[0] && \ + (NLBLOCK->nllen == 1 || UCHAR21TEST(p - NLBLOCK->nllen + 1) == NLBLOCK->nl[1]) \ + ) \ + ) + +/* When PCRE is compiled as a C++ library, the subject pointer can be replaced +with a custom type. This makes it possible, for example, to allow pcre_exec() +to process subject strings that are discontinuous by using a smart pointer +class. It must always be possible to inspect all of the subject string in +pcre_exec() because of the way it backtracks. Two macros are required in the +normal case, for sign-unspecified and unsigned char pointers. The former is +used for the external interface and appears in pcre.h, which is why its name +must begin with PCRE_. */ + +#ifdef CUSTOM_SUBJECT_PTR +#define PCRE_PUCHAR CUSTOM_SUBJECT_PTR +#else +#define PCRE_PUCHAR const pcre_uchar * +#endif + +/* Include the public PCRE header and the definitions of UCP character property +values. */ + +#include "pcre.h" +#include "ucp.h" + +#ifdef COMPILE_PCRE32 +/* Assert that the public PCRE_UCHAR32 is a 32-bit type */ +typedef int __assert_pcre_uchar32_size[sizeof(PCRE_UCHAR32) == 4 ? 1 : -1]; +#endif + +/* When compiling for use with the Virtual Pascal compiler, these functions +need to have their names changed. PCRE must be compiled with the -DVPCOMPAT +option on the command line. */ + +#ifdef VPCOMPAT +#define strlen(s) _strlen(s) +#define strncmp(s1,s2,m) _strncmp(s1,s2,m) +#define memcmp(s,c,n) _memcmp(s,c,n) +#define memcpy(d,s,n) _memcpy(d,s,n) +#define memmove(d,s,n) _memmove(d,s,n) +#define memset(s,c,n) _memset(s,c,n) +#else /* VPCOMPAT */ + +/* To cope with SunOS4 and other systems that lack memmove() but have bcopy(), +define a macro for memmove() if HAVE_MEMMOVE is false, provided that HAVE_BCOPY +is set. Otherwise, include an emulating function for those systems that have +neither (there some non-Unix environments where this is the case). */ + +#ifndef HAVE_MEMMOVE +#undef memmove /* some systems may have a macro */ +#ifdef HAVE_BCOPY +#define memmove(a, b, c) bcopy(b, a, c) +#else /* HAVE_BCOPY */ +static void * +pcre_memmove(void *d, const void *s, size_t n) +{ +size_t i; +unsigned char *dest = (unsigned char *)d; +const unsigned char *src = (const unsigned char *)s; +if (dest > src) + { + dest += n; + src += n; + for (i = 0; i < n; ++i) *(--dest) = *(--src); + return (void *)dest; + } +else + { + for (i = 0; i < n; ++i) *dest++ = *src++; + return (void *)(dest - n); + } +} +#define memmove(a, b, c) pcre_memmove(a, b, c) +#endif /* not HAVE_BCOPY */ +#endif /* not HAVE_MEMMOVE */ +#endif /* not VPCOMPAT */ + + +/* PCRE keeps offsets in its compiled code as 2-byte quantities (always stored +in big-endian order) by default. These are used, for example, to link from the +start of a subpattern to its alternatives and its end. The use of 2 bytes per +offset limits the size of the compiled regex to around 64K, which is big enough +for almost everybody. However, I received a request for an even bigger limit. +For this reason, and also to make the code easier to maintain, the storing and +loading of offsets from the byte string is now handled by the macros that are +defined here. + +The macros are controlled by the value of LINK_SIZE. This defaults to 2 in +the config.h file, but can be overridden by using -D on the command line. This +is automated on Unix systems via the "configure" command. */ + +#if defined COMPILE_PCRE8 + +#if LINK_SIZE == 2 + +#define PUT(a,n,d) \ + (a[n] = (d) >> 8), \ + (a[(n)+1] = (d) & 255) + +#define GET(a,n) \ + (((a)[n] << 8) | (a)[(n)+1]) + +#define MAX_PATTERN_SIZE (1 << 16) + + +#elif LINK_SIZE == 3 + +#define PUT(a,n,d) \ + (a[n] = (d) >> 16), \ + (a[(n)+1] = (d) >> 8), \ + (a[(n)+2] = (d) & 255) + +#define GET(a,n) \ + (((a)[n] << 16) | ((a)[(n)+1] << 8) | (a)[(n)+2]) + +#define MAX_PATTERN_SIZE (1 << 24) + + +#elif LINK_SIZE == 4 + +#define PUT(a,n,d) \ + (a[n] = (d) >> 24), \ + (a[(n)+1] = (d) >> 16), \ + (a[(n)+2] = (d) >> 8), \ + (a[(n)+3] = (d) & 255) + +#define GET(a,n) \ + (((a)[n] << 24) | ((a)[(n)+1] << 16) | ((a)[(n)+2] << 8) | (a)[(n)+3]) + +/* Keep it positive */ +#define MAX_PATTERN_SIZE (1 << 30) + +#else +#error LINK_SIZE must be either 2, 3, or 4 +#endif + +#elif defined COMPILE_PCRE16 + +#if LINK_SIZE == 2 + +/* Redefine LINK_SIZE as a multiple of sizeof(pcre_uchar) */ +#undef LINK_SIZE +#define LINK_SIZE 1 + +#define PUT(a,n,d) \ + (a[n] = (d)) + +#define GET(a,n) \ + (a[n]) + +#define MAX_PATTERN_SIZE (1 << 16) + +#elif LINK_SIZE == 3 || LINK_SIZE == 4 + +/* Redefine LINK_SIZE as a multiple of sizeof(pcre_uchar) */ +#undef LINK_SIZE +#define LINK_SIZE 2 + +#define PUT(a,n,d) \ + (a[n] = (d) >> 16), \ + (a[(n)+1] = (d) & 65535) + +#define GET(a,n) \ + (((a)[n] << 16) | (a)[(n)+1]) + +/* Keep it positive */ +#define MAX_PATTERN_SIZE (1 << 30) + +#else +#error LINK_SIZE must be either 2, 3, or 4 +#endif + +#elif defined COMPILE_PCRE32 + +/* Only supported LINK_SIZE is 4 */ +/* Redefine LINK_SIZE as a multiple of sizeof(pcre_uchar) */ +#undef LINK_SIZE +#define LINK_SIZE 1 + +#define PUT(a,n,d) \ + (a[n] = (d)) + +#define GET(a,n) \ + (a[n]) + +/* Keep it positive */ +#define MAX_PATTERN_SIZE (1 << 30) + +#else +#error Unsupported compiling mode +#endif /* COMPILE_PCRE[8|16|32] */ + +/* Convenience macro defined in terms of the others */ + +#define PUTINC(a,n,d) PUT(a,n,d), a += LINK_SIZE + + +/* PCRE uses some other 2-byte quantities that do not change when the size of +offsets changes. There are used for repeat counts and for other things such as +capturing parenthesis numbers in back references. */ + +#if defined COMPILE_PCRE8 + +#define IMM2_SIZE 2 + +#define PUT2(a,n,d) \ + a[n] = (d) >> 8; \ + a[(n)+1] = (d) & 255 + +/* For reasons that I do not understand, the expression in this GET2 macro is +treated by gcc as a signed expression, even when a is declared as unsigned. It +seems that any kind of arithmetic results in a signed value. */ + +#define GET2(a,n) \ + (unsigned int)(((a)[n] << 8) | (a)[(n)+1]) + +#elif defined COMPILE_PCRE16 + +#define IMM2_SIZE 1 + +#define PUT2(a,n,d) \ + a[n] = d + +#define GET2(a,n) \ + a[n] + +#elif defined COMPILE_PCRE32 + +#define IMM2_SIZE 1 + +#define PUT2(a,n,d) \ + a[n] = d + +#define GET2(a,n) \ + a[n] + +#else +#error Unsupported compiling mode +#endif /* COMPILE_PCRE[8|16|32] */ + +#define PUT2INC(a,n,d) PUT2(a,n,d), a += IMM2_SIZE + +/* The maximum length of a MARK name is currently one data unit; it may be +changed in future to be a fixed number of bytes or to depend on LINK_SIZE. */ + +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 +#define MAX_MARK ((1u << 16) - 1) +#else +#define MAX_MARK ((1u << 8) - 1) +#endif + +/* There is a proposed future special "UTF-21" mode, in which only the lowest +21 bits of a 32-bit character are interpreted as UTF, with the remaining 11 +high-order bits available to the application for other uses. In preparation for +the future implementation of this mode, there are macros that load a data item +and, if in this special mode, mask it to 21 bits. These macros all have names +starting with UCHAR21. In all other modes, including the normal 32-bit +library, the macros all have the same simple definitions. When the new mode is +implemented, it is expected that these definitions will be varied appropriately +using #ifdef when compiling the library that supports the special mode. */ + +#define UCHAR21(eptr) (*(eptr)) +#define UCHAR21TEST(eptr) (*(eptr)) +#define UCHAR21INC(eptr) (*(eptr)++) +#define UCHAR21INCTEST(eptr) (*(eptr)++) + +/* When UTF encoding is being used, a character is no longer just a single +byte in 8-bit mode or a single short in 16-bit mode. The macros for character +handling generate simple sequences when used in the basic mode, and more +complicated ones for UTF characters. GETCHARLENTEST and other macros are not +used when UTF is not supported. To make sure they can never even appear when +UTF support is omitted, we don't even define them. */ + +#ifndef SUPPORT_UTF + +/* #define MAX_VALUE_FOR_SINGLE_CHAR */ +/* #define HAS_EXTRALEN(c) */ +/* #define GET_EXTRALEN(c) */ +/* #define NOT_FIRSTCHAR(c) */ +#define GETCHAR(c, eptr) c = *eptr; +#define GETCHARTEST(c, eptr) c = *eptr; +#define GETCHARINC(c, eptr) c = *eptr++; +#define GETCHARINCTEST(c, eptr) c = *eptr++; +#define GETCHARLEN(c, eptr, len) c = *eptr; +/* #define GETCHARLENTEST(c, eptr, len) */ +/* #define BACKCHAR(eptr) */ +/* #define FORWARDCHAR(eptr) */ +/* #define ACROSSCHAR(condition, eptr, action) */ + +#else /* SUPPORT_UTF */ + +/* Tests whether the code point needs extra characters to decode. */ + +#define HASUTF8EXTRALEN(c) ((c) >= 0xc0) + +/* Base macro to pick up the remaining bytes of a UTF-8 character, not +advancing the pointer. */ + +#define GETUTF8(c, eptr) \ + { \ + if ((c & 0x20) == 0) \ + c = ((c & 0x1f) << 6) | (eptr[1] & 0x3f); \ + else if ((c & 0x10) == 0) \ + c = ((c & 0x0f) << 12) | ((eptr[1] & 0x3f) << 6) | (eptr[2] & 0x3f); \ + else if ((c & 0x08) == 0) \ + c = ((c & 0x07) << 18) | ((eptr[1] & 0x3f) << 12) | \ + ((eptr[2] & 0x3f) << 6) | (eptr[3] & 0x3f); \ + else if ((c & 0x04) == 0) \ + c = ((c & 0x03) << 24) | ((eptr[1] & 0x3f) << 18) | \ + ((eptr[2] & 0x3f) << 12) | ((eptr[3] & 0x3f) << 6) | \ + (eptr[4] & 0x3f); \ + else \ + c = ((c & 0x01) << 30) | ((eptr[1] & 0x3f) << 24) | \ + ((eptr[2] & 0x3f) << 18) | ((eptr[3] & 0x3f) << 12) | \ + ((eptr[4] & 0x3f) << 6) | (eptr[5] & 0x3f); \ + } + +/* Base macro to pick up the remaining bytes of a UTF-8 character, advancing +the pointer. */ + +#define GETUTF8INC(c, eptr) \ + { \ + if ((c & 0x20) == 0) \ + c = ((c & 0x1f) << 6) | (*eptr++ & 0x3f); \ + else if ((c & 0x10) == 0) \ + { \ + c = ((c & 0x0f) << 12) | ((*eptr & 0x3f) << 6) | (eptr[1] & 0x3f); \ + eptr += 2; \ + } \ + else if ((c & 0x08) == 0) \ + { \ + c = ((c & 0x07) << 18) | ((*eptr & 0x3f) << 12) | \ + ((eptr[1] & 0x3f) << 6) | (eptr[2] & 0x3f); \ + eptr += 3; \ + } \ + else if ((c & 0x04) == 0) \ + { \ + c = ((c & 0x03) << 24) | ((*eptr & 0x3f) << 18) | \ + ((eptr[1] & 0x3f) << 12) | ((eptr[2] & 0x3f) << 6) | \ + (eptr[3] & 0x3f); \ + eptr += 4; \ + } \ + else \ + { \ + c = ((c & 0x01) << 30) | ((*eptr & 0x3f) << 24) | \ + ((eptr[1] & 0x3f) << 18) | ((eptr[2] & 0x3f) << 12) | \ + ((eptr[3] & 0x3f) << 6) | (eptr[4] & 0x3f); \ + eptr += 5; \ + } \ + } + +#if defined COMPILE_PCRE8 + +/* These macros were originally written in the form of loops that used data +from the tables whose names start with PRIV(utf8_table). They were rewritten by +a user so as not to use loops, because in some environments this gives a +significant performance advantage, and it seems never to do any harm. */ + +/* Tells the biggest code point which can be encoded as a single character. */ + +#define MAX_VALUE_FOR_SINGLE_CHAR 127 + +/* Tests whether the code point needs extra characters to decode. */ + +#define HAS_EXTRALEN(c) ((c) >= 0xc0) + +/* Returns with the additional number of characters if IS_MULTICHAR(c) is TRUE. +Otherwise it has an undefined behaviour. */ + +#define GET_EXTRALEN(c) (PRIV(utf8_table4)[(c) & 0x3f]) + +/* Returns TRUE, if the given character is not the first character +of a UTF sequence. */ + +#define NOT_FIRSTCHAR(c) (((c) & 0xc0) == 0x80) + +/* Get the next UTF-8 character, not advancing the pointer. This is called when +we know we are in UTF-8 mode. */ + +#define GETCHAR(c, eptr) \ + c = *eptr; \ + if (c >= 0xc0) GETUTF8(c, eptr); + +/* Get the next UTF-8 character, testing for UTF-8 mode, and not advancing the +pointer. */ + +#define GETCHARTEST(c, eptr) \ + c = *eptr; \ + if (utf && c >= 0xc0) GETUTF8(c, eptr); + +/* Get the next UTF-8 character, advancing the pointer. This is called when we +know we are in UTF-8 mode. */ + +#define GETCHARINC(c, eptr) \ + c = *eptr++; \ + if (c >= 0xc0) GETUTF8INC(c, eptr); + +/* Get the next character, testing for UTF-8 mode, and advancing the pointer. +This is called when we don't know if we are in UTF-8 mode. */ + +#define GETCHARINCTEST(c, eptr) \ + c = *eptr++; \ + if (utf && c >= 0xc0) GETUTF8INC(c, eptr); + +/* Base macro to pick up the remaining bytes of a UTF-8 character, not +advancing the pointer, incrementing the length. */ + +#define GETUTF8LEN(c, eptr, len) \ + { \ + if ((c & 0x20) == 0) \ + { \ + c = ((c & 0x1f) << 6) | (eptr[1] & 0x3f); \ + len++; \ + } \ + else if ((c & 0x10) == 0) \ + { \ + c = ((c & 0x0f) << 12) | ((eptr[1] & 0x3f) << 6) | (eptr[2] & 0x3f); \ + len += 2; \ + } \ + else if ((c & 0x08) == 0) \ + {\ + c = ((c & 0x07) << 18) | ((eptr[1] & 0x3f) << 12) | \ + ((eptr[2] & 0x3f) << 6) | (eptr[3] & 0x3f); \ + len += 3; \ + } \ + else if ((c & 0x04) == 0) \ + { \ + c = ((c & 0x03) << 24) | ((eptr[1] & 0x3f) << 18) | \ + ((eptr[2] & 0x3f) << 12) | ((eptr[3] & 0x3f) << 6) | \ + (eptr[4] & 0x3f); \ + len += 4; \ + } \ + else \ + {\ + c = ((c & 0x01) << 30) | ((eptr[1] & 0x3f) << 24) | \ + ((eptr[2] & 0x3f) << 18) | ((eptr[3] & 0x3f) << 12) | \ + ((eptr[4] & 0x3f) << 6) | (eptr[5] & 0x3f); \ + len += 5; \ + } \ + } + +/* Get the next UTF-8 character, not advancing the pointer, incrementing length +if there are extra bytes. This is called when we know we are in UTF-8 mode. */ + +#define GETCHARLEN(c, eptr, len) \ + c = *eptr; \ + if (c >= 0xc0) GETUTF8LEN(c, eptr, len); + +/* Get the next UTF-8 character, testing for UTF-8 mode, not advancing the +pointer, incrementing length if there are extra bytes. This is called when we +do not know if we are in UTF-8 mode. */ + +#define GETCHARLENTEST(c, eptr, len) \ + c = *eptr; \ + if (utf && c >= 0xc0) GETUTF8LEN(c, eptr, len); + +/* If the pointer is not at the start of a character, move it back until +it is. This is called only in UTF-8 mode - we don't put a test within the macro +because almost all calls are already within a block of UTF-8 only code. */ + +#define BACKCHAR(eptr) while((*eptr & 0xc0) == 0x80) eptr-- + +/* Same as above, just in the other direction. */ +#define FORWARDCHAR(eptr) while((*eptr & 0xc0) == 0x80) eptr++ + +/* Same as above, but it allows a fully customizable form. */ +#define ACROSSCHAR(condition, eptr, action) \ + while((condition) && ((eptr) & 0xc0) == 0x80) action + +#elif defined COMPILE_PCRE16 + +/* Tells the biggest code point which can be encoded as a single character. */ + +#define MAX_VALUE_FOR_SINGLE_CHAR 65535 + +/* Tests whether the code point needs extra characters to decode. */ + +#define HAS_EXTRALEN(c) (((c) & 0xfc00) == 0xd800) + +/* Returns with the additional number of characters if IS_MULTICHAR(c) is TRUE. +Otherwise it has an undefined behaviour. */ + +#define GET_EXTRALEN(c) 1 + +/* Returns TRUE, if the given character is not the first character +of a UTF sequence. */ + +#define NOT_FIRSTCHAR(c) (((c) & 0xfc00) == 0xdc00) + +/* Base macro to pick up the low surrogate of a UTF-16 character, not +advancing the pointer. */ + +#define GETUTF16(c, eptr) \ + { c = (((c & 0x3ff) << 10) | (eptr[1] & 0x3ff)) + 0x10000; } + +/* Get the next UTF-16 character, not advancing the pointer. This is called when +we know we are in UTF-16 mode. */ + +#define GETCHAR(c, eptr) \ + c = *eptr; \ + if ((c & 0xfc00) == 0xd800) GETUTF16(c, eptr); + +/* Get the next UTF-16 character, testing for UTF-16 mode, and not advancing the +pointer. */ + +#define GETCHARTEST(c, eptr) \ + c = *eptr; \ + if (utf && (c & 0xfc00) == 0xd800) GETUTF16(c, eptr); + +/* Base macro to pick up the low surrogate of a UTF-16 character, advancing +the pointer. */ + +#define GETUTF16INC(c, eptr) \ + { c = (((c & 0x3ff) << 10) | (*eptr++ & 0x3ff)) + 0x10000; } + +/* Get the next UTF-16 character, advancing the pointer. This is called when we +know we are in UTF-16 mode. */ + +#define GETCHARINC(c, eptr) \ + c = *eptr++; \ + if ((c & 0xfc00) == 0xd800) GETUTF16INC(c, eptr); + +/* Get the next character, testing for UTF-16 mode, and advancing the pointer. +This is called when we don't know if we are in UTF-16 mode. */ + +#define GETCHARINCTEST(c, eptr) \ + c = *eptr++; \ + if (utf && (c & 0xfc00) == 0xd800) GETUTF16INC(c, eptr); + +/* Base macro to pick up the low surrogate of a UTF-16 character, not +advancing the pointer, incrementing the length. */ + +#define GETUTF16LEN(c, eptr, len) \ + { c = (((c & 0x3ff) << 10) | (eptr[1] & 0x3ff)) + 0x10000; len++; } + +/* Get the next UTF-16 character, not advancing the pointer, incrementing +length if there is a low surrogate. This is called when we know we are in +UTF-16 mode. */ + +#define GETCHARLEN(c, eptr, len) \ + c = *eptr; \ + if ((c & 0xfc00) == 0xd800) GETUTF16LEN(c, eptr, len); + +/* Get the next UTF-816character, testing for UTF-16 mode, not advancing the +pointer, incrementing length if there is a low surrogate. This is called when +we do not know if we are in UTF-16 mode. */ + +#define GETCHARLENTEST(c, eptr, len) \ + c = *eptr; \ + if (utf && (c & 0xfc00) == 0xd800) GETUTF16LEN(c, eptr, len); + +/* If the pointer is not at the start of a character, move it back until +it is. This is called only in UTF-16 mode - we don't put a test within the +macro because almost all calls are already within a block of UTF-16 only +code. */ + +#define BACKCHAR(eptr) if ((*eptr & 0xfc00) == 0xdc00) eptr-- + +/* Same as above, just in the other direction. */ +#define FORWARDCHAR(eptr) if ((*eptr & 0xfc00) == 0xdc00) eptr++ + +/* Same as above, but it allows a fully customizable form. */ +#define ACROSSCHAR(condition, eptr, action) \ + if ((condition) && ((eptr) & 0xfc00) == 0xdc00) action + +#elif defined COMPILE_PCRE32 + +/* These are trivial for the 32-bit library, since all UTF-32 characters fit +into one pcre_uchar unit. */ +#define MAX_VALUE_FOR_SINGLE_CHAR (0x10ffffu) +#define HAS_EXTRALEN(c) (0) +#define GET_EXTRALEN(c) (0) +#define NOT_FIRSTCHAR(c) (0) + +/* Get the next UTF-32 character, not advancing the pointer. This is called when +we know we are in UTF-32 mode. */ + +#define GETCHAR(c, eptr) \ + c = *(eptr); + +/* Get the next UTF-32 character, testing for UTF-32 mode, and not advancing the +pointer. */ + +#define GETCHARTEST(c, eptr) \ + c = *(eptr); + +/* Get the next UTF-32 character, advancing the pointer. This is called when we +know we are in UTF-32 mode. */ + +#define GETCHARINC(c, eptr) \ + c = *((eptr)++); + +/* Get the next character, testing for UTF-32 mode, and advancing the pointer. +This is called when we don't know if we are in UTF-32 mode. */ + +#define GETCHARINCTEST(c, eptr) \ + c = *((eptr)++); + +/* Get the next UTF-32 character, not advancing the pointer, not incrementing +length (since all UTF-32 is of length 1). This is called when we know we are in +UTF-32 mode. */ + +#define GETCHARLEN(c, eptr, len) \ + GETCHAR(c, eptr) + +/* Get the next UTF-32character, testing for UTF-32 mode, not advancing the +pointer, not incrementing the length (since all UTF-32 is of length 1). +This is called when we do not know if we are in UTF-32 mode. */ + +#define GETCHARLENTEST(c, eptr, len) \ + GETCHARTEST(c, eptr) + +/* If the pointer is not at the start of a character, move it back until +it is. This is called only in UTF-32 mode - we don't put a test within the +macro because almost all calls are already within a block of UTF-32 only +code. +These are all no-ops since all UTF-32 characters fit into one pcre_uchar. */ + +#define BACKCHAR(eptr) do { } while (0) + +/* Same as above, just in the other direction. */ +#define FORWARDCHAR(eptr) do { } while (0) + +/* Same as above, but it allows a fully customizable form. */ +#define ACROSSCHAR(condition, eptr, action) do { } while (0) + +#else +#error Unsupported compiling mode +#endif /* COMPILE_PCRE[8|16|32] */ + +#endif /* SUPPORT_UTF */ + +/* Tests for Unicode horizontal and vertical whitespace characters must check a +number of different values. Using a switch statement for this generates the +fastest code (no loop, no memory access), and there are several places in the +interpreter code where this happens. In order to ensure that all the case lists +remain in step, we use macros so that there is only one place where the lists +are defined. + +These values are also required as lists in pcre_compile.c when processing \h, +\H, \v and \V in a character class. The lists are defined in pcre_tables.c, but +macros that define the values are here so that all the definitions are +together. The lists must be in ascending character order, terminated by +NOTACHAR (which is 0xffffffff). + +Any changes should ensure that the various macros are kept in step with each +other. NOTE: The values also appear in pcre_jit_compile.c. */ + +/* ------ ASCII/Unicode environments ------ */ + +#ifndef EBCDIC + +#define HSPACE_LIST \ + CHAR_HT, CHAR_SPACE, CHAR_NBSP, \ + 0x1680, 0x180e, 0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, \ + 0x2006, 0x2007, 0x2008, 0x2009, 0x200A, 0x202f, 0x205f, 0x3000, \ + NOTACHAR + +#define HSPACE_MULTIBYTE_CASES \ + case 0x1680: /* OGHAM SPACE MARK */ \ + case 0x180e: /* MONGOLIAN VOWEL SEPARATOR */ \ + case 0x2000: /* EN QUAD */ \ + case 0x2001: /* EM QUAD */ \ + case 0x2002: /* EN SPACE */ \ + case 0x2003: /* EM SPACE */ \ + case 0x2004: /* THREE-PER-EM SPACE */ \ + case 0x2005: /* FOUR-PER-EM SPACE */ \ + case 0x2006: /* SIX-PER-EM SPACE */ \ + case 0x2007: /* FIGURE SPACE */ \ + case 0x2008: /* PUNCTUATION SPACE */ \ + case 0x2009: /* THIN SPACE */ \ + case 0x200A: /* HAIR SPACE */ \ + case 0x202f: /* NARROW NO-BREAK SPACE */ \ + case 0x205f: /* MEDIUM MATHEMATICAL SPACE */ \ + case 0x3000 /* IDEOGRAPHIC SPACE */ + +#define HSPACE_BYTE_CASES \ + case CHAR_HT: \ + case CHAR_SPACE: \ + case CHAR_NBSP + +#define HSPACE_CASES \ + HSPACE_BYTE_CASES: \ + HSPACE_MULTIBYTE_CASES + +#define VSPACE_LIST \ + CHAR_LF, CHAR_VT, CHAR_FF, CHAR_CR, CHAR_NEL, 0x2028, 0x2029, NOTACHAR + +#define VSPACE_MULTIBYTE_CASES \ + case 0x2028: /* LINE SEPARATOR */ \ + case 0x2029 /* PARAGRAPH SEPARATOR */ + +#define VSPACE_BYTE_CASES \ + case CHAR_LF: \ + case CHAR_VT: \ + case CHAR_FF: \ + case CHAR_CR: \ + case CHAR_NEL + +#define VSPACE_CASES \ + VSPACE_BYTE_CASES: \ + VSPACE_MULTIBYTE_CASES + +/* ------ EBCDIC environments ------ */ + +#else +#define HSPACE_LIST CHAR_HT, CHAR_SPACE, CHAR_NBSP, NOTACHAR + +#define HSPACE_BYTE_CASES \ + case CHAR_HT: \ + case CHAR_SPACE: \ + case CHAR_NBSP + +#define HSPACE_CASES HSPACE_BYTE_CASES + +#ifdef EBCDIC_NL25 +#define VSPACE_LIST \ + CHAR_VT, CHAR_FF, CHAR_CR, CHAR_NEL, CHAR_LF, NOTACHAR +#else +#define VSPACE_LIST \ + CHAR_VT, CHAR_FF, CHAR_CR, CHAR_LF, CHAR_NEL, NOTACHAR +#endif + +#define VSPACE_BYTE_CASES \ + case CHAR_LF: \ + case CHAR_VT: \ + case CHAR_FF: \ + case CHAR_CR: \ + case CHAR_NEL + +#define VSPACE_CASES VSPACE_BYTE_CASES +#endif /* EBCDIC */ + +/* ------ End of whitespace macros ------ */ + + + +/* Private flags containing information about the compiled regex. They used to +live at the top end of the options word, but that got almost full, so they were +moved to a 16-bit flags word - which got almost full, so now they are in a +32-bit flags word. From release 8.00, PCRE_NOPARTIAL is unused, as the +restrictions on partial matching have been lifted. It remains for backwards +compatibility. */ + +#define PCRE_MODE8 0x00000001 /* compiled in 8 bit mode */ +#define PCRE_MODE16 0x00000002 /* compiled in 16 bit mode */ +#define PCRE_MODE32 0x00000004 /* compiled in 32 bit mode */ +#define PCRE_FIRSTSET 0x00000010 /* first_char is set */ +#define PCRE_FCH_CASELESS 0x00000020 /* caseless first char */ +#define PCRE_REQCHSET 0x00000040 /* req_byte is set */ +#define PCRE_RCH_CASELESS 0x00000080 /* caseless requested char */ +#define PCRE_STARTLINE 0x00000100 /* start after \n for multiline */ +#define PCRE_NOPARTIAL 0x00000200 /* can't use partial with this regex */ +#define PCRE_JCHANGED 0x00000400 /* j option used in regex */ +#define PCRE_HASCRORLF 0x00000800 /* explicit \r or \n in pattern */ +#define PCRE_HASTHEN 0x00001000 /* pattern contains (*THEN) */ +#define PCRE_MLSET 0x00002000 /* match limit set by regex */ +#define PCRE_RLSET 0x00004000 /* recursion limit set by regex */ +#define PCRE_MATCH_EMPTY 0x00008000 /* pattern can match empty string */ + +#if defined COMPILE_PCRE8 +#define PCRE_MODE PCRE_MODE8 +#elif defined COMPILE_PCRE16 +#define PCRE_MODE PCRE_MODE16 +#elif defined COMPILE_PCRE32 +#define PCRE_MODE PCRE_MODE32 +#endif +#define PCRE_MODE_MASK (PCRE_MODE8 | PCRE_MODE16 | PCRE_MODE32) + +/* Flags for the "extra" block produced by pcre_study(). */ + +#define PCRE_STUDY_MAPPED 0x0001 /* a map of starting chars exists */ +#define PCRE_STUDY_MINLEN 0x0002 /* a minimum length field exists */ + +/* Masks for identifying the public options that are permitted at compile +time, run time, or study time, respectively. */ + +#define PCRE_NEWLINE_BITS (PCRE_NEWLINE_CR|PCRE_NEWLINE_LF|PCRE_NEWLINE_ANY| \ + PCRE_NEWLINE_ANYCRLF) + +#define PUBLIC_COMPILE_OPTIONS \ + (PCRE_CASELESS|PCRE_EXTENDED|PCRE_ANCHORED|PCRE_MULTILINE| \ + PCRE_DOTALL|PCRE_DOLLAR_ENDONLY|PCRE_EXTRA|PCRE_UNGREEDY|PCRE_UTF8| \ + PCRE_NO_AUTO_CAPTURE|PCRE_NO_AUTO_POSSESS| \ + PCRE_NO_UTF8_CHECK|PCRE_AUTO_CALLOUT|PCRE_FIRSTLINE| \ + PCRE_DUPNAMES|PCRE_NEWLINE_BITS|PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE| \ + PCRE_JAVASCRIPT_COMPAT|PCRE_UCP|PCRE_NO_START_OPTIMIZE|PCRE_NEVER_UTF) + +#define PUBLIC_EXEC_OPTIONS \ + (PCRE_ANCHORED|PCRE_NOTBOL|PCRE_NOTEOL|PCRE_NOTEMPTY|PCRE_NOTEMPTY_ATSTART| \ + PCRE_NO_UTF8_CHECK|PCRE_PARTIAL_HARD|PCRE_PARTIAL_SOFT|PCRE_NEWLINE_BITS| \ + PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE|PCRE_NO_START_OPTIMIZE) + +#define PUBLIC_DFA_EXEC_OPTIONS \ + (PCRE_ANCHORED|PCRE_NOTBOL|PCRE_NOTEOL|PCRE_NOTEMPTY|PCRE_NOTEMPTY_ATSTART| \ + PCRE_NO_UTF8_CHECK|PCRE_PARTIAL_HARD|PCRE_PARTIAL_SOFT|PCRE_DFA_SHORTEST| \ + PCRE_DFA_RESTART|PCRE_NEWLINE_BITS|PCRE_BSR_ANYCRLF|PCRE_BSR_UNICODE| \ + PCRE_NO_START_OPTIMIZE) + +#define PUBLIC_STUDY_OPTIONS \ + (PCRE_STUDY_JIT_COMPILE|PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE| \ + PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE|PCRE_STUDY_EXTRA_NEEDED) + +#define PUBLIC_JIT_EXEC_OPTIONS \ + (PCRE_NO_UTF8_CHECK|PCRE_NOTBOL|PCRE_NOTEOL|PCRE_NOTEMPTY|\ + PCRE_NOTEMPTY_ATSTART|PCRE_PARTIAL_SOFT|PCRE_PARTIAL_HARD) + +/* Magic number to provide a small check against being handed junk. */ + +#define MAGIC_NUMBER 0x50435245UL /* 'PCRE' */ + +/* This variable is used to detect a loaded regular expression +in different endianness. */ + +#define REVERSED_MAGIC_NUMBER 0x45524350UL /* 'ERCP' */ + +/* The maximum remaining length of subject we are prepared to search for a +req_byte match. */ + +#define REQ_BYTE_MAX 1000 + +/* Miscellaneous definitions. The #ifndef is to pacify compiler warnings in +environments where these macros are defined elsewhere. Unfortunately, there +is no way to do the same for the typedef. */ + +typedef int BOOL; + +#ifndef FALSE +#define FALSE 0 +#define TRUE 1 +#endif + +/* If PCRE is to support UTF-8 on EBCDIC platforms, we cannot use normal +character constants like '*' because the compiler would emit their EBCDIC code, +which is different from their ASCII/UTF-8 code. Instead we define macros for +the characters so that they always use the ASCII/UTF-8 code when UTF-8 support +is enabled. When UTF-8 support is not enabled, the definitions use character +literals. Both character and string versions of each character are needed, and +there are some longer strings as well. + +This means that, on EBCDIC platforms, the PCRE library can handle either +EBCDIC, or UTF-8, but not both. To support both in the same compiled library +would need different lookups depending on whether PCRE_UTF8 was set or not. +This would make it impossible to use characters in switch/case statements, +which would reduce performance. For a theoretical use (which nobody has asked +for) in a minority area (EBCDIC platforms), this is not sensible. Any +application that did need both could compile two versions of the library, using +macros to give the functions distinct names. */ + +#ifndef SUPPORT_UTF + +/* UTF-8 support is not enabled; use the platform-dependent character literals +so that PCRE works in both ASCII and EBCDIC environments, but only in non-UTF +mode. Newline characters are problematic in EBCDIC. Though it has CR and LF +characters, a common practice has been to use its NL (0x15) character as the +line terminator in C-like processing environments. However, sometimes the LF +(0x25) character is used instead, according to this Unicode document: + +http://unicode.org/standard/reports/tr13/tr13-5.html + +PCRE defaults EBCDIC NL to 0x15, but has a build-time option to select 0x25 +instead. Whichever is *not* chosen is defined as NEL. + +In both ASCII and EBCDIC environments, CHAR_NL and CHAR_LF are synonyms for the +same code point. */ + +#ifdef EBCDIC + +#ifndef EBCDIC_NL25 +#define CHAR_NL '\x15' +#define CHAR_NEL '\x25' +#define STR_NL "\x15" +#define STR_NEL "\x25" +#else +#define CHAR_NL '\x25' +#define CHAR_NEL '\x15' +#define STR_NL "\x25" +#define STR_NEL "\x15" +#endif + +#define CHAR_LF CHAR_NL +#define STR_LF STR_NL + +#define CHAR_ESC '\047' +#define CHAR_DEL '\007' +#define CHAR_NBSP '\x41' +#define STR_ESC "\047" +#define STR_DEL "\007" + +#else /* Not EBCDIC */ + +/* In ASCII/Unicode, linefeed is '\n' and we equate this to NL for +compatibility. NEL is the Unicode newline character; make sure it is +a positive value. */ + +#define CHAR_LF '\n' +#define CHAR_NL CHAR_LF +#define CHAR_NEL ((unsigned char)'\x85') +#define CHAR_ESC '\033' +#define CHAR_DEL '\177' +#define CHAR_NBSP ((unsigned char)'\xa0') + +#define STR_LF "\n" +#define STR_NL STR_LF +#define STR_NEL "\x85" +#define STR_ESC "\033" +#define STR_DEL "\177" + +#endif /* EBCDIC */ + +/* The remaining definitions work in both environments. */ + +#define CHAR_NULL '\0' +#define CHAR_HT '\t' +#define CHAR_VT '\v' +#define CHAR_FF '\f' +#define CHAR_CR '\r' +#define CHAR_BS '\b' +#define CHAR_BEL '\a' + +#define CHAR_SPACE ' ' +#define CHAR_EXCLAMATION_MARK '!' +#define CHAR_QUOTATION_MARK '"' +#define CHAR_NUMBER_SIGN '#' +#define CHAR_DOLLAR_SIGN '$' +#define CHAR_PERCENT_SIGN '%' +#define CHAR_AMPERSAND '&' +#define CHAR_APOSTROPHE '\'' +#define CHAR_LEFT_PARENTHESIS '(' +#define CHAR_RIGHT_PARENTHESIS ')' +#define CHAR_ASTERISK '*' +#define CHAR_PLUS '+' +#define CHAR_COMMA ',' +#define CHAR_MINUS '-' +#define CHAR_DOT '.' +#define CHAR_SLASH '/' +#define CHAR_0 '0' +#define CHAR_1 '1' +#define CHAR_2 '2' +#define CHAR_3 '3' +#define CHAR_4 '4' +#define CHAR_5 '5' +#define CHAR_6 '6' +#define CHAR_7 '7' +#define CHAR_8 '8' +#define CHAR_9 '9' +#define CHAR_COLON ':' +#define CHAR_SEMICOLON ';' +#define CHAR_LESS_THAN_SIGN '<' +#define CHAR_EQUALS_SIGN '=' +#define CHAR_GREATER_THAN_SIGN '>' +#define CHAR_QUESTION_MARK '?' +#define CHAR_COMMERCIAL_AT '@' +#define CHAR_A 'A' +#define CHAR_B 'B' +#define CHAR_C 'C' +#define CHAR_D 'D' +#define CHAR_E 'E' +#define CHAR_F 'F' +#define CHAR_G 'G' +#define CHAR_H 'H' +#define CHAR_I 'I' +#define CHAR_J 'J' +#define CHAR_K 'K' +#define CHAR_L 'L' +#define CHAR_M 'M' +#define CHAR_N 'N' +#define CHAR_O 'O' +#define CHAR_P 'P' +#define CHAR_Q 'Q' +#define CHAR_R 'R' +#define CHAR_S 'S' +#define CHAR_T 'T' +#define CHAR_U 'U' +#define CHAR_V 'V' +#define CHAR_W 'W' +#define CHAR_X 'X' +#define CHAR_Y 'Y' +#define CHAR_Z 'Z' +#define CHAR_LEFT_SQUARE_BRACKET '[' +#define CHAR_BACKSLASH '\\' +#define CHAR_RIGHT_SQUARE_BRACKET ']' +#define CHAR_CIRCUMFLEX_ACCENT '^' +#define CHAR_UNDERSCORE '_' +#define CHAR_GRAVE_ACCENT '`' +#define CHAR_a 'a' +#define CHAR_b 'b' +#define CHAR_c 'c' +#define CHAR_d 'd' +#define CHAR_e 'e' +#define CHAR_f 'f' +#define CHAR_g 'g' +#define CHAR_h 'h' +#define CHAR_i 'i' +#define CHAR_j 'j' +#define CHAR_k 'k' +#define CHAR_l 'l' +#define CHAR_m 'm' +#define CHAR_n 'n' +#define CHAR_o 'o' +#define CHAR_p 'p' +#define CHAR_q 'q' +#define CHAR_r 'r' +#define CHAR_s 's' +#define CHAR_t 't' +#define CHAR_u 'u' +#define CHAR_v 'v' +#define CHAR_w 'w' +#define CHAR_x 'x' +#define CHAR_y 'y' +#define CHAR_z 'z' +#define CHAR_LEFT_CURLY_BRACKET '{' +#define CHAR_VERTICAL_LINE '|' +#define CHAR_RIGHT_CURLY_BRACKET '}' +#define CHAR_TILDE '~' + +#define STR_HT "\t" +#define STR_VT "\v" +#define STR_FF "\f" +#define STR_CR "\r" +#define STR_BS "\b" +#define STR_BEL "\a" + +#define STR_SPACE " " +#define STR_EXCLAMATION_MARK "!" +#define STR_QUOTATION_MARK "\"" +#define STR_NUMBER_SIGN "#" +#define STR_DOLLAR_SIGN "$" +#define STR_PERCENT_SIGN "%" +#define STR_AMPERSAND "&" +#define STR_APOSTROPHE "'" +#define STR_LEFT_PARENTHESIS "(" +#define STR_RIGHT_PARENTHESIS ")" +#define STR_ASTERISK "*" +#define STR_PLUS "+" +#define STR_COMMA "," +#define STR_MINUS "-" +#define STR_DOT "." +#define STR_SLASH "/" +#define STR_0 "0" +#define STR_1 "1" +#define STR_2 "2" +#define STR_3 "3" +#define STR_4 "4" +#define STR_5 "5" +#define STR_6 "6" +#define STR_7 "7" +#define STR_8 "8" +#define STR_9 "9" +#define STR_COLON ":" +#define STR_SEMICOLON ";" +#define STR_LESS_THAN_SIGN "<" +#define STR_EQUALS_SIGN "=" +#define STR_GREATER_THAN_SIGN ">" +#define STR_QUESTION_MARK "?" +#define STR_COMMERCIAL_AT "@" +#define STR_A "A" +#define STR_B "B" +#define STR_C "C" +#define STR_D "D" +#define STR_E "E" +#define STR_F "F" +#define STR_G "G" +#define STR_H "H" +#define STR_I "I" +#define STR_J "J" +#define STR_K "K" +#define STR_L "L" +#define STR_M "M" +#define STR_N "N" +#define STR_O "O" +#define STR_P "P" +#define STR_Q "Q" +#define STR_R "R" +#define STR_S "S" +#define STR_T "T" +#define STR_U "U" +#define STR_V "V" +#define STR_W "W" +#define STR_X "X" +#define STR_Y "Y" +#define STR_Z "Z" +#define STR_LEFT_SQUARE_BRACKET "[" +#define STR_BACKSLASH "\\" +#define STR_RIGHT_SQUARE_BRACKET "]" +#define STR_CIRCUMFLEX_ACCENT "^" +#define STR_UNDERSCORE "_" +#define STR_GRAVE_ACCENT "`" +#define STR_a "a" +#define STR_b "b" +#define STR_c "c" +#define STR_d "d" +#define STR_e "e" +#define STR_f "f" +#define STR_g "g" +#define STR_h "h" +#define STR_i "i" +#define STR_j "j" +#define STR_k "k" +#define STR_l "l" +#define STR_m "m" +#define STR_n "n" +#define STR_o "o" +#define STR_p "p" +#define STR_q "q" +#define STR_r "r" +#define STR_s "s" +#define STR_t "t" +#define STR_u "u" +#define STR_v "v" +#define STR_w "w" +#define STR_x "x" +#define STR_y "y" +#define STR_z "z" +#define STR_LEFT_CURLY_BRACKET "{" +#define STR_VERTICAL_LINE "|" +#define STR_RIGHT_CURLY_BRACKET "}" +#define STR_TILDE "~" + +#define STRING_ACCEPT0 "ACCEPT\0" +#define STRING_COMMIT0 "COMMIT\0" +#define STRING_F0 "F\0" +#define STRING_FAIL0 "FAIL\0" +#define STRING_MARK0 "MARK\0" +#define STRING_PRUNE0 "PRUNE\0" +#define STRING_SKIP0 "SKIP\0" +#define STRING_THEN "THEN" + +#define STRING_alpha0 "alpha\0" +#define STRING_lower0 "lower\0" +#define STRING_upper0 "upper\0" +#define STRING_alnum0 "alnum\0" +#define STRING_ascii0 "ascii\0" +#define STRING_blank0 "blank\0" +#define STRING_cntrl0 "cntrl\0" +#define STRING_digit0 "digit\0" +#define STRING_graph0 "graph\0" +#define STRING_print0 "print\0" +#define STRING_punct0 "punct\0" +#define STRING_space0 "space\0" +#define STRING_word0 "word\0" +#define STRING_xdigit "xdigit" + +#define STRING_DEFINE "DEFINE" +#define STRING_WEIRD_STARTWORD "[:<:]]" +#define STRING_WEIRD_ENDWORD "[:>:]]" + +#define STRING_CR_RIGHTPAR "CR)" +#define STRING_LF_RIGHTPAR "LF)" +#define STRING_CRLF_RIGHTPAR "CRLF)" +#define STRING_ANY_RIGHTPAR "ANY)" +#define STRING_ANYCRLF_RIGHTPAR "ANYCRLF)" +#define STRING_BSR_ANYCRLF_RIGHTPAR "BSR_ANYCRLF)" +#define STRING_BSR_UNICODE_RIGHTPAR "BSR_UNICODE)" +#define STRING_UTF8_RIGHTPAR "UTF8)" +#define STRING_UTF16_RIGHTPAR "UTF16)" +#define STRING_UTF32_RIGHTPAR "UTF32)" +#define STRING_UTF_RIGHTPAR "UTF)" +#define STRING_UCP_RIGHTPAR "UCP)" +#define STRING_NO_AUTO_POSSESS_RIGHTPAR "NO_AUTO_POSSESS)" +#define STRING_NO_START_OPT_RIGHTPAR "NO_START_OPT)" +#define STRING_LIMIT_MATCH_EQ "LIMIT_MATCH=" +#define STRING_LIMIT_RECURSION_EQ "LIMIT_RECURSION=" + +#else /* SUPPORT_UTF */ + +/* UTF-8 support is enabled; always use UTF-8 (=ASCII) character codes. This +works in both modes non-EBCDIC platforms, and on EBCDIC platforms in UTF-8 mode +only. */ + +#define CHAR_HT '\011' +#define CHAR_VT '\013' +#define CHAR_FF '\014' +#define CHAR_CR '\015' +#define CHAR_LF '\012' +#define CHAR_NL CHAR_LF +#define CHAR_NEL ((unsigned char)'\x85') +#define CHAR_BS '\010' +#define CHAR_BEL '\007' +#define CHAR_ESC '\033' +#define CHAR_DEL '\177' + +#define CHAR_NULL '\0' +#define CHAR_SPACE '\040' +#define CHAR_EXCLAMATION_MARK '\041' +#define CHAR_QUOTATION_MARK '\042' +#define CHAR_NUMBER_SIGN '\043' +#define CHAR_DOLLAR_SIGN '\044' +#define CHAR_PERCENT_SIGN '\045' +#define CHAR_AMPERSAND '\046' +#define CHAR_APOSTROPHE '\047' +#define CHAR_LEFT_PARENTHESIS '\050' +#define CHAR_RIGHT_PARENTHESIS '\051' +#define CHAR_ASTERISK '\052' +#define CHAR_PLUS '\053' +#define CHAR_COMMA '\054' +#define CHAR_MINUS '\055' +#define CHAR_DOT '\056' +#define CHAR_SLASH '\057' +#define CHAR_0 '\060' +#define CHAR_1 '\061' +#define CHAR_2 '\062' +#define CHAR_3 '\063' +#define CHAR_4 '\064' +#define CHAR_5 '\065' +#define CHAR_6 '\066' +#define CHAR_7 '\067' +#define CHAR_8 '\070' +#define CHAR_9 '\071' +#define CHAR_COLON '\072' +#define CHAR_SEMICOLON '\073' +#define CHAR_LESS_THAN_SIGN '\074' +#define CHAR_EQUALS_SIGN '\075' +#define CHAR_GREATER_THAN_SIGN '\076' +#define CHAR_QUESTION_MARK '\077' +#define CHAR_COMMERCIAL_AT '\100' +#define CHAR_A '\101' +#define CHAR_B '\102' +#define CHAR_C '\103' +#define CHAR_D '\104' +#define CHAR_E '\105' +#define CHAR_F '\106' +#define CHAR_G '\107' +#define CHAR_H '\110' +#define CHAR_I '\111' +#define CHAR_J '\112' +#define CHAR_K '\113' +#define CHAR_L '\114' +#define CHAR_M '\115' +#define CHAR_N '\116' +#define CHAR_O '\117' +#define CHAR_P '\120' +#define CHAR_Q '\121' +#define CHAR_R '\122' +#define CHAR_S '\123' +#define CHAR_T '\124' +#define CHAR_U '\125' +#define CHAR_V '\126' +#define CHAR_W '\127' +#define CHAR_X '\130' +#define CHAR_Y '\131' +#define CHAR_Z '\132' +#define CHAR_LEFT_SQUARE_BRACKET '\133' +#define CHAR_BACKSLASH '\134' +#define CHAR_RIGHT_SQUARE_BRACKET '\135' +#define CHAR_CIRCUMFLEX_ACCENT '\136' +#define CHAR_UNDERSCORE '\137' +#define CHAR_GRAVE_ACCENT '\140' +#define CHAR_a '\141' +#define CHAR_b '\142' +#define CHAR_c '\143' +#define CHAR_d '\144' +#define CHAR_e '\145' +#define CHAR_f '\146' +#define CHAR_g '\147' +#define CHAR_h '\150' +#define CHAR_i '\151' +#define CHAR_j '\152' +#define CHAR_k '\153' +#define CHAR_l '\154' +#define CHAR_m '\155' +#define CHAR_n '\156' +#define CHAR_o '\157' +#define CHAR_p '\160' +#define CHAR_q '\161' +#define CHAR_r '\162' +#define CHAR_s '\163' +#define CHAR_t '\164' +#define CHAR_u '\165' +#define CHAR_v '\166' +#define CHAR_w '\167' +#define CHAR_x '\170' +#define CHAR_y '\171' +#define CHAR_z '\172' +#define CHAR_LEFT_CURLY_BRACKET '\173' +#define CHAR_VERTICAL_LINE '\174' +#define CHAR_RIGHT_CURLY_BRACKET '\175' +#define CHAR_TILDE '\176' +#define CHAR_NBSP ((unsigned char)'\xa0') + +#define STR_HT "\011" +#define STR_VT "\013" +#define STR_FF "\014" +#define STR_CR "\015" +#define STR_NL "\012" +#define STR_BS "\010" +#define STR_BEL "\007" +#define STR_ESC "\033" +#define STR_DEL "\177" + +#define STR_SPACE "\040" +#define STR_EXCLAMATION_MARK "\041" +#define STR_QUOTATION_MARK "\042" +#define STR_NUMBER_SIGN "\043" +#define STR_DOLLAR_SIGN "\044" +#define STR_PERCENT_SIGN "\045" +#define STR_AMPERSAND "\046" +#define STR_APOSTROPHE "\047" +#define STR_LEFT_PARENTHESIS "\050" +#define STR_RIGHT_PARENTHESIS "\051" +#define STR_ASTERISK "\052" +#define STR_PLUS "\053" +#define STR_COMMA "\054" +#define STR_MINUS "\055" +#define STR_DOT "\056" +#define STR_SLASH "\057" +#define STR_0 "\060" +#define STR_1 "\061" +#define STR_2 "\062" +#define STR_3 "\063" +#define STR_4 "\064" +#define STR_5 "\065" +#define STR_6 "\066" +#define STR_7 "\067" +#define STR_8 "\070" +#define STR_9 "\071" +#define STR_COLON "\072" +#define STR_SEMICOLON "\073" +#define STR_LESS_THAN_SIGN "\074" +#define STR_EQUALS_SIGN "\075" +#define STR_GREATER_THAN_SIGN "\076" +#define STR_QUESTION_MARK "\077" +#define STR_COMMERCIAL_AT "\100" +#define STR_A "\101" +#define STR_B "\102" +#define STR_C "\103" +#define STR_D "\104" +#define STR_E "\105" +#define STR_F "\106" +#define STR_G "\107" +#define STR_H "\110" +#define STR_I "\111" +#define STR_J "\112" +#define STR_K "\113" +#define STR_L "\114" +#define STR_M "\115" +#define STR_N "\116" +#define STR_O "\117" +#define STR_P "\120" +#define STR_Q "\121" +#define STR_R "\122" +#define STR_S "\123" +#define STR_T "\124" +#define STR_U "\125" +#define STR_V "\126" +#define STR_W "\127" +#define STR_X "\130" +#define STR_Y "\131" +#define STR_Z "\132" +#define STR_LEFT_SQUARE_BRACKET "\133" +#define STR_BACKSLASH "\134" +#define STR_RIGHT_SQUARE_BRACKET "\135" +#define STR_CIRCUMFLEX_ACCENT "\136" +#define STR_UNDERSCORE "\137" +#define STR_GRAVE_ACCENT "\140" +#define STR_a "\141" +#define STR_b "\142" +#define STR_c "\143" +#define STR_d "\144" +#define STR_e "\145" +#define STR_f "\146" +#define STR_g "\147" +#define STR_h "\150" +#define STR_i "\151" +#define STR_j "\152" +#define STR_k "\153" +#define STR_l "\154" +#define STR_m "\155" +#define STR_n "\156" +#define STR_o "\157" +#define STR_p "\160" +#define STR_q "\161" +#define STR_r "\162" +#define STR_s "\163" +#define STR_t "\164" +#define STR_u "\165" +#define STR_v "\166" +#define STR_w "\167" +#define STR_x "\170" +#define STR_y "\171" +#define STR_z "\172" +#define STR_LEFT_CURLY_BRACKET "\173" +#define STR_VERTICAL_LINE "\174" +#define STR_RIGHT_CURLY_BRACKET "\175" +#define STR_TILDE "\176" + +#define STRING_ACCEPT0 STR_A STR_C STR_C STR_E STR_P STR_T "\0" +#define STRING_COMMIT0 STR_C STR_O STR_M STR_M STR_I STR_T "\0" +#define STRING_F0 STR_F "\0" +#define STRING_FAIL0 STR_F STR_A STR_I STR_L "\0" +#define STRING_MARK0 STR_M STR_A STR_R STR_K "\0" +#define STRING_PRUNE0 STR_P STR_R STR_U STR_N STR_E "\0" +#define STRING_SKIP0 STR_S STR_K STR_I STR_P "\0" +#define STRING_THEN STR_T STR_H STR_E STR_N + +#define STRING_alpha0 STR_a STR_l STR_p STR_h STR_a "\0" +#define STRING_lower0 STR_l STR_o STR_w STR_e STR_r "\0" +#define STRING_upper0 STR_u STR_p STR_p STR_e STR_r "\0" +#define STRING_alnum0 STR_a STR_l STR_n STR_u STR_m "\0" +#define STRING_ascii0 STR_a STR_s STR_c STR_i STR_i "\0" +#define STRING_blank0 STR_b STR_l STR_a STR_n STR_k "\0" +#define STRING_cntrl0 STR_c STR_n STR_t STR_r STR_l "\0" +#define STRING_digit0 STR_d STR_i STR_g STR_i STR_t "\0" +#define STRING_graph0 STR_g STR_r STR_a STR_p STR_h "\0" +#define STRING_print0 STR_p STR_r STR_i STR_n STR_t "\0" +#define STRING_punct0 STR_p STR_u STR_n STR_c STR_t "\0" +#define STRING_space0 STR_s STR_p STR_a STR_c STR_e "\0" +#define STRING_word0 STR_w STR_o STR_r STR_d "\0" +#define STRING_xdigit STR_x STR_d STR_i STR_g STR_i STR_t + +#define STRING_DEFINE STR_D STR_E STR_F STR_I STR_N STR_E +#define STRING_WEIRD_STARTWORD STR_LEFT_SQUARE_BRACKET STR_COLON STR_LESS_THAN_SIGN STR_COLON STR_RIGHT_SQUARE_BRACKET STR_RIGHT_SQUARE_BRACKET +#define STRING_WEIRD_ENDWORD STR_LEFT_SQUARE_BRACKET STR_COLON STR_GREATER_THAN_SIGN STR_COLON STR_RIGHT_SQUARE_BRACKET STR_RIGHT_SQUARE_BRACKET + +#define STRING_CR_RIGHTPAR STR_C STR_R STR_RIGHT_PARENTHESIS +#define STRING_LF_RIGHTPAR STR_L STR_F STR_RIGHT_PARENTHESIS +#define STRING_CRLF_RIGHTPAR STR_C STR_R STR_L STR_F STR_RIGHT_PARENTHESIS +#define STRING_ANY_RIGHTPAR STR_A STR_N STR_Y STR_RIGHT_PARENTHESIS +#define STRING_ANYCRLF_RIGHTPAR STR_A STR_N STR_Y STR_C STR_R STR_L STR_F STR_RIGHT_PARENTHESIS +#define STRING_BSR_ANYCRLF_RIGHTPAR STR_B STR_S STR_R STR_UNDERSCORE STR_A STR_N STR_Y STR_C STR_R STR_L STR_F STR_RIGHT_PARENTHESIS +#define STRING_BSR_UNICODE_RIGHTPAR STR_B STR_S STR_R STR_UNDERSCORE STR_U STR_N STR_I STR_C STR_O STR_D STR_E STR_RIGHT_PARENTHESIS +#define STRING_UTF8_RIGHTPAR STR_U STR_T STR_F STR_8 STR_RIGHT_PARENTHESIS +#define STRING_UTF16_RIGHTPAR STR_U STR_T STR_F STR_1 STR_6 STR_RIGHT_PARENTHESIS +#define STRING_UTF32_RIGHTPAR STR_U STR_T STR_F STR_3 STR_2 STR_RIGHT_PARENTHESIS +#define STRING_UTF_RIGHTPAR STR_U STR_T STR_F STR_RIGHT_PARENTHESIS +#define STRING_UCP_RIGHTPAR STR_U STR_C STR_P STR_RIGHT_PARENTHESIS +#define STRING_NO_AUTO_POSSESS_RIGHTPAR STR_N STR_O STR_UNDERSCORE STR_A STR_U STR_T STR_O STR_UNDERSCORE STR_P STR_O STR_S STR_S STR_E STR_S STR_S STR_RIGHT_PARENTHESIS +#define STRING_NO_START_OPT_RIGHTPAR STR_N STR_O STR_UNDERSCORE STR_S STR_T STR_A STR_R STR_T STR_UNDERSCORE STR_O STR_P STR_T STR_RIGHT_PARENTHESIS +#define STRING_LIMIT_MATCH_EQ STR_L STR_I STR_M STR_I STR_T STR_UNDERSCORE STR_M STR_A STR_T STR_C STR_H STR_EQUALS_SIGN +#define STRING_LIMIT_RECURSION_EQ STR_L STR_I STR_M STR_I STR_T STR_UNDERSCORE STR_R STR_E STR_C STR_U STR_R STR_S STR_I STR_O STR_N STR_EQUALS_SIGN + +#endif /* SUPPORT_UTF */ + +/* Escape items that are just an encoding of a particular data value. */ + +#ifndef ESC_a +#define ESC_a CHAR_BEL +#endif + +#ifndef ESC_e +#define ESC_e CHAR_ESC +#endif + +#ifndef ESC_f +#define ESC_f CHAR_FF +#endif + +#ifndef ESC_n +#define ESC_n CHAR_LF +#endif + +#ifndef ESC_r +#define ESC_r CHAR_CR +#endif + +/* We can't officially use ESC_t because it is a POSIX reserved identifier +(presumably because of all the others like size_t). */ + +#ifndef ESC_tee +#define ESC_tee CHAR_HT +#endif + +/* Codes for different types of Unicode property */ + +#define PT_ANY 0 /* Any property - matches all chars */ +#define PT_LAMP 1 /* L& - the union of Lu, Ll, Lt */ +#define PT_GC 2 /* Specified general characteristic (e.g. L) */ +#define PT_PC 3 /* Specified particular characteristic (e.g. Lu) */ +#define PT_SC 4 /* Script (e.g. Han) */ +#define PT_ALNUM 5 /* Alphanumeric - the union of L and N */ +#define PT_SPACE 6 /* Perl space - Z plus 9,10,12,13 */ +#define PT_PXSPACE 7 /* POSIX space - Z plus 9,10,11,12,13 */ +#define PT_WORD 8 /* Word - L plus N plus underscore */ +#define PT_CLIST 9 /* Pseudo-property: match character list */ +#define PT_UCNC 10 /* Universal Character nameable character */ +#define PT_TABSIZE 11 /* Size of square table for autopossessify tests */ + +/* The following special properties are used only in XCLASS items, when POSIX +classes are specified and PCRE_UCP is set - in other words, for Unicode +handling of these classes. They are not available via the \p or \P escapes like +those in the above list, and so they do not take part in the autopossessifying +table. */ + +#define PT_PXGRAPH 11 /* [:graph:] - characters that mark the paper */ +#define PT_PXPRINT 12 /* [:print:] - [:graph:] plus non-control spaces */ +#define PT_PXPUNCT 13 /* [:punct:] - punctuation characters */ + +/* Flag bits and data types for the extended class (OP_XCLASS) for classes that +contain characters with values greater than 255. */ + +#define XCL_NOT 0x01 /* Flag: this is a negative class */ +#define XCL_MAP 0x02 /* Flag: a 32-byte map is present */ +#define XCL_HASPROP 0x04 /* Flag: property checks are present. */ + +#define XCL_END 0 /* Marks end of individual items */ +#define XCL_SINGLE 1 /* Single item (one multibyte char) follows */ +#define XCL_RANGE 2 /* A range (two multibyte chars) follows */ +#define XCL_PROP 3 /* Unicode property (2-byte property code follows) */ +#define XCL_NOTPROP 4 /* Unicode inverted property (ditto) */ + +/* These are escaped items that aren't just an encoding of a particular data +value such as \n. They must have non-zero values, as check_escape() returns 0 +for a data character. Also, they must appear in the same order as in the +opcode definitions below, up to ESC_z. There's a dummy for OP_ALLANY because it +corresponds to "." in DOTALL mode rather than an escape sequence. It is also +used for [^] in JavaScript compatibility mode, and for \C in non-utf mode. In +non-DOTALL mode, "." behaves like \N. + +The special values ESC_DU, ESC_du, etc. are used instead of ESC_D, ESC_d, etc. +when PCRE_UCP is set and replacement of \d etc by \p sequences is required. +They must be contiguous, and remain in order so that the replacements can be +looked up from a table. + +Negative numbers are used to encode a backreference (\1, \2, \3, etc.) in +check_escape(). There are two tests in the code for an escape +greater than ESC_b and less than ESC_Z to detect the types that may be +repeated. These are the types that consume characters. If any new escapes are +put in between that don't consume a character, that code will have to change. +*/ + +enum { ESC_A = 1, ESC_G, ESC_K, ESC_B, ESC_b, ESC_D, ESC_d, ESC_S, ESC_s, + ESC_W, ESC_w, ESC_N, ESC_dum, ESC_C, ESC_P, ESC_p, ESC_R, ESC_H, + ESC_h, ESC_V, ESC_v, ESC_X, ESC_Z, ESC_z, + ESC_E, ESC_Q, ESC_g, ESC_k, + ESC_DU, ESC_du, ESC_SU, ESC_su, ESC_WU, ESC_wu }; + + +/********************** Opcode definitions ******************/ + +/****** NOTE NOTE NOTE ****** + +Starting from 1 (i.e. after OP_END), the values up to OP_EOD must correspond in +order to the list of escapes immediately above. Furthermore, values up to +OP_DOLLM must not be changed without adjusting the table called autoposstab in +pcre_compile.c + +Whenever this list is updated, the two macro definitions that follow must be +updated to match. The possessification table called "opcode_possessify" in +pcre_compile.c must also be updated, and also the tables called "coptable" +and "poptable" in pcre_dfa_exec.c. + +****** NOTE NOTE NOTE ******/ + + +/* The values between FIRST_AUTOTAB_OP and LAST_AUTOTAB_RIGHT_OP, inclusive, +are used in a table for deciding whether a repeated character type can be +auto-possessified. */ + +#define FIRST_AUTOTAB_OP OP_NOT_DIGIT +#define LAST_AUTOTAB_LEFT_OP OP_EXTUNI +#define LAST_AUTOTAB_RIGHT_OP OP_DOLLM + +enum { + OP_END, /* 0 End of pattern */ + + /* Values corresponding to backslashed metacharacters */ + + OP_SOD, /* 1 Start of data: \A */ + OP_SOM, /* 2 Start of match (subject + offset): \G */ + OP_SET_SOM, /* 3 Set start of match (\K) */ + OP_NOT_WORD_BOUNDARY, /* 4 \B */ + OP_WORD_BOUNDARY, /* 5 \b */ + OP_NOT_DIGIT, /* 6 \D */ + OP_DIGIT, /* 7 \d */ + OP_NOT_WHITESPACE, /* 8 \S */ + OP_WHITESPACE, /* 9 \s */ + OP_NOT_WORDCHAR, /* 10 \W */ + OP_WORDCHAR, /* 11 \w */ + + OP_ANY, /* 12 Match any character except newline (\N) */ + OP_ALLANY, /* 13 Match any character */ + OP_ANYBYTE, /* 14 Match any byte (\C); different to OP_ANY for UTF-8 */ + OP_NOTPROP, /* 15 \P (not Unicode property) */ + OP_PROP, /* 16 \p (Unicode property) */ + OP_ANYNL, /* 17 \R (any newline sequence) */ + OP_NOT_HSPACE, /* 18 \H (not horizontal whitespace) */ + OP_HSPACE, /* 19 \h (horizontal whitespace) */ + OP_NOT_VSPACE, /* 20 \V (not vertical whitespace) */ + OP_VSPACE, /* 21 \v (vertical whitespace) */ + OP_EXTUNI, /* 22 \X (extended Unicode sequence */ + OP_EODN, /* 23 End of data or \n at end of data (\Z) */ + OP_EOD, /* 24 End of data (\z) */ + + /* Line end assertions */ + + OP_DOLL, /* 25 End of line - not multiline */ + OP_DOLLM, /* 26 End of line - multiline */ + OP_CIRC, /* 27 Start of line - not multiline */ + OP_CIRCM, /* 28 Start of line - multiline */ + + /* Single characters; caseful must precede the caseless ones */ + + OP_CHAR, /* 29 Match one character, casefully */ + OP_CHARI, /* 30 Match one character, caselessly */ + OP_NOT, /* 31 Match one character, not the given one, casefully */ + OP_NOTI, /* 32 Match one character, not the given one, caselessly */ + + /* The following sets of 13 opcodes must always be kept in step because + the offset from the first one is used to generate the others. */ + + /* Repeated characters; caseful must precede the caseless ones */ + + OP_STAR, /* 33 The maximizing and minimizing versions of */ + OP_MINSTAR, /* 34 these six opcodes must come in pairs, with */ + OP_PLUS, /* 35 the minimizing one second. */ + OP_MINPLUS, /* 36 */ + OP_QUERY, /* 37 */ + OP_MINQUERY, /* 38 */ + + OP_UPTO, /* 39 From 0 to n matches of one character, caseful*/ + OP_MINUPTO, /* 40 */ + OP_EXACT, /* 41 Exactly n matches */ + + OP_POSSTAR, /* 42 Possessified star, caseful */ + OP_POSPLUS, /* 43 Possessified plus, caseful */ + OP_POSQUERY, /* 44 Posesssified query, caseful */ + OP_POSUPTO, /* 45 Possessified upto, caseful */ + + /* Repeated characters; caseless must follow the caseful ones */ + + OP_STARI, /* 46 */ + OP_MINSTARI, /* 47 */ + OP_PLUSI, /* 48 */ + OP_MINPLUSI, /* 49 */ + OP_QUERYI, /* 50 */ + OP_MINQUERYI, /* 51 */ + + OP_UPTOI, /* 52 From 0 to n matches of one character, caseless */ + OP_MINUPTOI, /* 53 */ + OP_EXACTI, /* 54 */ + + OP_POSSTARI, /* 55 Possessified star, caseless */ + OP_POSPLUSI, /* 56 Possessified plus, caseless */ + OP_POSQUERYI, /* 57 Posesssified query, caseless */ + OP_POSUPTOI, /* 58 Possessified upto, caseless */ + + /* The negated ones must follow the non-negated ones, and match them */ + /* Negated repeated character, caseful; must precede the caseless ones */ + + OP_NOTSTAR, /* 59 The maximizing and minimizing versions of */ + OP_NOTMINSTAR, /* 60 these six opcodes must come in pairs, with */ + OP_NOTPLUS, /* 61 the minimizing one second. They must be in */ + OP_NOTMINPLUS, /* 62 exactly the same order as those above. */ + OP_NOTQUERY, /* 63 */ + OP_NOTMINQUERY, /* 64 */ + + OP_NOTUPTO, /* 65 From 0 to n matches, caseful */ + OP_NOTMINUPTO, /* 66 */ + OP_NOTEXACT, /* 67 Exactly n matches */ + + OP_NOTPOSSTAR, /* 68 Possessified versions, caseful */ + OP_NOTPOSPLUS, /* 69 */ + OP_NOTPOSQUERY, /* 70 */ + OP_NOTPOSUPTO, /* 71 */ + + /* Negated repeated character, caseless; must follow the caseful ones */ + + OP_NOTSTARI, /* 72 */ + OP_NOTMINSTARI, /* 73 */ + OP_NOTPLUSI, /* 74 */ + OP_NOTMINPLUSI, /* 75 */ + OP_NOTQUERYI, /* 76 */ + OP_NOTMINQUERYI, /* 77 */ + + OP_NOTUPTOI, /* 78 From 0 to n matches, caseless */ + OP_NOTMINUPTOI, /* 79 */ + OP_NOTEXACTI, /* 80 Exactly n matches */ + + OP_NOTPOSSTARI, /* 81 Possessified versions, caseless */ + OP_NOTPOSPLUSI, /* 82 */ + OP_NOTPOSQUERYI, /* 83 */ + OP_NOTPOSUPTOI, /* 84 */ + + /* Character types */ + + OP_TYPESTAR, /* 85 The maximizing and minimizing versions of */ + OP_TYPEMINSTAR, /* 86 these six opcodes must come in pairs, with */ + OP_TYPEPLUS, /* 87 the minimizing one second. These codes must */ + OP_TYPEMINPLUS, /* 88 be in exactly the same order as those above. */ + OP_TYPEQUERY, /* 89 */ + OP_TYPEMINQUERY, /* 90 */ + + OP_TYPEUPTO, /* 91 From 0 to n matches */ + OP_TYPEMINUPTO, /* 92 */ + OP_TYPEEXACT, /* 93 Exactly n matches */ + + OP_TYPEPOSSTAR, /* 94 Possessified versions */ + OP_TYPEPOSPLUS, /* 95 */ + OP_TYPEPOSQUERY, /* 96 */ + OP_TYPEPOSUPTO, /* 97 */ + + /* These are used for character classes and back references; only the + first six are the same as the sets above. */ + + OP_CRSTAR, /* 98 The maximizing and minimizing versions of */ + OP_CRMINSTAR, /* 99 all these opcodes must come in pairs, with */ + OP_CRPLUS, /* 100 the minimizing one second. These codes must */ + OP_CRMINPLUS, /* 101 be in exactly the same order as those above. */ + OP_CRQUERY, /* 102 */ + OP_CRMINQUERY, /* 103 */ + + OP_CRRANGE, /* 104 These are different to the three sets above. */ + OP_CRMINRANGE, /* 105 */ + + OP_CRPOSSTAR, /* 106 Possessified versions */ + OP_CRPOSPLUS, /* 107 */ + OP_CRPOSQUERY, /* 108 */ + OP_CRPOSRANGE, /* 109 */ + + /* End of quantifier opcodes */ + + OP_CLASS, /* 110 Match a character class, chars < 256 only */ + OP_NCLASS, /* 111 Same, but the bitmap was created from a negative + class - the difference is relevant only when a + character > 255 is encountered. */ + OP_XCLASS, /* 112 Extended class for handling > 255 chars within the + class. This does both positive and negative. */ + OP_REF, /* 113 Match a back reference, casefully */ + OP_REFI, /* 114 Match a back reference, caselessly */ + OP_DNREF, /* 115 Match a duplicate name backref, casefully */ + OP_DNREFI, /* 116 Match a duplicate name backref, caselessly */ + OP_RECURSE, /* 117 Match a numbered subpattern (possibly recursive) */ + OP_CALLOUT, /* 118 Call out to external function if provided */ + + OP_ALT, /* 119 Start of alternation */ + OP_KET, /* 120 End of group that doesn't have an unbounded repeat */ + OP_KETRMAX, /* 121 These two must remain together and in this */ + OP_KETRMIN, /* 122 order. They are for groups the repeat for ever. */ + OP_KETRPOS, /* 123 Possessive unlimited repeat. */ + + /* The assertions must come before BRA, CBRA, ONCE, and COND, and the four + asserts must remain in order. */ + + OP_REVERSE, /* 124 Move pointer back - used in lookbehind assertions */ + OP_ASSERT, /* 125 Positive lookahead */ + OP_ASSERT_NOT, /* 126 Negative lookahead */ + OP_ASSERTBACK, /* 127 Positive lookbehind */ + OP_ASSERTBACK_NOT, /* 128 Negative lookbehind */ + + /* ONCE, ONCE_NC, BRA, BRAPOS, CBRA, CBRAPOS, and COND must come immediately + after the assertions, with ONCE first, as there's a test for >= ONCE for a + subpattern that isn't an assertion. The POS versions must immediately follow + the non-POS versions in each case. */ + + OP_ONCE, /* 129 Atomic group, contains captures */ + OP_ONCE_NC, /* 130 Atomic group containing no captures */ + OP_BRA, /* 131 Start of non-capturing bracket */ + OP_BRAPOS, /* 132 Ditto, with unlimited, possessive repeat */ + OP_CBRA, /* 133 Start of capturing bracket */ + OP_CBRAPOS, /* 134 Ditto, with unlimited, possessive repeat */ + OP_COND, /* 135 Conditional group */ + + /* These five must follow the previous five, in the same order. There's a + check for >= SBRA to distinguish the two sets. */ + + OP_SBRA, /* 136 Start of non-capturing bracket, check empty */ + OP_SBRAPOS, /* 137 Ditto, with unlimited, possessive repeat */ + OP_SCBRA, /* 138 Start of capturing bracket, check empty */ + OP_SCBRAPOS, /* 139 Ditto, with unlimited, possessive repeat */ + OP_SCOND, /* 140 Conditional group, check empty */ + + /* The next two pairs must (respectively) be kept together. */ + + OP_CREF, /* 141 Used to hold a capture number as condition */ + OP_DNCREF, /* 142 Used to point to duplicate names as a condition */ + OP_RREF, /* 143 Used to hold a recursion number as condition */ + OP_DNRREF, /* 144 Used to point to duplicate names as a condition */ + OP_DEF, /* 145 The DEFINE condition */ + + OP_BRAZERO, /* 146 These two must remain together and in this */ + OP_BRAMINZERO, /* 147 order. */ + OP_BRAPOSZERO, /* 148 */ + + /* These are backtracking control verbs */ + + OP_MARK, /* 149 always has an argument */ + OP_PRUNE, /* 150 */ + OP_PRUNE_ARG, /* 151 same, but with argument */ + OP_SKIP, /* 152 */ + OP_SKIP_ARG, /* 153 same, but with argument */ + OP_THEN, /* 154 */ + OP_THEN_ARG, /* 155 same, but with argument */ + OP_COMMIT, /* 156 */ + + /* These are forced failure and success verbs */ + + OP_FAIL, /* 157 */ + OP_ACCEPT, /* 158 */ + OP_ASSERT_ACCEPT, /* 159 Used inside assertions */ + OP_CLOSE, /* 160 Used before OP_ACCEPT to close open captures */ + + /* This is used to skip a subpattern with a {0} quantifier */ + + OP_SKIPZERO, /* 161 */ + + /* This is not an opcode, but is used to check that tables indexed by opcode + are the correct length, in order to catch updating errors - there have been + some in the past. */ + + OP_TABLE_LENGTH +}; + +/* *** NOTE NOTE NOTE *** Whenever the list above is updated, the two macro +definitions that follow must also be updated to match. There are also tables +called "opcode_possessify" in pcre_compile.c and "coptable" and "poptable" in +pcre_dfa_exec.c that must be updated. */ + + +/* This macro defines textual names for all the opcodes. These are used only +for debugging, and some of them are only partial names. The macro is referenced +only in pcre_printint.c, which fills out the full names in many cases (and in +some cases doesn't actually use these names at all). */ + +#define OP_NAME_LIST \ + "End", "\\A", "\\G", "\\K", "\\B", "\\b", "\\D", "\\d", \ + "\\S", "\\s", "\\W", "\\w", "Any", "AllAny", "Anybyte", \ + "notprop", "prop", "\\R", "\\H", "\\h", "\\V", "\\v", \ + "extuni", "\\Z", "\\z", \ + "$", "$", "^", "^", "char", "chari", "not", "noti", \ + "*", "*?", "+", "+?", "?", "??", \ + "{", "{", "{", \ + "*+","++", "?+", "{", \ + "*", "*?", "+", "+?", "?", "??", \ + "{", "{", "{", \ + "*+","++", "?+", "{", \ + "*", "*?", "+", "+?", "?", "??", \ + "{", "{", "{", \ + "*+","++", "?+", "{", \ + "*", "*?", "+", "+?", "?", "??", \ + "{", "{", "{", \ + "*+","++", "?+", "{", \ + "*", "*?", "+", "+?", "?", "??", "{", "{", "{", \ + "*+","++", "?+", "{", \ + "*", "*?", "+", "+?", "?", "??", "{", "{", \ + "*+","++", "?+", "{", \ + "class", "nclass", "xclass", "Ref", "Refi", "DnRef", "DnRefi", \ + "Recurse", "Callout", \ + "Alt", "Ket", "KetRmax", "KetRmin", "KetRpos", \ + "Reverse", "Assert", "Assert not", "AssertB", "AssertB not", \ + "Once", "Once_NC", \ + "Bra", "BraPos", "CBra", "CBraPos", \ + "Cond", \ + "SBra", "SBraPos", "SCBra", "SCBraPos", \ + "SCond", \ + "Cond ref", "Cond dnref", "Cond rec", "Cond dnrec", "Cond def", \ + "Brazero", "Braminzero", "Braposzero", \ + "*MARK", "*PRUNE", "*PRUNE", "*SKIP", "*SKIP", \ + "*THEN", "*THEN", "*COMMIT", "*FAIL", \ + "*ACCEPT", "*ASSERT_ACCEPT", \ + "Close", "Skip zero" + + +/* This macro defines the length of fixed length operations in the compiled +regex. The lengths are used when searching for specific things, and also in the +debugging printing of a compiled regex. We use a macro so that it can be +defined close to the definitions of the opcodes themselves. + +As things have been extended, some of these are no longer fixed lenths, but are +minima instead. For example, the length of a single-character repeat may vary +in UTF-8 mode. The code that uses this table must know about such things. */ + +#define OP_LENGTHS \ + 1, /* End */ \ + 1, 1, 1, 1, 1, /* \A, \G, \K, \B, \b */ \ + 1, 1, 1, 1, 1, 1, /* \D, \d, \S, \s, \W, \w */ \ + 1, 1, 1, /* Any, AllAny, Anybyte */ \ + 3, 3, /* \P, \p */ \ + 1, 1, 1, 1, 1, /* \R, \H, \h, \V, \v */ \ + 1, /* \X */ \ + 1, 1, 1, 1, 1, 1, /* \Z, \z, $, $M ^, ^M */ \ + 2, /* Char - the minimum length */ \ + 2, /* Chari - the minimum length */ \ + 2, /* not */ \ + 2, /* noti */ \ + /* Positive single-char repeats ** These are */ \ + 2, 2, 2, 2, 2, 2, /* *, *?, +, +?, ?, ?? ** minima in */ \ + 2+IMM2_SIZE, 2+IMM2_SIZE, /* upto, minupto ** mode */ \ + 2+IMM2_SIZE, /* exact */ \ + 2, 2, 2, 2+IMM2_SIZE, /* *+, ++, ?+, upto+ */ \ + 2, 2, 2, 2, 2, 2, /* *I, *?I, +I, +?I, ?I, ??I ** UTF-8 */ \ + 2+IMM2_SIZE, 2+IMM2_SIZE, /* upto I, minupto I */ \ + 2+IMM2_SIZE, /* exact I */ \ + 2, 2, 2, 2+IMM2_SIZE, /* *+I, ++I, ?+I, upto+I */ \ + /* Negative single-char repeats - only for chars < 256 */ \ + 2, 2, 2, 2, 2, 2, /* NOT *, *?, +, +?, ?, ?? */ \ + 2+IMM2_SIZE, 2+IMM2_SIZE, /* NOT upto, minupto */ \ + 2+IMM2_SIZE, /* NOT exact */ \ + 2, 2, 2, 2+IMM2_SIZE, /* Possessive NOT *, +, ?, upto */ \ + 2, 2, 2, 2, 2, 2, /* NOT *I, *?I, +I, +?I, ?I, ??I */ \ + 2+IMM2_SIZE, 2+IMM2_SIZE, /* NOT upto I, minupto I */ \ + 2+IMM2_SIZE, /* NOT exact I */ \ + 2, 2, 2, 2+IMM2_SIZE, /* Possessive NOT *I, +I, ?I, upto I */ \ + /* Positive type repeats */ \ + 2, 2, 2, 2, 2, 2, /* Type *, *?, +, +?, ?, ?? */ \ + 2+IMM2_SIZE, 2+IMM2_SIZE, /* Type upto, minupto */ \ + 2+IMM2_SIZE, /* Type exact */ \ + 2, 2, 2, 2+IMM2_SIZE, /* Possessive *+, ++, ?+, upto+ */ \ + /* Character class & ref repeats */ \ + 1, 1, 1, 1, 1, 1, /* *, *?, +, +?, ?, ?? */ \ + 1+2*IMM2_SIZE, 1+2*IMM2_SIZE, /* CRRANGE, CRMINRANGE */ \ + 1, 1, 1, 1+2*IMM2_SIZE, /* Possessive *+, ++, ?+, CRPOSRANGE */ \ + 1+(32/sizeof(pcre_uchar)), /* CLASS */ \ + 1+(32/sizeof(pcre_uchar)), /* NCLASS */ \ + 0, /* XCLASS - variable length */ \ + 1+IMM2_SIZE, /* REF */ \ + 1+IMM2_SIZE, /* REFI */ \ + 1+2*IMM2_SIZE, /* DNREF */ \ + 1+2*IMM2_SIZE, /* DNREFI */ \ + 1+LINK_SIZE, /* RECURSE */ \ + 2+2*LINK_SIZE, /* CALLOUT */ \ + 1+LINK_SIZE, /* Alt */ \ + 1+LINK_SIZE, /* Ket */ \ + 1+LINK_SIZE, /* KetRmax */ \ + 1+LINK_SIZE, /* KetRmin */ \ + 1+LINK_SIZE, /* KetRpos */ \ + 1+LINK_SIZE, /* Reverse */ \ + 1+LINK_SIZE, /* Assert */ \ + 1+LINK_SIZE, /* Assert not */ \ + 1+LINK_SIZE, /* Assert behind */ \ + 1+LINK_SIZE, /* Assert behind not */ \ + 1+LINK_SIZE, /* ONCE */ \ + 1+LINK_SIZE, /* ONCE_NC */ \ + 1+LINK_SIZE, /* BRA */ \ + 1+LINK_SIZE, /* BRAPOS */ \ + 1+LINK_SIZE+IMM2_SIZE, /* CBRA */ \ + 1+LINK_SIZE+IMM2_SIZE, /* CBRAPOS */ \ + 1+LINK_SIZE, /* COND */ \ + 1+LINK_SIZE, /* SBRA */ \ + 1+LINK_SIZE, /* SBRAPOS */ \ + 1+LINK_SIZE+IMM2_SIZE, /* SCBRA */ \ + 1+LINK_SIZE+IMM2_SIZE, /* SCBRAPOS */ \ + 1+LINK_SIZE, /* SCOND */ \ + 1+IMM2_SIZE, 1+2*IMM2_SIZE, /* CREF, DNCREF */ \ + 1+IMM2_SIZE, 1+2*IMM2_SIZE, /* RREF, DNRREF */ \ + 1, /* DEF */ \ + 1, 1, 1, /* BRAZERO, BRAMINZERO, BRAPOSZERO */ \ + 3, 1, 3, /* MARK, PRUNE, PRUNE_ARG */ \ + 1, 3, /* SKIP, SKIP_ARG */ \ + 1, 3, /* THEN, THEN_ARG */ \ + 1, 1, 1, 1, /* COMMIT, FAIL, ACCEPT, ASSERT_ACCEPT */ \ + 1+IMM2_SIZE, 1 /* CLOSE, SKIPZERO */ + +/* A magic value for OP_RREF to indicate the "any recursion" condition. */ + +#define RREF_ANY 0xffff + +/* Compile time error code numbers. They are given names so that they can more +easily be tracked. When a new number is added, the table called eint in +pcreposix.c must be updated. */ + +enum { ERR0, ERR1, ERR2, ERR3, ERR4, ERR5, ERR6, ERR7, ERR8, ERR9, + ERR10, ERR11, ERR12, ERR13, ERR14, ERR15, ERR16, ERR17, ERR18, ERR19, + ERR20, ERR21, ERR22, ERR23, ERR24, ERR25, ERR26, ERR27, ERR28, ERR29, + ERR30, ERR31, ERR32, ERR33, ERR34, ERR35, ERR36, ERR37, ERR38, ERR39, + ERR40, ERR41, ERR42, ERR43, ERR44, ERR45, ERR46, ERR47, ERR48, ERR49, + ERR50, ERR51, ERR52, ERR53, ERR54, ERR55, ERR56, ERR57, ERR58, ERR59, + ERR60, ERR61, ERR62, ERR63, ERR64, ERR65, ERR66, ERR67, ERR68, ERR69, + ERR70, ERR71, ERR72, ERR73, ERR74, ERR75, ERR76, ERR77, ERR78, ERR79, + ERR80, ERR81, ERR82, ERR83, ERR84, ERR85, ERR86, ERR87, ERRCOUNT }; + +/* JIT compiling modes. The function list is indexed by them. */ + +enum { JIT_COMPILE, JIT_PARTIAL_SOFT_COMPILE, JIT_PARTIAL_HARD_COMPILE, + JIT_NUMBER_OF_COMPILE_MODES }; + +/* The real format of the start of the pcre block; the index of names and the +code vector run on as long as necessary after the end. We store an explicit +offset to the name table so that if a regex is compiled on one host, saved, and +then run on another where the size of pointers is different, all might still +be well. + +The size of the structure must be a multiple of 8 bytes. For the case of +compiled-on-4 and run-on-8, we include an extra pointer that is always NULL so +that there are an even number of pointers which therefore are a multiple of 8 +bytes. + +It is necessary to fork the struct for the 32 bit library, since it needs to +use pcre_uint32 for first_char and req_char. We can't put an ifdef inside the +typedef because pcretest needs access to the struct of the 8-, 16- and 32-bit +variants. + +*** WARNING *** +When new fields are added to these structures, remember to adjust the code in +pcre_byte_order.c that is concerned with swapping the byte order of the fields +when a compiled regex is reloaded on a host with different endianness. +*** WARNING *** +There is also similar byte-flipping code in pcretest.c, which is used for +testing the byte-flipping features. It must also be kept in step. +*** WARNING *** +*/ + +typedef struct real_pcre8_or_16 { + pcre_uint32 magic_number; + pcre_uint32 size; /* Total that was malloced */ + pcre_uint32 options; /* Public options */ + pcre_uint32 flags; /* Private flags */ + pcre_uint32 limit_match; /* Limit set from regex */ + pcre_uint32 limit_recursion; /* Limit set from regex */ + pcre_uint16 first_char; /* Starting character */ + pcre_uint16 req_char; /* This character must be seen */ + pcre_uint16 max_lookbehind; /* Longest lookbehind (characters) */ + pcre_uint16 top_bracket; /* Highest numbered group */ + pcre_uint16 top_backref; /* Highest numbered back reference */ + pcre_uint16 name_table_offset; /* Offset to name table that follows */ + pcre_uint16 name_entry_size; /* Size of any name items */ + pcre_uint16 name_count; /* Number of name items */ + pcre_uint16 ref_count; /* Reference count */ + pcre_uint16 dummy1; /* To ensure size is a multiple of 8 */ + pcre_uint16 dummy2; /* To ensure size is a multiple of 8 */ + pcre_uint16 dummy3; /* To ensure size is a multiple of 8 */ + const pcre_uint8 *tables; /* Pointer to tables or NULL for std */ + void *nullpad; /* NULL padding */ +} real_pcre8_or_16; + +typedef struct real_pcre8_or_16 real_pcre; +typedef struct real_pcre8_or_16 real_pcre16; + +typedef struct real_pcre32 { + pcre_uint32 magic_number; + pcre_uint32 size; /* Total that was malloced */ + pcre_uint32 options; /* Public options */ + pcre_uint32 flags; /* Private flags */ + pcre_uint32 limit_match; /* Limit set from regex */ + pcre_uint32 limit_recursion; /* Limit set from regex */ + pcre_uint32 first_char; /* Starting character */ + pcre_uint32 req_char; /* This character must be seen */ + pcre_uint16 max_lookbehind; /* Longest lookbehind (characters) */ + pcre_uint16 top_bracket; /* Highest numbered group */ + pcre_uint16 top_backref; /* Highest numbered back reference */ + pcre_uint16 name_table_offset; /* Offset to name table that follows */ + pcre_uint16 name_entry_size; /* Size of any name items */ + pcre_uint16 name_count; /* Number of name items */ + pcre_uint16 ref_count; /* Reference count */ + pcre_uint16 dummy; /* To ensure size is a multiple of 8 */ + const pcre_uint8 *tables; /* Pointer to tables or NULL for std */ + void *nullpad; /* NULL padding */ +} real_pcre32; + +#if defined COMPILE_PCRE8 +#define REAL_PCRE real_pcre +#elif defined COMPILE_PCRE16 +#define REAL_PCRE real_pcre16 +#elif defined COMPILE_PCRE32 +#define REAL_PCRE real_pcre32 +#endif + +/* Assert that the size of REAL_PCRE is divisible by 8 */ +typedef int __assert_real_pcre_size_divisible_8[(sizeof(REAL_PCRE) % 8) == 0 ? 1 : -1]; + +/* Needed in pcretest to access some fields in the real_pcre* structures + * directly. They're unified for 8/16/32 bits since the structs only differ + * after these fields; if that ever changes, need to fork those defines into + * 8/16 and 32 bit versions. */ +#define REAL_PCRE_MAGIC(re) (((REAL_PCRE*)re)->magic_number) +#define REAL_PCRE_SIZE(re) (((REAL_PCRE*)re)->size) +#define REAL_PCRE_OPTIONS(re) (((REAL_PCRE*)re)->options) +#define REAL_PCRE_FLAGS(re) (((REAL_PCRE*)re)->flags) + +/* The format of the block used to store data from pcre_study(). The same +remark (see NOTE above) about extending this structure applies. */ + +typedef struct pcre_study_data { + pcre_uint32 size; /* Total that was malloced */ + pcre_uint32 flags; /* Private flags */ + pcre_uint8 start_bits[32]; /* Starting char bits */ + pcre_uint32 minlength; /* Minimum subject length */ +} pcre_study_data; + +/* Structure for building a chain of open capturing subpatterns during +compiling, so that instructions to close them can be compiled when (*ACCEPT) is +encountered. This is also used to identify subpatterns that contain recursive +back references to themselves, so that they can be made atomic. */ + +typedef struct open_capitem { + struct open_capitem *next; /* Chain link */ + pcre_uint16 number; /* Capture number */ + pcre_uint16 flag; /* Set TRUE if recursive back ref */ +} open_capitem; + +/* Structure for building a list of named groups during the first pass of +compiling. */ + +typedef struct named_group { + const pcre_uchar *name; /* Points to the name in the pattern */ + int length; /* Length of the name */ + pcre_uint32 number; /* Group number */ +} named_group; + +/* Structure for passing "static" information around between the functions +doing the compiling, so that they are thread-safe. */ + +typedef struct compile_data { + const pcre_uint8 *lcc; /* Points to lower casing table */ + const pcre_uint8 *fcc; /* Points to case-flipping table */ + const pcre_uint8 *cbits; /* Points to character type table */ + const pcre_uint8 *ctypes; /* Points to table of type maps */ + const pcre_uchar *start_workspace;/* The start of working space */ + const pcre_uchar *start_code; /* The start of the compiled code */ + const pcre_uchar *start_pattern; /* The start of the pattern */ + const pcre_uchar *end_pattern; /* The end of the pattern */ + pcre_uchar *hwm; /* High watermark of workspace */ + open_capitem *open_caps; /* Chain of open capture items */ + named_group *named_groups; /* Points to vector in pre-compile */ + pcre_uchar *name_table; /* The name/number table */ + int names_found; /* Number of entries so far */ + int name_entry_size; /* Size of each entry */ + int named_group_list_size; /* Number of entries in the list */ + int workspace_size; /* Size of workspace */ + unsigned int bracount; /* Count of capturing parens as we compile */ + int final_bracount; /* Saved value after first pass */ + int max_lookbehind; /* Maximum lookbehind (characters) */ + int top_backref; /* Maximum back reference */ + unsigned int backref_map; /* Bitmap of low back refs */ + unsigned int namedrefcount; /* Number of backreferences by name */ + int parens_depth; /* Depth of nested parentheses */ + int assert_depth; /* Depth of nested assertions */ + pcre_uint32 external_options; /* External (initial) options */ + pcre_uint32 external_flags; /* External flag bits to be set */ + int req_varyopt; /* "After variable item" flag for reqbyte */ + BOOL had_accept; /* (*ACCEPT) encountered */ + BOOL had_pruneorskip; /* (*PRUNE) or (*SKIP) encountered */ + BOOL check_lookbehind; /* Lookbehinds need later checking */ + BOOL dupnames; /* Duplicate names exist */ + BOOL dupgroups; /* Duplicate groups exist: (?| found */ + BOOL iscondassert; /* Next assert is a condition */ + int nltype; /* Newline type */ + int nllen; /* Newline string length */ + pcre_uchar nl[4]; /* Newline string when fixed length */ +} compile_data; + +/* Structure for maintaining a chain of pointers to the currently incomplete +branches, for testing for left recursion while compiling. */ + +typedef struct branch_chain { + struct branch_chain *outer; + pcre_uchar *current_branch; +} branch_chain; + +/* Structure for mutual recursion detection. */ + +typedef struct recurse_check { + struct recurse_check *prev; + const pcre_uchar *group; +} recurse_check; + +/* Structure for items in a linked list that represents an explicit recursive +call within the pattern; used by pcre_exec(). */ + +typedef struct recursion_info { + struct recursion_info *prevrec; /* Previous recursion record (or NULL) */ + unsigned int group_num; /* Number of group that was called */ + int *offset_save; /* Pointer to start of saved offsets */ + int saved_max; /* Number of saved offsets */ + int saved_capture_last; /* Last capture number */ + PCRE_PUCHAR subject_position; /* Position at start of recursion */ +} recursion_info; + +/* A similar structure for pcre_dfa_exec(). */ + +typedef struct dfa_recursion_info { + struct dfa_recursion_info *prevrec; + int group_num; + PCRE_PUCHAR subject_position; +} dfa_recursion_info; + +/* Structure for building a chain of data for holding the values of the subject +pointer at the start of each subpattern, so as to detect when an empty string +has been matched by a subpattern - to break infinite loops; used by +pcre_exec(). */ + +typedef struct eptrblock { + struct eptrblock *epb_prev; + PCRE_PUCHAR epb_saved_eptr; +} eptrblock; + + +/* Structure for passing "static" information around between the functions +doing traditional NFA matching, so that they are thread-safe. */ + +typedef struct match_data { + unsigned long int match_call_count; /* As it says */ + unsigned long int match_limit; /* As it says */ + unsigned long int match_limit_recursion; /* As it says */ + int *offset_vector; /* Offset vector */ + int offset_end; /* One past the end */ + int offset_max; /* The maximum usable for return data */ + int nltype; /* Newline type */ + int nllen; /* Newline string length */ + int name_count; /* Number of names in name table */ + int name_entry_size; /* Size of entry in names table */ + unsigned int skip_arg_count; /* For counting SKIP_ARGs */ + unsigned int ignore_skip_arg; /* For re-run when SKIP arg name not found */ + pcre_uchar *name_table; /* Table of names */ + pcre_uchar nl[4]; /* Newline string when fixed */ + const pcre_uint8 *lcc; /* Points to lower casing table */ + const pcre_uint8 *fcc; /* Points to case-flipping table */ + const pcre_uint8 *ctypes; /* Points to table of type maps */ + BOOL notbol; /* NOTBOL flag */ + BOOL noteol; /* NOTEOL flag */ + BOOL utf; /* UTF-8 / UTF-16 flag */ + BOOL jscript_compat; /* JAVASCRIPT_COMPAT flag */ + BOOL use_ucp; /* PCRE_UCP flag */ + BOOL endonly; /* Dollar not before final \n */ + BOOL notempty; /* Empty string match not wanted */ + BOOL notempty_atstart; /* Empty string match at start not wanted */ + BOOL hitend; /* Hit the end of the subject at some point */ + BOOL bsr_anycrlf; /* \R is just any CRLF, not full Unicode */ + BOOL hasthen; /* Pattern contains (*THEN) */ + const pcre_uchar *start_code; /* For use when recursing */ + PCRE_PUCHAR start_subject; /* Start of the subject string */ + PCRE_PUCHAR end_subject; /* End of the subject string */ + PCRE_PUCHAR start_match_ptr; /* Start of matched string */ + PCRE_PUCHAR end_match_ptr; /* Subject position at end match */ + PCRE_PUCHAR start_used_ptr; /* Earliest consulted character */ + int partial; /* PARTIAL options */ + int end_offset_top; /* Highwater mark at end of match */ + pcre_int32 capture_last; /* Most recent capture number + overflow flag */ + int start_offset; /* The start offset value */ + int match_function_type; /* Set for certain special calls of MATCH() */ + eptrblock *eptrchain; /* Chain of eptrblocks for tail recursions */ + int eptrn; /* Next free eptrblock */ + recursion_info *recursive; /* Linked list of recursion data */ + void *callout_data; /* To pass back to callouts */ + const pcre_uchar *mark; /* Mark pointer to pass back on success */ + const pcre_uchar *nomatch_mark;/* Mark pointer to pass back on failure */ + const pcre_uchar *once_target; /* Where to back up to for atomic groups */ +#ifdef NO_RECURSE + void *match_frames_base; /* For remembering malloc'd frames */ +#endif +} match_data; + +/* A similar structure is used for the same purpose by the DFA matching +functions. */ + +typedef struct dfa_match_data { + const pcre_uchar *start_code; /* Start of the compiled pattern */ + const pcre_uchar *start_subject ; /* Start of the subject string */ + const pcre_uchar *end_subject; /* End of subject string */ + const pcre_uchar *start_used_ptr; /* Earliest consulted character */ + const pcre_uint8 *tables; /* Character tables */ + int start_offset; /* The start offset value */ + int moptions; /* Match options */ + int poptions; /* Pattern options */ + int nltype; /* Newline type */ + int nllen; /* Newline string length */ + pcre_uchar nl[4]; /* Newline string when fixed */ + void *callout_data; /* To pass back to callouts */ + dfa_recursion_info *recursive; /* Linked list of recursion data */ +} dfa_match_data; + +/* Bit definitions for entries in the pcre_ctypes table. */ + +#define ctype_space 0x01 +#define ctype_letter 0x02 +#define ctype_digit 0x04 +#define ctype_xdigit 0x08 +#define ctype_word 0x10 /* alphanumeric or '_' */ +#define ctype_meta 0x80 /* regexp meta char or zero (end pattern) */ + +/* Offsets for the bitmap tables in pcre_cbits. Each table contains a set +of bits for a class map. Some classes are built by combining these tables. */ + +#define cbit_space 0 /* [:space:] or \s */ +#define cbit_xdigit 32 /* [:xdigit:] */ +#define cbit_digit 64 /* [:digit:] or \d */ +#define cbit_upper 96 /* [:upper:] */ +#define cbit_lower 128 /* [:lower:] */ +#define cbit_word 160 /* [:word:] or \w */ +#define cbit_graph 192 /* [:graph:] */ +#define cbit_print 224 /* [:print:] */ +#define cbit_punct 256 /* [:punct:] */ +#define cbit_cntrl 288 /* [:cntrl:] */ +#define cbit_length 320 /* Length of the cbits table */ + +/* Offsets of the various tables from the base tables pointer, and +total length. */ + +#define lcc_offset 0 +#define fcc_offset 256 +#define cbits_offset 512 +#define ctypes_offset (cbits_offset + cbit_length) +#define tables_length (ctypes_offset + 256) + +/* Internal function and data prefixes. */ + +#if defined COMPILE_PCRE8 +#ifndef PUBL +#define PUBL(name) pcre_##name +#endif +#ifndef PRIV +#define PRIV(name) _pcre_##name +#endif +#elif defined COMPILE_PCRE16 +#ifndef PUBL +#define PUBL(name) pcre16_##name +#endif +#ifndef PRIV +#define PRIV(name) _pcre16_##name +#endif +#elif defined COMPILE_PCRE32 +#ifndef PUBL +#define PUBL(name) pcre32_##name +#endif +#ifndef PRIV +#define PRIV(name) _pcre32_##name +#endif +#else +#error Unsupported compiling mode +#endif /* COMPILE_PCRE[8|16|32] */ + +/* Layout of the UCP type table that translates property names into types and +codes. Each entry used to point directly to a name, but to reduce the number of +relocations in shared libraries, it now has an offset into a single string +instead. */ + +typedef struct { + pcre_uint16 name_offset; + pcre_uint16 type; + pcre_uint16 value; +} ucp_type_table; + + +/* Internal shared data tables. These are tables that are used by more than one +of the exported public functions. They have to be "external" in the C sense, +but are not part of the PCRE public API. The data for these tables is in the +pcre_tables.c module. */ + +#ifdef COMPILE_PCRE8 +extern const int PRIV(utf8_table1)[]; +extern const int PRIV(utf8_table1_size); +extern const int PRIV(utf8_table2)[]; +extern const int PRIV(utf8_table3)[]; +extern const pcre_uint8 PRIV(utf8_table4)[]; +#endif /* COMPILE_PCRE8 */ + +extern const char PRIV(utt_names)[]; +extern const ucp_type_table PRIV(utt)[]; +extern const int PRIV(utt_size); + +extern const pcre_uint8 PRIV(OP_lengths)[]; +extern const pcre_uint8 PRIV(default_tables)[]; + +extern const pcre_uint32 PRIV(hspace_list)[]; +extern const pcre_uint32 PRIV(vspace_list)[]; + + +/* Internal shared functions. These are functions that are used by more than +one of the exported public functions. They have to be "external" in the C +sense, but are not part of the PCRE public API. */ + +/* String comparison functions. */ +#if defined COMPILE_PCRE8 + +#define STRCMP_UC_UC(str1, str2) \ + strcmp((char *)(str1), (char *)(str2)) +#define STRCMP_UC_C8(str1, str2) \ + strcmp((char *)(str1), (str2)) +#define STRNCMP_UC_UC(str1, str2, num) \ + strncmp((char *)(str1), (char *)(str2), (num)) +#define STRNCMP_UC_C8(str1, str2, num) \ + strncmp((char *)(str1), (str2), (num)) +#define STRLEN_UC(str) strlen((const char *)str) + +#elif defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + +extern int PRIV(strcmp_uc_uc)(const pcre_uchar *, + const pcre_uchar *); +extern int PRIV(strcmp_uc_c8)(const pcre_uchar *, + const char *); +extern int PRIV(strncmp_uc_uc)(const pcre_uchar *, + const pcre_uchar *, unsigned int num); +extern int PRIV(strncmp_uc_c8)(const pcre_uchar *, + const char *, unsigned int num); +extern unsigned int PRIV(strlen_uc)(const pcre_uchar *str); + +#define STRCMP_UC_UC(str1, str2) \ + PRIV(strcmp_uc_uc)((str1), (str2)) +#define STRCMP_UC_C8(str1, str2) \ + PRIV(strcmp_uc_c8)((str1), (str2)) +#define STRNCMP_UC_UC(str1, str2, num) \ + PRIV(strncmp_uc_uc)((str1), (str2), (num)) +#define STRNCMP_UC_C8(str1, str2, num) \ + PRIV(strncmp_uc_c8)((str1), (str2), (num)) +#define STRLEN_UC(str) PRIV(strlen_uc)(str) + +#endif /* COMPILE_PCRE[8|16|32] */ + +#if defined COMPILE_PCRE8 || defined COMPILE_PCRE16 + +#define STRCMP_UC_UC_TEST(str1, str2) STRCMP_UC_UC(str1, str2) +#define STRCMP_UC_C8_TEST(str1, str2) STRCMP_UC_C8(str1, str2) + +#elif defined COMPILE_PCRE32 + +extern int PRIV(strcmp_uc_uc_utf)(const pcre_uchar *, + const pcre_uchar *); +extern int PRIV(strcmp_uc_c8_utf)(const pcre_uchar *, + const char *); + +#define STRCMP_UC_UC_TEST(str1, str2) \ + (utf ? PRIV(strcmp_uc_uc_utf)((str1), (str2)) : PRIV(strcmp_uc_uc)((str1), (str2))) +#define STRCMP_UC_C8_TEST(str1, str2) \ + (utf ? PRIV(strcmp_uc_c8_utf)((str1), (str2)) : PRIV(strcmp_uc_c8)((str1), (str2))) + +#endif /* COMPILE_PCRE[8|16|32] */ + +extern const pcre_uchar *PRIV(find_bracket)(const pcre_uchar *, BOOL, int); +extern BOOL PRIV(is_newline)(PCRE_PUCHAR, int, PCRE_PUCHAR, + int *, BOOL); +extern unsigned int PRIV(ord2utf)(pcre_uint32, pcre_uchar *); +extern int PRIV(valid_utf)(PCRE_PUCHAR, int, int *); +extern BOOL PRIV(was_newline)(PCRE_PUCHAR, int, PCRE_PUCHAR, + int *, BOOL); +extern BOOL PRIV(xclass)(pcre_uint32, const pcre_uchar *, BOOL); + +#ifdef SUPPORT_JIT +extern void PRIV(jit_compile)(const REAL_PCRE *, + PUBL(extra) *, int); +extern int PRIV(jit_exec)(const PUBL(extra) *, + const pcre_uchar *, int, int, int, int *, int); +extern void PRIV(jit_free)(void *); +extern int PRIV(jit_get_size)(void *); +extern const char* PRIV(jit_get_target)(void); +#endif + +/* Unicode character database (UCD) */ + +typedef struct { + pcre_uint8 script; /* ucp_Arabic, etc. */ + pcre_uint8 chartype; /* ucp_Cc, etc. (general categories) */ + pcre_uint8 gbprop; /* ucp_gbControl, etc. (grapheme break property) */ + pcre_uint8 caseset; /* offset to multichar other cases or zero */ + pcre_int32 other_case; /* offset to other case, or zero if none */ +} ucd_record; + +extern const pcre_uint32 PRIV(ucd_caseless_sets)[]; +extern const ucd_record PRIV(ucd_records)[]; +extern const pcre_uint8 PRIV(ucd_stage1)[]; +extern const pcre_uint16 PRIV(ucd_stage2)[]; +extern const pcre_uint32 PRIV(ucp_gentype)[]; +extern const pcre_uint32 PRIV(ucp_gbtable)[]; +#ifdef COMPILE_PCRE32 +extern const ucd_record PRIV(dummy_ucd_record)[]; +#endif +#ifdef SUPPORT_JIT +extern const int PRIV(ucp_typerange)[]; +#endif + +#ifdef SUPPORT_UCP +/* UCD access macros */ + +#define UCD_BLOCK_SIZE 128 +#define REAL_GET_UCD(ch) (PRIV(ucd_records) + \ + PRIV(ucd_stage2)[PRIV(ucd_stage1)[(int)(ch) / UCD_BLOCK_SIZE] * \ + UCD_BLOCK_SIZE + (int)(ch) % UCD_BLOCK_SIZE]) + +#ifdef COMPILE_PCRE32 +#define GET_UCD(ch) ((ch > 0x10ffff)? PRIV(dummy_ucd_record) : REAL_GET_UCD(ch)) +#else +#define GET_UCD(ch) REAL_GET_UCD(ch) +#endif + +#define UCD_CHARTYPE(ch) GET_UCD(ch)->chartype +#define UCD_SCRIPT(ch) GET_UCD(ch)->script +#define UCD_CATEGORY(ch) PRIV(ucp_gentype)[UCD_CHARTYPE(ch)] +#define UCD_GRAPHBREAK(ch) GET_UCD(ch)->gbprop +#define UCD_CASESET(ch) GET_UCD(ch)->caseset +#define UCD_OTHERCASE(ch) ((pcre_uint32)((int)ch + (int)(GET_UCD(ch)->other_case))) + +#endif /* SUPPORT_UCP */ + +#endif + +/* End of pcre_internal.h */ diff --git a/src/third_party/pcre-8.41/pcre_jit_compile.c b/src/third_party/pcre-8.41/pcre_jit_compile.c new file mode 100644 index 00000000000..249edbe8e7f --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_jit_compile.c @@ -0,0 +1,11744 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2013 University of Cambridge + + The machine code generator part (this module) was written by Zoltan Herczeg + Copyright (c) 2010-2013 + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "pcre_internal.h" + +#if defined SUPPORT_JIT + +/* All-in-one: Since we use the JIT compiler only from here, +we just include it. This way we don't need to touch the build +system files. */ + +#define SLJIT_MALLOC(size, allocator_data) (PUBL(malloc))(size) +#define SLJIT_FREE(ptr, allocator_data) (PUBL(free))(ptr) +#define SLJIT_CONFIG_AUTO 1 +#define SLJIT_CONFIG_STATIC 1 +#define SLJIT_VERBOSE 0 +#define SLJIT_DEBUG 0 + +#include "sljit/sljitLir.c" + +#if defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED +#error Unsupported architecture +#endif + +/* Defines for debugging purposes. */ + +/* 1 - Use unoptimized capturing brackets. + 2 - Enable capture_last_ptr (includes option 1). */ +/* #define DEBUG_FORCE_UNOPTIMIZED_CBRAS 2 */ + +/* 1 - Always have a control head. */ +/* #define DEBUG_FORCE_CONTROL_HEAD 1 */ + +/* Allocate memory for the regex stack on the real machine stack. +Fast, but limited size. */ +#define MACHINE_STACK_SIZE 32768 + +/* Growth rate for stack allocated by the OS. Should be the multiply +of page size. */ +#define STACK_GROWTH_RATE 8192 + +/* Enable to check that the allocation could destroy temporaries. */ +#if defined SLJIT_DEBUG && SLJIT_DEBUG +#define DESTROY_REGISTERS 1 +#endif + +/* +Short summary about the backtracking mechanism empolyed by the jit code generator: + +The code generator follows the recursive nature of the PERL compatible regular +expressions. The basic blocks of regular expressions are condition checkers +whose execute different commands depending on the result of the condition check. +The relationship between the operators can be horizontal (concatenation) and +vertical (sub-expression) (See struct backtrack_common for more details). + + 'ab' - 'a' and 'b' regexps are concatenated + 'a+' - 'a' is the sub-expression of the '+' operator + +The condition checkers are boolean (true/false) checkers. Machine code is generated +for the checker itself and for the actions depending on the result of the checker. +The 'true' case is called as the matching path (expected path), and the other is called as +the 'backtrack' path. Branch instructions are expesive for all CPUs, so we avoid taken +branches on the matching path. + + Greedy star operator (*) : + Matching path: match happens. + Backtrack path: match failed. + Non-greedy star operator (*?) : + Matching path: no need to perform a match. + Backtrack path: match is required. + +The following example shows how the code generated for a capturing bracket +with two alternatives. Let A, B, C, D are arbirary regular expressions, and +we have the following regular expression: + + A(B|C)D + +The generated code will be the following: + + A matching path + '(' matching path (pushing arguments to the stack) + B matching path + ')' matching path (pushing arguments to the stack) + D matching path + return with successful match + + D backtrack path + ')' backtrack path (If we arrived from "C" jump to the backtrack of "C") + B backtrack path + C expected path + jump to D matching path + C backtrack path + A backtrack path + + Notice, that the order of backtrack code paths are the opposite of the fast + code paths. In this way the topmost value on the stack is always belong + to the current backtrack code path. The backtrack path must check + whether there is a next alternative. If so, it needs to jump back to + the matching path eventually. Otherwise it needs to clear out its own stack + frame and continue the execution on the backtrack code paths. +*/ + +/* +Saved stack frames: + +Atomic blocks and asserts require reloading the values of private data +when the backtrack mechanism performed. Because of OP_RECURSE, the data +are not necessarly known in compile time, thus we need a dynamic restore +mechanism. + +The stack frames are stored in a chain list, and have the following format: +([ capturing bracket offset ][ start value ][ end value ])+ ... [ 0 ] [ previous head ] + +Thus we can restore the private data to a particular point in the stack. +*/ + +typedef struct jit_arguments { + /* Pointers first. */ + struct sljit_stack *stack; + const pcre_uchar *str; + const pcre_uchar *begin; + const pcre_uchar *end; + int *offsets; + pcre_uchar *uchar_ptr; + pcre_uchar *mark_ptr; + void *callout_data; + /* Everything else after. */ + sljit_u32 limit_match; + int real_offset_count; + int offset_count; + sljit_u8 notbol; + sljit_u8 noteol; + sljit_u8 notempty; + sljit_u8 notempty_atstart; +} jit_arguments; + +typedef struct executable_functions { + void *executable_funcs[JIT_NUMBER_OF_COMPILE_MODES]; + void *read_only_data_heads[JIT_NUMBER_OF_COMPILE_MODES]; + sljit_uw executable_sizes[JIT_NUMBER_OF_COMPILE_MODES]; + PUBL(jit_callback) callback; + void *userdata; + sljit_u32 top_bracket; + sljit_u32 limit_match; +} executable_functions; + +typedef struct jump_list { + struct sljit_jump *jump; + struct jump_list *next; +} jump_list; + +typedef struct stub_list { + struct sljit_jump *start; + struct sljit_label *quit; + struct stub_list *next; +} stub_list; + +typedef struct label_addr_list { + struct sljit_label *label; + sljit_uw *update_addr; + struct label_addr_list *next; +} label_addr_list; + +enum frame_types { + no_frame = -1, + no_stack = -2 +}; + +enum control_types { + type_mark = 0, + type_then_trap = 1 +}; + +typedef int (SLJIT_CALL *jit_function)(jit_arguments *args); + +/* The following structure is the key data type for the recursive +code generator. It is allocated by compile_matchingpath, and contains +the arguments for compile_backtrackingpath. Must be the first member +of its descendants. */ +typedef struct backtrack_common { + /* Concatenation stack. */ + struct backtrack_common *prev; + jump_list *nextbacktracks; + /* Internal stack (for component operators). */ + struct backtrack_common *top; + jump_list *topbacktracks; + /* Opcode pointer. */ + pcre_uchar *cc; +} backtrack_common; + +typedef struct assert_backtrack { + backtrack_common common; + jump_list *condfailed; + /* Less than 0 if a frame is not needed. */ + int framesize; + /* Points to our private memory word on the stack. */ + int private_data_ptr; + /* For iterators. */ + struct sljit_label *matchingpath; +} assert_backtrack; + +typedef struct bracket_backtrack { + backtrack_common common; + /* Where to coninue if an alternative is successfully matched. */ + struct sljit_label *alternative_matchingpath; + /* For rmin and rmax iterators. */ + struct sljit_label *recursive_matchingpath; + /* For greedy ? operator. */ + struct sljit_label *zero_matchingpath; + /* Contains the branches of a failed condition. */ + union { + /* Both for OP_COND, OP_SCOND. */ + jump_list *condfailed; + assert_backtrack *assert; + /* For OP_ONCE. Less than 0 if not needed. */ + int framesize; + } u; + /* Points to our private memory word on the stack. */ + int private_data_ptr; +} bracket_backtrack; + +typedef struct bracketpos_backtrack { + backtrack_common common; + /* Points to our private memory word on the stack. */ + int private_data_ptr; + /* Reverting stack is needed. */ + int framesize; + /* Allocated stack size. */ + int stacksize; +} bracketpos_backtrack; + +typedef struct braminzero_backtrack { + backtrack_common common; + struct sljit_label *matchingpath; +} braminzero_backtrack; + +typedef struct char_iterator_backtrack { + backtrack_common common; + /* Next iteration. */ + struct sljit_label *matchingpath; + union { + jump_list *backtracks; + struct { + unsigned int othercasebit; + pcre_uchar chr; + BOOL enabled; + } charpos; + } u; +} char_iterator_backtrack; + +typedef struct ref_iterator_backtrack { + backtrack_common common; + /* Next iteration. */ + struct sljit_label *matchingpath; +} ref_iterator_backtrack; + +typedef struct recurse_entry { + struct recurse_entry *next; + /* Contains the function entry. */ + struct sljit_label *entry; + /* Collects the calls until the function is not created. */ + jump_list *calls; + /* Points to the starting opcode. */ + sljit_sw start; +} recurse_entry; + +typedef struct recurse_backtrack { + backtrack_common common; + BOOL inlined_pattern; +} recurse_backtrack; + +#define OP_THEN_TRAP OP_TABLE_LENGTH + +typedef struct then_trap_backtrack { + backtrack_common common; + /* If then_trap is not NULL, this structure contains the real + then_trap for the backtracking path. */ + struct then_trap_backtrack *then_trap; + /* Points to the starting opcode. */ + sljit_sw start; + /* Exit point for the then opcodes of this alternative. */ + jump_list *quit; + /* Frame size of the current alternative. */ + int framesize; +} then_trap_backtrack; + +#define MAX_RANGE_SIZE 4 + +typedef struct compiler_common { + /* The sljit ceneric compiler. */ + struct sljit_compiler *compiler; + /* First byte code. */ + pcre_uchar *start; + /* Maps private data offset to each opcode. */ + sljit_s32 *private_data_ptrs; + /* Chain list of read-only data ptrs. */ + void *read_only_data_head; + /* Tells whether the capturing bracket is optimized. */ + sljit_u8 *optimized_cbracket; + /* Tells whether the starting offset is a target of then. */ + sljit_u8 *then_offsets; + /* Current position where a THEN must jump. */ + then_trap_backtrack *then_trap; + /* Starting offset of private data for capturing brackets. */ + sljit_s32 cbra_ptr; + /* Output vector starting point. Must be divisible by 2. */ + sljit_s32 ovector_start; + /* Points to the starting character of the current match. */ + sljit_s32 start_ptr; + /* Last known position of the requested byte. */ + sljit_s32 req_char_ptr; + /* Head of the last recursion. */ + sljit_s32 recursive_head_ptr; + /* First inspected character for partial matching. + (Needed for avoiding zero length partial matches.) */ + sljit_s32 start_used_ptr; + /* Starting pointer for partial soft matches. */ + sljit_s32 hit_start; + /* Pointer of the match end position. */ + sljit_s32 match_end_ptr; + /* Points to the marked string. */ + sljit_s32 mark_ptr; + /* Recursive control verb management chain. */ + sljit_s32 control_head_ptr; + /* Points to the last matched capture block index. */ + sljit_s32 capture_last_ptr; + /* Fast forward skipping byte code pointer. */ + pcre_uchar *fast_forward_bc_ptr; + /* Locals used by fast fail optimization. */ + sljit_s32 fast_fail_start_ptr; + sljit_s32 fast_fail_end_ptr; + + /* Flipped and lower case tables. */ + const sljit_u8 *fcc; + sljit_sw lcc; + /* Mode can be PCRE_STUDY_JIT_COMPILE and others. */ + int mode; + /* TRUE, when minlength is greater than 0. */ + BOOL might_be_empty; + /* \K is found in the pattern. */ + BOOL has_set_som; + /* (*SKIP:arg) is found in the pattern. */ + BOOL has_skip_arg; + /* (*THEN) is found in the pattern. */ + BOOL has_then; + /* (*SKIP) or (*SKIP:arg) is found in lookbehind assertion. */ + BOOL has_skip_in_assert_back; + /* Currently in recurse or negative assert. */ + BOOL local_exit; + /* Currently in a positive assert. */ + BOOL positive_assert; + /* Newline control. */ + int nltype; + sljit_u32 nlmax; + sljit_u32 nlmin; + int newline; + int bsr_nltype; + sljit_u32 bsr_nlmax; + sljit_u32 bsr_nlmin; + /* Dollar endonly. */ + int endonly; + /* Tables. */ + sljit_sw ctypes; + /* Named capturing brackets. */ + pcre_uchar *name_table; + sljit_sw name_count; + sljit_sw name_entry_size; + + /* Labels and jump lists. */ + struct sljit_label *partialmatchlabel; + struct sljit_label *quit_label; + struct sljit_label *forced_quit_label; + struct sljit_label *accept_label; + struct sljit_label *ff_newline_shortcut; + stub_list *stubs; + label_addr_list *label_addrs; + recurse_entry *entries; + recurse_entry *currententry; + jump_list *partialmatch; + jump_list *quit; + jump_list *positive_assert_quit; + jump_list *forced_quit; + jump_list *accept; + jump_list *calllimit; + jump_list *stackalloc; + jump_list *revertframes; + jump_list *wordboundary; + jump_list *anynewline; + jump_list *hspace; + jump_list *vspace; + jump_list *casefulcmp; + jump_list *caselesscmp; + jump_list *reset_match; + BOOL jscript_compat; +#ifdef SUPPORT_UTF + BOOL utf; +#ifdef SUPPORT_UCP + BOOL use_ucp; + jump_list *getucd; +#endif +#ifdef COMPILE_PCRE8 + jump_list *utfreadchar; + jump_list *utfreadchar16; + jump_list *utfreadtype8; +#endif +#endif /* SUPPORT_UTF */ +} compiler_common; + +/* For byte_sequence_compare. */ + +typedef struct compare_context { + int length; + int sourcereg; +#if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED + int ucharptr; + union { + sljit_s32 asint; + sljit_u16 asushort; +#if defined COMPILE_PCRE8 + sljit_u8 asbyte; + sljit_u8 asuchars[4]; +#elif defined COMPILE_PCRE16 + sljit_u16 asuchars[2]; +#elif defined COMPILE_PCRE32 + sljit_u32 asuchars[1]; +#endif + } c; + union { + sljit_s32 asint; + sljit_u16 asushort; +#if defined COMPILE_PCRE8 + sljit_u8 asbyte; + sljit_u8 asuchars[4]; +#elif defined COMPILE_PCRE16 + sljit_u16 asuchars[2]; +#elif defined COMPILE_PCRE32 + sljit_u32 asuchars[1]; +#endif + } oc; +#endif +} compare_context; + +/* Undefine sljit macros. */ +#undef CMP + +/* Used for accessing the elements of the stack. */ +#define STACK(i) ((i) * (int)sizeof(sljit_sw)) + +#define TMP1 SLJIT_R0 +#define TMP2 SLJIT_R2 +#define TMP3 SLJIT_R3 +#define STR_PTR SLJIT_S0 +#define STR_END SLJIT_S1 +#define STACK_TOP SLJIT_R1 +#define STACK_LIMIT SLJIT_S2 +#define COUNT_MATCH SLJIT_S3 +#define ARGUMENTS SLJIT_S4 +#define RETURN_ADDR SLJIT_R4 + +/* Local space layout. */ +/* These two locals can be used by the current opcode. */ +#define LOCALS0 (0 * sizeof(sljit_sw)) +#define LOCALS1 (1 * sizeof(sljit_sw)) +/* Two local variables for possessive quantifiers (char1 cannot use them). */ +#define POSSESSIVE0 (2 * sizeof(sljit_sw)) +#define POSSESSIVE1 (3 * sizeof(sljit_sw)) +/* Max limit of recursions. */ +#define LIMIT_MATCH (4 * sizeof(sljit_sw)) +/* The output vector is stored on the stack, and contains pointers +to characters. The vector data is divided into two groups: the first +group contains the start / end character pointers, and the second is +the start pointers when the end of the capturing group has not yet reached. */ +#define OVECTOR_START (common->ovector_start) +#define OVECTOR(i) (OVECTOR_START + (i) * (sljit_sw)sizeof(sljit_sw)) +#define OVECTOR_PRIV(i) (common->cbra_ptr + (i) * (sljit_sw)sizeof(sljit_sw)) +#define PRIVATE_DATA(cc) (common->private_data_ptrs[(cc) - common->start]) + +#if defined COMPILE_PCRE8 +#define MOV_UCHAR SLJIT_MOV_U8 +#define MOVU_UCHAR SLJIT_MOVU_U8 +#elif defined COMPILE_PCRE16 +#define MOV_UCHAR SLJIT_MOV_U16 +#define MOVU_UCHAR SLJIT_MOVU_U16 +#elif defined COMPILE_PCRE32 +#define MOV_UCHAR SLJIT_MOV_U32 +#define MOVU_UCHAR SLJIT_MOVU_U32 +#else +#error Unsupported compiling mode +#endif + +/* Shortcuts. */ +#define DEFINE_COMPILER \ + struct sljit_compiler *compiler = common->compiler +#define OP1(op, dst, dstw, src, srcw) \ + sljit_emit_op1(compiler, (op), (dst), (dstw), (src), (srcw)) +#define OP2(op, dst, dstw, src1, src1w, src2, src2w) \ + sljit_emit_op2(compiler, (op), (dst), (dstw), (src1), (src1w), (src2), (src2w)) +#define LABEL() \ + sljit_emit_label(compiler) +#define JUMP(type) \ + sljit_emit_jump(compiler, (type)) +#define JUMPTO(type, label) \ + sljit_set_label(sljit_emit_jump(compiler, (type)), (label)) +#define JUMPHERE(jump) \ + sljit_set_label((jump), sljit_emit_label(compiler)) +#define SET_LABEL(jump, label) \ + sljit_set_label((jump), (label)) +#define CMP(type, src1, src1w, src2, src2w) \ + sljit_emit_cmp(compiler, (type), (src1), (src1w), (src2), (src2w)) +#define CMPTO(type, src1, src1w, src2, src2w, label) \ + sljit_set_label(sljit_emit_cmp(compiler, (type), (src1), (src1w), (src2), (src2w)), (label)) +#define OP_FLAGS(op, dst, dstw, type) \ + sljit_emit_op_flags(compiler, (op), (dst), (dstw), (type)) +#define GET_LOCAL_BASE(dst, dstw, offset) \ + sljit_get_local_base(compiler, (dst), (dstw), (offset)) + +#define READ_CHAR_MAX 0x7fffffff + +#define INVALID_UTF_CHAR 888 + +static pcre_uchar *bracketend(pcre_uchar *cc) +{ +SLJIT_ASSERT((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) || (*cc >= OP_ONCE && *cc <= OP_SCOND)); +do cc += GET(cc, 1); while (*cc == OP_ALT); +SLJIT_ASSERT(*cc >= OP_KET && *cc <= OP_KETRPOS); +cc += 1 + LINK_SIZE; +return cc; +} + +static int no_alternatives(pcre_uchar *cc) +{ +int count = 0; +SLJIT_ASSERT((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) || (*cc >= OP_ONCE && *cc <= OP_SCOND)); +do + { + cc += GET(cc, 1); + count++; + } +while (*cc == OP_ALT); +SLJIT_ASSERT(*cc >= OP_KET && *cc <= OP_KETRPOS); +return count; +} + +/* Functions whose might need modification for all new supported opcodes: + next_opcode + check_opcode_types + set_private_data_ptrs + get_framesize + init_frame + get_private_data_copy_length + copy_private_data + compile_matchingpath + compile_backtrackingpath +*/ + +static pcre_uchar *next_opcode(compiler_common *common, pcre_uchar *cc) +{ +SLJIT_UNUSED_ARG(common); +switch(*cc) + { + case OP_SOD: + case OP_SOM: + case OP_SET_SOM: + case OP_NOT_WORD_BOUNDARY: + case OP_WORD_BOUNDARY: + case OP_NOT_DIGIT: + case OP_DIGIT: + case OP_NOT_WHITESPACE: + case OP_WHITESPACE: + case OP_NOT_WORDCHAR: + case OP_WORDCHAR: + case OP_ANY: + case OP_ALLANY: + case OP_NOTPROP: + case OP_PROP: + case OP_ANYNL: + case OP_NOT_HSPACE: + case OP_HSPACE: + case OP_NOT_VSPACE: + case OP_VSPACE: + case OP_EXTUNI: + case OP_EODN: + case OP_EOD: + case OP_CIRC: + case OP_CIRCM: + case OP_DOLL: + case OP_DOLLM: + case OP_CRSTAR: + case OP_CRMINSTAR: + case OP_CRPLUS: + case OP_CRMINPLUS: + case OP_CRQUERY: + case OP_CRMINQUERY: + case OP_CRRANGE: + case OP_CRMINRANGE: + case OP_CRPOSSTAR: + case OP_CRPOSPLUS: + case OP_CRPOSQUERY: + case OP_CRPOSRANGE: + case OP_CLASS: + case OP_NCLASS: + case OP_REF: + case OP_REFI: + case OP_DNREF: + case OP_DNREFI: + case OP_RECURSE: + case OP_CALLOUT: + case OP_ALT: + case OP_KET: + case OP_KETRMAX: + case OP_KETRMIN: + case OP_KETRPOS: + case OP_REVERSE: + case OP_ASSERT: + case OP_ASSERT_NOT: + case OP_ASSERTBACK: + case OP_ASSERTBACK_NOT: + case OP_ONCE: + case OP_ONCE_NC: + case OP_BRA: + case OP_BRAPOS: + case OP_CBRA: + case OP_CBRAPOS: + case OP_COND: + case OP_SBRA: + case OP_SBRAPOS: + case OP_SCBRA: + case OP_SCBRAPOS: + case OP_SCOND: + case OP_CREF: + case OP_DNCREF: + case OP_RREF: + case OP_DNRREF: + case OP_DEF: + case OP_BRAZERO: + case OP_BRAMINZERO: + case OP_BRAPOSZERO: + case OP_PRUNE: + case OP_SKIP: + case OP_THEN: + case OP_COMMIT: + case OP_FAIL: + case OP_ACCEPT: + case OP_ASSERT_ACCEPT: + case OP_CLOSE: + case OP_SKIPZERO: + return cc + PRIV(OP_lengths)[*cc]; + + case OP_CHAR: + case OP_CHARI: + case OP_NOT: + case OP_NOTI: + case OP_STAR: + case OP_MINSTAR: + case OP_PLUS: + case OP_MINPLUS: + case OP_QUERY: + case OP_MINQUERY: + case OP_UPTO: + case OP_MINUPTO: + case OP_EXACT: + case OP_POSSTAR: + case OP_POSPLUS: + case OP_POSQUERY: + case OP_POSUPTO: + case OP_STARI: + case OP_MINSTARI: + case OP_PLUSI: + case OP_MINPLUSI: + case OP_QUERYI: + case OP_MINQUERYI: + case OP_UPTOI: + case OP_MINUPTOI: + case OP_EXACTI: + case OP_POSSTARI: + case OP_POSPLUSI: + case OP_POSQUERYI: + case OP_POSUPTOI: + case OP_NOTSTAR: + case OP_NOTMINSTAR: + case OP_NOTPLUS: + case OP_NOTMINPLUS: + case OP_NOTQUERY: + case OP_NOTMINQUERY: + case OP_NOTUPTO: + case OP_NOTMINUPTO: + case OP_NOTEXACT: + case OP_NOTPOSSTAR: + case OP_NOTPOSPLUS: + case OP_NOTPOSQUERY: + case OP_NOTPOSUPTO: + case OP_NOTSTARI: + case OP_NOTMINSTARI: + case OP_NOTPLUSI: + case OP_NOTMINPLUSI: + case OP_NOTQUERYI: + case OP_NOTMINQUERYI: + case OP_NOTUPTOI: + case OP_NOTMINUPTOI: + case OP_NOTEXACTI: + case OP_NOTPOSSTARI: + case OP_NOTPOSPLUSI: + case OP_NOTPOSQUERYI: + case OP_NOTPOSUPTOI: + cc += PRIV(OP_lengths)[*cc]; +#ifdef SUPPORT_UTF + if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); +#endif + return cc; + + /* Special cases. */ + case OP_TYPESTAR: + case OP_TYPEMINSTAR: + case OP_TYPEPLUS: + case OP_TYPEMINPLUS: + case OP_TYPEQUERY: + case OP_TYPEMINQUERY: + case OP_TYPEUPTO: + case OP_TYPEMINUPTO: + case OP_TYPEEXACT: + case OP_TYPEPOSSTAR: + case OP_TYPEPOSPLUS: + case OP_TYPEPOSQUERY: + case OP_TYPEPOSUPTO: + return cc + PRIV(OP_lengths)[*cc] - 1; + + case OP_ANYBYTE: +#ifdef SUPPORT_UTF + if (common->utf) return NULL; +#endif + return cc + 1; + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: + return cc + GET(cc, 1); +#endif + + case OP_MARK: + case OP_PRUNE_ARG: + case OP_SKIP_ARG: + case OP_THEN_ARG: + return cc + 1 + 2 + cc[1]; + + default: + /* All opcodes are supported now! */ + SLJIT_UNREACHABLE(); + return NULL; + } +} + +static BOOL check_opcode_types(compiler_common *common, pcre_uchar *cc, pcre_uchar *ccend) +{ +int count; +pcre_uchar *slot; +pcre_uchar *assert_back_end = cc - 1; + +/* Calculate important variables (like stack size) and checks whether all opcodes are supported. */ +while (cc < ccend) + { + switch(*cc) + { + case OP_SET_SOM: + common->has_set_som = TRUE; + common->might_be_empty = TRUE; + cc += 1; + break; + + case OP_REF: + case OP_REFI: + common->optimized_cbracket[GET2(cc, 1)] = 0; + cc += 1 + IMM2_SIZE; + break; + + case OP_CBRAPOS: + case OP_SCBRAPOS: + common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] = 0; + cc += 1 + LINK_SIZE + IMM2_SIZE; + break; + + case OP_COND: + case OP_SCOND: + /* Only AUTO_CALLOUT can insert this opcode. We do + not intend to support this case. */ + if (cc[1 + LINK_SIZE] == OP_CALLOUT) + return FALSE; + cc += 1 + LINK_SIZE; + break; + + case OP_CREF: + common->optimized_cbracket[GET2(cc, 1)] = 0; + cc += 1 + IMM2_SIZE; + break; + + case OP_DNREF: + case OP_DNREFI: + case OP_DNCREF: + count = GET2(cc, 1 + IMM2_SIZE); + slot = common->name_table + GET2(cc, 1) * common->name_entry_size; + while (count-- > 0) + { + common->optimized_cbracket[GET2(slot, 0)] = 0; + slot += common->name_entry_size; + } + cc += 1 + 2 * IMM2_SIZE; + break; + + case OP_RECURSE: + /* Set its value only once. */ + if (common->recursive_head_ptr == 0) + { + common->recursive_head_ptr = common->ovector_start; + common->ovector_start += sizeof(sljit_sw); + } + cc += 1 + LINK_SIZE; + break; + + case OP_CALLOUT: + if (common->capture_last_ptr == 0) + { + common->capture_last_ptr = common->ovector_start; + common->ovector_start += sizeof(sljit_sw); + } + cc += 2 + 2 * LINK_SIZE; + break; + + case OP_ASSERTBACK: + slot = bracketend(cc); + if (slot > assert_back_end) + assert_back_end = slot; + cc += 1 + LINK_SIZE; + break; + + case OP_THEN_ARG: + common->has_then = TRUE; + common->control_head_ptr = 1; + /* Fall through. */ + + case OP_PRUNE_ARG: + case OP_MARK: + if (common->mark_ptr == 0) + { + common->mark_ptr = common->ovector_start; + common->ovector_start += sizeof(sljit_sw); + } + cc += 1 + 2 + cc[1]; + break; + + case OP_THEN: + common->has_then = TRUE; + common->control_head_ptr = 1; + cc += 1; + break; + + case OP_SKIP: + if (cc < assert_back_end) + common->has_skip_in_assert_back = TRUE; + cc += 1; + break; + + case OP_SKIP_ARG: + common->control_head_ptr = 1; + common->has_skip_arg = TRUE; + if (cc < assert_back_end) + common->has_skip_in_assert_back = TRUE; + cc += 1 + 2 + cc[1]; + break; + + default: + cc = next_opcode(common, cc); + if (cc == NULL) + return FALSE; + break; + } + } +return TRUE; +} + +static BOOL is_accelerated_repeat(pcre_uchar *cc) +{ +switch(*cc) + { + case OP_TYPESTAR: + case OP_TYPEMINSTAR: + case OP_TYPEPLUS: + case OP_TYPEMINPLUS: + case OP_TYPEPOSSTAR: + case OP_TYPEPOSPLUS: + return (cc[1] != OP_ANYNL && cc[1] != OP_EXTUNI); + + case OP_STAR: + case OP_MINSTAR: + case OP_PLUS: + case OP_MINPLUS: + case OP_POSSTAR: + case OP_POSPLUS: + + case OP_STARI: + case OP_MINSTARI: + case OP_PLUSI: + case OP_MINPLUSI: + case OP_POSSTARI: + case OP_POSPLUSI: + + case OP_NOTSTAR: + case OP_NOTMINSTAR: + case OP_NOTPLUS: + case OP_NOTMINPLUS: + case OP_NOTPOSSTAR: + case OP_NOTPOSPLUS: + + case OP_NOTSTARI: + case OP_NOTMINSTARI: + case OP_NOTPLUSI: + case OP_NOTMINPLUSI: + case OP_NOTPOSSTARI: + case OP_NOTPOSPLUSI: + return TRUE; + + case OP_CLASS: + case OP_NCLASS: +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: + cc += (*cc == OP_XCLASS) ? GET(cc, 1) : (int)(1 + (32 / sizeof(pcre_uchar))); +#else + cc += (1 + (32 / sizeof(pcre_uchar))); +#endif + + switch(*cc) + { + case OP_CRSTAR: + case OP_CRMINSTAR: + case OP_CRPLUS: + case OP_CRMINPLUS: + case OP_CRPOSSTAR: + case OP_CRPOSPLUS: + return TRUE; + } + break; + } +return FALSE; +} + +static SLJIT_INLINE BOOL detect_fast_forward_skip(compiler_common *common, int *private_data_start) +{ +pcre_uchar *cc = common->start; +pcre_uchar *end; + +/* Skip not repeated brackets. */ +while (TRUE) + { + switch(*cc) + { + case OP_SOD: + case OP_SOM: + case OP_SET_SOM: + case OP_NOT_WORD_BOUNDARY: + case OP_WORD_BOUNDARY: + case OP_EODN: + case OP_EOD: + case OP_CIRC: + case OP_CIRCM: + case OP_DOLL: + case OP_DOLLM: + /* Zero width assertions. */ + cc++; + continue; + } + + if (*cc != OP_BRA && *cc != OP_CBRA) + break; + + end = cc + GET(cc, 1); + if (*end != OP_KET || PRIVATE_DATA(end) != 0) + return FALSE; + if (*cc == OP_CBRA) + { + if (common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0) + return FALSE; + cc += IMM2_SIZE; + } + cc += 1 + LINK_SIZE; + } + +if (is_accelerated_repeat(cc)) + { + common->fast_forward_bc_ptr = cc; + common->private_data_ptrs[(cc + 1) - common->start] = *private_data_start; + *private_data_start += sizeof(sljit_sw); + return TRUE; + } +return FALSE; +} + +static SLJIT_INLINE void detect_fast_fail(compiler_common *common, pcre_uchar *cc, int *private_data_start, sljit_s32 depth) +{ + pcre_uchar *next_alt; + + SLJIT_ASSERT(*cc == OP_BRA || *cc == OP_CBRA); + + if (*cc == OP_CBRA && common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0) + return; + + next_alt = bracketend(cc) - (1 + LINK_SIZE); + if (*next_alt != OP_KET || PRIVATE_DATA(next_alt) != 0) + return; + + do + { + next_alt = cc + GET(cc, 1); + + cc += 1 + LINK_SIZE + ((*cc == OP_CBRA) ? IMM2_SIZE : 0); + + while (TRUE) + { + switch(*cc) + { + case OP_SOD: + case OP_SOM: + case OP_SET_SOM: + case OP_NOT_WORD_BOUNDARY: + case OP_WORD_BOUNDARY: + case OP_EODN: + case OP_EOD: + case OP_CIRC: + case OP_CIRCM: + case OP_DOLL: + case OP_DOLLM: + /* Zero width assertions. */ + cc++; + continue; + } + break; + } + + if (depth > 0 && (*cc == OP_BRA || *cc == OP_CBRA)) + detect_fast_fail(common, cc, private_data_start, depth - 1); + + if (is_accelerated_repeat(cc)) + { + common->private_data_ptrs[(cc + 1) - common->start] = *private_data_start; + + if (common->fast_fail_start_ptr == 0) + common->fast_fail_start_ptr = *private_data_start; + + *private_data_start += sizeof(sljit_sw); + common->fast_fail_end_ptr = *private_data_start; + + if (*private_data_start > SLJIT_MAX_LOCAL_SIZE) + return; + } + + cc = next_alt; + } + while (*cc == OP_ALT); +} + +static int get_class_iterator_size(pcre_uchar *cc) +{ +sljit_u32 min; +sljit_u32 max; +switch(*cc) + { + case OP_CRSTAR: + case OP_CRPLUS: + return 2; + + case OP_CRMINSTAR: + case OP_CRMINPLUS: + case OP_CRQUERY: + case OP_CRMINQUERY: + return 1; + + case OP_CRRANGE: + case OP_CRMINRANGE: + min = GET2(cc, 1); + max = GET2(cc, 1 + IMM2_SIZE); + if (max == 0) + return (*cc == OP_CRRANGE) ? 2 : 1; + max -= min; + if (max > 2) + max = 2; + return max; + + default: + return 0; + } +} + +static BOOL detect_repeat(compiler_common *common, pcre_uchar *begin) +{ +pcre_uchar *end = bracketend(begin); +pcre_uchar *next; +pcre_uchar *next_end; +pcre_uchar *max_end; +pcre_uchar type; +sljit_sw length = end - begin; +int min, max, i; + +/* Detect fixed iterations first. */ +if (end[-(1 + LINK_SIZE)] != OP_KET) + return FALSE; + +/* Already detected repeat. */ +if (common->private_data_ptrs[end - common->start - LINK_SIZE] != 0) + return TRUE; + +next = end; +min = 1; +while (1) + { + if (*next != *begin) + break; + next_end = bracketend(next); + if (next_end - next != length || memcmp(begin, next, IN_UCHARS(length)) != 0) + break; + next = next_end; + min++; + } + +if (min == 2) + return FALSE; + +max = 0; +max_end = next; +if (*next == OP_BRAZERO || *next == OP_BRAMINZERO) + { + type = *next; + while (1) + { + if (next[0] != type || next[1] != OP_BRA || next[2 + LINK_SIZE] != *begin) + break; + next_end = bracketend(next + 2 + LINK_SIZE); + if (next_end - next != (length + 2 + LINK_SIZE) || memcmp(begin, next + 2 + LINK_SIZE, IN_UCHARS(length)) != 0) + break; + next = next_end; + max++; + } + + if (next[0] == type && next[1] == *begin && max >= 1) + { + next_end = bracketend(next + 1); + if (next_end - next == (length + 1) && memcmp(begin, next + 1, IN_UCHARS(length)) == 0) + { + for (i = 0; i < max; i++, next_end += 1 + LINK_SIZE) + if (*next_end != OP_KET) + break; + + if (i == max) + { + common->private_data_ptrs[max_end - common->start - LINK_SIZE] = next_end - max_end; + common->private_data_ptrs[max_end - common->start - LINK_SIZE + 1] = (type == OP_BRAZERO) ? OP_UPTO : OP_MINUPTO; + /* +2 the original and the last. */ + common->private_data_ptrs[max_end - common->start - LINK_SIZE + 2] = max + 2; + if (min == 1) + return TRUE; + min--; + max_end -= (1 + LINK_SIZE) + GET(max_end, -LINK_SIZE); + } + } + } + } + +if (min >= 3) + { + common->private_data_ptrs[end - common->start - LINK_SIZE] = max_end - end; + common->private_data_ptrs[end - common->start - LINK_SIZE + 1] = OP_EXACT; + common->private_data_ptrs[end - common->start - LINK_SIZE + 2] = min; + return TRUE; + } + +return FALSE; +} + +#define CASE_ITERATOR_PRIVATE_DATA_1 \ + case OP_MINSTAR: \ + case OP_MINPLUS: \ + case OP_QUERY: \ + case OP_MINQUERY: \ + case OP_MINSTARI: \ + case OP_MINPLUSI: \ + case OP_QUERYI: \ + case OP_MINQUERYI: \ + case OP_NOTMINSTAR: \ + case OP_NOTMINPLUS: \ + case OP_NOTQUERY: \ + case OP_NOTMINQUERY: \ + case OP_NOTMINSTARI: \ + case OP_NOTMINPLUSI: \ + case OP_NOTQUERYI: \ + case OP_NOTMINQUERYI: + +#define CASE_ITERATOR_PRIVATE_DATA_2A \ + case OP_STAR: \ + case OP_PLUS: \ + case OP_STARI: \ + case OP_PLUSI: \ + case OP_NOTSTAR: \ + case OP_NOTPLUS: \ + case OP_NOTSTARI: \ + case OP_NOTPLUSI: + +#define CASE_ITERATOR_PRIVATE_DATA_2B \ + case OP_UPTO: \ + case OP_MINUPTO: \ + case OP_UPTOI: \ + case OP_MINUPTOI: \ + case OP_NOTUPTO: \ + case OP_NOTMINUPTO: \ + case OP_NOTUPTOI: \ + case OP_NOTMINUPTOI: + +#define CASE_ITERATOR_TYPE_PRIVATE_DATA_1 \ + case OP_TYPEMINSTAR: \ + case OP_TYPEMINPLUS: \ + case OP_TYPEQUERY: \ + case OP_TYPEMINQUERY: + +#define CASE_ITERATOR_TYPE_PRIVATE_DATA_2A \ + case OP_TYPESTAR: \ + case OP_TYPEPLUS: + +#define CASE_ITERATOR_TYPE_PRIVATE_DATA_2B \ + case OP_TYPEUPTO: \ + case OP_TYPEMINUPTO: + +static void set_private_data_ptrs(compiler_common *common, int *private_data_start, pcre_uchar *ccend) +{ +pcre_uchar *cc = common->start; +pcre_uchar *alternative; +pcre_uchar *end = NULL; +int private_data_ptr = *private_data_start; +int space, size, bracketlen; +BOOL repeat_check = TRUE; + +while (cc < ccend) + { + space = 0; + size = 0; + bracketlen = 0; + if (private_data_ptr > SLJIT_MAX_LOCAL_SIZE) + break; + + if (repeat_check && (*cc == OP_ONCE || *cc == OP_ONCE_NC || *cc == OP_BRA || *cc == OP_CBRA || *cc == OP_COND)) + { + if (detect_repeat(common, cc)) + { + /* These brackets are converted to repeats, so no global + based single character repeat is allowed. */ + if (cc >= end) + end = bracketend(cc); + } + } + repeat_check = TRUE; + + switch(*cc) + { + case OP_KET: + if (common->private_data_ptrs[cc + 1 - common->start] != 0) + { + common->private_data_ptrs[cc - common->start] = private_data_ptr; + private_data_ptr += sizeof(sljit_sw); + cc += common->private_data_ptrs[cc + 1 - common->start]; + } + cc += 1 + LINK_SIZE; + break; + + case OP_ASSERT: + case OP_ASSERT_NOT: + case OP_ASSERTBACK: + case OP_ASSERTBACK_NOT: + case OP_ONCE: + case OP_ONCE_NC: + case OP_BRAPOS: + case OP_SBRA: + case OP_SBRAPOS: + case OP_SCOND: + common->private_data_ptrs[cc - common->start] = private_data_ptr; + private_data_ptr += sizeof(sljit_sw); + bracketlen = 1 + LINK_SIZE; + break; + + case OP_CBRAPOS: + case OP_SCBRAPOS: + common->private_data_ptrs[cc - common->start] = private_data_ptr; + private_data_ptr += sizeof(sljit_sw); + bracketlen = 1 + LINK_SIZE + IMM2_SIZE; + break; + + case OP_COND: + /* Might be a hidden SCOND. */ + alternative = cc + GET(cc, 1); + if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN) + { + common->private_data_ptrs[cc - common->start] = private_data_ptr; + private_data_ptr += sizeof(sljit_sw); + } + bracketlen = 1 + LINK_SIZE; + break; + + case OP_BRA: + bracketlen = 1 + LINK_SIZE; + break; + + case OP_CBRA: + case OP_SCBRA: + bracketlen = 1 + LINK_SIZE + IMM2_SIZE; + break; + + case OP_BRAZERO: + case OP_BRAMINZERO: + case OP_BRAPOSZERO: + repeat_check = FALSE; + size = 1; + break; + + CASE_ITERATOR_PRIVATE_DATA_1 + space = 1; + size = -2; + break; + + CASE_ITERATOR_PRIVATE_DATA_2A + space = 2; + size = -2; + break; + + CASE_ITERATOR_PRIVATE_DATA_2B + space = 2; + size = -(2 + IMM2_SIZE); + break; + + CASE_ITERATOR_TYPE_PRIVATE_DATA_1 + space = 1; + size = 1; + break; + + CASE_ITERATOR_TYPE_PRIVATE_DATA_2A + if (cc[1] != OP_ANYNL && cc[1] != OP_EXTUNI) + space = 2; + size = 1; + break; + + case OP_TYPEUPTO: + if (cc[1 + IMM2_SIZE] != OP_ANYNL && cc[1 + IMM2_SIZE] != OP_EXTUNI) + space = 2; + size = 1 + IMM2_SIZE; + break; + + case OP_TYPEMINUPTO: + space = 2; + size = 1 + IMM2_SIZE; + break; + + case OP_CLASS: + case OP_NCLASS: + space = get_class_iterator_size(cc + size); + size = 1 + 32 / sizeof(pcre_uchar); + break; + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: + space = get_class_iterator_size(cc + size); + size = GET(cc, 1); + break; +#endif + + default: + cc = next_opcode(common, cc); + SLJIT_ASSERT(cc != NULL); + break; + } + + /* Character iterators, which are not inside a repeated bracket, + gets a private slot instead of allocating it on the stack. */ + if (space > 0 && cc >= end) + { + common->private_data_ptrs[cc - common->start] = private_data_ptr; + private_data_ptr += sizeof(sljit_sw) * space; + } + + if (size != 0) + { + if (size < 0) + { + cc += -size; +#ifdef SUPPORT_UTF + if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); +#endif + } + else + cc += size; + } + + if (bracketlen > 0) + { + if (cc >= end) + { + end = bracketend(cc); + if (end[-1 - LINK_SIZE] == OP_KET) + end = NULL; + } + cc += bracketlen; + } + } +*private_data_start = private_data_ptr; +} + +/* Returns with a frame_types (always < 0) if no need for frame. */ +static int get_framesize(compiler_common *common, pcre_uchar *cc, pcre_uchar *ccend, BOOL recursive, BOOL *needs_control_head) +{ +int length = 0; +int possessive = 0; +BOOL stack_restore = FALSE; +BOOL setsom_found = recursive; +BOOL setmark_found = recursive; +/* The last capture is a local variable even for recursions. */ +BOOL capture_last_found = FALSE; + +#if defined DEBUG_FORCE_CONTROL_HEAD && DEBUG_FORCE_CONTROL_HEAD +SLJIT_ASSERT(common->control_head_ptr != 0); +*needs_control_head = TRUE; +#else +*needs_control_head = FALSE; +#endif + +if (ccend == NULL) + { + ccend = bracketend(cc) - (1 + LINK_SIZE); + if (!recursive && (*cc == OP_CBRAPOS || *cc == OP_SCBRAPOS)) + { + possessive = length = (common->capture_last_ptr != 0) ? 5 : 3; + /* This is correct regardless of common->capture_last_ptr. */ + capture_last_found = TRUE; + } + cc = next_opcode(common, cc); + } + +SLJIT_ASSERT(cc != NULL); +while (cc < ccend) + switch(*cc) + { + case OP_SET_SOM: + SLJIT_ASSERT(common->has_set_som); + stack_restore = TRUE; + if (!setsom_found) + { + length += 2; + setsom_found = TRUE; + } + cc += 1; + break; + + case OP_MARK: + case OP_PRUNE_ARG: + case OP_THEN_ARG: + SLJIT_ASSERT(common->mark_ptr != 0); + stack_restore = TRUE; + if (!setmark_found) + { + length += 2; + setmark_found = TRUE; + } + if (common->control_head_ptr != 0) + *needs_control_head = TRUE; + cc += 1 + 2 + cc[1]; + break; + + case OP_RECURSE: + stack_restore = TRUE; + if (common->has_set_som && !setsom_found) + { + length += 2; + setsom_found = TRUE; + } + if (common->mark_ptr != 0 && !setmark_found) + { + length += 2; + setmark_found = TRUE; + } + if (common->capture_last_ptr != 0 && !capture_last_found) + { + length += 2; + capture_last_found = TRUE; + } + cc += 1 + LINK_SIZE; + break; + + case OP_CBRA: + case OP_CBRAPOS: + case OP_SCBRA: + case OP_SCBRAPOS: + stack_restore = TRUE; + if (common->capture_last_ptr != 0 && !capture_last_found) + { + length += 2; + capture_last_found = TRUE; + } + length += 3; + cc += 1 + LINK_SIZE + IMM2_SIZE; + break; + + case OP_THEN: + stack_restore = TRUE; + if (common->control_head_ptr != 0) + *needs_control_head = TRUE; + cc ++; + break; + + default: + stack_restore = TRUE; + /* Fall through. */ + + case OP_NOT_WORD_BOUNDARY: + case OP_WORD_BOUNDARY: + case OP_NOT_DIGIT: + case OP_DIGIT: + case OP_NOT_WHITESPACE: + case OP_WHITESPACE: + case OP_NOT_WORDCHAR: + case OP_WORDCHAR: + case OP_ANY: + case OP_ALLANY: + case OP_ANYBYTE: + case OP_NOTPROP: + case OP_PROP: + case OP_ANYNL: + case OP_NOT_HSPACE: + case OP_HSPACE: + case OP_NOT_VSPACE: + case OP_VSPACE: + case OP_EXTUNI: + case OP_EODN: + case OP_EOD: + case OP_CIRC: + case OP_CIRCM: + case OP_DOLL: + case OP_DOLLM: + case OP_CHAR: + case OP_CHARI: + case OP_NOT: + case OP_NOTI: + + case OP_EXACT: + case OP_POSSTAR: + case OP_POSPLUS: + case OP_POSQUERY: + case OP_POSUPTO: + + case OP_EXACTI: + case OP_POSSTARI: + case OP_POSPLUSI: + case OP_POSQUERYI: + case OP_POSUPTOI: + + case OP_NOTEXACT: + case OP_NOTPOSSTAR: + case OP_NOTPOSPLUS: + case OP_NOTPOSQUERY: + case OP_NOTPOSUPTO: + + case OP_NOTEXACTI: + case OP_NOTPOSSTARI: + case OP_NOTPOSPLUSI: + case OP_NOTPOSQUERYI: + case OP_NOTPOSUPTOI: + + case OP_TYPEEXACT: + case OP_TYPEPOSSTAR: + case OP_TYPEPOSPLUS: + case OP_TYPEPOSQUERY: + case OP_TYPEPOSUPTO: + + case OP_CLASS: + case OP_NCLASS: + case OP_XCLASS: + case OP_CALLOUT: + + cc = next_opcode(common, cc); + SLJIT_ASSERT(cc != NULL); + break; + } + +/* Possessive quantifiers can use a special case. */ +if (SLJIT_UNLIKELY(possessive == length)) + return stack_restore ? no_frame : no_stack; + +if (length > 0) + return length + 1; +return stack_restore ? no_frame : no_stack; +} + +static void init_frame(compiler_common *common, pcre_uchar *cc, pcre_uchar *ccend, int stackpos, int stacktop, BOOL recursive) +{ +DEFINE_COMPILER; +BOOL setsom_found = recursive; +BOOL setmark_found = recursive; +/* The last capture is a local variable even for recursions. */ +BOOL capture_last_found = FALSE; +int offset; + +/* >= 1 + shortest item size (2) */ +SLJIT_UNUSED_ARG(stacktop); +SLJIT_ASSERT(stackpos >= stacktop + 2); + +stackpos = STACK(stackpos); +if (ccend == NULL) + { + ccend = bracketend(cc) - (1 + LINK_SIZE); + if (recursive || (*cc != OP_CBRAPOS && *cc != OP_SCBRAPOS)) + cc = next_opcode(common, cc); + } + +SLJIT_ASSERT(cc != NULL); +while (cc < ccend) + switch(*cc) + { + case OP_SET_SOM: + SLJIT_ASSERT(common->has_set_som); + if (!setsom_found) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0)); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -OVECTOR(0)); + stackpos -= (int)sizeof(sljit_sw); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); + stackpos -= (int)sizeof(sljit_sw); + setsom_found = TRUE; + } + cc += 1; + break; + + case OP_MARK: + case OP_PRUNE_ARG: + case OP_THEN_ARG: + SLJIT_ASSERT(common->mark_ptr != 0); + if (!setmark_found) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->mark_ptr); + stackpos -= (int)sizeof(sljit_sw); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); + stackpos -= (int)sizeof(sljit_sw); + setmark_found = TRUE; + } + cc += 1 + 2 + cc[1]; + break; + + case OP_RECURSE: + if (common->has_set_som && !setsom_found) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0)); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -OVECTOR(0)); + stackpos -= (int)sizeof(sljit_sw); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); + stackpos -= (int)sizeof(sljit_sw); + setsom_found = TRUE; + } + if (common->mark_ptr != 0 && !setmark_found) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->mark_ptr); + stackpos -= (int)sizeof(sljit_sw); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); + stackpos -= (int)sizeof(sljit_sw); + setmark_found = TRUE; + } + if (common->capture_last_ptr != 0 && !capture_last_found) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->capture_last_ptr); + stackpos -= (int)sizeof(sljit_sw); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); + stackpos -= (int)sizeof(sljit_sw); + capture_last_found = TRUE; + } + cc += 1 + LINK_SIZE; + break; + + case OP_CBRA: + case OP_CBRAPOS: + case OP_SCBRA: + case OP_SCBRAPOS: + if (common->capture_last_ptr != 0 && !capture_last_found) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, -common->capture_last_ptr); + stackpos -= (int)sizeof(sljit_sw); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); + stackpos -= (int)sizeof(sljit_sw); + capture_last_found = TRUE; + } + offset = (GET2(cc, 1 + LINK_SIZE)) << 1; + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, OVECTOR(offset)); + stackpos -= (int)sizeof(sljit_sw); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset)); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP1, 0); + stackpos -= (int)sizeof(sljit_sw); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, TMP2, 0); + stackpos -= (int)sizeof(sljit_sw); + + cc += 1 + LINK_SIZE + IMM2_SIZE; + break; + + default: + cc = next_opcode(common, cc); + SLJIT_ASSERT(cc != NULL); + break; + } + +OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackpos, SLJIT_IMM, 0); +SLJIT_ASSERT(stackpos == STACK(stacktop)); +} + +static SLJIT_INLINE int get_private_data_copy_length(compiler_common *common, pcre_uchar *cc, pcre_uchar *ccend, BOOL needs_control_head) +{ +int private_data_length = needs_control_head ? 3 : 2; +int size; +pcre_uchar *alternative; +/* Calculate the sum of the private machine words. */ +while (cc < ccend) + { + size = 0; + switch(*cc) + { + case OP_KET: + if (PRIVATE_DATA(cc) != 0) + { + private_data_length++; + SLJIT_ASSERT(PRIVATE_DATA(cc + 1) != 0); + cc += PRIVATE_DATA(cc + 1); + } + cc += 1 + LINK_SIZE; + break; + + case OP_ASSERT: + case OP_ASSERT_NOT: + case OP_ASSERTBACK: + case OP_ASSERTBACK_NOT: + case OP_ONCE: + case OP_ONCE_NC: + case OP_BRAPOS: + case OP_SBRA: + case OP_SBRAPOS: + case OP_SCOND: + private_data_length++; + SLJIT_ASSERT(PRIVATE_DATA(cc) != 0); + cc += 1 + LINK_SIZE; + break; + + case OP_CBRA: + case OP_SCBRA: + if (common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0) + private_data_length++; + cc += 1 + LINK_SIZE + IMM2_SIZE; + break; + + case OP_CBRAPOS: + case OP_SCBRAPOS: + private_data_length += 2; + cc += 1 + LINK_SIZE + IMM2_SIZE; + break; + + case OP_COND: + /* Might be a hidden SCOND. */ + alternative = cc + GET(cc, 1); + if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN) + private_data_length++; + cc += 1 + LINK_SIZE; + break; + + CASE_ITERATOR_PRIVATE_DATA_1 + if (PRIVATE_DATA(cc)) + private_data_length++; + cc += 2; +#ifdef SUPPORT_UTF + if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); +#endif + break; + + CASE_ITERATOR_PRIVATE_DATA_2A + if (PRIVATE_DATA(cc)) + private_data_length += 2; + cc += 2; +#ifdef SUPPORT_UTF + if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); +#endif + break; + + CASE_ITERATOR_PRIVATE_DATA_2B + if (PRIVATE_DATA(cc)) + private_data_length += 2; + cc += 2 + IMM2_SIZE; +#ifdef SUPPORT_UTF + if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); +#endif + break; + + CASE_ITERATOR_TYPE_PRIVATE_DATA_1 + if (PRIVATE_DATA(cc)) + private_data_length++; + cc += 1; + break; + + CASE_ITERATOR_TYPE_PRIVATE_DATA_2A + if (PRIVATE_DATA(cc)) + private_data_length += 2; + cc += 1; + break; + + CASE_ITERATOR_TYPE_PRIVATE_DATA_2B + if (PRIVATE_DATA(cc)) + private_data_length += 2; + cc += 1 + IMM2_SIZE; + break; + + case OP_CLASS: + case OP_NCLASS: +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: + size = (*cc == OP_XCLASS) ? GET(cc, 1) : 1 + 32 / (int)sizeof(pcre_uchar); +#else + size = 1 + 32 / (int)sizeof(pcre_uchar); +#endif + if (PRIVATE_DATA(cc)) + private_data_length += get_class_iterator_size(cc + size); + cc += size; + break; + + default: + cc = next_opcode(common, cc); + SLJIT_ASSERT(cc != NULL); + break; + } + } +SLJIT_ASSERT(cc == ccend); +return private_data_length; +} + +static void copy_private_data(compiler_common *common, pcre_uchar *cc, pcre_uchar *ccend, + BOOL save, int stackptr, int stacktop, BOOL needs_control_head) +{ +DEFINE_COMPILER; +int srcw[2]; +int count, size; +BOOL tmp1next = TRUE; +BOOL tmp1empty = TRUE; +BOOL tmp2empty = TRUE; +pcre_uchar *alternative; +enum { + loop, + end +} status; + +status = loop; +stackptr = STACK(stackptr); +stacktop = STACK(stacktop - 1); + +if (!save) + { + stacktop -= (needs_control_head ? 2 : 1) * sizeof(sljit_sw); + if (stackptr < stacktop) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), stackptr); + stackptr += sizeof(sljit_sw); + tmp1empty = FALSE; + } + if (stackptr < stacktop) + { + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), stackptr); + stackptr += sizeof(sljit_sw); + tmp2empty = FALSE; + } + /* The tmp1next must be TRUE in either way. */ + } + +SLJIT_ASSERT(common->recursive_head_ptr != 0); + +do + { + count = 0; + if (cc >= ccend) + { + if (!save) + break; + + count = 1; + srcw[0] = common->recursive_head_ptr; + if (needs_control_head) + { + SLJIT_ASSERT(common->control_head_ptr != 0); + count = 2; + srcw[0] = common->control_head_ptr; + srcw[1] = common->recursive_head_ptr; + } + status = end; + } + else switch(*cc) + { + case OP_KET: + if (PRIVATE_DATA(cc) != 0) + { + count = 1; + srcw[0] = PRIVATE_DATA(cc); + SLJIT_ASSERT(PRIVATE_DATA(cc + 1) != 0); + cc += PRIVATE_DATA(cc + 1); + } + cc += 1 + LINK_SIZE; + break; + + case OP_ASSERT: + case OP_ASSERT_NOT: + case OP_ASSERTBACK: + case OP_ASSERTBACK_NOT: + case OP_ONCE: + case OP_ONCE_NC: + case OP_BRAPOS: + case OP_SBRA: + case OP_SBRAPOS: + case OP_SCOND: + count = 1; + srcw[0] = PRIVATE_DATA(cc); + SLJIT_ASSERT(srcw[0] != 0); + cc += 1 + LINK_SIZE; + break; + + case OP_CBRA: + case OP_SCBRA: + if (common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0) + { + count = 1; + srcw[0] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE)); + } + cc += 1 + LINK_SIZE + IMM2_SIZE; + break; + + case OP_CBRAPOS: + case OP_SCBRAPOS: + count = 2; + srcw[0] = PRIVATE_DATA(cc); + srcw[1] = OVECTOR_PRIV(GET2(cc, 1 + LINK_SIZE)); + SLJIT_ASSERT(srcw[0] != 0 && srcw[1] != 0); + cc += 1 + LINK_SIZE + IMM2_SIZE; + break; + + case OP_COND: + /* Might be a hidden SCOND. */ + alternative = cc + GET(cc, 1); + if (*alternative == OP_KETRMAX || *alternative == OP_KETRMIN) + { + count = 1; + srcw[0] = PRIVATE_DATA(cc); + SLJIT_ASSERT(srcw[0] != 0); + } + cc += 1 + LINK_SIZE; + break; + + CASE_ITERATOR_PRIVATE_DATA_1 + if (PRIVATE_DATA(cc)) + { + count = 1; + srcw[0] = PRIVATE_DATA(cc); + } + cc += 2; +#ifdef SUPPORT_UTF + if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); +#endif + break; + + CASE_ITERATOR_PRIVATE_DATA_2A + if (PRIVATE_DATA(cc)) + { + count = 2; + srcw[0] = PRIVATE_DATA(cc); + srcw[1] = PRIVATE_DATA(cc) + sizeof(sljit_sw); + } + cc += 2; +#ifdef SUPPORT_UTF + if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); +#endif + break; + + CASE_ITERATOR_PRIVATE_DATA_2B + if (PRIVATE_DATA(cc)) + { + count = 2; + srcw[0] = PRIVATE_DATA(cc); + srcw[1] = PRIVATE_DATA(cc) + sizeof(sljit_sw); + } + cc += 2 + IMM2_SIZE; +#ifdef SUPPORT_UTF + if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); +#endif + break; + + CASE_ITERATOR_TYPE_PRIVATE_DATA_1 + if (PRIVATE_DATA(cc)) + { + count = 1; + srcw[0] = PRIVATE_DATA(cc); + } + cc += 1; + break; + + CASE_ITERATOR_TYPE_PRIVATE_DATA_2A + if (PRIVATE_DATA(cc)) + { + count = 2; + srcw[0] = PRIVATE_DATA(cc); + srcw[1] = srcw[0] + sizeof(sljit_sw); + } + cc += 1; + break; + + CASE_ITERATOR_TYPE_PRIVATE_DATA_2B + if (PRIVATE_DATA(cc)) + { + count = 2; + srcw[0] = PRIVATE_DATA(cc); + srcw[1] = srcw[0] + sizeof(sljit_sw); + } + cc += 1 + IMM2_SIZE; + break; + + case OP_CLASS: + case OP_NCLASS: +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: + size = (*cc == OP_XCLASS) ? GET(cc, 1) : 1 + 32 / (int)sizeof(pcre_uchar); +#else + size = 1 + 32 / (int)sizeof(pcre_uchar); +#endif + if (PRIVATE_DATA(cc)) + switch(get_class_iterator_size(cc + size)) + { + case 1: + count = 1; + srcw[0] = PRIVATE_DATA(cc); + break; + + case 2: + count = 2; + srcw[0] = PRIVATE_DATA(cc); + srcw[1] = srcw[0] + sizeof(sljit_sw); + break; + + default: + SLJIT_UNREACHABLE(); + break; + } + cc += size; + break; + + default: + cc = next_opcode(common, cc); + SLJIT_ASSERT(cc != NULL); + break; + } + + while (count > 0) + { + count--; + if (save) + { + if (tmp1next) + { + if (!tmp1empty) + { + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP1, 0); + stackptr += sizeof(sljit_sw); + } + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), srcw[count]); + tmp1empty = FALSE; + tmp1next = FALSE; + } + else + { + if (!tmp2empty) + { + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP2, 0); + stackptr += sizeof(sljit_sw); + } + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), srcw[count]); + tmp2empty = FALSE; + tmp1next = TRUE; + } + } + else + { + if (tmp1next) + { + SLJIT_ASSERT(!tmp1empty); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), srcw[count], TMP1, 0); + tmp1empty = stackptr >= stacktop; + if (!tmp1empty) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), stackptr); + stackptr += sizeof(sljit_sw); + } + tmp1next = FALSE; + } + else + { + SLJIT_ASSERT(!tmp2empty); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), srcw[count], TMP2, 0); + tmp2empty = stackptr >= stacktop; + if (!tmp2empty) + { + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), stackptr); + stackptr += sizeof(sljit_sw); + } + tmp1next = TRUE; + } + } + } + } +while (status != end); + +if (save) + { + if (tmp1next) + { + if (!tmp1empty) + { + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP1, 0); + stackptr += sizeof(sljit_sw); + } + if (!tmp2empty) + { + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP2, 0); + stackptr += sizeof(sljit_sw); + } + } + else + { + if (!tmp2empty) + { + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP2, 0); + stackptr += sizeof(sljit_sw); + } + if (!tmp1empty) + { + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), stackptr, TMP1, 0); + stackptr += sizeof(sljit_sw); + } + } + } +SLJIT_ASSERT(cc == ccend && stackptr == stacktop && (save || (tmp1empty && tmp2empty))); +} + +static SLJIT_INLINE pcre_uchar *set_then_offsets(compiler_common *common, pcre_uchar *cc, sljit_u8 *current_offset) +{ +pcre_uchar *end = bracketend(cc); +BOOL has_alternatives = cc[GET(cc, 1)] == OP_ALT; + +/* Assert captures then. */ +if (*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) + current_offset = NULL; +/* Conditional block does not. */ +if (*cc == OP_COND || *cc == OP_SCOND) + has_alternatives = FALSE; + +cc = next_opcode(common, cc); +if (has_alternatives) + current_offset = common->then_offsets + (cc - common->start); + +while (cc < end) + { + if ((*cc >= OP_ASSERT && *cc <= OP_ASSERTBACK_NOT) || (*cc >= OP_ONCE && *cc <= OP_SCOND)) + cc = set_then_offsets(common, cc, current_offset); + else + { + if (*cc == OP_ALT && has_alternatives) + current_offset = common->then_offsets + (cc + 1 + LINK_SIZE - common->start); + if (*cc >= OP_THEN && *cc <= OP_THEN_ARG && current_offset != NULL) + *current_offset = 1; + cc = next_opcode(common, cc); + } + } + +return end; +} + +#undef CASE_ITERATOR_PRIVATE_DATA_1 +#undef CASE_ITERATOR_PRIVATE_DATA_2A +#undef CASE_ITERATOR_PRIVATE_DATA_2B +#undef CASE_ITERATOR_TYPE_PRIVATE_DATA_1 +#undef CASE_ITERATOR_TYPE_PRIVATE_DATA_2A +#undef CASE_ITERATOR_TYPE_PRIVATE_DATA_2B + +static SLJIT_INLINE BOOL is_powerof2(unsigned int value) +{ +return (value & (value - 1)) == 0; +} + +static SLJIT_INLINE void set_jumps(jump_list *list, struct sljit_label *label) +{ +while (list) + { + /* sljit_set_label is clever enough to do nothing + if either the jump or the label is NULL. */ + SET_LABEL(list->jump, label); + list = list->next; + } +} + +static SLJIT_INLINE void add_jump(struct sljit_compiler *compiler, jump_list **list, struct sljit_jump *jump) +{ +jump_list *list_item = sljit_alloc_memory(compiler, sizeof(jump_list)); +if (list_item) + { + list_item->next = *list; + list_item->jump = jump; + *list = list_item; + } +} + +static void add_stub(compiler_common *common, struct sljit_jump *start) +{ +DEFINE_COMPILER; +stub_list *list_item = sljit_alloc_memory(compiler, sizeof(stub_list)); + +if (list_item) + { + list_item->start = start; + list_item->quit = LABEL(); + list_item->next = common->stubs; + common->stubs = list_item; + } +} + +static void flush_stubs(compiler_common *common) +{ +DEFINE_COMPILER; +stub_list *list_item = common->stubs; + +while (list_item) + { + JUMPHERE(list_item->start); + add_jump(compiler, &common->stackalloc, JUMP(SLJIT_FAST_CALL)); + JUMPTO(SLJIT_JUMP, list_item->quit); + list_item = list_item->next; + } +common->stubs = NULL; +} + +static void add_label_addr(compiler_common *common, sljit_uw *update_addr) +{ +DEFINE_COMPILER; +label_addr_list *label_addr; + +label_addr = sljit_alloc_memory(compiler, sizeof(label_addr_list)); +if (label_addr == NULL) + return; +label_addr->label = LABEL(); +label_addr->update_addr = update_addr; +label_addr->next = common->label_addrs; +common->label_addrs = label_addr; +} + +static SLJIT_INLINE void count_match(compiler_common *common) +{ +DEFINE_COMPILER; + +OP2(SLJIT_SUB | SLJIT_SET_Z, COUNT_MATCH, 0, COUNT_MATCH, 0, SLJIT_IMM, 1); +add_jump(compiler, &common->calllimit, JUMP(SLJIT_ZERO)); +} + +static SLJIT_INLINE void allocate_stack(compiler_common *common, int size) +{ +/* May destroy all locals and registers except TMP2. */ +DEFINE_COMPILER; + +SLJIT_ASSERT(size > 0); +OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_sw)); +#ifdef DESTROY_REGISTERS +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 12345); +OP1(SLJIT_MOV, TMP3, 0, TMP1, 0); +OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0); +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, TMP1, 0); +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP1, 0); +#endif +add_stub(common, CMP(SLJIT_LESS, STACK_TOP, 0, STACK_LIMIT, 0)); +} + +static SLJIT_INLINE void free_stack(compiler_common *common, int size) +{ +DEFINE_COMPILER; + +SLJIT_ASSERT(size > 0); +OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_sw)); +} + +static sljit_uw * allocate_read_only_data(compiler_common *common, sljit_uw size) +{ +DEFINE_COMPILER; +sljit_uw *result; + +if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) + return NULL; + +result = (sljit_uw *)SLJIT_MALLOC(size + sizeof(sljit_uw), compiler->allocator_data); +if (SLJIT_UNLIKELY(result == NULL)) + { + sljit_set_compiler_memory_error(compiler); + return NULL; + } + +*(void**)result = common->read_only_data_head; +common->read_only_data_head = (void *)result; +return result + 1; +} + +static void free_read_only_data(void *current, void *allocator_data) +{ +void *next; + +SLJIT_UNUSED_ARG(allocator_data); + +while (current != NULL) + { + next = *(void**)current; + SLJIT_FREE(current, allocator_data); + current = next; + } +} + +static SLJIT_INLINE void reset_ovector(compiler_common *common, int length) +{ +DEFINE_COMPILER; +struct sljit_label *loop; +int i; + +/* At this point we can freely use all temporary registers. */ +SLJIT_ASSERT(length > 1); +/* TMP1 returns with begin - 1. */ +OP2(SLJIT_SUB, SLJIT_R0, 0, SLJIT_MEM1(SLJIT_S0), SLJIT_OFFSETOF(jit_arguments, begin), SLJIT_IMM, IN_UCHARS(1)); +if (length < 8) + { + for (i = 1; i < length; i++) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(i), SLJIT_R0, 0); + } +else + { + GET_LOCAL_BASE(SLJIT_R1, 0, OVECTOR_START); + OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_IMM, length - 1); + loop = LABEL(); + OP1(SLJIT_MOVU, SLJIT_MEM1(SLJIT_R1), sizeof(sljit_sw), SLJIT_R0, 0); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, 1); + JUMPTO(SLJIT_NOT_ZERO, loop); + } +} + +static SLJIT_INLINE void reset_fast_fail(compiler_common *common) +{ +DEFINE_COMPILER; +sljit_s32 i; + +SLJIT_ASSERT(common->fast_fail_start_ptr < common->fast_fail_end_ptr); + +OP2(SLJIT_SUB, TMP1, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +for (i = common->fast_fail_start_ptr; i < common->fast_fail_end_ptr; i += sizeof(sljit_sw)) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), i, TMP1, 0); +} + +static SLJIT_INLINE void do_reset_match(compiler_common *common, int length) +{ +DEFINE_COMPILER; +struct sljit_label *loop; +int i; + +SLJIT_ASSERT(length > 1); +/* OVECTOR(1) contains the "string begin - 1" constant. */ +if (length > 2) + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1)); +if (length < 8) + { + for (i = 2; i < length; i++) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(i), TMP1, 0); + } +else + { + GET_LOCAL_BASE(TMP2, 0, OVECTOR_START + sizeof(sljit_sw)); + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_IMM, length - 2); + loop = LABEL(); + OP1(SLJIT_MOVU, SLJIT_MEM1(TMP2), sizeof(sljit_sw), TMP1, 0); + OP2(SLJIT_SUB | SLJIT_SET_Z, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 1); + JUMPTO(SLJIT_NOT_ZERO, loop); + } + +OP1(SLJIT_MOV, STACK_TOP, 0, ARGUMENTS, 0); +if (common->mark_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, SLJIT_IMM, 0); +if (common->control_head_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0); +OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), SLJIT_OFFSETOF(jit_arguments, stack)); +OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->start_ptr); +OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), SLJIT_OFFSETOF(struct sljit_stack, base)); +} + +static sljit_sw SLJIT_CALL do_search_mark(sljit_sw *current, const pcre_uchar *skip_arg) +{ +while (current != NULL) + { + switch (current[1]) + { + case type_then_trap: + break; + + case type_mark: + if (STRCMP_UC_UC(skip_arg, (pcre_uchar *)current[2]) == 0) + return current[3]; + break; + + default: + SLJIT_UNREACHABLE(); + break; + } + SLJIT_ASSERT(current[0] == 0 || current < (sljit_sw*)current[0]); + current = (sljit_sw*)current[0]; + } +return -1; +} + +static SLJIT_INLINE void copy_ovector(compiler_common *common, int topbracket) +{ +DEFINE_COMPILER; +struct sljit_label *loop; +struct sljit_jump *early_quit; + +/* At this point we can freely use all registers. */ +OP1(SLJIT_MOV, SLJIT_S2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1)); +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(1), STR_PTR, 0); + +OP1(SLJIT_MOV, SLJIT_R0, 0, ARGUMENTS, 0); +if (common->mark_ptr != 0) + OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr); +OP1(SLJIT_MOV_S32, SLJIT_R1, 0, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, offset_count)); +if (common->mark_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, mark_ptr), SLJIT_R2, 0); +OP2(SLJIT_SUB, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, offsets), SLJIT_IMM, sizeof(int)); +OP1(SLJIT_MOV, SLJIT_R0, 0, SLJIT_MEM1(SLJIT_R0), SLJIT_OFFSETOF(jit_arguments, begin)); +GET_LOCAL_BASE(SLJIT_S0, 0, OVECTOR_START); +/* Unlikely, but possible */ +early_quit = CMP(SLJIT_EQUAL, SLJIT_R1, 0, SLJIT_IMM, 0); +loop = LABEL(); +OP2(SLJIT_SUB, SLJIT_S1, 0, SLJIT_MEM1(SLJIT_S0), 0, SLJIT_R0, 0); +OP2(SLJIT_ADD, SLJIT_S0, 0, SLJIT_S0, 0, SLJIT_IMM, sizeof(sljit_sw)); +/* Copy the integer value to the output buffer */ +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 +OP2(SLJIT_ASHR, SLJIT_S1, 0, SLJIT_S1, 0, SLJIT_IMM, UCHAR_SHIFT); +#endif +OP1(SLJIT_MOVU_S32, SLJIT_MEM1(SLJIT_R2), sizeof(int), SLJIT_S1, 0); +OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_R1, 0, SLJIT_R1, 0, SLJIT_IMM, 1); +JUMPTO(SLJIT_NOT_ZERO, loop); +JUMPHERE(early_quit); + +/* Calculate the return value, which is the maximum ovector value. */ +if (topbracket > 1) + { + GET_LOCAL_BASE(SLJIT_R0, 0, OVECTOR_START + topbracket * 2 * sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_R1, 0, SLJIT_IMM, topbracket + 1); + + /* OVECTOR(0) is never equal to SLJIT_S2. */ + loop = LABEL(); + OP1(SLJIT_MOVU, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_R0), -(2 * (sljit_sw)sizeof(sljit_sw))); + OP2(SLJIT_SUB, SLJIT_R1, 0, SLJIT_R1, 0, SLJIT_IMM, 1); + CMPTO(SLJIT_EQUAL, SLJIT_R2, 0, SLJIT_S2, 0, loop); + OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_R1, 0); + } +else + OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, 1); +} + +static SLJIT_INLINE void return_with_partial_match(compiler_common *common, struct sljit_label *quit) +{ +DEFINE_COMPILER; +struct sljit_jump *jump; + +SLJIT_COMPILE_ASSERT(STR_END == SLJIT_S1, str_end_must_be_saved_reg2); +SLJIT_ASSERT(common->start_used_ptr != 0 && common->start_ptr != 0 + && (common->mode == JIT_PARTIAL_SOFT_COMPILE ? common->hit_start != 0 : common->hit_start == 0)); + +OP1(SLJIT_MOV, SLJIT_R1, 0, ARGUMENTS, 0); +OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_PARTIAL); +OP1(SLJIT_MOV_S32, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_R1), SLJIT_OFFSETOF(jit_arguments, real_offset_count)); +CMPTO(SLJIT_SIG_LESS, SLJIT_R2, 0, SLJIT_IMM, 2, quit); + +/* Store match begin and end. */ +OP1(SLJIT_MOV, SLJIT_S0, 0, SLJIT_MEM1(SLJIT_R1), SLJIT_OFFSETOF(jit_arguments, begin)); +OP1(SLJIT_MOV, SLJIT_R1, 0, SLJIT_MEM1(SLJIT_R1), SLJIT_OFFSETOF(jit_arguments, offsets)); + +jump = CMP(SLJIT_SIG_LESS, SLJIT_R2, 0, SLJIT_IMM, 3); +OP2(SLJIT_SUB, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), common->mode == JIT_PARTIAL_HARD_COMPILE ? common->start_ptr : (common->hit_start + (int)sizeof(sljit_sw)), SLJIT_S0, 0); +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 +OP2(SLJIT_ASHR, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, UCHAR_SHIFT); +#endif +OP1(SLJIT_MOV_S32, SLJIT_MEM1(SLJIT_R1), 2 * sizeof(int), SLJIT_R2, 0); +JUMPHERE(jump); + +OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), common->mode == JIT_PARTIAL_HARD_COMPILE ? common->start_used_ptr : common->hit_start); +OP2(SLJIT_SUB, SLJIT_S1, 0, STR_END, 0, SLJIT_S0, 0); +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 +OP2(SLJIT_ASHR, SLJIT_S1, 0, SLJIT_S1, 0, SLJIT_IMM, UCHAR_SHIFT); +#endif +OP1(SLJIT_MOV_S32, SLJIT_MEM1(SLJIT_R1), sizeof(int), SLJIT_S1, 0); + +OP2(SLJIT_SUB, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_S0, 0); +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 +OP2(SLJIT_ASHR, SLJIT_R2, 0, SLJIT_R2, 0, SLJIT_IMM, UCHAR_SHIFT); +#endif +OP1(SLJIT_MOV_S32, SLJIT_MEM1(SLJIT_R1), 0, SLJIT_R2, 0); + +JUMPTO(SLJIT_JUMP, quit); +} + +static SLJIT_INLINE void check_start_used_ptr(compiler_common *common) +{ +/* May destroy TMP1. */ +DEFINE_COMPILER; +struct sljit_jump *jump; + +if (common->mode == JIT_PARTIAL_SOFT_COMPILE) + { + /* The value of -1 must be kept for start_used_ptr! */ + OP2(SLJIT_ADD, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, SLJIT_IMM, 1); + /* Jumps if start_used_ptr < STR_PTR, or start_used_ptr == -1. Although overwriting + is not necessary if start_used_ptr == STR_PTR, it does not hurt as well. */ + jump = CMP(SLJIT_LESS_EQUAL, TMP1, 0, STR_PTR, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0); + JUMPHERE(jump); + } +else if (common->mode == JIT_PARTIAL_HARD_COMPILE) + { + jump = CMP(SLJIT_LESS_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0); + JUMPHERE(jump); + } +} + +static SLJIT_INLINE BOOL char_has_othercase(compiler_common *common, pcre_uchar *cc) +{ +/* Detects if the character has an othercase. */ +unsigned int c; + +#ifdef SUPPORT_UTF +if (common->utf) + { + GETCHAR(c, cc); + if (c > 127) + { +#ifdef SUPPORT_UCP + return c != UCD_OTHERCASE(c); +#else + return FALSE; +#endif + } +#ifndef COMPILE_PCRE8 + return common->fcc[c] != c; +#endif + } +else +#endif + c = *cc; +return MAX_255(c) ? common->fcc[c] != c : FALSE; +} + +static SLJIT_INLINE unsigned int char_othercase(compiler_common *common, unsigned int c) +{ +/* Returns with the othercase. */ +#ifdef SUPPORT_UTF +if (common->utf && c > 127) + { +#ifdef SUPPORT_UCP + return UCD_OTHERCASE(c); +#else + return c; +#endif + } +#endif +return TABLE_GET(c, common->fcc, c); +} + +static unsigned int char_get_othercase_bit(compiler_common *common, pcre_uchar *cc) +{ +/* Detects if the character and its othercase has only 1 bit difference. */ +unsigned int c, oc, bit; +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 +int n; +#endif + +#ifdef SUPPORT_UTF +if (common->utf) + { + GETCHAR(c, cc); + if (c <= 127) + oc = common->fcc[c]; + else + { +#ifdef SUPPORT_UCP + oc = UCD_OTHERCASE(c); +#else + oc = c; +#endif + } + } +else + { + c = *cc; + oc = TABLE_GET(c, common->fcc, c); + } +#else +c = *cc; +oc = TABLE_GET(c, common->fcc, c); +#endif + +SLJIT_ASSERT(c != oc); + +bit = c ^ oc; +/* Optimized for English alphabet. */ +if (c <= 127 && bit == 0x20) + return (0 << 8) | 0x20; + +/* Since c != oc, they must have at least 1 bit difference. */ +if (!is_powerof2(bit)) + return 0; + +#if defined COMPILE_PCRE8 + +#ifdef SUPPORT_UTF +if (common->utf && c > 127) + { + n = GET_EXTRALEN(*cc); + while ((bit & 0x3f) == 0) + { + n--; + bit >>= 6; + } + return (n << 8) | bit; + } +#endif /* SUPPORT_UTF */ +return (0 << 8) | bit; + +#elif defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + +#ifdef SUPPORT_UTF +if (common->utf && c > 65535) + { + if (bit >= (1 << 10)) + bit >>= 10; + else + return (bit < 256) ? ((2 << 8) | bit) : ((3 << 8) | (bit >> 8)); + } +#endif /* SUPPORT_UTF */ +return (bit < 256) ? ((0 << 8) | bit) : ((1 << 8) | (bit >> 8)); + +#endif /* COMPILE_PCRE[8|16|32] */ +} + +static void check_partial(compiler_common *common, BOOL force) +{ +/* Checks whether a partial matching is occurred. Does not modify registers. */ +DEFINE_COMPILER; +struct sljit_jump *jump = NULL; + +SLJIT_ASSERT(!force || common->mode != JIT_COMPILE); + +if (common->mode == JIT_COMPILE) + return; + +if (!force) + jump = CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0); +else if (common->mode == JIT_PARTIAL_SOFT_COMPILE) + jump = CMP(SLJIT_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, SLJIT_IMM, -1); + +if (common->mode == JIT_PARTIAL_SOFT_COMPILE) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, 0); +else + { + if (common->partialmatchlabel != NULL) + JUMPTO(SLJIT_JUMP, common->partialmatchlabel); + else + add_jump(compiler, &common->partialmatch, JUMP(SLJIT_JUMP)); + } + +if (jump != NULL) + JUMPHERE(jump); +} + +static void check_str_end(compiler_common *common, jump_list **end_reached) +{ +/* Does not affect registers. Usually used in a tight spot. */ +DEFINE_COMPILER; +struct sljit_jump *jump; + +if (common->mode == JIT_COMPILE) + { + add_jump(compiler, end_reached, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); + return; + } + +jump = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0); +if (common->mode == JIT_PARTIAL_SOFT_COMPILE) + { + add_jump(compiler, end_reached, CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, 0); + add_jump(compiler, end_reached, JUMP(SLJIT_JUMP)); + } +else + { + add_jump(compiler, end_reached, CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0)); + if (common->partialmatchlabel != NULL) + JUMPTO(SLJIT_JUMP, common->partialmatchlabel); + else + add_jump(compiler, &common->partialmatch, JUMP(SLJIT_JUMP)); + } +JUMPHERE(jump); +} + +static void detect_partial_match(compiler_common *common, jump_list **backtracks) +{ +DEFINE_COMPILER; +struct sljit_jump *jump; + +if (common->mode == JIT_COMPILE) + { + add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); + return; + } + +/* Partial matching mode. */ +jump = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0); +add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0)); +if (common->mode == JIT_PARTIAL_SOFT_COMPILE) + { + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, 0); + add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); + } +else + { + if (common->partialmatchlabel != NULL) + JUMPTO(SLJIT_JUMP, common->partialmatchlabel); + else + add_jump(compiler, &common->partialmatch, JUMP(SLJIT_JUMP)); + } +JUMPHERE(jump); +} + +static void peek_char(compiler_common *common, sljit_u32 max) +{ +/* Reads the character into TMP1, keeps STR_PTR. +Does not check STR_END. TMP2 Destroyed. */ +DEFINE_COMPILER; +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 +struct sljit_jump *jump; +#endif + +SLJIT_UNUSED_ARG(max); + +OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 +if (common->utf) + { + if (max < 128) return; + + jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + add_jump(compiler, &common->utfreadchar, JUMP(SLJIT_FAST_CALL)); + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0); + JUMPHERE(jump); + } +#endif /* SUPPORT_UTF && !COMPILE_PCRE32 */ + +#if defined SUPPORT_UTF && defined COMPILE_PCRE16 +if (common->utf) + { + if (max < 0xd800) return; + + OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800); + jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xdc00 - 0xd800 - 1); + /* TMP2 contains the high surrogate. */ + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x40); + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 10); + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3ff); + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + JUMPHERE(jump); + } +#endif +} + +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 + +static BOOL is_char7_bitset(const sljit_u8 *bitset, BOOL nclass) +{ +/* Tells whether the character codes below 128 are enough +to determine a match. */ +const sljit_u8 value = nclass ? 0xff : 0; +const sljit_u8 *end = bitset + 32; + +bitset += 16; +do + { + if (*bitset++ != value) + return FALSE; + } +while (bitset < end); +return TRUE; +} + +static void read_char7_type(compiler_common *common, BOOL full_read) +{ +/* Reads the precise character type of a character into TMP1, if the character +is less than 128. Otherwise it returns with zero. Does not check STR_END. The +full_read argument tells whether characters above max are accepted or not. */ +DEFINE_COMPILER; +struct sljit_jump *jump; + +SLJIT_ASSERT(common->utf); + +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), 0); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + +OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes); + +if (full_read) + { + jump = CMP(SLJIT_LESS, TMP2, 0, SLJIT_IMM, 0xc0); + OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(utf8_table4) - 0xc0); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); + JUMPHERE(jump); + } +} + +#endif /* SUPPORT_UTF && COMPILE_PCRE8 */ + +static void read_char_range(compiler_common *common, sljit_u32 min, sljit_u32 max, BOOL update_str_ptr) +{ +/* Reads the precise value of a character into TMP1, if the character is +between min and max (c >= min && c <= max). Otherwise it returns with a value +outside the range. Does not check STR_END. */ +DEFINE_COMPILER; +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 +struct sljit_jump *jump; +#endif +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 +struct sljit_jump *jump2; +#endif + +SLJIT_UNUSED_ARG(update_str_ptr); +SLJIT_UNUSED_ARG(min); +SLJIT_UNUSED_ARG(max); +SLJIT_ASSERT(min <= max); + +OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 +if (common->utf) + { + if (max < 128 && !update_str_ptr) return; + + jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0); + if (min >= 0x10000) + { + OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xf0); + if (update_str_ptr) + OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + jump2 = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0x7); + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6); + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); + OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(2)); + if (!update_str_ptr) + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(3)); + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); + OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + JUMPHERE(jump2); + if (update_str_ptr) + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, RETURN_ADDR, 0); + } + else if (min >= 0x800 && max <= 0xffff) + { + OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xe0); + if (update_str_ptr) + OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + jump2 = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xf); + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6); + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); + if (!update_str_ptr) + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); + OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + JUMPHERE(jump2); + if (update_str_ptr) + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, RETURN_ADDR, 0); + } + else if (max >= 0x800) + add_jump(compiler, (max < 0x10000) ? &common->utfreadchar16 : &common->utfreadchar, JUMP(SLJIT_FAST_CALL)); + else if (max < 128) + { + OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); + } + else + { + OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + if (!update_str_ptr) + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + else + OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); + OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + if (update_str_ptr) + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, RETURN_ADDR, 0); + } + JUMPHERE(jump); + } +#endif + +#if defined SUPPORT_UTF && defined COMPILE_PCRE16 +if (common->utf) + { + if (max >= 0x10000) + { + OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800); + jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xdc00 - 0xd800 - 1); + /* TMP2 contains the high surrogate. */ + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x40); + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 10); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3ff); + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + JUMPHERE(jump); + return; + } + + if (max < 0xd800 && !update_str_ptr) return; + + /* Skip low surrogate if necessary. */ + OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, 0xd800); + jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xdc00 - 0xd800 - 1); + if (update_str_ptr) + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + if (max >= 0xd800) + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0x10000); + JUMPHERE(jump); + } +#endif +} + +static SLJIT_INLINE void read_char(compiler_common *common) +{ +read_char_range(common, 0, READ_CHAR_MAX, TRUE); +} + +static void read_char8_type(compiler_common *common, BOOL update_str_ptr) +{ +/* Reads the character type into TMP1, updates STR_PTR. Does not check STR_END. */ +DEFINE_COMPILER; +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 +struct sljit_jump *jump; +#endif +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 +struct sljit_jump *jump2; +#endif + +SLJIT_UNUSED_ARG(update_str_ptr); + +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), 0); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 +if (common->utf) + { + /* This can be an extra read in some situations, but hopefully + it is needed in most cases. */ + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes); + jump = CMP(SLJIT_LESS, TMP2, 0, SLJIT_IMM, 0xc0); + if (!update_str_ptr) + { + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6); + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); + OP2(SLJIT_OR, TMP2, 0, TMP2, 0, TMP1, 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); + jump2 = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 255); + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes); + JUMPHERE(jump2); + } + else + add_jump(compiler, &common->utfreadtype8, JUMP(SLJIT_FAST_CALL)); + JUMPHERE(jump); + return; + } +#endif /* SUPPORT_UTF && COMPILE_PCRE8 */ + +#if !defined COMPILE_PCRE8 +/* The ctypes array contains only 256 values. */ +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); +jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 255); +#endif +OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes); +#if !defined COMPILE_PCRE8 +JUMPHERE(jump); +#endif + +#if defined SUPPORT_UTF && defined COMPILE_PCRE16 +if (common->utf && update_str_ptr) + { + /* Skip low surrogate if necessary. */ + OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xd800); + jump = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0xdc00 - 0xd800 - 1); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + JUMPHERE(jump); + } +#endif /* SUPPORT_UTF && COMPILE_PCRE16 */ +} + +static void skip_char_back(compiler_common *common) +{ +/* Goes one character back. Affects STR_PTR and TMP1. Does not check begin. */ +DEFINE_COMPILER; +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 +#if defined COMPILE_PCRE8 +struct sljit_label *label; + +if (common->utf) + { + label = LABEL(); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), -IN_UCHARS(1)); + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0); + CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0x80, label); + return; + } +#elif defined COMPILE_PCRE16 +if (common->utf) + { + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), -IN_UCHARS(1)); + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + /* Skip low surrogate if necessary. */ + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xdc00); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP1, 0); + return; + } +#endif /* COMPILE_PCRE[8|16] */ +#endif /* SUPPORT_UTF && !COMPILE_PCRE32 */ +OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +} + +static void check_newlinechar(compiler_common *common, int nltype, jump_list **backtracks, BOOL jumpifmatch) +{ +/* Character comes in TMP1. Checks if it is a newline. TMP2 may be destroyed. */ +DEFINE_COMPILER; +struct sljit_jump *jump; + +if (nltype == NLTYPE_ANY) + { + add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL)); + sljit_set_current_flags(compiler, SLJIT_SET_Z); + add_jump(compiler, backtracks, JUMP(jumpifmatch ? SLJIT_NOT_ZERO : SLJIT_ZERO)); + } +else if (nltype == NLTYPE_ANYCRLF) + { + if (jumpifmatch) + { + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR)); + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL)); + } + else + { + jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL)); + JUMPHERE(jump); + } + } +else + { + SLJIT_ASSERT(nltype == NLTYPE_FIXED && common->newline < 256); + add_jump(compiler, backtracks, CMP(jumpifmatch ? SLJIT_EQUAL : SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, common->newline)); + } +} + +#ifdef SUPPORT_UTF + +#if defined COMPILE_PCRE8 +static void do_utfreadchar(compiler_common *common) +{ +/* Fast decoding a UTF-8 character. TMP1 contains the first byte +of the character (>= 0xc0). Return char value in TMP1, length in TMP2. */ +DEFINE_COMPILER; +struct sljit_jump *jump; + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); +OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); +OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); +OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); +OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + +/* Searching for the first zero. */ +OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); +jump = JUMP(SLJIT_NOT_ZERO); +/* Two byte sequence. */ +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, IN_UCHARS(2)); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +JUMPHERE(jump); +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); +OP2(SLJIT_XOR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x800); +OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); +OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); +OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + +OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x10000); +jump = JUMP(SLJIT_NOT_ZERO); +/* Three byte sequence. */ +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); +OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, IN_UCHARS(3)); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +/* Four byte sequence. */ +JUMPHERE(jump); +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(2)); +OP2(SLJIT_XOR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x10000); +OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(3)); +OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); +OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); +OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, IN_UCHARS(4)); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +} + +static void do_utfreadchar16(compiler_common *common) +{ +/* Fast decoding a UTF-8 character. TMP1 contains the first byte +of the character (>= 0xc0). Return value in TMP1. */ +DEFINE_COMPILER; +struct sljit_jump *jump; + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); +OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); +OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); +OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); +OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + +/* Searching for the first zero. */ +OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x800); +jump = JUMP(SLJIT_NOT_ZERO); +/* Two byte sequence. */ +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +JUMPHERE(jump); +OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x400); +OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_NOT_ZERO); +/* This code runs only in 8 bit mode. No need to shift the value. */ +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); +OP2(SLJIT_XOR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x800); +OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 6); +OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x3f); +OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); +/* Three byte sequence. */ +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +} + +static void do_utfreadtype8(compiler_common *common) +{ +/* Fast decoding a UTF-8 character type. TMP2 contains the first byte +of the character (>= 0xc0). Return value in TMP1. */ +DEFINE_COMPILER; +struct sljit_jump *jump; +struct sljit_jump *compare; + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); + +OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, 0x20); +jump = JUMP(SLJIT_NOT_ZERO); +/* Two byte sequence. */ +OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x1f); +/* The upper 5 bits are known at this point. */ +compare = CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, 0x3); +OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 6); +OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x3f); +OP2(SLJIT_OR, TMP2, 0, TMP2, 0, TMP1, 0); +OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP2), common->ctypes); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +JUMPHERE(compare); +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +/* We only have types for characters less than 256. */ +JUMPHERE(jump); +OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(utf8_table4) - 0xc0); +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +} + +#endif /* COMPILE_PCRE8 */ + +#endif /* SUPPORT_UTF */ + +#ifdef SUPPORT_UCP + +/* UCD_BLOCK_SIZE must be 128 (see the assert below). */ +#define UCD_BLOCK_MASK 127 +#define UCD_BLOCK_SHIFT 7 + +static void do_getucd(compiler_common *common) +{ +/* Search the UCD record for the character comes in TMP1. +Returns chartype in TMP1 and UCD offset in TMP2. */ +DEFINE_COMPILER; +#ifdef COMPILE_PCRE32 +struct sljit_jump *jump; +#endif + +#if defined SLJIT_DEBUG && SLJIT_DEBUG +/* dummy_ucd_record */ +const ucd_record *record = GET_UCD(INVALID_UTF_CHAR); +SLJIT_ASSERT(record->script == ucp_Common && record->chartype == ucp_Cn && record->gbprop == ucp_gbOther); +SLJIT_ASSERT(record->caseset == 0 && record->other_case == 0); +#endif + +SLJIT_ASSERT(UCD_BLOCK_SIZE == 128 && sizeof(ucd_record) == 8); + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); + +#ifdef COMPILE_PCRE32 +if (!common->utf) + { + jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x10ffff + 1); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); + JUMPHERE(jump); + } +#endif + +OP2(SLJIT_LSHR, TMP2, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_SHIFT); +OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_stage1)); +OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_MASK); +OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCD_BLOCK_SHIFT); +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); +OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_stage2)); +OP1(SLJIT_MOV_U16, TMP2, 0, SLJIT_MEM2(TMP2, TMP1), 1); +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype)); +OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +} +#endif + +static SLJIT_INLINE struct sljit_label *mainloop_entry(compiler_common *common, BOOL hascrorlf) +{ +DEFINE_COMPILER; +struct sljit_label *mainloop; +struct sljit_label *newlinelabel = NULL; +struct sljit_jump *start; +struct sljit_jump *end = NULL; +struct sljit_jump *end2 = NULL; +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 +struct sljit_jump *singlechar; +#endif +jump_list *newline = NULL; +BOOL newlinecheck = FALSE; +BOOL readuchar = FALSE; + +if (!(hascrorlf || (common->match_end_ptr != 0)) && + (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF || common->newline > 255)) + newlinecheck = TRUE; + +if (common->match_end_ptr != 0) + { + /* Search for the end of the first line. */ + OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0); + + if (common->nltype == NLTYPE_FIXED && common->newline > 255) + { + mainloop = LABEL(); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + end = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1)); + OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, mainloop); + CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff, mainloop); + JUMPHERE(end); + OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + } + else + { + end = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + mainloop = LABEL(); + /* Continual stores does not cause data dependency. */ + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, STR_PTR, 0); + read_char_range(common, common->nlmin, common->nlmax, TRUE); + check_newlinechar(common, common->nltype, &newline, TRUE); + CMPTO(SLJIT_LESS, STR_PTR, 0, STR_END, 0, mainloop); + JUMPHERE(end); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, STR_PTR, 0); + set_jumps(newline, LABEL()); + } + + OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0); + } + +start = JUMP(SLJIT_JUMP); + +if (newlinecheck) + { + newlinelabel = LABEL(); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + end = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, common->newline & 0xff); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT); +#endif + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); + end2 = JUMP(SLJIT_JUMP); + } + +mainloop = LABEL(); + +/* Increasing the STR_PTR here requires one less jump in the most common case. */ +#ifdef SUPPORT_UTF +if (common->utf) readuchar = TRUE; +#endif +if (newlinecheck) readuchar = TRUE; + +if (readuchar) + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); + +if (newlinecheck) + CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, newlinelabel); + +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 +#if defined COMPILE_PCRE8 +if (common->utf) + { + singlechar = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0); + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); + JUMPHERE(singlechar); + } +#elif defined COMPILE_PCRE16 +if (common->utf) + { + singlechar = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800); + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); + JUMPHERE(singlechar); + } +#endif /* COMPILE_PCRE[8|16] */ +#endif /* SUPPORT_UTF && !COMPILE_PCRE32 */ +JUMPHERE(start); + +if (newlinecheck) + { + JUMPHERE(end); + JUMPHERE(end2); + } + +return mainloop; +} + +#define MAX_N_CHARS 16 +#define MAX_DIFF_CHARS 6 + +static SLJIT_INLINE void add_prefix_char(pcre_uchar chr, pcre_uchar *chars) +{ +pcre_uchar i, len; + +len = chars[0]; +if (len == 255) + return; + +if (len == 0) + { + chars[0] = 1; + chars[1] = chr; + return; + } + +for (i = len; i > 0; i--) + if (chars[i] == chr) + return; + +if (len >= MAX_DIFF_CHARS - 1) + { + chars[0] = 255; + return; + } + +len++; +chars[len] = chr; +chars[0] = len; +} + +static int scan_prefix(compiler_common *common, pcre_uchar *cc, pcre_uchar *chars, int max_chars, sljit_u32 *rec_count) +{ +/* Recursive function, which scans prefix literals. */ +BOOL last, any, class, caseless; +int len, repeat, len_save, consumed = 0; +sljit_u32 chr; /* Any unicode character. */ +sljit_u8 *bytes, *bytes_end, byte; +pcre_uchar *alternative, *cc_save, *oc; +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 +pcre_uchar othercase[8]; +#elif defined SUPPORT_UTF && defined COMPILE_PCRE16 +pcre_uchar othercase[2]; +#else +pcre_uchar othercase[1]; +#endif + +repeat = 1; +while (TRUE) + { + if (*rec_count == 0) + return 0; + (*rec_count)--; + + last = TRUE; + any = FALSE; + class = FALSE; + caseless = FALSE; + + switch (*cc) + { + case OP_CHARI: + caseless = TRUE; + case OP_CHAR: + last = FALSE; + cc++; + break; + + case OP_SOD: + case OP_SOM: + case OP_SET_SOM: + case OP_NOT_WORD_BOUNDARY: + case OP_WORD_BOUNDARY: + case OP_EODN: + case OP_EOD: + case OP_CIRC: + case OP_CIRCM: + case OP_DOLL: + case OP_DOLLM: + /* Zero width assertions. */ + cc++; + continue; + + case OP_ASSERT: + case OP_ASSERT_NOT: + case OP_ASSERTBACK: + case OP_ASSERTBACK_NOT: + cc = bracketend(cc); + continue; + + case OP_PLUSI: + case OP_MINPLUSI: + case OP_POSPLUSI: + caseless = TRUE; + case OP_PLUS: + case OP_MINPLUS: + case OP_POSPLUS: + cc++; + break; + + case OP_EXACTI: + caseless = TRUE; + case OP_EXACT: + repeat = GET2(cc, 1); + last = FALSE; + cc += 1 + IMM2_SIZE; + break; + + case OP_QUERYI: + case OP_MINQUERYI: + case OP_POSQUERYI: + caseless = TRUE; + case OP_QUERY: + case OP_MINQUERY: + case OP_POSQUERY: + len = 1; + cc++; +#ifdef SUPPORT_UTF + if (common->utf && HAS_EXTRALEN(*cc)) len += GET_EXTRALEN(*cc); +#endif + max_chars = scan_prefix(common, cc + len, chars, max_chars, rec_count); + if (max_chars == 0) + return consumed; + last = FALSE; + break; + + case OP_KET: + cc += 1 + LINK_SIZE; + continue; + + case OP_ALT: + cc += GET(cc, 1); + continue; + + case OP_ONCE: + case OP_ONCE_NC: + case OP_BRA: + case OP_BRAPOS: + case OP_CBRA: + case OP_CBRAPOS: + alternative = cc + GET(cc, 1); + while (*alternative == OP_ALT) + { + max_chars = scan_prefix(common, alternative + 1 + LINK_SIZE, chars, max_chars, rec_count); + if (max_chars == 0) + return consumed; + alternative += GET(alternative, 1); + } + + if (*cc == OP_CBRA || *cc == OP_CBRAPOS) + cc += IMM2_SIZE; + cc += 1 + LINK_SIZE; + continue; + + case OP_CLASS: +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 + if (common->utf && !is_char7_bitset((const sljit_u8 *)(cc + 1), FALSE)) + return consumed; +#endif + class = TRUE; + break; + + case OP_NCLASS: +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (common->utf) return consumed; +#endif + class = TRUE; + break; + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (common->utf) return consumed; +#endif + any = TRUE; + cc += GET(cc, 1); + break; +#endif + + case OP_DIGIT: +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 + if (common->utf && !is_char7_bitset((const sljit_u8 *)common->ctypes - cbit_length + cbit_digit, FALSE)) + return consumed; +#endif + any = TRUE; + cc++; + break; + + case OP_WHITESPACE: +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 + if (common->utf && !is_char7_bitset((const sljit_u8 *)common->ctypes - cbit_length + cbit_space, FALSE)) + return consumed; +#endif + any = TRUE; + cc++; + break; + + case OP_WORDCHAR: +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 + if (common->utf && !is_char7_bitset((const sljit_u8 *)common->ctypes - cbit_length + cbit_word, FALSE)) + return consumed; +#endif + any = TRUE; + cc++; + break; + + case OP_NOT: + case OP_NOTI: + cc++; + /* Fall through. */ + case OP_NOT_DIGIT: + case OP_NOT_WHITESPACE: + case OP_NOT_WORDCHAR: + case OP_ANY: + case OP_ALLANY: +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (common->utf) return consumed; +#endif + any = TRUE; + cc++; + break; + +#ifdef SUPPORT_UTF + case OP_NOTPROP: + case OP_PROP: +#ifndef COMPILE_PCRE32 + if (common->utf) return consumed; +#endif + any = TRUE; + cc += 1 + 2; + break; +#endif + + case OP_TYPEEXACT: + repeat = GET2(cc, 1); + cc += 1 + IMM2_SIZE; + continue; + + case OP_NOTEXACT: + case OP_NOTEXACTI: +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (common->utf) return consumed; +#endif + any = TRUE; + repeat = GET2(cc, 1); + cc += 1 + IMM2_SIZE + 1; + break; + + default: + return consumed; + } + + if (any) + { + do + { + chars[0] = 255; + + consumed++; + if (--max_chars == 0) + return consumed; + chars += MAX_DIFF_CHARS; + } + while (--repeat > 0); + + repeat = 1; + continue; + } + + if (class) + { + bytes = (sljit_u8*) (cc + 1); + cc += 1 + 32 / sizeof(pcre_uchar); + + switch (*cc) + { + case OP_CRSTAR: + case OP_CRMINSTAR: + case OP_CRPOSSTAR: + case OP_CRQUERY: + case OP_CRMINQUERY: + case OP_CRPOSQUERY: + max_chars = scan_prefix(common, cc + 1, chars, max_chars, rec_count); + if (max_chars == 0) + return consumed; + break; + + default: + case OP_CRPLUS: + case OP_CRMINPLUS: + case OP_CRPOSPLUS: + break; + + case OP_CRRANGE: + case OP_CRMINRANGE: + case OP_CRPOSRANGE: + repeat = GET2(cc, 1); + if (repeat <= 0) + return consumed; + break; + } + + do + { + if (bytes[31] & 0x80) + chars[0] = 255; + else if (chars[0] != 255) + { + bytes_end = bytes + 32; + chr = 0; + do + { + byte = *bytes++; + SLJIT_ASSERT((chr & 0x7) == 0); + if (byte == 0) + chr += 8; + else + { + do + { + if ((byte & 0x1) != 0) + add_prefix_char(chr, chars); + byte >>= 1; + chr++; + } + while (byte != 0); + chr = (chr + 7) & ~7; + } + } + while (chars[0] != 255 && bytes < bytes_end); + bytes = bytes_end - 32; + } + + consumed++; + if (--max_chars == 0) + return consumed; + chars += MAX_DIFF_CHARS; + } + while (--repeat > 0); + + switch (*cc) + { + case OP_CRSTAR: + case OP_CRMINSTAR: + case OP_CRPOSSTAR: + return consumed; + + case OP_CRQUERY: + case OP_CRMINQUERY: + case OP_CRPOSQUERY: + cc++; + break; + + case OP_CRRANGE: + case OP_CRMINRANGE: + case OP_CRPOSRANGE: + if (GET2(cc, 1) != GET2(cc, 1 + IMM2_SIZE)) + return consumed; + cc += 1 + 2 * IMM2_SIZE; + break; + } + + repeat = 1; + continue; + } + + len = 1; +#ifdef SUPPORT_UTF + if (common->utf && HAS_EXTRALEN(*cc)) len += GET_EXTRALEN(*cc); +#endif + + if (caseless && char_has_othercase(common, cc)) + { +#ifdef SUPPORT_UTF + if (common->utf) + { + GETCHAR(chr, cc); + if ((int)PRIV(ord2utf)(char_othercase(common, chr), othercase) != len) + return consumed; + } + else +#endif + { + chr = *cc; + othercase[0] = TABLE_GET(chr, common->fcc, chr); + } + } + else + { + caseless = FALSE; + othercase[0] = 0; /* Stops compiler warning - PH */ + } + + len_save = len; + cc_save = cc; + while (TRUE) + { + oc = othercase; + do + { + chr = *cc; + add_prefix_char(*cc, chars); + + if (caseless) + add_prefix_char(*oc, chars); + + len--; + consumed++; + if (--max_chars == 0) + return consumed; + chars += MAX_DIFF_CHARS; + cc++; + oc++; + } + while (len > 0); + + if (--repeat == 0) + break; + + len = len_save; + cc = cc_save; + } + + repeat = 1; + if (last) + return consumed; + } +} + +#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) && !(defined SUPPORT_VALGRIND) + +static sljit_s32 character_to_int32(pcre_uchar chr) +{ +sljit_s32 value = (sljit_s32)chr; +#if defined COMPILE_PCRE8 +#define SSE2_COMPARE_TYPE_INDEX 0 +return (value << 24) | (value << 16) | (value << 8) | value; +#elif defined COMPILE_PCRE16 +#define SSE2_COMPARE_TYPE_INDEX 1 +return (value << 16) | value; +#elif defined COMPILE_PCRE32 +#define SSE2_COMPARE_TYPE_INDEX 2 +return value; +#else +#error "Unsupported unit width" +#endif +} + +static SLJIT_INLINE void fast_forward_first_char2_sse2(compiler_common *common, pcre_uchar char1, pcre_uchar char2) +{ +DEFINE_COMPILER; +struct sljit_label *start; +struct sljit_jump *quit[3]; +struct sljit_jump *nomatch; +sljit_u8 instruction[8]; +sljit_s32 tmp1_ind = sljit_get_register_index(TMP1); +sljit_s32 tmp2_ind = sljit_get_register_index(TMP2); +sljit_s32 str_ptr_ind = sljit_get_register_index(STR_PTR); +BOOL load_twice = FALSE; +pcre_uchar bit; + +bit = char1 ^ char2; +if (!is_powerof2(bit)) + bit = 0; + +if ((char1 != char2) && bit == 0) + load_twice = TRUE; + +quit[0] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + +/* First part (unaligned start) */ + +OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char1 | bit)); + +SLJIT_ASSERT(tmp1_ind < 8 && tmp2_ind == 1); + +/* MOVD xmm, r/m32 */ +instruction[0] = 0x66; +instruction[1] = 0x0f; +instruction[2] = 0x6e; +instruction[3] = 0xc0 | (2 << 3) | tmp1_ind; +sljit_emit_op_custom(compiler, instruction, 4); + +if (char1 != char2) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(bit != 0 ? bit : char2)); + + /* MOVD xmm, r/m32 */ + instruction[3] = 0xc0 | (3 << 3) | tmp1_ind; + sljit_emit_op_custom(compiler, instruction, 4); + } + +/* PSHUFD xmm1, xmm2/m128, imm8 */ +instruction[2] = 0x70; +instruction[3] = 0xc0 | (2 << 3) | 2; +instruction[4] = 0; +sljit_emit_op_custom(compiler, instruction, 5); + +if (char1 != char2) + { + /* PSHUFD xmm1, xmm2/m128, imm8 */ + instruction[3] = 0xc0 | (3 << 3) | 3; + instruction[4] = 0; + sljit_emit_op_custom(compiler, instruction, 5); + } + +OP2(SLJIT_AND, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 0xf); +OP2(SLJIT_AND, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, ~0xf); + +/* MOVDQA xmm1, xmm2/m128 */ +#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) + +if (str_ptr_ind < 8) + { + instruction[2] = 0x6f; + instruction[3] = (0 << 3) | str_ptr_ind; + sljit_emit_op_custom(compiler, instruction, 4); + + if (load_twice) + { + instruction[3] = (1 << 3) | str_ptr_ind; + sljit_emit_op_custom(compiler, instruction, 4); + } + } +else + { + instruction[1] = 0x41; + instruction[2] = 0x0f; + instruction[3] = 0x6f; + instruction[4] = (0 << 3) | (str_ptr_ind & 0x7); + sljit_emit_op_custom(compiler, instruction, 5); + + if (load_twice) + { + instruction[4] = (1 << 3) | str_ptr_ind; + sljit_emit_op_custom(compiler, instruction, 5); + } + instruction[1] = 0x0f; + } + +#else + +instruction[2] = 0x6f; +instruction[3] = (0 << 3) | str_ptr_ind; +sljit_emit_op_custom(compiler, instruction, 4); + +if (load_twice) + { + instruction[3] = (1 << 3) | str_ptr_ind; + sljit_emit_op_custom(compiler, instruction, 4); + } + +#endif + +if (bit != 0) + { + /* POR xmm1, xmm2/m128 */ + instruction[2] = 0xeb; + instruction[3] = 0xc0 | (0 << 3) | 3; + sljit_emit_op_custom(compiler, instruction, 4); + } + +/* PCMPEQB/W/D xmm1, xmm2/m128 */ +instruction[2] = 0x74 + SSE2_COMPARE_TYPE_INDEX; +instruction[3] = 0xc0 | (0 << 3) | 2; +sljit_emit_op_custom(compiler, instruction, 4); + +if (load_twice) + { + instruction[3] = 0xc0 | (1 << 3) | 3; + sljit_emit_op_custom(compiler, instruction, 4); + } + +/* PMOVMSKB reg, xmm */ +instruction[2] = 0xd7; +instruction[3] = 0xc0 | (tmp1_ind << 3) | 0; +sljit_emit_op_custom(compiler, instruction, 4); + +if (load_twice) + { + OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP2, 0); + instruction[3] = 0xc0 | (tmp2_ind << 3) | 1; + sljit_emit_op_custom(compiler, instruction, 4); + + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + OP1(SLJIT_MOV, TMP2, 0, RETURN_ADDR, 0); + } + +OP2(SLJIT_ASHR, TMP1, 0, TMP1, 0, TMP2, 0); + +/* BSF r32, r/m32 */ +instruction[0] = 0x0f; +instruction[1] = 0xbc; +instruction[2] = 0xc0 | (tmp1_ind << 3) | tmp1_ind; +sljit_emit_op_custom(compiler, instruction, 3); +sljit_set_current_flags(compiler, SLJIT_SET_Z); + +nomatch = JUMP(SLJIT_ZERO); + +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); +quit[1] = JUMP(SLJIT_JUMP); + +JUMPHERE(nomatch); + +start = LABEL(); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 16); +quit[2] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + +/* Second part (aligned) */ + +instruction[0] = 0x66; +instruction[1] = 0x0f; + +/* MOVDQA xmm1, xmm2/m128 */ +#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64) + +if (str_ptr_ind < 8) + { + instruction[2] = 0x6f; + instruction[3] = (0 << 3) | str_ptr_ind; + sljit_emit_op_custom(compiler, instruction, 4); + + if (load_twice) + { + instruction[3] = (1 << 3) | str_ptr_ind; + sljit_emit_op_custom(compiler, instruction, 4); + } + } +else + { + instruction[1] = 0x41; + instruction[2] = 0x0f; + instruction[3] = 0x6f; + instruction[4] = (0 << 3) | (str_ptr_ind & 0x7); + sljit_emit_op_custom(compiler, instruction, 5); + + if (load_twice) + { + instruction[4] = (1 << 3) | str_ptr_ind; + sljit_emit_op_custom(compiler, instruction, 5); + } + instruction[1] = 0x0f; + } + +#else + +instruction[2] = 0x6f; +instruction[3] = (0 << 3) | str_ptr_ind; +sljit_emit_op_custom(compiler, instruction, 4); + +if (load_twice) + { + instruction[3] = (1 << 3) | str_ptr_ind; + sljit_emit_op_custom(compiler, instruction, 4); + } + +#endif + +if (bit != 0) + { + /* POR xmm1, xmm2/m128 */ + instruction[2] = 0xeb; + instruction[3] = 0xc0 | (0 << 3) | 3; + sljit_emit_op_custom(compiler, instruction, 4); + } + +/* PCMPEQB/W/D xmm1, xmm2/m128 */ +instruction[2] = 0x74 + SSE2_COMPARE_TYPE_INDEX; +instruction[3] = 0xc0 | (0 << 3) | 2; +sljit_emit_op_custom(compiler, instruction, 4); + +if (load_twice) + { + instruction[3] = 0xc0 | (1 << 3) | 3; + sljit_emit_op_custom(compiler, instruction, 4); + } + +/* PMOVMSKB reg, xmm */ +instruction[2] = 0xd7; +instruction[3] = 0xc0 | (tmp1_ind << 3) | 0; +sljit_emit_op_custom(compiler, instruction, 4); + +if (load_twice) + { + instruction[3] = 0xc0 | (tmp2_ind << 3) | 1; + sljit_emit_op_custom(compiler, instruction, 4); + + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, TMP2, 0); + } + +/* BSF r32, r/m32 */ +instruction[0] = 0x0f; +instruction[1] = 0xbc; +instruction[2] = 0xc0 | (tmp1_ind << 3) | tmp1_ind; +sljit_emit_op_custom(compiler, instruction, 3); +sljit_set_current_flags(compiler, SLJIT_SET_Z); + +JUMPTO(SLJIT_ZERO, start); + +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); + +start = LABEL(); +SET_LABEL(quit[0], start); +SET_LABEL(quit[1], start); +SET_LABEL(quit[2], start); +} + +#undef SSE2_COMPARE_TYPE_INDEX + +#endif + +static void fast_forward_first_char2(compiler_common *common, pcre_uchar char1, pcre_uchar char2, sljit_s32 offset) +{ +DEFINE_COMPILER; +struct sljit_label *start; +struct sljit_jump *quit; +struct sljit_jump *found; +pcre_uchar mask; +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 +struct sljit_label *utf_start = NULL; +struct sljit_jump *utf_quit = NULL; +#endif +BOOL has_match_end = (common->match_end_ptr != 0); + +if (offset > 0) + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset)); + +if (has_match_end) + { + OP1(SLJIT_MOV, TMP3, 0, STR_END, 0); + + OP2(SLJIT_ADD, STR_END, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, SLJIT_IMM, IN_UCHARS(offset + 1)); + OP2(SLJIT_SUB | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, STR_END, 0, TMP3, 0); + sljit_emit_cmov(compiler, SLJIT_GREATER, STR_END, TMP3, 0); + } + +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 +if (common->utf && offset > 0) + utf_start = LABEL(); +#endif + +#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) && !(defined SUPPORT_VALGRIND) + +/* SSE2 accelerated first character search. */ + +if (sljit_has_cpu_feature(SLJIT_HAS_SSE2)) + { + fast_forward_first_char2_sse2(common, char1, char2); + + SLJIT_ASSERT(common->mode == JIT_COMPILE || offset == 0); + if (common->mode == JIT_COMPILE) + { + /* In complete mode, we don't need to run a match when STR_PTR == STR_END. */ + SLJIT_ASSERT(common->forced_quit_label == NULL); + OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_NOMATCH); + add_jump(compiler, &common->forced_quit, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); + +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (common->utf && offset > 0) + { + SLJIT_ASSERT(common->mode == JIT_COMPILE); + + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-offset)); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +#if defined COMPILE_PCRE8 + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0); + CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0x80, utf_start); +#elif defined COMPILE_PCRE16 + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); + CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0xdc00, utf_start); +#else +#error "Unknown code width" +#endif + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + } +#endif + + if (offset > 0) + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset)); + } + else + { + OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, STR_PTR, 0, STR_END, 0); + if (has_match_end) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); + sljit_emit_cmov(compiler, SLJIT_GREATER_EQUAL, STR_PTR, TMP1, 0); + } + else + sljit_emit_cmov(compiler, SLJIT_GREATER_EQUAL, STR_PTR, STR_END, 0); + } + + if (has_match_end) + OP1(SLJIT_MOV, STR_END, 0, TMP3, 0); + return; + } + +#endif + +quit = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + +start = LABEL(); +OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); + +if (char1 == char2) + found = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, char1); +else + { + mask = char1 ^ char2; + if (is_powerof2(mask)) + { + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, mask); + found = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, char1 | mask); + } + else + { + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char1); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, char2); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); + found = JUMP(SLJIT_NOT_ZERO); + } + } + +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +CMPTO(SLJIT_LESS, STR_PTR, 0, STR_END, 0, start); + +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 +if (common->utf && offset > 0) + utf_quit = JUMP(SLJIT_JUMP); +#endif + +JUMPHERE(found); + +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 +if (common->utf && offset > 0) + { + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-offset)); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +#if defined COMPILE_PCRE8 + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0); + CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0x80, utf_start); +#elif defined COMPILE_PCRE16 + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); + CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0xdc00, utf_start); +#else +#error "Unknown code width" +#endif + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + JUMPHERE(utf_quit); + } +#endif + +JUMPHERE(quit); + +if (has_match_end) + { + quit = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); + if (offset > 0) + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset)); + JUMPHERE(quit); + OP1(SLJIT_MOV, STR_END, 0, TMP3, 0); + } + +if (offset > 0) + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset)); +} + +static SLJIT_INLINE BOOL fast_forward_first_n_chars(compiler_common *common) +{ +DEFINE_COMPILER; +struct sljit_label *start; +struct sljit_jump *quit; +struct sljit_jump *match; +/* bytes[0] represent the number of characters between 0 +and MAX_N_BYTES - 1, 255 represents any character. */ +pcre_uchar chars[MAX_N_CHARS * MAX_DIFF_CHARS]; +sljit_s32 offset; +pcre_uchar mask; +pcre_uchar *char_set, *char_set_end; +int i, max, from; +int range_right = -1, range_len; +sljit_u8 *update_table = NULL; +BOOL in_range; +sljit_u32 rec_count; + +for (i = 0; i < MAX_N_CHARS; i++) + chars[i * MAX_DIFF_CHARS] = 0; + +rec_count = 10000; +max = scan_prefix(common, common->start, chars, MAX_N_CHARS, &rec_count); + +if (max < 1) + return FALSE; + +in_range = FALSE; +/* Prevent compiler "uninitialized" warning */ +from = 0; +range_len = 4 /* minimum length */ - 1; +for (i = 0; i <= max; i++) + { + if (in_range && (i - from) > range_len && (chars[(i - 1) * MAX_DIFF_CHARS] < 255)) + { + range_len = i - from; + range_right = i - 1; + } + + if (i < max && chars[i * MAX_DIFF_CHARS] < 255) + { + SLJIT_ASSERT(chars[i * MAX_DIFF_CHARS] > 0); + if (!in_range) + { + in_range = TRUE; + from = i; + } + } + else + in_range = FALSE; + } + +if (range_right >= 0) + { + update_table = (sljit_u8 *)allocate_read_only_data(common, 256); + if (update_table == NULL) + return TRUE; + memset(update_table, IN_UCHARS(range_len), 256); + + for (i = 0; i < range_len; i++) + { + char_set = chars + ((range_right - i) * MAX_DIFF_CHARS); + SLJIT_ASSERT(char_set[0] > 0 && char_set[0] < 255); + char_set_end = char_set + char_set[0]; + char_set++; + while (char_set <= char_set_end) + { + if (update_table[(*char_set) & 0xff] > IN_UCHARS(i)) + update_table[(*char_set) & 0xff] = IN_UCHARS(i); + char_set++; + } + } + } + +offset = -1; +/* Scan forward. */ +for (i = 0; i < max; i++) + { + if (offset == -1) + { + if (chars[i * MAX_DIFF_CHARS] <= 2) + offset = i; + } + else if (chars[offset * MAX_DIFF_CHARS] == 2 && chars[i * MAX_DIFF_CHARS] <= 2) + { + if (chars[i * MAX_DIFF_CHARS] == 1) + offset = i; + else + { + mask = chars[offset * MAX_DIFF_CHARS + 1] ^ chars[offset * MAX_DIFF_CHARS + 2]; + if (!is_powerof2(mask)) + { + mask = chars[i * MAX_DIFF_CHARS + 1] ^ chars[i * MAX_DIFF_CHARS + 2]; + if (is_powerof2(mask)) + offset = i; + } + } + } + } + +if (range_right < 0) + { + if (offset < 0) + return FALSE; + SLJIT_ASSERT(chars[offset * MAX_DIFF_CHARS] >= 1 && chars[offset * MAX_DIFF_CHARS] <= 2); + /* Works regardless the value is 1 or 2. */ + mask = chars[offset * MAX_DIFF_CHARS + chars[offset * MAX_DIFF_CHARS]]; + fast_forward_first_char2(common, chars[offset * MAX_DIFF_CHARS + 1], mask, offset); + return TRUE; + } + +if (range_right == offset) + offset = -1; + +SLJIT_ASSERT(offset == -1 || (chars[offset * MAX_DIFF_CHARS] >= 1 && chars[offset * MAX_DIFF_CHARS] <= 2)); + +max -= 1; +SLJIT_ASSERT(max > 0); +if (common->match_end_ptr != 0) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); + OP1(SLJIT_MOV, TMP3, 0, STR_END, 0); + OP2(SLJIT_SUB, STR_END, 0, STR_END, 0, SLJIT_IMM, IN_UCHARS(max)); + quit = CMP(SLJIT_LESS_EQUAL, STR_END, 0, TMP1, 0); + OP1(SLJIT_MOV, STR_END, 0, TMP1, 0); + JUMPHERE(quit); + } +else + OP2(SLJIT_SUB, STR_END, 0, STR_END, 0, SLJIT_IMM, IN_UCHARS(max)); + +SLJIT_ASSERT(range_right >= 0); + +#if !(defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) +OP1(SLJIT_MOV, RETURN_ADDR, 0, SLJIT_IMM, (sljit_sw)update_table); +#endif + +start = LABEL(); +quit = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + +#if defined COMPILE_PCRE8 || (defined SLJIT_LITTLE_ENDIAN && SLJIT_LITTLE_ENDIAN) +OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(range_right)); +#else +OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(range_right + 1) - 1); +#endif + +#if !(defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) +OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(RETURN_ADDR, TMP1), 0); +#else +OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)update_table); +#endif +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); +CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0, start); + +if (offset >= 0) + { + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(offset)); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + + if (chars[offset * MAX_DIFF_CHARS] == 1) + CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, chars[offset * MAX_DIFF_CHARS + 1], start); + else + { + mask = chars[offset * MAX_DIFF_CHARS + 1] ^ chars[offset * MAX_DIFF_CHARS + 2]; + if (is_powerof2(mask)) + { + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, mask); + CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, chars[offset * MAX_DIFF_CHARS + 1] | mask, start); + } + else + { + match = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, chars[offset * MAX_DIFF_CHARS + 1]); + CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, chars[offset * MAX_DIFF_CHARS + 2], start); + JUMPHERE(match); + } + } + } + +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 +if (common->utf && offset != 0) + { + if (offset < 0) + { + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + } + else + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1)); +#if defined COMPILE_PCRE8 + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0); + CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0x80, start); +#elif defined COMPILE_PCRE16 + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); + CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0xdc00, start); +#else +#error "Unknown code width" +#endif + if (offset < 0) + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + } +#endif + +if (offset >= 0) + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + +JUMPHERE(quit); + +if (common->match_end_ptr != 0) + { + if (range_right >= 0) + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); + OP1(SLJIT_MOV, STR_END, 0, TMP3, 0); + if (range_right >= 0) + { + quit = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP1, 0); + OP1(SLJIT_MOV, STR_PTR, 0, TMP1, 0); + JUMPHERE(quit); + } + } +else + OP2(SLJIT_ADD, STR_END, 0, STR_END, 0, SLJIT_IMM, IN_UCHARS(max)); +return TRUE; +} + +#undef MAX_N_CHARS +#undef MAX_DIFF_CHARS + +static SLJIT_INLINE void fast_forward_first_char(compiler_common *common, pcre_uchar first_char, BOOL caseless) +{ +pcre_uchar oc; + +oc = first_char; +if (caseless) + { + oc = TABLE_GET(first_char, common->fcc, first_char); +#if defined SUPPORT_UCP && !defined COMPILE_PCRE8 + if (first_char > 127 && common->utf) + oc = UCD_OTHERCASE(first_char); +#endif + } + +fast_forward_first_char2(common, first_char, oc, 0); +} + +static SLJIT_INLINE void fast_forward_newline(compiler_common *common) +{ +DEFINE_COMPILER; +struct sljit_label *loop; +struct sljit_jump *lastchar; +struct sljit_jump *firstchar; +struct sljit_jump *quit; +struct sljit_jump *foundcr = NULL; +struct sljit_jump *notfoundnl; +jump_list *newline = NULL; + +if (common->match_end_ptr != 0) + { + OP1(SLJIT_MOV, TMP3, 0, STR_END, 0); + OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); + } + +if (common->nltype == NLTYPE_FIXED && common->newline > 255) + { + lastchar = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); + firstchar = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0); + + OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(2)); + OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, STR_PTR, 0, TMP1, 0); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_GREATER_EQUAL); +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCHAR_SHIFT); +#endif + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0); + + loop = LABEL(); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + quit = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2)); + OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1)); + CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, loop); + CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff, loop); + + JUMPHERE(quit); + JUMPHERE(firstchar); + JUMPHERE(lastchar); + + if (common->match_end_ptr != 0) + OP1(SLJIT_MOV, STR_END, 0, TMP3, 0); + return; + } + +OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); +OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); +firstchar = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0); +skip_char_back(common); + +loop = LABEL(); +common->ff_newline_shortcut = loop; + +read_char_range(common, common->nlmin, common->nlmax, TRUE); +lastchar = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); +if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF) + foundcr = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR); +check_newlinechar(common, common->nltype, &newline, FALSE); +set_jumps(newline, loop); + +if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF) + { + quit = JUMP(SLJIT_JUMP); + JUMPHERE(foundcr); + notfoundnl = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, CHAR_NL); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT); +#endif + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); + JUMPHERE(notfoundnl); + JUMPHERE(quit); + } +JUMPHERE(lastchar); +JUMPHERE(firstchar); + +if (common->match_end_ptr != 0) + OP1(SLJIT_MOV, STR_END, 0, TMP3, 0); +} + +static BOOL check_class_ranges(compiler_common *common, const sljit_u8 *bits, BOOL nclass, BOOL invert, jump_list **backtracks); + +static SLJIT_INLINE void fast_forward_start_bits(compiler_common *common, const sljit_u8 *start_bits) +{ +DEFINE_COMPILER; +struct sljit_label *start; +struct sljit_jump *quit; +struct sljit_jump *found = NULL; +jump_list *matches = NULL; +#ifndef COMPILE_PCRE8 +struct sljit_jump *jump; +#endif + +if (common->match_end_ptr != 0) + { + OP1(SLJIT_MOV, RETURN_ADDR, 0, STR_END, 0); + OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); + } + +start = LABEL(); +quit = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); +OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); +#ifdef SUPPORT_UTF +if (common->utf) + OP1(SLJIT_MOV, TMP3, 0, TMP1, 0); +#endif + +if (!check_class_ranges(common, start_bits, (start_bits[31] & 0x80) != 0, TRUE, &matches)) + { +#ifndef COMPILE_PCRE8 + jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 255); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 255); + JUMPHERE(jump); +#endif + OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7); + OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3); + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)start_bits); + OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); + found = JUMP(SLJIT_NOT_ZERO); + } + +#ifdef SUPPORT_UTF +if (common->utf) + OP1(SLJIT_MOV, TMP1, 0, TMP3, 0); +#endif +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +#ifdef SUPPORT_UTF +#if defined COMPILE_PCRE8 +if (common->utf) + { + CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0, start); + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); + } +#elif defined COMPILE_PCRE16 +if (common->utf) + { + CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800, start); + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); + } +#endif /* COMPILE_PCRE[8|16] */ +#endif /* SUPPORT_UTF */ +JUMPTO(SLJIT_JUMP, start); +if (found != NULL) + JUMPHERE(found); +if (matches != NULL) + set_jumps(matches, LABEL()); +JUMPHERE(quit); + +if (common->match_end_ptr != 0) + OP1(SLJIT_MOV, STR_END, 0, RETURN_ADDR, 0); +} + +static SLJIT_INLINE struct sljit_jump *search_requested_char(compiler_common *common, pcre_uchar req_char, BOOL caseless, BOOL has_firstchar) +{ +DEFINE_COMPILER; +struct sljit_label *loop; +struct sljit_jump *toolong; +struct sljit_jump *alreadyfound; +struct sljit_jump *found; +struct sljit_jump *foundoc = NULL; +struct sljit_jump *notfound; +sljit_u32 oc, bit; + +SLJIT_ASSERT(common->req_char_ptr != 0); +OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->req_char_ptr); +OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, REQ_BYTE_MAX); +toolong = CMP(SLJIT_LESS, TMP1, 0, STR_END, 0); +alreadyfound = CMP(SLJIT_LESS, STR_PTR, 0, TMP2, 0); + +if (has_firstchar) + OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +else + OP1(SLJIT_MOV, TMP1, 0, STR_PTR, 0); + +loop = LABEL(); +notfound = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, STR_END, 0); + +OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(TMP1), 0); +oc = req_char; +if (caseless) + { + oc = TABLE_GET(req_char, common->fcc, req_char); +#if defined SUPPORT_UCP && !(defined COMPILE_PCRE8) + if (req_char > 127 && common->utf) + oc = UCD_OTHERCASE(req_char); +#endif + } +if (req_char == oc) + found = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, req_char); +else + { + bit = req_char ^ oc; + if (is_powerof2(bit)) + { + OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, bit); + found = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, req_char | bit); + } + else + { + found = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, req_char); + foundoc = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, oc); + } + } +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1)); +JUMPTO(SLJIT_JUMP, loop); + +JUMPHERE(found); +if (foundoc) + JUMPHERE(foundoc); +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->req_char_ptr, TMP1, 0); +JUMPHERE(alreadyfound); +JUMPHERE(toolong); +return notfound; +} + +static void do_revertframes(compiler_common *common) +{ +DEFINE_COMPILER; +struct sljit_jump *jump; +struct sljit_label *mainloop; + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); +OP1(SLJIT_MOV, TMP3, 0, STACK_TOP, 0); +GET_LOCAL_BASE(TMP1, 0, 0); + +/* Drop frames until we reach STACK_TOP. */ +mainloop = LABEL(); +OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), -sizeof(sljit_sw)); +jump = CMP(SLJIT_SIG_LESS_EQUAL, TMP2, 0, SLJIT_IMM, 0); + +OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); +OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(STACK_TOP), -2 * sizeof(sljit_sw)); +OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), sizeof(sljit_sw), SLJIT_MEM1(STACK_TOP), -3 * sizeof(sljit_sw)); +OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 3 * sizeof(sljit_sw)); +JUMPTO(SLJIT_JUMP, mainloop); + +JUMPHERE(jump); +jump = CMP(SLJIT_NOT_ZERO /* SIG_LESS */, TMP2, 0, SLJIT_IMM, 0); +/* End of reverting values. */ +OP1(SLJIT_MOV, STACK_TOP, 0, TMP3, 0); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); + +JUMPHERE(jump); +OP1(SLJIT_NEG, TMP2, 0, TMP2, 0); +OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0); +OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), 0, SLJIT_MEM1(STACK_TOP), -2 * sizeof(sljit_sw)); +OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 2 * sizeof(sljit_sw)); +JUMPTO(SLJIT_JUMP, mainloop); +} + +static void check_wordboundary(compiler_common *common) +{ +DEFINE_COMPILER; +struct sljit_jump *skipread; +jump_list *skipread_list = NULL; +#if !(defined COMPILE_PCRE8) || defined SUPPORT_UTF +struct sljit_jump *jump; +#endif + +SLJIT_COMPILE_ASSERT(ctype_word == 0x10, ctype_word_must_be_16); + +sljit_emit_fast_enter(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0); +/* Get type of the previous char, and put it to LOCALS1. */ +OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); +OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, SLJIT_IMM, 0); +skipread = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP1, 0); +skip_char_back(common); +check_start_used_ptr(common); +read_char(common); + +/* Testing char type. */ +#ifdef SUPPORT_UCP +if (common->use_ucp) + { + OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 1); + jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE); + add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL)); + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Nd - ucp_Ll); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL); + JUMPHERE(jump); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP2, 0); + } +else +#endif + { +#ifndef COMPILE_PCRE8 + jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255); +#elif defined SUPPORT_UTF + /* Here LOCALS1 has already been zeroed. */ + jump = NULL; + if (common->utf) + jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255); +#endif /* COMPILE_PCRE8 */ + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), common->ctypes); + OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 4 /* ctype_word */); + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP1, 0); +#ifndef COMPILE_PCRE8 + JUMPHERE(jump); +#elif defined SUPPORT_UTF + if (jump != NULL) + JUMPHERE(jump); +#endif /* COMPILE_PCRE8 */ + } +JUMPHERE(skipread); + +OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0); +check_str_end(common, &skipread_list); +peek_char(common, READ_CHAR_MAX); + +/* Testing char type. This is a code duplication. */ +#ifdef SUPPORT_UCP +if (common->use_ucp) + { + OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 1); + jump = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_UNDERSCORE); + add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL)); + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Ll); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ucp_Nd - ucp_Ll); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ucp_No - ucp_Nd); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL); + JUMPHERE(jump); + } +else +#endif + { +#ifndef COMPILE_PCRE8 + /* TMP2 may be destroyed by peek_char. */ + OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0); + jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255); +#elif defined SUPPORT_UTF + OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, 0); + jump = NULL; + if (common->utf) + jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255); +#endif + OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), common->ctypes); + OP2(SLJIT_LSHR, TMP2, 0, TMP2, 0, SLJIT_IMM, 4 /* ctype_word */); + OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 1); +#ifndef COMPILE_PCRE8 + JUMPHERE(jump); +#elif defined SUPPORT_UTF + if (jump != NULL) + JUMPHERE(jump); +#endif /* COMPILE_PCRE8 */ + } +set_jumps(skipread_list, LABEL()); + +OP2(SLJIT_XOR | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); +sljit_emit_fast_return(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0); +} + +static BOOL check_class_ranges(compiler_common *common, const sljit_u8 *bits, BOOL nclass, BOOL invert, jump_list **backtracks) +{ +/* May destroy TMP1. */ +DEFINE_COMPILER; +int ranges[MAX_RANGE_SIZE]; +sljit_u8 bit, cbit, all; +int i, byte, length = 0; + +bit = bits[0] & 0x1; +/* All bits will be zero or one (since bit is zero or one). */ +all = -bit; + +for (i = 0; i < 256; ) + { + byte = i >> 3; + if ((i & 0x7) == 0 && bits[byte] == all) + i += 8; + else + { + cbit = (bits[byte] >> (i & 0x7)) & 0x1; + if (cbit != bit) + { + if (length >= MAX_RANGE_SIZE) + return FALSE; + ranges[length] = i; + length++; + bit = cbit; + all = -cbit; + } + i++; + } + } + +if (((bit == 0) && nclass) || ((bit == 1) && !nclass)) + { + if (length >= MAX_RANGE_SIZE) + return FALSE; + ranges[length] = 256; + length++; + } + +if (length < 0 || length > 4) + return FALSE; + +bit = bits[0] & 0x1; +if (invert) bit ^= 0x1; + +/* No character is accepted. */ +if (length == 0 && bit == 0) + add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); + +switch(length) + { + case 0: + /* When bit != 0, all characters are accepted. */ + return TRUE; + + case 1: + add_jump(compiler, backtracks, CMP(bit == 0 ? SLJIT_LESS : SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, ranges[0])); + return TRUE; + + case 2: + if (ranges[0] + 1 != ranges[1]) + { + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[0]); + add_jump(compiler, backtracks, CMP(bit != 0 ? SLJIT_LESS : SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, ranges[1] - ranges[0])); + } + else + add_jump(compiler, backtracks, CMP(bit != 0 ? SLJIT_EQUAL : SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[0])); + return TRUE; + + case 3: + if (bit != 0) + { + add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, ranges[2])); + if (ranges[0] + 1 != ranges[1]) + { + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[0]); + add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, ranges[1] - ranges[0])); + } + else + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[0])); + return TRUE; + } + + add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, ranges[0])); + if (ranges[1] + 1 != ranges[2]) + { + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[1]); + add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, ranges[2] - ranges[1])); + } + else + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[1])); + return TRUE; + + case 4: + if ((ranges[1] - ranges[0]) == (ranges[3] - ranges[2]) + && (ranges[0] | (ranges[2] - ranges[0])) == ranges[2] + && (ranges[1] & (ranges[2] - ranges[0])) == 0 + && is_powerof2(ranges[2] - ranges[0])) + { + SLJIT_ASSERT((ranges[0] & (ranges[2] - ranges[0])) == 0 && (ranges[2] & ranges[3] & (ranges[2] - ranges[0])) != 0); + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[2] - ranges[0]); + if (ranges[2] + 1 != ranges[3]) + { + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[2]); + add_jump(compiler, backtracks, CMP(bit != 0 ? SLJIT_LESS : SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, ranges[3] - ranges[2])); + } + else + add_jump(compiler, backtracks, CMP(bit != 0 ? SLJIT_EQUAL : SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[2])); + return TRUE; + } + + if (bit != 0) + { + i = 0; + if (ranges[0] + 1 != ranges[1]) + { + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[0]); + add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, ranges[1] - ranges[0])); + i = ranges[0]; + } + else + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[0])); + + if (ranges[2] + 1 != ranges[3]) + { + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[2] - i); + add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, ranges[3] - ranges[2])); + } + else + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[2] - i)); + return TRUE; + } + + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[0]); + add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, ranges[3] - ranges[0])); + if (ranges[1] + 1 != ranges[2]) + { + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, ranges[1] - ranges[0]); + add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, ranges[2] - ranges[1])); + } + else + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, ranges[1] - ranges[0])); + return TRUE; + + default: + SLJIT_UNREACHABLE(); + return FALSE; + } +} + +static void check_anynewline(compiler_common *common) +{ +/* Check whether TMP1 contains a newline character. TMP2 destroyed. */ +DEFINE_COMPILER; + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); + +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0a); +OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a); +OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); +OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a); +#if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 +#ifdef COMPILE_PCRE8 +if (common->utf) + { +#endif + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a); +#ifdef COMPILE_PCRE8 + } +#endif +#endif /* SUPPORT_UTF || COMPILE_PCRE16 || COMPILE_PCRE32 */ +OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +} + +static void check_hspace(compiler_common *common) +{ +/* Check whether TMP1 contains a newline character. TMP2 destroyed. */ +DEFINE_COMPILER; + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); + +OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x09); +OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); +OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x20); +OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); +OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xa0); +#if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 +#ifdef COMPILE_PCRE8 +if (common->utf) + { +#endif + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x1680); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x2000); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x200A - 0x2000); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x202f - 0x2000); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x205f - 0x2000); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x3000 - 0x2000); +#ifdef COMPILE_PCRE8 + } +#endif +#endif /* SUPPORT_UTF || COMPILE_PCRE16 || COMPILE_PCRE32 */ +OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); + +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +} + +static void check_vspace(compiler_common *common) +{ +/* Check whether TMP1 contains a newline character. TMP2 destroyed. */ +DEFINE_COMPILER; + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); + +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x0a); +OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x0d - 0x0a); +OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); +OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x0a); +#if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 +#ifdef COMPILE_PCRE8 +if (common->utf) + { +#endif + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x1); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2029 - 0x0a); +#ifdef COMPILE_PCRE8 + } +#endif +#endif /* SUPPORT_UTF || COMPILE_PCRE16 || COMPILE_PCRE32 */ +OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); + +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +} + +#define CHAR1 STR_END +#define CHAR2 STACK_TOP + +static void do_casefulcmp(compiler_common *common) +{ +DEFINE_COMPILER; +struct sljit_jump *jump; +struct sljit_label *label; + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); +OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0); +OP1(SLJIT_MOV, TMP3, 0, CHAR1, 0); +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, CHAR2, 0); +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1)); +OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + +label = LABEL(); +OP1(MOVU_UCHAR, CHAR1, 0, SLJIT_MEM1(TMP1), IN_UCHARS(1)); +OP1(MOVU_UCHAR, CHAR2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); +jump = CMP(SLJIT_NOT_EQUAL, CHAR1, 0, CHAR2, 0); +OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1)); +JUMPTO(SLJIT_NOT_ZERO, label); + +JUMPHERE(jump); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +OP1(SLJIT_MOV, CHAR1, 0, TMP3, 0); +OP1(SLJIT_MOV, CHAR2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +} + +#define LCC_TABLE STACK_LIMIT + +static void do_caselesscmp(compiler_common *common) +{ +DEFINE_COMPILER; +struct sljit_jump *jump; +struct sljit_label *label; + +sljit_emit_fast_enter(compiler, RETURN_ADDR, 0); +OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0); + +OP1(SLJIT_MOV, TMP3, 0, LCC_TABLE, 0); +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, CHAR1, 0); +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, CHAR2, 0); +OP1(SLJIT_MOV, LCC_TABLE, 0, SLJIT_IMM, common->lcc); +OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(1)); +OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + +label = LABEL(); +OP1(MOVU_UCHAR, CHAR1, 0, SLJIT_MEM1(TMP1), IN_UCHARS(1)); +OP1(MOVU_UCHAR, CHAR2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); +#ifndef COMPILE_PCRE8 +jump = CMP(SLJIT_GREATER, CHAR1, 0, SLJIT_IMM, 255); +#endif +OP1(SLJIT_MOV_U8, CHAR1, 0, SLJIT_MEM2(LCC_TABLE, CHAR1), 0); +#ifndef COMPILE_PCRE8 +JUMPHERE(jump); +jump = CMP(SLJIT_GREATER, CHAR2, 0, SLJIT_IMM, 255); +#endif +OP1(SLJIT_MOV_U8, CHAR2, 0, SLJIT_MEM2(LCC_TABLE, CHAR2), 0); +#ifndef COMPILE_PCRE8 +JUMPHERE(jump); +#endif +jump = CMP(SLJIT_NOT_EQUAL, CHAR1, 0, CHAR2, 0); +OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, IN_UCHARS(1)); +JUMPTO(SLJIT_NOT_ZERO, label); + +JUMPHERE(jump); +OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +OP1(SLJIT_MOV, LCC_TABLE, 0, TMP3, 0); +OP1(SLJIT_MOV, CHAR1, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); +OP1(SLJIT_MOV, CHAR2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); +sljit_emit_fast_return(compiler, RETURN_ADDR, 0); +} + +#undef LCC_TABLE +#undef CHAR1 +#undef CHAR2 + +#if defined SUPPORT_UTF && defined SUPPORT_UCP + +static const pcre_uchar * SLJIT_CALL do_utf_caselesscmp(pcre_uchar *src1, jit_arguments *args, pcre_uchar *end1) +{ +/* This function would be ineffective to do in JIT level. */ +sljit_u32 c1, c2; +const pcre_uchar *src2 = args->uchar_ptr; +const pcre_uchar *end2 = args->end; +const ucd_record *ur; +const sljit_u32 *pp; + +while (src1 < end1) + { + if (src2 >= end2) + return (pcre_uchar*)1; + GETCHARINC(c1, src1); + GETCHARINC(c2, src2); + ur = GET_UCD(c2); + if (c1 != c2 && c1 != c2 + ur->other_case) + { + pp = PRIV(ucd_caseless_sets) + ur->caseset; + for (;;) + { + if (c1 < *pp) return NULL; + if (c1 == *pp++) break; + } + } + } +return src2; +} + +#endif /* SUPPORT_UTF && SUPPORT_UCP */ + +static pcre_uchar *byte_sequence_compare(compiler_common *common, BOOL caseless, pcre_uchar *cc, + compare_context *context, jump_list **backtracks) +{ +DEFINE_COMPILER; +unsigned int othercasebit = 0; +pcre_uchar *othercasechar = NULL; +#ifdef SUPPORT_UTF +int utflength; +#endif + +if (caseless && char_has_othercase(common, cc)) + { + othercasebit = char_get_othercase_bit(common, cc); + SLJIT_ASSERT(othercasebit); + /* Extracting bit difference info. */ +#if defined COMPILE_PCRE8 + othercasechar = cc + (othercasebit >> 8); + othercasebit &= 0xff; +#elif defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + /* Note that this code only handles characters in the BMP. If there + ever are characters outside the BMP whose othercase differs in only one + bit from itself (there currently are none), this code will need to be + revised for COMPILE_PCRE32. */ + othercasechar = cc + (othercasebit >> 9); + if ((othercasebit & 0x100) != 0) + othercasebit = (othercasebit & 0xff) << 8; + else + othercasebit &= 0xff; +#endif /* COMPILE_PCRE[8|16|32] */ + } + +if (context->sourcereg == -1) + { +#if defined COMPILE_PCRE8 +#if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED + if (context->length >= 4) + OP1(SLJIT_MOV_S32, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length); + else if (context->length >= 2) + OP1(SLJIT_MOV_U16, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length); + else +#endif + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length); +#elif defined COMPILE_PCRE16 +#if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED + if (context->length >= 4) + OP1(SLJIT_MOV_S32, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length); + else +#endif + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length); +#elif defined COMPILE_PCRE32 + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), -context->length); +#endif /* COMPILE_PCRE[8|16|32] */ + context->sourcereg = TMP2; + } + +#ifdef SUPPORT_UTF +utflength = 1; +if (common->utf && HAS_EXTRALEN(*cc)) + utflength += GET_EXTRALEN(*cc); + +do + { +#endif + + context->length -= IN_UCHARS(1); +#if (defined SLJIT_UNALIGNED && SLJIT_UNALIGNED) && (defined COMPILE_PCRE8 || defined COMPILE_PCRE16) + + /* Unaligned read is supported. */ + if (othercasebit != 0 && othercasechar == cc) + { + context->c.asuchars[context->ucharptr] = *cc | othercasebit; + context->oc.asuchars[context->ucharptr] = othercasebit; + } + else + { + context->c.asuchars[context->ucharptr] = *cc; + context->oc.asuchars[context->ucharptr] = 0; + } + context->ucharptr++; + +#if defined COMPILE_PCRE8 + if (context->ucharptr >= 4 || context->length == 0 || (context->ucharptr == 2 && context->length == 1)) +#else + if (context->ucharptr >= 2 || context->length == 0) +#endif + { + if (context->length >= 4) + OP1(SLJIT_MOV_S32, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length); + else if (context->length >= 2) + OP1(SLJIT_MOV_U16, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length); +#if defined COMPILE_PCRE8 + else if (context->length >= 1) + OP1(SLJIT_MOV_U8, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length); +#endif /* COMPILE_PCRE8 */ + context->sourcereg = context->sourcereg == TMP1 ? TMP2 : TMP1; + + switch(context->ucharptr) + { + case 4 / sizeof(pcre_uchar): + if (context->oc.asint != 0) + OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, context->oc.asint); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asint | context->oc.asint)); + break; + + case 2 / sizeof(pcre_uchar): + if (context->oc.asushort != 0) + OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, context->oc.asushort); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asushort | context->oc.asushort)); + break; + +#ifdef COMPILE_PCRE8 + case 1: + if (context->oc.asbyte != 0) + OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, context->oc.asbyte); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, context->c.asbyte | context->oc.asbyte)); + break; +#endif + + default: + SLJIT_UNREACHABLE(); + break; + } + context->ucharptr = 0; + } + +#else + + /* Unaligned read is unsupported or in 32 bit mode. */ + if (context->length >= 1) + OP1(MOV_UCHAR, context->sourcereg, 0, SLJIT_MEM1(STR_PTR), -context->length); + + context->sourcereg = context->sourcereg == TMP1 ? TMP2 : TMP1; + + if (othercasebit != 0 && othercasechar == cc) + { + OP2(SLJIT_OR, context->sourcereg, 0, context->sourcereg, 0, SLJIT_IMM, othercasebit); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, *cc | othercasebit)); + } + else + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, context->sourcereg, 0, SLJIT_IMM, *cc)); + +#endif + + cc++; +#ifdef SUPPORT_UTF + utflength--; + } +while (utflength > 0); +#endif + +return cc; +} + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + +#define SET_TYPE_OFFSET(value) \ + if ((value) != typeoffset) \ + { \ + if ((value) < typeoffset) \ + OP2(SLJIT_ADD, typereg, 0, typereg, 0, SLJIT_IMM, typeoffset - (value)); \ + else \ + OP2(SLJIT_SUB, typereg, 0, typereg, 0, SLJIT_IMM, (value) - typeoffset); \ + } \ + typeoffset = (value); + +#define SET_CHAR_OFFSET(value) \ + if ((value) != charoffset) \ + { \ + if ((value) < charoffset) \ + OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(charoffset - (value))); \ + else \ + OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)((value) - charoffset)); \ + } \ + charoffset = (value); + +static pcre_uchar *compile_char1_matchingpath(compiler_common *common, pcre_uchar type, pcre_uchar *cc, jump_list **backtracks, BOOL check_str_ptr); + +static void compile_xclass_matchingpath(compiler_common *common, pcre_uchar *cc, jump_list **backtracks) +{ +DEFINE_COMPILER; +jump_list *found = NULL; +jump_list **list = (cc[0] & XCL_NOT) == 0 ? &found : backtracks; +sljit_uw c, charoffset, max = 256, min = READ_CHAR_MAX; +struct sljit_jump *jump = NULL; +pcre_uchar *ccbegin; +int compares, invertcmp, numberofcmps; +#if defined SUPPORT_UTF && (defined COMPILE_PCRE8 || defined COMPILE_PCRE16) +BOOL utf = common->utf; +#endif + +#ifdef SUPPORT_UCP +BOOL needstype = FALSE, needsscript = FALSE, needschar = FALSE; +BOOL charsaved = FALSE; +int typereg = TMP1; +const sljit_u32 *other_cases; +sljit_uw typeoffset; +#endif + +/* Scanning the necessary info. */ +cc++; +ccbegin = cc; +compares = 0; +if (cc[-1] & XCL_MAP) + { + min = 0; + cc += 32 / sizeof(pcre_uchar); + } + +while (*cc != XCL_END) + { + compares++; + if (*cc == XCL_SINGLE) + { + cc ++; + GETCHARINCTEST(c, cc); + if (c > max) max = c; + if (c < min) min = c; +#ifdef SUPPORT_UCP + needschar = TRUE; +#endif + } + else if (*cc == XCL_RANGE) + { + cc ++; + GETCHARINCTEST(c, cc); + if (c < min) min = c; + GETCHARINCTEST(c, cc); + if (c > max) max = c; +#ifdef SUPPORT_UCP + needschar = TRUE; +#endif + } +#ifdef SUPPORT_UCP + else + { + SLJIT_ASSERT(*cc == XCL_PROP || *cc == XCL_NOTPROP); + cc++; + if (*cc == PT_CLIST) + { + other_cases = PRIV(ucd_caseless_sets) + cc[1]; + while (*other_cases != NOTACHAR) + { + if (*other_cases > max) max = *other_cases; + if (*other_cases < min) min = *other_cases; + other_cases++; + } + } + else + { + max = READ_CHAR_MAX; + min = 0; + } + + switch(*cc) + { + case PT_ANY: + /* Any either accepts everything or ignored. */ + if (cc[-1] == XCL_PROP) + { + compile_char1_matchingpath(common, OP_ALLANY, cc, backtracks, FALSE); + if (list == backtracks) + add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); + return; + } + break; + + case PT_LAMP: + case PT_GC: + case PT_PC: + case PT_ALNUM: + needstype = TRUE; + break; + + case PT_SC: + needsscript = TRUE; + break; + + case PT_SPACE: + case PT_PXSPACE: + case PT_WORD: + case PT_PXGRAPH: + case PT_PXPRINT: + case PT_PXPUNCT: + needstype = TRUE; + needschar = TRUE; + break; + + case PT_CLIST: + case PT_UCNC: + needschar = TRUE; + break; + + default: + SLJIT_UNREACHABLE(); + break; + } + cc += 2; + } +#endif + } +SLJIT_ASSERT(compares > 0); + +/* We are not necessary in utf mode even in 8 bit mode. */ +cc = ccbegin; +read_char_range(common, min, max, (cc[-1] & XCL_NOT) != 0); + +if ((cc[-1] & XCL_HASPROP) == 0) + { + if ((cc[-1] & XCL_MAP) != 0) + { + jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255); + if (!check_class_ranges(common, (const sljit_u8 *)cc, (((const sljit_u8 *)cc)[31] & 0x80) != 0, TRUE, &found)) + { + OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7); + OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3); + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc); + OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); + add_jump(compiler, &found, JUMP(SLJIT_NOT_ZERO)); + } + + add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); + JUMPHERE(jump); + + cc += 32 / sizeof(pcre_uchar); + } + else + { + OP2(SLJIT_SUB, TMP2, 0, TMP1, 0, SLJIT_IMM, min); + add_jump(compiler, (cc[-1] & XCL_NOT) == 0 ? backtracks : &found, CMP(SLJIT_GREATER, TMP2, 0, SLJIT_IMM, max - min)); + } + } +else if ((cc[-1] & XCL_MAP) != 0) + { + OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0); +#ifdef SUPPORT_UCP + charsaved = TRUE; +#endif + if (!check_class_ranges(common, (const sljit_u8 *)cc, FALSE, TRUE, list)) + { +#ifdef COMPILE_PCRE8 + jump = NULL; + if (common->utf) +#endif + jump = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255); + + OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7); + OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3); + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc); + OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); + add_jump(compiler, list, JUMP(SLJIT_NOT_ZERO)); + +#ifdef COMPILE_PCRE8 + if (common->utf) +#endif + JUMPHERE(jump); + } + + OP1(SLJIT_MOV, TMP1, 0, RETURN_ADDR, 0); + cc += 32 / sizeof(pcre_uchar); + } + +#ifdef SUPPORT_UCP +if (needstype || needsscript) + { + if (needschar && !charsaved) + OP1(SLJIT_MOV, RETURN_ADDR, 0, TMP1, 0); + +#ifdef COMPILE_PCRE32 + if (!common->utf) + { + jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x10ffff + 1); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR); + JUMPHERE(jump); + } +#endif + + OP2(SLJIT_LSHR, TMP2, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_SHIFT); + OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_stage1)); + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, UCD_BLOCK_MASK); + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCD_BLOCK_SHIFT); + OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_stage2)); + OP1(SLJIT_MOV_U16, TMP2, 0, SLJIT_MEM2(TMP2, TMP1), 1); + + /* Before anything else, we deal with scripts. */ + if (needsscript) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, script)); + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3); + + ccbegin = cc; + + while (*cc != XCL_END) + { + if (*cc == XCL_SINGLE) + { + cc ++; + GETCHARINCTEST(c, cc); + } + else if (*cc == XCL_RANGE) + { + cc ++; + GETCHARINCTEST(c, cc); + GETCHARINCTEST(c, cc); + } + else + { + SLJIT_ASSERT(*cc == XCL_PROP || *cc == XCL_NOTPROP); + cc++; + if (*cc == PT_SC) + { + compares--; + invertcmp = (compares == 0 && list != backtracks); + if (cc[-1] == XCL_NOTPROP) + invertcmp ^= 0x1; + jump = CMP(SLJIT_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, (int)cc[1]); + add_jump(compiler, compares > 0 ? list : backtracks, jump); + } + cc += 2; + } + } + + cc = ccbegin; + } + + if (needschar) + { + OP1(SLJIT_MOV, TMP1, 0, RETURN_ADDR, 0); + } + + if (needstype) + { + if (!needschar) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype)); + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM2(TMP1, TMP2), 3); + } + else + { + OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 3); + OP1(SLJIT_MOV_U8, RETURN_ADDR, 0, SLJIT_MEM1(TMP2), (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, chartype)); + typereg = RETURN_ADDR; + } + } + } +#endif + +/* Generating code. */ +charoffset = 0; +numberofcmps = 0; +#ifdef SUPPORT_UCP +typeoffset = 0; +#endif + +while (*cc != XCL_END) + { + compares--; + invertcmp = (compares == 0 && list != backtracks); + jump = NULL; + + if (*cc == XCL_SINGLE) + { + cc ++; + GETCHARINCTEST(c, cc); + + if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE)) + { + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); + OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + numberofcmps++; + } + else if (numberofcmps > 0) + { + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); + jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); + numberofcmps = 0; + } + else + { + jump = CMP(SLJIT_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); + numberofcmps = 0; + } + } + else if (*cc == XCL_RANGE) + { + cc ++; + GETCHARINCTEST(c, cc); + SET_CHAR_OFFSET(c); + GETCHARINCTEST(c, cc); + + if (numberofcmps < 3 && (*cc == XCL_SINGLE || *cc == XCL_RANGE)) + { + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); + OP_FLAGS(numberofcmps == 0 ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL); + numberofcmps++; + } + else if (numberofcmps > 0) + { + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL); + jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); + numberofcmps = 0; + } + else + { + jump = CMP(SLJIT_LESS_EQUAL ^ invertcmp, TMP1, 0, SLJIT_IMM, (sljit_sw)(c - charoffset)); + numberofcmps = 0; + } + } +#ifdef SUPPORT_UCP + else + { + SLJIT_ASSERT(*cc == XCL_PROP || *cc == XCL_NOTPROP); + if (*cc == XCL_NOTPROP) + invertcmp ^= 0x1; + cc++; + switch(*cc) + { + case PT_ANY: + if (!invertcmp) + jump = JUMP(SLJIT_JUMP); + break; + + case PT_LAMP: + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - typeoffset); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ll - typeoffset); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lt - typeoffset); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_EQUAL); + jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); + break; + + case PT_GC: + c = PRIV(ucp_typerange)[(int)cc[1] * 2]; + SET_TYPE_OFFSET(c); + jump = CMP(SLJIT_LESS_EQUAL ^ invertcmp, typereg, 0, SLJIT_IMM, PRIV(ucp_typerange)[(int)cc[1] * 2 + 1] - c); + break; + + case PT_PC: + jump = CMP(SLJIT_EQUAL ^ invertcmp, typereg, 0, SLJIT_IMM, (int)cc[1] - typeoffset); + break; + + case PT_SC: + compares++; + /* Do nothing. */ + break; + + case PT_SPACE: + case PT_PXSPACE: + SET_CHAR_OFFSET(9); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd - 0x9); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); + + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x85 - 0x9); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e - 0x9); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + + SET_TYPE_OFFSET(ucp_Zl); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Zl); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL); + jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); + break; + + case PT_WORD: + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_UNDERSCORE - charoffset)); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); + /* Fall through. */ + + case PT_ALNUM: + SET_TYPE_OFFSET(ucp_Ll); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Lu - ucp_Ll); + OP_FLAGS((*cc == PT_ALNUM) ? SLJIT_MOV : SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL); + SET_TYPE_OFFSET(ucp_Nd); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_No - ucp_Nd); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL); + jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); + break; + + case PT_CLIST: + other_cases = PRIV(ucd_caseless_sets) + cc[1]; + + /* At least three characters are required. + Otherwise this case would be handled by the normal code path. */ + SLJIT_ASSERT(other_cases[0] != NOTACHAR && other_cases[1] != NOTACHAR && other_cases[2] != NOTACHAR); + SLJIT_ASSERT(other_cases[0] < other_cases[1] && other_cases[1] < other_cases[2]); + + /* Optimizing character pairs, if their difference is power of 2. */ + if (is_powerof2(other_cases[1] ^ other_cases[0])) + { + if (charoffset == 0) + OP2(SLJIT_OR, TMP2, 0, TMP1, 0, SLJIT_IMM, other_cases[1] ^ other_cases[0]); + else + { + OP2(SLJIT_ADD, TMP2, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)charoffset); + OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, other_cases[1] ^ other_cases[0]); + } + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, other_cases[1]); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); + other_cases += 2; + } + else if (is_powerof2(other_cases[2] ^ other_cases[1])) + { + if (charoffset == 0) + OP2(SLJIT_OR, TMP2, 0, TMP1, 0, SLJIT_IMM, other_cases[2] ^ other_cases[1]); + else + { + OP2(SLJIT_ADD, TMP2, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)charoffset); + OP2(SLJIT_OR, TMP2, 0, TMP2, 0, SLJIT_IMM, other_cases[1] ^ other_cases[0]); + } + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP2, 0, SLJIT_IMM, other_cases[2]); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); + + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(other_cases[0] - charoffset)); + OP_FLAGS(SLJIT_OR | ((other_cases[3] == NOTACHAR) ? SLJIT_SET_Z : 0), TMP2, 0, SLJIT_EQUAL); + + other_cases += 3; + } + else + { + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset)); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); + } + + while (*other_cases != NOTACHAR) + { + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(*other_cases++ - charoffset)); + OP_FLAGS(SLJIT_OR | ((*other_cases == NOTACHAR) ? SLJIT_SET_Z : 0), TMP2, 0, SLJIT_EQUAL); + } + jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); + break; + + case PT_UCNC: + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_DOLLAR_SIGN - charoffset)); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_COMMERCIAL_AT - charoffset)); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(CHAR_GRAVE_ACCENT - charoffset)); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + + SET_CHAR_OFFSET(0xa0); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (sljit_sw)(0xd7ff - charoffset)); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_LESS_EQUAL); + SET_CHAR_OFFSET(0); + OP2(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xe000 - 0); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_GREATER_EQUAL); + jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); + break; + + case PT_PXGRAPH: + /* C and Z groups are the farthest two groups. */ + SET_TYPE_OFFSET(ucp_Ll); + OP2(SLJIT_SUB | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_GREATER); + + jump = CMP(SLJIT_NOT_EQUAL, typereg, 0, SLJIT_IMM, ucp_Cf - ucp_Ll); + + /* In case of ucp_Cf, we overwrite the result. */ + SET_CHAR_OFFSET(0x2066); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); + + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x180e - 0x2066); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + + JUMPHERE(jump); + jump = CMP(SLJIT_ZERO ^ invertcmp, TMP2, 0, SLJIT_IMM, 0); + break; + + case PT_PXPRINT: + /* C and Z groups are the farthest two groups. */ + SET_TYPE_OFFSET(ucp_Ll); + OP2(SLJIT_SUB | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Ll); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_GREATER); + + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Zs - ucp_Ll); + OP_FLAGS(SLJIT_AND, TMP2, 0, SLJIT_NOT_EQUAL); + + jump = CMP(SLJIT_NOT_EQUAL, typereg, 0, SLJIT_IMM, ucp_Cf - ucp_Ll); + + /* In case of ucp_Cf, we overwrite the result. */ + SET_CHAR_OFFSET(0x2066); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x2069 - 0x2066); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); + + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x061c - 0x2066); + OP_FLAGS(SLJIT_OR, TMP2, 0, SLJIT_EQUAL); + + JUMPHERE(jump); + jump = CMP(SLJIT_ZERO ^ invertcmp, TMP2, 0, SLJIT_IMM, 0); + break; + + case PT_PXPUNCT: + SET_TYPE_OFFSET(ucp_Sc); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_So - ucp_Sc); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS_EQUAL); + + SET_CHAR_OFFSET(0); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0x7f); + OP_FLAGS(SLJIT_AND, TMP2, 0, SLJIT_LESS_EQUAL); + + SET_TYPE_OFFSET(ucp_Pc); + OP2(SLJIT_SUB | SLJIT_SET_LESS_EQUAL, SLJIT_UNUSED, 0, typereg, 0, SLJIT_IMM, ucp_Ps - ucp_Pc); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_LESS_EQUAL); + jump = JUMP(SLJIT_NOT_ZERO ^ invertcmp); + break; + + default: + SLJIT_UNREACHABLE(); + break; + } + cc += 2; + } +#endif + + if (jump != NULL) + add_jump(compiler, compares > 0 ? list : backtracks, jump); + } + +if (found != NULL) + set_jumps(found, LABEL()); +} + +#undef SET_TYPE_OFFSET +#undef SET_CHAR_OFFSET + +#endif + +static pcre_uchar *compile_simple_assertion_matchingpath(compiler_common *common, pcre_uchar type, pcre_uchar *cc, jump_list **backtracks) +{ +DEFINE_COMPILER; +int length; +struct sljit_jump *jump[4]; +#ifdef SUPPORT_UTF +struct sljit_label *label; +#endif /* SUPPORT_UTF */ + +switch(type) + { + case OP_SOD: + OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, TMP1, 0)); + return cc; + + case OP_SOM: + OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, TMP1, 0)); + return cc; + + case OP_NOT_WORD_BOUNDARY: + case OP_WORD_BOUNDARY: + add_jump(compiler, &common->wordboundary, JUMP(SLJIT_FAST_CALL)); + sljit_set_current_flags(compiler, SLJIT_SET_Z); + add_jump(compiler, backtracks, JUMP(type == OP_NOT_WORD_BOUNDARY ? SLJIT_NOT_ZERO : SLJIT_ZERO)); + return cc; + + case OP_EODN: + /* Requires rather complex checks. */ + jump[0] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + if (common->nltype == NLTYPE_FIXED && common->newline > 255) + { + OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + if (common->mode == JIT_COMPILE) + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, STR_END, 0)); + else + { + jump[1] = CMP(SLJIT_EQUAL, TMP2, 0, STR_END, 0); + OP2(SLJIT_SUB | SLJIT_SET_LESS, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0); + OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_LESS); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff); + OP_FLAGS(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, SLJIT_NOT_EQUAL); + add_jump(compiler, backtracks, JUMP(SLJIT_NOT_EQUAL)); + check_partial(common, TRUE); + add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); + JUMPHERE(jump[1]); + } + OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff)); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff)); + } + else if (common->nltype == NLTYPE_FIXED) + { + OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, STR_END, 0)); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, common->newline)); + } + else + { + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + jump[1] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR); + OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); + OP2(SLJIT_SUB | SLJIT_SET_Z | SLJIT_SET_GREATER, SLJIT_UNUSED, 0, TMP2, 0, STR_END, 0); + jump[2] = JUMP(SLJIT_GREATER); + add_jump(compiler, backtracks, JUMP(SLJIT_NOT_EQUAL) /* LESS */); + /* Equal. */ + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); + jump[3] = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL); + add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); + + JUMPHERE(jump[1]); + if (common->nltype == NLTYPE_ANYCRLF) + { + OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP2, 0, STR_END, 0)); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL)); + } + else + { + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, STR_PTR, 0); + read_char_range(common, common->nlmin, common->nlmax, TRUE); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, STR_END, 0)); + add_jump(compiler, &common->anynewline, JUMP(SLJIT_FAST_CALL)); + sljit_set_current_flags(compiler, SLJIT_SET_Z); + add_jump(compiler, backtracks, JUMP(SLJIT_ZERO)); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); + } + JUMPHERE(jump[2]); + JUMPHERE(jump[3]); + } + JUMPHERE(jump[0]); + check_partial(common, FALSE); + return cc; + + case OP_EOD: + add_jump(compiler, backtracks, CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0)); + check_partial(common, FALSE); + return cc; + + case OP_DOLL: + OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, noteol)); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0)); + + if (!common->endonly) + compile_simple_assertion_matchingpath(common, OP_EODN, cc, backtracks); + else + { + add_jump(compiler, backtracks, CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0)); + check_partial(common, FALSE); + } + return cc; + + case OP_DOLLM: + jump[1] = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0); + OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, noteol)); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0)); + check_partial(common, FALSE); + jump[0] = JUMP(SLJIT_JUMP); + JUMPHERE(jump[1]); + + if (common->nltype == NLTYPE_FIXED && common->newline > 255) + { + OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + if (common->mode == JIT_COMPILE) + add_jump(compiler, backtracks, CMP(SLJIT_GREATER, TMP2, 0, STR_END, 0)); + else + { + jump[1] = CMP(SLJIT_LESS_EQUAL, TMP2, 0, STR_END, 0); + /* STR_PTR = STR_END - IN_UCHARS(1) */ + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff)); + check_partial(common, TRUE); + add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); + JUMPHERE(jump[1]); + } + + OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1)); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff)); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff)); + } + else + { + peek_char(common, common->nlmax); + check_newlinechar(common, common->nltype, backtracks, FALSE); + } + JUMPHERE(jump[0]); + return cc; + + case OP_CIRC: + OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin)); + add_jump(compiler, backtracks, CMP(SLJIT_GREATER, STR_PTR, 0, TMP1, 0)); + OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, notbol)); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0)); + return cc; + + case OP_CIRCM: + OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin)); + jump[1] = CMP(SLJIT_GREATER, STR_PTR, 0, TMP1, 0); + OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, notbol)); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0)); + jump[0] = JUMP(SLJIT_JUMP); + JUMPHERE(jump[1]); + + add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); + if (common->nltype == NLTYPE_FIXED && common->newline > 255) + { + OP2(SLJIT_SUB, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2)); + add_jump(compiler, backtracks, CMP(SLJIT_LESS, TMP2, 0, TMP1, 0)); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2)); + OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1)); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff)); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff)); + } + else + { + skip_char_back(common); + read_char_range(common, common->nlmin, common->nlmax, TRUE); + check_newlinechar(common, common->nltype, backtracks, FALSE); + } + JUMPHERE(jump[0]); + return cc; + + case OP_REVERSE: + length = GET(cc, 0); + if (length == 0) + return cc + LINK_SIZE; + OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); +#ifdef SUPPORT_UTF + if (common->utf) + { + OP1(SLJIT_MOV, TMP3, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, length); + label = LABEL(); + add_jump(compiler, backtracks, CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP3, 0)); + skip_char_back(common); + OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, TMP2, 0, SLJIT_IMM, 1); + JUMPTO(SLJIT_NOT_ZERO, label); + } + else +#endif + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin)); + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(length)); + add_jump(compiler, backtracks, CMP(SLJIT_LESS, STR_PTR, 0, TMP1, 0)); + } + check_start_used_ptr(common); + return cc + LINK_SIZE; + } +SLJIT_UNREACHABLE(); +return cc; +} + +static pcre_uchar *compile_char1_matchingpath(compiler_common *common, pcre_uchar type, pcre_uchar *cc, jump_list **backtracks, BOOL check_str_ptr) +{ +DEFINE_COMPILER; +int length; +unsigned int c, oc, bit; +compare_context context; +struct sljit_jump *jump[3]; +jump_list *end_list; +#ifdef SUPPORT_UTF +struct sljit_label *label; +#ifdef SUPPORT_UCP +pcre_uchar propdata[5]; +#endif +#endif /* SUPPORT_UTF */ + +switch(type) + { + case OP_NOT_DIGIT: + case OP_DIGIT: + /* Digits are usually 0-9, so it is worth to optimize them. */ + if (check_str_ptr) + detect_partial_match(common, backtracks); +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 + if (common->utf && is_char7_bitset((const sljit_u8 *)common->ctypes - cbit_length + cbit_digit, FALSE)) + read_char7_type(common, type == OP_NOT_DIGIT); + else +#endif + read_char8_type(common, type == OP_NOT_DIGIT); + /* Flip the starting bit in the negative case. */ + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_digit); + add_jump(compiler, backtracks, JUMP(type == OP_DIGIT ? SLJIT_ZERO : SLJIT_NOT_ZERO)); + return cc; + + case OP_NOT_WHITESPACE: + case OP_WHITESPACE: + if (check_str_ptr) + detect_partial_match(common, backtracks); +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 + if (common->utf && is_char7_bitset((const sljit_u8 *)common->ctypes - cbit_length + cbit_space, FALSE)) + read_char7_type(common, type == OP_NOT_WHITESPACE); + else +#endif + read_char8_type(common, type == OP_NOT_WHITESPACE); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_space); + add_jump(compiler, backtracks, JUMP(type == OP_WHITESPACE ? SLJIT_ZERO : SLJIT_NOT_ZERO)); + return cc; + + case OP_NOT_WORDCHAR: + case OP_WORDCHAR: + if (check_str_ptr) + detect_partial_match(common, backtracks); +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 + if (common->utf && is_char7_bitset((const sljit_u8 *)common->ctypes - cbit_length + cbit_word, FALSE)) + read_char7_type(common, type == OP_NOT_WORDCHAR); + else +#endif + read_char8_type(common, type == OP_NOT_WORDCHAR); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, ctype_word); + add_jump(compiler, backtracks, JUMP(type == OP_WORDCHAR ? SLJIT_ZERO : SLJIT_NOT_ZERO)); + return cc; + + case OP_ANY: + if (check_str_ptr) + detect_partial_match(common, backtracks); + read_char_range(common, common->nlmin, common->nlmax, TRUE); + if (common->nltype == NLTYPE_FIXED && common->newline > 255) + { + jump[0] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff); + end_list = NULL; + if (common->mode != JIT_PARTIAL_HARD_COMPILE) + add_jump(compiler, &end_list, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); + else + check_str_end(common, &end_list); + + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, common->newline & 0xff)); + set_jumps(end_list, LABEL()); + JUMPHERE(jump[0]); + } + else + check_newlinechar(common, common->nltype, backtracks, TRUE); + return cc; + + case OP_ALLANY: + if (check_str_ptr) + detect_partial_match(common, backtracks); +#ifdef SUPPORT_UTF + if (common->utf) + { + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); +#if defined COMPILE_PCRE8 || defined COMPILE_PCRE16 +#if defined COMPILE_PCRE8 + jump[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0); + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); +#elif defined COMPILE_PCRE16 + jump[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800); + OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, SLJIT_IMM, 0xd800); + OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL); + OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); +#endif + JUMPHERE(jump[0]); +#endif /* COMPILE_PCRE[8|16] */ + return cc; + } +#endif + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + return cc; + + case OP_ANYBYTE: + if (check_str_ptr) + detect_partial_match(common, backtracks); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + return cc; + +#ifdef SUPPORT_UTF +#ifdef SUPPORT_UCP + case OP_NOTPROP: + case OP_PROP: + propdata[0] = XCL_HASPROP; + propdata[1] = type == OP_NOTPROP ? XCL_NOTPROP : XCL_PROP; + propdata[2] = cc[0]; + propdata[3] = cc[1]; + propdata[4] = XCL_END; + if (check_str_ptr) + detect_partial_match(common, backtracks); + compile_xclass_matchingpath(common, propdata, backtracks); + return cc + 2; +#endif +#endif + + case OP_ANYNL: + if (check_str_ptr) + detect_partial_match(common, backtracks); + read_char_range(common, common->bsr_nlmin, common->bsr_nlmax, FALSE); + jump[0] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR); + /* We don't need to handle soft partial matching case. */ + end_list = NULL; + if (common->mode != JIT_PARTIAL_HARD_COMPILE) + add_jump(compiler, &end_list, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0)); + else + check_str_end(common, &end_list); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); + jump[1] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + jump[2] = JUMP(SLJIT_JUMP); + JUMPHERE(jump[0]); + check_newlinechar(common, common->bsr_nltype, backtracks, FALSE); + set_jumps(end_list, LABEL()); + JUMPHERE(jump[1]); + JUMPHERE(jump[2]); + return cc; + + case OP_NOT_HSPACE: + case OP_HSPACE: + if (check_str_ptr) + detect_partial_match(common, backtracks); + read_char_range(common, 0x9, 0x3000, type == OP_NOT_HSPACE); + add_jump(compiler, &common->hspace, JUMP(SLJIT_FAST_CALL)); + sljit_set_current_flags(compiler, SLJIT_SET_Z); + add_jump(compiler, backtracks, JUMP(type == OP_NOT_HSPACE ? SLJIT_NOT_ZERO : SLJIT_ZERO)); + return cc; + + case OP_NOT_VSPACE: + case OP_VSPACE: + if (check_str_ptr) + detect_partial_match(common, backtracks); + read_char_range(common, 0xa, 0x2029, type == OP_NOT_VSPACE); + add_jump(compiler, &common->vspace, JUMP(SLJIT_FAST_CALL)); + sljit_set_current_flags(compiler, SLJIT_SET_Z); + add_jump(compiler, backtracks, JUMP(type == OP_NOT_VSPACE ? SLJIT_NOT_ZERO : SLJIT_ZERO)); + return cc; + +#ifdef SUPPORT_UCP + case OP_EXTUNI: + if (check_str_ptr) + detect_partial_match(common, backtracks); + read_char(common); + add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL)); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, gbprop)); + /* Optimize register allocation: use a real register. */ + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STACK_TOP, 0); + OP1(SLJIT_MOV_U8, STACK_TOP, 0, SLJIT_MEM2(TMP1, TMP2), 3); + + label = LABEL(); + jump[0] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0); + OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0); + read_char(common); + add_jump(compiler, &common->getucd, JUMP(SLJIT_FAST_CALL)); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, (sljit_sw)PRIV(ucd_records) + SLJIT_OFFSETOF(ucd_record, gbprop)); + OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM2(TMP1, TMP2), 3); + + OP2(SLJIT_SHL, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, 2); + OP1(SLJIT_MOV_U32, TMP1, 0, SLJIT_MEM1(STACK_TOP), (sljit_sw)PRIV(ucp_gbtable)); + OP1(SLJIT_MOV, STACK_TOP, 0, TMP2, 0); + OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); + JUMPTO(SLJIT_NOT_ZERO, label); + + OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0); + JUMPHERE(jump[0]); + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); + + if (common->mode == JIT_PARTIAL_HARD_COMPILE) + { + jump[0] = CMP(SLJIT_LESS, STR_PTR, 0, STR_END, 0); + /* Since we successfully read a char above, partial matching must occure. */ + check_partial(common, TRUE); + JUMPHERE(jump[0]); + } + return cc; +#endif + + case OP_CHAR: + case OP_CHARI: + length = 1; +#ifdef SUPPORT_UTF + if (common->utf && HAS_EXTRALEN(*cc)) length += GET_EXTRALEN(*cc); +#endif + if (common->mode == JIT_COMPILE && check_str_ptr + && (type == OP_CHAR || !char_has_othercase(common, cc) || char_get_othercase_bit(common, cc) != 0)) + { + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(length)); + add_jump(compiler, backtracks, CMP(SLJIT_GREATER, STR_PTR, 0, STR_END, 0)); + + context.length = IN_UCHARS(length); + context.sourcereg = -1; +#if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED + context.ucharptr = 0; +#endif + return byte_sequence_compare(common, type == OP_CHARI, cc, &context, backtracks); + } + + if (check_str_ptr) + detect_partial_match(common, backtracks); +#ifdef SUPPORT_UTF + if (common->utf) + { + GETCHAR(c, cc); + } + else +#endif + c = *cc; + + if (type == OP_CHAR || !char_has_othercase(common, cc)) + { + read_char_range(common, c, c, FALSE); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, c)); + return cc + length; + } + oc = char_othercase(common, c); + read_char_range(common, c < oc ? c : oc, c > oc ? c : oc, FALSE); + bit = c ^ oc; + if (is_powerof2(bit)) + { + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, bit); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, c | bit)); + return cc + length; + } + jump[0] = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, oc)); + JUMPHERE(jump[0]); + return cc + length; + + case OP_NOT: + case OP_NOTI: + if (check_str_ptr) + detect_partial_match(common, backtracks); + length = 1; +#ifdef SUPPORT_UTF + if (common->utf) + { +#ifdef COMPILE_PCRE8 + c = *cc; + if (c < 128) + { + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(STR_PTR), 0); + if (type == OP_NOT || !char_has_othercase(common, cc)) + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c)); + else + { + /* Since UTF8 code page is fixed, we know that c is in [a-z] or [A-Z] range. */ + OP2(SLJIT_OR, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x20); + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, c | 0x20)); + } + /* Skip the variable-length character. */ + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + jump[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0); + JUMPHERE(jump[0]); + return cc + 1; + } + else +#endif /* COMPILE_PCRE8 */ + { + GETCHARLEN(c, cc, length); + } + } + else +#endif /* SUPPORT_UTF */ + c = *cc; + + if (type == OP_NOT || !char_has_othercase(common, cc)) + { + read_char_range(common, c, c, TRUE); + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c)); + } + else + { + oc = char_othercase(common, c); + read_char_range(common, c < oc ? c : oc, c > oc ? c : oc, TRUE); + bit = c ^ oc; + if (is_powerof2(bit)) + { + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, bit); + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c | bit)); + } + else + { + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c)); + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, oc)); + } + } + return cc + length; + + case OP_CLASS: + case OP_NCLASS: + if (check_str_ptr) + detect_partial_match(common, backtracks); + +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 + bit = (common->utf && is_char7_bitset((const sljit_u8 *)cc, type == OP_NCLASS)) ? 127 : 255; + read_char_range(common, 0, bit, type == OP_NCLASS); +#else + read_char_range(common, 0, 255, type == OP_NCLASS); +#endif + + if (check_class_ranges(common, (const sljit_u8 *)cc, type == OP_NCLASS, FALSE, backtracks)) + return cc + 32 / sizeof(pcre_uchar); + +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 + jump[0] = NULL; + if (common->utf) + { + jump[0] = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, bit); + if (type == OP_CLASS) + { + add_jump(compiler, backtracks, jump[0]); + jump[0] = NULL; + } + } +#elif !defined COMPILE_PCRE8 + jump[0] = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255); + if (type == OP_CLASS) + { + add_jump(compiler, backtracks, jump[0]); + jump[0] = NULL; + } +#endif /* SUPPORT_UTF && COMPILE_PCRE8 */ + + OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7); + OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3); + OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc); + OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0); + OP2(SLJIT_AND | SLJIT_SET_Z, SLJIT_UNUSED, 0, TMP1, 0, TMP2, 0); + add_jump(compiler, backtracks, JUMP(SLJIT_ZERO)); + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + if (jump[0] != NULL) + JUMPHERE(jump[0]); +#endif + return cc + 32 / sizeof(pcre_uchar); + +#if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + case OP_XCLASS: + if (check_str_ptr) + detect_partial_match(common, backtracks); + compile_xclass_matchingpath(common, cc + LINK_SIZE, backtracks); + return cc + GET(cc, 0) - 1; +#endif + } +SLJIT_UNREACHABLE(); +return cc; +} + +static SLJIT_INLINE pcre_uchar *compile_charn_matchingpath(compiler_common *common, pcre_uchar *cc, pcre_uchar *ccend, jump_list **backtracks) +{ +/* This function consumes at least one input character. */ +/* To decrease the number of length checks, we try to concatenate the fixed length character sequences. */ +DEFINE_COMPILER; +pcre_uchar *ccbegin = cc; +compare_context context; +int size; + +context.length = 0; +do + { + if (cc >= ccend) + break; + + if (*cc == OP_CHAR) + { + size = 1; +#ifdef SUPPORT_UTF + if (common->utf && HAS_EXTRALEN(cc[1])) + size += GET_EXTRALEN(cc[1]); +#endif + } + else if (*cc == OP_CHARI) + { + size = 1; +#ifdef SUPPORT_UTF + if (common->utf) + { + if (char_has_othercase(common, cc + 1) && char_get_othercase_bit(common, cc + 1) == 0) + size = 0; + else if (HAS_EXTRALEN(cc[1])) + size += GET_EXTRALEN(cc[1]); + } + else +#endif + if (char_has_othercase(common, cc + 1) && char_get_othercase_bit(common, cc + 1) == 0) + size = 0; + } + else + size = 0; + + cc += 1 + size; + context.length += IN_UCHARS(size); + } +while (size > 0 && context.length <= 128); + +cc = ccbegin; +if (context.length > 0) + { + /* We have a fixed-length byte sequence. */ + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, context.length); + add_jump(compiler, backtracks, CMP(SLJIT_GREATER, STR_PTR, 0, STR_END, 0)); + + context.sourcereg = -1; +#if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED + context.ucharptr = 0; +#endif + do cc = byte_sequence_compare(common, *cc == OP_CHARI, cc + 1, &context, backtracks); while (context.length > 0); + return cc; + } + +/* A non-fixed length character will be checked if length == 0. */ +return compile_char1_matchingpath(common, *cc, cc + 1, backtracks, TRUE); +} + +/* Forward definitions. */ +static void compile_matchingpath(compiler_common *, pcre_uchar *, pcre_uchar *, backtrack_common *); +static void compile_backtrackingpath(compiler_common *, struct backtrack_common *); + +#define PUSH_BACKTRACK(size, ccstart, error) \ + do \ + { \ + backtrack = sljit_alloc_memory(compiler, (size)); \ + if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) \ + return error; \ + memset(backtrack, 0, size); \ + backtrack->prev = parent->top; \ + backtrack->cc = (ccstart); \ + parent->top = backtrack; \ + } \ + while (0) + +#define PUSH_BACKTRACK_NOVALUE(size, ccstart) \ + do \ + { \ + backtrack = sljit_alloc_memory(compiler, (size)); \ + if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) \ + return; \ + memset(backtrack, 0, size); \ + backtrack->prev = parent->top; \ + backtrack->cc = (ccstart); \ + parent->top = backtrack; \ + } \ + while (0) + +#define BACKTRACK_AS(type) ((type *)backtrack) + +static void compile_dnref_search(compiler_common *common, pcre_uchar *cc, jump_list **backtracks) +{ +/* The OVECTOR offset goes to TMP2. */ +DEFINE_COMPILER; +int count = GET2(cc, 1 + IMM2_SIZE); +pcre_uchar *slot = common->name_table + GET2(cc, 1) * common->name_entry_size; +unsigned int offset; +jump_list *found = NULL; + +SLJIT_ASSERT(*cc == OP_DNREF || *cc == OP_DNREFI); + +OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1)); + +count--; +while (count-- > 0) + { + offset = GET2(slot, 0) << 1; + GET_LOCAL_BASE(TMP2, 0, OVECTOR(offset)); + add_jump(compiler, &found, CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0)); + slot += common->name_entry_size; + } + +offset = GET2(slot, 0) << 1; +GET_LOCAL_BASE(TMP2, 0, OVECTOR(offset)); +if (backtracks != NULL && !common->jscript_compat) + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0)); + +set_jumps(found, LABEL()); +} + +static void compile_ref_matchingpath(compiler_common *common, pcre_uchar *cc, jump_list **backtracks, BOOL withchecks, BOOL emptyfail) +{ +DEFINE_COMPILER; +BOOL ref = (*cc == OP_REF || *cc == OP_REFI); +int offset = 0; +struct sljit_jump *jump = NULL; +struct sljit_jump *partial; +struct sljit_jump *nopartial; + +if (ref) + { + offset = GET2(cc, 1) << 1; + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset)); + /* OVECTOR(1) contains the "string begin - 1" constant. */ + if (withchecks && !common->jscript_compat) + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1))); + } +else + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), 0); + +#if defined SUPPORT_UTF && defined SUPPORT_UCP +if (common->utf && *cc == OP_REFI) + { + SLJIT_ASSERT(TMP1 == SLJIT_R0 && STACK_TOP == SLJIT_R1 && TMP2 == SLJIT_R2); + if (ref) + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); + else + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw)); + + if (withchecks) + jump = CMP(SLJIT_EQUAL, TMP1, 0, TMP2, 0); + + /* Needed to save important temporary registers. */ + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STACK_TOP, 0); + OP1(SLJIT_MOV, SLJIT_R1, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_R1), SLJIT_OFFSETOF(jit_arguments, uchar_ptr), STR_PTR, 0); + sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_utf_caselesscmp)); + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); + if (common->mode == JIT_COMPILE) + add_jump(compiler, backtracks, CMP(SLJIT_LESS_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 1)); + else + { + add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0)); + nopartial = CMP(SLJIT_NOT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 1); + check_partial(common, FALSE); + add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); + JUMPHERE(nopartial); + } + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_RETURN_REG, 0); + } +else +#endif /* SUPPORT_UTF && SUPPORT_UCP */ + { + if (ref) + OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP1, 0); + else + OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw), TMP1, 0); + + if (withchecks) + jump = JUMP(SLJIT_ZERO); + + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0); + partial = CMP(SLJIT_GREATER, STR_PTR, 0, STR_END, 0); + if (common->mode == JIT_COMPILE) + add_jump(compiler, backtracks, partial); + + add_jump(compiler, *cc == OP_REF ? &common->casefulcmp : &common->caselesscmp, JUMP(SLJIT_FAST_CALL)); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0)); + + if (common->mode != JIT_COMPILE) + { + nopartial = JUMP(SLJIT_JUMP); + JUMPHERE(partial); + /* TMP2 -= STR_END - STR_PTR */ + OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, STR_PTR, 0); + OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, STR_END, 0); + partial = CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, 0); + OP1(SLJIT_MOV, STR_PTR, 0, STR_END, 0); + add_jump(compiler, *cc == OP_REF ? &common->casefulcmp : &common->caselesscmp, JUMP(SLJIT_FAST_CALL)); + add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0)); + JUMPHERE(partial); + check_partial(common, FALSE); + add_jump(compiler, backtracks, JUMP(SLJIT_JUMP)); + JUMPHERE(nopartial); + } + } + +if (jump != NULL) + { + if (emptyfail) + add_jump(compiler, backtracks, jump); + else + JUMPHERE(jump); + } +} + +static SLJIT_INLINE pcre_uchar *compile_ref_iterator_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent) +{ +DEFINE_COMPILER; +BOOL ref = (*cc == OP_REF || *cc == OP_REFI); +backtrack_common *backtrack; +pcre_uchar type; +int offset = 0; +struct sljit_label *label; +struct sljit_jump *zerolength; +struct sljit_jump *jump = NULL; +pcre_uchar *ccbegin = cc; +int min = 0, max = 0; +BOOL minimize; + +PUSH_BACKTRACK(sizeof(ref_iterator_backtrack), cc, NULL); + +if (ref) + offset = GET2(cc, 1) << 1; +else + cc += IMM2_SIZE; +type = cc[1 + IMM2_SIZE]; + +SLJIT_COMPILE_ASSERT((OP_CRSTAR & 0x1) == 0, crstar_opcode_must_be_even); +minimize = (type & 0x1) != 0; +switch(type) + { + case OP_CRSTAR: + case OP_CRMINSTAR: + min = 0; + max = 0; + cc += 1 + IMM2_SIZE + 1; + break; + case OP_CRPLUS: + case OP_CRMINPLUS: + min = 1; + max = 0; + cc += 1 + IMM2_SIZE + 1; + break; + case OP_CRQUERY: + case OP_CRMINQUERY: + min = 0; + max = 1; + cc += 1 + IMM2_SIZE + 1; + break; + case OP_CRRANGE: + case OP_CRMINRANGE: + min = GET2(cc, 1 + IMM2_SIZE + 1); + max = GET2(cc, 1 + IMM2_SIZE + 1 + IMM2_SIZE); + cc += 1 + IMM2_SIZE + 1 + 2 * IMM2_SIZE; + break; + default: + SLJIT_UNREACHABLE(); + break; + } + +if (!minimize) + { + if (min == 0) + { + allocate_stack(common, 2); + if (ref) + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset)); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0); + /* Temporary release of STR_PTR. */ + OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); + /* Handles both invalid and empty cases. Since the minimum repeat, + is zero the invalid case is basically the same as an empty case. */ + if (ref) + zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); + else + { + compile_dnref_search(common, ccbegin, NULL); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1, TMP2, 0); + zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw)); + } + /* Restore if not zero length. */ + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); + } + else + { + allocate_stack(common, 1); + if (ref) + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset)); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); + if (ref) + { + add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1))); + zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); + } + else + { + compile_dnref_search(common, ccbegin, &backtrack->topbacktracks); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1, TMP2, 0); + zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw)); + } + } + + if (min > 1 || max > 1) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0, SLJIT_IMM, 0); + + label = LABEL(); + if (!ref) + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1); + compile_ref_matchingpath(common, ccbegin, &backtrack->topbacktracks, FALSE, FALSE); + + if (min > 1 || max > 1) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0); + OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0, TMP1, 0); + if (min > 1) + CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, min, label); + if (max > 1) + { + jump = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, max); + allocate_stack(common, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); + JUMPTO(SLJIT_JUMP, label); + JUMPHERE(jump); + } + } + + if (max == 0) + { + /* Includes min > 1 case as well. */ + allocate_stack(common, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); + JUMPTO(SLJIT_JUMP, label); + } + + JUMPHERE(zerolength); + BACKTRACK_AS(ref_iterator_backtrack)->matchingpath = LABEL(); + + count_match(common); + return cc; + } + +allocate_stack(common, ref ? 2 : 3); +if (ref) + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset)); +OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); +if (type != OP_CRMINSTAR) + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0); + +if (min == 0) + { + /* Handles both invalid and empty cases. Since the minimum repeat, + is zero the invalid case is basically the same as an empty case. */ + if (ref) + zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); + else + { + compile_dnref_search(common, ccbegin, NULL); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), 0); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(2), TMP2, 0); + zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw)); + } + /* Length is non-zero, we can match real repeats. */ + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); + jump = JUMP(SLJIT_JUMP); + } +else + { + if (ref) + { + add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1))); + zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); + } + else + { + compile_dnref_search(common, ccbegin, &backtrack->topbacktracks); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), 0); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(2), TMP2, 0); + zerolength = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_MEM1(TMP2), sizeof(sljit_sw)); + } + } + +BACKTRACK_AS(ref_iterator_backtrack)->matchingpath = LABEL(); +if (max > 0) + add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, max)); + +if (!ref) + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(2)); +compile_ref_matchingpath(common, ccbegin, &backtrack->topbacktracks, TRUE, TRUE); +OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); + +if (min > 1) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); + OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0); + CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, min, BACKTRACK_AS(ref_iterator_backtrack)->matchingpath); + } +else if (max > 0) + OP2(SLJIT_ADD, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 1); + +if (jump != NULL) + JUMPHERE(jump); +JUMPHERE(zerolength); + +count_match(common); +return cc; +} + +static SLJIT_INLINE pcre_uchar *compile_recurse_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent) +{ +DEFINE_COMPILER; +backtrack_common *backtrack; +recurse_entry *entry = common->entries; +recurse_entry *prev = NULL; +sljit_sw start = GET(cc, 1); +pcre_uchar *start_cc; +BOOL needs_control_head; + +PUSH_BACKTRACK(sizeof(recurse_backtrack), cc, NULL); + +/* Inlining simple patterns. */ +if (get_framesize(common, common->start + start, NULL, TRUE, &needs_control_head) == no_stack) + { + start_cc = common->start + start; + compile_matchingpath(common, next_opcode(common, start_cc), bracketend(start_cc) - (1 + LINK_SIZE), backtrack); + BACKTRACK_AS(recurse_backtrack)->inlined_pattern = TRUE; + return cc + 1 + LINK_SIZE; + } + +while (entry != NULL) + { + if (entry->start == start) + break; + prev = entry; + entry = entry->next; + } + +if (entry == NULL) + { + entry = sljit_alloc_memory(compiler, sizeof(recurse_entry)); + if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) + return NULL; + entry->next = NULL; + entry->entry = NULL; + entry->calls = NULL; + entry->start = start; + + if (prev != NULL) + prev->next = entry; + else + common->entries = entry; + } + +if (common->has_set_som && common->mark_ptr != 0) + { + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0)); + allocate_stack(common, 2); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0); + } +else if (common->has_set_som || common->mark_ptr != 0) + { + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->has_set_som ? (int)(OVECTOR(0)) : common->mark_ptr); + allocate_stack(common, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); + } + +if (entry->entry == NULL) + add_jump(compiler, &entry->calls, JUMP(SLJIT_FAST_CALL)); +else + JUMPTO(SLJIT_FAST_CALL, entry->entry); +/* Leave if the match is failed. */ +add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0)); +return cc + 1 + LINK_SIZE; +} + +static int SLJIT_CALL do_callout(struct jit_arguments *arguments, PUBL(callout_block) *callout_block, pcre_uchar **jit_ovector) +{ +const pcre_uchar *begin = arguments->begin; +int *offset_vector = arguments->offsets; +int offset_count = arguments->offset_count; +int i; + +if (PUBL(callout) == NULL) + return 0; + +callout_block->version = 2; +callout_block->callout_data = arguments->callout_data; + +/* Offsets in subject. */ +callout_block->subject_length = arguments->end - arguments->begin; +callout_block->start_match = (pcre_uchar*)callout_block->subject - arguments->begin; +callout_block->current_position = (pcre_uchar*)callout_block->offset_vector - arguments->begin; +#if defined COMPILE_PCRE8 +callout_block->subject = (PCRE_SPTR)begin; +#elif defined COMPILE_PCRE16 +callout_block->subject = (PCRE_SPTR16)begin; +#elif defined COMPILE_PCRE32 +callout_block->subject = (PCRE_SPTR32)begin; +#endif + +/* Convert and copy the JIT offset vector to the offset_vector array. */ +callout_block->capture_top = 0; +callout_block->offset_vector = offset_vector; +for (i = 2; i < offset_count; i += 2) + { + offset_vector[i] = jit_ovector[i] - begin; + offset_vector[i + 1] = jit_ovector[i + 1] - begin; + if (jit_ovector[i] >= begin) + callout_block->capture_top = i; + } + +callout_block->capture_top = (callout_block->capture_top >> 1) + 1; +if (offset_count > 0) + offset_vector[0] = -1; +if (offset_count > 1) + offset_vector[1] = -1; +return (*PUBL(callout))(callout_block); +} + +/* Aligning to 8 byte. */ +#define CALLOUT_ARG_SIZE \ + (((int)sizeof(PUBL(callout_block)) + 7) & ~7) + +#define CALLOUT_ARG_OFFSET(arg) \ + SLJIT_OFFSETOF(PUBL(callout_block), arg) + +static SLJIT_INLINE pcre_uchar *compile_callout_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent) +{ +DEFINE_COMPILER; +backtrack_common *backtrack; + +PUSH_BACKTRACK(sizeof(backtrack_common), cc, NULL); + +allocate_stack(common, CALLOUT_ARG_SIZE / sizeof(sljit_sw)); + +SLJIT_ASSERT(common->capture_last_ptr != 0); +OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr); +OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); +OP1(SLJIT_MOV_S32, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(callout_number), SLJIT_IMM, cc[1]); +OP1(SLJIT_MOV_S32, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(capture_last), TMP2, 0); + +/* These pointer sized fields temporarly stores internal variables. */ +OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0)); +OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(offset_vector), STR_PTR, 0); +OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(subject), TMP2, 0); + +if (common->mark_ptr != 0) + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, mark_ptr)); +OP1(SLJIT_MOV_S32, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(pattern_position), SLJIT_IMM, GET(cc, 2)); +OP1(SLJIT_MOV_S32, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(next_item_length), SLJIT_IMM, GET(cc, 2 + LINK_SIZE)); +OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), CALLOUT_ARG_OFFSET(mark), (common->mark_ptr != 0) ? TMP2 : SLJIT_IMM, 0); + +/* Needed to save important temporary registers. */ +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STACK_TOP, 0); +/* SLJIT_R0 = arguments */ +OP1(SLJIT_MOV, SLJIT_R1, 0, STACK_TOP, 0); +GET_LOCAL_BASE(SLJIT_R2, 0, OVECTOR_START); +sljit_emit_ijump(compiler, SLJIT_CALL3, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_callout)); +OP1(SLJIT_MOV_S32, SLJIT_RETURN_REG, 0, SLJIT_RETURN_REG, 0); +OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); +free_stack(common, CALLOUT_ARG_SIZE / sizeof(sljit_sw)); + +/* Check return value. */ +OP2(SLJIT_SUB | SLJIT_SET_Z | SLJIT_SET_SIG_GREATER, SLJIT_UNUSED, 0, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0); +add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_SIG_GREATER)); +if (common->forced_quit_label == NULL) + add_jump(compiler, &common->forced_quit, JUMP(SLJIT_NOT_EQUAL) /* SIG_LESS */); +else + JUMPTO(SLJIT_NOT_EQUAL /* SIG_LESS */, common->forced_quit_label); +return cc + 2 + 2 * LINK_SIZE; +} + +#undef CALLOUT_ARG_SIZE +#undef CALLOUT_ARG_OFFSET + +static SLJIT_INLINE BOOL assert_needs_str_ptr_saving(pcre_uchar *cc) +{ +while (TRUE) + { + switch (*cc) + { + case OP_NOT_WORD_BOUNDARY: + case OP_WORD_BOUNDARY: + case OP_CIRC: + case OP_CIRCM: + case OP_DOLL: + case OP_DOLLM: + case OP_CALLOUT: + case OP_ALT: + cc += PRIV(OP_lengths)[*cc]; + break; + + case OP_KET: + return FALSE; + + default: + return TRUE; + } + } +} + +static pcre_uchar *compile_assert_matchingpath(compiler_common *common, pcre_uchar *cc, assert_backtrack *backtrack, BOOL conditional) +{ +DEFINE_COMPILER; +int framesize; +int extrasize; +BOOL needs_control_head; +int private_data_ptr; +backtrack_common altbacktrack; +pcre_uchar *ccbegin; +pcre_uchar opcode; +pcre_uchar bra = OP_BRA; +jump_list *tmp = NULL; +jump_list **target = (conditional) ? &backtrack->condfailed : &backtrack->common.topbacktracks; +jump_list **found; +/* Saving previous accept variables. */ +BOOL save_local_exit = common->local_exit; +BOOL save_positive_assert = common->positive_assert; +then_trap_backtrack *save_then_trap = common->then_trap; +struct sljit_label *save_quit_label = common->quit_label; +struct sljit_label *save_accept_label = common->accept_label; +jump_list *save_quit = common->quit; +jump_list *save_positive_assert_quit = common->positive_assert_quit; +jump_list *save_accept = common->accept; +struct sljit_jump *jump; +struct sljit_jump *brajump = NULL; + +/* Assert captures then. */ +common->then_trap = NULL; + +if (*cc == OP_BRAZERO || *cc == OP_BRAMINZERO) + { + SLJIT_ASSERT(!conditional); + bra = *cc; + cc++; + } +private_data_ptr = PRIVATE_DATA(cc); +SLJIT_ASSERT(private_data_ptr != 0); +framesize = get_framesize(common, cc, NULL, FALSE, &needs_control_head); +backtrack->framesize = framesize; +backtrack->private_data_ptr = private_data_ptr; +opcode = *cc; +SLJIT_ASSERT(opcode >= OP_ASSERT && opcode <= OP_ASSERTBACK_NOT); +found = (opcode == OP_ASSERT || opcode == OP_ASSERTBACK) ? &tmp : target; +ccbegin = cc; +cc += GET(cc, 1); + +if (bra == OP_BRAMINZERO) + { + /* This is a braminzero backtrack path. */ + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + free_stack(common, 1); + brajump = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0); + } + +if (framesize < 0) + { + extrasize = 1; + if (bra == OP_BRA && !assert_needs_str_ptr_saving(ccbegin + 1 + LINK_SIZE)) + extrasize = 0; + + if (needs_control_head) + extrasize++; + + if (framesize == no_frame) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0); + + if (extrasize > 0) + allocate_stack(common, extrasize); + + if (needs_control_head) + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); + + if (extrasize > 0) + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); + + if (needs_control_head) + { + SLJIT_ASSERT(extrasize == 2); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0); + } + } +else + { + extrasize = needs_control_head ? 3 : 2; + allocate_stack(common, framesize + extrasize); + + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + OP2(SLJIT_ADD, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + extrasize) * sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP2, 0); + if (needs_control_head) + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); + + if (needs_control_head) + { + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(2), TMP1, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP2, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0); + } + else + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0); + + init_frame(common, ccbegin, NULL, framesize + extrasize - 1, extrasize, FALSE); + } + +memset(&altbacktrack, 0, sizeof(backtrack_common)); +if (opcode == OP_ASSERT_NOT || opcode == OP_ASSERTBACK_NOT) + { + /* Negative assert is stronger than positive assert. */ + common->local_exit = TRUE; + common->quit_label = NULL; + common->quit = NULL; + common->positive_assert = FALSE; + } +else + common->positive_assert = TRUE; +common->positive_assert_quit = NULL; + +while (1) + { + common->accept_label = NULL; + common->accept = NULL; + altbacktrack.top = NULL; + altbacktrack.topbacktracks = NULL; + + if (*ccbegin == OP_ALT && extrasize > 0) + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + + altbacktrack.cc = ccbegin; + compile_matchingpath(common, ccbegin + 1 + LINK_SIZE, cc, &altbacktrack); + if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) + { + if (opcode == OP_ASSERT_NOT || opcode == OP_ASSERTBACK_NOT) + { + common->local_exit = save_local_exit; + common->quit_label = save_quit_label; + common->quit = save_quit; + } + common->positive_assert = save_positive_assert; + common->then_trap = save_then_trap; + common->accept_label = save_accept_label; + common->positive_assert_quit = save_positive_assert_quit; + common->accept = save_accept; + return NULL; + } + common->accept_label = LABEL(); + if (common->accept != NULL) + set_jumps(common->accept, common->accept_label); + + /* Reset stack. */ + if (framesize < 0) + { + if (framesize == no_frame) + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + else if (extrasize > 0) + free_stack(common, extrasize); + + if (needs_control_head) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(-1)); + } + else + { + if ((opcode != OP_ASSERT_NOT && opcode != OP_ASSERTBACK_NOT) || conditional) + { + /* We don't need to keep the STR_PTR, only the previous private_data_ptr. */ + OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw)); + if (needs_control_head) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(-1)); + } + else + { + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + if (needs_control_head) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(-framesize - 2)); + add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); + } + } + + if (opcode == OP_ASSERT_NOT || opcode == OP_ASSERTBACK_NOT) + { + /* We know that STR_PTR was stored on the top of the stack. */ + if (conditional) + { + if (extrasize > 0) + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), needs_control_head ? STACK(-2) : STACK(-1)); + } + else if (bra == OP_BRAZERO) + { + if (framesize < 0) + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(-extrasize)); + else + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(-framesize - 1)); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(-framesize - extrasize)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0); + } + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); + } + else if (framesize >= 0) + { + /* For OP_BRA and OP_BRAMINZERO. */ + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-framesize - 1)); + } + } + add_jump(compiler, found, JUMP(SLJIT_JUMP)); + + compile_backtrackingpath(common, altbacktrack.top); + if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) + { + if (opcode == OP_ASSERT_NOT || opcode == OP_ASSERTBACK_NOT) + { + common->local_exit = save_local_exit; + common->quit_label = save_quit_label; + common->quit = save_quit; + } + common->positive_assert = save_positive_assert; + common->then_trap = save_then_trap; + common->accept_label = save_accept_label; + common->positive_assert_quit = save_positive_assert_quit; + common->accept = save_accept; + return NULL; + } + set_jumps(altbacktrack.topbacktracks, LABEL()); + + if (*cc != OP_ALT) + break; + + ccbegin = cc; + cc += GET(cc, 1); + } + +if (opcode == OP_ASSERT_NOT || opcode == OP_ASSERTBACK_NOT) + { + SLJIT_ASSERT(common->positive_assert_quit == NULL); + /* Makes the check less complicated below. */ + common->positive_assert_quit = common->quit; + } + +/* None of them matched. */ +if (common->positive_assert_quit != NULL) + { + jump = JUMP(SLJIT_JUMP); + set_jumps(common->positive_assert_quit, LABEL()); + SLJIT_ASSERT(framesize != no_stack); + if (framesize < 0) + OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, extrasize * sizeof(sljit_sw)); + else + { + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + extrasize) * sizeof(sljit_sw)); + } + JUMPHERE(jump); + } + +if (needs_control_head) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(1)); + +if (opcode == OP_ASSERT || opcode == OP_ASSERTBACK) + { + /* Assert is failed. */ + if ((conditional && extrasize > 0) || bra == OP_BRAZERO) + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + + if (framesize < 0) + { + /* The topmost item should be 0. */ + if (bra == OP_BRAZERO) + { + if (extrasize == 2) + free_stack(common, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); + } + else if (extrasize > 0) + free_stack(common, extrasize); + } + else + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(extrasize - 1)); + /* The topmost item should be 0. */ + if (bra == OP_BRAZERO) + { + free_stack(common, framesize + extrasize - 1); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); + } + else + free_stack(common, framesize + extrasize); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0); + } + jump = JUMP(SLJIT_JUMP); + if (bra != OP_BRAZERO) + add_jump(compiler, target, jump); + + /* Assert is successful. */ + set_jumps(tmp, LABEL()); + if (framesize < 0) + { + /* We know that STR_PTR was stored on the top of the stack. */ + if (extrasize > 0) + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(-extrasize)); + + /* Keep the STR_PTR on the top of the stack. */ + if (bra == OP_BRAZERO) + { + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); + if (extrasize == 2) + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); + } + else if (bra == OP_BRAMINZERO) + { + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); + } + } + else + { + if (bra == OP_BRA) + { + /* We don't need to keep the STR_PTR, only the previous private_data_ptr. */ + OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 1) * sizeof(sljit_sw)); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(-extrasize + 1)); + } + else + { + /* We don't need to keep the STR_PTR, only the previous private_data_ptr. */ + OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + 2) * sizeof(sljit_sw)); + if (extrasize == 2) + { + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + if (bra == OP_BRAMINZERO) + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); + } + else + { + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), 0); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), bra == OP_BRAZERO ? STR_PTR : SLJIT_IMM, 0); + } + } + } + + if (bra == OP_BRAZERO) + { + backtrack->matchingpath = LABEL(); + SET_LABEL(jump, backtrack->matchingpath); + } + else if (bra == OP_BRAMINZERO) + { + JUMPTO(SLJIT_JUMP, backtrack->matchingpath); + JUMPHERE(brajump); + if (framesize >= 0) + { + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-framesize - 1)); + } + set_jumps(backtrack->common.topbacktracks, LABEL()); + } + } +else + { + /* AssertNot is successful. */ + if (framesize < 0) + { + if (extrasize > 0) + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + + if (bra != OP_BRA) + { + if (extrasize == 2) + free_stack(common, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); + } + else if (extrasize > 0) + free_stack(common, extrasize); + } + else + { + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(extrasize - 1)); + /* The topmost item should be 0. */ + if (bra != OP_BRA) + { + free_stack(common, framesize + extrasize - 1); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); + } + else + free_stack(common, framesize + extrasize); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0); + } + + if (bra == OP_BRAZERO) + backtrack->matchingpath = LABEL(); + else if (bra == OP_BRAMINZERO) + { + JUMPTO(SLJIT_JUMP, backtrack->matchingpath); + JUMPHERE(brajump); + } + + if (bra != OP_BRA) + { + SLJIT_ASSERT(found == &backtrack->common.topbacktracks); + set_jumps(backtrack->common.topbacktracks, LABEL()); + backtrack->common.topbacktracks = NULL; + } + } + +if (opcode == OP_ASSERT_NOT || opcode == OP_ASSERTBACK_NOT) + { + common->local_exit = save_local_exit; + common->quit_label = save_quit_label; + common->quit = save_quit; + } +common->positive_assert = save_positive_assert; +common->then_trap = save_then_trap; +common->accept_label = save_accept_label; +common->positive_assert_quit = save_positive_assert_quit; +common->accept = save_accept; +return cc + 1 + LINK_SIZE; +} + +static SLJIT_INLINE void match_once_common(compiler_common *common, pcre_uchar ket, int framesize, int private_data_ptr, BOOL has_alternatives, BOOL needs_control_head) +{ +DEFINE_COMPILER; +int stacksize; + +if (framesize < 0) + { + if (framesize == no_frame) + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + else + { + stacksize = needs_control_head ? 1 : 0; + if (ket != OP_KET || has_alternatives) + stacksize++; + + if (stacksize > 0) + free_stack(common, stacksize); + } + + if (needs_control_head) + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), (ket != OP_KET || has_alternatives) ? STACK(-2) : STACK(-1)); + + /* TMP2 which is set here used by OP_KETRMAX below. */ + if (ket == OP_KETRMAX) + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(-1)); + else if (ket == OP_KETRMIN) + { + /* Move the STR_PTR to the private_data_ptr. */ + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-1)); + } + } +else + { + stacksize = (ket != OP_KET || has_alternatives) ? 2 : 1; + OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, (framesize + stacksize) * sizeof(sljit_sw)); + if (needs_control_head) + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(-1)); + + if (ket == OP_KETRMAX) + { + /* TMP2 which is set here used by OP_KETRMAX below. */ + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + } + } +if (needs_control_head) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, TMP1, 0); +} + +static SLJIT_INLINE int match_capture_common(compiler_common *common, int stacksize, int offset, int private_data_ptr) +{ +DEFINE_COMPILER; + +if (common->capture_last_ptr != 0) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, SLJIT_IMM, offset >> 1); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), TMP1, 0); + stacksize++; + } +if (common->optimized_cbracket[offset >> 1] == 0) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset)); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), TMP1, 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize + 1), TMP2, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0); + stacksize += 2; + } +return stacksize; +} + +/* + Handling bracketed expressions is probably the most complex part. + + Stack layout naming characters: + S - Push the current STR_PTR + 0 - Push a 0 (NULL) + A - Push the current STR_PTR. Needed for restoring the STR_PTR + before the next alternative. Not pushed if there are no alternatives. + M - Any values pushed by the current alternative. Can be empty, or anything. + C - Push the previous OVECTOR(i), OVECTOR(i+1) and OVECTOR_PRIV(i) to the stack. + L - Push the previous local (pointed by localptr) to the stack + () - opional values stored on the stack + ()* - optonal, can be stored multiple times + + The following list shows the regular expression templates, their PCRE byte codes + and stack layout supported by pcre-sljit. + + (?:) OP_BRA | OP_KET A M + () OP_CBRA | OP_KET C M + (?:)+ OP_BRA | OP_KETRMAX 0 A M S ( A M S )* + OP_SBRA | OP_KETRMAX 0 L M S ( L M S )* + (?:)+? OP_BRA | OP_KETRMIN 0 A M S ( A M S )* + OP_SBRA | OP_KETRMIN 0 L M S ( L M S )* + ()+ OP_CBRA | OP_KETRMAX 0 C M S ( C M S )* + OP_SCBRA | OP_KETRMAX 0 C M S ( C M S )* + ()+? OP_CBRA | OP_KETRMIN 0 C M S ( C M S )* + OP_SCBRA | OP_KETRMIN 0 C M S ( C M S )* + (?:)? OP_BRAZERO | OP_BRA | OP_KET S ( A M 0 ) + (?:)?? OP_BRAMINZERO | OP_BRA | OP_KET S ( A M 0 ) + ()? OP_BRAZERO | OP_CBRA | OP_KET S ( C M 0 ) + ()?? OP_BRAMINZERO | OP_CBRA | OP_KET S ( C M 0 ) + (?:)* OP_BRAZERO | OP_BRA | OP_KETRMAX S 0 ( A M S )* + OP_BRAZERO | OP_SBRA | OP_KETRMAX S 0 ( L M S )* + (?:)*? OP_BRAMINZERO | OP_BRA | OP_KETRMIN S 0 ( A M S )* + OP_BRAMINZERO | OP_SBRA | OP_KETRMIN S 0 ( L M S )* + ()* OP_BRAZERO | OP_CBRA | OP_KETRMAX S 0 ( C M S )* + OP_BRAZERO | OP_SCBRA | OP_KETRMAX S 0 ( C M S )* + ()*? OP_BRAMINZERO | OP_CBRA | OP_KETRMIN S 0 ( C M S )* + OP_BRAMINZERO | OP_SCBRA | OP_KETRMIN S 0 ( C M S )* + + + Stack layout naming characters: + A - Push the alternative index (starting from 0) on the stack. + Not pushed if there is no alternatives. + M - Any values pushed by the current alternative. Can be empty, or anything. + + The next list shows the possible content of a bracket: + (|) OP_*BRA | OP_ALT ... M A + (?()|) OP_*COND | OP_ALT M A + (?>|) OP_ONCE | OP_ALT ... [stack trace] M A + (?>|) OP_ONCE_NC | OP_ALT ... [stack trace] M A + Or nothing, if trace is unnecessary +*/ + +static pcre_uchar *compile_bracket_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent) +{ +DEFINE_COMPILER; +backtrack_common *backtrack; +pcre_uchar opcode; +int private_data_ptr = 0; +int offset = 0; +int i, stacksize; +int repeat_ptr = 0, repeat_length = 0; +int repeat_type = 0, repeat_count = 0; +pcre_uchar *ccbegin; +pcre_uchar *matchingpath; +pcre_uchar *slot; +pcre_uchar bra = OP_BRA; +pcre_uchar ket; +assert_backtrack *assert; +BOOL has_alternatives; +BOOL needs_control_head = FALSE; +struct sljit_jump *jump; +struct sljit_jump *skip; +struct sljit_label *rmax_label = NULL; +struct sljit_jump *braminzero = NULL; + +PUSH_BACKTRACK(sizeof(bracket_backtrack), cc, NULL); + +if (*cc == OP_BRAZERO || *cc == OP_BRAMINZERO) + { + bra = *cc; + cc++; + opcode = *cc; + } + +opcode = *cc; +ccbegin = cc; +matchingpath = bracketend(cc) - 1 - LINK_SIZE; +ket = *matchingpath; +if (ket == OP_KET && PRIVATE_DATA(matchingpath) != 0) + { + repeat_ptr = PRIVATE_DATA(matchingpath); + repeat_length = PRIVATE_DATA(matchingpath + 1); + repeat_type = PRIVATE_DATA(matchingpath + 2); + repeat_count = PRIVATE_DATA(matchingpath + 3); + SLJIT_ASSERT(repeat_length != 0 && repeat_type != 0 && repeat_count != 0); + if (repeat_type == OP_UPTO) + ket = OP_KETRMAX; + if (repeat_type == OP_MINUPTO) + ket = OP_KETRMIN; + } + +if ((opcode == OP_COND || opcode == OP_SCOND) && cc[1 + LINK_SIZE] == OP_DEF) + { + /* Drop this bracket_backtrack. */ + parent->top = backtrack->prev; + return matchingpath + 1 + LINK_SIZE + repeat_length; + } + +matchingpath = ccbegin + 1 + LINK_SIZE; +SLJIT_ASSERT(ket == OP_KET || ket == OP_KETRMAX || ket == OP_KETRMIN); +SLJIT_ASSERT(!((bra == OP_BRAZERO && ket == OP_KETRMIN) || (bra == OP_BRAMINZERO && ket == OP_KETRMAX))); +cc += GET(cc, 1); + +has_alternatives = *cc == OP_ALT; +if (SLJIT_UNLIKELY(opcode == OP_COND || opcode == OP_SCOND)) + has_alternatives = (*matchingpath == OP_RREF || *matchingpath == OP_DNRREF || *matchingpath == OP_FAIL) ? FALSE : TRUE; + +if (SLJIT_UNLIKELY(opcode == OP_COND) && (*cc == OP_KETRMAX || *cc == OP_KETRMIN)) + opcode = OP_SCOND; +if (SLJIT_UNLIKELY(opcode == OP_ONCE_NC)) + opcode = OP_ONCE; + +if (opcode == OP_CBRA || opcode == OP_SCBRA) + { + /* Capturing brackets has a pre-allocated space. */ + offset = GET2(ccbegin, 1 + LINK_SIZE); + if (common->optimized_cbracket[offset] == 0) + { + private_data_ptr = OVECTOR_PRIV(offset); + offset <<= 1; + } + else + { + offset <<= 1; + private_data_ptr = OVECTOR(offset); + } + BACKTRACK_AS(bracket_backtrack)->private_data_ptr = private_data_ptr; + matchingpath += IMM2_SIZE; + } +else if (opcode == OP_ONCE || opcode == OP_SBRA || opcode == OP_SCOND) + { + /* Other brackets simply allocate the next entry. */ + private_data_ptr = PRIVATE_DATA(ccbegin); + SLJIT_ASSERT(private_data_ptr != 0); + BACKTRACK_AS(bracket_backtrack)->private_data_ptr = private_data_ptr; + if (opcode == OP_ONCE) + BACKTRACK_AS(bracket_backtrack)->u.framesize = get_framesize(common, ccbegin, NULL, FALSE, &needs_control_head); + } + +/* Instructions before the first alternative. */ +stacksize = 0; +if (ket == OP_KETRMAX || (ket == OP_KETRMIN && bra != OP_BRAMINZERO)) + stacksize++; +if (bra == OP_BRAZERO) + stacksize++; + +if (stacksize > 0) + allocate_stack(common, stacksize); + +stacksize = 0; +if (ket == OP_KETRMAX || (ket == OP_KETRMIN && bra != OP_BRAMINZERO)) + { + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0); + stacksize++; + } + +if (bra == OP_BRAZERO) + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0); + +if (bra == OP_BRAMINZERO) + { + /* This is a backtrack path! (Since the try-path of OP_BRAMINZERO matches to the empty string) */ + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + if (ket != OP_KETRMIN) + { + free_stack(common, 1); + braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0); + } + else + { + if (opcode == OP_ONCE || opcode >= OP_SBRA) + { + jump = CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); + /* Nothing stored during the first run. */ + skip = JUMP(SLJIT_JUMP); + JUMPHERE(jump); + /* Checking zero-length iteration. */ + if (opcode != OP_ONCE || BACKTRACK_AS(bracket_backtrack)->u.framesize < 0) + { + /* When we come from outside, private_data_ptr contains the previous STR_PTR. */ + braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + } + else + { + /* Except when the whole stack frame must be saved. */ + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + braminzero = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), STACK(-BACKTRACK_AS(bracket_backtrack)->u.framesize - 2)); + } + JUMPHERE(skip); + } + else + { + jump = CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); + JUMPHERE(jump); + } + } + } + +if (repeat_type != 0) + { + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, repeat_count); + if (repeat_type == OP_EXACT) + rmax_label = LABEL(); + } + +if (ket == OP_KETRMIN) + BACKTRACK_AS(bracket_backtrack)->recursive_matchingpath = LABEL(); + +if (ket == OP_KETRMAX) + { + rmax_label = LABEL(); + if (has_alternatives && opcode != OP_ONCE && opcode < OP_SBRA && repeat_type == 0) + BACKTRACK_AS(bracket_backtrack)->alternative_matchingpath = rmax_label; + } + +/* Handling capturing brackets and alternatives. */ +if (opcode == OP_ONCE) + { + stacksize = 0; + if (needs_control_head) + { + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); + stacksize++; + } + + if (BACKTRACK_AS(bracket_backtrack)->u.framesize < 0) + { + /* Neither capturing brackets nor recursions are found in the block. */ + if (ket == OP_KETRMIN) + { + stacksize += 2; + if (!needs_control_head) + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + } + else + { + if (BACKTRACK_AS(bracket_backtrack)->u.framesize == no_frame) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0); + if (ket == OP_KETRMAX || has_alternatives) + stacksize++; + } + + if (stacksize > 0) + allocate_stack(common, stacksize); + + stacksize = 0; + if (needs_control_head) + { + stacksize++; + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); + } + + if (ket == OP_KETRMIN) + { + if (needs_control_head) + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0); + if (BACKTRACK_AS(bracket_backtrack)->u.framesize == no_frame) + OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, needs_control_head ? (2 * sizeof(sljit_sw)) : sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize + 1), TMP2, 0); + } + else if (ket == OP_KETRMAX || has_alternatives) + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0); + } + else + { + if (ket != OP_KET || has_alternatives) + stacksize++; + + stacksize += BACKTRACK_AS(bracket_backtrack)->u.framesize + 1; + allocate_stack(common, stacksize); + + if (needs_control_head) + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); + + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + OP2(SLJIT_ADD, TMP2, 0, STACK_TOP, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw)); + + stacksize = needs_control_head ? 1 : 0; + if (ket != OP_KET || has_alternatives) + { + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP2, 0); + stacksize++; + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), TMP1, 0); + } + else + { + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP2, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), TMP1, 0); + } + init_frame(common, ccbegin, NULL, BACKTRACK_AS(bracket_backtrack)->u.framesize + stacksize, stacksize + 1, FALSE); + } + } +else if (opcode == OP_CBRA || opcode == OP_SCBRA) + { + /* Saving the previous values. */ + if (common->optimized_cbracket[offset >> 1] != 0) + { + SLJIT_ASSERT(private_data_ptr == OVECTOR(offset)); + allocate_stack(common, 2); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr + sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STR_PTR, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP2, 0); + } + else + { + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + allocate_stack(common, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STR_PTR, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); + } + } +else if (opcode == OP_SBRA || opcode == OP_SCOND) + { + /* Saving the previous value. */ + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + allocate_stack(common, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STR_PTR, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); + } +else if (has_alternatives) + { + /* Pushing the starting string pointer. */ + allocate_stack(common, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); + } + +/* Generating code for the first alternative. */ +if (opcode == OP_COND || opcode == OP_SCOND) + { + if (*matchingpath == OP_CREF) + { + SLJIT_ASSERT(has_alternatives); + add_jump(compiler, &(BACKTRACK_AS(bracket_backtrack)->u.condfailed), + CMP(SLJIT_EQUAL, SLJIT_MEM1(SLJIT_SP), OVECTOR(GET2(matchingpath, 1) << 1), SLJIT_MEM1(SLJIT_SP), OVECTOR(1))); + matchingpath += 1 + IMM2_SIZE; + } + else if (*matchingpath == OP_DNCREF) + { + SLJIT_ASSERT(has_alternatives); + + i = GET2(matchingpath, 1 + IMM2_SIZE); + slot = common->name_table + GET2(matchingpath, 1) * common->name_entry_size; + OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(1)); + OP2(SLJIT_SUB | SLJIT_SET_Z, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(GET2(slot, 0) << 1), TMP1, 0); + slot += common->name_entry_size; + i--; + while (i-- > 0) + { + OP2(SLJIT_SUB, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(GET2(slot, 0) << 1), TMP1, 0); + OP2(SLJIT_OR | SLJIT_SET_Z, TMP2, 0, TMP2, 0, STR_PTR, 0); + slot += common->name_entry_size; + } + OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0); + add_jump(compiler, &(BACKTRACK_AS(bracket_backtrack)->u.condfailed), JUMP(SLJIT_ZERO)); + matchingpath += 1 + 2 * IMM2_SIZE; + } + else if (*matchingpath == OP_RREF || *matchingpath == OP_DNRREF || *matchingpath == OP_FAIL) + { + /* Never has other case. */ + BACKTRACK_AS(bracket_backtrack)->u.condfailed = NULL; + SLJIT_ASSERT(!has_alternatives); + + if (*matchingpath == OP_FAIL) + stacksize = 0; + else if (*matchingpath == OP_RREF) + { + stacksize = GET2(matchingpath, 1); + if (common->currententry == NULL) + stacksize = 0; + else if (stacksize == RREF_ANY) + stacksize = 1; + else if (common->currententry->start == 0) + stacksize = stacksize == 0; + else + stacksize = stacksize == (int)GET2(common->start, common->currententry->start + 1 + LINK_SIZE); + + if (stacksize != 0) + matchingpath += 1 + IMM2_SIZE; + } + else + { + if (common->currententry == NULL || common->currententry->start == 0) + stacksize = 0; + else + { + stacksize = GET2(matchingpath, 1 + IMM2_SIZE); + slot = common->name_table + GET2(matchingpath, 1) * common->name_entry_size; + i = (int)GET2(common->start, common->currententry->start + 1 + LINK_SIZE); + while (stacksize > 0) + { + if ((int)GET2(slot, 0) == i) + break; + slot += common->name_entry_size; + stacksize--; + } + } + + if (stacksize != 0) + matchingpath += 1 + 2 * IMM2_SIZE; + } + + /* The stacksize == 0 is a common "else" case. */ + if (stacksize == 0) + { + if (*cc == OP_ALT) + { + matchingpath = cc + 1 + LINK_SIZE; + cc += GET(cc, 1); + } + else + matchingpath = cc; + } + } + else + { + SLJIT_ASSERT(has_alternatives && *matchingpath >= OP_ASSERT && *matchingpath <= OP_ASSERTBACK_NOT); + /* Similar code as PUSH_BACKTRACK macro. */ + assert = sljit_alloc_memory(compiler, sizeof(assert_backtrack)); + if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) + return NULL; + memset(assert, 0, sizeof(assert_backtrack)); + assert->common.cc = matchingpath; + BACKTRACK_AS(bracket_backtrack)->u.assert = assert; + matchingpath = compile_assert_matchingpath(common, matchingpath, assert, TRUE); + } + } + +compile_matchingpath(common, matchingpath, cc, backtrack); +if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) + return NULL; + +if (opcode == OP_ONCE) + match_once_common(common, ket, BACKTRACK_AS(bracket_backtrack)->u.framesize, private_data_ptr, has_alternatives, needs_control_head); + +stacksize = 0; +if (repeat_type == OP_MINUPTO) + { + /* We need to preserve the counter. TMP2 will be used below. */ + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), repeat_ptr); + stacksize++; + } +if (ket != OP_KET || bra != OP_BRA) + stacksize++; +if (offset != 0) + { + if (common->capture_last_ptr != 0) + stacksize++; + if (common->optimized_cbracket[offset >> 1] == 0) + stacksize += 2; + } +if (has_alternatives && opcode != OP_ONCE) + stacksize++; + +if (stacksize > 0) + allocate_stack(common, stacksize); + +stacksize = 0; +if (repeat_type == OP_MINUPTO) + { + /* TMP2 was set above. */ + OP2(SLJIT_SUB, SLJIT_MEM1(STACK_TOP), STACK(stacksize), TMP2, 0, SLJIT_IMM, 1); + stacksize++; + } + +if (ket != OP_KET || bra != OP_BRA) + { + if (ket != OP_KET) + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0); + else + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0); + stacksize++; + } + +if (offset != 0) + stacksize = match_capture_common(common, stacksize, offset, private_data_ptr); + +if (has_alternatives) + { + if (opcode != OP_ONCE) + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0); + if (ket != OP_KETRMAX) + BACKTRACK_AS(bracket_backtrack)->alternative_matchingpath = LABEL(); + } + +/* Must be after the matchingpath label. */ +if (offset != 0 && common->optimized_cbracket[offset >> 1] != 0) + { + SLJIT_ASSERT(private_data_ptr == OVECTOR(offset + 0)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0); + } + +if (ket == OP_KETRMAX) + { + if (repeat_type != 0) + { + if (has_alternatives) + BACKTRACK_AS(bracket_backtrack)->alternative_matchingpath = LABEL(); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1); + JUMPTO(SLJIT_NOT_ZERO, rmax_label); + /* Drop STR_PTR for greedy plus quantifier. */ + if (opcode != OP_ONCE) + free_stack(common, 1); + } + else if (opcode == OP_ONCE || opcode >= OP_SBRA) + { + if (has_alternatives) + BACKTRACK_AS(bracket_backtrack)->alternative_matchingpath = LABEL(); + /* Checking zero-length iteration. */ + if (opcode != OP_ONCE) + { + CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STR_PTR, 0, rmax_label); + /* Drop STR_PTR for greedy plus quantifier. */ + if (bra != OP_BRAZERO) + free_stack(common, 1); + } + else + /* TMP2 must contain the starting STR_PTR. */ + CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, STR_PTR, 0, rmax_label); + } + else + JUMPTO(SLJIT_JUMP, rmax_label); + BACKTRACK_AS(bracket_backtrack)->recursive_matchingpath = LABEL(); + } + +if (repeat_type == OP_EXACT) + { + count_match(common); + OP2(SLJIT_SUB | SLJIT_SET_Z, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1); + JUMPTO(SLJIT_NOT_ZERO, rmax_label); + } +else if (repeat_type == OP_UPTO) + { + /* We need to preserve the counter. */ + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), repeat_ptr); + allocate_stack(common, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); + } + +if (bra == OP_BRAZERO) + BACKTRACK_AS(bracket_backtrack)->zero_matchingpath = LABEL(); + +if (bra == OP_BRAMINZERO) + { + /* This is a backtrack path! (From the viewpoint of OP_BRAMINZERO) */ + JUMPTO(SLJIT_JUMP, ((braminzero_backtrack *)parent)->matchingpath); + if (braminzero != NULL) + { + JUMPHERE(braminzero); + /* We need to release the end pointer to perform the + backtrack for the zero-length iteration. When + framesize is < 0, OP_ONCE will do the release itself. */ + if (opcode == OP_ONCE && BACKTRACK_AS(bracket_backtrack)->u.framesize >= 0) + { + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); + } + else if (ket == OP_KETRMIN && opcode != OP_ONCE) + free_stack(common, 1); + } + /* Continue to the normal backtrack. */ + } + +if ((ket != OP_KET && bra != OP_BRAMINZERO) || bra == OP_BRAZERO) + count_match(common); + +/* Skip the other alternatives. */ +while (*cc == OP_ALT) + cc += GET(cc, 1); +cc += 1 + LINK_SIZE; + +if (opcode == OP_ONCE) + { + /* We temporarily encode the needs_control_head in the lowest bit. + Note: on the target architectures of SLJIT the ((x << 1) >> 1) returns + the same value for small signed numbers (including negative numbers). */ + BACKTRACK_AS(bracket_backtrack)->u.framesize = (BACKTRACK_AS(bracket_backtrack)->u.framesize << 1) | (needs_control_head ? 1 : 0); + } +return cc + repeat_length; +} + +static pcre_uchar *compile_bracketpos_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent) +{ +DEFINE_COMPILER; +backtrack_common *backtrack; +pcre_uchar opcode; +int private_data_ptr; +int cbraprivptr = 0; +BOOL needs_control_head; +int framesize; +int stacksize; +int offset = 0; +BOOL zero = FALSE; +pcre_uchar *ccbegin = NULL; +int stack; /* Also contains the offset of control head. */ +struct sljit_label *loop = NULL; +struct jump_list *emptymatch = NULL; + +PUSH_BACKTRACK(sizeof(bracketpos_backtrack), cc, NULL); +if (*cc == OP_BRAPOSZERO) + { + zero = TRUE; + cc++; + } + +opcode = *cc; +private_data_ptr = PRIVATE_DATA(cc); +SLJIT_ASSERT(private_data_ptr != 0); +BACKTRACK_AS(bracketpos_backtrack)->private_data_ptr = private_data_ptr; +switch(opcode) + { + case OP_BRAPOS: + case OP_SBRAPOS: + ccbegin = cc + 1 + LINK_SIZE; + break; + + case OP_CBRAPOS: + case OP_SCBRAPOS: + offset = GET2(cc, 1 + LINK_SIZE); + /* This case cannot be optimized in the same was as + normal capturing brackets. */ + SLJIT_ASSERT(common->optimized_cbracket[offset] == 0); + cbraprivptr = OVECTOR_PRIV(offset); + offset <<= 1; + ccbegin = cc + 1 + LINK_SIZE + IMM2_SIZE; + break; + + default: + SLJIT_UNREACHABLE(); + break; + } + +framesize = get_framesize(common, cc, NULL, FALSE, &needs_control_head); +BACKTRACK_AS(bracketpos_backtrack)->framesize = framesize; +if (framesize < 0) + { + if (offset != 0) + { + stacksize = 2; + if (common->capture_last_ptr != 0) + stacksize++; + } + else + stacksize = 1; + + if (needs_control_head) + stacksize++; + if (!zero) + stacksize++; + + BACKTRACK_AS(bracketpos_backtrack)->stacksize = stacksize; + allocate_stack(common, stacksize); + if (framesize == no_frame) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0); + + stack = 0; + if (offset != 0) + { + stack = 2; + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset)); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1)); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0); + if (common->capture_last_ptr != 0) + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), TMP2, 0); + if (needs_control_head) + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); + if (common->capture_last_ptr != 0) + { + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(2), TMP1, 0); + stack = 3; + } + } + else + { + if (needs_control_head) + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); + stack = 1; + } + + if (needs_control_head) + stack++; + if (!zero) + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stack), SLJIT_IMM, 1); + if (needs_control_head) + { + stack--; + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stack), TMP2, 0); + } + } +else + { + stacksize = framesize + 1; + if (!zero) + stacksize++; + if (needs_control_head) + stacksize++; + if (offset == 0) + stacksize++; + BACKTRACK_AS(bracketpos_backtrack)->stacksize = stacksize; + + allocate_stack(common, stacksize); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + if (needs_control_head) + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); + OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_SP), private_data_ptr, STACK_TOP, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw)); + + stack = 0; + if (!zero) + { + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 1); + stack = 1; + } + if (needs_control_head) + { + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stack), TMP2, 0); + stack++; + } + if (offset == 0) + { + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stack), STR_PTR, 0); + stack++; + } + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stack), TMP1, 0); + init_frame(common, cc, NULL, stacksize - 1, stacksize - framesize, FALSE); + stack -= 1 + (offset == 0); + } + +if (offset != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), cbraprivptr, STR_PTR, 0); + +loop = LABEL(); +while (*cc != OP_KETRPOS) + { + backtrack->top = NULL; + backtrack->topbacktracks = NULL; + cc += GET(cc, 1); + + compile_matchingpath(common, ccbegin, cc, backtrack); + if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) + return NULL; + + if (framesize < 0) + { + if (framesize == no_frame) + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + + if (offset != 0) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), cbraprivptr); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), cbraprivptr, STR_PTR, 0); + if (common->capture_last_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, SLJIT_IMM, offset >> 1); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0); + } + else + { + if (opcode == OP_SBRAPOS) + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); + } + + /* Even if the match is empty, we need to reset the control head. */ + if (needs_control_head) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(stack)); + + if (opcode == OP_SBRAPOS || opcode == OP_SCBRAPOS) + add_jump(compiler, &emptymatch, CMP(SLJIT_EQUAL, TMP1, 0, STR_PTR, 0)); + + if (!zero) + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0); + } + else + { + if (offset != 0) + { + OP2(SLJIT_SUB, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_IMM, stacksize * sizeof(sljit_sw)); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), cbraprivptr); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), cbraprivptr, STR_PTR, 0); + if (common->capture_last_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, SLJIT_IMM, offset >> 1); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0); + } + else + { + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + OP2(SLJIT_SUB, STACK_TOP, 0, TMP2, 0, SLJIT_IMM, stacksize * sizeof(sljit_sw)); + if (opcode == OP_SBRAPOS) + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), STACK(-framesize - 2)); + OP1(SLJIT_MOV, SLJIT_MEM1(TMP2), STACK(-framesize - 2), STR_PTR, 0); + } + + /* Even if the match is empty, we need to reset the control head. */ + if (needs_control_head) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_MEM1(STACK_TOP), STACK(stack)); + + if (opcode == OP_SBRAPOS || opcode == OP_SCBRAPOS) + add_jump(compiler, &emptymatch, CMP(SLJIT_EQUAL, TMP1, 0, STR_PTR, 0)); + + if (!zero) + { + if (framesize < 0) + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0); + else + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); + } + } + + JUMPTO(SLJIT_JUMP, loop); + flush_stubs(common); + + compile_backtrackingpath(common, backtrack->top); + if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) + return NULL; + set_jumps(backtrack->topbacktracks, LABEL()); + + if (framesize < 0) + { + if (offset != 0) + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), cbraprivptr); + else + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + } + else + { + if (offset != 0) + { + /* Last alternative. */ + if (*cc == OP_KETRPOS) + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), cbraprivptr); + } + else + { + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP2), STACK(-framesize - 2)); + } + } + + if (*cc == OP_KETRPOS) + break; + ccbegin = cc + 1 + LINK_SIZE; + } + +/* We don't have to restore the control head in case of a failed match. */ + +backtrack->topbacktracks = NULL; +if (!zero) + { + if (framesize < 0) + add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(stacksize - 1), SLJIT_IMM, 0)); + else /* TMP2 is set to [private_data_ptr] above. */ + add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(TMP2), STACK(-stacksize), SLJIT_IMM, 0)); + } + +/* None of them matched. */ +set_jumps(emptymatch, LABEL()); +count_match(common); +return cc + 1 + LINK_SIZE; +} + +static SLJIT_INLINE pcre_uchar *get_iterator_parameters(compiler_common *common, pcre_uchar *cc, pcre_uchar *opcode, pcre_uchar *type, sljit_u32 *max, sljit_u32 *exact, pcre_uchar **end) +{ +int class_len; + +*opcode = *cc; +*exact = 0; + +if (*opcode >= OP_STAR && *opcode <= OP_POSUPTO) + { + cc++; + *type = OP_CHAR; + } +else if (*opcode >= OP_STARI && *opcode <= OP_POSUPTOI) + { + cc++; + *type = OP_CHARI; + *opcode -= OP_STARI - OP_STAR; + } +else if (*opcode >= OP_NOTSTAR && *opcode <= OP_NOTPOSUPTO) + { + cc++; + *type = OP_NOT; + *opcode -= OP_NOTSTAR - OP_STAR; + } +else if (*opcode >= OP_NOTSTARI && *opcode <= OP_NOTPOSUPTOI) + { + cc++; + *type = OP_NOTI; + *opcode -= OP_NOTSTARI - OP_STAR; + } +else if (*opcode >= OP_TYPESTAR && *opcode <= OP_TYPEPOSUPTO) + { + cc++; + *opcode -= OP_TYPESTAR - OP_STAR; + *type = OP_END; + } +else + { + SLJIT_ASSERT(*opcode == OP_CLASS || *opcode == OP_NCLASS || *opcode == OP_XCLASS); + *type = *opcode; + cc++; + class_len = (*type < OP_XCLASS) ? (int)(1 + (32 / sizeof(pcre_uchar))) : GET(cc, 0); + *opcode = cc[class_len - 1]; + + if (*opcode >= OP_CRSTAR && *opcode <= OP_CRMINQUERY) + { + *opcode -= OP_CRSTAR - OP_STAR; + *end = cc + class_len; + + if (*opcode == OP_PLUS || *opcode == OP_MINPLUS) + { + *exact = 1; + *opcode -= OP_PLUS - OP_STAR; + } + } + else if (*opcode >= OP_CRPOSSTAR && *opcode <= OP_CRPOSQUERY) + { + *opcode -= OP_CRPOSSTAR - OP_POSSTAR; + *end = cc + class_len; + + if (*opcode == OP_POSPLUS) + { + *exact = 1; + *opcode = OP_POSSTAR; + } + } + else + { + SLJIT_ASSERT(*opcode == OP_CRRANGE || *opcode == OP_CRMINRANGE || *opcode == OP_CRPOSRANGE); + *max = GET2(cc, (class_len + IMM2_SIZE)); + *exact = GET2(cc, class_len); + + if (*max == 0) + { + if (*opcode == OP_CRPOSRANGE) + *opcode = OP_POSSTAR; + else + *opcode -= OP_CRRANGE - OP_STAR; + } + else + { + *max -= *exact; + if (*max == 0) + *opcode = OP_EXACT; + else if (*max == 1) + { + if (*opcode == OP_CRPOSRANGE) + *opcode = OP_POSQUERY; + else + *opcode -= OP_CRRANGE - OP_QUERY; + } + else + { + if (*opcode == OP_CRPOSRANGE) + *opcode = OP_POSUPTO; + else + *opcode -= OP_CRRANGE - OP_UPTO; + } + } + *end = cc + class_len + 2 * IMM2_SIZE; + } + return cc; + } + +switch(*opcode) + { + case OP_EXACT: + *exact = GET2(cc, 0); + cc += IMM2_SIZE; + break; + + case OP_PLUS: + case OP_MINPLUS: + *exact = 1; + *opcode -= OP_PLUS - OP_STAR; + break; + + case OP_POSPLUS: + *exact = 1; + *opcode = OP_POSSTAR; + break; + + case OP_UPTO: + case OP_MINUPTO: + case OP_POSUPTO: + *max = GET2(cc, 0); + cc += IMM2_SIZE; + break; + } + +if (*type == OP_END) + { + *type = *cc; + *end = next_opcode(common, cc); + cc++; + return cc; + } + +*end = cc + 1; +#ifdef SUPPORT_UTF +if (common->utf && HAS_EXTRALEN(*cc)) *end += GET_EXTRALEN(*cc); +#endif +return cc; +} + +static pcre_uchar *compile_iterator_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent) +{ +DEFINE_COMPILER; +backtrack_common *backtrack; +pcre_uchar opcode; +pcre_uchar type; +sljit_u32 max = 0, exact; +BOOL fast_fail; +sljit_s32 fast_str_ptr; +BOOL charpos_enabled; +pcre_uchar charpos_char; +unsigned int charpos_othercasebit; +pcre_uchar *end; +jump_list *no_match = NULL; +jump_list *no_char1_match = NULL; +struct sljit_jump *jump = NULL; +struct sljit_label *label; +int private_data_ptr = PRIVATE_DATA(cc); +int base = (private_data_ptr == 0) ? SLJIT_MEM1(STACK_TOP) : SLJIT_MEM1(SLJIT_SP); +int offset0 = (private_data_ptr == 0) ? STACK(0) : private_data_ptr; +int offset1 = (private_data_ptr == 0) ? STACK(1) : private_data_ptr + (int)sizeof(sljit_sw); +int tmp_base, tmp_offset; + +PUSH_BACKTRACK(sizeof(char_iterator_backtrack), cc, NULL); + +fast_str_ptr = PRIVATE_DATA(cc + 1); +fast_fail = TRUE; + +SLJIT_ASSERT(common->fast_forward_bc_ptr == NULL || fast_str_ptr == 0 || cc == common->fast_forward_bc_ptr); + +if (cc == common->fast_forward_bc_ptr) + fast_fail = FALSE; +else if (common->fast_fail_start_ptr == 0) + fast_str_ptr = 0; + +SLJIT_ASSERT(common->fast_forward_bc_ptr != NULL || fast_str_ptr == 0 + || (fast_str_ptr >= common->fast_fail_start_ptr && fast_str_ptr <= common->fast_fail_end_ptr)); + +cc = get_iterator_parameters(common, cc, &opcode, &type, &max, &exact, &end); + +if (type != OP_EXTUNI) + { + tmp_base = TMP3; + tmp_offset = 0; + } +else + { + tmp_base = SLJIT_MEM1(SLJIT_SP); + tmp_offset = POSSESSIVE0; + } + +if (fast_fail && fast_str_ptr != 0) + add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), fast_str_ptr)); + +/* Handle fixed part first. */ +if (exact > 1) + { + SLJIT_ASSERT(fast_str_ptr == 0); + if (common->mode == JIT_COMPILE +#ifdef SUPPORT_UTF + && !common->utf +#endif + ) + { + OP2(SLJIT_ADD, TMP1, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(exact)); + add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_GREATER, TMP1, 0, STR_END, 0)); + OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, exact); + label = LABEL(); + compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, FALSE); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + JUMPTO(SLJIT_NOT_ZERO, label); + } + else + { + OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, exact); + label = LABEL(); + compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, TRUE); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + JUMPTO(SLJIT_NOT_ZERO, label); + } + } +else if (exact == 1) + compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, TRUE); + +switch(opcode) + { + case OP_STAR: + case OP_UPTO: + SLJIT_ASSERT(fast_str_ptr == 0 || opcode == OP_STAR); + + if (type == OP_ANYNL || type == OP_EXTUNI) + { + SLJIT_ASSERT(private_data_ptr == 0); + SLJIT_ASSERT(fast_str_ptr == 0); + + allocate_stack(common, 2); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, 0); + + if (opcode == OP_UPTO) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0, SLJIT_IMM, max); + + label = LABEL(); + compile_char1_matchingpath(common, type, cc, &BACKTRACK_AS(char_iterator_backtrack)->u.backtracks, TRUE); + if (opcode == OP_UPTO) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0); + OP2(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); + jump = JUMP(SLJIT_ZERO); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE0, TMP1, 0); + } + + /* We cannot use TMP3 because of this allocate_stack. */ + allocate_stack(common, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); + JUMPTO(SLJIT_JUMP, label); + if (jump != NULL) + JUMPHERE(jump); + } + else + { + charpos_enabled = FALSE; + charpos_char = 0; + charpos_othercasebit = 0; + + if ((type != OP_CHAR && type != OP_CHARI) && (*end == OP_CHAR || *end == OP_CHARI)) + { + charpos_enabled = TRUE; +#ifdef SUPPORT_UTF + charpos_enabled = !common->utf || !HAS_EXTRALEN(end[1]); +#endif + if (charpos_enabled && *end == OP_CHARI && char_has_othercase(common, end + 1)) + { + charpos_othercasebit = char_get_othercase_bit(common, end + 1); + if (charpos_othercasebit == 0) + charpos_enabled = FALSE; + } + + if (charpos_enabled) + { + charpos_char = end[1]; + /* Consumpe the OP_CHAR opcode. */ + end += 2; +#if defined COMPILE_PCRE8 + SLJIT_ASSERT((charpos_othercasebit >> 8) == 0); +#elif defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + SLJIT_ASSERT((charpos_othercasebit >> 9) == 0); + if ((charpos_othercasebit & 0x100) != 0) + charpos_othercasebit = (charpos_othercasebit & 0xff) << 8; +#endif + if (charpos_othercasebit != 0) + charpos_char |= charpos_othercasebit; + + BACKTRACK_AS(char_iterator_backtrack)->u.charpos.enabled = TRUE; + BACKTRACK_AS(char_iterator_backtrack)->u.charpos.chr = charpos_char; + BACKTRACK_AS(char_iterator_backtrack)->u.charpos.othercasebit = charpos_othercasebit; + } + } + + if (charpos_enabled) + { + if (opcode == OP_UPTO) + OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max + 1); + + /* Search the first instance of charpos_char. */ + jump = JUMP(SLJIT_JUMP); + label = LABEL(); + if (opcode == OP_UPTO) + { + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_ZERO)); + } + compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, FALSE); + if (fast_str_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); + JUMPHERE(jump); + + detect_partial_match(common, &backtrack->topbacktracks); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + if (charpos_othercasebit != 0) + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, charpos_othercasebit); + CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, charpos_char, label); + + if (private_data_ptr == 0) + allocate_stack(common, 2); + OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); + OP1(SLJIT_MOV, base, offset1, STR_PTR, 0); + if (opcode == OP_UPTO) + { + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + add_jump(compiler, &no_match, JUMP(SLJIT_ZERO)); + } + + /* Search the last instance of charpos_char. */ + label = LABEL(); + compile_char1_matchingpath(common, type, cc, &no_match, FALSE); + if (fast_str_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); + detect_partial_match(common, &no_match); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); + if (charpos_othercasebit != 0) + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, charpos_othercasebit); + if (opcode == OP_STAR) + { + CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, charpos_char, label); + OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); + } + else + { + jump = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, charpos_char); + OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); + JUMPHERE(jump); + } + + if (opcode == OP_UPTO) + { + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + JUMPTO(SLJIT_NOT_ZERO, label); + } + else + JUMPTO(SLJIT_JUMP, label); + + set_jumps(no_match, LABEL()); + OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); + } +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + else if (common->utf) + { + if (private_data_ptr == 0) + allocate_stack(common, 2); + + OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); + OP1(SLJIT_MOV, base, offset1, STR_PTR, 0); + + if (opcode == OP_UPTO) + OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max); + + label = LABEL(); + compile_char1_matchingpath(common, type, cc, &no_match, TRUE); + OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); + + if (opcode == OP_UPTO) + { + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + JUMPTO(SLJIT_NOT_ZERO, label); + } + else + JUMPTO(SLJIT_JUMP, label); + + set_jumps(no_match, LABEL()); + OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); + if (fast_str_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); + } +#endif + else + { + if (private_data_ptr == 0) + allocate_stack(common, 2); + + OP1(SLJIT_MOV, base, offset1, STR_PTR, 0); + if (opcode == OP_UPTO) + OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max); + + label = LABEL(); + detect_partial_match(common, &no_match); + compile_char1_matchingpath(common, type, cc, &no_char1_match, FALSE); + if (opcode == OP_UPTO) + { + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + JUMPTO(SLJIT_NOT_ZERO, label); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + } + else + JUMPTO(SLJIT_JUMP, label); + + set_jumps(no_char1_match, LABEL()); + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + set_jumps(no_match, LABEL()); + OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); + if (fast_str_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); + } + } + BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL(); + break; + + case OP_MINSTAR: + if (private_data_ptr == 0) + allocate_stack(common, 1); + OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); + BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL(); + if (fast_str_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); + break; + + case OP_MINUPTO: + SLJIT_ASSERT(fast_str_ptr == 0); + if (private_data_ptr == 0) + allocate_stack(common, 2); + OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); + OP1(SLJIT_MOV, base, offset1, SLJIT_IMM, max + 1); + BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL(); + break; + + case OP_QUERY: + case OP_MINQUERY: + SLJIT_ASSERT(fast_str_ptr == 0); + if (private_data_ptr == 0) + allocate_stack(common, 1); + OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); + if (opcode == OP_QUERY) + compile_char1_matchingpath(common, type, cc, &BACKTRACK_AS(char_iterator_backtrack)->u.backtracks, TRUE); + BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL(); + break; + + case OP_EXACT: + break; + + case OP_POSSTAR: +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (common->utf) + { + OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0); + label = LABEL(); + compile_char1_matchingpath(common, type, cc, &no_match, TRUE); + OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0); + JUMPTO(SLJIT_JUMP, label); + set_jumps(no_match, LABEL()); + OP1(SLJIT_MOV, STR_PTR, 0, tmp_base, tmp_offset); + if (fast_str_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); + break; + } +#endif + label = LABEL(); + detect_partial_match(common, &no_match); + compile_char1_matchingpath(common, type, cc, &no_char1_match, FALSE); + JUMPTO(SLJIT_JUMP, label); + set_jumps(no_char1_match, LABEL()); + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + set_jumps(no_match, LABEL()); + if (fast_str_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); + break; + + case OP_POSUPTO: + SLJIT_ASSERT(fast_str_ptr == 0); +#if defined SUPPORT_UTF && !defined COMPILE_PCRE32 + if (common->utf) + { + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1, STR_PTR, 0); + OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max); + label = LABEL(); + compile_char1_matchingpath(common, type, cc, &no_match, TRUE); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1, STR_PTR, 0); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + JUMPTO(SLJIT_NOT_ZERO, label); + set_jumps(no_match, LABEL()); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), POSSESSIVE1); + break; + } +#endif + OP1(SLJIT_MOV, tmp_base, tmp_offset, SLJIT_IMM, max); + label = LABEL(); + detect_partial_match(common, &no_match); + compile_char1_matchingpath(common, type, cc, &no_char1_match, FALSE); + OP2(SLJIT_SUB | SLJIT_SET_Z, tmp_base, tmp_offset, tmp_base, tmp_offset, SLJIT_IMM, 1); + JUMPTO(SLJIT_NOT_ZERO, label); + OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + set_jumps(no_char1_match, LABEL()); + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + set_jumps(no_match, LABEL()); + break; + + case OP_POSQUERY: + SLJIT_ASSERT(fast_str_ptr == 0); + OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0); + compile_char1_matchingpath(common, type, cc, &no_match, TRUE); + OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0); + set_jumps(no_match, LABEL()); + OP1(SLJIT_MOV, STR_PTR, 0, tmp_base, tmp_offset); + break; + + default: + SLJIT_UNREACHABLE(); + break; + } + +count_match(common); +return end; +} + +static SLJIT_INLINE pcre_uchar *compile_fail_accept_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent) +{ +DEFINE_COMPILER; +backtrack_common *backtrack; + +PUSH_BACKTRACK(sizeof(backtrack_common), cc, NULL); + +if (*cc == OP_FAIL) + { + add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_JUMP)); + return cc + 1; + } + +if (*cc == OP_ASSERT_ACCEPT || common->currententry != NULL || !common->might_be_empty) + { + /* No need to check notempty conditions. */ + if (common->accept_label == NULL) + add_jump(compiler, &common->accept, JUMP(SLJIT_JUMP)); + else + JUMPTO(SLJIT_JUMP, common->accept_label); + return cc + 1; + } + +if (common->accept_label == NULL) + add_jump(compiler, &common->accept, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0))); +else + CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0), common->accept_label); +OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); +OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty)); +add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0)); +OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty_atstart)); +if (common->accept_label == NULL) + add_jump(compiler, &common->accept, CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, 0)); +else + CMPTO(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, 0, common->accept_label); +OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); +if (common->accept_label == NULL) + add_jump(compiler, &common->accept, CMP(SLJIT_NOT_EQUAL, TMP2, 0, STR_PTR, 0)); +else + CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, STR_PTR, 0, common->accept_label); +add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_JUMP)); +return cc + 1; +} + +static SLJIT_INLINE pcre_uchar *compile_close_matchingpath(compiler_common *common, pcre_uchar *cc) +{ +DEFINE_COMPILER; +int offset = GET2(cc, 1); +BOOL optimized_cbracket = common->optimized_cbracket[offset] != 0; + +/* Data will be discarded anyway... */ +if (common->currententry != NULL) + return cc + 1 + IMM2_SIZE; + +if (!optimized_cbracket) + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR_PRIV(offset)); +offset <<= 1; +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0); +if (!optimized_cbracket) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0); +return cc + 1 + IMM2_SIZE; +} + +static SLJIT_INLINE pcre_uchar *compile_control_verb_matchingpath(compiler_common *common, pcre_uchar *cc, backtrack_common *parent) +{ +DEFINE_COMPILER; +backtrack_common *backtrack; +pcre_uchar opcode = *cc; +pcre_uchar *ccend = cc + 1; + +if (opcode == OP_PRUNE_ARG || opcode == OP_SKIP_ARG || opcode == OP_THEN_ARG) + ccend += 2 + cc[1]; + +PUSH_BACKTRACK(sizeof(backtrack_common), cc, NULL); + +if (opcode == OP_SKIP) + { + allocate_stack(common, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); + return ccend; + } + +if (opcode == OP_PRUNE_ARG || opcode == OP_THEN_ARG) + { + OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_sw)(cc + 2)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, TMP2, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, mark_ptr), TMP2, 0); + } + +return ccend; +} + +static pcre_uchar then_trap_opcode[1] = { OP_THEN_TRAP }; + +static SLJIT_INLINE void compile_then_trap_matchingpath(compiler_common *common, pcre_uchar *cc, pcre_uchar *ccend, backtrack_common *parent) +{ +DEFINE_COMPILER; +backtrack_common *backtrack; +BOOL needs_control_head; +int size; + +PUSH_BACKTRACK_NOVALUE(sizeof(then_trap_backtrack), cc); +common->then_trap = BACKTRACK_AS(then_trap_backtrack); +BACKTRACK_AS(then_trap_backtrack)->common.cc = then_trap_opcode; +BACKTRACK_AS(then_trap_backtrack)->start = (sljit_sw)(cc - common->start); +BACKTRACK_AS(then_trap_backtrack)->framesize = get_framesize(common, cc, ccend, FALSE, &needs_control_head); + +size = BACKTRACK_AS(then_trap_backtrack)->framesize; +size = 3 + (size < 0 ? 0 : size); + +OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); +allocate_stack(common, size); +if (size > 3) + OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, STACK_TOP, 0, SLJIT_IMM, (size - 3) * sizeof(sljit_sw)); +else + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, STACK_TOP, 0); +OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(size - 1), SLJIT_IMM, BACKTRACK_AS(then_trap_backtrack)->start); +OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(size - 2), SLJIT_IMM, type_then_trap); +OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(size - 3), TMP2, 0); + +size = BACKTRACK_AS(then_trap_backtrack)->framesize; +if (size >= 0) + init_frame(common, cc, ccend, size - 1, 0, FALSE); +} + +static void compile_matchingpath(compiler_common *common, pcre_uchar *cc, pcre_uchar *ccend, backtrack_common *parent) +{ +DEFINE_COMPILER; +backtrack_common *backtrack; +BOOL has_then_trap = FALSE; +then_trap_backtrack *save_then_trap = NULL; + +SLJIT_ASSERT(*ccend == OP_END || (*ccend >= OP_ALT && *ccend <= OP_KETRPOS)); + +if (common->has_then && common->then_offsets[cc - common->start] != 0) + { + SLJIT_ASSERT(*ccend != OP_END && common->control_head_ptr != 0); + has_then_trap = TRUE; + save_then_trap = common->then_trap; + /* Tail item on backtrack. */ + compile_then_trap_matchingpath(common, cc, ccend, parent); + } + +while (cc < ccend) + { + switch(*cc) + { + case OP_SOD: + case OP_SOM: + case OP_NOT_WORD_BOUNDARY: + case OP_WORD_BOUNDARY: + case OP_EODN: + case OP_EOD: + case OP_DOLL: + case OP_DOLLM: + case OP_CIRC: + case OP_CIRCM: + case OP_REVERSE: + cc = compile_simple_assertion_matchingpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks); + break; + + case OP_NOT_DIGIT: + case OP_DIGIT: + case OP_NOT_WHITESPACE: + case OP_WHITESPACE: + case OP_NOT_WORDCHAR: + case OP_WORDCHAR: + case OP_ANY: + case OP_ALLANY: + case OP_ANYBYTE: + case OP_NOTPROP: + case OP_PROP: + case OP_ANYNL: + case OP_NOT_HSPACE: + case OP_HSPACE: + case OP_NOT_VSPACE: + case OP_VSPACE: + case OP_EXTUNI: + case OP_NOT: + case OP_NOTI: + cc = compile_char1_matchingpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, TRUE); + break; + + case OP_SET_SOM: + PUSH_BACKTRACK_NOVALUE(sizeof(backtrack_common), cc); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0)); + allocate_stack(common, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(0), STR_PTR, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP2, 0); + cc++; + break; + + case OP_CHAR: + case OP_CHARI: + if (common->mode == JIT_COMPILE) + cc = compile_charn_matchingpath(common, cc, ccend, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks); + else + cc = compile_char1_matchingpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, TRUE); + break; + + case OP_STAR: + case OP_MINSTAR: + case OP_PLUS: + case OP_MINPLUS: + case OP_QUERY: + case OP_MINQUERY: + case OP_UPTO: + case OP_MINUPTO: + case OP_EXACT: + case OP_POSSTAR: + case OP_POSPLUS: + case OP_POSQUERY: + case OP_POSUPTO: + case OP_STARI: + case OP_MINSTARI: + case OP_PLUSI: + case OP_MINPLUSI: + case OP_QUERYI: + case OP_MINQUERYI: + case OP_UPTOI: + case OP_MINUPTOI: + case OP_EXACTI: + case OP_POSSTARI: + case OP_POSPLUSI: + case OP_POSQUERYI: + case OP_POSUPTOI: + case OP_NOTSTAR: + case OP_NOTMINSTAR: + case OP_NOTPLUS: + case OP_NOTMINPLUS: + case OP_NOTQUERY: + case OP_NOTMINQUERY: + case OP_NOTUPTO: + case OP_NOTMINUPTO: + case OP_NOTEXACT: + case OP_NOTPOSSTAR: + case OP_NOTPOSPLUS: + case OP_NOTPOSQUERY: + case OP_NOTPOSUPTO: + case OP_NOTSTARI: + case OP_NOTMINSTARI: + case OP_NOTPLUSI: + case OP_NOTMINPLUSI: + case OP_NOTQUERYI: + case OP_NOTMINQUERYI: + case OP_NOTUPTOI: + case OP_NOTMINUPTOI: + case OP_NOTEXACTI: + case OP_NOTPOSSTARI: + case OP_NOTPOSPLUSI: + case OP_NOTPOSQUERYI: + case OP_NOTPOSUPTOI: + case OP_TYPESTAR: + case OP_TYPEMINSTAR: + case OP_TYPEPLUS: + case OP_TYPEMINPLUS: + case OP_TYPEQUERY: + case OP_TYPEMINQUERY: + case OP_TYPEUPTO: + case OP_TYPEMINUPTO: + case OP_TYPEEXACT: + case OP_TYPEPOSSTAR: + case OP_TYPEPOSPLUS: + case OP_TYPEPOSQUERY: + case OP_TYPEPOSUPTO: + cc = compile_iterator_matchingpath(common, cc, parent); + break; + + case OP_CLASS: + case OP_NCLASS: + if (cc[1 + (32 / sizeof(pcre_uchar))] >= OP_CRSTAR && cc[1 + (32 / sizeof(pcre_uchar))] <= OP_CRPOSRANGE) + cc = compile_iterator_matchingpath(common, cc, parent); + else + cc = compile_char1_matchingpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, TRUE); + break; + +#if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + case OP_XCLASS: + if (*(cc + GET(cc, 1)) >= OP_CRSTAR && *(cc + GET(cc, 1)) <= OP_CRPOSRANGE) + cc = compile_iterator_matchingpath(common, cc, parent); + else + cc = compile_char1_matchingpath(common, *cc, cc + 1, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, TRUE); + break; +#endif + + case OP_REF: + case OP_REFI: + if (cc[1 + IMM2_SIZE] >= OP_CRSTAR && cc[1 + IMM2_SIZE] <= OP_CRPOSRANGE) + cc = compile_ref_iterator_matchingpath(common, cc, parent); + else + { + compile_ref_matchingpath(common, cc, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, TRUE, FALSE); + cc += 1 + IMM2_SIZE; + } + break; + + case OP_DNREF: + case OP_DNREFI: + if (cc[1 + 2 * IMM2_SIZE] >= OP_CRSTAR && cc[1 + 2 * IMM2_SIZE] <= OP_CRPOSRANGE) + cc = compile_ref_iterator_matchingpath(common, cc, parent); + else + { + compile_dnref_search(common, cc, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks); + compile_ref_matchingpath(common, cc, parent->top != NULL ? &parent->top->nextbacktracks : &parent->topbacktracks, TRUE, FALSE); + cc += 1 + 2 * IMM2_SIZE; + } + break; + + case OP_RECURSE: + cc = compile_recurse_matchingpath(common, cc, parent); + break; + + case OP_CALLOUT: + cc = compile_callout_matchingpath(common, cc, parent); + break; + + case OP_ASSERT: + case OP_ASSERT_NOT: + case OP_ASSERTBACK: + case OP_ASSERTBACK_NOT: + PUSH_BACKTRACK_NOVALUE(sizeof(assert_backtrack), cc); + cc = compile_assert_matchingpath(common, cc, BACKTRACK_AS(assert_backtrack), FALSE); + break; + + case OP_BRAMINZERO: + PUSH_BACKTRACK_NOVALUE(sizeof(braminzero_backtrack), cc); + cc = bracketend(cc + 1); + if (*(cc - 1 - LINK_SIZE) != OP_KETRMIN) + { + allocate_stack(common, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); + } + else + { + allocate_stack(common, 2); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), STR_PTR, 0); + } + BACKTRACK_AS(braminzero_backtrack)->matchingpath = LABEL(); + count_match(common); + break; + + case OP_ONCE: + case OP_ONCE_NC: + case OP_BRA: + case OP_CBRA: + case OP_COND: + case OP_SBRA: + case OP_SCBRA: + case OP_SCOND: + cc = compile_bracket_matchingpath(common, cc, parent); + break; + + case OP_BRAZERO: + if (cc[1] > OP_ASSERTBACK_NOT) + cc = compile_bracket_matchingpath(common, cc, parent); + else + { + PUSH_BACKTRACK_NOVALUE(sizeof(assert_backtrack), cc); + cc = compile_assert_matchingpath(common, cc, BACKTRACK_AS(assert_backtrack), FALSE); + } + break; + + case OP_BRAPOS: + case OP_CBRAPOS: + case OP_SBRAPOS: + case OP_SCBRAPOS: + case OP_BRAPOSZERO: + cc = compile_bracketpos_matchingpath(common, cc, parent); + break; + + case OP_MARK: + PUSH_BACKTRACK_NOVALUE(sizeof(backtrack_common), cc); + SLJIT_ASSERT(common->mark_ptr != 0); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->mark_ptr); + allocate_stack(common, common->has_skip_arg ? 5 : 1); + OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(common->has_skip_arg ? 4 : 0), TMP2, 0); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, (sljit_sw)(cc + 2)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, TMP2, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, mark_ptr), TMP2, 0); + if (common->has_skip_arg) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, STACK_TOP, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(1), SLJIT_IMM, type_mark); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(2), SLJIT_IMM, (sljit_sw)(cc + 2)); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(3), STR_PTR, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), TMP1, 0); + } + cc += 1 + 2 + cc[1]; + break; + + case OP_PRUNE: + case OP_PRUNE_ARG: + case OP_SKIP: + case OP_SKIP_ARG: + case OP_THEN: + case OP_THEN_ARG: + case OP_COMMIT: + cc = compile_control_verb_matchingpath(common, cc, parent); + break; + + case OP_FAIL: + case OP_ACCEPT: + case OP_ASSERT_ACCEPT: + cc = compile_fail_accept_matchingpath(common, cc, parent); + break; + + case OP_CLOSE: + cc = compile_close_matchingpath(common, cc); + break; + + case OP_SKIPZERO: + cc = bracketend(cc + 1); + break; + + default: + SLJIT_UNREACHABLE(); + return; + } + if (cc == NULL) + return; + } + +if (has_then_trap) + { + /* Head item on backtrack. */ + PUSH_BACKTRACK_NOVALUE(sizeof(then_trap_backtrack), cc); + BACKTRACK_AS(then_trap_backtrack)->common.cc = then_trap_opcode; + BACKTRACK_AS(then_trap_backtrack)->then_trap = common->then_trap; + common->then_trap = save_then_trap; + } +SLJIT_ASSERT(cc == ccend); +} + +#undef PUSH_BACKTRACK +#undef PUSH_BACKTRACK_NOVALUE +#undef BACKTRACK_AS + +#define COMPILE_BACKTRACKINGPATH(current) \ + do \ + { \ + compile_backtrackingpath(common, (current)); \ + if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) \ + return; \ + } \ + while (0) + +#define CURRENT_AS(type) ((type *)current) + +static void compile_iterator_backtrackingpath(compiler_common *common, struct backtrack_common *current) +{ +DEFINE_COMPILER; +pcre_uchar *cc = current->cc; +pcre_uchar opcode; +pcre_uchar type; +sljit_u32 max = 0, exact; +struct sljit_label *label = NULL; +struct sljit_jump *jump = NULL; +jump_list *jumplist = NULL; +pcre_uchar *end; +int private_data_ptr = PRIVATE_DATA(cc); +int base = (private_data_ptr == 0) ? SLJIT_MEM1(STACK_TOP) : SLJIT_MEM1(SLJIT_SP); +int offset0 = (private_data_ptr == 0) ? STACK(0) : private_data_ptr; +int offset1 = (private_data_ptr == 0) ? STACK(1) : private_data_ptr + (int)sizeof(sljit_sw); + +cc = get_iterator_parameters(common, cc, &opcode, &type, &max, &exact, &end); + +switch(opcode) + { + case OP_STAR: + case OP_UPTO: + if (type == OP_ANYNL || type == OP_EXTUNI) + { + SLJIT_ASSERT(private_data_ptr == 0); + set_jumps(CURRENT_AS(char_iterator_backtrack)->u.backtracks, LABEL()); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + free_stack(common, 1); + CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(char_iterator_backtrack)->matchingpath); + } + else + { + if (CURRENT_AS(char_iterator_backtrack)->u.charpos.enabled) + { + OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); + OP1(SLJIT_MOV, TMP2, 0, base, offset1); + OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); + + jump = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0); + label = LABEL(); + OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1)); + OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); + if (CURRENT_AS(char_iterator_backtrack)->u.charpos.othercasebit != 0) + OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, CURRENT_AS(char_iterator_backtrack)->u.charpos.othercasebit); + CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CURRENT_AS(char_iterator_backtrack)->u.charpos.chr, CURRENT_AS(char_iterator_backtrack)->matchingpath); + skip_char_back(common); + CMPTO(SLJIT_GREATER, STR_PTR, 0, TMP2, 0, label); + } + else + { + OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); + jump = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, base, offset1); + skip_char_back(common); + OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); + JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath); + } + JUMPHERE(jump); + if (private_data_ptr == 0) + free_stack(common, 2); + } + break; + + case OP_MINSTAR: + OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); + compile_char1_matchingpath(common, type, cc, &jumplist, TRUE); + OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); + JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath); + set_jumps(jumplist, LABEL()); + if (private_data_ptr == 0) + free_stack(common, 1); + break; + + case OP_MINUPTO: + OP1(SLJIT_MOV, TMP1, 0, base, offset1); + OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); + OP2(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); + add_jump(compiler, &jumplist, JUMP(SLJIT_ZERO)); + + OP1(SLJIT_MOV, base, offset1, TMP1, 0); + compile_char1_matchingpath(common, type, cc, &jumplist, TRUE); + OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); + JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath); + + set_jumps(jumplist, LABEL()); + if (private_data_ptr == 0) + free_stack(common, 2); + break; + + case OP_QUERY: + OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); + OP1(SLJIT_MOV, base, offset0, SLJIT_IMM, 0); + CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(char_iterator_backtrack)->matchingpath); + jump = JUMP(SLJIT_JUMP); + set_jumps(CURRENT_AS(char_iterator_backtrack)->u.backtracks, LABEL()); + OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); + OP1(SLJIT_MOV, base, offset0, SLJIT_IMM, 0); + JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath); + JUMPHERE(jump); + if (private_data_ptr == 0) + free_stack(common, 1); + break; + + case OP_MINQUERY: + OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); + OP1(SLJIT_MOV, base, offset0, SLJIT_IMM, 0); + jump = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0); + compile_char1_matchingpath(common, type, cc, &jumplist, TRUE); + JUMPTO(SLJIT_JUMP, CURRENT_AS(char_iterator_backtrack)->matchingpath); + set_jumps(jumplist, LABEL()); + JUMPHERE(jump); + if (private_data_ptr == 0) + free_stack(common, 1); + break; + + case OP_EXACT: + case OP_POSSTAR: + case OP_POSQUERY: + case OP_POSUPTO: + break; + + default: + SLJIT_UNREACHABLE(); + break; + } + +set_jumps(current->topbacktracks, LABEL()); +} + +static SLJIT_INLINE void compile_ref_iterator_backtrackingpath(compiler_common *common, struct backtrack_common *current) +{ +DEFINE_COMPILER; +pcre_uchar *cc = current->cc; +BOOL ref = (*cc == OP_REF || *cc == OP_REFI); +pcre_uchar type; + +type = cc[ref ? 1 + IMM2_SIZE : 1 + 2 * IMM2_SIZE]; + +if ((type & 0x1) == 0) + { + /* Maximize case. */ + set_jumps(current->topbacktracks, LABEL()); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + free_stack(common, 1); + CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(ref_iterator_backtrack)->matchingpath); + return; + } + +OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); +CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(ref_iterator_backtrack)->matchingpath); +set_jumps(current->topbacktracks, LABEL()); +free_stack(common, ref ? 2 : 3); +} + +static SLJIT_INLINE void compile_recurse_backtrackingpath(compiler_common *common, struct backtrack_common *current) +{ +DEFINE_COMPILER; + +if (CURRENT_AS(recurse_backtrack)->inlined_pattern) + compile_backtrackingpath(common, current->top); +set_jumps(current->topbacktracks, LABEL()); +if (CURRENT_AS(recurse_backtrack)->inlined_pattern) + return; + +if (common->has_set_som && common->mark_ptr != 0) + { + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); + free_stack(common, 2); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(0), TMP2, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, TMP1, 0); + } +else if (common->has_set_som || common->mark_ptr != 0) + { + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + free_stack(common, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->has_set_som ? (int)(OVECTOR(0)) : common->mark_ptr, TMP2, 0); + } +} + +static void compile_assert_backtrackingpath(compiler_common *common, struct backtrack_common *current) +{ +DEFINE_COMPILER; +pcre_uchar *cc = current->cc; +pcre_uchar bra = OP_BRA; +struct sljit_jump *brajump = NULL; + +SLJIT_ASSERT(*cc != OP_BRAMINZERO); +if (*cc == OP_BRAZERO) + { + bra = *cc; + cc++; + } + +if (bra == OP_BRAZERO) + { + SLJIT_ASSERT(current->topbacktracks == NULL); + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + } + +if (CURRENT_AS(assert_backtrack)->framesize < 0) + { + set_jumps(current->topbacktracks, LABEL()); + + if (bra == OP_BRAZERO) + { + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); + CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(assert_backtrack)->matchingpath); + free_stack(common, 1); + } + return; + } + +if (bra == OP_BRAZERO) + { + if (*cc == OP_ASSERT_NOT || *cc == OP_ASSERTBACK_NOT) + { + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); + CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(assert_backtrack)->matchingpath); + free_stack(common, 1); + return; + } + free_stack(common, 1); + brajump = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0); + } + +if (*cc == OP_ASSERT || *cc == OP_ASSERTBACK) + { + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(assert_backtrack)->private_data_ptr); + add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(assert_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-CURRENT_AS(assert_backtrack)->framesize - 1)); + + set_jumps(current->topbacktracks, LABEL()); + } +else + set_jumps(current->topbacktracks, LABEL()); + +if (bra == OP_BRAZERO) + { + /* We know there is enough place on the stack. */ + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, sizeof(sljit_sw)); + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), SLJIT_IMM, 0); + JUMPTO(SLJIT_JUMP, CURRENT_AS(assert_backtrack)->matchingpath); + JUMPHERE(brajump); + } +} + +static void compile_bracket_backtrackingpath(compiler_common *common, struct backtrack_common *current) +{ +DEFINE_COMPILER; +int opcode, stacksize, alt_count, alt_max; +int offset = 0; +int private_data_ptr = CURRENT_AS(bracket_backtrack)->private_data_ptr; +int repeat_ptr = 0, repeat_type = 0, repeat_count = 0; +pcre_uchar *cc = current->cc; +pcre_uchar *ccbegin; +pcre_uchar *ccprev; +pcre_uchar bra = OP_BRA; +pcre_uchar ket; +assert_backtrack *assert; +sljit_uw *next_update_addr = NULL; +BOOL has_alternatives; +BOOL needs_control_head = FALSE; +struct sljit_jump *brazero = NULL; +struct sljit_jump *alt1 = NULL; +struct sljit_jump *alt2 = NULL; +struct sljit_jump *once = NULL; +struct sljit_jump *cond = NULL; +struct sljit_label *rmin_label = NULL; +struct sljit_label *exact_label = NULL; + +if (*cc == OP_BRAZERO || *cc == OP_BRAMINZERO) + { + bra = *cc; + cc++; + } + +opcode = *cc; +ccbegin = bracketend(cc) - 1 - LINK_SIZE; +ket = *ccbegin; +if (ket == OP_KET && PRIVATE_DATA(ccbegin) != 0) + { + repeat_ptr = PRIVATE_DATA(ccbegin); + repeat_type = PRIVATE_DATA(ccbegin + 2); + repeat_count = PRIVATE_DATA(ccbegin + 3); + SLJIT_ASSERT(repeat_type != 0 && repeat_count != 0); + if (repeat_type == OP_UPTO) + ket = OP_KETRMAX; + if (repeat_type == OP_MINUPTO) + ket = OP_KETRMIN; + } +ccbegin = cc; +cc += GET(cc, 1); +has_alternatives = *cc == OP_ALT; +if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND)) + has_alternatives = (ccbegin[1 + LINK_SIZE] >= OP_ASSERT && ccbegin[1 + LINK_SIZE] <= OP_ASSERTBACK_NOT) || CURRENT_AS(bracket_backtrack)->u.condfailed != NULL; +if (opcode == OP_CBRA || opcode == OP_SCBRA) + offset = (GET2(ccbegin, 1 + LINK_SIZE)) << 1; +if (SLJIT_UNLIKELY(opcode == OP_COND) && (*cc == OP_KETRMAX || *cc == OP_KETRMIN)) + opcode = OP_SCOND; +if (SLJIT_UNLIKELY(opcode == OP_ONCE_NC)) + opcode = OP_ONCE; + +alt_max = has_alternatives ? no_alternatives(ccbegin) : 0; + +/* Decoding the needs_control_head in framesize. */ +if (opcode == OP_ONCE) + { + needs_control_head = (CURRENT_AS(bracket_backtrack)->u.framesize & 0x1) != 0; + CURRENT_AS(bracket_backtrack)->u.framesize >>= 1; + } + +if (ket != OP_KET && repeat_type != 0) + { + /* TMP1 is used in OP_KETRMIN below. */ + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + free_stack(common, 1); + if (repeat_type == OP_UPTO) + OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_SP), repeat_ptr, TMP1, 0, SLJIT_IMM, 1); + else + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), repeat_ptr, TMP1, 0); + } + +if (ket == OP_KETRMAX) + { + if (bra == OP_BRAZERO) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + free_stack(common, 1); + brazero = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, 0); + } + } +else if (ket == OP_KETRMIN) + { + if (bra != OP_BRAMINZERO) + { + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + if (repeat_type != 0) + { + /* TMP1 was set a few lines above. */ + CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0, CURRENT_AS(bracket_backtrack)->recursive_matchingpath); + /* Drop STR_PTR for non-greedy plus quantifier. */ + if (opcode != OP_ONCE) + free_stack(common, 1); + } + else if (opcode >= OP_SBRA || opcode == OP_ONCE) + { + /* Checking zero-length iteration. */ + if (opcode != OP_ONCE || CURRENT_AS(bracket_backtrack)->u.framesize < 0) + CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr, CURRENT_AS(bracket_backtrack)->recursive_matchingpath); + else + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_MEM1(TMP1), STACK(-CURRENT_AS(bracket_backtrack)->u.framesize - 2), CURRENT_AS(bracket_backtrack)->recursive_matchingpath); + } + /* Drop STR_PTR for non-greedy plus quantifier. */ + if (opcode != OP_ONCE) + free_stack(common, 1); + } + else + JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_backtrack)->recursive_matchingpath); + } + rmin_label = LABEL(); + if (repeat_type != 0) + OP2(SLJIT_ADD, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1); + } +else if (bra == OP_BRAZERO) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + free_stack(common, 1); + brazero = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0); + } +else if (repeat_type == OP_EXACT) + { + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1); + exact_label = LABEL(); + } + +if (offset != 0) + { + if (common->capture_last_ptr != 0) + { + SLJIT_ASSERT(common->optimized_cbracket[offset >> 1] == 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, TMP1, 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(2)); + free_stack(common, 3); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP2, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP1, 0); + } + else if (common->optimized_cbracket[offset >> 1] == 0) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); + free_stack(common, 2); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP2, 0); + } + } + +if (SLJIT_UNLIKELY(opcode == OP_ONCE)) + { + if (CURRENT_AS(bracket_backtrack)->u.framesize >= 0) + { + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); + } + once = JUMP(SLJIT_JUMP); + } +else if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND)) + { + if (has_alternatives) + { + /* Always exactly one alternative. */ + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + free_stack(common, 1); + + alt_max = 2; + alt1 = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, sizeof(sljit_uw)); + } + } +else if (has_alternatives) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + free_stack(common, 1); + + if (alt_max > 4) + { + /* Table jump if alt_max is greater than 4. */ + next_update_addr = allocate_read_only_data(common, alt_max * sizeof(sljit_uw)); + if (SLJIT_UNLIKELY(next_update_addr == NULL)) + return; + sljit_emit_ijump(compiler, SLJIT_JUMP, SLJIT_MEM1(TMP1), (sljit_sw)next_update_addr); + add_label_addr(common, next_update_addr++); + } + else + { + if (alt_max == 4) + alt2 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_uw)); + alt1 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, sizeof(sljit_uw)); + } + } + +COMPILE_BACKTRACKINGPATH(current->top); +if (current->topbacktracks) + set_jumps(current->topbacktracks, LABEL()); + +if (SLJIT_UNLIKELY(opcode == OP_COND) || SLJIT_UNLIKELY(opcode == OP_SCOND)) + { + /* Conditional block always has at most one alternative. */ + if (ccbegin[1 + LINK_SIZE] >= OP_ASSERT && ccbegin[1 + LINK_SIZE] <= OP_ASSERTBACK_NOT) + { + SLJIT_ASSERT(has_alternatives); + assert = CURRENT_AS(bracket_backtrack)->u.assert; + if (assert->framesize >= 0 && (ccbegin[1 + LINK_SIZE] == OP_ASSERT || ccbegin[1 + LINK_SIZE] == OP_ASSERTBACK)) + { + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr); + add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-assert->framesize - 1)); + } + cond = JUMP(SLJIT_JUMP); + set_jumps(CURRENT_AS(bracket_backtrack)->u.assert->condfailed, LABEL()); + } + else if (CURRENT_AS(bracket_backtrack)->u.condfailed != NULL) + { + SLJIT_ASSERT(has_alternatives); + cond = JUMP(SLJIT_JUMP); + set_jumps(CURRENT_AS(bracket_backtrack)->u.condfailed, LABEL()); + } + else + SLJIT_ASSERT(!has_alternatives); + } + +if (has_alternatives) + { + alt_count = sizeof(sljit_uw); + do + { + current->top = NULL; + current->topbacktracks = NULL; + current->nextbacktracks = NULL; + /* Conditional blocks always have an additional alternative, even if it is empty. */ + if (*cc == OP_ALT) + { + ccprev = cc + 1 + LINK_SIZE; + cc += GET(cc, 1); + if (opcode != OP_COND && opcode != OP_SCOND) + { + if (opcode != OP_ONCE) + { + if (private_data_ptr != 0) + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), private_data_ptr); + else + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + } + else + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(needs_control_head ? 1 : 0)); + } + compile_matchingpath(common, ccprev, cc, current); + if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) + return; + } + + /* Instructions after the current alternative is successfully matched. */ + /* There is a similar code in compile_bracket_matchingpath. */ + if (opcode == OP_ONCE) + match_once_common(common, ket, CURRENT_AS(bracket_backtrack)->u.framesize, private_data_ptr, has_alternatives, needs_control_head); + + stacksize = 0; + if (repeat_type == OP_MINUPTO) + { + /* We need to preserve the counter. TMP2 will be used below. */ + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), repeat_ptr); + stacksize++; + } + if (ket != OP_KET || bra != OP_BRA) + stacksize++; + if (offset != 0) + { + if (common->capture_last_ptr != 0) + stacksize++; + if (common->optimized_cbracket[offset >> 1] == 0) + stacksize += 2; + } + if (opcode != OP_ONCE) + stacksize++; + + if (stacksize > 0) + allocate_stack(common, stacksize); + + stacksize = 0; + if (repeat_type == OP_MINUPTO) + { + /* TMP2 was set above. */ + OP2(SLJIT_SUB, SLJIT_MEM1(STACK_TOP), STACK(stacksize), TMP2, 0, SLJIT_IMM, 1); + stacksize++; + } + + if (ket != OP_KET || bra != OP_BRA) + { + if (ket != OP_KET) + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), STR_PTR, 0); + else + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, 0); + stacksize++; + } + + if (offset != 0) + stacksize = match_capture_common(common, stacksize, offset, private_data_ptr); + + if (opcode != OP_ONCE) + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(stacksize), SLJIT_IMM, alt_count); + + if (offset != 0 && ket == OP_KETRMAX && common->optimized_cbracket[offset >> 1] != 0) + { + /* If ket is not OP_KETRMAX, this code path is executed after the jump to alternative_matchingpath. */ + SLJIT_ASSERT(private_data_ptr == OVECTOR(offset + 0)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), STR_PTR, 0); + } + + JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_backtrack)->alternative_matchingpath); + + if (opcode != OP_ONCE) + { + if (alt_max > 4) + add_label_addr(common, next_update_addr++); + else + { + if (alt_count != 2 * sizeof(sljit_uw)) + { + JUMPHERE(alt1); + if (alt_max == 3 && alt_count == sizeof(sljit_uw)) + alt2 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 2 * sizeof(sljit_uw)); + } + else + { + JUMPHERE(alt2); + if (alt_max == 4) + alt1 = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 3 * sizeof(sljit_uw)); + } + } + alt_count += sizeof(sljit_uw); + } + + COMPILE_BACKTRACKINGPATH(current->top); + if (current->topbacktracks) + set_jumps(current->topbacktracks, LABEL()); + SLJIT_ASSERT(!current->nextbacktracks); + } + while (*cc == OP_ALT); + + if (cond != NULL) + { + SLJIT_ASSERT(opcode == OP_COND || opcode == OP_SCOND); + assert = CURRENT_AS(bracket_backtrack)->u.assert; + if ((ccbegin[1 + LINK_SIZE] == OP_ASSERT_NOT || ccbegin[1 + LINK_SIZE] == OP_ASSERTBACK_NOT) && assert->framesize >= 0) + { + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr); + add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), assert->private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-assert->framesize - 1)); + } + JUMPHERE(cond); + } + + /* Free the STR_PTR. */ + if (private_data_ptr == 0) + free_stack(common, 1); + } + +if (offset != 0) + { + /* Using both tmp register is better for instruction scheduling. */ + if (common->optimized_cbracket[offset >> 1] != 0) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); + free_stack(common, 2); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP2, 0); + } + else + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + free_stack(common, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0); + } + } +else if (opcode == OP_SBRA || opcode == OP_SCOND) + { + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(0)); + free_stack(common, 1); + } +else if (opcode == OP_ONCE) + { + cc = ccbegin + GET(ccbegin, 1); + stacksize = needs_control_head ? 1 : 0; + + if (CURRENT_AS(bracket_backtrack)->u.framesize >= 0) + { + /* Reset head and drop saved frame. */ + stacksize += CURRENT_AS(bracket_backtrack)->u.framesize + ((ket != OP_KET || *cc == OP_ALT) ? 2 : 1); + } + else if (ket == OP_KETRMAX || (*cc == OP_ALT && ket != OP_KETRMIN)) + { + /* The STR_PTR must be released. */ + stacksize++; + } + + if (stacksize > 0) + free_stack(common, stacksize); + + JUMPHERE(once); + /* Restore previous private_data_ptr */ + if (CURRENT_AS(bracket_backtrack)->u.framesize >= 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-CURRENT_AS(bracket_backtrack)->u.framesize - 1)); + else if (ket == OP_KETRMIN) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); + /* See the comment below. */ + free_stack(common, 2); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), private_data_ptr, TMP1, 0); + } + } + +if (repeat_type == OP_EXACT) + { + OP2(SLJIT_ADD, TMP1, 0, SLJIT_MEM1(SLJIT_SP), repeat_ptr, SLJIT_IMM, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), repeat_ptr, TMP1, 0); + CMPTO(SLJIT_LESS_EQUAL, TMP1, 0, SLJIT_IMM, repeat_count, exact_label); + } +else if (ket == OP_KETRMAX) + { + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + if (bra != OP_BRAZERO) + free_stack(common, 1); + + CMPTO(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, 0, CURRENT_AS(bracket_backtrack)->recursive_matchingpath); + if (bra == OP_BRAZERO) + { + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); + JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_backtrack)->zero_matchingpath); + JUMPHERE(brazero); + free_stack(common, 1); + } + } +else if (ket == OP_KETRMIN) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + + /* OP_ONCE removes everything in case of a backtrack, so we don't + need to explicitly release the STR_PTR. The extra release would + affect badly the free_stack(2) above. */ + if (opcode != OP_ONCE) + free_stack(common, 1); + CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, 0, rmin_label); + if (opcode == OP_ONCE) + free_stack(common, bra == OP_BRAMINZERO ? 2 : 1); + else if (bra == OP_BRAMINZERO) + free_stack(common, 1); + } +else if (bra == OP_BRAZERO) + { + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + JUMPTO(SLJIT_JUMP, CURRENT_AS(bracket_backtrack)->zero_matchingpath); + JUMPHERE(brazero); + } +} + +static SLJIT_INLINE void compile_bracketpos_backtrackingpath(compiler_common *common, struct backtrack_common *current) +{ +DEFINE_COMPILER; +int offset; +struct sljit_jump *jump; + +if (CURRENT_AS(bracketpos_backtrack)->framesize < 0) + { + if (*current->cc == OP_CBRAPOS || *current->cc == OP_SCBRAPOS) + { + offset = (GET2(current->cc, 1 + LINK_SIZE)) << 1; + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(1)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset), TMP1, 0); + if (common->capture_last_ptr != 0) + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(2)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(offset + 1), TMP2, 0); + if (common->capture_last_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, TMP1, 0); + } + set_jumps(current->topbacktracks, LABEL()); + free_stack(common, CURRENT_AS(bracketpos_backtrack)->stacksize); + return; + } + +OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(bracketpos_backtrack)->private_data_ptr); +add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); + +if (current->topbacktracks) + { + jump = JUMP(SLJIT_JUMP); + set_jumps(current->topbacktracks, LABEL()); + /* Drop the stack frame. */ + free_stack(common, CURRENT_AS(bracketpos_backtrack)->stacksize); + JUMPHERE(jump); + } +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), CURRENT_AS(bracketpos_backtrack)->private_data_ptr, SLJIT_MEM1(STACK_TOP), STACK(-CURRENT_AS(bracketpos_backtrack)->framesize - 1)); +} + +static SLJIT_INLINE void compile_braminzero_backtrackingpath(compiler_common *common, struct backtrack_common *current) +{ +assert_backtrack backtrack; + +current->top = NULL; +current->topbacktracks = NULL; +current->nextbacktracks = NULL; +if (current->cc[1] > OP_ASSERTBACK_NOT) + { + /* Manual call of compile_bracket_matchingpath and compile_bracket_backtrackingpath. */ + compile_bracket_matchingpath(common, current->cc, current); + compile_bracket_backtrackingpath(common, current->top); + } +else + { + memset(&backtrack, 0, sizeof(backtrack)); + backtrack.common.cc = current->cc; + backtrack.matchingpath = CURRENT_AS(braminzero_backtrack)->matchingpath; + /* Manual call of compile_assert_matchingpath. */ + compile_assert_matchingpath(common, current->cc, &backtrack, FALSE); + } +SLJIT_ASSERT(!current->nextbacktracks && !current->topbacktracks); +} + +static SLJIT_INLINE void compile_control_verb_backtrackingpath(compiler_common *common, struct backtrack_common *current) +{ +DEFINE_COMPILER; +pcre_uchar opcode = *current->cc; +struct sljit_label *loop; +struct sljit_jump *jump; + +if (opcode == OP_THEN || opcode == OP_THEN_ARG) + { + if (common->then_trap != NULL) + { + SLJIT_ASSERT(common->control_head_ptr != 0); + + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, type_then_trap); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, common->then_trap->start); + jump = JUMP(SLJIT_JUMP); + + loop = LABEL(); + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + JUMPHERE(jump); + CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(1), TMP1, 0, loop); + CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(STACK_TOP), STACK(2), TMP2, 0, loop); + add_jump(compiler, &common->then_trap->quit, JUMP(SLJIT_JUMP)); + return; + } + else if (common->positive_assert) + { + add_jump(compiler, &common->positive_assert_quit, JUMP(SLJIT_JUMP)); + return; + } + } + +if (common->local_exit) + { + if (common->quit_label == NULL) + add_jump(compiler, &common->quit, JUMP(SLJIT_JUMP)); + else + JUMPTO(SLJIT_JUMP, common->quit_label); + return; + } + +if (opcode == OP_SKIP_ARG) + { + SLJIT_ASSERT(common->control_head_ptr != 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STACK_TOP, 0); + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_IMM, (sljit_sw)(current->cc + 2)); + sljit_emit_ijump(compiler, SLJIT_CALL2, SLJIT_IMM, SLJIT_FUNC_OFFSET(do_search_mark)); + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0); + + OP1(SLJIT_MOV, STR_PTR, 0, TMP1, 0); + add_jump(compiler, &common->reset_match, CMP(SLJIT_NOT_EQUAL, STR_PTR, 0, SLJIT_IMM, -1)); + return; + } + +if (opcode == OP_SKIP) + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); +else + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_IMM, 0); +add_jump(compiler, &common->reset_match, JUMP(SLJIT_JUMP)); +} + +static SLJIT_INLINE void compile_then_trap_backtrackingpath(compiler_common *common, struct backtrack_common *current) +{ +DEFINE_COMPILER; +struct sljit_jump *jump; +int size; + +if (CURRENT_AS(then_trap_backtrack)->then_trap) + { + common->then_trap = CURRENT_AS(then_trap_backtrack)->then_trap; + return; + } + +size = CURRENT_AS(then_trap_backtrack)->framesize; +size = 3 + (size < 0 ? 0 : size); + +OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(size - 3)); +free_stack(common, size); +jump = JUMP(SLJIT_JUMP); + +set_jumps(CURRENT_AS(then_trap_backtrack)->quit, LABEL()); +/* STACK_TOP is set by THEN. */ +if (CURRENT_AS(then_trap_backtrack)->framesize >= 0) + add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); +OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); +free_stack(common, 3); + +JUMPHERE(jump); +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, TMP1, 0); +} + +static void compile_backtrackingpath(compiler_common *common, struct backtrack_common *current) +{ +DEFINE_COMPILER; +then_trap_backtrack *save_then_trap = common->then_trap; + +while (current) + { + if (current->nextbacktracks != NULL) + set_jumps(current->nextbacktracks, LABEL()); + switch(*current->cc) + { + case OP_SET_SOM: + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + free_stack(common, 1); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(0), TMP1, 0); + break; + + case OP_STAR: + case OP_MINSTAR: + case OP_PLUS: + case OP_MINPLUS: + case OP_QUERY: + case OP_MINQUERY: + case OP_UPTO: + case OP_MINUPTO: + case OP_EXACT: + case OP_POSSTAR: + case OP_POSPLUS: + case OP_POSQUERY: + case OP_POSUPTO: + case OP_STARI: + case OP_MINSTARI: + case OP_PLUSI: + case OP_MINPLUSI: + case OP_QUERYI: + case OP_MINQUERYI: + case OP_UPTOI: + case OP_MINUPTOI: + case OP_EXACTI: + case OP_POSSTARI: + case OP_POSPLUSI: + case OP_POSQUERYI: + case OP_POSUPTOI: + case OP_NOTSTAR: + case OP_NOTMINSTAR: + case OP_NOTPLUS: + case OP_NOTMINPLUS: + case OP_NOTQUERY: + case OP_NOTMINQUERY: + case OP_NOTUPTO: + case OP_NOTMINUPTO: + case OP_NOTEXACT: + case OP_NOTPOSSTAR: + case OP_NOTPOSPLUS: + case OP_NOTPOSQUERY: + case OP_NOTPOSUPTO: + case OP_NOTSTARI: + case OP_NOTMINSTARI: + case OP_NOTPLUSI: + case OP_NOTMINPLUSI: + case OP_NOTQUERYI: + case OP_NOTMINQUERYI: + case OP_NOTUPTOI: + case OP_NOTMINUPTOI: + case OP_NOTEXACTI: + case OP_NOTPOSSTARI: + case OP_NOTPOSPLUSI: + case OP_NOTPOSQUERYI: + case OP_NOTPOSUPTOI: + case OP_TYPESTAR: + case OP_TYPEMINSTAR: + case OP_TYPEPLUS: + case OP_TYPEMINPLUS: + case OP_TYPEQUERY: + case OP_TYPEMINQUERY: + case OP_TYPEUPTO: + case OP_TYPEMINUPTO: + case OP_TYPEEXACT: + case OP_TYPEPOSSTAR: + case OP_TYPEPOSPLUS: + case OP_TYPEPOSQUERY: + case OP_TYPEPOSUPTO: + case OP_CLASS: + case OP_NCLASS: +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: +#endif + compile_iterator_backtrackingpath(common, current); + break; + + case OP_REF: + case OP_REFI: + case OP_DNREF: + case OP_DNREFI: + compile_ref_iterator_backtrackingpath(common, current); + break; + + case OP_RECURSE: + compile_recurse_backtrackingpath(common, current); + break; + + case OP_ASSERT: + case OP_ASSERT_NOT: + case OP_ASSERTBACK: + case OP_ASSERTBACK_NOT: + compile_assert_backtrackingpath(common, current); + break; + + case OP_ONCE: + case OP_ONCE_NC: + case OP_BRA: + case OP_CBRA: + case OP_COND: + case OP_SBRA: + case OP_SCBRA: + case OP_SCOND: + compile_bracket_backtrackingpath(common, current); + break; + + case OP_BRAZERO: + if (current->cc[1] > OP_ASSERTBACK_NOT) + compile_bracket_backtrackingpath(common, current); + else + compile_assert_backtrackingpath(common, current); + break; + + case OP_BRAPOS: + case OP_CBRAPOS: + case OP_SBRAPOS: + case OP_SCBRAPOS: + case OP_BRAPOSZERO: + compile_bracketpos_backtrackingpath(common, current); + break; + + case OP_BRAMINZERO: + compile_braminzero_backtrackingpath(common, current); + break; + + case OP_MARK: + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(common->has_skip_arg ? 4 : 0)); + if (common->has_skip_arg) + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + free_stack(common, common->has_skip_arg ? 5 : 1); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, TMP1, 0); + if (common->has_skip_arg) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, TMP2, 0); + break; + + case OP_THEN: + case OP_THEN_ARG: + case OP_PRUNE: + case OP_PRUNE_ARG: + case OP_SKIP: + case OP_SKIP_ARG: + compile_control_verb_backtrackingpath(common, current); + break; + + case OP_COMMIT: + if (!common->local_exit) + OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_NOMATCH); + if (common->quit_label == NULL) + add_jump(compiler, &common->quit, JUMP(SLJIT_JUMP)); + else + JUMPTO(SLJIT_JUMP, common->quit_label); + break; + + case OP_CALLOUT: + case OP_FAIL: + case OP_ACCEPT: + case OP_ASSERT_ACCEPT: + set_jumps(current->topbacktracks, LABEL()); + break; + + case OP_THEN_TRAP: + /* A virtual opcode for then traps. */ + compile_then_trap_backtrackingpath(common, current); + break; + + default: + SLJIT_UNREACHABLE(); + break; + } + current = current->prev; + } +common->then_trap = save_then_trap; +} + +static SLJIT_INLINE void compile_recurse(compiler_common *common) +{ +DEFINE_COMPILER; +pcre_uchar *cc = common->start + common->currententry->start; +pcre_uchar *ccbegin = cc + 1 + LINK_SIZE + (*cc == OP_BRA ? 0 : IMM2_SIZE); +pcre_uchar *ccend = bracketend(cc) - (1 + LINK_SIZE); +BOOL needs_control_head; +int framesize = get_framesize(common, cc, NULL, TRUE, &needs_control_head); +int private_data_size = get_private_data_copy_length(common, ccbegin, ccend, needs_control_head); +int alternativesize; +BOOL needs_frame; +backtrack_common altbacktrack; +struct sljit_jump *jump; + +/* Recurse captures then. */ +common->then_trap = NULL; + +SLJIT_ASSERT(*cc == OP_BRA || *cc == OP_CBRA || *cc == OP_CBRAPOS || *cc == OP_SCBRA || *cc == OP_SCBRAPOS); +needs_frame = framesize >= 0; +if (!needs_frame) + framesize = 0; +alternativesize = *(cc + GET(cc, 1)) == OP_ALT ? 1 : 0; + +SLJIT_ASSERT(common->currententry->entry == NULL && common->recursive_head_ptr != 0); +common->currententry->entry = LABEL(); +set_jumps(common->currententry->calls, common->currententry->entry); + +sljit_emit_fast_enter(compiler, TMP2, 0); +count_match(common); +allocate_stack(common, private_data_size + framesize + alternativesize); +OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(private_data_size + framesize + alternativesize - 1), TMP2, 0); +copy_private_data(common, ccbegin, ccend, TRUE, framesize + alternativesize, private_data_size + framesize + alternativesize, needs_control_head); +if (needs_control_head) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0); +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr, STACK_TOP, 0); +if (needs_frame) + init_frame(common, cc, NULL, framesize + alternativesize - 1, alternativesize, TRUE); + +if (alternativesize > 0) + OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); + +memset(&altbacktrack, 0, sizeof(backtrack_common)); +common->quit_label = NULL; +common->accept_label = NULL; +common->quit = NULL; +common->accept = NULL; +altbacktrack.cc = ccbegin; +cc += GET(cc, 1); +while (1) + { + altbacktrack.top = NULL; + altbacktrack.topbacktracks = NULL; + + if (altbacktrack.cc != ccbegin) + OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(STACK_TOP), STACK(0)); + + compile_matchingpath(common, altbacktrack.cc, cc, &altbacktrack); + if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) + return; + + add_jump(compiler, &common->accept, JUMP(SLJIT_JUMP)); + + compile_backtrackingpath(common, altbacktrack.top); + if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) + return; + set_jumps(altbacktrack.topbacktracks, LABEL()); + + if (*cc != OP_ALT) + break; + + altbacktrack.cc = cc + 1 + LINK_SIZE; + cc += GET(cc, 1); + } + +/* None of them matched. */ +OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 0); +jump = JUMP(SLJIT_JUMP); + +if (common->quit != NULL) + { + set_jumps(common->quit, LABEL()); + OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr); + if (needs_frame) + { + OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); + add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); + } + OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 0); + common->quit = NULL; + add_jump(compiler, &common->quit, JUMP(SLJIT_JUMP)); + } + +set_jumps(common->accept, LABEL()); +OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr); +if (needs_frame) + { + OP2(SLJIT_ADD, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); + add_jump(compiler, &common->revertframes, JUMP(SLJIT_FAST_CALL)); + OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, (framesize + alternativesize) * sizeof(sljit_sw)); + } +OP1(SLJIT_MOV, TMP3, 0, SLJIT_IMM, 1); + +JUMPHERE(jump); +if (common->quit != NULL) + set_jumps(common->quit, LABEL()); +copy_private_data(common, ccbegin, ccend, FALSE, framesize + alternativesize, private_data_size + framesize + alternativesize, needs_control_head); +free_stack(common, private_data_size + framesize + alternativesize); +if (needs_control_head) + { + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(STACK_TOP), STACK(-3)); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(-2)); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr, TMP1, 0); + OP1(SLJIT_MOV, TMP1, 0, TMP3, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, TMP2, 0); + } +else + { + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(STACK_TOP), STACK(-2)); + OP1(SLJIT_MOV, TMP1, 0, TMP3, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->recursive_head_ptr, TMP2, 0); + } +sljit_emit_fast_return(compiler, SLJIT_MEM1(STACK_TOP), STACK(-1)); +} + +#undef COMPILE_BACKTRACKINGPATH +#undef CURRENT_AS + +void +PRIV(jit_compile)(const REAL_PCRE *re, PUBL(extra) *extra, int mode) +{ +struct sljit_compiler *compiler; +backtrack_common rootbacktrack; +compiler_common common_data; +compiler_common *common = &common_data; +const sljit_u8 *tables = re->tables; +pcre_study_data *study; +int private_data_size; +pcre_uchar *ccend; +executable_functions *functions; +void *executable_func; +sljit_uw executable_size; +sljit_uw total_length; +label_addr_list *label_addr; +struct sljit_label *mainloop_label = NULL; +struct sljit_label *continue_match_label; +struct sljit_label *empty_match_found_label = NULL; +struct sljit_label *empty_match_backtrack_label = NULL; +struct sljit_label *reset_match_label; +struct sljit_label *quit_label; +struct sljit_jump *jump; +struct sljit_jump *minlength_check_failed = NULL; +struct sljit_jump *reqbyte_notfound = NULL; +struct sljit_jump *empty_match = NULL; + +SLJIT_ASSERT((extra->flags & PCRE_EXTRA_STUDY_DATA) != 0); +study = extra->study_data; + +if (!tables) + tables = PRIV(default_tables); + +memset(&rootbacktrack, 0, sizeof(backtrack_common)); +memset(common, 0, sizeof(compiler_common)); +rootbacktrack.cc = (pcre_uchar *)re + re->name_table_offset + re->name_count * re->name_entry_size; + +common->start = rootbacktrack.cc; +common->read_only_data_head = NULL; +common->fcc = tables + fcc_offset; +common->lcc = (sljit_sw)(tables + lcc_offset); +common->mode = mode; +common->might_be_empty = study->minlength == 0; +common->nltype = NLTYPE_FIXED; +switch(re->options & PCRE_NEWLINE_BITS) + { + case 0: + /* Compile-time default */ + switch(NEWLINE) + { + case -1: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANY; break; + case -2: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANYCRLF; break; + default: common->newline = NEWLINE; break; + } + break; + case PCRE_NEWLINE_CR: common->newline = CHAR_CR; break; + case PCRE_NEWLINE_LF: common->newline = CHAR_NL; break; + case PCRE_NEWLINE_CR+ + PCRE_NEWLINE_LF: common->newline = (CHAR_CR << 8) | CHAR_NL; break; + case PCRE_NEWLINE_ANY: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANY; break; + case PCRE_NEWLINE_ANYCRLF: common->newline = (CHAR_CR << 8) | CHAR_NL; common->nltype = NLTYPE_ANYCRLF; break; + default: return; + } +common->nlmax = READ_CHAR_MAX; +common->nlmin = 0; +if ((re->options & PCRE_BSR_ANYCRLF) != 0) + common->bsr_nltype = NLTYPE_ANYCRLF; +else if ((re->options & PCRE_BSR_UNICODE) != 0) + common->bsr_nltype = NLTYPE_ANY; +else + { +#ifdef BSR_ANYCRLF + common->bsr_nltype = NLTYPE_ANYCRLF; +#else + common->bsr_nltype = NLTYPE_ANY; +#endif + } +common->bsr_nlmax = READ_CHAR_MAX; +common->bsr_nlmin = 0; +common->endonly = (re->options & PCRE_DOLLAR_ENDONLY) != 0; +common->ctypes = (sljit_sw)(tables + ctypes_offset); +common->name_table = ((pcre_uchar *)re) + re->name_table_offset; +common->name_count = re->name_count; +common->name_entry_size = re->name_entry_size; +common->jscript_compat = (re->options & PCRE_JAVASCRIPT_COMPAT) != 0; +#ifdef SUPPORT_UTF +/* PCRE_UTF[16|32] have the same value as PCRE_UTF8. */ +common->utf = (re->options & PCRE_UTF8) != 0; +#ifdef SUPPORT_UCP +common->use_ucp = (re->options & PCRE_UCP) != 0; +#endif +if (common->utf) + { + if (common->nltype == NLTYPE_ANY) + common->nlmax = 0x2029; + else if (common->nltype == NLTYPE_ANYCRLF) + common->nlmax = (CHAR_CR > CHAR_NL) ? CHAR_CR : CHAR_NL; + else + { + /* We only care about the first newline character. */ + common->nlmax = common->newline & 0xff; + } + + if (common->nltype == NLTYPE_FIXED) + common->nlmin = common->newline & 0xff; + else + common->nlmin = (CHAR_CR < CHAR_NL) ? CHAR_CR : CHAR_NL; + + if (common->bsr_nltype == NLTYPE_ANY) + common->bsr_nlmax = 0x2029; + else + common->bsr_nlmax = (CHAR_CR > CHAR_NL) ? CHAR_CR : CHAR_NL; + common->bsr_nlmin = (CHAR_CR < CHAR_NL) ? CHAR_CR : CHAR_NL; + } +#endif /* SUPPORT_UTF */ +ccend = bracketend(common->start); + +/* Calculate the local space size on the stack. */ +common->ovector_start = LIMIT_MATCH + sizeof(sljit_sw); +common->optimized_cbracket = (sljit_u8 *)SLJIT_MALLOC(re->top_bracket + 1, compiler->allocator_data); +if (!common->optimized_cbracket) + return; +#if defined DEBUG_FORCE_UNOPTIMIZED_CBRAS && DEBUG_FORCE_UNOPTIMIZED_CBRAS == 1 +memset(common->optimized_cbracket, 0, re->top_bracket + 1); +#else +memset(common->optimized_cbracket, 1, re->top_bracket + 1); +#endif + +SLJIT_ASSERT(*common->start == OP_BRA && ccend[-(1 + LINK_SIZE)] == OP_KET); +#if defined DEBUG_FORCE_UNOPTIMIZED_CBRAS && DEBUG_FORCE_UNOPTIMIZED_CBRAS == 2 +common->capture_last_ptr = common->ovector_start; +common->ovector_start += sizeof(sljit_sw); +#endif +if (!check_opcode_types(common, common->start, ccend)) + { + SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data); + return; + } + +/* Checking flags and updating ovector_start. */ +if (mode == JIT_COMPILE && (re->flags & PCRE_REQCHSET) != 0 && (re->options & PCRE_NO_START_OPTIMIZE) == 0) + { + common->req_char_ptr = common->ovector_start; + common->ovector_start += sizeof(sljit_sw); + } +if (mode != JIT_COMPILE) + { + common->start_used_ptr = common->ovector_start; + common->ovector_start += sizeof(sljit_sw); + if (mode == JIT_PARTIAL_SOFT_COMPILE) + { + common->hit_start = common->ovector_start; + common->ovector_start += 2 * sizeof(sljit_sw); + } + } +if ((re->options & PCRE_FIRSTLINE) != 0) + { + common->match_end_ptr = common->ovector_start; + common->ovector_start += sizeof(sljit_sw); + } +#if defined DEBUG_FORCE_CONTROL_HEAD && DEBUG_FORCE_CONTROL_HEAD +common->control_head_ptr = 1; +#endif +if (common->control_head_ptr != 0) + { + common->control_head_ptr = common->ovector_start; + common->ovector_start += sizeof(sljit_sw); + } +if (common->has_set_som) + { + /* Saving the real start pointer is necessary. */ + common->start_ptr = common->ovector_start; + common->ovector_start += sizeof(sljit_sw); + } + +/* Aligning ovector to even number of sljit words. */ +if ((common->ovector_start & sizeof(sljit_sw)) != 0) + common->ovector_start += sizeof(sljit_sw); + +if (common->start_ptr == 0) + common->start_ptr = OVECTOR(0); + +/* Capturing brackets cannot be optimized if callouts are allowed. */ +if (common->capture_last_ptr != 0) + memset(common->optimized_cbracket, 0, re->top_bracket + 1); + +SLJIT_ASSERT(!(common->req_char_ptr != 0 && common->start_used_ptr != 0)); +common->cbra_ptr = OVECTOR_START + (re->top_bracket + 1) * 2 * sizeof(sljit_sw); + +total_length = ccend - common->start; +common->private_data_ptrs = (sljit_s32 *)SLJIT_MALLOC(total_length * (sizeof(sljit_s32) + (common->has_then ? 1 : 0)), compiler->allocator_data); +if (!common->private_data_ptrs) + { + SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data); + return; + } +memset(common->private_data_ptrs, 0, total_length * sizeof(sljit_s32)); + +private_data_size = common->cbra_ptr + (re->top_bracket + 1) * sizeof(sljit_sw); +set_private_data_ptrs(common, &private_data_size, ccend); +if ((re->options & PCRE_ANCHORED) == 0 && (re->options & PCRE_NO_START_OPTIMIZE) == 0) + { + if (!detect_fast_forward_skip(common, &private_data_size) && !common->has_skip_in_assert_back) + detect_fast_fail(common, common->start, &private_data_size, 4); + } + +SLJIT_ASSERT(common->fast_fail_start_ptr <= common->fast_fail_end_ptr); + +if (private_data_size > SLJIT_MAX_LOCAL_SIZE) + { + SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data); + SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data); + return; + } + +if (common->has_then) + { + common->then_offsets = (sljit_u8 *)(common->private_data_ptrs + total_length); + memset(common->then_offsets, 0, total_length); + set_then_offsets(common, common->start, NULL); + } + +compiler = sljit_create_compiler(NULL); +if (!compiler) + { + SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data); + SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data); + return; + } +common->compiler = compiler; + +/* Main pcre_jit_exec entry. */ +sljit_emit_enter(compiler, 0, 1, 5, 5, 0, 0, private_data_size); + +/* Register init. */ +reset_ovector(common, (re->top_bracket + 1) * 2); +if (common->req_char_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->req_char_ptr, SLJIT_R0, 0); + +OP1(SLJIT_MOV, ARGUMENTS, 0, SLJIT_S0, 0); +OP1(SLJIT_MOV, TMP1, 0, SLJIT_S0, 0); +OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); +OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, end)); +OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack)); +OP1(SLJIT_MOV_U32, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, limit_match)); +OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(struct sljit_stack, base)); +OP1(SLJIT_MOV, STACK_LIMIT, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(struct sljit_stack, limit)); +OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LIMIT_MATCH, TMP1, 0); + +if (common->fast_fail_start_ptr < common->fast_fail_end_ptr) + reset_fast_fail(common); + +if (mode == JIT_PARTIAL_SOFT_COMPILE) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, -1); +if (common->mark_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->mark_ptr, SLJIT_IMM, 0); +if (common->control_head_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->control_head_ptr, SLJIT_IMM, 0); + +/* Main part of the matching */ +if ((re->options & PCRE_ANCHORED) == 0) + { + mainloop_label = mainloop_entry(common, (re->flags & PCRE_HASCRORLF) != 0); + continue_match_label = LABEL(); + /* Forward search if possible. */ + if ((re->options & PCRE_NO_START_OPTIMIZE) == 0) + { + if (mode == JIT_COMPILE && fast_forward_first_n_chars(common)) + ; + else if ((re->flags & PCRE_FIRSTSET) != 0) + fast_forward_first_char(common, (pcre_uchar)re->first_char, (re->flags & PCRE_FCH_CASELESS) != 0); + else if ((re->flags & PCRE_STARTLINE) != 0) + fast_forward_newline(common); + else if (study != NULL && (study->flags & PCRE_STUDY_MAPPED) != 0) + fast_forward_start_bits(common, study->start_bits); + } + } +else + continue_match_label = LABEL(); + +if (mode == JIT_COMPILE && study->minlength > 0 && (re->options & PCRE_NO_START_OPTIMIZE) == 0) + { + OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_NOMATCH); + OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(study->minlength)); + minlength_check_failed = CMP(SLJIT_GREATER, TMP2, 0, STR_END, 0); + } +if (common->req_char_ptr != 0) + reqbyte_notfound = search_requested_char(common, (pcre_uchar)re->req_char, (re->flags & PCRE_RCH_CASELESS) != 0, (re->flags & PCRE_FIRSTSET) != 0); + +/* Store the current STR_PTR in OVECTOR(0). */ +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), OVECTOR(0), STR_PTR, 0); +/* Copy the limit of allowed recursions. */ +OP1(SLJIT_MOV, COUNT_MATCH, 0, SLJIT_MEM1(SLJIT_SP), LIMIT_MATCH); +if (common->capture_last_ptr != 0) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, SLJIT_IMM, -1); +if (common->fast_forward_bc_ptr != NULL) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), PRIVATE_DATA(common->fast_forward_bc_ptr + 1), STR_PTR, 0); + +if (common->start_ptr != OVECTOR(0)) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_ptr, STR_PTR, 0); + +/* Copy the beginning of the string. */ +if (mode == JIT_PARTIAL_SOFT_COMPILE) + { + jump = CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, -1); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start + sizeof(sljit_sw), STR_PTR, 0); + JUMPHERE(jump); + } +else if (mode == JIT_PARTIAL_HARD_COMPILE) + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0); + +compile_matchingpath(common, common->start, ccend, &rootbacktrack); +if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) + { + sljit_free_compiler(compiler); + SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data); + SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data); + free_read_only_data(common->read_only_data_head, compiler->allocator_data); + return; + } + +if (common->might_be_empty) + { + empty_match = CMP(SLJIT_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), OVECTOR(0)); + empty_match_found_label = LABEL(); + } + +common->accept_label = LABEL(); +if (common->accept != NULL) + set_jumps(common->accept, common->accept_label); + +/* This means we have a match. Update the ovector. */ +copy_ovector(common, re->top_bracket + 1); +common->quit_label = common->forced_quit_label = LABEL(); +if (common->quit != NULL) + set_jumps(common->quit, common->quit_label); +if (common->forced_quit != NULL) + set_jumps(common->forced_quit, common->forced_quit_label); +if (minlength_check_failed != NULL) + SET_LABEL(minlength_check_failed, common->forced_quit_label); +sljit_emit_return(compiler, SLJIT_MOV, SLJIT_RETURN_REG, 0); + +if (mode != JIT_COMPILE) + { + common->partialmatchlabel = LABEL(); + set_jumps(common->partialmatch, common->partialmatchlabel); + return_with_partial_match(common, common->quit_label); + } + +if (common->might_be_empty) + empty_match_backtrack_label = LABEL(); +compile_backtrackingpath(common, rootbacktrack.top); +if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) + { + sljit_free_compiler(compiler); + SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data); + SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data); + free_read_only_data(common->read_only_data_head, compiler->allocator_data); + return; + } + +SLJIT_ASSERT(rootbacktrack.prev == NULL); +reset_match_label = LABEL(); + +if (mode == JIT_PARTIAL_SOFT_COMPILE) + { + /* Update hit_start only in the first time. */ + jump = CMP(SLJIT_NOT_EQUAL, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, SLJIT_IMM, -1); + OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, TMP1, 0); + JUMPHERE(jump); + } + +/* Check we have remaining characters. */ +if ((re->options & PCRE_ANCHORED) == 0 && (re->options & PCRE_FIRSTLINE) != 0) + { + SLJIT_ASSERT(common->match_end_ptr != 0); + OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr); + } + +OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), + (common->fast_forward_bc_ptr != NULL) ? (PRIVATE_DATA(common->fast_forward_bc_ptr + 1)) : common->start_ptr); + +if ((re->options & PCRE_ANCHORED) == 0) + { + if (common->ff_newline_shortcut != NULL) + { + if ((re->options & PCRE_FIRSTLINE) == 0) + CMPTO(SLJIT_LESS, STR_PTR, 0, STR_END, 0, common->ff_newline_shortcut); + /* There cannot be more newlines here. */ + } + else + CMPTO(SLJIT_LESS, STR_PTR, 0, ((re->options & PCRE_FIRSTLINE) == 0) ? STR_END : TMP1, 0, mainloop_label); + } + +/* No more remaining characters. */ +if (reqbyte_notfound != NULL) + JUMPHERE(reqbyte_notfound); + +if (mode == JIT_PARTIAL_SOFT_COMPILE) + CMPTO(SLJIT_NOT_EQUAL, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, -1, common->partialmatchlabel); + +OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_NOMATCH); +JUMPTO(SLJIT_JUMP, common->quit_label); + +flush_stubs(common); + +if (common->might_be_empty) + { + JUMPHERE(empty_match); + OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); + OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty)); + CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, 0, empty_match_backtrack_label); + OP1(SLJIT_MOV_U8, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, notempty_atstart)); + CMPTO(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, 0, empty_match_found_label); + OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str)); + CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, STR_PTR, 0, empty_match_found_label); + JUMPTO(SLJIT_JUMP, empty_match_backtrack_label); + } + +common->fast_forward_bc_ptr = NULL; +common->fast_fail_start_ptr = 0; +common->fast_fail_end_ptr = 0; +common->currententry = common->entries; +common->local_exit = TRUE; +quit_label = common->quit_label; +while (common->currententry != NULL) + { + /* Might add new entries. */ + compile_recurse(common); + if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler))) + { + sljit_free_compiler(compiler); + SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data); + SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data); + free_read_only_data(common->read_only_data_head, compiler->allocator_data); + return; + } + flush_stubs(common); + common->currententry = common->currententry->next; + } +common->local_exit = FALSE; +common->quit_label = quit_label; + +/* Allocating stack, returns with PCRE_ERROR_JIT_STACKLIMIT if fails. */ +/* This is a (really) rare case. */ +set_jumps(common->stackalloc, LABEL()); +/* RETURN_ADDR is not a saved register. */ +sljit_emit_fast_enter(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0); +OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP2, 0); +OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); +OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack)); +OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, top), STACK_TOP, 0); +OP2(SLJIT_SUB, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit), SLJIT_IMM, STACK_GROWTH_RATE); + +sljit_emit_ijump(compiler, SLJIT_CALL2, SLJIT_IMM, SLJIT_FUNC_OFFSET(sljit_stack_resize)); +jump = CMP(SLJIT_NOT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0); +OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0); +OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, stack)); +OP1(SLJIT_MOV, STACK_TOP, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, top)); +OP1(SLJIT_MOV, STACK_LIMIT, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(struct sljit_stack, limit)); +OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1); +sljit_emit_fast_return(compiler, SLJIT_MEM1(SLJIT_SP), LOCALS0); + +/* Allocation failed. */ +JUMPHERE(jump); +/* We break the return address cache here, but this is a really rare case. */ +OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_JIT_STACKLIMIT); +JUMPTO(SLJIT_JUMP, common->quit_label); + +/* Call limit reached. */ +set_jumps(common->calllimit, LABEL()); +OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE_ERROR_MATCHLIMIT); +JUMPTO(SLJIT_JUMP, common->quit_label); + +if (common->revertframes != NULL) + { + set_jumps(common->revertframes, LABEL()); + do_revertframes(common); + } +if (common->wordboundary != NULL) + { + set_jumps(common->wordboundary, LABEL()); + check_wordboundary(common); + } +if (common->anynewline != NULL) + { + set_jumps(common->anynewline, LABEL()); + check_anynewline(common); + } +if (common->hspace != NULL) + { + set_jumps(common->hspace, LABEL()); + check_hspace(common); + } +if (common->vspace != NULL) + { + set_jumps(common->vspace, LABEL()); + check_vspace(common); + } +if (common->casefulcmp != NULL) + { + set_jumps(common->casefulcmp, LABEL()); + do_casefulcmp(common); + } +if (common->caselesscmp != NULL) + { + set_jumps(common->caselesscmp, LABEL()); + do_caselesscmp(common); + } +if (common->reset_match != NULL) + { + set_jumps(common->reset_match, LABEL()); + do_reset_match(common, (re->top_bracket + 1) * 2); + CMPTO(SLJIT_GREATER, STR_PTR, 0, TMP1, 0, continue_match_label); + OP1(SLJIT_MOV, STR_PTR, 0, TMP1, 0); + JUMPTO(SLJIT_JUMP, reset_match_label); + } +#ifdef SUPPORT_UTF +#ifdef COMPILE_PCRE8 +if (common->utfreadchar != NULL) + { + set_jumps(common->utfreadchar, LABEL()); + do_utfreadchar(common); + } +if (common->utfreadchar16 != NULL) + { + set_jumps(common->utfreadchar16, LABEL()); + do_utfreadchar16(common); + } +if (common->utfreadtype8 != NULL) + { + set_jumps(common->utfreadtype8, LABEL()); + do_utfreadtype8(common); + } +#endif /* COMPILE_PCRE8 */ +#endif /* SUPPORT_UTF */ +#ifdef SUPPORT_UCP +if (common->getucd != NULL) + { + set_jumps(common->getucd, LABEL()); + do_getucd(common); + } +#endif + +SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data); +SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data); + +executable_func = sljit_generate_code(compiler); +executable_size = sljit_get_generated_code_size(compiler); +label_addr = common->label_addrs; +while (label_addr != NULL) + { + *label_addr->update_addr = sljit_get_label_addr(label_addr->label); + label_addr = label_addr->next; + } +sljit_free_compiler(compiler); +if (executable_func == NULL) + { + free_read_only_data(common->read_only_data_head, compiler->allocator_data); + return; + } + +/* Reuse the function descriptor if possible. */ +if ((extra->flags & PCRE_EXTRA_EXECUTABLE_JIT) != 0 && extra->executable_jit != NULL) + functions = (executable_functions *)extra->executable_jit; +else + { + /* Note: If your memory-checker has flagged the allocation below as a + * memory leak, it is probably because you either forgot to call + * pcre_free_study() (or pcre16_free_study()) on the pcre_extra (or + * pcre16_extra) object, or you called said function after having + * cleared the PCRE_EXTRA_EXECUTABLE_JIT bit from the "flags" field + * of the object. (The function will only free the JIT data if the + * bit remains set, as the bit indicates that the pointer to the data + * is valid.) + */ + functions = SLJIT_MALLOC(sizeof(executable_functions), compiler->allocator_data); + if (functions == NULL) + { + /* This case is highly unlikely since we just recently + freed a lot of memory. Not impossible though. */ + sljit_free_code(executable_func); + free_read_only_data(common->read_only_data_head, compiler->allocator_data); + return; + } + memset(functions, 0, sizeof(executable_functions)); + functions->top_bracket = (re->top_bracket + 1) * 2; + functions->limit_match = (re->flags & PCRE_MLSET) != 0 ? re->limit_match : 0; + extra->executable_jit = functions; + extra->flags |= PCRE_EXTRA_EXECUTABLE_JIT; + } + +functions->executable_funcs[mode] = executable_func; +functions->read_only_data_heads[mode] = common->read_only_data_head; +functions->executable_sizes[mode] = executable_size; +} + +static SLJIT_NOINLINE int jit_machine_stack_exec(jit_arguments *arguments, void *executable_func) +{ +union { + void *executable_func; + jit_function call_executable_func; +} convert_executable_func; +sljit_u8 local_space[MACHINE_STACK_SIZE]; +struct sljit_stack local_stack; + +local_stack.max_limit = local_space; +local_stack.limit = local_space; +local_stack.base = local_space + MACHINE_STACK_SIZE; +local_stack.top = local_space + MACHINE_STACK_SIZE; +arguments->stack = &local_stack; +convert_executable_func.executable_func = executable_func; +return convert_executable_func.call_executable_func(arguments); +} + +int +PRIV(jit_exec)(const PUBL(extra) *extra_data, const pcre_uchar *subject, + int length, int start_offset, int options, int *offsets, int offset_count) +{ +executable_functions *functions = (executable_functions *)extra_data->executable_jit; +union { + void *executable_func; + jit_function call_executable_func; +} convert_executable_func; +jit_arguments arguments; +int max_offset_count; +int retval; +int mode = JIT_COMPILE; + +if ((options & PCRE_PARTIAL_HARD) != 0) + mode = JIT_PARTIAL_HARD_COMPILE; +else if ((options & PCRE_PARTIAL_SOFT) != 0) + mode = JIT_PARTIAL_SOFT_COMPILE; + +if (functions->executable_funcs[mode] == NULL) + return PCRE_ERROR_JIT_BADOPTION; + +/* Sanity checks should be handled by pcre_exec. */ +arguments.str = subject + start_offset; +arguments.begin = subject; +arguments.end = subject + length; +arguments.mark_ptr = NULL; +/* JIT decreases this value less frequently than the interpreter. */ +arguments.limit_match = ((extra_data->flags & PCRE_EXTRA_MATCH_LIMIT) == 0) ? MATCH_LIMIT : (sljit_u32)(extra_data->match_limit); +if (functions->limit_match != 0 && functions->limit_match < arguments.limit_match) + arguments.limit_match = functions->limit_match; +arguments.notbol = (options & PCRE_NOTBOL) != 0; +arguments.noteol = (options & PCRE_NOTEOL) != 0; +arguments.notempty = (options & PCRE_NOTEMPTY) != 0; +arguments.notempty_atstart = (options & PCRE_NOTEMPTY_ATSTART) != 0; +arguments.offsets = offsets; +arguments.callout_data = (extra_data->flags & PCRE_EXTRA_CALLOUT_DATA) != 0 ? extra_data->callout_data : NULL; +arguments.real_offset_count = offset_count; + +/* pcre_exec() rounds offset_count to a multiple of 3, and then uses only 2/3 of +the output vector for storing captured strings, with the remainder used as +workspace. We don't need the workspace here. For compatibility, we limit the +number of captured strings in the same way as pcre_exec(), so that the user +gets the same result with and without JIT. */ + +if (offset_count != 2) + offset_count = ((offset_count - (offset_count % 3)) * 2) / 3; +max_offset_count = functions->top_bracket; +if (offset_count > max_offset_count) + offset_count = max_offset_count; +arguments.offset_count = offset_count; + +if (functions->callback) + arguments.stack = (struct sljit_stack *)functions->callback(functions->userdata); +else + arguments.stack = (struct sljit_stack *)functions->userdata; + +if (arguments.stack == NULL) + retval = jit_machine_stack_exec(&arguments, functions->executable_funcs[mode]); +else + { + convert_executable_func.executable_func = functions->executable_funcs[mode]; + retval = convert_executable_func.call_executable_func(&arguments); + } + +if (retval * 2 > offset_count) + retval = 0; +if ((extra_data->flags & PCRE_EXTRA_MARK) != 0) + *(extra_data->mark) = arguments.mark_ptr; + +return retval; +} + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre_jit_exec(const pcre *argument_re, const pcre_extra *extra_data, + PCRE_SPTR subject, int length, int start_offset, int options, + int *offsets, int offset_count, pcre_jit_stack *stack) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre16_jit_exec(const pcre16 *argument_re, const pcre16_extra *extra_data, + PCRE_SPTR16 subject, int length, int start_offset, int options, + int *offsets, int offset_count, pcre16_jit_stack *stack) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre32_jit_exec(const pcre32 *argument_re, const pcre32_extra *extra_data, + PCRE_SPTR32 subject, int length, int start_offset, int options, + int *offsets, int offset_count, pcre32_jit_stack *stack) +#endif +{ +pcre_uchar *subject_ptr = (pcre_uchar *)subject; +executable_functions *functions = (executable_functions *)extra_data->executable_jit; +union { + void *executable_func; + jit_function call_executable_func; +} convert_executable_func; +jit_arguments arguments; +int max_offset_count; +int retval; +int mode = JIT_COMPILE; + +SLJIT_UNUSED_ARG(argument_re); + +/* Plausibility checks */ +if ((options & ~PUBLIC_JIT_EXEC_OPTIONS) != 0) return PCRE_ERROR_JIT_BADOPTION; + +if ((options & PCRE_PARTIAL_HARD) != 0) + mode = JIT_PARTIAL_HARD_COMPILE; +else if ((options & PCRE_PARTIAL_SOFT) != 0) + mode = JIT_PARTIAL_SOFT_COMPILE; + +if (functions->executable_funcs[mode] == NULL) + return PCRE_ERROR_JIT_BADOPTION; + +/* Sanity checks should be handled by pcre_exec. */ +arguments.stack = (struct sljit_stack *)stack; +arguments.str = subject_ptr + start_offset; +arguments.begin = subject_ptr; +arguments.end = subject_ptr + length; +arguments.mark_ptr = NULL; +/* JIT decreases this value less frequently than the interpreter. */ +arguments.limit_match = ((extra_data->flags & PCRE_EXTRA_MATCH_LIMIT) == 0) ? MATCH_LIMIT : (sljit_u32)(extra_data->match_limit); +if (functions->limit_match != 0 && functions->limit_match < arguments.limit_match) + arguments.limit_match = functions->limit_match; +arguments.notbol = (options & PCRE_NOTBOL) != 0; +arguments.noteol = (options & PCRE_NOTEOL) != 0; +arguments.notempty = (options & PCRE_NOTEMPTY) != 0; +arguments.notempty_atstart = (options & PCRE_NOTEMPTY_ATSTART) != 0; +arguments.offsets = offsets; +arguments.callout_data = (extra_data->flags & PCRE_EXTRA_CALLOUT_DATA) != 0 ? extra_data->callout_data : NULL; +arguments.real_offset_count = offset_count; + +/* pcre_exec() rounds offset_count to a multiple of 3, and then uses only 2/3 of +the output vector for storing captured strings, with the remainder used as +workspace. We don't need the workspace here. For compatibility, we limit the +number of captured strings in the same way as pcre_exec(), so that the user +gets the same result with and without JIT. */ + +if (offset_count != 2) + offset_count = ((offset_count - (offset_count % 3)) * 2) / 3; +max_offset_count = functions->top_bracket; +if (offset_count > max_offset_count) + offset_count = max_offset_count; +arguments.offset_count = offset_count; + +convert_executable_func.executable_func = functions->executable_funcs[mode]; +retval = convert_executable_func.call_executable_func(&arguments); + +if (retval * 2 > offset_count) + retval = 0; +if ((extra_data->flags & PCRE_EXTRA_MARK) != 0) + *(extra_data->mark) = arguments.mark_ptr; + +return retval; +} + +void +PRIV(jit_free)(void *executable_funcs) +{ +int i; +executable_functions *functions = (executable_functions *)executable_funcs; +for (i = 0; i < JIT_NUMBER_OF_COMPILE_MODES; i++) + { + if (functions->executable_funcs[i] != NULL) + sljit_free_code(functions->executable_funcs[i]); + free_read_only_data(functions->read_only_data_heads[i], NULL); + } +SLJIT_FREE(functions, compiler->allocator_data); +} + +int +PRIV(jit_get_size)(void *executable_funcs) +{ +int i; +sljit_uw size = 0; +sljit_uw *executable_sizes = ((executable_functions *)executable_funcs)->executable_sizes; +for (i = 0; i < JIT_NUMBER_OF_COMPILE_MODES; i++) + size += executable_sizes[i]; +return (int)size; +} + +const char* +PRIV(jit_get_target)(void) +{ +return sljit_get_platform_name(); +} + +#if defined COMPILE_PCRE8 +PCRE_EXP_DECL pcre_jit_stack * +pcre_jit_stack_alloc(int startsize, int maxsize) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DECL pcre16_jit_stack * +pcre16_jit_stack_alloc(int startsize, int maxsize) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DECL pcre32_jit_stack * +pcre32_jit_stack_alloc(int startsize, int maxsize) +#endif +{ +if (startsize < 1 || maxsize < 1) + return NULL; +if (startsize > maxsize) + startsize = maxsize; +startsize = (startsize + STACK_GROWTH_RATE - 1) & ~(STACK_GROWTH_RATE - 1); +maxsize = (maxsize + STACK_GROWTH_RATE - 1) & ~(STACK_GROWTH_RATE - 1); +return (PUBL(jit_stack)*)sljit_allocate_stack(startsize, maxsize, NULL); +} + +#if defined COMPILE_PCRE8 +PCRE_EXP_DECL void +pcre_jit_stack_free(pcre_jit_stack *stack) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DECL void +pcre16_jit_stack_free(pcre16_jit_stack *stack) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DECL void +pcre32_jit_stack_free(pcre32_jit_stack *stack) +#endif +{ +sljit_free_stack((struct sljit_stack *)stack, NULL); +} + +#if defined COMPILE_PCRE8 +PCRE_EXP_DECL void +pcre_assign_jit_stack(pcre_extra *extra, pcre_jit_callback callback, void *userdata) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DECL void +pcre16_assign_jit_stack(pcre16_extra *extra, pcre16_jit_callback callback, void *userdata) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DECL void +pcre32_assign_jit_stack(pcre32_extra *extra, pcre32_jit_callback callback, void *userdata) +#endif +{ +executable_functions *functions; +if (extra != NULL && + (extra->flags & PCRE_EXTRA_EXECUTABLE_JIT) != 0 && + extra->executable_jit != NULL) + { + functions = (executable_functions *)extra->executable_jit; + functions->callback = callback; + functions->userdata = userdata; + } +} + +#if defined COMPILE_PCRE8 +PCRE_EXP_DECL void +pcre_jit_free_unused_memory(void) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DECL void +pcre16_jit_free_unused_memory(void) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DECL void +pcre32_jit_free_unused_memory(void) +#endif +{ +sljit_free_unused_memory_exec(); +} + +#else /* SUPPORT_JIT */ + +/* These are dummy functions to avoid linking errors when JIT support is not +being compiled. */ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DECL pcre_jit_stack * +pcre_jit_stack_alloc(int startsize, int maxsize) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DECL pcre16_jit_stack * +pcre16_jit_stack_alloc(int startsize, int maxsize) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DECL pcre32_jit_stack * +pcre32_jit_stack_alloc(int startsize, int maxsize) +#endif +{ +(void)startsize; +(void)maxsize; +return NULL; +} + +#if defined COMPILE_PCRE8 +PCRE_EXP_DECL void +pcre_jit_stack_free(pcre_jit_stack *stack) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DECL void +pcre16_jit_stack_free(pcre16_jit_stack *stack) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DECL void +pcre32_jit_stack_free(pcre32_jit_stack *stack) +#endif +{ +(void)stack; +} + +#if defined COMPILE_PCRE8 +PCRE_EXP_DECL void +pcre_assign_jit_stack(pcre_extra *extra, pcre_jit_callback callback, void *userdata) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DECL void +pcre16_assign_jit_stack(pcre16_extra *extra, pcre16_jit_callback callback, void *userdata) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DECL void +pcre32_assign_jit_stack(pcre32_extra *extra, pcre32_jit_callback callback, void *userdata) +#endif +{ +(void)extra; +(void)callback; +(void)userdata; +} + +#if defined COMPILE_PCRE8 +PCRE_EXP_DECL void +pcre_jit_free_unused_memory(void) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DECL void +pcre16_jit_free_unused_memory(void) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DECL void +pcre32_jit_free_unused_memory(void) +#endif +{ +} + +#endif + +/* End of pcre_jit_compile.c */ diff --git a/src/third_party/pcre-8.41/pcre_jit_test.c b/src/third_party/pcre-8.41/pcre_jit_test.c new file mode 100644 index 00000000000..034cb52697f --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_jit_test.c @@ -0,0 +1,1765 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Main Library written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + + This JIT compiler regression test program was written by Zoltan Herczeg + Copyright (c) 2010-2012 + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include "pcre.h" + + +#include "pcre_internal.h" + +/* + Letter characters: + \xe6\x92\xad = 0x64ad = 25773 (kanji) + Non-letter characters: + \xc2\xa1 = 0xa1 = (Inverted Exclamation Mark) + \xf3\xa9\xb7\x80 = 0xe9dc0 = 957888 + \xed\xa0\x80 = 55296 = 0xd800 (Invalid UTF character) + \xed\xb0\x80 = 56320 = 0xdc00 (Invalid UTF character) + Newlines: + \xc2\x85 = 0x85 = 133 (NExt Line = NEL) + \xe2\x80\xa8 = 0x2028 = 8232 (Line Separator) + Othercase pairs: + \xc3\xa9 = 0xe9 = 233 (e') + \xc3\x89 = 0xc9 = 201 (E') + \xc3\xa1 = 0xe1 = 225 (a') + \xc3\x81 = 0xc1 = 193 (A') + \x53 = 0x53 = S + \x73 = 0x73 = s + \xc5\xbf = 0x17f = 383 (long S) + \xc8\xba = 0x23a = 570 + \xe2\xb1\xa5 = 0x2c65 = 11365 + \xe1\xbd\xb8 = 0x1f78 = 8056 + \xe1\xbf\xb8 = 0x1ff8 = 8184 + \xf0\x90\x90\x80 = 0x10400 = 66560 + \xf0\x90\x90\xa8 = 0x10428 = 66600 + \xc7\x84 = 0x1c4 = 452 + \xc7\x85 = 0x1c5 = 453 + \xc7\x86 = 0x1c6 = 454 + Caseless sets: + ucp_Armenian - \x{531}-\x{556} -> \x{561}-\x{586} + ucp_Coptic - \x{2c80}-\x{2ce3} -> caseless: XOR 0x1 + ucp_Latin - \x{ff21}-\x{ff3a} -> \x{ff41]-\x{ff5a} + + Mark property: + \xcc\x8d = 0x30d = 781 + Special: + \xc2\x80 = 0x80 = 128 (lowest 2 byte character) + \xdf\xbf = 0x7ff = 2047 (highest 2 byte character) + \xe0\xa0\x80 = 0x800 = 2048 (lowest 2 byte character) + \xef\xbf\xbf = 0xffff = 65535 (highest 3 byte character) + \xf0\x90\x80\x80 = 0x10000 = 65536 (lowest 4 byte character) + \xf4\x8f\xbf\xbf = 0x10ffff = 1114111 (highest allowed utf character) +*/ + +static int regression_tests(void); + +int main(void) +{ + int jit = 0; +#if defined SUPPORT_PCRE8 + pcre_config(PCRE_CONFIG_JIT, &jit); +#elif defined SUPPORT_PCRE16 + pcre16_config(PCRE_CONFIG_JIT, &jit); +#elif defined SUPPORT_PCRE32 + pcre32_config(PCRE_CONFIG_JIT, &jit); +#endif + if (!jit) { + printf("JIT must be enabled to run pcre_jit_test\n"); + return 1; + } + return regression_tests(); +} + +/* --------------------------------------------------------------------------------------- */ + +#if !(defined SUPPORT_PCRE8) && !(defined SUPPORT_PCRE16) && !(defined SUPPORT_PCRE32) +#error SUPPORT_PCRE8 or SUPPORT_PCRE16 or SUPPORT_PCRE32 must be defined +#endif + +#define MUA (PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANYCRLF) +#define MUAP (PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANYCRLF | PCRE_UCP) +#define CMUA (PCRE_CASELESS | PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANYCRLF) +#define CMUAP (PCRE_CASELESS | PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANYCRLF | PCRE_UCP) +#define MA (PCRE_MULTILINE | PCRE_NEWLINE_ANYCRLF) +#define MAP (PCRE_MULTILINE | PCRE_NEWLINE_ANYCRLF | PCRE_UCP) +#define CMA (PCRE_CASELESS | PCRE_MULTILINE | PCRE_NEWLINE_ANYCRLF) + +#define OFFSET_MASK 0x00ffff +#define F_NO8 0x010000 +#define F_NO16 0x020000 +#define F_NO32 0x020000 +#define F_NOMATCH 0x040000 +#define F_DIFF 0x080000 +#define F_FORCECONV 0x100000 +#define F_PROPERTY 0x200000 +#define F_STUDY 0x400000 + +struct regression_test_case { + int flags; + int start_offset; + const char *pattern; + const char *input; +}; + +static struct regression_test_case regression_test_cases[] = { + /* Constant strings. */ + { MUA, 0, "AbC", "AbAbC" }, + { MUA, 0, "ACCEPT", "AACACCACCEACCEPACCEPTACCEPTT" }, + { CMUA, 0, "aA#\xc3\xa9\xc3\x81", "aA#Aa#\xc3\x89\xc3\xa1" }, + { MA, 0, "[^a]", "aAbB" }, + { CMA, 0, "[^m]", "mMnN" }, + { MA, 0, "a[^b][^#]", "abacd" }, + { CMA, 0, "A[^B][^E]", "abacd" }, + { CMUA, 0, "[^x][^#]", "XxBll" }, + { MUA, 0, "[^a]", "aaa\xc3\xa1#Ab" }, + { CMUA, 0, "[^A]", "aA\xe6\x92\xad" }, + { MUA, 0, "\\W(\\W)?\\w", "\r\n+bc" }, + { MUA, 0, "\\W(\\W)?\\w", "\n\r+bc" }, + { MUA, 0, "\\W(\\W)?\\w", "\r\r+bc" }, + { MUA, 0, "\\W(\\W)?\\w", "\n\n+bc" }, + { MUA, 0, "[axd]", "sAXd" }, + { CMUA, 0, "[axd]", "sAXd" }, + { CMUA, 0 | F_NOMATCH, "[^axd]", "DxA" }, + { MUA, 0, "[a-dA-C]", "\xe6\x92\xad\xc3\xa9.B" }, + { MUA, 0, "[^a-dA-C]", "\xe6\x92\xad\xc3\xa9" }, + { CMUA, 0, "[^\xc3\xa9]", "\xc3\xa9\xc3\x89." }, + { MUA, 0, "[^\xc3\xa9]", "\xc3\xa9\xc3\x89." }, + { MUA, 0, "[^a]", "\xc2\x80[]" }, + { CMUA, 0, "\xf0\x90\x90\xa7", "\xf0\x90\x91\x8f" }, + { CMA, 0, "1a2b3c4", "1a2B3c51A2B3C4" }, + { PCRE_CASELESS, 0, "\xff#a", "\xff#\xff\xfe##\xff#A" }, + { PCRE_CASELESS, 0, "\xfe", "\xff\xfc#\xfe\xfe" }, + { PCRE_CASELESS, 0, "a1", "Aa1" }, + { MA, 0, "\\Ca", "cda" }, + { CMA, 0, "\\Ca", "CDA" }, + { MA, 0 | F_NOMATCH, "\\Cx", "cda" }, + { CMA, 0 | F_NOMATCH, "\\Cx", "CDA" }, + { CMUAP, 0, "\xf0\x90\x90\x80\xf0\x90\x90\xa8", "\xf0\x90\x90\xa8\xf0\x90\x90\x80" }, + { CMUAP, 0, "\xf0\x90\x90\x80{2}", "\xf0\x90\x90\x80#\xf0\x90\x90\xa8\xf0\x90\x90\x80" }, + { CMUAP, 0, "\xf0\x90\x90\xa8{2}", "\xf0\x90\x90\x80#\xf0\x90\x90\xa8\xf0\x90\x90\x80" }, + { CMUAP, 0, "\xe1\xbd\xb8\xe1\xbf\xb8", "\xe1\xbf\xb8\xe1\xbd\xb8" }, + { MA, 0, "[3-57-9]", "5" }, + + /* Assertions. */ + { MUA, 0, "\\b[^A]", "A_B#" }, + { MA, 0 | F_NOMATCH, "\\b\\W", "\n*" }, + { MUA, 0, "\\B[^,]\\b[^s]\\b", "#X" }, + { MAP, 0, "\\B", "_\xa1" }, + { MAP, 0, "\\b_\\b[,A]\\B", "_," }, + { MUAP, 0, "\\b", "\xe6\x92\xad!" }, + { MUAP, 0, "\\B", "_\xc2\xa1\xc3\xa1\xc2\x85" }, + { MUAP, 0, "\\b[^A]\\B[^c]\\b[^_]\\B", "_\xc3\xa1\xe2\x80\xa8" }, + { MUAP, 0, "\\b\\w+\\B", "\xc3\x89\xc2\xa1\xe6\x92\xad\xc3\x81\xc3\xa1" }, + { MUA, 0 | F_NOMATCH, "\\b.", "\xcd\xbe" }, + { CMUAP, 0, "\\By", "\xf0\x90\x90\xa8y" }, + { MA, 0 | F_NOMATCH, "\\R^", "\n" }, + { MA, 1 | F_NOMATCH, "^", "\n" }, + { 0, 0, "^ab", "ab" }, + { 0, 0 | F_NOMATCH, "^ab", "aab" }, + { PCRE_MULTILINE | PCRE_NEWLINE_CRLF, 0, "^a", "\r\raa\n\naa\r\naa" }, + { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANYCRLF, 0, "^-", "\xe2\x80\xa8--\xc2\x85-\r\n-" }, + { PCRE_MULTILINE | PCRE_NEWLINE_ANY, 0, "^-", "a--b--\x85--" }, + { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANY, 0, "^-", "a--\xe2\x80\xa8--" }, + { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANY, 0, "^-", "a--\xc2\x85--" }, + { 0, 0, "ab$", "ab" }, + { 0, 0 | F_NOMATCH, "ab$", "abab\n\n" }, + { PCRE_DOLLAR_ENDONLY, 0 | F_NOMATCH, "ab$", "abab\r\n" }, + { PCRE_MULTILINE | PCRE_NEWLINE_CRLF, 0, "a$", "\r\raa\n\naa\r\naa" }, + { PCRE_MULTILINE | PCRE_NEWLINE_ANY, 0, "a$", "aaa" }, + { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANYCRLF, 0, "#$", "#\xc2\x85###\r#" }, + { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANY, 0, "#$", "#\xe2\x80\xa9" }, + { PCRE_NOTBOL | PCRE_NEWLINE_ANY, 0 | F_NOMATCH, "^a", "aa\naa" }, + { PCRE_NOTBOL | PCRE_MULTILINE | PCRE_NEWLINE_ANY, 0, "^a", "aa\naa" }, + { PCRE_NOTEOL | PCRE_NEWLINE_ANY, 0 | F_NOMATCH, "a$", "aa\naa" }, + { PCRE_NOTEOL | PCRE_NEWLINE_ANY, 0 | F_NOMATCH, "a$", "aa\r\n" }, + { PCRE_UTF8 | PCRE_DOLLAR_ENDONLY | PCRE_NEWLINE_ANY, 0 | F_PROPERTY, "\\p{Any}{2,}$", "aa\r\n" }, + { PCRE_NOTEOL | PCRE_MULTILINE | PCRE_NEWLINE_ANY, 0, "a$", "aa\naa" }, + { PCRE_NEWLINE_CR, 0, ".\\Z", "aaa" }, + { PCRE_NEWLINE_CR | PCRE_UTF8, 0, "a\\Z", "aaa\r" }, + { PCRE_NEWLINE_CR, 0, ".\\Z", "aaa\n" }, + { PCRE_NEWLINE_CRLF, 0, ".\\Z", "aaa\r" }, + { PCRE_NEWLINE_CRLF | PCRE_UTF8, 0, ".\\Z", "aaa\n" }, + { PCRE_NEWLINE_CRLF, 0, ".\\Z", "aaa\r\n" }, + { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".\\Z", "aaa" }, + { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".\\Z", "aaa\r" }, + { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".\\Z", "aaa\n" }, + { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".\\Z", "aaa\r\n" }, + { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".\\Z", "aaa\xe2\x80\xa8" }, + { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".\\Z", "aaa" }, + { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".\\Z", "aaa\r" }, + { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".\\Z", "aaa\n" }, + { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".\\Z", "aaa\r\n" }, + { PCRE_NEWLINE_ANY | PCRE_UTF8, 0, ".\\Z", "aaa\xc2\x85" }, + { PCRE_NEWLINE_ANY | PCRE_UTF8, 0, ".\\Z", "aaa\xe2\x80\xa8" }, + { MA, 0, "\\Aa", "aaa" }, + { MA, 1 | F_NOMATCH, "\\Aa", "aaa" }, + { MA, 1, "\\Ga", "aaa" }, + { MA, 1 | F_NOMATCH, "\\Ga", "aba" }, + { MA, 0, "a\\z", "aaa" }, + { MA, 0 | F_NOMATCH, "a\\z", "aab" }, + + /* Brackets and alternatives. */ + { MUA, 0, "(ab|bb|cd)", "bacde" }, + { MUA, 0, "(?:ab|a)(bc|c)", "ababc" }, + { MUA, 0, "((ab|(cc))|(bb)|(?:cd|efg))", "abac" }, + { CMUA, 0, "((aB|(Cc))|(bB)|(?:cd|EFg))", "AcCe" }, + { MUA, 0, "((ab|(cc))|(bb)|(?:cd|ebg))", "acebebg" }, + { MUA, 0, "(?:(a)|(?:b))(cc|(?:d|e))(a|b)k", "accabdbbccbk" }, + { MUA, 0, "\xc7\x82|\xc6\x82", "\xf1\x83\x82\x82\xc7\x82\xc7\x83" }, + { MUA, 0, "=\xc7\x82|#\xc6\x82", "\xf1\x83\x82\x82=\xc7\x82\xc7\x83" }, + { MUA, 0, "\xc7\x82\xc7\x83|\xc6\x82\xc6\x82", "\xf1\x83\x82\x82\xc7\x82\xc7\x83" }, + { MUA, 0, "\xc6\x82\xc6\x82|\xc7\x83\xc7\x83|\xc8\x84\xc8\x84", "\xf1\x83\x82\x82\xc8\x84\xc8\x84" }, + + /* Greedy and non-greedy ? operators. */ + { MUA, 0, "(?:a)?a", "laab" }, + { CMUA, 0, "(A)?A", "llaab" }, + { MUA, 0, "(a)?\?a", "aab" }, /* ?? is the prefix of trygraphs in GCC. */ + { MUA, 0, "(a)?a", "manm" }, + { CMUA, 0, "(a|b)?\?d((?:e)?)", "ABABdx" }, + { MUA, 0, "(a|b)?\?d((?:e)?)", "abcde" }, + { MUA, 0, "((?:ab)?\?g|b(?:g(nn|d)?\?)?)?\?(?:n)?m", "abgnbgnnbgdnmm" }, + + /* Greedy and non-greedy + operators */ + { MUA, 0, "(aa)+aa", "aaaaaaa" }, + { MUA, 0, "(aa)+?aa", "aaaaaaa" }, + { MUA, 0, "(?:aba|ab|a)+l", "ababamababal" }, + { MUA, 0, "(?:aba|ab|a)+?l", "ababamababal" }, + { MUA, 0, "(a(?:bc|cb|b|c)+?|ss)+e", "accssabccbcacbccbbXaccssabccbcacbccbbe" }, + { MUA, 0, "(a(?:bc|cb|b|c)+|ss)+?e", "accssabccbcacbccbbXaccssabccbcacbccbbe" }, + { MUA, 0, "(?:(b(c)+?)+)?\?(?:(bc)+|(cb)+)+(?:m)+", "bccbcccbcbccbcbPbccbcccbcbccbcbmmn" }, + + /* Greedy and non-greedy * operators */ + { CMUA, 0, "(?:AA)*AB", "aaaaaaamaaaaaaab" }, + { MUA, 0, "(?:aa)*?ab", "aaaaaaamaaaaaaab" }, + { MUA, 0, "(aa|ab)*ab", "aaabaaab" }, + { CMUA, 0, "(aa|Ab)*?aB", "aaabaaab" }, + { MUA, 0, "(a|b)*(?:a)*(?:b)*m", "abbbaaababanabbbaaababamm" }, + { MUA, 0, "(a|b)*?(?:a)*?(?:b)*?m", "abbbaaababanabbbaaababamm" }, + { MA, 0, "a(a(\\1*)a|(b)b+){0}a", "aa" }, + { MA, 0, "((?:a|)*){0}a", "a" }, + + /* Combining ? + * operators */ + { MUA, 0, "((bm)+)?\?(?:a)*(bm)+n|((am)+?)?(?:a)+(am)*n", "bmbmabmamaaamambmaman" }, + { MUA, 0, "(((ab)?cd)*ef)+g", "abcdcdefcdefefmabcdcdefcdefefgg" }, + { MUA, 0, "(((ab)?\?cd)*?ef)+?g", "abcdcdefcdefefmabcdcdefcdefefgg" }, + { MUA, 0, "(?:(ab)?c|(?:ab)+?d)*g", "ababcdccababddg" }, + { MUA, 0, "(?:(?:ab)?\?c|(ab)+d)*?g", "ababcdccababddg" }, + + /* Single character iterators. */ + { MUA, 0, "(a+aab)+aaaab", "aaaabcaaaabaabcaabcaaabaaaab" }, + { MUA, 0, "(a*a*aab)+x", "aaaaabaabaaabmaabx" }, + { MUA, 0, "(a*?(b|ab)a*?)+x", "aaaabcxbbaabaacbaaabaabax" }, + { MUA, 0, "(a+(ab|ad)a+)+x", "aaabaaaadaabaaabaaaadaaax" }, + { MUA, 0, "(a?(a)a?)+(aaa)", "abaaabaaaaaaaa" }, + { MUA, 0, "(a?\?(a)a?\?)+(b)", "aaaacaaacaacacbaaab" }, + { MUA, 0, "(a{0,4}(b))+d", "aaaaaabaabcaaaaabaaaaabd" }, + { MUA, 0, "(a{0,4}?[^b])+d+(a{0,4}[^b])d+", "aaaaadaaaacaadddaaddd" }, + { MUA, 0, "(ba{2})+c", "baabaaabacbaabaac" }, + { MUA, 0, "(a*+bc++)+", "aaabbcaaabcccab" }, + { MUA, 0, "(a?+[^b])+", "babaacacb" }, + { MUA, 0, "(a{0,3}+b)(a{0,3}+b)(a{0,3}+)[^c]", "abaabaaacbaabaaaac" }, + { CMUA, 0, "([a-c]+[d-f]+?)+?g", "aBdacdehAbDaFgA" }, + { CMUA, 0, "[c-f]+k", "DemmFke" }, + { MUA, 0, "([DGH]{0,4}M)+", "GGDGHDGMMHMDHHGHM" }, + { MUA, 0, "([a-c]{4,}s)+", "abasabbasbbaabsbba" }, + { CMUA, 0, "[ace]{3,7}", "AcbDAcEEcEd" }, + { CMUA, 0, "[ace]{3,7}?", "AcbDAcEEcEd" }, + { CMUA, 0, "[ace]{3,}", "AcbDAcEEcEd" }, + { CMUA, 0, "[ace]{3,}?", "AcbDAcEEcEd" }, + { MUA, 0, "[ckl]{2,}?g", "cdkkmlglglkcg" }, + { CMUA, 0, "[ace]{5}?", "AcCebDAcEEcEd" }, + { MUA, 0, "([AbC]{3,5}?d)+", "BACaAbbAEAACCbdCCbdCCAAbb" }, + { MUA, 0, "([^ab]{0,}s){2}", "abaabcdsABamsDDs" }, + { MUA, 0, "\\b\\w+\\B", "x,a_cd" }, + { MUAP, 0, "\\b[^\xc2\xa1]+\\B", "\xc3\x89\xc2\xa1\xe6\x92\xad\xc3\x81\xc3\xa1" }, + { CMUA, 0, "[^b]+(a*)([^c]?d{3})", "aaaaddd" }, + { CMUAP, 0, "\xe1\xbd\xb8{2}", "\xe1\xbf\xb8#\xe1\xbf\xb8\xe1\xbd\xb8" }, + { CMUA, 0, "[^\xf0\x90\x90\x80]{2,4}@", "\xf0\x90\x90\xa8\xf0\x90\x90\x80###\xf0\x90\x90\x80@@@" }, + { CMUA, 0, "[^\xe1\xbd\xb8][^\xc3\xa9]", "\xe1\xbd\xb8\xe1\xbf\xb8\xc3\xa9\xc3\x89#" }, + { MUA, 0, "[^\xe1\xbd\xb8][^\xc3\xa9]", "\xe1\xbd\xb8\xe1\xbf\xb8\xc3\xa9\xc3\x89#" }, + { MUA, 0, "[^\xe1\xbd\xb8]{3,}?", "##\xe1\xbd\xb8#\xe1\xbd\xb8#\xc3\x89#\xe1\xbd\xb8" }, + { MUA, 0, "\\d+123", "987654321,01234" }, + { MUA, 0, "abcd*|\\w+xy", "aaaaa,abxyz" }, + { MUA, 0, "(?:abc|((?:amc|\\b\\w*xy)))", "aaaaa,abxyz" }, + { MUA, 0, "a(?R)|([a-z]++)#", ".abcd.abcd#."}, + { MUA, 0, "a(?R)|([a-z]++)#", ".abcd.mbcd#."}, + { MUA, 0, ".[ab]*.", "xx" }, + { MUA, 0, ".[ab]*a", "xxa" }, + { MUA, 0, ".[ab]?.", "xx" }, + + /* Bracket repeats with limit. */ + { MUA, 0, "(?:(ab){2}){5}M", "abababababababababababM" }, + { MUA, 0, "(?:ab|abab){1,5}M", "abababababababababababM" }, + { MUA, 0, "(?>ab|abab){1,5}M", "abababababababababababM" }, + { MUA, 0, "(?:ab|abab){1,5}?M", "abababababababababababM" }, + { MUA, 0, "(?>ab|abab){1,5}?M", "abababababababababababM" }, + { MUA, 0, "(?:(ab){1,4}?){1,3}?M", "abababababababababababababM" }, + { MUA, 0, "(?:(ab){1,4}){1,3}abababababababababababM", "ababababababababababababM" }, + { MUA, 0 | F_NOMATCH, "(?:(ab){1,4}){1,3}abababababababababababM", "abababababababababababM" }, + { MUA, 0, "(ab){4,6}?M", "abababababababM" }, + + /* Basic character sets. */ + { MUA, 0, "(?:\\s)+(?:\\S)+", "ab \t\xc3\xa9\xe6\x92\xad " }, + { MUA, 0, "(\\w)*(k)(\\W)?\?", "abcdef abck11" }, + { MUA, 0, "\\((\\d)+\\)\\D", "a() (83 (8)2 (9)ab" }, + { MUA, 0, "\\w(\\s|(?:\\d)*,)+\\w\\wb", "a 5, 4,, bb 5, 4,, aab" }, + { MUA, 0, "(\\v+)(\\V+)", "\x0e\xc2\x85\xe2\x80\xa8\x0b\x09\xe2\x80\xa9" }, + { MUA, 0, "(\\h+)(\\H+)", "\xe2\x80\xa8\xe2\x80\x80\x20\xe2\x80\x8a\xe2\x81\x9f\xe3\x80\x80\x09\x20\xc2\xa0\x0a" }, + { MUA, 0, "x[bcef]+", "xaxdxecbfg" }, + { MUA, 0, "x[bcdghij]+", "xaxexfxdgbjk" }, + { MUA, 0, "x[^befg]+", "xbxexacdhg" }, + { MUA, 0, "x[^bcdl]+", "xlxbxaekmd" }, + { MUA, 0, "x[^bcdghi]+", "xbxdxgxaefji" }, + { MUA, 0, "x[B-Fb-f]+", "xaxAxgxbfBFG" }, + { CMUA, 0, "\\x{e9}+", "#\xf0\x90\x90\xa8\xc3\xa8\xc3\xa9\xc3\x89\xc3\x88" }, + { CMUA, 0, "[^\\x{e9}]+", "\xc3\xa9#\xf0\x90\x90\xa8\xc3\xa8\xc3\x88\xc3\x89" }, + { MUA, 0, "[\\x02\\x7e]+", "\xc3\x81\xe1\xbf\xb8\xf0\x90\x90\xa8\x01\x02\x7e\x7f" }, + { MUA, 0, "[^\\x02\\x7e]+", "\x02\xc3\x81\xe1\xbf\xb8\xf0\x90\x90\xa8\x01\x7f\x7e" }, + { MUA, 0, "[\\x{81}-\\x{7fe}]+", "#\xe1\xbf\xb8\xf0\x90\x90\xa8\xc2\x80\xc2\x81\xdf\xbe\xdf\xbf" }, + { MUA, 0, "[^\\x{81}-\\x{7fe}]+", "\xc2\x81#\xe1\xbf\xb8\xf0\x90\x90\xa8\xc2\x80\xdf\xbf\xdf\xbe" }, + { MUA, 0, "[\\x{801}-\\x{fffe}]+", "#\xc3\xa9\xf0\x90\x90\x80\xe0\xa0\x80\xe0\xa0\x81\xef\xbf\xbe\xef\xbf\xbf" }, + { MUA, 0, "[^\\x{801}-\\x{fffe}]+", "\xe0\xa0\x81#\xc3\xa9\xf0\x90\x90\x80\xe0\xa0\x80\xef\xbf\xbf\xef\xbf\xbe" }, + { MUA, 0, "[\\x{10001}-\\x{10fffe}]+", "#\xc3\xa9\xe2\xb1\xa5\xf0\x90\x80\x80\xf0\x90\x80\x81\xf4\x8f\xbf\xbe\xf4\x8f\xbf\xbf" }, + { MUA, 0, "[^\\x{10001}-\\x{10fffe}]+", "\xf0\x90\x80\x81#\xc3\xa9\xe2\xb1\xa5\xf0\x90\x80\x80\xf4\x8f\xbf\xbf\xf4\x8f\xbf\xbe" }, + + /* Unicode properties. */ + { MUAP, 0, "[1-5\xc3\xa9\\w]", "\xc3\xa1_" }, + { MUAP, 0 | F_PROPERTY, "[\xc3\x81\\p{Ll}]", "A_\xc3\x89\xc3\xa1" }, + { MUAP, 0, "[\\Wd-h_x-z]+", "a\xc2\xa1#_yhzdxi" }, + { MUAP, 0 | F_NOMATCH | F_PROPERTY, "[\\P{Any}]", "abc" }, + { MUAP, 0 | F_NOMATCH | F_PROPERTY, "[^\\p{Any}]", "abc" }, + { MUAP, 0 | F_NOMATCH | F_PROPERTY, "[\\P{Any}\xc3\xa1-\xc3\xa8]", "abc" }, + { MUAP, 0 | F_NOMATCH | F_PROPERTY, "[^\\p{Any}\xc3\xa1-\xc3\xa8]", "abc" }, + { MUAP, 0 | F_NOMATCH | F_PROPERTY, "[\xc3\xa1-\xc3\xa8\\P{Any}]", "abc" }, + { MUAP, 0 | F_NOMATCH | F_PROPERTY, "[^\xc3\xa1-\xc3\xa8\\p{Any}]", "abc" }, + { MUAP, 0 | F_PROPERTY, "[\xc3\xa1-\xc3\xa8\\p{Any}]", "abc" }, + { MUAP, 0 | F_PROPERTY, "[^\xc3\xa1-\xc3\xa8\\P{Any}]", "abc" }, + { MUAP, 0, "[b-\xc3\xa9\\s]", "a\xc\xe6\x92\xad" }, + { CMUAP, 0, "[\xc2\x85-\xc2\x89\xc3\x89]", "\xc2\x84\xc3\xa9" }, + { MUAP, 0, "[^b-d^&\\s]{3,}", "db^ !a\xe2\x80\xa8_ae" }, + { MUAP, 0 | F_PROPERTY, "[^\\S\\P{Any}][\\sN]{1,3}[\\P{N}]{4}", "\xe2\x80\xaa\xa N\x9\xc3\xa9_0" }, + { MUA, 0 | F_PROPERTY, "[^\\P{L}\x9!D-F\xa]{2,3}", "\x9,.DF\xa.CG\xc3\x81" }, + { CMUAP, 0, "[\xc3\xa1-\xc3\xa9_\xe2\x80\xa0-\xe2\x80\xaf]{1,5}[^\xe2\x80\xa0-\xe2\x80\xaf]", "\xc2\xa1\xc3\x89\xc3\x89\xe2\x80\xaf_\xe2\x80\xa0" }, + { MUAP, 0 | F_PROPERTY, "[\xc3\xa2-\xc3\xa6\xc3\x81-\xc3\x84\xe2\x80\xa8-\xe2\x80\xa9\xe6\x92\xad\\p{Zs}]{2,}", "\xe2\x80\xa7\xe2\x80\xa9\xe6\x92\xad \xe6\x92\xae" }, + { MUAP, 0 | F_PROPERTY, "[\\P{L&}]{2}[^\xc2\x85-\xc2\x89\\p{Ll}\\p{Lu}]{2}", "\xc3\xa9\xe6\x92\xad.a\xe6\x92\xad|\xc2\x8a#" }, + { PCRE_UCP, 0, "[a-b\\s]{2,5}[^a]", "AB baaa" }, + + /* Possible empty brackets. */ + { MUA, 0, "(?:|ab||bc|a)+d", "abcxabcabd" }, + { MUA, 0, "(|ab||bc|a)+d", "abcxabcabd" }, + { MUA, 0, "(?:|ab||bc|a)*d", "abcxabcabd" }, + { MUA, 0, "(|ab||bc|a)*d", "abcxabcabd" }, + { MUA, 0, "(?:|ab||bc|a)+?d", "abcxabcabd" }, + { MUA, 0, "(|ab||bc|a)+?d", "abcxabcabd" }, + { MUA, 0, "(?:|ab||bc|a)*?d", "abcxabcabd" }, + { MUA, 0, "(|ab||bc|a)*?d", "abcxabcabd" }, + { MUA, 0, "(((a)*?|(?:ba)+)+?|(?:|c|ca)*)*m", "abaacaccabacabalabaacaccabacabamm" }, + { MUA, 0, "(?:((?:a)*|(ba)+?)+|(|c|ca)*?)*?m", "abaacaccabacabalabaacaccabacabamm" }, + + /* Start offset. */ + { MUA, 3, "(\\d|(?:\\w)*\\w)+", "0ac01Hb" }, + { MUA, 4 | F_NOMATCH, "(\\w\\W\\w)+", "ab#d" }, + { MUA, 2 | F_NOMATCH, "(\\w\\W\\w)+", "ab#d" }, + { MUA, 1, "(\\w\\W\\w)+", "ab#d" }, + + /* Newline. */ + { PCRE_MULTILINE | PCRE_NEWLINE_CRLF, 0, "\\W{0,2}[^#]{3}", "\r\n#....." }, + { PCRE_MULTILINE | PCRE_NEWLINE_CR, 0, "\\W{0,2}[^#]{3}", "\r\n#....." }, + { PCRE_MULTILINE | PCRE_NEWLINE_CRLF, 0, "\\W{1,3}[^#]", "\r\n##...." }, + { MUA | PCRE_NO_UTF8_CHECK, 1, "^.a", "\n\x80\nxa" }, + { MUA, 1, "^", "\r\n" }, + { PCRE_MULTILINE | PCRE_NEWLINE_CRLF, 1 | F_NOMATCH, "^", "\r\n" }, + { PCRE_MULTILINE | PCRE_NEWLINE_CRLF, 1, "^", "\r\na" }, + + /* Any character except newline or any newline. */ + { PCRE_NEWLINE_CRLF, 0, ".", "\r" }, + { PCRE_NEWLINE_CRLF | PCRE_UTF8, 0, ".(.).", "a\xc3\xa1\r\n\n\r\r" }, + { PCRE_NEWLINE_ANYCRLF, 0, ".(.)", "a\rb\nc\r\n\xc2\x85\xe2\x80\xa8" }, + { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0, ".(.)", "a\rb\nc\r\n\xc2\x85\xe2\x80\xa8" }, + { PCRE_NEWLINE_ANY | PCRE_UTF8, 0, "(.).", "a\rb\nc\r\n\xc2\x85\xe2\x80\xa9$de" }, + { PCRE_NEWLINE_ANYCRLF | PCRE_UTF8, 0 | F_NOMATCH, ".(.).", "\xe2\x80\xa8\nb\r" }, + { PCRE_NEWLINE_ANY, 0, "(.)(.)", "#\x85#\r#\n#\r\n#\x84" }, + { PCRE_NEWLINE_ANY | PCRE_UTF8, 0, "(.+)#", "#\rMn\xc2\x85#\n###" }, + { PCRE_BSR_ANYCRLF, 0, "\\R", "\r" }, + { PCRE_BSR_ANYCRLF, 0, "\\R", "\x85#\r\n#" }, + { PCRE_BSR_UNICODE | PCRE_UTF8, 0, "\\R", "ab\xe2\x80\xa8#c" }, + { PCRE_BSR_UNICODE | PCRE_UTF8, 0, "\\R", "ab\r\nc" }, + { PCRE_NEWLINE_CRLF | PCRE_BSR_UNICODE | PCRE_UTF8, 0, "(\\R.)+", "\xc2\x85\r\n#\xe2\x80\xa8\n\r\n\r" }, + { MUA, 0 | F_NOMATCH, "\\R+", "ab" }, + { MUA, 0, "\\R+", "ab\r\n\r" }, + { MUA, 0, "\\R*", "ab\r\n\r" }, + { MUA, 0, "\\R*", "\r\n\r" }, + { MUA, 0, "\\R{2,4}", "\r\nab\r\r" }, + { MUA, 0, "\\R{2,4}", "\r\nab\n\n\n\r\r\r" }, + { MUA, 0, "\\R{2,}", "\r\nab\n\n\n\r\r\r" }, + { MUA, 0, "\\R{0,3}", "\r\n\r\n\r\n\r\n\r\n" }, + { MUA, 0 | F_NOMATCH, "\\R+\\R\\R", "\r\n\r\n" }, + { MUA, 0, "\\R+\\R\\R", "\r\r\r" }, + { MUA, 0, "\\R*\\R\\R", "\n\r" }, + { MUA, 0 | F_NOMATCH, "\\R{2,4}\\R\\R", "\r\r\r" }, + { MUA, 0, "\\R{2,4}\\R\\R", "\r\r\r\r" }, + + /* Atomic groups (no fallback from "next" direction). */ + { MUA, 0 | F_NOMATCH, "(?>ab)ab", "bab" }, + { MUA, 0 | F_NOMATCH, "(?>(ab))ab", "bab" }, + { MUA, 0, "(?>ab)+abc(?>de)*def(?>gh)?ghe(?>ij)+?k(?>lm)*?n(?>op)?\?op", + "bababcdedefgheijijklmlmnop" }, + { MUA, 0, "(?>a(b)+a|(ab)?\?(b))an", "abban" }, + { MUA, 0, "(?>ab+a|(?:ab)?\?b)an", "abban" }, + { MUA, 0, "((?>ab|ad|)*?)(?>|c)*abad", "abababcababad" }, + { MUA, 0, "(?>(aa|b|)*+(?>(##)|###)*d|(aa)(?>(baa)?)m)", "aabaa#####da" }, + { MUA, 0, "((?>a|)+?)b", "aaacaaab" }, + { MUA, 0, "(?>x|)*$", "aaa" }, + { MUA, 0, "(?>(x)|)*$", "aaa" }, + { MUA, 0, "(?>x|())*$", "aaa" }, + { MUA, 0, "((?>[cxy]a|[a-d])*?)b", "aaa+ aaab" }, + { MUA, 0, "((?>[cxy](a)|[a-d])*?)b", "aaa+ aaab" }, + { MUA, 0, "(?>((?>(a+))))bab|(?>((?>(a+))))bb", "aaaabaaabaabab" }, + { MUA, 0, "(?>(?>a+))bab|(?>(?>a+))bb", "aaaabaaabaabab" }, + { MUA, 0, "(?>(a)c|(?>(c)|(a))a)b*?bab", "aaaabaaabaabab" }, + { MUA, 0, "(?>ac|(?>c|a)a)b*?bab", "aaaabaaabaabab" }, + { MUA, 0, "(?>(b)b|(a))*b(?>(c)|d)?x", "ababcaaabdbx" }, + { MUA, 0, "(?>bb|a)*b(?>c|d)?x", "ababcaaabdbx" }, + { MUA, 0, "(?>(bb)|a)*b(?>c|(d))?x", "ababcaaabdbx" }, + { MUA, 0, "(?>(a))*?(?>(a))+?(?>(a))??x", "aaaaaacccaaaaabax" }, + { MUA, 0, "(?>a)*?(?>a)+?(?>a)??x", "aaaaaacccaaaaabax" }, + { MUA, 0, "(?>(a)|)*?(?>(a)|)+?(?>(a)|)??x", "aaaaaacccaaaaabax" }, + { MUA, 0, "(?>a|)*?(?>a|)+?(?>a|)??x", "aaaaaacccaaaaabax" }, + { MUA, 0, "(?>a(?>(a{0,2}))*?b|aac)+b", "aaaaaaacaaaabaaaaacaaaabaacaaabb" }, + { CMA, 0, "(?>((?>a{32}|b+|(a*))?(?>c+|d*)?\?)+e)+?f", "aaccebbdde bbdaaaccebbdee bbdaaaccebbdeef" }, + { MUA, 0, "(?>(?:(?>aa|a||x)+?b|(?>aa|a||(x))+?c)?(?>[ad]{0,2})*?d)+d", "aaacdbaabdcabdbaaacd aacaabdbdcdcaaaadaabcbaadd" }, + { MUA, 0, "(?>(?:(?>aa|a||(x))+?b|(?>aa|a||x)+?c)?(?>[ad]{0,2})*?d)+d", "aaacdbaabdcabdbaaacd aacaabdbdcdcaaaadaabcbaadd" }, + { MUA, 0 | F_PROPERTY, "\\X", "\xcc\x8d\xcc\x8d" }, + { MUA, 0 | F_PROPERTY, "\\X", "\xcc\x8d\xcc\x8d#\xcc\x8d\xcc\x8d" }, + { MUA, 0 | F_PROPERTY, "\\X+..", "\xcc\x8d#\xcc\x8d#\xcc\x8d\xcc\x8d" }, + { MUA, 0 | F_PROPERTY, "\\X{2,4}", "abcdef" }, + { MUA, 0 | F_PROPERTY, "\\X{2,4}?", "abcdef" }, + { MUA, 0 | F_NOMATCH | F_PROPERTY, "\\X{2,4}..", "#\xcc\x8d##" }, + { MUA, 0 | F_PROPERTY, "\\X{2,4}..", "#\xcc\x8d#\xcc\x8d##" }, + { MUA, 0, "(c(ab)?+ab)+", "cabcababcab" }, + { MUA, 0, "(?>(a+)b)+aabab", "aaaabaaabaabab" }, + + /* Possessive quantifiers. */ + { MUA, 0, "(?:a|b)++m", "mababbaaxababbaam" }, + { MUA, 0, "(?:a|b)*+m", "mababbaaxababbaam" }, + { MUA, 0, "(?:a|b)*+m", "ababbaaxababbaam" }, + { MUA, 0, "(a|b)++m", "mababbaaxababbaam" }, + { MUA, 0, "(a|b)*+m", "mababbaaxababbaam" }, + { MUA, 0, "(a|b)*+m", "ababbaaxababbaam" }, + { MUA, 0, "(a|b(*ACCEPT))++m", "maaxab" }, + { MUA, 0, "(?:b*)++m", "bxbbxbbbxm" }, + { MUA, 0, "(?:b*)++m", "bxbbxbbbxbbm" }, + { MUA, 0, "(?:b*)*+m", "bxbbxbbbxm" }, + { MUA, 0, "(?:b*)*+m", "bxbbxbbbxbbm" }, + { MUA, 0, "(b*)++m", "bxbbxbbbxm" }, + { MUA, 0, "(b*)++m", "bxbbxbbbxbbm" }, + { MUA, 0, "(b*)*+m", "bxbbxbbbxm" }, + { MUA, 0, "(b*)*+m", "bxbbxbbbxbbm" }, + { MUA, 0, "(?:a|(b))++m", "mababbaaxababbaam" }, + { MUA, 0, "(?:(a)|b)*+m", "mababbaaxababbaam" }, + { MUA, 0, "(?:(a)|(b))*+m", "ababbaaxababbaam" }, + { MUA, 0, "(a|(b))++m", "mababbaaxababbaam" }, + { MUA, 0, "((a)|b)*+m", "mababbaaxababbaam" }, + { MUA, 0, "((a)|(b))*+m", "ababbaaxababbaam" }, + { MUA, 0, "(a|(b)(*ACCEPT))++m", "maaxab" }, + { MUA, 0, "(?:(b*))++m", "bxbbxbbbxm" }, + { MUA, 0, "(?:(b*))++m", "bxbbxbbbxbbm" }, + { MUA, 0, "(?:(b*))*+m", "bxbbxbbbxm" }, + { MUA, 0, "(?:(b*))*+m", "bxbbxbbbxbbm" }, + { MUA, 0, "((b*))++m", "bxbbxbbbxm" }, + { MUA, 0, "((b*))++m", "bxbbxbbbxbbm" }, + { MUA, 0, "((b*))*+m", "bxbbxbbbxm" }, + { MUA, 0, "((b*))*+m", "bxbbxbbbxbbm" }, + { MUA, 0 | F_NOMATCH, "(?>(b{2,4}))(?:(?:(aa|c))++m|(?:(aa|c))+n)", "bbaacaaccaaaacxbbbmbn" }, + { MUA, 0, "((?:b)++a)+(cd)*+m", "bbababbacdcdnbbababbacdcdm" }, + { MUA, 0, "((?:(b))++a)+((c)d)*+m", "bbababbacdcdnbbababbacdcdm" }, + { MUA, 0, "(?:(?:(?:ab)*+k)++(?:n(?:cd)++)*+)*+m", "ababkkXababkkabkncXababkkabkncdcdncdXababkkabkncdcdncdkkabkncdXababkkabkncdcdncdkkabkncdm" }, + { MUA, 0, "(?:((ab)*+(k))++(n(?:c(d))++)*+)*+m", "ababkkXababkkabkncXababkkabkncdcdncdXababkkabkncdcdncdkkabkncdXababkkabkncdcdncdkkabkncdm" }, + + /* Back references. */ + { MUA, 0, "(aa|bb)(\\1*)(ll|)(\\3*)bbbbbbc", "aaaaaabbbbbbbbc" }, + { CMUA, 0, "(aa|bb)(\\1+)(ll|)(\\3+)bbbbbbc", "bBbbBbCbBbbbBbbcbbBbbbBBbbC" }, + { CMA, 0, "(a{2,4})\\1", "AaAaaAaA" }, + { MUA, 0, "(aa|bb)(\\1?)aa(\\1?)(ll|)(\\4+)bbc", "aaaaaaaabbaabbbbaabbbbc" }, + { MUA, 0, "(aa|bb)(\\1{0,5})(ll|)(\\3{0,5})cc", "bbxxbbbbxxaaaaaaaaaaaaaaaacc" }, + { MUA, 0, "(aa|bb)(\\1{3,5})(ll|)(\\3{3,5})cc", "bbbbbbbbbbbbaaaaaaccbbbbbbbbbbbbbbcc" }, + { MUA, 0, "(aa|bb)(\\1{3,})(ll|)(\\3{3,})cc", "bbbbbbbbbbbbaaaaaaccbbbbbbbbbbbbbbcc" }, + { MUA, 0, "(\\w+)b(\\1+)c", "GabGaGaDbGaDGaDc" }, + { MUA, 0, "(?:(aa)|b)\\1?b", "bb" }, + { CMUA, 0, "(aa|bb)(\\1*?)aa(\\1+?)", "bBBbaaAAaaAAaa" }, + { MUA, 0, "(aa|bb)(\\1*?)(dd|)cc(\\3+?)", "aaaaaccdd" }, + { CMUA, 0, "(?:(aa|bb)(\\1?\?)cc){2}(\\1?\?)", "aAaABBbbAAaAcCaAcCaA" }, + { MUA, 0, "(?:(aa|bb)(\\1{3,5}?)){2}(dd|)(\\3{3,5}?)", "aaaaaabbbbbbbbbbaaaaaaaaaaaaaa" }, + { CMA, 0, "(?:(aa|bb)(\\1{3,}?)){2}(dd|)(\\3{3,}?)", "aaaaaabbbbbbbbbbaaaaaaaaaaaaaa" }, + { MUA, 0, "(?:(aa|bb)(\\1{0,3}?)){2}(dd|)(\\3{0,3}?)b(\\1{0,3}?)(\\1{0,3})", "aaaaaaaaaaaaaaabaaaaa" }, + { MUA, 0, "(a(?:\\1|)a){3}b", "aaaaaaaaaaab" }, + { MA, 0, "(a?)b(\\1\\1*\\1+\\1?\\1*?\\1+?\\1??\\1*+\\1++\\1?+\\1{4}\\1{3,5}\\1{4,}\\1{0,5}\\1{3,5}?\\1{4,}?\\1{0,5}?\\1{3,5}+\\1{4,}+\\1{0,5}+#){2}d", "bb#b##d" }, + { MUAP, 0 | F_PROPERTY, "(\\P{N})\\1{2,}", ".www." }, + { MUAP, 0 | F_PROPERTY, "(\\P{N})\\1{0,2}", "wwwww." }, + { MUAP, 0 | F_PROPERTY, "(\\P{N})\\1{1,2}ww", "wwww" }, + { MUAP, 0 | F_PROPERTY, "(\\P{N})\\1{1,2}ww", "wwwww" }, + { PCRE_UCP, 0 | F_PROPERTY, "(\\P{N})\\1{2,}", ".www." }, + { CMUAP, 0, "(\xf0\x90\x90\x80)\\1", "\xf0\x90\x90\xa8\xf0\x90\x90\xa8" }, + { MUA | PCRE_DUPNAMES, 0 | F_NOMATCH, "\\k{1,3}(?aa)(?bb)", "aabb" }, + { MUA | PCRE_DUPNAMES | PCRE_JAVASCRIPT_COMPAT, 0, "\\k{1,3}(?aa)(?bb)", "aabb" }, + { MUA | PCRE_DUPNAMES | PCRE_JAVASCRIPT_COMPAT, 0, "\\k*(?aa)(?bb)", "aabb" }, + { MUA | PCRE_DUPNAMES, 0, "(?aa)(?bb)\\k{0,3}aaaaaa", "aabbaaaaaa" }, + { MUA | PCRE_DUPNAMES, 0, "(?aa)(?bb)\\k{2,5}bb", "aabbaaaabb" }, + { MUA | PCRE_DUPNAMES, 0, "(?:(?aa)|(?bb))\\k{0,3}m", "aaaaaaaabbbbaabbbbm" }, + { MUA | PCRE_DUPNAMES, 0 | F_NOMATCH, "\\k{1,3}?(?aa)(?bb)", "aabb" }, + { MUA | PCRE_DUPNAMES | PCRE_JAVASCRIPT_COMPAT, 0, "\\k{1,3}?(?aa)(?bb)", "aabb" }, + { MUA | PCRE_DUPNAMES, 0, "\\k*?(?aa)(?bb)", "aabb" }, + { MUA | PCRE_DUPNAMES, 0, "(?:(?aa)|(?bb))\\k{0,3}?m", "aaaaaabbbbbbaabbbbbbbbbbm" }, + { MUA | PCRE_DUPNAMES, 0, "(?:(?aa)|(?bb))\\k*?m", "aaaaaabbbbbbaabbbbbbbbbbm" }, + { MUA | PCRE_DUPNAMES, 0, "(?:(?aa)|(?bb))\\k{2,3}?", "aaaabbbbaaaabbbbbbbbbb" }, + { CMUA | PCRE_DUPNAMES, 0, "(?:(?AA)|(?BB))\\k{0,3}M", "aaaaaaaabbbbaabbbbm" }, + { CMUA | PCRE_DUPNAMES, 0, "(?:(?AA)|(?BB))\\k{1,3}M", "aaaaaaaabbbbaabbbbm" }, + { CMUA | PCRE_DUPNAMES, 0, "(?:(?AA)|(?BB))\\k{0,3}?M", "aaaaaabbbbbbaabbbbbbbbbbm" }, + { CMUA | PCRE_DUPNAMES, 0, "(?:(?AA)|(?BB))\\k{2,3}?", "aaaabbbbaaaabbbbbbbbbb" }, + + /* Assertions. */ + { MUA, 0, "(?=xx|yy|zz)\\w{4}", "abczzdefg" }, + { MUA, 0, "(?=((\\w+)b){3}|ab)", "dbbbb ab" }, + { MUA, 0, "(?!ab|bc|cd)[a-z]{2}", "Xabcdef" }, + { MUA, 0, "(?<=aaa|aa|a)a", "aaa" }, + { MUA, 2, "(?<=aaa|aa|a)a", "aaa" }, + { MA, 0, "(?<=aaa|aa|a)a", "aaa" }, + { MA, 2, "(?<=aaa|aa|a)a", "aaa" }, + { MUA, 0, "(\\d{2})(?!\\w+c|(((\\w?)m){2}n)+|\\1)", "x5656" }, + { MUA, 0, "((?=((\\d{2,6}\\w){2,}))\\w{5,20}K){2,}", "567v09708K12l00M00 567v09708K12l00M00K45K" }, + { MUA, 0, "(?=(?:(?=\\S+a)\\w*(b)){3})\\w+\\d", "bba bbab nbbkba nbbkba0kl" }, + { MUA, 0, "(?>a(?>(b+))a(?=(..)))*?k", "acabbcabbaabacabaabbakk" }, + { MUA, 0, "((?(?=(a))a)+k)", "bbak" }, + { MUA, 0, "((?(?=a)a)+k)", "bbak" }, + { MUA, 0 | F_NOMATCH, "(?=(?>(a))m)amk", "a k" }, + { MUA, 0 | F_NOMATCH, "(?!(?>(a))m)amk", "a k" }, + { MUA, 0 | F_NOMATCH, "(?>(?=(a))am)amk", "a k" }, + { MUA, 0, "(?=(?>a|(?=(?>(b+))a|c)[a-c]+)*?m)[a-cm]+k", "aaam bbam baaambaam abbabba baaambaamk" }, + { MUA, 0, "(?> ?\?\\b(?(?=\\w{1,4}(a))m)\\w{0,8}bc){2,}?", "bca ssbc mabd ssbc mabc" }, + { MUA, 0, "(?:(?=ab)?[^n][^n])+m", "ababcdabcdcdabnababcdabcdcdabm" }, + { MUA, 0, "(?:(?=a(b))?[^n][^n])+m", "ababcdabcdcdabnababcdabcdcdabm" }, + { MUA, 0, "(?:(?=.(.))??\\1.)+m", "aabbbcbacccanaabbbcbacccam" }, + { MUA, 0, "(?:(?=.)??[a-c])+m", "abacdcbacacdcaccam" }, + { MUA, 0, "((?!a)?(?!([^a]))?)+$", "acbab" }, + { MUA, 0, "((?!a)?\?(?!([^a]))?\?)+$", "acbab" }, + { MUA, 0, "a(?=(?C)\\B)b", "ab" }, + { MUA, 0, "a(?!(?C)\\B)bb|ab", "abb" }, + { MUA, 0, "a(?=\\b|(?C)\\B)b", "ab" }, + { MUA, 0, "a(?!\\b|(?C)\\B)bb|ab", "abb" }, + { MUA, 0, "c(?(?=(?C)\\B)ab|a)", "cab" }, + { MUA, 0, "c(?(?!(?C)\\B)ab|a)", "cab" }, + { MUA, 0, "c(?(?=\\b|(?C)\\B)ab|a)", "cab" }, + { MUA, 0, "c(?(?!\\b|(?C)\\B)ab|a)", "cab" }, + { MUA, 0, "a(?=)b", "ab" }, + { MUA, 0 | F_NOMATCH, "a(?!)b", "ab" }, + + /* Not empty, ACCEPT, FAIL */ + { MUA | PCRE_NOTEMPTY, 0 | F_NOMATCH, "a*", "bcx" }, + { MUA | PCRE_NOTEMPTY, 0, "a*", "bcaad" }, + { MUA | PCRE_NOTEMPTY, 0, "a*?", "bcaad" }, + { MUA | PCRE_NOTEMPTY_ATSTART, 0, "a*", "bcaad" }, + { MUA, 0, "a(*ACCEPT)b", "ab" }, + { MUA | PCRE_NOTEMPTY, 0 | F_NOMATCH, "a*(*ACCEPT)b", "bcx" }, + { MUA | PCRE_NOTEMPTY, 0, "a*(*ACCEPT)b", "bcaad" }, + { MUA | PCRE_NOTEMPTY, 0, "a*?(*ACCEPT)b", "bcaad" }, + { MUA | PCRE_NOTEMPTY, 0 | F_NOMATCH, "(?:z|a*(*ACCEPT)b)", "bcx" }, + { MUA | PCRE_NOTEMPTY, 0, "(?:z|a*(*ACCEPT)b)", "bcaad" }, + { MUA | PCRE_NOTEMPTY, 0, "(?:z|a*?(*ACCEPT)b)", "bcaad" }, + { MUA | PCRE_NOTEMPTY_ATSTART, 0, "a*(*ACCEPT)b", "bcx" }, + { MUA | PCRE_NOTEMPTY_ATSTART, 0 | F_NOMATCH, "a*(*ACCEPT)b", "" }, + { MUA, 0, "((a(*ACCEPT)b))", "ab" }, + { MUA, 0, "(a(*FAIL)a|a)", "aaa" }, + { MUA, 0, "(?=ab(*ACCEPT)b)a", "ab" }, + { MUA, 0, "(?=(?:x|ab(*ACCEPT)b))", "ab" }, + { MUA, 0, "(?=(a(b(*ACCEPT)b)))a", "ab" }, + { MUA | PCRE_NOTEMPTY, 0, "(?=a*(*ACCEPT))c", "c" }, + + /* Conditional blocks. */ + { MUA, 0, "(?(?=(a))a|b)+k", "ababbalbbadabak" }, + { MUA, 0, "(?(?!(b))a|b)+k", "ababbalbbadabak" }, + { MUA, 0, "(?(?=a)a|b)+k", "ababbalbbadabak" }, + { MUA, 0, "(?(?!b)a|b)+k", "ababbalbbadabak" }, + { MUA, 0, "(?(?=(a))a*|b*)+k", "ababbalbbadabak" }, + { MUA, 0, "(?(?!(b))a*|b*)+k", "ababbalbbadabak" }, + { MUA, 0, "(?(?!(b))(?:aaaaaa|a)|(?:bbbbbb|b))+aaaak", "aaaaaaaaaaaaaa bbbbbbbbbbbbbbb aaaaaaak" }, + { MUA, 0, "(?(?!b)(?:aaaaaa|a)|(?:bbbbbb|b))+aaaak", "aaaaaaaaaaaaaa bbbbbbbbbbbbbbb aaaaaaak" }, + { MUA, 0 | F_DIFF, "(?(?!(b))(?:aaaaaa|a)|(?:bbbbbb|b))+bbbbk", "aaaaaaaaaaaaaa bbbbbbbbbbbbbbb bbbbbbbk" }, + { MUA, 0, "(?(?!b)(?:aaaaaa|a)|(?:bbbbbb|b))+bbbbk", "aaaaaaaaaaaaaa bbbbbbbbbbbbbbb bbbbbbbk" }, + { MUA, 0, "(?(?=a)a*|b*)+k", "ababbalbbadabak" }, + { MUA, 0, "(?(?!b)a*|b*)+k", "ababbalbbadabak" }, + { MUA, 0, "(?(?=a)ab)", "a" }, + { MUA, 0, "(?(?a)?(?Pb)?(?(Name)c|d)*l", "bc ddd abccabccl" }, + { MUA, 0, "(?Pa)?(?Pb)?(?(Name)c|d)+?dd", "bcabcacdb bdddd" }, + { MUA, 0, "(?Pa)?(?Pb)?(?(Name)c|d)+l", "ababccddabdbccd abcccl" }, + { MUA, 0, "((?:a|aa)(?(1)aaa))x", "aax" }, + { MUA, 0, "(?(?!)a|b)", "ab" }, + { MUA, 0, "(?(?!)a)", "ab" }, + { MUA, 0 | F_NOMATCH, "(?(?!)a|b)", "ac" }, + + /* Set start of match. */ + { MUA, 0, "(?:\\Ka)*aaaab", "aaaaaaaa aaaaaaabb" }, + { MUA, 0, "(?>\\Ka\\Ka)*aaaab", "aaaaaaaa aaaaaaaaaabb" }, + { MUA, 0, "a+\\K(?<=\\Gaa)a", "aaaaaa" }, + { MUA | PCRE_NOTEMPTY, 0 | F_NOMATCH, "a\\K(*ACCEPT)b", "aa" }, + { MUA | PCRE_NOTEMPTY_ATSTART, 0, "a\\K(*ACCEPT)b", "aa" }, + + /* First line. */ + { MUA | PCRE_FIRSTLINE, 0 | F_PROPERTY, "\\p{Any}a", "bb\naaa" }, + { MUA | PCRE_FIRSTLINE, 0 | F_NOMATCH | F_PROPERTY, "\\p{Any}a", "bb\r\naaa" }, + { MUA | PCRE_FIRSTLINE, 0, "(?<=a)", "a" }, + { MUA | PCRE_FIRSTLINE, 0 | F_NOMATCH, "[^a][^b]", "ab" }, + { MUA | PCRE_FIRSTLINE, 0 | F_NOMATCH, "a", "\na" }, + { MUA | PCRE_FIRSTLINE, 0 | F_NOMATCH, "[abc]", "\na" }, + { MUA | PCRE_FIRSTLINE, 0 | F_NOMATCH, "^a", "\na" }, + { MUA | PCRE_FIRSTLINE, 0 | F_NOMATCH, "^(?<=\n)", "\na" }, + { MUA | PCRE_FIRSTLINE, 0, "\xf0\x90\x90\x80", "\xf0\x90\x90\x80" }, + { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANY | PCRE_FIRSTLINE, 0 | F_NOMATCH, "#", "\xc2\x85#" }, + { PCRE_MULTILINE | PCRE_NEWLINE_ANY | PCRE_FIRSTLINE, 0 | F_NOMATCH, "#", "\x85#" }, + { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_ANY | PCRE_FIRSTLINE, 0 | F_NOMATCH, "^#", "\xe2\x80\xa8#" }, + { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_CRLF | PCRE_FIRSTLINE, 0 | F_PROPERTY, "\\p{Any}", "\r\na" }, + { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_CRLF | PCRE_FIRSTLINE, 0, ".", "\r" }, + { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_CRLF | PCRE_FIRSTLINE, 0, "a", "\ra" }, + { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_CRLF | PCRE_FIRSTLINE, 0 | F_NOMATCH, "ba", "bbb\r\nba" }, + { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_CRLF | PCRE_FIRSTLINE, 0 | F_NOMATCH | F_PROPERTY, "\\p{Any}{4}|a", "\r\na" }, + { PCRE_MULTILINE | PCRE_UTF8 | PCRE_NEWLINE_CRLF | PCRE_FIRSTLINE, 1, ".", "\r\n" }, + { PCRE_FIRSTLINE | PCRE_NEWLINE_LF | PCRE_DOTALL, 0 | F_NOMATCH, "ab.", "ab" }, + { MUA | PCRE_FIRSTLINE, 1 | F_NOMATCH, "^[a-d0-9]", "\nxx\nd" }, + { PCRE_NEWLINE_ANY | PCRE_FIRSTLINE | PCRE_DOTALL, 0, "....a", "012\n0a" }, + { MUA | PCRE_FIRSTLINE, 0, "[aC]", "a" }, + + /* Recurse. */ + { MUA, 0, "(a)(?1)", "aa" }, + { MUA, 0, "((a))(?1)", "aa" }, + { MUA, 0, "(b|a)(?1)", "aa" }, + { MUA, 0, "(b|(a))(?1)", "aa" }, + { MUA, 0 | F_NOMATCH, "((a)(b)(?:a*))(?1)", "aba" }, + { MUA, 0, "((a)(b)(?:a*))(?1)", "abab" }, + { MUA, 0, "((a+)c(?2))b(?1)", "aacaabaca" }, + { MUA, 0, "((?2)b|(a)){2}(?1)", "aabab" }, + { MUA, 0, "(?1)(a)*+(?2)(b(?1))", "aababa" }, + { MUA, 0, "(?1)(((a(*ACCEPT)))b)", "axaa" }, + { MUA, 0, "(?1)(?(DEFINE) (((ac(*ACCEPT)))b) )", "akaac" }, + { MUA, 0, "(a+)b(?1)b\\1", "abaaabaaaaa" }, + { MUA, 0 | F_NOMATCH, "(?(DEFINE)(aa|a))(?1)ab", "aab" }, + { MUA, 0, "(?(DEFINE)(a\\Kb))(?1)+ababc", "abababxabababc" }, + { MUA, 0, "(a\\Kb)(?1)+ababc", "abababxababababc" }, + { MUA, 0 | F_NOMATCH, "(a\\Kb)(?1)+ababc", "abababxababababxc" }, + { MUA, 0, "b|<(?R)*>", "<" }, + { MUA, 0, "(a\\K){0}(?:(?1)b|ac)", "ac" }, + { MUA, 0, "(?(DEFINE)(a(?2)|b)(b(?1)|(a)))(?:(?1)|(?2))m", "ababababnababababaam" }, + { MUA, 0, "(a)((?(R)a|b))(?2)", "aabbabaa" }, + { MUA, 0, "(a)((?(R2)a|b))(?2)", "aabbabaa" }, + { MUA, 0, "(a)((?(R1)a|b))(?2)", "ababba" }, + { MUA, 0, "(?(R0)aa|bb(?R))", "abba aabb bbaa" }, + { MUA, 0, "((?(R)(?:aaaa|a)|(?:(aaaa)|(a)))+)(?1)$", "aaaaaaaaaa aaaa" }, + { MUA, 0, "(?Pa(?(R&Name)a|b))(?1)", "aab abb abaa" }, + { MUA, 0, "((?(R)a|(?1)){3})", "XaaaaaaaaaX" }, + { MUA, 0, "((?:(?(R)a|(?1))){3})", "XaaaaaaaaaX" }, + { MUA, 0, "((?(R)a|(?1)){1,3})aaaaaa", "aaaaaaaaXaaaaaaaaa" }, + { MUA, 0, "((?(R)a|(?1)){1,3}?)M", "aaaM" }, + + /* 16 bit specific tests. */ + { CMA, 0 | F_FORCECONV, "\xc3\xa1", "\xc3\x81\xc3\xa1" }, + { CMA, 0 | F_FORCECONV, "\xe1\xbd\xb8", "\xe1\xbf\xb8\xe1\xbd\xb8" }, + { CMA, 0 | F_FORCECONV, "[\xc3\xa1]", "\xc3\x81\xc3\xa1" }, + { CMA, 0 | F_FORCECONV, "[\xe1\xbd\xb8]", "\xe1\xbf\xb8\xe1\xbd\xb8" }, + { CMA, 0 | F_FORCECONV, "[a-\xed\xb0\x80]", "A" }, + { CMA, 0 | F_NO8 | F_FORCECONV, "[a-\\x{dc00}]", "B" }, + { CMA, 0 | F_NO8 | F_NOMATCH | F_FORCECONV, "[b-\\x{dc00}]", "a" }, + { CMA, 0 | F_NO8 | F_FORCECONV, "\xed\xa0\x80\\x{d800}\xed\xb0\x80\\x{dc00}", "\xed\xa0\x80\xed\xa0\x80\xed\xb0\x80\xed\xb0\x80" }, + { CMA, 0 | F_NO8 | F_FORCECONV, "[\xed\xa0\x80\\x{d800}]{1,2}?[\xed\xb0\x80\\x{dc00}]{1,2}?#", "\xed\xa0\x80\xed\xa0\x80\xed\xb0\x80\xed\xb0\x80#" }, + { CMA, 0 | F_FORCECONV, "[\xed\xa0\x80\xed\xb0\x80#]{0,3}(?<=\xed\xb0\x80.)", "\xed\xa0\x80#\xed\xa0\x80##\xed\xb0\x80\xed\xa0\x80" }, + { CMA, 0 | F_FORCECONV, "[\xed\xa0\x80-\xed\xb3\xbf]", "\xed\x9f\xbf\xed\xa0\x83" }, + { CMA, 0 | F_FORCECONV, "[\xed\xa0\x80-\xed\xb3\xbf]", "\xed\xb4\x80\xed\xb3\xb0" }, + { CMA, 0 | F_NO8 | F_FORCECONV, "[\\x{d800}-\\x{dcff}]", "\xed\x9f\xbf\xed\xa0\x83" }, + { CMA, 0 | F_NO8 | F_FORCECONV, "[\\x{d800}-\\x{dcff}]", "\xed\xb4\x80\xed\xb3\xb0" }, + { CMA, 0 | F_FORCECONV, "[\xed\xa0\x80-\xef\xbf\xbf]+[\x1-\xed\xb0\x80]+#", "\xed\xa0\x85\xc3\x81\xed\xa0\x85\xef\xbf\xb0\xc2\x85\xed\xa9\x89#" }, + { CMA, 0 | F_FORCECONV, "[\xed\xa0\x80][\xed\xb0\x80]{2,}", "\xed\xa0\x80\xed\xb0\x80\xed\xa0\x80\xed\xb0\x80\xed\xb0\x80\xed\xb0\x80" }, + { MA, 0 | F_FORCECONV, "[^\xed\xb0\x80]{3,}?", "##\xed\xb0\x80#\xed\xb0\x80#\xc3\x89#\xed\xb0\x80" }, + { MA, 0 | F_NO8 | F_FORCECONV, "[^\\x{dc00}]{3,}?", "##\xed\xb0\x80#\xed\xb0\x80#\xc3\x89#\xed\xb0\x80" }, + { CMA, 0 | F_FORCECONV, ".\\B.", "\xed\xa0\x80\xed\xb0\x80" }, + { CMA, 0 | F_FORCECONV, "\\D+(?:\\d+|.)\\S+(?:\\s+|.)\\W+(?:\\w+|.)\xed\xa0\x80\xed\xa0\x80", "\xed\xa0\x80\xed\xa0\x80\xed\xa0\x80\xed\xa0\x80\xed\xa0\x80\xed\xa0\x80\xed\xa0\x80\xed\xa0\x80" }, + { CMA, 0 | F_FORCECONV, "\\d*\\s*\\w*\xed\xa0\x80\xed\xa0\x80", "\xed\xa0\x80\xed\xa0\x80" }, + { CMA, 0 | F_FORCECONV | F_NOMATCH, "\\d*?\\D*?\\s*?\\S*?\\w*?\\W*?##", "\xed\xa0\x80\xed\xa0\x80\xed\xa0\x80\xed\xa0\x80#" }, + { CMA | PCRE_EXTENDED, 0 | F_FORCECONV, "\xed\xa0\x80 \xed\xb0\x80 !", "\xed\xa0\x80\xed\xb0\x80!" }, + { CMA, 0 | F_FORCECONV, "\xed\xa0\x80+#[^#]+\xed\xa0\x80", "\xed\xa0\x80#a\xed\xa0\x80" }, + { CMA, 0 | F_FORCECONV, "(\xed\xa0\x80+)#\\1", "\xed\xa0\x80\xed\xa0\x80#\xed\xa0\x80\xed\xa0\x80" }, + { PCRE_MULTILINE | PCRE_NEWLINE_ANY, 0 | F_NO8 | F_FORCECONV, "^-", "a--\xe2\x80\xa8--" }, + { PCRE_BSR_UNICODE, 0 | F_NO8 | F_FORCECONV, "\\R", "ab\xe2\x80\xa8" }, + { 0, 0 | F_NO8 | F_FORCECONV, "\\v", "ab\xe2\x80\xa9" }, + { 0, 0 | F_NO8 | F_FORCECONV, "\\h", "ab\xe1\xa0\x8e" }, + { 0, 0 | F_NO8 | F_FORCECONV, "\\v+?\\V+?#", "\xe2\x80\xa9\xe2\x80\xa9\xef\xbf\xbf\xef\xbf\xbf#" }, + { 0, 0 | F_NO8 | F_FORCECONV, "\\h+?\\H+?#", "\xe1\xa0\x8e\xe1\xa0\x8e\xef\xbf\xbf\xef\xbf\xbf#" }, + + /* Partial matching. */ + { MUA | PCRE_PARTIAL_SOFT, 0, "ab", "a" }, + { MUA | PCRE_PARTIAL_SOFT, 0, "ab|a", "a" }, + { MUA | PCRE_PARTIAL_HARD, 0, "ab|a", "a" }, + { MUA | PCRE_PARTIAL_SOFT, 0, "\\b#", "a" }, + { MUA | PCRE_PARTIAL_SOFT, 0, "(?<=a)b", "a" }, + { MUA | PCRE_PARTIAL_SOFT, 0, "abc|(?<=xxa)bc", "xxab" }, + { MUA | PCRE_PARTIAL_SOFT, 0, "a\\B", "a" }, + { MUA | PCRE_PARTIAL_HARD, 0, "a\\b", "a" }, + + /* (*MARK) verb. */ + { MUA, 0, "a(*MARK:aa)a", "ababaa" }, + { MUA, 0 | F_NOMATCH, "a(*:aa)a", "abab" }, + { MUA, 0, "a(*:aa)(b(*:bb)b|bc)", "abc" }, + { MUA, 0 | F_NOMATCH, "a(*:1)x|b(*:2)y", "abc" }, + { MUA, 0, "(?>a(*:aa))b|ac", "ac" }, + { MUA, 0, "(?(DEFINE)(a(*:aa)))(?1)", "a" }, + { MUA, 0 | F_NOMATCH, "(?(DEFINE)((a)(*:aa)))(?1)b", "aa" }, + { MUA, 0, "(?(DEFINE)(a(*:aa)))a(?1)b|aac", "aac" }, + { MUA, 0, "(a(*:aa)){0}(?:b(?1)b|c)+c", "babbab cc" }, + { MUA, 0, "(a(*:aa)){0}(?:b(?1)b)+", "babba" }, + { MUA, 0 | F_NOMATCH | F_STUDY, "(a(*:aa)){0}(?:b(?1)b)+", "ba" }, + { MUA, 0, "(a\\K(*:aa)){0}(?:b(?1)b|c)+c", "babbab cc" }, + { MUA, 0, "(a\\K(*:aa)){0}(?:b(?1)b)+", "babba" }, + { MUA, 0 | F_NOMATCH | F_STUDY, "(a\\K(*:aa)){0}(?:b(?1)b)+", "ba" }, + { MUA, 0 | F_NOMATCH | F_STUDY, "(*:mark)m", "a" }, + + /* (*COMMIT) verb. */ + { MUA, 0 | F_NOMATCH, "a(*COMMIT)b", "ac" }, + { MUA, 0, "aa(*COMMIT)b", "xaxaab" }, + { MUA, 0 | F_NOMATCH, "a(*COMMIT)(*:msg)b|ac", "ac" }, + { MUA, 0 | F_NOMATCH, "(a(*COMMIT)b)++", "abac" }, + { MUA, 0 | F_NOMATCH, "((a)(*COMMIT)b)++", "abac" }, + { MUA, 0 | F_NOMATCH, "(?=a(*COMMIT)b)ab|ad", "ad" }, + + /* (*PRUNE) verb. */ + { MUA, 0, "aa\\K(*PRUNE)b", "aaab" }, + { MUA, 0, "aa(*PRUNE:bb)b|a", "aa" }, + { MUA, 0, "(a)(a)(*PRUNE)b|(a)", "aa" }, + { MUA, 0, "(a)(a)(a)(a)(a)(a)(a)(a)(*PRUNE)b|(a)", "aaaaaaaa" }, + { MUA | PCRE_PARTIAL_SOFT, 0, "a(*PRUNE)a|", "a" }, + { MUA | PCRE_PARTIAL_SOFT, 0, "a(*PRUNE)a|m", "a" }, + { MUA, 0 | F_NOMATCH, "(?=a(*PRUNE)b)ab|ad", "ad" }, + { MUA, 0, "a(*COMMIT)(*PRUNE)d|bc", "abc" }, + { MUA, 0, "(?=a(*COMMIT)b)a(*PRUNE)c|bc", "abc" }, + { MUA, 0 | F_NOMATCH, "(*COMMIT)(?=a(*COMMIT)b)a(*PRUNE)c|bc", "abc" }, + { MUA, 0, "(?=(a)(*COMMIT)b)a(*PRUNE)c|bc", "abc" }, + { MUA, 0 | F_NOMATCH, "(*COMMIT)(?=(a)(*COMMIT)b)a(*PRUNE)c|bc", "abc" }, + { MUA, 0, "(a(*COMMIT)b){0}a(?1)(*PRUNE)c|bc", "abc" }, + { MUA, 0 | F_NOMATCH, "(a(*COMMIT)b){0}a(*COMMIT)(?1)(*PRUNE)c|bc", "abc" }, + { MUA, 0, "(a(*COMMIT)b)++(*PRUNE)d|c", "ababc" }, + { MUA, 0 | F_NOMATCH, "(*COMMIT)(a(*COMMIT)b)++(*PRUNE)d|c", "ababc" }, + { MUA, 0, "((a)(*COMMIT)b)++(*PRUNE)d|c", "ababc" }, + { MUA, 0 | F_NOMATCH, "(*COMMIT)((a)(*COMMIT)b)++(*PRUNE)d|c", "ababc" }, + { MUA, 0, "(?>a(*COMMIT)b)*abab(*PRUNE)d|ba", "ababab" }, + { MUA, 0 | F_NOMATCH, "(*COMMIT)(?>a(*COMMIT)b)*abab(*PRUNE)d|ba", "ababab" }, + { MUA, 0, "(?>a(*COMMIT)b)+abab(*PRUNE)d|ba", "ababab" }, + { MUA, 0 | F_NOMATCH, "(*COMMIT)(?>a(*COMMIT)b)+abab(*PRUNE)d|ba", "ababab" }, + { MUA, 0, "(?>a(*COMMIT)b)?ab(*PRUNE)d|ba", "aba" }, + { MUA, 0 | F_NOMATCH, "(*COMMIT)(?>a(*COMMIT)b)?ab(*PRUNE)d|ba", "aba" }, + { MUA, 0, "(?>a(*COMMIT)b)*?n(*PRUNE)d|ba", "abababn" }, + { MUA, 0 | F_NOMATCH, "(*COMMIT)(?>a(*COMMIT)b)*?n(*PRUNE)d|ba", "abababn" }, + { MUA, 0, "(?>a(*COMMIT)b)+?n(*PRUNE)d|ba", "abababn" }, + { MUA, 0 | F_NOMATCH, "(*COMMIT)(?>a(*COMMIT)b)+?n(*PRUNE)d|ba", "abababn" }, + { MUA, 0, "(?>a(*COMMIT)b)??n(*PRUNE)d|bn", "abn" }, + { MUA, 0 | F_NOMATCH, "(*COMMIT)(?>a(*COMMIT)b)??n(*PRUNE)d|bn", "abn" }, + + /* (*SKIP) verb. */ + { MUA, 0 | F_NOMATCH, "(?=a(*SKIP)b)ab|ad", "ad" }, + { MUA, 0, "(\\w+(*SKIP)#)", "abcd,xyz#," }, + { MUA, 0, "\\w+(*SKIP)#|mm", "abcd,xyz#," }, + { MUA, 0 | F_NOMATCH, "b+(?<=(*SKIP)#c)|b+", "#bbb" }, + + /* (*THEN) verb. */ + { MUA, 0, "((?:a(*THEN)|aab)(*THEN)c|a+)+m", "aabcaabcaabcaabcnacm" }, + { MUA, 0 | F_NOMATCH, "((?:a(*THEN)|aab)(*THEN)c|a+)+m", "aabcm" }, + { MUA, 0, "((?:a(*THEN)|aab)c|a+)+m", "aabcaabcnmaabcaabcm" }, + { MUA, 0, "((?:a|aab)(*THEN)c|a+)+m", "aam" }, + { MUA, 0, "((?:a(*COMMIT)|aab)(*THEN)c|a+)+m", "aam" }, + { MUA, 0, "(?(?=a(*THEN)b)ab|ad)", "ad" }, + { MUA, 0, "(?(?!a(*THEN)b)ad|add)", "add" }, + { MUA, 0 | F_NOMATCH, "(?(?=a)a(*THEN)b|ad)", "ad" }, + { MUA, 0, "(?!(?(?=a)ab|b(*THEN)d))bn|bnn", "bnn" }, + + /* Deep recursion. */ + { MUA, 0, "((((?:(?:(?:\\w)+)?)*|(?>\\w)+?)+|(?>\\w)?\?)*)?\\s", "aaaaa+ " }, + { MUA, 0, "(?:((?:(?:(?:\\w*?)+)??|(?>\\w)?|\\w*+)*)+)+?\\s", "aa+ " }, + { MUA, 0, "((a?)+)+b", "aaaaaaaaaaaa b" }, + + /* Deep recursion: Stack limit reached. */ + { MA, 0 | F_NOMATCH, "a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?aaaaaaaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaaaaaaa" }, + { MA, 0 | F_NOMATCH, "(?:a+)+b", "aaaaaaaaaaaaaaaaaaaaaaaa b" }, + { MA, 0 | F_NOMATCH, "(?:a+?)+?b", "aaaaaaaaaaaaaaaaaaaaaaaa b" }, + { MA, 0 | F_NOMATCH, "(?:a*)*b", "aaaaaaaaaaaaaaaaaaaaaaaa b" }, + { MA, 0 | F_NOMATCH, "(?:a*?)*?b", "aaaaaaaaaaaaaaaaaaaaaaaa b" }, + + { 0, 0, NULL, NULL } +}; + +static const unsigned char *tables(int mode) +{ + /* The purpose of this function to allow valgrind + for reporting invalid reads and writes. */ + static unsigned char *tables_copy; + const char *errorptr; + int erroroffset; + unsigned char *default_tables; +#if defined SUPPORT_PCRE8 + pcre *regex; + char null_str[1] = { 0 }; +#elif defined SUPPORT_PCRE16 + pcre16 *regex; + PCRE_UCHAR16 null_str[1] = { 0 }; +#elif defined SUPPORT_PCRE32 + pcre32 *regex; + PCRE_UCHAR32 null_str[1] = { 0 }; +#endif + + if (mode) { + if (tables_copy) + free(tables_copy); + tables_copy = NULL; + return NULL; + } + + if (tables_copy) + return tables_copy; + + default_tables = NULL; +#if defined SUPPORT_PCRE8 + regex = pcre_compile(null_str, 0, &errorptr, &erroroffset, NULL); + if (regex) { + pcre_fullinfo(regex, NULL, PCRE_INFO_DEFAULT_TABLES, &default_tables); + pcre_free(regex); + } +#elif defined SUPPORT_PCRE16 + regex = pcre16_compile(null_str, 0, &errorptr, &erroroffset, NULL); + if (regex) { + pcre16_fullinfo(regex, NULL, PCRE_INFO_DEFAULT_TABLES, &default_tables); + pcre16_free(regex); + } +#elif defined SUPPORT_PCRE32 + regex = pcre32_compile(null_str, 0, &errorptr, &erroroffset, NULL); + if (regex) { + pcre32_fullinfo(regex, NULL, PCRE_INFO_DEFAULT_TABLES, &default_tables); + pcre32_free(regex); + } +#endif + /* Shouldn't ever happen. */ + if (!default_tables) + return NULL; + + /* Unfortunately this value cannot get from pcre_fullinfo. + Since this is a test program, this is acceptable at the moment. */ + tables_copy = (unsigned char *)malloc(1088); + if (!tables_copy) + return NULL; + + memcpy(tables_copy, default_tables, 1088); + return tables_copy; +} + +#ifdef SUPPORT_PCRE8 +static pcre_jit_stack* callback8(void *arg) +{ + return (pcre_jit_stack *)arg; +} +#endif + +#ifdef SUPPORT_PCRE16 +static pcre16_jit_stack* callback16(void *arg) +{ + return (pcre16_jit_stack *)arg; +} +#endif + +#ifdef SUPPORT_PCRE32 +static pcre32_jit_stack* callback32(void *arg) +{ + return (pcre32_jit_stack *)arg; +} +#endif + +#ifdef SUPPORT_PCRE8 +static pcre_jit_stack *stack8; + +static pcre_jit_stack *getstack8(void) +{ + if (!stack8) + stack8 = pcre_jit_stack_alloc(1, 1024 * 1024); + return stack8; +} + +static void setstack8(pcre_extra *extra) +{ + if (!extra) { + if (stack8) + pcre_jit_stack_free(stack8); + stack8 = NULL; + return; + } + + pcre_assign_jit_stack(extra, callback8, getstack8()); +} +#endif /* SUPPORT_PCRE8 */ + +#ifdef SUPPORT_PCRE16 +static pcre16_jit_stack *stack16; + +static pcre16_jit_stack *getstack16(void) +{ + if (!stack16) + stack16 = pcre16_jit_stack_alloc(1, 1024 * 1024); + return stack16; +} + +static void setstack16(pcre16_extra *extra) +{ + if (!extra) { + if (stack16) + pcre16_jit_stack_free(stack16); + stack16 = NULL; + return; + } + + pcre16_assign_jit_stack(extra, callback16, getstack16()); +} +#endif /* SUPPORT_PCRE16 */ + +#ifdef SUPPORT_PCRE32 +static pcre32_jit_stack *stack32; + +static pcre32_jit_stack *getstack32(void) +{ + if (!stack32) + stack32 = pcre32_jit_stack_alloc(1, 1024 * 1024); + return stack32; +} + +static void setstack32(pcre32_extra *extra) +{ + if (!extra) { + if (stack32) + pcre32_jit_stack_free(stack32); + stack32 = NULL; + return; + } + + pcre32_assign_jit_stack(extra, callback32, getstack32()); +} +#endif /* SUPPORT_PCRE32 */ + +#ifdef SUPPORT_PCRE16 + +static int convert_utf8_to_utf16(const char *input, PCRE_UCHAR16 *output, int *offsetmap, int max_length) +{ + unsigned char *iptr = (unsigned char*)input; + PCRE_UCHAR16 *optr = output; + unsigned int c; + + if (max_length == 0) + return 0; + + while (*iptr && max_length > 1) { + c = 0; + if (offsetmap) + *offsetmap++ = (int)(iptr - (unsigned char*)input); + + if (*iptr < 0xc0) + c = *iptr++; + else if (!(*iptr & 0x20)) { + c = ((iptr[0] & 0x1f) << 6) | (iptr[1] & 0x3f); + iptr += 2; + } else if (!(*iptr & 0x10)) { + c = ((iptr[0] & 0x0f) << 12) | ((iptr[1] & 0x3f) << 6) | (iptr[2] & 0x3f); + iptr += 3; + } else if (!(*iptr & 0x08)) { + c = ((iptr[0] & 0x07) << 18) | ((iptr[1] & 0x3f) << 12) | ((iptr[2] & 0x3f) << 6) | (iptr[3] & 0x3f); + iptr += 4; + } + + if (c < 65536) { + *optr++ = c; + max_length--; + } else if (max_length <= 2) { + *optr = '\0'; + return (int)(optr - output); + } else { + c -= 0x10000; + *optr++ = 0xd800 | ((c >> 10) & 0x3ff); + *optr++ = 0xdc00 | (c & 0x3ff); + max_length -= 2; + if (offsetmap) + offsetmap++; + } + } + if (offsetmap) + *offsetmap = (int)(iptr - (unsigned char*)input); + *optr = '\0'; + return (int)(optr - output); +} + +static int copy_char8_to_char16(const char *input, PCRE_UCHAR16 *output, int max_length) +{ + unsigned char *iptr = (unsigned char*)input; + PCRE_UCHAR16 *optr = output; + + if (max_length == 0) + return 0; + + while (*iptr && max_length > 1) { + *optr++ = *iptr++; + max_length--; + } + *optr = '\0'; + return (int)(optr - output); +} + +#define REGTEST_MAX_LENGTH16 4096 +static PCRE_UCHAR16 regtest_buf16[REGTEST_MAX_LENGTH16]; +static int regtest_offsetmap16[REGTEST_MAX_LENGTH16]; + +#endif /* SUPPORT_PCRE16 */ + +#ifdef SUPPORT_PCRE32 + +static int convert_utf8_to_utf32(const char *input, PCRE_UCHAR32 *output, int *offsetmap, int max_length) +{ + unsigned char *iptr = (unsigned char*)input; + PCRE_UCHAR32 *optr = output; + unsigned int c; + + if (max_length == 0) + return 0; + + while (*iptr && max_length > 1) { + c = 0; + if (offsetmap) + *offsetmap++ = (int)(iptr - (unsigned char*)input); + + if (*iptr < 0xc0) + c = *iptr++; + else if (!(*iptr & 0x20)) { + c = ((iptr[0] & 0x1f) << 6) | (iptr[1] & 0x3f); + iptr += 2; + } else if (!(*iptr & 0x10)) { + c = ((iptr[0] & 0x0f) << 12) | ((iptr[1] & 0x3f) << 6) | (iptr[2] & 0x3f); + iptr += 3; + } else if (!(*iptr & 0x08)) { + c = ((iptr[0] & 0x07) << 18) | ((iptr[1] & 0x3f) << 12) | ((iptr[2] & 0x3f) << 6) | (iptr[3] & 0x3f); + iptr += 4; + } + + *optr++ = c; + max_length--; + } + if (offsetmap) + *offsetmap = (int)(iptr - (unsigned char*)input); + *optr = 0; + return (int)(optr - output); +} + +static int copy_char8_to_char32(const char *input, PCRE_UCHAR32 *output, int max_length) +{ + unsigned char *iptr = (unsigned char*)input; + PCRE_UCHAR32 *optr = output; + + if (max_length == 0) + return 0; + + while (*iptr && max_length > 1) { + *optr++ = *iptr++; + max_length--; + } + *optr = '\0'; + return (int)(optr - output); +} + +#define REGTEST_MAX_LENGTH32 4096 +static PCRE_UCHAR32 regtest_buf32[REGTEST_MAX_LENGTH32]; +static int regtest_offsetmap32[REGTEST_MAX_LENGTH32]; + +#endif /* SUPPORT_PCRE32 */ + +static int check_ascii(const char *input) +{ + const unsigned char *ptr = (unsigned char *)input; + while (*ptr) { + if (*ptr > 127) + return 0; + ptr++; + } + return 1; +} + +static int regression_tests(void) +{ + struct regression_test_case *current = regression_test_cases; + const char *error; + char *cpu_info; + int i, err_offs; + int is_successful, is_ascii; + int total = 0; + int successful = 0; + int successful_row = 0; + int counter = 0; + int study_mode; + int utf = 0, ucp = 0; + int disabled_flags = 0; +#ifdef SUPPORT_PCRE8 + pcre *re8; + pcre_extra *extra8; + pcre_extra dummy_extra8; + int ovector8_1[32]; + int ovector8_2[32]; + int return_value8[2]; + unsigned char *mark8_1, *mark8_2; +#endif +#ifdef SUPPORT_PCRE16 + pcre16 *re16; + pcre16_extra *extra16; + pcre16_extra dummy_extra16; + int ovector16_1[32]; + int ovector16_2[32]; + int return_value16[2]; + PCRE_UCHAR16 *mark16_1, *mark16_2; + int length16; +#endif +#ifdef SUPPORT_PCRE32 + pcre32 *re32; + pcre32_extra *extra32; + pcre32_extra dummy_extra32; + int ovector32_1[32]; + int ovector32_2[32]; + int return_value32[2]; + PCRE_UCHAR32 *mark32_1, *mark32_2; + int length32; +#endif + + /* This test compares the behaviour of interpreter and JIT. Although disabling + utf or ucp may make tests fail, if the pcre_exec result is the SAME, it is + still considered successful from pcre_jit_test point of view. */ + +#if defined SUPPORT_PCRE8 + pcre_config(PCRE_CONFIG_JITTARGET, &cpu_info); +#elif defined SUPPORT_PCRE16 + pcre16_config(PCRE_CONFIG_JITTARGET, &cpu_info); +#elif defined SUPPORT_PCRE32 + pcre32_config(PCRE_CONFIG_JITTARGET, &cpu_info); +#endif + + printf("Running JIT regression tests\n"); + printf(" target CPU of SLJIT compiler: %s\n", cpu_info); + +#if defined SUPPORT_PCRE8 + pcre_config(PCRE_CONFIG_UTF8, &utf); + pcre_config(PCRE_CONFIG_UNICODE_PROPERTIES, &ucp); +#elif defined SUPPORT_PCRE16 + pcre16_config(PCRE_CONFIG_UTF16, &utf); + pcre16_config(PCRE_CONFIG_UNICODE_PROPERTIES, &ucp); +#elif defined SUPPORT_PCRE32 + pcre32_config(PCRE_CONFIG_UTF32, &utf); + pcre32_config(PCRE_CONFIG_UNICODE_PROPERTIES, &ucp); +#endif + + if (!utf) + disabled_flags |= PCRE_UTF8 | PCRE_UTF16 | PCRE_UTF32; + if (!ucp) + disabled_flags |= PCRE_UCP; +#ifdef SUPPORT_PCRE8 + printf(" in 8 bit mode with UTF-8 %s and ucp %s:\n", utf ? "enabled" : "disabled", ucp ? "enabled" : "disabled"); +#endif +#ifdef SUPPORT_PCRE16 + printf(" in 16 bit mode with UTF-16 %s and ucp %s:\n", utf ? "enabled" : "disabled", ucp ? "enabled" : "disabled"); +#endif +#ifdef SUPPORT_PCRE32 + printf(" in 32 bit mode with UTF-32 %s and ucp %s:\n", utf ? "enabled" : "disabled", ucp ? "enabled" : "disabled"); +#endif + + while (current->pattern) { + /* printf("\nPattern: %s :\n", current->pattern); */ + total++; + is_ascii = 0; + if (!(current->start_offset & F_PROPERTY)) + is_ascii = check_ascii(current->pattern) && check_ascii(current->input); + + if (current->flags & PCRE_PARTIAL_SOFT) + study_mode = PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE; + else if (current->flags & PCRE_PARTIAL_HARD) + study_mode = PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE; + else + study_mode = PCRE_STUDY_JIT_COMPILE; + error = NULL; +#ifdef SUPPORT_PCRE8 + re8 = NULL; + if (!(current->start_offset & F_NO8)) + re8 = pcre_compile(current->pattern, + current->flags & ~(PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | disabled_flags), + &error, &err_offs, tables(0)); + + extra8 = NULL; + if (re8) { + error = NULL; + extra8 = pcre_study(re8, study_mode, &error); + if (!extra8) { + printf("\n8 bit: Cannot study pattern: %s\n", current->pattern); + pcre_free(re8); + re8 = NULL; + } + else if (!(extra8->flags & PCRE_EXTRA_EXECUTABLE_JIT)) { + printf("\n8 bit: JIT compiler does not support: %s\n", current->pattern); + pcre_free_study(extra8); + pcre_free(re8); + re8 = NULL; + } + extra8->flags |= PCRE_EXTRA_MARK; + } else if (((utf && ucp) || is_ascii) && !(current->start_offset & F_NO8)) + printf("\n8 bit: Cannot compile pattern \"%s\": %s\n", current->pattern, error); +#endif +#ifdef SUPPORT_PCRE16 + if ((current->flags & PCRE_UTF16) || (current->start_offset & F_FORCECONV)) + convert_utf8_to_utf16(current->pattern, regtest_buf16, NULL, REGTEST_MAX_LENGTH16); + else + copy_char8_to_char16(current->pattern, regtest_buf16, REGTEST_MAX_LENGTH16); + + re16 = NULL; + if (!(current->start_offset & F_NO16)) + re16 = pcre16_compile(regtest_buf16, + current->flags & ~(PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | disabled_flags), + &error, &err_offs, tables(0)); + + extra16 = NULL; + if (re16) { + error = NULL; + extra16 = pcre16_study(re16, study_mode, &error); + if (!extra16) { + printf("\n16 bit: Cannot study pattern: %s\n", current->pattern); + pcre16_free(re16); + re16 = NULL; + } + else if (!(extra16->flags & PCRE_EXTRA_EXECUTABLE_JIT)) { + printf("\n16 bit: JIT compiler does not support: %s\n", current->pattern); + pcre16_free_study(extra16); + pcre16_free(re16); + re16 = NULL; + } + extra16->flags |= PCRE_EXTRA_MARK; + } else if (((utf && ucp) || is_ascii) && !(current->start_offset & F_NO16)) + printf("\n16 bit: Cannot compile pattern \"%s\": %s\n", current->pattern, error); +#endif +#ifdef SUPPORT_PCRE32 + if ((current->flags & PCRE_UTF32) || (current->start_offset & F_FORCECONV)) + convert_utf8_to_utf32(current->pattern, regtest_buf32, NULL, REGTEST_MAX_LENGTH32); + else + copy_char8_to_char32(current->pattern, regtest_buf32, REGTEST_MAX_LENGTH32); + + re32 = NULL; + if (!(current->start_offset & F_NO32)) + re32 = pcre32_compile(regtest_buf32, + current->flags & ~(PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | disabled_flags), + &error, &err_offs, tables(0)); + + extra32 = NULL; + if (re32) { + error = NULL; + extra32 = pcre32_study(re32, study_mode, &error); + if (!extra32) { + printf("\n32 bit: Cannot study pattern: %s\n", current->pattern); + pcre32_free(re32); + re32 = NULL; + } + if (!(extra32->flags & PCRE_EXTRA_EXECUTABLE_JIT)) { + printf("\n32 bit: JIT compiler does not support: %s\n", current->pattern); + pcre32_free_study(extra32); + pcre32_free(re32); + re32 = NULL; + } + extra32->flags |= PCRE_EXTRA_MARK; + } else if (((utf && ucp) || is_ascii) && !(current->start_offset & F_NO32)) + printf("\n32 bit: Cannot compile pattern \"%s\": %s\n", current->pattern, error); +#endif + + counter++; + if ((counter & 0x3) != 0) { +#ifdef SUPPORT_PCRE8 + setstack8(NULL); +#endif +#ifdef SUPPORT_PCRE16 + setstack16(NULL); +#endif +#ifdef SUPPORT_PCRE32 + setstack32(NULL); +#endif + } + +#ifdef SUPPORT_PCRE8 + return_value8[0] = -1000; + return_value8[1] = -1000; + for (i = 0; i < 32; ++i) + ovector8_1[i] = -2; + for (i = 0; i < 32; ++i) + ovector8_2[i] = -2; + if (re8) { + mark8_1 = NULL; + mark8_2 = NULL; + extra8->mark = &mark8_1; + + if ((counter & 0x1) != 0) { + setstack8(extra8); + return_value8[0] = pcre_exec(re8, extra8, current->input, strlen(current->input), current->start_offset & OFFSET_MASK, + current->flags & (PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | PCRE_NO_UTF8_CHECK), ovector8_1, 32); + } else + return_value8[0] = pcre_jit_exec(re8, extra8, current->input, strlen(current->input), current->start_offset & OFFSET_MASK, + current->flags & (PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | PCRE_NO_UTF8_CHECK), ovector8_1, 32, getstack8()); + memset(&dummy_extra8, 0, sizeof(pcre_extra)); + dummy_extra8.flags = PCRE_EXTRA_MARK; + if (current->start_offset & F_STUDY) { + dummy_extra8.flags |= PCRE_EXTRA_STUDY_DATA; + dummy_extra8.study_data = extra8->study_data; + } + dummy_extra8.mark = &mark8_2; + return_value8[1] = pcre_exec(re8, &dummy_extra8, current->input, strlen(current->input), current->start_offset & OFFSET_MASK, + current->flags & (PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | PCRE_NO_UTF8_CHECK), ovector8_2, 32); + } +#endif + +#ifdef SUPPORT_PCRE16 + return_value16[0] = -1000; + return_value16[1] = -1000; + for (i = 0; i < 32; ++i) + ovector16_1[i] = -2; + for (i = 0; i < 32; ++i) + ovector16_2[i] = -2; + if (re16) { + mark16_1 = NULL; + mark16_2 = NULL; + if ((current->flags & PCRE_UTF16) || (current->start_offset & F_FORCECONV)) + length16 = convert_utf8_to_utf16(current->input, regtest_buf16, regtest_offsetmap16, REGTEST_MAX_LENGTH16); + else + length16 = copy_char8_to_char16(current->input, regtest_buf16, REGTEST_MAX_LENGTH16); + extra16->mark = &mark16_1; + if ((counter & 0x1) != 0) { + setstack16(extra16); + return_value16[0] = pcre16_exec(re16, extra16, regtest_buf16, length16, current->start_offset & OFFSET_MASK, + current->flags & (PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | PCRE_NO_UTF8_CHECK), ovector16_1, 32); + } else + return_value16[0] = pcre16_jit_exec(re16, extra16, regtest_buf16, length16, current->start_offset & OFFSET_MASK, + current->flags & (PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | PCRE_NO_UTF8_CHECK), ovector16_1, 32, getstack16()); + memset(&dummy_extra16, 0, sizeof(pcre16_extra)); + dummy_extra16.flags = PCRE_EXTRA_MARK; + if (current->start_offset & F_STUDY) { + dummy_extra16.flags |= PCRE_EXTRA_STUDY_DATA; + dummy_extra16.study_data = extra16->study_data; + } + dummy_extra16.mark = &mark16_2; + return_value16[1] = pcre16_exec(re16, &dummy_extra16, regtest_buf16, length16, current->start_offset & OFFSET_MASK, + current->flags & (PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | PCRE_NO_UTF8_CHECK), ovector16_2, 32); + } +#endif + +#ifdef SUPPORT_PCRE32 + return_value32[0] = -1000; + return_value32[1] = -1000; + for (i = 0; i < 32; ++i) + ovector32_1[i] = -2; + for (i = 0; i < 32; ++i) + ovector32_2[i] = -2; + if (re32) { + mark32_1 = NULL; + mark32_2 = NULL; + if ((current->flags & PCRE_UTF32) || (current->start_offset & F_FORCECONV)) + length32 = convert_utf8_to_utf32(current->input, regtest_buf32, regtest_offsetmap32, REGTEST_MAX_LENGTH32); + else + length32 = copy_char8_to_char32(current->input, regtest_buf32, REGTEST_MAX_LENGTH32); + extra32->mark = &mark32_1; + if ((counter & 0x1) != 0) { + setstack32(extra32); + return_value32[0] = pcre32_exec(re32, extra32, regtest_buf32, length32, current->start_offset & OFFSET_MASK, + current->flags & (PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | PCRE_NO_UTF8_CHECK), ovector32_1, 32); + } else + return_value32[0] = pcre32_jit_exec(re32, extra32, regtest_buf32, length32, current->start_offset & OFFSET_MASK, + current->flags & (PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | PCRE_NO_UTF8_CHECK), ovector32_1, 32, getstack32()); + memset(&dummy_extra32, 0, sizeof(pcre32_extra)); + dummy_extra32.flags = PCRE_EXTRA_MARK; + if (current->start_offset & F_STUDY) { + dummy_extra32.flags |= PCRE_EXTRA_STUDY_DATA; + dummy_extra32.study_data = extra32->study_data; + } + dummy_extra32.mark = &mark32_2; + return_value32[1] = pcre32_exec(re32, &dummy_extra32, regtest_buf32, length32, current->start_offset & OFFSET_MASK, + current->flags & (PCRE_NOTBOL | PCRE_NOTEOL | PCRE_NOTEMPTY | PCRE_NOTEMPTY_ATSTART | PCRE_PARTIAL_SOFT | PCRE_PARTIAL_HARD | PCRE_NO_UTF8_CHECK), ovector32_2, 32); + } +#endif + + /* printf("[%d-%d-%d|%d-%d|%d-%d|%d-%d]%s", + return_value8[0], return_value16[0], return_value32[0], + ovector8_1[0], ovector8_1[1], + ovector16_1[0], ovector16_1[1], + ovector32_1[0], ovector32_1[1], + (current->flags & PCRE_CASELESS) ? "C" : ""); */ + + /* If F_DIFF is set, just run the test, but do not compare the results. + Segfaults can still be captured. */ + + is_successful = 1; + if (!(current->start_offset & F_DIFF)) { +#if defined SUPPORT_UTF && ((defined(SUPPORT_PCRE8) + defined(SUPPORT_PCRE16) + defined(SUPPORT_PCRE32)) >= 2) + if (!(current->start_offset & F_FORCECONV)) { + int return_value; + + /* All results must be the same. */ +#ifdef SUPPORT_PCRE8 + if ((return_value = return_value8[0]) != return_value8[1]) { + printf("\n8 bit: Return value differs(J8:%d,I8:%d): [%d] '%s' @ '%s'\n", + return_value8[0], return_value8[1], total, current->pattern, current->input); + is_successful = 0; + } else +#endif +#ifdef SUPPORT_PCRE16 + if ((return_value = return_value16[0]) != return_value16[1]) { + printf("\n16 bit: Return value differs(J16:%d,I16:%d): [%d] '%s' @ '%s'\n", + return_value16[0], return_value16[1], total, current->pattern, current->input); + is_successful = 0; + } else +#endif +#ifdef SUPPORT_PCRE32 + if ((return_value = return_value32[0]) != return_value32[1]) { + printf("\n32 bit: Return value differs(J32:%d,I32:%d): [%d] '%s' @ '%s'\n", + return_value32[0], return_value32[1], total, current->pattern, current->input); + is_successful = 0; + } else +#endif +#if defined SUPPORT_PCRE8 && defined SUPPORT_PCRE16 + if (return_value8[0] != return_value16[0]) { + printf("\n8 and 16 bit: Return value differs(J8:%d,J16:%d): [%d] '%s' @ '%s'\n", + return_value8[0], return_value16[0], + total, current->pattern, current->input); + is_successful = 0; + } else +#endif +#if defined SUPPORT_PCRE8 && defined SUPPORT_PCRE32 + if (return_value8[0] != return_value32[0]) { + printf("\n8 and 32 bit: Return value differs(J8:%d,J32:%d): [%d] '%s' @ '%s'\n", + return_value8[0], return_value32[0], + total, current->pattern, current->input); + is_successful = 0; + } else +#endif +#if defined SUPPORT_PCRE16 && defined SUPPORT_PCRE32 + if (return_value16[0] != return_value32[0]) { + printf("\n16 and 32 bit: Return value differs(J16:%d,J32:%d): [%d] '%s' @ '%s'\n", + return_value16[0], return_value32[0], + total, current->pattern, current->input); + is_successful = 0; + } else +#endif + if (return_value >= 0 || return_value == PCRE_ERROR_PARTIAL) { + if (return_value == PCRE_ERROR_PARTIAL) { + return_value = 2; + } else { + return_value *= 2; + } +#ifdef SUPPORT_PCRE8 + return_value8[0] = return_value; +#endif +#ifdef SUPPORT_PCRE16 + return_value16[0] = return_value; +#endif +#ifdef SUPPORT_PCRE32 + return_value32[0] = return_value; +#endif + /* Transform back the results. */ + if (current->flags & PCRE_UTF8) { +#ifdef SUPPORT_PCRE16 + for (i = 0; i < return_value; ++i) { + if (ovector16_1[i] >= 0) + ovector16_1[i] = regtest_offsetmap16[ovector16_1[i]]; + if (ovector16_2[i] >= 0) + ovector16_2[i] = regtest_offsetmap16[ovector16_2[i]]; + } +#endif +#ifdef SUPPORT_PCRE32 + for (i = 0; i < return_value; ++i) { + if (ovector32_1[i] >= 0) + ovector32_1[i] = regtest_offsetmap32[ovector32_1[i]]; + if (ovector32_2[i] >= 0) + ovector32_2[i] = regtest_offsetmap32[ovector32_2[i]]; + } +#endif + } + + for (i = 0; i < return_value; ++i) { +#if defined SUPPORT_PCRE8 && defined SUPPORT_PCRE16 + if (ovector8_1[i] != ovector8_2[i] || ovector8_1[i] != ovector16_1[i] || ovector8_1[i] != ovector16_2[i]) { + printf("\n8 and 16 bit: Ovector[%d] value differs(J8:%d,I8:%d,J16:%d,I16:%d): [%d] '%s' @ '%s' \n", + i, ovector8_1[i], ovector8_2[i], ovector16_1[i], ovector16_2[i], + total, current->pattern, current->input); + is_successful = 0; + } +#endif +#if defined SUPPORT_PCRE8 && defined SUPPORT_PCRE32 + if (ovector8_1[i] != ovector8_2[i] || ovector8_1[i] != ovector32_1[i] || ovector8_1[i] != ovector32_2[i]) { + printf("\n8 and 32 bit: Ovector[%d] value differs(J8:%d,I8:%d,J32:%d,I32:%d): [%d] '%s' @ '%s' \n", + i, ovector8_1[i], ovector8_2[i], ovector32_1[i], ovector32_2[i], + total, current->pattern, current->input); + is_successful = 0; + } +#endif +#if defined SUPPORT_PCRE16 && defined SUPPORT_PCRE32 + if (ovector16_1[i] != ovector16_2[i] || ovector16_1[i] != ovector32_1[i] || ovector16_1[i] != ovector32_2[i]) { + printf("\n16 and 32 bit: Ovector[%d] value differs(J16:%d,I16:%d,J32:%d,I32:%d): [%d] '%s' @ '%s' \n", + i, ovector16_1[i], ovector16_2[i], ovector32_1[i], ovector32_2[i], + total, current->pattern, current->input); + is_successful = 0; + } +#endif + } + } + } else +#endif /* more than one of SUPPORT_PCRE8, SUPPORT_PCRE16 and SUPPORT_PCRE32 */ + { + /* Only the 8 bit and 16 bit results must be equal. */ +#ifdef SUPPORT_PCRE8 + if (return_value8[0] != return_value8[1]) { + printf("\n8 bit: Return value differs(%d:%d): [%d] '%s' @ '%s'\n", + return_value8[0], return_value8[1], total, current->pattern, current->input); + is_successful = 0; + } else if (return_value8[0] >= 0 || return_value8[0] == PCRE_ERROR_PARTIAL) { + if (return_value8[0] == PCRE_ERROR_PARTIAL) + return_value8[0] = 2; + else + return_value8[0] *= 2; + + for (i = 0; i < return_value8[0]; ++i) + if (ovector8_1[i] != ovector8_2[i]) { + printf("\n8 bit: Ovector[%d] value differs(%d:%d): [%d] '%s' @ '%s'\n", + i, ovector8_1[i], ovector8_2[i], total, current->pattern, current->input); + is_successful = 0; + } + } +#endif + +#ifdef SUPPORT_PCRE16 + if (return_value16[0] != return_value16[1]) { + printf("\n16 bit: Return value differs(%d:%d): [%d] '%s' @ '%s'\n", + return_value16[0], return_value16[1], total, current->pattern, current->input); + is_successful = 0; + } else if (return_value16[0] >= 0 || return_value16[0] == PCRE_ERROR_PARTIAL) { + if (return_value16[0] == PCRE_ERROR_PARTIAL) + return_value16[0] = 2; + else + return_value16[0] *= 2; + + for (i = 0; i < return_value16[0]; ++i) + if (ovector16_1[i] != ovector16_2[i]) { + printf("\n16 bit: Ovector[%d] value differs(%d:%d): [%d] '%s' @ '%s'\n", + i, ovector16_1[i], ovector16_2[i], total, current->pattern, current->input); + is_successful = 0; + } + } +#endif + +#ifdef SUPPORT_PCRE32 + if (return_value32[0] != return_value32[1]) { + printf("\n32 bit: Return value differs(%d:%d): [%d] '%s' @ '%s'\n", + return_value32[0], return_value32[1], total, current->pattern, current->input); + is_successful = 0; + } else if (return_value32[0] >= 0 || return_value32[0] == PCRE_ERROR_PARTIAL) { + if (return_value32[0] == PCRE_ERROR_PARTIAL) + return_value32[0] = 2; + else + return_value32[0] *= 2; + + for (i = 0; i < return_value32[0]; ++i) + if (ovector32_1[i] != ovector32_2[i]) { + printf("\n32 bit: Ovector[%d] value differs(%d:%d): [%d] '%s' @ '%s'\n", + i, ovector32_1[i], ovector32_2[i], total, current->pattern, current->input); + is_successful = 0; + } + } +#endif + } + } + + if (is_successful) { +#ifdef SUPPORT_PCRE8 + if (!(current->start_offset & F_NO8) && ((utf && ucp) || is_ascii)) { + if (return_value8[0] < 0 && !(current->start_offset & F_NOMATCH)) { + printf("8 bit: Test should match: [%d] '%s' @ '%s'\n", + total, current->pattern, current->input); + is_successful = 0; + } + + if (return_value8[0] >= 0 && (current->start_offset & F_NOMATCH)) { + printf("8 bit: Test should not match: [%d] '%s' @ '%s'\n", + total, current->pattern, current->input); + is_successful = 0; + } + } +#endif +#ifdef SUPPORT_PCRE16 + if (!(current->start_offset & F_NO16) && ((utf && ucp) || is_ascii)) { + if (return_value16[0] < 0 && !(current->start_offset & F_NOMATCH)) { + printf("16 bit: Test should match: [%d] '%s' @ '%s'\n", + total, current->pattern, current->input); + is_successful = 0; + } + + if (return_value16[0] >= 0 && (current->start_offset & F_NOMATCH)) { + printf("16 bit: Test should not match: [%d] '%s' @ '%s'\n", + total, current->pattern, current->input); + is_successful = 0; + } + } +#endif +#ifdef SUPPORT_PCRE32 + if (!(current->start_offset & F_NO32) && ((utf && ucp) || is_ascii)) { + if (return_value32[0] < 0 && !(current->start_offset & F_NOMATCH)) { + printf("32 bit: Test should match: [%d] '%s' @ '%s'\n", + total, current->pattern, current->input); + is_successful = 0; + } + + if (return_value32[0] >= 0 && (current->start_offset & F_NOMATCH)) { + printf("32 bit: Test should not match: [%d] '%s' @ '%s'\n", + total, current->pattern, current->input); + is_successful = 0; + } + } +#endif + } + + if (is_successful) { +#ifdef SUPPORT_PCRE8 + if (mark8_1 != mark8_2) { + printf("8 bit: Mark value mismatch: [%d] '%s' @ '%s'\n", + total, current->pattern, current->input); + is_successful = 0; + } +#endif +#ifdef SUPPORT_PCRE16 + if (mark16_1 != mark16_2) { + printf("16 bit: Mark value mismatch: [%d] '%s' @ '%s'\n", + total, current->pattern, current->input); + is_successful = 0; + } +#endif +#ifdef SUPPORT_PCRE32 + if (mark32_1 != mark32_2) { + printf("32 bit: Mark value mismatch: [%d] '%s' @ '%s'\n", + total, current->pattern, current->input); + is_successful = 0; + } +#endif + } + +#ifdef SUPPORT_PCRE8 + if (re8) { + pcre_free_study(extra8); + pcre_free(re8); + } +#endif +#ifdef SUPPORT_PCRE16 + if (re16) { + pcre16_free_study(extra16); + pcre16_free(re16); + } +#endif +#ifdef SUPPORT_PCRE32 + if (re32) { + pcre32_free_study(extra32); + pcre32_free(re32); + } +#endif + + if (is_successful) { + successful++; + successful_row++; + printf("."); + if (successful_row >= 60) { + successful_row = 0; + printf("\n"); + } + } else + successful_row = 0; + + fflush(stdout); + current++; + } + tables(1); +#ifdef SUPPORT_PCRE8 + setstack8(NULL); +#endif +#ifdef SUPPORT_PCRE16 + setstack16(NULL); +#endif +#ifdef SUPPORT_PCRE32 + setstack32(NULL); +#endif + + if (total == successful) { + printf("\nAll JIT regression tests are successfully passed.\n"); + return 0; + } else { + printf("\nSuccessful test ratio: %d%% (%d failed)\n", successful * 100 / total, total - successful); + return 1; + } +} + +/* End of pcre_jit_test.c */ diff --git a/src/third_party/pcre-8.41/pcre_maketables.c b/src/third_party/pcre-8.41/pcre_maketables.c new file mode 100644 index 00000000000..a44a6eaa905 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_maketables.c @@ -0,0 +1,156 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This module contains the external function pcre_maketables(), which builds +character tables for PCRE in the current locale. The file is compiled on its +own as part of the PCRE library. However, it is also included in the +compilation of dftables.c, in which case the macro DFTABLES is defined. */ + + +#ifndef DFTABLES +# ifdef HAVE_CONFIG_H +# include "config.h" +# endif +# include "pcre_internal.h" +#endif + + +/************************************************* +* Create PCRE character tables * +*************************************************/ + +/* This function builds a set of character tables for use by PCRE and returns +a pointer to them. They are build using the ctype functions, and consequently +their contents will depend upon the current locale setting. When compiled as +part of the library, the store is obtained via PUBL(malloc)(), but when +compiled inside dftables, use malloc(). + +Arguments: none +Returns: pointer to the contiguous block of data +*/ + +#if defined COMPILE_PCRE8 +const unsigned char * +pcre_maketables(void) +#elif defined COMPILE_PCRE16 +const unsigned char * +pcre16_maketables(void) +#elif defined COMPILE_PCRE32 +const unsigned char * +pcre32_maketables(void) +#endif +{ +unsigned char *yield, *p; +int i; + +#ifndef DFTABLES +yield = (unsigned char*)(PUBL(malloc))(tables_length); +#else +yield = (unsigned char*)malloc(tables_length); +#endif + +if (yield == NULL) return NULL; +p = yield; + +/* First comes the lower casing table */ + +for (i = 0; i < 256; i++) *p++ = tolower(i); + +/* Next the case-flipping table */ + +for (i = 0; i < 256; i++) *p++ = islower(i)? toupper(i) : tolower(i); + +/* Then the character class tables. Don't try to be clever and save effort on +exclusive ones - in some locales things may be different. + +Note that the table for "space" includes everything "isspace" gives, including +VT in the default locale. This makes it work for the POSIX class [:space:]. +From release 8.34 is is also correct for Perl space, because Perl added VT at +release 5.18. + +Note also that it is possible for a character to be alnum or alpha without +being lower or upper, such as "male and female ordinals" (\xAA and \xBA) in the +fr_FR locale (at least under Debian Linux's locales as of 12/2005). So we must +test for alnum specially. */ + +memset(p, 0, cbit_length); +for (i = 0; i < 256; i++) + { + if (isdigit(i)) p[cbit_digit + i/8] |= 1 << (i&7); + if (isupper(i)) p[cbit_upper + i/8] |= 1 << (i&7); + if (islower(i)) p[cbit_lower + i/8] |= 1 << (i&7); + if (isalnum(i)) p[cbit_word + i/8] |= 1 << (i&7); + if (i == '_') p[cbit_word + i/8] |= 1 << (i&7); + if (isspace(i)) p[cbit_space + i/8] |= 1 << (i&7); + if (isxdigit(i))p[cbit_xdigit + i/8] |= 1 << (i&7); + if (isgraph(i)) p[cbit_graph + i/8] |= 1 << (i&7); + if (isprint(i)) p[cbit_print + i/8] |= 1 << (i&7); + if (ispunct(i)) p[cbit_punct + i/8] |= 1 << (i&7); + if (iscntrl(i)) p[cbit_cntrl + i/8] |= 1 << (i&7); + } +p += cbit_length; + +/* Finally, the character type table. In this, we used to exclude VT from the +white space chars, because Perl didn't recognize it as such for \s and for +comments within regexes. However, Perl changed at release 5.18, so PCRE changed +at release 8.34. */ + +for (i = 0; i < 256; i++) + { + int x = 0; + if (isspace(i)) x += ctype_space; + if (isalpha(i)) x += ctype_letter; + if (isdigit(i)) x += ctype_digit; + if (isxdigit(i)) x += ctype_xdigit; + if (isalnum(i) || i == '_') x += ctype_word; + + /* Note: strchr includes the terminating zero in the characters it considers. + In this instance, that is ok because we want binary zero to be flagged as a + meta-character, which in this sense is any character that terminates a run + of data characters. */ + + if (strchr("\\*+?{^.$|()[", i) != 0) x += ctype_meta; + *p++ = x; + } + +return yield; +} + +/* End of pcre_maketables.c */ diff --git a/src/third_party/pcre-8.41/pcre_newline.c b/src/third_party/pcre-8.41/pcre_newline.c new file mode 100644 index 00000000000..b8f5a4de19c --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_newline.c @@ -0,0 +1,210 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This module contains internal functions for testing newlines when more than +one kind of newline is to be recognized. When a newline is found, its length is +returned. In principle, we could implement several newline "types", each +referring to a different set of newline characters. At present, PCRE supports +only NLTYPE_FIXED, which gets handled without these functions, NLTYPE_ANYCRLF, +and NLTYPE_ANY. The full list of Unicode newline characters is taken from +http://unicode.org/unicode/reports/tr18/. */ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "pcre_internal.h" + + + +/************************************************* +* Check for newline at given position * +*************************************************/ + +/* It is guaranteed that the initial value of ptr is less than the end of the +string that is being processed. + +Arguments: + ptr pointer to possible newline + type the newline type + endptr pointer to the end of the string + lenptr where to return the length + utf TRUE if in utf mode + +Returns: TRUE or FALSE +*/ + +BOOL +PRIV(is_newline)(PCRE_PUCHAR ptr, int type, PCRE_PUCHAR endptr, int *lenptr, + BOOL utf) +{ +pcre_uint32 c; +(void)utf; +#ifdef SUPPORT_UTF +if (utf) + { + GETCHAR(c, ptr); + } +else +#endif /* SUPPORT_UTF */ + c = *ptr; + +/* Note that this function is called only for ANY or ANYCRLF. */ + +if (type == NLTYPE_ANYCRLF) switch(c) + { + case CHAR_LF: *lenptr = 1; return TRUE; + case CHAR_CR: *lenptr = (ptr < endptr - 1 && ptr[1] == CHAR_LF)? 2 : 1; + return TRUE; + default: return FALSE; + } + +/* NLTYPE_ANY */ + +else switch(c) + { +#ifdef EBCDIC + case CHAR_NEL: +#endif + case CHAR_LF: + case CHAR_VT: + case CHAR_FF: *lenptr = 1; return TRUE; + + case CHAR_CR: + *lenptr = (ptr < endptr - 1 && ptr[1] == CHAR_LF)? 2 : 1; + return TRUE; + +#ifndef EBCDIC +#ifdef COMPILE_PCRE8 + case CHAR_NEL: *lenptr = utf? 2 : 1; return TRUE; + case 0x2028: /* LS */ + case 0x2029: *lenptr = 3; return TRUE; /* PS */ +#else /* COMPILE_PCRE16 || COMPILE_PCRE32 */ + case CHAR_NEL: + case 0x2028: /* LS */ + case 0x2029: *lenptr = 1; return TRUE; /* PS */ +#endif /* COMPILE_PCRE8 */ +#endif /* Not EBCDIC */ + + default: return FALSE; + } +} + + + +/************************************************* +* Check for newline at previous position * +*************************************************/ + +/* It is guaranteed that the initial value of ptr is greater than the start of +the string that is being processed. + +Arguments: + ptr pointer to possible newline + type the newline type + startptr pointer to the start of the string + lenptr where to return the length + utf TRUE if in utf mode + +Returns: TRUE or FALSE +*/ + +BOOL +PRIV(was_newline)(PCRE_PUCHAR ptr, int type, PCRE_PUCHAR startptr, int *lenptr, + BOOL utf) +{ +pcre_uint32 c; +(void)utf; +ptr--; +#ifdef SUPPORT_UTF +if (utf) + { + BACKCHAR(ptr); + GETCHAR(c, ptr); + } +else +#endif /* SUPPORT_UTF */ + c = *ptr; + +/* Note that this function is called only for ANY or ANYCRLF. */ + +if (type == NLTYPE_ANYCRLF) switch(c) + { + case CHAR_LF: + *lenptr = (ptr > startptr && ptr[-1] == CHAR_CR)? 2 : 1; + return TRUE; + + case CHAR_CR: *lenptr = 1; return TRUE; + default: return FALSE; + } + +/* NLTYPE_ANY */ + +else switch(c) + { + case CHAR_LF: + *lenptr = (ptr > startptr && ptr[-1] == CHAR_CR)? 2 : 1; + return TRUE; + +#ifdef EBCDIC + case CHAR_NEL: +#endif + case CHAR_VT: + case CHAR_FF: + case CHAR_CR: *lenptr = 1; return TRUE; + +#ifndef EBCDIC +#ifdef COMPILE_PCRE8 + case CHAR_NEL: *lenptr = utf? 2 : 1; return TRUE; + case 0x2028: /* LS */ + case 0x2029: *lenptr = 3; return TRUE; /* PS */ +#else /* COMPILE_PCRE16 || COMPILE_PCRE32 */ + case CHAR_NEL: + case 0x2028: /* LS */ + case 0x2029: *lenptr = 1; return TRUE; /* PS */ +#endif /* COMPILE_PCRE8 */ +#endif /* NotEBCDIC */ + + default: return FALSE; + } +} + +/* End of pcre_newline.c */ diff --git a/src/third_party/pcre-8.41/pcre_ord2utf8.c b/src/third_party/pcre-8.41/pcre_ord2utf8.c new file mode 100644 index 00000000000..95f1beb963e --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_ord2utf8.c @@ -0,0 +1,94 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This file contains a private PCRE function that converts an ordinal +character value into a UTF8 string. */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define COMPILE_PCRE8 + +#include "pcre_internal.h" + +/************************************************* +* Convert character value to UTF-8 * +*************************************************/ + +/* This function takes an integer value in the range 0 - 0x10ffff +and encodes it as a UTF-8 character in 1 to 4 pcre_uchars. + +Arguments: + cvalue the character value + buffer pointer to buffer for result - at least 6 pcre_uchars long + +Returns: number of characters placed in the buffer +*/ + +unsigned +int +PRIV(ord2utf)(pcre_uint32 cvalue, pcre_uchar *buffer) +{ +#ifdef SUPPORT_UTF + +register int i, j; + +for (i = 0; i < PRIV(utf8_table1_size); i++) + if ((int)cvalue <= PRIV(utf8_table1)[i]) break; +buffer += i; +for (j = i; j > 0; j--) + { + *buffer-- = 0x80 | (cvalue & 0x3f); + cvalue >>= 6; + } +*buffer = PRIV(utf8_table2)[i] | cvalue; +return i + 1; + +#else + +(void)(cvalue); /* Keep compiler happy; this function won't ever be */ +(void)(buffer); /* called when SUPPORT_UTF is not defined. */ +return 0; + +#endif +} + +/* End of pcre_ord2utf8.c */ diff --git a/src/third_party/pcre-8.41/pcre_printint.c b/src/third_party/pcre-8.41/pcre_printint.c new file mode 100644 index 00000000000..60dcb55efbf --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_printint.c @@ -0,0 +1,834 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This module contains a PCRE private debugging function for printing out the +internal form of a compiled regular expression, along with some supporting +local functions. This source file is used in two places: + +(1) It is #included by pcre_compile.c when it is compiled in debugging mode +(PCRE_DEBUG defined in pcre_internal.h). It is not included in production +compiles. In this case PCRE_INCLUDED is defined. + +(2) It is also compiled separately and linked with pcretest.c, which can be +asked to print out a compiled regex for debugging purposes. */ + +#ifndef PCRE_INCLUDED + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +/* For pcretest program. */ +#define PRIV(name) name + +/* We have to include pcre_internal.h because we need the internal info for +displaying the results of pcre_study() and we also need to know about the +internal macros, structures, and other internal data values; pcretest has +"inside information" compared to a program that strictly follows the PCRE API. + +Although pcre_internal.h does itself include pcre.h, we explicitly include it +here before pcre_internal.h so that the PCRE_EXP_xxx macros get set +appropriately for an application, not for building PCRE. */ + +#include "pcre.h" +#include "pcre_internal.h" + +/* These are the funtions that are contained within. It doesn't seem worth +having a separate .h file just for this. */ + +#endif /* PCRE_INCLUDED */ + +#ifdef PCRE_INCLUDED +static /* Keep the following function as private. */ +#endif + +#if defined COMPILE_PCRE8 +void pcre_printint(pcre *external_re, FILE *f, BOOL print_lengths); +#elif defined COMPILE_PCRE16 +void pcre16_printint(pcre *external_re, FILE *f, BOOL print_lengths); +#elif defined COMPILE_PCRE32 +void pcre32_printint(pcre *external_re, FILE *f, BOOL print_lengths); +#endif + +/* Macro that decides whether a character should be output as a literal or in +hexadecimal. We don't use isprint() because that can vary from system to system +(even without the use of locales) and we want the output always to be the same, +for testing purposes. */ + +#ifdef EBCDIC +#define PRINTABLE(c) ((c) >= 64 && (c) < 255) +#else +#define PRINTABLE(c) ((c) >= 32 && (c) < 127) +#endif + +/* The table of operator names. */ + +static const char *priv_OP_names[] = { OP_NAME_LIST }; + +/* This table of operator lengths is not actually used by the working code, +but its size is needed for a check that ensures it is the correct size for the +number of opcodes (thus catching update omissions). */ + +static const pcre_uint8 priv_OP_lengths[] = { OP_LENGTHS }; + + + +/************************************************* +* Print single- or multi-byte character * +*************************************************/ + +static unsigned int +print_char(FILE *f, pcre_uchar *ptr, BOOL utf) +{ +pcre_uint32 c = *ptr; + +#ifndef SUPPORT_UTF + +(void)utf; /* Avoid compiler warning */ +if (PRINTABLE(c)) fprintf(f, "%c", (char)c); +else if (c <= 0x80) fprintf(f, "\\x%02x", c); +else fprintf(f, "\\x{%x}", c); +return 0; + +#else + +#if defined COMPILE_PCRE8 + +if (!utf || (c & 0xc0) != 0xc0) + { + if (PRINTABLE(c)) fprintf(f, "%c", (char)c); + else if (c < 0x80) fprintf(f, "\\x%02x", c); + else fprintf(f, "\\x{%02x}", c); + return 0; + } +else + { + int i; + int a = PRIV(utf8_table4)[c & 0x3f]; /* Number of additional bytes */ + int s = 6*a; + c = (c & PRIV(utf8_table3)[a]) << s; + for (i = 1; i <= a; i++) + { + /* This is a check for malformed UTF-8; it should only occur if the sanity + check has been turned off. Rather than swallow random bytes, just stop if + we hit a bad one. Print it with \X instead of \x as an indication. */ + + if ((ptr[i] & 0xc0) != 0x80) + { + fprintf(f, "\\X{%x}", c); + return i - 1; + } + + /* The byte is OK */ + + s -= 6; + c |= (ptr[i] & 0x3f) << s; + } + fprintf(f, "\\x{%x}", c); + return a; + } + +#elif defined COMPILE_PCRE16 + +if (!utf || (c & 0xfc00) != 0xd800) + { + if (PRINTABLE(c)) fprintf(f, "%c", (char)c); + else if (c <= 0x80) fprintf(f, "\\x%02x", c); + else fprintf(f, "\\x{%02x}", c); + return 0; + } +else + { + /* This is a check for malformed UTF-16; it should only occur if the sanity + check has been turned off. Rather than swallow a low surrogate, just stop if + we hit a bad one. Print it with \X instead of \x as an indication. */ + + if ((ptr[1] & 0xfc00) != 0xdc00) + { + fprintf(f, "\\X{%x}", c); + return 0; + } + + c = (((c & 0x3ff) << 10) | (ptr[1] & 0x3ff)) + 0x10000; + fprintf(f, "\\x{%x}", c); + return 1; + } + +#elif defined COMPILE_PCRE32 + +if (!utf || (c & 0xfffff800u) != 0xd800u) + { + if (PRINTABLE(c)) fprintf(f, "%c", (char)c); + else if (c <= 0x80) fprintf(f, "\\x%02x", c); + else fprintf(f, "\\x{%x}", c); + return 0; + } +else + { + /* This is a check for malformed UTF-32; it should only occur if the sanity + check has been turned off. Rather than swallow a surrogate, just stop if + we hit one. Print it with \X instead of \x as an indication. */ + fprintf(f, "\\X{%x}", c); + return 0; + } + +#endif /* COMPILE_PCRE[8|16|32] */ + +#endif /* SUPPORT_UTF */ +} + +/************************************************* +* Print uchar string (regardless of utf) * +*************************************************/ + +static void +print_puchar(FILE *f, PCRE_PUCHAR ptr) +{ +while (*ptr != '\0') + { + register pcre_uint32 c = *ptr++; + if (PRINTABLE(c)) fprintf(f, "%c", c); else fprintf(f, "\\x{%x}", c); + } +} + +/************************************************* +* Find Unicode property name * +*************************************************/ + +static const char * +get_ucpname(unsigned int ptype, unsigned int pvalue) +{ +#ifdef SUPPORT_UCP +int i; +for (i = PRIV(utt_size) - 1; i >= 0; i--) + { + if (ptype == PRIV(utt)[i].type && pvalue == PRIV(utt)[i].value) break; + } +return (i >= 0)? PRIV(utt_names) + PRIV(utt)[i].name_offset : "??"; +#else +/* It gets harder and harder to shut off unwanted compiler warnings. */ +ptype = ptype * pvalue; +return (ptype == pvalue)? "??" : "??"; +#endif +} + + +/************************************************* +* Print Unicode property value * +*************************************************/ + +/* "Normal" properties can be printed from tables. The PT_CLIST property is a +pseudo-property that contains a pointer to a list of case-equivalent +characters. This is used only when UCP support is available and UTF mode is +selected. It should never occur otherwise, but just in case it does, have +something ready to print. */ + +static void +print_prop(FILE *f, pcre_uchar *code, const char *before, const char *after) +{ +if (code[1] != PT_CLIST) + { + fprintf(f, "%s%s %s%s", before, priv_OP_names[*code], get_ucpname(code[1], + code[2]), after); + } +else + { + const char *not = (*code == OP_PROP)? "" : "not "; +#ifndef SUPPORT_UCP + fprintf(f, "%s%sclist %d%s", before, not, code[2], after); +#else + const pcre_uint32 *p = PRIV(ucd_caseless_sets) + code[2]; + fprintf (f, "%s%sclist", before, not); + while (*p < NOTACHAR) fprintf(f, " %04x", *p++); + fprintf(f, "%s", after); +#endif + } +} + + + + +/************************************************* +* Print compiled regex * +*************************************************/ + +/* Make this function work for a regex with integers either byte order. +However, we assume that what we are passed is a compiled regex. The +print_lengths flag controls whether offsets and lengths of items are printed. +They can be turned off from pcretest so that automatic tests on bytecode can be +written that do not depend on the value of LINK_SIZE. */ + +#ifdef PCRE_INCLUDED +static /* Keep the following function as private. */ +#endif +#if defined COMPILE_PCRE8 +void +pcre_printint(pcre *external_re, FILE *f, BOOL print_lengths) +#elif defined COMPILE_PCRE16 +void +pcre16_printint(pcre *external_re, FILE *f, BOOL print_lengths) +#elif defined COMPILE_PCRE32 +void +pcre32_printint(pcre *external_re, FILE *f, BOOL print_lengths) +#endif +{ +REAL_PCRE *re = (REAL_PCRE *)external_re; +pcre_uchar *codestart, *code; +BOOL utf; + +unsigned int options = re->options; +int offset = re->name_table_offset; +int count = re->name_count; +int size = re->name_entry_size; + +if (re->magic_number != MAGIC_NUMBER) + { + offset = ((offset << 8) & 0xff00) | ((offset >> 8) & 0xff); + count = ((count << 8) & 0xff00) | ((count >> 8) & 0xff); + size = ((size << 8) & 0xff00) | ((size >> 8) & 0xff); + options = ((options << 24) & 0xff000000) | + ((options << 8) & 0x00ff0000) | + ((options >> 8) & 0x0000ff00) | + ((options >> 24) & 0x000000ff); + } + +code = codestart = (pcre_uchar *)re + offset + count * size; +/* PCRE_UTF(16|32) have the same value as PCRE_UTF8. */ +utf = (options & PCRE_UTF8) != 0; + +for(;;) + { + pcre_uchar *ccode; + const char *flag = " "; + pcre_uint32 c; + unsigned int extra = 0; + + if (print_lengths) + fprintf(f, "%3d ", (int)(code - codestart)); + else + fprintf(f, " "); + + switch(*code) + { +/* ========================================================================== */ + /* These cases are never obeyed. This is a fudge that causes a compile- + time error if the vectors OP_names or OP_lengths, which are indexed + by opcode, are not the correct length. It seems to be the only way to do + such a check at compile time, as the sizeof() operator does not work in + the C preprocessor. */ + + case OP_TABLE_LENGTH: + case OP_TABLE_LENGTH + + ((sizeof(priv_OP_names)/sizeof(const char *) == OP_TABLE_LENGTH) && + (sizeof(priv_OP_lengths) == OP_TABLE_LENGTH)): + break; +/* ========================================================================== */ + + case OP_END: + fprintf(f, " %s\n", priv_OP_names[*code]); + fprintf(f, "------------------------------------------------------------------\n"); + return; + + case OP_CHAR: + fprintf(f, " "); + do + { + code++; + code += 1 + print_char(f, code, utf); + } + while (*code == OP_CHAR); + fprintf(f, "\n"); + continue; + + case OP_CHARI: + fprintf(f, " /i "); + do + { + code++; + code += 1 + print_char(f, code, utf); + } + while (*code == OP_CHARI); + fprintf(f, "\n"); + continue; + + case OP_CBRA: + case OP_CBRAPOS: + case OP_SCBRA: + case OP_SCBRAPOS: + if (print_lengths) fprintf(f, "%3d ", GET(code, 1)); + else fprintf(f, " "); + fprintf(f, "%s %d", priv_OP_names[*code], GET2(code, 1+LINK_SIZE)); + break; + + case OP_BRA: + case OP_BRAPOS: + case OP_SBRA: + case OP_SBRAPOS: + case OP_KETRMAX: + case OP_KETRMIN: + case OP_KETRPOS: + case OP_ALT: + case OP_KET: + case OP_ASSERT: + case OP_ASSERT_NOT: + case OP_ASSERTBACK: + case OP_ASSERTBACK_NOT: + case OP_ONCE: + case OP_ONCE_NC: + case OP_COND: + case OP_SCOND: + case OP_REVERSE: + if (print_lengths) fprintf(f, "%3d ", GET(code, 1)); + else fprintf(f, " "); + fprintf(f, "%s", priv_OP_names[*code]); + break; + + case OP_CLOSE: + fprintf(f, " %s %d", priv_OP_names[*code], GET2(code, 1)); + break; + + case OP_CREF: + fprintf(f, "%3d %s", GET2(code,1), priv_OP_names[*code]); + break; + + case OP_DNCREF: + { + pcre_uchar *entry = (pcre_uchar *)re + offset + (GET2(code, 1) * size) + + IMM2_SIZE; + fprintf(f, " %s Cond ref <", flag); + print_puchar(f, entry); + fprintf(f, ">%d", GET2(code, 1 + IMM2_SIZE)); + } + break; + + case OP_RREF: + c = GET2(code, 1); + if (c == RREF_ANY) + fprintf(f, " Cond recurse any"); + else + fprintf(f, " Cond recurse %d", c); + break; + + case OP_DNRREF: + { + pcre_uchar *entry = (pcre_uchar *)re + offset + (GET2(code, 1) * size) + + IMM2_SIZE; + fprintf(f, " %s Cond recurse <", flag); + print_puchar(f, entry); + fprintf(f, ">%d", GET2(code, 1 + IMM2_SIZE)); + } + break; + + case OP_DEF: + fprintf(f, " Cond def"); + break; + + case OP_STARI: + case OP_MINSTARI: + case OP_POSSTARI: + case OP_PLUSI: + case OP_MINPLUSI: + case OP_POSPLUSI: + case OP_QUERYI: + case OP_MINQUERYI: + case OP_POSQUERYI: + flag = "/i"; + /* Fall through */ + case OP_STAR: + case OP_MINSTAR: + case OP_POSSTAR: + case OP_PLUS: + case OP_MINPLUS: + case OP_POSPLUS: + case OP_QUERY: + case OP_MINQUERY: + case OP_POSQUERY: + case OP_TYPESTAR: + case OP_TYPEMINSTAR: + case OP_TYPEPOSSTAR: + case OP_TYPEPLUS: + case OP_TYPEMINPLUS: + case OP_TYPEPOSPLUS: + case OP_TYPEQUERY: + case OP_TYPEMINQUERY: + case OP_TYPEPOSQUERY: + fprintf(f, " %s ", flag); + if (*code >= OP_TYPESTAR) + { + if (code[1] == OP_PROP || code[1] == OP_NOTPROP) + { + print_prop(f, code + 1, "", " "); + extra = 2; + } + else fprintf(f, "%s", priv_OP_names[code[1]]); + } + else extra = print_char(f, code+1, utf); + fprintf(f, "%s", priv_OP_names[*code]); + break; + + case OP_EXACTI: + case OP_UPTOI: + case OP_MINUPTOI: + case OP_POSUPTOI: + flag = "/i"; + /* Fall through */ + case OP_EXACT: + case OP_UPTO: + case OP_MINUPTO: + case OP_POSUPTO: + fprintf(f, " %s ", flag); + extra = print_char(f, code + 1 + IMM2_SIZE, utf); + fprintf(f, "{"); + if (*code != OP_EXACT && *code != OP_EXACTI) fprintf(f, "0,"); + fprintf(f, "%d}", GET2(code,1)); + if (*code == OP_MINUPTO || *code == OP_MINUPTOI) fprintf(f, "?"); + else if (*code == OP_POSUPTO || *code == OP_POSUPTOI) fprintf(f, "+"); + break; + + case OP_TYPEEXACT: + case OP_TYPEUPTO: + case OP_TYPEMINUPTO: + case OP_TYPEPOSUPTO: + if (code[1 + IMM2_SIZE] == OP_PROP || code[1 + IMM2_SIZE] == OP_NOTPROP) + { + print_prop(f, code + IMM2_SIZE + 1, " ", " "); + extra = 2; + } + else fprintf(f, " %s", priv_OP_names[code[1 + IMM2_SIZE]]); + fprintf(f, "{"); + if (*code != OP_TYPEEXACT) fprintf(f, "0,"); + fprintf(f, "%d}", GET2(code,1)); + if (*code == OP_TYPEMINUPTO) fprintf(f, "?"); + else if (*code == OP_TYPEPOSUPTO) fprintf(f, "+"); + break; + + case OP_NOTI: + flag = "/i"; + /* Fall through */ + case OP_NOT: + fprintf(f, " %s [^", flag); + extra = print_char(f, code + 1, utf); + fprintf(f, "]"); + break; + + case OP_NOTSTARI: + case OP_NOTMINSTARI: + case OP_NOTPOSSTARI: + case OP_NOTPLUSI: + case OP_NOTMINPLUSI: + case OP_NOTPOSPLUSI: + case OP_NOTQUERYI: + case OP_NOTMINQUERYI: + case OP_NOTPOSQUERYI: + flag = "/i"; + /* Fall through */ + + case OP_NOTSTAR: + case OP_NOTMINSTAR: + case OP_NOTPOSSTAR: + case OP_NOTPLUS: + case OP_NOTMINPLUS: + case OP_NOTPOSPLUS: + case OP_NOTQUERY: + case OP_NOTMINQUERY: + case OP_NOTPOSQUERY: + fprintf(f, " %s [^", flag); + extra = print_char(f, code + 1, utf); + fprintf(f, "]%s", priv_OP_names[*code]); + break; + + case OP_NOTEXACTI: + case OP_NOTUPTOI: + case OP_NOTMINUPTOI: + case OP_NOTPOSUPTOI: + flag = "/i"; + /* Fall through */ + + case OP_NOTEXACT: + case OP_NOTUPTO: + case OP_NOTMINUPTO: + case OP_NOTPOSUPTO: + fprintf(f, " %s [^", flag); + extra = print_char(f, code + 1 + IMM2_SIZE, utf); + fprintf(f, "]{"); + if (*code != OP_NOTEXACT && *code != OP_NOTEXACTI) fprintf(f, "0,"); + fprintf(f, "%d}", GET2(code,1)); + if (*code == OP_NOTMINUPTO || *code == OP_NOTMINUPTOI) fprintf(f, "?"); + else + if (*code == OP_NOTPOSUPTO || *code == OP_NOTPOSUPTOI) fprintf(f, "+"); + break; + + case OP_RECURSE: + if (print_lengths) fprintf(f, "%3d ", GET(code, 1)); + else fprintf(f, " "); + fprintf(f, "%s", priv_OP_names[*code]); + break; + + case OP_REFI: + flag = "/i"; + /* Fall through */ + case OP_REF: + fprintf(f, " %s \\%d", flag, GET2(code,1)); + ccode = code + priv_OP_lengths[*code]; + goto CLASS_REF_REPEAT; + + case OP_DNREFI: + flag = "/i"; + /* Fall through */ + case OP_DNREF: + { + pcre_uchar *entry = (pcre_uchar *)re + offset + (GET2(code, 1) * size) + + IMM2_SIZE; + fprintf(f, " %s \\k<", flag); + print_puchar(f, entry); + fprintf(f, ">%d", GET2(code, 1 + IMM2_SIZE)); + } + ccode = code + priv_OP_lengths[*code]; + goto CLASS_REF_REPEAT; + + case OP_CALLOUT: + fprintf(f, " %s %d %d %d", priv_OP_names[*code], code[1], GET(code,2), + GET(code, 2 + LINK_SIZE)); + break; + + case OP_PROP: + case OP_NOTPROP: + print_prop(f, code, " ", ""); + break; + + /* OP_XCLASS cannot occur in 8-bit, non-UTF mode. However, there's no harm + in having this code always here, and it makes it less messy without all + those #ifdefs. */ + + case OP_CLASS: + case OP_NCLASS: + case OP_XCLASS: + { + int i; + unsigned int min, max; + BOOL printmap; + BOOL invertmap = FALSE; + pcre_uint8 *map; + pcre_uint8 inverted_map[32]; + + fprintf(f, " ["); + + if (*code == OP_XCLASS) + { + extra = GET(code, 1); + ccode = code + LINK_SIZE + 1; + printmap = (*ccode & XCL_MAP) != 0; + if ((*ccode & XCL_NOT) != 0) + { + invertmap = (*ccode & XCL_HASPROP) == 0; + fprintf(f, "^"); + } + ccode++; + } + else + { + printmap = TRUE; + ccode = code + 1; + } + + /* Print a bit map */ + + if (printmap) + { + map = (pcre_uint8 *)ccode; + if (invertmap) + { + for (i = 0; i < 32; i++) inverted_map[i] = ~map[i]; + map = inverted_map; + } + + for (i = 0; i < 256; i++) + { + if ((map[i/8] & (1 << (i&7))) != 0) + { + int j; + for (j = i+1; j < 256; j++) + if ((map[j/8] & (1 << (j&7))) == 0) break; + if (i == '-' || i == ']') fprintf(f, "\\"); + if (PRINTABLE(i)) fprintf(f, "%c", i); + else fprintf(f, "\\x%02x", i); + if (--j > i) + { + if (j != i + 1) fprintf(f, "-"); + if (j == '-' || j == ']') fprintf(f, "\\"); + if (PRINTABLE(j)) fprintf(f, "%c", j); + else fprintf(f, "\\x%02x", j); + } + i = j; + } + } + ccode += 32 / sizeof(pcre_uchar); + } + + /* For an XCLASS there is always some additional data */ + + if (*code == OP_XCLASS) + { + pcre_uchar ch; + while ((ch = *ccode++) != XCL_END) + { + BOOL not = FALSE; + const char *notch = ""; + + switch(ch) + { + case XCL_NOTPROP: + not = TRUE; + notch = "^"; + /* Fall through */ + + case XCL_PROP: + { + unsigned int ptype = *ccode++; + unsigned int pvalue = *ccode++; + + switch(ptype) + { + case PT_PXGRAPH: + fprintf(f, "[:%sgraph:]", notch); + break; + + case PT_PXPRINT: + fprintf(f, "[:%sprint:]", notch); + break; + + case PT_PXPUNCT: + fprintf(f, "[:%spunct:]", notch); + break; + + default: + fprintf(f, "\\%c{%s}", (not? 'P':'p'), + get_ucpname(ptype, pvalue)); + break; + } + } + break; + + default: + ccode += 1 + print_char(f, ccode, utf); + if (ch == XCL_RANGE) + { + fprintf(f, "-"); + ccode += 1 + print_char(f, ccode, utf); + } + break; + } + } + } + + /* Indicate a non-UTF class which was created by negation */ + + fprintf(f, "]%s", (*code == OP_NCLASS)? " (neg)" : ""); + + /* Handle repeats after a class or a back reference */ + + CLASS_REF_REPEAT: + switch(*ccode) + { + case OP_CRSTAR: + case OP_CRMINSTAR: + case OP_CRPLUS: + case OP_CRMINPLUS: + case OP_CRQUERY: + case OP_CRMINQUERY: + case OP_CRPOSSTAR: + case OP_CRPOSPLUS: + case OP_CRPOSQUERY: + fprintf(f, "%s", priv_OP_names[*ccode]); + extra += priv_OP_lengths[*ccode]; + break; + + case OP_CRRANGE: + case OP_CRMINRANGE: + case OP_CRPOSRANGE: + min = GET2(ccode,1); + max = GET2(ccode,1 + IMM2_SIZE); + if (max == 0) fprintf(f, "{%u,}", min); + else fprintf(f, "{%u,%u}", min, max); + if (*ccode == OP_CRMINRANGE) fprintf(f, "?"); + else if (*ccode == OP_CRPOSRANGE) fprintf(f, "+"); + extra += priv_OP_lengths[*ccode]; + break; + + /* Do nothing if it's not a repeat; this code stops picky compilers + warning about the lack of a default code path. */ + + default: + break; + } + } + break; + + case OP_MARK: + case OP_PRUNE_ARG: + case OP_SKIP_ARG: + case OP_THEN_ARG: + fprintf(f, " %s ", priv_OP_names[*code]); + print_puchar(f, code + 2); + extra += code[1]; + break; + + case OP_THEN: + fprintf(f, " %s", priv_OP_names[*code]); + break; + + case OP_CIRCM: + case OP_DOLLM: + flag = "/m"; + /* Fall through */ + + /* Anything else is just an item with no data, but possibly a flag. */ + + default: + fprintf(f, " %s %s", flag, priv_OP_names[*code]); + break; + } + + code += priv_OP_lengths[*code] + extra; + fprintf(f, "\n"); + } +} + +/* End of pcre_printint.src */ diff --git a/src/third_party/pcre-8.41/pcre_refcount.c b/src/third_party/pcre-8.41/pcre_refcount.c new file mode 100644 index 00000000000..79efa90f216 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_refcount.c @@ -0,0 +1,92 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This module contains the external function pcre_refcount(), which is an +auxiliary function that can be used to maintain a reference count in a compiled +pattern data block. This might be helpful in applications where the block is +shared by different users. */ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "pcre_internal.h" + + +/************************************************* +* Maintain reference count * +*************************************************/ + +/* The reference count is a 16-bit field, initialized to zero. It is not +possible to transfer a non-zero count from one host to a different host that +has a different byte order - though I can't see why anyone in their right mind +would ever want to do that! + +Arguments: + argument_re points to compiled code + adjust value to add to the count + +Returns: the (possibly updated) count value (a non-negative number), or + a negative error number +*/ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre_refcount(pcre *argument_re, int adjust) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre16_refcount(pcre16 *argument_re, int adjust) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN int PCRE_CALL_CONVENTION +pcre32_refcount(pcre32 *argument_re, int adjust) +#endif +{ +REAL_PCRE *re = (REAL_PCRE *)argument_re; +if (re == NULL) return PCRE_ERROR_NULL; +if (re->magic_number != MAGIC_NUMBER) return PCRE_ERROR_BADMAGIC; +if ((re->flags & PCRE_MODE) == 0) return PCRE_ERROR_BADMODE; +re->ref_count = (-adjust > re->ref_count)? 0 : + (adjust + re->ref_count > 65535)? 65535 : + re->ref_count + adjust; +return re->ref_count; +} + +/* End of pcre_refcount.c */ diff --git a/src/third_party/pcre-8.41/pcre_scanner.cc b/src/third_party/pcre-8.41/pcre_scanner.cc new file mode 100644 index 00000000000..6be2be6829b --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_scanner.cc @@ -0,0 +1,199 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Sanjay Ghemawat + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +#include "pcrecpp_internal.h" +#include "pcre_scanner.h" + +using std::vector; + +namespace pcrecpp { + +Scanner::Scanner() + : data_(), + input_(data_), + skip_(NULL), + should_skip_(false), + skip_repeat_(false), + save_comments_(false), + comments_(NULL), + comments_offset_(0) { +} + +Scanner::Scanner(const string& in) + : data_(in), + input_(data_), + skip_(NULL), + should_skip_(false), + skip_repeat_(false), + save_comments_(false), + comments_(NULL), + comments_offset_(0) { +} + +Scanner::~Scanner() { + delete skip_; + delete comments_; +} + +void Scanner::SetSkipExpression(const char* re) { + delete skip_; + if (re != NULL) { + skip_ = new RE(re); + should_skip_ = true; + skip_repeat_ = true; + ConsumeSkip(); + } else { + skip_ = NULL; + should_skip_ = false; + skip_repeat_ = false; + } +} + +void Scanner::Skip(const char* re) { + delete skip_; + if (re != NULL) { + skip_ = new RE(re); + should_skip_ = true; + skip_repeat_ = false; + ConsumeSkip(); + } else { + skip_ = NULL; + should_skip_ = false; + skip_repeat_ = false; + } +} + +void Scanner::DisableSkip() { + assert(skip_ != NULL); + should_skip_ = false; +} + +void Scanner::EnableSkip() { + assert(skip_ != NULL); + should_skip_ = true; + ConsumeSkip(); +} + +int Scanner::LineNumber() const { + // TODO: Make it more efficient by keeping track of the last point + // where we computed line numbers and counting newlines since then. + // We could use std:count, but not all systems have it. :-( + int count = 1; + for (const char* p = data_.data(); p < input_.data(); ++p) + if (*p == '\n') + ++count; + return count; +} + +int Scanner::Offset() const { + return (int)(input_.data() - data_.c_str()); +} + +bool Scanner::LookingAt(const RE& re) const { + int consumed; + return re.DoMatch(input_, RE::ANCHOR_START, &consumed, 0, 0); +} + + +bool Scanner::Consume(const RE& re, + const Arg& arg0, + const Arg& arg1, + const Arg& arg2) { + const bool result = re.Consume(&input_, arg0, arg1, arg2); + if (result && should_skip_) ConsumeSkip(); + return result; +} + +// helper function to consume *skip_ and honour save_comments_ +void Scanner::ConsumeSkip() { + const char* start_data = input_.data(); + while (skip_->Consume(&input_)) { + if (!skip_repeat_) { + // Only one skip allowed. + break; + } + } + if (save_comments_) { + if (comments_ == NULL) { + comments_ = new vector; + } + // already pointing one past end, so no need to +1 + int length = (int)(input_.data() - start_data); + if (length > 0) { + comments_->push_back(StringPiece(start_data, length)); + } + } +} + + +void Scanner::GetComments(int start, int end, vector *ranges) { + // short circuit out if we've not yet initialized comments_ + // (e.g., when save_comments is false) + if (!comments_) { + return; + } + // TODO: if we guarantee that comments_ will contain StringPieces + // that are ordered by their start, then we can do a binary search + // for the first StringPiece at or past start and then scan for the + // ones contained in the range, quit early (use equal_range or + // lower_bound) + for (vector::const_iterator it = comments_->begin(); + it != comments_->end(); ++it) { + if ((it->data() >= data_.c_str() + start && + it->data() + it->size() <= data_.c_str() + end)) { + ranges->push_back(*it); + } + } +} + + +void Scanner::GetNextComments(vector *ranges) { + // short circuit out if we've not yet initialized comments_ + // (e.g., when save_comments is false) + if (!comments_) { + return; + } + for (vector::const_iterator it = + comments_->begin() + comments_offset_; + it != comments_->end(); ++it) { + ranges->push_back(*it); + ++comments_offset_; + } +} + +} // namespace pcrecpp diff --git a/src/third_party/pcre-8.41/pcre_scanner.h b/src/third_party/pcre-8.41/pcre_scanner.h new file mode 100644 index 00000000000..5617e4515cb --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_scanner.h @@ -0,0 +1,172 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Sanjay Ghemawat +// +// Regular-expression based scanner for parsing an input stream. +// +// Example 1: parse a sequence of "var = number" entries from input: +// +// Scanner scanner(input); +// string var; +// int number; +// scanner.SetSkipExpression("\\s+"); // Skip any white space we encounter +// while (scanner.Consume("(\\w+) = (\\d+)", &var, &number)) { +// ...; +// } + +#ifndef _PCRE_SCANNER_H +#define _PCRE_SCANNER_H + +#include +#include +#include + +#include +#include + +namespace pcrecpp { + +class PCRECPP_EXP_DEFN Scanner { + public: + Scanner(); + explicit Scanner(const std::string& input); + ~Scanner(); + + // Return current line number. The returned line-number is + // one-based. I.e. it returns 1 + the number of consumed newlines. + // + // Note: this method may be slow. It may take time proportional to + // the size of the input. + int LineNumber() const; + + // Return the byte-offset that the scanner is looking in the + // input data; + int Offset() const; + + // Return true iff the start of the remaining input matches "re" + bool LookingAt(const RE& re) const; + + // Return true iff all of the following are true + // a. the start of the remaining input matches "re", + // b. if any arguments are supplied, matched sub-patterns can be + // parsed and stored into the arguments. + // If it returns true, it skips over the matched input and any + // following input that matches the "skip" regular expression. + bool Consume(const RE& re, + const Arg& arg0 = RE::no_arg, + const Arg& arg1 = RE::no_arg, + const Arg& arg2 = RE::no_arg + // TODO: Allow more arguments? + ); + + // Set the "skip" regular expression. If after consuming some data, + // a prefix of the input matches this RE, it is automatically + // skipped. For example, a programming language scanner would use + // a skip RE that matches white space and comments. + // + // scanner.SetSkipExpression("\\s+|//.*|/[*](.|\n)*?[*]/"); + // + // Skipping repeats as long as it succeeds. We used to let people do + // this by writing "(...)*" in the regular expression, but that added + // up to lots of recursive calls within the pcre library, so now we + // control repetition explicitly via the function call API. + // + // You can pass NULL for "re" if you do not want any data to be skipped. + void Skip(const char* re); // DEPRECATED; does *not* repeat + void SetSkipExpression(const char* re); + + // Temporarily pause "skip"ing. This + // Skip("Foo"); code ; DisableSkip(); code; EnableSkip() + // is similar to + // Skip("Foo"); code ; Skip(NULL); code ; Skip("Foo"); + // but avoids creating/deleting new RE objects. + void DisableSkip(); + + // Reenable previously paused skipping. Any prefix of the input + // that matches the skip pattern is immediately dropped. + void EnableSkip(); + + /***** Special wrappers around SetSkip() for some common idioms *****/ + + // Arranges to skip whitespace, C comments, C++ comments. + // The overall RE is a disjunction of the following REs: + // \\s whitespace + // //.*\n C++ comment + // /[*](.|\n)*?[*]/ C comment (x*? means minimal repetitions of x) + // We get repetition via the semantics of SetSkipExpression, not by using * + void SkipCXXComments() { + SetSkipExpression("\\s|//.*\n|/[*](?:\n|.)*?[*]/"); + } + + void set_save_comments(bool comments) { + save_comments_ = comments; + } + + bool save_comments() { + return save_comments_; + } + + // Append to vector ranges the comments found in the + // byte range [start,end] (inclusive) of the input data. + // Only comments that were extracted entirely within that + // range are returned: no range splitting of atomically-extracted + // comments is performed. + void GetComments(int start, int end, std::vector *ranges); + + // Append to vector ranges the comments added + // since the last time this was called. This + // functionality is provided for efficiency when + // interleaving scanning with parsing. + void GetNextComments(std::vector *ranges); + + private: + std::string data_; // All the input data + StringPiece input_; // Unprocessed input + RE* skip_; // If non-NULL, RE for skipping input + bool should_skip_; // If true, use skip_ + bool skip_repeat_; // If true, repeat skip_ as long as it works + bool save_comments_; // If true, aggregate the skip expression + + // the skipped comments + // TODO: later consider requiring that the StringPieces be added + // in order by their start position + std::vector *comments_; + + // the offset into comments_ that has been returned by GetNextComments + int comments_offset_; + + // helper function to consume *skip_ and honour + // save_comments_ + void ConsumeSkip(); +}; + +} // namespace pcrecpp + +#endif /* _PCRE_SCANNER_H */ diff --git a/src/third_party/pcre-8.41/pcre_scanner_unittest.cc b/src/third_party/pcre-8.41/pcre_scanner_unittest.cc new file mode 100644 index 00000000000..623e2afda80 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_scanner_unittest.cc @@ -0,0 +1,162 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Greg J. Badros +// +// Unittest for scanner, especially GetNextComments and GetComments() +// functionality. + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include /* for strchr */ +#include +#include + +#include "pcrecpp.h" +#include "pcre_stringpiece.h" +#include "pcre_scanner.h" + +#define FLAGS_unittest_stack_size 49152 + +// Dies with a fatal error if the two values are not equal. +#define CHECK_EQ(a, b) do { \ + if ( (a) != (b) ) { \ + fprintf(stderr, "%s:%d: Check failed because %s != %s\n", \ + __FILE__, __LINE__, #a, #b); \ + exit(1); \ + } \ +} while (0) + +using std::vector; +using std::string; +using pcrecpp::StringPiece; +using pcrecpp::Scanner; + +static void TestScanner() { + const char input[] = "\n" + "alpha = 1; // this sets alpha\n" + "bravo = 2; // bravo is set here\n" + "gamma = 33; /* and here is gamma */\n"; + + const char *re = "(\\w+) = (\\d+);"; + + Scanner s(input); + string var; + int number; + s.SkipCXXComments(); + s.set_save_comments(true); + vector comments; + + s.Consume(re, &var, &number); + CHECK_EQ(var, "alpha"); + CHECK_EQ(number, 1); + CHECK_EQ(s.LineNumber(), 3); + s.GetNextComments(&comments); + CHECK_EQ(comments.size(), 1); + CHECK_EQ(comments[0].as_string(), " // this sets alpha\n"); + comments.resize(0); + + s.Consume(re, &var, &number); + CHECK_EQ(var, "bravo"); + CHECK_EQ(number, 2); + s.GetNextComments(&comments); + CHECK_EQ(comments.size(), 1); + CHECK_EQ(comments[0].as_string(), " // bravo is set here\n"); + comments.resize(0); + + s.Consume(re, &var, &number); + CHECK_EQ(var, "gamma"); + CHECK_EQ(number, 33); + s.GetNextComments(&comments); + CHECK_EQ(comments.size(), 1); + CHECK_EQ(comments[0].as_string(), " /* and here is gamma */\n"); + comments.resize(0); + + s.GetComments(0, sizeof(input), &comments); + CHECK_EQ(comments.size(), 3); + CHECK_EQ(comments[0].as_string(), " // this sets alpha\n"); + CHECK_EQ(comments[1].as_string(), " // bravo is set here\n"); + CHECK_EQ(comments[2].as_string(), " /* and here is gamma */\n"); + comments.resize(0); + + s.GetComments(0, (int)(strchr(input, '/') - input), &comments); + CHECK_EQ(comments.size(), 0); + comments.resize(0); + + s.GetComments((int)(strchr(input, '/') - input - 1), sizeof(input), + &comments); + CHECK_EQ(comments.size(), 3); + CHECK_EQ(comments[0].as_string(), " // this sets alpha\n"); + CHECK_EQ(comments[1].as_string(), " // bravo is set here\n"); + CHECK_EQ(comments[2].as_string(), " /* and here is gamma */\n"); + comments.resize(0); + + s.GetComments((int)(strchr(input, '/') - input - 1), + (int)(strchr(input + 1, '\n') - input + 1), &comments); + CHECK_EQ(comments.size(), 1); + CHECK_EQ(comments[0].as_string(), " // this sets alpha\n"); + comments.resize(0); +} + +static void TestBigComment() { + string input; + for (int i = 0; i < 1024; ++i) { + char buf[1024]; // definitely big enough + sprintf(buf, " # Comment %d\n", i); + input += buf; + } + input += "name = value;\n"; + + Scanner s(input.c_str()); + s.SetSkipExpression("\\s+|#.*\n"); + + string name; + string value; + s.Consume("(\\w+) = (\\w+);", &name, &value); + CHECK_EQ(name, "name"); + CHECK_EQ(value, "value"); +} + +// TODO: also test scanner and big-comment in a thread with a +// small stack size + +int main(int argc, char** argv) { + (void)argc; + (void)argv; + TestScanner(); + TestBigComment(); + + // Done + printf("OK\n"); + + return 0; +} diff --git a/src/third_party/pcre-8.41/pcre_string_utils.c b/src/third_party/pcre-8.41/pcre_string_utils.c new file mode 100644 index 00000000000..25eacc85073 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_string_utils.c @@ -0,0 +1,211 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2014 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This module contains internal functions for comparing and finding the length +of strings for different data item sizes. */ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "pcre_internal.h" + +#ifndef COMPILE_PCRE8 + +/************************************************* +* Compare string utilities * +*************************************************/ + +/* The following two functions compares two strings. Basically a strcmp +for non 8 bit characters. + +Arguments: + str1 first string + str2 second string + +Returns: 0 if both string are equal (like strcmp), 1 otherwise +*/ + +int +PRIV(strcmp_uc_uc)(const pcre_uchar *str1, const pcre_uchar *str2) +{ +pcre_uchar c1; +pcre_uchar c2; + +while (*str1 != '\0' || *str2 != '\0') + { + c1 = *str1++; + c2 = *str2++; + if (c1 != c2) + return ((c1 > c2) << 1) - 1; + } +/* Both length and characters must be equal. */ +return 0; +} + +#ifdef COMPILE_PCRE32 + +int +PRIV(strcmp_uc_uc_utf)(const pcre_uchar *str1, const pcre_uchar *str2) +{ +pcre_uchar c1; +pcre_uchar c2; + +while (*str1 != '\0' || *str2 != '\0') + { + c1 = UCHAR21INC(str1); + c2 = UCHAR21INC(str2); + if (c1 != c2) + return ((c1 > c2) << 1) - 1; + } +/* Both length and characters must be equal. */ +return 0; +} + +#endif /* COMPILE_PCRE32 */ + +int +PRIV(strcmp_uc_c8)(const pcre_uchar *str1, const char *str2) +{ +const pcre_uint8 *ustr2 = (pcre_uint8 *)str2; +pcre_uchar c1; +pcre_uchar c2; + +while (*str1 != '\0' || *ustr2 != '\0') + { + c1 = *str1++; + c2 = (pcre_uchar)*ustr2++; + if (c1 != c2) + return ((c1 > c2) << 1) - 1; + } +/* Both length and characters must be equal. */ +return 0; +} + +#ifdef COMPILE_PCRE32 + +int +PRIV(strcmp_uc_c8_utf)(const pcre_uchar *str1, const char *str2) +{ +const pcre_uint8 *ustr2 = (pcre_uint8 *)str2; +pcre_uchar c1; +pcre_uchar c2; + +while (*str1 != '\0' || *ustr2 != '\0') + { + c1 = UCHAR21INC(str1); + c2 = (pcre_uchar)*ustr2++; + if (c1 != c2) + return ((c1 > c2) << 1) - 1; + } +/* Both length and characters must be equal. */ +return 0; +} + +#endif /* COMPILE_PCRE32 */ + +/* The following two functions compares two, fixed length +strings. Basically an strncmp for non 8 bit characters. + +Arguments: + str1 first string + str2 second string + num size of the string + +Returns: 0 if both string are equal (like strcmp), 1 otherwise +*/ + +int +PRIV(strncmp_uc_uc)(const pcre_uchar *str1, const pcre_uchar *str2, unsigned int num) +{ +pcre_uchar c1; +pcre_uchar c2; + +while (num-- > 0) + { + c1 = *str1++; + c2 = *str2++; + if (c1 != c2) + return ((c1 > c2) << 1) - 1; + } +/* Both length and characters must be equal. */ +return 0; +} + +int +PRIV(strncmp_uc_c8)(const pcre_uchar *str1, const char *str2, unsigned int num) +{ +const pcre_uint8 *ustr2 = (pcre_uint8 *)str2; +pcre_uchar c1; +pcre_uchar c2; + +while (num-- > 0) + { + c1 = *str1++; + c2 = (pcre_uchar)*ustr2++; + if (c1 != c2) + return ((c1 > c2) << 1) - 1; + } +/* Both length and characters must be equal. */ +return 0; +} + +/* The following function returns with the length of +a zero terminated string. Basically an strlen for non 8 bit characters. + +Arguments: + str string + +Returns: length of the string +*/ + +unsigned int +PRIV(strlen_uc)(const pcre_uchar *str) +{ +unsigned int len = 0; +while (*str++ != 0) + len++; +return len; +} + +#endif /* !COMPILE_PCRE8 */ + +/* End of pcre_string_utils.c */ diff --git a/src/third_party/pcre-8.41/pcre_stringpiece.cc b/src/third_party/pcre-8.41/pcre_stringpiece.cc new file mode 100644 index 00000000000..67c0f1fc0e5 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_stringpiece.cc @@ -0,0 +1,43 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wilsonh@google.com (Wilson Hsieh) +// + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "pcrecpp_internal.h" +#include "pcre_stringpiece.h" + +std::ostream& operator<<(std::ostream& o, const pcrecpp::StringPiece& piece) { + return (o << piece.as_string()); +} diff --git a/src/third_party/pcre-8.41/pcre_stringpiece.h b/src/third_party/pcre-8.41/pcre_stringpiece.h new file mode 100644 index 00000000000..cb94f52a010 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_stringpiece.h @@ -0,0 +1,180 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Sanjay Ghemawat +// +// A string like object that points into another piece of memory. +// Useful for providing an interface that allows clients to easily +// pass in either a "const char*" or a "string". +// +// Arghh! I wish C++ literals were automatically of type "string". + +#ifndef _PCRE_STRINGPIECE_H +#define _PCRE_STRINGPIECE_H + +#include +#include +#include // for ostream forward-declaration + +#if 0 +#define HAVE_TYPE_TRAITS +#include +#elif 0 +#define HAVE_TYPE_TRAITS +#include +#endif + +#include + +namespace pcrecpp { + +using std::memcmp; +using std::strlen; +using std::string; + +class PCRECPP_EXP_DEFN StringPiece { + private: + const char* ptr_; + int length_; + + public: + // We provide non-explicit singleton constructors so users can pass + // in a "const char*" or a "string" wherever a "StringPiece" is + // expected. + StringPiece() + : ptr_(NULL), length_(0) { } + StringPiece(const char* str) + : ptr_(str), length_(static_cast(strlen(ptr_))) { } + StringPiece(const unsigned char* str) + : ptr_(reinterpret_cast(str)), + length_(static_cast(strlen(ptr_))) { } + StringPiece(const string& str) + : ptr_(str.data()), length_(static_cast(str.size())) { } + StringPiece(const char* offset, int len) + : ptr_(offset), length_(len) { } + + // data() may return a pointer to a buffer with embedded NULs, and the + // returned buffer may or may not be null terminated. Therefore it is + // typically a mistake to pass data() to a routine that expects a NUL + // terminated string. Use "as_string().c_str()" if you really need to do + // this. Or better yet, change your routine so it does not rely on NUL + // termination. + const char* data() const { return ptr_; } + int size() const { return length_; } + bool empty() const { return length_ == 0; } + + void clear() { ptr_ = NULL; length_ = 0; } + void set(const char* buffer, int len) { ptr_ = buffer; length_ = len; } + void set(const char* str) { + ptr_ = str; + length_ = static_cast(strlen(str)); + } + void set(const void* buffer, int len) { + ptr_ = reinterpret_cast(buffer); + length_ = len; + } + + char operator[](int i) const { return ptr_[i]; } + + void remove_prefix(int n) { + ptr_ += n; + length_ -= n; + } + + void remove_suffix(int n) { + length_ -= n; + } + + bool operator==(const StringPiece& x) const { + return ((length_ == x.length_) && + (memcmp(ptr_, x.ptr_, length_) == 0)); + } + bool operator!=(const StringPiece& x) const { + return !(*this == x); + } + +#define STRINGPIECE_BINARY_PREDICATE(cmp,auxcmp) \ + bool operator cmp (const StringPiece& x) const { \ + int r = memcmp(ptr_, x.ptr_, length_ < x.length_ ? length_ : x.length_); \ + return ((r auxcmp 0) || ((r == 0) && (length_ cmp x.length_))); \ + } + STRINGPIECE_BINARY_PREDICATE(<, <); + STRINGPIECE_BINARY_PREDICATE(<=, <); + STRINGPIECE_BINARY_PREDICATE(>=, >); + STRINGPIECE_BINARY_PREDICATE(>, >); +#undef STRINGPIECE_BINARY_PREDICATE + + int compare(const StringPiece& x) const { + int r = memcmp(ptr_, x.ptr_, length_ < x.length_ ? length_ : x.length_); + if (r == 0) { + if (length_ < x.length_) r = -1; + else if (length_ > x.length_) r = +1; + } + return r; + } + + string as_string() const { + return string(data(), size()); + } + + void CopyToString(string* target) const { + target->assign(ptr_, length_); + } + + // Does "this" start with "x" + bool starts_with(const StringPiece& x) const { + return ((length_ >= x.length_) && (memcmp(ptr_, x.ptr_, x.length_) == 0)); + } +}; + +} // namespace pcrecpp + +// ------------------------------------------------------------------ +// Functions used to create STL containers that use StringPiece +// Remember that a StringPiece's lifetime had better be less than +// that of the underlying string or char*. If it is not, then you +// cannot safely store a StringPiece into an STL container +// ------------------------------------------------------------------ + +#ifdef HAVE_TYPE_TRAITS +// This makes vector really fast for some STL implementations +template<> struct __type_traits { + typedef __true_type has_trivial_default_constructor; + typedef __true_type has_trivial_copy_constructor; + typedef __true_type has_trivial_assignment_operator; + typedef __true_type has_trivial_destructor; + typedef __true_type is_POD_type; +}; +#endif + +// allow StringPiece to be logged +PCRECPP_EXP_DECL std::ostream& operator<<(std::ostream& o, + const pcrecpp::StringPiece& piece); + +#endif /* _PCRE_STRINGPIECE_H */ diff --git a/src/third_party/pcre-8.41/pcre_stringpiece.h.in b/src/third_party/pcre-8.41/pcre_stringpiece.h.in new file mode 100644 index 00000000000..f54f3f3b31b --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_stringpiece.h.in @@ -0,0 +1,180 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Sanjay Ghemawat +// +// A string like object that points into another piece of memory. +// Useful for providing an interface that allows clients to easily +// pass in either a "const char*" or a "string". +// +// Arghh! I wish C++ literals were automatically of type "string". + +#ifndef _PCRE_STRINGPIECE_H +#define _PCRE_STRINGPIECE_H + +#include +#include +#include // for ostream forward-declaration + +#if @pcre_have_type_traits@ +#define HAVE_TYPE_TRAITS +#include +#elif @pcre_have_bits_type_traits@ +#define HAVE_TYPE_TRAITS +#include +#endif + +#include + +namespace pcrecpp { + +using std::memcmp; +using std::strlen; +using std::string; + +class PCRECPP_EXP_DEFN StringPiece { + private: + const char* ptr_; + int length_; + + public: + // We provide non-explicit singleton constructors so users can pass + // in a "const char*" or a "string" wherever a "StringPiece" is + // expected. + StringPiece() + : ptr_(NULL), length_(0) { } + StringPiece(const char* str) + : ptr_(str), length_(static_cast(strlen(ptr_))) { } + StringPiece(const unsigned char* str) + : ptr_(reinterpret_cast(str)), + length_(static_cast(strlen(ptr_))) { } + StringPiece(const string& str) + : ptr_(str.data()), length_(static_cast(str.size())) { } + StringPiece(const char* offset, int len) + : ptr_(offset), length_(len) { } + + // data() may return a pointer to a buffer with embedded NULs, and the + // returned buffer may or may not be null terminated. Therefore it is + // typically a mistake to pass data() to a routine that expects a NUL + // terminated string. Use "as_string().c_str()" if you really need to do + // this. Or better yet, change your routine so it does not rely on NUL + // termination. + const char* data() const { return ptr_; } + int size() const { return length_; } + bool empty() const { return length_ == 0; } + + void clear() { ptr_ = NULL; length_ = 0; } + void set(const char* buffer, int len) { ptr_ = buffer; length_ = len; } + void set(const char* str) { + ptr_ = str; + length_ = static_cast(strlen(str)); + } + void set(const void* buffer, int len) { + ptr_ = reinterpret_cast(buffer); + length_ = len; + } + + char operator[](int i) const { return ptr_[i]; } + + void remove_prefix(int n) { + ptr_ += n; + length_ -= n; + } + + void remove_suffix(int n) { + length_ -= n; + } + + bool operator==(const StringPiece& x) const { + return ((length_ == x.length_) && + (memcmp(ptr_, x.ptr_, length_) == 0)); + } + bool operator!=(const StringPiece& x) const { + return !(*this == x); + } + +#define STRINGPIECE_BINARY_PREDICATE(cmp,auxcmp) \ + bool operator cmp (const StringPiece& x) const { \ + int r = memcmp(ptr_, x.ptr_, length_ < x.length_ ? length_ : x.length_); \ + return ((r auxcmp 0) || ((r == 0) && (length_ cmp x.length_))); \ + } + STRINGPIECE_BINARY_PREDICATE(<, <); + STRINGPIECE_BINARY_PREDICATE(<=, <); + STRINGPIECE_BINARY_PREDICATE(>=, >); + STRINGPIECE_BINARY_PREDICATE(>, >); +#undef STRINGPIECE_BINARY_PREDICATE + + int compare(const StringPiece& x) const { + int r = memcmp(ptr_, x.ptr_, length_ < x.length_ ? length_ : x.length_); + if (r == 0) { + if (length_ < x.length_) r = -1; + else if (length_ > x.length_) r = +1; + } + return r; + } + + string as_string() const { + return string(data(), size()); + } + + void CopyToString(string* target) const { + target->assign(ptr_, length_); + } + + // Does "this" start with "x" + bool starts_with(const StringPiece& x) const { + return ((length_ >= x.length_) && (memcmp(ptr_, x.ptr_, x.length_) == 0)); + } +}; + +} // namespace pcrecpp + +// ------------------------------------------------------------------ +// Functions used to create STL containers that use StringPiece +// Remember that a StringPiece's lifetime had better be less than +// that of the underlying string or char*. If it is not, then you +// cannot safely store a StringPiece into an STL container +// ------------------------------------------------------------------ + +#ifdef HAVE_TYPE_TRAITS +// This makes vector really fast for some STL implementations +template<> struct __type_traits { + typedef __true_type has_trivial_default_constructor; + typedef __true_type has_trivial_copy_constructor; + typedef __true_type has_trivial_assignment_operator; + typedef __true_type has_trivial_destructor; + typedef __true_type is_POD_type; +}; +#endif + +// allow StringPiece to be logged +PCRECPP_EXP_DECL std::ostream& operator<<(std::ostream& o, + const pcrecpp::StringPiece& piece); + +#endif /* _PCRE_STRINGPIECE_H */ diff --git a/src/third_party/pcre-8.41/pcre_stringpiece_unittest.cc b/src/third_party/pcre-8.41/pcre_stringpiece_unittest.cc new file mode 100644 index 00000000000..88e73a1f976 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_stringpiece_unittest.cc @@ -0,0 +1,153 @@ +// Copyright 2003 and onwards Google Inc. +// Author: Sanjay Ghemawat + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include // for make_pair + +#include "pcrecpp.h" +#include "pcre_stringpiece.h" + +// CHECK dies with a fatal error if condition is not true. It is *not* +// controlled by NDEBUG, so the check will be executed regardless of +// compilation mode. Therefore, it is safe to do things like: +// CHECK(fp->Write(x) == 4) +#define CHECK(condition) do { \ + if (!(condition)) { \ + fprintf(stderr, "%s:%d: Check failed: %s\n", \ + __FILE__, __LINE__, #condition); \ + exit(1); \ + } \ +} while (0) + +using std::string; +using pcrecpp::StringPiece; + +static void CheckSTLComparator() { + string s1("foo"); + string s2("bar"); + string s3("baz"); + + StringPiece p1(s1); + StringPiece p2(s2); + StringPiece p3(s3); + + typedef std::map TestMap; + TestMap map; + + map.insert(std::make_pair(p1, 0)); + map.insert(std::make_pair(p2, 1)); + map.insert(std::make_pair(p3, 2)); + + CHECK(map.size() == 3); + + TestMap::const_iterator iter = map.begin(); + CHECK(iter->second == 1); + ++iter; + CHECK(iter->second == 2); + ++iter; + CHECK(iter->second == 0); + ++iter; + CHECK(iter == map.end()); + + TestMap::iterator new_iter = map.find("zot"); + CHECK(new_iter == map.end()); + + new_iter = map.find("bar"); + CHECK(new_iter != map.end()); + + map.erase(new_iter); + CHECK(map.size() == 2); + + iter = map.begin(); + CHECK(iter->second == 2); + ++iter; + CHECK(iter->second == 0); + ++iter; + CHECK(iter == map.end()); +} + +static void CheckComparisonOperators() { +#define CMP_Y(op, x, y) \ + CHECK( (StringPiece((x)) op StringPiece((y)))); \ + CHECK( (StringPiece((x)).compare(StringPiece((y))) op 0)) + +#define CMP_N(op, x, y) \ + CHECK(!(StringPiece((x)) op StringPiece((y)))); \ + CHECK(!(StringPiece((x)).compare(StringPiece((y))) op 0)) + + CMP_Y(==, "", ""); + CMP_Y(==, "a", "a"); + CMP_Y(==, "aa", "aa"); + CMP_N(==, "a", ""); + CMP_N(==, "", "a"); + CMP_N(==, "a", "b"); + CMP_N(==, "a", "aa"); + CMP_N(==, "aa", "a"); + + CMP_N(!=, "", ""); + CMP_N(!=, "a", "a"); + CMP_N(!=, "aa", "aa"); + CMP_Y(!=, "a", ""); + CMP_Y(!=, "", "a"); + CMP_Y(!=, "a", "b"); + CMP_Y(!=, "a", "aa"); + CMP_Y(!=, "aa", "a"); + + CMP_Y(<, "a", "b"); + CMP_Y(<, "a", "aa"); + CMP_Y(<, "aa", "b"); + CMP_Y(<, "aa", "bb"); + CMP_N(<, "a", "a"); + CMP_N(<, "b", "a"); + CMP_N(<, "aa", "a"); + CMP_N(<, "b", "aa"); + CMP_N(<, "bb", "aa"); + + CMP_Y(<=, "a", "a"); + CMP_Y(<=, "a", "b"); + CMP_Y(<=, "a", "aa"); + CMP_Y(<=, "aa", "b"); + CMP_Y(<=, "aa", "bb"); + CMP_N(<=, "b", "a"); + CMP_N(<=, "aa", "a"); + CMP_N(<=, "b", "aa"); + CMP_N(<=, "bb", "aa"); + + CMP_N(>=, "a", "b"); + CMP_N(>=, "a", "aa"); + CMP_N(>=, "aa", "b"); + CMP_N(>=, "aa", "bb"); + CMP_Y(>=, "a", "a"); + CMP_Y(>=, "b", "a"); + CMP_Y(>=, "aa", "a"); + CMP_Y(>=, "b", "aa"); + CMP_Y(>=, "bb", "aa"); + + CMP_N(>, "a", "a"); + CMP_N(>, "a", "b"); + CMP_N(>, "a", "aa"); + CMP_N(>, "aa", "b"); + CMP_N(>, "aa", "bb"); + CMP_Y(>, "b", "a"); + CMP_Y(>, "aa", "a"); + CMP_Y(>, "b", "aa"); + CMP_Y(>, "bb", "aa"); + +#undef CMP_Y +#undef CMP_N +} + +int main(int argc, char** argv) { + (void)argc; + (void)argv; + CheckComparisonOperators(); + CheckSTLComparator(); + + printf("OK\n"); + return 0; +} diff --git a/src/third_party/pcre-8.41/pcre_study.c b/src/third_party/pcre-8.41/pcre_study.c new file mode 100644 index 00000000000..d9d4960d84e --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_study.c @@ -0,0 +1,1686 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This module contains the external function pcre_study(), along with local +supporting functions. */ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "pcre_internal.h" + +#define SET_BIT(c) start_bits[c/8] |= (1 << (c&7)) + +/* Returns from set_start_bits() */ + +enum { SSB_FAIL, SSB_DONE, SSB_CONTINUE, SSB_UNKNOWN }; + + + +/************************************************* +* Find the minimum subject length for a group * +*************************************************/ + +/* Scan a parenthesized group and compute the minimum length of subject that +is needed to match it. This is a lower bound; it does not mean there is a +string of that length that matches. In UTF8 mode, the result is in characters +rather than bytes. + +Arguments: + re compiled pattern block + code pointer to start of group (the bracket) + startcode pointer to start of the whole pattern's code + options the compiling options + recurses chain of recurse_check to catch mutual recursion + countptr pointer to call count (to catch over complexity) + +Returns: the minimum length + -1 if \C in UTF-8 mode or (*ACCEPT) was encountered + -2 internal error (missing capturing bracket) + -3 internal error (opcode not listed) +*/ + +static int +find_minlength(const REAL_PCRE *re, const pcre_uchar *code, + const pcre_uchar *startcode, int options, recurse_check *recurses, + int *countptr) +{ +int length = -1; +/* PCRE_UTF16 has the same value as PCRE_UTF8. */ +BOOL utf = (options & PCRE_UTF8) != 0; +BOOL had_recurse = FALSE; +recurse_check this_recurse; +register int branchlength = 0; +register pcre_uchar *cc = (pcre_uchar *)code + 1 + LINK_SIZE; + +if ((*countptr)++ > 1000) return -1; /* too complex */ + +if (*code == OP_CBRA || *code == OP_SCBRA || + *code == OP_CBRAPOS || *code == OP_SCBRAPOS) cc += IMM2_SIZE; + +/* Scan along the opcodes for this branch. If we get to the end of the +branch, check the length against that of the other branches. */ + +for (;;) + { + int d, min; + pcre_uchar *cs, *ce; + register pcre_uchar op = *cc; + + switch (op) + { + case OP_COND: + case OP_SCOND: + + /* If there is only one branch in a condition, the implied branch has zero + length, so we don't add anything. This covers the DEFINE "condition" + automatically. */ + + cs = cc + GET(cc, 1); + if (*cs != OP_ALT) + { + cc = cs + 1 + LINK_SIZE; + break; + } + + /* Otherwise we can fall through and treat it the same as any other + subpattern. */ + + case OP_CBRA: + case OP_SCBRA: + case OP_BRA: + case OP_SBRA: + case OP_CBRAPOS: + case OP_SCBRAPOS: + case OP_BRAPOS: + case OP_SBRAPOS: + case OP_ONCE: + case OP_ONCE_NC: + d = find_minlength(re, cc, startcode, options, recurses, countptr); + if (d < 0) return d; + branchlength += d; + do cc += GET(cc, 1); while (*cc == OP_ALT); + cc += 1 + LINK_SIZE; + break; + + /* ACCEPT makes things far too complicated; we have to give up. */ + + case OP_ACCEPT: + case OP_ASSERT_ACCEPT: + return -1; + + /* Reached end of a branch; if it's a ket it is the end of a nested + call. If it's ALT it is an alternation in a nested call. If it is END it's + the end of the outer call. All can be handled by the same code. If an + ACCEPT was previously encountered, use the length that was in force at that + time, and pass back the shortest ACCEPT length. */ + + case OP_ALT: + case OP_KET: + case OP_KETRMAX: + case OP_KETRMIN: + case OP_KETRPOS: + case OP_END: + if (length < 0 || (!had_recurse && branchlength < length)) + length = branchlength; + if (op != OP_ALT) return length; + cc += 1 + LINK_SIZE; + branchlength = 0; + had_recurse = FALSE; + break; + + /* Skip over assertive subpatterns */ + + case OP_ASSERT: + case OP_ASSERT_NOT: + case OP_ASSERTBACK: + case OP_ASSERTBACK_NOT: + do cc += GET(cc, 1); while (*cc == OP_ALT); + /* Fall through */ + + /* Skip over things that don't match chars */ + + case OP_REVERSE: + case OP_CREF: + case OP_DNCREF: + case OP_RREF: + case OP_DNRREF: + case OP_DEF: + case OP_CALLOUT: + case OP_SOD: + case OP_SOM: + case OP_EOD: + case OP_EODN: + case OP_CIRC: + case OP_CIRCM: + case OP_DOLL: + case OP_DOLLM: + case OP_NOT_WORD_BOUNDARY: + case OP_WORD_BOUNDARY: + cc += PRIV(OP_lengths)[*cc]; + break; + + /* Skip over a subpattern that has a {0} or {0,x} quantifier */ + + case OP_BRAZERO: + case OP_BRAMINZERO: + case OP_BRAPOSZERO: + case OP_SKIPZERO: + cc += PRIV(OP_lengths)[*cc]; + do cc += GET(cc, 1); while (*cc == OP_ALT); + cc += 1 + LINK_SIZE; + break; + + /* Handle literal characters and + repetitions */ + + case OP_CHAR: + case OP_CHARI: + case OP_NOT: + case OP_NOTI: + case OP_PLUS: + case OP_PLUSI: + case OP_MINPLUS: + case OP_MINPLUSI: + case OP_POSPLUS: + case OP_POSPLUSI: + case OP_NOTPLUS: + case OP_NOTPLUSI: + case OP_NOTMINPLUS: + case OP_NOTMINPLUSI: + case OP_NOTPOSPLUS: + case OP_NOTPOSPLUSI: + branchlength++; + cc += 2; +#ifdef SUPPORT_UTF + if (utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); +#endif + break; + + case OP_TYPEPLUS: + case OP_TYPEMINPLUS: + case OP_TYPEPOSPLUS: + branchlength++; + cc += (cc[1] == OP_PROP || cc[1] == OP_NOTPROP)? 4 : 2; + break; + + /* Handle exact repetitions. The count is already in characters, but we + need to skip over a multibyte character in UTF8 mode. */ + + case OP_EXACT: + case OP_EXACTI: + case OP_NOTEXACT: + case OP_NOTEXACTI: + branchlength += GET2(cc,1); + cc += 2 + IMM2_SIZE; +#ifdef SUPPORT_UTF + if (utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); +#endif + break; + + case OP_TYPEEXACT: + branchlength += GET2(cc,1); + cc += 2 + IMM2_SIZE + ((cc[1 + IMM2_SIZE] == OP_PROP + || cc[1 + IMM2_SIZE] == OP_NOTPROP)? 2 : 0); + break; + + /* Handle single-char non-literal matchers */ + + case OP_PROP: + case OP_NOTPROP: + cc += 2; + /* Fall through */ + + case OP_NOT_DIGIT: + case OP_DIGIT: + case OP_NOT_WHITESPACE: + case OP_WHITESPACE: + case OP_NOT_WORDCHAR: + case OP_WORDCHAR: + case OP_ANY: + case OP_ALLANY: + case OP_EXTUNI: + case OP_HSPACE: + case OP_NOT_HSPACE: + case OP_VSPACE: + case OP_NOT_VSPACE: + branchlength++; + cc++; + break; + + /* "Any newline" might match two characters, but it also might match just + one. */ + + case OP_ANYNL: + branchlength += 1; + cc++; + break; + + /* The single-byte matcher means we can't proceed in UTF-8 mode. (In + non-UTF-8 mode \C will actually be turned into OP_ALLANY, so won't ever + appear, but leave the code, just in case.) */ + + case OP_ANYBYTE: +#ifdef SUPPORT_UTF + if (utf) return -1; +#endif + branchlength++; + cc++; + break; + + /* For repeated character types, we have to test for \p and \P, which have + an extra two bytes of parameters. */ + + case OP_TYPESTAR: + case OP_TYPEMINSTAR: + case OP_TYPEQUERY: + case OP_TYPEMINQUERY: + case OP_TYPEPOSSTAR: + case OP_TYPEPOSQUERY: + if (cc[1] == OP_PROP || cc[1] == OP_NOTPROP) cc += 2; + cc += PRIV(OP_lengths)[op]; + break; + + case OP_TYPEUPTO: + case OP_TYPEMINUPTO: + case OP_TYPEPOSUPTO: + if (cc[1 + IMM2_SIZE] == OP_PROP + || cc[1 + IMM2_SIZE] == OP_NOTPROP) cc += 2; + cc += PRIV(OP_lengths)[op]; + break; + + /* Check a class for variable quantification */ + + case OP_CLASS: + case OP_NCLASS: +#if defined SUPPORT_UTF || defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + case OP_XCLASS: + /* The original code caused an unsigned overflow in 64 bit systems, + so now we use a conditional statement. */ + if (op == OP_XCLASS) + cc += GET(cc, 1); + else + cc += PRIV(OP_lengths)[OP_CLASS]; +#else + cc += PRIV(OP_lengths)[OP_CLASS]; +#endif + + switch (*cc) + { + case OP_CRPLUS: + case OP_CRMINPLUS: + case OP_CRPOSPLUS: + branchlength++; + /* Fall through */ + + case OP_CRSTAR: + case OP_CRMINSTAR: + case OP_CRQUERY: + case OP_CRMINQUERY: + case OP_CRPOSSTAR: + case OP_CRPOSQUERY: + cc++; + break; + + case OP_CRRANGE: + case OP_CRMINRANGE: + case OP_CRPOSRANGE: + branchlength += GET2(cc,1); + cc += 1 + 2 * IMM2_SIZE; + break; + + default: + branchlength++; + break; + } + break; + + /* Backreferences and subroutine calls are treated in the same way: we find + the minimum length for the subpattern. A recursion, however, causes an + a flag to be set that causes the length of this branch to be ignored. The + logic is that a recursion can only make sense if there is another + alternation that stops the recursing. That will provide the minimum length + (when no recursion happens). A backreference within the group that it is + referencing behaves in the same way. + + If PCRE_JAVASCRIPT_COMPAT is set, a backreference to an unset bracket + matches an empty string (by default it causes a matching failure), so in + that case we must set the minimum length to zero. */ + + case OP_DNREF: /* Duplicate named pattern back reference */ + case OP_DNREFI: + if ((options & PCRE_JAVASCRIPT_COMPAT) == 0) + { + int count = GET2(cc, 1+IMM2_SIZE); + pcre_uchar *slot = (pcre_uchar *)re + + re->name_table_offset + GET2(cc, 1) * re->name_entry_size; + d = INT_MAX; + while (count-- > 0) + { + ce = cs = (pcre_uchar *)PRIV(find_bracket)(startcode, utf, GET2(slot, 0)); + if (cs == NULL) return -2; + do ce += GET(ce, 1); while (*ce == OP_ALT); + if (cc > cs && cc < ce) /* Simple recursion */ + { + d = 0; + had_recurse = TRUE; + break; + } + else + { + recurse_check *r = recurses; + for (r = recurses; r != NULL; r = r->prev) if (r->group == cs) break; + if (r != NULL) /* Mutual recursion */ + { + d = 0; + had_recurse = TRUE; + break; + } + else + { + int dd; + this_recurse.prev = recurses; + this_recurse.group = cs; + dd = find_minlength(re, cs, startcode, options, &this_recurse, + countptr); + if (dd < d) d = dd; + } + } + slot += re->name_entry_size; + } + } + else d = 0; + cc += 1 + 2*IMM2_SIZE; + goto REPEAT_BACK_REFERENCE; + + case OP_REF: /* Single back reference */ + case OP_REFI: + if ((options & PCRE_JAVASCRIPT_COMPAT) == 0) + { + ce = cs = (pcre_uchar *)PRIV(find_bracket)(startcode, utf, GET2(cc, 1)); + if (cs == NULL) return -2; + do ce += GET(ce, 1); while (*ce == OP_ALT); + if (cc > cs && cc < ce) /* Simple recursion */ + { + d = 0; + had_recurse = TRUE; + } + else + { + recurse_check *r = recurses; + for (r = recurses; r != NULL; r = r->prev) if (r->group == cs) break; + if (r != NULL) /* Mutual recursion */ + { + d = 0; + had_recurse = TRUE; + } + else + { + this_recurse.prev = recurses; + this_recurse.group = cs; + d = find_minlength(re, cs, startcode, options, &this_recurse, + countptr); + } + } + } + else d = 0; + cc += 1 + IMM2_SIZE; + + /* Handle repeated back references */ + + REPEAT_BACK_REFERENCE: + switch (*cc) + { + case OP_CRSTAR: + case OP_CRMINSTAR: + case OP_CRQUERY: + case OP_CRMINQUERY: + case OP_CRPOSSTAR: + case OP_CRPOSQUERY: + min = 0; + cc++; + break; + + case OP_CRPLUS: + case OP_CRMINPLUS: + case OP_CRPOSPLUS: + min = 1; + cc++; + break; + + case OP_CRRANGE: + case OP_CRMINRANGE: + case OP_CRPOSRANGE: + min = GET2(cc, 1); + cc += 1 + 2 * IMM2_SIZE; + break; + + default: + min = 1; + break; + } + + branchlength += min * d; + break; + + /* We can easily detect direct recursion, but not mutual recursion. This is + caught by a recursion depth count. */ + + case OP_RECURSE: + cs = ce = (pcre_uchar *)startcode + GET(cc, 1); + do ce += GET(ce, 1); while (*ce == OP_ALT); + if (cc > cs && cc < ce) /* Simple recursion */ + had_recurse = TRUE; + else + { + recurse_check *r = recurses; + for (r = recurses; r != NULL; r = r->prev) if (r->group == cs) break; + if (r != NULL) /* Mutual recursion */ + had_recurse = TRUE; + else + { + this_recurse.prev = recurses; + this_recurse.group = cs; + branchlength += find_minlength(re, cs, startcode, options, + &this_recurse, countptr); + } + } + cc += 1 + LINK_SIZE; + break; + + /* Anything else does not or need not match a character. We can get the + item's length from the table, but for those that can match zero occurrences + of a character, we must take special action for UTF-8 characters. As it + happens, the "NOT" versions of these opcodes are used at present only for + ASCII characters, so they could be omitted from this list. However, in + future that may change, so we include them here so as not to leave a + gotcha for a future maintainer. */ + + case OP_UPTO: + case OP_UPTOI: + case OP_NOTUPTO: + case OP_NOTUPTOI: + case OP_MINUPTO: + case OP_MINUPTOI: + case OP_NOTMINUPTO: + case OP_NOTMINUPTOI: + case OP_POSUPTO: + case OP_POSUPTOI: + case OP_NOTPOSUPTO: + case OP_NOTPOSUPTOI: + + case OP_STAR: + case OP_STARI: + case OP_NOTSTAR: + case OP_NOTSTARI: + case OP_MINSTAR: + case OP_MINSTARI: + case OP_NOTMINSTAR: + case OP_NOTMINSTARI: + case OP_POSSTAR: + case OP_POSSTARI: + case OP_NOTPOSSTAR: + case OP_NOTPOSSTARI: + + case OP_QUERY: + case OP_QUERYI: + case OP_NOTQUERY: + case OP_NOTQUERYI: + case OP_MINQUERY: + case OP_MINQUERYI: + case OP_NOTMINQUERY: + case OP_NOTMINQUERYI: + case OP_POSQUERY: + case OP_POSQUERYI: + case OP_NOTPOSQUERY: + case OP_NOTPOSQUERYI: + + cc += PRIV(OP_lengths)[op]; +#ifdef SUPPORT_UTF + if (utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); +#endif + break; + + /* Skip these, but we need to add in the name length. */ + + case OP_MARK: + case OP_PRUNE_ARG: + case OP_SKIP_ARG: + case OP_THEN_ARG: + cc += PRIV(OP_lengths)[op] + cc[1]; + break; + + /* The remaining opcodes are just skipped over. */ + + case OP_CLOSE: + case OP_COMMIT: + case OP_FAIL: + case OP_PRUNE: + case OP_SET_SOM: + case OP_SKIP: + case OP_THEN: + cc += PRIV(OP_lengths)[op]; + break; + + /* This should not occur: we list all opcodes explicitly so that when + new ones get added they are properly considered. */ + + default: + return -3; + } + } +/* Control never gets here */ +} + + + +/************************************************* +* Set a bit and maybe its alternate case * +*************************************************/ + +/* Given a character, set its first byte's bit in the table, and also the +corresponding bit for the other version of a letter if we are caseless. In +UTF-8 mode, for characters greater than 127, we can only do the caseless thing +when Unicode property support is available. + +Arguments: + start_bits points to the bit map + p points to the character + caseless the caseless flag + cd the block with char table pointers + utf TRUE for UTF-8 / UTF-16 / UTF-32 mode + +Returns: pointer after the character +*/ + +static const pcre_uchar * +set_table_bit(pcre_uint8 *start_bits, const pcre_uchar *p, BOOL caseless, + compile_data *cd, BOOL utf) +{ +pcre_uint32 c = *p; + +#ifdef COMPILE_PCRE8 +SET_BIT(c); + +#ifdef SUPPORT_UTF +if (utf && c > 127) + { + GETCHARINC(c, p); +#ifdef SUPPORT_UCP + if (caseless) + { + pcre_uchar buff[6]; + c = UCD_OTHERCASE(c); + (void)PRIV(ord2utf)(c, buff); + SET_BIT(buff[0]); + } +#endif /* Not SUPPORT_UCP */ + return p; + } +#else /* Not SUPPORT_UTF */ +(void)(utf); /* Stops warning for unused parameter */ +#endif /* SUPPORT_UTF */ + +/* Not UTF-8 mode, or character is less than 127. */ + +if (caseless && (cd->ctypes[c] & ctype_letter) != 0) SET_BIT(cd->fcc[c]); +return p + 1; +#endif /* COMPILE_PCRE8 */ + +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 +if (c > 0xff) + { + c = 0xff; + caseless = FALSE; + } +SET_BIT(c); + +#ifdef SUPPORT_UTF +if (utf && c > 127) + { + GETCHARINC(c, p); +#ifdef SUPPORT_UCP + if (caseless) + { + c = UCD_OTHERCASE(c); + if (c > 0xff) + c = 0xff; + SET_BIT(c); + } +#endif /* SUPPORT_UCP */ + return p; + } +#else /* Not SUPPORT_UTF */ +(void)(utf); /* Stops warning for unused parameter */ +#endif /* SUPPORT_UTF */ + +if (caseless && (cd->ctypes[c] & ctype_letter) != 0) SET_BIT(cd->fcc[c]); +return p + 1; +#endif +} + + + +/************************************************* +* Set bits for a positive character type * +*************************************************/ + +/* This function sets starting bits for a character type. In UTF-8 mode, we can +only do a direct setting for bytes less than 128, as otherwise there can be +confusion with bytes in the middle of UTF-8 characters. In a "traditional" +environment, the tables will only recognize ASCII characters anyway, but in at +least one Windows environment, some higher bytes bits were set in the tables. +So we deal with that case by considering the UTF-8 encoding. + +Arguments: + start_bits the starting bitmap + cbit type the type of character wanted + table_limit 32 for non-UTF-8; 16 for UTF-8 + cd the block with char table pointers + +Returns: nothing +*/ + +static void +set_type_bits(pcre_uint8 *start_bits, int cbit_type, unsigned int table_limit, + compile_data *cd) +{ +register pcre_uint32 c; +for (c = 0; c < table_limit; c++) start_bits[c] |= cd->cbits[c+cbit_type]; +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 +if (table_limit == 32) return; +for (c = 128; c < 256; c++) + { + if ((cd->cbits[c/8] & (1 << (c&7))) != 0) + { + pcre_uchar buff[6]; + (void)PRIV(ord2utf)(c, buff); + SET_BIT(buff[0]); + } + } +#endif +} + + +/************************************************* +* Set bits for a negative character type * +*************************************************/ + +/* This function sets starting bits for a negative character type such as \D. +In UTF-8 mode, we can only do a direct setting for bytes less than 128, as +otherwise there can be confusion with bytes in the middle of UTF-8 characters. +Unlike in the positive case, where we can set appropriate starting bits for +specific high-valued UTF-8 characters, in this case we have to set the bits for +all high-valued characters. The lowest is 0xc2, but we overkill by starting at +0xc0 (192) for simplicity. + +Arguments: + start_bits the starting bitmap + cbit type the type of character wanted + table_limit 32 for non-UTF-8; 16 for UTF-8 + cd the block with char table pointers + +Returns: nothing +*/ + +static void +set_nottype_bits(pcre_uint8 *start_bits, int cbit_type, unsigned int table_limit, + compile_data *cd) +{ +register pcre_uint32 c; +for (c = 0; c < table_limit; c++) start_bits[c] |= ~cd->cbits[c+cbit_type]; +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 +if (table_limit != 32) for (c = 24; c < 32; c++) start_bits[c] = 0xff; +#endif +} + + + +/************************************************* +* Create bitmap of starting bytes * +*************************************************/ + +/* This function scans a compiled unanchored expression recursively and +attempts to build a bitmap of the set of possible starting bytes. As time goes +by, we may be able to get more clever at doing this. The SSB_CONTINUE return is +useful for parenthesized groups in patterns such as (a*)b where the group +provides some optional starting bytes but scanning must continue at the outer +level to find at least one mandatory byte. At the outermost level, this +function fails unless the result is SSB_DONE. + +Arguments: + code points to an expression + start_bits points to a 32-byte table, initialized to 0 + utf TRUE if in UTF-8 / UTF-16 / UTF-32 mode + cd the block with char table pointers + +Returns: SSB_FAIL => Failed to find any starting bytes + SSB_DONE => Found mandatory starting bytes + SSB_CONTINUE => Found optional starting bytes + SSB_UNKNOWN => Hit an unrecognized opcode +*/ + +static int +set_start_bits(const pcre_uchar *code, pcre_uint8 *start_bits, BOOL utf, + compile_data *cd) +{ +register pcre_uint32 c; +int yield = SSB_DONE; +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 +int table_limit = utf? 16:32; +#else +int table_limit = 32; +#endif + +#if 0 +/* ========================================================================= */ +/* The following comment and code was inserted in January 1999. In May 2006, +when it was observed to cause compiler warnings about unused values, I took it +out again. If anybody is still using OS/2, they will have to put it back +manually. */ + +/* This next statement and the later reference to dummy are here in order to +trick the optimizer of the IBM C compiler for OS/2 into generating correct +code. Apparently IBM isn't going to fix the problem, and we would rather not +disable optimization (in this module it actually makes a big difference, and +the pcre module can use all the optimization it can get). */ + +volatile int dummy; +/* ========================================================================= */ +#endif + +do + { + BOOL try_next = TRUE; + const pcre_uchar *tcode = code + 1 + LINK_SIZE; + + if (*code == OP_CBRA || *code == OP_SCBRA || + *code == OP_CBRAPOS || *code == OP_SCBRAPOS) tcode += IMM2_SIZE; + + while (try_next) /* Loop for items in this branch */ + { + int rc; + + switch(*tcode) + { + /* If we reach something we don't understand, it means a new opcode has + been created that hasn't been added to this code. Hopefully this problem + will be discovered during testing. */ + + default: + return SSB_UNKNOWN; + + /* Fail for a valid opcode that implies no starting bits. */ + + case OP_ACCEPT: + case OP_ASSERT_ACCEPT: + case OP_ALLANY: + case OP_ANY: + case OP_ANYBYTE: + case OP_CIRC: + case OP_CIRCM: + case OP_CLOSE: + case OP_COMMIT: + case OP_COND: + case OP_CREF: + case OP_DEF: + case OP_DNCREF: + case OP_DNREF: + case OP_DNREFI: + case OP_DNRREF: + case OP_DOLL: + case OP_DOLLM: + case OP_END: + case OP_EOD: + case OP_EODN: + case OP_EXTUNI: + case OP_FAIL: + case OP_MARK: + case OP_NOT: + case OP_NOTEXACT: + case OP_NOTEXACTI: + case OP_NOTI: + case OP_NOTMINPLUS: + case OP_NOTMINPLUSI: + case OP_NOTMINQUERY: + case OP_NOTMINQUERYI: + case OP_NOTMINSTAR: + case OP_NOTMINSTARI: + case OP_NOTMINUPTO: + case OP_NOTMINUPTOI: + case OP_NOTPLUS: + case OP_NOTPLUSI: + case OP_NOTPOSPLUS: + case OP_NOTPOSPLUSI: + case OP_NOTPOSQUERY: + case OP_NOTPOSQUERYI: + case OP_NOTPOSSTAR: + case OP_NOTPOSSTARI: + case OP_NOTPOSUPTO: + case OP_NOTPOSUPTOI: + case OP_NOTPROP: + case OP_NOTQUERY: + case OP_NOTQUERYI: + case OP_NOTSTAR: + case OP_NOTSTARI: + case OP_NOTUPTO: + case OP_NOTUPTOI: + case OP_NOT_HSPACE: + case OP_NOT_VSPACE: + case OP_PRUNE: + case OP_PRUNE_ARG: + case OP_RECURSE: + case OP_REF: + case OP_REFI: + case OP_REVERSE: + case OP_RREF: + case OP_SCOND: + case OP_SET_SOM: + case OP_SKIP: + case OP_SKIP_ARG: + case OP_SOD: + case OP_SOM: + case OP_THEN: + case OP_THEN_ARG: + return SSB_FAIL; + + /* A "real" property test implies no starting bits, but the fake property + PT_CLIST identifies a list of characters. These lists are short, as they + are used for characters with more than one "other case", so there is no + point in recognizing them for OP_NOTPROP. */ + + case OP_PROP: + if (tcode[1] != PT_CLIST) return SSB_FAIL; + { + const pcre_uint32 *p = PRIV(ucd_caseless_sets) + tcode[2]; + while ((c = *p++) < NOTACHAR) + { +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 + if (utf) + { + pcre_uchar buff[6]; + (void)PRIV(ord2utf)(c, buff); + c = buff[0]; + } +#endif + if (c > 0xff) SET_BIT(0xff); else SET_BIT(c); + } + } + try_next = FALSE; + break; + + /* We can ignore word boundary tests. */ + + case OP_WORD_BOUNDARY: + case OP_NOT_WORD_BOUNDARY: + tcode++; + break; + + /* If we hit a bracket or a positive lookahead assertion, recurse to set + bits from within the subpattern. If it can't find anything, we have to + give up. If it finds some mandatory character(s), we are done for this + branch. Otherwise, carry on scanning after the subpattern. */ + + case OP_BRA: + case OP_SBRA: + case OP_CBRA: + case OP_SCBRA: + case OP_BRAPOS: + case OP_SBRAPOS: + case OP_CBRAPOS: + case OP_SCBRAPOS: + case OP_ONCE: + case OP_ONCE_NC: + case OP_ASSERT: + rc = set_start_bits(tcode, start_bits, utf, cd); + if (rc == SSB_FAIL || rc == SSB_UNKNOWN) return rc; + if (rc == SSB_DONE) try_next = FALSE; else + { + do tcode += GET(tcode, 1); while (*tcode == OP_ALT); + tcode += 1 + LINK_SIZE; + } + break; + + /* If we hit ALT or KET, it means we haven't found anything mandatory in + this branch, though we might have found something optional. For ALT, we + continue with the next alternative, but we have to arrange that the final + result from subpattern is SSB_CONTINUE rather than SSB_DONE. For KET, + return SSB_CONTINUE: if this is the top level, that indicates failure, + but after a nested subpattern, it causes scanning to continue. */ + + case OP_ALT: + yield = SSB_CONTINUE; + try_next = FALSE; + break; + + case OP_KET: + case OP_KETRMAX: + case OP_KETRMIN: + case OP_KETRPOS: + return SSB_CONTINUE; + + /* Skip over callout */ + + case OP_CALLOUT: + tcode += 2 + 2*LINK_SIZE; + break; + + /* Skip over lookbehind and negative lookahead assertions */ + + case OP_ASSERT_NOT: + case OP_ASSERTBACK: + case OP_ASSERTBACK_NOT: + do tcode += GET(tcode, 1); while (*tcode == OP_ALT); + tcode += 1 + LINK_SIZE; + break; + + /* BRAZERO does the bracket, but carries on. */ + + case OP_BRAZERO: + case OP_BRAMINZERO: + case OP_BRAPOSZERO: + rc = set_start_bits(++tcode, start_bits, utf, cd); + if (rc == SSB_FAIL || rc == SSB_UNKNOWN) return rc; +/* ========================================================================= + See the comment at the head of this function concerning the next line, + which was an old fudge for the benefit of OS/2. + dummy = 1; + ========================================================================= */ + do tcode += GET(tcode,1); while (*tcode == OP_ALT); + tcode += 1 + LINK_SIZE; + break; + + /* SKIPZERO skips the bracket. */ + + case OP_SKIPZERO: + tcode++; + do tcode += GET(tcode,1); while (*tcode == OP_ALT); + tcode += 1 + LINK_SIZE; + break; + + /* Single-char * or ? sets the bit and tries the next item */ + + case OP_STAR: + case OP_MINSTAR: + case OP_POSSTAR: + case OP_QUERY: + case OP_MINQUERY: + case OP_POSQUERY: + tcode = set_table_bit(start_bits, tcode + 1, FALSE, cd, utf); + break; + + case OP_STARI: + case OP_MINSTARI: + case OP_POSSTARI: + case OP_QUERYI: + case OP_MINQUERYI: + case OP_POSQUERYI: + tcode = set_table_bit(start_bits, tcode + 1, TRUE, cd, utf); + break; + + /* Single-char upto sets the bit and tries the next */ + + case OP_UPTO: + case OP_MINUPTO: + case OP_POSUPTO: + tcode = set_table_bit(start_bits, tcode + 1 + IMM2_SIZE, FALSE, cd, utf); + break; + + case OP_UPTOI: + case OP_MINUPTOI: + case OP_POSUPTOI: + tcode = set_table_bit(start_bits, tcode + 1 + IMM2_SIZE, TRUE, cd, utf); + break; + + /* At least one single char sets the bit and stops */ + + case OP_EXACT: + tcode += IMM2_SIZE; + /* Fall through */ + case OP_CHAR: + case OP_PLUS: + case OP_MINPLUS: + case OP_POSPLUS: + (void)set_table_bit(start_bits, tcode + 1, FALSE, cd, utf); + try_next = FALSE; + break; + + case OP_EXACTI: + tcode += IMM2_SIZE; + /* Fall through */ + case OP_CHARI: + case OP_PLUSI: + case OP_MINPLUSI: + case OP_POSPLUSI: + (void)set_table_bit(start_bits, tcode + 1, TRUE, cd, utf); + try_next = FALSE; + break; + + /* Special spacing and line-terminating items. These recognize specific + lists of characters. The difference between VSPACE and ANYNL is that the + latter can match the two-character CRLF sequence, but that is not + relevant for finding the first character, so their code here is + identical. */ + + case OP_HSPACE: + SET_BIT(CHAR_HT); + SET_BIT(CHAR_SPACE); +#ifdef SUPPORT_UTF + if (utf) + { +#ifdef COMPILE_PCRE8 + SET_BIT(0xC2); /* For U+00A0 */ + SET_BIT(0xE1); /* For U+1680, U+180E */ + SET_BIT(0xE2); /* For U+2000 - U+200A, U+202F, U+205F */ + SET_BIT(0xE3); /* For U+3000 */ +#elif defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + SET_BIT(0xA0); + SET_BIT(0xFF); /* For characters > 255 */ +#endif /* COMPILE_PCRE[8|16|32] */ + } + else +#endif /* SUPPORT_UTF */ + { +#ifndef EBCDIC + SET_BIT(0xA0); +#endif /* Not EBCDIC */ +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + SET_BIT(0xFF); /* For characters > 255 */ +#endif /* COMPILE_PCRE[16|32] */ + } + try_next = FALSE; + break; + + case OP_ANYNL: + case OP_VSPACE: + SET_BIT(CHAR_LF); + SET_BIT(CHAR_VT); + SET_BIT(CHAR_FF); + SET_BIT(CHAR_CR); +#ifdef SUPPORT_UTF + if (utf) + { +#ifdef COMPILE_PCRE8 + SET_BIT(0xC2); /* For U+0085 */ + SET_BIT(0xE2); /* For U+2028, U+2029 */ +#elif defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + SET_BIT(CHAR_NEL); + SET_BIT(0xFF); /* For characters > 255 */ +#endif /* COMPILE_PCRE[8|16|32] */ + } + else +#endif /* SUPPORT_UTF */ + { + SET_BIT(CHAR_NEL); +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + SET_BIT(0xFF); /* For characters > 255 */ +#endif + } + try_next = FALSE; + break; + + /* Single character types set the bits and stop. Note that if PCRE_UCP + is set, we do not see these op codes because \d etc are converted to + properties. Therefore, these apply in the case when only characters less + than 256 are recognized to match the types. */ + + case OP_NOT_DIGIT: + set_nottype_bits(start_bits, cbit_digit, table_limit, cd); + try_next = FALSE; + break; + + case OP_DIGIT: + set_type_bits(start_bits, cbit_digit, table_limit, cd); + try_next = FALSE; + break; + + /* The cbit_space table has vertical tab as whitespace; we no longer + have to play fancy tricks because Perl added VT to its whitespace at + release 5.18. PCRE added it at release 8.34. */ + + case OP_NOT_WHITESPACE: + set_nottype_bits(start_bits, cbit_space, table_limit, cd); + try_next = FALSE; + break; + + case OP_WHITESPACE: + set_type_bits(start_bits, cbit_space, table_limit, cd); + try_next = FALSE; + break; + + case OP_NOT_WORDCHAR: + set_nottype_bits(start_bits, cbit_word, table_limit, cd); + try_next = FALSE; + break; + + case OP_WORDCHAR: + set_type_bits(start_bits, cbit_word, table_limit, cd); + try_next = FALSE; + break; + + /* One or more character type fudges the pointer and restarts, knowing + it will hit a single character type and stop there. */ + + case OP_TYPEPLUS: + case OP_TYPEMINPLUS: + case OP_TYPEPOSPLUS: + tcode++; + break; + + case OP_TYPEEXACT: + tcode += 1 + IMM2_SIZE; + break; + + /* Zero or more repeats of character types set the bits and then + try again. */ + + case OP_TYPEUPTO: + case OP_TYPEMINUPTO: + case OP_TYPEPOSUPTO: + tcode += IMM2_SIZE; /* Fall through */ + + case OP_TYPESTAR: + case OP_TYPEMINSTAR: + case OP_TYPEPOSSTAR: + case OP_TYPEQUERY: + case OP_TYPEMINQUERY: + case OP_TYPEPOSQUERY: + switch(tcode[1]) + { + default: + case OP_ANY: + case OP_ALLANY: + return SSB_FAIL; + + case OP_HSPACE: + SET_BIT(CHAR_HT); + SET_BIT(CHAR_SPACE); +#ifdef SUPPORT_UTF + if (utf) + { +#ifdef COMPILE_PCRE8 + SET_BIT(0xC2); /* For U+00A0 */ + SET_BIT(0xE1); /* For U+1680, U+180E */ + SET_BIT(0xE2); /* For U+2000 - U+200A, U+202F, U+205F */ + SET_BIT(0xE3); /* For U+3000 */ +#elif defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + SET_BIT(0xA0); + SET_BIT(0xFF); /* For characters > 255 */ +#endif /* COMPILE_PCRE[8|16|32] */ + } + else +#endif /* SUPPORT_UTF */ +#ifndef EBCDIC + SET_BIT(0xA0); +#endif /* Not EBCDIC */ + break; + + case OP_ANYNL: + case OP_VSPACE: + SET_BIT(CHAR_LF); + SET_BIT(CHAR_VT); + SET_BIT(CHAR_FF); + SET_BIT(CHAR_CR); +#ifdef SUPPORT_UTF + if (utf) + { +#ifdef COMPILE_PCRE8 + SET_BIT(0xC2); /* For U+0085 */ + SET_BIT(0xE2); /* For U+2028, U+2029 */ +#elif defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + SET_BIT(CHAR_NEL); + SET_BIT(0xFF); /* For characters > 255 */ +#endif /* COMPILE_PCRE16 */ + } + else +#endif /* SUPPORT_UTF */ + SET_BIT(CHAR_NEL); + break; + + case OP_NOT_DIGIT: + set_nottype_bits(start_bits, cbit_digit, table_limit, cd); + break; + + case OP_DIGIT: + set_type_bits(start_bits, cbit_digit, table_limit, cd); + break; + + /* The cbit_space table has vertical tab as whitespace; we no longer + have to play fancy tricks because Perl added VT to its whitespace at + release 5.18. PCRE added it at release 8.34. */ + + case OP_NOT_WHITESPACE: + set_nottype_bits(start_bits, cbit_space, table_limit, cd); + break; + + case OP_WHITESPACE: + set_type_bits(start_bits, cbit_space, table_limit, cd); + break; + + case OP_NOT_WORDCHAR: + set_nottype_bits(start_bits, cbit_word, table_limit, cd); + break; + + case OP_WORDCHAR: + set_type_bits(start_bits, cbit_word, table_limit, cd); + break; + } + + tcode += 2; + break; + + /* Character class where all the information is in a bit map: set the + bits and either carry on or not, according to the repeat count. If it was + a negative class, and we are operating with UTF-8 characters, any byte + with a value >= 0xc4 is a potentially valid starter because it starts a + character with a value > 255. */ + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + case OP_XCLASS: + if ((tcode[1 + LINK_SIZE] & XCL_HASPROP) != 0) + return SSB_FAIL; + /* All bits are set. */ + if ((tcode[1 + LINK_SIZE] & XCL_MAP) == 0 && (tcode[1 + LINK_SIZE] & XCL_NOT) != 0) + return SSB_FAIL; +#endif + /* Fall through */ + + case OP_NCLASS: +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 + if (utf) + { + start_bits[24] |= 0xf0; /* Bits for 0xc4 - 0xc8 */ + memset(start_bits+25, 0xff, 7); /* Bits for 0xc9 - 0xff */ + } +#endif +#if defined COMPILE_PCRE16 || defined COMPILE_PCRE32 + SET_BIT(0xFF); /* For characters > 255 */ +#endif + /* Fall through */ + + case OP_CLASS: + { + pcre_uint8 *map; +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + map = NULL; + if (*tcode == OP_XCLASS) + { + if ((tcode[1 + LINK_SIZE] & XCL_MAP) != 0) + map = (pcre_uint8 *)(tcode + 1 + LINK_SIZE + 1); + tcode += GET(tcode, 1); + } + else +#endif + { + tcode++; + map = (pcre_uint8 *)tcode; + tcode += 32 / sizeof(pcre_uchar); + } + + /* In UTF-8 mode, the bits in a bit map correspond to character + values, not to byte values. However, the bit map we are constructing is + for byte values. So we have to do a conversion for characters whose + value is > 127. In fact, there are only two possible starting bytes for + characters in the range 128 - 255. */ + +#if defined SUPPORT_UTF || !defined COMPILE_PCRE8 + if (map != NULL) +#endif + { +#if defined SUPPORT_UTF && defined COMPILE_PCRE8 + if (utf) + { + for (c = 0; c < 16; c++) start_bits[c] |= map[c]; + for (c = 128; c < 256; c++) + { + if ((map[c/8] & (1 << (c&7))) != 0) + { + int d = (c >> 6) | 0xc0; /* Set bit for this starter */ + start_bits[d/8] |= (1 << (d&7)); /* and then skip on to the */ + c = (c & 0xc0) + 0x40 - 1; /* next relevant character. */ + } + } + } + else +#endif + { + /* In non-UTF-8 mode, the two bit maps are completely compatible. */ + for (c = 0; c < 32; c++) start_bits[c] |= map[c]; + } + } + + /* Advance past the bit map, and act on what follows. For a zero + minimum repeat, continue; otherwise stop processing. */ + + switch (*tcode) + { + case OP_CRSTAR: + case OP_CRMINSTAR: + case OP_CRQUERY: + case OP_CRMINQUERY: + case OP_CRPOSSTAR: + case OP_CRPOSQUERY: + tcode++; + break; + + case OP_CRRANGE: + case OP_CRMINRANGE: + case OP_CRPOSRANGE: + if (GET2(tcode, 1) == 0) tcode += 1 + 2 * IMM2_SIZE; + else try_next = FALSE; + break; + + default: + try_next = FALSE; + break; + } + } + break; /* End of bitmap class handling */ + + } /* End of switch */ + } /* End of try_next loop */ + + code += GET(code, 1); /* Advance to next branch */ + } +while (*code == OP_ALT); +return yield; +} + + + + + +/************************************************* +* Study a compiled expression * +*************************************************/ + +/* This function is handed a compiled expression that it must study to produce +information that will speed up the matching. It returns a pcre[16]_extra block +which then gets handed back to pcre_exec(). + +Arguments: + re points to the compiled expression + options contains option bits + errorptr points to where to place error messages; + set NULL unless error + +Returns: pointer to a pcre[16]_extra block, with study_data filled in and + the appropriate flags set; + NULL on error or if no optimization possible +*/ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN pcre_extra * PCRE_CALL_CONVENTION +pcre_study(const pcre *external_re, int options, const char **errorptr) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN pcre16_extra * PCRE_CALL_CONVENTION +pcre16_study(const pcre16 *external_re, int options, const char **errorptr) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN pcre32_extra * PCRE_CALL_CONVENTION +pcre32_study(const pcre32 *external_re, int options, const char **errorptr) +#endif +{ +int min; +int count = 0; +BOOL bits_set = FALSE; +pcre_uint8 start_bits[32]; +PUBL(extra) *extra = NULL; +pcre_study_data *study; +const pcre_uint8 *tables; +pcre_uchar *code; +compile_data compile_block; +const REAL_PCRE *re = (const REAL_PCRE *)external_re; + + +*errorptr = NULL; + +if (re == NULL || re->magic_number != MAGIC_NUMBER) + { + *errorptr = "argument is not a compiled regular expression"; + return NULL; + } + +if ((re->flags & PCRE_MODE) == 0) + { +#if defined COMPILE_PCRE8 + *errorptr = "argument not compiled in 8 bit mode"; +#elif defined COMPILE_PCRE16 + *errorptr = "argument not compiled in 16 bit mode"; +#elif defined COMPILE_PCRE32 + *errorptr = "argument not compiled in 32 bit mode"; +#endif + return NULL; + } + +if ((options & ~PUBLIC_STUDY_OPTIONS) != 0) + { + *errorptr = "unknown or incorrect option bit(s) set"; + return NULL; + } + +code = (pcre_uchar *)re + re->name_table_offset + + (re->name_count * re->name_entry_size); + +/* For an anchored pattern, or an unanchored pattern that has a first char, or +a multiline pattern that matches only at "line starts", there is no point in +seeking a list of starting bytes. */ + +if ((re->options & PCRE_ANCHORED) == 0 && + (re->flags & (PCRE_FIRSTSET|PCRE_STARTLINE)) == 0) + { + int rc; + + /* Set the character tables in the block that is passed around */ + + tables = re->tables; + +#if defined COMPILE_PCRE8 + if (tables == NULL) + (void)pcre_fullinfo(external_re, NULL, PCRE_INFO_DEFAULT_TABLES, + (void *)(&tables)); +#elif defined COMPILE_PCRE16 + if (tables == NULL) + (void)pcre16_fullinfo(external_re, NULL, PCRE_INFO_DEFAULT_TABLES, + (void *)(&tables)); +#elif defined COMPILE_PCRE32 + if (tables == NULL) + (void)pcre32_fullinfo(external_re, NULL, PCRE_INFO_DEFAULT_TABLES, + (void *)(&tables)); +#endif + + compile_block.lcc = tables + lcc_offset; + compile_block.fcc = tables + fcc_offset; + compile_block.cbits = tables + cbits_offset; + compile_block.ctypes = tables + ctypes_offset; + + /* See if we can find a fixed set of initial characters for the pattern. */ + + memset(start_bits, 0, 32 * sizeof(pcre_uint8)); + rc = set_start_bits(code, start_bits, (re->options & PCRE_UTF8) != 0, + &compile_block); + bits_set = rc == SSB_DONE; + if (rc == SSB_UNKNOWN) + { + *errorptr = "internal error: opcode not recognized"; + return NULL; + } + } + +/* Find the minimum length of subject string. */ + +switch(min = find_minlength(re, code, code, re->options, NULL, &count)) + { + case -2: *errorptr = "internal error: missing capturing bracket"; return NULL; + case -3: *errorptr = "internal error: opcode not recognized"; return NULL; + default: break; + } + +/* If a set of starting bytes has been identified, or if the minimum length is +greater than zero, or if JIT optimization has been requested, or if +PCRE_STUDY_EXTRA_NEEDED is set, get a pcre[16]_extra block and a +pcre_study_data block. The study data is put in the latter, which is pointed to +by the former, which may also get additional data set later by the calling +program. At the moment, the size of pcre_study_data is fixed. We nevertheless +save it in a field for returning via the pcre_fullinfo() function so that if it +becomes variable in the future, we don't have to change that code. */ + +if (bits_set || min > 0 || (options & ( +#ifdef SUPPORT_JIT + PCRE_STUDY_JIT_COMPILE | PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE | + PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE | +#endif + PCRE_STUDY_EXTRA_NEEDED)) != 0) + { + extra = (PUBL(extra) *)(PUBL(malloc)) + (sizeof(PUBL(extra)) + sizeof(pcre_study_data)); + if (extra == NULL) + { + *errorptr = "failed to get memory"; + return NULL; + } + + study = (pcre_study_data *)((char *)extra + sizeof(PUBL(extra))); + extra->flags = PCRE_EXTRA_STUDY_DATA; + extra->study_data = study; + + study->size = sizeof(pcre_study_data); + study->flags = 0; + + /* Set the start bits always, to avoid unset memory errors if the + study data is written to a file, but set the flag only if any of the bits + are set, to save time looking when none are. */ + + if (bits_set) + { + study->flags |= PCRE_STUDY_MAPPED; + memcpy(study->start_bits, start_bits, sizeof(start_bits)); + } + else memset(study->start_bits, 0, 32 * sizeof(pcre_uint8)); + +#ifdef PCRE_DEBUG + if (bits_set) + { + pcre_uint8 *ptr = start_bits; + int i; + + printf("Start bits:\n"); + for (i = 0; i < 32; i++) + printf("%3d: %02x%s", i * 8, *ptr++, ((i + 1) & 0x7) != 0? " " : "\n"); + } +#endif + + /* Always set the minlength value in the block, because the JIT compiler + makes use of it. However, don't set the bit unless the length is greater than + zero - the interpretive pcre_exec() and pcre_dfa_exec() needn't waste time + checking the zero case. */ + + if (min > 0) + { + study->flags |= PCRE_STUDY_MINLEN; + study->minlength = min; + } + else study->minlength = 0; + + /* If JIT support was compiled and requested, attempt the JIT compilation. + If no starting bytes were found, and the minimum length is zero, and JIT + compilation fails, abandon the extra block and return NULL, unless + PCRE_STUDY_EXTRA_NEEDED is set. */ + +#ifdef SUPPORT_JIT + extra->executable_jit = NULL; + if ((options & PCRE_STUDY_JIT_COMPILE) != 0) + PRIV(jit_compile)(re, extra, JIT_COMPILE); + if ((options & PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE) != 0) + PRIV(jit_compile)(re, extra, JIT_PARTIAL_SOFT_COMPILE); + if ((options & PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE) != 0) + PRIV(jit_compile)(re, extra, JIT_PARTIAL_HARD_COMPILE); + + if (study->flags == 0 && (extra->flags & PCRE_EXTRA_EXECUTABLE_JIT) == 0 && + (options & PCRE_STUDY_EXTRA_NEEDED) == 0) + { +#if defined COMPILE_PCRE8 + pcre_free_study(extra); +#elif defined COMPILE_PCRE16 + pcre16_free_study(extra); +#elif defined COMPILE_PCRE32 + pcre32_free_study(extra); +#endif + extra = NULL; + } +#endif + } + +return extra; +} + + +/************************************************* +* Free the study data * +*************************************************/ + +/* This function frees the memory that was obtained by pcre_study(). + +Argument: a pointer to the pcre[16]_extra block +Returns: nothing +*/ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN void +pcre_free_study(pcre_extra *extra) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN void +pcre16_free_study(pcre16_extra *extra) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN void +pcre32_free_study(pcre32_extra *extra) +#endif +{ +if (extra == NULL) + return; +#ifdef SUPPORT_JIT +if ((extra->flags & PCRE_EXTRA_EXECUTABLE_JIT) != 0 && + extra->executable_jit != NULL) + PRIV(jit_free)(extra->executable_jit); +#endif +PUBL(free)(extra); +} + +/* End of pcre_study.c */ diff --git a/src/third_party/pcre-8.41/pcre_tables.c b/src/third_party/pcre-8.41/pcre_tables.c new file mode 100644 index 00000000000..5e18e8cf904 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_tables.c @@ -0,0 +1,727 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2017 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +#ifndef PCRE_INCLUDED + +/* This module contains some fixed tables that are used by more than one of the +PCRE code modules. The tables are also #included by the pcretest program, which +uses macros to change their names from _pcre_xxx to xxxx, thereby avoiding name +clashes with the library. */ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "pcre_internal.h" + +#endif /* PCRE_INCLUDED */ + +/* Table of sizes for the fixed-length opcodes. It's defined in a macro so that +the definition is next to the definition of the opcodes in pcre_internal.h. */ + +const pcre_uint8 PRIV(OP_lengths)[] = { OP_LENGTHS }; + +/* Tables of horizontal and vertical whitespace characters, suitable for +adding to classes. */ + +const pcre_uint32 PRIV(hspace_list)[] = { HSPACE_LIST }; +const pcre_uint32 PRIV(vspace_list)[] = { VSPACE_LIST }; + + + +/************************************************* +* Tables for UTF-8 support * +*************************************************/ + +/* These are the breakpoints for different numbers of bytes in a UTF-8 +character. */ + +#if (defined SUPPORT_UTF && defined COMPILE_PCRE8) \ + || (defined PCRE_INCLUDED && (defined SUPPORT_PCRE16 || defined SUPPORT_PCRE32)) + +/* These tables are also required by pcretest in 16- or 32-bit mode. */ + +const int PRIV(utf8_table1)[] = + { 0x7f, 0x7ff, 0xffff, 0x1fffff, 0x3ffffff, 0x7fffffff}; + +const int PRIV(utf8_table1_size) = sizeof(PRIV(utf8_table1)) / sizeof(int); + +/* These are the indicator bits and the mask for the data bits to set in the +first byte of a character, indexed by the number of additional bytes. */ + +const int PRIV(utf8_table2)[] = { 0, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc}; +const int PRIV(utf8_table3)[] = { 0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01}; + +/* Table of the number of extra bytes, indexed by the first byte masked with +0x3f. The highest number for a valid UTF-8 first byte is in fact 0x3d. */ + +const pcre_uint8 PRIV(utf8_table4)[] = { + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, + 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 }; + +#endif /* (SUPPORT_UTF && COMPILE_PCRE8) || (PCRE_INCLUDED && SUPPORT_PCRE[16|32])*/ + +#ifdef SUPPORT_UTF + +/* Table to translate from particular type value to the general value. */ + +const pcre_uint32 PRIV(ucp_gentype)[] = { + ucp_C, ucp_C, ucp_C, ucp_C, ucp_C, /* Cc, Cf, Cn, Co, Cs */ + ucp_L, ucp_L, ucp_L, ucp_L, ucp_L, /* Ll, Lu, Lm, Lo, Lt */ + ucp_M, ucp_M, ucp_M, /* Mc, Me, Mn */ + ucp_N, ucp_N, ucp_N, /* Nd, Nl, No */ + ucp_P, ucp_P, ucp_P, ucp_P, ucp_P, /* Pc, Pd, Pe, Pf, Pi */ + ucp_P, ucp_P, /* Ps, Po */ + ucp_S, ucp_S, ucp_S, ucp_S, /* Sc, Sk, Sm, So */ + ucp_Z, ucp_Z, ucp_Z /* Zl, Zp, Zs */ +}; + +/* This table encodes the rules for finding the end of an extended grapheme +cluster. Every code point has a grapheme break property which is one of the +ucp_gbXX values defined in ucp.h. The 2-dimensional table is indexed by the +properties of two adjacent code points. The left property selects a word from +the table, and the right property selects a bit from that word like this: + + ucp_gbtable[left-property] & (1 << right-property) + +The value is non-zero if a grapheme break is NOT permitted between the relevant +two code points. The breaking rules are as follows: + +1. Break at the start and end of text (pretty obviously). + +2. Do not break between a CR and LF; otherwise, break before and after + controls. + +3. Do not break Hangul syllable sequences, the rules for which are: + + L may be followed by L, V, LV or LVT + LV or V may be followed by V or T + LVT or T may be followed by T + +4. Do not break before extending characters. + +The next two rules are only for extended grapheme clusters (but that's what we +are implementing). + +5. Do not break before SpacingMarks. + +6. Do not break after Prepend characters. + +7. Otherwise, break everywhere. +*/ + +const pcre_uint32 PRIV(ucp_gbtable[]) = { + (1< 0x10ffff is not permitted +PCRE_UTF8_ERR14 3-byte character with value 0xd000-0xdfff is not permitted +PCRE_UTF8_ERR15 Overlong 2-byte sequence +PCRE_UTF8_ERR16 Overlong 3-byte sequence +PCRE_UTF8_ERR17 Overlong 4-byte sequence +PCRE_UTF8_ERR18 Overlong 5-byte sequence (won't ever occur) +PCRE_UTF8_ERR19 Overlong 6-byte sequence (won't ever occur) +PCRE_UTF8_ERR20 Isolated 0x80 byte (not within UTF-8 character) +PCRE_UTF8_ERR21 Byte with the illegal value 0xfe or 0xff +PCRE_UTF8_ERR22 Unused (was non-character) + +Arguments: + string points to the string + length length of string, or -1 if the string is zero-terminated + errp pointer to an error position offset variable + +Returns: = 0 if the string is a valid UTF-8 string + > 0 otherwise, setting the offset of the bad character +*/ + +int +PRIV(valid_utf)(PCRE_PUCHAR string, int length, int *erroroffset) +{ +#ifdef SUPPORT_UTF +register PCRE_PUCHAR p; + +if (length < 0) + { + for (p = string; *p != 0; p++); + length = (int)(p - string); + } + +for (p = string; length-- > 0; p++) + { + register pcre_uchar ab, c, d; + + c = *p; + if (c < 128) continue; /* ASCII character */ + + if (c < 0xc0) /* Isolated 10xx xxxx byte */ + { + *erroroffset = (int)(p - string); + return PCRE_UTF8_ERR20; + } + + if (c >= 0xfe) /* Invalid 0xfe or 0xff bytes */ + { + *erroroffset = (int)(p - string); + return PCRE_UTF8_ERR21; + } + + ab = PRIV(utf8_table4)[c & 0x3f]; /* Number of additional bytes */ + if (length < ab) + { + *erroroffset = (int)(p - string); /* Missing bytes */ + return ab - length; /* Codes ERR1 to ERR5 */ + } + length -= ab; /* Length remaining */ + + /* Check top bits in the second byte */ + + if (((d = *(++p)) & 0xc0) != 0x80) + { + *erroroffset = (int)(p - string) - 1; + return PCRE_UTF8_ERR6; + } + + /* For each length, check that the remaining bytes start with the 0x80 bit + set and not the 0x40 bit. Then check for an overlong sequence, and for the + excluded range 0xd800 to 0xdfff. */ + + switch (ab) + { + /* 2-byte character. No further bytes to check for 0x80. Check first byte + for for xx00 000x (overlong sequence). */ + + case 1: if ((c & 0x3e) == 0) + { + *erroroffset = (int)(p - string) - 1; + return PCRE_UTF8_ERR15; + } + break; + + /* 3-byte character. Check third byte for 0x80. Then check first 2 bytes + for 1110 0000, xx0x xxxx (overlong sequence) or + 1110 1101, 1010 xxxx (0xd800 - 0xdfff) */ + + case 2: + if ((*(++p) & 0xc0) != 0x80) /* Third byte */ + { + *erroroffset = (int)(p - string) - 2; + return PCRE_UTF8_ERR7; + } + if (c == 0xe0 && (d & 0x20) == 0) + { + *erroroffset = (int)(p - string) - 2; + return PCRE_UTF8_ERR16; + } + if (c == 0xed && d >= 0xa0) + { + *erroroffset = (int)(p - string) - 2; + return PCRE_UTF8_ERR14; + } + break; + + /* 4-byte character. Check 3rd and 4th bytes for 0x80. Then check first 2 + bytes for for 1111 0000, xx00 xxxx (overlong sequence), then check for a + character greater than 0x0010ffff (f4 8f bf bf) */ + + case 3: + if ((*(++p) & 0xc0) != 0x80) /* Third byte */ + { + *erroroffset = (int)(p - string) - 2; + return PCRE_UTF8_ERR7; + } + if ((*(++p) & 0xc0) != 0x80) /* Fourth byte */ + { + *erroroffset = (int)(p - string) - 3; + return PCRE_UTF8_ERR8; + } + if (c == 0xf0 && (d & 0x30) == 0) + { + *erroroffset = (int)(p - string) - 3; + return PCRE_UTF8_ERR17; + } + if (c > 0xf4 || (c == 0xf4 && d > 0x8f)) + { + *erroroffset = (int)(p - string) - 3; + return PCRE_UTF8_ERR13; + } + break; + + /* 5-byte and 6-byte characters are not allowed by RFC 3629, and will be + rejected by the length test below. However, we do the appropriate tests + here so that overlong sequences get diagnosed, and also in case there is + ever an option for handling these larger code points. */ + + /* 5-byte character. Check 3rd, 4th, and 5th bytes for 0x80. Then check for + 1111 1000, xx00 0xxx */ + + case 4: + if ((*(++p) & 0xc0) != 0x80) /* Third byte */ + { + *erroroffset = (int)(p - string) - 2; + return PCRE_UTF8_ERR7; + } + if ((*(++p) & 0xc0) != 0x80) /* Fourth byte */ + { + *erroroffset = (int)(p - string) - 3; + return PCRE_UTF8_ERR8; + } + if ((*(++p) & 0xc0) != 0x80) /* Fifth byte */ + { + *erroroffset = (int)(p - string) - 4; + return PCRE_UTF8_ERR9; + } + if (c == 0xf8 && (d & 0x38) == 0) + { + *erroroffset = (int)(p - string) - 4; + return PCRE_UTF8_ERR18; + } + break; + + /* 6-byte character. Check 3rd-6th bytes for 0x80. Then check for + 1111 1100, xx00 00xx. */ + + case 5: + if ((*(++p) & 0xc0) != 0x80) /* Third byte */ + { + *erroroffset = (int)(p - string) - 2; + return PCRE_UTF8_ERR7; + } + if ((*(++p) & 0xc0) != 0x80) /* Fourth byte */ + { + *erroroffset = (int)(p - string) - 3; + return PCRE_UTF8_ERR8; + } + if ((*(++p) & 0xc0) != 0x80) /* Fifth byte */ + { + *erroroffset = (int)(p - string) - 4; + return PCRE_UTF8_ERR9; + } + if ((*(++p) & 0xc0) != 0x80) /* Sixth byte */ + { + *erroroffset = (int)(p - string) - 5; + return PCRE_UTF8_ERR10; + } + if (c == 0xfc && (d & 0x3c) == 0) + { + *erroroffset = (int)(p - string) - 5; + return PCRE_UTF8_ERR19; + } + break; + } + + /* Character is valid under RFC 2279, but 4-byte and 5-byte characters are + excluded by RFC 3629. The pointer p is currently at the last byte of the + character. */ + + if (ab > 3) + { + *erroroffset = (int)(p - string) - ab; + return (ab == 4)? PCRE_UTF8_ERR11 : PCRE_UTF8_ERR12; + } + } + +#else /* Not SUPPORT_UTF */ +(void)(string); /* Keep picky compilers happy */ +(void)(length); +(void)(erroroffset); +#endif + +return PCRE_UTF8_ERR0; /* This indicates success */ +} + +/* End of pcre_valid_utf8.c */ diff --git a/src/third_party/pcre-8.41/pcre_version.c b/src/third_party/pcre-8.41/pcre_version.c new file mode 100644 index 00000000000..ae86ff28bc8 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_version.c @@ -0,0 +1,98 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This module contains the external function pcre_version(), which returns a +string that identifies the PCRE version that is in use. */ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "pcre_internal.h" + + +/************************************************* +* Return version string * +*************************************************/ + +/* These macros are the standard way of turning unquoted text into C strings. +They allow macros like PCRE_MAJOR to be defined without quotes, which is +convenient for user programs that want to test its value. */ + +#define STRING(a) # a +#define XSTRING(s) STRING(s) + +/* A problem turned up with PCRE_PRERELEASE, which is defined empty for +production releases. Originally, it was used naively in this code: + + return XSTRING(PCRE_MAJOR) + "." XSTRING(PCRE_MINOR) + XSTRING(PCRE_PRERELEASE) + " " XSTRING(PCRE_DATE); + +However, when PCRE_PRERELEASE is empty, this leads to an attempted expansion of +STRING(). The C standard states: "If (before argument substitution) any +argument consists of no preprocessing tokens, the behavior is undefined." It +turns out the gcc treats this case as a single empty string - which is what we +really want - but Visual C grumbles about the lack of an argument for the +macro. Unfortunately, both are within their rights. To cope with both ways of +handling this, I had resort to some messy hackery that does a test at run time. +I could find no way of detecting that a macro is defined as an empty string at +pre-processor time. This hack uses a standard trick for avoiding calling +the STRING macro with an empty argument when doing the test. */ + +#if defined COMPILE_PCRE8 +PCRE_EXP_DEFN const char * PCRE_CALL_CONVENTION +pcre_version(void) +#elif defined COMPILE_PCRE16 +PCRE_EXP_DEFN const char * PCRE_CALL_CONVENTION +pcre16_version(void) +#elif defined COMPILE_PCRE32 +PCRE_EXP_DEFN const char * PCRE_CALL_CONVENTION +pcre32_version(void) +#endif +{ +return (XSTRING(Z PCRE_PRERELEASE)[1] == 0)? + XSTRING(PCRE_MAJOR.PCRE_MINOR PCRE_DATE) : + XSTRING(PCRE_MAJOR.PCRE_MINOR) XSTRING(PCRE_PRERELEASE PCRE_DATE); +} + +/* End of pcre_version.c */ diff --git a/src/third_party/pcre-8.41/pcre_xclass.c b/src/third_party/pcre-8.41/pcre_xclass.c new file mode 100644 index 00000000000..ef759a589a6 --- /dev/null +++ b/src/third_party/pcre-8.41/pcre_xclass.c @@ -0,0 +1,268 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2013 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This module contains an internal function that is used to match an extended +class. It is used by both pcre_exec() and pcre_def_exec(). */ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "pcre_internal.h" + + +/************************************************* +* Match character against an XCLASS * +*************************************************/ + +/* This function is called to match a character against an extended class that +might contain values > 255 and/or Unicode properties. + +Arguments: + c the character + data points to the flag byte of the XCLASS data + +Returns: TRUE if character matches, else FALSE +*/ + +BOOL +PRIV(xclass)(pcre_uint32 c, const pcre_uchar *data, BOOL utf) +{ +pcre_uchar t; +BOOL negated = (*data & XCL_NOT) != 0; + +(void)utf; +#ifdef COMPILE_PCRE8 +/* In 8 bit mode, this must always be TRUE. Help the compiler to know that. */ +utf = TRUE; +#endif + +/* Character values < 256 are matched against a bitmap, if one is present. If +not, we still carry on, because there may be ranges that start below 256 in the +additional data. */ + +if (c < 256) + { + if ((*data & XCL_HASPROP) == 0) + { + if ((*data & XCL_MAP) == 0) return negated; + return (((pcre_uint8 *)(data + 1))[c/8] & (1 << (c&7))) != 0; + } + if ((*data & XCL_MAP) != 0 && + (((pcre_uint8 *)(data + 1))[c/8] & (1 << (c&7))) != 0) + return !negated; /* char found */ + } + +/* First skip the bit map if present. Then match against the list of Unicode +properties or large chars or ranges that end with a large char. We won't ever +encounter XCL_PROP or XCL_NOTPROP when UCP support is not compiled. */ + +if ((*data++ & XCL_MAP) != 0) data += 32 / sizeof(pcre_uchar); + +while ((t = *data++) != XCL_END) + { + pcre_uint32 x, y; + if (t == XCL_SINGLE) + { +#ifdef SUPPORT_UTF + if (utf) + { + GETCHARINC(x, data); /* macro generates multiple statements */ + } + else +#endif + x = *data++; + if (c == x) return !negated; + } + else if (t == XCL_RANGE) + { +#ifdef SUPPORT_UTF + if (utf) + { + GETCHARINC(x, data); /* macro generates multiple statements */ + GETCHARINC(y, data); /* macro generates multiple statements */ + } + else +#endif + { + x = *data++; + y = *data++; + } + if (c >= x && c <= y) return !negated; + } + +#ifdef SUPPORT_UCP + else /* XCL_PROP & XCL_NOTPROP */ + { + const ucd_record *prop = GET_UCD(c); + BOOL isprop = t == XCL_PROP; + + switch(*data) + { + case PT_ANY: + if (isprop) return !negated; + break; + + case PT_LAMP: + if ((prop->chartype == ucp_Lu || prop->chartype == ucp_Ll || + prop->chartype == ucp_Lt) == isprop) return !negated; + break; + + case PT_GC: + if ((data[1] == PRIV(ucp_gentype)[prop->chartype]) == isprop) + return !negated; + break; + + case PT_PC: + if ((data[1] == prop->chartype) == isprop) return !negated; + break; + + case PT_SC: + if ((data[1] == prop->script) == isprop) return !negated; + break; + + case PT_ALNUM: + if ((PRIV(ucp_gentype)[prop->chartype] == ucp_L || + PRIV(ucp_gentype)[prop->chartype] == ucp_N) == isprop) + return !negated; + break; + + /* Perl space used to exclude VT, but from Perl 5.18 it is included, + which means that Perl space and POSIX space are now identical. PCRE + was changed at release 8.34. */ + + case PT_SPACE: /* Perl space */ + case PT_PXSPACE: /* POSIX space */ + switch(c) + { + HSPACE_CASES: + VSPACE_CASES: + if (isprop) return !negated; + break; + + default: + if ((PRIV(ucp_gentype)[prop->chartype] == ucp_Z) == isprop) + return !negated; + break; + } + break; + + case PT_WORD: + if ((PRIV(ucp_gentype)[prop->chartype] == ucp_L || + PRIV(ucp_gentype)[prop->chartype] == ucp_N || c == CHAR_UNDERSCORE) + == isprop) + return !negated; + break; + + case PT_UCNC: + if (c < 0xa0) + { + if ((c == CHAR_DOLLAR_SIGN || c == CHAR_COMMERCIAL_AT || + c == CHAR_GRAVE_ACCENT) == isprop) + return !negated; + } + else + { + if ((c < 0xd800 || c > 0xdfff) == isprop) + return !negated; + } + break; + + /* The following three properties can occur only in an XCLASS, as there + is no \p or \P coding for them. */ + + /* Graphic character. Implement this as not Z (space or separator) and + not C (other), except for Cf (format) with a few exceptions. This seems + to be what Perl does. The exceptional characters are: + + U+061C Arabic Letter Mark + U+180E Mongolian Vowel Separator + U+2066 - U+2069 Various "isolate"s + */ + + case PT_PXGRAPH: + if ((PRIV(ucp_gentype)[prop->chartype] != ucp_Z && + (PRIV(ucp_gentype)[prop->chartype] != ucp_C || + (prop->chartype == ucp_Cf && + c != 0x061c && c != 0x180e && (c < 0x2066 || c > 0x2069)) + )) == isprop) + return !negated; + break; + + /* Printable character: same as graphic, with the addition of Zs, i.e. + not Zl and not Zp, and U+180E. */ + + case PT_PXPRINT: + if ((prop->chartype != ucp_Zl && + prop->chartype != ucp_Zp && + (PRIV(ucp_gentype)[prop->chartype] != ucp_C || + (prop->chartype == ucp_Cf && + c != 0x061c && (c < 0x2066 || c > 0x2069)) + )) == isprop) + return !negated; + break; + + /* Punctuation: all Unicode punctuation, plus ASCII characters that + Unicode treats as symbols rather than punctuation, for Perl + compatibility (these are $+<=>^`|~). */ + + case PT_PXPUNCT: + if ((PRIV(ucp_gentype)[prop->chartype] == ucp_P || + (c < 128 && PRIV(ucp_gentype)[prop->chartype] == ucp_S)) == isprop) + return !negated; + break; + + /* This should never occur, but compilers may mutter if there is no + default. */ + + default: + return FALSE; + } + + data += 2; + } +#endif /* SUPPORT_UCP */ + } + +return negated; /* char did not match */ +} + +/* End of pcre_xclass.c */ diff --git a/src/third_party/pcre-8.41/pcrecpp.cc b/src/third_party/pcre-8.41/pcrecpp.cc new file mode 100644 index 00000000000..d09c9abc516 --- /dev/null +++ b/src/third_party/pcre-8.41/pcrecpp.cc @@ -0,0 +1,924 @@ +// Copyright (c) 2010, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Sanjay Ghemawat + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include /* for SHRT_MIN, USHRT_MAX, etc */ +#include /* for memcpy */ +#include +#include +#include +#include + +#include "pcrecpp_internal.h" +#include "pcre.h" +#include "pcrecpp.h" +#include "pcre_stringpiece.h" + + +namespace pcrecpp { + +// Maximum number of args we can set +static const int kMaxArgs = 16; +static const int kVecSize = (1 + kMaxArgs) * 3; // results + PCRE workspace + +// Special object that stands-in for no argument +Arg RE::no_arg((void*)NULL); + +// This is for ABI compatibility with old versions of pcre (pre-7.6), +// which defined a global no_arg variable instead of putting it in the +// RE class. This works on GCC >= 3, at least. It definitely works +// for ELF, but may not for other object formats (Mach-O, for +// instance, does not support aliases.) We could probably have a more +// inclusive test if we ever needed it. (Note that not only the +// __attribute__ syntax, but also __USER_LABEL_PREFIX__, are +// gnu-specific.) +#if defined(__GNUC__) && __GNUC__ >= 3 && defined(__ELF__) && !defined(__INTEL_COMPILER) +# define ULP_AS_STRING(x) ULP_AS_STRING_INTERNAL(x) +# define ULP_AS_STRING_INTERNAL(x) #x +# define USER_LABEL_PREFIX_STR ULP_AS_STRING(__USER_LABEL_PREFIX__) +extern Arg no_arg + __attribute__((alias(USER_LABEL_PREFIX_STR "_ZN7pcrecpp2RE6no_argE"))); +#endif + +// If a regular expression has no error, its error_ field points here +static const string empty_string; + +// If the user doesn't ask for any options, we just use this one +static RE_Options default_options; + +void RE::Init(const string& pat, const RE_Options* options) { + pattern_ = pat; + if (options == NULL) { + options_ = default_options; + } else { + options_ = *options; + } + error_ = &empty_string; + re_full_ = NULL; + re_partial_ = NULL; + + re_partial_ = Compile(UNANCHORED); + if (re_partial_ != NULL) { + re_full_ = Compile(ANCHOR_BOTH); + } +} + +void RE::Cleanup() { + if (re_full_ != NULL) (*pcre_free)(re_full_); + if (re_partial_ != NULL) (*pcre_free)(re_partial_); + if (error_ != &empty_string) delete error_; +} + + +RE::~RE() { + Cleanup(); +} + + +pcre* RE::Compile(Anchor anchor) { + // First, convert RE_Options into pcre options + int pcre_options = 0; + pcre_options = options_.all_options(); + + // Special treatment for anchoring. This is needed because at + // runtime pcre only provides an option for anchoring at the + // beginning of a string (unless you use offset). + // + // There are three types of anchoring we want: + // UNANCHORED Compile the original pattern, and use + // a pcre unanchored match. + // ANCHOR_START Compile the original pattern, and use + // a pcre anchored match. + // ANCHOR_BOTH Tack a "\z" to the end of the original pattern + // and use a pcre anchored match. + + const char* compile_error; + int eoffset; + pcre* re; + if (anchor != ANCHOR_BOTH) { + re = pcre_compile(pattern_.c_str(), pcre_options, + &compile_error, &eoffset, NULL); + } else { + // Tack a '\z' at the end of RE. Parenthesize it first so that + // the '\z' applies to all top-level alternatives in the regexp. + string wrapped = "(?:"; // A non-counting grouping operator + wrapped += pattern_; + wrapped += ")\\z"; + re = pcre_compile(wrapped.c_str(), pcre_options, + &compile_error, &eoffset, NULL); + } + if (re == NULL) { + if (error_ == &empty_string) error_ = new string(compile_error); + } + return re; +} + +/***** Matching interfaces *****/ + +bool RE::FullMatch(const StringPiece& text, + const Arg& ptr1, + const Arg& ptr2, + const Arg& ptr3, + const Arg& ptr4, + const Arg& ptr5, + const Arg& ptr6, + const Arg& ptr7, + const Arg& ptr8, + const Arg& ptr9, + const Arg& ptr10, + const Arg& ptr11, + const Arg& ptr12, + const Arg& ptr13, + const Arg& ptr14, + const Arg& ptr15, + const Arg& ptr16) const { + const Arg* args[kMaxArgs]; + int n = 0; + if (&ptr1 == &no_arg) { goto done; } args[n++] = &ptr1; + if (&ptr2 == &no_arg) { goto done; } args[n++] = &ptr2; + if (&ptr3 == &no_arg) { goto done; } args[n++] = &ptr3; + if (&ptr4 == &no_arg) { goto done; } args[n++] = &ptr4; + if (&ptr5 == &no_arg) { goto done; } args[n++] = &ptr5; + if (&ptr6 == &no_arg) { goto done; } args[n++] = &ptr6; + if (&ptr7 == &no_arg) { goto done; } args[n++] = &ptr7; + if (&ptr8 == &no_arg) { goto done; } args[n++] = &ptr8; + if (&ptr9 == &no_arg) { goto done; } args[n++] = &ptr9; + if (&ptr10 == &no_arg) { goto done; } args[n++] = &ptr10; + if (&ptr11 == &no_arg) { goto done; } args[n++] = &ptr11; + if (&ptr12 == &no_arg) { goto done; } args[n++] = &ptr12; + if (&ptr13 == &no_arg) { goto done; } args[n++] = &ptr13; + if (&ptr14 == &no_arg) { goto done; } args[n++] = &ptr14; + if (&ptr15 == &no_arg) { goto done; } args[n++] = &ptr15; + if (&ptr16 == &no_arg) { goto done; } args[n++] = &ptr16; + done: + + int consumed; + int vec[kVecSize]; + return DoMatchImpl(text, ANCHOR_BOTH, &consumed, args, n, vec, kVecSize); +} + +bool RE::PartialMatch(const StringPiece& text, + const Arg& ptr1, + const Arg& ptr2, + const Arg& ptr3, + const Arg& ptr4, + const Arg& ptr5, + const Arg& ptr6, + const Arg& ptr7, + const Arg& ptr8, + const Arg& ptr9, + const Arg& ptr10, + const Arg& ptr11, + const Arg& ptr12, + const Arg& ptr13, + const Arg& ptr14, + const Arg& ptr15, + const Arg& ptr16) const { + const Arg* args[kMaxArgs]; + int n = 0; + if (&ptr1 == &no_arg) { goto done; } args[n++] = &ptr1; + if (&ptr2 == &no_arg) { goto done; } args[n++] = &ptr2; + if (&ptr3 == &no_arg) { goto done; } args[n++] = &ptr3; + if (&ptr4 == &no_arg) { goto done; } args[n++] = &ptr4; + if (&ptr5 == &no_arg) { goto done; } args[n++] = &ptr5; + if (&ptr6 == &no_arg) { goto done; } args[n++] = &ptr6; + if (&ptr7 == &no_arg) { goto done; } args[n++] = &ptr7; + if (&ptr8 == &no_arg) { goto done; } args[n++] = &ptr8; + if (&ptr9 == &no_arg) { goto done; } args[n++] = &ptr9; + if (&ptr10 == &no_arg) { goto done; } args[n++] = &ptr10; + if (&ptr11 == &no_arg) { goto done; } args[n++] = &ptr11; + if (&ptr12 == &no_arg) { goto done; } args[n++] = &ptr12; + if (&ptr13 == &no_arg) { goto done; } args[n++] = &ptr13; + if (&ptr14 == &no_arg) { goto done; } args[n++] = &ptr14; + if (&ptr15 == &no_arg) { goto done; } args[n++] = &ptr15; + if (&ptr16 == &no_arg) { goto done; } args[n++] = &ptr16; + done: + + int consumed; + int vec[kVecSize]; + return DoMatchImpl(text, UNANCHORED, &consumed, args, n, vec, kVecSize); +} + +bool RE::Consume(StringPiece* input, + const Arg& ptr1, + const Arg& ptr2, + const Arg& ptr3, + const Arg& ptr4, + const Arg& ptr5, + const Arg& ptr6, + const Arg& ptr7, + const Arg& ptr8, + const Arg& ptr9, + const Arg& ptr10, + const Arg& ptr11, + const Arg& ptr12, + const Arg& ptr13, + const Arg& ptr14, + const Arg& ptr15, + const Arg& ptr16) const { + const Arg* args[kMaxArgs]; + int n = 0; + if (&ptr1 == &no_arg) { goto done; } args[n++] = &ptr1; + if (&ptr2 == &no_arg) { goto done; } args[n++] = &ptr2; + if (&ptr3 == &no_arg) { goto done; } args[n++] = &ptr3; + if (&ptr4 == &no_arg) { goto done; } args[n++] = &ptr4; + if (&ptr5 == &no_arg) { goto done; } args[n++] = &ptr5; + if (&ptr6 == &no_arg) { goto done; } args[n++] = &ptr6; + if (&ptr7 == &no_arg) { goto done; } args[n++] = &ptr7; + if (&ptr8 == &no_arg) { goto done; } args[n++] = &ptr8; + if (&ptr9 == &no_arg) { goto done; } args[n++] = &ptr9; + if (&ptr10 == &no_arg) { goto done; } args[n++] = &ptr10; + if (&ptr11 == &no_arg) { goto done; } args[n++] = &ptr11; + if (&ptr12 == &no_arg) { goto done; } args[n++] = &ptr12; + if (&ptr13 == &no_arg) { goto done; } args[n++] = &ptr13; + if (&ptr14 == &no_arg) { goto done; } args[n++] = &ptr14; + if (&ptr15 == &no_arg) { goto done; } args[n++] = &ptr15; + if (&ptr16 == &no_arg) { goto done; } args[n++] = &ptr16; + done: + + int consumed; + int vec[kVecSize]; + if (DoMatchImpl(*input, ANCHOR_START, &consumed, + args, n, vec, kVecSize)) { + input->remove_prefix(consumed); + return true; + } else { + return false; + } +} + +bool RE::FindAndConsume(StringPiece* input, + const Arg& ptr1, + const Arg& ptr2, + const Arg& ptr3, + const Arg& ptr4, + const Arg& ptr5, + const Arg& ptr6, + const Arg& ptr7, + const Arg& ptr8, + const Arg& ptr9, + const Arg& ptr10, + const Arg& ptr11, + const Arg& ptr12, + const Arg& ptr13, + const Arg& ptr14, + const Arg& ptr15, + const Arg& ptr16) const { + const Arg* args[kMaxArgs]; + int n = 0; + if (&ptr1 == &no_arg) { goto done; } args[n++] = &ptr1; + if (&ptr2 == &no_arg) { goto done; } args[n++] = &ptr2; + if (&ptr3 == &no_arg) { goto done; } args[n++] = &ptr3; + if (&ptr4 == &no_arg) { goto done; } args[n++] = &ptr4; + if (&ptr5 == &no_arg) { goto done; } args[n++] = &ptr5; + if (&ptr6 == &no_arg) { goto done; } args[n++] = &ptr6; + if (&ptr7 == &no_arg) { goto done; } args[n++] = &ptr7; + if (&ptr8 == &no_arg) { goto done; } args[n++] = &ptr8; + if (&ptr9 == &no_arg) { goto done; } args[n++] = &ptr9; + if (&ptr10 == &no_arg) { goto done; } args[n++] = &ptr10; + if (&ptr11 == &no_arg) { goto done; } args[n++] = &ptr11; + if (&ptr12 == &no_arg) { goto done; } args[n++] = &ptr12; + if (&ptr13 == &no_arg) { goto done; } args[n++] = &ptr13; + if (&ptr14 == &no_arg) { goto done; } args[n++] = &ptr14; + if (&ptr15 == &no_arg) { goto done; } args[n++] = &ptr15; + if (&ptr16 == &no_arg) { goto done; } args[n++] = &ptr16; + done: + + int consumed; + int vec[kVecSize]; + if (DoMatchImpl(*input, UNANCHORED, &consumed, + args, n, vec, kVecSize)) { + input->remove_prefix(consumed); + return true; + } else { + return false; + } +} + +bool RE::Replace(const StringPiece& rewrite, + string *str) const { + int vec[kVecSize]; + int matches = TryMatch(*str, 0, UNANCHORED, true, vec, kVecSize); + if (matches == 0) + return false; + + string s; + if (!Rewrite(&s, rewrite, *str, vec, matches)) + return false; + + assert(vec[0] >= 0); + assert(vec[1] >= 0); + str->replace(vec[0], vec[1] - vec[0], s); + return true; +} + +// Returns PCRE_NEWLINE_CRLF, PCRE_NEWLINE_CR, or PCRE_NEWLINE_LF. +// Note that PCRE_NEWLINE_CRLF is defined to be P_N_CR | P_N_LF. +// Modified by PH to add PCRE_NEWLINE_ANY and PCRE_NEWLINE_ANYCRLF. + +static int NewlineMode(int pcre_options) { + // TODO: if we can make it threadsafe, cache this var + int newline_mode = 0; + /* if (newline_mode) return newline_mode; */ // do this once it's cached + if (pcre_options & (PCRE_NEWLINE_CRLF|PCRE_NEWLINE_CR|PCRE_NEWLINE_LF| + PCRE_NEWLINE_ANY|PCRE_NEWLINE_ANYCRLF)) { + newline_mode = (pcre_options & + (PCRE_NEWLINE_CRLF|PCRE_NEWLINE_CR|PCRE_NEWLINE_LF| + PCRE_NEWLINE_ANY|PCRE_NEWLINE_ANYCRLF)); + } else { + int newline; + pcre_config(PCRE_CONFIG_NEWLINE, &newline); + if (newline == 10) + newline_mode = PCRE_NEWLINE_LF; + else if (newline == 13) + newline_mode = PCRE_NEWLINE_CR; + else if (newline == 3338) + newline_mode = PCRE_NEWLINE_CRLF; + else if (newline == -1) + newline_mode = PCRE_NEWLINE_ANY; + else if (newline == -2) + newline_mode = PCRE_NEWLINE_ANYCRLF; + else + assert(NULL == "Unexpected return value from pcre_config(NEWLINE)"); + } + return newline_mode; +} + +int RE::GlobalReplace(const StringPiece& rewrite, + string *str) const { + int count = 0; + int vec[kVecSize]; + string out; + int start = 0; + bool last_match_was_empty_string = false; + + while (start <= static_cast(str->length())) { + // If the previous match was for the empty string, we shouldn't + // just match again: we'll match in the same way and get an + // infinite loop. Instead, we do the match in a special way: + // anchored -- to force another try at the same position -- + // and with a flag saying that this time, ignore empty matches. + // If this special match returns, that means there's a non-empty + // match at this position as well, and we can continue. If not, + // we do what perl does, and just advance by one. + // Notice that perl prints '@@@' for this; + // perl -le '$_ = "aa"; s/b*|aa/@/g; print' + int matches; + if (last_match_was_empty_string) { + matches = TryMatch(*str, start, ANCHOR_START, false, vec, kVecSize); + if (matches <= 0) { + int matchend = start + 1; // advance one character. + // If the current char is CR and we're in CRLF mode, skip LF too. + // Note it's better to call pcre_fullinfo() than to examine + // all_options(), since options_ could have changed bewteen + // compile-time and now, but this is simpler and safe enough. + // Modified by PH to add ANY and ANYCRLF. + if (matchend < static_cast(str->length()) && + (*str)[start] == '\r' && (*str)[matchend] == '\n' && + (NewlineMode(options_.all_options()) == PCRE_NEWLINE_CRLF || + NewlineMode(options_.all_options()) == PCRE_NEWLINE_ANY || + NewlineMode(options_.all_options()) == PCRE_NEWLINE_ANYCRLF)) { + matchend++; + } + // We also need to advance more than one char if we're in utf8 mode. +#ifdef SUPPORT_UTF8 + if (options_.utf8()) { + while (matchend < static_cast(str->length()) && + ((*str)[matchend] & 0xc0) == 0x80) + matchend++; + } +#endif + if (start < static_cast(str->length())) + out.append(*str, start, matchend - start); + start = matchend; + last_match_was_empty_string = false; + continue; + } + } else { + matches = TryMatch(*str, start, UNANCHORED, true, vec, kVecSize); + if (matches <= 0) + break; + } + int matchstart = vec[0], matchend = vec[1]; + assert(matchstart >= start); + assert(matchend >= matchstart); + out.append(*str, start, matchstart - start); + Rewrite(&out, rewrite, *str, vec, matches); + start = matchend; + count++; + last_match_was_empty_string = (matchstart == matchend); + } + + if (count == 0) + return 0; + + if (start < static_cast(str->length())) + out.append(*str, start, str->length() - start); + swap(out, *str); + return count; +} + +bool RE::Extract(const StringPiece& rewrite, + const StringPiece& text, + string *out) const { + int vec[kVecSize]; + int matches = TryMatch(text, 0, UNANCHORED, true, vec, kVecSize); + if (matches == 0) + return false; + out->erase(); + return Rewrite(out, rewrite, text, vec, matches); +} + +/*static*/ string RE::QuoteMeta(const StringPiece& unquoted) { + string result; + + // Escape any ascii character not in [A-Za-z_0-9]. + // + // Note that it's legal to escape a character even if it has no + // special meaning in a regular expression -- so this function does + // that. (This also makes it identical to the perl function of the + // same name; see `perldoc -f quotemeta`.) The one exception is + // escaping NUL: rather than doing backslash + NUL, like perl does, + // we do '\0', because pcre itself doesn't take embedded NUL chars. + for (int ii = 0; ii < unquoted.size(); ++ii) { + // Note that using 'isalnum' here raises the benchmark time from + // 32ns to 58ns: + if (unquoted[ii] == '\0') { + result += "\\0"; + } else if ((unquoted[ii] < 'a' || unquoted[ii] > 'z') && + (unquoted[ii] < 'A' || unquoted[ii] > 'Z') && + (unquoted[ii] < '0' || unquoted[ii] > '9') && + unquoted[ii] != '_' && + // If this is the part of a UTF8 or Latin1 character, we need + // to copy this byte without escaping. Experimentally this is + // what works correctly with the regexp library. + !(unquoted[ii] & 128)) { + result += '\\'; + result += unquoted[ii]; + } else { + result += unquoted[ii]; + } + } + + return result; +} + +/***** Actual matching and rewriting code *****/ + +int RE::TryMatch(const StringPiece& text, + int startpos, + Anchor anchor, + bool empty_ok, + int *vec, + int vecsize) const { + pcre* re = (anchor == ANCHOR_BOTH) ? re_full_ : re_partial_; + if (re == NULL) { + //fprintf(stderr, "Matching against invalid re: %s\n", error_->c_str()); + return 0; + } + + pcre_extra extra = { 0, 0, 0, 0, 0, 0, 0, 0 }; + if (options_.match_limit() > 0) { + extra.flags |= PCRE_EXTRA_MATCH_LIMIT; + extra.match_limit = options_.match_limit(); + } + if (options_.match_limit_recursion() > 0) { + extra.flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION; + extra.match_limit_recursion = options_.match_limit_recursion(); + } + + // int options = 0; + // Changed by PH as a result of bugzilla #1288 + int options = (options_.all_options() & PCRE_NO_UTF8_CHECK); + + if (anchor != UNANCHORED) + options |= PCRE_ANCHORED; + if (!empty_ok) + options |= PCRE_NOTEMPTY; + + int rc = pcre_exec(re, // The regular expression object + &extra, + (text.data() == NULL) ? "" : text.data(), + text.size(), + startpos, + options, + vec, + vecsize); + + // Handle errors + if (rc == PCRE_ERROR_NOMATCH) { + return 0; + } else if (rc < 0) { + //fprintf(stderr, "Unexpected return code: %d when matching '%s'\n", + // re, pattern_.c_str()); + return 0; + } else if (rc == 0) { + // pcre_exec() returns 0 as a special case when the number of + // capturing subpatterns exceeds the size of the vector. + // When this happens, there is a match and the output vector + // is filled, but we miss out on the positions of the extra subpatterns. + rc = vecsize / 2; + } + + return rc; +} + +bool RE::DoMatchImpl(const StringPiece& text, + Anchor anchor, + int* consumed, + const Arg* const* args, + int n, + int* vec, + int vecsize) const { + assert((1 + n) * 3 <= vecsize); // results + PCRE workspace + int matches = TryMatch(text, 0, anchor, true, vec, vecsize); + assert(matches >= 0); // TryMatch never returns negatives + if (matches == 0) + return false; + + *consumed = vec[1]; + + if (n == 0 || args == NULL) { + // We are not interested in results + return true; + } + + if (NumberOfCapturingGroups() < n) { + // RE has fewer capturing groups than number of arg pointers passed in + return false; + } + + // If we got here, we must have matched the whole pattern. + // We do not need (can not do) any more checks on the value of 'matches' here + // -- see the comment for TryMatch. + for (int i = 0; i < n; i++) { + const int start = vec[2*(i+1)]; + const int limit = vec[2*(i+1)+1]; + if (!args[i]->Parse(text.data() + start, limit-start)) { + // TODO: Should we indicate what the error was? + return false; + } + } + + return true; +} + +bool RE::DoMatch(const StringPiece& text, + Anchor anchor, + int* consumed, + const Arg* const args[], + int n) const { + assert(n >= 0); + size_t const vecsize = (1 + n) * 3; // results + PCRE workspace + // (as for kVecSize) + int space[21]; // use stack allocation for small vecsize (common case) + int* vec = vecsize <= 21 ? space : new int[vecsize]; + bool retval = DoMatchImpl(text, anchor, consumed, args, n, vec, (int)vecsize); + if (vec != space) delete [] vec; + return retval; +} + +bool RE::Rewrite(string *out, const StringPiece &rewrite, + const StringPiece &text, int *vec, int veclen) const { + for (const char *s = rewrite.data(), *end = s + rewrite.size(); + s < end; s++) { + int c = *s; + if (c == '\\') { + c = *++s; + if (isdigit(c)) { + int n = (c - '0'); + if (n >= veclen) { + //fprintf(stderr, requested group %d in regexp %.*s\n", + // n, rewrite.size(), rewrite.data()); + return false; + } + int start = vec[2 * n]; + if (start >= 0) + out->append(text.data() + start, vec[2 * n + 1] - start); + } else if (c == '\\') { + *out += '\\'; + } else { + //fprintf(stderr, "invalid rewrite pattern: %.*s\n", + // rewrite.size(), rewrite.data()); + return false; + } + } else { + *out += c; + } + } + return true; +} + +// Return the number of capturing subpatterns, or -1 if the +// regexp wasn't valid on construction. +int RE::NumberOfCapturingGroups() const { + if (re_partial_ == NULL) return -1; + + int result; + int pcre_retval = pcre_fullinfo(re_partial_, // The regular expression object + NULL, // We did not study the pattern + PCRE_INFO_CAPTURECOUNT, + &result); + assert(pcre_retval == 0); + return result; +} + +/***** Parsers for various types *****/ + +bool Arg::parse_null(const char* str, int n, void* dest) { + (void)str; + (void)n; + // We fail if somebody asked us to store into a non-NULL void* pointer + return (dest == NULL); +} + +bool Arg::parse_string(const char* str, int n, void* dest) { + if (dest == NULL) return true; + reinterpret_cast(dest)->assign(str, n); + return true; +} + +bool Arg::parse_stringpiece(const char* str, int n, void* dest) { + if (dest == NULL) return true; + reinterpret_cast(dest)->set(str, n); + return true; +} + +bool Arg::parse_char(const char* str, int n, void* dest) { + if (n != 1) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = str[0]; + return true; +} + +bool Arg::parse_uchar(const char* str, int n, void* dest) { + if (n != 1) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = str[0]; + return true; +} + +// Largest number spec that we are willing to parse +static const int kMaxNumberLength = 32; + +// REQUIRES "buf" must have length at least kMaxNumberLength+1 +// REQUIRES "n > 0" +// Copies "str" into "buf" and null-terminates if necessary. +// Returns one of: +// a. "str" if no termination is needed +// b. "buf" if the string was copied and null-terminated +// c. "" if the input was invalid and has no hope of being parsed +static const char* TerminateNumber(char* buf, const char* str, int n) { + if ((n > 0) && isspace(*str)) { + // We are less forgiving than the strtoxxx() routines and do not + // allow leading spaces. + return ""; + } + + // See if the character right after the input text may potentially + // look like a digit. + if (isdigit(str[n]) || + ((str[n] >= 'a') && (str[n] <= 'f')) || + ((str[n] >= 'A') && (str[n] <= 'F'))) { + if (n > kMaxNumberLength) return ""; // Input too big to be a valid number + memcpy(buf, str, n); + buf[n] = '\0'; + return buf; + } else { + // We can parse right out of the supplied string, so return it. + return str; + } +} + +bool Arg::parse_long_radix(const char* str, + int n, + void* dest, + int radix) { + if (n == 0) return false; + char buf[kMaxNumberLength+1]; + str = TerminateNumber(buf, str, n); + char* end; + errno = 0; + long r = strtol(str, &end, radix); + if (end != str + n) return false; // Leftover junk + if (errno) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = r; + return true; +} + +bool Arg::parse_ulong_radix(const char* str, + int n, + void* dest, + int radix) { + if (n == 0) return false; + char buf[kMaxNumberLength+1]; + str = TerminateNumber(buf, str, n); + if (str[0] == '-') return false; // strtoul() on a negative number?! + char* end; + errno = 0; + unsigned long r = strtoul(str, &end, radix); + if (end != str + n) return false; // Leftover junk + if (errno) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = r; + return true; +} + +bool Arg::parse_short_radix(const char* str, + int n, + void* dest, + int radix) { + long r; + if (!parse_long_radix(str, n, &r, radix)) return false; // Could not parse + if (r < SHRT_MIN || r > SHRT_MAX) return false; // Out of range + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = static_cast(r); + return true; +} + +bool Arg::parse_ushort_radix(const char* str, + int n, + void* dest, + int radix) { + unsigned long r; + if (!parse_ulong_radix(str, n, &r, radix)) return false; // Could not parse + if (r > USHRT_MAX) return false; // Out of range + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = static_cast(r); + return true; +} + +bool Arg::parse_int_radix(const char* str, + int n, + void* dest, + int radix) { + long r; + if (!parse_long_radix(str, n, &r, radix)) return false; // Could not parse + if (r < INT_MIN || r > INT_MAX) return false; // Out of range + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = r; + return true; +} + +bool Arg::parse_uint_radix(const char* str, + int n, + void* dest, + int radix) { + unsigned long r; + if (!parse_ulong_radix(str, n, &r, radix)) return false; // Could not parse + if (r > UINT_MAX) return false; // Out of range + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = r; + return true; +} + +bool Arg::parse_longlong_radix(const char* str, + int n, + void* dest, + int radix) { +#ifndef HAVE_LONG_LONG + return false; +#else + if (n == 0) return false; + char buf[kMaxNumberLength+1]; + str = TerminateNumber(buf, str, n); + char* end; + errno = 0; +#if defined HAVE_STRTOQ + long long r = strtoq(str, &end, radix); +#elif defined HAVE_STRTOLL + long long r = strtoll(str, &end, radix); +#elif defined HAVE__STRTOI64 + long long r = _strtoi64(str, &end, radix); +#elif defined HAVE_STRTOIMAX + long long r = strtoimax(str, &end, radix); +#else +#error parse_longlong_radix: cannot convert input to a long-long +#endif + if (end != str + n) return false; // Leftover junk + if (errno) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = r; + return true; +#endif /* HAVE_LONG_LONG */ +} + +bool Arg::parse_ulonglong_radix(const char* str, + int n, + void* dest, + int radix) { +#ifndef HAVE_UNSIGNED_LONG_LONG + return false; +#else + if (n == 0) return false; + char buf[kMaxNumberLength+1]; + str = TerminateNumber(buf, str, n); + if (str[0] == '-') return false; // strtoull() on a negative number?! + char* end; + errno = 0; +#if defined HAVE_STRTOQ + unsigned long long r = strtouq(str, &end, radix); +#elif defined HAVE_STRTOLL + unsigned long long r = strtoull(str, &end, radix); +#elif defined HAVE__STRTOI64 + unsigned long long r = _strtoui64(str, &end, radix); +#elif defined HAVE_STRTOIMAX + unsigned long long r = strtoumax(str, &end, radix); +#else +#error parse_ulonglong_radix: cannot convert input to a long-long +#endif + if (end != str + n) return false; // Leftover junk + if (errno) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = r; + return true; +#endif /* HAVE_UNSIGNED_LONG_LONG */ +} + +bool Arg::parse_double(const char* str, int n, void* dest) { + if (n == 0) return false; + static const int kMaxLength = 200; + char buf[kMaxLength]; + if (n >= kMaxLength) return false; + memcpy(buf, str, n); + buf[n] = '\0'; + errno = 0; + char* end; + double r = strtod(buf, &end); + if (end != buf + n) return false; // Leftover junk + if (errno) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = r; + return true; +} + +bool Arg::parse_float(const char* str, int n, void* dest) { + double r; + if (!parse_double(str, n, &r)) return false; + if (dest == NULL) return true; + *(reinterpret_cast(dest)) = static_cast(r); + return true; +} + + +#define DEFINE_INTEGER_PARSERS(name) \ + bool Arg::parse_##name(const char* str, int n, void* dest) { \ + return parse_##name##_radix(str, n, dest, 10); \ + } \ + bool Arg::parse_##name##_hex(const char* str, int n, void* dest) { \ + return parse_##name##_radix(str, n, dest, 16); \ + } \ + bool Arg::parse_##name##_octal(const char* str, int n, void* dest) { \ + return parse_##name##_radix(str, n, dest, 8); \ + } \ + bool Arg::parse_##name##_cradix(const char* str, int n, void* dest) { \ + return parse_##name##_radix(str, n, dest, 0); \ + } + +DEFINE_INTEGER_PARSERS(short) /* */ +DEFINE_INTEGER_PARSERS(ushort) /* */ +DEFINE_INTEGER_PARSERS(int) /* Don't use semicolons after these */ +DEFINE_INTEGER_PARSERS(uint) /* statements because they can cause */ +DEFINE_INTEGER_PARSERS(long) /* compiler warnings if the checking */ +DEFINE_INTEGER_PARSERS(ulong) /* level is turned up high enough. */ +DEFINE_INTEGER_PARSERS(longlong) /* */ +DEFINE_INTEGER_PARSERS(ulonglong) /* */ + +#undef DEFINE_INTEGER_PARSERS + +} // namespace pcrecpp diff --git a/src/third_party/pcre-8.41/pcrecpp.h b/src/third_party/pcre-8.41/pcrecpp.h new file mode 100644 index 00000000000..3e594b0d439 --- /dev/null +++ b/src/third_party/pcre-8.41/pcrecpp.h @@ -0,0 +1,710 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Sanjay Ghemawat +// Support for PCRE_XXX modifiers added by Giuseppe Maxia, July 2005 + +#ifndef _PCRECPP_H +#define _PCRECPP_H + +// C++ interface to the pcre regular-expression library. RE supports +// Perl-style regular expressions (with extensions like \d, \w, \s, +// ...). +// +// ----------------------------------------------------------------------- +// REGEXP SYNTAX: +// +// This module is part of the pcre library and hence supports its syntax +// for regular expressions. +// +// The syntax is pretty similar to Perl's. For those not familiar +// with Perl's regular expressions, here are some examples of the most +// commonly used extensions: +// +// "hello (\\w+) world" -- \w matches a "word" character +// "version (\\d+)" -- \d matches a digit +// "hello\\s+world" -- \s matches any whitespace character +// "\\b(\\w+)\\b" -- \b matches empty string at a word boundary +// "(?i)hello" -- (?i) turns on case-insensitive matching +// "/\\*(.*?)\\*/" -- .*? matches . minimum no. of times possible +// +// ----------------------------------------------------------------------- +// MATCHING INTERFACE: +// +// The "FullMatch" operation checks that supplied text matches a +// supplied pattern exactly. +// +// Example: successful match +// pcrecpp::RE re("h.*o"); +// re.FullMatch("hello"); +// +// Example: unsuccessful match (requires full match): +// pcrecpp::RE re("e"); +// !re.FullMatch("hello"); +// +// Example: creating a temporary RE object: +// pcrecpp::RE("h.*o").FullMatch("hello"); +// +// You can pass in a "const char*" or a "string" for "text". The +// examples below tend to use a const char*. +// +// You can, as in the different examples above, store the RE object +// explicitly in a variable or use a temporary RE object. The +// examples below use one mode or the other arbitrarily. Either +// could correctly be used for any of these examples. +// +// ----------------------------------------------------------------------- +// MATCHING WITH SUB-STRING EXTRACTION: +// +// You can supply extra pointer arguments to extract matched subpieces. +// +// Example: extracts "ruby" into "s" and 1234 into "i" +// int i; +// string s; +// pcrecpp::RE re("(\\w+):(\\d+)"); +// re.FullMatch("ruby:1234", &s, &i); +// +// Example: does not try to extract any extra sub-patterns +// re.FullMatch("ruby:1234", &s); +// +// Example: does not try to extract into NULL +// re.FullMatch("ruby:1234", NULL, &i); +// +// Example: integer overflow causes failure +// !re.FullMatch("ruby:1234567891234", NULL, &i); +// +// Example: fails because there aren't enough sub-patterns: +// !pcrecpp::RE("\\w+:\\d+").FullMatch("ruby:1234", &s); +// +// Example: fails because string cannot be stored in integer +// !pcrecpp::RE("(.*)").FullMatch("ruby", &i); +// +// The provided pointer arguments can be pointers to any scalar numeric +// type, or one of +// string (matched piece is copied to string) +// StringPiece (StringPiece is mutated to point to matched piece) +// T (where "bool T::ParseFrom(const char*, int)" exists) +// NULL (the corresponding matched sub-pattern is not copied) +// +// CAVEAT: An optional sub-pattern that does not exist in the matched +// string is assigned the empty string. Therefore, the following will +// return false (because the empty string is not a valid number): +// int number; +// pcrecpp::RE::FullMatch("abc", "[a-z]+(\\d+)?", &number); +// +// ----------------------------------------------------------------------- +// DO_MATCH +// +// The matching interface supports at most 16 arguments per call. +// If you need more, consider using the more general interface +// pcrecpp::RE::DoMatch(). See pcrecpp.h for the signature for DoMatch. +// +// ----------------------------------------------------------------------- +// PARTIAL MATCHES +// +// You can use the "PartialMatch" operation when you want the pattern +// to match any substring of the text. +// +// Example: simple search for a string: +// pcrecpp::RE("ell").PartialMatch("hello"); +// +// Example: find first number in a string: +// int number; +// pcrecpp::RE re("(\\d+)"); +// re.PartialMatch("x*100 + 20", &number); +// assert(number == 100); +// +// ----------------------------------------------------------------------- +// UTF-8 AND THE MATCHING INTERFACE: +// +// By default, pattern and text are plain text, one byte per character. +// The UTF8 flag, passed to the constructor, causes both pattern +// and string to be treated as UTF-8 text, still a byte stream but +// potentially multiple bytes per character. In practice, the text +// is likelier to be UTF-8 than the pattern, but the match returned +// may depend on the UTF8 flag, so always use it when matching +// UTF8 text. E.g., "." will match one byte normally but with UTF8 +// set may match up to three bytes of a multi-byte character. +// +// Example: +// pcrecpp::RE_Options options; +// options.set_utf8(); +// pcrecpp::RE re(utf8_pattern, options); +// re.FullMatch(utf8_string); +// +// Example: using the convenience function UTF8(): +// pcrecpp::RE re(utf8_pattern, pcrecpp::UTF8()); +// re.FullMatch(utf8_string); +// +// NOTE: The UTF8 option is ignored if pcre was not configured with the +// --enable-utf8 flag. +// +// ----------------------------------------------------------------------- +// PASSING MODIFIERS TO THE REGULAR EXPRESSION ENGINE +// +// PCRE defines some modifiers to change the behavior of the regular +// expression engine. +// The C++ wrapper defines an auxiliary class, RE_Options, as a vehicle +// to pass such modifiers to a RE class. +// +// Currently, the following modifiers are supported +// +// modifier description Perl corresponding +// +// PCRE_CASELESS case insensitive match /i +// PCRE_MULTILINE multiple lines match /m +// PCRE_DOTALL dot matches newlines /s +// PCRE_DOLLAR_ENDONLY $ matches only at end N/A +// PCRE_EXTRA strict escape parsing N/A +// PCRE_EXTENDED ignore whitespaces /x +// PCRE_UTF8 handles UTF8 chars built-in +// PCRE_UNGREEDY reverses * and *? N/A +// PCRE_NO_AUTO_CAPTURE disables matching parens N/A (*) +// +// (For a full account on how each modifier works, please check the +// PCRE API reference manual). +// +// (*) Both Perl and PCRE allow non matching parentheses by means of the +// "?:" modifier within the pattern itself. e.g. (?:ab|cd) does not +// capture, while (ab|cd) does. +// +// For each modifier, there are two member functions whose name is made +// out of the modifier in lowercase, without the "PCRE_" prefix. For +// instance, PCRE_CASELESS is handled by +// bool caseless(), +// which returns true if the modifier is set, and +// RE_Options & set_caseless(bool), +// which sets or unsets the modifier. +// +// Moreover, PCRE_EXTRA_MATCH_LIMIT can be accessed through the +// set_match_limit() and match_limit() member functions. +// Setting match_limit to a non-zero value will limit the executation of +// pcre to keep it from doing bad things like blowing the stack or taking +// an eternity to return a result. A value of 5000 is good enough to stop +// stack blowup in a 2MB thread stack. Setting match_limit to zero will +// disable match limiting. Alternately, you can set match_limit_recursion() +// which uses PCRE_EXTRA_MATCH_LIMIT_RECURSION to limit how much pcre +// recurses. match_limit() caps the number of matches pcre does; +// match_limit_recrusion() caps the depth of recursion. +// +// Normally, to pass one or more modifiers to a RE class, you declare +// a RE_Options object, set the appropriate options, and pass this +// object to a RE constructor. Example: +// +// RE_options opt; +// opt.set_caseless(true); +// +// if (RE("HELLO", opt).PartialMatch("hello world")) ... +// +// RE_options has two constructors. The default constructor takes no +// arguments and creates a set of flags that are off by default. +// +// The optional parameter 'option_flags' is to facilitate transfer +// of legacy code from C programs. This lets you do +// RE(pattern, RE_Options(PCRE_CASELESS|PCRE_MULTILINE)).PartialMatch(str); +// +// But new code is better off doing +// RE(pattern, +// RE_Options().set_caseless(true).set_multiline(true)).PartialMatch(str); +// (See below) +// +// If you are going to pass one of the most used modifiers, there are some +// convenience functions that return a RE_Options class with the +// appropriate modifier already set: +// CASELESS(), UTF8(), MULTILINE(), DOTALL(), EXTENDED() +// +// If you need to set several options at once, and you don't want to go +// through the pains of declaring a RE_Options object and setting several +// options, there is a parallel method that give you such ability on the +// fly. You can concatenate several set_xxxxx member functions, since each +// of them returns a reference to its class object. e.g.: to pass +// PCRE_CASELESS, PCRE_EXTENDED, and PCRE_MULTILINE to a RE with one +// statement, you may write +// +// RE(" ^ xyz \\s+ .* blah$", RE_Options() +// .set_caseless(true) +// .set_extended(true) +// .set_multiline(true)).PartialMatch(sometext); +// +// ----------------------------------------------------------------------- +// SCANNING TEXT INCREMENTALLY +// +// The "Consume" operation may be useful if you want to repeatedly +// match regular expressions at the front of a string and skip over +// them as they match. This requires use of the "StringPiece" type, +// which represents a sub-range of a real string. Like RE, StringPiece +// is defined in the pcrecpp namespace. +// +// Example: read lines of the form "var = value" from a string. +// string contents = ...; // Fill string somehow +// pcrecpp::StringPiece input(contents); // Wrap in a StringPiece +// +// string var; +// int value; +// pcrecpp::RE re("(\\w+) = (\\d+)\n"); +// while (re.Consume(&input, &var, &value)) { +// ...; +// } +// +// Each successful call to "Consume" will set "var/value", and also +// advance "input" so it points past the matched text. +// +// The "FindAndConsume" operation is similar to "Consume" but does not +// anchor your match at the beginning of the string. For example, you +// could extract all words from a string by repeatedly calling +// pcrecpp::RE("(\\w+)").FindAndConsume(&input, &word) +// +// ----------------------------------------------------------------------- +// PARSING HEX/OCTAL/C-RADIX NUMBERS +// +// By default, if you pass a pointer to a numeric value, the +// corresponding text is interpreted as a base-10 number. You can +// instead wrap the pointer with a call to one of the operators Hex(), +// Octal(), or CRadix() to interpret the text in another base. The +// CRadix operator interprets C-style "0" (base-8) and "0x" (base-16) +// prefixes, but defaults to base-10. +// +// Example: +// int a, b, c, d; +// pcrecpp::RE re("(.*) (.*) (.*) (.*)"); +// re.FullMatch("100 40 0100 0x40", +// pcrecpp::Octal(&a), pcrecpp::Hex(&b), +// pcrecpp::CRadix(&c), pcrecpp::CRadix(&d)); +// will leave 64 in a, b, c, and d. +// +// ----------------------------------------------------------------------- +// REPLACING PARTS OF STRINGS +// +// You can replace the first match of "pattern" in "str" with +// "rewrite". Within "rewrite", backslash-escaped digits (\1 to \9) +// can be used to insert text matching corresponding parenthesized +// group from the pattern. \0 in "rewrite" refers to the entire +// matching text. E.g., +// +// string s = "yabba dabba doo"; +// pcrecpp::RE("b+").Replace("d", &s); +// +// will leave "s" containing "yada dabba doo". The result is true if +// the pattern matches and a replacement occurs, or false otherwise. +// +// GlobalReplace() is like Replace(), except that it replaces all +// occurrences of the pattern in the string with the rewrite. +// Replacements are not subject to re-matching. E.g., +// +// string s = "yabba dabba doo"; +// pcrecpp::RE("b+").GlobalReplace("d", &s); +// +// will leave "s" containing "yada dada doo". It returns the number +// of replacements made. +// +// Extract() is like Replace(), except that if the pattern matches, +// "rewrite" is copied into "out" (an additional argument) with +// substitutions. The non-matching portions of "text" are ignored. +// Returns true iff a match occurred and the extraction happened +// successfully. If no match occurs, the string is left unaffected. + + +#include +#include +#include // defines the Arg class +// This isn't technically needed here, but we include it +// anyway so folks who include pcrecpp.h don't have to. +#include + +namespace pcrecpp { + +#define PCRE_SET_OR_CLEAR(b, o) \ + if (b) all_options_ |= (o); else all_options_ &= ~(o); \ + return *this + +#define PCRE_IS_SET(o) \ + (all_options_ & o) == o + +/***** Compiling regular expressions: the RE class *****/ + +// RE_Options allow you to set options to be passed along to pcre, +// along with other options we put on top of pcre. +// Only 9 modifiers, plus match_limit and match_limit_recursion, +// are supported now. +class PCRECPP_EXP_DEFN RE_Options { + public: + // constructor + RE_Options() : match_limit_(0), match_limit_recursion_(0), all_options_(0) {} + + // alternative constructor. + // To facilitate transfer of legacy code from C programs + // + // This lets you do + // RE(pattern, RE_Options(PCRE_CASELESS|PCRE_MULTILINE)).PartialMatch(str); + // But new code is better off doing + // RE(pattern, + // RE_Options().set_caseless(true).set_multiline(true)).PartialMatch(str); + RE_Options(int option_flags) : match_limit_(0), match_limit_recursion_(0), + all_options_(option_flags) {} + // we're fine with the default destructor, copy constructor, etc. + + // accessors and mutators + int match_limit() const { return match_limit_; }; + RE_Options &set_match_limit(int limit) { + match_limit_ = limit; + return *this; + } + + int match_limit_recursion() const { return match_limit_recursion_; }; + RE_Options &set_match_limit_recursion(int limit) { + match_limit_recursion_ = limit; + return *this; + } + + bool caseless() const { + return PCRE_IS_SET(PCRE_CASELESS); + } + RE_Options &set_caseless(bool x) { + PCRE_SET_OR_CLEAR(x, PCRE_CASELESS); + } + + bool multiline() const { + return PCRE_IS_SET(PCRE_MULTILINE); + } + RE_Options &set_multiline(bool x) { + PCRE_SET_OR_CLEAR(x, PCRE_MULTILINE); + } + + bool dotall() const { + return PCRE_IS_SET(PCRE_DOTALL); + } + RE_Options &set_dotall(bool x) { + PCRE_SET_OR_CLEAR(x, PCRE_DOTALL); + } + + bool extended() const { + return PCRE_IS_SET(PCRE_EXTENDED); + } + RE_Options &set_extended(bool x) { + PCRE_SET_OR_CLEAR(x, PCRE_EXTENDED); + } + + bool dollar_endonly() const { + return PCRE_IS_SET(PCRE_DOLLAR_ENDONLY); + } + RE_Options &set_dollar_endonly(bool x) { + PCRE_SET_OR_CLEAR(x, PCRE_DOLLAR_ENDONLY); + } + + bool extra() const { + return PCRE_IS_SET(PCRE_EXTRA); + } + RE_Options &set_extra(bool x) { + PCRE_SET_OR_CLEAR(x, PCRE_EXTRA); + } + + bool ungreedy() const { + return PCRE_IS_SET(PCRE_UNGREEDY); + } + RE_Options &set_ungreedy(bool x) { + PCRE_SET_OR_CLEAR(x, PCRE_UNGREEDY); + } + + bool utf8() const { + return PCRE_IS_SET(PCRE_UTF8); + } + RE_Options &set_utf8(bool x) { + PCRE_SET_OR_CLEAR(x, PCRE_UTF8); + } + + bool no_auto_capture() const { + return PCRE_IS_SET(PCRE_NO_AUTO_CAPTURE); + } + RE_Options &set_no_auto_capture(bool x) { + PCRE_SET_OR_CLEAR(x, PCRE_NO_AUTO_CAPTURE); + } + + RE_Options &set_all_options(int opt) { + all_options_ = opt; + return *this; + } + int all_options() const { + return all_options_ ; + } + + // TODO: add other pcre flags + + private: + int match_limit_; + int match_limit_recursion_; + int all_options_; +}; + +// These functions return some common RE_Options +static inline RE_Options UTF8() { + return RE_Options().set_utf8(true); +} + +static inline RE_Options CASELESS() { + return RE_Options().set_caseless(true); +} +static inline RE_Options MULTILINE() { + return RE_Options().set_multiline(true); +} + +static inline RE_Options DOTALL() { + return RE_Options().set_dotall(true); +} + +static inline RE_Options EXTENDED() { + return RE_Options().set_extended(true); +} + +// Interface for regular expression matching. Also corresponds to a +// pre-compiled regular expression. An "RE" object is safe for +// concurrent use by multiple threads. +class PCRECPP_EXP_DEFN RE { + public: + // We provide implicit conversions from strings so that users can + // pass in a string or a "const char*" wherever an "RE" is expected. + RE(const string& pat) { Init(pat, NULL); } + RE(const string& pat, const RE_Options& option) { Init(pat, &option); } + RE(const char* pat) { Init(pat, NULL); } + RE(const char* pat, const RE_Options& option) { Init(pat, &option); } + RE(const unsigned char* pat) { + Init(reinterpret_cast(pat), NULL); + } + RE(const unsigned char* pat, const RE_Options& option) { + Init(reinterpret_cast(pat), &option); + } + + // Copy constructor & assignment - note that these are expensive + // because they recompile the expression. + RE(const RE& re) { Init(re.pattern_, &re.options_); } + const RE& operator=(const RE& re) { + if (this != &re) { + Cleanup(); + + // This is the code that originally came from Google + // Init(re.pattern_.c_str(), &re.options_); + + // This is the replacement from Ari Pollak + Init(re.pattern_, &re.options_); + } + return *this; + } + + + ~RE(); + + // The string specification for this RE. E.g. + // RE re("ab*c?d+"); + // re.pattern(); // "ab*c?d+" + const string& pattern() const { return pattern_; } + + // If RE could not be created properly, returns an error string. + // Else returns the empty string. + const string& error() const { return *error_; } + + /***** The useful part: the matching interface *****/ + + // This is provided so one can do pattern.ReplaceAll() just as + // easily as ReplaceAll(pattern-text, ....) + + bool FullMatch(const StringPiece& text, + const Arg& ptr1 = no_arg, + const Arg& ptr2 = no_arg, + const Arg& ptr3 = no_arg, + const Arg& ptr4 = no_arg, + const Arg& ptr5 = no_arg, + const Arg& ptr6 = no_arg, + const Arg& ptr7 = no_arg, + const Arg& ptr8 = no_arg, + const Arg& ptr9 = no_arg, + const Arg& ptr10 = no_arg, + const Arg& ptr11 = no_arg, + const Arg& ptr12 = no_arg, + const Arg& ptr13 = no_arg, + const Arg& ptr14 = no_arg, + const Arg& ptr15 = no_arg, + const Arg& ptr16 = no_arg) const; + + bool PartialMatch(const StringPiece& text, + const Arg& ptr1 = no_arg, + const Arg& ptr2 = no_arg, + const Arg& ptr3 = no_arg, + const Arg& ptr4 = no_arg, + const Arg& ptr5 = no_arg, + const Arg& ptr6 = no_arg, + const Arg& ptr7 = no_arg, + const Arg& ptr8 = no_arg, + const Arg& ptr9 = no_arg, + const Arg& ptr10 = no_arg, + const Arg& ptr11 = no_arg, + const Arg& ptr12 = no_arg, + const Arg& ptr13 = no_arg, + const Arg& ptr14 = no_arg, + const Arg& ptr15 = no_arg, + const Arg& ptr16 = no_arg) const; + + bool Consume(StringPiece* input, + const Arg& ptr1 = no_arg, + const Arg& ptr2 = no_arg, + const Arg& ptr3 = no_arg, + const Arg& ptr4 = no_arg, + const Arg& ptr5 = no_arg, + const Arg& ptr6 = no_arg, + const Arg& ptr7 = no_arg, + const Arg& ptr8 = no_arg, + const Arg& ptr9 = no_arg, + const Arg& ptr10 = no_arg, + const Arg& ptr11 = no_arg, + const Arg& ptr12 = no_arg, + const Arg& ptr13 = no_arg, + const Arg& ptr14 = no_arg, + const Arg& ptr15 = no_arg, + const Arg& ptr16 = no_arg) const; + + bool FindAndConsume(StringPiece* input, + const Arg& ptr1 = no_arg, + const Arg& ptr2 = no_arg, + const Arg& ptr3 = no_arg, + const Arg& ptr4 = no_arg, + const Arg& ptr5 = no_arg, + const Arg& ptr6 = no_arg, + const Arg& ptr7 = no_arg, + const Arg& ptr8 = no_arg, + const Arg& ptr9 = no_arg, + const Arg& ptr10 = no_arg, + const Arg& ptr11 = no_arg, + const Arg& ptr12 = no_arg, + const Arg& ptr13 = no_arg, + const Arg& ptr14 = no_arg, + const Arg& ptr15 = no_arg, + const Arg& ptr16 = no_arg) const; + + bool Replace(const StringPiece& rewrite, + string *str) const; + + int GlobalReplace(const StringPiece& rewrite, + string *str) const; + + bool Extract(const StringPiece &rewrite, + const StringPiece &text, + string *out) const; + + // Escapes all potentially meaningful regexp characters in + // 'unquoted'. The returned string, used as a regular expression, + // will exactly match the original string. For example, + // 1.5-2.0? + // may become: + // 1\.5\-2\.0\? + // Note QuoteMeta behaves the same as perl's QuoteMeta function, + // *except* that it escapes the NUL character (\0) as backslash + 0, + // rather than backslash + NUL. + static string QuoteMeta(const StringPiece& unquoted); + + + /***** Generic matching interface *****/ + + // Type of match (TODO: Should be restructured as part of RE_Options) + enum Anchor { + UNANCHORED, // No anchoring + ANCHOR_START, // Anchor at start only + ANCHOR_BOTH // Anchor at start and end + }; + + // General matching routine. Stores the length of the match in + // "*consumed" if successful. + bool DoMatch(const StringPiece& text, + Anchor anchor, + int* consumed, + const Arg* const* args, int n) const; + + // Return the number of capturing subpatterns, or -1 if the + // regexp wasn't valid on construction. + int NumberOfCapturingGroups() const; + + // The default value for an argument, to indicate the end of the argument + // list. This must be used only in optional argument defaults. It should NOT + // be passed explicitly. Some people have tried to use it like this: + // + // FullMatch(x, y, &z, no_arg, &w); + // + // This is a mistake, and will not work. + static Arg no_arg; + + private: + + void Init(const string& pattern, const RE_Options* options); + void Cleanup(); + + // Match against "text", filling in "vec" (up to "vecsize" * 2/3) with + // pairs of integers for the beginning and end positions of matched + // text. The first pair corresponds to the entire matched text; + // subsequent pairs correspond, in order, to parentheses-captured + // matches. Returns the number of pairs (one more than the number of + // the last subpattern with a match) if matching was successful + // and zero if the match failed. + // I.e. for RE("(foo)|(bar)|(baz)") it will return 2, 3, and 4 when matching + // against "foo", "bar", and "baz" respectively. + // When matching RE("(foo)|hello") against "hello", it will return 1. + // But the values for all subpattern are filled in into "vec". + int TryMatch(const StringPiece& text, + int startpos, + Anchor anchor, + bool empty_ok, + int *vec, + int vecsize) const; + + // Append the "rewrite" string, with backslash subsitutions from "text" + // and "vec", to string "out". + bool Rewrite(string *out, + const StringPiece& rewrite, + const StringPiece& text, + int *vec, + int veclen) const; + + // internal implementation for DoMatch + bool DoMatchImpl(const StringPiece& text, + Anchor anchor, + int* consumed, + const Arg* const args[], + int n, + int* vec, + int vecsize) const; + + // Compile the regexp for the specified anchoring mode + pcre* Compile(Anchor anchor); + + string pattern_; + RE_Options options_; + pcre* re_full_; // For full matches + pcre* re_partial_; // For partial matches + const string* error_; // Error indicator (or points to empty string) +}; + +} // namespace pcrecpp + +#endif /* _PCRECPP_H */ diff --git a/src/third_party/pcre-8.41/pcrecpp_internal.h b/src/third_party/pcre-8.41/pcrecpp_internal.h new file mode 100644 index 00000000000..827f9e04e2a --- /dev/null +++ b/src/third_party/pcre-8.41/pcrecpp_internal.h @@ -0,0 +1,71 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* +Copyright (c) 2005, Google Inc. +All rights reserved. + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +#ifndef PCRECPP_INTERNAL_H +#define PCRECPP_INTERNAL_H + +/* When compiling a DLL for Windows, the exported symbols have to be declared +using some MS magic. I found some useful information on this web page: +http://msdn2.microsoft.com/en-us/library/y4h7bcy6(VS.80).aspx. According to the +information there, using __declspec(dllexport) without "extern" we have a +definition; with "extern" we have a declaration. The settings here override the +setting in pcre.h. We use: + + PCRECPP_EXP_DECL for declarations + PCRECPP_EXP_DEFN for definitions of exported functions + +*/ + +#ifndef PCRECPP_EXP_DECL +# ifdef _WIN32 +# ifndef PCRE_STATIC +# define PCRECPP_EXP_DECL extern __declspec(dllexport) +# define PCRECPP_EXP_DEFN __declspec(dllexport) +# else +# define PCRECPP_EXP_DECL extern +# define PCRECPP_EXP_DEFN +# endif +# else +# define PCRECPP_EXP_DECL extern +# define PCRECPP_EXP_DEFN +# endif +#endif + +#endif /* PCRECPP_INTERNAL_H */ + +/* End of pcrecpp_internal.h */ diff --git a/src/third_party/pcre-8.41/pcrecpp_unittest.cc b/src/third_party/pcre-8.41/pcrecpp_unittest.cc new file mode 100644 index 00000000000..4b15fbef1c3 --- /dev/null +++ b/src/third_party/pcre-8.41/pcrecpp_unittest.cc @@ -0,0 +1,1292 @@ +// -*- coding: utf-8 -*- +// +// Copyright (c) 2005 - 2010, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Sanjay Ghemawat +// +// TODO: Test extractions for PartialMatch/Consume + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include /* for memset and strcmp */ +#include +#include +#include "pcrecpp.h" + +using std::string; +using pcrecpp::StringPiece; +using pcrecpp::RE; +using pcrecpp::RE_Options; +using pcrecpp::Hex; +using pcrecpp::Octal; +using pcrecpp::CRadix; + +static bool VERBOSE_TEST = false; + +// CHECK dies with a fatal error if condition is not true. It is *not* +// controlled by NDEBUG, so the check will be executed regardless of +// compilation mode. Therefore, it is safe to do things like: +// CHECK_EQ(fp->Write(x), 4) +#define CHECK(condition) do { \ + if (!(condition)) { \ + fprintf(stderr, "%s:%d: Check failed: %s\n", \ + __FILE__, __LINE__, #condition); \ + exit(1); \ + } \ +} while (0) + +#define CHECK_EQ(a, b) CHECK(a == b) + +static void Timing1(int num_iters) { + // Same pattern lots of times + RE pattern("ruby:\\d+"); + StringPiece p("ruby:1234"); + for (int j = num_iters; j > 0; j--) { + CHECK(pattern.FullMatch(p)); + } +} + +static void Timing2(int num_iters) { + // Same pattern lots of times + RE pattern("ruby:(\\d+)"); + int i; + for (int j = num_iters; j > 0; j--) { + CHECK(pattern.FullMatch("ruby:1234", &i)); + CHECK_EQ(i, 1234); + } +} + +static void Timing3(int num_iters) { + string text_string; + for (int j = num_iters; j > 0; j--) { + text_string += "this is another line\n"; + } + + RE line_matcher(".*\n"); + string line; + StringPiece text(text_string); + int counter = 0; + while (line_matcher.Consume(&text)) { + counter++; + } + printf("Matched %d lines\n", counter); +} + +#if 0 // uncomment this if you have a way of defining VirtualProcessSize() + +static void LeakTest() { + // Check for memory leaks + unsigned long long initial_size = 0; + for (int i = 0; i < 100000; i++) { + if (i == 50000) { + initial_size = VirtualProcessSize(); + printf("Size after 50000: %llu\n", initial_size); + } + char buf[100]; // definitely big enough + sprintf(buf, "pat%09d", i); + RE newre(buf); + } + uint64 final_size = VirtualProcessSize(); + printf("Size after 100000: %llu\n", final_size); + const double growth = double(final_size - initial_size) / final_size; + printf("Growth: %0.2f%%", growth * 100); + CHECK(growth < 0.02); // Allow < 2% growth +} + +#endif + +static void RadixTests() { + printf("Testing hex\n"); + +#define CHECK_HEX(type, value) \ + do { \ + type v; \ + CHECK(RE("([0-9a-fA-F]+)[uUlL]*").FullMatch(#value, Hex(&v))); \ + CHECK_EQ(v, 0x ## value); \ + CHECK(RE("([0-9a-fA-FxX]+)[uUlL]*").FullMatch("0x" #value, CRadix(&v))); \ + CHECK_EQ(v, 0x ## value); \ + } while(0) + + CHECK_HEX(short, 2bad); + CHECK_HEX(unsigned short, 2badU); + CHECK_HEX(int, dead); + CHECK_HEX(unsigned int, deadU); + CHECK_HEX(long, 7eadbeefL); + CHECK_HEX(unsigned long, deadbeefUL); +#ifdef HAVE_LONG_LONG + CHECK_HEX(long long, 12345678deadbeefLL); +#endif +#ifdef HAVE_UNSIGNED_LONG_LONG + CHECK_HEX(unsigned long long, cafebabedeadbeefULL); +#endif + +#undef CHECK_HEX + + printf("Testing octal\n"); + +#define CHECK_OCTAL(type, value) \ + do { \ + type v; \ + CHECK(RE("([0-7]+)[uUlL]*").FullMatch(#value, Octal(&v))); \ + CHECK_EQ(v, 0 ## value); \ + CHECK(RE("([0-9a-fA-FxX]+)[uUlL]*").FullMatch("0" #value, CRadix(&v))); \ + CHECK_EQ(v, 0 ## value); \ + } while(0) + + CHECK_OCTAL(short, 77777); + CHECK_OCTAL(unsigned short, 177777U); + CHECK_OCTAL(int, 17777777777); + CHECK_OCTAL(unsigned int, 37777777777U); + CHECK_OCTAL(long, 17777777777L); + CHECK_OCTAL(unsigned long, 37777777777UL); +#ifdef HAVE_LONG_LONG + CHECK_OCTAL(long long, 777777777777777777777LL); +#endif +#ifdef HAVE_UNSIGNED_LONG_LONG + CHECK_OCTAL(unsigned long long, 1777777777777777777777ULL); +#endif + +#undef CHECK_OCTAL + + printf("Testing decimal\n"); + +#define CHECK_DECIMAL(type, value) \ + do { \ + type v; \ + CHECK(RE("(-?[0-9]+)[uUlL]*").FullMatch(#value, &v)); \ + CHECK_EQ(v, value); \ + CHECK(RE("(-?[0-9a-fA-FxX]+)[uUlL]*").FullMatch(#value, CRadix(&v))); \ + CHECK_EQ(v, value); \ + } while(0) + + CHECK_DECIMAL(short, -1); + CHECK_DECIMAL(unsigned short, 9999); + CHECK_DECIMAL(int, -1000); + CHECK_DECIMAL(unsigned int, 12345U); + CHECK_DECIMAL(long, -10000000L); + CHECK_DECIMAL(unsigned long, 3083324652U); +#ifdef HAVE_LONG_LONG + CHECK_DECIMAL(long long, -100000000000000LL); +#endif +#ifdef HAVE_UNSIGNED_LONG_LONG + CHECK_DECIMAL(unsigned long long, 1234567890987654321ULL); +#endif + +#undef CHECK_DECIMAL + +} + +static void TestReplace() { + printf("Testing Replace\n"); + + struct ReplaceTest { + const char *regexp; + const char *rewrite; + const char *original; + const char *single; + const char *global; + int global_count; // the expected return value from ReplaceAll + }; + static const ReplaceTest tests[] = { + { "(qu|[b-df-hj-np-tv-z]*)([a-z]+)", + "\\2\\1ay", + "the quick brown fox jumps over the lazy dogs.", + "ethay quick brown fox jumps over the lazy dogs.", + "ethay ickquay ownbray oxfay umpsjay overay ethay azylay ogsday.", + 9 }, + { "\\w+", + "\\0-NOSPAM", + "paul.haahr@google.com", + "paul-NOSPAM.haahr@google.com", + "paul-NOSPAM.haahr-NOSPAM@google-NOSPAM.com-NOSPAM", + 4 }, + { "^", + "(START)", + "foo", + "(START)foo", + "(START)foo", + 1 }, + { "^", + "(START)", + "", + "(START)", + "(START)", + 1 }, + { "$", + "(END)", + "", + "(END)", + "(END)", + 1 }, + { "b", + "bb", + "ababababab", + "abbabababab", + "abbabbabbabbabb", + 5 }, + { "b", + "bb", + "bbbbbb", + "bbbbbbb", + "bbbbbbbbbbbb", + 6 }, + { "b+", + "bb", + "bbbbbb", + "bb", + "bb", + 1 }, + { "b*", + "bb", + "bbbbbb", + "bb", + "bbbb", + 2 }, + { "b*", + "bb", + "aaaaa", + "bbaaaaa", + "bbabbabbabbabbabb", + 6 }, + { "b*", + "bb", + "aa\naa\n", + "bbaa\naa\n", + "bbabbabb\nbbabbabb\nbb", + 7 }, + { "b*", + "bb", + "aa\raa\r", + "bbaa\raa\r", + "bbabbabb\rbbabbabb\rbb", + 7 }, + { "b*", + "bb", + "aa\r\naa\r\n", + "bbaa\r\naa\r\n", + "bbabbabb\r\nbbabbabb\r\nbb", + 7 }, + // Check empty-string matching (it's tricky!) + { "aa|b*", + "@", + "aa", + "@", + "@@", + 2 }, + { "b*|aa", + "@", + "aa", + "@aa", + "@@@", + 3 }, +#ifdef SUPPORT_UTF8 + { "b*", + "bb", + "\xE3\x83\x9B\xE3\x83\xBC\xE3\x83\xA0\xE3\x81\xB8", // utf8 + "bb\xE3\x83\x9B\xE3\x83\xBC\xE3\x83\xA0\xE3\x81\xB8", + "bb\xE3\x83\x9B""bb""\xE3\x83\xBC""bb""\xE3\x83\xA0""bb""\xE3\x81\xB8""bb", + 5 }, + { "b*", + "bb", + "\xE3\x83\x9B\r\n\xE3\x83\xBC\r\xE3\x83\xA0\n\xE3\x81\xB8\r\n", // utf8 + "bb\xE3\x83\x9B\r\n\xE3\x83\xBC\r\xE3\x83\xA0\n\xE3\x81\xB8\r\n", + ("bb\xE3\x83\x9B""bb\r\nbb""\xE3\x83\xBC""bb\rbb""\xE3\x83\xA0" + "bb\nbb""\xE3\x81\xB8""bb\r\nbb"), + 9 }, +#endif + { "", NULL, NULL, NULL, NULL, 0 } + }; + +#ifdef SUPPORT_UTF8 + const bool support_utf8 = true; +#else + const bool support_utf8 = false; +#endif + + for (const ReplaceTest *t = tests; t->original != NULL; ++t) { + RE re(t->regexp, RE_Options(PCRE_NEWLINE_CRLF).set_utf8(support_utf8)); + assert(re.error().empty()); + string one(t->original); + CHECK(re.Replace(t->rewrite, &one)); + CHECK_EQ(one, t->single); + string all(t->original); + const int replace_count = re.GlobalReplace(t->rewrite, &all); + CHECK_EQ(all, t->global); + CHECK_EQ(replace_count, t->global_count); + } + + // One final test: test \r\n replacement when we're not in CRLF mode + { + RE re("b*", RE_Options(PCRE_NEWLINE_CR).set_utf8(support_utf8)); + assert(re.error().empty()); + string all("aa\r\naa\r\n"); + CHECK_EQ(re.GlobalReplace("bb", &all), 9); + CHECK_EQ(all, string("bbabbabb\rbb\nbbabbabb\rbb\nbb")); + } + { + RE re("b*", RE_Options(PCRE_NEWLINE_LF).set_utf8(support_utf8)); + assert(re.error().empty()); + string all("aa\r\naa\r\n"); + CHECK_EQ(re.GlobalReplace("bb", &all), 9); + CHECK_EQ(all, string("bbabbabb\rbb\nbbabbabb\rbb\nbb")); + } + // TODO: test what happens when no PCRE_NEWLINE_* flag is set. + // Alas, the answer depends on how pcre was compiled. +} + +static void TestExtract() { + printf("Testing Extract\n"); + + string s; + + CHECK(RE("(.*)@([^.]*)").Extract("\\2!\\1", "boris@kremvax.ru", &s)); + CHECK_EQ(s, "kremvax!boris"); + + // check the RE interface as well + CHECK(RE(".*").Extract("'\\0'", "foo", &s)); + CHECK_EQ(s, "'foo'"); + CHECK(!RE("bar").Extract("'\\0'", "baz", &s)); + CHECK_EQ(s, "'foo'"); +} + +static void TestConsume() { + printf("Testing Consume\n"); + + string word; + + string s(" aaa b!@#$@#$cccc"); + StringPiece input(s); + + RE r("\\s*(\\w+)"); // matches a word, possibly proceeded by whitespace + CHECK(r.Consume(&input, &word)); + CHECK_EQ(word, "aaa"); + CHECK(r.Consume(&input, &word)); + CHECK_EQ(word, "b"); + CHECK(! r.Consume(&input, &word)); +} + +static void TestFindAndConsume() { + printf("Testing FindAndConsume\n"); + + string word; + + string s(" aaa b!@#$@#$cccc"); + StringPiece input(s); + + RE r("(\\w+)"); // matches a word + CHECK(r.FindAndConsume(&input, &word)); + CHECK_EQ(word, "aaa"); + CHECK(r.FindAndConsume(&input, &word)); + CHECK_EQ(word, "b"); + CHECK(r.FindAndConsume(&input, &word)); + CHECK_EQ(word, "cccc"); + CHECK(! r.FindAndConsume(&input, &word)); +} + +static void TestMatchNumberPeculiarity() { + printf("Testing match-number peculiarity\n"); + + string word1; + string word2; + string word3; + + RE r("(foo)|(bar)|(baz)"); + CHECK(r.PartialMatch("foo", &word1, &word2, &word3)); + CHECK_EQ(word1, "foo"); + CHECK_EQ(word2, ""); + CHECK_EQ(word3, ""); + CHECK(r.PartialMatch("bar", &word1, &word2, &word3)); + CHECK_EQ(word1, ""); + CHECK_EQ(word2, "bar"); + CHECK_EQ(word3, ""); + CHECK(r.PartialMatch("baz", &word1, &word2, &word3)); + CHECK_EQ(word1, ""); + CHECK_EQ(word2, ""); + CHECK_EQ(word3, "baz"); + CHECK(!r.PartialMatch("f", &word1, &word2, &word3)); + + string a; + CHECK(RE("(foo)|hello").FullMatch("hello", &a)); + CHECK_EQ(a, ""); +} + +static void TestRecursion() { + printf("Testing recursion\n"); + + // Get one string that passes (sometimes), one that never does. + string text_good("abcdefghijk"); + string text_bad("acdefghijkl"); + + // According to pcretest, matching text_good against (\w+)*b + // requires match_limit of at least 8192, and match_recursion_limit + // of at least 37. + + RE_Options options_ml; + options_ml.set_match_limit(8192); + RE re("(\\w+)*b", options_ml); + CHECK(re.PartialMatch(text_good) == true); + CHECK(re.PartialMatch(text_bad) == false); + CHECK(re.FullMatch(text_good) == false); + CHECK(re.FullMatch(text_bad) == false); + + options_ml.set_match_limit(1024); + RE re2("(\\w+)*b", options_ml); + CHECK(re2.PartialMatch(text_good) == false); // because of match_limit + CHECK(re2.PartialMatch(text_bad) == false); + CHECK(re2.FullMatch(text_good) == false); + CHECK(re2.FullMatch(text_bad) == false); + + RE_Options options_mlr; + options_mlr.set_match_limit_recursion(50); + RE re3("(\\w+)*b", options_mlr); + CHECK(re3.PartialMatch(text_good) == true); + CHECK(re3.PartialMatch(text_bad) == false); + CHECK(re3.FullMatch(text_good) == false); + CHECK(re3.FullMatch(text_bad) == false); + + options_mlr.set_match_limit_recursion(10); + RE re4("(\\w+)*b", options_mlr); + CHECK(re4.PartialMatch(text_good) == false); + CHECK(re4.PartialMatch(text_bad) == false); + CHECK(re4.FullMatch(text_good) == false); + CHECK(re4.FullMatch(text_bad) == false); +} + +// A meta-quoted string, interpreted as a pattern, should always match +// the original unquoted string. +static void TestQuoteMeta(string unquoted, RE_Options options = RE_Options()) { + string quoted = RE::QuoteMeta(unquoted); + RE re(quoted, options); + CHECK(re.FullMatch(unquoted)); +} + +// A string containing meaningful regexp characters, which is then meta- +// quoted, should not generally match a string the unquoted string does. +static void NegativeTestQuoteMeta(string unquoted, string should_not_match, + RE_Options options = RE_Options()) { + string quoted = RE::QuoteMeta(unquoted); + RE re(quoted, options); + CHECK(!re.FullMatch(should_not_match)); +} + +// Tests that quoted meta characters match their original strings, +// and that a few things that shouldn't match indeed do not. +static void TestQuotaMetaSimple() { + TestQuoteMeta("foo"); + TestQuoteMeta("foo.bar"); + TestQuoteMeta("foo\\.bar"); + TestQuoteMeta("[1-9]"); + TestQuoteMeta("1.5-2.0?"); + TestQuoteMeta("\\d"); + TestQuoteMeta("Who doesn't like ice cream?"); + TestQuoteMeta("((a|b)c?d*e+[f-h]i)"); + TestQuoteMeta("((?!)xxx).*yyy"); + TestQuoteMeta("(["); + TestQuoteMeta(string("foo\0bar", 7)); +} + +static void TestQuoteMetaSimpleNegative() { + NegativeTestQuoteMeta("foo", "bar"); + NegativeTestQuoteMeta("...", "bar"); + NegativeTestQuoteMeta("\\.", "."); + NegativeTestQuoteMeta("\\.", ".."); + NegativeTestQuoteMeta("(a)", "a"); + NegativeTestQuoteMeta("(a|b)", "a"); + NegativeTestQuoteMeta("(a|b)", "(a)"); + NegativeTestQuoteMeta("(a|b)", "a|b"); + NegativeTestQuoteMeta("[0-9]", "0"); + NegativeTestQuoteMeta("[0-9]", "0-9"); + NegativeTestQuoteMeta("[0-9]", "[9]"); + NegativeTestQuoteMeta("((?!)xxx)", "xxx"); +} + +static void TestQuoteMetaLatin1() { + TestQuoteMeta("3\xb2 = 9"); +} + +static void TestQuoteMetaUtf8() { +#ifdef SUPPORT_UTF8 + TestQuoteMeta("Pl\xc3\xa1\x63ido Domingo", pcrecpp::UTF8()); + TestQuoteMeta("xyz", pcrecpp::UTF8()); // No fancy utf8 + TestQuoteMeta("\xc2\xb0", pcrecpp::UTF8()); // 2-byte utf8 (degree symbol) + TestQuoteMeta("27\xc2\xb0 degrees", pcrecpp::UTF8()); // As a middle character + TestQuoteMeta("\xe2\x80\xb3", pcrecpp::UTF8()); // 3-byte utf8 (double prime) + TestQuoteMeta("\xf0\x9d\x85\x9f", pcrecpp::UTF8()); // 4-byte utf8 (music note) + TestQuoteMeta("27\xc2\xb0"); // Interpreted as Latin-1, but should still work + NegativeTestQuoteMeta("27\xc2\xb0", // 2-byte utf (degree symbol) + "27\\\xc2\\\xb0", + pcrecpp::UTF8()); +#endif +} + +static void TestQuoteMetaAll() { + printf("Testing QuoteMeta\n"); + TestQuotaMetaSimple(); + TestQuoteMetaSimpleNegative(); + TestQuoteMetaLatin1(); + TestQuoteMetaUtf8(); +} + +// +// Options tests contributed by +// Giuseppe Maxia, CTO, Stardata s.r.l. +// July 2005 +// +static void GetOneOptionResult( + const char *option_name, + const char *regex, + const char *str, + RE_Options options, + bool full, + string expected) { + + printf("Testing Option <%s>\n", option_name); + if(VERBOSE_TEST) + printf("/%s/ finds \"%s\" within \"%s\" \n", + regex, + expected.c_str(), + str); + string captured(""); + if (full) + RE(regex,options).FullMatch(str, &captured); + else + RE(regex,options).PartialMatch(str, &captured); + CHECK_EQ(captured, expected); +} + +static void TestOneOption( + const char *option_name, + const char *regex, + const char *str, + RE_Options options, + bool full, + bool assertive = true) { + + printf("Testing Option <%s>\n", option_name); + if (VERBOSE_TEST) + printf("'%s' %s /%s/ \n", + str, + (assertive? "matches" : "doesn't match"), + regex); + if (assertive) { + if (full) + CHECK(RE(regex,options).FullMatch(str)); + else + CHECK(RE(regex,options).PartialMatch(str)); + } else { + if (full) + CHECK(!RE(regex,options).FullMatch(str)); + else + CHECK(!RE(regex,options).PartialMatch(str)); + } +} + +static void Test_CASELESS() { + RE_Options options; + RE_Options options2; + + options.set_caseless(true); + TestOneOption("CASELESS (class)", "HELLO", "hello", options, false); + TestOneOption("CASELESS (class2)", "HELLO", "hello", options2.set_caseless(true), false); + TestOneOption("CASELESS (class)", "^[A-Z]+$", "Hello", options, false); + + TestOneOption("CASELESS (function)", "HELLO", "hello", pcrecpp::CASELESS(), false); + TestOneOption("CASELESS (function)", "^[A-Z]+$", "Hello", pcrecpp::CASELESS(), false); + options.set_caseless(false); + TestOneOption("no CASELESS", "HELLO", "hello", options, false, false); +} + +static void Test_MULTILINE() { + RE_Options options; + RE_Options options2; + const char *str = "HELLO\n" "cruel\n" "world\n"; + + options.set_multiline(true); + TestOneOption("MULTILINE (class)", "^cruel$", str, options, false); + TestOneOption("MULTILINE (class2)", "^cruel$", str, options2.set_multiline(true), false); + TestOneOption("MULTILINE (function)", "^cruel$", str, pcrecpp::MULTILINE(), false); + options.set_multiline(false); + TestOneOption("no MULTILINE", "^cruel$", str, options, false, false); +} + +static void Test_DOTALL() { + RE_Options options; + RE_Options options2; + const char *str = "HELLO\n" "cruel\n" "world"; + + options.set_dotall(true); + TestOneOption("DOTALL (class)", "HELLO.*world", str, options, true); + TestOneOption("DOTALL (class2)", "HELLO.*world", str, options2.set_dotall(true), true); + TestOneOption("DOTALL (function)", "HELLO.*world", str, pcrecpp::DOTALL(), true); + options.set_dotall(false); + TestOneOption("no DOTALL", "HELLO.*world", str, options, true, false); +} + +static void Test_DOLLAR_ENDONLY() { + RE_Options options; + RE_Options options2; + const char *str = "HELLO world\n"; + + TestOneOption("no DOLLAR_ENDONLY", "world$", str, options, false); + options.set_dollar_endonly(true); + TestOneOption("DOLLAR_ENDONLY 1", "world$", str, options, false, false); + TestOneOption("DOLLAR_ENDONLY 2", "world$", str, options2.set_dollar_endonly(true), false, false); +} + +static void Test_EXTRA() { + RE_Options options; + const char *str = "HELLO"; + + options.set_extra(true); + TestOneOption("EXTRA 1", "\\HELL\\O", str, options, true, false ); + TestOneOption("EXTRA 2", "\\HELL\\O", str, RE_Options().set_extra(true), true, false ); + options.set_extra(false); + TestOneOption("no EXTRA", "\\HELL\\O", str, options, true ); +} + +static void Test_EXTENDED() { + RE_Options options; + RE_Options options2; + const char *str = "HELLO world"; + + options.set_extended(true); + TestOneOption("EXTENDED (class)", "HELLO world", str, options, false, false); + TestOneOption("EXTENDED (class2)", "HELLO world", str, options2.set_extended(true), false, false); + TestOneOption("EXTENDED (class)", + "^ HE L{2} O " + "\\s+ " + "\\w+ $ ", + str, + options, + false); + + TestOneOption("EXTENDED (function)", "HELLO world", str, pcrecpp::EXTENDED(), false, false); + TestOneOption("EXTENDED (function)", + "^ HE L{2} O " + "\\s+ " + "\\w+ $ ", + str, + pcrecpp::EXTENDED(), + false); + + options.set_extended(false); + TestOneOption("no EXTENDED", "HELLO world", str, options, false); +} + +static void Test_NO_AUTO_CAPTURE() { + RE_Options options; + const char *str = "HELLO world"; + string captured; + + printf("Testing Option \n"); + if (VERBOSE_TEST) + printf("parentheses capture text\n"); + RE re("(world|universe)$", options); + CHECK(re.Extract("\\1", str , &captured)); + CHECK_EQ(captured, "world"); + options.set_no_auto_capture(true); + printf("testing Option \n"); + if (VERBOSE_TEST) + printf("parentheses do not capture text\n"); + re.Extract("\\1",str, &captured ); + CHECK_EQ(captured, "world"); +} + +static void Test_UNGREEDY() { + RE_Options options; + const char *str = "HELLO, 'this' is the 'world'"; + + options.set_ungreedy(true); + GetOneOptionResult("UNGREEDY 1", "('.*')", str, options, false, "'this'" ); + GetOneOptionResult("UNGREEDY 2", "('.*')", str, RE_Options().set_ungreedy(true), false, "'this'" ); + GetOneOptionResult("UNGREEDY", "('.*?')", str, options, false, "'this' is the 'world'" ); + + options.set_ungreedy(false); + GetOneOptionResult("no UNGREEDY", "('.*')", str, options, false, "'this' is the 'world'" ); + GetOneOptionResult("no UNGREEDY", "('.*?')", str, options, false, "'this'" ); +} + +static void Test_all_options() { + const char *str = "HELLO\n" "cruel\n" "world"; + RE_Options options; + options.set_all_options(PCRE_CASELESS | PCRE_DOTALL); + + TestOneOption("all_options (CASELESS|DOTALL)", "^hello.*WORLD", str , options, false); + options.set_all_options(0); + TestOneOption("all_options (0)", "^hello.*WORLD", str , options, false, false); + options.set_all_options(PCRE_MULTILINE | PCRE_EXTENDED); + + TestOneOption("all_options (MULTILINE|EXTENDED)", " ^ c r u e l $ ", str, options, false); + TestOneOption("all_options (MULTILINE|EXTENDED) with constructor", + " ^ c r u e l $ ", + str, + RE_Options(PCRE_MULTILINE | PCRE_EXTENDED), + false); + + TestOneOption("all_options (MULTILINE|EXTENDED) with concatenation", + " ^ c r u e l $ ", + str, + RE_Options() + .set_multiline(true) + .set_extended(true), + false); + + options.set_all_options(0); + TestOneOption("all_options (0)", "^ c r u e l $", str, options, false, false); + +} + +static void TestOptions() { + printf("Testing Options\n"); + Test_CASELESS(); + Test_MULTILINE(); + Test_DOTALL(); + Test_DOLLAR_ENDONLY(); + Test_EXTENDED(); + Test_NO_AUTO_CAPTURE(); + Test_UNGREEDY(); + Test_EXTRA(); + Test_all_options(); +} + +static void TestConstructors() { + printf("Testing constructors\n"); + + RE_Options options; + options.set_dotall(true); + const char *str = "HELLO\n" "cruel\n" "world"; + + RE orig("HELLO.*world", options); + CHECK(orig.FullMatch(str)); + + RE copy1(orig); + CHECK(copy1.FullMatch(str)); + + RE copy2("not a match"); + CHECK(!copy2.FullMatch(str)); + copy2 = copy1; + CHECK(copy2.FullMatch(str)); + copy2 = orig; + CHECK(copy2.FullMatch(str)); + + // Make sure when we assign to ourselves, nothing bad happens + orig = orig; + copy1 = copy1; + copy2 = copy2; + CHECK(orig.FullMatch(str)); + CHECK(copy1.FullMatch(str)); + CHECK(copy2.FullMatch(str)); +} + +int main(int argc, char** argv) { + // Treat any flag as --help + if (argc > 1 && argv[1][0] == '-') { + printf("Usage: %s [timing1|timing2|timing3 num-iters]\n" + " If 'timingX ###' is specified, run the given timing test\n" + " with the given number of iterations, rather than running\n" + " the default corectness test.\n", argv[0]); + return 0; + } + + if (argc > 1) { + if ( argc == 2 || atoi(argv[2]) == 0) { + printf("timing mode needs a num-iters argument\n"); + return 1; + } + if (!strcmp(argv[1], "timing1")) + Timing1(atoi(argv[2])); + else if (!strcmp(argv[1], "timing2")) + Timing2(atoi(argv[2])); + else if (!strcmp(argv[1], "timing3")) + Timing3(atoi(argv[2])); + else + printf("Unknown argument '%s'\n", argv[1]); + return 0; + } + + printf("PCRE C++ wrapper tests\n"); + printf("Testing FullMatch\n"); + + int i; + string s; + + /***** FullMatch with no args *****/ + + CHECK(RE("h.*o").FullMatch("hello")); + CHECK(!RE("h.*o").FullMatch("othello")); // Must be anchored at front + CHECK(!RE("h.*o").FullMatch("hello!")); // Must be anchored at end + CHECK(RE("a*").FullMatch("aaaa")); // Fullmatch with normal op + CHECK(RE("a*?").FullMatch("aaaa")); // Fullmatch with nongreedy op + CHECK(RE("a*?\\z").FullMatch("aaaa")); // Two unusual ops + + /***** FullMatch with args *****/ + + // Zero-arg + CHECK(RE("\\d+").FullMatch("1001")); + + // Single-arg + CHECK(RE("(\\d+)").FullMatch("1001", &i)); + CHECK_EQ(i, 1001); + CHECK(RE("(-?\\d+)").FullMatch("-123", &i)); + CHECK_EQ(i, -123); + CHECK(!RE("()\\d+").FullMatch("10", &i)); + CHECK(!RE("(\\d+)").FullMatch("1234567890123456789012345678901234567890", + &i)); + + // Digits surrounding integer-arg + CHECK(RE("1(\\d*)4").FullMatch("1234", &i)); + CHECK_EQ(i, 23); + CHECK(RE("(\\d)\\d+").FullMatch("1234", &i)); + CHECK_EQ(i, 1); + CHECK(RE("(-\\d)\\d+").FullMatch("-1234", &i)); + CHECK_EQ(i, -1); + CHECK(RE("(\\d)").PartialMatch("1234", &i)); + CHECK_EQ(i, 1); + CHECK(RE("(-\\d)").PartialMatch("-1234", &i)); + CHECK_EQ(i, -1); + + // String-arg + CHECK(RE("h(.*)o").FullMatch("hello", &s)); + CHECK_EQ(s, string("ell")); + + // StringPiece-arg + StringPiece sp; + CHECK(RE("(\\w+):(\\d+)").FullMatch("ruby:1234", &sp, &i)); + CHECK_EQ(sp.size(), 4); + CHECK(memcmp(sp.data(), "ruby", 4) == 0); + CHECK_EQ(i, 1234); + + // Multi-arg + CHECK(RE("(\\w+):(\\d+)").FullMatch("ruby:1234", &s, &i)); + CHECK_EQ(s, string("ruby")); + CHECK_EQ(i, 1234); + + // Ignore non-void* NULL arg + CHECK(RE("he(.*)lo").FullMatch("hello", (char*)NULL)); + CHECK(RE("h(.*)o").FullMatch("hello", (string*)NULL)); + CHECK(RE("h(.*)o").FullMatch("hello", (StringPiece*)NULL)); + CHECK(RE("(.*)").FullMatch("1234", (int*)NULL)); +#ifdef HAVE_LONG_LONG + CHECK(RE("(.*)").FullMatch("1234567890123456", (long long*)NULL)); +#endif + CHECK(RE("(.*)").FullMatch("123.4567890123456", (double*)NULL)); + CHECK(RE("(.*)").FullMatch("123.4567890123456", (float*)NULL)); + + // Fail on non-void* NULL arg if the match doesn't parse for the given type. + CHECK(!RE("h(.*)lo").FullMatch("hello", &s, (char*)NULL)); + CHECK(!RE("(.*)").FullMatch("hello", (int*)NULL)); + CHECK(!RE("(.*)").FullMatch("1234567890123456", (int*)NULL)); + CHECK(!RE("(.*)").FullMatch("hello", (double*)NULL)); + CHECK(!RE("(.*)").FullMatch("hello", (float*)NULL)); + + // Ignored arg + CHECK(RE("(\\w+)(:)(\\d+)").FullMatch("ruby:1234", &s, (void*)NULL, &i)); + CHECK_EQ(s, string("ruby")); + CHECK_EQ(i, 1234); + + // Type tests + { + char c; + CHECK(RE("(H)ello").FullMatch("Hello", &c)); + CHECK_EQ(c, 'H'); + } + { + unsigned char c; + CHECK(RE("(H)ello").FullMatch("Hello", &c)); + CHECK_EQ(c, static_cast('H')); + } + { + short v; + CHECK(RE("(-?\\d+)").FullMatch("100", &v)); CHECK_EQ(v, 100); + CHECK(RE("(-?\\d+)").FullMatch("-100", &v)); CHECK_EQ(v, -100); + CHECK(RE("(-?\\d+)").FullMatch("32767", &v)); CHECK_EQ(v, 32767); + CHECK(RE("(-?\\d+)").FullMatch("-32768", &v)); CHECK_EQ(v, -32768); + CHECK(!RE("(-?\\d+)").FullMatch("-32769", &v)); + CHECK(!RE("(-?\\d+)").FullMatch("32768", &v)); + } + { + unsigned short v; + CHECK(RE("(\\d+)").FullMatch("100", &v)); CHECK_EQ(v, 100); + CHECK(RE("(\\d+)").FullMatch("32767", &v)); CHECK_EQ(v, 32767); + CHECK(RE("(\\d+)").FullMatch("65535", &v)); CHECK_EQ(v, 65535); + CHECK(!RE("(\\d+)").FullMatch("65536", &v)); + } + { + int v; + static const int max_value = 0x7fffffff; + static const int min_value = -max_value - 1; + CHECK(RE("(-?\\d+)").FullMatch("100", &v)); CHECK_EQ(v, 100); + CHECK(RE("(-?\\d+)").FullMatch("-100", &v)); CHECK_EQ(v, -100); + CHECK(RE("(-?\\d+)").FullMatch("2147483647", &v)); CHECK_EQ(v, max_value); + CHECK(RE("(-?\\d+)").FullMatch("-2147483648", &v)); CHECK_EQ(v, min_value); + CHECK(!RE("(-?\\d+)").FullMatch("-2147483649", &v)); + CHECK(!RE("(-?\\d+)").FullMatch("2147483648", &v)); + } + { + unsigned int v; + static const unsigned int max_value = 0xfffffffful; + CHECK(RE("(\\d+)").FullMatch("100", &v)); CHECK_EQ(v, 100); + CHECK(RE("(\\d+)").FullMatch("4294967295", &v)); CHECK_EQ(v, max_value); + CHECK(!RE("(\\d+)").FullMatch("4294967296", &v)); + } +#ifdef HAVE_LONG_LONG +# if defined(__MINGW__) || defined(__MINGW32__) +# define LLD "%I64d" +# define LLU "%I64u" +# else +# define LLD "%lld" +# define LLU "%llu" +# endif + { + long long v; + static const long long max_value = 0x7fffffffffffffffLL; + static const long long min_value = -max_value - 1; + char buf[32]; // definitely big enough for a long long + + CHECK(RE("(-?\\d+)").FullMatch("100", &v)); CHECK_EQ(v, 100); + CHECK(RE("(-?\\d+)").FullMatch("-100",&v)); CHECK_EQ(v, -100); + + sprintf(buf, LLD, max_value); + CHECK(RE("(-?\\d+)").FullMatch(buf,&v)); CHECK_EQ(v, max_value); + + sprintf(buf, LLD, min_value); + CHECK(RE("(-?\\d+)").FullMatch(buf,&v)); CHECK_EQ(v, min_value); + + sprintf(buf, LLD, max_value); + assert(buf[strlen(buf)-1] != '9'); + buf[strlen(buf)-1]++; + CHECK(!RE("(-?\\d+)").FullMatch(buf, &v)); + + sprintf(buf, LLD, min_value); + assert(buf[strlen(buf)-1] != '9'); + buf[strlen(buf)-1]++; + CHECK(!RE("(-?\\d+)").FullMatch(buf, &v)); + } +#endif +#if defined HAVE_UNSIGNED_LONG_LONG && defined HAVE_LONG_LONG + { + unsigned long long v; + long long v2; + static const unsigned long long max_value = 0xffffffffffffffffULL; + char buf[32]; // definitely big enough for a unsigned long long + + CHECK(RE("(-?\\d+)").FullMatch("100",&v)); CHECK_EQ(v, 100); + CHECK(RE("(-?\\d+)").FullMatch("-100",&v2)); CHECK_EQ(v2, -100); + + sprintf(buf, LLU, max_value); + CHECK(RE("(-?\\d+)").FullMatch(buf,&v)); CHECK_EQ(v, max_value); + + assert(buf[strlen(buf)-1] != '9'); + buf[strlen(buf)-1]++; + CHECK(!RE("(-?\\d+)").FullMatch(buf, &v)); + } +#endif + { + float v; + CHECK(RE("(.*)").FullMatch("100", &v)); + CHECK(RE("(.*)").FullMatch("-100.", &v)); + CHECK(RE("(.*)").FullMatch("1e23", &v)); + } + { + double v; + CHECK(RE("(.*)").FullMatch("100", &v)); + CHECK(RE("(.*)").FullMatch("-100.", &v)); + CHECK(RE("(.*)").FullMatch("1e23", &v)); + } + + // Check that matching is fully anchored + CHECK(!RE("(\\d+)").FullMatch("x1001", &i)); + CHECK(!RE("(\\d+)").FullMatch("1001x", &i)); + CHECK(RE("x(\\d+)").FullMatch("x1001", &i)); CHECK_EQ(i, 1001); + CHECK(RE("(\\d+)x").FullMatch("1001x", &i)); CHECK_EQ(i, 1001); + + // Braces + CHECK(RE("[0-9a-f+.-]{5,}").FullMatch("0abcd")); + CHECK(RE("[0-9a-f+.-]{5,}").FullMatch("0abcde")); + CHECK(!RE("[0-9a-f+.-]{5,}").FullMatch("0abc")); + + // Complicated RE + CHECK(RE("foo|bar|[A-Z]").FullMatch("foo")); + CHECK(RE("foo|bar|[A-Z]").FullMatch("bar")); + CHECK(RE("foo|bar|[A-Z]").FullMatch("X")); + CHECK(!RE("foo|bar|[A-Z]").FullMatch("XY")); + + // Check full-match handling (needs '$' tacked on internally) + CHECK(RE("fo|foo").FullMatch("fo")); + CHECK(RE("fo|foo").FullMatch("foo")); + CHECK(RE("fo|foo$").FullMatch("fo")); + CHECK(RE("fo|foo$").FullMatch("foo")); + CHECK(RE("foo$").FullMatch("foo")); + CHECK(!RE("foo\\$").FullMatch("foo$bar")); + CHECK(!RE("fo|bar").FullMatch("fox")); + + // Uncomment the following if we change the handling of '$' to + // prevent it from matching a trailing newline + if (false) { + // Check that we don't get bitten by pcre's special handling of a + // '\n' at the end of the string matching '$' + CHECK(!RE("foo$").PartialMatch("foo\n")); + } + + // Number of args + int a[16]; + CHECK(RE("").FullMatch("")); + + memset(a, 0, sizeof(0)); + CHECK(RE("(\\d){1}").FullMatch("1", + &a[0])); + CHECK_EQ(a[0], 1); + + memset(a, 0, sizeof(0)); + CHECK(RE("(\\d)(\\d)").FullMatch("12", + &a[0], &a[1])); + CHECK_EQ(a[0], 1); + CHECK_EQ(a[1], 2); + + memset(a, 0, sizeof(0)); + CHECK(RE("(\\d)(\\d)(\\d)").FullMatch("123", + &a[0], &a[1], &a[2])); + CHECK_EQ(a[0], 1); + CHECK_EQ(a[1], 2); + CHECK_EQ(a[2], 3); + + memset(a, 0, sizeof(0)); + CHECK(RE("(\\d)(\\d)(\\d)(\\d)").FullMatch("1234", + &a[0], &a[1], &a[2], &a[3])); + CHECK_EQ(a[0], 1); + CHECK_EQ(a[1], 2); + CHECK_EQ(a[2], 3); + CHECK_EQ(a[3], 4); + + memset(a, 0, sizeof(0)); + CHECK(RE("(\\d)(\\d)(\\d)(\\d)(\\d)").FullMatch("12345", + &a[0], &a[1], &a[2], + &a[3], &a[4])); + CHECK_EQ(a[0], 1); + CHECK_EQ(a[1], 2); + CHECK_EQ(a[2], 3); + CHECK_EQ(a[3], 4); + CHECK_EQ(a[4], 5); + + memset(a, 0, sizeof(0)); + CHECK(RE("(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)").FullMatch("123456", + &a[0], &a[1], &a[2], + &a[3], &a[4], &a[5])); + CHECK_EQ(a[0], 1); + CHECK_EQ(a[1], 2); + CHECK_EQ(a[2], 3); + CHECK_EQ(a[3], 4); + CHECK_EQ(a[4], 5); + CHECK_EQ(a[5], 6); + + memset(a, 0, sizeof(0)); + CHECK(RE("(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)").FullMatch("1234567", + &a[0], &a[1], &a[2], &a[3], + &a[4], &a[5], &a[6])); + CHECK_EQ(a[0], 1); + CHECK_EQ(a[1], 2); + CHECK_EQ(a[2], 3); + CHECK_EQ(a[3], 4); + CHECK_EQ(a[4], 5); + CHECK_EQ(a[5], 6); + CHECK_EQ(a[6], 7); + + memset(a, 0, sizeof(0)); + CHECK(RE("(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)" + "(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)(\\d)").FullMatch( + "1234567890123456", + &a[0], &a[1], &a[2], &a[3], + &a[4], &a[5], &a[6], &a[7], + &a[8], &a[9], &a[10], &a[11], + &a[12], &a[13], &a[14], &a[15])); + CHECK_EQ(a[0], 1); + CHECK_EQ(a[1], 2); + CHECK_EQ(a[2], 3); + CHECK_EQ(a[3], 4); + CHECK_EQ(a[4], 5); + CHECK_EQ(a[5], 6); + CHECK_EQ(a[6], 7); + CHECK_EQ(a[7], 8); + CHECK_EQ(a[8], 9); + CHECK_EQ(a[9], 0); + CHECK_EQ(a[10], 1); + CHECK_EQ(a[11], 2); + CHECK_EQ(a[12], 3); + CHECK_EQ(a[13], 4); + CHECK_EQ(a[14], 5); + CHECK_EQ(a[15], 6); + + /***** PartialMatch *****/ + + printf("Testing PartialMatch\n"); + + CHECK(RE("h.*o").PartialMatch("hello")); + CHECK(RE("h.*o").PartialMatch("othello")); + CHECK(RE("h.*o").PartialMatch("hello!")); + CHECK(RE("((((((((((((((((((((x))))))))))))))))))))").PartialMatch("x")); + + /***** other tests *****/ + + RadixTests(); + TestReplace(); + TestExtract(); + TestConsume(); + TestFindAndConsume(); + TestQuoteMetaAll(); + TestMatchNumberPeculiarity(); + + // Check the pattern() accessor + { + const string kPattern = "http://([^/]+)/.*"; + const RE re(kPattern); + CHECK_EQ(kPattern, re.pattern()); + } + + // Check RE error field. + { + RE re("foo"); + CHECK(re.error().empty()); // Must have no error + } + +#ifdef SUPPORT_UTF8 + // Check UTF-8 handling + { + printf("Testing UTF-8 handling\n"); + + // Three Japanese characters (nihongo) + const unsigned char utf8_string[] = { + 0xe6, 0x97, 0xa5, // 65e5 + 0xe6, 0x9c, 0xac, // 627c + 0xe8, 0xaa, 0x9e, // 8a9e + 0 + }; + const unsigned char utf8_pattern[] = { + '.', + 0xe6, 0x9c, 0xac, // 627c + '.', + 0 + }; + + // Both should match in either mode, bytes or UTF-8 + RE re_test1("........."); + CHECK(re_test1.FullMatch(utf8_string)); + RE re_test2("...", pcrecpp::UTF8()); + CHECK(re_test2.FullMatch(utf8_string)); + + // Check that '.' matches one byte or UTF-8 character + // according to the mode. + string ss; + RE re_test3("(.)"); + CHECK(re_test3.PartialMatch(utf8_string, &ss)); + CHECK_EQ(ss, string("\xe6")); + RE re_test4("(.)", pcrecpp::UTF8()); + CHECK(re_test4.PartialMatch(utf8_string, &ss)); + CHECK_EQ(ss, string("\xe6\x97\xa5")); + + // Check that string matches itself in either mode + RE re_test5(utf8_string); + CHECK(re_test5.FullMatch(utf8_string)); + RE re_test6(utf8_string, pcrecpp::UTF8()); + CHECK(re_test6.FullMatch(utf8_string)); + + // Check that pattern matches string only in UTF8 mode + RE re_test7(utf8_pattern); + CHECK(!re_test7.FullMatch(utf8_string)); + RE re_test8(utf8_pattern, pcrecpp::UTF8()); + CHECK(re_test8.FullMatch(utf8_string)); + } + + // Check that ungreedy, UTF8 regular expressions don't match when they + // oughtn't -- see bug 82246. + { + // This code always worked. + const char* pattern = "\\w+X"; + const string target = "a aX"; + RE match_sentence(pattern); + RE match_sentence_re(pattern, pcrecpp::UTF8()); + + CHECK(!match_sentence.FullMatch(target)); + CHECK(!match_sentence_re.FullMatch(target)); + } + + { + const char* pattern = "(?U)\\w+X"; + const string target = "a aX"; + RE match_sentence(pattern); + RE match_sentence_re(pattern, pcrecpp::UTF8()); + + CHECK(!match_sentence.FullMatch(target)); + CHECK(!match_sentence_re.FullMatch(target)); + } +#endif /* def SUPPORT_UTF8 */ + + printf("Testing error reporting\n"); + + { RE re("a\\1"); CHECK(!re.error().empty()); } + { + RE re("a[x"); + CHECK(!re.error().empty()); + } + { + RE re("a[z-a]"); + CHECK(!re.error().empty()); + } + { + RE re("a[[:foobar:]]"); + CHECK(!re.error().empty()); + } + { + RE re("a(b"); + CHECK(!re.error().empty()); + } + { + RE re("a\\"); + CHECK(!re.error().empty()); + } + + // Test that recursion is stopped + TestRecursion(); + + // Test Options + if (getenv("VERBOSE_TEST") != NULL) + VERBOSE_TEST = true; + TestOptions(); + + // Test the constructors + TestConstructors(); + + // Done + printf("OK\n"); + + return 0; +} diff --git a/src/third_party/pcre-8.41/pcrecpparg.h b/src/third_party/pcre-8.41/pcrecpparg.h new file mode 100644 index 00000000000..b4f9c3f4989 --- /dev/null +++ b/src/third_party/pcre-8.41/pcrecpparg.h @@ -0,0 +1,174 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Sanjay Ghemawat + +#ifndef _PCRECPPARG_H +#define _PCRECPPARG_H + +#include // for NULL +#include + +#include + +namespace pcrecpp { + +class StringPiece; + +// Hex/Octal/Binary? + +// Special class for parsing into objects that define a ParseFrom() method +template +class _RE_MatchObject { + public: + static inline bool Parse(const char* str, int n, void* dest) { + if (dest == NULL) return true; + T* object = reinterpret_cast(dest); + return object->ParseFrom(str, n); + } +}; + +class PCRECPP_EXP_DEFN Arg { + public: + // Empty constructor so we can declare arrays of Arg + Arg(); + + // Constructor specially designed for NULL arguments + Arg(void*); + + typedef bool (*Parser)(const char* str, int n, void* dest); + +// Type-specific parsers +#define PCRE_MAKE_PARSER(type,name) \ + Arg(type* p) : arg_(p), parser_(name) { } \ + Arg(type* p, Parser parser) : arg_(p), parser_(parser) { } + + + PCRE_MAKE_PARSER(char, parse_char); + PCRE_MAKE_PARSER(unsigned char, parse_uchar); + PCRE_MAKE_PARSER(short, parse_short); + PCRE_MAKE_PARSER(unsigned short, parse_ushort); + PCRE_MAKE_PARSER(int, parse_int); + PCRE_MAKE_PARSER(unsigned int, parse_uint); + PCRE_MAKE_PARSER(long, parse_long); + PCRE_MAKE_PARSER(unsigned long, parse_ulong); +#if 1 + PCRE_MAKE_PARSER(long long, parse_longlong); +#endif +#if 1 + PCRE_MAKE_PARSER(unsigned long long, parse_ulonglong); +#endif + PCRE_MAKE_PARSER(float, parse_float); + PCRE_MAKE_PARSER(double, parse_double); + PCRE_MAKE_PARSER(std::string, parse_string); + PCRE_MAKE_PARSER(StringPiece, parse_stringpiece); + +#undef PCRE_MAKE_PARSER + + // Generic constructor + template Arg(T*, Parser parser); + // Generic constructor template + template Arg(T* p) + : arg_(p), parser_(_RE_MatchObject::Parse) { + } + + // Parse the data + bool Parse(const char* str, int n) const; + + private: + void* arg_; + Parser parser_; + + static bool parse_null (const char* str, int n, void* dest); + static bool parse_char (const char* str, int n, void* dest); + static bool parse_uchar (const char* str, int n, void* dest); + static bool parse_float (const char* str, int n, void* dest); + static bool parse_double (const char* str, int n, void* dest); + static bool parse_string (const char* str, int n, void* dest); + static bool parse_stringpiece (const char* str, int n, void* dest); + +#define PCRE_DECLARE_INTEGER_PARSER(name) \ + private: \ + static bool parse_ ## name(const char* str, int n, void* dest); \ + static bool parse_ ## name ## _radix( \ + const char* str, int n, void* dest, int radix); \ + public: \ + static bool parse_ ## name ## _hex(const char* str, int n, void* dest); \ + static bool parse_ ## name ## _octal(const char* str, int n, void* dest); \ + static bool parse_ ## name ## _cradix(const char* str, int n, void* dest) + + PCRE_DECLARE_INTEGER_PARSER(short); + PCRE_DECLARE_INTEGER_PARSER(ushort); + PCRE_DECLARE_INTEGER_PARSER(int); + PCRE_DECLARE_INTEGER_PARSER(uint); + PCRE_DECLARE_INTEGER_PARSER(long); + PCRE_DECLARE_INTEGER_PARSER(ulong); + PCRE_DECLARE_INTEGER_PARSER(longlong); + PCRE_DECLARE_INTEGER_PARSER(ulonglong); + +#undef PCRE_DECLARE_INTEGER_PARSER +}; + +inline Arg::Arg() : arg_(NULL), parser_(parse_null) { } +inline Arg::Arg(void* p) : arg_(p), parser_(parse_null) { } + +inline bool Arg::Parse(const char* str, int n) const { + return (*parser_)(str, n, arg_); +} + +// This part of the parser, appropriate only for ints, deals with bases +#define MAKE_INTEGER_PARSER(type, name) \ + inline Arg Hex(type* ptr) { \ + return Arg(ptr, Arg::parse_ ## name ## _hex); } \ + inline Arg Octal(type* ptr) { \ + return Arg(ptr, Arg::parse_ ## name ## _octal); } \ + inline Arg CRadix(type* ptr) { \ + return Arg(ptr, Arg::parse_ ## name ## _cradix); } + +MAKE_INTEGER_PARSER(short, short) /* */ +MAKE_INTEGER_PARSER(unsigned short, ushort) /* */ +MAKE_INTEGER_PARSER(int, int) /* Don't use semicolons */ +MAKE_INTEGER_PARSER(unsigned int, uint) /* after these statement */ +MAKE_INTEGER_PARSER(long, long) /* because they can cause */ +MAKE_INTEGER_PARSER(unsigned long, ulong) /* compiler warnings if */ +#if 1 /* the checking level is */ +MAKE_INTEGER_PARSER(long long, longlong) /* turned up high enough. */ +#endif /* */ +#if 1 /* */ +MAKE_INTEGER_PARSER(unsigned long long, ulonglong) /* */ +#endif + +#undef PCRE_IS_SET +#undef PCRE_SET_OR_CLEAR +#undef MAKE_INTEGER_PARSER + +} // namespace pcrecpp + + +#endif /* _PCRECPPARG_H */ diff --git a/src/third_party/pcre-8.41/pcrecpparg.h.in b/src/third_party/pcre-8.41/pcrecpparg.h.in new file mode 100644 index 00000000000..61bcab5402c --- /dev/null +++ b/src/third_party/pcre-8.41/pcrecpparg.h.in @@ -0,0 +1,174 @@ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Sanjay Ghemawat + +#ifndef _PCRECPPARG_H +#define _PCRECPPARG_H + +#include // for NULL +#include + +#include + +namespace pcrecpp { + +class StringPiece; + +// Hex/Octal/Binary? + +// Special class for parsing into objects that define a ParseFrom() method +template +class _RE_MatchObject { + public: + static inline bool Parse(const char* str, int n, void* dest) { + if (dest == NULL) return true; + T* object = reinterpret_cast(dest); + return object->ParseFrom(str, n); + } +}; + +class PCRECPP_EXP_DEFN Arg { + public: + // Empty constructor so we can declare arrays of Arg + Arg(); + + // Constructor specially designed for NULL arguments + Arg(void*); + + typedef bool (*Parser)(const char* str, int n, void* dest); + +// Type-specific parsers +#define PCRE_MAKE_PARSER(type,name) \ + Arg(type* p) : arg_(p), parser_(name) { } \ + Arg(type* p, Parser parser) : arg_(p), parser_(parser) { } + + + PCRE_MAKE_PARSER(char, parse_char); + PCRE_MAKE_PARSER(unsigned char, parse_uchar); + PCRE_MAKE_PARSER(short, parse_short); + PCRE_MAKE_PARSER(unsigned short, parse_ushort); + PCRE_MAKE_PARSER(int, parse_int); + PCRE_MAKE_PARSER(unsigned int, parse_uint); + PCRE_MAKE_PARSER(long, parse_long); + PCRE_MAKE_PARSER(unsigned long, parse_ulong); +#if @pcre_have_long_long@ + PCRE_MAKE_PARSER(long long, parse_longlong); +#endif +#if @pcre_have_ulong_long@ + PCRE_MAKE_PARSER(unsigned long long, parse_ulonglong); +#endif + PCRE_MAKE_PARSER(float, parse_float); + PCRE_MAKE_PARSER(double, parse_double); + PCRE_MAKE_PARSER(std::string, parse_string); + PCRE_MAKE_PARSER(StringPiece, parse_stringpiece); + +#undef PCRE_MAKE_PARSER + + // Generic constructor + template Arg(T*, Parser parser); + // Generic constructor template + template Arg(T* p) + : arg_(p), parser_(_RE_MatchObject::Parse) { + } + + // Parse the data + bool Parse(const char* str, int n) const; + + private: + void* arg_; + Parser parser_; + + static bool parse_null (const char* str, int n, void* dest); + static bool parse_char (const char* str, int n, void* dest); + static bool parse_uchar (const char* str, int n, void* dest); + static bool parse_float (const char* str, int n, void* dest); + static bool parse_double (const char* str, int n, void* dest); + static bool parse_string (const char* str, int n, void* dest); + static bool parse_stringpiece (const char* str, int n, void* dest); + +#define PCRE_DECLARE_INTEGER_PARSER(name) \ + private: \ + static bool parse_ ## name(const char* str, int n, void* dest); \ + static bool parse_ ## name ## _radix( \ + const char* str, int n, void* dest, int radix); \ + public: \ + static bool parse_ ## name ## _hex(const char* str, int n, void* dest); \ + static bool parse_ ## name ## _octal(const char* str, int n, void* dest); \ + static bool parse_ ## name ## _cradix(const char* str, int n, void* dest) + + PCRE_DECLARE_INTEGER_PARSER(short); + PCRE_DECLARE_INTEGER_PARSER(ushort); + PCRE_DECLARE_INTEGER_PARSER(int); + PCRE_DECLARE_INTEGER_PARSER(uint); + PCRE_DECLARE_INTEGER_PARSER(long); + PCRE_DECLARE_INTEGER_PARSER(ulong); + PCRE_DECLARE_INTEGER_PARSER(longlong); + PCRE_DECLARE_INTEGER_PARSER(ulonglong); + +#undef PCRE_DECLARE_INTEGER_PARSER +}; + +inline Arg::Arg() : arg_(NULL), parser_(parse_null) { } +inline Arg::Arg(void* p) : arg_(p), parser_(parse_null) { } + +inline bool Arg::Parse(const char* str, int n) const { + return (*parser_)(str, n, arg_); +} + +// This part of the parser, appropriate only for ints, deals with bases +#define MAKE_INTEGER_PARSER(type, name) \ + inline Arg Hex(type* ptr) { \ + return Arg(ptr, Arg::parse_ ## name ## _hex); } \ + inline Arg Octal(type* ptr) { \ + return Arg(ptr, Arg::parse_ ## name ## _octal); } \ + inline Arg CRadix(type* ptr) { \ + return Arg(ptr, Arg::parse_ ## name ## _cradix); } + +MAKE_INTEGER_PARSER(short, short) /* */ +MAKE_INTEGER_PARSER(unsigned short, ushort) /* */ +MAKE_INTEGER_PARSER(int, int) /* Don't use semicolons */ +MAKE_INTEGER_PARSER(unsigned int, uint) /* after these statement */ +MAKE_INTEGER_PARSER(long, long) /* because they can cause */ +MAKE_INTEGER_PARSER(unsigned long, ulong) /* compiler warnings if */ +#if @pcre_have_long_long@ /* the checking level is */ +MAKE_INTEGER_PARSER(long long, longlong) /* turned up high enough. */ +#endif /* */ +#if @pcre_have_ulong_long@ /* */ +MAKE_INTEGER_PARSER(unsigned long long, ulonglong) /* */ +#endif + +#undef PCRE_IS_SET +#undef PCRE_SET_OR_CLEAR +#undef MAKE_INTEGER_PARSER + +} // namespace pcrecpp + + +#endif /* _PCRECPPARG_H */ diff --git a/src/third_party/pcre-8.41/pcredemo.c b/src/third_party/pcre-8.41/pcredemo.c new file mode 100644 index 00000000000..946aba45cdc --- /dev/null +++ b/src/third_party/pcre-8.41/pcredemo.c @@ -0,0 +1,406 @@ +/************************************************* +* PCRE DEMONSTRATION PROGRAM * +*************************************************/ + +/* This is a demonstration program to illustrate the most straightforward ways +of calling the PCRE regular expression library from a C program. See the +pcresample documentation for a short discussion ("man pcresample" if you have +the PCRE man pages installed). + +In Unix-like environments, if PCRE is installed in your standard system +libraries, you should be able to compile this program using this command: + +gcc -Wall pcredemo.c -lpcre -o pcredemo + +If PCRE is not installed in a standard place, it is likely to be installed with +support for the pkg-config mechanism. If you have pkg-config, you can compile +this program using this command: + +gcc -Wall pcredemo.c `pkg-config --cflags --libs libpcre` -o pcredemo + +If you do not have pkg-config, you may have to use this: + +gcc -Wall pcredemo.c -I/usr/local/include -L/usr/local/lib \ + -R/usr/local/lib -lpcre -o pcredemo + +Replace "/usr/local/include" and "/usr/local/lib" with wherever the include and +library files for PCRE are installed on your system. Only some operating +systems (e.g. Solaris) use the -R option. + +Building under Windows: + +If you want to statically link this program against a non-dll .a file, you must +define PCRE_STATIC before including pcre.h, otherwise the pcre_malloc() and +pcre_free() exported functions will be declared __declspec(dllimport), with +unwanted results. So in this environment, uncomment the following line. */ + +/* #define PCRE_STATIC */ + +#include +#include +#include + +#define OVECCOUNT 30 /* should be a multiple of 3 */ + + +int main(int argc, char **argv) +{ +pcre *re; +const char *error; +char *pattern; +char *subject; +unsigned char *name_table; +unsigned int option_bits; +int erroffset; +int find_all; +int crlf_is_newline; +int namecount; +int name_entry_size; +int ovector[OVECCOUNT]; +int subject_length; +int rc, i; +int utf8; + + +/************************************************************************** +* First, sort out the command line. There is only one possible option at * +* the moment, "-g" to request repeated matching to find all occurrences, * +* like Perl's /g option. We set the variable find_all to a non-zero value * +* if the -g option is present. Apart from that, there must be exactly two * +* arguments. * +**************************************************************************/ + +find_all = 0; +for (i = 1; i < argc; i++) + { + if (strcmp(argv[i], "-g") == 0) find_all = 1; + else break; + } + +/* After the options, we require exactly two arguments, which are the pattern, +and the subject string. */ + +if (argc - i != 2) + { + printf("Two arguments required: a regex and a subject string\n"); + return 1; + } + +pattern = argv[i]; +subject = argv[i+1]; +subject_length = (int)strlen(subject); + + +/************************************************************************* +* Now we are going to compile the regular expression pattern, and handle * +* and errors that are detected. * +*************************************************************************/ + +re = pcre_compile( + pattern, /* the pattern */ + 0, /* default options */ + &error, /* for error message */ + &erroffset, /* for error offset */ + NULL); /* use default character tables */ + +/* Compilation failed: print the error message and exit */ + +if (re == NULL) + { + printf("PCRE compilation failed at offset %d: %s\n", erroffset, error); + return 1; + } + + +/************************************************************************* +* If the compilation succeeded, we call PCRE again, in order to do a * +* pattern match against the subject string. This does just ONE match. If * +* further matching is needed, it will be done below. * +*************************************************************************/ + +rc = pcre_exec( + re, /* the compiled pattern */ + NULL, /* no extra data - we didn't study the pattern */ + subject, /* the subject string */ + subject_length, /* the length of the subject */ + 0, /* start at offset 0 in the subject */ + 0, /* default options */ + ovector, /* output vector for substring information */ + OVECCOUNT); /* number of elements in the output vector */ + +/* Matching failed: handle error cases */ + +if (rc < 0) + { + switch(rc) + { + case PCRE_ERROR_NOMATCH: printf("No match\n"); break; + /* + Handle other special cases if you like + */ + default: printf("Matching error %d\n", rc); break; + } + pcre_free(re); /* Release memory used for the compiled pattern */ + return 1; + } + +/* Match succeded */ + +printf("\nMatch succeeded at offset %d\n", ovector[0]); + + +/************************************************************************* +* We have found the first match within the subject string. If the output * +* vector wasn't big enough, say so. Then output any substrings that were * +* captured. * +*************************************************************************/ + +/* The output vector wasn't big enough */ + +if (rc == 0) + { + rc = OVECCOUNT/3; + printf("ovector only has room for %d captured substrings\n", rc - 1); + } + +/* Show substrings stored in the output vector by number. Obviously, in a real +application you might want to do things other than print them. */ + +for (i = 0; i < rc; i++) + { + char *substring_start = subject + ovector[2*i]; + int substring_length = ovector[2*i+1] - ovector[2*i]; + printf("%2d: %.*s\n", i, substring_length, substring_start); + } + + +/************************************************************************** +* That concludes the basic part of this demonstration program. We have * +* compiled a pattern, and performed a single match. The code that follows * +* shows first how to access named substrings, and then how to code for * +* repeated matches on the same subject. * +**************************************************************************/ + +/* See if there are any named substrings, and if so, show them by name. First +we have to extract the count of named parentheses from the pattern. */ + +(void)pcre_fullinfo( + re, /* the compiled pattern */ + NULL, /* no extra data - we didn't study the pattern */ + PCRE_INFO_NAMECOUNT, /* number of named substrings */ + &namecount); /* where to put the answer */ + +if (namecount <= 0) printf("No named substrings\n"); else + { + unsigned char *tabptr; + printf("Named substrings\n"); + + /* Before we can access the substrings, we must extract the table for + translating names to numbers, and the size of each entry in the table. */ + + (void)pcre_fullinfo( + re, /* the compiled pattern */ + NULL, /* no extra data - we didn't study the pattern */ + PCRE_INFO_NAMETABLE, /* address of the table */ + &name_table); /* where to put the answer */ + + (void)pcre_fullinfo( + re, /* the compiled pattern */ + NULL, /* no extra data - we didn't study the pattern */ + PCRE_INFO_NAMEENTRYSIZE, /* size of each entry in the table */ + &name_entry_size); /* where to put the answer */ + + /* Now we can scan the table and, for each entry, print the number, the name, + and the substring itself. */ + + tabptr = name_table; + for (i = 0; i < namecount; i++) + { + int n = (tabptr[0] << 8) | tabptr[1]; + printf("(%d) %*s: %.*s\n", n, name_entry_size - 3, tabptr + 2, + ovector[2*n+1] - ovector[2*n], subject + ovector[2*n]); + tabptr += name_entry_size; + } + } + + +/************************************************************************* +* If the "-g" option was given on the command line, we want to continue * +* to search for additional matches in the subject string, in a similar * +* way to the /g option in Perl. This turns out to be trickier than you * +* might think because of the possibility of matching an empty string. * +* What happens is as follows: * +* * +* If the previous match was NOT for an empty string, we can just start * +* the next match at the end of the previous one. * +* * +* If the previous match WAS for an empty string, we can't do that, as it * +* would lead to an infinite loop. Instead, a special call of pcre_exec() * +* is made with the PCRE_NOTEMPTY_ATSTART and PCRE_ANCHORED flags set. * +* The first of these tells PCRE that an empty string at the start of the * +* subject is not a valid match; other possibilities must be tried. The * +* second flag restricts PCRE to one match attempt at the initial string * +* position. If this match succeeds, an alternative to the empty string * +* match has been found, and we can print it and proceed round the loop, * +* advancing by the length of whatever was found. If this match does not * +* succeed, we still stay in the loop, advancing by just one character. * +* In UTF-8 mode, which can be set by (*UTF8) in the pattern, this may be * +* more than one byte. * +* * +* However, there is a complication concerned with newlines. When the * +* newline convention is such that CRLF is a valid newline, we must * +* advance by two characters rather than one. The newline convention can * +* be set in the regex by (*CR), etc.; if not, we must find the default. * +*************************************************************************/ + +if (!find_all) /* Check for -g */ + { + pcre_free(re); /* Release the memory used for the compiled pattern */ + return 0; /* Finish unless -g was given */ + } + +/* Before running the loop, check for UTF-8 and whether CRLF is a valid newline +sequence. First, find the options with which the regex was compiled; extract +the UTF-8 state, and mask off all but the newline options. */ + +(void)pcre_fullinfo(re, NULL, PCRE_INFO_OPTIONS, &option_bits); +utf8 = option_bits & PCRE_UTF8; +option_bits &= PCRE_NEWLINE_CR|PCRE_NEWLINE_LF|PCRE_NEWLINE_CRLF| + PCRE_NEWLINE_ANY|PCRE_NEWLINE_ANYCRLF; + +/* If no newline options were set, find the default newline convention from the +build configuration. */ + +if (option_bits == 0) + { + int d; + (void)pcre_config(PCRE_CONFIG_NEWLINE, &d); + /* Note that these values are always the ASCII ones, even in + EBCDIC environments. CR = 13, NL = 10. */ + option_bits = (d == 13)? PCRE_NEWLINE_CR : + (d == 10)? PCRE_NEWLINE_LF : + (d == (13<<8 | 10))? PCRE_NEWLINE_CRLF : + (d == -2)? PCRE_NEWLINE_ANYCRLF : + (d == -1)? PCRE_NEWLINE_ANY : 0; + } + +/* See if CRLF is a valid newline sequence. */ + +crlf_is_newline = + option_bits == PCRE_NEWLINE_ANY || + option_bits == PCRE_NEWLINE_CRLF || + option_bits == PCRE_NEWLINE_ANYCRLF; + +/* Loop for second and subsequent matches */ + +for (;;) + { + int options = 0; /* Normally no options */ + int start_offset = ovector[1]; /* Start at end of previous match */ + + /* If the previous match was for an empty string, we are finished if we are + at the end of the subject. Otherwise, arrange to run another match at the + same point to see if a non-empty match can be found. */ + + if (ovector[0] == ovector[1]) + { + if (ovector[0] == subject_length) break; + options = PCRE_NOTEMPTY_ATSTART | PCRE_ANCHORED; + } + + /* Run the next matching operation */ + + rc = pcre_exec( + re, /* the compiled pattern */ + NULL, /* no extra data - we didn't study the pattern */ + subject, /* the subject string */ + subject_length, /* the length of the subject */ + start_offset, /* starting offset in the subject */ + options, /* options */ + ovector, /* output vector for substring information */ + OVECCOUNT); /* number of elements in the output vector */ + + /* This time, a result of NOMATCH isn't an error. If the value in "options" + is zero, it just means we have found all possible matches, so the loop ends. + Otherwise, it means we have failed to find a non-empty-string match at a + point where there was a previous empty-string match. In this case, we do what + Perl does: advance the matching position by one character, and continue. We + do this by setting the "end of previous match" offset, because that is picked + up at the top of the loop as the point at which to start again. + + There are two complications: (a) When CRLF is a valid newline sequence, and + the current position is just before it, advance by an extra byte. (b) + Otherwise we must ensure that we skip an entire UTF-8 character if we are in + UTF-8 mode. */ + + if (rc == PCRE_ERROR_NOMATCH) + { + if (options == 0) break; /* All matches found */ + ovector[1] = start_offset + 1; /* Advance one byte */ + if (crlf_is_newline && /* If CRLF is newline & */ + start_offset < subject_length - 1 && /* we are at CRLF, */ + subject[start_offset] == '\r' && + subject[start_offset + 1] == '\n') + ovector[1] += 1; /* Advance by one more. */ + else if (utf8) /* Otherwise, ensure we */ + { /* advance a whole UTF-8 */ + while (ovector[1] < subject_length) /* character. */ + { + if ((subject[ovector[1]] & 0xc0) != 0x80) break; + ovector[1] += 1; + } + } + continue; /* Go round the loop again */ + } + + /* Other matching errors are not recoverable. */ + + if (rc < 0) + { + printf("Matching error %d\n", rc); + pcre_free(re); /* Release memory used for the compiled pattern */ + return 1; + } + + /* Match succeded */ + + printf("\nMatch succeeded again at offset %d\n", ovector[0]); + + /* The match succeeded, but the output vector wasn't big enough. */ + + if (rc == 0) + { + rc = OVECCOUNT/3; + printf("ovector only has room for %d captured substrings\n", rc - 1); + } + + /* As before, show substrings stored in the output vector by number, and then + also any named substrings. */ + + for (i = 0; i < rc; i++) + { + char *substring_start = subject + ovector[2*i]; + int substring_length = ovector[2*i+1] - ovector[2*i]; + printf("%2d: %.*s\n", i, substring_length, substring_start); + } + + if (namecount <= 0) printf("No named substrings\n"); else + { + unsigned char *tabptr = name_table; + printf("Named substrings\n"); + for (i = 0; i < namecount; i++) + { + int n = (tabptr[0] << 8) | tabptr[1]; + printf("(%d) %*s: %.*s\n", n, name_entry_size - 3, tabptr + 2, + ovector[2*n+1] - ovector[2*n], subject + ovector[2*n]); + tabptr += name_entry_size; + } + } + } /* End of loop to find second and subsequent matches */ + +printf("\n"); +pcre_free(re); /* Release memory used for the compiled pattern */ +return 0; +} + +/* End of pcredemo.c */ diff --git a/src/third_party/pcre-8.41/pcregexp.pas b/src/third_party/pcre-8.41/pcregexp.pas new file mode 100644 index 00000000000..bb2b3da8f3d --- /dev/null +++ b/src/third_party/pcre-8.41/pcregexp.pas @@ -0,0 +1,845 @@ +{ + pcRegExp - Perl compatible regular expressions for Virtual Pascal + (c) 2001 Peter S. Voronov aka Chem O'Dun + + Based on PCRE library interface unit for Virtual Pascal. + (c) 2001 Alexander Tokarev + + The current PCRE version is: 3.7 + + This software may be distributed under the terms of the modified BSD license + Copyright (c) 2001, Alexander Tokarev + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + The PCRE library is written by: Philip Hazel + Copyright (c) 1997-2004 University of Cambridge + + AngelsHolocaust 4-11-04 updated to use version v5.0 + (INFO: this is regex-directed, NFA) + AH: 9-11-04 - pcre_free: removed var, pcre already gives the ptr, now + everything works as it should (no more crashes) + -> removed CheckRegExp because pcre handles errors perfectly + 10-11-04 - added pcError (errorhandling), pcInit + 13-11-04 - removed the ErrorPos = 0 check -> always print erroroffset + 17-10-05 - support for \1-\9 backreferences in TpcRegExp.GetReplStr + 17-02-06 - added RunTimeOptions: caller can set options while searching + 19-02-06 - added SearchOfs(): let PCRE use the complete string and offset + into the string itself + 20-12-06 - support for version 7.0 + 27.08.08 - support for v7.7 +} + +{$H+} {$DEFINE PCRE_3_7} {$DEFINE PCRE_5_0} {$DEFINE PCRE_7_0} {$DEFINE PCRE_7_7} + +Unit pcregexp; + +Interface + +uses objects; + +Type + PpcRegExp = ^TpcRegExp; +// TpcRegExp = object + TpcRegExp = object(TObject) + MatchesCount: integer; + RegExpC, RegExpExt : Pointer; + Matches:Pointer; + RegExp: shortstring; + SourceLen: integer; + PartialMatch : boolean; + Error : boolean; + ErrorMsg : Pchar; + ErrorPos : integer; + RunTimeOptions: Integer; // options which can be set by the caller + constructor Init(const ARegExp : shortstring; AOptions : integer; ALocale : Pointer); + function Search(AStr: Pchar; ALen : longint) : boolean; virtual; + function SearchNext( AStr: Pchar; ALen : longint) : boolean; virtual; + function SearchOfs ( AStr: Pchar; ALen, AOfs : longint) : boolean; virtual; + function MatchSub(ANom: integer; var Pos, Len : longint) : boolean; virtual; + function MatchFull(var Pos, Len : longint) : boolean; virtual; + function GetSubStr(ANom: integer; AStr: Pchar) : string; virtual; + function GetFullStr(AStr: Pchar) : string; virtual; + function GetReplStr(AStr: Pchar; const ARepl: string) : string; virtual; + function GetPreSubStr(AStr: Pchar) : string; virtual; + function GetPostSubStr(AStr: Pchar) : string; virtual; + function ErrorStr : string; virtual; + destructor Done; virtual; + end; + + function pcGrepMatch(WildCard, aStr: string; AOptions:integer; ALocale : Pointer): Boolean; + function pcGrepSub(WildCard, aStr, aRepl: string; AOptions:integer; ALocale : Pointer): string; + + function pcFastGrepMatch(WildCard, aStr: string): Boolean; + function pcFastGrepSub(WildCard, aStr, aRepl: string): string; + +{$IFDEF PCRE_5_0} + function pcGetVersion : pchar; +{$ENDIF} + + function pcError (var pRegExp : Pointer) : Boolean; + function pcInit (const Pattern: Shortstring; CaseSens: Boolean) : Pointer; + +Const { Options } + PCRE_CASELESS = $0001; + PCRE_MULTILINE = $0002; + PCRE_DOTALL = $0004; + PCRE_EXTENDED = $0008; + PCRE_ANCHORED = $0010; + PCRE_DOLLAR_ENDONLY = $0020; + PCRE_EXTRA = $0040; + PCRE_NOTBOL = $0080; + PCRE_NOTEOL = $0100; + PCRE_UNGREEDY = $0200; + PCRE_NOTEMPTY = $0400; +{$IFDEF PCRE_5_0} + PCRE_UTF8 = $0800; + PCRE_NO_AUTO_CAPTURE = $1000; + PCRE_NO_UTF8_CHECK = $2000; + PCRE_AUTO_CALLOUT = $4000; + PCRE_PARTIAL = $8000; +{$ENDIF} +{$IFDEF PCRE_7_0} + PCRE_DFA_SHORTEST = $00010000; + PCRE_DFA_RESTART = $00020000; + PCRE_FIRSTLINE = $00040000; + PCRE_DUPNAMES = $00080000; + PCRE_NEWLINE_CR = $00100000; + PCRE_NEWLINE_LF = $00200000; + PCRE_NEWLINE_CRLF = $00300000; + PCRE_NEWLINE_ANY = $00400000; + PCRE_NEWLINE_ANYCRLF = $00500000; + + PCRE_NEWLINE_BITS = PCRE_NEWLINE_CR or PCRE_NEWLINE_LF or PCRE_NEWLINE_ANY; + +{$ENDIF} +{$IFDEF PCRE_7_7} + PCRE_BSR_ANYCRLF = $00800000; + PCRE_BSR_UNICODE = $01000000; + PCRE_JAVASCRIPT_COMPAT= $02000000; +{$ENDIF} + + PCRE_COMPILE_ALLOWED_OPTIONS = PCRE_ANCHORED + PCRE_AUTO_CALLOUT + PCRE_CASELESS + + PCRE_DOLLAR_ENDONLY + PCRE_DOTALL + PCRE_EXTENDED + + PCRE_EXTRA + PCRE_MULTILINE + PCRE_NO_AUTO_CAPTURE + + PCRE_UNGREEDY + PCRE_UTF8 + PCRE_NO_UTF8_CHECK + {$IFDEF PCRE_7_0} + + PCRE_DUPNAMES + PCRE_FIRSTLINE + PCRE_NEWLINE_BITS + {$ENDIF} + {$IFDEF PCRE_7_7} + + PCRE_BSR_ANYCRLF + PCRE_BSR_UNICODE + PCRE_JAVASCRIPT_COMPAT + {$ENDIF} + ; + + PCRE_EXEC_ALLOWED_OPTIONS = PCRE_ANCHORED + PCRE_NOTBOL + PCRE_NOTEOL + + PCRE_NOTEMPTY + PCRE_NO_UTF8_CHECK + PCRE_PARTIAL + {$IFDEF PCRE_7_0} + + PCRE_NEWLINE_BITS + {$ENDIF} + {$IFDEF PCRE_7_7} + + PCRE_BSR_ANYCRLF + PCRE_BSR_UNICODE + {$ENDIF} + ; + +{$IFDEF PCRE_7_0} + PCRE_DFA_EXEC_ALLOWED_OPTIONS = PCRE_ANCHORED + PCRE_NOTBOL + PCRE_NOTEOL + + PCRE_NOTEMPTY + PCRE_NO_UTF8_CHECK + PCRE_PARTIAL + + PCRE_DFA_SHORTEST + PCRE_DFA_RESTART + + PCRE_NEWLINE_BITS + {$IFDEF PCRE_7_7} + + PCRE_BSR_ANYCRLF + PCRE_BSR_UNICODE + {$ENDIF} + ; +{$ENDIF} + +{ Exec-time and get/set-time error codes } + PCRE_ERROR_NOMATCH = -1; + PCRE_ERROR_NULL = -2; + PCRE_ERROR_BADOPTION = -3; + PCRE_ERROR_BADMAGIC = -4; + PCRE_ERROR_UNKNOWN_MODE = -5; + PCRE_ERROR_NOMEMORY = -6; + PCRE_ERROR_NOSUBSTRING = -7; +{$IFDEF PCRE_5_0} + PCRE_ERROR_MATCHLIMIT = -8; + PCRE_ERROR_CALLOUT = -9; { Never used by PCRE itself } + PCRE_ERROR_BADUTF8 = -10; + PCRE_ERROR_BADUTF8_OFFSET = -11; + PCRE_ERROR_PARTIAL = -12; + PCRE_ERROR_BADPARTIAL = -13; + PCRE_ERROR_INTERNAL = -14; + PCRE_ERROR_BADCOUNT = -15; +{$ENDIF} +{$IFDEF PCRE_7_0} + PCRE_ERROR_DFA_UITEM = -16; + PCRE_ERROR_DFA_UCOND = -17; + PCRE_ERROR_DFA_UMLIMIT = -18; + PCRE_ERROR_DFA_WSSIZE = -19; + PCRE_ERROR_DFA_RECURSE = -20; + PCRE_ERROR_RECURSIONLIMIT = -21; + PCRE_ERROR_NULLWSLIMIT = -22; + PCRE_ERROR_BADNEWLINE = -23; +{$ENDIF} + +{ Request types for pcre_fullinfo() } + + PCRE_INFO_OPTIONS = 0; + PCRE_INFO_SIZE = 1; + PCRE_INFO_CAPTURECOUNT = 2; + PCRE_INFO_BACKREFMAX = 3; + PCRE_INFO_FIRSTBYTE = 4; + PCRE_INFO_FIRSTCHAR = 4; { For backwards compatibility } + PCRE_INFO_FIRSTTABLE = 5; +{$IFDEF PCRE_5_0} + PCRE_INFO_LASTLITERAL = 6; + PCRE_INFO_NAMEENTRYSIZE = 7; + PCRE_INFO_NAMECOUNT = 8; + PCRE_INFO_NAMETABLE = 9; + PCRE_INFO_STUDYSIZE = 10; + PCRE_INFO_DEFAULT_TABLES = 11; +{$ENDIF PCRE_5_0} +{$IFDEF PCRE_7_7} + PCRE_INFO_OKPARTIAL = 12; + PCRE_INFO_JCHANGED = 13; + PCRE_INFO_HASCRORLF = 14; +{$ENDIF} + +{ Request types for pcre_config() } +{$IFDEF PCRE_5_0} + PCRE_CONFIG_UTF8 = 0; + PCRE_CONFIG_NEWLINE = 1; + PCRE_CONFIG_LINK_SIZE = 2; + PCRE_CONFIG_POSIX_MALLOC_THRESHOLD = 3; + PCRE_CONFIG_MATCH_LIMIT = 4; + PCRE_CONFIG_STACKRECURSE = 5; + PCRE_CONFIG_UNICODE_PROPERTIES = 6; +{$ENDIF PCRE_5_0} +{$IFDEF PCRE_7_0} + PCRE_CONFIG_MATCH_LIMIT_RECURSION = 7; +{$ENDIF} +{$IFDEF PCRE_7_7} + PCRE_CONFIG_BSR = 8; +{$ENDIF} + +{ Bit flags for the pcre_extra structure } +{$IFDEF PCRE_5_0} + PCRE_EXTRA_STUDY_DATA = $0001; + PCRE_EXTRA_MATCH_LIMIT = $0002; + PCRE_EXTRA_CALLOUT_DATA = $0004; + PCRE_EXTRA_TABLES = $0008; +{$ENDIF PCRE_5_0} +{$IFDEF PCRE_7_0} + PCRE_EXTRA_MATCH_LIMIT_RECURSION = $0010; +{$ENDIF} + +Const +// DefaultOptions : integer = 0; + DefaultLocaleTable : pointer = nil; + +{$IFDEF PCRE_5_0} +{ The structure for passing additional data to pcre_exec(). This is defined in +such as way as to be extensible. Always add new fields at the end, in order to +remain compatible. } + +type ppcre_extra = ^tpcre_extra; + tpcre_extra = record + flags : longint; { Bits for which fields are set } + study_data : pointer; { Opaque data from pcre_study() } + match_limit : longint; { Maximum number of calls to match() } + callout_data : pointer; { Data passed back in callouts } + tables : pointer; { Pointer to character tables } + match_limit_recursion: longint; { Max recursive calls to match() } + end; + +type ppcre_callout_block = ^pcre_callout_block; + pcre_callout_block = record + version, + (* ------------------------ Version 0 ------------------------------- *) + callout_number : integer; + offset_vector : pointer; + subject : pchar; + subject_length, start_match, current_position, capture_top, + capture_last : integer; + callout_data : pointer; + (* ------------------- Added for Version 1 -------------------------- *) + pattern_position, next_item_length : integer; + end; +{$ENDIF PCRE_5_0} + +{$OrgName+} +{$IFDEF VIRTUALPASCAL} {&Cdecl+} {$ENDIF VIRTUALPASCAL} + + { local replacement of external pcre memory management functions } + function pcre_malloc( size : integer ) : pointer; + procedure pcre_free( {var} p : pointer ); +{$IFDEF PCRE_5_0} + const pcre_stack_malloc: function ( size : integer ): pointer = pcre_malloc; + pcre_stack_free: procedure ( {var} p : pointer ) = pcre_free; + function pcre_callout(var p : ppcre_callout_block) : integer; +{$ENDIF PCRE_5_0} +{$IFDEF VIRTUALPASCAL} {&Cdecl-} {$ENDIF VIRTUALPASCAL} + +Implementation + +Uses strings, collect, messages, dnapp, commands, advance0, stringsx + {$IFDEF VIRTUALPASCAL} ,vpsyslow {$ENDIF VIRTUALPASCAL}; + +Const + MAGIC_NUMBER = $50435245; { 'PCRE' } + MAX_MATCHES = 90; { changed in 3.5 version; should be divisible by 3, was 64} + +Type + PMatchArray = ^TMatchArray; + TMatchArray = array[0..( MAX_MATCHES * 3 )] of integer; + + PRegExpCollection = ^TRegExpCollection; + TRegExpCollection = object(TSortedCollection) + MaxRegExp : integer; + SearchRegExp : shortstring; + CompareModeInsert : boolean; + constructor Init(AMaxRegExp:integer); + procedure FreeItem(P: Pointer); virtual; + function Compare(P1, P2: Pointer): Integer; virtual; + function Find(ARegExp:shortstring;var P: PpcRegExp):boolean; virtual; + function CheckNew(ARegExp:shortstring):PpcRegExp;virtual; + end; + +Var + PRegExpCache : PRegExpCollection; + + +{$IFDEF VIRTUALPASCAL} {&Cdecl+} {$ENDIF VIRTUALPASCAL} + + { imported original pcre functions } + + function pcre_compile( const pattern : PChar; options : integer; + var errorptr : PChar; var erroroffset : integer; + const tables : PChar ) : pointer {pcre}; external; +{$IFDEF PCRE_7_0} + function pcre_compile2( const pattern : PChar; options : integer; + var errorcodeptr : Integer; + var errorptr : PChar; var erroroffset : integer; + const tables : PChar ) : pointer {pcre}; external; +{$ENDIF} +{$IFDEF PCRE_5_0} + function pcre_config( what : integer; where : pointer) : integer; external; + function pcre_copy_named_substring( const code : pointer {pcre}; + const subject : pchar; + var ovector : integer; + stringcount : integer; + const stringname : pchar; + var buffer : pchar; + size : integer) : integer; external; + function pcre_copy_substring( const subject : pchar; var ovector : integer; + stringcount, stringnumber : integer; + var buffer : pchar; size : integer ) + : integer; external; + function pcre_exec( const argument_re : pointer {pcre}; + const extra_data : pointer {pcre_extra}; +{$ELSE} + function pcre_exec( const external_re : pointer; + const external_extra : pointer; +{$ENDIF} + const subject : PChar; + length, start_offset, options : integer; + offsets : pointer; + offsetcount : integer ) : integer; external; +{$IFDEF PCRE_7_0} + function pcre_dfa_exec( const argument_re : pointer {pcre}; + const extra_data : pointer {pcre_extra}; + const subject : pchar; + length, start_offset, options : integer; + offsets : pointer; + offsetcount : integer; + workspace : pointer; + wscount : integer ) : integer; external; +{$ENDIF} +{$IFDEF PCRE_5_0} + procedure pcre_free_substring( const p : pchar ); external; + procedure pcre_free_substring_list( var p : pchar ); external; + function pcre_fullinfo( const argument_re : pointer {pcre}; + const extra_data : pointer {pcre_extra}; + what : integer; + where : pointer ) : integer; external; + function pcre_get_named_substring( const code : pointer {pcre}; + const subject : pchar; + var ovector : integer; + stringcount : integer; + const stringname : pchar; + var stringptr : pchar ) : integer; external; + function pcre_get_stringnumber( const code : pointer {pcre}; + const stringname : pchar ) : integer; external; + function pcre_get_stringtable_entries( const code : pointer {pcre}; + const stringname : pchar; + var firstptr, + lastptr : pchar ) : integer; external; + function pcre_get_substring( const subject : pchar; var ovector : integer; + stringcount, stringnumber : integer; + var stringptr : pchar ) : integer; external; + function pcre_get_substring_list( const subject : pchar; var ovector : integer; + stringcount : integer; + listptr : pointer {const char ***listptr}) : integer; external; + function pcre_info( const argument_re : pointer {pcre}; + var optptr : integer; + var first_byte : integer ) : integer; external; + function pcre_maketables : pchar; external; +{$ENDIF} +{$IFDEF PCRE_7_0} + function pcre_refcount( const argument_re : pointer {pcre}; + adjust : integer ) : pchar; external; +{$ENDIF} + function pcre_study( const external_re : pointer {pcre}; + options : integer; + var errorptr : PChar ) : pointer {pcre_extra}; external; +{$IFDEF PCRE_5_0} + function pcre_version : pchar; external; +{$ENDIF} + + function pcre_malloc( size : integer ) : pointer; + begin + GetMem( result, size ); + end; + + procedure pcre_free( {var} p : pointer ); + begin + if (p <> nil) then + FreeMem( p, 0 ); + {@p := nil;} + end; + +{$IFDEF PCRE_5_0} +(* Called from PCRE as a result of the (?C) item. We print out where we are in +the match. Yield zero unless more callouts than the fail count, or the callout +data is not zero. *) + + function pcre_callout; + begin + end; +{$ENDIF} + +{$IFDEF VIRTUALPASCAL} {&Cdecl-} {$ENDIF VIRTUALPASCAL} + +// Always include the newest version of the library +{$IFDEF PCRE_7_7} + {$L pcre77.lib} +{$ELSE} + {$IFDEF PCRE_7_0} + {$L pcre70.lib} + {$ELSE} + {$IFDEF PCRE_5_0} + {$L pcre50.lib} + {$ELSE} + {$IFDEF PCRE_3_7} + {$L pcre37.lib} + {$ENDIF PCRE_3_7} + {$ENDIF PCRE_5_0} + {$ENDIF PCRE_7_0} +{$ENDIF PCRE_7_7} + +{TpcRegExp} + + constructor TpcRegExp.Init(const ARegExp:shortstring; AOptions:integer; ALocale : Pointer); + var + pRegExp : PChar; + begin + RegExp:=ARegExp; + RegExpC:=nil; + RegExpExt:=nil; + Matches:=nil; + MatchesCount:=0; + Error:=true; + ErrorMsg:=nil; + ErrorPos:=0; + RunTimeOptions := 0; + if length(RegExp) < 255 then + begin + RegExp[length(RegExp)+1]:=#0; + pRegExp:=@RegExp[1]; + end + else + begin + GetMem(pRegExp,length(RegExp)+1); + pRegExp:=strpcopy(pRegExp,RegExp); + end; + RegExpC := pcre_compile( pRegExp, + AOptions and PCRE_COMPILE_ALLOWED_OPTIONS, + ErrorMsg, ErrorPos, ALocale); + if length(RegExp) = 255 then + StrDispose(pRegExp); + if RegExpC = nil then + exit; + ErrorMsg:=nil; + RegExpExt := pcre_study( RegExpC, 0, ErrorMsg ); + if (RegExpExt = nil) and (ErrorMsg <> nil) then + begin + pcre_free(RegExpC); + exit; + end; + GetMem(Matches,SizeOf(TMatchArray)); + Error:=false; + end; + + destructor TpcRegExp.Done; + begin + if RegExpC <> nil then + pcre_free(RegExpC); + if RegExpExt <> nil then + pcre_free(RegExpExt); + if Matches <> nil then + FreeMem(Matches,SizeOf(TMatchArray)); + end; + + function TpcRegExp.SearchNext( AStr: Pchar; ALen : longint ) : boolean; + var Options: Integer; + begin // must handle PCRE_ERROR_PARTIAL here + Options := (RunTimeOptions or startup.MiscMultiData.cfgRegEx.DefaultOptions) and + PCRE_EXEC_ALLOWED_OPTIONS; + if MatchesCount > 0 then + MatchesCount:=pcre_exec( RegExpC, RegExpExt, AStr, ALen, PMatchArray(Matches)^[1], + Options, Matches, MAX_MATCHES ) else + MatchesCount:=pcre_exec( RegExpC, RegExpExt, AStr, ALen, 0, + Options, Matches, MAX_MATCHES ); +{ if MatchesCount = 0 then + MatchesCount := MatchesCount div 3;} + PartialMatch := MatchesCount = PCRE_ERROR_PARTIAL; + SearchNext := MatchesCount > 0; + end; + + function TpcRegExp.Search( AStr: Pchar; ALen : longint):boolean; + begin + MatchesCount:=0; + Search:=SearchNext(AStr,ALen); + SourceLen:=ALen; + end; + + function TpcRegExp.SearchOfs( AStr: Pchar; ALen, AOfs: longint ) : boolean; + var Options: Integer; + begin + MatchesCount:=0; + Options := (RunTimeOptions or startup.MiscMultiData.cfgRegEx.DefaultOptions) and + PCRE_EXEC_ALLOWED_OPTIONS; + MatchesCount:=pcre_exec( RegExpC, RegExpExt, AStr, ALen, AOfs, + Options, Matches, MAX_MATCHES ); + PartialMatch := MatchesCount = PCRE_ERROR_PARTIAL; + SearchOfs := MatchesCount > 0; + SourceLen := ALen-AOfs; + end; + + function TpcRegExp.MatchSub(ANom:integer; var Pos,Len:longint):boolean; + begin + if (MatchesCount > 0) and (ANom <= (MatchesCount-1)) then + begin + ANom:=ANom*2; + Pos:=PMatchArray(Matches)^[ANom]; + Len:=PMatchArray(Matches)^[ANom+1]-Pos; + MatchSub:=true; + end + else + MatchSub:=false; + end; + + function TpcRegExp.MatchFull(var Pos,Len:longint):boolean; + begin + MatchFull:=MatchSub(0,Pos,Len); + end; + + function TpcRegExp.GetSubStr(ANom: integer; AStr: Pchar):string; + var + s: ansistring; + pos,len: longint; + begin + s:=''; + if MatchSub(ANom, pos, len) then + begin + setlength(s, len); + Move(AStr[pos], s[1], len); + end; + GetSubStr:=s; + end; + + function TpcRegExp.GetPreSubStr(AStr: Pchar):string; + var + s: ansistring; + l: longint; + begin + s:=''; + if (MatchesCount > 0) then + begin + l:=PMatchArray(Matches)^[0]-1; + if l > 0 then + begin + setlength(s,l); + Move(AStr[1],s[1],l); + end; + end; + GetPreSubStr:=s; + end; + + function TpcRegExp.GetPostSubStr(AStr: Pchar):string; + var + s: ansistring; + l: longint; + ANom: integer; + begin + s:=''; + if (MatchesCount > 0) then + begin + ANom:=(MatchesCount-1){*2} shl 1; + l:=SourceLen-PMatchArray(Matches)^[ANom+1]+1; + if l > 0 then + begin + setlength(s,l); + Move(AStr[PMatchArray(Matches)^[ANom+1]],s[1],l); + end; + end; + GetPostSubStr:=s; + end; + + + function TpcRegExp.GetFullStr(AStr: Pchar):string; + var + s: ansistring; + l: longint; + begin + GetFullStr:=GetSubStr(0,AStr); + end; + + function TpcRegExp.GetReplStr(AStr: Pchar; const ARepl: string):string; + var + s: ansistring; + l,i,lasti: longint; + begin + l:=length(ARepl); + i:=1; + lasti:=1; + s:=''; + while i <= l do + begin + case ARepl[i] of + '\' : + begin + if i < l then + begin + s:=s+copy(ARepl,lasti,i-lasti){+ARepl[i+1]}; + {AH 17-10-05 support for POSIX \1-\9 backreferences} + case ARepl[i+1] of + '0' : s:=s+GetFullStr(AStr); + '1'..'9' : s:=s+GetSubStr(ord(ARepl[i+1])-ord('0'),AStr); + else s:=s+ARepl[i+1]; // copy the escaped character + end; + end; + inc(i); + lasti:=i+1; + end; + '$' : + begin + if i < l then + begin + s:=s+copy(ARepl,lasti,i-lasti); + case ARepl[i+1] of + '&' : s:=s+GetFullStr(AStr); + '1'..'9' : s:=s+GetSubStr(ord(ARepl[i+1])-ord('0'),AStr); + '`' : s:=s+GetPreSubStr(AStr); + #39 : s:=s+GetPostSubStr(AStr); + end; + end; + inc(i); + lasti:=i+1; + end; + end; + inc(i); + end; + if lasti <= {AH 25-10-2004 added =, else l==1 won't work} l then + s:=s+copy(ARepl,lasti,l-lasti+1); + GetReplStr:=s; + end; + + function TpcRegExp.ErrorStr:string; + begin + ErrorStr:=StrPas(ErrorMsg); + end; + +{TRegExpCollection} + +constructor TRegExpCollection.Init(AMaxRegExp: integer); +begin + Inherited Init(1,1); + MaxRegExp:=AMaxRegExp; + CompareModeInsert:=true; +end; + +procedure TRegExpCollection.FreeItem(P: Pointer); +begin + if P <> nil then + begin + Dispose(PpcRegExp(P),Done); + end; +end; + +function TRegExpCollection.Compare(P1, P2: Pointer): Integer; +//var +// l,l1,l2,i : byte; +//// wPos: pchar; +begin + if CompareModeInsert then + begin +// l1:=length(PpcRegExp(P1)^.RegExp); +// l2:=length(PpcRegExp(P2)^.RegExp); +// if l1 > l2 then l:=l2 else +// l:=l1; +// for i:=1 to l do +// if PpcRegExp(P1).RegExp[i] <> PpcRegExp(P2).RegExp[i] then break; +// if i <=l then +// Compare:=ord(PpcRegExp(P1).RegExp[i])-ord(PpcRegExp(P2).RegExp[i]) else +// Compare:=l1-l2; + Compare := stringsx.PasStrCmp(PpcRegExp(P1).RegExp, PpcRegExp(P2).RegExp, False); + end + else + begin +// l1:=length(PpcRegExp(P1)^.RegExp); +// l2:=length(SearchRegExp); +// if l1 > l2 then l:=l2 else +// l:=l1; +// for i:=1 to l do +// if PpcRegExp(P1).RegExp[i] <> SearchRegExp[i] then +// begin +// Compare:=ord(PpcRegExp(P1).RegExp[i])-ord(SearchRegExp[i]); +// break; +// end; +// if i > l then Compare:=l1-l2; + Compare := stringsx.PasStrCmp(PpcRegExp(P1).RegExp, SearchRegExp, False); + end; +end; + +function TRegExpCollection.Find(ARegExp:shortstring;var P: PpcRegExp):boolean; +var I : integer; +begin + CompareModeInsert:=false; + SearchRegExp:=ARegExp; + if Search(nil,I) then + begin + P:=PpcRegExp(At(I)); + Find:=true; + end + else + begin + P:=nil; + Find:=false; + end; + CompareModeInsert:=true; +end; + +function TRegExpCollection.CheckNew(ARegExp:shortstring):PpcRegExp; +var + P : PpcRegExp; +begin + if not Find(ARegExp,P) then + begin + if Count = MaxRegExp then + AtFree(0); + P:=New(ppcRegExp,Init(ARegExp,PCRE_CASELESS,nil)); + Insert(P); + end; + CheckNew:=P; +end; + +function pcGrepMatch(WildCard, aStr: string; AOptions:integer; ALocale : Pointer): Boolean; +var + PpcRE:PpcRegExp; +begin + PpcRE:=New(ppcRegExp,Init(WildCard,AOptions,Alocale)); + pcGrepMatch:=PpcRE^.Search(pchar(AStr),Length(AStr)); + Dispose(PpcRE,Done); +end; + +function pcGrepSub(WildCard, aStr, aRepl: string; AOptions:integer; ALocale : Pointer): string; +var + PpcRE:PpcRegExp; +begin + PpcRE:=New(ppcRegExp,Init(WildCard,AOptions,Alocale)); + if PpcRE^.Search(pchar(AStr),Length(AStr)) then + pcGrepSub:=PpcRE^.GetReplStr(pchar(AStr),ARepl) + else + pcGrepSub:=''; + Dispose(PpcRE,Done); +end; + +function pcFastGrepMatch(WildCard, aStr: string): Boolean; +var + PpcRE:PpcRegExp; +begin + PpcRE:=PRegExpCache^.CheckNew(WildCard); + pcFastGrepMatch:=PpcRE^.Search(pchar(AStr),Length(AStr)); +end; + +function pcFastGrepSub(WildCard, aStr, aRepl: string): string; +var + PpcRE:PpcRegExp; +begin + PpcRE:=PRegExpCache^.CheckNew(WildCard); + if PpcRE^.Search(pchar(AStr),Length(AStr)) then + pcFastGrepSub:=PpcRE^.GetReplStr(pchar(AStr),ARepl) + else + pcFastGrepSub:=''; +end; + +{$IFDEF PCRE_5_0} +function pcGetVersion : pchar; assembler; {$FRAME-}{$USES none} +asm + call pcre_version +end; +{$ENDIF PCRE_5_0} + +function pcError; +var P: ppcRegExp absolute pRegExp; +begin + Result := (P = nil) or P^.Error; + If Result and (P <> nil) then + begin +{ if P^.ErrorPos = 0 then + MessageBox(GetString(erRegExpCompile)+'"'+P^.ErrorStr+'"', nil,mfConfirmation+mfOkButton) + else} + MessageBox(GetString(erRegExpCompile)+'"'+P^.ErrorStr+'"'+GetString(erRegExpCompPos), + @P^.ErrorPos,mfConfirmation+mfOkButton); + Dispose(P, Done); + P:=nil; + end; +end; + +function pcInit; +var Options : Integer; +begin + If CaseSens then Options := 0 else Options := PCRE_CASELESS; + Result := New( PpcRegExp, Init( Pattern, + {DefaultOptions} + startup.MiscMultiData.cfgRegEx.DefaultOptions or Options, + DefaultLocaleTable) ); +end; + +Initialization + PRegExpCache:=New(PRegExpCollection,Init(64)); +Finalization + Dispose(PRegExpCache,Done); +End. diff --git a/src/third_party/pcre-8.41/pcregrep.c b/src/third_party/pcre-8.41/pcregrep.c new file mode 100644 index 00000000000..317f7454e13 --- /dev/null +++ b/src/third_party/pcre-8.41/pcregrep.c @@ -0,0 +1,3344 @@ +/************************************************* +* pcregrep program * +*************************************************/ + +/* This is a grep program that uses the PCRE regular expression library to do +its pattern matching. On Unix-like, Windows, and native z/OS systems it can +recurse into directories, and in z/OS it can handle PDS files. + +Note that for native z/OS, in addition to defining the NATIVE_ZOS macro, an +additional header is required. That header is not included in the main PCRE +distribution because other apparatus is needed to compile pcregrep for z/OS. +The header can be found in the special z/OS distribution, which is available +from www.zaconsultants.net or from www.cbttape.org. + + Copyright (c) 1997-2014 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include + +#include +#include + +#ifdef HAVE_UNISTD_H +#include +#endif + +#ifdef SUPPORT_LIBZ +#include +#endif + +#ifdef SUPPORT_LIBBZ2 +#include +#endif + +#include "pcre.h" + +#define FALSE 0 +#define TRUE 1 + +typedef int BOOL; + +#define OFFSET_SIZE 99 + +#if BUFSIZ > 8192 +#define MAXPATLEN BUFSIZ +#else +#define MAXPATLEN 8192 +#endif + +#define PATBUFSIZE (MAXPATLEN + 10) /* Allows for prefix+suffix */ + +/* Values for the "filenames" variable, which specifies options for file name +output. The order is important; it is assumed that a file name is wanted for +all values greater than FN_DEFAULT. */ + +enum { FN_NONE, FN_DEFAULT, FN_MATCH_ONLY, FN_NOMATCH_ONLY, FN_FORCE }; + +/* File reading styles */ + +enum { FR_PLAIN, FR_LIBZ, FR_LIBBZ2 }; + +/* Actions for the -d and -D options */ + +enum { dee_READ, dee_SKIP, dee_RECURSE }; +enum { DEE_READ, DEE_SKIP }; + +/* Actions for special processing options (flag bits) */ + +#define PO_WORD_MATCH 0x0001 +#define PO_LINE_MATCH 0x0002 +#define PO_FIXED_STRINGS 0x0004 + +/* Line ending types */ + +enum { EL_LF, EL_CR, EL_CRLF, EL_ANY, EL_ANYCRLF }; + +/* Binary file options */ + +enum { BIN_BINARY, BIN_NOMATCH, BIN_TEXT }; + +/* In newer versions of gcc, with FORTIFY_SOURCE set (the default in some +environments), a warning is issued if the value of fwrite() is ignored. +Unfortunately, casting to (void) does not suppress the warning. To get round +this, we use a macro that compiles a fudge. Oddly, this does not also seem to +apply to fprintf(). */ + +#define FWRITE(a,b,c,d) if (fwrite(a,b,c,d)) {} + + + +/************************************************* +* Global variables * +*************************************************/ + +/* Jeffrey Friedl has some debugging requirements that are not part of the +regular code. */ + +#ifdef JFRIEDL_DEBUG +static int S_arg = -1; +static unsigned int jfriedl_XR = 0; /* repeat regex attempt this many times */ +static unsigned int jfriedl_XT = 0; /* replicate text this many times */ +static const char *jfriedl_prefix = ""; +static const char *jfriedl_postfix = ""; +#endif + +static int endlinetype; + +static char *colour_string = (char *)"1;31"; +static char *colour_option = NULL; +static char *dee_option = NULL; +static char *DEE_option = NULL; +static char *locale = NULL; +static char *main_buffer = NULL; +static char *newline = NULL; +static char *om_separator = (char *)""; +static char *stdin_name = (char *)"(standard input)"; + +static const unsigned char *pcretables = NULL; + +static int after_context = 0; +static int before_context = 0; +static int binary_files = BIN_BINARY; +static int both_context = 0; +static int bufthird = PCREGREP_BUFSIZE; +static int bufsize = 3*PCREGREP_BUFSIZE; + +#if defined HAVE_WINDOWS_H && HAVE_WINDOWS_H +static int dee_action = dee_SKIP; +#else +static int dee_action = dee_READ; +#endif + +static int DEE_action = DEE_READ; +static int error_count = 0; +static int filenames = FN_DEFAULT; +static int pcre_options = 0; +static int process_options = 0; + +#ifdef SUPPORT_PCREGREP_JIT +static int study_options = PCRE_STUDY_JIT_COMPILE; +#else +static int study_options = 0; +#endif + +static unsigned long int match_limit = 0; +static unsigned long int match_limit_recursion = 0; + +static BOOL count_only = FALSE; +static BOOL do_colour = FALSE; +static BOOL file_offsets = FALSE; +static BOOL hyphenpending = FALSE; +static BOOL invert = FALSE; +static BOOL line_buffered = FALSE; +static BOOL line_offsets = FALSE; +static BOOL multiline = FALSE; +static BOOL number = FALSE; +static BOOL omit_zero_count = FALSE; +static BOOL resource_error = FALSE; +static BOOL quiet = FALSE; +static BOOL show_only_matching = FALSE; +static BOOL silent = FALSE; +static BOOL utf8 = FALSE; + +/* Structure for list of --only-matching capturing numbers. */ + +typedef struct omstr { + struct omstr *next; + int groupnum; +} omstr; + +static omstr *only_matching = NULL; +static omstr *only_matching_last = NULL; + +/* Structure for holding the two variables that describe a number chain. */ + +typedef struct omdatastr { + omstr **anchor; + omstr **lastptr; +} omdatastr; + +static omdatastr only_matching_data = { &only_matching, &only_matching_last }; + +/* Structure for list of file names (for -f and --{in,ex}clude-from) */ + +typedef struct fnstr { + struct fnstr *next; + char *name; +} fnstr; + +static fnstr *exclude_from = NULL; +static fnstr *exclude_from_last = NULL; +static fnstr *include_from = NULL; +static fnstr *include_from_last = NULL; + +static fnstr *file_lists = NULL; +static fnstr *file_lists_last = NULL; +static fnstr *pattern_files = NULL; +static fnstr *pattern_files_last = NULL; + +/* Structure for holding the two variables that describe a file name chain. */ + +typedef struct fndatastr { + fnstr **anchor; + fnstr **lastptr; +} fndatastr; + +static fndatastr exclude_from_data = { &exclude_from, &exclude_from_last }; +static fndatastr include_from_data = { &include_from, &include_from_last }; +static fndatastr file_lists_data = { &file_lists, &file_lists_last }; +static fndatastr pattern_files_data = { &pattern_files, &pattern_files_last }; + +/* Structure for pattern and its compiled form; used for matching patterns and +also for include/exclude patterns. */ + +typedef struct patstr { + struct patstr *next; + char *string; + pcre *compiled; + pcre_extra *hint; +} patstr; + +static patstr *patterns = NULL; +static patstr *patterns_last = NULL; +static patstr *include_patterns = NULL; +static patstr *include_patterns_last = NULL; +static patstr *exclude_patterns = NULL; +static patstr *exclude_patterns_last = NULL; +static patstr *include_dir_patterns = NULL; +static patstr *include_dir_patterns_last = NULL; +static patstr *exclude_dir_patterns = NULL; +static patstr *exclude_dir_patterns_last = NULL; + +/* Structure holding the two variables that describe a pattern chain. A pointer +to such structures is used for each appropriate option. */ + +typedef struct patdatastr { + patstr **anchor; + patstr **lastptr; +} patdatastr; + +static patdatastr match_patdata = { &patterns, &patterns_last }; +static patdatastr include_patdata = { &include_patterns, &include_patterns_last }; +static patdatastr exclude_patdata = { &exclude_patterns, &exclude_patterns_last }; +static patdatastr include_dir_patdata = { &include_dir_patterns, &include_dir_patterns_last }; +static patdatastr exclude_dir_patdata = { &exclude_dir_patterns, &exclude_dir_patterns_last }; + +static patstr **incexlist[4] = { &include_patterns, &exclude_patterns, + &include_dir_patterns, &exclude_dir_patterns }; + +static const char *incexname[4] = { "--include", "--exclude", + "--include-dir", "--exclude-dir" }; + +/* Structure for options and list of them */ + +enum { OP_NODATA, OP_STRING, OP_OP_STRING, OP_NUMBER, OP_LONGNUMBER, + OP_OP_NUMBER, OP_OP_NUMBERS, OP_PATLIST, OP_FILELIST, OP_BINFILES }; + +typedef struct option_item { + int type; + int one_char; + void *dataptr; + const char *long_name; + const char *help_text; +} option_item; + +/* Options without a single-letter equivalent get a negative value. This can be +used to identify them. */ + +#define N_COLOUR (-1) +#define N_EXCLUDE (-2) +#define N_EXCLUDE_DIR (-3) +#define N_HELP (-4) +#define N_INCLUDE (-5) +#define N_INCLUDE_DIR (-6) +#define N_LABEL (-7) +#define N_LOCALE (-8) +#define N_NULL (-9) +#define N_LOFFSETS (-10) +#define N_FOFFSETS (-11) +#define N_LBUFFER (-12) +#define N_M_LIMIT (-13) +#define N_M_LIMIT_REC (-14) +#define N_BUFSIZE (-15) +#define N_NOJIT (-16) +#define N_FILE_LIST (-17) +#define N_BINARY_FILES (-18) +#define N_EXCLUDE_FROM (-19) +#define N_INCLUDE_FROM (-20) +#define N_OM_SEPARATOR (-21) + +static option_item optionlist[] = { + { OP_NODATA, N_NULL, NULL, "", "terminate options" }, + { OP_NODATA, N_HELP, NULL, "help", "display this help and exit" }, + { OP_NUMBER, 'A', &after_context, "after-context=number", "set number of following context lines" }, + { OP_NODATA, 'a', NULL, "text", "treat binary files as text" }, + { OP_NUMBER, 'B', &before_context, "before-context=number", "set number of prior context lines" }, + { OP_BINFILES, N_BINARY_FILES, NULL, "binary-files=word", "set treatment of binary files" }, + { OP_NUMBER, N_BUFSIZE,&bufthird, "buffer-size=number", "set processing buffer size parameter" }, + { OP_OP_STRING, N_COLOUR, &colour_option, "color=option", "matched text color option" }, + { OP_OP_STRING, N_COLOUR, &colour_option, "colour=option", "matched text colour option" }, + { OP_NUMBER, 'C', &both_context, "context=number", "set number of context lines, before & after" }, + { OP_NODATA, 'c', NULL, "count", "print only a count of matching lines per FILE" }, + { OP_STRING, 'D', &DEE_option, "devices=action","how to handle devices, FIFOs, and sockets" }, + { OP_STRING, 'd', &dee_option, "directories=action", "how to handle directories" }, + { OP_PATLIST, 'e', &match_patdata, "regex(p)=pattern", "specify pattern (may be used more than once)" }, + { OP_NODATA, 'F', NULL, "fixed-strings", "patterns are sets of newline-separated strings" }, + { OP_FILELIST, 'f', &pattern_files_data, "file=path", "read patterns from file" }, + { OP_FILELIST, N_FILE_LIST, &file_lists_data, "file-list=path","read files to search from file" }, + { OP_NODATA, N_FOFFSETS, NULL, "file-offsets", "output file offsets, not text" }, + { OP_NODATA, 'H', NULL, "with-filename", "force the prefixing filename on output" }, + { OP_NODATA, 'h', NULL, "no-filename", "suppress the prefixing filename on output" }, + { OP_NODATA, 'I', NULL, "", "treat binary files as not matching (ignore)" }, + { OP_NODATA, 'i', NULL, "ignore-case", "ignore case distinctions" }, +#ifdef SUPPORT_PCREGREP_JIT + { OP_NODATA, N_NOJIT, NULL, "no-jit", "do not use just-in-time compiler optimization" }, +#else + { OP_NODATA, N_NOJIT, NULL, "no-jit", "ignored: this pcregrep does not support JIT" }, +#endif + { OP_NODATA, 'l', NULL, "files-with-matches", "print only FILE names containing matches" }, + { OP_NODATA, 'L', NULL, "files-without-match","print only FILE names not containing matches" }, + { OP_STRING, N_LABEL, &stdin_name, "label=name", "set name for standard input" }, + { OP_NODATA, N_LBUFFER, NULL, "line-buffered", "use line buffering" }, + { OP_NODATA, N_LOFFSETS, NULL, "line-offsets", "output line numbers and offsets, not text" }, + { OP_STRING, N_LOCALE, &locale, "locale=locale", "use the named locale" }, + { OP_LONGNUMBER, N_M_LIMIT, &match_limit, "match-limit=number", "set PCRE match limit option" }, + { OP_LONGNUMBER, N_M_LIMIT_REC, &match_limit_recursion, "recursion-limit=number", "set PCRE match recursion limit option" }, + { OP_NODATA, 'M', NULL, "multiline", "run in multiline mode" }, + { OP_STRING, 'N', &newline, "newline=type", "set newline type (CR, LF, CRLF, ANYCRLF or ANY)" }, + { OP_NODATA, 'n', NULL, "line-number", "print line number with output lines" }, + { OP_OP_NUMBERS, 'o', &only_matching_data, "only-matching=n", "show only the part of the line that matched" }, + { OP_STRING, N_OM_SEPARATOR, &om_separator, "om-separator=text", "set separator for multiple -o output" }, + { OP_NODATA, 'q', NULL, "quiet", "suppress output, just set return code" }, + { OP_NODATA, 'r', NULL, "recursive", "recursively scan sub-directories" }, + { OP_PATLIST, N_EXCLUDE,&exclude_patdata, "exclude=pattern","exclude matching files when recursing" }, + { OP_PATLIST, N_INCLUDE,&include_patdata, "include=pattern","include matching files when recursing" }, + { OP_PATLIST, N_EXCLUDE_DIR,&exclude_dir_patdata, "exclude-dir=pattern","exclude matching directories when recursing" }, + { OP_PATLIST, N_INCLUDE_DIR,&include_dir_patdata, "include-dir=pattern","include matching directories when recursing" }, + { OP_FILELIST, N_EXCLUDE_FROM,&exclude_from_data, "exclude-from=path", "read exclude list from file" }, + { OP_FILELIST, N_INCLUDE_FROM,&include_from_data, "include-from=path", "read include list from file" }, + + /* These two were accidentally implemented with underscores instead of + hyphens in the option names. As this was not discovered for several releases, + the incorrect versions are left in the table for compatibility. However, the + --help function misses out any option that has an underscore in its name. */ + + { OP_PATLIST, N_EXCLUDE_DIR,&exclude_dir_patdata, "exclude_dir=pattern","exclude matching directories when recursing" }, + { OP_PATLIST, N_INCLUDE_DIR,&include_dir_patdata, "include_dir=pattern","include matching directories when recursing" }, + +#ifdef JFRIEDL_DEBUG + { OP_OP_NUMBER, 'S', &S_arg, "jeffS", "replace matched (sub)string with X" }, +#endif + { OP_NODATA, 's', NULL, "no-messages", "suppress error messages" }, + { OP_NODATA, 'u', NULL, "utf-8", "use UTF-8 mode" }, + { OP_NODATA, 'V', NULL, "version", "print version information and exit" }, + { OP_NODATA, 'v', NULL, "invert-match", "select non-matching lines" }, + { OP_NODATA, 'w', NULL, "word-regex(p)", "force patterns to match only as words" }, + { OP_NODATA, 'x', NULL, "line-regex(p)", "force patterns to match only whole lines" }, + { OP_NODATA, 0, NULL, NULL, NULL } +}; + +/* Tables for prefixing and suffixing patterns, according to the -w, -x, and -F +options. These set the 1, 2, and 4 bits in process_options, respectively. Note +that the combination of -w and -x has the same effect as -x on its own, so we +can treat them as the same. Note that the MAXPATLEN macro assumes the longest +prefix+suffix is 10 characters; if anything longer is added, it must be +adjusted. */ + +static const char *prefix[] = { + "", "\\b", "^(?:", "^(?:", "\\Q", "\\b\\Q", "^(?:\\Q", "^(?:\\Q" }; + +static const char *suffix[] = { + "", "\\b", ")$", ")$", "\\E", "\\E\\b", "\\E)$", "\\E)$" }; + +/* UTF-8 tables - used only when the newline setting is "any". */ + +const int utf8_table3[] = { 0xff, 0x1f, 0x0f, 0x07, 0x03, 0x01}; + +const char utf8_table4[] = { + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, + 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 }; + + + +/************************************************* +* Exit from the program * +*************************************************/ + +/* If there has been a resource error, give a suitable message. + +Argument: the return code +Returns: does not return +*/ + +static void +pcregrep_exit(int rc) +{ +if (resource_error) + { + fprintf(stderr, "pcregrep: Error %d, %d or %d means that a resource limit " + "was exceeded.\n", PCRE_ERROR_MATCHLIMIT, PCRE_ERROR_RECURSIONLIMIT, + PCRE_ERROR_JIT_STACKLIMIT); + fprintf(stderr, "pcregrep: Check your regex for nested unlimited loops.\n"); + } +exit(rc); +} + + +/************************************************* +* Add item to chain of patterns * +*************************************************/ + +/* Used to add an item onto a chain, or just return an unconnected item if the +"after" argument is NULL. + +Arguments: + s pattern string to add + after if not NULL points to item to insert after + +Returns: new pattern block or NULL on error +*/ + +static patstr * +add_pattern(char *s, patstr *after) +{ +patstr *p = (patstr *)malloc(sizeof(patstr)); +if (p == NULL) + { + fprintf(stderr, "pcregrep: malloc failed\n"); + pcregrep_exit(2); + } +if (strlen(s) > MAXPATLEN) + { + fprintf(stderr, "pcregrep: pattern is too long (limit is %d bytes)\n", + MAXPATLEN); + free(p); + return NULL; + } +p->next = NULL; +p->string = s; +p->compiled = NULL; +p->hint = NULL; + +if (after != NULL) + { + p->next = after->next; + after->next = p; + } +return p; +} + + +/************************************************* +* Free chain of patterns * +*************************************************/ + +/* Used for several chains of patterns. + +Argument: pointer to start of chain +Returns: nothing +*/ + +static void +free_pattern_chain(patstr *pc) +{ +while (pc != NULL) + { + patstr *p = pc; + pc = p->next; + if (p->hint != NULL) pcre_free_study(p->hint); + if (p->compiled != NULL) pcre_free(p->compiled); + free(p); + } +} + + +/************************************************* +* Free chain of file names * +*************************************************/ + +/* +Argument: pointer to start of chain +Returns: nothing +*/ + +static void +free_file_chain(fnstr *fn) +{ +while (fn != NULL) + { + fnstr *f = fn; + fn = f->next; + free(f); + } +} + + +/************************************************* +* OS-specific functions * +*************************************************/ + +/* These functions are defined so that they can be made system specific. +At present there are versions for Unix-style environments, Windows, native +z/OS, and "no support". */ + + +/************* Directory scanning Unix-style and z/OS ***********/ + +#if (defined HAVE_SYS_STAT_H && defined HAVE_DIRENT_H && defined HAVE_SYS_TYPES_H) || defined NATIVE_ZOS +#include +#include +#include + +#if defined NATIVE_ZOS +/************* Directory and PDS/E scanning for z/OS ***********/ +/************* z/OS looks mostly like Unix with USS ************/ +/* However, z/OS needs the #include statements in this header */ +#include "pcrzosfs.h" +/* That header is not included in the main PCRE distribution because + other apparatus is needed to compile pcregrep for z/OS. The header + can be found in the special z/OS distribution, which is available + from www.zaconsultants.net or from www.cbttape.org. */ +#endif + +typedef DIR directory_type; +#define FILESEP '/' + +static int +isdirectory(char *filename) +{ +struct stat statbuf; +if (stat(filename, &statbuf) < 0) + return 0; /* In the expectation that opening as a file will fail */ +return (statbuf.st_mode & S_IFMT) == S_IFDIR; +} + +static directory_type * +opendirectory(char *filename) +{ +return opendir(filename); +} + +static char * +readdirectory(directory_type *dir) +{ +for (;;) + { + struct dirent *dent = readdir(dir); + if (dent == NULL) return NULL; + if (strcmp(dent->d_name, ".") != 0 && strcmp(dent->d_name, "..") != 0) + return dent->d_name; + } +/* Control never reaches here */ +} + +static void +closedirectory(directory_type *dir) +{ +closedir(dir); +} + + +/************* Test for regular file, Unix-style **********/ + +static int +isregfile(char *filename) +{ +struct stat statbuf; +if (stat(filename, &statbuf) < 0) + return 1; /* In the expectation that opening as a file will fail */ +return (statbuf.st_mode & S_IFMT) == S_IFREG; +} + + +#if defined NATIVE_ZOS +/************* Test for a terminal in z/OS **********/ +/* isatty() does not work in a TSO environment, so always give FALSE.*/ + +static BOOL +is_stdout_tty(void) +{ +return FALSE; +} + +static BOOL +is_file_tty(FILE *f) +{ +return FALSE; +} + + +/************* Test for a terminal, Unix-style **********/ + +#else +static BOOL +is_stdout_tty(void) +{ +return isatty(fileno(stdout)); +} + +static BOOL +is_file_tty(FILE *f) +{ +return isatty(fileno(f)); +} +#endif + +/* End of Unix-style or native z/OS environment functions. */ + + +/************* Directory scanning in Windows ***********/ + +/* I (Philip Hazel) have no means of testing this code. It was contributed by +Lionel Fourquaux. David Burgess added a patch to define INVALID_FILE_ATTRIBUTES +when it did not exist. David Byron added a patch that moved the #include of + to before the INVALID_FILE_ATTRIBUTES definition rather than after. +The double test below stops gcc 4.4.4 grumbling that HAVE_WINDOWS_H is +undefined when it is indeed undefined. */ + +#elif defined HAVE_WINDOWS_H && HAVE_WINDOWS_H + +#ifndef STRICT +# define STRICT +#endif +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif + +#include + +#ifndef INVALID_FILE_ATTRIBUTES +#define INVALID_FILE_ATTRIBUTES 0xFFFFFFFF +#endif + +typedef struct directory_type +{ +HANDLE handle; +BOOL first; +WIN32_FIND_DATA data; +} directory_type; + +#define FILESEP '/' + +int +isdirectory(char *filename) +{ +DWORD attr = GetFileAttributes(filename); +if (attr == INVALID_FILE_ATTRIBUTES) + return 0; +return (attr & FILE_ATTRIBUTE_DIRECTORY) != 0; +} + +directory_type * +opendirectory(char *filename) +{ +size_t len; +char *pattern; +directory_type *dir; +DWORD err; +len = strlen(filename); +pattern = (char *)malloc(len + 3); +dir = (directory_type *)malloc(sizeof(*dir)); +if ((pattern == NULL) || (dir == NULL)) + { + fprintf(stderr, "pcregrep: malloc failed\n"); + pcregrep_exit(2); + } +memcpy(pattern, filename, len); +memcpy(&(pattern[len]), "\\*", 3); +dir->handle = FindFirstFile(pattern, &(dir->data)); +if (dir->handle != INVALID_HANDLE_VALUE) + { + free(pattern); + dir->first = TRUE; + return dir; + } +err = GetLastError(); +free(pattern); +free(dir); +errno = (err == ERROR_ACCESS_DENIED) ? EACCES : ENOENT; +return NULL; +} + +char * +readdirectory(directory_type *dir) +{ +for (;;) + { + if (!dir->first) + { + if (!FindNextFile(dir->handle, &(dir->data))) + return NULL; + } + else + { + dir->first = FALSE; + } + if (strcmp(dir->data.cFileName, ".") != 0 && strcmp(dir->data.cFileName, "..") != 0) + return dir->data.cFileName; + } +#ifndef _MSC_VER +return NULL; /* Keep compiler happy; never executed */ +#endif +} + +void +closedirectory(directory_type *dir) +{ +FindClose(dir->handle); +free(dir); +} + + +/************* Test for regular file in Windows **********/ + +/* I don't know how to do this, or if it can be done; assume all paths are +regular if they are not directories. */ + +int isregfile(char *filename) +{ +return !isdirectory(filename); +} + + +/************* Test for a terminal in Windows **********/ + +/* I don't know how to do this; assume never */ + +static BOOL +is_stdout_tty(void) +{ +return FALSE; +} + +static BOOL +is_file_tty(FILE *f) +{ +return FALSE; +} + +/* End of Windows functions */ + + +/************* Directory scanning when we can't do it ***********/ + +/* The type is void, and apart from isdirectory(), the functions do nothing. */ + +#else + +#define FILESEP 0 +typedef void directory_type; + +int isdirectory(char *filename) { return 0; } +directory_type * opendirectory(char *filename) { return (directory_type*)0;} +char *readdirectory(directory_type *dir) { return (char*)0;} +void closedirectory(directory_type *dir) {} + + +/************* Test for regular file when we can't do it **********/ + +/* Assume all files are regular. */ + +int isregfile(char *filename) { return 1; } + + +/************* Test for a terminal when we can't do it **********/ + +static BOOL +is_stdout_tty(void) +{ +return FALSE; +} + +static BOOL +is_file_tty(FILE *f) +{ +return FALSE; +} + +#endif /* End of system-specific functions */ + + + +#ifndef HAVE_STRERROR +/************************************************* +* Provide strerror() for non-ANSI libraries * +*************************************************/ + +/* Some old-fashioned systems still around (e.g. SunOS4) don't have strerror() +in their libraries, but can provide the same facility by this simple +alternative function. */ + +extern int sys_nerr; +extern char *sys_errlist[]; + +char * +strerror(int n) +{ +if (n < 0 || n >= sys_nerr) return "unknown error number"; +return sys_errlist[n]; +} +#endif /* HAVE_STRERROR */ + + + +/************************************************* +* Usage function * +*************************************************/ + +static int +usage(int rc) +{ +option_item *op; +fprintf(stderr, "Usage: pcregrep [-"); +for (op = optionlist; op->one_char != 0; op++) + { + if (op->one_char > 0) fprintf(stderr, "%c", op->one_char); + } +fprintf(stderr, "] [long options] [pattern] [files]\n"); +fprintf(stderr, "Type `pcregrep --help' for more information and the long " + "options.\n"); +return rc; +} + + + +/************************************************* +* Help function * +*************************************************/ + +static void +help(void) +{ +option_item *op; + +printf("Usage: pcregrep [OPTION]... [PATTERN] [FILE1 FILE2 ...]\n"); +printf("Search for PATTERN in each FILE or standard input.\n"); +printf("PATTERN must be present if neither -e nor -f is used.\n"); +printf("\"-\" can be used as a file name to mean STDIN.\n"); + +#ifdef SUPPORT_LIBZ +printf("Files whose names end in .gz are read using zlib.\n"); +#endif + +#ifdef SUPPORT_LIBBZ2 +printf("Files whose names end in .bz2 are read using bzlib2.\n"); +#endif + +#if defined SUPPORT_LIBZ || defined SUPPORT_LIBBZ2 +printf("Other files and the standard input are read as plain files.\n\n"); +#else +printf("All files are read as plain files, without any interpretation.\n\n"); +#endif + +printf("Example: pcregrep -i 'hello.*world' menu.h main.c\n\n"); +printf("Options:\n"); + +for (op = optionlist; op->one_char != 0; op++) + { + int n; + char s[4]; + + /* Two options were accidentally implemented and documented with underscores + instead of hyphens in their names, something that was not noticed for quite a + few releases. When fixing this, I left the underscored versions in the list + in case people were using them. However, we don't want to display them in the + help data. There are no other options that contain underscores, and we do not + expect ever to implement such options. Therefore, just omit any option that + contains an underscore. */ + + if (strchr(op->long_name, '_') != NULL) continue; + + if (op->one_char > 0 && (op->long_name)[0] == 0) + n = 31 - printf(" -%c", op->one_char); + else + { + if (op->one_char > 0) sprintf(s, "-%c,", op->one_char); + else strcpy(s, " "); + n = 31 - printf(" %s --%s", s, op->long_name); + } + + if (n < 1) n = 1; + printf("%.*s%s\n", n, " ", op->help_text); + } + +printf("\nNumbers may be followed by K or M, e.g. --buffer-size=100K.\n"); +printf("The default value for --buffer-size is %d.\n", PCREGREP_BUFSIZE); +printf("When reading patterns or file names from a file, trailing white\n"); +printf("space is removed and blank lines are ignored.\n"); +printf("The maximum size of any pattern is %d bytes.\n", MAXPATLEN); + +printf("\nWith no FILEs, read standard input. If fewer than two FILEs given, assume -h.\n"); +printf("Exit status is 0 if any matches, 1 if no matches, and 2 if trouble.\n"); +} + + + +/************************************************* +* Test exclude/includes * +*************************************************/ + +/* If any exclude pattern matches, the path is excluded. Otherwise, unless +there are no includes, the path must match an include pattern. + +Arguments: + path the path to be matched + ip the chain of include patterns + ep the chain of exclude patterns + +Returns: TRUE if the path is not excluded +*/ + +static BOOL +test_incexc(char *path, patstr *ip, patstr *ep) +{ +int plen = strlen(path); + +for (; ep != NULL; ep = ep->next) + { + if (pcre_exec(ep->compiled, NULL, path, plen, 0, 0, NULL, 0) >= 0) + return FALSE; + } + +if (ip == NULL) return TRUE; + +for (; ip != NULL; ip = ip->next) + { + if (pcre_exec(ip->compiled, NULL, path, plen, 0, 0, NULL, 0) >= 0) + return TRUE; + } + +return FALSE; +} + + + +/************************************************* +* Decode integer argument value * +*************************************************/ + +/* Integer arguments can be followed by K or M. Avoid the use of strtoul() +because SunOS4 doesn't have it. This is used only for unpicking arguments, so +just keep it simple. + +Arguments: + option_data the option data string + op the option item (for error messages) + longop TRUE if option given in long form + +Returns: a long integer +*/ + +static long int +decode_number(char *option_data, option_item *op, BOOL longop) +{ +unsigned long int n = 0; +char *endptr = option_data; +while (*endptr != 0 && isspace((unsigned char)(*endptr))) endptr++; +while (isdigit((unsigned char)(*endptr))) + n = n * 10 + (int)(*endptr++ - '0'); +if (toupper(*endptr) == 'K') + { + n *= 1024; + endptr++; + } +else if (toupper(*endptr) == 'M') + { + n *= 1024*1024; + endptr++; + } + +if (*endptr != 0) /* Error */ + { + if (longop) + { + char *equals = strchr(op->long_name, '='); + int nlen = (equals == NULL)? (int)strlen(op->long_name) : + (int)(equals - op->long_name); + fprintf(stderr, "pcregrep: Malformed number \"%s\" after --%.*s\n", + option_data, nlen, op->long_name); + } + else + fprintf(stderr, "pcregrep: Malformed number \"%s\" after -%c\n", + option_data, op->one_char); + pcregrep_exit(usage(2)); + } + +return n; +} + + + +/************************************************* +* Add item to a chain of numbers * +*************************************************/ + +/* Used to add an item onto a chain, or just return an unconnected item if the +"after" argument is NULL. + +Arguments: + n the number to add + after if not NULL points to item to insert after + +Returns: new number block +*/ + +static omstr * +add_number(int n, omstr *after) +{ +omstr *om = (omstr *)malloc(sizeof(omstr)); + +if (om == NULL) + { + fprintf(stderr, "pcregrep: malloc failed\n"); + pcregrep_exit(2); + } +om->next = NULL; +om->groupnum = n; + +if (after != NULL) + { + om->next = after->next; + after->next = om; + } +return om; +} + + + +/************************************************* +* Read one line of input * +*************************************************/ + +/* Normally, input is read using fread() into a large buffer, so many lines may +be read at once. However, doing this for tty input means that no output appears +until a lot of input has been typed. Instead, tty input is handled line by +line. We cannot use fgets() for this, because it does not stop at a binary +zero, and therefore there is no way of telling how many characters it has read, +because there may be binary zeros embedded in the data. + +Arguments: + buffer the buffer to read into + length the maximum number of characters to read + f the file + +Returns: the number of characters read, zero at end of file +*/ + +static unsigned int +read_one_line(char *buffer, int length, FILE *f) +{ +int c; +int yield = 0; +while ((c = fgetc(f)) != EOF) + { + buffer[yield++] = c; + if (c == '\n' || yield >= length) break; + } +return yield; +} + + + +/************************************************* +* Find end of line * +*************************************************/ + +/* The length of the endline sequence that is found is set via lenptr. This may +be zero at the very end of the file if there is no line-ending sequence there. + +Arguments: + p current position in line + endptr end of available data + lenptr where to put the length of the eol sequence + +Returns: pointer after the last byte of the line, + including the newline byte(s) +*/ + +static char * +end_of_line(char *p, char *endptr, int *lenptr) +{ +switch(endlinetype) + { + default: /* Just in case */ + case EL_LF: + while (p < endptr && *p != '\n') p++; + if (p < endptr) + { + *lenptr = 1; + return p + 1; + } + *lenptr = 0; + return endptr; + + case EL_CR: + while (p < endptr && *p != '\r') p++; + if (p < endptr) + { + *lenptr = 1; + return p + 1; + } + *lenptr = 0; + return endptr; + + case EL_CRLF: + for (;;) + { + while (p < endptr && *p != '\r') p++; + if (++p >= endptr) + { + *lenptr = 0; + return endptr; + } + if (*p == '\n') + { + *lenptr = 2; + return p + 1; + } + } + break; + + case EL_ANYCRLF: + while (p < endptr) + { + int extra = 0; + register int c = *((unsigned char *)p); + + if (utf8 && c >= 0xc0) + { + int gcii, gcss; + extra = utf8_table4[c & 0x3f]; /* Number of additional bytes */ + gcss = 6*extra; + c = (c & utf8_table3[extra]) << gcss; + for (gcii = 1; gcii <= extra; gcii++) + { + gcss -= 6; + c |= (p[gcii] & 0x3f) << gcss; + } + } + + p += 1 + extra; + + switch (c) + { + case '\n': + *lenptr = 1; + return p; + + case '\r': + if (p < endptr && *p == '\n') + { + *lenptr = 2; + p++; + } + else *lenptr = 1; + return p; + + default: + break; + } + } /* End of loop for ANYCRLF case */ + + *lenptr = 0; /* Must have hit the end */ + return endptr; + + case EL_ANY: + while (p < endptr) + { + int extra = 0; + register int c = *((unsigned char *)p); + + if (utf8 && c >= 0xc0) + { + int gcii, gcss; + extra = utf8_table4[c & 0x3f]; /* Number of additional bytes */ + gcss = 6*extra; + c = (c & utf8_table3[extra]) << gcss; + for (gcii = 1; gcii <= extra; gcii++) + { + gcss -= 6; + c |= (p[gcii] & 0x3f) << gcss; + } + } + + p += 1 + extra; + + switch (c) + { + case '\n': /* LF */ + case '\v': /* VT */ + case '\f': /* FF */ + *lenptr = 1; + return p; + + case '\r': /* CR */ + if (p < endptr && *p == '\n') + { + *lenptr = 2; + p++; + } + else *lenptr = 1; + return p; + +#ifndef EBCDIC + case 0x85: /* Unicode NEL */ + *lenptr = utf8? 2 : 1; + return p; + + case 0x2028: /* Unicode LS */ + case 0x2029: /* Unicode PS */ + *lenptr = 3; + return p; +#endif /* Not EBCDIC */ + + default: + break; + } + } /* End of loop for ANY case */ + + *lenptr = 0; /* Must have hit the end */ + return endptr; + } /* End of overall switch */ +} + + + +/************************************************* +* Find start of previous line * +*************************************************/ + +/* This is called when looking back for before lines to print. + +Arguments: + p start of the subsequent line + startptr start of available data + +Returns: pointer to the start of the previous line +*/ + +static char * +previous_line(char *p, char *startptr) +{ +switch(endlinetype) + { + default: /* Just in case */ + case EL_LF: + p--; + while (p > startptr && p[-1] != '\n') p--; + return p; + + case EL_CR: + p--; + while (p > startptr && p[-1] != '\n') p--; + return p; + + case EL_CRLF: + for (;;) + { + p -= 2; + while (p > startptr && p[-1] != '\n') p--; + if (p <= startptr + 1 || p[-2] == '\r') return p; + } + /* Control can never get here */ + + case EL_ANY: + case EL_ANYCRLF: + if (*(--p) == '\n' && p > startptr && p[-1] == '\r') p--; + if (utf8) while ((*p & 0xc0) == 0x80) p--; + + while (p > startptr) + { + register unsigned int c; + char *pp = p - 1; + + if (utf8) + { + int extra = 0; + while ((*pp & 0xc0) == 0x80) pp--; + c = *((unsigned char *)pp); + if (c >= 0xc0) + { + int gcii, gcss; + extra = utf8_table4[c & 0x3f]; /* Number of additional bytes */ + gcss = 6*extra; + c = (c & utf8_table3[extra]) << gcss; + for (gcii = 1; gcii <= extra; gcii++) + { + gcss -= 6; + c |= (pp[gcii] & 0x3f) << gcss; + } + } + } + else c = *((unsigned char *)pp); + + if (endlinetype == EL_ANYCRLF) switch (c) + { + case '\n': /* LF */ + case '\r': /* CR */ + return p; + + default: + break; + } + + else switch (c) + { + case '\n': /* LF */ + case '\v': /* VT */ + case '\f': /* FF */ + case '\r': /* CR */ +#ifndef EBCDIE + case 0x85: /* Unicode NEL */ + case 0x2028: /* Unicode LS */ + case 0x2029: /* Unicode PS */ +#endif /* Not EBCDIC */ + return p; + + default: + break; + } + + p = pp; /* Back one character */ + } /* End of loop for ANY case */ + + return startptr; /* Hit start of data */ + } /* End of overall switch */ +} + + + + + +/************************************************* +* Print the previous "after" lines * +*************************************************/ + +/* This is called if we are about to lose said lines because of buffer filling, +and at the end of the file. The data in the line is written using fwrite() so +that a binary zero does not terminate it. + +Arguments: + lastmatchnumber the number of the last matching line, plus one + lastmatchrestart where we restarted after the last match + endptr end of available data + printname filename for printing + +Returns: nothing +*/ + +static void +do_after_lines(int lastmatchnumber, char *lastmatchrestart, char *endptr, + char *printname) +{ +if (after_context > 0 && lastmatchnumber > 0) + { + int count = 0; + while (lastmatchrestart < endptr && count++ < after_context) + { + int ellength; + char *pp = lastmatchrestart; + if (printname != NULL) fprintf(stdout, "%s-", printname); + if (number) fprintf(stdout, "%d-", lastmatchnumber++); + pp = end_of_line(pp, endptr, &ellength); + FWRITE(lastmatchrestart, 1, pp - lastmatchrestart, stdout); + lastmatchrestart = pp; + } + hyphenpending = TRUE; + } +} + + + +/************************************************* +* Apply patterns to subject till one matches * +*************************************************/ + +/* This function is called to run through all patterns, looking for a match. It +is used multiple times for the same subject when colouring is enabled, in order +to find all possible matches. + +Arguments: + matchptr the start of the subject + length the length of the subject to match + options options for pcre_exec + startoffset where to start matching + offsets the offets vector to fill in + mrc address of where to put the result of pcre_exec() + +Returns: TRUE if there was a match + FALSE if there was no match + invert if there was a non-fatal error +*/ + +static BOOL +match_patterns(char *matchptr, size_t length, unsigned int options, + int startoffset, int *offsets, int *mrc) +{ +int i; +size_t slen = length; +patstr *p = patterns; +const char *msg = "this text:\n\n"; + +if (slen > 200) + { + slen = 200; + msg = "text that starts:\n\n"; + } +for (i = 1; p != NULL; p = p->next, i++) + { + *mrc = pcre_exec(p->compiled, p->hint, matchptr, (int)length, + startoffset, options, offsets, OFFSET_SIZE); + if (*mrc >= 0) return TRUE; + if (*mrc == PCRE_ERROR_NOMATCH) continue; + fprintf(stderr, "pcregrep: pcre_exec() gave error %d while matching ", *mrc); + if (patterns->next != NULL) fprintf(stderr, "pattern number %d to ", i); + fprintf(stderr, "%s", msg); + FWRITE(matchptr, 1, slen, stderr); /* In case binary zero included */ + fprintf(stderr, "\n\n"); + if (*mrc == PCRE_ERROR_MATCHLIMIT || *mrc == PCRE_ERROR_RECURSIONLIMIT || + *mrc == PCRE_ERROR_JIT_STACKLIMIT) + resource_error = TRUE; + if (error_count++ > 20) + { + fprintf(stderr, "pcregrep: Too many errors - abandoned.\n"); + pcregrep_exit(2); + } + return invert; /* No more matching; don't show the line again */ + } + +return FALSE; /* No match, no errors */ +} + + + +/************************************************* +* Grep an individual file * +*************************************************/ + +/* This is called from grep_or_recurse() below. It uses a buffer that is three +times the value of bufthird. The matching point is never allowed to stray into +the top third of the buffer, thus keeping more of the file available for +context printing or for multiline scanning. For large files, the pointer will +be in the middle third most of the time, so the bottom third is available for +"before" context printing. + +Arguments: + handle the fopened FILE stream for a normal file + the gzFile pointer when reading is via libz + the BZFILE pointer when reading is via libbz2 + frtype FR_PLAIN, FR_LIBZ, or FR_LIBBZ2 + filename the file name or NULL (for errors) + printname the file name if it is to be printed for each match + or NULL if the file name is not to be printed + it cannot be NULL if filenames[_nomatch]_only is set + +Returns: 0 if there was at least one match + 1 otherwise (no matches) + 2 if an overlong line is encountered + 3 if there is a read error on a .bz2 file +*/ + +static int +pcregrep(void *handle, int frtype, char *filename, char *printname) +{ +int rc = 1; +int linenumber = 1; +int lastmatchnumber = 0; +int count = 0; +int filepos = 0; +int offsets[OFFSET_SIZE]; +char *lastmatchrestart = NULL; +char *ptr = main_buffer; +char *endptr; +size_t bufflength; +BOOL binary = FALSE; +BOOL endhyphenpending = FALSE; +BOOL input_line_buffered = line_buffered; +FILE *in = NULL; /* Ensure initialized */ + +#ifdef SUPPORT_LIBZ +gzFile ingz = NULL; +#endif + +#ifdef SUPPORT_LIBBZ2 +BZFILE *inbz2 = NULL; +#endif + + +/* Do the first read into the start of the buffer and set up the pointer to end +of what we have. In the case of libz, a non-zipped .gz file will be read as a +plain file. However, if a .bz2 file isn't actually bzipped, the first read will +fail. */ + +(void)frtype; + +#ifdef SUPPORT_LIBZ +if (frtype == FR_LIBZ) + { + ingz = (gzFile)handle; + bufflength = gzread (ingz, main_buffer, bufsize); + } +else +#endif + +#ifdef SUPPORT_LIBBZ2 +if (frtype == FR_LIBBZ2) + { + inbz2 = (BZFILE *)handle; + bufflength = BZ2_bzread(inbz2, main_buffer, bufsize); + if ((int)bufflength < 0) return 2; /* Gotcha: bufflength is size_t; */ + } /* without the cast it is unsigned. */ +else +#endif + + { + in = (FILE *)handle; + if (is_file_tty(in)) input_line_buffered = TRUE; + bufflength = input_line_buffered? + read_one_line(main_buffer, bufsize, in) : + fread(main_buffer, 1, bufsize, in); + } + +endptr = main_buffer + bufflength; + +/* Unless binary-files=text, see if we have a binary file. This uses the same +rule as GNU grep, namely, a search for a binary zero byte near the start of the +file. */ + +if (binary_files != BIN_TEXT) + { + binary = + memchr(main_buffer, 0, (bufflength > 1024)? 1024 : bufflength) != NULL; + if (binary && binary_files == BIN_NOMATCH) return 1; + } + +/* Loop while the current pointer is not at the end of the file. For large +files, endptr will be at the end of the buffer when we are in the middle of the +file, but ptr will never get there, because as soon as it gets over 2/3 of the +way, the buffer is shifted left and re-filled. */ + +while (ptr < endptr) + { + int endlinelength; + int mrc = 0; + int startoffset = 0; + int prevoffsets[2]; + unsigned int options = 0; + BOOL match; + char *matchptr = ptr; + char *t = ptr; + size_t length, linelength; + + prevoffsets[0] = prevoffsets[1] = -1; + + /* At this point, ptr is at the start of a line. We need to find the length + of the subject string to pass to pcre_exec(). In multiline mode, it is the + length remainder of the data in the buffer. Otherwise, it is the length of + the next line, excluding the terminating newline. After matching, we always + advance by the length of the next line. In multiline mode the PCRE_FIRSTLINE + option is used for compiling, so that any match is constrained to be in the + first line. */ + + t = end_of_line(t, endptr, &endlinelength); + linelength = t - ptr - endlinelength; + length = multiline? (size_t)(endptr - ptr) : linelength; + + /* Check to see if the line we are looking at extends right to the very end + of the buffer without a line terminator. This means the line is too long to + handle. */ + + if (endlinelength == 0 && t == main_buffer + bufsize) + { + fprintf(stderr, "pcregrep: line %d%s%s is too long for the internal buffer\n" + "pcregrep: check the --buffer-size option\n", + linenumber, + (filename == NULL)? "" : " of file ", + (filename == NULL)? "" : filename); + return 2; + } + + /* Extra processing for Jeffrey Friedl's debugging. */ + +#ifdef JFRIEDL_DEBUG + if (jfriedl_XT || jfriedl_XR) + { +# include +# include + struct timeval start_time, end_time; + struct timezone dummy; + int i; + + if (jfriedl_XT) + { + unsigned long newlen = length * jfriedl_XT + strlen(jfriedl_prefix) + strlen(jfriedl_postfix); + const char *orig = ptr; + ptr = malloc(newlen + 1); + if (!ptr) { + printf("out of memory"); + pcregrep_exit(2); + } + endptr = ptr; + strcpy(endptr, jfriedl_prefix); endptr += strlen(jfriedl_prefix); + for (i = 0; i < jfriedl_XT; i++) { + strncpy(endptr, orig, length); + endptr += length; + } + strcpy(endptr, jfriedl_postfix); endptr += strlen(jfriedl_postfix); + length = newlen; + } + + if (gettimeofday(&start_time, &dummy) != 0) + perror("bad gettimeofday"); + + + for (i = 0; i < jfriedl_XR; i++) + match = (pcre_exec(patterns->compiled, patterns->hint, ptr, length, 0, + PCRE_NOTEMPTY, offsets, OFFSET_SIZE) >= 0); + + if (gettimeofday(&end_time, &dummy) != 0) + perror("bad gettimeofday"); + + double delta = ((end_time.tv_sec + (end_time.tv_usec / 1000000.0)) + - + (start_time.tv_sec + (start_time.tv_usec / 1000000.0))); + + printf("%s TIMER[%.4f]\n", match ? "MATCH" : "FAIL", delta); + return 0; + } +#endif + + /* We come back here after a match when show_only_matching is set, in order + to find any further matches in the same line. This applies to + --only-matching, --file-offsets, and --line-offsets. */ + + ONLY_MATCHING_RESTART: + + /* Run through all the patterns until one matches or there is an error other + than NOMATCH. This code is in a subroutine so that it can be re-used for + finding subsequent matches when colouring matched lines. After finding one + match, set PCRE_NOTEMPTY to disable any further matches of null strings in + this line. */ + + match = match_patterns(matchptr, length, options, startoffset, offsets, &mrc); + options = PCRE_NOTEMPTY; + + /* If it's a match or a not-match (as required), do what's wanted. */ + + if (match != invert) + { + BOOL hyphenprinted = FALSE; + + /* We've failed if we want a file that doesn't have any matches. */ + + if (filenames == FN_NOMATCH_ONLY) return 1; + + /* If all we want is a yes/no answer, stop now. */ + + if (quiet) return 0; + + /* Just count if just counting is wanted. */ + + else if (count_only) count++; + + /* When handling a binary file and binary-files==binary, the "binary" + variable will be set true (it's false in all other cases). In this + situation we just want to output the file name. No need to scan further. */ + + else if (binary) + { + fprintf(stdout, "Binary file %s matches\n", filename); + return 0; + } + + /* If all we want is a file name, there is no need to scan any more lines + in the file. */ + + else if (filenames == FN_MATCH_ONLY) + { + fprintf(stdout, "%s\n", printname); + return 0; + } + + /* The --only-matching option prints just the substring that matched, + and/or one or more captured portions of it, as long as these strings are + not empty. The --file-offsets and --line-offsets options output offsets for + the matching substring (all three set show_only_matching). None of these + mutually exclusive options prints any context. Afterwards, adjust the start + and then jump back to look for further matches in the same line. If we are + in invert mode, however, nothing is printed and we do not restart - this + could still be useful because the return code is set. */ + + else if (show_only_matching) + { + if (!invert) + { + int oldstartoffset = startoffset; + + /* It is possible, when a lookbehind assertion contains \K, for the + same string to be found again. The code below advances startoffset, but + until it is past the "bumpalong" offset that gave the match, the same + substring will be returned. The PCRE1 library does not return the + bumpalong offset, so all we can do is ignore repeated strings. (PCRE2 + does this better.) */ + + if (prevoffsets[0] != offsets[0] || prevoffsets[1] != offsets[1]) + { + prevoffsets[0] = offsets[0]; + prevoffsets[1] = offsets[1]; + + if (printname != NULL) fprintf(stdout, "%s:", printname); + if (number) fprintf(stdout, "%d:", linenumber); + + /* Handle --line-offsets */ + + if (line_offsets) + fprintf(stdout, "%d,%d\n", (int)(matchptr + offsets[0] - ptr), + offsets[1] - offsets[0]); + + /* Handle --file-offsets */ + + else if (file_offsets) + fprintf(stdout, "%d,%d\n", + (int)(filepos + matchptr + offsets[0] - ptr), + offsets[1] - offsets[0]); + + /* Handle --only-matching, which may occur many times */ + + else + { + BOOL printed = FALSE; + omstr *om; + + for (om = only_matching; om != NULL; om = om->next) + { + int n = om->groupnum; + if (n < mrc) + { + int plen = offsets[2*n + 1] - offsets[2*n]; + if (plen > 0) + { + if (printed) fprintf(stdout, "%s", om_separator); + if (do_colour) fprintf(stdout, "%c[%sm", 0x1b, colour_string); + FWRITE(matchptr + offsets[n*2], 1, plen, stdout); + if (do_colour) fprintf(stdout, "%c[00m", 0x1b); + printed = TRUE; + } + } + } + + if (printed || printname != NULL || number) fprintf(stdout, "\n"); + } + } + + /* Prepare to repeat to find the next match. If the patterned contained + a lookbehind tht included \K, it is possible that the end of the match + might be at or before the actual strting offset we have just used. We + need to start one character further on. Unfortunately, for unanchored + patterns, the actual start offset can be greater that the one that was + set as a result of "bumpalong". PCRE1 does not return the actual start + offset, so we have to check against the original start offset. This may + lead to duplicates - we we need the fudge above to avoid printing them. + (PCRE2 does this better.) */ + + match = FALSE; + if (line_buffered) fflush(stdout); + rc = 0; /* Had some success */ + + startoffset = offsets[1]; /* Restart after the match */ + if (startoffset <= oldstartoffset) + { + if ((size_t)startoffset >= length) + goto END_ONE_MATCH; /* We were at the end */ + startoffset = oldstartoffset + 1; + if (utf8) + while ((matchptr[startoffset] & 0xc0) == 0x80) startoffset++; + } + + /* If the current match ended past the end of the line (only possible + in multiline mode), we must move on to the line in which it did end + before searching for more matches. */ + + while (startoffset > (int)linelength) + { + matchptr = ptr += linelength + endlinelength; + filepos += (int)(linelength + endlinelength); + linenumber++; + startoffset -= (int)(linelength + endlinelength); + t = end_of_line(ptr, endptr, &endlinelength); + linelength = t - ptr - endlinelength; + length = (size_t)(endptr - ptr); + } + + goto ONLY_MATCHING_RESTART; + } + } + + /* This is the default case when none of the above options is set. We print + the matching lines(s), possibly preceded and/or followed by other lines of + context. */ + + else + { + /* See if there is a requirement to print some "after" lines from a + previous match. We never print any overlaps. */ + + if (after_context > 0 && lastmatchnumber > 0) + { + int ellength; + int linecount = 0; + char *p = lastmatchrestart; + + while (p < ptr && linecount < after_context) + { + p = end_of_line(p, ptr, &ellength); + linecount++; + } + + /* It is important to advance lastmatchrestart during this printing so + that it interacts correctly with any "before" printing below. Print + each line's data using fwrite() in case there are binary zeroes. */ + + while (lastmatchrestart < p) + { + char *pp = lastmatchrestart; + if (printname != NULL) fprintf(stdout, "%s-", printname); + if (number) fprintf(stdout, "%d-", lastmatchnumber++); + pp = end_of_line(pp, endptr, &ellength); + FWRITE(lastmatchrestart, 1, pp - lastmatchrestart, stdout); + lastmatchrestart = pp; + } + if (lastmatchrestart != ptr) hyphenpending = TRUE; + } + + /* If there were non-contiguous lines printed above, insert hyphens. */ + + if (hyphenpending) + { + fprintf(stdout, "--\n"); + hyphenpending = FALSE; + hyphenprinted = TRUE; + } + + /* See if there is a requirement to print some "before" lines for this + match. Again, don't print overlaps. */ + + if (before_context > 0) + { + int linecount = 0; + char *p = ptr; + + while (p > main_buffer && (lastmatchnumber == 0 || p > lastmatchrestart) && + linecount < before_context) + { + linecount++; + p = previous_line(p, main_buffer); + } + + if (lastmatchnumber > 0 && p > lastmatchrestart && !hyphenprinted) + fprintf(stdout, "--\n"); + + while (p < ptr) + { + int ellength; + char *pp = p; + if (printname != NULL) fprintf(stdout, "%s-", printname); + if (number) fprintf(stdout, "%d-", linenumber - linecount--); + pp = end_of_line(pp, endptr, &ellength); + FWRITE(p, 1, pp - p, stdout); + p = pp; + } + } + + /* Now print the matching line(s); ensure we set hyphenpending at the end + of the file if any context lines are being output. */ + + if (after_context > 0 || before_context > 0) + endhyphenpending = TRUE; + + if (printname != NULL) fprintf(stdout, "%s:", printname); + if (number) fprintf(stdout, "%d:", linenumber); + + /* In multiline mode, we want to print to the end of the line in which + the end of the matched string is found, so we adjust linelength and the + line number appropriately, but only when there actually was a match + (invert not set). Because the PCRE_FIRSTLINE option is set, the start of + the match will always be before the first newline sequence. */ + + if (multiline & !invert) + { + char *endmatch = ptr + offsets[1]; + t = ptr; + while (t <= endmatch) + { + t = end_of_line(t, endptr, &endlinelength); + if (t < endmatch) linenumber++; else break; + } + linelength = t - ptr - endlinelength; + } + + /*** NOTE: Use only fwrite() to output the data line, so that binary + zeroes are treated as just another data character. */ + + /* This extra option, for Jeffrey Friedl's debugging requirements, + replaces the matched string, or a specific captured string if it exists, + with X. When this happens, colouring is ignored. */ + +#ifdef JFRIEDL_DEBUG + if (S_arg >= 0 && S_arg < mrc) + { + int first = S_arg * 2; + int last = first + 1; + FWRITE(ptr, 1, offsets[first], stdout); + fprintf(stdout, "X"); + FWRITE(ptr + offsets[last], 1, linelength - offsets[last], stdout); + } + else +#endif + + /* We have to split the line(s) up if colouring, and search for further + matches, but not of course if the line is a non-match. */ + + if (do_colour && !invert) + { + int plength; + FWRITE(ptr, 1, offsets[0], stdout); + fprintf(stdout, "%c[%sm", 0x1b, colour_string); + FWRITE(ptr + offsets[0], 1, offsets[1] - offsets[0], stdout); + fprintf(stdout, "%c[00m", 0x1b); + for (;;) + { + startoffset = offsets[1]; + if (startoffset >= (int)linelength + endlinelength || + !match_patterns(matchptr, length, options, startoffset, offsets, + &mrc)) + break; + FWRITE(matchptr + startoffset, 1, offsets[0] - startoffset, stdout); + fprintf(stdout, "%c[%sm", 0x1b, colour_string); + FWRITE(matchptr + offsets[0], 1, offsets[1] - offsets[0], stdout); + fprintf(stdout, "%c[00m", 0x1b); + } + + /* In multiline mode, we may have already printed the complete line + and its line-ending characters (if they matched the pattern), so there + may be no more to print. */ + + plength = (int)((linelength + endlinelength) - startoffset); + if (plength > 0) FWRITE(ptr + startoffset, 1, plength, stdout); + } + + /* Not colouring; no need to search for further matches */ + + else FWRITE(ptr, 1, linelength + endlinelength, stdout); + } + + /* End of doing what has to be done for a match. If --line-buffered was + given, flush the output. */ + + if (line_buffered) fflush(stdout); + rc = 0; /* Had some success */ + + /* Remember where the last match happened for after_context. We remember + where we are about to restart, and that line's number. */ + + lastmatchrestart = ptr + linelength + endlinelength; + lastmatchnumber = linenumber + 1; + } + + /* For a match in multiline inverted mode (which of course did not cause + anything to be printed), we have to move on to the end of the match before + proceeding. */ + + if (multiline && invert && match) + { + int ellength; + char *endmatch = ptr + offsets[1]; + t = ptr; + while (t < endmatch) + { + t = end_of_line(t, endptr, &ellength); + if (t <= endmatch) linenumber++; else break; + } + endmatch = end_of_line(endmatch, endptr, &ellength); + linelength = endmatch - ptr - ellength; + } + + /* Advance to after the newline and increment the line number. The file + offset to the current line is maintained in filepos. */ + + END_ONE_MATCH: + ptr += linelength + endlinelength; + filepos += (int)(linelength + endlinelength); + linenumber++; + + /* If input is line buffered, and the buffer is not yet full, read another + line and add it into the buffer. */ + + if (input_line_buffered && bufflength < (size_t)bufsize) + { + int add = read_one_line(ptr, bufsize - (int)(ptr - main_buffer), in); + bufflength += add; + endptr += add; + } + + /* If we haven't yet reached the end of the file (the buffer is full), and + the current point is in the top 1/3 of the buffer, slide the buffer down by + 1/3 and refill it. Before we do this, if some unprinted "after" lines are + about to be lost, print them. */ + + if (bufflength >= (size_t)bufsize && ptr > main_buffer + 2*bufthird) + { + if (after_context > 0 && + lastmatchnumber > 0 && + lastmatchrestart < main_buffer + bufthird) + { + do_after_lines(lastmatchnumber, lastmatchrestart, endptr, printname); + lastmatchnumber = 0; + } + + /* Now do the shuffle */ + + memmove(main_buffer, main_buffer + bufthird, 2*bufthird); + ptr -= bufthird; + +#ifdef SUPPORT_LIBZ + if (frtype == FR_LIBZ) + bufflength = 2*bufthird + + gzread (ingz, main_buffer + 2*bufthird, bufthird); + else +#endif + +#ifdef SUPPORT_LIBBZ2 + if (frtype == FR_LIBBZ2) + bufflength = 2*bufthird + + BZ2_bzread(inbz2, main_buffer + 2*bufthird, bufthird); + else +#endif + + bufflength = 2*bufthird + + (input_line_buffered? + read_one_line(main_buffer + 2*bufthird, bufthird, in) : + fread(main_buffer + 2*bufthird, 1, bufthird, in)); + endptr = main_buffer + bufflength; + + /* Adjust any last match point */ + + if (lastmatchnumber > 0) lastmatchrestart -= bufthird; + } + } /* Loop through the whole file */ + +/* End of file; print final "after" lines if wanted; do_after_lines sets +hyphenpending if it prints something. */ + +if (!show_only_matching && !count_only) + { + do_after_lines(lastmatchnumber, lastmatchrestart, endptr, printname); + hyphenpending |= endhyphenpending; + } + +/* Print the file name if we are looking for those without matches and there +were none. If we found a match, we won't have got this far. */ + +if (filenames == FN_NOMATCH_ONLY) + { + fprintf(stdout, "%s\n", printname); + return 0; + } + +/* Print the match count if wanted */ + +if (count_only && !quiet) + { + if (count > 0 || !omit_zero_count) + { + if (printname != NULL && filenames != FN_NONE) + fprintf(stdout, "%s:", printname); + fprintf(stdout, "%d\n", count); + } + } + +return rc; +} + + + +/************************************************* +* Grep a file or recurse into a directory * +*************************************************/ + +/* Given a path name, if it's a directory, scan all the files if we are +recursing; if it's a file, grep it. + +Arguments: + pathname the path to investigate + dir_recurse TRUE if recursing is wanted (-r or -drecurse) + only_one_at_top TRUE if the path is the only one at toplevel + +Returns: -1 the file/directory was skipped + 0 if there was at least one match + 1 if there were no matches + 2 there was some kind of error + +However, file opening failures are suppressed if "silent" is set. +*/ + +static int +grep_or_recurse(char *pathname, BOOL dir_recurse, BOOL only_one_at_top) +{ +int rc = 1; +int frtype; +void *handle; +char *lastcomp; +FILE *in = NULL; /* Ensure initialized */ + +#ifdef SUPPORT_LIBZ +gzFile ingz = NULL; +#endif + +#ifdef SUPPORT_LIBBZ2 +BZFILE *inbz2 = NULL; +#endif + +#if defined SUPPORT_LIBZ || defined SUPPORT_LIBBZ2 +int pathlen; +#endif + +#if defined NATIVE_ZOS +int zos_type; +FILE *zos_test_file; +#endif + +/* If the file name is "-" we scan stdin */ + +if (strcmp(pathname, "-") == 0) + { + return pcregrep(stdin, FR_PLAIN, stdin_name, + (filenames > FN_DEFAULT || (filenames == FN_DEFAULT && !only_one_at_top))? + stdin_name : NULL); + } + +/* Inclusion and exclusion: --include-dir and --exclude-dir apply only to +directories, whereas --include and --exclude apply to everything else. The test +is against the final component of the path. */ + +lastcomp = strrchr(pathname, FILESEP); +lastcomp = (lastcomp == NULL)? pathname : lastcomp + 1; + +/* If the file is a directory, skip if not recursing or if explicitly excluded. +Otherwise, scan the directory and recurse for each path within it. The scanning +code is localized so it can be made system-specific. */ + + +/* For z/OS, determine the file type. */ + +#if defined NATIVE_ZOS +zos_test_file = fopen(pathname,"rb"); + +if (zos_test_file == NULL) + { + if (!silent) fprintf(stderr, "pcregrep: failed to test next file %s\n", + pathname, strerror(errno)); + return -1; + } +zos_type = identifyzosfiletype (zos_test_file); +fclose (zos_test_file); + +/* Handle a PDS in separate code */ + +if (zos_type == __ZOS_PDS || zos_type == __ZOS_PDSE) + { + return travelonpdsdir (pathname, only_one_at_top); + } + +/* Deal with regular files in the normal way below. These types are: + zos_type == __ZOS_PDS_MEMBER + zos_type == __ZOS_PS + zos_type == __ZOS_VSAM_KSDS + zos_type == __ZOS_VSAM_ESDS + zos_type == __ZOS_VSAM_RRDS +*/ + +/* Handle a z/OS directory using common code. */ + +else if (zos_type == __ZOS_HFS) + { +#endif /* NATIVE_ZOS */ + + +/* Handle directories: common code for all OS */ + +if (isdirectory(pathname)) + { + if (dee_action == dee_SKIP || + !test_incexc(lastcomp, include_dir_patterns, exclude_dir_patterns)) + return -1; + + if (dee_action == dee_RECURSE) + { + char buffer[1024]; + char *nextfile; + directory_type *dir = opendirectory(pathname); + + if (dir == NULL) + { + if (!silent) + fprintf(stderr, "pcregrep: Failed to open directory %s: %s\n", pathname, + strerror(errno)); + return 2; + } + + while ((nextfile = readdirectory(dir)) != NULL) + { + int frc; + sprintf(buffer, "%.512s%c%.128s", pathname, FILESEP, nextfile); + frc = grep_or_recurse(buffer, dir_recurse, FALSE); + if (frc > 1) rc = frc; + else if (frc == 0 && rc == 1) rc = 0; + } + + closedirectory(dir); + return rc; + } + } + +#if defined NATIVE_ZOS + } +#endif + +/* If the file is not a directory, check for a regular file, and if it is not, +skip it if that's been requested. Otherwise, check for an explicit inclusion or +exclusion. */ + +else if ( +#if defined NATIVE_ZOS + (zos_type == __ZOS_NOFILE && DEE_action == DEE_SKIP) || +#else /* all other OS */ + (!isregfile(pathname) && DEE_action == DEE_SKIP) || +#endif + !test_incexc(lastcomp, include_patterns, exclude_patterns)) + return -1; /* File skipped */ + +/* Control reaches here if we have a regular file, or if we have a directory +and recursion or skipping was not requested, or if we have anything else and +skipping was not requested. The scan proceeds. If this is the first and only +argument at top level, we don't show the file name, unless we are only showing +the file name, or the filename was forced (-H). */ + +#if defined SUPPORT_LIBZ || defined SUPPORT_LIBBZ2 +pathlen = (int)(strlen(pathname)); +#endif + +/* Open using zlib if it is supported and the file name ends with .gz. */ + +#ifdef SUPPORT_LIBZ +if (pathlen > 3 && strcmp(pathname + pathlen - 3, ".gz") == 0) + { + ingz = gzopen(pathname, "rb"); + if (ingz == NULL) + { + if (!silent) + fprintf(stderr, "pcregrep: Failed to open %s: %s\n", pathname, + strerror(errno)); + return 2; + } + handle = (void *)ingz; + frtype = FR_LIBZ; + } +else +#endif + +/* Otherwise open with bz2lib if it is supported and the name ends with .bz2. */ + +#ifdef SUPPORT_LIBBZ2 +if (pathlen > 4 && strcmp(pathname + pathlen - 4, ".bz2") == 0) + { + inbz2 = BZ2_bzopen(pathname, "rb"); + handle = (void *)inbz2; + frtype = FR_LIBBZ2; + } +else +#endif + +/* Otherwise use plain fopen(). The label is so that we can come back here if +an attempt to read a .bz2 file indicates that it really is a plain file. */ + +#ifdef SUPPORT_LIBBZ2 +PLAIN_FILE: +#endif + { + in = fopen(pathname, "rb"); + handle = (void *)in; + frtype = FR_PLAIN; + } + +/* All the opening methods return errno when they fail. */ + +if (handle == NULL) + { + if (!silent) + fprintf(stderr, "pcregrep: Failed to open %s: %s\n", pathname, + strerror(errno)); + return 2; + } + +/* Now grep the file */ + +rc = pcregrep(handle, frtype, pathname, (filenames > FN_DEFAULT || + (filenames == FN_DEFAULT && !only_one_at_top))? pathname : NULL); + +/* Close in an appropriate manner. */ + +#ifdef SUPPORT_LIBZ +if (frtype == FR_LIBZ) + gzclose(ingz); +else +#endif + +/* If it is a .bz2 file and the result is 3, it means that the first attempt to +read failed. If the error indicates that the file isn't in fact bzipped, try +again as a normal file. */ + +#ifdef SUPPORT_LIBBZ2 +if (frtype == FR_LIBBZ2) + { + if (rc == 3) + { + int errnum; + const char *err = BZ2_bzerror(inbz2, &errnum); + if (errnum == BZ_DATA_ERROR_MAGIC) + { + BZ2_bzclose(inbz2); + goto PLAIN_FILE; + } + else if (!silent) + fprintf(stderr, "pcregrep: Failed to read %s using bzlib: %s\n", + pathname, err); + rc = 2; /* The normal "something went wrong" code */ + } + BZ2_bzclose(inbz2); + } +else +#endif + +/* Normal file close */ + +fclose(in); + +/* Pass back the yield from pcregrep(). */ + +return rc; +} + + + +/************************************************* +* Handle a single-letter, no data option * +*************************************************/ + +static int +handle_option(int letter, int options) +{ +switch(letter) + { + case N_FOFFSETS: file_offsets = TRUE; break; + case N_HELP: help(); pcregrep_exit(0); + case N_LBUFFER: line_buffered = TRUE; break; + case N_LOFFSETS: line_offsets = number = TRUE; break; + case N_NOJIT: study_options &= ~PCRE_STUDY_JIT_COMPILE; break; + case 'a': binary_files = BIN_TEXT; break; + case 'c': count_only = TRUE; break; + case 'F': process_options |= PO_FIXED_STRINGS; break; + case 'H': filenames = FN_FORCE; break; + case 'I': binary_files = BIN_NOMATCH; break; + case 'h': filenames = FN_NONE; break; + case 'i': options |= PCRE_CASELESS; break; + case 'l': omit_zero_count = TRUE; filenames = FN_MATCH_ONLY; break; + case 'L': filenames = FN_NOMATCH_ONLY; break; + case 'M': multiline = TRUE; options |= PCRE_MULTILINE|PCRE_FIRSTLINE; break; + case 'n': number = TRUE; break; + + case 'o': + only_matching_last = add_number(0, only_matching_last); + if (only_matching == NULL) only_matching = only_matching_last; + break; + + case 'q': quiet = TRUE; break; + case 'r': dee_action = dee_RECURSE; break; + case 's': silent = TRUE; break; + case 'u': options |= PCRE_UTF8; utf8 = TRUE; break; + case 'v': invert = TRUE; break; + case 'w': process_options |= PO_WORD_MATCH; break; + case 'x': process_options |= PO_LINE_MATCH; break; + + case 'V': + fprintf(stdout, "pcregrep version %s\n", pcre_version()); + pcregrep_exit(0); + break; + + default: + fprintf(stderr, "pcregrep: Unknown option -%c\n", letter); + pcregrep_exit(usage(2)); + } + +return options; +} + + + + +/************************************************* +* Construct printed ordinal * +*************************************************/ + +/* This turns a number into "1st", "3rd", etc. */ + +static char * +ordin(int n) +{ +static char buffer[14]; +char *p = buffer; +sprintf(p, "%d", n); +while (*p != 0) p++; +switch (n%10) + { + case 1: strcpy(p, "st"); break; + case 2: strcpy(p, "nd"); break; + case 3: strcpy(p, "rd"); break; + default: strcpy(p, "th"); break; + } +return buffer; +} + + + +/************************************************* +* Compile a single pattern * +*************************************************/ + +/* Do nothing if the pattern has already been compiled. This is the case for +include/exclude patterns read from a file. + +When the -F option has been used, each "pattern" may be a list of strings, +separated by line breaks. They will be matched literally. We split such a +string and compile the first substring, inserting an additional block into the +pattern chain. + +Arguments: + p points to the pattern block + options the PCRE options + popts the processing options + fromfile TRUE if the pattern was read from a file + fromtext file name or identifying text (e.g. "include") + count 0 if this is the only command line pattern, or + number of the command line pattern, or + linenumber for a pattern from a file + +Returns: TRUE on success, FALSE after an error +*/ + +static BOOL +compile_pattern(patstr *p, int options, int popts, int fromfile, + const char *fromtext, int count) +{ +char buffer[PATBUFSIZE]; +const char *error; +char *ps = p->string; +int patlen = strlen(ps); +int errptr; + +if (p->compiled != NULL) return TRUE; + +if ((popts & PO_FIXED_STRINGS) != 0) + { + int ellength; + char *eop = ps + patlen; + char *pe = end_of_line(ps, eop, &ellength); + + if (ellength != 0) + { + if (add_pattern(pe, p) == NULL) return FALSE; + patlen = (int)(pe - ps - ellength); + } + } + +sprintf(buffer, "%s%.*s%s", prefix[popts], patlen, ps, suffix[popts]); +p->compiled = pcre_compile(buffer, options, &error, &errptr, pcretables); +if (p->compiled != NULL) return TRUE; + +/* Handle compile errors */ + +errptr -= (int)strlen(prefix[popts]); +if (errptr > patlen) errptr = patlen; + +if (fromfile) + { + fprintf(stderr, "pcregrep: Error in regex in line %d of %s " + "at offset %d: %s\n", count, fromtext, errptr, error); + } +else + { + if (count == 0) + fprintf(stderr, "pcregrep: Error in %s regex at offset %d: %s\n", + fromtext, errptr, error); + else + fprintf(stderr, "pcregrep: Error in %s %s regex at offset %d: %s\n", + ordin(count), fromtext, errptr, error); + } + +return FALSE; +} + + + +/************************************************* +* Read and compile a file of patterns * +*************************************************/ + +/* This is used for --filelist, --include-from, and --exclude-from. + +Arguments: + name the name of the file; "-" is stdin + patptr pointer to the pattern chain anchor + patlastptr pointer to the last pattern pointer + popts the process options to pass to pattern_compile() + +Returns: TRUE if all went well +*/ + +static BOOL +read_pattern_file(char *name, patstr **patptr, patstr **patlastptr, int popts) +{ +int linenumber = 0; +FILE *f; +char *filename; +char buffer[PATBUFSIZE]; + +if (strcmp(name, "-") == 0) + { + f = stdin; + filename = stdin_name; + } +else + { + f = fopen(name, "r"); + if (f == NULL) + { + fprintf(stderr, "pcregrep: Failed to open %s: %s\n", name, strerror(errno)); + return FALSE; + } + filename = name; + } + +while (fgets(buffer, PATBUFSIZE, f) != NULL) + { + char *s = buffer + (int)strlen(buffer); + while (s > buffer && isspace((unsigned char)(s[-1]))) s--; + *s = 0; + linenumber++; + if (buffer[0] == 0) continue; /* Skip blank lines */ + + /* Note: this call to add_pattern() puts a pointer to the local variable + "buffer" into the pattern chain. However, that pointer is used only when + compiling the pattern, which happens immediately below, so we flatten it + afterwards, as a precaution against any later code trying to use it. */ + + *patlastptr = add_pattern(buffer, *patlastptr); + if (*patlastptr == NULL) + { + if (f != stdin) fclose(f); + return FALSE; + } + if (*patptr == NULL) *patptr = *patlastptr; + + /* This loop is needed because compiling a "pattern" when -F is set may add + on additional literal patterns if the original contains a newline. In the + common case, it never will, because fgets() stops at a newline. However, + the -N option can be used to give pcregrep a different newline setting. */ + + for(;;) + { + if (!compile_pattern(*patlastptr, pcre_options, popts, TRUE, filename, + linenumber)) + { + if (f != stdin) fclose(f); + return FALSE; + } + (*patlastptr)->string = NULL; /* Insurance */ + if ((*patlastptr)->next == NULL) break; + *patlastptr = (*patlastptr)->next; + } + } + +if (f != stdin) fclose(f); +return TRUE; +} + + + +/************************************************* +* Main program * +*************************************************/ + +/* Returns 0 if something matched, 1 if nothing matched, 2 after an error. */ + +int +main(int argc, char **argv) +{ +int i, j; +int rc = 1; +BOOL only_one_at_top; +patstr *cp; +fnstr *fn; +const char *locale_from = "--locale"; +const char *error; + +#ifdef SUPPORT_PCREGREP_JIT +pcre_jit_stack *jit_stack = NULL; +#endif + +/* Set the default line ending value from the default in the PCRE library; +"lf", "cr", "crlf", and "any" are supported. Anything else is treated as "lf". +Note that the return values from pcre_config(), though derived from the ASCII +codes, are the same in EBCDIC environments, so we must use the actual values +rather than escapes such as as '\r'. */ + +(void)pcre_config(PCRE_CONFIG_NEWLINE, &i); +switch(i) + { + default: newline = (char *)"lf"; break; + case 13: newline = (char *)"cr"; break; + case (13 << 8) | 10: newline = (char *)"crlf"; break; + case -1: newline = (char *)"any"; break; + case -2: newline = (char *)"anycrlf"; break; + } + +/* Process the options */ + +for (i = 1; i < argc; i++) + { + option_item *op = NULL; + char *option_data = (char *)""; /* default to keep compiler happy */ + BOOL longop; + BOOL longopwasequals = FALSE; + + if (argv[i][0] != '-') break; + + /* If we hit an argument that is just "-", it may be a reference to STDIN, + but only if we have previously had -e or -f to define the patterns. */ + + if (argv[i][1] == 0) + { + if (pattern_files != NULL || patterns != NULL) break; + else pcregrep_exit(usage(2)); + } + + /* Handle a long name option, or -- to terminate the options */ + + if (argv[i][1] == '-') + { + char *arg = argv[i] + 2; + char *argequals = strchr(arg, '='); + + if (*arg == 0) /* -- terminates options */ + { + i++; + break; /* out of the options-handling loop */ + } + + longop = TRUE; + + /* Some long options have data that follows after =, for example file=name. + Some options have variations in the long name spelling: specifically, we + allow "regexp" because GNU grep allows it, though I personally go along + with Jeffrey Friedl and Larry Wall in preferring "regex" without the "p". + These options are entered in the table as "regex(p)". Options can be in + both these categories. */ + + for (op = optionlist; op->one_char != 0; op++) + { + char *opbra = strchr(op->long_name, '('); + char *equals = strchr(op->long_name, '='); + + /* Handle options with only one spelling of the name */ + + if (opbra == NULL) /* Does not contain '(' */ + { + if (equals == NULL) /* Not thing=data case */ + { + if (strcmp(arg, op->long_name) == 0) break; + } + else /* Special case xxx=data */ + { + int oplen = (int)(equals - op->long_name); + int arglen = (argequals == NULL)? + (int)strlen(arg) : (int)(argequals - arg); + if (oplen == arglen && strncmp(arg, op->long_name, oplen) == 0) + { + option_data = arg + arglen; + if (*option_data == '=') + { + option_data++; + longopwasequals = TRUE; + } + break; + } + } + } + + /* Handle options with an alternate spelling of the name */ + + else + { + char buff1[24]; + char buff2[24]; + + int baselen = (int)(opbra - op->long_name); + int fulllen = (int)(strchr(op->long_name, ')') - op->long_name + 1); + int arglen = (argequals == NULL || equals == NULL)? + (int)strlen(arg) : (int)(argequals - arg); + + sprintf(buff1, "%.*s", baselen, op->long_name); + sprintf(buff2, "%s%.*s", buff1, fulllen - baselen - 2, opbra + 1); + + if (strncmp(arg, buff1, arglen) == 0 || + strncmp(arg, buff2, arglen) == 0) + { + if (equals != NULL && argequals != NULL) + { + option_data = argequals; + if (*option_data == '=') + { + option_data++; + longopwasequals = TRUE; + } + } + break; + } + } + } + + if (op->one_char == 0) + { + fprintf(stderr, "pcregrep: Unknown option %s\n", argv[i]); + pcregrep_exit(usage(2)); + } + } + + /* Jeffrey Friedl's debugging harness uses these additional options which + are not in the right form for putting in the option table because they use + only one hyphen, yet are more than one character long. By putting them + separately here, they will not get displayed as part of the help() output, + but I don't think Jeffrey will care about that. */ + +#ifdef JFRIEDL_DEBUG + else if (strcmp(argv[i], "-pre") == 0) { + jfriedl_prefix = argv[++i]; + continue; + } else if (strcmp(argv[i], "-post") == 0) { + jfriedl_postfix = argv[++i]; + continue; + } else if (strcmp(argv[i], "-XT") == 0) { + sscanf(argv[++i], "%d", &jfriedl_XT); + continue; + } else if (strcmp(argv[i], "-XR") == 0) { + sscanf(argv[++i], "%d", &jfriedl_XR); + continue; + } +#endif + + + /* One-char options; many that have no data may be in a single argument; we + continue till we hit the last one or one that needs data. */ + + else + { + char *s = argv[i] + 1; + longop = FALSE; + + while (*s != 0) + { + for (op = optionlist; op->one_char != 0; op++) + { + if (*s == op->one_char) break; + } + if (op->one_char == 0) + { + fprintf(stderr, "pcregrep: Unknown option letter '%c' in \"%s\"\n", + *s, argv[i]); + pcregrep_exit(usage(2)); + } + + option_data = s+1; + + /* Break out if this is the last character in the string; it's handled + below like a single multi-char option. */ + + if (*option_data == 0) break; + + /* Check for a single-character option that has data: OP_OP_NUMBER(S) + are used for ones that either have a numerical number or defaults, i.e. + the data is optional. If a digit follows, there is data; if not, carry on + with other single-character options in the same string. */ + + if (op->type == OP_OP_NUMBER || op->type == OP_OP_NUMBERS) + { + if (isdigit((unsigned char)s[1])) break; + } + else /* Check for an option with data */ + { + if (op->type != OP_NODATA) break; + } + + /* Handle a single-character option with no data, then loop for the + next character in the string. */ + + pcre_options = handle_option(*s++, pcre_options); + } + } + + /* At this point we should have op pointing to a matched option. If the type + is NO_DATA, it means that there is no data, and the option might set + something in the PCRE options. */ + + if (op->type == OP_NODATA) + { + pcre_options = handle_option(op->one_char, pcre_options); + continue; + } + + /* If the option type is OP_OP_STRING or OP_OP_NUMBER(S), it's an option that + either has a value or defaults to something. It cannot have data in a + separate item. At the moment, the only such options are "colo(u)r", + "only-matching", and Jeffrey Friedl's special -S debugging option. */ + + if (*option_data == 0 && + (op->type == OP_OP_STRING || op->type == OP_OP_NUMBER || + op->type == OP_OP_NUMBERS)) + { + switch (op->one_char) + { + case N_COLOUR: + colour_option = (char *)"auto"; + break; + + case 'o': + only_matching_last = add_number(0, only_matching_last); + if (only_matching == NULL) only_matching = only_matching_last; + break; + +#ifdef JFRIEDL_DEBUG + case 'S': + S_arg = 0; + break; +#endif + } + continue; + } + + /* Otherwise, find the data string for the option. */ + + if (*option_data == 0) + { + if (i >= argc - 1 || longopwasequals) + { + fprintf(stderr, "pcregrep: Data missing after %s\n", argv[i]); + pcregrep_exit(usage(2)); + } + option_data = argv[++i]; + } + + /* If the option type is OP_OP_NUMBERS, the value is a number that is to be + added to a chain of numbers. */ + + if (op->type == OP_OP_NUMBERS) + { + unsigned long int n = decode_number(option_data, op, longop); + omdatastr *omd = (omdatastr *)op->dataptr; + *(omd->lastptr) = add_number((int)n, *(omd->lastptr)); + if (*(omd->anchor) == NULL) *(omd->anchor) = *(omd->lastptr); + } + + /* If the option type is OP_PATLIST, it's the -e option, or one of the + include/exclude options, which can be called multiple times to create lists + of patterns. */ + + else if (op->type == OP_PATLIST) + { + patdatastr *pd = (patdatastr *)op->dataptr; + *(pd->lastptr) = add_pattern(option_data, *(pd->lastptr)); + if (*(pd->lastptr) == NULL) goto EXIT2; + if (*(pd->anchor) == NULL) *(pd->anchor) = *(pd->lastptr); + } + + /* If the option type is OP_FILELIST, it's one of the options that names a + file. */ + + else if (op->type == OP_FILELIST) + { + fndatastr *fd = (fndatastr *)op->dataptr; + fn = (fnstr *)malloc(sizeof(fnstr)); + if (fn == NULL) + { + fprintf(stderr, "pcregrep: malloc failed\n"); + goto EXIT2; + } + fn->next = NULL; + fn->name = option_data; + if (*(fd->anchor) == NULL) + *(fd->anchor) = fn; + else + (*(fd->lastptr))->next = fn; + *(fd->lastptr) = fn; + } + + /* Handle OP_BINARY_FILES */ + + else if (op->type == OP_BINFILES) + { + if (strcmp(option_data, "binary") == 0) + binary_files = BIN_BINARY; + else if (strcmp(option_data, "without-match") == 0) + binary_files = BIN_NOMATCH; + else if (strcmp(option_data, "text") == 0) + binary_files = BIN_TEXT; + else + { + fprintf(stderr, "pcregrep: unknown value \"%s\" for binary-files\n", + option_data); + pcregrep_exit(usage(2)); + } + } + + /* Otherwise, deal with a single string or numeric data value. */ + + else if (op->type != OP_NUMBER && op->type != OP_LONGNUMBER && + op->type != OP_OP_NUMBER) + { + *((char **)op->dataptr) = option_data; + } + else + { + unsigned long int n = decode_number(option_data, op, longop); + if (op->type == OP_LONGNUMBER) *((unsigned long int *)op->dataptr) = n; + else *((int *)op->dataptr) = n; + } + } + +/* Options have been decoded. If -C was used, its value is used as a default +for -A and -B. */ + +if (both_context > 0) + { + if (after_context == 0) after_context = both_context; + if (before_context == 0) before_context = both_context; + } + +/* Only one of --only-matching, --file-offsets, or --line-offsets is permitted. +However, all three set show_only_matching because they display, each in their +own way, only the data that has matched. */ + +if ((only_matching != NULL && (file_offsets || line_offsets)) || + (file_offsets && line_offsets)) + { + fprintf(stderr, "pcregrep: Cannot mix --only-matching, --file-offsets " + "and/or --line-offsets\n"); + pcregrep_exit(usage(2)); + } + +if (only_matching != NULL || file_offsets || line_offsets) + show_only_matching = TRUE; + +/* If a locale has not been provided as an option, see if the LC_CTYPE or +LC_ALL environment variable is set, and if so, use it. */ + +if (locale == NULL) + { + locale = getenv("LC_ALL"); + locale_from = "LCC_ALL"; + } + +if (locale == NULL) + { + locale = getenv("LC_CTYPE"); + locale_from = "LC_CTYPE"; + } + +/* If a locale is set, use it to generate the tables the PCRE needs. Otherwise, +pcretables==NULL, which causes the use of default tables. */ + +if (locale != NULL) + { + if (setlocale(LC_CTYPE, locale) == NULL) + { + fprintf(stderr, "pcregrep: Failed to set locale %s (obtained from %s)\n", + locale, locale_from); + goto EXIT2; + } + pcretables = pcre_maketables(); + } + +/* Sort out colouring */ + +if (colour_option != NULL && strcmp(colour_option, "never") != 0) + { + if (strcmp(colour_option, "always") == 0) do_colour = TRUE; + else if (strcmp(colour_option, "auto") == 0) do_colour = is_stdout_tty(); + else + { + fprintf(stderr, "pcregrep: Unknown colour setting \"%s\"\n", + colour_option); + goto EXIT2; + } + if (do_colour) + { + char *cs = getenv("PCREGREP_COLOUR"); + if (cs == NULL) cs = getenv("PCREGREP_COLOR"); + if (cs != NULL) colour_string = cs; + } + } + +/* Interpret the newline type; the default settings are Unix-like. */ + +if (strcmp(newline, "cr") == 0 || strcmp(newline, "CR") == 0) + { + pcre_options |= PCRE_NEWLINE_CR; + endlinetype = EL_CR; + } +else if (strcmp(newline, "lf") == 0 || strcmp(newline, "LF") == 0) + { + pcre_options |= PCRE_NEWLINE_LF; + endlinetype = EL_LF; + } +else if (strcmp(newline, "crlf") == 0 || strcmp(newline, "CRLF") == 0) + { + pcre_options |= PCRE_NEWLINE_CRLF; + endlinetype = EL_CRLF; + } +else if (strcmp(newline, "any") == 0 || strcmp(newline, "ANY") == 0) + { + pcre_options |= PCRE_NEWLINE_ANY; + endlinetype = EL_ANY; + } +else if (strcmp(newline, "anycrlf") == 0 || strcmp(newline, "ANYCRLF") == 0) + { + pcre_options |= PCRE_NEWLINE_ANYCRLF; + endlinetype = EL_ANYCRLF; + } +else + { + fprintf(stderr, "pcregrep: Invalid newline specifier \"%s\"\n", newline); + goto EXIT2; + } + +/* Interpret the text values for -d and -D */ + +if (dee_option != NULL) + { + if (strcmp(dee_option, "read") == 0) dee_action = dee_READ; + else if (strcmp(dee_option, "recurse") == 0) dee_action = dee_RECURSE; + else if (strcmp(dee_option, "skip") == 0) dee_action = dee_SKIP; + else + { + fprintf(stderr, "pcregrep: Invalid value \"%s\" for -d\n", dee_option); + goto EXIT2; + } + } + +if (DEE_option != NULL) + { + if (strcmp(DEE_option, "read") == 0) DEE_action = DEE_READ; + else if (strcmp(DEE_option, "skip") == 0) DEE_action = DEE_SKIP; + else + { + fprintf(stderr, "pcregrep: Invalid value \"%s\" for -D\n", DEE_option); + goto EXIT2; + } + } + +/* Check the values for Jeffrey Friedl's debugging options. */ + +#ifdef JFRIEDL_DEBUG +if (S_arg > 9) + { + fprintf(stderr, "pcregrep: bad value for -S option\n"); + return 2; + } +if (jfriedl_XT != 0 || jfriedl_XR != 0) + { + if (jfriedl_XT == 0) jfriedl_XT = 1; + if (jfriedl_XR == 0) jfriedl_XR = 1; + } +#endif + +/* Get memory for the main buffer. */ + +bufsize = 3*bufthird; +main_buffer = (char *)malloc(bufsize); + +if (main_buffer == NULL) + { + fprintf(stderr, "pcregrep: malloc failed\n"); + goto EXIT2; + } + +/* If no patterns were provided by -e, and there are no files provided by -f, +the first argument is the one and only pattern, and it must exist. */ + +if (patterns == NULL && pattern_files == NULL) + { + if (i >= argc) return usage(2); + patterns = patterns_last = add_pattern(argv[i++], NULL); + if (patterns == NULL) goto EXIT2; + } + +/* Compile the patterns that were provided on the command line, either by +multiple uses of -e or as a single unkeyed pattern. We cannot do this until +after all the command-line options are read so that we know which PCRE options +to use. When -F is used, compile_pattern() may add another block into the +chain, so we must not access the next pointer till after the compile. */ + +for (j = 1, cp = patterns; cp != NULL; j++, cp = cp->next) + { + if (!compile_pattern(cp, pcre_options, process_options, FALSE, "command-line", + (j == 1 && patterns->next == NULL)? 0 : j)) + goto EXIT2; + } + +/* Read and compile the regular expressions that are provided in files. */ + +for (fn = pattern_files; fn != NULL; fn = fn->next) + { + if (!read_pattern_file(fn->name, &patterns, &patterns_last, process_options)) + goto EXIT2; + } + +/* Study the regular expressions, as we will be running them many times. If an +extra block is needed for a limit, set PCRE_STUDY_EXTRA_NEEDED so that one is +returned, even if studying produces no data. */ + +if (match_limit > 0 || match_limit_recursion > 0) + study_options |= PCRE_STUDY_EXTRA_NEEDED; + +/* Unless JIT has been explicitly disabled, arrange a stack for it to use. */ + +#ifdef SUPPORT_PCREGREP_JIT +if ((study_options & PCRE_STUDY_JIT_COMPILE) != 0) + jit_stack = pcre_jit_stack_alloc(32*1024, 1024*1024); +#endif + +for (j = 1, cp = patterns; cp != NULL; j++, cp = cp->next) + { + cp->hint = pcre_study(cp->compiled, study_options, &error); + if (error != NULL) + { + if (patterns->next == NULL) + fprintf(stderr, "pcregrep: Error while studying regex: %s\n", error); + else + fprintf(stderr, "pcregrep: Error while studying regex number %d: %s\n", + j, error); + goto EXIT2; + } +#ifdef SUPPORT_PCREGREP_JIT + if (jit_stack != NULL && cp->hint != NULL) + pcre_assign_jit_stack(cp->hint, NULL, jit_stack); +#endif + } + +/* If --match-limit or --recursion-limit was set, put the value(s) into the +pcre_extra block for each pattern. There will always be an extra block because +of the use of PCRE_STUDY_EXTRA_NEEDED above. */ + +for (cp = patterns; cp != NULL; cp = cp->next) + { + if (match_limit > 0) + { + cp->hint->flags |= PCRE_EXTRA_MATCH_LIMIT; + cp->hint->match_limit = match_limit; + } + + if (match_limit_recursion > 0) + { + cp->hint->flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION; + cp->hint->match_limit_recursion = match_limit_recursion; + } + } + +/* If there are include or exclude patterns read from the command line, compile +them. -F, -w, and -x do not apply, so the third argument of compile_pattern is +0. */ + +for (j = 0; j < 4; j++) + { + int k; + for (k = 1, cp = *(incexlist[j]); cp != NULL; k++, cp = cp->next) + { + if (!compile_pattern(cp, pcre_options, 0, FALSE, incexname[j], + (k == 1 && cp->next == NULL)? 0 : k)) + goto EXIT2; + } + } + +/* Read and compile include/exclude patterns from files. */ + +for (fn = include_from; fn != NULL; fn = fn->next) + { + if (!read_pattern_file(fn->name, &include_patterns, &include_patterns_last, 0)) + goto EXIT2; + } + +for (fn = exclude_from; fn != NULL; fn = fn->next) + { + if (!read_pattern_file(fn->name, &exclude_patterns, &exclude_patterns_last, 0)) + goto EXIT2; + } + +/* If there are no files that contain lists of files to search, and there are +no file arguments, search stdin, and then exit. */ + +if (file_lists == NULL && i >= argc) + { + rc = pcregrep(stdin, FR_PLAIN, stdin_name, + (filenames > FN_DEFAULT)? stdin_name : NULL); + goto EXIT; + } + +/* If any files that contains a list of files to search have been specified, +read them line by line and search the given files. */ + +for (fn = file_lists; fn != NULL; fn = fn->next) + { + char buffer[PATBUFSIZE]; + FILE *fl; + if (strcmp(fn->name, "-") == 0) fl = stdin; else + { + fl = fopen(fn->name, "rb"); + if (fl == NULL) + { + fprintf(stderr, "pcregrep: Failed to open %s: %s\n", fn->name, + strerror(errno)); + goto EXIT2; + } + } + while (fgets(buffer, PATBUFSIZE, fl) != NULL) + { + int frc; + char *end = buffer + (int)strlen(buffer); + while (end > buffer && isspace(end[-1])) end--; + *end = 0; + if (*buffer != 0) + { + frc = grep_or_recurse(buffer, dee_action == dee_RECURSE, FALSE); + if (frc > 1) rc = frc; + else if (frc == 0 && rc == 1) rc = 0; + } + } + if (fl != stdin) fclose(fl); + } + +/* After handling file-list, work through remaining arguments. Pass in the fact +that there is only one argument at top level - this suppresses the file name if +the argument is not a directory and filenames are not otherwise forced. */ + +only_one_at_top = i == argc - 1 && file_lists == NULL; + +for (; i < argc; i++) + { + int frc = grep_or_recurse(argv[i], dee_action == dee_RECURSE, + only_one_at_top); + if (frc > 1) rc = frc; + else if (frc == 0 && rc == 1) rc = 0; + } + +EXIT: +#ifdef SUPPORT_PCREGREP_JIT +if (jit_stack != NULL) pcre_jit_stack_free(jit_stack); +#endif + +free(main_buffer); +free((void *)pcretables); + +free_pattern_chain(patterns); +free_pattern_chain(include_patterns); +free_pattern_chain(include_dir_patterns); +free_pattern_chain(exclude_patterns); +free_pattern_chain(exclude_dir_patterns); + +free_file_chain(exclude_from); +free_file_chain(include_from); +free_file_chain(pattern_files); +free_file_chain(file_lists); + +while (only_matching != NULL) + { + omstr *this = only_matching; + only_matching = this->next; + free(this); + } + +pcregrep_exit(rc); + +EXIT2: +rc = 2; +goto EXIT; +} + +/* End of pcregrep */ diff --git a/src/third_party/pcre-8.41/pcreposix.c b/src/third_party/pcre-8.41/pcreposix.c new file mode 100644 index 00000000000..7b404a71100 --- /dev/null +++ b/src/third_party/pcre-8.41/pcreposix.c @@ -0,0 +1,432 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +/* PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + + Written by Philip Hazel + Copyright (c) 1997-2017 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + + +/* This module is a wrapper that provides a POSIX API to the underlying PCRE +functions. */ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + + +/* Ensure that the PCREPOSIX_EXP_xxx macros are set appropriately for +compiling these functions. This must come before including pcreposix.h, where +they are set for an application (using these functions) if they have not +previously been set. */ + +#if defined(_WIN32) && !defined(PCRE_STATIC) +# define PCREPOSIX_EXP_DECL extern __declspec(dllexport) +# define PCREPOSIX_EXP_DEFN __declspec(dllexport) +#endif + +/* We include pcre.h before pcre_internal.h so that the PCRE library functions +are declared as "import" for Windows by defining PCRE_EXP_DECL as "import". +This is needed even though pcre_internal.h itself includes pcre.h, because it +does so after it has set PCRE_EXP_DECL to "export" if it is not already set. */ + +#include "pcre.h" +#include "pcre_internal.h" +#include "pcreposix.h" + + +/* Table to translate PCRE compile time error codes into POSIX error codes. */ + +static const int eint[] = { + 0, /* no error */ + REG_EESCAPE, /* \ at end of pattern */ + REG_EESCAPE, /* \c at end of pattern */ + REG_EESCAPE, /* unrecognized character follows \ */ + REG_BADBR, /* numbers out of order in {} quantifier */ + /* 5 */ + REG_BADBR, /* number too big in {} quantifier */ + REG_EBRACK, /* missing terminating ] for character class */ + REG_ECTYPE, /* invalid escape sequence in character class */ + REG_ERANGE, /* range out of order in character class */ + REG_BADRPT, /* nothing to repeat */ + /* 10 */ + REG_BADRPT, /* operand of unlimited repeat could match the empty string */ + REG_ASSERT, /* internal error: unexpected repeat */ + REG_BADPAT, /* unrecognized character after (? */ + REG_BADPAT, /* POSIX named classes are supported only within a class */ + REG_EPAREN, /* missing ) */ + /* 15 */ + REG_ESUBREG, /* reference to non-existent subpattern */ + REG_INVARG, /* erroffset passed as NULL */ + REG_INVARG, /* unknown option bit(s) set */ + REG_EPAREN, /* missing ) after comment */ + REG_ESIZE, /* parentheses nested too deeply */ + /* 20 */ + REG_ESIZE, /* regular expression too large */ + REG_ESPACE, /* failed to get memory */ + REG_EPAREN, /* unmatched parentheses */ + REG_ASSERT, /* internal error: code overflow */ + REG_BADPAT, /* unrecognized character after (?< */ + /* 25 */ + REG_BADPAT, /* lookbehind assertion is not fixed length */ + REG_BADPAT, /* malformed number or name after (?( */ + REG_BADPAT, /* conditional group contains more than two branches */ + REG_BADPAT, /* assertion expected after (?( */ + REG_BADPAT, /* (?R or (?[+-]digits must be followed by ) */ + /* 30 */ + REG_ECTYPE, /* unknown POSIX class name */ + REG_BADPAT, /* POSIX collating elements are not supported */ + REG_INVARG, /* this version of PCRE is not compiled with PCRE_UTF8 support */ + REG_BADPAT, /* spare error */ + REG_BADPAT, /* character value in \x{} or \o{} is too large */ + /* 35 */ + REG_BADPAT, /* invalid condition (?(0) */ + REG_BADPAT, /* \C not allowed in lookbehind assertion */ + REG_EESCAPE, /* PCRE does not support \L, \l, \N, \U, or \u */ + REG_BADPAT, /* number after (?C is > 255 */ + REG_BADPAT, /* closing ) for (?C expected */ + /* 40 */ + REG_BADPAT, /* recursive call could loop indefinitely */ + REG_BADPAT, /* unrecognized character after (?P */ + REG_BADPAT, /* syntax error in subpattern name (missing terminator) */ + REG_BADPAT, /* two named subpatterns have the same name */ + REG_BADPAT, /* invalid UTF-8 string */ + /* 45 */ + REG_BADPAT, /* support for \P, \p, and \X has not been compiled */ + REG_BADPAT, /* malformed \P or \p sequence */ + REG_BADPAT, /* unknown property name after \P or \p */ + REG_BADPAT, /* subpattern name is too long (maximum 32 characters) */ + REG_BADPAT, /* too many named subpatterns (maximum 10,000) */ + /* 50 */ + REG_BADPAT, /* repeated subpattern is too long */ + REG_BADPAT, /* octal value is greater than \377 (not in UTF-8 mode) */ + REG_BADPAT, /* internal error: overran compiling workspace */ + REG_BADPAT, /* internal error: previously-checked referenced subpattern not found */ + REG_BADPAT, /* DEFINE group contains more than one branch */ + /* 55 */ + REG_BADPAT, /* repeating a DEFINE group is not allowed */ + REG_INVARG, /* inconsistent NEWLINE options */ + REG_BADPAT, /* \g is not followed followed by an (optionally braced) non-zero number */ + REG_BADPAT, /* a numbered reference must not be zero */ + REG_BADPAT, /* an argument is not allowed for (*ACCEPT), (*FAIL), or (*COMMIT) */ + /* 60 */ + REG_BADPAT, /* (*VERB) not recognized */ + REG_BADPAT, /* number is too big */ + REG_BADPAT, /* subpattern name expected */ + REG_BADPAT, /* digit expected after (?+ */ + REG_BADPAT, /* ] is an invalid data character in JavaScript compatibility mode */ + /* 65 */ + REG_BADPAT, /* different names for subpatterns of the same number are not allowed */ + REG_BADPAT, /* (*MARK) must have an argument */ + REG_INVARG, /* this version of PCRE is not compiled with PCRE_UCP support */ + REG_BADPAT, /* \c must be followed by an ASCII character */ + REG_BADPAT, /* \k is not followed by a braced, angle-bracketed, or quoted name */ + /* 70 */ + REG_BADPAT, /* internal error: unknown opcode in find_fixedlength() */ + REG_BADPAT, /* \N is not supported in a class */ + REG_BADPAT, /* too many forward references */ + REG_BADPAT, /* disallowed UTF-8/16/32 code point (>= 0xd800 && <= 0xdfff) */ + REG_BADPAT, /* invalid UTF-16 string (should not occur) */ + /* 75 */ + REG_BADPAT, /* overlong MARK name */ + REG_BADPAT, /* character value in \u.... sequence is too large */ + REG_BADPAT, /* invalid UTF-32 string (should not occur) */ + REG_BADPAT, /* setting UTF is disabled by the application */ + REG_BADPAT, /* non-hex character in \\x{} (closing brace missing?) */ + /* 80 */ + REG_BADPAT, /* non-octal character in \o{} (closing brace missing?) */ + REG_BADPAT, /* missing opening brace after \o */ + REG_BADPAT, /* parentheses too deeply nested */ + REG_BADPAT, /* invalid range in character class */ + REG_BADPAT, /* group name must start with a non-digit */ + /* 85 */ + REG_BADPAT, /* parentheses too deeply nested (stack check) */ + REG_BADPAT, /* missing digits in \x{} or \o{} */ + REG_BADPAT /* pattern too complicated */ +}; + +/* Table of texts corresponding to POSIX error codes */ + +static const char *const pstring[] = { + "", /* Dummy for value 0 */ + "internal error", /* REG_ASSERT */ + "invalid repeat counts in {}", /* BADBR */ + "pattern error", /* BADPAT */ + "? * + invalid", /* BADRPT */ + "unbalanced {}", /* EBRACE */ + "unbalanced []", /* EBRACK */ + "collation error - not relevant", /* ECOLLATE */ + "bad class", /* ECTYPE */ + "bad escape sequence", /* EESCAPE */ + "empty expression", /* EMPTY */ + "unbalanced ()", /* EPAREN */ + "bad range inside []", /* ERANGE */ + "expression too big", /* ESIZE */ + "failed to get memory", /* ESPACE */ + "bad back reference", /* ESUBREG */ + "bad argument", /* INVARG */ + "match failed" /* NOMATCH */ +}; + + + + +/************************************************* +* Translate error code to string * +*************************************************/ + +PCREPOSIX_EXP_DEFN size_t PCRE_CALL_CONVENTION +regerror(int errcode, const regex_t *preg, char *errbuf, size_t errbuf_size) +{ +const char *message, *addmessage; +size_t length, addlength; + +message = (errcode >= (int)(sizeof(pstring)/sizeof(char *)))? + "unknown error code" : pstring[errcode]; +length = strlen(message) + 1; + +addmessage = " at offset "; +addlength = (preg != NULL && (int)preg->re_erroffset != -1)? + strlen(addmessage) + 6 : 0; + +if (errbuf_size > 0) + { + if (addlength > 0 && errbuf_size >= length + addlength) + sprintf(errbuf, "%s%s%-6d", message, addmessage, (int)preg->re_erroffset); + else + { + strncpy(errbuf, message, errbuf_size - 1); + errbuf[errbuf_size-1] = 0; + } + } + +return length + addlength; +} + + + + +/************************************************* +* Free store held by a regex * +*************************************************/ + +PCREPOSIX_EXP_DEFN void PCRE_CALL_CONVENTION +regfree(regex_t *preg) +{ +(PUBL(free))(preg->re_pcre); +} + + + + +/************************************************* +* Compile a regular expression * +*************************************************/ + +/* +Arguments: + preg points to a structure for recording the compiled expression + pattern the pattern to compile + cflags compilation flags + +Returns: 0 on success + various non-zero codes on failure +*/ + +PCREPOSIX_EXP_DEFN int PCRE_CALL_CONVENTION +regcomp(regex_t *preg, const char *pattern, int cflags) +{ +const char *errorptr; +int erroffset; +int errorcode; +int options = 0; +int re_nsub = 0; + +if ((cflags & REG_ICASE) != 0) options |= PCRE_CASELESS; +if ((cflags & REG_NEWLINE) != 0) options |= PCRE_MULTILINE; +if ((cflags & REG_DOTALL) != 0) options |= PCRE_DOTALL; +if ((cflags & REG_NOSUB) != 0) options |= PCRE_NO_AUTO_CAPTURE; +if ((cflags & REG_UTF8) != 0) options |= PCRE_UTF8; +if ((cflags & REG_UCP) != 0) options |= PCRE_UCP; +if ((cflags & REG_UNGREEDY) != 0) options |= PCRE_UNGREEDY; + +preg->re_pcre = pcre_compile2(pattern, options, &errorcode, &errorptr, + &erroffset, NULL); +preg->re_erroffset = erroffset; + +/* Safety: if the error code is too big for the translation vector (which +should not happen, but we all make mistakes), return REG_BADPAT. */ + +if (preg->re_pcre == NULL) + { + return (errorcode < (int)(sizeof(eint)/sizeof(const int)))? + eint[errorcode] : REG_BADPAT; + } + +(void)pcre_fullinfo((const pcre *)preg->re_pcre, NULL, PCRE_INFO_CAPTURECOUNT, + &re_nsub); +preg->re_nsub = (size_t)re_nsub; +return 0; +} + + + + +/************************************************* +* Match a regular expression * +*************************************************/ + +/* Unfortunately, PCRE requires 3 ints of working space for each captured +substring, so we have to get and release working store instead of just using +the POSIX structures as was done in earlier releases when PCRE needed only 2 +ints. However, if the number of possible capturing brackets is small, use a +block of store on the stack, to reduce the use of malloc/free. The threshold is +in a macro that can be changed at configure time. + +If REG_NOSUB was specified at compile time, the PCRE_NO_AUTO_CAPTURE flag will +be set. When this is the case, the nmatch and pmatch arguments are ignored, and +the only result is yes/no/error. */ + +PCREPOSIX_EXP_DEFN int PCRE_CALL_CONVENTION +regexec(const regex_t *preg, const char *string, size_t nmatch, + regmatch_t pmatch[], int eflags) +{ +int rc, so, eo; +int options = 0; +int *ovector = NULL; +int small_ovector[POSIX_MALLOC_THRESHOLD * 3]; +BOOL allocated_ovector = FALSE; +BOOL nosub = + (REAL_PCRE_OPTIONS((const pcre *)preg->re_pcre) & PCRE_NO_AUTO_CAPTURE) != 0; + +if ((eflags & REG_NOTBOL) != 0) options |= PCRE_NOTBOL; +if ((eflags & REG_NOTEOL) != 0) options |= PCRE_NOTEOL; +if ((eflags & REG_NOTEMPTY) != 0) options |= PCRE_NOTEMPTY; + +((regex_t *)preg)->re_erroffset = (size_t)(-1); /* Only has meaning after compile */ + +/* When no string data is being returned, or no vector has been passed in which +to put it, ensure that nmatch is zero. Otherwise, ensure the vector for holding +the return data is large enough. */ + +if (nosub || pmatch == NULL) nmatch = 0; + +else if (nmatch > 0) + { + if (nmatch <= POSIX_MALLOC_THRESHOLD) + { + ovector = &(small_ovector[0]); + } + else + { + if (nmatch > INT_MAX/(sizeof(int) * 3)) return REG_ESPACE; + ovector = (int *)malloc(sizeof(int) * nmatch * 3); + if (ovector == NULL) return REG_ESPACE; + allocated_ovector = TRUE; + } + } + +/* REG_STARTEND is a BSD extension, to allow for non-NUL-terminated strings. +The man page from OS X says "REG_STARTEND affects only the location of the +string, not how it is matched". That is why the "so" value is used to bump the +start location rather than being passed as a PCRE "starting offset". */ + +if ((eflags & REG_STARTEND) != 0) + { + if (pmatch == NULL) return REG_INVARG; + so = pmatch[0].rm_so; + eo = pmatch[0].rm_eo; + } +else + { + so = 0; + eo = (int)strlen(string); + } + +rc = pcre_exec((const pcre *)preg->re_pcre, NULL, string + so, (eo - so), + 0, options, ovector, (int)(nmatch * 3)); + +if (rc == 0) rc = (int)nmatch; /* All captured slots were filled in */ + +/* Successful match */ + +if (rc >= 0) + { + size_t i; + if (!nosub) + { + for (i = 0; i < (size_t)rc; i++) + { + pmatch[i].rm_so = ovector[i*2] + so; + pmatch[i].rm_eo = ovector[i*2+1] + so; + } + if (allocated_ovector) free(ovector); + for (; i < nmatch; i++) pmatch[i].rm_so = pmatch[i].rm_eo = -1; + } + return 0; + } + +/* Unsuccessful match */ + +if (allocated_ovector) free(ovector); +switch(rc) + { +/* ========================================================================== */ + /* These cases are never obeyed. This is a fudge that causes a compile-time + error if the vector eint, which is indexed by compile-time error number, is + not the correct length. It seems to be the only way to do such a check at + compile time, as the sizeof() operator does not work in the C preprocessor. + As all the PCRE_ERROR_xxx values are negative, we can use 0 and 1. */ + + case 0: + case (sizeof(eint)/sizeof(int) == ERRCOUNT): + return REG_ASSERT; +/* ========================================================================== */ + + case PCRE_ERROR_NOMATCH: return REG_NOMATCH; + case PCRE_ERROR_NULL: return REG_INVARG; + case PCRE_ERROR_BADOPTION: return REG_INVARG; + case PCRE_ERROR_BADMAGIC: return REG_INVARG; + case PCRE_ERROR_UNKNOWN_NODE: return REG_ASSERT; + case PCRE_ERROR_NOMEMORY: return REG_ESPACE; + case PCRE_ERROR_MATCHLIMIT: return REG_ESPACE; + case PCRE_ERROR_BADUTF8: return REG_INVARG; + case PCRE_ERROR_BADUTF8_OFFSET: return REG_INVARG; + case PCRE_ERROR_BADMODE: return REG_INVARG; + default: return REG_ASSERT; + } +} + +/* End of pcreposix.c */ diff --git a/src/third_party/pcre-8.41/pcreposix.h b/src/third_party/pcre-8.41/pcreposix.h new file mode 100644 index 00000000000..c77c0b0523c --- /dev/null +++ b/src/third_party/pcre-8.41/pcreposix.h @@ -0,0 +1,146 @@ +/************************************************* +* Perl-Compatible Regular Expressions * +*************************************************/ + +#ifndef _PCREPOSIX_H +#define _PCREPOSIX_H + +/* This is the header for the POSIX wrapper interface to the PCRE Perl- +Compatible Regular Expression library. It defines the things POSIX says should +be there. I hope. + + Copyright (c) 1997-2012 University of Cambridge + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* Have to include stdlib.h in order to ensure that size_t is defined. */ + +#include + +/* Allow for C++ users */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Options, mostly defined by POSIX, but with some extras. */ + +#define REG_ICASE 0x0001 /* Maps to PCRE_CASELESS */ +#define REG_NEWLINE 0x0002 /* Maps to PCRE_MULTILINE */ +#define REG_NOTBOL 0x0004 /* Maps to PCRE_NOTBOL */ +#define REG_NOTEOL 0x0008 /* Maps to PCRE_NOTEOL */ +#define REG_DOTALL 0x0010 /* NOT defined by POSIX; maps to PCRE_DOTALL */ +#define REG_NOSUB 0x0020 /* Maps to PCRE_NO_AUTO_CAPTURE */ +#define REG_UTF8 0x0040 /* NOT defined by POSIX; maps to PCRE_UTF8 */ +#define REG_STARTEND 0x0080 /* BSD feature: pass subject string by so,eo */ +#define REG_NOTEMPTY 0x0100 /* NOT defined by POSIX; maps to PCRE_NOTEMPTY */ +#define REG_UNGREEDY 0x0200 /* NOT defined by POSIX; maps to PCRE_UNGREEDY */ +#define REG_UCP 0x0400 /* NOT defined by POSIX; maps to PCRE_UCP */ + +/* This is not used by PCRE, but by defining it we make it easier +to slot PCRE into existing programs that make POSIX calls. */ + +#define REG_EXTENDED 0 + +/* Error values. Not all these are relevant or used by the wrapper. */ + +enum { + REG_ASSERT = 1, /* internal error ? */ + REG_BADBR, /* invalid repeat counts in {} */ + REG_BADPAT, /* pattern error */ + REG_BADRPT, /* ? * + invalid */ + REG_EBRACE, /* unbalanced {} */ + REG_EBRACK, /* unbalanced [] */ + REG_ECOLLATE, /* collation error - not relevant */ + REG_ECTYPE, /* bad class */ + REG_EESCAPE, /* bad escape sequence */ + REG_EMPTY, /* empty expression */ + REG_EPAREN, /* unbalanced () */ + REG_ERANGE, /* bad range inside [] */ + REG_ESIZE, /* expression too big */ + REG_ESPACE, /* failed to get memory */ + REG_ESUBREG, /* bad back reference */ + REG_INVARG, /* bad argument */ + REG_NOMATCH /* match failed */ +}; + + +/* The structure representing a compiled regular expression. */ + +typedef struct { + void *re_pcre; + size_t re_nsub; + size_t re_erroffset; +} regex_t; + +/* The structure in which a captured offset is returned. */ + +typedef int regoff_t; + +typedef struct { + regoff_t rm_so; + regoff_t rm_eo; +} regmatch_t; + +/* When an application links to a PCRE DLL in Windows, the symbols that are +imported have to be identified as such. When building PCRE, the appropriate +export settings are needed, and are set in pcreposix.c before including this +file. */ + +#if defined(_WIN32) && !defined(PCRE_STATIC) && !defined(PCREPOSIX_EXP_DECL) +# define PCREPOSIX_EXP_DECL extern __declspec(dllimport) +# define PCREPOSIX_EXP_DEFN __declspec(dllimport) +#endif + +/* By default, we use the standard "extern" declarations. */ + +#ifndef PCREPOSIX_EXP_DECL +# ifdef __cplusplus +# define PCREPOSIX_EXP_DECL extern "C" +# define PCREPOSIX_EXP_DEFN extern "C" +# else +# define PCREPOSIX_EXP_DECL extern +# define PCREPOSIX_EXP_DEFN extern +# endif +#endif + +/* The functions */ + +PCREPOSIX_EXP_DECL int regcomp(regex_t *, const char *, int); +PCREPOSIX_EXP_DECL int regexec(const regex_t *, const char *, size_t, + regmatch_t *, int); +PCREPOSIX_EXP_DECL size_t regerror(int, const regex_t *, char *, size_t); +PCREPOSIX_EXP_DECL void regfree(regex_t *); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* End of pcreposix.h */ diff --git a/src/third_party/pcre-8.41/pcretest.c b/src/third_party/pcre-8.41/pcretest.c new file mode 100644 index 00000000000..f1303037281 --- /dev/null +++ b/src/third_party/pcre-8.41/pcretest.c @@ -0,0 +1,5773 @@ +/************************************************* +* PCRE testing program * +*************************************************/ + +/* This program was hacked up as a tester for PCRE. I really should have +written it more tidily in the first place. Will I ever learn? It has grown and +been extended and consequently is now rather, er, *very* untidy in places. The +addition of 16-bit support has made it even worse. :-( + +----------------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +----------------------------------------------------------------------------- +*/ + +/* This program now supports the testing of all of the 8-bit, 16-bit, and +32-bit PCRE libraries in a single program. This is different from the modules +such as pcre_compile.c in the library itself, which are compiled separately for +each mode. If two modes are enabled, for example, pcre_compile.c is compiled +twice. By contrast, pcretest.c is compiled only once. Therefore, it must not +make use of any of the macros from pcre_internal.h that depend on +COMPILE_PCRE8, COMPILE_PCRE16, or COMPILE_PCRE32. It does, however, make use of +SUPPORT_PCRE8, SUPPORT_PCRE16, and SUPPORT_PCRE32 to ensure that it calls only +supported library functions. */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include + +/* Both libreadline and libedit are optionally supported. The user-supplied +original patch uses readline/readline.h for libedit, but in at least one system +it is installed as editline/readline.h, so the configuration code now looks for +that first, falling back to readline/readline.h. */ + +#if defined(SUPPORT_LIBREADLINE) || defined(SUPPORT_LIBEDIT) +#ifdef HAVE_UNISTD_H +#include +#endif +#if defined(SUPPORT_LIBREADLINE) +#include +#include +#else +#if defined(HAVE_EDITLINE_READLINE_H) +#include +#else +#include +#endif +#endif +#endif + +/* A number of things vary for Windows builds. Originally, pcretest opened its +input and output without "b"; then I was told that "b" was needed in some +environments, so it was added for release 5.0 to both the input and output. (It +makes no difference on Unix-like systems.) Later I was told that it is wrong +for the input on Windows. I've now abstracted the modes into two macros that +are set here, to make it easier to fiddle with them, and removed "b" from the +input mode under Windows. */ + +#if defined(_WIN32) || defined(WIN32) +#include /* For _setmode() */ +#include /* For _O_BINARY */ +#define INPUT_MODE "r" +#define OUTPUT_MODE "wb" + +#ifndef isatty +#define isatty _isatty /* This is what Windows calls them, I'm told, */ +#endif /* though in some environments they seem to */ + /* be already defined, hence the #ifndefs. */ +#ifndef fileno +#define fileno _fileno +#endif + +/* A user sent this fix for Borland Builder 5 under Windows. */ + +#ifdef __BORLANDC__ +#define _setmode(handle, mode) setmode(handle, mode) +#endif + +/* Not Windows */ + +#else +#include /* These two includes are needed */ +#include /* for setrlimit(). */ +#if defined NATIVE_ZOS /* z/OS uses non-binary I/O */ +#define INPUT_MODE "r" +#define OUTPUT_MODE "w" +#else +#define INPUT_MODE "rb" +#define OUTPUT_MODE "wb" +#endif +#endif + +#ifdef __VMS +#include +void vms_setsymbol( char *, char *, int ); +#endif + + +#define PRIV(name) name + +/* We have to include pcre_internal.h because we need the internal info for +displaying the results of pcre_study() and we also need to know about the +internal macros, structures, and other internal data values; pcretest has +"inside information" compared to a program that strictly follows the PCRE API. + +Although pcre_internal.h does itself include pcre.h, we explicitly include it +here before pcre_internal.h so that the PCRE_EXP_xxx macros get set +appropriately for an application, not for building PCRE. */ + +#include "pcre.h" +#include "pcre_internal.h" + +/* The pcre_printint() function, which prints the internal form of a compiled +regex, is held in a separate file so that (a) it can be compiled in either +8-, 16- or 32-bit mode, and (b) it can be #included directly in pcre_compile.c +when that is compiled in debug mode. */ + +#ifdef SUPPORT_PCRE8 +void pcre_printint(pcre *external_re, FILE *f, BOOL print_lengths); +#endif +#ifdef SUPPORT_PCRE16 +void pcre16_printint(pcre *external_re, FILE *f, BOOL print_lengths); +#endif +#ifdef SUPPORT_PCRE32 +void pcre32_printint(pcre *external_re, FILE *f, BOOL print_lengths); +#endif + +/* We need access to some of the data tables that PCRE uses. So as not to have +to keep two copies, we include the source files here, changing the names of the +external symbols to prevent clashes. */ + +#define PCRE_INCLUDED + +#include "pcre_tables.c" +#include "pcre_ucd.c" + +/* The definition of the macro PRINTABLE, which determines whether to print an +output character as-is or as a hex value when showing compiled patterns, is +the same as in the printint.src file. We uses it here in cases when the locale +has not been explicitly changed, so as to get consistent output from systems +that differ in their output from isprint() even in the "C" locale. */ + +#ifdef EBCDIC +#define PRINTABLE(c) ((c) >= 64 && (c) < 255) +#else +#define PRINTABLE(c) ((c) >= 32 && (c) < 127) +#endif + +#define PRINTOK(c) (locale_set? (((c) < 256) && isprint(c)) : PRINTABLE(c)) + +/* Posix support is disabled in 16 or 32 bit only mode. */ +#if !defined SUPPORT_PCRE8 && !defined NOPOSIX +#define NOPOSIX +#endif + +/* It is possible to compile this test program without including support for +testing the POSIX interface, though this is not available via the standard +Makefile. */ + +#if !defined NOPOSIX +#include "pcreposix.h" +#endif + +/* It is also possible, originally for the benefit of a version that was +imported into Exim, to build pcretest without support for UTF8 or UTF16 (define +NOUTF), without the interface to the DFA matcher (NODFA). In fact, we +automatically cut out the UTF support if PCRE is built without it. */ + +#ifndef SUPPORT_UTF +#ifndef NOUTF +#define NOUTF +#endif +#endif + +/* To make the code a bit tidier for 8/16/32-bit support, we define macros +for all the pcre[16]_xxx functions (except pcre16_fullinfo, which is called +only from one place and is handled differently). I couldn't dream up any way of +using a single macro to do this in a generic way, because of the many different +argument requirements. We know that at least one of SUPPORT_PCRE8 and +SUPPORT_PCRE16 must be set. First define macros for each individual mode; then +use these in the definitions of generic macros. + +**** Special note about the PCHARSxxx macros: the address of the string to be +printed is always given as two arguments: a base address followed by an offset. +The base address is cast to the correct data size for 8 or 16 bit data; the +offset is in units of this size. If the string were given as base+offset in one +argument, the casting might be incorrectly applied. */ + +#ifdef SUPPORT_PCRE8 + +#define PCHARS8(lv, p, offset, len, f) \ + lv = pchars((pcre_uint8 *)(p) + offset, len, f) + +#define PCHARSV8(p, offset, len, f) \ + (void)pchars((pcre_uint8 *)(p) + offset, len, f) + +#define READ_CAPTURE_NAME8(p, cn8, cn16, cn32, re) \ + p = read_capture_name8(p, cn8, re) + +#define STRLEN8(p) ((int)strlen((char *)p)) + +#define SET_PCRE_CALLOUT8(callout) \ + pcre_callout = callout + +#define SET_PCRE_STACK_GUARD8(stack_guard) \ + pcre_stack_guard = stack_guard + +#define PCRE_ASSIGN_JIT_STACK8(extra, callback, userdata) \ + pcre_assign_jit_stack(extra, callback, userdata) + +#define PCRE_COMPILE8(re, pat, options, error, erroffset, tables) \ + re = pcre_compile((char *)pat, options, error, erroffset, tables) + +#define PCRE_COPY_NAMED_SUBSTRING8(rc, re, bptr, offsets, count, \ + namesptr, cbuffer, size) \ + rc = pcre_copy_named_substring(re, (char *)bptr, offsets, count, \ + (char *)namesptr, cbuffer, size) + +#define PCRE_COPY_SUBSTRING8(rc, bptr, offsets, count, i, cbuffer, size) \ + rc = pcre_copy_substring((char *)bptr, offsets, count, i, cbuffer, size) + +#define PCRE_DFA_EXEC8(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets, workspace, size_workspace) \ + count = pcre_dfa_exec(re, extra, (char *)bptr, len, start_offset, options, \ + offsets, size_offsets, workspace, size_workspace) + +#define PCRE_EXEC8(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets) \ + count = pcre_exec(re, extra, (char *)bptr, len, start_offset, options, \ + offsets, size_offsets) + +#define PCRE_FREE_STUDY8(extra) \ + pcre_free_study(extra) + +#define PCRE_FREE_SUBSTRING8(substring) \ + pcre_free_substring(substring) + +#define PCRE_FREE_SUBSTRING_LIST8(listptr) \ + pcre_free_substring_list(listptr) + +#define PCRE_GET_NAMED_SUBSTRING8(rc, re, bptr, offsets, count, \ + getnamesptr, subsptr) \ + rc = pcre_get_named_substring(re, (char *)bptr, offsets, count, \ + (char *)getnamesptr, subsptr) + +#define PCRE_GET_STRINGNUMBER8(n, rc, ptr) \ + n = pcre_get_stringnumber(re, (char *)ptr) + +#define PCRE_GET_SUBSTRING8(rc, bptr, offsets, count, i, subsptr) \ + rc = pcre_get_substring((char *)bptr, offsets, count, i, subsptr) + +#define PCRE_GET_SUBSTRING_LIST8(rc, bptr, offsets, count, listptr) \ + rc = pcre_get_substring_list((const char *)bptr, offsets, count, listptr) + +#define PCRE_PATTERN_TO_HOST_BYTE_ORDER8(rc, re, extra, tables) \ + rc = pcre_pattern_to_host_byte_order(re, extra, tables) + +#define PCRE_PRINTINT8(re, outfile, debug_lengths) \ + pcre_printint(re, outfile, debug_lengths) + +#define PCRE_STUDY8(extra, re, options, error) \ + extra = pcre_study(re, options, error) + +#define PCRE_JIT_STACK_ALLOC8(startsize, maxsize) \ + pcre_jit_stack_alloc(startsize, maxsize) + +#define PCRE_JIT_STACK_FREE8(stack) \ + pcre_jit_stack_free(stack) + +#define pcre8_maketables pcre_maketables + +#endif /* SUPPORT_PCRE8 */ + +/* -----------------------------------------------------------*/ + +#ifdef SUPPORT_PCRE16 + +#define PCHARS16(lv, p, offset, len, f) \ + lv = pchars16((PCRE_SPTR16)(p) + offset, len, f) + +#define PCHARSV16(p, offset, len, f) \ + (void)pchars16((PCRE_SPTR16)(p) + offset, len, f) + +#define READ_CAPTURE_NAME16(p, cn8, cn16, cn32, re) \ + p = read_capture_name16(p, cn16, re) + +#define STRLEN16(p) ((int)strlen16((PCRE_SPTR16)p)) + +#define SET_PCRE_CALLOUT16(callout) \ + pcre16_callout = (int (*)(pcre16_callout_block *))callout + +#define SET_PCRE_STACK_GUARD16(stack_guard) \ + pcre16_stack_guard = (int (*)(void))stack_guard + +#define PCRE_ASSIGN_JIT_STACK16(extra, callback, userdata) \ + pcre16_assign_jit_stack((pcre16_extra *)extra, \ + (pcre16_jit_callback)callback, userdata) + +#define PCRE_COMPILE16(re, pat, options, error, erroffset, tables) \ + re = (pcre *)pcre16_compile((PCRE_SPTR16)pat, options, error, erroffset, \ + tables) + +#define PCRE_COPY_NAMED_SUBSTRING16(rc, re, bptr, offsets, count, \ + namesptr, cbuffer, size) \ + rc = pcre16_copy_named_substring((pcre16 *)re, (PCRE_SPTR16)bptr, offsets, \ + count, (PCRE_SPTR16)namesptr, (PCRE_UCHAR16 *)cbuffer, size/2) + +#define PCRE_COPY_SUBSTRING16(rc, bptr, offsets, count, i, cbuffer, size) \ + rc = pcre16_copy_substring((PCRE_SPTR16)bptr, offsets, count, i, \ + (PCRE_UCHAR16 *)cbuffer, size/2) + +#define PCRE_DFA_EXEC16(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets, workspace, size_workspace) \ + count = pcre16_dfa_exec((pcre16 *)re, (pcre16_extra *)extra, \ + (PCRE_SPTR16)bptr, len, start_offset, options, offsets, size_offsets, \ + workspace, size_workspace) + +#define PCRE_EXEC16(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets) \ + count = pcre16_exec((pcre16 *)re, (pcre16_extra *)extra, (PCRE_SPTR16)bptr, \ + len, start_offset, options, offsets, size_offsets) + +#define PCRE_FREE_STUDY16(extra) \ + pcre16_free_study((pcre16_extra *)extra) + +#define PCRE_FREE_SUBSTRING16(substring) \ + pcre16_free_substring((PCRE_SPTR16)substring) + +#define PCRE_FREE_SUBSTRING_LIST16(listptr) \ + pcre16_free_substring_list((PCRE_SPTR16 *)listptr) + +#define PCRE_GET_NAMED_SUBSTRING16(rc, re, bptr, offsets, count, \ + getnamesptr, subsptr) \ + rc = pcre16_get_named_substring((pcre16 *)re, (PCRE_SPTR16)bptr, offsets, \ + count, (PCRE_SPTR16)getnamesptr, (PCRE_SPTR16 *)(void*)subsptr) + +#define PCRE_GET_STRINGNUMBER16(n, rc, ptr) \ + n = pcre16_get_stringnumber(re, (PCRE_SPTR16)ptr) + +#define PCRE_GET_SUBSTRING16(rc, bptr, offsets, count, i, subsptr) \ + rc = pcre16_get_substring((PCRE_SPTR16)bptr, offsets, count, i, \ + (PCRE_SPTR16 *)(void*)subsptr) + +#define PCRE_GET_SUBSTRING_LIST16(rc, bptr, offsets, count, listptr) \ + rc = pcre16_get_substring_list((PCRE_SPTR16)bptr, offsets, count, \ + (PCRE_SPTR16 **)(void*)listptr) + +#define PCRE_PATTERN_TO_HOST_BYTE_ORDER16(rc, re, extra, tables) \ + rc = pcre16_pattern_to_host_byte_order((pcre16 *)re, (pcre16_extra *)extra, \ + tables) + +#define PCRE_PRINTINT16(re, outfile, debug_lengths) \ + pcre16_printint(re, outfile, debug_lengths) + +#define PCRE_STUDY16(extra, re, options, error) \ + extra = (pcre_extra *)pcre16_study((pcre16 *)re, options, error) + +#define PCRE_JIT_STACK_ALLOC16(startsize, maxsize) \ + (pcre_jit_stack *)pcre16_jit_stack_alloc(startsize, maxsize) + +#define PCRE_JIT_STACK_FREE16(stack) \ + pcre16_jit_stack_free((pcre16_jit_stack *)stack) + +#endif /* SUPPORT_PCRE16 */ + +/* -----------------------------------------------------------*/ + +#ifdef SUPPORT_PCRE32 + +#define PCHARS32(lv, p, offset, len, f) \ + lv = pchars32((PCRE_SPTR32)(p) + offset, len, use_utf, f) + +#define PCHARSV32(p, offset, len, f) \ + (void)pchars32((PCRE_SPTR32)(p) + offset, len, use_utf, f) + +#define READ_CAPTURE_NAME32(p, cn8, cn16, cn32, re) \ + p = read_capture_name32(p, cn32, re) + +#define STRLEN32(p) ((int)strlen32((PCRE_SPTR32)p)) + +#define SET_PCRE_CALLOUT32(callout) \ + pcre32_callout = (int (*)(pcre32_callout_block *))callout + +#define SET_PCRE_STACK_GUARD32(stack_guard) \ + pcre32_stack_guard = (int (*)(void))stack_guard + +#define PCRE_ASSIGN_JIT_STACK32(extra, callback, userdata) \ + pcre32_assign_jit_stack((pcre32_extra *)extra, \ + (pcre32_jit_callback)callback, userdata) + +#define PCRE_COMPILE32(re, pat, options, error, erroffset, tables) \ + re = (pcre *)pcre32_compile((PCRE_SPTR32)pat, options, error, erroffset, \ + tables) + +#define PCRE_COPY_NAMED_SUBSTRING32(rc, re, bptr, offsets, count, \ + namesptr, cbuffer, size) \ + rc = pcre32_copy_named_substring((pcre32 *)re, (PCRE_SPTR32)bptr, offsets, \ + count, (PCRE_SPTR32)namesptr, (PCRE_UCHAR32 *)cbuffer, size/4) + +#define PCRE_COPY_SUBSTRING32(rc, bptr, offsets, count, i, cbuffer, size) \ + rc = pcre32_copy_substring((PCRE_SPTR32)bptr, offsets, count, i, \ + (PCRE_UCHAR32 *)cbuffer, size/4) + +#define PCRE_DFA_EXEC32(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets, workspace, size_workspace) \ + count = pcre32_dfa_exec((pcre32 *)re, (pcre32_extra *)extra, \ + (PCRE_SPTR32)bptr, len, start_offset, options, offsets, size_offsets, \ + workspace, size_workspace) + +#define PCRE_EXEC32(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets) \ + count = pcre32_exec((pcre32 *)re, (pcre32_extra *)extra, (PCRE_SPTR32)bptr, \ + len, start_offset, options, offsets, size_offsets) + +#define PCRE_FREE_STUDY32(extra) \ + pcre32_free_study((pcre32_extra *)extra) + +#define PCRE_FREE_SUBSTRING32(substring) \ + pcre32_free_substring((PCRE_SPTR32)substring) + +#define PCRE_FREE_SUBSTRING_LIST32(listptr) \ + pcre32_free_substring_list((PCRE_SPTR32 *)listptr) + +#define PCRE_GET_NAMED_SUBSTRING32(rc, re, bptr, offsets, count, \ + getnamesptr, subsptr) \ + rc = pcre32_get_named_substring((pcre32 *)re, (PCRE_SPTR32)bptr, offsets, \ + count, (PCRE_SPTR32)getnamesptr, (PCRE_SPTR32 *)(void*)subsptr) + +#define PCRE_GET_STRINGNUMBER32(n, rc, ptr) \ + n = pcre32_get_stringnumber(re, (PCRE_SPTR32)ptr) + +#define PCRE_GET_SUBSTRING32(rc, bptr, offsets, count, i, subsptr) \ + rc = pcre32_get_substring((PCRE_SPTR32)bptr, offsets, count, i, \ + (PCRE_SPTR32 *)(void*)subsptr) + +#define PCRE_GET_SUBSTRING_LIST32(rc, bptr, offsets, count, listptr) \ + rc = pcre32_get_substring_list((PCRE_SPTR32)bptr, offsets, count, \ + (PCRE_SPTR32 **)(void*)listptr) + +#define PCRE_PATTERN_TO_HOST_BYTE_ORDER32(rc, re, extra, tables) \ + rc = pcre32_pattern_to_host_byte_order((pcre32 *)re, (pcre32_extra *)extra, \ + tables) + +#define PCRE_PRINTINT32(re, outfile, debug_lengths) \ + pcre32_printint(re, outfile, debug_lengths) + +#define PCRE_STUDY32(extra, re, options, error) \ + extra = (pcre_extra *)pcre32_study((pcre32 *)re, options, error) + +#define PCRE_JIT_STACK_ALLOC32(startsize, maxsize) \ + (pcre_jit_stack *)pcre32_jit_stack_alloc(startsize, maxsize) + +#define PCRE_JIT_STACK_FREE32(stack) \ + pcre32_jit_stack_free((pcre32_jit_stack *)stack) + +#endif /* SUPPORT_PCRE32 */ + + +/* ----- More than one mode is supported; a runtime test is needed, except for +pcre_config(), and the JIT stack functions, when it doesn't matter which +available version is called. ----- */ + +enum { + PCRE8_MODE, + PCRE16_MODE, + PCRE32_MODE +}; + +#if (defined (SUPPORT_PCRE8) + defined (SUPPORT_PCRE16) + \ + defined (SUPPORT_PCRE32)) >= 2 + +#define CHAR_SIZE (1 << pcre_mode) + +/* There doesn't seem to be an easy way of writing these macros that can cope +with the 3 pairs of bit sizes plus all three bit sizes. So just handle all the +cases separately. */ + +/* ----- All three modes supported ----- */ + +#if defined(SUPPORT_PCRE8) && defined(SUPPORT_PCRE16) && defined(SUPPORT_PCRE32) + +#define PCHARS(lv, p, offset, len, f) \ + if (pcre_mode == PCRE32_MODE) \ + PCHARS32(lv, p, offset, len, f); \ + else if (pcre_mode == PCRE16_MODE) \ + PCHARS16(lv, p, offset, len, f); \ + else \ + PCHARS8(lv, p, offset, len, f) + +#define PCHARSV(p, offset, len, f) \ + if (pcre_mode == PCRE32_MODE) \ + PCHARSV32(p, offset, len, f); \ + else if (pcre_mode == PCRE16_MODE) \ + PCHARSV16(p, offset, len, f); \ + else \ + PCHARSV8(p, offset, len, f) + +#define READ_CAPTURE_NAME(p, cn8, cn16, cn32, re) \ + if (pcre_mode == PCRE32_MODE) \ + READ_CAPTURE_NAME32(p, cn8, cn16, cn32, re); \ + else if (pcre_mode == PCRE16_MODE) \ + READ_CAPTURE_NAME16(p, cn8, cn16, cn32, re); \ + else \ + READ_CAPTURE_NAME8(p, cn8, cn16, cn32, re) + +#define SET_PCRE_CALLOUT(callout) \ + if (pcre_mode == PCRE32_MODE) \ + SET_PCRE_CALLOUT32(callout); \ + else if (pcre_mode == PCRE16_MODE) \ + SET_PCRE_CALLOUT16(callout); \ + else \ + SET_PCRE_CALLOUT8(callout) + +#define SET_PCRE_STACK_GUARD(stack_guard) \ + if (pcre_mode == PCRE32_MODE) \ + SET_PCRE_STACK_GUARD32(stack_guard); \ + else if (pcre_mode == PCRE16_MODE) \ + SET_PCRE_STACK_GUARD16(stack_guard); \ + else \ + SET_PCRE_STACK_GUARD8(stack_guard) + +#define STRLEN(p) (pcre_mode == PCRE32_MODE ? STRLEN32(p) : pcre_mode == PCRE16_MODE ? STRLEN16(p) : STRLEN8(p)) + +#define PCRE_ASSIGN_JIT_STACK(extra, callback, userdata) \ + if (pcre_mode == PCRE32_MODE) \ + PCRE_ASSIGN_JIT_STACK32(extra, callback, userdata); \ + else if (pcre_mode == PCRE16_MODE) \ + PCRE_ASSIGN_JIT_STACK16(extra, callback, userdata); \ + else \ + PCRE_ASSIGN_JIT_STACK8(extra, callback, userdata) + +#define PCRE_COMPILE(re, pat, options, error, erroffset, tables) \ + if (pcre_mode == PCRE32_MODE) \ + PCRE_COMPILE32(re, pat, options, error, erroffset, tables); \ + else if (pcre_mode == PCRE16_MODE) \ + PCRE_COMPILE16(re, pat, options, error, erroffset, tables); \ + else \ + PCRE_COMPILE8(re, pat, options, error, erroffset, tables) + +#define PCRE_CONFIG pcre_config + +#define PCRE_COPY_NAMED_SUBSTRING(rc, re, bptr, offsets, count, \ + namesptr, cbuffer, size) \ + if (pcre_mode == PCRE32_MODE) \ + PCRE_COPY_NAMED_SUBSTRING32(rc, re, bptr, offsets, count, \ + namesptr, cbuffer, size); \ + else if (pcre_mode == PCRE16_MODE) \ + PCRE_COPY_NAMED_SUBSTRING16(rc, re, bptr, offsets, count, \ + namesptr, cbuffer, size); \ + else \ + PCRE_COPY_NAMED_SUBSTRING8(rc, re, bptr, offsets, count, \ + namesptr, cbuffer, size) + +#define PCRE_COPY_SUBSTRING(rc, bptr, offsets, count, i, cbuffer, size) \ + if (pcre_mode == PCRE32_MODE) \ + PCRE_COPY_SUBSTRING32(rc, bptr, offsets, count, i, cbuffer, size); \ + else if (pcre_mode == PCRE16_MODE) \ + PCRE_COPY_SUBSTRING16(rc, bptr, offsets, count, i, cbuffer, size); \ + else \ + PCRE_COPY_SUBSTRING8(rc, bptr, offsets, count, i, cbuffer, size) + +#define PCRE_DFA_EXEC(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets, workspace, size_workspace) \ + if (pcre_mode == PCRE32_MODE) \ + PCRE_DFA_EXEC32(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets, workspace, size_workspace); \ + else if (pcre_mode == PCRE16_MODE) \ + PCRE_DFA_EXEC16(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets, workspace, size_workspace); \ + else \ + PCRE_DFA_EXEC8(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets, workspace, size_workspace) + +#define PCRE_EXEC(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets) \ + if (pcre_mode == PCRE32_MODE) \ + PCRE_EXEC32(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets); \ + else if (pcre_mode == PCRE16_MODE) \ + PCRE_EXEC16(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets); \ + else \ + PCRE_EXEC8(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets) + +#define PCRE_FREE_STUDY(extra) \ + if (pcre_mode == PCRE32_MODE) \ + PCRE_FREE_STUDY32(extra); \ + else if (pcre_mode == PCRE16_MODE) \ + PCRE_FREE_STUDY16(extra); \ + else \ + PCRE_FREE_STUDY8(extra) + +#define PCRE_FREE_SUBSTRING(substring) \ + if (pcre_mode == PCRE32_MODE) \ + PCRE_FREE_SUBSTRING32(substring); \ + else if (pcre_mode == PCRE16_MODE) \ + PCRE_FREE_SUBSTRING16(substring); \ + else \ + PCRE_FREE_SUBSTRING8(substring) + +#define PCRE_FREE_SUBSTRING_LIST(listptr) \ + if (pcre_mode == PCRE32_MODE) \ + PCRE_FREE_SUBSTRING_LIST32(listptr); \ + else if (pcre_mode == PCRE16_MODE) \ + PCRE_FREE_SUBSTRING_LIST16(listptr); \ + else \ + PCRE_FREE_SUBSTRING_LIST8(listptr) + +#define PCRE_GET_NAMED_SUBSTRING(rc, re, bptr, offsets, count, \ + getnamesptr, subsptr) \ + if (pcre_mode == PCRE32_MODE) \ + PCRE_GET_NAMED_SUBSTRING32(rc, re, bptr, offsets, count, \ + getnamesptr, subsptr); \ + else if (pcre_mode == PCRE16_MODE) \ + PCRE_GET_NAMED_SUBSTRING16(rc, re, bptr, offsets, count, \ + getnamesptr, subsptr); \ + else \ + PCRE_GET_NAMED_SUBSTRING8(rc, re, bptr, offsets, count, \ + getnamesptr, subsptr) + +#define PCRE_GET_STRINGNUMBER(n, rc, ptr) \ + if (pcre_mode == PCRE32_MODE) \ + PCRE_GET_STRINGNUMBER32(n, rc, ptr); \ + else if (pcre_mode == PCRE16_MODE) \ + PCRE_GET_STRINGNUMBER16(n, rc, ptr); \ + else \ + PCRE_GET_STRINGNUMBER8(n, rc, ptr) + +#define PCRE_GET_SUBSTRING(rc, bptr, use_offsets, count, i, subsptr) \ + if (pcre_mode == PCRE32_MODE) \ + PCRE_GET_SUBSTRING32(rc, bptr, use_offsets, count, i, subsptr); \ + else if (pcre_mode == PCRE16_MODE) \ + PCRE_GET_SUBSTRING16(rc, bptr, use_offsets, count, i, subsptr); \ + else \ + PCRE_GET_SUBSTRING8(rc, bptr, use_offsets, count, i, subsptr) + +#define PCRE_GET_SUBSTRING_LIST(rc, bptr, offsets, count, listptr) \ + if (pcre_mode == PCRE32_MODE) \ + PCRE_GET_SUBSTRING_LIST32(rc, bptr, offsets, count, listptr); \ + else if (pcre_mode == PCRE16_MODE) \ + PCRE_GET_SUBSTRING_LIST16(rc, bptr, offsets, count, listptr); \ + else \ + PCRE_GET_SUBSTRING_LIST8(rc, bptr, offsets, count, listptr) + +#define PCRE_JIT_STACK_ALLOC(startsize, maxsize) \ + (pcre_mode == PCRE32_MODE ? \ + PCRE_JIT_STACK_ALLOC32(startsize, maxsize) \ + : pcre_mode == PCRE16_MODE ? \ + PCRE_JIT_STACK_ALLOC16(startsize, maxsize) \ + : PCRE_JIT_STACK_ALLOC8(startsize, maxsize)) + +#define PCRE_JIT_STACK_FREE(stack) \ + if (pcre_mode == PCRE32_MODE) \ + PCRE_JIT_STACK_FREE32(stack); \ + else if (pcre_mode == PCRE16_MODE) \ + PCRE_JIT_STACK_FREE16(stack); \ + else \ + PCRE_JIT_STACK_FREE8(stack) + +#define PCRE_MAKETABLES \ + (pcre_mode == PCRE32_MODE ? pcre32_maketables() : pcre_mode == PCRE16_MODE ? pcre16_maketables() : pcre_maketables()) + +#define PCRE_PATTERN_TO_HOST_BYTE_ORDER(rc, re, extra, tables) \ + if (pcre_mode == PCRE32_MODE) \ + PCRE_PATTERN_TO_HOST_BYTE_ORDER32(rc, re, extra, tables); \ + else if (pcre_mode == PCRE16_MODE) \ + PCRE_PATTERN_TO_HOST_BYTE_ORDER16(rc, re, extra, tables); \ + else \ + PCRE_PATTERN_TO_HOST_BYTE_ORDER8(rc, re, extra, tables) + +#define PCRE_PRINTINT(re, outfile, debug_lengths) \ + if (pcre_mode == PCRE32_MODE) \ + PCRE_PRINTINT32(re, outfile, debug_lengths); \ + else if (pcre_mode == PCRE16_MODE) \ + PCRE_PRINTINT16(re, outfile, debug_lengths); \ + else \ + PCRE_PRINTINT8(re, outfile, debug_lengths) + +#define PCRE_STUDY(extra, re, options, error) \ + if (pcre_mode == PCRE32_MODE) \ + PCRE_STUDY32(extra, re, options, error); \ + else if (pcre_mode == PCRE16_MODE) \ + PCRE_STUDY16(extra, re, options, error); \ + else \ + PCRE_STUDY8(extra, re, options, error) + + +/* ----- Two out of three modes are supported ----- */ + +#else + +/* We can use some macro trickery to make a single set of definitions work in +the three different cases. */ + +/* ----- 32-bit and 16-bit but not 8-bit supported ----- */ + +#if defined(SUPPORT_PCRE32) && defined(SUPPORT_PCRE16) +#define BITONE 32 +#define BITTWO 16 + +/* ----- 32-bit and 8-bit but not 16-bit supported ----- */ + +#elif defined(SUPPORT_PCRE32) && defined(SUPPORT_PCRE8) +#define BITONE 32 +#define BITTWO 8 + +/* ----- 16-bit and 8-bit but not 32-bit supported ----- */ + +#else +#define BITONE 16 +#define BITTWO 8 +#endif + +#define glue(a,b) a##b +#define G(a,b) glue(a,b) + + +/* ----- Common macros for two-mode cases ----- */ + +#define PCHARS(lv, p, offset, len, f) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(PCHARS,BITONE)(lv, p, offset, len, f); \ + else \ + G(PCHARS,BITTWO)(lv, p, offset, len, f) + +#define PCHARSV(p, offset, len, f) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(PCHARSV,BITONE)(p, offset, len, f); \ + else \ + G(PCHARSV,BITTWO)(p, offset, len, f) + +#define READ_CAPTURE_NAME(p, cn8, cn16, cn32, re) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(READ_CAPTURE_NAME,BITONE)(p, cn8, cn16, cn32, re); \ + else \ + G(READ_CAPTURE_NAME,BITTWO)(p, cn8, cn16, cn32, re) + +#define SET_PCRE_CALLOUT(callout) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(SET_PCRE_CALLOUT,BITONE)(callout); \ + else \ + G(SET_PCRE_CALLOUT,BITTWO)(callout) + +#define SET_PCRE_STACK_GUARD(stack_guard) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(SET_PCRE_STACK_GUARD,BITONE)(stack_guard); \ + else \ + G(SET_PCRE_STACK_GUARD,BITTWO)(stack_guard) + +#define STRLEN(p) ((pcre_mode == G(G(PCRE,BITONE),_MODE)) ? \ + G(STRLEN,BITONE)(p) : G(STRLEN,BITTWO)(p)) + +#define PCRE_ASSIGN_JIT_STACK(extra, callback, userdata) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(PCRE_ASSIGN_JIT_STACK,BITONE)(extra, callback, userdata); \ + else \ + G(PCRE_ASSIGN_JIT_STACK,BITTWO)(extra, callback, userdata) + +#define PCRE_COMPILE(re, pat, options, error, erroffset, tables) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(PCRE_COMPILE,BITONE)(re, pat, options, error, erroffset, tables); \ + else \ + G(PCRE_COMPILE,BITTWO)(re, pat, options, error, erroffset, tables) + +#define PCRE_CONFIG G(G(pcre,BITONE),_config) + +#define PCRE_COPY_NAMED_SUBSTRING(rc, re, bptr, offsets, count, \ + namesptr, cbuffer, size) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(PCRE_COPY_NAMED_SUBSTRING,BITONE)(rc, re, bptr, offsets, count, \ + namesptr, cbuffer, size); \ + else \ + G(PCRE_COPY_NAMED_SUBSTRING,BITTWO)(rc, re, bptr, offsets, count, \ + namesptr, cbuffer, size) + +#define PCRE_COPY_SUBSTRING(rc, bptr, offsets, count, i, cbuffer, size) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(PCRE_COPY_SUBSTRING,BITONE)(rc, bptr, offsets, count, i, cbuffer, size); \ + else \ + G(PCRE_COPY_SUBSTRING,BITTWO)(rc, bptr, offsets, count, i, cbuffer, size) + +#define PCRE_DFA_EXEC(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets, workspace, size_workspace) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(PCRE_DFA_EXEC,BITONE)(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets, workspace, size_workspace); \ + else \ + G(PCRE_DFA_EXEC,BITTWO)(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets, workspace, size_workspace) + +#define PCRE_EXEC(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(PCRE_EXEC,BITONE)(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets); \ + else \ + G(PCRE_EXEC,BITTWO)(count, re, extra, bptr, len, start_offset, options, \ + offsets, size_offsets) + +#define PCRE_FREE_STUDY(extra) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(PCRE_FREE_STUDY,BITONE)(extra); \ + else \ + G(PCRE_FREE_STUDY,BITTWO)(extra) + +#define PCRE_FREE_SUBSTRING(substring) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(PCRE_FREE_SUBSTRING,BITONE)(substring); \ + else \ + G(PCRE_FREE_SUBSTRING,BITTWO)(substring) + +#define PCRE_FREE_SUBSTRING_LIST(listptr) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(PCRE_FREE_SUBSTRING_LIST,BITONE)(listptr); \ + else \ + G(PCRE_FREE_SUBSTRING_LIST,BITTWO)(listptr) + +#define PCRE_GET_NAMED_SUBSTRING(rc, re, bptr, offsets, count, \ + getnamesptr, subsptr) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(PCRE_GET_NAMED_SUBSTRING,BITONE)(rc, re, bptr, offsets, count, \ + getnamesptr, subsptr); \ + else \ + G(PCRE_GET_NAMED_SUBSTRING,BITTWO)(rc, re, bptr, offsets, count, \ + getnamesptr, subsptr) + +#define PCRE_GET_STRINGNUMBER(n, rc, ptr) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(PCRE_GET_STRINGNUMBER,BITONE)(n, rc, ptr); \ + else \ + G(PCRE_GET_STRINGNUMBER,BITTWO)(n, rc, ptr) + +#define PCRE_GET_SUBSTRING(rc, bptr, use_offsets, count, i, subsptr) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(PCRE_GET_SUBSTRING,BITONE)(rc, bptr, use_offsets, count, i, subsptr); \ + else \ + G(PCRE_GET_SUBSTRING,BITTWO)(rc, bptr, use_offsets, count, i, subsptr) + +#define PCRE_GET_SUBSTRING_LIST(rc, bptr, offsets, count, listptr) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(PCRE_GET_SUBSTRING_LIST,BITONE)(rc, bptr, offsets, count, listptr); \ + else \ + G(PCRE_GET_SUBSTRING_LIST,BITTWO)(rc, bptr, offsets, count, listptr) + +#define PCRE_JIT_STACK_ALLOC(startsize, maxsize) \ + (pcre_mode == G(G(PCRE,BITONE),_MODE)) ? \ + G(PCRE_JIT_STACK_ALLOC,BITONE)(startsize, maxsize) \ + : G(PCRE_JIT_STACK_ALLOC,BITTWO)(startsize, maxsize) + +#define PCRE_JIT_STACK_FREE(stack) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(PCRE_JIT_STACK_FREE,BITONE)(stack); \ + else \ + G(PCRE_JIT_STACK_FREE,BITTWO)(stack) + +#define PCRE_MAKETABLES \ + (pcre_mode == G(G(PCRE,BITONE),_MODE)) ? \ + G(G(pcre,BITONE),_maketables)() : G(G(pcre,BITTWO),_maketables)() + +#define PCRE_PATTERN_TO_HOST_BYTE_ORDER(rc, re, extra, tables) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(PCRE_PATTERN_TO_HOST_BYTE_ORDER,BITONE)(rc, re, extra, tables); \ + else \ + G(PCRE_PATTERN_TO_HOST_BYTE_ORDER,BITTWO)(rc, re, extra, tables) + +#define PCRE_PRINTINT(re, outfile, debug_lengths) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(PCRE_PRINTINT,BITONE)(re, outfile, debug_lengths); \ + else \ + G(PCRE_PRINTINT,BITTWO)(re, outfile, debug_lengths) + +#define PCRE_STUDY(extra, re, options, error) \ + if (pcre_mode == G(G(PCRE,BITONE),_MODE)) \ + G(PCRE_STUDY,BITONE)(extra, re, options, error); \ + else \ + G(PCRE_STUDY,BITTWO)(extra, re, options, error) + +#endif /* Two out of three modes */ + +/* ----- End of cases where more than one mode is supported ----- */ + + +/* ----- Only 8-bit mode is supported ----- */ + +#elif defined SUPPORT_PCRE8 +#define CHAR_SIZE 1 +#define PCHARS PCHARS8 +#define PCHARSV PCHARSV8 +#define READ_CAPTURE_NAME READ_CAPTURE_NAME8 +#define SET_PCRE_CALLOUT SET_PCRE_CALLOUT8 +#define SET_PCRE_STACK_GUARD SET_PCRE_STACK_GUARD8 +#define STRLEN STRLEN8 +#define PCRE_ASSIGN_JIT_STACK PCRE_ASSIGN_JIT_STACK8 +#define PCRE_COMPILE PCRE_COMPILE8 +#define PCRE_CONFIG pcre_config +#define PCRE_COPY_NAMED_SUBSTRING PCRE_COPY_NAMED_SUBSTRING8 +#define PCRE_COPY_SUBSTRING PCRE_COPY_SUBSTRING8 +#define PCRE_DFA_EXEC PCRE_DFA_EXEC8 +#define PCRE_EXEC PCRE_EXEC8 +#define PCRE_FREE_STUDY PCRE_FREE_STUDY8 +#define PCRE_FREE_SUBSTRING PCRE_FREE_SUBSTRING8 +#define PCRE_FREE_SUBSTRING_LIST PCRE_FREE_SUBSTRING_LIST8 +#define PCRE_GET_NAMED_SUBSTRING PCRE_GET_NAMED_SUBSTRING8 +#define PCRE_GET_STRINGNUMBER PCRE_GET_STRINGNUMBER8 +#define PCRE_GET_SUBSTRING PCRE_GET_SUBSTRING8 +#define PCRE_GET_SUBSTRING_LIST PCRE_GET_SUBSTRING_LIST8 +#define PCRE_JIT_STACK_ALLOC PCRE_JIT_STACK_ALLOC8 +#define PCRE_JIT_STACK_FREE PCRE_JIT_STACK_FREE8 +#define PCRE_MAKETABLES pcre_maketables() +#define PCRE_PATTERN_TO_HOST_BYTE_ORDER PCRE_PATTERN_TO_HOST_BYTE_ORDER8 +#define PCRE_PRINTINT PCRE_PRINTINT8 +#define PCRE_STUDY PCRE_STUDY8 + +/* ----- Only 16-bit mode is supported ----- */ + +#elif defined SUPPORT_PCRE16 +#define CHAR_SIZE 2 +#define PCHARS PCHARS16 +#define PCHARSV PCHARSV16 +#define READ_CAPTURE_NAME READ_CAPTURE_NAME16 +#define SET_PCRE_CALLOUT SET_PCRE_CALLOUT16 +#define SET_PCRE_STACK_GUARD SET_PCRE_STACK_GUARD16 +#define STRLEN STRLEN16 +#define PCRE_ASSIGN_JIT_STACK PCRE_ASSIGN_JIT_STACK16 +#define PCRE_COMPILE PCRE_COMPILE16 +#define PCRE_CONFIG pcre16_config +#define PCRE_COPY_NAMED_SUBSTRING PCRE_COPY_NAMED_SUBSTRING16 +#define PCRE_COPY_SUBSTRING PCRE_COPY_SUBSTRING16 +#define PCRE_DFA_EXEC PCRE_DFA_EXEC16 +#define PCRE_EXEC PCRE_EXEC16 +#define PCRE_FREE_STUDY PCRE_FREE_STUDY16 +#define PCRE_FREE_SUBSTRING PCRE_FREE_SUBSTRING16 +#define PCRE_FREE_SUBSTRING_LIST PCRE_FREE_SUBSTRING_LIST16 +#define PCRE_GET_NAMED_SUBSTRING PCRE_GET_NAMED_SUBSTRING16 +#define PCRE_GET_STRINGNUMBER PCRE_GET_STRINGNUMBER16 +#define PCRE_GET_SUBSTRING PCRE_GET_SUBSTRING16 +#define PCRE_GET_SUBSTRING_LIST PCRE_GET_SUBSTRING_LIST16 +#define PCRE_JIT_STACK_ALLOC PCRE_JIT_STACK_ALLOC16 +#define PCRE_JIT_STACK_FREE PCRE_JIT_STACK_FREE16 +#define PCRE_MAKETABLES pcre16_maketables() +#define PCRE_PATTERN_TO_HOST_BYTE_ORDER PCRE_PATTERN_TO_HOST_BYTE_ORDER16 +#define PCRE_PRINTINT PCRE_PRINTINT16 +#define PCRE_STUDY PCRE_STUDY16 + +/* ----- Only 32-bit mode is supported ----- */ + +#elif defined SUPPORT_PCRE32 +#define CHAR_SIZE 4 +#define PCHARS PCHARS32 +#define PCHARSV PCHARSV32 +#define READ_CAPTURE_NAME READ_CAPTURE_NAME32 +#define SET_PCRE_CALLOUT SET_PCRE_CALLOUT32 +#define SET_PCRE_STACK_GUARD SET_PCRE_STACK_GUARD32 +#define STRLEN STRLEN32 +#define PCRE_ASSIGN_JIT_STACK PCRE_ASSIGN_JIT_STACK32 +#define PCRE_COMPILE PCRE_COMPILE32 +#define PCRE_CONFIG pcre32_config +#define PCRE_COPY_NAMED_SUBSTRING PCRE_COPY_NAMED_SUBSTRING32 +#define PCRE_COPY_SUBSTRING PCRE_COPY_SUBSTRING32 +#define PCRE_DFA_EXEC PCRE_DFA_EXEC32 +#define PCRE_EXEC PCRE_EXEC32 +#define PCRE_FREE_STUDY PCRE_FREE_STUDY32 +#define PCRE_FREE_SUBSTRING PCRE_FREE_SUBSTRING32 +#define PCRE_FREE_SUBSTRING_LIST PCRE_FREE_SUBSTRING_LIST32 +#define PCRE_GET_NAMED_SUBSTRING PCRE_GET_NAMED_SUBSTRING32 +#define PCRE_GET_STRINGNUMBER PCRE_GET_STRINGNUMBER32 +#define PCRE_GET_SUBSTRING PCRE_GET_SUBSTRING32 +#define PCRE_GET_SUBSTRING_LIST PCRE_GET_SUBSTRING_LIST32 +#define PCRE_JIT_STACK_ALLOC PCRE_JIT_STACK_ALLOC32 +#define PCRE_JIT_STACK_FREE PCRE_JIT_STACK_FREE32 +#define PCRE_MAKETABLES pcre32_maketables() +#define PCRE_PATTERN_TO_HOST_BYTE_ORDER PCRE_PATTERN_TO_HOST_BYTE_ORDER32 +#define PCRE_PRINTINT PCRE_PRINTINT32 +#define PCRE_STUDY PCRE_STUDY32 + +#endif + +/* ----- End of mode-specific function call macros ----- */ + + +/* Other parameters */ + +#ifndef CLOCKS_PER_SEC +#ifdef CLK_TCK +#define CLOCKS_PER_SEC CLK_TCK +#else +#define CLOCKS_PER_SEC 100 +#endif +#endif + +#if !defined NODFA +#define DFA_WS_DIMENSION 1000 +#endif + +/* This is the default loop count for timing. */ + +#define LOOPREPEAT 500000 + +/* Static variables */ + +static FILE *outfile; +static int log_store = 0; +static int callout_count; +static int callout_extra; +static int callout_fail_count; +static int callout_fail_id; +static int debug_lengths; +static int first_callout; +static int jit_was_used; +static int locale_set = 0; +static int show_malloc; +static int stack_guard_return; +static int use_utf; +static const unsigned char *last_callout_mark = NULL; + +/* The buffers grow automatically if very long input lines are encountered. */ + +static int buffer_size = 50000; +static pcre_uint8 *buffer = NULL; +static pcre_uint8 *pbuffer = NULL; + +/* Just as a safety check, make sure that COMPILE_PCRE[16|32] are *not* set. */ + +#ifdef COMPILE_PCRE16 +#error COMPILE_PCRE16 must not be set when compiling pcretest.c +#endif + +#ifdef COMPILE_PCRE32 +#error COMPILE_PCRE32 must not be set when compiling pcretest.c +#endif + +/* We need buffers for building 16/32-bit strings, and the tables of operator +lengths that are used for 16/32-bit compiling, in order to swap bytes in a +pattern for saving/reloading testing. Luckily, the data for these tables is +defined as a macro. However, we must ensure that LINK_SIZE and IMM2_SIZE (which +are used in the tables) are adjusted appropriately for the 16/32-bit world. +LINK_SIZE is also used later in this program. */ + +#ifdef SUPPORT_PCRE16 +#undef IMM2_SIZE +#define IMM2_SIZE 1 + +#if LINK_SIZE == 2 +#undef LINK_SIZE +#define LINK_SIZE 1 +#elif LINK_SIZE == 3 || LINK_SIZE == 4 +#undef LINK_SIZE +#define LINK_SIZE 2 +#else +#error LINK_SIZE must be either 2, 3, or 4 +#endif + +static int buffer16_size = 0; +static pcre_uint16 *buffer16 = NULL; +static const pcre_uint16 OP_lengths16[] = { OP_LENGTHS }; +#endif /* SUPPORT_PCRE16 */ + +#ifdef SUPPORT_PCRE32 +#undef IMM2_SIZE +#define IMM2_SIZE 1 +#undef LINK_SIZE +#define LINK_SIZE 1 + +static int buffer32_size = 0; +static pcre_uint32 *buffer32 = NULL; +static const pcre_uint32 OP_lengths32[] = { OP_LENGTHS }; +#endif /* SUPPORT_PCRE32 */ + +/* If we have 8-bit support, default to it; if there is also 16-or 32-bit +support, it can be changed by an option. If there is no 8-bit support, there +must be 16-or 32-bit support, so default it to 1. */ + +#if defined SUPPORT_PCRE8 +static int pcre_mode = PCRE8_MODE; +#elif defined SUPPORT_PCRE16 +static int pcre_mode = PCRE16_MODE; +#elif defined SUPPORT_PCRE32 +static int pcre_mode = PCRE32_MODE; +#endif + +/* JIT study options for -s+n and /S+n where '1' <= n <= '7'. */ + +static int jit_study_bits[] = + { + PCRE_STUDY_JIT_COMPILE, + PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE, + PCRE_STUDY_JIT_COMPILE + PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE, + PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE, + PCRE_STUDY_JIT_COMPILE + PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE, + PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE + PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE, + PCRE_STUDY_JIT_COMPILE + PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE + + PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE +}; + +#define PCRE_STUDY_ALLJIT (PCRE_STUDY_JIT_COMPILE | \ + PCRE_STUDY_JIT_PARTIAL_SOFT_COMPILE | PCRE_STUDY_JIT_PARTIAL_HARD_COMPILE) + +/* Textual explanations for runtime error codes */ + +static const char *errtexts[] = { + NULL, /* 0 is no error */ + NULL, /* NOMATCH is handled specially */ + "NULL argument passed", + "bad option value", + "magic number missing", + "unknown opcode - pattern overwritten?", + "no more memory", + NULL, /* never returned by pcre_exec() or pcre_dfa_exec() */ + "match limit exceeded", + "callout error code", + NULL, /* BADUTF8/16 is handled specially */ + NULL, /* BADUTF8/16 offset is handled specially */ + NULL, /* PARTIAL is handled specially */ + "not used - internal error", + "internal error - pattern overwritten?", + "bad count value", + "item unsupported for DFA matching", + "backreference condition or recursion test not supported for DFA matching", + "match limit not supported for DFA matching", + "workspace size exceeded in DFA matching", + "too much recursion for DFA matching", + "recursion limit exceeded", + "not used - internal error", + "invalid combination of newline options", + "bad offset value", + NULL, /* SHORTUTF8/16 is handled specially */ + "nested recursion at the same subject position", + "JIT stack limit reached", + "pattern compiled in wrong mode: 8-bit/16-bit error", + "pattern compiled with other endianness", + "invalid data in workspace for DFA restart", + "bad JIT option", + "bad length" +}; + + +/************************************************* +* Alternate character tables * +*************************************************/ + +/* By default, the "tables" pointer when calling PCRE is set to NULL, thereby +using the default tables of the library. However, the T option can be used to +select alternate sets of tables, for different kinds of testing. Note also that +the L (locale) option also adjusts the tables. */ + +/* This is the set of tables distributed as default with PCRE. It recognizes +only ASCII characters. */ + +static const pcre_uint8 tables0[] = { + +/* This table is a lower casing table. */ + + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, + 64, 97, 98, 99,100,101,102,103, + 104,105,106,107,108,109,110,111, + 112,113,114,115,116,117,118,119, + 120,121,122, 91, 92, 93, 94, 95, + 96, 97, 98, 99,100,101,102,103, + 104,105,106,107,108,109,110,111, + 112,113,114,115,116,117,118,119, + 120,121,122,123,124,125,126,127, + 128,129,130,131,132,133,134,135, + 136,137,138,139,140,141,142,143, + 144,145,146,147,148,149,150,151, + 152,153,154,155,156,157,158,159, + 160,161,162,163,164,165,166,167, + 168,169,170,171,172,173,174,175, + 176,177,178,179,180,181,182,183, + 184,185,186,187,188,189,190,191, + 192,193,194,195,196,197,198,199, + 200,201,202,203,204,205,206,207, + 208,209,210,211,212,213,214,215, + 216,217,218,219,220,221,222,223, + 224,225,226,227,228,229,230,231, + 232,233,234,235,236,237,238,239, + 240,241,242,243,244,245,246,247, + 248,249,250,251,252,253,254,255, + +/* This table is a case flipping table. */ + + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, + 64, 97, 98, 99,100,101,102,103, + 104,105,106,107,108,109,110,111, + 112,113,114,115,116,117,118,119, + 120,121,122, 91, 92, 93, 94, 95, + 96, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 74, 75, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 87, + 88, 89, 90,123,124,125,126,127, + 128,129,130,131,132,133,134,135, + 136,137,138,139,140,141,142,143, + 144,145,146,147,148,149,150,151, + 152,153,154,155,156,157,158,159, + 160,161,162,163,164,165,166,167, + 168,169,170,171,172,173,174,175, + 176,177,178,179,180,181,182,183, + 184,185,186,187,188,189,190,191, + 192,193,194,195,196,197,198,199, + 200,201,202,203,204,205,206,207, + 208,209,210,211,212,213,214,215, + 216,217,218,219,220,221,222,223, + 224,225,226,227,228,229,230,231, + 232,233,234,235,236,237,238,239, + 240,241,242,243,244,245,246,247, + 248,249,250,251,252,253,254,255, + +/* This table contains bit maps for various character classes. Each map is 32 +bytes long and the bits run from the least significant end of each byte. The +classes that have their own maps are: space, xdigit, digit, upper, lower, word, +graph, print, punct, and cntrl. Other classes are built from combinations. */ + + 0x00,0x3e,0x00,0x00,0x01,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, + 0x7e,0x00,0x00,0x00,0x7e,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xfe,0xff,0xff,0x07,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0x07, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x03, + 0xfe,0xff,0xff,0x87,0xfe,0xff,0xff,0x07, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0xff,0xff,0xff,0xff, + 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x7f, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0x00,0x00,0x00,0x00,0xfe,0xff,0x00,0xfc, + 0x01,0x00,0x00,0xf8,0x01,0x00,0x00,0x78, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + + 0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x80, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + +/* This table identifies various classes of character by individual bits: + 0x01 white space character + 0x02 letter + 0x04 decimal digit + 0x08 hexadecimal digit + 0x10 alphanumeric or '_' + 0x80 regular expression metacharacter or binary zero +*/ + + 0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 0- 7 */ + 0x00,0x01,0x01,0x01,0x01,0x01,0x00,0x00, /* 8- 15 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 16- 23 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 24- 31 */ + 0x01,0x00,0x00,0x00,0x80,0x00,0x00,0x00, /* - ' */ + 0x80,0x80,0x80,0x80,0x00,0x00,0x80,0x00, /* ( - / */ + 0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c,0x1c, /* 0 - 7 */ + 0x1c,0x1c,0x00,0x00,0x00,0x00,0x00,0x80, /* 8 - ? */ + 0x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /* @ - G */ + 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* H - O */ + 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* P - W */ + 0x12,0x12,0x12,0x80,0x80,0x00,0x80,0x10, /* X - _ */ + 0x00,0x1a,0x1a,0x1a,0x1a,0x1a,0x1a,0x12, /* ` - g */ + 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* h - o */ + 0x12,0x12,0x12,0x12,0x12,0x12,0x12,0x12, /* p - w */ + 0x12,0x12,0x12,0x80,0x80,0x00,0x00,0x00, /* x -127 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 128-135 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 136-143 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 144-151 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 152-159 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 160-167 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 168-175 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 176-183 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 184-191 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 192-199 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 200-207 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 208-215 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 216-223 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 224-231 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 232-239 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, /* 240-247 */ + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00};/* 248-255 */ + +/* This is a set of tables that came originally from a Windows user. It seems +to be at least an approximation of ISO 8859. In particular, there are +characters greater than 128 that are marked as spaces, letters, etc. */ + +static const pcre_uint8 tables1[] = { +0,1,2,3,4,5,6,7, +8,9,10,11,12,13,14,15, +16,17,18,19,20,21,22,23, +24,25,26,27,28,29,30,31, +32,33,34,35,36,37,38,39, +40,41,42,43,44,45,46,47, +48,49,50,51,52,53,54,55, +56,57,58,59,60,61,62,63, +64,97,98,99,100,101,102,103, +104,105,106,107,108,109,110,111, +112,113,114,115,116,117,118,119, +120,121,122,91,92,93,94,95, +96,97,98,99,100,101,102,103, +104,105,106,107,108,109,110,111, +112,113,114,115,116,117,118,119, +120,121,122,123,124,125,126,127, +128,129,130,131,132,133,134,135, +136,137,138,139,140,141,142,143, +144,145,146,147,148,149,150,151, +152,153,154,155,156,157,158,159, +160,161,162,163,164,165,166,167, +168,169,170,171,172,173,174,175, +176,177,178,179,180,181,182,183, +184,185,186,187,188,189,190,191, +224,225,226,227,228,229,230,231, +232,233,234,235,236,237,238,239, +240,241,242,243,244,245,246,215, +248,249,250,251,252,253,254,223, +224,225,226,227,228,229,230,231, +232,233,234,235,236,237,238,239, +240,241,242,243,244,245,246,247, +248,249,250,251,252,253,254,255, +0,1,2,3,4,5,6,7, +8,9,10,11,12,13,14,15, +16,17,18,19,20,21,22,23, +24,25,26,27,28,29,30,31, +32,33,34,35,36,37,38,39, +40,41,42,43,44,45,46,47, +48,49,50,51,52,53,54,55, +56,57,58,59,60,61,62,63, +64,97,98,99,100,101,102,103, +104,105,106,107,108,109,110,111, +112,113,114,115,116,117,118,119, +120,121,122,91,92,93,94,95, +96,65,66,67,68,69,70,71, +72,73,74,75,76,77,78,79, +80,81,82,83,84,85,86,87, +88,89,90,123,124,125,126,127, +128,129,130,131,132,133,134,135, +136,137,138,139,140,141,142,143, +144,145,146,147,148,149,150,151, +152,153,154,155,156,157,158,159, +160,161,162,163,164,165,166,167, +168,169,170,171,172,173,174,175, +176,177,178,179,180,181,182,183, +184,185,186,187,188,189,190,191, +224,225,226,227,228,229,230,231, +232,233,234,235,236,237,238,239, +240,241,242,243,244,245,246,215, +248,249,250,251,252,253,254,223, +192,193,194,195,196,197,198,199, +200,201,202,203,204,205,206,207, +208,209,210,211,212,213,214,247, +216,217,218,219,220,221,222,255, +0,62,0,0,1,0,0,0, +0,0,0,0,0,0,0,0, +32,0,0,0,1,0,0,0, +0,0,0,0,0,0,0,0, +0,0,0,0,0,0,255,3, +126,0,0,0,126,0,0,0, +0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0, +0,0,0,0,0,0,255,3, +0,0,0,0,0,0,0,0, +0,0,0,0,0,0,12,2, +0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0, +254,255,255,7,0,0,0,0, +0,0,0,0,0,0,0,0, +255,255,127,127,0,0,0,0, +0,0,0,0,0,0,0,0, +0,0,0,0,254,255,255,7, +0,0,0,0,0,4,32,4, +0,0,0,128,255,255,127,255, +0,0,0,0,0,0,255,3, +254,255,255,135,254,255,255,7, +0,0,0,0,0,4,44,6, +255,255,127,255,255,255,127,255, +0,0,0,0,254,255,255,255, +255,255,255,255,255,255,255,127, +0,0,0,0,254,255,255,255, +255,255,255,255,255,255,255,255, +0,2,0,0,255,255,255,255, +255,255,255,255,255,255,255,127, +0,0,0,0,255,255,255,255, +255,255,255,255,255,255,255,255, +0,0,0,0,254,255,0,252, +1,0,0,248,1,0,0,120, +0,0,0,0,254,255,255,255, +0,0,128,0,0,0,128,0, +255,255,255,255,0,0,0,0, +0,0,0,0,0,0,0,128, +255,255,255,255,0,0,0,0, +0,0,0,0,0,0,0,0, +128,0,0,0,0,0,0,0, +0,1,1,0,1,1,0,0, +0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0, +1,0,0,0,128,0,0,0, +128,128,128,128,0,0,128,0, +28,28,28,28,28,28,28,28, +28,28,0,0,0,0,0,128, +0,26,26,26,26,26,26,18, +18,18,18,18,18,18,18,18, +18,18,18,18,18,18,18,18, +18,18,18,128,128,0,128,16, +0,26,26,26,26,26,26,18, +18,18,18,18,18,18,18,18, +18,18,18,18,18,18,18,18, +18,18,18,128,128,0,0,0, +0,0,0,0,0,1,0,0, +0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0, +1,0,0,0,0,0,0,0, +0,0,18,0,0,0,0,0, +0,0,20,20,0,18,0,0, +0,20,18,0,0,0,0,0, +18,18,18,18,18,18,18,18, +18,18,18,18,18,18,18,18, +18,18,18,18,18,18,18,0, +18,18,18,18,18,18,18,18, +18,18,18,18,18,18,18,18, +18,18,18,18,18,18,18,18, +18,18,18,18,18,18,18,0, +18,18,18,18,18,18,18,18 +}; + + + + +#ifndef HAVE_STRERROR +/************************************************* +* Provide strerror() for non-ANSI libraries * +*************************************************/ + +/* Some old-fashioned systems still around (e.g. SunOS4) don't have strerror() +in their libraries, but can provide the same facility by this simple +alternative function. */ + +extern int sys_nerr; +extern char *sys_errlist[]; + +char * +strerror(int n) +{ +if (n < 0 || n >= sys_nerr) return "unknown error number"; +return sys_errlist[n]; +} +#endif /* HAVE_STRERROR */ + + + +/************************************************* +* Print newline configuration * +*************************************************/ + +/* +Arguments: + rc the return code from PCRE_CONFIG_NEWLINE + isc TRUE if called from "-C newline" +Returns: nothing +*/ + +static void +print_newline_config(int rc, BOOL isc) +{ +const char *s = NULL; +if (!isc) printf(" Newline sequence is "); +switch(rc) + { + case CHAR_CR: s = "CR"; break; + case CHAR_LF: s = "LF"; break; + case (CHAR_CR<<8 | CHAR_LF): s = "CRLF"; break; + case -1: s = "ANY"; break; + case -2: s = "ANYCRLF"; break; + + default: + printf("a non-standard value: 0x%04x\n", rc); + return; + } + +printf("%s\n", s); +} + + + +/************************************************* +* JIT memory callback * +*************************************************/ + +static pcre_jit_stack* jit_callback(void *arg) +{ +jit_was_used = TRUE; +return (pcre_jit_stack *)arg; +} + + +#if !defined NOUTF || defined SUPPORT_PCRE16 || defined SUPPORT_PCRE32 +/************************************************* +* Convert UTF-8 string to value * +*************************************************/ + +/* This function takes one or more bytes that represents a UTF-8 character, +and returns the value of the character. + +Argument: + utf8bytes a pointer to the byte vector + vptr a pointer to an int to receive the value + +Returns: > 0 => the number of bytes consumed + -6 to 0 => malformed UTF-8 character at offset = (-return) +*/ + +static int +utf82ord(pcre_uint8 *utf8bytes, pcre_uint32 *vptr) +{ +pcre_uint32 c = *utf8bytes++; +pcre_uint32 d = c; +int i, j, s; + +for (i = -1; i < 6; i++) /* i is number of additional bytes */ + { + if ((d & 0x80) == 0) break; + d <<= 1; + } + +if (i == -1) { *vptr = c; return 1; } /* ascii character */ +if (i == 0 || i == 6) return 0; /* invalid UTF-8 */ + +/* i now has a value in the range 1-5 */ + +s = 6*i; +d = (c & utf8_table3[i]) << s; + +for (j = 0; j < i; j++) + { + c = *utf8bytes++; + if ((c & 0xc0) != 0x80) return -(j+1); + s -= 6; + d |= (c & 0x3f) << s; + } + +/* Check that encoding was the correct unique one */ + +for (j = 0; j < utf8_table1_size; j++) + if (d <= (pcre_uint32)utf8_table1[j]) break; +if (j != i) return -(i+1); + +/* Valid value */ + +*vptr = d; +return i+1; +} +#endif /* NOUTF || SUPPORT_PCRE16 */ + + + +#if defined SUPPORT_PCRE8 && !defined NOUTF +/************************************************* +* Convert character value to UTF-8 * +*************************************************/ + +/* This function takes an integer value in the range 0 - 0x7fffffff +and encodes it as a UTF-8 character in 0 to 6 bytes. + +Arguments: + cvalue the character value + utf8bytes pointer to buffer for result - at least 6 bytes long + +Returns: number of characters placed in the buffer +*/ + +static int +ord2utf8(pcre_uint32 cvalue, pcre_uint8 *utf8bytes) +{ +register int i, j; +if (cvalue > 0x7fffffffu) + return -1; +for (i = 0; i < utf8_table1_size; i++) + if (cvalue <= (pcre_uint32)utf8_table1[i]) break; +utf8bytes += i; +for (j = i; j > 0; j--) + { + *utf8bytes-- = 0x80 | (cvalue & 0x3f); + cvalue >>= 6; + } +*utf8bytes = utf8_table2[i] | cvalue; +return i + 1; +} +#endif + + +#ifdef SUPPORT_PCRE16 +/************************************************* +* Convert a string to 16-bit * +*************************************************/ + +/* In non-UTF mode, the space needed for a 16-bit string is exactly double the +8-bit size. For a UTF-8 string, the size needed for UTF-16 is no more than +double, because up to 0xffff uses no more than 3 bytes in UTF-8 but possibly 4 +in UTF-16. Higher values use 4 bytes in UTF-8 and up to 4 bytes in UTF-16. The +result is always left in buffer16. + +Note that this function does not object to surrogate values. This is +deliberate; it makes it possible to construct UTF-16 strings that are invalid, +for the purpose of testing that they are correctly faulted. + +Patterns to be converted are either plain ASCII or UTF-8; data lines are always +in UTF-8 so that values greater than 255 can be handled. + +Arguments: + data TRUE if converting a data line; FALSE for a regex + p points to a byte string + utf true if UTF-8 (to be converted to UTF-16) + len number of bytes in the string (excluding trailing zero) + +Returns: number of 16-bit data items used (excluding trailing zero) + OR -1 if a UTF-8 string is malformed + OR -2 if a value > 0x10ffff is encountered + OR -3 if a value > 0xffff is encountered when not in UTF mode +*/ + +static int +to16(int data, pcre_uint8 *p, int utf, int len) +{ +pcre_uint16 *pp; + +if (buffer16_size < 2*len + 2) + { + if (buffer16 != NULL) free(buffer16); + buffer16_size = 2*len + 2; + buffer16 = (pcre_uint16 *)malloc(buffer16_size); + if (buffer16 == NULL) + { + fprintf(stderr, "pcretest: malloc(%d) failed for buffer16\n", buffer16_size); + exit(1); + } + } + +pp = buffer16; + +if (!utf && !data) + { + while (len-- > 0) *pp++ = *p++; + } + +else + { + pcre_uint32 c = 0; + while (len > 0) + { + int chlen = utf82ord(p, &c); + if (chlen <= 0) return -1; + if (c > 0x10ffff) return -2; + p += chlen; + len -= chlen; + if (c < 0x10000) *pp++ = c; else + { + if (!utf) return -3; + c -= 0x10000; + *pp++ = 0xD800 | (c >> 10); + *pp++ = 0xDC00 | (c & 0x3ff); + } + } + } + +*pp = 0; +return pp - buffer16; +} +#endif + +#ifdef SUPPORT_PCRE32 +/************************************************* +* Convert a string to 32-bit * +*************************************************/ + +/* In non-UTF mode, the space needed for a 32-bit string is exactly four times the +8-bit size. For a UTF-8 string, the size needed for UTF-32 is no more than four +times, because up to 0xffff uses no more than 3 bytes in UTF-8 but possibly 4 +in UTF-32. Higher values use 4 bytes in UTF-8 and up to 4 bytes in UTF-32. The +result is always left in buffer32. + +Note that this function does not object to surrogate values. This is +deliberate; it makes it possible to construct UTF-32 strings that are invalid, +for the purpose of testing that they are correctly faulted. + +Patterns to be converted are either plain ASCII or UTF-8; data lines are always +in UTF-8 so that values greater than 255 can be handled. + +Arguments: + data TRUE if converting a data line; FALSE for a regex + p points to a byte string + utf true if UTF-8 (to be converted to UTF-32) + len number of bytes in the string (excluding trailing zero) + +Returns: number of 32-bit data items used (excluding trailing zero) + OR -1 if a UTF-8 string is malformed + OR -2 if a value > 0x10ffff is encountered + OR -3 if an ill-formed value is encountered (i.e. a surrogate) +*/ + +static int +to32(int data, pcre_uint8 *p, int utf, int len) +{ +pcre_uint32 *pp; + +if (buffer32_size < 4*len + 4) + { + if (buffer32 != NULL) free(buffer32); + buffer32_size = 4*len + 4; + buffer32 = (pcre_uint32 *)malloc(buffer32_size); + if (buffer32 == NULL) + { + fprintf(stderr, "pcretest: malloc(%d) failed for buffer32\n", buffer32_size); + exit(1); + } + } + +pp = buffer32; + +if (!utf && !data) + { + while (len-- > 0) *pp++ = *p++; + } + +else + { + pcre_uint32 c = 0; + while (len > 0) + { + int chlen = utf82ord(p, &c); + if (chlen <= 0) return -1; + if (utf) + { + if (c > 0x10ffff) return -2; + if (!data && (c & 0xfffff800u) == 0xd800u) return -3; + } + + p += chlen; + len -= chlen; + *pp++ = c; + } + } + +*pp = 0; +return pp - buffer32; +} + +/* Check that a 32-bit character string is valid UTF-32. + +Arguments: + string points to the string + length length of string, or -1 if the string is zero-terminated + +Returns: TRUE if the string is a valid UTF-32 string + FALSE otherwise +*/ + +#ifdef NEVER /* Not used */ +#ifdef SUPPORT_UTF +static BOOL +valid_utf32(pcre_uint32 *string, int length) +{ +register pcre_uint32 *p; +register pcre_uint32 c; + +for (p = string; length-- > 0; p++) + { + c = *p; + if (c > 0x10ffffu) return FALSE; /* Too big */ + if ((c & 0xfffff800u) == 0xd800u) return FALSE; /* Surrogate */ + } + +return TRUE; +} +#endif /* SUPPORT_UTF */ +#endif /* NEVER */ +#endif /* SUPPORT_PCRE32 */ + + +/************************************************* +* Read or extend an input line * +*************************************************/ + +/* Input lines are read into buffer, but both patterns and data lines can be +continued over multiple input lines. In addition, if the buffer fills up, we +want to automatically expand it so as to be able to handle extremely large +lines that are needed for certain stress tests. When the input buffer is +expanded, the other two buffers must also be expanded likewise, and the +contents of pbuffer, which are a copy of the input for callouts, must be +preserved (for when expansion happens for a data line). This is not the most +optimal way of handling this, but hey, this is just a test program! + +Arguments: + f the file to read + start where in buffer to start (this *must* be within buffer) + prompt for stdin or readline() + +Returns: pointer to the start of new data + could be a copy of start, or could be moved + NULL if no data read and EOF reached +*/ + +static pcre_uint8 * +extend_inputline(FILE *f, pcre_uint8 *start, const char *prompt) +{ +pcre_uint8 *here = start; + +for (;;) + { + size_t rlen = (size_t)(buffer_size - (here - buffer)); + + if (rlen > 1000) + { + int dlen; + + /* If libreadline or libedit support is required, use readline() to read a + line if the input is a terminal. Note that readline() removes the trailing + newline, so we must put it back again, to be compatible with fgets(). */ + +#if defined(SUPPORT_LIBREADLINE) || defined(SUPPORT_LIBEDIT) + if (isatty(fileno(f))) + { + size_t len; + char *s = readline(prompt); + if (s == NULL) return (here == start)? NULL : start; + len = strlen(s); + if (len > 0) add_history(s); + if (len > rlen - 1) len = rlen - 1; + memcpy(here, s, len); + here[len] = '\n'; + here[len+1] = 0; + free(s); + } + else +#endif + + /* Read the next line by normal means, prompting if the file is stdin. */ + + { + if (f == stdin) printf("%s", prompt); + if (fgets((char *)here, rlen, f) == NULL) + return (here == start)? NULL : start; + } + + dlen = (int)strlen((char *)here); + if (dlen > 0 && here[dlen - 1] == '\n') return start; + here += dlen; + } + + else + { + int new_buffer_size = 2*buffer_size; + pcre_uint8 *new_buffer = (pcre_uint8 *)malloc(new_buffer_size); + pcre_uint8 *new_pbuffer = (pcre_uint8 *)malloc(new_buffer_size); + + if (new_buffer == NULL || new_pbuffer == NULL) + { + fprintf(stderr, "pcretest: malloc(%d) failed\n", new_buffer_size); + exit(1); + } + + memcpy(new_buffer, buffer, buffer_size); + memcpy(new_pbuffer, pbuffer, buffer_size); + + buffer_size = new_buffer_size; + + start = new_buffer + (start - buffer); + here = new_buffer + (here - buffer); + + free(buffer); + free(pbuffer); + + buffer = new_buffer; + pbuffer = new_pbuffer; + } + } + +/* Control never gets here */ +} + + + +/************************************************* +* Read number from string * +*************************************************/ + +/* We don't use strtoul() because SunOS4 doesn't have it. Rather than mess +around with conditional compilation, just do the job by hand. It is only used +for unpicking arguments, so just keep it simple. + +Arguments: + str string to be converted + endptr where to put the end pointer + +Returns: the unsigned long +*/ + +static int +get_value(pcre_uint8 *str, pcre_uint8 **endptr) +{ +int result = 0; +while(*str != 0 && isspace(*str)) str++; +while (isdigit(*str)) result = result * 10 + (int)(*str++ - '0'); +*endptr = str; +return(result); +} + + + +/************************************************* +* Print one character * +*************************************************/ + +/* Print a single character either literally, or as a hex escape. */ + +static int pchar(pcre_uint32 c, FILE *f) +{ +int n = 0; +char tempbuffer[16]; +if (PRINTOK(c)) + { + if (f != NULL) fprintf(f, "%c", c); + return 1; + } + +if (c < 0x100) + { + if (use_utf) + { + if (f != NULL) fprintf(f, "\\x{%02x}", c); + return 6; + } + else + { + if (f != NULL) fprintf(f, "\\x%02x", c); + return 4; + } + } + +if (f != NULL) n = fprintf(f, "\\x{%02x}", c); + else n = sprintf(tempbuffer, "\\x{%02x}", c); + +return n >= 0 ? n : 0; +} + + + +#ifdef SUPPORT_PCRE8 +/************************************************* +* Print 8-bit character string * +*************************************************/ + +/* Must handle UTF-8 strings in utf8 mode. Yields number of characters printed. +If handed a NULL file, just counts chars without printing. */ + +static int pchars(pcre_uint8 *p, int length, FILE *f) +{ +pcre_uint32 c = 0; +int yield = 0; + +if (length < 0) + length = strlen((char *)p); + +while (length-- > 0) + { +#if !defined NOUTF + if (use_utf) + { + int rc = utf82ord(p, &c); + if (rc > 0 && rc <= length + 1) /* Mustn't run over the end */ + { + length -= rc - 1; + p += rc; + yield += pchar(c, f); + continue; + } + } +#endif + c = *p++; + yield += pchar(c, f); + } + +return yield; +} +#endif + + + +#ifdef SUPPORT_PCRE16 +/************************************************* +* Find length of 0-terminated 16-bit string * +*************************************************/ + +static int strlen16(PCRE_SPTR16 p) +{ +PCRE_SPTR16 pp = p; +while (*pp != 0) pp++; +return (int)(pp - p); +} +#endif /* SUPPORT_PCRE16 */ + + + +#ifdef SUPPORT_PCRE32 +/************************************************* +* Find length of 0-terminated 32-bit string * +*************************************************/ + +static int strlen32(PCRE_SPTR32 p) +{ +PCRE_SPTR32 pp = p; +while (*pp != 0) pp++; +return (int)(pp - p); +} +#endif /* SUPPORT_PCRE32 */ + + + +#ifdef SUPPORT_PCRE16 +/************************************************* +* Print 16-bit character string * +*************************************************/ + +/* Must handle UTF-16 strings in utf mode. Yields number of characters printed. +If handed a NULL file, just counts chars without printing. */ + +static int pchars16(PCRE_SPTR16 p, int length, FILE *f) +{ +int yield = 0; + +if (length < 0) + length = strlen16(p); + +while (length-- > 0) + { + pcre_uint32 c = *p++ & 0xffff; +#if !defined NOUTF + if (use_utf && c >= 0xD800 && c < 0xDC00 && length > 0) + { + int d = *p & 0xffff; + if (d >= 0xDC00 && d <= 0xDFFF) + { + c = ((c & 0x3ff) << 10) + (d & 0x3ff) + 0x10000; + length--; + p++; + } + } +#endif + yield += pchar(c, f); + } + +return yield; +} +#endif /* SUPPORT_PCRE16 */ + + + +#ifdef SUPPORT_PCRE32 +/************************************************* +* Print 32-bit character string * +*************************************************/ + +/* Must handle UTF-32 strings in utf mode. Yields number of characters printed. +If handed a NULL file, just counts chars without printing. */ + +static int pchars32(PCRE_SPTR32 p, int length, BOOL utf, FILE *f) +{ +int yield = 0; + +(void)(utf); /* Avoid compiler warning */ + +if (length < 0) + length = strlen32(p); + +while (length-- > 0) + { + pcre_uint32 c = *p++; + yield += pchar(c, f); + } + +return yield; +} +#endif /* SUPPORT_PCRE32 */ + + + +#ifdef SUPPORT_PCRE8 +/************************************************* +* Read a capture name (8-bit) and check it * +*************************************************/ + +static pcre_uint8 * +read_capture_name8(pcre_uint8 *p, pcre_uint8 **pp, pcre *re) +{ +pcre_uint8 *npp = *pp; +while (isalnum(*p)) *npp++ = *p++; +*npp++ = 0; +*npp = 0; +if (pcre_get_stringnumber(re, (char *)(*pp)) < 0) + { + fprintf(outfile, "no parentheses with name \""); + PCHARSV(*pp, 0, -1, outfile); + fprintf(outfile, "\"\n"); + } + +*pp = npp; +return p; +} +#endif /* SUPPORT_PCRE8 */ + + + +#ifdef SUPPORT_PCRE16 +/************************************************* +* Read a capture name (16-bit) and check it * +*************************************************/ + +/* Note that the text being read is 8-bit. */ + +static pcre_uint8 * +read_capture_name16(pcre_uint8 *p, pcre_uint16 **pp, pcre *re) +{ +pcre_uint16 *npp = *pp; +while (isalnum(*p)) *npp++ = *p++; +*npp++ = 0; +*npp = 0; +if (pcre16_get_stringnumber((pcre16 *)re, (PCRE_SPTR16)(*pp)) < 0) + { + fprintf(outfile, "no parentheses with name \""); + PCHARSV(*pp, 0, -1, outfile); + fprintf(outfile, "\"\n"); + } +*pp = npp; +return p; +} +#endif /* SUPPORT_PCRE16 */ + + + +#ifdef SUPPORT_PCRE32 +/************************************************* +* Read a capture name (32-bit) and check it * +*************************************************/ + +/* Note that the text being read is 8-bit. */ + +static pcre_uint8 * +read_capture_name32(pcre_uint8 *p, pcre_uint32 **pp, pcre *re) +{ +pcre_uint32 *npp = *pp; +while (isalnum(*p)) *npp++ = *p++; +*npp++ = 0; +*npp = 0; +if (pcre32_get_stringnumber((pcre32 *)re, (PCRE_SPTR32)(*pp)) < 0) + { + fprintf(outfile, "no parentheses with name \""); + PCHARSV(*pp, 0, -1, outfile); + fprintf(outfile, "\"\n"); + } +*pp = npp; +return p; +} +#endif /* SUPPORT_PCRE32 */ + + + +/************************************************* +* Stack guard function * +*************************************************/ + +/* Called from PCRE when set in pcre_stack_guard. We give an error (non-zero) +return when a count overflows. */ + +static int stack_guard(void) +{ +return stack_guard_return; +} + +/************************************************* +* Callout function * +*************************************************/ + +/* Called from PCRE as a result of the (?C) item. We print out where we are in +the match. Yield zero unless more callouts than the fail count, or the callout +data is not zero. */ + +static int callout(pcre_callout_block *cb) +{ +FILE *f = (first_callout | callout_extra)? outfile : NULL; +int i, current_position, pre_start, post_start, subject_length; + +if (callout_extra) + { + fprintf(f, "Callout %d: last capture = %d\n", + cb->callout_number, cb->capture_last); + + if (cb->offset_vector != NULL) + { + for (i = 0; i < cb->capture_top * 2; i += 2) + { + if (cb->offset_vector[i] < 0) + fprintf(f, "%2d: \n", i/2); + else + { + fprintf(f, "%2d: ", i/2); + PCHARSV(cb->subject, cb->offset_vector[i], + cb->offset_vector[i+1] - cb->offset_vector[i], f); + fprintf(f, "\n"); + } + } + } + } + +/* Re-print the subject in canonical form, the first time or if giving full +datails. On subsequent calls in the same match, we use pchars just to find the +printed lengths of the substrings. */ + +if (f != NULL) fprintf(f, "--->"); + +/* If a lookbehind is involved, the current position may be earlier than the +match start. If so, use the match start instead. */ + +current_position = (cb->current_position >= cb->start_match)? + cb->current_position : cb->start_match; + +PCHARS(pre_start, cb->subject, 0, cb->start_match, f); +PCHARS(post_start, cb->subject, cb->start_match, + current_position - cb->start_match, f); + +PCHARS(subject_length, cb->subject, 0, cb->subject_length, NULL); + +PCHARSV(cb->subject, current_position, cb->subject_length - current_position, f); + +if (f != NULL) fprintf(f, "\n"); + +/* Always print appropriate indicators, with callout number if not already +shown. For automatic callouts, show the pattern offset. */ + +if (cb->callout_number == 255) + { + fprintf(outfile, "%+3d ", cb->pattern_position); + if (cb->pattern_position > 99) fprintf(outfile, "\n "); + } +else + { + if (callout_extra) fprintf(outfile, " "); + else fprintf(outfile, "%3d ", cb->callout_number); + } + +for (i = 0; i < pre_start; i++) fprintf(outfile, " "); +fprintf(outfile, "^"); + +if (post_start > 0) + { + for (i = 0; i < post_start - 1; i++) fprintf(outfile, " "); + fprintf(outfile, "^"); + } + +for (i = 0; i < subject_length - pre_start - post_start + 4; i++) + fprintf(outfile, " "); + +fprintf(outfile, "%.*s", (cb->next_item_length == 0)? 1 : cb->next_item_length, + pbuffer + cb->pattern_position); + +fprintf(outfile, "\n"); +first_callout = 0; + +if (cb->mark != last_callout_mark) + { + if (cb->mark == NULL) + fprintf(outfile, "Latest Mark: \n"); + else + { + fprintf(outfile, "Latest Mark: "); + PCHARSV(cb->mark, 0, -1, outfile); + putc('\n', outfile); + } + last_callout_mark = cb->mark; + } + +if (cb->callout_data != NULL) + { + int callout_data = *((int *)(cb->callout_data)); + if (callout_data != 0) + { + fprintf(outfile, "Callout data = %d\n", callout_data); + return callout_data; + } + } + +return (cb->callout_number != callout_fail_id)? 0 : + (++callout_count >= callout_fail_count)? 1 : 0; +} + + +/************************************************* +* Local malloc functions * +*************************************************/ + +/* Alternative malloc function, to test functionality and save the size of a +compiled re, which is the first store request that pcre_compile() makes. The +show_malloc variable is set only during matching. */ + +static void *new_malloc(size_t size) +{ +void *block = malloc(size); +if (show_malloc) + fprintf(outfile, "malloc %3d %p\n", (int)size, block); +return block; +} + +static void new_free(void *block) +{ +if (show_malloc) + fprintf(outfile, "free %p\n", block); +free(block); +} + +/* For recursion malloc/free, to test stacking calls */ + +static void *stack_malloc(size_t size) +{ +void *block = malloc(size); +if (show_malloc) + fprintf(outfile, "stack_malloc %3d %p\n", (int)size, block); +return block; +} + +static void stack_free(void *block) +{ +if (show_malloc) + fprintf(outfile, "stack_free %p\n", block); +free(block); +} + + +/************************************************* +* Call pcre_fullinfo() * +*************************************************/ + +/* Get one piece of information from the pcre_fullinfo() function. When only +one of 8-, 16- or 32-bit is supported, pcre_mode should always have the correct +value, but the code is defensive. + +Arguments: + re compiled regex + study study data + option PCRE_INFO_xxx option + ptr where to put the data + +Returns: 0 when OK, < 0 on error +*/ + +static int +new_info(pcre *re, pcre_extra *study, int option, void *ptr) +{ +int rc; + +if (pcre_mode == PCRE32_MODE) +#ifdef SUPPORT_PCRE32 + rc = pcre32_fullinfo((pcre32 *)re, (pcre32_extra *)study, option, ptr); +#else + rc = PCRE_ERROR_BADMODE; +#endif +else if (pcre_mode == PCRE16_MODE) +#ifdef SUPPORT_PCRE16 + rc = pcre16_fullinfo((pcre16 *)re, (pcre16_extra *)study, option, ptr); +#else + rc = PCRE_ERROR_BADMODE; +#endif +else +#ifdef SUPPORT_PCRE8 + rc = pcre_fullinfo(re, study, option, ptr); +#else + rc = PCRE_ERROR_BADMODE; +#endif + +if (rc < 0 && rc != PCRE_ERROR_UNSET) + { + fprintf(outfile, "Error %d from pcre%s_fullinfo(%d)\n", rc, + pcre_mode == PCRE32_MODE ? "32" : pcre_mode == PCRE16_MODE ? "16" : "", option); + if (rc == PCRE_ERROR_BADMODE) + fprintf(outfile, "Running in %d-bit mode but pattern was compiled in " + "%d-bit mode\n", 8 * CHAR_SIZE, + 8 * (REAL_PCRE_FLAGS(re) & PCRE_MODE_MASK)); + } + +return rc; +} + + + +/************************************************* +* Swap byte functions * +*************************************************/ + +/* The following functions swap the bytes of a pcre_uint16 and pcre_uint32 +value, respectively. + +Arguments: + value any number + +Returns: the byte swapped value +*/ + +static pcre_uint32 +swap_uint32(pcre_uint32 value) +{ +return ((value & 0x000000ff) << 24) | + ((value & 0x0000ff00) << 8) | + ((value & 0x00ff0000) >> 8) | + (value >> 24); +} + +static pcre_uint16 +swap_uint16(pcre_uint16 value) +{ +return (value >> 8) | (value << 8); +} + + + +/************************************************* +* Flip bytes in a compiled pattern * +*************************************************/ + +/* This function is called if the 'F' option was present on a pattern that is +to be written to a file. We flip the bytes of all the integer fields in the +regex data block and the study block. In 16-bit mode this also flips relevant +bytes in the pattern itself. This is to make it possible to test PCRE's +ability to reload byte-flipped patterns, e.g. those compiled on a different +architecture. */ + +#if defined SUPPORT_PCRE8 || defined SUPPORT_PCRE16 +static void +regexflip8_or_16(pcre *ere, pcre_extra *extra) +{ +real_pcre8_or_16 *re = (real_pcre8_or_16 *)ere; +#ifdef SUPPORT_PCRE16 +int op; +pcre_uint16 *ptr = (pcre_uint16 *)re + re->name_table_offset; +int length = re->name_count * re->name_entry_size; +#ifdef SUPPORT_UTF +BOOL utf = (re->options & PCRE_UTF16) != 0; +BOOL utf16_char = FALSE; +#endif /* SUPPORT_UTF */ +#endif /* SUPPORT_PCRE16 */ + +/* Always flip the bytes in the main data block and study blocks. */ + +re->magic_number = REVERSED_MAGIC_NUMBER; +re->size = swap_uint32(re->size); +re->options = swap_uint32(re->options); +re->flags = swap_uint32(re->flags); +re->limit_match = swap_uint32(re->limit_match); +re->limit_recursion = swap_uint32(re->limit_recursion); +re->first_char = swap_uint16(re->first_char); +re->req_char = swap_uint16(re->req_char); +re->max_lookbehind = swap_uint16(re->max_lookbehind); +re->top_bracket = swap_uint16(re->top_bracket); +re->top_backref = swap_uint16(re->top_backref); +re->name_table_offset = swap_uint16(re->name_table_offset); +re->name_entry_size = swap_uint16(re->name_entry_size); +re->name_count = swap_uint16(re->name_count); +re->ref_count = swap_uint16(re->ref_count); + +if (extra != NULL && (extra->flags & PCRE_EXTRA_STUDY_DATA) != 0) + { + pcre_study_data *rsd = (pcre_study_data *)(extra->study_data); + rsd->size = swap_uint32(rsd->size); + rsd->flags = swap_uint32(rsd->flags); + rsd->minlength = swap_uint32(rsd->minlength); + } + +/* In 8-bit mode, that is all we need to do. In 16-bit mode we must swap bytes +in the name table, if present, and then in the pattern itself. */ + +#ifdef SUPPORT_PCRE16 +if (pcre_mode != PCRE16_MODE) return; + +while(TRUE) + { + /* Swap previous characters. */ + while (length-- > 0) + { + *ptr = swap_uint16(*ptr); + ptr++; + } +#ifdef SUPPORT_UTF + if (utf16_char) + { + if ((ptr[-1] & 0xfc00) == 0xd800) + { + /* We know that there is only one extra character in UTF-16. */ + *ptr = swap_uint16(*ptr); + ptr++; + } + } + utf16_char = FALSE; +#endif /* SUPPORT_UTF */ + + /* Get next opcode. */ + + length = 0; + op = *ptr; + *ptr++ = swap_uint16(op); + + switch (op) + { + case OP_END: + return; + +#ifdef SUPPORT_UTF + case OP_CHAR: + case OP_CHARI: + case OP_NOT: + case OP_NOTI: + case OP_STAR: + case OP_MINSTAR: + case OP_PLUS: + case OP_MINPLUS: + case OP_QUERY: + case OP_MINQUERY: + case OP_UPTO: + case OP_MINUPTO: + case OP_EXACT: + case OP_POSSTAR: + case OP_POSPLUS: + case OP_POSQUERY: + case OP_POSUPTO: + case OP_STARI: + case OP_MINSTARI: + case OP_PLUSI: + case OP_MINPLUSI: + case OP_QUERYI: + case OP_MINQUERYI: + case OP_UPTOI: + case OP_MINUPTOI: + case OP_EXACTI: + case OP_POSSTARI: + case OP_POSPLUSI: + case OP_POSQUERYI: + case OP_POSUPTOI: + case OP_NOTSTAR: + case OP_NOTMINSTAR: + case OP_NOTPLUS: + case OP_NOTMINPLUS: + case OP_NOTQUERY: + case OP_NOTMINQUERY: + case OP_NOTUPTO: + case OP_NOTMINUPTO: + case OP_NOTEXACT: + case OP_NOTPOSSTAR: + case OP_NOTPOSPLUS: + case OP_NOTPOSQUERY: + case OP_NOTPOSUPTO: + case OP_NOTSTARI: + case OP_NOTMINSTARI: + case OP_NOTPLUSI: + case OP_NOTMINPLUSI: + case OP_NOTQUERYI: + case OP_NOTMINQUERYI: + case OP_NOTUPTOI: + case OP_NOTMINUPTOI: + case OP_NOTEXACTI: + case OP_NOTPOSSTARI: + case OP_NOTPOSPLUSI: + case OP_NOTPOSQUERYI: + case OP_NOTPOSUPTOI: + if (utf) utf16_char = TRUE; +#endif + /* Fall through. */ + + default: + length = OP_lengths16[op] - 1; + break; + + case OP_CLASS: + case OP_NCLASS: + /* Skip the character bit map. */ + ptr += 32/sizeof(pcre_uint16); + length = 0; + break; + + case OP_XCLASS: + /* LINK_SIZE can be 1 or 2 in 16 bit mode. */ + if (LINK_SIZE > 1) + length = (int)((((unsigned int)(ptr[0]) << 16) | (unsigned int)(ptr[1])) + - (1 + LINK_SIZE + 1)); + else + length = (int)((unsigned int)(ptr[0]) - (1 + LINK_SIZE + 1)); + + /* Reverse the size of the XCLASS instance. */ + *ptr = swap_uint16(*ptr); + ptr++; + if (LINK_SIZE > 1) + { + *ptr = swap_uint16(*ptr); + ptr++; + } + + op = *ptr; + *ptr = swap_uint16(op); + ptr++; + if ((op & XCL_MAP) != 0) + { + /* Skip the character bit map. */ + ptr += 32/sizeof(pcre_uint16); + length -= 32/sizeof(pcre_uint16); + } + break; + } + } +/* Control should never reach here in 16 bit mode. */ +#endif /* SUPPORT_PCRE16 */ +} +#endif /* SUPPORT_PCRE[8|16] */ + + + +#if defined SUPPORT_PCRE32 +static void +regexflip_32(pcre *ere, pcre_extra *extra) +{ +real_pcre32 *re = (real_pcre32 *)ere; +int op; +pcre_uint32 *ptr = (pcre_uint32 *)re + re->name_table_offset; +int length = re->name_count * re->name_entry_size; + +/* Always flip the bytes in the main data block and study blocks. */ + +re->magic_number = REVERSED_MAGIC_NUMBER; +re->size = swap_uint32(re->size); +re->options = swap_uint32(re->options); +re->flags = swap_uint32(re->flags); +re->limit_match = swap_uint32(re->limit_match); +re->limit_recursion = swap_uint32(re->limit_recursion); +re->first_char = swap_uint32(re->first_char); +re->req_char = swap_uint32(re->req_char); +re->max_lookbehind = swap_uint16(re->max_lookbehind); +re->top_bracket = swap_uint16(re->top_bracket); +re->top_backref = swap_uint16(re->top_backref); +re->name_table_offset = swap_uint16(re->name_table_offset); +re->name_entry_size = swap_uint16(re->name_entry_size); +re->name_count = swap_uint16(re->name_count); +re->ref_count = swap_uint16(re->ref_count); + +if (extra != NULL && (extra->flags & PCRE_EXTRA_STUDY_DATA) != 0) + { + pcre_study_data *rsd = (pcre_study_data *)(extra->study_data); + rsd->size = swap_uint32(rsd->size); + rsd->flags = swap_uint32(rsd->flags); + rsd->minlength = swap_uint32(rsd->minlength); + } + +/* In 32-bit mode we must swap bytes in the name table, if present, and then in +the pattern itself. */ + +while(TRUE) + { + /* Swap previous characters. */ + while (length-- > 0) + { + *ptr = swap_uint32(*ptr); + ptr++; + } + + /* Get next opcode. */ + + length = 0; + op = *ptr; + *ptr++ = swap_uint32(op); + + switch (op) + { + case OP_END: + return; + + default: + length = OP_lengths32[op] - 1; + break; + + case OP_CLASS: + case OP_NCLASS: + /* Skip the character bit map. */ + ptr += 32/sizeof(pcre_uint32); + length = 0; + break; + + case OP_XCLASS: + /* LINK_SIZE can only be 1 in 32-bit mode. */ + length = (int)((unsigned int)(ptr[0]) - (1 + LINK_SIZE + 1)); + + /* Reverse the size of the XCLASS instance. */ + *ptr = swap_uint32(*ptr); + ptr++; + + op = *ptr; + *ptr = swap_uint32(op); + ptr++; + if ((op & XCL_MAP) != 0) + { + /* Skip the character bit map. */ + ptr += 32/sizeof(pcre_uint32); + length -= 32/sizeof(pcre_uint32); + } + break; + } + } +/* Control should never reach here in 32 bit mode. */ +} + +#endif /* SUPPORT_PCRE32 */ + + + +static void +regexflip(pcre *ere, pcre_extra *extra) +{ +#if defined SUPPORT_PCRE32 + if (REAL_PCRE_FLAGS(ere) & PCRE_MODE32) + regexflip_32(ere, extra); +#endif +#if defined SUPPORT_PCRE8 || defined SUPPORT_PCRE16 + if (REAL_PCRE_FLAGS(ere) & (PCRE_MODE8 | PCRE_MODE16)) + regexflip8_or_16(ere, extra); +#endif +} + + + +/************************************************* +* Check match or recursion limit * +*************************************************/ + +static int +check_match_limit(pcre *re, pcre_extra *extra, pcre_uint8 *bptr, int len, + int start_offset, int options, int *use_offsets, int use_size_offsets, + int flag, unsigned long int *limit, int errnumber, const char *msg) +{ +int count; +int min = 0; +int mid = 64; +int max = -1; + +extra->flags |= flag; + +for (;;) + { + *limit = mid; + + PCRE_EXEC(count, re, extra, bptr, len, start_offset, options, + use_offsets, use_size_offsets); + + if (count == errnumber) + { + /* fprintf(outfile, "Testing %s limit = %d\n", msg, mid); */ + min = mid; + mid = (mid == max - 1)? max : (max > 0)? (min + max)/2 : mid*2; + } + + else if (count >= 0 || count == PCRE_ERROR_NOMATCH || + count == PCRE_ERROR_PARTIAL) + { + if (mid == min + 1) + { + fprintf(outfile, "Minimum %s limit = %d\n", msg, mid); + break; + } + /* fprintf(outfile, "Testing %s limit = %d\n", msg, mid); */ + max = mid; + mid = (min + mid)/2; + } + else break; /* Some other error */ + } + +extra->flags &= ~flag; +return count; +} + + + +/************************************************* +* Case-independent strncmp() function * +*************************************************/ + +/* +Arguments: + s first string + t second string + n number of characters to compare + +Returns: < 0, = 0, or > 0, according to the comparison +*/ + +static int +strncmpic(pcre_uint8 *s, pcre_uint8 *t, int n) +{ +while (n--) + { + int c = tolower(*s++) - tolower(*t++); + if (c) return c; + } +return 0; +} + + + +/************************************************* +* Check multicharacter option * +*************************************************/ + +/* This is used both at compile and run-time to check for escapes. Print +a message and return 0 if there is no match. + +Arguments: + p points after the leading '<' + f file for error message + nl TRUE to check only for newline settings + stype "modifier" or "escape sequence" + +Returns: appropriate PCRE_NEWLINE_xxx flags, or 0 +*/ + +static int +check_mc_option(pcre_uint8 *p, FILE *f, BOOL nl, const char *stype) +{ +if (strncmpic(p, (pcre_uint8 *)"cr>", 3) == 0) return PCRE_NEWLINE_CR; +if (strncmpic(p, (pcre_uint8 *)"lf>", 3) == 0) return PCRE_NEWLINE_LF; +if (strncmpic(p, (pcre_uint8 *)"crlf>", 5) == 0) return PCRE_NEWLINE_CRLF; +if (strncmpic(p, (pcre_uint8 *)"anycrlf>", 8) == 0) return PCRE_NEWLINE_ANYCRLF; +if (strncmpic(p, (pcre_uint8 *)"any>", 4) == 0) return PCRE_NEWLINE_ANY; +if (strncmpic(p, (pcre_uint8 *)"bsr_anycrlf>", 12) == 0) return PCRE_BSR_ANYCRLF; +if (strncmpic(p, (pcre_uint8 *)"bsr_unicode>", 12) == 0) return PCRE_BSR_UNICODE; + +if (!nl) + { + if (strncmpic(p, (pcre_uint8 *)"JS>", 3) == 0) return PCRE_JAVASCRIPT_COMPAT; + } + +fprintf(f, "Unknown %s at: <%s\n", stype, p); +return 0; +} + + + +/************************************************* +* Usage function * +*************************************************/ + +static void +usage(void) +{ +printf("Usage: pcretest [options] [ []]\n\n"); +printf("Input and output default to stdin and stdout.\n"); +#if defined(SUPPORT_LIBREADLINE) || defined(SUPPORT_LIBEDIT) +printf("If input is a terminal, readline() is used to read from it.\n"); +#else +printf("This version of pcretest is not linked with readline().\n"); +#endif +printf("\nOptions:\n"); +#ifdef SUPPORT_PCRE16 +printf(" -16 use the 16-bit library\n"); +#endif +#ifdef SUPPORT_PCRE32 +printf(" -32 use the 32-bit library\n"); +#endif +printf(" -b show compiled code\n"); +printf(" -C show PCRE compile-time options and exit\n"); +printf(" -C arg show a specific compile-time option and exit\n"); +printf(" with its value if numeric (else 0). The arg can be:\n"); +printf(" linksize internal link size [2, 3, 4]\n"); +printf(" pcre8 8 bit library support enabled [0, 1]\n"); +printf(" pcre16 16 bit library support enabled [0, 1]\n"); +printf(" pcre32 32 bit library support enabled [0, 1]\n"); +printf(" utf Unicode Transformation Format supported [0, 1]\n"); +printf(" ucp Unicode Properties supported [0, 1]\n"); +printf(" jit Just-in-time compiler supported [0, 1]\n"); +printf(" newline Newline type [CR, LF, CRLF, ANYCRLF, ANY]\n"); +printf(" bsr \\R type [ANYCRLF, ANY]\n"); +printf(" -d debug: show compiled code and information (-b and -i)\n"); +#if !defined NODFA +printf(" -dfa force DFA matching for all subjects\n"); +#endif +printf(" -help show usage information\n"); +printf(" -i show information about compiled patterns\n" + " -M find MATCH_LIMIT minimum for each subject\n" + " -m output memory used information\n" + " -O set PCRE_NO_AUTO_POSSESS on each pattern\n" + " -o set size of offsets vector to \n"); +#if !defined NOPOSIX +printf(" -p use POSIX interface\n"); +#endif +printf(" -q quiet: do not output PCRE version number at start\n"); +printf(" -S set stack size to megabytes\n"); +printf(" -s force each pattern to be studied at basic level\n" + " -s+ force each pattern to be studied, using JIT if available\n" + " -s++ ditto, verifying when JIT was actually used\n" + " -s+n force each pattern to be studied, using JIT if available,\n" + " where 1 <= n <= 7 selects JIT options\n" + " -s++n ditto, verifying when JIT was actually used\n" + " -t time compilation and execution\n"); +printf(" -t time compilation and execution, repeating times\n"); +printf(" -tm time execution (matching) only\n"); +printf(" -tm time execution (matching) only, repeating times\n"); +printf(" -T same as -t, but show total times at the end\n"); +printf(" -TM same as -tm, but show total time at the end\n"); +} + + + +/************************************************* +* Main Program * +*************************************************/ + +/* Read lines from named file or stdin and write to named file or stdout; lines +consist of a regular expression, in delimiters and optionally followed by +options, followed by a set of test data, terminated by an empty line. */ + +int main(int argc, char **argv) +{ +FILE *infile = stdin; +const char *version; +int options = 0; +int study_options = 0; +int default_find_match_limit = FALSE; +pcre_uint32 default_options = 0; +int op = 1; +int timeit = 0; +int timeitm = 0; +int showtotaltimes = 0; +int showinfo = 0; +int showstore = 0; +int force_study = -1; +int force_study_options = 0; +int quiet = 0; +int size_offsets = 45; +int size_offsets_max; +int *offsets = NULL; +int debug = 0; +int done = 0; +int all_use_dfa = 0; +int verify_jit = 0; +int yield = 0; +int stack_size; +pcre_uint8 *dbuffer = NULL; +pcre_uint8 lockout[24] = { 0 }; +size_t dbuffer_size = 1u << 14; +clock_t total_compile_time = 0; +clock_t total_study_time = 0; +clock_t total_match_time = 0; + +#if !defined NOPOSIX +int posix = 0; +#endif +#if !defined NODFA +int *dfa_workspace = NULL; +#endif + +pcre_jit_stack *jit_stack = NULL; + +/* These vectors store, end-to-end, a list of zero-terminated captured +substring names, each list itself being terminated by an empty name. Assume +that 1024 is plenty long enough for the few names we'll be testing. It is +easiest to keep separate 8-, 16- and 32-bit versions, using the 32-bit version +for the actual memory, to ensure alignment. */ + +pcre_uint32 copynames[1024]; +pcre_uint32 getnames[1024]; + +#ifdef SUPPORT_PCRE32 +pcre_uint32 *cn32ptr; +pcre_uint32 *gn32ptr; +#endif + +#ifdef SUPPORT_PCRE16 +pcre_uint16 *copynames16 = (pcre_uint16 *)copynames; +pcre_uint16 *getnames16 = (pcre_uint16 *)getnames; +pcre_uint16 *cn16ptr; +pcre_uint16 *gn16ptr; +#endif + +#ifdef SUPPORT_PCRE8 +pcre_uint8 *copynames8 = (pcre_uint8 *)copynames; +pcre_uint8 *getnames8 = (pcre_uint8 *)getnames; +pcre_uint8 *cn8ptr; +pcre_uint8 *gn8ptr; +#endif + +/* Get buffers from malloc() so that valgrind will check their misuse when +debugging. They grow automatically when very long lines are read. The 16- +and 32-bit buffers (buffer16, buffer32) are obtained only if needed. */ + +buffer = (pcre_uint8 *)malloc(buffer_size); +pbuffer = (pcre_uint8 *)malloc(buffer_size); + +/* The outfile variable is static so that new_malloc can use it. */ + +outfile = stdout; + +/* The following _setmode() stuff is some Windows magic that tells its runtime +library to translate CRLF into a single LF character. At least, that's what +I've been told: never having used Windows I take this all on trust. Originally +it set 0x8000, but then I was advised that _O_BINARY was better. */ + +#if defined(_WIN32) || defined(WIN32) +_setmode( _fileno( stdout ), _O_BINARY ); +#endif + +/* Get the version number: both pcre_version() and pcre16_version() give the +same answer. We just need to ensure that we call one that is available. */ + +#if defined SUPPORT_PCRE8 +version = pcre_version(); +#elif defined SUPPORT_PCRE16 +version = pcre16_version(); +#elif defined SUPPORT_PCRE32 +version = pcre32_version(); +#endif + +/* Scan options */ + +while (argc > 1 && argv[op][0] == '-') + { + pcre_uint8 *endptr; + char *arg = argv[op]; + + if (strcmp(arg, "-m") == 0) showstore = 1; + else if (strcmp(arg, "-s") == 0) force_study = 0; + + else if (strncmp(arg, "-s+", 3) == 0) + { + arg += 3; + if (*arg == '+') { arg++; verify_jit = TRUE; } + force_study = 1; + if (*arg == 0) + force_study_options = jit_study_bits[6]; + else if (*arg >= '1' && *arg <= '7') + force_study_options = jit_study_bits[*arg - '1']; + else goto BAD_ARG; + } + else if (strcmp(arg, "-8") == 0) + { +#ifdef SUPPORT_PCRE8 + pcre_mode = PCRE8_MODE; +#else + printf("** This version of PCRE was built without 8-bit support\n"); + exit(1); +#endif + } + else if (strcmp(arg, "-16") == 0) + { +#ifdef SUPPORT_PCRE16 + pcre_mode = PCRE16_MODE; +#else + printf("** This version of PCRE was built without 16-bit support\n"); + exit(1); +#endif + } + else if (strcmp(arg, "-32") == 0) + { +#ifdef SUPPORT_PCRE32 + pcre_mode = PCRE32_MODE; +#else + printf("** This version of PCRE was built without 32-bit support\n"); + exit(1); +#endif + } + else if (strcmp(arg, "-q") == 0) quiet = 1; + else if (strcmp(arg, "-b") == 0) debug = 1; + else if (strcmp(arg, "-i") == 0) showinfo = 1; + else if (strcmp(arg, "-d") == 0) showinfo = debug = 1; + else if (strcmp(arg, "-M") == 0) default_find_match_limit = TRUE; + else if (strcmp(arg, "-O") == 0) default_options |= PCRE_NO_AUTO_POSSESS; +#if !defined NODFA + else if (strcmp(arg, "-dfa") == 0) all_use_dfa = 1; +#endif + else if (strcmp(arg, "-o") == 0 && argc > 2 && + ((size_offsets = get_value((pcre_uint8 *)argv[op+1], &endptr)), + *endptr == 0)) + { + op++; + argc--; + } + else if (strcmp(arg, "-t") == 0 || strcmp(arg, "-tm") == 0 || + strcmp(arg, "-T") == 0 || strcmp(arg, "-TM") == 0) + { + int temp; + int both = arg[2] == 0; + showtotaltimes = arg[1] == 'T'; + if (argc > 2 && (temp = get_value((pcre_uint8 *)argv[op+1], &endptr), + *endptr == 0)) + { + timeitm = temp; + op++; + argc--; + } + else timeitm = LOOPREPEAT; + if (both) timeit = timeitm; + } + else if (strcmp(arg, "-S") == 0 && argc > 2 && + ((stack_size = get_value((pcre_uint8 *)argv[op+1], &endptr)), + *endptr == 0)) + { +#if defined(_WIN32) || defined(WIN32) || defined(__minix) || defined(NATIVE_ZOS) || defined(__VMS) + printf("PCRE: -S not supported on this OS\n"); + exit(1); +#else + int rc; + struct rlimit rlim; + getrlimit(RLIMIT_STACK, &rlim); + rlim.rlim_cur = stack_size * 1024 * 1024; + rc = setrlimit(RLIMIT_STACK, &rlim); + if (rc != 0) + { + printf("PCRE: setrlimit() failed with error %d\n", rc); + exit(1); + } + op++; + argc--; +#endif + } +#if !defined NOPOSIX + else if (strcmp(arg, "-p") == 0) posix = 1; +#endif + else if (strcmp(arg, "-C") == 0) + { + int rc; + unsigned long int lrc; + + if (argc > 2) + { + if (strcmp(argv[op + 1], "linksize") == 0) + { + (void)PCRE_CONFIG(PCRE_CONFIG_LINK_SIZE, &rc); + printf("%d\n", rc); + yield = rc; + +#ifdef __VMS + vms_setsymbol("LINKSIZE",0,yield ); +#endif + } + else if (strcmp(argv[op + 1], "pcre8") == 0) + { +#ifdef SUPPORT_PCRE8 + printf("1\n"); + yield = 1; +#else + printf("0\n"); + yield = 0; +#endif +#ifdef __VMS + vms_setsymbol("PCRE8",0,yield ); +#endif + } + else if (strcmp(argv[op + 1], "pcre16") == 0) + { +#ifdef SUPPORT_PCRE16 + printf("1\n"); + yield = 1; +#else + printf("0\n"); + yield = 0; +#endif +#ifdef __VMS + vms_setsymbol("PCRE16",0,yield ); +#endif + } + else if (strcmp(argv[op + 1], "pcre32") == 0) + { +#ifdef SUPPORT_PCRE32 + printf("1\n"); + yield = 1; +#else + printf("0\n"); + yield = 0; +#endif +#ifdef __VMS + vms_setsymbol("PCRE32",0,yield ); +#endif + } + else if (strcmp(argv[op + 1], "utf") == 0) + { +#ifdef SUPPORT_PCRE8 + if (pcre_mode == PCRE8_MODE) + (void)pcre_config(PCRE_CONFIG_UTF8, &rc); +#endif +#ifdef SUPPORT_PCRE16 + if (pcre_mode == PCRE16_MODE) + (void)pcre16_config(PCRE_CONFIG_UTF16, &rc); +#endif +#ifdef SUPPORT_PCRE32 + if (pcre_mode == PCRE32_MODE) + (void)pcre32_config(PCRE_CONFIG_UTF32, &rc); +#endif + printf("%d\n", rc); + yield = rc; +#ifdef __VMS + vms_setsymbol("UTF",0,yield ); +#endif + } + else if (strcmp(argv[op + 1], "ucp") == 0) + { + (void)PCRE_CONFIG(PCRE_CONFIG_UNICODE_PROPERTIES, &rc); + printf("%d\n", rc); + yield = rc; + } + else if (strcmp(argv[op + 1], "jit") == 0) + { + (void)PCRE_CONFIG(PCRE_CONFIG_JIT, &rc); + printf("%d\n", rc); + yield = rc; + } + else if (strcmp(argv[op + 1], "newline") == 0) + { + (void)PCRE_CONFIG(PCRE_CONFIG_NEWLINE, &rc); + print_newline_config(rc, TRUE); + } + else if (strcmp(argv[op + 1], "bsr") == 0) + { + (void)PCRE_CONFIG(PCRE_CONFIG_BSR, &rc); + printf("%s\n", rc? "ANYCRLF" : "ANY"); + } + else if (strcmp(argv[op + 1], "ebcdic") == 0) + { +#ifdef EBCDIC + printf("1\n"); + yield = 1; +#else + printf("0\n"); +#endif + } + else if (strcmp(argv[op + 1], "ebcdic-nl") == 0) + { +#ifdef EBCDIC + printf("0x%02x\n", CHAR_LF); +#else + printf("0\n"); +#endif + } + else + { + printf("Unknown -C option: %s\n", argv[op + 1]); + } + goto EXIT; + } + + /* No argument for -C: output all configuration information. */ + + printf("PCRE version %s\n", version); + printf("Compiled with\n"); + +#ifdef EBCDIC + printf(" EBCDIC code support: LF is 0x%02x\n", CHAR_LF); +#endif + +/* At least one of SUPPORT_PCRE8 and SUPPORT_PCRE16 will be set. If both +are set, either both UTFs are supported or both are not supported. */ + +#ifdef SUPPORT_PCRE8 + printf(" 8-bit support\n"); + (void)pcre_config(PCRE_CONFIG_UTF8, &rc); + printf (" %sUTF-8 support\n", rc ? "" : "No "); +#endif +#ifdef SUPPORT_PCRE16 + printf(" 16-bit support\n"); + (void)pcre16_config(PCRE_CONFIG_UTF16, &rc); + printf (" %sUTF-16 support\n", rc ? "" : "No "); +#endif +#ifdef SUPPORT_PCRE32 + printf(" 32-bit support\n"); + (void)pcre32_config(PCRE_CONFIG_UTF32, &rc); + printf (" %sUTF-32 support\n", rc ? "" : "No "); +#endif + + (void)PCRE_CONFIG(PCRE_CONFIG_UNICODE_PROPERTIES, &rc); + printf(" %sUnicode properties support\n", rc? "" : "No "); + (void)PCRE_CONFIG(PCRE_CONFIG_JIT, &rc); + if (rc) + { + const char *arch; + (void)PCRE_CONFIG(PCRE_CONFIG_JITTARGET, (void *)(&arch)); + printf(" Just-in-time compiler support: %s\n", arch); + } + else + printf(" No just-in-time compiler support\n"); + (void)PCRE_CONFIG(PCRE_CONFIG_NEWLINE, &rc); + print_newline_config(rc, FALSE); + (void)PCRE_CONFIG(PCRE_CONFIG_BSR, &rc); + printf(" \\R matches %s\n", rc? "CR, LF, or CRLF only" : + "all Unicode newlines"); + (void)PCRE_CONFIG(PCRE_CONFIG_LINK_SIZE, &rc); + printf(" Internal link size = %d\n", rc); + (void)PCRE_CONFIG(PCRE_CONFIG_POSIX_MALLOC_THRESHOLD, &rc); + printf(" POSIX malloc threshold = %d\n", rc); + (void)PCRE_CONFIG(PCRE_CONFIG_PARENS_LIMIT, &lrc); + printf(" Parentheses nest limit = %ld\n", lrc); + (void)PCRE_CONFIG(PCRE_CONFIG_MATCH_LIMIT, &lrc); + printf(" Default match limit = %ld\n", lrc); + (void)PCRE_CONFIG(PCRE_CONFIG_MATCH_LIMIT_RECURSION, &lrc); + printf(" Default recursion depth limit = %ld\n", lrc); + (void)PCRE_CONFIG(PCRE_CONFIG_STACKRECURSE, &rc); + printf(" Match recursion uses %s", rc? "stack" : "heap"); + if (showstore) + { + PCRE_EXEC(stack_size, NULL, NULL, NULL, -999, -999, 0, NULL, 0); + printf(": %sframe size = %d bytes", rc? "approximate " : "", -stack_size); + } + printf("\n"); + goto EXIT; + } + else if (strcmp(arg, "-help") == 0 || + strcmp(arg, "--help") == 0) + { + usage(); + goto EXIT; + } + else + { + BAD_ARG: + printf("** Unknown or malformed option %s\n", arg); + usage(); + yield = 1; + goto EXIT; + } + op++; + argc--; + } + +/* Get the store for the offsets vector, and remember what it was */ + +size_offsets_max = size_offsets; +offsets = (int *)malloc(size_offsets_max * sizeof(int)); +if (offsets == NULL) + { + printf("** Failed to get %d bytes of memory for offsets vector\n", + (int)(size_offsets_max * sizeof(int))); + yield = 1; + goto EXIT; + } + +/* Sort out the input and output files */ + +if (argc > 1) + { + infile = fopen(argv[op], INPUT_MODE); + if (infile == NULL) + { + printf("** Failed to open %s\n", argv[op]); + yield = 1; + goto EXIT; + } + } + +if (argc > 2) + { + outfile = fopen(argv[op+1], OUTPUT_MODE); + if (outfile == NULL) + { + printf("** Failed to open %s\n", argv[op+1]); + yield = 1; + goto EXIT; + } + } + +/* Set alternative malloc function */ + +#ifdef SUPPORT_PCRE8 +pcre_malloc = new_malloc; +pcre_free = new_free; +pcre_stack_malloc = stack_malloc; +pcre_stack_free = stack_free; +#endif + +#ifdef SUPPORT_PCRE16 +pcre16_malloc = new_malloc; +pcre16_free = new_free; +pcre16_stack_malloc = stack_malloc; +pcre16_stack_free = stack_free; +#endif + +#ifdef SUPPORT_PCRE32 +pcre32_malloc = new_malloc; +pcre32_free = new_free; +pcre32_stack_malloc = stack_malloc; +pcre32_stack_free = stack_free; +#endif + +/* Heading line unless quiet */ + +if (!quiet) fprintf(outfile, "PCRE version %s\n\n", version); + +/* Main loop */ + +while (!done) + { + pcre *re = NULL; + pcre_extra *extra = NULL; + +#if !defined NOPOSIX /* There are still compilers that require no indent */ + regex_t preg = { NULL, 0, 0} ; + int do_posix = 0; +#endif + + const char *error; + pcre_uint8 *markptr; + pcre_uint8 *p, *pp, *ppp; + pcre_uint8 *to_file = NULL; + const pcre_uint8 *tables = NULL; + unsigned long int get_options; + unsigned long int true_size, true_study_size = 0; + size_t size; + int do_allcaps = 0; + int do_mark = 0; + int do_study = 0; + int no_force_study = 0; + int do_debug = debug; + int do_G = 0; + int do_g = 0; + int do_showinfo = showinfo; + int do_showrest = 0; + int do_showcaprest = 0; + int do_flip = 0; + int erroroffset, len, delimiter, poffset; + +#if !defined NODFA + int dfa_matched = 0; +#endif + + use_utf = 0; + debug_lengths = 1; + SET_PCRE_STACK_GUARD(NULL); + + if (extend_inputline(infile, buffer, " re> ") == NULL) break; + if (infile != stdin) fprintf(outfile, "%s", (char *)buffer); + fflush(outfile); + + p = buffer; + while (isspace(*p)) p++; + if (*p == 0) continue; + + /* Handle option lock-out setting */ + + if (*p == '<' && p[1] == ' ') + { + p += 2; + while (isspace(*p)) p++; + if (strncmp((char *)p, "forbid ", 7) == 0) + { + p += 7; + while (isspace(*p)) p++; + pp = lockout; + while (!isspace(*p) && pp < lockout + sizeof(lockout) - 1) + *pp++ = *p++; + *pp = 0; + } + else + { + printf("** Unrecognized special command '%s'\n", p); + yield = 1; + goto EXIT; + } + continue; + } + + /* See if the pattern is to be loaded pre-compiled from a file. */ + + if (*p == '<' && strchr((char *)(p+1), '<') == NULL) + { + pcre_uint32 magic; + pcre_uint8 sbuf[8]; + FILE *f; + + p++; + if (*p == '!') + { + do_debug = TRUE; + do_showinfo = TRUE; + p++; + } + + pp = p + (int)strlen((char *)p); + while (isspace(pp[-1])) pp--; + *pp = 0; + + f = fopen((char *)p, "rb"); + if (f == NULL) + { + fprintf(outfile, "Failed to open %s: %s\n", p, strerror(errno)); + continue; + } + if (fread(sbuf, 1, 8, f) != 8) goto FAIL_READ; + + true_size = + (sbuf[0] << 24) | (sbuf[1] << 16) | (sbuf[2] << 8) | sbuf[3]; + true_study_size = + (sbuf[4] << 24) | (sbuf[5] << 16) | (sbuf[6] << 8) | sbuf[7]; + + re = (pcre *)new_malloc(true_size); + if (re == NULL) + { + printf("** Failed to get %d bytes of memory for pcre object\n", + (int)true_size); + yield = 1; + goto EXIT; + } + if (fread(re, 1, true_size, f) != true_size) goto FAIL_READ; + + magic = REAL_PCRE_MAGIC(re); + if (magic != MAGIC_NUMBER) + { + if (swap_uint32(magic) == MAGIC_NUMBER) + { + do_flip = 1; + } + else + { + fprintf(outfile, "Data in %s is not a compiled PCRE regex\n", p); + new_free(re); + fclose(f); + continue; + } + } + + /* We hide the byte-invert info for little and big endian tests. */ + fprintf(outfile, "Compiled pattern%s loaded from %s\n", + do_flip && (p[-1] == '<') ? " (byte-inverted)" : "", p); + + /* Now see if there is any following study data. */ + + if (true_study_size != 0) + { + pcre_study_data *psd; + + extra = (pcre_extra *)new_malloc(sizeof(pcre_extra) + true_study_size); + extra->flags = PCRE_EXTRA_STUDY_DATA; + + psd = (pcre_study_data *)(((char *)extra) + sizeof(pcre_extra)); + extra->study_data = psd; + + if (fread(psd, 1, true_study_size, f) != true_study_size) + { + FAIL_READ: + fprintf(outfile, "Failed to read data from %s\n", p); + if (extra != NULL) + { + PCRE_FREE_STUDY(extra); + } + new_free(re); + fclose(f); + continue; + } + fprintf(outfile, "Study data loaded from %s\n", p); + do_study = 1; /* To get the data output if requested */ + } + else fprintf(outfile, "No study data\n"); + + /* Flip the necessary bytes. */ + if (do_flip) + { + int rc; + PCRE_PATTERN_TO_HOST_BYTE_ORDER(rc, re, extra, NULL); + if (rc == PCRE_ERROR_BADMODE) + { + pcre_uint32 flags_in_host_byte_order; + if (REAL_PCRE_MAGIC(re) == MAGIC_NUMBER) + flags_in_host_byte_order = REAL_PCRE_FLAGS(re); + else + flags_in_host_byte_order = swap_uint32(REAL_PCRE_FLAGS(re)); + /* Simulate the result of the function call below. */ + fprintf(outfile, "Error %d from pcre%s_fullinfo(%d)\n", rc, + pcre_mode == PCRE32_MODE ? "32" : pcre_mode == PCRE16_MODE ? "16" : "", + PCRE_INFO_OPTIONS); + fprintf(outfile, "Running in %d-bit mode but pattern was compiled in " + "%d-bit mode\n", 8 * CHAR_SIZE, 8 * (flags_in_host_byte_order & PCRE_MODE_MASK)); + new_free(re); + fclose(f); + continue; + } + } + + /* Need to know if UTF-8 for printing data strings. */ + + if (new_info(re, NULL, PCRE_INFO_OPTIONS, &get_options) < 0) + { + new_free(re); + fclose(f); + continue; + } + use_utf = (get_options & PCRE_UTF8) != 0; + + fclose(f); + goto SHOW_INFO; + } + + /* In-line pattern (the usual case). Get the delimiter and seek the end of + the pattern; if it isn't complete, read more. */ + + delimiter = *p++; + + if (isalnum(delimiter) || delimiter == '\\') + { + fprintf(outfile, "** Delimiter must not be alphanumeric or \\\n"); + goto SKIP_DATA; + } + + pp = p; + poffset = (int)(p - buffer); + + for(;;) + { + while (*pp != 0) + { + if (*pp == '\\' && pp[1] != 0) pp++; + else if (*pp == delimiter) break; + pp++; + } + if (*pp != 0) break; + if ((pp = extend_inputline(infile, pp, " > ")) == NULL) + { + fprintf(outfile, "** Unexpected EOF\n"); + done = 1; + goto CONTINUE; + } + if (infile != stdin) fprintf(outfile, "%s", (char *)pp); + } + + /* The buffer may have moved while being extended; reset the start of data + pointer to the correct relative point in the buffer. */ + + p = buffer + poffset; + + /* If the first character after the delimiter is backslash, make + the pattern end with backslash. This is purely to provide a way + of testing for the error message when a pattern ends with backslash. */ + + if (pp[1] == '\\') *pp++ = '\\'; + + /* Terminate the pattern at the delimiter, and save a copy of the pattern + for callouts. */ + + *pp++ = 0; + strcpy((char *)pbuffer, (char *)p); + + /* Look for modifiers and options after the final delimiter. */ + + options = default_options; + study_options = force_study_options; + log_store = showstore; /* default from command line */ + + while (*pp != 0) + { + /* Check to see whether this modifier has been locked out for this file. + This is complicated for the multi-character options that begin with '<'. + If there is no '>' in the lockout string, all multi-character modifiers are + locked out. */ + + if (strchr((char *)lockout, *pp) != NULL) + { + if (*pp == '<' && strchr((char *)lockout, '>') != NULL) + { + int x = check_mc_option(pp+1, outfile, FALSE, "modifier"); + if (x == 0) goto SKIP_DATA; + + for (ppp = lockout; *ppp != 0; ppp++) + { + if (*ppp == '<') + { + int y = check_mc_option(ppp+1, outfile, FALSE, "modifier"); + if (y == 0) + { + printf("** Error in modifier forbid data - giving up.\n"); + yield = 1; + goto EXIT; + } + if (x == y) + { + ppp = pp; + while (*ppp != '>') ppp++; + printf("** The %.*s modifier is locked out - giving up.\n", + (int)(ppp - pp + 1), pp); + yield = 1; + goto EXIT; + } + } + } + } + + /* The single-character modifiers are straightforward. */ + + else + { + printf("** The /%c modifier is locked out - giving up.\n", *pp); + yield = 1; + goto EXIT; + } + } + + /* The modifier is not locked out; handle it. */ + + switch (*pp++) + { + case 'f': options |= PCRE_FIRSTLINE; break; + case 'g': do_g = 1; break; + case 'i': options |= PCRE_CASELESS; break; + case 'm': options |= PCRE_MULTILINE; break; + case 's': options |= PCRE_DOTALL; break; + case 'x': options |= PCRE_EXTENDED; break; + + case '+': + if (do_showrest) do_showcaprest = 1; else do_showrest = 1; + break; + + case '=': do_allcaps = 1; break; + case 'A': options |= PCRE_ANCHORED; break; + case 'B': do_debug = 1; break; + case 'C': options |= PCRE_AUTO_CALLOUT; break; + case 'D': do_debug = do_showinfo = 1; break; + case 'E': options |= PCRE_DOLLAR_ENDONLY; break; + case 'F': do_flip = 1; break; + case 'G': do_G = 1; break; + case 'I': do_showinfo = 1; break; + case 'J': options |= PCRE_DUPNAMES; break; + case 'K': do_mark = 1; break; + case 'M': log_store = 1; break; + case 'N': options |= PCRE_NO_AUTO_CAPTURE; break; + case 'O': options |= PCRE_NO_AUTO_POSSESS; break; + +#if !defined NOPOSIX + case 'P': do_posix = 1; break; +#endif + + case 'Q': + switch (*pp) + { + case '0': + case '1': + stack_guard_return = *pp++ - '0'; + break; + + default: + fprintf(outfile, "** Missing 0 or 1 after /Q\n"); + goto SKIP_DATA; + } + SET_PCRE_STACK_GUARD(stack_guard); + break; + + case 'S': + do_study = 1; + for (;;) + { + switch (*pp++) + { + case 'S': + do_study = 0; + no_force_study = 1; + break; + + case '!': + study_options |= PCRE_STUDY_EXTRA_NEEDED; + break; + + case '+': + if (*pp == '+') + { + verify_jit = TRUE; + pp++; + } + if (*pp >= '1' && *pp <= '7') + study_options |= jit_study_bits[*pp++ - '1']; + else + study_options |= jit_study_bits[6]; + break; + + case '-': + study_options &= ~PCRE_STUDY_ALLJIT; + break; + + default: + pp--; + goto ENDLOOP; + } + } + ENDLOOP: + break; + + case 'U': options |= PCRE_UNGREEDY; break; + case 'W': options |= PCRE_UCP; break; + case 'X': options |= PCRE_EXTRA; break; + case 'Y': options |= PCRE_NO_START_OPTIMISE; break; + case 'Z': debug_lengths = 0; break; + case '8': options |= PCRE_UTF8; use_utf = 1; break; + case '9': options |= PCRE_NEVER_UTF; break; + case '?': options |= PCRE_NO_UTF8_CHECK; break; + + case 'T': + switch (*pp++) + { + case '0': tables = tables0; break; + case '1': tables = tables1; break; + + case '\r': + case '\n': + case ' ': + case 0: + fprintf(outfile, "** Missing table number after /T\n"); + goto SKIP_DATA; + + default: + fprintf(outfile, "** Bad table number \"%c\" after /T\n", pp[-1]); + goto SKIP_DATA; + } + break; + + case 'L': + ppp = pp; + /* The '\r' test here is so that it works on Windows. */ + /* The '0' test is just in case this is an unterminated line. */ + while (*ppp != 0 && *ppp != '\n' && *ppp != '\r' && *ppp != ' ') ppp++; + *ppp = 0; + if (setlocale(LC_CTYPE, (const char *)pp) == NULL) + { + fprintf(outfile, "** Failed to set locale \"%s\"\n", pp); + goto SKIP_DATA; + } + locale_set = 1; + tables = PCRE_MAKETABLES; + pp = ppp; + break; + + case '>': + to_file = pp; + while (*pp != 0) pp++; + while (isspace(pp[-1])) pp--; + *pp = 0; + break; + + case '<': + { + int x = check_mc_option(pp, outfile, FALSE, "modifier"); + if (x == 0) goto SKIP_DATA; + options |= x; + while (*pp++ != '>'); + } + break; + + case '\r': /* So that it works in Windows */ + case '\n': + case ' ': + break; + + default: + fprintf(outfile, "** Unknown modifier '%c'\n", pp[-1]); + goto SKIP_DATA; + } + } + + /* Handle compiling via the POSIX interface, which doesn't support the + timing, showing, or debugging options, nor the ability to pass over + local character tables. Neither does it have 16-bit support. */ + +#if !defined NOPOSIX + if (posix || do_posix) + { + int rc; + int cflags = 0; + + if ((options & PCRE_CASELESS) != 0) cflags |= REG_ICASE; + if ((options & PCRE_MULTILINE) != 0) cflags |= REG_NEWLINE; + if ((options & PCRE_DOTALL) != 0) cflags |= REG_DOTALL; + if ((options & PCRE_NO_AUTO_CAPTURE) != 0) cflags |= REG_NOSUB; + if ((options & PCRE_UTF8) != 0) cflags |= REG_UTF8; + if ((options & PCRE_UCP) != 0) cflags |= REG_UCP; + if ((options & PCRE_UNGREEDY) != 0) cflags |= REG_UNGREEDY; + + rc = regcomp(&preg, (char *)p, cflags); + + /* Compilation failed; go back for another re, skipping to blank line + if non-interactive. */ + + if (rc != 0) + { + (void)regerror(rc, &preg, (char *)buffer, buffer_size); + fprintf(outfile, "Failed: POSIX code %d: %s\n", rc, buffer); + goto SKIP_DATA; + } + } + + /* Handle compiling via the native interface */ + + else +#endif /* !defined NOPOSIX */ + + { + /* In 16- or 32-bit mode, convert the input. */ + +#ifdef SUPPORT_PCRE16 + if (pcre_mode == PCRE16_MODE) + { + switch(to16(FALSE, p, options & PCRE_UTF8, (int)strlen((char *)p))) + { + case -1: + fprintf(outfile, "**Failed: invalid UTF-8 string cannot be " + "converted to UTF-16\n"); + goto SKIP_DATA; + + case -2: + fprintf(outfile, "**Failed: character value greater than 0x10ffff " + "cannot be converted to UTF-16\n"); + goto SKIP_DATA; + + case -3: /* "Impossible error" when to16 is called arg1 FALSE */ + fprintf(outfile, "**Failed: character value greater than 0xffff " + "cannot be converted to 16-bit in non-UTF mode\n"); + goto SKIP_DATA; + + default: + break; + } + p = (pcre_uint8 *)buffer16; + } +#endif + +#ifdef SUPPORT_PCRE32 + if (pcre_mode == PCRE32_MODE) + { + switch(to32(FALSE, p, options & PCRE_UTF32, (int)strlen((char *)p))) + { + case -1: + fprintf(outfile, "**Failed: invalid UTF-8 string cannot be " + "converted to UTF-32\n"); + goto SKIP_DATA; + + case -2: + fprintf(outfile, "**Failed: character value greater than 0x10ffff " + "cannot be converted to UTF-32\n"); + goto SKIP_DATA; + + case -3: + fprintf(outfile, "**Failed: character value is ill-formed UTF-32\n"); + goto SKIP_DATA; + + default: + break; + } + p = (pcre_uint8 *)buffer32; + } +#endif + + /* Compile many times when timing */ + + if (timeit > 0) + { + register int i; + clock_t time_taken; + clock_t start_time = clock(); + for (i = 0; i < timeit; i++) + { + PCRE_COMPILE(re, p, options, &error, &erroroffset, tables); + if (re != NULL) free(re); + } + total_compile_time += (time_taken = clock() - start_time); + fprintf(outfile, "Compile time %.4f milliseconds\n", + (((double)time_taken * 1000.0) / (double)timeit) / + (double)CLOCKS_PER_SEC); + } + + PCRE_COMPILE(re, p, options, &error, &erroroffset, tables); + + /* Compilation failed; go back for another re, skipping to blank line + if non-interactive. */ + + if (re == NULL) + { + fprintf(outfile, "Failed: %s at offset %d\n", error, erroroffset); + SKIP_DATA: + if (infile != stdin) + { + for (;;) + { + if (extend_inputline(infile, buffer, NULL) == NULL) + { + done = 1; + goto CONTINUE; + } + len = (int)strlen((char *)buffer); + while (len > 0 && isspace(buffer[len-1])) len--; + if (len == 0) break; + } + fprintf(outfile, "\n"); + } + goto CONTINUE; + } + + /* Compilation succeeded. It is now possible to set the UTF-8 option from + within the regex; check for this so that we know how to process the data + lines. */ + + if (new_info(re, NULL, PCRE_INFO_OPTIONS, &get_options) < 0) + goto SKIP_DATA; + if ((get_options & PCRE_UTF8) != 0) use_utf = 1; + + /* Extract the size for possible writing before possibly flipping it, + and remember the store that was got. */ + + true_size = REAL_PCRE_SIZE(re); + + /* Output code size information if requested */ + + if (log_store) + { + int name_count, name_entry_size, real_pcre_size; + + new_info(re, NULL, PCRE_INFO_NAMECOUNT, &name_count); + new_info(re, NULL, PCRE_INFO_NAMEENTRYSIZE, &name_entry_size); + real_pcre_size = 0; +#ifdef SUPPORT_PCRE8 + if (REAL_PCRE_FLAGS(re) & PCRE_MODE8) + real_pcre_size = sizeof(real_pcre); +#endif +#ifdef SUPPORT_PCRE16 + if (REAL_PCRE_FLAGS(re) & PCRE_MODE16) + real_pcre_size = sizeof(real_pcre16); +#endif +#ifdef SUPPORT_PCRE32 + if (REAL_PCRE_FLAGS(re) & PCRE_MODE32) + real_pcre_size = sizeof(real_pcre32); +#endif + new_info(re, NULL, PCRE_INFO_SIZE, &size); + fprintf(outfile, "Memory allocation (code space): %d\n", + (int)(size - real_pcre_size - name_count * name_entry_size)); + } + + /* If -s or /S was present, study the regex to generate additional info to + help with the matching, unless the pattern has the SS option, which + suppresses the effect of /S (used for a few test patterns where studying is + never sensible). */ + + if (do_study || (force_study >= 0 && !no_force_study)) + { + if (timeit > 0) + { + register int i; + clock_t time_taken; + clock_t start_time = clock(); + for (i = 0; i < timeit; i++) + { + PCRE_STUDY(extra, re, study_options, &error); + } + total_study_time = (time_taken = clock() - start_time); + if (extra != NULL) + { + PCRE_FREE_STUDY(extra); + } + fprintf(outfile, " Study time %.4f milliseconds\n", + (((double)time_taken * 1000.0) / (double)timeit) / + (double)CLOCKS_PER_SEC); + } + PCRE_STUDY(extra, re, study_options, &error); + if (error != NULL) + fprintf(outfile, "Failed to study: %s\n", error); + else if (extra != NULL) + { + true_study_size = ((pcre_study_data *)(extra->study_data))->size; + if (log_store) + { + size_t jitsize; + if (new_info(re, extra, PCRE_INFO_JITSIZE, &jitsize) == 0 && + jitsize != 0) + fprintf(outfile, "Memory allocation (JIT code): %d\n", (int)jitsize); + } + } + } + + /* If /K was present, we set up for handling MARK data. */ + + if (do_mark) + { + if (extra == NULL) + { + extra = (pcre_extra *)malloc(sizeof(pcre_extra)); + extra->flags = 0; + } + extra->mark = &markptr; + extra->flags |= PCRE_EXTRA_MARK; + } + + /* Extract and display information from the compiled data if required. */ + + SHOW_INFO: + + if (do_debug) + { + fprintf(outfile, "------------------------------------------------------------------\n"); + PCRE_PRINTINT(re, outfile, debug_lengths); + } + + /* We already have the options in get_options (see above) */ + + if (do_showinfo) + { + unsigned long int all_options; + pcre_uint32 first_char, need_char; + pcre_uint32 match_limit, recursion_limit; + int count, backrefmax, first_char_set, need_char_set, okpartial, jchanged, + hascrorlf, maxlookbehind, match_empty; + int nameentrysize, namecount; + const pcre_uint8 *nametable; + + if (new_info(re, NULL, PCRE_INFO_CAPTURECOUNT, &count) + + new_info(re, NULL, PCRE_INFO_BACKREFMAX, &backrefmax) + + new_info(re, NULL, PCRE_INFO_FIRSTCHARACTER, &first_char) + + new_info(re, NULL, PCRE_INFO_FIRSTCHARACTERFLAGS, &first_char_set) + + new_info(re, NULL, PCRE_INFO_REQUIREDCHAR, &need_char) + + new_info(re, NULL, PCRE_INFO_REQUIREDCHARFLAGS, &need_char_set) + + new_info(re, NULL, PCRE_INFO_NAMEENTRYSIZE, &nameentrysize) + + new_info(re, NULL, PCRE_INFO_NAMECOUNT, &namecount) + + new_info(re, NULL, PCRE_INFO_NAMETABLE, (void *)&nametable) + + new_info(re, NULL, PCRE_INFO_OKPARTIAL, &okpartial) + + new_info(re, NULL, PCRE_INFO_JCHANGED, &jchanged) + + new_info(re, NULL, PCRE_INFO_HASCRORLF, &hascrorlf) + + new_info(re, NULL, PCRE_INFO_MATCH_EMPTY, &match_empty) + + new_info(re, NULL, PCRE_INFO_MAXLOOKBEHIND, &maxlookbehind) + != 0) + goto SKIP_DATA; + + fprintf(outfile, "Capturing subpattern count = %d\n", count); + + if (backrefmax > 0) + fprintf(outfile, "Max back reference = %d\n", backrefmax); + + if (maxlookbehind > 0) + fprintf(outfile, "Max lookbehind = %d\n", maxlookbehind); + + if (new_info(re, NULL, PCRE_INFO_MATCHLIMIT, &match_limit) == 0) + fprintf(outfile, "Match limit = %u\n", match_limit); + + if (new_info(re, NULL, PCRE_INFO_RECURSIONLIMIT, &recursion_limit) == 0) + fprintf(outfile, "Recursion limit = %u\n", recursion_limit); + + if (namecount > 0) + { + fprintf(outfile, "Named capturing subpatterns:\n"); + while (namecount-- > 0) + { + int imm2_size = pcre_mode == PCRE8_MODE ? 2 : 1; + int length = (int)STRLEN(nametable + imm2_size); + fprintf(outfile, " "); + PCHARSV(nametable, imm2_size, length, outfile); + while (length++ < nameentrysize - imm2_size) putc(' ', outfile); +#ifdef SUPPORT_PCRE32 + if (pcre_mode == PCRE32_MODE) + fprintf(outfile, "%3d\n", (int)(((PCRE_SPTR32)nametable)[0])); +#endif +#ifdef SUPPORT_PCRE16 + if (pcre_mode == PCRE16_MODE) + fprintf(outfile, "%3d\n", (int)(((PCRE_SPTR16)nametable)[0])); +#endif +#ifdef SUPPORT_PCRE8 + if (pcre_mode == PCRE8_MODE) + fprintf(outfile, "%3d\n", ((int)nametable[0] << 8) | (int)nametable[1]); +#endif + nametable += nameentrysize * CHAR_SIZE; + } + } + + if (!okpartial) fprintf(outfile, "Partial matching not supported\n"); + if (hascrorlf) fprintf(outfile, "Contains explicit CR or LF match\n"); + if (match_empty) fprintf(outfile, "May match empty string\n"); + + all_options = REAL_PCRE_OPTIONS(re); + if (do_flip) all_options = swap_uint32(all_options); + + if (get_options == 0) fprintf(outfile, "No options\n"); + else fprintf(outfile, "Options:%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n", + ((get_options & PCRE_ANCHORED) != 0)? " anchored" : "", + ((get_options & PCRE_CASELESS) != 0)? " caseless" : "", + ((get_options & PCRE_EXTENDED) != 0)? " extended" : "", + ((get_options & PCRE_MULTILINE) != 0)? " multiline" : "", + ((get_options & PCRE_FIRSTLINE) != 0)? " firstline" : "", + ((get_options & PCRE_DOTALL) != 0)? " dotall" : "", + ((get_options & PCRE_BSR_ANYCRLF) != 0)? " bsr_anycrlf" : "", + ((get_options & PCRE_BSR_UNICODE) != 0)? " bsr_unicode" : "", + ((get_options & PCRE_DOLLAR_ENDONLY) != 0)? " dollar_endonly" : "", + ((get_options & PCRE_EXTRA) != 0)? " extra" : "", + ((get_options & PCRE_UNGREEDY) != 0)? " ungreedy" : "", + ((get_options & PCRE_NO_AUTO_CAPTURE) != 0)? " no_auto_capture" : "", + ((get_options & PCRE_NO_AUTO_POSSESS) != 0)? " no_auto_possessify" : "", + ((get_options & PCRE_UTF8) != 0)? " utf" : "", + ((get_options & PCRE_UCP) != 0)? " ucp" : "", + ((get_options & PCRE_NO_UTF8_CHECK) != 0)? " no_utf_check" : "", + ((get_options & PCRE_NO_START_OPTIMIZE) != 0)? " no_start_optimize" : "", + ((get_options & PCRE_DUPNAMES) != 0)? " dupnames" : "", + ((get_options & PCRE_NEVER_UTF) != 0)? " never_utf" : ""); + + if (jchanged) fprintf(outfile, "Duplicate name status changes\n"); + + switch (get_options & PCRE_NEWLINE_BITS) + { + case PCRE_NEWLINE_CR: + fprintf(outfile, "Forced newline sequence: CR\n"); + break; + + case PCRE_NEWLINE_LF: + fprintf(outfile, "Forced newline sequence: LF\n"); + break; + + case PCRE_NEWLINE_CRLF: + fprintf(outfile, "Forced newline sequence: CRLF\n"); + break; + + case PCRE_NEWLINE_ANYCRLF: + fprintf(outfile, "Forced newline sequence: ANYCRLF\n"); + break; + + case PCRE_NEWLINE_ANY: + fprintf(outfile, "Forced newline sequence: ANY\n"); + break; + + default: + break; + } + + if (first_char_set == 2) + { + fprintf(outfile, "First char at start or follows newline\n"); + } + else if (first_char_set == 1) + { + const char *caseless = + ((REAL_PCRE_FLAGS(re) & PCRE_FCH_CASELESS) == 0)? + "" : " (caseless)"; + + if (PRINTOK(first_char)) + fprintf(outfile, "First char = \'%c\'%s\n", first_char, caseless); + else + { + fprintf(outfile, "First char = "); + pchar(first_char, outfile); + fprintf(outfile, "%s\n", caseless); + } + } + else + { + fprintf(outfile, "No first char\n"); + } + + if (need_char_set == 0) + { + fprintf(outfile, "No need char\n"); + } + else + { + const char *caseless = + ((REAL_PCRE_FLAGS(re) & PCRE_RCH_CASELESS) == 0)? + "" : " (caseless)"; + + if (PRINTOK(need_char)) + fprintf(outfile, "Need char = \'%c\'%s\n", need_char, caseless); + else + { + fprintf(outfile, "Need char = "); + pchar(need_char, outfile); + fprintf(outfile, "%s\n", caseless); + } + } + + /* Don't output study size; at present it is in any case a fixed + value, but it varies, depending on the computer architecture, and + so messes up the test suite. (And with the /F option, it might be + flipped.) If study was forced by an external -s, don't show this + information unless -i or -d was also present. This means that, except + when auto-callouts are involved, the output from runs with and without + -s should be identical. */ + + if (do_study || (force_study >= 0 && showinfo && !no_force_study)) + { + if (extra == NULL) + fprintf(outfile, "Study returned NULL\n"); + else + { + pcre_uint8 *start_bits = NULL; + int minlength; + + if (new_info(re, extra, PCRE_INFO_MINLENGTH, &minlength) == 0) + fprintf(outfile, "Subject length lower bound = %d\n", minlength); + + if (new_info(re, extra, PCRE_INFO_FIRSTTABLE, &start_bits) == 0) + { + if (start_bits == NULL) + fprintf(outfile, "No starting char list\n"); + else + { + int i; + int c = 24; + fprintf(outfile, "Starting chars: "); + for (i = 0; i < 256; i++) + { + if ((start_bits[i/8] & (1<<(i&7))) != 0) + { + if (c > 75) + { + fprintf(outfile, "\n "); + c = 2; + } + if (PRINTOK(i) && i != ' ') + { + fprintf(outfile, "%c ", i); + c += 2; + } + else + { + fprintf(outfile, "\\x%02x ", i); + c += 5; + } + } + } + fprintf(outfile, "\n"); + } + } + } + + /* Show this only if the JIT was set by /S, not by -s. */ + + if ((study_options & PCRE_STUDY_ALLJIT) != 0 && + (force_study_options & PCRE_STUDY_ALLJIT) == 0) + { + int jit; + if (new_info(re, extra, PCRE_INFO_JIT, &jit) == 0) + { + if (jit) + fprintf(outfile, "JIT study was successful\n"); + else +#ifdef SUPPORT_JIT + fprintf(outfile, "JIT study was not successful\n"); +#else + fprintf(outfile, "JIT support is not available in this version of PCRE\n"); +#endif + } + } + } + } + + /* If the '>' option was present, we write out the regex to a file, and + that is all. The first 8 bytes of the file are the regex length and then + the study length, in big-endian order. */ + + if (to_file != NULL) + { + FILE *f = fopen((char *)to_file, "wb"); + if (f == NULL) + { + fprintf(outfile, "Unable to open %s: %s\n", to_file, strerror(errno)); + } + else + { + pcre_uint8 sbuf[8]; + + if (do_flip) regexflip(re, extra); + sbuf[0] = (pcre_uint8)((true_size >> 24) & 255); + sbuf[1] = (pcre_uint8)((true_size >> 16) & 255); + sbuf[2] = (pcre_uint8)((true_size >> 8) & 255); + sbuf[3] = (pcre_uint8)((true_size) & 255); + sbuf[4] = (pcre_uint8)((true_study_size >> 24) & 255); + sbuf[5] = (pcre_uint8)((true_study_size >> 16) & 255); + sbuf[6] = (pcre_uint8)((true_study_size >> 8) & 255); + sbuf[7] = (pcre_uint8)((true_study_size) & 255); + + if (fwrite(sbuf, 1, 8, f) < 8 || + fwrite(re, 1, true_size, f) < true_size) + { + fprintf(outfile, "Write error on %s: %s\n", to_file, strerror(errno)); + } + else + { + fprintf(outfile, "Compiled pattern written to %s\n", to_file); + + /* If there is study data, write it. */ + + if (extra != NULL) + { + if (fwrite(extra->study_data, 1, true_study_size, f) < + true_study_size) + { + fprintf(outfile, "Write error on %s: %s\n", to_file, + strerror(errno)); + } + else fprintf(outfile, "Study data written to %s\n", to_file); + } + } + fclose(f); + } + + new_free(re); + if (extra != NULL) + { + PCRE_FREE_STUDY(extra); + } + if (locale_set) + { + new_free((void *)tables); + setlocale(LC_CTYPE, "C"); + locale_set = 0; + } + continue; /* With next regex */ + } + } /* End of non-POSIX compile */ + + /* Read data lines and test them */ + + for (;;) + { +#ifdef SUPPORT_PCRE8 + pcre_uint8 *q8; +#endif +#ifdef SUPPORT_PCRE16 + pcre_uint16 *q16; +#endif +#ifdef SUPPORT_PCRE32 + pcre_uint32 *q32; +#endif + pcre_uint8 *bptr; + int *use_offsets = offsets; + int use_size_offsets = size_offsets; + int callout_data = 0; + int callout_data_set = 0; + int count; + pcre_uint32 c; + int copystrings = 0; + int find_match_limit = default_find_match_limit; + int getstrings = 0; + int getlist = 0; + int gmatched = 0; + int start_offset = 0; + int start_offset_sign = 1; + int g_notempty = 0; + int use_dfa = 0; + + *copynames = 0; + *getnames = 0; + +#ifdef SUPPORT_PCRE32 + cn32ptr = copynames; + gn32ptr = getnames; +#endif +#ifdef SUPPORT_PCRE16 + cn16ptr = copynames16; + gn16ptr = getnames16; +#endif +#ifdef SUPPORT_PCRE8 + cn8ptr = copynames8; + gn8ptr = getnames8; +#endif + + SET_PCRE_CALLOUT(callout); + first_callout = 1; + last_callout_mark = NULL; + callout_extra = 0; + callout_count = 0; + callout_fail_count = 999999; + callout_fail_id = -1; + show_malloc = 0; + options = 0; + + if (extra != NULL) extra->flags &= + ~(PCRE_EXTRA_MATCH_LIMIT|PCRE_EXTRA_MATCH_LIMIT_RECURSION); + + len = 0; + for (;;) + { + if (extend_inputline(infile, buffer + len, "data> ") == NULL) + { + if (len > 0) /* Reached EOF without hitting a newline */ + { + fprintf(outfile, "\n"); + break; + } + done = 1; + goto CONTINUE; + } + if (infile != stdin) fprintf(outfile, "%s", (char *)buffer); + len = (int)strlen((char *)buffer); + if (buffer[len-1] == '\n') break; + } + + while (len > 0 && isspace(buffer[len-1])) len--; + buffer[len] = 0; + if (len == 0) break; + + p = buffer; + while (isspace(*p)) p++; + +#ifndef NOUTF + /* Check that the data is well-formed UTF-8 if we're in UTF mode. To create + invalid input to pcre_exec, you must use \x?? or \x{} sequences. */ + + if (use_utf) + { + pcre_uint8 *q; + pcre_uint32 cc; + int n = 1; + + for (q = p; n > 0 && *q; q += n) n = utf82ord(q, &cc); + if (n <= 0) + { + fprintf(outfile, "**Failed: invalid UTF-8 string cannot be used as input in UTF mode\n"); + goto NEXT_DATA; + } + } +#endif + +#ifdef SUPPORT_VALGRIND + /* Mark the dbuffer as addressable but undefined again. */ + + if (dbuffer != NULL) + { + VALGRIND_MAKE_MEM_UNDEFINED(dbuffer, dbuffer_size * CHAR_SIZE); + } +#endif + + /* Allocate a buffer to hold the data line; len+1 is an upper bound on + the number of pcre_uchar units that will be needed. */ + + while (dbuffer == NULL || (size_t)len >= dbuffer_size) + { + dbuffer_size *= 2; + dbuffer = (pcre_uint8 *)realloc(dbuffer, dbuffer_size * CHAR_SIZE); + if (dbuffer == NULL) + { + fprintf(stderr, "pcretest: realloc(%d) failed\n", (int)dbuffer_size); + exit(1); + } + } + +#ifdef SUPPORT_PCRE8 + q8 = (pcre_uint8 *) dbuffer; +#endif +#ifdef SUPPORT_PCRE16 + q16 = (pcre_uint16 *) dbuffer; +#endif +#ifdef SUPPORT_PCRE32 + q32 = (pcre_uint32 *) dbuffer; +#endif + + while ((c = *p++) != 0) + { + int i = 0; + int n = 0; + + /* In UTF mode, input can be UTF-8, so just copy all non-backslash bytes. + In non-UTF mode, allow the value of the byte to fall through to later, + where values greater than 127 are turned into UTF-8 when running in + 16-bit or 32-bit mode. */ + + if (c != '\\') + { +#ifndef NOUTF + if (use_utf && HASUTF8EXTRALEN(c)) { GETUTF8INC(c, p); } +#endif + } + + /* Handle backslash escapes */ + + else switch ((c = *p++)) + { + case 'a': c = CHAR_BEL; break; + case 'b': c = '\b'; break; + case 'e': c = CHAR_ESC; break; + case 'f': c = '\f'; break; + case 'n': c = '\n'; break; + case 'r': c = '\r'; break; + case 't': c = '\t'; break; + case 'v': c = '\v'; break; + + case '0': case '1': case '2': case '3': + case '4': case '5': case '6': case '7': + c -= '0'; + while (i++ < 2 && isdigit(*p) && *p != '8' && *p != '9') + c = c * 8 + *p++ - '0'; + break; + + case 'o': + if (*p == '{') + { + pcre_uint8 *pt = p; + c = 0; + for (pt++; isdigit(*pt) && *pt != '8' && *pt != '9'; pt++) + { + if (++i == 12) + fprintf(outfile, "** Too many octal digits in \\o{...} item; " + "using only the first twelve.\n"); + else c = c * 8 + *pt - '0'; + } + if (*pt == '}') p = pt + 1; + else fprintf(outfile, "** Missing } after \\o{ (assumed)\n"); + } + break; + + case 'x': + if (*p == '{') + { + pcre_uint8 *pt = p; + c = 0; + + /* We used to have "while (isxdigit(*(++pt)))" here, but it fails + when isxdigit() is a macro that refers to its argument more than + once. This is banned by the C Standard, but apparently happens in at + least one MacOS environment. */ + + for (pt++; isxdigit(*pt); pt++) + { + if (++i == 9) + fprintf(outfile, "** Too many hex digits in \\x{...} item; " + "using only the first eight.\n"); + else c = c * 16 + tolower(*pt) - ((isdigit(*pt))? '0' : 'a' - 10); + } + if (*pt == '}') + { + p = pt + 1; + break; + } + /* Not correct form for \x{...}; fall through */ + } + + /* \x without {} always defines just one byte in 8-bit mode. This + allows UTF-8 characters to be constructed byte by byte, and also allows + invalid UTF-8 sequences to be made. Just copy the byte in UTF mode. + Otherwise, pass it down to later code so that it can be turned into + UTF-8 when running in 16/32-bit mode. */ + + c = 0; + while (i++ < 2 && isxdigit(*p)) + { + c = c * 16 + tolower(*p) - ((isdigit(*p))? '0' : 'a' - 10); + p++; + } +#if !defined NOUTF && defined SUPPORT_PCRE8 + if (use_utf && (pcre_mode == PCRE8_MODE)) + { + *q8++ = c; + continue; + } +#endif + break; + + case 0: /* \ followed by EOF allows for an empty line */ + p--; + continue; + + case '>': + if (*p == '-') + { + start_offset_sign = -1; + p++; + } + while(isdigit(*p)) start_offset = start_offset * 10 + *p++ - '0'; + start_offset *= start_offset_sign; + continue; + + case 'A': /* Option setting */ + options |= PCRE_ANCHORED; + continue; + + case 'B': + options |= PCRE_NOTBOL; + continue; + + case 'C': + if (isdigit(*p)) /* Set copy string */ + { + while(isdigit(*p)) n = n * 10 + *p++ - '0'; + copystrings |= 1 << n; + } + else if (isalnum(*p)) + { + READ_CAPTURE_NAME(p, &cn8ptr, &cn16ptr, &cn32ptr, re); + } + else if (*p == '+') + { + callout_extra = 1; + p++; + } + else if (*p == '-') + { + SET_PCRE_CALLOUT(NULL); + p++; + } + else if (*p == '!') + { + callout_fail_id = 0; + p++; + while(isdigit(*p)) + callout_fail_id = callout_fail_id * 10 + *p++ - '0'; + callout_fail_count = 0; + if (*p == '!') + { + p++; + while(isdigit(*p)) + callout_fail_count = callout_fail_count * 10 + *p++ - '0'; + } + } + else if (*p == '*') + { + int sign = 1; + callout_data = 0; + if (*(++p) == '-') { sign = -1; p++; } + while(isdigit(*p)) + callout_data = callout_data * 10 + *p++ - '0'; + callout_data *= sign; + callout_data_set = 1; + } + continue; + +#if !defined NODFA + case 'D': +#if !defined NOPOSIX + if (posix || do_posix) + printf("** Can't use dfa matching in POSIX mode: \\D ignored\n"); + else +#endif + use_dfa = 1; + continue; +#endif + +#if !defined NODFA + case 'F': + options |= PCRE_DFA_SHORTEST; + continue; +#endif + + case 'G': + if (isdigit(*p)) + { + while(isdigit(*p)) n = n * 10 + *p++ - '0'; + getstrings |= 1 << n; + } + else if (isalnum(*p)) + { + READ_CAPTURE_NAME(p, &gn8ptr, &gn16ptr, &gn32ptr, re); + } + continue; + + case 'J': + while(isdigit(*p)) n = n * 10 + *p++ - '0'; + if (extra != NULL + && (extra->flags & PCRE_EXTRA_EXECUTABLE_JIT) != 0 + && extra->executable_jit != NULL) + { + if (jit_stack != NULL) { PCRE_JIT_STACK_FREE(jit_stack); } + jit_stack = PCRE_JIT_STACK_ALLOC(1, n * 1024); + PCRE_ASSIGN_JIT_STACK(extra, jit_callback, jit_stack); + } + continue; + + case 'L': + getlist = 1; + continue; + + case 'M': + find_match_limit = 1; + continue; + + case 'N': + if ((options & PCRE_NOTEMPTY) != 0) + options = (options & ~PCRE_NOTEMPTY) | PCRE_NOTEMPTY_ATSTART; + else + options |= PCRE_NOTEMPTY; + continue; + + case 'O': + while(isdigit(*p)) + { + if (n > (INT_MAX-10)/10) /* Hack to stop fuzzers */ + { + printf("** \\O argument is too big\n"); + yield = 1; + goto EXIT; + } + n = n * 10 + *p++ - '0'; + } + if (n > size_offsets_max) + { + size_offsets_max = n; + free(offsets); + use_offsets = offsets = (int *)malloc(size_offsets_max * sizeof(int)); + if (offsets == NULL) + { + printf("** Failed to get %d bytes of memory for offsets vector\n", + (int)(size_offsets_max * sizeof(int))); + yield = 1; + goto EXIT; + } + } + use_size_offsets = n; + if (n == 0) use_offsets = NULL; /* Ensures it can't write to it */ + else use_offsets = offsets + size_offsets_max - n; /* To catch overruns */ + continue; + + case 'P': + options |= ((options & PCRE_PARTIAL_SOFT) == 0)? + PCRE_PARTIAL_SOFT : PCRE_PARTIAL_HARD; + continue; + + case 'Q': + while(isdigit(*p)) n = n * 10 + *p++ - '0'; + if (extra == NULL) + { + extra = (pcre_extra *)malloc(sizeof(pcre_extra)); + extra->flags = 0; + } + extra->flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION; + extra->match_limit_recursion = n; + continue; + + case 'q': + while(isdigit(*p)) n = n * 10 + *p++ - '0'; + if (extra == NULL) + { + extra = (pcre_extra *)malloc(sizeof(pcre_extra)); + extra->flags = 0; + } + extra->flags |= PCRE_EXTRA_MATCH_LIMIT; + extra->match_limit = n; + continue; + +#if !defined NODFA + case 'R': + options |= PCRE_DFA_RESTART; + continue; +#endif + + case 'S': + show_malloc = 1; + continue; + + case 'Y': + options |= PCRE_NO_START_OPTIMIZE; + continue; + + case 'Z': + options |= PCRE_NOTEOL; + continue; + + case '?': + options |= PCRE_NO_UTF8_CHECK; + continue; + + case '<': + { + int x = check_mc_option(p, outfile, TRUE, "escape sequence"); + if (x == 0) goto NEXT_DATA; + options |= x; + while (*p++ != '>'); + } + continue; + } + + /* We now have a character value in c that may be greater than 255. + In 8-bit mode we convert to UTF-8 if we are in UTF mode. Values greater + than 127 in UTF mode must have come from \x{...} or octal constructs + because values from \x.. get this far only in non-UTF mode. */ + +#ifdef SUPPORT_PCRE8 + if (pcre_mode == PCRE8_MODE) + { +#ifndef NOUTF + if (use_utf) + { + if (c > 0x7fffffff) + { + fprintf(outfile, "** Character \\x{%x} is greater than 0x7fffffff " + "and so cannot be converted to UTF-8\n", c); + goto NEXT_DATA; + } + q8 += ord2utf8(c, q8); + } + else +#endif + { + if (c > 0xffu) + { + fprintf(outfile, "** Character \\x{%x} is greater than 255 " + "and UTF-8 mode is not enabled.\n", c); + fprintf(outfile, "** Truncation will probably give the wrong " + "result.\n"); + } + *q8++ = c; + } + } +#endif +#ifdef SUPPORT_PCRE16 + if (pcre_mode == PCRE16_MODE) + { +#ifndef NOUTF + if (use_utf) + { + if (c > 0x10ffffu) + { + fprintf(outfile, "** Failed: character \\x{%x} is greater than " + "0x10ffff and so cannot be converted to UTF-16\n", c); + goto NEXT_DATA; + } + else if (c >= 0x10000u) + { + c-= 0x10000u; + *q16++ = 0xD800 | (c >> 10); + *q16++ = 0xDC00 | (c & 0x3ff); + } + else + *q16++ = c; + } + else +#endif + { + if (c > 0xffffu) + { + fprintf(outfile, "** Character \\x{%x} is greater than 0xffff " + "and UTF-16 mode is not enabled.\n", c); + fprintf(outfile, "** Truncation will probably give the wrong " + "result.\n"); + } + + *q16++ = c; + } + } +#endif +#ifdef SUPPORT_PCRE32 + if (pcre_mode == PCRE32_MODE) + { + *q32++ = c; + } +#endif + + } + + /* Reached end of subject string */ + +#ifdef SUPPORT_PCRE8 + if (pcre_mode == PCRE8_MODE) + { + *q8 = 0; + len = (int)(q8 - (pcre_uint8 *)dbuffer); + } +#endif +#ifdef SUPPORT_PCRE16 + if (pcre_mode == PCRE16_MODE) + { + *q16 = 0; + len = (int)(q16 - (pcre_uint16 *)dbuffer); + } +#endif +#ifdef SUPPORT_PCRE32 + if (pcre_mode == PCRE32_MODE) + { + *q32 = 0; + len = (int)(q32 - (pcre_uint32 *)dbuffer); + } +#endif + + /* If we're compiling with explicit valgrind support, Mark the data from after + its end to the end of the buffer as unaddressable, so that a read over the end + of the buffer will be seen by valgrind, even if it doesn't cause a crash. + If we're not building with valgrind support, at least move the data to the end + of the buffer so that it might at least cause a crash. + If we are using the POSIX interface, we must include the terminating zero. */ + + bptr = dbuffer; + +#if !defined NOPOSIX + if (posix || do_posix) + { +#ifdef SUPPORT_VALGRIND + VALGRIND_MAKE_MEM_NOACCESS(dbuffer + len + 1, dbuffer_size - (len + 1)); +#else + memmove(bptr + dbuffer_size - len - 1, bptr, len + 1); + bptr += dbuffer_size - len - 1; +#endif + } + else +#endif + { +#ifdef SUPPORT_VALGRIND + VALGRIND_MAKE_MEM_NOACCESS(dbuffer + len * CHAR_SIZE, (dbuffer_size - len) * CHAR_SIZE); +#else + bptr = memmove(bptr + (dbuffer_size - len) * CHAR_SIZE, bptr, len * CHAR_SIZE); +#endif + } + + if ((all_use_dfa || use_dfa) && find_match_limit) + { + printf("** Match limit not relevant for DFA matching: ignored\n"); + find_match_limit = 0; + } + + /* Handle matching via the POSIX interface, which does not + support timing or playing with the match limit or callout data. */ + +#if !defined NOPOSIX + if (posix || do_posix) + { + int rc; + int eflags = 0; + regmatch_t *pmatch = NULL; + if (use_size_offsets > 0) + pmatch = (regmatch_t *)malloc(sizeof(regmatch_t) * use_size_offsets); + if ((options & PCRE_NOTBOL) != 0) eflags |= REG_NOTBOL; + if ((options & PCRE_NOTEOL) != 0) eflags |= REG_NOTEOL; + if ((options & PCRE_NOTEMPTY) != 0) eflags |= REG_NOTEMPTY; + + rc = regexec(&preg, (const char *)bptr, use_size_offsets, pmatch, eflags); + + if (rc != 0) + { + (void)regerror(rc, &preg, (char *)buffer, buffer_size); + fprintf(outfile, "No match: POSIX code %d: %s\n", rc, buffer); + } + else if ((REAL_PCRE_OPTIONS(preg.re_pcre) & PCRE_NO_AUTO_CAPTURE) != 0) + { + fprintf(outfile, "Matched with REG_NOSUB\n"); + } + else + { + size_t i; + for (i = 0; i < (size_t)use_size_offsets; i++) + { + if (pmatch[i].rm_so >= 0) + { + fprintf(outfile, "%2d: ", (int)i); + PCHARSV(dbuffer, pmatch[i].rm_so, + pmatch[i].rm_eo - pmatch[i].rm_so, outfile); + fprintf(outfile, "\n"); + if (do_showcaprest || (i == 0 && do_showrest)) + { + fprintf(outfile, "%2d+ ", (int)i); + PCHARSV(dbuffer, pmatch[i].rm_eo, len - pmatch[i].rm_eo, + outfile); + fprintf(outfile, "\n"); + } + } + } + } + free(pmatch); + goto NEXT_DATA; + } + +#endif /* !defined NOPOSIX */ + + /* Handle matching via the native interface - repeats for /g and /G */ + + /* Ensure that there is a JIT callback if we want to verify that JIT was + actually used. If jit_stack == NULL, no stack has yet been assigned. */ + + if (verify_jit && jit_stack == NULL && extra != NULL) + { PCRE_ASSIGN_JIT_STACK(extra, jit_callback, jit_stack); } + + for (;; gmatched++) /* Loop for /g or /G */ + { + markptr = NULL; + jit_was_used = FALSE; + + if (timeitm > 0) + { + register int i; + clock_t time_taken; + clock_t start_time = clock(); + +#if !defined NODFA + if (all_use_dfa || use_dfa) + { + if ((options & PCRE_DFA_RESTART) != 0) + { + fprintf(outfile, "Timing DFA restarts is not supported\n"); + break; + } + if (dfa_workspace == NULL) + dfa_workspace = (int *)malloc(DFA_WS_DIMENSION*sizeof(int)); + for (i = 0; i < timeitm; i++) + { + PCRE_DFA_EXEC(count, re, extra, bptr, len, start_offset, + (options | g_notempty), use_offsets, use_size_offsets, + dfa_workspace, DFA_WS_DIMENSION); + } + } + else +#endif + + for (i = 0; i < timeitm; i++) + { + PCRE_EXEC(count, re, extra, bptr, len, start_offset, + (options | g_notempty), use_offsets, use_size_offsets); + } + total_match_time += (time_taken = clock() - start_time); + fprintf(outfile, "Execute time %.4f milliseconds\n", + (((double)time_taken * 1000.0) / (double)timeitm) / + (double)CLOCKS_PER_SEC); + } + + /* If find_match_limit is set, we want to do repeated matches with + varying limits in order to find the minimum value for the match limit and + for the recursion limit. The match limits are relevant only to the normal + running of pcre_exec(), so disable the JIT optimization. This makes it + possible to run the same set of tests with and without JIT externally + requested. */ + + if (find_match_limit) + { + if (extra != NULL) { PCRE_FREE_STUDY(extra); } + extra = (pcre_extra *)malloc(sizeof(pcre_extra)); + extra->flags = 0; + + (void)check_match_limit(re, extra, bptr, len, start_offset, + options|g_notempty, use_offsets, use_size_offsets, + PCRE_EXTRA_MATCH_LIMIT, &(extra->match_limit), + PCRE_ERROR_MATCHLIMIT, "match()"); + + count = check_match_limit(re, extra, bptr, len, start_offset, + options|g_notempty, use_offsets, use_size_offsets, + PCRE_EXTRA_MATCH_LIMIT_RECURSION, &(extra->match_limit_recursion), + PCRE_ERROR_RECURSIONLIMIT, "match() recursion"); + } + + /* If callout_data is set, use the interface with additional data */ + + else if (callout_data_set) + { + if (extra == NULL) + { + extra = (pcre_extra *)malloc(sizeof(pcre_extra)); + extra->flags = 0; + } + extra->flags |= PCRE_EXTRA_CALLOUT_DATA; + extra->callout_data = &callout_data; + PCRE_EXEC(count, re, extra, bptr, len, start_offset, + options | g_notempty, use_offsets, use_size_offsets); + extra->flags &= ~PCRE_EXTRA_CALLOUT_DATA; + } + + /* The normal case is just to do the match once, with the default + value of match_limit. */ + +#if !defined NODFA + else if (all_use_dfa || use_dfa) + { + if (dfa_workspace == NULL) + dfa_workspace = (int *)malloc(DFA_WS_DIMENSION*sizeof(int)); + if (dfa_matched++ == 0) + dfa_workspace[0] = -1; /* To catch bad restart */ + PCRE_DFA_EXEC(count, re, extra, bptr, len, start_offset, + (options | g_notempty), use_offsets, use_size_offsets, dfa_workspace, + DFA_WS_DIMENSION); + if (count == 0) + { + fprintf(outfile, "Matched, but offsets vector is too small to show all matches\n"); + count = use_size_offsets/2; + } + } +#endif + + else + { + PCRE_EXEC(count, re, extra, bptr, len, start_offset, + options | g_notempty, use_offsets, use_size_offsets); + if (count == 0) + { + fprintf(outfile, "Matched, but too many substrings\n"); + /* 2 is a special case; match can be returned */ + count = (use_size_offsets == 2)? 1 : use_size_offsets/3; + } + } + + /* Matched */ + + if (count >= 0) + { + int i, maxcount; + void *cnptr, *gnptr; + +#if !defined NODFA + if (all_use_dfa || use_dfa) maxcount = use_size_offsets/2; else +#endif + /* 2 is a special case; match can be returned */ + maxcount = (use_size_offsets == 2)? 1 : use_size_offsets/3; + + /* This is a check against a lunatic return value. */ + + if (count > maxcount) + { + fprintf(outfile, + "** PCRE error: returned count %d is too big for offset size %d\n", + count, use_size_offsets); + count = use_size_offsets/3; + if (do_g || do_G) + { + fprintf(outfile, "** /%c loop abandoned\n", do_g? 'g' : 'G'); + do_g = do_G = FALSE; /* Break g/G loop */ + } + } + + /* do_allcaps requests showing of all captures in the pattern, to check + unset ones at the end. */ + + if (do_allcaps) + { + if (all_use_dfa || use_dfa) + { + fprintf(outfile, "** Show all captures ignored after DFA matching\n"); + } + else + { + if (new_info(re, NULL, PCRE_INFO_CAPTURECOUNT, &count) < 0) + goto SKIP_DATA; + count++; /* Allow for full match */ + if (count * 2 > use_size_offsets) count = use_size_offsets/2; + } + } + + /* Output the captured substrings. Note that, for the matched string, + the use of \K in an assertion can make the start later than the end. */ + + for (i = 0; i < count * 2; i += 2) + { + if (use_offsets[i] < 0) + { + if (use_offsets[i] != -1) + fprintf(outfile, "ERROR: bad negative value %d for offset %d\n", + use_offsets[i], i); + if (use_offsets[i+1] != -1) + fprintf(outfile, "ERROR: bad negative value %d for offset %d\n", + use_offsets[i+1], i+1); + fprintf(outfile, "%2d: \n", i/2); + } + else + { + int start = use_offsets[i]; + int end = use_offsets[i+1]; + + if (start > end) + { + start = use_offsets[i+1]; + end = use_offsets[i]; + fprintf(outfile, "Start of matched string is beyond its end - " + "displaying from end to start.\n"); + } + + fprintf(outfile, "%2d: ", i/2); + PCHARSV(bptr, start, end - start, outfile); + if (verify_jit && jit_was_used) fprintf(outfile, " (JIT)"); + fprintf(outfile, "\n"); + + /* Note: don't use the start/end variables here because we want to + show the text from what is reported as the end. */ + + if (do_showcaprest || (i == 0 && do_showrest)) + { + fprintf(outfile, "%2d+ ", i/2); + PCHARSV(bptr, use_offsets[i+1], len - use_offsets[i+1], + outfile); + fprintf(outfile, "\n"); + } + } + } + + if (markptr != NULL) + { + fprintf(outfile, "MK: "); + PCHARSV(markptr, 0, -1, outfile); + fprintf(outfile, "\n"); + } + + for (i = 0; i < 32; i++) + { + if ((copystrings & (1 << i)) != 0) + { + int rc; + char copybuffer[256]; + PCRE_COPY_SUBSTRING(rc, bptr, use_offsets, count, i, + copybuffer, sizeof(copybuffer)); + if (rc < 0) + fprintf(outfile, "copy substring %d failed %d\n", i, rc); + else + { + fprintf(outfile, "%2dC ", i); + PCHARSV(copybuffer, 0, rc, outfile); + fprintf(outfile, " (%d)\n", rc); + } + } + } + + cnptr = copynames; + for (;;) + { + int rc; + char copybuffer[256]; + +#ifdef SUPPORT_PCRE32 + if (pcre_mode == PCRE32_MODE) + { + if (*(pcre_uint32 *)cnptr == 0) break; + } +#endif +#ifdef SUPPORT_PCRE16 + if (pcre_mode == PCRE16_MODE) + { + if (*(pcre_uint16 *)cnptr == 0) break; + } +#endif +#ifdef SUPPORT_PCRE8 + if (pcre_mode == PCRE8_MODE) + { + if (*(pcre_uint8 *)cnptr == 0) break; + } +#endif + + PCRE_COPY_NAMED_SUBSTRING(rc, re, bptr, use_offsets, count, + cnptr, copybuffer, sizeof(copybuffer)); + + if (rc < 0) + { + fprintf(outfile, "copy substring "); + PCHARSV(cnptr, 0, -1, outfile); + fprintf(outfile, " failed %d\n", rc); + } + else + { + fprintf(outfile, " C "); + PCHARSV(copybuffer, 0, rc, outfile); + fprintf(outfile, " (%d) ", rc); + PCHARSV(cnptr, 0, -1, outfile); + putc('\n', outfile); + } + + cnptr = (char *)cnptr + (STRLEN(cnptr) + 1) * CHAR_SIZE; + } + + for (i = 0; i < 32; i++) + { + if ((getstrings & (1 << i)) != 0) + { + int rc; + const char *substring; + PCRE_GET_SUBSTRING(rc, bptr, use_offsets, count, i, &substring); + if (rc < 0) + fprintf(outfile, "get substring %d failed %d\n", i, rc); + else + { + fprintf(outfile, "%2dG ", i); + PCHARSV(substring, 0, rc, outfile); + fprintf(outfile, " (%d)\n", rc); + PCRE_FREE_SUBSTRING(substring); + } + } + } + + gnptr = getnames; + for (;;) + { + int rc; + const char *substring; + +#ifdef SUPPORT_PCRE32 + if (pcre_mode == PCRE32_MODE) + { + if (*(pcre_uint32 *)gnptr == 0) break; + } +#endif +#ifdef SUPPORT_PCRE16 + if (pcre_mode == PCRE16_MODE) + { + if (*(pcre_uint16 *)gnptr == 0) break; + } +#endif +#ifdef SUPPORT_PCRE8 + if (pcre_mode == PCRE8_MODE) + { + if (*(pcre_uint8 *)gnptr == 0) break; + } +#endif + + PCRE_GET_NAMED_SUBSTRING(rc, re, bptr, use_offsets, count, + gnptr, &substring); + if (rc < 0) + { + fprintf(outfile, "get substring "); + PCHARSV(gnptr, 0, -1, outfile); + fprintf(outfile, " failed %d\n", rc); + } + else + { + fprintf(outfile, " G "); + PCHARSV(substring, 0, rc, outfile); + fprintf(outfile, " (%d) ", rc); + PCHARSV(gnptr, 0, -1, outfile); + PCRE_FREE_SUBSTRING(substring); + putc('\n', outfile); + } + + gnptr = (char *)gnptr + (STRLEN(gnptr) + 1) * CHAR_SIZE; + } + + if (getlist) + { + int rc; + const char **stringlist; + PCRE_GET_SUBSTRING_LIST(rc, bptr, use_offsets, count, &stringlist); + if (rc < 0) + fprintf(outfile, "get substring list failed %d\n", rc); + else + { + for (i = 0; i < count; i++) + { + fprintf(outfile, "%2dL ", i); + PCHARSV(stringlist[i], 0, -1, outfile); + putc('\n', outfile); + } + if (stringlist[i] != NULL) + fprintf(outfile, "string list not terminated by NULL\n"); + PCRE_FREE_SUBSTRING_LIST(stringlist); + } + } + } + + /* There was a partial match. If the bumpalong point is not the same as + the first inspected character, show the offset explicitly. */ + + else if (count == PCRE_ERROR_PARTIAL) + { + fprintf(outfile, "Partial match"); + if (use_size_offsets > 2 && use_offsets[0] != use_offsets[2]) + fprintf(outfile, " at offset %d", use_offsets[2]); + if (markptr != NULL) + { + fprintf(outfile, ", mark="); + PCHARSV(markptr, 0, -1, outfile); + } + if (use_size_offsets > 1) + { + fprintf(outfile, ": "); + PCHARSV(bptr, use_offsets[0], use_offsets[1] - use_offsets[0], + outfile); + } + if (verify_jit && jit_was_used) fprintf(outfile, " (JIT)"); + fprintf(outfile, "\n"); + break; /* Out of the /g loop */ + } + + /* Failed to match. If this is a /g or /G loop and we previously set + g_notempty after a null match, this is not necessarily the end. We want + to advance the start offset, and continue. We won't be at the end of the + string - that was checked before setting g_notempty. + + Complication arises in the case when the newline convention is "any", + "crlf", or "anycrlf". If the previous match was at the end of a line + terminated by CRLF, an advance of one character just passes the \r, + whereas we should prefer the longer newline sequence, as does the code in + pcre_exec(). Fudge the offset value to achieve this. We check for a + newline setting in the pattern; if none was set, use PCRE_CONFIG() to + find the default. + + Otherwise, in the case of UTF-8 matching, the advance must be one + character, not one byte. */ + + else + { + if (g_notempty != 0) + { + int onechar = 1; + unsigned int obits = REAL_PCRE_OPTIONS(re); + use_offsets[0] = start_offset; + if ((obits & PCRE_NEWLINE_BITS) == 0) + { + int d; + (void)PCRE_CONFIG(PCRE_CONFIG_NEWLINE, &d); + /* Note that these values are always the ASCII ones, even in + EBCDIC environments. CR = 13, NL = 10. */ + obits = (d == 13)? PCRE_NEWLINE_CR : + (d == 10)? PCRE_NEWLINE_LF : + (d == (13<<8 | 10))? PCRE_NEWLINE_CRLF : + (d == -2)? PCRE_NEWLINE_ANYCRLF : + (d == -1)? PCRE_NEWLINE_ANY : 0; + } + if (((obits & PCRE_NEWLINE_BITS) == PCRE_NEWLINE_ANY || + (obits & PCRE_NEWLINE_BITS) == PCRE_NEWLINE_CRLF || + (obits & PCRE_NEWLINE_BITS) == PCRE_NEWLINE_ANYCRLF) + && + start_offset < len - 1 && ( +#ifdef SUPPORT_PCRE8 + (pcre_mode == PCRE8_MODE && + bptr[start_offset] == '\r' && + bptr[start_offset + 1] == '\n') || +#endif +#ifdef SUPPORT_PCRE16 + (pcre_mode == PCRE16_MODE && + ((PCRE_SPTR16)bptr)[start_offset] == '\r' && + ((PCRE_SPTR16)bptr)[start_offset + 1] == '\n') || +#endif +#ifdef SUPPORT_PCRE32 + (pcre_mode == PCRE32_MODE && + ((PCRE_SPTR32)bptr)[start_offset] == '\r' && + ((PCRE_SPTR32)bptr)[start_offset + 1] == '\n') || +#endif + 0)) + onechar++; + else if (use_utf) + { + while (start_offset + onechar < len) + { + if ((bptr[start_offset+onechar] & 0xc0) != 0x80) break; + onechar++; + } + } + use_offsets[1] = start_offset + onechar; + } + else + { + switch(count) + { + case PCRE_ERROR_NOMATCH: + if (gmatched == 0) + { + if (markptr == NULL) + { + fprintf(outfile, "No match"); + } + else + { + fprintf(outfile, "No match, mark = "); + PCHARSV(markptr, 0, -1, outfile); + } + if (verify_jit && jit_was_used) fprintf(outfile, " (JIT)"); + putc('\n', outfile); + } + break; + + case PCRE_ERROR_BADUTF8: + case PCRE_ERROR_SHORTUTF8: + fprintf(outfile, "Error %d (%s UTF-%d string)", count, + (count == PCRE_ERROR_BADUTF8)? "bad" : "short", + 8 * CHAR_SIZE); + if (use_size_offsets >= 2) + fprintf(outfile, " offset=%d reason=%d", use_offsets[0], + use_offsets[1]); + fprintf(outfile, "\n"); + break; + + case PCRE_ERROR_BADUTF8_OFFSET: + fprintf(outfile, "Error %d (bad UTF-%d offset)\n", count, + 8 * CHAR_SIZE); + break; + + default: + if (count < 0 && + (-count) < (int)(sizeof(errtexts)/sizeof(const char *))) + fprintf(outfile, "Error %d (%s)\n", count, errtexts[-count]); + else + fprintf(outfile, "Error %d (Unexpected value)\n", count); + break; + } + + break; /* Out of the /g loop */ + } + } + + /* If not /g or /G we are done */ + + if (!do_g && !do_G) break; + + if (use_offsets == NULL) + { + fprintf(outfile, "Cannot do global matching without an ovector\n"); + break; + } + + if (use_size_offsets < 2) + { + fprintf(outfile, "Cannot do global matching with an ovector size < 2\n"); + break; + } + + /* If we have matched an empty string, first check to see if we are at + the end of the subject. If so, the /g loop is over. Otherwise, mimic what + Perl's /g options does. This turns out to be rather cunning. First we set + PCRE_NOTEMPTY_ATSTART and PCRE_ANCHORED and try the match again at the + same point. If this fails (picked up above) we advance to the next + character. */ + + g_notempty = 0; + + if (use_offsets[0] == use_offsets[1]) + { + if (use_offsets[0] == len) break; + g_notempty = PCRE_NOTEMPTY_ATSTART | PCRE_ANCHORED; + } + + /* For /g, update the start offset, leaving the rest alone. There is a + tricky case when \K is used in a positive lookbehind assertion. This can + cause the end of the match to be less than or equal to the start offset. + In this case we restart at one past the start offset. This may return the + same match if the original start offset was bumped along during the + match, but eventually the new start offset will hit the actual start + offset. (In PCRE2 the true start offset is available, and this can be + done better. It is not worth doing more than making sure we do not loop + at this stage in the life of PCRE1.) */ + + if (do_g) + { + if (g_notempty == 0 && use_offsets[1] <= start_offset) + { + if (start_offset >= len) break; /* End of subject */ + start_offset++; + if (use_utf) + { + while (start_offset < len) + { + if ((bptr[start_offset] & 0xc0) != 0x80) break; + start_offset++; + } + } + } + else start_offset = use_offsets[1]; + } + + /* For /G, update the pointer and length */ + + else + { + bptr += use_offsets[1] * CHAR_SIZE; + len -= use_offsets[1]; + } + } /* End of loop for /g and /G */ + + NEXT_DATA: continue; + } /* End of loop for data lines */ + + CONTINUE: + +#if !defined NOPOSIX + if ((posix || do_posix) && preg.re_pcre != 0) regfree(&preg); +#endif + + if (re != NULL) new_free(re); + if (extra != NULL) + { + PCRE_FREE_STUDY(extra); + } + if (locale_set) + { + new_free((void *)tables); + setlocale(LC_CTYPE, "C"); + locale_set = 0; + } + if (jit_stack != NULL) + { + PCRE_JIT_STACK_FREE(jit_stack); + jit_stack = NULL; + } + } + +if (infile == stdin) fprintf(outfile, "\n"); + +if (showtotaltimes) + { + fprintf(outfile, "--------------------------------------\n"); + if (timeit > 0) + { + fprintf(outfile, "Total compile time %.4f milliseconds\n", + (((double)total_compile_time * 1000.0) / (double)timeit) / + (double)CLOCKS_PER_SEC); + fprintf(outfile, "Total study time %.4f milliseconds\n", + (((double)total_study_time * 1000.0) / (double)timeit) / + (double)CLOCKS_PER_SEC); + } + fprintf(outfile, "Total execute time %.4f milliseconds\n", + (((double)total_match_time * 1000.0) / (double)timeitm) / + (double)CLOCKS_PER_SEC); + } + +EXIT: + +if (infile != NULL && infile != stdin) fclose(infile); +if (outfile != NULL && outfile != stdout) fclose(outfile); + +free(buffer); +free(dbuffer); +free(pbuffer); +free(offsets); + +#ifdef SUPPORT_PCRE16 +if (buffer16 != NULL) free(buffer16); +#endif +#ifdef SUPPORT_PCRE32 +if (buffer32 != NULL) free(buffer32); +#endif + +#if !defined NODFA +if (dfa_workspace != NULL) + free(dfa_workspace); +#endif + +#if defined(__VMS) + yield = SS$_NORMAL; /* Return values via DCL symbols */ +#endif + +return yield; +} + +/* End of pcretest.c */ + diff --git a/src/third_party/pcre-8.41/perltest.pl b/src/third_party/pcre-8.41/perltest.pl new file mode 100755 index 00000000000..29b808b5293 --- /dev/null +++ b/src/third_party/pcre-8.41/perltest.pl @@ -0,0 +1,242 @@ +#! /usr/bin/env perl + +# Program for testing regular expressions with perl to check that PCRE handles +# them the same. This version needs to have "use utf8" at the start for running +# the UTF-8 tests, but *not* for the other tests. The only way I've found for +# doing this is to cat this line in explicitly in the RunPerlTest script. I've +# also used this method to supply "require Encode" for the UTF-8 tests, so that +# the main test will still run where Encode is not installed. + +#use utf8; +#require Encode; + +# Function for turning a string into a string of printing chars. + +sub pchars { +my($t) = ""; + +if ($utf8) + { + @p = unpack('U*', $_[0]); + foreach $c (@p) + { + if ($c >= 32 && $c < 127) { $t .= chr $c; } + else { $t .= sprintf("\\x{%02x}", $c); + } + } + } +else + { + foreach $c (split(//, $_[0])) + { + if (ord $c >= 32 && ord $c < 127) { $t .= $c; } + else { $t .= sprintf("\\x%02x", ord $c); } + } + } + +$t; +} + + +# Read lines from named file or stdin and write to named file or stdout; lines +# consist of a regular expression, in delimiters and optionally followed by +# options, followed by a set of test data, terminated by an empty line. + +# Sort out the input and output files + +if (@ARGV > 0) + { + open(INFILE, "<$ARGV[0]") || die "Failed to open $ARGV[0]\n"; + $infile = "INFILE"; + } +else { $infile = "STDIN"; } + +if (@ARGV > 1) + { + open(OUTFILE, ">$ARGV[1]") || die "Failed to open $ARGV[1]\n"; + $outfile = "OUTFILE"; + } +else { $outfile = "STDOUT"; } + +printf($outfile "Perl $] Regular Expressions\n\n"); + +# Main loop + +NEXT_RE: +for (;;) + { + printf " re> " if $infile eq "STDIN"; + last if ! ($_ = <$infile>); + printf $outfile "$_" if $infile ne "STDIN"; + next if ($_ =~ /^\s*$/ || $_ =~ /^< forbid/); + + $pattern = $_; + + while ($pattern !~ /^\s*(.).*\1/s) + { + printf " > " if $infile eq "STDIN"; + last if ! ($_ = <$infile>); + printf $outfile "$_" if $infile ne "STDIN"; + $pattern .= $_; + } + + chomp($pattern); + $pattern =~ s/\s+$//; + + # The private /+ modifier means "print $' afterwards". + + $showrest = ($pattern =~ s/\+(?=[a-zA-Z]*$)//); + + # A doubled version is used by pcretest to print remainders after captures + + $pattern =~ s/\+(?=[a-zA-Z]*$)//; + + # Remove /8 from a UTF-8 pattern. + + $utf8 = $pattern =~ s/8(?=[a-zA-Z]*$)//; + + # Remove /J from a pattern with duplicate names. + + $pattern =~ s/J(?=[a-zA-Z]*$)//; + + # Remove /K from a pattern (asks pcretest to check MARK data) */ + + $pattern =~ s/K(?=[a-zA-Z]*$)//; + + # /W asks pcretest to set PCRE_UCP; change this to /u for Perl + + $pattern =~ s/W(?=[a-zA-Z]*$)/u/; + + # Remove /S or /SS from a pattern (asks pcretest to study or not to study) + + $pattern =~ s/S(?=[a-zA-Z]*$)//g; + + # Remove /Y and /O from a pattern (disable PCRE optimizations) + + $pattern =~ s/[YO](?=[a-zA-Z]*$)//; + + # Check that the pattern is valid + + eval "\$_ =~ ${pattern}"; + if ($@) + { + printf $outfile "Error: $@"; + if ($infile != "STDIN") + { + for (;;) + { + last if ! ($_ = <$infile>); + last if $_ =~ /^\s*$/; + } + } + next NEXT_RE; + } + + # If the /g modifier is present, we want to put a loop round the matching; + # otherwise just a single "if". + + $cmd = ($pattern =~ /g[a-z]*$/)? "while" : "if"; + + # If the pattern is actually the null string, Perl uses the most recently + # executed (and successfully compiled) regex is used instead. This is a + # nasty trap for the unwary! The PCRE test suite does contain null strings + # in places - if they are allowed through here all sorts of weird and + # unexpected effects happen. To avoid this, we replace such patterns with + # a non-null pattern that has the same effect. + + $pattern = "/(?#)/$2" if ($pattern =~ /^(.)\1(.*)$/); + + # Read data lines and test them + + for (;;) + { + printf "data> " if $infile eq "STDIN"; + last NEXT_RE if ! ($_ = <$infile>); + chomp; + printf $outfile "$_\n" if $infile ne "STDIN"; + + s/\s+$//; # Remove trailing space + s/^\s+//; # Remove leading space + s/\\Y//g; # Remove \Y (pcretest flag to set PCRE_NO_START_OPTIMIZE) + + last if ($_ eq ""); + $x = eval "\"$_\""; # To get escapes processed + + # Empty array for holding results, ensure $REGERROR and $REGMARK are + # unset, then do the matching. + + @subs = (); + + $pushes = "push \@subs,\$&;" . + "push \@subs,\$1;" . + "push \@subs,\$2;" . + "push \@subs,\$3;" . + "push \@subs,\$4;" . + "push \@subs,\$5;" . + "push \@subs,\$6;" . + "push \@subs,\$7;" . + "push \@subs,\$8;" . + "push \@subs,\$9;" . + "push \@subs,\$10;" . + "push \@subs,\$11;" . + "push \@subs,\$12;" . + "push \@subs,\$13;" . + "push \@subs,\$14;" . + "push \@subs,\$15;" . + "push \@subs,\$16;" . + "push \@subs,\$'; }"; + + undef $REGERROR; + undef $REGMARK; + + eval "${cmd} (\$x =~ ${pattern}) {" . $pushes; + + if ($@) + { + printf $outfile "Error: $@\n"; + next NEXT_RE; + } + elsif (scalar(@subs) == 0) + { + printf $outfile "No match"; + if (defined $REGERROR && $REGERROR != 1) + { printf $outfile (", mark = %s", &pchars($REGERROR)); } + printf $outfile "\n"; + } + else + { + while (scalar(@subs) != 0) + { + printf $outfile (" 0: %s\n", &pchars($subs[0])); + printf $outfile (" 0+ %s\n", &pchars($subs[17])) if $showrest; + $last_printed = 0; + for ($i = 1; $i <= 16; $i++) + { + if (defined $subs[$i]) + { + while ($last_printed++ < $i-1) + { printf $outfile ("%2d: \n", $last_printed); } + printf $outfile ("%2d: %s\n", $i, &pchars($subs[$i])); + $last_printed = $i; + } + } + splice(@subs, 0, 18); + } + + # It seems that $REGMARK is not marked as UTF-8 even when use utf8 is + # set and the input pattern was a UTF-8 string. We can, however, force + # it to be so marked. + + if (defined $REGMARK && $REGMARK != 1) + { + $xx = $REGMARK; + $xx = Encode::decode_utf8($xx) if $utf8; + printf $outfile ("MK: %s\n", &pchars($xx)); + } + } + } + } + +# printf $outfile "\n"; + +# End diff --git a/src/third_party/pcre-8.41/ucp.h b/src/third_party/pcre-8.41/ucp.h new file mode 100644 index 00000000000..2fa00296e42 --- /dev/null +++ b/src/third_party/pcre-8.41/ucp.h @@ -0,0 +1,224 @@ +/************************************************* +* Unicode Property Table handler * +*************************************************/ + +#ifndef _UCP_H +#define _UCP_H + +/* This file contains definitions of the property values that are returned by +the UCD access macros. New values that are added for new releases of Unicode +should always be at the end of each enum, for backwards compatibility. + +IMPORTANT: Note also that the specific numeric values of the enums have to be +the same as the values that are generated by the maint/MultiStage2.py script, +where the equivalent property descriptive names are listed in vectors. + +ALSO: The specific values of the first two enums are assumed for the table +called catposstab in pcre_compile.c. */ + +/* These are the general character categories. */ + +enum { + ucp_C, /* Other */ + ucp_L, /* Letter */ + ucp_M, /* Mark */ + ucp_N, /* Number */ + ucp_P, /* Punctuation */ + ucp_S, /* Symbol */ + ucp_Z /* Separator */ +}; + +/* These are the particular character categories. */ + +enum { + ucp_Cc, /* Control */ + ucp_Cf, /* Format */ + ucp_Cn, /* Unassigned */ + ucp_Co, /* Private use */ + ucp_Cs, /* Surrogate */ + ucp_Ll, /* Lower case letter */ + ucp_Lm, /* Modifier letter */ + ucp_Lo, /* Other letter */ + ucp_Lt, /* Title case letter */ + ucp_Lu, /* Upper case letter */ + ucp_Mc, /* Spacing mark */ + ucp_Me, /* Enclosing mark */ + ucp_Mn, /* Non-spacing mark */ + ucp_Nd, /* Decimal number */ + ucp_Nl, /* Letter number */ + ucp_No, /* Other number */ + ucp_Pc, /* Connector punctuation */ + ucp_Pd, /* Dash punctuation */ + ucp_Pe, /* Close punctuation */ + ucp_Pf, /* Final punctuation */ + ucp_Pi, /* Initial punctuation */ + ucp_Po, /* Other punctuation */ + ucp_Ps, /* Open punctuation */ + ucp_Sc, /* Currency symbol */ + ucp_Sk, /* Modifier symbol */ + ucp_Sm, /* Mathematical symbol */ + ucp_So, /* Other symbol */ + ucp_Zl, /* Line separator */ + ucp_Zp, /* Paragraph separator */ + ucp_Zs /* Space separator */ +}; + +/* These are grapheme break properties. Note that the code for processing them +assumes that the values are less than 16. If more values are added that take +the number to 16 or more, the code will have to be rewritten. */ + +enum { + ucp_gbCR, /* 0 */ + ucp_gbLF, /* 1 */ + ucp_gbControl, /* 2 */ + ucp_gbExtend, /* 3 */ + ucp_gbPrepend, /* 4 */ + ucp_gbSpacingMark, /* 5 */ + ucp_gbL, /* 6 Hangul syllable type L */ + ucp_gbV, /* 7 Hangul syllable type V */ + ucp_gbT, /* 8 Hangul syllable type T */ + ucp_gbLV, /* 9 Hangul syllable type LV */ + ucp_gbLVT, /* 10 Hangul syllable type LVT */ + ucp_gbRegionalIndicator, /* 11 */ + ucp_gbOther /* 12 */ +}; + +/* These are the script identifications. */ + +enum { + ucp_Arabic, + ucp_Armenian, + ucp_Bengali, + ucp_Bopomofo, + ucp_Braille, + ucp_Buginese, + ucp_Buhid, + ucp_Canadian_Aboriginal, + ucp_Cherokee, + ucp_Common, + ucp_Coptic, + ucp_Cypriot, + ucp_Cyrillic, + ucp_Deseret, + ucp_Devanagari, + ucp_Ethiopic, + ucp_Georgian, + ucp_Glagolitic, + ucp_Gothic, + ucp_Greek, + ucp_Gujarati, + ucp_Gurmukhi, + ucp_Han, + ucp_Hangul, + ucp_Hanunoo, + ucp_Hebrew, + ucp_Hiragana, + ucp_Inherited, + ucp_Kannada, + ucp_Katakana, + ucp_Kharoshthi, + ucp_Khmer, + ucp_Lao, + ucp_Latin, + ucp_Limbu, + ucp_Linear_B, + ucp_Malayalam, + ucp_Mongolian, + ucp_Myanmar, + ucp_New_Tai_Lue, + ucp_Ogham, + ucp_Old_Italic, + ucp_Old_Persian, + ucp_Oriya, + ucp_Osmanya, + ucp_Runic, + ucp_Shavian, + ucp_Sinhala, + ucp_Syloti_Nagri, + ucp_Syriac, + ucp_Tagalog, + ucp_Tagbanwa, + ucp_Tai_Le, + ucp_Tamil, + ucp_Telugu, + ucp_Thaana, + ucp_Thai, + ucp_Tibetan, + ucp_Tifinagh, + ucp_Ugaritic, + ucp_Yi, + /* New for Unicode 5.0: */ + ucp_Balinese, + ucp_Cuneiform, + ucp_Nko, + ucp_Phags_Pa, + ucp_Phoenician, + /* New for Unicode 5.1: */ + ucp_Carian, + ucp_Cham, + ucp_Kayah_Li, + ucp_Lepcha, + ucp_Lycian, + ucp_Lydian, + ucp_Ol_Chiki, + ucp_Rejang, + ucp_Saurashtra, + ucp_Sundanese, + ucp_Vai, + /* New for Unicode 5.2: */ + ucp_Avestan, + ucp_Bamum, + ucp_Egyptian_Hieroglyphs, + ucp_Imperial_Aramaic, + ucp_Inscriptional_Pahlavi, + ucp_Inscriptional_Parthian, + ucp_Javanese, + ucp_Kaithi, + ucp_Lisu, + ucp_Meetei_Mayek, + ucp_Old_South_Arabian, + ucp_Old_Turkic, + ucp_Samaritan, + ucp_Tai_Tham, + ucp_Tai_Viet, + /* New for Unicode 6.0.0: */ + ucp_Batak, + ucp_Brahmi, + ucp_Mandaic, + /* New for Unicode 6.1.0: */ + ucp_Chakma, + ucp_Meroitic_Cursive, + ucp_Meroitic_Hieroglyphs, + ucp_Miao, + ucp_Sharada, + ucp_Sora_Sompeng, + ucp_Takri, + /* New for Unicode 7.0.0: */ + ucp_Bassa_Vah, + ucp_Caucasian_Albanian, + ucp_Duployan, + ucp_Elbasan, + ucp_Grantha, + ucp_Khojki, + ucp_Khudawadi, + ucp_Linear_A, + ucp_Mahajani, + ucp_Manichaean, + ucp_Mende_Kikakui, + ucp_Modi, + ucp_Mro, + ucp_Nabataean, + ucp_Old_North_Arabian, + ucp_Old_Permic, + ucp_Pahawh_Hmong, + ucp_Palmyrene, + ucp_Psalter_Pahlavi, + ucp_Pau_Cin_Hau, + ucp_Siddham, + ucp_Tirhuta, + ucp_Warang_Citi +}; + +#endif + +/* End of ucp.h */ diff --git a/src/third_party/scripts/pcre_get_sources.sh b/src/third_party/scripts/pcre_get_sources.sh index da73652ef66..16c79a3e911 100644 --- a/src/third_party/scripts/pcre_get_sources.sh +++ b/src/third_party/scripts/pcre_get_sources.sh @@ -16,7 +16,7 @@ if [ "$#" -ne 0 ]; then exit 1 fi -VERSION=8.39 +VERSION=8.41 NAME=pcre TARBALL=$NAME-$VERSION.tar.gz TARBALL_DIR=$NAME-$VERSION diff --git a/src/third_party/wiredtiger/build_linux/wiredtiger_config.h b/src/third_party/wiredtiger/build_linux/wiredtiger_config.h index 1122e1e319d..6fffea61ad1 100644 --- a/src/third_party/wiredtiger/build_linux/wiredtiger_config.h +++ b/src/third_party/wiredtiger/build_linux/wiredtiger_config.h @@ -82,6 +82,9 @@ /* Define to 1 if you have the `posix_memalign' function. */ #define HAVE_POSIX_MEMALIGN 1 +/* Define to 1 if pthread condition variables support monotonic clocks. */ +#define HAVE_PTHREAD_COND_MONOTONIC 1 + /* Define to 1 if you have the header file. */ /* #undef HAVE_PTHREAD_NP_H */ diff --git a/src/third_party/wiredtiger/build_posix/configure.ac.in b/src/third_party/wiredtiger/build_posix/configure.ac.in index 0fef587b4b8..415545a0d56 100644 --- a/src/third_party/wiredtiger/build_posix/configure.ac.in +++ b/src/third_party/wiredtiger/build_posix/configure.ac.in @@ -160,6 +160,44 @@ AS_CASE([$host_os], [darwin*], [], [AC_CHECK_FUNCS([fdatasync])]) # the generic declaration in AC_CHECK_FUNCS is incompatible. AX_FUNC_POSIX_MEMALIGN +# Check for POSIX condition variables with monotonic clock support +AC_CACHE_CHECK([for condition waits with monotonic clock support], + [wt_cv_pthread_cond_monotonic], + [AC_RUN_IFELSE([AC_LANG_SOURCE([[ +#include +#include +#include +#include + +int main() +{ + int ret; + pthread_condattr_t condattr; + pthread_cond_t cond; + pthread_mutex_t mtx; + struct timespec ts; + + if ((ret = pthread_condattr_init(&condattr)) != 0) exit(1); + if ((ret = pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC)) != 0) exit(1); + if ((ret = pthread_cond_init(&cond, &condattr)) != 0) exit(1); + if ((ret = pthread_mutex_init(&mtx, NULL)) != 0) exit(1); + if ((ret = clock_gettime(CLOCK_MONOTONIC, &ts)) != 0) exit(1); + ts.tv_sec += 1; + if ((ret = pthread_mutex_lock(&mtx)) != 0) exit(1); + if ((ret = pthread_cond_timedwait(&cond, &mtx, &ts)) != 0 && ret != EINTR && ret != ETIMEDOUT) exit(1); + + exit(0); +} + ]])], + [wt_pthread_cond_monotonic=yes], + [wt_pthread_cond_monotonic=no], + [wt_pthread_cond_monotonic=no])]) +AC_MSG_RESULT($wt_pthread_cond_monotonic) +if test "$wt_pthread_cond_monotonic" = "yes" ; then + AC_DEFINE([HAVE_PTHREAD_COND_MONOTONIC], [1], + [Define to 1 if pthread condition variables support monotonic clocks.]) +fi + AC_SYS_LARGEFILE AC_C_BIGENDIAN diff --git a/src/third_party/wiredtiger/build_win/wiredtiger_config.h b/src/third_party/wiredtiger/build_win/wiredtiger_config.h index 78d2784cb70..8babdbfdc1b 100644 --- a/src/third_party/wiredtiger/build_win/wiredtiger_config.h +++ b/src/third_party/wiredtiger/build_win/wiredtiger_config.h @@ -79,6 +79,9 @@ /* Define to 1 if you have the header file. */ /* #undef HAVE_MEMORY_H */ +/* Define to 1 if pthread condition variables support monotonic clocks. */ +/* #undef HAVE_PTHREAD_COND_MONOTONIC */ + /* Define to 1 if you have the `posix_fadvise' function. */ /* #undef HAVE_POSIX_FADVISE */ diff --git a/src/third_party/wiredtiger/dist/api_data.py b/src/third_party/wiredtiger/dist/api_data.py index 22600dd5e29..596099647be 100644 --- a/src/third_party/wiredtiger/dist/api_data.py +++ b/src/third_party/wiredtiger/dist/api_data.py @@ -753,6 +753,9 @@ wiredtiger_open_common =\ Config('session_scratch_max', '2MB', r''' maximum memory to cache in each session''', type='int', undoc=True), + Config('session_table_cache', 'true', r''' + Maintain a per-session cache of tables''', + type='boolean'), Config('transaction_sync', '', r''' how to sync log records when the transaction commits''', type='category', subconfig=[ diff --git a/src/third_party/wiredtiger/dist/filelist b/src/third_party/wiredtiger/dist/filelist index 5a3348b940a..f53509e96ec 100644 --- a/src/third_party/wiredtiger/dist/filelist +++ b/src/third_party/wiredtiger/dist/filelist @@ -191,6 +191,7 @@ src/support/rand.c src/support/scratch.c src/support/stat.c src/support/thread_group.c +src/support/time.c src/txn/txn.c src/txn/txn_ckpt.c src/txn/txn_ext.c diff --git a/src/third_party/wiredtiger/dist/flags.py b/src/third_party/wiredtiger/dist/flags.py index 64b5d789e72..1ce717f3586 100644 --- a/src/third_party/wiredtiger/dist/flags.py +++ b/src/third_party/wiredtiger/dist/flags.py @@ -32,7 +32,6 @@ flags = { 'READ_PREV', 'READ_RESTART_OK', 'READ_SKIP_INTL', - 'READ_SKIP_LEAF', 'READ_TRUNCATE', 'READ_WONT_NEED', ], @@ -111,6 +110,7 @@ flags = { 'CONN_SERVER_LSM', 'CONN_SERVER_STATISTICS', 'CONN_SERVER_SWEEP', + 'CONN_TABLE_CACHE', 'CONN_WAS_BACKUP', ], 'session' : [ diff --git a/src/third_party/wiredtiger/dist/s_string.ok b/src/third_party/wiredtiger/dist/s_string.ok index f3852d00ac8..fad8c2f021a 100644 --- a/src/third_party/wiredtiger/dist/s_string.ok +++ b/src/third_party/wiredtiger/dist/s_string.ok @@ -807,6 +807,7 @@ intl intnum intpack intptr +intr intrin inuse io @@ -1060,6 +1061,7 @@ rebalancing recno recnos reconfig +reconfigures reconfiguring recsize rectype diff --git a/src/third_party/wiredtiger/dist/stat_data.py b/src/third_party/wiredtiger/dist/stat_data.py index 512892eb44d..b66e95ce49b 100644 --- a/src/third_party/wiredtiger/dist/stat_data.py +++ b/src/third_party/wiredtiger/dist/stat_data.py @@ -431,11 +431,19 @@ connection_stats = [ ########################################## YieldStat('application_cache_time', 'application thread time waiting for cache (usecs)'), YieldStat('application_evict_time', 'application thread time evicting (usecs)'), + YieldStat('child_modify_blocked_page', 'page reconciliation yielded due to child modification'), + YieldStat('conn_close_blocked_lsm', 'connection close yielded for lsm manager shutdown'), + YieldStat('dhandle_lock_blocked', 'data handle lock yielded'), + YieldStat('log_server_sync_blocked', 'log server sync yielded for log write'), YieldStat('page_busy_blocked', 'page acquire busy blocked'), + YieldStat('page_del_rollback_blocked', 'page delete rollback yielded for instantiation'), YieldStat('page_forcible_evict_blocked', 'page acquire eviction blocked'), + YieldStat('page_index_slot_blocked', 'reference for page index and slot yielded'), YieldStat('page_locked_blocked', 'page acquire locked blocked'), YieldStat('page_read_blocked', 'page acquire read blocked'), YieldStat('page_sleep', 'page acquire time sleeping (usecs)'), + YieldStat('tree_descend_blocked', 'tree descend one level yielded for split page index update'), + YieldStat('txn_release_blocked', 'connection close blocked waiting for transaction state stabilization'), ] connection_stats = sorted(connection_stats, key=attrgetter('desc')) diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index 03d5746ffde..e867147e59e 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "d2f2eae6d7718a53ac5bacf7141773ee0696f3c6", + "commit": "0b36171f4aa0bea6ab6118b0d3bcf6329a24939e", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-3.4" diff --git a/src/third_party/wiredtiger/src/async/async_worker.c b/src/third_party/wiredtiger/src/async/async_worker.c index 11f59ed14f1..89877652f1e 100644 --- a/src/third_party/wiredtiger/src/async/async_worker.c +++ b/src/third_party/wiredtiger/src/async/async_worker.c @@ -301,11 +301,10 @@ __wt_async_worker(void *arg) WT_ERR(__async_op_dequeue(conn, session, &op)); if (op != NULL && op != &async->flush_op) { /* - * If an operation fails, we want the worker thread to - * keep running, unless there is a panic. + * Operation failure doesn't cause the worker thread to + * exit. */ (void)__async_worker_op(session, op, &worker); - WT_ERR(WT_SESSION_CHECK_PANIC(session)); } else if (async->flush_state == WT_ASYNC_FLUSHING) { /* * Worker flushing going on. Last worker to the party diff --git a/src/third_party/wiredtiger/src/btree/bt_cursor.c b/src/third_party/wiredtiger/src/btree/bt_cursor.c index f0aa632551b..28977eee9f5 100644 --- a/src/third_party/wiredtiger/src/btree/bt_cursor.c +++ b/src/third_party/wiredtiger/src/btree/bt_cursor.c @@ -52,15 +52,18 @@ __cursor_state_restore(WT_CURSOR *cursor, WT_CURFILE_STATE *state) /* * __cursor_page_pinned -- - * Return if we have a page pinned and it's not been flagged for forced - * eviction (the forced eviction test is so we periodically release pages - * grown too large). + * Return if we have a page pinned. */ static inline bool -__cursor_page_pinned(WT_CURSOR_BTREE *cbt) +__cursor_page_pinned(WT_CURSOR_BTREE *cbt, bool eviction_ok) { + /* + * Optionally fail the page-pinned test when the page is flagged for + * forced eviction (so we periodically release pages grown too large). + * The test is optional as not all callers can release pinned pages. + */ return (F_ISSET(cbt, WT_CBT_ACTIVE) && - cbt->ref->page->read_gen != WT_READGEN_OLDEST); + (!eviction_ok || cbt->ref->page->read_gen != WT_READGEN_OLDEST)); } /* @@ -156,8 +159,10 @@ __cursor_disable_bulk(WT_SESSION_IMPL *session, WT_BTREE *btree) * into a tree. Eviction is disabled when an empty tree is opened, and * it must only be enabled once. */ - if (__wt_atomic_cas8(&btree->original, 1, 0)) + if (__wt_atomic_cas8(&btree->original, 1, 0)) { + btree->evict_disabled_open = false; __wt_evict_file_exclusive_off(session); + } } /* @@ -443,7 +448,7 @@ __wt_btcur_search(WT_CURSOR_BTREE *cbt) * from the root. */ valid = false; - if (__cursor_page_pinned(cbt)) { + if (__cursor_page_pinned(cbt, true)) { __wt_txn_cursor_op(session); WT_ERR(btree->type == BTREE_ROW ? @@ -535,7 +540,7 @@ __wt_btcur_search_near(WT_CURSOR_BTREE *cbt, int *exactp) * existing record. */ valid = false; - if (btree->type == BTREE_ROW && __cursor_page_pinned(cbt)) { + if (btree->type == BTREE_ROW && __cursor_page_pinned(cbt, true)) { __wt_txn_cursor_op(session); WT_ERR(__cursor_row_search(session, cbt, cbt->ref, true)); @@ -665,7 +670,7 @@ __wt_btcur_insert(WT_CURSOR_BTREE *cbt) * configured for append aren't included, regardless of whether or not * they meet all other criteria. */ - if (__cursor_page_pinned(cbt) && + if (__cursor_page_pinned(cbt, true) && F_ISSET_ALL(cursor, WT_CURSTD_KEY_INT | WT_CURSTD_OVERWRITE) && !append_key) { WT_ERR(__wt_txn_autocommit_check(session)); @@ -880,8 +885,22 @@ __wt_btcur_remove(WT_CURSOR_BTREE *cbt) * removed, and the record must exist with a positioned cursor. The * cursor won't be positioned on a page with an external key set, but * be sure. + * + * There's trickiness in the page-pinned check. By definition a remove + * operation leaves a cursor positioned if it's initially positioned. + * However, if every item on the page is deleted and we unpin the page, + * eviction might delete the page and our search will re-instantiate an + * empty page for us. Cursor remove returns not-found whether or not + * that eviction/deletion happens and it's OK unless cursor-overwrite + * is configured (which means we return success even if there's no item + * to delete). In that case, we'll fail when we try to point the cursor + * at the key on the page to satisfy the positioned requirement. It's + * arguably safe to simply leave the key initialized in the cursor (as + * that's all a positioned cursor implies), but it's probably safer to + * avoid page eviction entirely in the positioned case. */ - if (__cursor_page_pinned(cbt) && F_ISSET(cursor, WT_CURSTD_KEY_INT)) { + if (__cursor_page_pinned(cbt, !positioned) && + F_ISSET(cursor, WT_CURSTD_KEY_INT)) { WT_ERR(__wt_txn_autocommit_check(session)); /* @@ -1024,7 +1043,8 @@ __wt_btcur_update(WT_CURSOR_BTREE *cbt) * cursor won't be positioned on a page with an external key set, but * be sure. */ - if (__cursor_page_pinned(cbt) && F_ISSET(cursor, WT_CURSTD_KEY_INT)) { + if (__cursor_page_pinned(cbt, true) && + F_ISSET(cursor, WT_CURSTD_KEY_INT)) { WT_ERR(__wt_txn_autocommit_check(session)); /* * The cursor position may not be exact (the cursor's comparison diff --git a/src/third_party/wiredtiger/src/btree/bt_delete.c b/src/third_party/wiredtiger/src/btree/bt_delete.c index b55ad291c5e..5c4625044d3 100644 --- a/src/third_party/wiredtiger/src/btree/bt_delete.c +++ b/src/third_party/wiredtiger/src/btree/bt_delete.c @@ -153,6 +153,7 @@ void __wt_delete_page_rollback(WT_SESSION_IMPL *session, WT_REF *ref) { WT_UPDATE **upd; + uint64_t yield_count; /* * If the page is still "deleted", it's as we left it, reset the state @@ -160,7 +161,7 @@ __wt_delete_page_rollback(WT_SESSION_IMPL *session, WT_REF *ref) * instantiated or being instantiated. Loop because it's possible for * the page to return to the deleted state if instantiation fails. */ - for (;; __wt_yield()) + for (yield_count = 0;; yield_count++, __wt_yield()) switch (ref->state) { case WT_REF_DISK: case WT_REF_READING: @@ -173,7 +174,7 @@ __wt_delete_page_rollback(WT_SESSION_IMPL *session, WT_REF *ref) */ if (__wt_atomic_casv32( &ref->state, WT_REF_DELETED, WT_REF_DISK)) - return; + goto done; break; case WT_REF_LOCKED: /* @@ -203,8 +204,10 @@ __wt_delete_page_rollback(WT_SESSION_IMPL *session, WT_REF *ref) */ __wt_free(session, ref->page_del->update_list); __wt_free(session, ref->page_del); - return; + goto done; } + +done: WT_STAT_CONN_INCRV(session, page_del_rollback_blocked, yield_count); } /* diff --git a/src/third_party/wiredtiger/src/btree/bt_handle.c b/src/third_party/wiredtiger/src/btree/bt_handle.c index a0da7df0998..8637a8cd751 100644 --- a/src/third_party/wiredtiger/src/btree/bt_handle.c +++ b/src/third_party/wiredtiger/src/btree/bt_handle.c @@ -66,7 +66,6 @@ __wt_btree_open(WT_SESSION_IMPL *session, const char *op_cfg[]) WT_DATA_HANDLE *dhandle; WT_DECL_RET; size_t root_addr_size; - uint32_t mask; uint8_t root_addr[WT_BTREE_MAX_ADDR_COOKIE]; const char *filename; bool creation, forced_salvage, readonly; @@ -75,15 +74,14 @@ __wt_btree_open(WT_SESSION_IMPL *session, const char *op_cfg[]) dhandle = session->dhandle; /* - * This may be a re-open of an underlying object and we have to clean - * up. We can't clear the operation flags, however, they're set by the - * connection handle software that called us. + * This may be a re-open, clean up the btree structure. + * Clear the fields that don't persist across a re-open. + * Clear all flags other than the operation flags (which are set by the + * connection handle software that called us). */ WT_RET(__btree_clear(session)); - - mask = F_MASK(btree, WT_BTREE_SPECIAL_FLAGS); - memset(btree, 0, sizeof(*btree)); - btree->flags = mask; + memset(btree, 0, WT_BTREE_CLEAR_SIZE); + F_CLR(btree, ~WT_BTREE_SPECIAL_FLAGS); /* Set the data handle first, our called functions reasonably use it. */ btree->dhandle = dhandle; @@ -185,13 +183,19 @@ __wt_btree_open(WT_SESSION_IMPL *session, const char *op_cfg[]) * * Files that can still be bulk-loaded cannot be evicted. * Permanently cache-resident files can never be evicted. - * Special operations don't enable eviction. (The underlying commands - * may turn on eviction, but it's their decision.) + * Special operations don't enable eviction. The underlying commands may + * turn on eviction (for example, verify turns on eviction while working + * a file to keep from consuming the cache), but it's their decision. If + * an underlying command reconfigures eviction, it must either clear the + * evict-disabled-open flag or restore the eviction configuration when + * finished so that handle close behaves correctly. */ if (btree->original || F_ISSET(btree, WT_BTREE_IN_MEMORY | WT_BTREE_REBALANCE | - WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY)) + WT_BTREE_SALVAGE | WT_BTREE_UPGRADE | WT_BTREE_VERIFY)) { WT_ERR(__wt_evict_file_exclusive_on(session)); + btree->evict_disabled_open = true; + } if (0) { err: WT_TRET(__wt_btree_close(session)); @@ -228,6 +232,15 @@ __wt_btree_close(WT_SESSION_IMPL *session) return (0); F_SET(btree, WT_BTREE_CLOSED); + /* + * If we turned eviction off and never turned it back on, do that now, + * otherwise the counter will be off. + */ + if (btree->evict_disabled_open) { + btree->evict_disabled_open = false; + __wt_evict_file_exclusive_off(session); + } + /* Discard any underlying block manager resources. */ if ((bm = btree->bm) != NULL) { btree->bm = NULL; @@ -447,9 +460,11 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt) WT_RET(__wt_rwlock_init(session, &btree->ovfl_lock)); WT_RET(__wt_spin_init(session, &btree->flush_lock, "btree flush")); - btree->checkpointing = WT_CKPT_OFF; /* Not checkpointing */ btree->modified = false; /* Clean */ - btree->write_gen = ckpt->write_gen; /* Write generation */ + + btree->checkpointing = WT_CKPT_OFF; /* Not checkpointing */ + btree->write_gen = ckpt->write_gen; /* Write generation */ + btree->checkpoint_gen = S2C(session)->txn_global.checkpoint_gen; return (0); } diff --git a/src/third_party/wiredtiger/src/btree/bt_random.c b/src/third_party/wiredtiger/src/btree/bt_random.c index c5948ec4ab5..b4f05c440ba 100644 --- a/src/third_party/wiredtiger/src/btree/bt_random.c +++ b/src/third_party/wiredtiger/src/btree/bt_random.c @@ -395,8 +395,7 @@ __wt_btcur_next_random(WT_CURSOR_BTREE *cbt) */ for (skip = cbt->next_random_leaf_skip; cbt->ref == NULL || skip > 0;) { n = skip; - WT_ERR(__wt_tree_walk_skip(session, &cbt->ref, &skip, - WT_READ_NO_GEN | WT_READ_SKIP_INTL | WT_READ_WONT_NEED)); + WT_ERR(__wt_tree_walk_skip(session, &cbt->ref, &skip)); if (n == skip) { if (skip == 0) break; diff --git a/src/third_party/wiredtiger/src/btree/bt_walk.c b/src/third_party/wiredtiger/src/btree/bt_walk.c index 86484feb7c9..c22b99c55d0 100644 --- a/src/third_party/wiredtiger/src/btree/bt_walk.c +++ b/src/third_party/wiredtiger/src/btree/bt_walk.c @@ -18,9 +18,16 @@ __ref_index_slot(WT_SESSION_IMPL *session, { WT_PAGE_INDEX *pindex; WT_REF **start, **stop, **p, **t; + uint64_t yield_count; uint32_t entries, slot; - for (;;) { + /* + * If we don't find our reference, the page split and our home + * pointer references the wrong page. When internal pages + * split, their WT_REF structure home values are updated; yield + * and wait for that to happen. + */ + for (yield_count = 0;; yield_count++, __wt_yield()) { /* * Copy the parent page's index value: the page can split at * any time, but the index's value is always valid, even if @@ -59,18 +66,13 @@ __ref_index_slot(WT_SESSION_IMPL *session, } } - /* - * If we don't find our reference, the page split and our home - * pointer references the wrong page. When internal pages - * split, their WT_REF structure home values are updated; yield - * and wait for that to happen. - */ - __wt_yield(); } found: WT_ASSERT(session, pindex->index[slot] == ref); *pindexp = pindex; *slotp = slot; + + WT_STAT_CONN_INCRV(session, page_index_slot_blocked, yield_count); } /* @@ -177,12 +179,13 @@ __ref_descend_prev( WT_SESSION_IMPL *session, WT_REF *ref, WT_PAGE_INDEX **pindexp) { WT_PAGE_INDEX *pindex; + uint64_t yield_count; /* * We're passed a child page into which we're descending, and on which * we have a hazard pointer. */ - for (;; __wt_yield()) { + for (yield_count = 0;; yield_count++, __wt_yield()) { /* * There's a split race when a cursor moving backwards through * the tree descends the tree. If we're splitting an internal @@ -242,6 +245,7 @@ __ref_descend_prev( break; } *pindexp = pindex; + WT_STAT_CONN_INCRV(session, tree_descend_blocked, yield_count); } /* @@ -497,29 +501,21 @@ restart: /* } /* - * Optionally skip leaf pages: skip all leaf pages if - * WT_READ_SKIP_LEAF is set, when the skip-leaf-count - * variable is non-zero, skip some count of leaf pages. - * If this page is disk-based, crack the cell to figure - * out it's a leaf page without reading it. + * Optionally skip leaf pages: when the skip-leaf-count + * variable is non-zero, skip some count of leaf pages, + * then take the next leaf page we can. * - * If skipping some number of leaf pages, decrement the - * count of pages to zero, and then take the next leaf - * page we can. Be cautious around the page decrement, - * if for some reason don't take this particular page, - * we can take the next one, and, there are additional - * tests/decrements when we're about to return a leaf - * page. + * The reason to do some of this work here (rather than + * in our caller), is because we can look at the cell + * and know it's a leaf page without reading it into + * memory. If this page is disk-based, crack the cell + * to figure out it's a leaf page without reading it. */ - if (skipleafcntp != NULL || LF_ISSET(WT_READ_SKIP_LEAF)) - if (__ref_is_leaf(ref)) { - if (LF_ISSET(WT_READ_SKIP_LEAF)) - break; - if (*skipleafcntp > 0) { - --*skipleafcntp; - break; - } - } + if (skipleafcntp != NULL && + *skipleafcntp > 0 && __ref_is_leaf(ref)) { + --*skipleafcntp; + break; + } ret = __wt_page_swap(session, couple, ref, WT_READ_NOTFOUND_OK | WT_READ_RESTART_OK | flags); @@ -626,34 +622,18 @@ descend: empty_internal = true; session, ref, &pindex); slot = pindex->entries - 1; } - } else { - /* - * At the lowest tree level (considering a leaf - * page), turn off the initial-descent state. - * Descent race tests are different when moving - * through the tree vs. the initial descent. - */ - initial_descent = false; - - /* - * Optionally skip leaf pages, the second half. - * We didn't have an on-page cell to figure out - * if it was a leaf page, we had to acquire the - * hazard pointer and look at the page. - */ - if (skipleafcntp != NULL || - LF_ISSET(WT_READ_SKIP_LEAF)) { - if (LF_ISSET(WT_READ_SKIP_LEAF)) - break; - if (*skipleafcntp > 0) { - --*skipleafcntp; - break; - } - } - - *refp = ref; - goto done; + continue; } + + /* + * The tree-walk restart code knows we return any leaf + * page we acquire (never hazard-pointer coupling on + * after acquiring a leaf page), and asserts no restart + * happens while holding a leaf page. This page must be + * returned to our caller. + */ + *refp = ref; + goto done; } } @@ -690,8 +670,29 @@ __wt_tree_walk_count(WT_SESSION_IMPL *session, * of leaf pages before returning. */ int -__wt_tree_walk_skip(WT_SESSION_IMPL *session, - WT_REF **refp, uint64_t *skipleafcntp, uint32_t flags) +__wt_tree_walk_skip( + WT_SESSION_IMPL *session, WT_REF **refp, uint64_t *skipleafcntp) { - return (__tree_walk_internal(session, refp, NULL, skipleafcntp, flags)); + /* + * Optionally skip leaf pages, the second half. The tree-walk function + * didn't have an on-page cell it could use to figure out if the page + * was a leaf page or not, it had to acquire the hazard pointer and look + * at the page. The tree-walk code never acquires a hazard pointer on a + * leaf page without returning it, and it's not trivial to change that. + * So, the tree-walk code returns all leaf pages here and we deal with + * decrementing the count. + */ + do { + WT_RET(__tree_walk_internal(session, refp, NULL, skipleafcntp, + WT_READ_NO_GEN | WT_READ_SKIP_INTL | WT_READ_WONT_NEED)); + + /* + * The walk skipped internal pages, any page returned must be a + * leaf page. + */ + if (*skipleafcntp > 0) + --*skipleafcntp; + } while (*skipleafcntp > 0); + + return (0); } diff --git a/src/third_party/wiredtiger/src/config/config_def.c b/src/third_party/wiredtiger/src/config/config_def.c index f152fbacad4..c8a8c525751 100644 --- a/src/third_party/wiredtiger/src/config/config_def.c +++ b/src/third_party/wiredtiger/src/config/config_def.c @@ -733,6 +733,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { { "readonly", "boolean", NULL, NULL, NULL, 0 }, { "session_max", "int", NULL, "min=1", NULL, 0 }, { "session_scratch_max", "int", NULL, NULL, NULL, 0 }, + { "session_table_cache", "boolean", NULL, NULL, NULL, 0 }, { "shared_cache", "category", NULL, NULL, confchk_wiredtiger_open_shared_cache_subconfigs, 5 }, @@ -820,6 +821,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { { "readonly", "boolean", NULL, NULL, NULL, 0 }, { "session_max", "int", NULL, "min=1", NULL, 0 }, { "session_scratch_max", "int", NULL, NULL, NULL, 0 }, + { "session_table_cache", "boolean", NULL, NULL, NULL, 0 }, { "shared_cache", "category", NULL, NULL, confchk_wiredtiger_open_shared_cache_subconfigs, 5 }, @@ -904,6 +906,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { { "readonly", "boolean", NULL, NULL, NULL, 0 }, { "session_max", "int", NULL, "min=1", NULL, 0 }, { "session_scratch_max", "int", NULL, NULL, NULL, 0 }, + { "session_table_cache", "boolean", NULL, NULL, NULL, 0 }, { "shared_cache", "category", NULL, NULL, confchk_wiredtiger_open_shared_cache_subconfigs, 5 }, @@ -986,6 +989,7 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { { "readonly", "boolean", NULL, NULL, NULL, 0 }, { "session_max", "int", NULL, "min=1", NULL, 0 }, { "session_scratch_max", "int", NULL, NULL, NULL, 0 }, + { "session_table_cache", "boolean", NULL, NULL, NULL, 0 }, { "shared_cache", "category", NULL, NULL, confchk_wiredtiger_open_shared_cache_subconfigs, 5 }, @@ -1276,14 +1280,14 @@ static const WT_CONFIG_ENTRY config_entries[] = { "file_max=100MB,path=\".\",prealloc=true,recover=on," "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," "lsm_merge=true,mmap=true,multiprocess=false,readonly=false," - "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" - ",name=,quota=0,reserve=0,size=500MB),statistics=none," - "statistics_log=(json=false,on_close=false,path=\".\",sources=," - "timestamp=\"%b %d %H:%M:%S\",wait=0)," + "session_max=100,session_scratch_max=2MB,session_table_cache=true" + ",shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," + "statistics=none,statistics_log=(json=false,on_close=false," + "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "transaction_sync=(enabled=false,method=fsync)," "use_environment=true,use_environment_priv=false,verbose=," "write_through=", - confchk_wiredtiger_open, 40 + confchk_wiredtiger_open, 41 }, { "wiredtiger_open_all", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" @@ -1300,14 +1304,14 @@ static const WT_CONFIG_ENTRY config_entries[] = { "file_max=100MB,path=\".\",prealloc=true,recover=on," "zero_fill=false),lsm_manager=(merge=true,worker_thread_max=4)," "lsm_merge=true,mmap=true,multiprocess=false,readonly=false," - "session_max=100,session_scratch_max=2MB,shared_cache=(chunk=10MB" - ",name=,quota=0,reserve=0,size=500MB),statistics=none," - "statistics_log=(json=false,on_close=false,path=\".\",sources=," - "timestamp=\"%b %d %H:%M:%S\",wait=0)," + "session_max=100,session_scratch_max=2MB,session_table_cache=true" + ",shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," + "statistics=none,statistics_log=(json=false,on_close=false," + "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," "transaction_sync=(enabled=false,method=fsync)," "use_environment=true,use_environment_priv=false,verbose=," "version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_all, 41 + confchk_wiredtiger_open_all, 42 }, { "wiredtiger_open_basecfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" @@ -1323,12 +1327,13 @@ static const WT_CONFIG_ENTRY config_entries[] = { "path=\".\",prealloc=true,recover=on,zero_fill=false)," "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," "mmap=true,multiprocess=false,readonly=false,session_max=100," - "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," - "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" - ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" - ",wait=0),transaction_sync=(enabled=false,method=fsync),verbose=," + "session_scratch_max=2MB,session_table_cache=true," + "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," + "statistics=none,statistics_log=(json=false,on_close=false," + "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," + "transaction_sync=(enabled=false,method=fsync),verbose=," "version=(major=0,minor=0),write_through=", - confchk_wiredtiger_open_basecfg, 35 + confchk_wiredtiger_open_basecfg, 36 }, { "wiredtiger_open_usercfg", "async=(enabled=false,ops_max=1024,threads=2),buffer_alignment=-1" @@ -1344,12 +1349,13 @@ static const WT_CONFIG_ENTRY config_entries[] = { "path=\".\",prealloc=true,recover=on,zero_fill=false)," "lsm_manager=(merge=true,worker_thread_max=4),lsm_merge=true," "mmap=true,multiprocess=false,readonly=false,session_max=100," - "session_scratch_max=2MB,shared_cache=(chunk=10MB,name=,quota=0," - "reserve=0,size=500MB),statistics=none,statistics_log=(json=false" - ",on_close=false,path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\"" - ",wait=0),transaction_sync=(enabled=false,method=fsync),verbose=," + "session_scratch_max=2MB,session_table_cache=true," + "shared_cache=(chunk=10MB,name=,quota=0,reserve=0,size=500MB)," + "statistics=none,statistics_log=(json=false,on_close=false," + "path=\".\",sources=,timestamp=\"%b %d %H:%M:%S\",wait=0)," + "transaction_sync=(enabled=false,method=fsync),verbose=," "write_through=", - confchk_wiredtiger_open_usercfg, 34 + confchk_wiredtiger_open_usercfg, 35 }, { NULL, NULL, NULL, 0 } }; diff --git a/src/third_party/wiredtiger/src/conn/conn_api.c b/src/third_party/wiredtiger/src/conn/conn_api.c index 68d45678965..103e4a68f04 100644 --- a/src/third_party/wiredtiger/src/conn/conn_api.c +++ b/src/third_party/wiredtiger/src/conn/conn_api.c @@ -1086,6 +1086,41 @@ err: /* WT_TRET(wt_session->close(wt_session, config)); } + /* + * Perform a system-wide checkpoint so that all tables are consistent + * with each other. Do this before shutting down all the subsystems. + * We have shut down all user sessions, but send in true for waiting + * for internal races. + */ + if (!F_ISSET(conn, WT_CONN_IN_MEMORY | WT_CONN_READONLY)) { + s = NULL; + WT_TRET(__wt_open_internal_session( + conn, "close_ckpt", true, 0, &s)); + if (s != NULL) { + const char *checkpoint_cfg[] = { + WT_CONFIG_BASE(session, WT_SESSION_checkpoint), + NULL + }; + wt_session = &s->iface; + WT_TRET(__wt_txn_checkpoint(s, checkpoint_cfg, true)); + + /* + * Mark the metadata dirty so we flush it on close, + * allowing recovery to be skipped. + */ + WT_WITH_DHANDLE(s, WT_SESSION_META_DHANDLE(s), + __wt_tree_modify_set(s)); + + WT_TRET(wt_session->close(wt_session, config)); + } + } + + if (ret != 0) { + __wt_err(session, ret, + "failure during close, disabling further writes"); + F_SET(conn, WT_CONN_PANIC); + } + WT_TRET(__wt_connection_close(conn)); /* We no longer have a session, don't try to update it. */ @@ -2185,6 +2220,9 @@ wiredtiger_open(const char *home, WT_EVENT_HANDLER *event_handler, WT_ERR(__wt_config_gets(session, cfg, "readonly", &cval)); if (cval.val) F_SET(conn, WT_CONN_READONLY); + WT_ERR(__wt_config_gets(session, cfg, "session_table_cache", &cval)); + if (cval.val) + F_SET(conn, WT_CONN_TABLE_CACHE); /* Configure error messages so we get them right early. */ WT_ERR(__wt_config_gets(session, cfg, "error_prefix", &cval)); @@ -2475,8 +2513,15 @@ err: /* Discard the scratch buffers. */ __wt_scr_discard(session); __wt_scr_discard(&conn->dummy_session); - if (ret != 0) + if (ret != 0) { + /* + * Set panic if we're returning the run recovery error so that + * we don't try to checkpoint data handles. + */ + if (ret == WT_RUN_RECOVERY) + F_SET(conn, WT_CONN_PANIC); WT_TRET(__wt_connection_close(conn)); + } return (ret); } diff --git a/src/third_party/wiredtiger/src/conn/conn_dhandle.c b/src/third_party/wiredtiger/src/conn/conn_dhandle.c index 1816e66b0b7..181f12ab2dd 100644 --- a/src/third_party/wiredtiger/src/conn/conn_dhandle.c +++ b/src/third_party/wiredtiger/src/conn/conn_dhandle.c @@ -317,6 +317,9 @@ __wt_conn_btree_open( WT_ASSERT(session, !F_ISSET(S2C(session), WT_CONN_CLOSING_NO_MORE_OPENS)); + /* Turn off eviction. */ + WT_RET(__wt_evict_file_exclusive_on(session)); + /* * If the handle is already open, it has to be closed so it can be * reopened with a new configuration. @@ -330,11 +333,11 @@ __wt_conn_btree_open( * in the tree that can block the close. */ if (F_ISSET(dhandle, WT_DHANDLE_OPEN)) - WT_RET(__wt_conn_btree_sync_and_close(session, false, false)); + WT_ERR(__wt_conn_btree_sync_and_close(session, false, false)); /* Discard any previous configuration, set up the new configuration. */ __conn_btree_config_clear(session); - WT_RET(__conn_btree_config_set(session)); + WT_ERR(__conn_btree_config_set(session)); /* Set any special flags on the handle. */ F_SET(btree, LF_MASK(WT_BTREE_SPECIAL_FLAGS)); @@ -374,6 +377,8 @@ __wt_conn_btree_open( err: F_CLR(btree, WT_BTREE_SPECIAL_FLAGS); } + __wt_evict_file_exclusive_off(session); + return (ret); } @@ -577,7 +582,7 @@ __conn_dhandle_remove(WT_SESSION_IMPL *session, bool final) WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_HANDLE_LIST_WRITE)); - WT_ASSERT(session, dhandle != conn->cache->evict_file_next); + WT_ASSERT(session, dhandle != conn->cache->walk_tree); /* Check if the handle was reacquired by a session while we waited. */ if (!final && @@ -673,8 +678,8 @@ restart: continue; WT_WITH_DHANDLE(session, dhandle, - WT_TRET(__wt_conn_dhandle_discard_single( - session, true, F_ISSET(conn, WT_CONN_IN_MEMORY)))); + WT_TRET(__wt_conn_dhandle_discard_single(session, true, + F_ISSET(conn, WT_CONN_IN_MEMORY | WT_CONN_PANIC)))); goto restart; } @@ -699,8 +704,8 @@ restart: /* Close the metadata file handle. */ while ((dhandle = TAILQ_FIRST(&conn->dhqh)) != NULL) WT_WITH_DHANDLE(session, dhandle, - WT_TRET(__wt_conn_dhandle_discard_single( - session, true, F_ISSET(conn, WT_CONN_IN_MEMORY)))); + WT_TRET(__wt_conn_dhandle_discard_single(session, true, + F_ISSET(conn, WT_CONN_IN_MEMORY | WT_CONN_PANIC)))); return (ret); } diff --git a/src/third_party/wiredtiger/src/conn/conn_log.c b/src/third_party/wiredtiger/src/conn/conn_log.c index d2ed314fd2e..8b47d3b04c0 100644 --- a/src/third_party/wiredtiger/src/conn/conn_log.c +++ b/src/third_party/wiredtiger/src/conn/conn_log.c @@ -375,6 +375,7 @@ __log_file_server(void *arg) WT_LOG *log; WT_LSN close_end_lsn, min_lsn; WT_SESSION_IMPL *session; + uint64_t yield_count; uint32_t filenum; bool locked; @@ -382,6 +383,7 @@ __log_file_server(void *arg) conn = S2C(session); log = conn->log; locked = false; + yield_count = 0; while (F_ISSET(conn, WT_CONN_SERVER_LOG)) { /* * If there is a log file to close, make sure any outstanding @@ -512,6 +514,7 @@ __log_file_server(void *arg) * thread a chance to run and try again in * this case. */ + yield_count++; __wt_yield(); continue; } @@ -522,8 +525,9 @@ __log_file_server(void *arg) } if (0) { -err: __wt_err(session, ret, "log close server error"); +err: WT_PANIC_MSG(session, ret, "log close server error"); } + WT_STAT_CONN_INCRV(session, log_server_sync_blocked, yield_count); if (locked) __wt_spin_unlock(session, &log->log_sync_lock); return (WT_THREAD_RET_VALUE); @@ -740,7 +744,8 @@ __log_wrlsn_server(void *arg) WT_ERR(__wt_log_force_write(session, 1, NULL)); __wt_log_wrlsn(session, NULL); if (0) { -err: __wt_err(session, ret, "log wrlsn server error"); +err: WT_PANIC_MSG(session, ret, "log wrlsn server error"); + } return (WT_THREAD_RET_VALUE); } @@ -757,7 +762,7 @@ __log_server(void *arg) WT_DECL_RET; WT_LOG *log; WT_SESSION_IMPL *session; - uint64_t timediff; + uint64_t retry, timediff; bool did_work, signalled; session = arg; @@ -783,6 +788,7 @@ __log_server(void *arg) * takes to sync out an earlier file. */ did_work = true; + retry = 0; while (F_ISSET(conn, WT_CONN_SERVER_LOG)) { /* * Slots depend on future activity. Force out buffered @@ -827,7 +833,24 @@ __log_server(void *arg) ret = __log_archive_once(session, 0); __wt_writeunlock( session, &log->log_archive_lock); - WT_ERR(ret); + /* + * It is possible that an external + * process on some systems may prevent + * removal. If we get a permission + * error, retry a few times. + */ + if (ret == EACCES && + retry < WT_RETRY_MAX) { + retry++; + ret = 0; + } else { + /* + * Return the error if there is + * one or reset on success. + */ + WT_ERR(ret); + retry = 0; + } } else __wt_verbose(session, WT_VERB_LOG, "log_archive: Blocked due to open " @@ -844,7 +867,7 @@ __log_server(void *arg) } if (0) { -err: __wt_err(session, ret, "log server error"); +err: WT_PANIC_MSG(session, ret, "log server error"); } return (WT_THREAD_RET_VALUE); } @@ -902,7 +925,7 @@ __wt_logmgr_create(WT_SESSION_IMPL *session, const char *cfg[]) WT_RET(__wt_cond_alloc(session, "log sync", &log->log_sync_cond)); WT_RET(__wt_cond_alloc(session, "log write", &log->log_write_cond)); WT_RET(__wt_log_open(session)); - WT_RET(__wt_log_slot_init(session)); + WT_RET(__wt_log_slot_init(session, true)); return (0); } diff --git a/src/third_party/wiredtiger/src/conn/conn_open.c b/src/third_party/wiredtiger/src/conn/conn_open.c index eb3c79422a0..649bfa7c81f 100644 --- a/src/third_party/wiredtiger/src/conn/conn_open.c +++ b/src/third_party/wiredtiger/src/conn/conn_open.c @@ -91,6 +91,7 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) if (txn_global->oldest_id == txn_global->current && txn_global->metadata_pinned == txn_global->current) break; + WT_STAT_CONN_INCR(session, txn_release_blocked); __wt_yield(); } @@ -143,7 +144,7 @@ __wt_connection_close(WT_CONNECTION_IMPL *conn) * conditional because we allocate the log path so that printlog can * run without running logging or recovery. */ - if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED) && + if (ret == 0 && FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED) && FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_DONE)) WT_TRET(__wt_txn_checkpoint_log( session, true, WT_TXN_LOG_CKPT_STOP, NULL)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_index.c b/src/third_party/wiredtiger/src/cursor/cur_index.c index 6fc01c0421f..15c1271c992 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_index.c +++ b/src/third_party/wiredtiger/src/cursor/cur_index.c @@ -382,7 +382,7 @@ __curindex_close(WT_CURSOR *cursor) if (cindex->child != NULL) WT_TRET(cindex->child->close(cindex->child)); - __wt_schema_release_table(session, cindex->table); + WT_TRET(__wt_schema_release_table(session, cindex->table)); /* The URI is owned by the index. */ cursor->internal_uri = NULL; WT_TRET(__wt_cursor_close(cursor)); @@ -485,7 +485,7 @@ __wt_curindex_open(WT_SESSION_IMPL *session, if ((ret = __wt_schema_open_index( session, table, idxname, namesize, &idx)) != 0) { - __wt_schema_release_table(session, table); + WT_TRET(__wt_schema_release_table(session, table)); return (ret); } WT_RET(__wt_calloc_one(session, &cindex)); diff --git a/src/third_party/wiredtiger/src/cursor/cur_join.c b/src/third_party/wiredtiger/src/cursor/cur_join.c index 80afaf798dc..3681d36b452 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_join.c +++ b/src/third_party/wiredtiger/src/cursor/cur_join.c @@ -325,7 +325,7 @@ __curjoin_close(WT_CURSOR *cursor) JOINABLE_CURSOR_API_CALL(cursor, session, close, NULL); - __wt_schema_release_table(session, cjoin->table); + WT_TRET(__wt_schema_release_table(session, cjoin->table)); /* This is owned by the table */ cursor->key_format = NULL; if (cjoin->projection != NULL) { diff --git a/src/third_party/wiredtiger/src/cursor/cur_table.c b/src/third_party/wiredtiger/src/cursor/cur_table.c index f6855172e90..ffa9ca35926 100644 --- a/src/third_party/wiredtiger/src/cursor/cur_table.c +++ b/src/third_party/wiredtiger/src/cursor/cur_table.c @@ -786,7 +786,7 @@ __curtable_close(WT_CURSOR *cursor) __wt_free(session, ctable->cg_cursors); __wt_free(session, ctable->cg_valcopy); __wt_free(session, ctable->idx_cursors); - __wt_schema_release_table(session, ctable->table); + WT_TRET(__wt_schema_release_table(session, ctable->table)); /* The URI is owned by the table. */ cursor->internal_uri = NULL; WT_TRET(__wt_cursor_close(cursor)); @@ -942,7 +942,7 @@ __wt_curtable_open(WT_SESSION_IMPL *session, ret = __wt_open_cursor(session, table->cgroups[0]->source, NULL, cfg, cursorp); - __wt_schema_release_table(session, table); + WT_TRET(__wt_schema_release_table(session, table)); if (ret == 0) { /* Fix up the public URI to match what was passed in. */ cursor = *cursorp; diff --git a/src/third_party/wiredtiger/src/evict/evict_lru.c b/src/third_party/wiredtiger/src/evict/evict_lru.c index 26bbf9f679b..6e850f67b3e 100644 --- a/src/third_party/wiredtiger/src/evict/evict_lru.c +++ b/src/third_party/wiredtiger/src/evict/evict_lru.c @@ -17,7 +17,7 @@ static int __evict_pass(WT_SESSION_IMPL *); static int __evict_server(WT_SESSION_IMPL *, bool *); static int __evict_tune_workers(WT_SESSION_IMPL *session); static int __evict_walk(WT_SESSION_IMPL *, WT_EVICT_QUEUE *); -static int __evict_walk_file( +static int __evict_walk_tree( WT_SESSION_IMPL *, WT_EVICT_QUEUE *, u_int, u_int *); #define WT_EVICT_HAS_WORKERS(s) \ @@ -767,8 +767,10 @@ __evict_clear_walk(WT_SESSION_IMPL *session) cache = S2C(session)->cache; WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_PASS)); - if (session->dhandle == cache->evict_file_next) - cache->evict_file_next = NULL; + if (session->dhandle == cache->walk_tree) { + cache->walk_tree = NULL; + cache->walk_target = 0; + } if ((ref = btree->evict_ref) == NULL) return (0); @@ -880,10 +882,8 @@ void __wt_evict_file_exclusive_off(WT_SESSION_IMPL *session) { WT_BTREE *btree; - WT_CACHE *cache; btree = S2BT(session); - cache = S2C(session)->cache; /* * We have seen subtle bugs with multiple threads racing to turn @@ -891,12 +891,26 @@ __wt_evict_file_exclusive_off(WT_SESSION_IMPL *session) */ WT_DIAGNOSTIC_YIELD; - /* Hold the walk lock to turn on eviction. */ - __wt_spin_lock(session, &cache->evict_walk_lock); - WT_ASSERT(session, - btree->evict_ref == NULL && btree->evict_disabled > 0); - --btree->evict_disabled; - __wt_spin_unlock(session, &cache->evict_walk_lock); + /* + * Atomically decrement the evict-disabled count, without acquiring the + * eviction walk-lock. We can't acquire that lock here because there's + * a potential deadlock. When acquiring exclusive eviction access, we + * acquire the eviction walk-lock and then the cache's pass-intr lock. + * The current eviction implementation can hold the pass-intr lock and + * call into this function (see WT-3303 for the details), which might + * deadlock with another thread trying to get exclusive eviction access. + */ +#if defined(HAVE_DIAGNOSTIC) + { + int32_t v; + + WT_ASSERT(session, btree->evict_ref == NULL); + v = __wt_atomic_subi32(&btree->evict_disabled, 1); + WT_ASSERT(session, v >= 0); + } +#else + (void)__wt_atomic_subi32(&btree->evict_disabled, 1); +#endif } #define EVICT_TUNE_BATCH 1 /* Max workers to add each period */ @@ -941,6 +955,13 @@ __evict_tune_workers(WT_SESSION_IMPL *session) conn = S2C(session); cache = conn->cache; + /* + * If we have a fixed number of eviction threads, there is no value in + * calculating if we should do any tuning. + */ + if (conn->evict_threads_max == conn->evict_threads_min) + return (0); + WT_ASSERT(session, conn->evict_threads.threads[0]->session == session); pgs_evicted_cur = pgs_evicted_persec_cur = 0; @@ -1371,19 +1392,22 @@ retry: while (slot < max_entries) { * scan last time through. If we don't have a saved * handle, start from the beginning of the list. */ - if ((dhandle = cache->evict_file_next) != NULL) - cache->evict_file_next = NULL; - else + if ((dhandle = cache->walk_tree) != NULL) + cache->walk_tree = NULL; + else { dhandle = TAILQ_FIRST(&conn->dhqh); + cache->walk_target = 0; + } } else { if (incr) { WT_ASSERT(session, dhandle->session_inuse > 0); (void)__wt_atomic_subi32( &dhandle->session_inuse, 1); incr = false; - cache->evict_file_next = NULL; + cache->walk_tree = NULL; } dhandle = TAILQ_NEXT(dhandle, q); + cache->walk_target = 0; } /* If we reach the end of the list, we're done. */ @@ -1445,29 +1469,26 @@ retry: while (slot < max_entries) { /* * Re-check the "no eviction" flag, used to enforce exclusive - * access when a handle is being closed. If not set, remember - * the file to visit first, next loop. + * access when a handle is being closed. * * Only try to acquire the lock and simply continue if we fail; * the lock is held while the thread turning off eviction clears * the tree's current eviction point, and part of the process is * waiting on this thread to acknowledge that action. + * + * If a handle is being discarded, it will still be marked open, + * but won't have a root page. */ if (btree->evict_disabled == 0 && !__wt_spin_trylock(session, &cache->evict_walk_lock)) { - if (btree->evict_disabled == 0) { + if (btree->evict_disabled == 0 && + btree->root.page != NULL) { /* - * Assert the handle has a root page: eviction - * should have been locked out if the tree is - * being discarded or the root page is changing. - * As this has not always been the case, assert - * to debug that change. + * Remember the file to visit first, next loop. */ - WT_ASSERT(session, btree->root.page != NULL); - - cache->evict_file_next = dhandle; + cache->walk_tree = dhandle; WT_WITH_DHANDLE(session, dhandle, - ret = __evict_walk_file( + ret = __evict_walk_tree( session, queue, max_entries, &slot)); WT_ASSERT(session, session->split_gen == 0); @@ -1489,7 +1510,7 @@ retry: while (slot < max_entries) { * candidates and we aren't finding more. */ if (slot < max_entries && (retries < 2 || - (retries < 10 && + (retries < WT_RETRY_MAX && (slot == queue->evict_entries || slot > start_slot)))) { start_slot = slot; ++retries; @@ -1556,45 +1577,21 @@ __evict_push_candidate(WT_SESSION_IMPL *session, } /* - * __evict_walk_file -- - * Get a few page eviction candidates from a single underlying file. + * __evict_walk_target -- + * Calculate how many pages to queue for a given tree. */ -static int -__evict_walk_file(WT_SESSION_IMPL *session, - WT_EVICT_QUEUE *queue, u_int max_entries, u_int *slotp) +static uint32_t +__evict_walk_target( + WT_SESSION_IMPL *session, WT_EVICT_QUEUE *queue, u_int max_entries) { - WT_BTREE *btree; WT_CACHE *cache; - WT_CONNECTION_IMPL *conn; - WT_DECL_RET; - WT_EVICT_ENTRY *end, *evict, *start; - WT_PAGE *page; - WT_PAGE_MODIFY *mod; - WT_REF *ref; - WT_TXN_GLOBAL *txn_global; - uint64_t btree_inuse, bytes_per_slot, cache_inuse, min_pages; - uint64_t pages_seen, pages_queued, refs_walked; - uint32_t remaining_slots, total_slots, walk_flags; + uint64_t btree_inuse, bytes_per_slot, cache_inuse; uint32_t target_pages_clean, target_pages_dirty, target_pages; - int internal_pages, restarts; - bool give_up, modified, urgent_queued; + uint32_t total_slots; - conn = S2C(session); - btree = S2BT(session); - cache = conn->cache; - txn_global = &conn->txn_global; - internal_pages = restarts = 0; - give_up = urgent_queued = false; - - /* - * Figure out how many slots to fill from this tree. - * Note that some care is taken in the calculation to avoid overflow. - */ - start = queue->evict_queue + *slotp; - remaining_slots = max_entries - *slotp; - total_slots = max_entries - queue->evict_entries; - btree_inuse = cache_inuse = 0; + cache = S2C(session)->cache; target_pages_clean = target_pages_dirty = 0; + total_slots = max_entries - queue->evict_entries; /* * The number of times we should fill the queue by the end of @@ -1640,26 +1637,16 @@ __evict_walk_file(WT_SESSION_IMPL *session, QUEUE_FILLS_PER_PASS; /* - * Randomly walk trees with a small fraction of the cache in case there - * are so many trees that none of them use enough of the cache to be - * allocated slots. - * - * The chance of walking a tree is equal to the chance that a random - * byte in cache belongs to the tree, weighted by how many times we - * want to fill queues during a pass through all the trees in cache. + * Walk trees with a small fraction of the cache in case there are so + * many trees that none of them use enough of the cache to be allocated + * slots. Only skip a tree if it has no bytes of interest. */ if (target_pages == 0) { - if (F_ISSET(cache, WT_CACHE_EVICT_CLEAN)) { - btree_inuse = __wt_btree_bytes_evictable(session); - cache_inuse = __wt_cache_bytes_inuse(cache); - } else { - btree_inuse = __wt_btree_dirty_leaf_inuse(session); - cache_inuse = __wt_cache_dirty_leaf_inuse(cache); - } - if (btree_inuse == 0 || cache_inuse == 0) - return (0); - if (__wt_random64(&session->rnd) % cache_inuse > - btree_inuse * QUEUE_FILLS_PER_PASS) + btree_inuse = F_ISSET(cache, WT_CACHE_EVICT_CLEAN) ? + __wt_btree_bytes_evictable(session) : + __wt_btree_dirty_leaf_inuse(session); + + if (btree_inuse == 0) return (0); } @@ -1670,13 +1657,64 @@ __evict_walk_file(WT_SESSION_IMPL *session, if (target_pages < MIN_PAGES_PER_TREE) target_pages = MIN_PAGES_PER_TREE; + /* If the tree is dead, take a lot of pages. */ + if (F_ISSET(session->dhandle, WT_DHANDLE_DEAD)) + target_pages *= 10; + + return (target_pages); +} + +/* + * __evict_walk_tree -- + * Get a few page eviction candidates from a single underlying file. + */ +static int +__evict_walk_tree(WT_SESSION_IMPL *session, + WT_EVICT_QUEUE *queue, u_int max_entries, u_int *slotp) +{ + WT_BTREE *btree; + WT_CACHE *cache; + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + WT_EVICT_ENTRY *end, *evict, *start; + WT_PAGE *page; + WT_PAGE_MODIFY *mod; + WT_REF *ref; + WT_TXN_GLOBAL *txn_global; + uint64_t min_pages, pages_seen, pages_queued, refs_walked; + uint32_t remaining_slots, target_pages, walk_flags; + int internal_pages, restarts; + bool give_up, modified, urgent_queued; + + conn = S2C(session); + btree = S2BT(session); + cache = conn->cache; + txn_global = &conn->txn_global; + internal_pages = restarts = 0; + give_up = urgent_queued = false; + /* - * If the tree is dead or we're near the end of the queue, fill the - * remaining slots. + * Figure out how many slots to fill from this tree. + * Note that some care is taken in the calculation to avoid overflow. */ - if (F_ISSET(session->dhandle, WT_DHANDLE_DEAD) || - target_pages > remaining_slots) + start = queue->evict_queue + *slotp; + remaining_slots = max_entries - *slotp; + if (cache->walk_target != 0) { + WT_ASSERT(session, cache->walk_progress <= cache->walk_target); + target_pages = cache->walk_target - cache->walk_progress; + } else { + target_pages = cache->walk_target = + __evict_walk_target(session, queue, max_entries); + cache->walk_progress = 0; + } + + if (target_pages > remaining_slots) target_pages = remaining_slots; + + /* If we don't want any pages from this tree, move on. */ + if (target_pages == 0) + return (0); + end = start + target_pages; /* @@ -1849,6 +1887,7 @@ fast: /* If the page can't be evicted, give up. */ continue; ++evict; ++pages_queued; + ++cache->walk_progress; if (WT_PAGE_IS_INTERNAL(page)) ++internal_pages; @@ -1863,6 +1902,10 @@ fast: /* If the page can't be evicted, give up. */ WT_STAT_CONN_INCRV( session, cache_eviction_pages_queued, (u_int)(evict - start)); + __wt_verbose(session, WT_VERB_EVICTSERVER, + "%s walk: seen %" PRIu64 ", queued %" PRIu64, + session->dhandle->name, pages_seen, pages_queued); + /* * If we couldn't find the number of pages we were looking for, skip * the tree next time. @@ -2445,14 +2488,23 @@ __wt_verbose_dump_cache(WT_SESSION_IMPL *session) WT_CONNECTION_IMPL *conn; WT_DATA_HANDLE *dhandle; WT_DECL_RET; + u_int pct; uint64_t total_bytes, total_dirty_bytes; conn = S2C(session); total_bytes = total_dirty_bytes = 0; + pct = 0; /* [-Werror=uninitialized] */ WT_RET(__wt_msg(session, "%s", WT_DIVIDER)); WT_RET(__wt_msg(session, "cache dump")); + WT_RET(__wt_msg(session, + "cache full: %s", __wt_cache_full(session) ? "yes" : "no")); + WT_RET(__wt_msg(session, "cache clean check: %s (%u%%)", + __wt_eviction_clean_needed(session, &pct) ? "yes" : "no", pct)); + WT_RET(__wt_msg(session, "cache dirty check: %s (%u%%)", + __wt_eviction_dirty_needed(session, &pct) ? "yes" : "no", pct)); + for (dhandle = NULL;;) { WT_WITH_HANDLE_LIST_READ_LOCK(session, WT_DHANDLE_NEXT(session, dhandle, &conn->dhqh, q)); diff --git a/src/third_party/wiredtiger/src/evict/evict_page.c b/src/third_party/wiredtiger/src/evict/evict_page.c index 85689efd0b1..70a16442dc4 100644 --- a/src/third_party/wiredtiger/src/evict/evict_page.c +++ b/src/third_party/wiredtiger/src/evict/evict_page.c @@ -156,7 +156,7 @@ __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, bool closing) /* Update the reference and discard the page. */ if (__wt_ref_is_root(ref)) __wt_ref_out(session, ref); - else if ((clean_page && !LF_ISSET(WT_EVICT_IN_MEMORY)) || tree_dead) + else if ((clean_page && !F_ISSET(conn, WT_CONN_IN_MEMORY)) || tree_dead) /* * Pages that belong to dead trees never write back to disk * and can't support page splits. @@ -202,8 +202,8 @@ __evict_delete_ref(WT_SESSION_IMPL *session, WT_REF *ref, bool closing) return (0); /* - * Avoid doing reverse splits when closing the file, it is - * wasted work and some structure may already have been freed. + * Avoid doing reverse splits when closing the file, it is wasted work + * and some structures may have already been freed. */ if (!closing) { parent = ref->home; @@ -393,11 +393,13 @@ __evict_review( WT_SESSION_IMPL *session, WT_REF *ref, uint32_t *flagsp, bool closing) { WT_CACHE *cache; + WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_PAGE *page; uint32_t flags; - bool lookaside_retry, modified; + bool lookaside_retry, *lookaside_retryp, modified; + conn = S2C(session); flags = WT_EVICTING; *flagsp = flags; @@ -453,7 +455,7 @@ __evict_review( * Clean pages can't be evicted when running in memory only. This * should be uncommon - we don't add clean pages to the queue. */ - if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY) && !modified && !closing) + if (F_ISSET(conn, WT_CONN_IN_MEMORY) && !modified && !closing) return (EBUSY); /* Check if the page can be evicted. */ @@ -519,11 +521,14 @@ __evict_review( * Additionally, if we aren't trying to free space in the cache, scrub * the page and keep it in memory. */ - cache = S2C(session)->cache; + cache = conn->cache; + lookaside_retry = false; + lookaside_retryp = NULL; + if (closing) LF_SET(WT_VISIBILITY_ERR); else if (!WT_PAGE_IS_INTERNAL(page)) { - if (F_ISSET(S2C(session), WT_CONN_IN_MEMORY)) + if (F_ISSET(conn, WT_CONN_IN_MEMORY)) LF_SET(WT_EVICT_IN_MEMORY | WT_EVICT_SCRUB | WT_EVICT_UPDATE_RESTORE); else { @@ -531,21 +536,26 @@ __evict_review( if (F_ISSET(cache, WT_CACHE_EVICT_SCRUB)) LF_SET(WT_EVICT_SCRUB); + + /* + * Check if reconciliation suggests trying the + * lookaside table. + */ + lookaside_retryp = &lookaside_retry; } } /* Reconcile the page. */ - ret = __wt_reconcile(session, ref, NULL, flags, &lookaside_retry); + ret = __wt_reconcile(session, ref, NULL, flags, lookaside_retryp); /* - * If reconciliation fails, eviction is stuck and reconciliation reports - * it might succeed if we use the lookaside table (the page didn't have - * uncommitted updates, it was not-yet-globally visible updates causing - * the problem), configure reconciliation to write those updates to the - * lookaside table, allowing the eviction of pages we'd otherwise have - * to retain in cache to support older readers. + * If reconciliation fails, eviction is stuck and reconciliation + * reports it might succeed if we use the lookaside table, then + * configure reconciliation to write those updates to the lookaside + * table, allowing the eviction of pages we'd otherwise have to retain + * in cache to support older readers. */ - if (ret == EBUSY && __wt_cache_stuck(session) && lookaside_retry) { + if (ret == EBUSY && lookaside_retry && __wt_cache_stuck(session)) { LF_CLR(WT_EVICT_SCRUB | WT_EVICT_UPDATE_RESTORE); LF_SET(WT_EVICT_LOOKASIDE); ret = __wt_reconcile(session, ref, NULL, flags, NULL); diff --git a/src/third_party/wiredtiger/src/include/btmem.h b/src/third_party/wiredtiger/src/include/btmem.h index d0b21b17965..e965724dffe 100644 --- a/src/third_party/wiredtiger/src/include/btmem.h +++ b/src/third_party/wiredtiger/src/include/btmem.h @@ -714,7 +714,7 @@ struct __wt_page { * Related information for fast-delete, on-disk pages. */ struct __wt_page_deleted { - uint64_t txnid; /* Transaction ID */ + volatile uint64_t txnid; /* Transaction ID */ WT_UPDATE **update_list; /* List of updates for abort */ }; @@ -904,7 +904,7 @@ struct __wt_ikey { * list. */ WT_PACKED_STRUCT_BEGIN(__wt_update) - uint64_t txnid; /* update transaction */ + volatile uint64_t txnid; /* Transaction ID */ WT_UPDATE *next; /* forward-linked list */ diff --git a/src/third_party/wiredtiger/src/include/btree.h b/src/third_party/wiredtiger/src/include/btree.h index 28fe1b94b23..5b1b4d68976 100644 --- a/src/third_party/wiredtiger/src/include/btree.h +++ b/src/third_party/wiredtiger/src/include/btree.h @@ -134,34 +134,44 @@ struct __wt_btree { WT_BM *bm; /* Block manager reference */ u_int block_header; /* WT_PAGE_HEADER_BYTE_SIZE */ - uint64_t checkpoint_gen; /* Checkpoint generation */ - uint64_t rec_max_txn; /* Maximum txn seen (clean trees) */ uint64_t write_gen; /* Write generation */ + uint64_t rec_max_txn; /* Maximum txn seen (clean trees) */ + uint64_t checkpoint_gen; /* Checkpoint generation */ + volatile enum { + WT_CKPT_OFF, WT_CKPT_PREPARE, WT_CKPT_RUNNING + } checkpointing; /* Checkpoint in progress */ uint64_t bytes_inmem; /* Cache bytes in memory. */ uint64_t bytes_dirty_intl; /* Bytes in dirty internal pages. */ uint64_t bytes_dirty_leaf; /* Bytes in dirty leaf pages. */ + /* + * We flush pages from the tree (in order to make checkpoint faster), + * without a high-level lock. To avoid multiple threads flushing at + * the same time, lock the tree. + */ + WT_SPINLOCK flush_lock; /* Lock to flush the tree's pages */ + + /* + * All of the following fields live at the end of the structure so it's + * easier to clear everything but the fields that persist. + */ +#define WT_BTREE_CLEAR_SIZE (offsetof(WT_BTREE, evict_ref)) + + /* + * Eviction information is maintained in the btree handle, but owned by + * eviction, not the btree code. + */ WT_REF *evict_ref; /* Eviction thread's location */ uint64_t evict_priority; /* Relative priority of cached pages */ u_int evict_walk_period; /* Skip this many LRU walks */ u_int evict_walk_saved; /* Saved walk skips for checkpoints */ u_int evict_walk_skips; /* Number of walks skipped */ - int evict_disabled; /* Eviction disabled count */ + int32_t evict_disabled; /* Eviction disabled count */ + bool evict_disabled_open;/* Eviction disabled on open */ volatile uint32_t evict_busy; /* Count of threads in eviction */ int evict_start_type; /* Start position for eviction walk (see WT_EVICT_WALK_START). */ - enum { - WT_CKPT_OFF, WT_CKPT_PREPARE, WT_CKPT_RUNNING - } checkpointing; /* Checkpoint in progress */ - - /* - * We flush pages from the tree (in order to make checkpoint faster), - * without a high-level lock. To avoid multiple threads flushing at - * the same time, lock the tree. - */ - WT_SPINLOCK flush_lock; /* Lock to flush the tree's pages */ - /* Flags values up to 0xff are reserved for WT_DHANDLE_* */ #define WT_BTREE_ALLOW_SPLITS 0x000100 /* Allow splits, even with no evict */ #define WT_BTREE_BULK 0x000200 /* Bulk-load handle */ diff --git a/src/third_party/wiredtiger/src/include/cache.h b/src/third_party/wiredtiger/src/include/cache.h index 04920c3585a..42a152e5f10 100644 --- a/src/third_party/wiredtiger/src/include/cache.h +++ b/src/third_party/wiredtiger/src/include/cache.h @@ -133,7 +133,8 @@ struct __wt_cache { */ WT_SPINLOCK evict_pass_lock; /* Eviction pass lock */ WT_SESSION_IMPL *walk_session; /* Eviction pass session */ - WT_DATA_HANDLE *evict_file_next;/* LRU next file to search */ + WT_DATA_HANDLE *walk_tree; /* LRU walk current tree */ + uint32_t walk_progress, walk_target;/* Progress in current tree */ WT_SPINLOCK evict_queue_lock; /* Eviction current queue lock */ WT_EVICT_QUEUE evict_queues[WT_EVICT_QUEUE_MAX]; diff --git a/src/third_party/wiredtiger/src/include/connection.h b/src/third_party/wiredtiger/src/include/connection.h index f74732684f5..74611de1131 100644 --- a/src/third_party/wiredtiger/src/include/connection.h +++ b/src/third_party/wiredtiger/src/include/connection.h @@ -314,9 +314,10 @@ struct __wt_connection_impl { #define WT_CONN_LOG_ARCHIVE 0x01 /* Archive is enabled */ #define WT_CONN_LOG_ENABLED 0x02 /* Logging is enabled */ #define WT_CONN_LOG_EXISTED 0x04 /* Log files found */ -#define WT_CONN_LOG_RECOVER_DONE 0x08 /* Recovery completed */ -#define WT_CONN_LOG_RECOVER_ERR 0x10 /* Error if recovery required */ -#define WT_CONN_LOG_ZERO_FILL 0x20 /* Manually zero files */ +#define WT_CONN_LOG_RECOVER_DIRTY 0x08 /* Recovering unclean */ +#define WT_CONN_LOG_RECOVER_DONE 0x10 /* Recovery completed */ +#define WT_CONN_LOG_RECOVER_ERR 0x20 /* Error if recovery required */ +#define WT_CONN_LOG_ZERO_FILL 0x40 /* Manually zero files */ uint32_t log_flags; /* Global logging configuration */ WT_CONDVAR *log_cond; /* Log server wait mutex */ WT_SESSION_IMPL *log_session; /* Log server session */ diff --git a/src/third_party/wiredtiger/src/include/extern.h b/src/third_party/wiredtiger/src/include/extern.h index bf3279d0f94..3c45b290dd8 100644 --- a/src/third_party/wiredtiger/src/include/extern.h +++ b/src/third_party/wiredtiger/src/include/extern.h @@ -181,7 +181,7 @@ extern int __wt_verify_dsk_image(WT_SESSION_IMPL *session, const char *tag, cons extern int __wt_verify_dsk(WT_SESSION_IMPL *session, const char *tag, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_tree_walk(WT_SESSION_IMPL *session, WT_REF **refp, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_tree_walk_count(WT_SESSION_IMPL *session, WT_REF **refp, uint64_t *walkcntp, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_tree_walk_skip(WT_SESSION_IMPL *session, WT_REF **refp, uint64_t *skipleafcntp, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_tree_walk_skip( WT_SESSION_IMPL *session, WT_REF **refp, uint64_t *skipleafcntp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_col_modify(WT_SESSION_IMPL *session, WT_CURSOR_BTREE *cbt, uint64_t recno, WT_ITEM *value, WT_UPDATE *upd_arg, bool is_remove, bool exclusive) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_col_search(WT_SESSION_IMPL *session, uint64_t search_recno, WT_REF *leaf, WT_CURSOR_BTREE *cbt) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_row_leaf_keys(WT_SESSION_IMPL *session, WT_PAGE *page) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -369,6 +369,7 @@ extern int __wt_log_needs_recovery(WT_SESSION_IMPL *session, WT_LSN *ckp_lsn, bo extern void __wt_log_written_reset(WT_SESSION_IMPL *session); extern int __wt_log_get_all_files(WT_SESSION_IMPL *session, char ***filesp, u_int *countp, uint32_t *maxid, bool active_only) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_extract_lognum( WT_SESSION_IMPL *session, const char *name, uint32_t *id) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_reset(WT_SESSION_IMPL *session, uint32_t lognum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_acquire(WT_SESSION_IMPL *session, uint64_t recsize, WT_LOGSLOT *slot) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_allocfile( WT_SESSION_IMPL *session, uint32_t lognum, const char *dest) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_remove(WT_SESSION_IMPL *session, const char *file_prefix, uint32_t lognum) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -405,7 +406,7 @@ extern int __wt_logop_row_truncate_print(WT_SESSION_IMPL *session, const uint8_t extern int __wt_txn_op_printlog(WT_SESSION_IMPL *session, const uint8_t **pp, const uint8_t *end, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_log_slot_activate(WT_SESSION_IMPL *session, WT_LOGSLOT *slot); extern int __wt_log_slot_switch(WT_SESSION_IMPL *session, WT_MYSLOT *myslot, bool retry, bool forced, bool *did_work) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern int __wt_log_slot_init(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_log_slot_init(WT_SESSION_IMPL *session, bool alloc) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_slot_destroy(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_log_slot_join(WT_SESSION_IMPL *session, uint64_t mysize, uint32_t flags, WT_MYSLOT *myslot) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int64_t __wt_log_slot_release(WT_SESSION_IMPL *session, WT_MYSLOT *myslot, int64_t size); @@ -564,12 +565,13 @@ extern int __wt_schema_index_source(WT_SESSION_IMPL *session, WT_TABLE *table, c extern int __wt_schema_create( WT_SESSION_IMPL *session, const char *uri, const char *config) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_drop(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_get_table(WT_SESSION_IMPL *session, const char *name, size_t namelen, bool ok_incomplete, WT_TABLE **tablep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_schema_release_table(WT_SESSION_IMPL *session, WT_TABLE *table); +extern int __wt_schema_release_table(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_schema_destroy_colgroup(WT_SESSION_IMPL *session, WT_COLGROUP **colgroupp); extern int __wt_schema_destroy_index(WT_SESSION_IMPL *session, WT_INDEX **idxp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_destroy_table(WT_SESSION_IMPL *session, WT_TABLE **tablep) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_remove_table(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_close_tables(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_schema_sweep_tables(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_colgroup_name(WT_SESSION_IMPL *session, WT_TABLE *table, const char *cgname, size_t len, WT_ITEM *buf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_open_colgroups(WT_SESSION_IMPL *session, WT_TABLE *table) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_schema_open_index(WT_SESSION_IMPL *session, WT_TABLE *table, const char *idxname, size_t len, WT_INDEX **indexp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -691,7 +693,6 @@ extern uint32_t __wt_rduppo2(uint32_t n, uint32_t po2); extern void __wt_random_init(WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern void __wt_random_init_seed( WT_SESSION_IMPL *session, WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern uint32_t __wt_random(WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); -extern uint64_t __wt_random64(WT_RAND_STATE volatile *rnd_state) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); extern int __wt_buf_grow_worker(WT_SESSION_IMPL *session, WT_ITEM *buf, size_t size) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_buf_fmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_buf_catfmt(WT_SESSION_IMPL *session, WT_ITEM *buf, const char *fmt, ...) WT_GCC_FUNC_DECL_ATTRIBUTE((format (printf, 3, 4))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); @@ -732,6 +733,8 @@ extern int __wt_thread_group_create( WT_SESSION_IMPL *session, WT_THREAD_GROUP * extern int __wt_thread_group_destroy(WT_SESSION_IMPL *session, WT_THREAD_GROUP *group) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_group_start_one( WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, bool wait) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_group_stop_one( WT_SESSION_IMPL *session, WT_THREAD_GROUP *group, bool wait) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern void __wt_seconds(WT_SESSION_IMPL *session, time_t *timep); extern void __wt_txn_release_snapshot(WT_SESSION_IMPL *session); extern void __wt_txn_get_snapshot(WT_SESSION_IMPL *session); extern int __wt_txn_update_oldest(WT_SESSION_IMPL *session, uint32_t flags) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); diff --git a/src/third_party/wiredtiger/src/include/extern_posix.h b/src/third_party/wiredtiger/src/include/extern_posix.h index c0ed056c7b6..9e32e86e64c 100644 --- a/src/third_party/wiredtiger/src/include/extern_posix.h +++ b/src/third_party/wiredtiger/src/include/extern_posix.h @@ -28,5 +28,5 @@ extern int __wt_vsnprintf_len_incr( char *buf, size_t size, size_t *retsizep, co extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_id(char *buf, size_t buflen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); +extern void __wt_epoch_raw(WT_SESSION_IMPL *session, struct timespec *tsp); extern void __wt_yield(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))); diff --git a/src/third_party/wiredtiger/src/include/extern_win.h b/src/third_party/wiredtiger/src/include/extern_win.h index d548ee0b2ec..85db8175615 100644 --- a/src/third_party/wiredtiger/src/include/extern_win.h +++ b/src/third_party/wiredtiger/src/include/extern_win.h @@ -26,7 +26,7 @@ extern int __wt_vsnprintf_len_incr( char *buf, size_t size, size_t *retsizep, co extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_thread_id(char *buf, size_t buflen) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); -extern void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp); +extern void __wt_epoch_raw(WT_SESSION_IMPL *session, struct timespec *tsp); extern int __wt_to_utf16_string( WT_SESSION_IMPL *session, const char*utf8, WT_ITEM **outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_to_utf8_string( WT_SESSION_IMPL *session, const wchar_t*wide, WT_ITEM **outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern DWORD __wt_getlasterror(void); diff --git a/src/third_party/wiredtiger/src/include/flags.h b/src/third_party/wiredtiger/src/include/flags.h index f26a45c68f5..0e6cf7657ed 100644 --- a/src/third_party/wiredtiger/src/include/flags.h +++ b/src/third_party/wiredtiger/src/include/flags.h @@ -21,7 +21,8 @@ #define WT_CONN_SERVER_LSM 0x00008000 #define WT_CONN_SERVER_STATISTICS 0x00010000 #define WT_CONN_SERVER_SWEEP 0x00020000 -#define WT_CONN_WAS_BACKUP 0x00040000 +#define WT_CONN_TABLE_CACHE 0x00040000 +#define WT_CONN_WAS_BACKUP 0x00080000 #define WT_EVICTING 0x00000002 #define WT_EVICT_INMEM_SPLIT 0x00000004 #define WT_EVICT_IN_MEMORY 0x00000008 @@ -47,9 +48,8 @@ #define WT_READ_PREV 0x00000080 #define WT_READ_RESTART_OK 0x00000100 #define WT_READ_SKIP_INTL 0x00000200 -#define WT_READ_SKIP_LEAF 0x00000400 -#define WT_READ_TRUNCATE 0x00000800 -#define WT_READ_WONT_NEED 0x00001000 +#define WT_READ_TRUNCATE 0x00000400 +#define WT_READ_WONT_NEED 0x00000800 #define WT_SESSION_CAN_WAIT 0x00000001 #define WT_SESSION_INTERNAL 0x00000002 #define WT_SESSION_LOCKED_CHECKPOINT 0x00000004 diff --git a/src/third_party/wiredtiger/src/include/misc.i b/src/third_party/wiredtiger/src/include/misc.i index fad10f01103..eb99de3dcab 100644 --- a/src/third_party/wiredtiger/src/include/misc.i +++ b/src/third_party/wiredtiger/src/include/misc.i @@ -40,45 +40,6 @@ __wt_strdup(WT_SESSION_IMPL *session, const char *str, void *retp) session, str, (str == NULL) ? 0 : strlen(str), retp)); } -/* - * __wt_seconds -- - * Return the seconds since the Epoch. - */ -static inline void -__wt_seconds(WT_SESSION_IMPL *session, time_t *timep) -{ - struct timespec t; - - __wt_epoch(session, &t); - - *timep = t.tv_sec; -} - -/* - * __wt_time_check_monotonic -- - * Check and prevent time running backward. If we detect that it has, we - * set the time structure to the previous values, making time stand still - * until we see a time in the future of the highest value seen so far. - */ -static inline void -__wt_time_check_monotonic(WT_SESSION_IMPL *session, struct timespec *tsp) -{ - /* - * Detect time going backward. If so, use the last - * saved timestamp. - */ - if (session == NULL) - return; - - if (tsp->tv_sec < session->last_epoch.tv_sec || - (tsp->tv_sec == session->last_epoch.tv_sec && - tsp->tv_nsec < session->last_epoch.tv_nsec)) { - WT_STAT_CONN_INCR(session, time_travel); - *tsp = session->last_epoch; - } else - session->last_epoch = *tsp; -} - /* * __wt_verbose -- * Verbose message. diff --git a/src/third_party/wiredtiger/src/include/os.h b/src/third_party/wiredtiger/src/include/os.h index 73d89268392..2c03b115e7c 100644 --- a/src/third_party/wiredtiger/src/include/os.h +++ b/src/third_party/wiredtiger/src/include/os.h @@ -34,9 +34,11 @@ (ret) = __wt_errno(); \ } while (0) +#define WT_RETRY_MAX 10 + #define WT_SYSCALL_RETRY(call, ret) do { \ int __retry; \ - for (__retry = 0; __retry < 10; ++__retry) { \ + for (__retry = 0; __retry < WT_RETRY_MAX; ++__retry) { \ WT_SYSCALL(call, ret); \ switch (ret) { \ case EAGAIN: \ diff --git a/src/third_party/wiredtiger/src/include/session.h b/src/third_party/wiredtiger/src/include/session.h index 0074b4a9d43..d05dee68641 100644 --- a/src/third_party/wiredtiger/src/include/session.h +++ b/src/third_party/wiredtiger/src/include/session.h @@ -98,6 +98,12 @@ struct __wt_session_impl { */ TAILQ_HEAD(__tables, __wt_table) tables; + /* + * Updated when the table cache is swept of all tables older than the + * current schema generation. + */ + uint64_t table_sweep_gen; + /* Current rwlock for callback. */ WT_RWLOCK *current_rwlock; uint8_t current_rwticket; diff --git a/src/third_party/wiredtiger/src/include/stat.h b/src/third_party/wiredtiger/src/include/stat.h index db48a841571..01e622a5695 100644 --- a/src/third_party/wiredtiger/src/include/stat.h +++ b/src/third_party/wiredtiger/src/include/stat.h @@ -475,11 +475,19 @@ struct __wt_connection_stats { int64_t thread_write_active; int64_t application_evict_time; int64_t application_cache_time; + int64_t txn_release_blocked; + int64_t conn_close_blocked_lsm; + int64_t dhandle_lock_blocked; + int64_t log_server_sync_blocked; int64_t page_busy_blocked; int64_t page_forcible_evict_blocked; int64_t page_locked_blocked; int64_t page_read_blocked; int64_t page_sleep; + int64_t page_del_rollback_blocked; + int64_t child_modify_blocked_page; + int64_t page_index_slot_blocked; + int64_t tree_descend_blocked; int64_t txn_snapshots_created; int64_t txn_snapshots_dropped; int64_t txn_begin; diff --git a/src/third_party/wiredtiger/src/include/txn.h b/src/third_party/wiredtiger/src/include/txn.h index 7e802c188ab..fdf9c714afa 100644 --- a/src/third_party/wiredtiger/src/include/txn.h +++ b/src/third_party/wiredtiger/src/include/txn.h @@ -93,6 +93,8 @@ struct __wt_txn_global { * the global transaction state. */ WT_RWLOCK scan_rwlock; + /* Protects logging, checkpoints and transaction visibility. */ + WT_RWLOCK visibility_rwlock; /* * Track information about the running checkpoint. The transaction diff --git a/src/third_party/wiredtiger/src/include/txn.i b/src/third_party/wiredtiger/src/include/txn.i index 314c948e4d1..39273a1995c 100644 --- a/src/third_party/wiredtiger/src/include/txn.i +++ b/src/third_party/wiredtiger/src/include/txn.i @@ -147,16 +147,6 @@ __wt_txn_oldest_id(WT_SESSION_IMPL *session) return (checkpoint_pinned); } -/* - * __wt_txn_committed -- - * Return if a transaction has been committed. - */ -static inline bool -__wt_txn_committed(WT_SESSION_IMPL *session, uint64_t id) -{ - return (WT_TXNID_LT(id, S2C(session)->txn_global.last_running)); -} - /* * __wt_txn_visible_all -- * Check if a given transaction ID is "globally visible". This is, if diff --git a/src/third_party/wiredtiger/src/include/wiredtiger.in b/src/third_party/wiredtiger/src/include/wiredtiger.in index 821efdf5fa1..30bff37017a 100644 --- a/src/third_party/wiredtiger/src/include/wiredtiger.in +++ b/src/third_party/wiredtiger/src/include/wiredtiger.in @@ -2458,6 +2458,8 @@ struct __wt_connection { * readonly for more information., a boolean flag; default \c false.} * @config{session_max, maximum expected number of sessions (including server * threads)., an integer greater than or equal to 1; default \c 100.} + * @config{session_table_cache, Maintain a per-session cache of tables., a + * boolean flag; default \c true.} * @config{shared_cache = (, shared cache configuration options. A database * should configure either a cache_size or a shared_cache not both. Enabling a * shared cache uses a session from the configured session_max., a set of @@ -4806,72 +4808,94 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_CONN_APPLICATION_EVICT_TIME 1216 /*! thread-yield: application thread time waiting for cache (usecs) */ #define WT_STAT_CONN_APPLICATION_CACHE_TIME 1217 +/*! + * thread-yield: connection close blocked waiting for transaction state + * stabilization + */ +#define WT_STAT_CONN_TXN_RELEASE_BLOCKED 1218 +/*! thread-yield: connection close yielded for lsm manager shutdown */ +#define WT_STAT_CONN_CONN_CLOSE_BLOCKED_LSM 1219 +/*! thread-yield: data handle lock yielded */ +#define WT_STAT_CONN_DHANDLE_LOCK_BLOCKED 1220 +/*! thread-yield: log server sync yielded for log write */ +#define WT_STAT_CONN_LOG_SERVER_SYNC_BLOCKED 1221 /*! thread-yield: page acquire busy blocked */ -#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1218 +#define WT_STAT_CONN_PAGE_BUSY_BLOCKED 1222 /*! thread-yield: page acquire eviction blocked */ -#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1219 +#define WT_STAT_CONN_PAGE_FORCIBLE_EVICT_BLOCKED 1223 /*! thread-yield: page acquire locked blocked */ -#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1220 +#define WT_STAT_CONN_PAGE_LOCKED_BLOCKED 1224 /*! thread-yield: page acquire read blocked */ -#define WT_STAT_CONN_PAGE_READ_BLOCKED 1221 +#define WT_STAT_CONN_PAGE_READ_BLOCKED 1225 /*! thread-yield: page acquire time sleeping (usecs) */ -#define WT_STAT_CONN_PAGE_SLEEP 1222 +#define WT_STAT_CONN_PAGE_SLEEP 1226 +/*! thread-yield: page delete rollback yielded for instantiation */ +#define WT_STAT_CONN_PAGE_DEL_ROLLBACK_BLOCKED 1227 +/*! thread-yield: page reconciliation yielded due to child modification */ +#define WT_STAT_CONN_CHILD_MODIFY_BLOCKED_PAGE 1228 +/*! thread-yield: reference for page index and slot yielded */ +#define WT_STAT_CONN_PAGE_INDEX_SLOT_BLOCKED 1229 +/*! + * thread-yield: tree descend one level yielded for split page index + * update + */ +#define WT_STAT_CONN_TREE_DESCEND_BLOCKED 1230 /*! transaction: number of named snapshots created */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1223 +#define WT_STAT_CONN_TXN_SNAPSHOTS_CREATED 1231 /*! transaction: number of named snapshots dropped */ -#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1224 +#define WT_STAT_CONN_TXN_SNAPSHOTS_DROPPED 1232 /*! transaction: transaction begins */ -#define WT_STAT_CONN_TXN_BEGIN 1225 +#define WT_STAT_CONN_TXN_BEGIN 1233 /*! transaction: transaction checkpoint currently running */ -#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1226 +#define WT_STAT_CONN_TXN_CHECKPOINT_RUNNING 1234 /*! transaction: transaction checkpoint generation */ -#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1227 +#define WT_STAT_CONN_TXN_CHECKPOINT_GENERATION 1235 /*! transaction: transaction checkpoint max time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1228 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MAX 1236 /*! transaction: transaction checkpoint min time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1229 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_MIN 1237 /*! transaction: transaction checkpoint most recent time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1230 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_RECENT 1238 /*! transaction: transaction checkpoint scrub dirty target */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1231 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TARGET 1239 /*! transaction: transaction checkpoint scrub time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1232 +#define WT_STAT_CONN_TXN_CHECKPOINT_SCRUB_TIME 1240 /*! transaction: transaction checkpoint total time (msecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1233 +#define WT_STAT_CONN_TXN_CHECKPOINT_TIME_TOTAL 1241 /*! transaction: transaction checkpoints */ -#define WT_STAT_CONN_TXN_CHECKPOINT 1234 +#define WT_STAT_CONN_TXN_CHECKPOINT 1242 /*! * transaction: transaction checkpoints skipped because database was * clean */ -#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1235 +#define WT_STAT_CONN_TXN_CHECKPOINT_SKIPPED 1243 /*! transaction: transaction failures due to cache overflow */ -#define WT_STAT_CONN_TXN_FAIL_CACHE 1236 +#define WT_STAT_CONN_TXN_FAIL_CACHE 1244 /*! * transaction: transaction fsync calls for checkpoint after allocating * the transaction ID */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1237 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST 1245 /*! * transaction: transaction fsync duration for checkpoint after * allocating the transaction ID (usecs) */ -#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1238 +#define WT_STAT_CONN_TXN_CHECKPOINT_FSYNC_POST_DURATION 1246 /*! transaction: transaction range of IDs currently pinned */ -#define WT_STAT_CONN_TXN_PINNED_RANGE 1239 +#define WT_STAT_CONN_TXN_PINNED_RANGE 1247 /*! transaction: transaction range of IDs currently pinned by a checkpoint */ -#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1240 +#define WT_STAT_CONN_TXN_PINNED_CHECKPOINT_RANGE 1248 /*! * transaction: transaction range of IDs currently pinned by named * snapshots */ -#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1241 +#define WT_STAT_CONN_TXN_PINNED_SNAPSHOT_RANGE 1249 /*! transaction: transaction sync calls */ -#define WT_STAT_CONN_TXN_SYNC 1242 +#define WT_STAT_CONN_TXN_SYNC 1250 /*! transaction: transactions committed */ -#define WT_STAT_CONN_TXN_COMMIT 1243 +#define WT_STAT_CONN_TXN_COMMIT 1251 /*! transaction: transactions rolled back */ -#define WT_STAT_CONN_TXN_ROLLBACK 1244 +#define WT_STAT_CONN_TXN_ROLLBACK 1252 /*! * @} diff --git a/src/third_party/wiredtiger/src/log/log.c b/src/third_party/wiredtiger/src/log/log.c index 803d3e8dfab..da179400755 100644 --- a/src/third_party/wiredtiger/src/log/log.c +++ b/src/third_party/wiredtiger/src/log/log.c @@ -8,6 +8,7 @@ #include "wt_internal.h" +static int __log_newfile(WT_SESSION_IMPL *, bool, bool *); static int __log_openfile( WT_SESSION_IMPL *, WT_FH **, const char *, uint32_t, uint32_t); static int __log_write_internal( @@ -441,6 +442,59 @@ __wt_log_extract_lognum( return (0); } +/* + * __wt_log_reset -- + * Reset the existing log file to after the given file number. + * Called from recovery when toggling logging back on, it was off + * the previous open but it was on earlier before that toggle. + */ +int +__wt_log_reset(WT_SESSION_IMPL *session, uint32_t lognum) +{ + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + WT_LOG *log; + uint32_t old_lognum; + u_int i, logcount; + char **logfiles; + + conn = S2C(session); + log = conn->log; + + if (!FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED) || + log->fileid > lognum) + return (0); + + WT_ASSERT(session, F_ISSET(conn, WT_CONN_RECOVERING)); + WT_ASSERT(session, !F_ISSET(conn, WT_CONN_READONLY)); + /* + * We know we're single threaded and called from recovery only when + * toggling logging back on. Therefore the only log files we have are + * old and outdated and the new one created when logging opened before + * recovery. We have to remove all old log files first and then create + * the new one so that log file numbers are contiguous in the file + * system. + */ + WT_RET(__wt_close(session, &log->log_fh)); + WT_RET(__log_get_files(session, + WT_LOG_FILENAME, &logfiles, &logcount)); + for (i = 0; i < logcount; i++) { + WT_ERR(__wt_log_extract_lognum( + session, logfiles[i], &old_lognum)); + WT_ASSERT(session, old_lognum < lognum || lognum == 1); + WT_ERR(__wt_log_remove(session, WT_LOG_FILENAME, old_lognum)); + } + log->fileid = lognum; + + /* Send in true to update connection creation LSNs. */ + WT_WITH_SLOT_LOCK(session, log, + ret = __log_newfile(session, true, NULL)); + WT_ERR(__wt_log_slot_init(session, false)); +err: WT_TRET( + __wt_fs_directory_list_free(session, &logfiles, logcount)); + return (ret); +} + /* * __log_zero -- * Zero a log file. @@ -1611,11 +1665,6 @@ __wt_log_scan(WT_SESSION_IMPL *session, WT_LSN *lsnp, uint32_t flags, if (func == NULL) return (0); - if (LF_ISSET(WT_LOGSCAN_RECOVER)) - __wt_verbose(session, WT_VERB_LOG, - "__wt_log_scan truncating to %" PRIu32 "/%" PRIu32, - log->trunc_lsn.l.file, log->trunc_lsn.l.offset); - if (log != NULL) { allocsize = log->allocsize; @@ -1709,9 +1758,14 @@ advance: /* * Truncate this log file before we move to the next. */ - if (LF_ISSET(WT_LOGSCAN_RECOVER)) + if (LF_ISSET(WT_LOGSCAN_RECOVER) && + __wt_log_cmp(&rd_lsn, &log->trunc_lsn) < 0) { + __wt_verbose(session, WT_VERB_LOG, + "Truncate end of log %" PRIu32 "/%" PRIu32, + rd_lsn.l.file, rd_lsn.l.offset); WT_ERR(__log_truncate(session, &rd_lsn, WT_LOG_FILENAME, 1)); + } /* * If we had a partial record, we'll want to break * now after closing and truncating. Although for now diff --git a/src/third_party/wiredtiger/src/log/log_slot.c b/src/third_party/wiredtiger/src/log/log_slot.c index 97e317ce68c..b23d589c8e2 100644 --- a/src/third_party/wiredtiger/src/log/log_slot.c +++ b/src/third_party/wiredtiger/src/log/log_slot.c @@ -401,7 +401,7 @@ __wt_log_slot_switch(WT_SESSION_IMPL *session, * Initialize the slot array. */ int -__wt_log_slot_init(WT_SESSION_IMPL *session) +__wt_log_slot_init(WT_SESSION_IMPL *session, bool alloc) { WT_CONNECTION_IMPL *conn; WT_DECL_RET; @@ -423,15 +423,17 @@ __wt_log_slot_init(WT_SESSION_IMPL *session) * switch log files very aggressively. Scale back the buffer for * small log file sizes. */ - log->slot_buf_size = (uint32_t)WT_MIN( - (size_t)conn->log_file_max / 10, WT_LOG_SLOT_BUF_SIZE); - for (i = 0; i < WT_SLOT_POOL; i++) { - WT_ERR(__wt_buf_init(session, - &log->slot_pool[i].slot_buf, log->slot_buf_size)); - F_SET(&log->slot_pool[i], WT_SLOT_INIT_FLAGS); + if (alloc) { + log->slot_buf_size = (uint32_t)WT_MIN( + (size_t)conn->log_file_max / 10, WT_LOG_SLOT_BUF_SIZE); + for (i = 0; i < WT_SLOT_POOL; i++) { + WT_ERR(__wt_buf_init(session, + &log->slot_pool[i].slot_buf, log->slot_buf_size)); + F_SET(&log->slot_pool[i], WT_SLOT_INIT_FLAGS); + } + WT_STAT_CONN_SET(session, + log_buffer_size, log->slot_buf_size * WT_SLOT_POOL); } - WT_STAT_CONN_SET(session, - log_buffer_size, log->slot_buf_size * WT_SLOT_POOL); /* * Set up the available slot from the pool the first time. */ diff --git a/src/third_party/wiredtiger/src/lsm/lsm_manager.c b/src/third_party/wiredtiger/src/lsm/lsm_manager.c index b7d9086d10e..62da094b5f7 100644 --- a/src/third_party/wiredtiger/src/lsm/lsm_manager.c +++ b/src/third_party/wiredtiger/src/lsm/lsm_manager.c @@ -295,8 +295,10 @@ __wt_lsm_manager_destroy(WT_SESSION_IMPL *session) manager->lsm_workers == 0); if (manager->lsm_workers > 0) { /* Wait for the main LSM manager thread to finish. */ - while (!F_ISSET(manager, WT_LSM_MANAGER_SHUTDOWN)) + while (!F_ISSET(manager, WT_LSM_MANAGER_SHUTDOWN)) { + WT_STAT_CONN_INCR(session, conn_close_blocked_lsm); __wt_yield(); + } /* Clean up open LSM handles. */ ret = __wt_lsm_tree_close_all(session); diff --git a/src/third_party/wiredtiger/src/meta/meta_table.c b/src/third_party/wiredtiger/src/meta/meta_table.c index aca69d0e6a2..a970694b9d7 100644 --- a/src/third_party/wiredtiger/src/meta/meta_table.c +++ b/src/third_party/wiredtiger/src/meta/meta_table.c @@ -230,12 +230,23 @@ __wt_metadata_remove(WT_SESSION_IMPL *session, const char *key) WT_RET_MSG(session, EINVAL, "%s: remove not supported on the turtle file", key); + /* + * Take, release, and reacquire the metadata cursor. It's complicated, + * but that way the underlying meta-tracking function doesn't have to + * open a second metadata cursor, it can use the session's cached one. + */ WT_RET(__wt_metadata_cursor(session, &cursor)); cursor->set_key(cursor, key); WT_ERR(cursor->search(cursor)); + WT_ERR(__wt_metadata_cursor_release(session, &cursor)); + if (WT_META_TRACKING(session)) WT_ERR(__wt_meta_track_update(session, key)); - WT_ERR(cursor->remove(cursor)); + + WT_ERR(__wt_metadata_cursor(session, &cursor)); + cursor->set_key(cursor, key); + ret = cursor->remove(cursor); + err: WT_TRET(__wt_metadata_cursor_release(session, &cursor)); return (ret); } @@ -266,7 +277,9 @@ __wt_metadata_search(WT_SESSION_IMPL *session, const char *key, char **valuep) * that Coverity complains a lot, add an error check to get some * peace and quiet. */ - if ((ret = __wt_turtle_read(session, key, valuep)) != 0) + WT_WITH_TURTLE_LOCK(session, + ret = __wt_turtle_read(session, key, valuep)); + if (ret != 0) __wt_free(session, *valuep); return (ret); } diff --git a/src/third_party/wiredtiger/src/meta/meta_turtle.c b/src/third_party/wiredtiger/src/meta/meta_turtle.c index 5a089471059..f7ea6fe99c5 100644 --- a/src/third_party/wiredtiger/src/meta/meta_turtle.c +++ b/src/third_party/wiredtiger/src/meta/meta_turtle.c @@ -246,6 +246,9 @@ __wt_turtle_read(WT_SESSION_IMPL *session, const char *key, char **valuep) *valuep = NULL; + /* Require single-threading. */ + WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TURTLE)); + /* * Open the turtle file; there's one case where we won't find the turtle * file, yet still succeed. We create the metadata file before creating @@ -302,6 +305,9 @@ __wt_turtle_update(WT_SESSION_IMPL *session, const char *key, const char *value) fs = NULL; + /* Require single-threading. */ + WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TURTLE)); + /* * Create the turtle setup file: we currently re-write it from scratch * every time. diff --git a/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c b/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c index fe010b62305..e4a6683dee9 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c +++ b/src/third_party/wiredtiger/src/os_posix/os_mtx_cond.c @@ -19,11 +19,19 @@ __wt_cond_alloc(WT_SESSION_IMPL *session, const char *name, WT_CONDVAR **condp) WT_DECL_RET; WT_RET(__wt_calloc_one(session, &cond)); - WT_ERR(pthread_mutex_init(&cond->mtx, NULL)); - /* Initialize the condition variable to permit self-blocking. */ +#ifdef HAVE_PTHREAD_COND_MONOTONIC + { + pthread_condattr_t condattr; + + WT_ERR(pthread_condattr_init(&condattr)); + WT_ERR(pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC)); + WT_ERR(pthread_cond_init(&cond->cond, &condattr)); + } +#else WT_ERR(pthread_cond_init(&cond->cond, NULL)); +#endif cond->name = name; cond->waiters = 0; @@ -79,7 +87,26 @@ __wt_cond_wait_signal(WT_SESSION_IMPL *session, WT_CONDVAR *cond, goto skipping; if (usecs > 0) { - __wt_epoch(session, &ts); + /* + * Get the current time as the basis for calculating when the + * wait should end. Prefer a monotonic clock source to avoid + * unexpectedly long sleeps when the system clock is adjusted. + * + * Failing that, query the time directly and don't attempt to + * correct for the clock moving backwards, which would result + * in a sleep that is too long by however much the clock is + * updated. This isn't as good as a monotonic clock source but + * makes the window of vulnerability smaller (i.e., the + * calculated time is only incorrect if the system clock + * changes in between us querying it and waiting). + */ +#ifdef HAVE_PTHREAD_COND_MONOTONIC + WT_SYSCALL_RETRY(clock_gettime(CLOCK_MONOTONIC, &ts), ret); + if (ret != 0) + WT_PANIC_MSG(session, ret, "clock_gettime"); +#else + __wt_epoch_raw(session, &ts); +#endif ts.tv_sec += (time_t) (((uint64_t)ts.tv_nsec + WT_THOUSAND * usecs) / WT_BILLION); ts.tv_nsec = (long) diff --git a/src/third_party/wiredtiger/src/os_posix/os_time.c b/src/third_party/wiredtiger/src/os_posix/os_time.c index fe337fea7cf..25a08d62355 100644 --- a/src/third_party/wiredtiger/src/os_posix/os_time.c +++ b/src/third_party/wiredtiger/src/os_posix/os_time.c @@ -9,14 +9,12 @@ #include "wt_internal.h" /* - * __wt_epoch -- - * Return the time since the Epoch. + * __wt_epoch_raw -- + * Return the time since the Epoch as reported by a system call. */ void -__wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) - WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) +__wt_epoch_raw(WT_SESSION_IMPL *session, struct timespec *tsp) { - struct timespec tmp; WT_DECL_RET; /* @@ -28,19 +26,10 @@ __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) tsp->tv_sec = 0; tsp->tv_nsec = 0; - /* - * Read into a local variable so that we're comparing the correct - * value when we check for monotonic increasing time. There are - * many places we read into an unlocked global variable. - */ #if defined(HAVE_CLOCK_GETTIME) - WT_SYSCALL_RETRY(clock_gettime(CLOCK_REALTIME, &tmp), ret); - if (ret == 0) { - __wt_time_check_monotonic(session, &tmp); - tsp->tv_sec = tmp.tv_sec; - tsp->tv_nsec = tmp.tv_nsec; + WT_SYSCALL_RETRY(clock_gettime(CLOCK_REALTIME, tsp), ret); + if (ret == 0) return; - } WT_PANIC_MSG(session, ret, "clock_gettime"); #elif defined(HAVE_GETTIMEOFDAY) { @@ -48,10 +37,8 @@ __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) WT_SYSCALL_RETRY(gettimeofday(&v, NULL), ret); if (ret == 0) { - tmp.tv_sec = v.tv_sec; - tmp.tv_nsec = v.tv_usec * WT_THOUSAND; - __wt_time_check_monotonic(session, &tmp); - *tsp = tmp; + tsp->tv_sec = v.tv_sec; + tsp->tv_nsec = v.tv_usec * WT_THOUSAND; return; } WT_PANIC_MSG(session, ret, "gettimeofday"); diff --git a/src/third_party/wiredtiger/src/os_win/os_time.c b/src/third_party/wiredtiger/src/os_win/os_time.c index ba71341ab22..84c06bed6e5 100644 --- a/src/third_party/wiredtiger/src/os_win/os_time.c +++ b/src/third_party/wiredtiger/src/os_win/os_time.c @@ -9,24 +9,23 @@ #include "wt_internal.h" /* - * __wt_epoch -- - * Return the time since the Epoch. + * __wt_epoch_raw -- + * Return the time since the Epoch as reported by the system. */ void -__wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) +__wt_epoch_raw(WT_SESSION_IMPL *session, struct timespec *tsp) { - struct timespec tmp; FILETIME time; uint64_t ns100; + WT_UNUSED(session); + GetSystemTimeAsFileTime(&time); ns100 = (((int64_t)time.dwHighDateTime << 32) + time.dwLowDateTime) - 116444736000000000LL; - tmp.tv_sec = ns100 / 10000000; - tmp.tv_nsec = (long)((ns100 % 10000000) * 100); - __wt_time_check_monotonic(session, &tmp); - *tsp = tmp; + tsp->tv_sec = ns100 / 10000000; + tsp->tv_nsec = (long)((ns100 % 10000000) * 100); } /* diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index e59d9796352..688efa10398 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -45,12 +45,14 @@ typedef struct { uint64_t orig_btree_checkpoint_gen; uint64_t orig_txn_checkpoint_gen; - /* Track the page's maximum transaction ID. */ + /* Track the oldest transaction running when reconciliation starts. */ + uint64_t last_running; + uint64_t max_txn; - /* Track if all updates were skipped. */ - uint64_t update_cnt; - uint64_t update_skip_cnt; + uint64_t update_mem_all; /* Total update memory size */ + uint64_t update_mem_saved; /* Saved update memory size */ + uint64_t update_mem_uncommitted;/* Uncommitted update memory size */ /* * When we can't mark the page clean (for example, checkpoint found some @@ -338,7 +340,8 @@ static int __rec_split_write(WT_SESSION_IMPL *, WT_RECONCILE *, WT_BOUNDARY *, WT_ITEM *, bool); static int __rec_update_las( WT_SESSION_IMPL *, WT_RECONCILE *, uint32_t, WT_BOUNDARY *); -static int __rec_write_check_complete(WT_SESSION_IMPL *, WT_RECONCILE *); +static int __rec_write_check_complete( + WT_SESSION_IMPL *, WT_RECONCILE *, bool *); static int __rec_write_init(WT_SESSION_IMPL *, WT_REF *, uint32_t, WT_SALVAGE_COOKIE *, void *); static void __rec_write_page_status(WT_SESSION_IMPL *, WT_RECONCILE *); @@ -437,7 +440,7 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, /* Checks for a successful reconciliation. */ if (ret == 0) - ret = __rec_write_check_complete(session, r); + ret = __rec_write_check_complete(session, r, lookaside_retryp); /* Wrap up the page reconciliation. */ if (ret == 0 && (ret = __rec_write_wrapup(session, r, page)) == 0) @@ -448,14 +451,6 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, /* Release the reconciliation lock. */ WT_PAGE_UNLOCK(session, page); - /* - * If our caller can configure lookaside table reconciliation, flag if - * that's worth trying. The lookaside table doesn't help if we skipped - * updates, it can only help with older readers preventing eviction. - */ - if (lookaside_retryp != NULL && r->update_cnt == r->update_skip_cnt) - *lookaside_retryp = true; - /* Update statistics. */ WT_STAT_CONN_INCR(session, rec_pages); WT_STAT_DATA_INCR(session, rec_pages); @@ -558,13 +553,21 @@ __rec_las_checkpoint_test(WT_SESSION_IMPL *session, WT_RECONCILE *r) /* * __rec_write_check_complete -- - * Check that reconciliation should complete + * Check that reconciliation should complete. */ static int -__rec_write_check_complete(WT_SESSION_IMPL *session, WT_RECONCILE *r) +__rec_write_check_complete( + WT_SESSION_IMPL *session, WT_RECONCILE *r, bool *lookaside_retryp) { - WT_BOUNDARY *bnd; - size_t i; + /* + * Tests in this function are lookaside tests and tests to decide if + * rewriting a page in memory is worth doing. In-memory configurations + * can't use a lookaside table, and we ignore page rewrite desirability + * checks for in-memory eviction because a small cache can force us to + * rewrite every possible page. + */ + if (F_ISSET(r, WT_EVICT_IN_MEMORY)) + return (0); /* * If we have used the lookaside table, check for a lookaside table and @@ -574,19 +577,62 @@ __rec_write_check_complete(WT_SESSION_IMPL *session, WT_RECONCILE *r) return (EBUSY); /* - * If we are doing update/restore based eviction, confirm part of the - * page is being discarded, or at least 10% of the updates won't have - * to be re-instantiated. Otherwise, it isn't progress, don't bother. + * Eviction can configure lookaside table reconciliation, consider if + * it's worth giving up this reconciliation attempt and falling back to + * using the lookaside table. We continue with evict/restore if + * switching to the lookaside doesn't make sense for any reason: we + * won't retry an evict/restore reconciliation until/unless the + * transactional system moves forward, so at worst it's a single wasted + * effort. + * + * First, check if the lookaside table is a possible alternative. */ - if (F_ISSET(r, WT_EVICT_UPDATE_RESTORE)) { - for (bnd = r->bnd, i = 0; i < r->bnd_entries; ++bnd, ++i) - if (bnd->supd == NULL) - break; - if (i == r->bnd_entries && - r->update_cnt / 10 >= r->update_skip_cnt) - return (EBUSY); - } - return (0); + if (lookaside_retryp == NULL) + return (0); + + /* + * We only suggest lookaside if currently in an evict/restore attempt + * and some updates were saved. Our caller sets the evict/restore flag + * based on various conditions (like if this is a leaf page), which is + * why we're testing that flag instead of a set of other conditions. + * If no updates were saved, eviction will succeed without needing to + * restore anything. + */ + if (!F_ISSET(r, WT_EVICT_UPDATE_RESTORE) || r->bnd->supd == NULL) + return (0); + + /* + * Check if this reconciliation attempt is making progress. If there's + * any sign of progress, don't fall back to the lookaside table. + * + * Check if the current reconciliation split, in which case we'll + * likely get to write at least one of the blocks. If that page is + * empty, that's also progress. + */ + if (r->bnd_next != 1) + return (0); + + /* + * Check if the current reconciliation applied some updates, in which + * case evict/restore should gain us some space. + */ + if (r->update_mem_saved != r->update_mem_all) + return (0); + + /* + * Check if lookaside eviction is possible. If any of the updates we + * saw were uncommitted, the lookaside table cannot be used: it only + * helps with older readers preventing eviction. + */ + if (r->update_mem_uncommitted != 0) + return (0); + + /* + * The current evict/restore approach shows no signs of being useful, + * lookaside is possible, suggest the lookaside table. + */ + *lookaside_retryp = true; + return (EBUSY); } /* @@ -848,6 +894,16 @@ __rec_write_init(WT_SESSION_IMPL *session, r->orig_txn_checkpoint_gen = conn->txn_global.checkpoint_gen; WT_ORDERED_READ(r->orig_write_gen, page->modify->write_gen); + /* + * Cache the oldest running transaction ID. This is used to check + * whether updates seen by reconciliation have committed. We keep a + * cached copy to avoid races where a concurrent transaction could + * abort while reconciliation is examining its updates. This way, any + * transaction running when reconciliation starts is considered + * uncommitted. + */ + WT_ORDERED_READ(r->last_running, S2C(session)->txn_global.last_running); + /* * Lookaside table eviction is configured when eviction gets aggressive, * adjust the flags for cases we don't support. @@ -891,7 +947,7 @@ __rec_write_init(WT_SESSION_IMPL *session, r->max_txn = WT_TXN_NONE; /* Track if all updates were skipped. */ - r->update_cnt = r->update_skip_cnt = 0; + r->update_mem_all = r->update_mem_saved = r->update_mem_uncommitted = 0; /* Track if the page can be marked clean. */ r->leave_dirty = false; @@ -1115,7 +1171,7 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, WT_DECL_ITEM(tmp); WT_PAGE *page; WT_UPDATE *append, *upd, *upd_list; - size_t notused; + size_t notused, update_mem; uint64_t max_txn, min_txn, txnid; bool append_origv, skipped; @@ -1136,36 +1192,64 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, } else upd_list = ins->upd; - ++r->update_cnt; - for (skipped = false, - max_txn = WT_TXN_NONE, min_txn = UINT64_MAX, - upd = upd_list; upd != NULL; upd = upd->next) { - if ((txnid = upd->txnid) == WT_TXN_ABORTED) - continue; + skipped = false; + update_mem = 0; + max_txn = WT_TXN_NONE; + min_txn = UINT64_MAX; - /* Track the largest/smallest transaction IDs on the list. */ - if (WT_TXNID_LT(max_txn, txnid)) - max_txn = txnid; - if (WT_TXNID_LT(txnid, min_txn)) - min_txn = txnid; + if (F_ISSET(r, WT_EVICTING)) { + /* Discard obsolete updates. */ + if ((upd = __wt_update_obsolete_check( + session, page, upd_list->next)) != NULL) + __wt_update_obsolete_free(session, page, upd); + + for (upd = upd_list; upd != NULL; upd = upd->next) { + /* Track the total memory in the update chain. */ + update_mem += WT_UPDATE_MEMSIZE(upd); + + if ((txnid = upd->txnid) == WT_TXN_ABORTED) + continue; - /* - * Find the first update we can use. - */ - if (F_ISSET(r, WT_EVICTING)) { /* + * Track the largest/smallest transaction IDs on the + * list. + */ + if (WT_TXNID_LT(max_txn, txnid)) + max_txn = txnid; + if (WT_TXNID_LT(txnid, min_txn)) + min_txn = txnid; + + /* + * Find the first update we can use. + * * Eviction can write any committed update. * * When reconciling for eviction, track whether any * uncommitted updates are found. + * + * When reconciling for eviction, track the memory held + * by the update chain. */ - if (__wt_txn_committed(session, txnid)) { - if (*updp == NULL) - *updp = upd; - } else + if (WT_TXNID_LE(r->last_running, txnid)) { skipped = true; - } else { + continue; + } + + if (*updp == NULL) + *updp = upd; + } + } else + for (upd = upd_list; upd != NULL; upd = upd->next) { + if ((txnid = upd->txnid) == WT_TXN_ABORTED) + continue; + + /* Track the largest transaction ID on the list. */ + if (WT_TXNID_LT(max_txn, txnid)) + max_txn = txnid; + /* + * Find the first update we can use. + * * Checkpoint can only write updates visible as of its * snapshot. * @@ -1180,7 +1264,8 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, skipped = true; } } - } + + r->update_mem_all += update_mem; /* * If all of the updates were aborted, quit. This test is not strictly @@ -1227,12 +1312,6 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, txnid != S2C(session)->txn_global.checkpoint_txnid || WT_SESSION_IS_CHECKPOINT(session)); #endif - - /* - * Track how many update chains we saw vs. how many update - * chains had an entry we skipped. - */ - ++r->update_skip_cnt; return (0); } @@ -1276,6 +1355,23 @@ __rec_txn_read(WT_SESSION_IMPL *session, WT_RECONCILE *r, if (skipped && !F_ISSET(r, WT_EVICT_UPDATE_RESTORE)) return (EBUSY); + /* + * Track the memory required by the update chain. + * + * A page with no uncommitted (skipped) updates, that can't be evicted + * because some updates aren't yet globally visible, can be evicted by + * writing previous versions of the updates to the lookaside file. That + * test is just checking if the skipped updates memory is zero. + * + * If that's not possible (there are skipped updates), we can rewrite + * the pages in-memory, but we don't want to unless there's memory to + * recover. That test is comparing the memory we'd recover to the memory + * we'd have to re-instantiate as part of the rewrite. + */ + r->update_mem_saved += update_mem; + if (skipped) + r->update_mem_uncommitted += update_mem; + append_origv = false; if (F_ISSET(r, WT_EVICT_UPDATE_RESTORE)) { /* @@ -1562,7 +1658,7 @@ __rec_child_modify(WT_SESSION_IMPL *session, * not reserved for our exclusive use, there are other page states that * must be considered. */ - for (;; __wt_yield()) + for (;; __wt_yield()) { switch (r->tested_ref_state = ref->state) { case WT_REF_DISK: /* On disk, not modified by definition. */ @@ -1673,6 +1769,8 @@ __rec_child_modify(WT_SESSION_IMPL *session, WT_ILLEGAL_VALUE(session); } + WT_STAT_CONN_INCR(session, child_modify_blocked_page); + } in_memory: /* diff --git a/src/third_party/wiredtiger/src/schema/schema_alter.c b/src/third_party/wiredtiger/src/schema/schema_alter.c index 26d800aa98e..edb3c6f77ae 100644 --- a/src/third_party/wiredtiger/src/schema/schema_alter.c +++ b/src/third_party/wiredtiger/src/schema/schema_alter.c @@ -61,13 +61,16 @@ __alter_colgroup( { WT_COLGROUP *colgroup; WT_DECL_RET; + WT_TABLE *table; WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TABLE)); /* If we can get the colgroup, perform any potential alterations. */ if ((ret = __wt_schema_get_colgroup( - session, uri, false, NULL, &colgroup)) == 0) + session, uri, false, &table, &colgroup)) == 0) { WT_TRET(__wt_schema_alter(session, colgroup->source, cfg)); + WT_TRET(__wt_schema_release_table(session, table)); + } return (ret); } @@ -82,11 +85,14 @@ __alter_index( { WT_INDEX *idx; WT_DECL_RET; + WT_TABLE *table; /* If we can get the index, perform any potential alterations. */ if ((ret = __wt_schema_get_index( - session, uri, false, NULL, &idx)) == 0) + session, uri, false, &table, &idx)) == 0) { WT_TRET(__wt_schema_alter(session, idx->source, cfg)); + WT_TRET(__wt_schema_release_table(session, table)); + } return (ret); } @@ -127,7 +133,8 @@ __alter_table(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_ERR(__wt_schema_alter( session, colgroup->source, cfg)); } -err: __wt_schema_release_table(session, table); + +err: WT_TRET(__wt_schema_release_table(session, table)); return (ret); } diff --git a/src/third_party/wiredtiger/src/schema/schema_create.c b/src/third_party/wiredtiger/src/schema/schema_create.c index 0677fa711a5..49c0274994c 100644 --- a/src/third_party/wiredtiger/src/schema/schema_create.c +++ b/src/third_party/wiredtiger/src/schema/schema_create.c @@ -273,7 +273,7 @@ err: __wt_free(session, cgconf); __wt_buf_free(session, &fmt); __wt_buf_free(session, &namebuf); - __wt_schema_release_table(session, table); + WT_TRET(__wt_schema_release_table(session, table)); return (ret); } @@ -540,7 +540,7 @@ err: __wt_free(session, idxconf); __wt_buf_free(session, &fmt); __wt_buf_free(session, &namebuf); - __wt_schema_release_table(session, table); + WT_TRET(__wt_schema_release_table(session, table)); return (ret); } @@ -615,7 +615,7 @@ err: if (table != NULL) { } } if (table != NULL) - __wt_schema_release_table(session, table); + WT_TRET(__wt_schema_release_table(session, table)); __wt_free(session, cgname); __wt_free(session, tableconf); return (ret); diff --git a/src/third_party/wiredtiger/src/schema/schema_drop.c b/src/third_party/wiredtiger/src/schema/schema_drop.c index 49801e4e5f9..18817c4fa2d 100644 --- a/src/third_party/wiredtiger/src/schema/schema_drop.c +++ b/src/third_party/wiredtiger/src/schema/schema_drop.c @@ -67,6 +67,7 @@ __drop_colgroup( session, uri, force, &table, &colgroup)) == 0) { table->cg_complete = false; WT_TRET(__wt_schema_drop(session, colgroup->source, cfg)); + WT_TRET(__wt_schema_release_table(session, table)); } WT_TRET(__wt_metadata_remove(session, uri)); @@ -90,6 +91,7 @@ __drop_index( session, uri, force, &table, &idx)) == 0) { table->idx_complete = false; WT_TRET(__wt_schema_drop(session, idx->source, cfg)); + WT_TRET(__wt_schema_release_table(session, table)); } WT_TRET(__wt_metadata_remove(session, uri)); @@ -151,7 +153,7 @@ __drop_table(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_ERR(__wt_metadata_remove(session, uri)); err: if (table != NULL) - __wt_schema_release_table(session, table); + WT_TRET(__wt_schema_release_table(session, table)); return (ret); } diff --git a/src/third_party/wiredtiger/src/schema/schema_list.c b/src/third_party/wiredtiger/src/schema/schema_list.c index 74ef5135a4a..434038c6bc8 100644 --- a/src/third_party/wiredtiger/src/schema/schema_list.c +++ b/src/third_party/wiredtiger/src/schema/schema_list.c @@ -30,9 +30,12 @@ __schema_add_table(WT_SESSION_IMPL *session, session, name, namelen, ok_incomplete, &table)); WT_RET(ret); - bucket = table->name_hash % WT_HASH_ARRAY_SIZE; - TAILQ_INSERT_HEAD(&session->tables, table, q); - TAILQ_INSERT_HEAD(&session->tablehash[bucket], table, hashq); + if (!table->is_simple || F_ISSET(S2C(session), WT_CONN_TABLE_CACHE)) { + bucket = table->name_hash % WT_HASH_ARRAY_SIZE; + TAILQ_INSERT_HEAD(&session->tables, table, q); + TAILQ_INSERT_HEAD(&session->tablehash[bucket], table, hashq); + } + *tablep = table; return (0); @@ -112,11 +115,14 @@ __wt_schema_get_table(WT_SESSION_IMPL *session, * __wt_schema_release_table -- * Release a table handle. */ -void +int __wt_schema_release_table(WT_SESSION_IMPL *session, WT_TABLE *table) { WT_ASSERT(session, table->refcnt > 0); - --table->refcnt; + if (--table->refcnt == 0 && + table->is_simple && !F_ISSET(S2C(session), WT_CONN_TABLE_CACHE)) + WT_RET(__wt_schema_destroy_table(session, &table)); + return (0); } /* @@ -229,9 +235,11 @@ __wt_schema_remove_table(WT_SESSION_IMPL *session, WT_TABLE *table) uint64_t bucket; WT_ASSERT(session, table->refcnt <= 1); - bucket = table->name_hash % WT_HASH_ARRAY_SIZE; - TAILQ_REMOVE(&session->tables, table, q); - TAILQ_REMOVE(&session->tablehash[bucket], table, hashq); + if (!table->is_simple || F_ISSET(S2C(session), WT_CONN_TABLE_CACHE)) { + bucket = table->name_hash % WT_HASH_ARRAY_SIZE; + TAILQ_REMOVE(&session->tables, table, q); + TAILQ_REMOVE(&session->tablehash[bucket], table, hashq); + } return (__wt_schema_destroy_table(session, &table)); } @@ -249,3 +257,34 @@ __wt_schema_close_tables(WT_SESSION_IMPL *session) WT_TRET(__wt_schema_remove_table(session, table)); return (ret); } + +/* + * __wt_schema_sweep_tables -- + * Close all idle, obsolete tables in a session. + */ +int +__wt_schema_sweep_tables(WT_SESSION_IMPL *session) +{ + WT_TABLE *table, *next; + uint64_t schema_gen; + bool old_table_busy; + + WT_ORDERED_READ(schema_gen, S2C(session)->schema_gen); + if (schema_gen == session->table_sweep_gen) + return (0); + + old_table_busy = false; + TAILQ_FOREACH_SAFE(table, &session->tables, q, next) + if (table->schema_gen != schema_gen) { + if (table->refcnt == 0) + WT_RET(__wt_schema_remove_table( + session, table)); + else + old_table_busy = true; + } + + if (!old_table_busy) + session->table_sweep_gen = schema_gen; + + return (0); +} diff --git a/src/third_party/wiredtiger/src/schema/schema_open.c b/src/third_party/wiredtiger/src/schema/schema_open.c index 44bd66e011a..2df6bae45f3 100644 --- a/src/third_party/wiredtiger/src/schema/schema_open.c +++ b/src/third_party/wiredtiger/src/schema/schema_open.c @@ -425,37 +425,40 @@ __schema_open_table(WT_SESSION_IMPL *session, WT_DECL_RET; WT_TABLE *table; const char *tconfig; - char *tablename; *tablep = NULL; cursor = NULL; table = NULL; - tablename = NULL; WT_ASSERT(session, F_ISSET(session, WT_SESSION_LOCKED_TABLE)); + WT_ERR(__wt_calloc_one(session, &table)); + table->name_hash = __wt_hash_city64(name, namelen); + WT_ERR(__wt_scr_alloc(session, 0, &buf)); WT_ERR(__wt_buf_fmt(session, buf, "table:%.*s", (int)namelen, name)); - WT_ERR(__wt_strndup(session, buf->data, buf->size, &tablename)); + WT_ERR(__wt_strndup(session, buf->data, buf->size, &table->name)); + /* + * Don't hold the metadata cursor pinned, we call functions that use it + * to retrieve column group information. + */ WT_ERR(__wt_metadata_cursor(session, &cursor)); - cursor->set_key(cursor, tablename); - WT_ERR(cursor->search(cursor)); - WT_ERR(cursor->get_value(cursor, &tconfig)); - - WT_ERR(__wt_calloc_one(session, &table)); - table->name = tablename; - tablename = NULL; - table->name_hash = __wt_hash_city64(name, namelen); - - WT_ERR(__wt_config_getones(session, tconfig, "columns", &cval)); + cursor->set_key(cursor, table->name); + if ((ret = cursor->search(cursor)) == 0 && + (ret = cursor->get_value(cursor, &tconfig)) == 0) + ret = __wt_strdup(session, tconfig, &table->config); + WT_TRET(__wt_metadata_cursor_release(session, &cursor)); + WT_ERR(ret); - WT_ERR(__wt_config_getones(session, tconfig, "key_format", &cval)); + WT_ERR(__wt_config_getones(session, table->config, "columns", &cval)); + WT_ERR(__wt_config_getones( + session, table->config, "key_format", &cval)); WT_ERR(__wt_strndup(session, cval.str, cval.len, &table->key_format)); - WT_ERR(__wt_config_getones(session, tconfig, "value_format", &cval)); + WT_ERR(__wt_config_getones( + session, table->config, "value_format", &cval)); WT_ERR(__wt_strndup(session, cval.str, cval.len, &table->value_format)); - WT_ERR(__wt_strdup(session, tconfig, &table->config)); /* Point to some items in the copy to save re-parsing. */ WT_ERR(__wt_config_getones(session, table->config, @@ -491,7 +494,7 @@ __schema_open_table(WT_SESSION_IMPL *session, if (table->ncolgroups > 0 && table->is_simple) WT_ERR_MSG(session, EINVAL, - "%s requires a table with named columns", tablename); + "%s requires a table with named columns", table->name); WT_ERR(__wt_calloc_def(session, WT_COLGROUPS(table), &table->cgroups)); WT_ERR(__wt_schema_open_colgroups(session, table)); @@ -509,9 +512,7 @@ __schema_open_table(WT_SESSION_IMPL *session, if (0) { err: WT_TRET(__wt_schema_destroy_table(session, &table)); } - WT_TRET(__wt_metadata_cursor_release(session, &cursor)); - __wt_free(session, tablename); __wt_scr_free(session, &buf); return (ret); } @@ -529,8 +530,8 @@ __wt_schema_get_colgroup(WT_SESSION_IMPL *session, const char *tablename, *tend; u_int i; - if (tablep != NULL) - *tablep = NULL; + WT_ASSERT(session, tablep != NULL); + *tablep = NULL; *colgroupp = NULL; tablename = uri; @@ -547,15 +548,12 @@ __wt_schema_get_colgroup(WT_SESSION_IMPL *session, colgroup = table->cgroups[i]; if (strcmp(colgroup->name, uri) == 0) { *colgroupp = colgroup; - if (tablep != NULL) - *tablep = table; - else - __wt_schema_release_table(session, table); + *tablep = table; return (0); } } - __wt_schema_release_table(session, table); + WT_RET(__wt_schema_release_table(session, table)); if (quiet) WT_RET(ENOENT); WT_RET_MSG(session, ENOENT, "%s not found in table", uri); @@ -575,8 +573,8 @@ __wt_schema_get_index(WT_SESSION_IMPL *session, const char *tablename, *tend; u_int i; - if (tablep != NULL) - *tablep = NULL; + WT_ASSERT(session, tablep != NULL); + *tablep = NULL; *indexp = NULL; tablename = uri; @@ -591,11 +589,8 @@ __wt_schema_get_index(WT_SESSION_IMPL *session, for (i = 0; i < table->nindices; i++) { idx = table->indices[i]; if (idx != NULL && strcmp(idx->name, uri) == 0) { - if (tablep != NULL) - *tablep = table; - else - __wt_schema_release_table(session, table); *indexp = idx; + *tablep = table; return (0); } } @@ -603,15 +598,12 @@ __wt_schema_get_index(WT_SESSION_IMPL *session, /* Otherwise, open it. */ WT_ERR(__wt_schema_open_index( session, table, tend + 1, strlen(tend + 1), indexp)); - if (tablep != NULL) - *tablep = table; + *tablep = table; + return (0); -err: __wt_schema_release_table(session, table); +err: WT_TRET(__wt_schema_release_table(session, table)); WT_RET(ret); - if (*indexp != NULL) - return (0); - if (quiet) WT_RET(ENOENT); WT_RET_MSG(session, ENOENT, "%s not found in table", uri); diff --git a/src/third_party/wiredtiger/src/schema/schema_rename.c b/src/third_party/wiredtiger/src/schema/schema_rename.c index a374f4c2831..9effedd2cde 100644 --- a/src/third_party/wiredtiger/src/schema/schema_rename.c +++ b/src/third_party/wiredtiger/src/schema/schema_rename.c @@ -233,7 +233,7 @@ __rename_table(WT_SESSION_IMPL *session, WT_ERR(__metadata_rename(session, uri, newuri)); err: if (table != NULL) - __wt_schema_release_table(session, table); + WT_TRET(__wt_schema_release_table(session, table)); return (ret); } diff --git a/src/third_party/wiredtiger/src/schema/schema_stat.c b/src/third_party/wiredtiger/src/schema/schema_stat.c index 345f9164e9b..9bcd2439619 100644 --- a/src/third_party/wiredtiger/src/schema/schema_stat.c +++ b/src/third_party/wiredtiger/src/schema/schema_stat.c @@ -19,14 +19,17 @@ __wt_curstat_colgroup_init(WT_SESSION_IMPL *session, WT_COLGROUP *colgroup; WT_DECL_ITEM(buf); WT_DECL_RET; + WT_TABLE *table; - WT_RET(__wt_schema_get_colgroup(session, uri, false, NULL, &colgroup)); + WT_RET(__wt_schema_get_colgroup( + session, uri, false, &table, &colgroup)); WT_RET(__wt_scr_alloc(session, 0, &buf)); WT_ERR(__wt_buf_fmt(session, buf, "statistics:%s", colgroup->source)); ret = __wt_curstat_init(session, buf->data, NULL, cfg, cst); err: __wt_scr_free(session, &buf); + WT_TRET(__wt_schema_release_table(session, table)); return (ret); } @@ -41,14 +44,16 @@ __wt_curstat_index_init(WT_SESSION_IMPL *session, WT_DECL_ITEM(buf); WT_DECL_RET; WT_INDEX *idx; + WT_TABLE *table; - WT_RET(__wt_schema_get_index(session, uri, false, NULL, &idx)); + WT_RET(__wt_schema_get_index(session, uri, false, &table, &idx)); WT_RET(__wt_scr_alloc(session, 0, &buf)); WT_ERR(__wt_buf_fmt(session, buf, "statistics:%s", idx->source)); ret = __wt_curstat_init(session, buf->data, NULL, cfg, cst); err: __wt_scr_free(session, &buf); + WT_TRET(__wt_schema_release_table(session, table)); return (ret); } @@ -184,7 +189,7 @@ __wt_curstat_table_init(WT_SESSION_IMPL *session, __wt_curstat_dsrc_final(cst); -err: __wt_schema_release_table(session, table); +err: WT_TRET(__wt_schema_release_table(session, table)); __wt_scr_free(session, &buf); return (ret); diff --git a/src/third_party/wiredtiger/src/schema/schema_truncate.c b/src/third_party/wiredtiger/src/schema/schema_truncate.c index 563bafa8ffc..3046305e819 100644 --- a/src/third_party/wiredtiger/src/schema/schema_truncate.c +++ b/src/third_party/wiredtiger/src/schema/schema_truncate.c @@ -33,7 +33,7 @@ __truncate_table(WT_SESSION_IMPL *session, const char *uri, const char *cfg[]) WT_ERR(__wt_schema_truncate( session, table->indices[i]->source, cfg)); -err: __wt_schema_release_table(session, table); +err: WT_TRET(__wt_schema_release_table(session, table)); return (ret); } diff --git a/src/third_party/wiredtiger/src/schema/schema_worker.c b/src/third_party/wiredtiger/src/schema/schema_worker.c index 62cdd7d367b..9fb68720723 100644 --- a/src/third_party/wiredtiger/src/schema/schema_worker.c +++ b/src/third_party/wiredtiger/src/schema/schema_worker.c @@ -64,14 +64,17 @@ __wt_schema_worker(WT_SESSION_IMPL *session, } } else if (WT_PREFIX_MATCH(uri, "colgroup:")) { WT_ERR(__wt_schema_get_colgroup( - session, uri, false, NULL, &colgroup)); + session, uri, false, &table, &colgroup)); WT_ERR(__wt_schema_worker(session, colgroup->source, file_func, name_func, cfg, open_flags)); + WT_ERR(__wt_schema_release_table(session, table)); } else if (WT_PREFIX_SKIP(tablename, "index:")) { idx = NULL; - WT_ERR(__wt_schema_get_index(session, uri, false, NULL, &idx)); + WT_ERR(__wt_schema_get_index( + session, uri, false, &table, &idx)); WT_ERR(__wt_schema_worker(session, idx->source, file_func, name_func, cfg, open_flags)); + WT_ERR(__wt_schema_release_table(session, table)); } else if (WT_PREFIX_MATCH(uri, "lsm:")) { WT_ERR(__wt_lsm_tree_worker(session, uri, file_func, name_func, cfg, open_flags)); @@ -128,6 +131,6 @@ __wt_schema_worker(WT_SESSION_IMPL *session, WT_ERR(__wt_bad_object_type(session, uri)); err: if (table != NULL) - __wt_schema_release_table(session, table); + WT_TRET(__wt_schema_release_table(session, table)); return (ret); } diff --git a/src/third_party/wiredtiger/src/session/session_api.c b/src/third_party/wiredtiger/src/session/session_api.c index 89a5a2c633d..386084b78d9 100644 --- a/src/third_party/wiredtiger/src/session/session_api.c +++ b/src/third_party/wiredtiger/src/session/session_api.c @@ -328,6 +328,7 @@ __session_open_cursor_int(WT_SESSION_IMPL *session, const char *uri, WT_COLGROUP *colgroup; WT_DATA_SOURCE *dsrc; WT_DECL_RET; + WT_TABLE *table; *cursorp = NULL; @@ -355,9 +356,10 @@ __session_open_cursor_int(WT_SESSION_IMPL *session, const char *uri, * the underlying data source. */ WT_RET(__wt_schema_get_colgroup( - session, uri, false, NULL, &colgroup)); + session, uri, false, &table, &colgroup)); WT_RET(__wt_open_cursor( session, colgroup->source, owner, cfg, cursorp)); + WT_RET(__wt_schema_release_table(session, table)); } else if (WT_PREFIX_MATCH(uri, "config:")) WT_RET(__wt_curconfig_open( session, uri, cfg, cursorp)); @@ -818,6 +820,8 @@ __session_reset(WT_SESSION *wt_session) WT_TRET(__wt_session_reset_cursors(session, true)); + WT_TRET(__wt_schema_sweep_tables(session)); + /* Release common session resources. */ WT_TRET(__wt_session_release_resources(session)); @@ -1209,12 +1213,15 @@ __wt_session_range_truncate(WT_SESSION_IMPL *session, done: err: /* - * Close any locally-opened start cursor. Reset application cursors, - * they've possibly moved and the application cannot use them. + * Close any locally-opened start cursor. + * + * Reset application cursors, they've possibly moved and the + * application cannot use them. Note that we can make it here with a + * NULL start cursor (e.g., if the truncate range is empty). */ if (local_start) WT_TRET(start->close(start)); - else + else if (start != NULL) WT_TRET(start->reset(start)); if (stop != NULL) WT_TRET(stop->reset(stop)); diff --git a/src/third_party/wiredtiger/src/session/session_dhandle.c b/src/third_party/wiredtiger/src/session/session_dhandle.c index ffeb6137766..707e07ac11f 100644 --- a/src/third_party/wiredtiger/src/session/session_dhandle.c +++ b/src/third_party/wiredtiger/src/session/session_dhandle.c @@ -235,6 +235,7 @@ __wt_session_lock_dhandle( lock_busy = true; /* Give other threads a chance to make progress. */ + WT_STAT_CONN_INCR(session, dhandle_lock_blocked); __wt_yield(); } } @@ -597,7 +598,9 @@ __wt_session_lock_checkpoint(WT_SESSION_IMPL *session, const char *checkpoint) * the underlying file are visible to the in-memory pages. */ WT_ERR(__wt_evict_file_exclusive_on(session)); - WT_ERR(__wt_cache_op(session, WT_SYNC_DISCARD)); + ret = __wt_cache_op(session, WT_SYNC_DISCARD); + __wt_evict_file_exclusive_off(session); + WT_ERR(ret); /* * We lock checkpoint handles that we are overwriting, so the handle diff --git a/src/third_party/wiredtiger/src/support/err.c b/src/third_party/wiredtiger/src/support/err.c index 57efde72b23..f98b1943449 100644 --- a/src/third_party/wiredtiger/src/support/err.c +++ b/src/third_party/wiredtiger/src/support/err.c @@ -494,7 +494,18 @@ __wt_panic(WT_SESSION_IMPL *session) WT_GCC_FUNC_ATTRIBUTE((cold)) WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) { - F_SET(S2C(session), WT_CONN_PANIC); + WT_CONNECTION_IMPL *conn; + + conn = S2C(session); + + /* + * If the connection has already be marked for panic, just return the + * error. + */ + if (F_ISSET(conn, WT_CONN_PANIC)) + return (WT_PANIC); + + F_SET(conn, WT_CONN_PANIC); __wt_err(session, WT_PANIC, "the process must exit and restart"); #if defined(HAVE_DIAGNOSTIC) diff --git a/src/third_party/wiredtiger/src/support/rand.c b/src/third_party/wiredtiger/src/support/rand.c index 4fae43edc8e..a5b229b9abc 100644 --- a/src/third_party/wiredtiger/src/support/rand.c +++ b/src/third_party/wiredtiger/src/support/rand.c @@ -120,15 +120,3 @@ __wt_random(WT_RAND_STATE volatile * rnd_state) return ((z << 16) + (w & 65535)); } - -/* - * __wt_random64 -- - * Return a 64-bit pseudo-random number. - */ -uint64_t -__wt_random64(WT_RAND_STATE volatile * rnd_state) - WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) -{ - return (((uint64_t)__wt_random(rnd_state) << 32) + - __wt_random(rnd_state)); -} diff --git a/src/third_party/wiredtiger/src/support/stat.c b/src/third_party/wiredtiger/src/support/stat.c index 8b72e653658..c9e577ac3b6 100644 --- a/src/third_party/wiredtiger/src/support/stat.c +++ b/src/third_party/wiredtiger/src/support/stat.c @@ -842,11 +842,19 @@ static const char * const __stats_connection_desc[] = { "thread-state: active filesystem write calls", "thread-yield: application thread time evicting (usecs)", "thread-yield: application thread time waiting for cache (usecs)", + "thread-yield: connection close blocked waiting for transaction state stabilization", + "thread-yield: connection close yielded for lsm manager shutdown", + "thread-yield: data handle lock yielded", + "thread-yield: log server sync yielded for log write", "thread-yield: page acquire busy blocked", "thread-yield: page acquire eviction blocked", "thread-yield: page acquire locked blocked", "thread-yield: page acquire read blocked", "thread-yield: page acquire time sleeping (usecs)", + "thread-yield: page delete rollback yielded for instantiation", + "thread-yield: page reconciliation yielded due to child modification", + "thread-yield: reference for page index and slot yielded", + "thread-yield: tree descend one level yielded for split page index update", "transaction: number of named snapshots created", "transaction: number of named snapshots dropped", "transaction: transaction begins", @@ -1129,11 +1137,19 @@ __wt_stat_connection_clear_single(WT_CONNECTION_STATS *stats) /* not clearing thread_write_active */ stats->application_evict_time = 0; stats->application_cache_time = 0; + stats->txn_release_blocked = 0; + stats->conn_close_blocked_lsm = 0; + stats->dhandle_lock_blocked = 0; + stats->log_server_sync_blocked = 0; stats->page_busy_blocked = 0; stats->page_forcible_evict_blocked = 0; stats->page_locked_blocked = 0; stats->page_read_blocked = 0; stats->page_sleep = 0; + stats->page_del_rollback_blocked = 0; + stats->child_modify_blocked_page = 0; + stats->page_index_slot_blocked = 0; + stats->tree_descend_blocked = 0; stats->txn_snapshots_created = 0; stats->txn_snapshots_dropped = 0; stats->txn_begin = 0; @@ -1475,12 +1491,25 @@ __wt_stat_connection_aggregate( WT_STAT_READ(from, application_evict_time); to->application_cache_time += WT_STAT_READ(from, application_cache_time); + to->txn_release_blocked += WT_STAT_READ(from, txn_release_blocked); + to->conn_close_blocked_lsm += + WT_STAT_READ(from, conn_close_blocked_lsm); + to->dhandle_lock_blocked += WT_STAT_READ(from, dhandle_lock_blocked); + to->log_server_sync_blocked += + WT_STAT_READ(from, log_server_sync_blocked); to->page_busy_blocked += WT_STAT_READ(from, page_busy_blocked); to->page_forcible_evict_blocked += WT_STAT_READ(from, page_forcible_evict_blocked); to->page_locked_blocked += WT_STAT_READ(from, page_locked_blocked); to->page_read_blocked += WT_STAT_READ(from, page_read_blocked); to->page_sleep += WT_STAT_READ(from, page_sleep); + to->page_del_rollback_blocked += + WT_STAT_READ(from, page_del_rollback_blocked); + to->child_modify_blocked_page += + WT_STAT_READ(from, child_modify_blocked_page); + to->page_index_slot_blocked += + WT_STAT_READ(from, page_index_slot_blocked); + to->tree_descend_blocked += WT_STAT_READ(from, tree_descend_blocked); to->txn_snapshots_created += WT_STAT_READ(from, txn_snapshots_created); to->txn_snapshots_dropped += diff --git a/src/third_party/wiredtiger/src/support/time.c b/src/third_party/wiredtiger/src/support/time.c new file mode 100644 index 00000000000..0e4562c0234 --- /dev/null +++ b/src/third_party/wiredtiger/src/support/time.c @@ -0,0 +1,89 @@ +/*- + * Public Domain 2014-2017 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "wt_internal.h" + +/* + * __time_check_monotonic -- + * Check and prevent time running backward. If we detect that it has, we + * set the time structure to the previous values, making time stand still + * until we see a time in the future of the highest value seen so far. + */ +static void +__time_check_monotonic(WT_SESSION_IMPL *session, struct timespec *tsp) +{ + /* + * Detect time going backward. If so, use the last + * saved timestamp. + */ + if (session == NULL) + return; + + if (tsp->tv_sec < session->last_epoch.tv_sec || + (tsp->tv_sec == session->last_epoch.tv_sec && + tsp->tv_nsec < session->last_epoch.tv_nsec)) { + WT_STAT_CONN_INCR(session, time_travel); + *tsp = session->last_epoch; + } else + session->last_epoch = *tsp; +} + +/* + * __wt_epoch -- + * Return the time since the Epoch, adjusted so it never appears to go + * backwards. + */ +void +__wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) + WT_GCC_FUNC_ATTRIBUTE((visibility("default"))) +{ + struct timespec tmp; + + /* + * Read into a local variable so that we're comparing the correct + * value when we check for monotonic increasing time. There are + * many places we read into an unlocked global variable. + */ + __wt_epoch_raw(session, &tmp); + __time_check_monotonic(session, &tmp); + *tsp = tmp; +} + +/* + * __wt_seconds -- + * Return the seconds since the Epoch. + */ +void +__wt_seconds(WT_SESSION_IMPL *session, time_t *timep) +{ + struct timespec t; + + __wt_epoch(session, &t); + + *timep = t.tv_sec; +} diff --git a/src/third_party/wiredtiger/src/txn/txn.c b/src/third_party/wiredtiger/src/txn/txn.c index ea7faa2e966..7665ba56adc 100644 --- a/src/third_party/wiredtiger/src/txn/txn.c +++ b/src/third_party/wiredtiger/src/txn/txn.c @@ -503,13 +503,17 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) WT_CONNECTION_IMPL *conn; WT_DECL_RET; WT_TXN *txn; + WT_TXN_GLOBAL *txn_global; WT_TXN_OP *op; u_int i; - bool did_update; + bool did_update, locked; txn = &session->txn; conn = S2C(session); + txn_global = &conn->txn_global; did_update = txn->mod_count != 0; + locked = false; + WT_ASSERT(session, !F_ISSET(txn, WT_TXN_ERROR) || !did_update); if (!F_ISSET(txn, WT_TXN_RUNNING)) @@ -580,6 +584,14 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) * This is particularly important for checkpoints. */ __wt_txn_release_snapshot(session); + /* + * We hold the visibility lock for reading from the time + * we write our log record until the time we release our + * transaction so that the LSN any checkpoint gets will + * always reflect visible data. + */ + __wt_readlock(session, &txn_global->visibility_rwlock); + locked = true; ret = __wt_txn_log_commit(session, cfg); } @@ -590,6 +602,9 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) * Nothing can fail after this point. */ if (ret != 0) { + if (locked) + __wt_readunlock(session, + &txn_global->visibility_rwlock); WT_TRET(__wt_txn_rollback(session, cfg)); return (ret); } @@ -600,6 +615,8 @@ __wt_txn_commit(WT_SESSION_IMPL *session, const char *cfg[]) txn->mod_count = 0; __wt_txn_release(session); + if (locked) + __wt_readunlock(session, &txn_global->visibility_rwlock); return (0); } @@ -770,6 +787,7 @@ __wt_txn_global_init(WT_SESSION_IMPL *session, const char *cfg[]) &txn_global->id_lock, "transaction id lock")); WT_RET(__wt_rwlock_init(session, &txn_global->scan_rwlock)); WT_RET(__wt_rwlock_init(session, &txn_global->nsnap_rwlock)); + WT_RET(__wt_rwlock_init(session, &txn_global->visibility_rwlock)); txn_global->nsnap_oldest_id = WT_TXN_NONE; TAILQ_INIT(&txn_global->nsnaph); @@ -801,6 +819,7 @@ __wt_txn_global_destroy(WT_SESSION_IMPL *session) __wt_spin_destroy(session, &txn_global->id_lock); __wt_rwlock_destroy(session, &txn_global->scan_rwlock); __wt_rwlock_destroy(session, &txn_global->nsnap_rwlock); + __wt_rwlock_destroy(session, &txn_global->visibility_rwlock); __wt_free(session, txn_global->states); } diff --git a/src/third_party/wiredtiger/src/txn/txn_log.c b/src/third_party/wiredtiger/src/txn/txn_log.c index 2931dc1ce82..09a8c4d9663 100644 --- a/src/third_party/wiredtiger/src/txn/txn_log.c +++ b/src/third_party/wiredtiger/src/txn/txn_log.c @@ -289,16 +289,20 @@ int __wt_txn_checkpoint_log( WT_SESSION_IMPL *session, bool full, uint32_t flags, WT_LSN *lsnp) { + WT_CONNECTION_IMPL *conn; WT_DECL_ITEM(logrec); WT_DECL_RET; WT_ITEM *ckpt_snapshot, empty; WT_LSN *ckpt_lsn; WT_TXN *txn; + WT_TXN_GLOBAL *txn_global; uint8_t *end, *p; size_t recsize; uint32_t i, rectype = WT_LOGREC_CHECKPOINT; const char *fmt = WT_UNCHECKED_STRING(IIIIu); + conn = S2C(session); + txn_global = &conn->txn_global; txn = &session->txn; ckpt_lsn = &txn->ckpt_lsn; @@ -319,6 +323,15 @@ __wt_txn_checkpoint_log( case WT_TXN_LOG_CKPT_PREPARE: txn->full_ckpt = true; WT_ERR(__wt_log_flush_lsn(session, ckpt_lsn, true)); + /* + * We take and immediately release the visibility lock. + * Acquiring the write lock guarantees that any transaction + * that has written to the log has also made its transaction + * visible at this time. + */ + __wt_writelock(session, &txn_global->visibility_rwlock); + __wt_writeunlock(session, &txn_global->visibility_rwlock); + /* * We need to make sure that the log records in the checkpoint * LSN are on disk. In particular to make sure that the @@ -363,20 +376,20 @@ __wt_txn_checkpoint_log( txn->ckpt_nsnapshot, ckpt_snapshot)); logrec->size += (uint32_t)recsize; WT_ERR(__wt_log_write(session, logrec, lsnp, - F_ISSET(S2C(session), WT_CONN_CKPT_SYNC) ? + F_ISSET(conn, WT_CONN_CKPT_SYNC) ? WT_LOG_FSYNC : 0)); /* * If this full checkpoint completed successfully and there is - * no hot backup in progress and this is not recovery, tell - * the logging subsystem the checkpoint LSN so that it can - * archive. Do not update the logging checkpoint LSN if this - * is during a clean connection close, only during a full - * checkpoint. A clean close may not update any metadata LSN - * and we do not want to archive in that case. + * no hot backup in progress and this is not an unclean + * recovery, tell the logging subsystem the checkpoint LSN so + * that it can archive. Do not update the logging checkpoint + * LSN if this is during a clean connection close, only during + * a full checkpoint. A clean close may not update any + * metadata LSN and we do not want to archive in that case. */ - if (!S2C(session)->hot_backup && - !F_ISSET(S2C(session), WT_CONN_RECOVERING) && + if (!conn->hot_backup && + !FLD_ISSET(conn->log_flags, WT_CONN_LOG_RECOVER_DIRTY) && txn->full_ckpt) __wt_log_ckpt(session, ckpt_lsn); diff --git a/src/third_party/wiredtiger/src/txn/txn_recover.c b/src/third_party/wiredtiger/src/txn/txn_recover.c index 30932195b1e..29f4dee5199 100644 --- a/src/third_party/wiredtiger/src/txn/txn_recover.c +++ b/src/third_party/wiredtiger/src/txn/txn_recover.c @@ -20,6 +20,7 @@ typedef struct { } *files; size_t file_alloc; /* Allocated size of files array. */ u_int max_fileid; /* Maximum file ID seen. */ + WT_LSN max_lsn; /* Maximum checkpoint LSN seen. */ u_int nfiles; /* Number of files in the metadata. */ WT_LSN ckpt_lsn; /* Start LSN for main recovery loop. */ @@ -342,6 +343,10 @@ __recovery_setup_file(WT_RECOVERY *r, const char *uri, const char *config) "Recovering %s with id %" PRIu32 " @ (%" PRIu32 ", %" PRIu32 ")", uri, fileid, lsn.l.file, lsn.l.offset); + if ((!WT_IS_MAX_LSN(&lsn) && !WT_IS_INIT_LSN(&lsn)) && + (WT_IS_MAX_LSN(&r->max_lsn) || __wt_log_cmp(&lsn, &r->max_lsn) > 0)) + r->max_lsn = lsn; + return (0); } @@ -428,6 +433,7 @@ __wt_txn_recover(WT_SESSION_IMPL *session) WT_RET(__wt_open_internal_session(conn, "txn-recover", false, WT_SESSION_NO_LOGGING, &session)); r.session = session; + WT_MAX_LSN(&r.max_lsn); F_SET(conn, WT_CONN_RECOVERING); WT_ERR(__wt_metadata_search(session, WT_METAFILE_URI, &config)); @@ -443,9 +449,29 @@ __wt_txn_recover(WT_SESSION_IMPL *session) */ if (!FLD_ISSET(S2C(session)->log_flags, WT_CONN_LOG_EXISTED) || WT_IS_MAX_LSN(&metafile->ckpt_lsn)) { + /* + * Detect if we're going from logging disabled to enabled. + * We need to know this to verify LSNs and start at the correct + * log file later. If someone ran with logging, then disabled + * it and removed all the log files and then turned logging back + * on, we have to start logging in the log file number that is + * larger than any checkpoint LSN we have from the earlier time. + */ WT_ERR(__recovery_file_scan(&r)); + /* + * The array can be re-allocated in recovery_file_scan. Reset + * our pointer after scanning all the files. + */ + metafile = &r.files[WT_METAFILE_ID]; conn->next_file_id = r.max_fileid; - goto done; + + if (FLD_ISSET(conn->log_flags, WT_CONN_LOG_ENABLED) && + WT_IS_MAX_LSN(&metafile->ckpt_lsn) && + !WT_IS_MAX_LSN(&r.max_lsn)) { + WT_ERR(__wt_log_reset(session, r.max_lsn.l.file)); + goto ckpt; + } else + goto done; } /* @@ -488,6 +514,11 @@ __wt_txn_recover(WT_SESSION_IMPL *session) /* Scan the metadata to find the live files and their IDs. */ WT_ERR(__recovery_file_scan(&r)); + /* + * Clear this out. We no longer need it and it could have been + * re-allocated when scanning the files. + */ + metafile = NULL; /* * We no longer need the metadata cursor: close it to avoid pinning any @@ -535,6 +566,8 @@ __wt_txn_recover(WT_SESSION_IMPL *session) * this is not a read-only connection. * We can consider skipping it in the future. */ + if (needs_rec) + FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_DIRTY); if (WT_IS_INIT_LSN(&r.ckpt_lsn)) WT_ERR(__wt_log_scan(session, NULL, WT_LOGSCAN_FIRST | WT_LOGSCAN_RECOVER, @@ -554,11 +587,12 @@ __wt_txn_recover(WT_SESSION_IMPL *session) * open is fast and keep the metadata up to date with the checkpoint * LSN and archiving. */ - WT_ERR(session->iface.checkpoint(&session->iface, "force=1")); +ckpt: WT_ERR(session->iface.checkpoint(&session->iface, "force=1")); done: FLD_SET(conn->log_flags, WT_CONN_LOG_RECOVER_DONE); err: WT_TRET(__recovery_free(&r)); __wt_free(session, config); + FLD_CLR(conn->log_flags, WT_CONN_LOG_RECOVER_DIRTY); if (ret != 0) __wt_err(session, ret, "Recovery failed"); diff --git a/src/third_party/wiredtiger/test/mciproject.yml b/src/third_party/wiredtiger/test/mciproject.yml index 6456475aa00..50a910d9e58 100644 --- a/src/third_party/wiredtiger/test/mciproject.yml +++ b/src/third_party/wiredtiger/test/mciproject.yml @@ -157,20 +157,6 @@ buildvariants: - name: unit-test - name: fops -- name: solaris - display_name: Solaris - run_on: - - solaris - expansions: - make_command: PATH=/opt/mongodbtoolchain/bin:$PATH gmake - test_env_vars: LD_LIBRARY_PATH=`pwd`/.libs - smp_command: -j $(kstat cpu | sort -u | grep -c "^module") - configure_env_vars: PATH=/opt/mongodbtoolchain/bin:$PATH CFLAGS="-m64" - tasks: - - name: compile - - name: unit-test - - name: fops - - name: windows-64 display_name: Windows 64-bit run_on: diff --git a/src/third_party/wiredtiger/test/recovery/random-abort.c b/src/third_party/wiredtiger/test/recovery/random-abort.c index febe6530534..b53383e5730 100644 --- a/src/third_party/wiredtiger/test/recovery/random-abort.c +++ b/src/third_party/wiredtiger/test/recovery/random-abort.c @@ -47,9 +47,9 @@ static bool inmem; #define RECORDS_FILE "records-%" PRIu32 #define ENV_CONFIG_DEF \ - "create,log=(file_max=10M,archive=false,enabled)" + "create,log=(file_max=10M,enabled)" #define ENV_CONFIG_TXNSYNC \ - "create,log=(file_max=10M,archive=false,enabled)," \ + "create,log=(file_max=10M,enabled)," \ "transaction_sync=(enabled,method=none)" #define ENV_CONFIG_REC "log=(recover=on)" #define MAX_VAL 4096 diff --git a/src/third_party/wiredtiger/test/suite/test_bug018.py b/src/third_party/wiredtiger/test/suite/test_bug018.py new file mode 100644 index 00000000000..7d20ebcaacb --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_bug018.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from helper import copy_wiredtiger_home +import os +import wiredtiger, wttest + +# test_bug018.py +# JIRA WT-3590: if writing table data fails during close then tables +# that were updated within the same transaction could get out of sync with +# each other. +class test_bug018(wttest.WiredTigerTestCase): + '''Test closing/reopening/recovering tables when writes fail''' + + conn_config = 'log=(enabled)' + + def setUp(self): + # This test uses Linux-specific code so skip on any other system. + if os.name != 'posix' or os.uname()[0] != 'Linux': + self.skipTest('Linux-specific test skipped on ' + os.name) + super(test_bug018, self).setUp() + + def create_table(self, uri): + self.session.create(uri, 'key_format=S,value_format=S') + return self.session.open_cursor(uri) + + def test_bug018(self): + '''Test closing multiple tables''' + basename = 'bug018.' + baseuri = 'file:' + basename + c1 = self.create_table(baseuri + '01.wt') + c2 = self.create_table(baseuri + '02.wt') + + self.session.begin_transaction() + c1['key'] = 'value' + c2['key'] = 'value' + self.session.commit_transaction() + + # Simulate a write failure by closing the file descriptor for the second + # table out from underneath WiredTiger. We do this right before + # closing the connection so that the write error happens during close + # when writing out the final data. Allow table 1 to succeed and force + # an erorr writing out table 2. + # + # This is Linux-specific code to figure out the file descriptor. + for f in os.listdir('/proc/self/fd'): + try: + if os.readlink('/proc/self/fd/' + f).endswith(basename + '02.wt'): + os.close(int(f)) + except OSError: + pass + + # Expect an error and messages, so turn off stderr checking. + with self.expectedStderrPattern(''): + try: + self.close_conn() + except wiredtiger.WiredTigerError: + self.conn = None + + # Make a backup for forensics in case something goes wrong. + backup_dir = 'BACKUP' + copy_wiredtiger_home('.', backup_dir, True) + + # After reopening and running recovery both tables should be in + # sync even though table 1 was successfully written and table 2 + # had an error on close. + self.open_conn() + c1 = self.session.open_cursor(baseuri + '01.wt') + c2 = self.session.open_cursor(baseuri + '02.wt') + self.assertEqual(list(c1), list(c2)) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_inmem01.py b/src/third_party/wiredtiger/test/suite/test_inmem01.py index 388485db29b..d280642942a 100644 --- a/src/third_party/wiredtiger/test/suite/test_inmem01.py +++ b/src/third_party/wiredtiger/test/suite/test_inmem01.py @@ -108,12 +108,15 @@ class test_inmem01(wttest.WiredTigerTestCase): cursor.reset() # Spin inserting to give eviction a chance to reclaim space + sleeps = 0 inserted = False for i in range(1, 1000): try: cursor[ds.key(1)] = ds.value(1) except wiredtiger.WiredTigerError: cursor.reset() + sleeps = sleeps + 1 + self.assertLess(sleeps, 60 * 5) sleep(1) continue inserted = True diff --git a/src/third_party/wiredtiger/test/suite/test_las.py b/src/third_party/wiredtiger/test/suite/test_las.py new file mode 100644 index 00000000000..d0bd1d108fa --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_las.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import wiredtiger, wttest +from wtdataset import SimpleDataSet + +# test_las.py +# Smoke tests to ensure lookaside tables are working. +class test_las(wttest.WiredTigerTestCase): + # Force a small cache. + def conn_config(self): + return 'cache_size=1GB' + + @wttest.longtest('lookaside table smoke test') + def test_las(self): + # Create a small table. + uri = "table:test_las" + nrows = 100 + ds = SimpleDataSet(self, uri, nrows, key_format="S") + ds.populate() + + # Take a snapshot. + self.session.snapshot("name=xxx") + + # Insert a large number of records, we'll hang if the lookaside table + # isn't doing its thing. + c = self.session.open_cursor(uri) + bigvalue = "abcde" * 100 + for i in range(1, 1000000): + c.set_key(ds.key(nrows + i)) + c.set_value(bigvalue) + self.assertEquals(c.insert(), 0) + +if __name__ == '__main__': + wttest.run() diff --git a/src/third_party/wiredtiger/test/suite/test_txn02.py b/src/third_party/wiredtiger/test/suite/test_txn02.py index 01626057b9e..76a325743e9 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn02.py +++ b/src/third_party/wiredtiger/test/suite/test_txn02.py @@ -169,7 +169,6 @@ class test_txn02(wttest.WiredTigerTestCase, suite_subprocess): try: session = backup_conn.open_session() finally: - session.checkpoint("force") self.check(backup_conn.open_session(), None, committed) # Sleep long enough so that the archive thread is guaranteed # to run before we close the connection. diff --git a/src/third_party/wiredtiger/test/suite/test_txn05.py b/src/third_party/wiredtiger/test/suite/test_txn05.py index 7aaff221ba4..7099bc972aa 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn05.py +++ b/src/third_party/wiredtiger/test/suite/test_txn05.py @@ -134,12 +134,12 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): session = backup_conn.open_session() finally: self.check(session, None, committed) - # Force a checkpoint because we don't record the recovery - # checkpoint as available for archiving. - session.checkpoint("force") # Sleep long enough so that the archive thread is guaranteed # to run before we close the connection. time.sleep(1.0) + if count == 0: + first_logs = \ + fnmatch.filter(os.listdir(self.backup_dir), "*Log*") backup_conn.close() count += 1 # @@ -149,6 +149,11 @@ class test_txn05(wttest.WiredTigerTestCase, suite_subprocess): # cur_logs = fnmatch.filter(os.listdir(self.backup_dir), "*Log*") for o in orig_logs: + # Creating the backup was effectively an unclean shutdown so + # even after sleeping, we should never archive log files + # because a checkpoint has not run. Later opens and runs of + # recovery will detect a clean shutdown and allow archiving. + self.assertEqual(True, o in first_logs) if self.archive == 'true': self.assertEqual(False, o in cur_logs) else: diff --git a/src/third_party/wiredtiger/test/suite/test_txn09.py b/src/third_party/wiredtiger/test/suite/test_txn09.py index 768d714e248..b8a3d7f38ae 100644 --- a/src/third_party/wiredtiger/test/suite/test_txn09.py +++ b/src/third_party/wiredtiger/test/suite/test_txn09.py @@ -26,8 +26,8 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # -# test_txn02.py -# Transactions: commits and rollbacks +# test_txn09.py +# Transactions: recovery toggling logging # import fnmatch, os, shutil, time diff --git a/src/third_party/wiredtiger/test/suite/test_txn16.py b/src/third_party/wiredtiger/test/suite/test_txn16.py new file mode 100644 index 00000000000..929da2291c7 --- /dev/null +++ b/src/third_party/wiredtiger/test/suite/test_txn16.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# test_txn16.py +# Recovery: Test that toggling between logging and not logging does not +# continue to generate more log files. +# + +import fnmatch, os, shutil, time +from suite_subprocess import suite_subprocess +import wttest + +class test_txn16(wttest.WiredTigerTestCase, suite_subprocess): + t1 = 'table:test_txn16_1' + t2 = 'table:test_txn16_2' + t3 = 'table:test_txn16_3' + nentries = 1000 + create_params = 'key_format=i,value_format=i' + # Set the log file size small so we generate checkpoints + # with LSNs in different files. + conn_config = 'config_base=false,' + \ + 'log=(archive=false,enabled,file_max=100K),' + \ + 'transaction_sync=(method=dsync,enabled)' + conn_on = 'config_base=false,' + \ + 'log=(archive=false,enabled,file_max=100K),' + \ + 'transaction_sync=(method=dsync,enabled)' + conn_off = 'config_base=false,log=(enabled=false)' + + def populate_table(self, uri): + self.session.create(uri, self.create_params) + c = self.session.open_cursor(uri, None, None) + # Populate with an occasional checkpoint to generate + # some varying LSNs. + for i in range(self.nentries): + c[i] = i + 1 + if i % 900 == 0: + self.session.checkpoint() + c.close() + + def copy_dir(self, olddir, newdir): + ''' Simulate a crash from olddir and restart in newdir. ''' + # with the connection still open, copy files to new directory + shutil.rmtree(newdir, ignore_errors=True) + os.mkdir(newdir) + for fname in os.listdir(olddir): + fullname = os.path.join(olddir, fname) + # Skip lock file on Windows since it is locked + if os.path.isfile(fullname) and \ + "WiredTiger.lock" not in fullname and \ + "Tmplog" not in fullname and \ + "Preplog" not in fullname: + shutil.copy(fullname, newdir) + # close the original connection. + self.close_conn() + + def run_toggle(self, homedir): + loop = 0 + # Record original log files. There should never be overlap + # with these even after they're removed. + orig_logs = fnmatch.filter(os.listdir(homedir), "*Log*") + while loop < 3: + # Reopen with logging on to run recovery first time + on_conn = self.wiredtiger_open(homedir, self.conn_on) + on_conn.close() + if loop > 0: + # Get current log files. + cur_logs = fnmatch.filter(os.listdir(homedir), "*Log*") + scur = set(cur_logs) + sorig = set(orig_logs) + # There should never be overlap with the log files that + # were there originally. Mostly this checks that after + # opening with logging disabled and then re-enabled, we + # don't see log file 1. + self.assertEqual(scur.isdisjoint(sorig), True) + if loop > 1: + # We should be creating the same log files each time. + for l in cur_logs: + self.assertEqual(l in last_logs, True) + for l in last_logs: + self.assertEqual(l in cur_logs, True) + last_logs = cur_logs + loop += 1 + # Remove all log files before opening without logging. + cur_logs = fnmatch.filter(os.listdir(homedir), "*Log*") + for l in cur_logs: + path=homedir + "/" + l + os.remove(path) + off_conn = self.wiredtiger_open(homedir, self.conn_off) + off_conn.close() + + def test_recovery(self): + ''' Check log file creation when toggling. ''' + + # Here's the strategy: + # - With logging populate 4 tables. Checkpoint + # them at different times. + # - Copy to a new directory to simulate a crash. + # - Close the original connection. + # On both a "copy" to simulate a crash and the original (3x): + # - Record log files existing. + # - Reopen with logging to run recovery. Close connection. + # - Record log files existing. + # - Remove all log files. + # - Open connection with logging disabled. + # - Record log files existing. Verify we don't keep adding. + # + self.populate_table(self.t1) + self.populate_table(self.t2) + self.populate_table(self.t3) + self.copy_dir(".", "RESTART") + self.run_toggle(".") + self.run_toggle("RESTART") + +if __name__ == '__main__': + wttest.run() diff --git a/version.json b/version.json index aeb6f194474..16ecc44565a 100644 --- a/version.json +++ b/version.json @@ -1,4 +1,4 @@ { - "githash": "cf38c1b8a0a8dca4a11737581beafef4fe120bcd", - "version": "3.4.7" + "githash": "fd954412dfc10e4d1e3e2dd4fac040f8b476b268", + "version": "3.4.14" } \ No newline at end of file -- cgit v1.2.3

+Return to the PCRE index page. +